diff options
| author | Andrei Zmievski <andrei@php.net> | 2003-06-27 16:31:24 +0000 |
|---|---|---|
| committer | Andrei Zmievski <andrei@php.net> | 2003-06-27 16:31:24 +0000 |
| commit | 357f62734ade0f293454596c46fce51d21275734 (patch) | |
| tree | 2696f5e8696308f69a2c1e56b213609e2545d0e8 /ext/pcre | |
| parent | 2e4ef86e10897a0e64500d3aeeaa59d58f8f4f5b (diff) | |
| download | php-git-357f62734ade0f293454596c46fce51d21275734.tar.gz | |
Upgrading PCRE library to version 4.3.
Diffstat (limited to 'ext/pcre')
34 files changed, 5340 insertions, 8379 deletions
diff --git a/ext/pcre/config.m4 b/ext/pcre/config.m4 index 9e81ad2814..1e2fc6d23a 100644 --- a/ext/pcre/config.m4 +++ b/ext/pcre/config.m4 @@ -50,7 +50,7 @@ if test "$PHP_PCRE_REGEX" != "no"; then AC_DEFINE(HAVE_PCRE, 1, [ ]) PHP_ADD_INCLUDE($PCRE_INCDIR) - PHP_NEW_EXTENSION(pcre, php_pcre.c, $ext_shared,,-DSUPPORT_UTF8 -DLINK_SIZE=2) + PHP_NEW_EXTENSION(pcre, php_pcre.c, $ext_shared,,-DSUPPORT_UTF8 -DLINK_SIZE=2 -DPOSIX_MALLOC_THRESHOLD=10) fi PHP_SUBST(PCRE_SHARED_LIBADD) fi diff --git a/ext/pcre/config0.m4 b/ext/pcre/config0.m4 index 9e81ad2814..1e2fc6d23a 100644 --- a/ext/pcre/config0.m4 +++ b/ext/pcre/config0.m4 @@ -50,7 +50,7 @@ if test "$PHP_PCRE_REGEX" != "no"; then AC_DEFINE(HAVE_PCRE, 1, [ ]) PHP_ADD_INCLUDE($PCRE_INCDIR) - PHP_NEW_EXTENSION(pcre, php_pcre.c, $ext_shared,,-DSUPPORT_UTF8 -DLINK_SIZE=2) + PHP_NEW_EXTENSION(pcre, php_pcre.c, $ext_shared,,-DSUPPORT_UTF8 -DLINK_SIZE=2 -DPOSIX_MALLOC_THRESHOLD=10) fi PHP_SUBST(PCRE_SHARED_LIBADD) fi diff --git a/ext/pcre/pcrelib/COPYING b/ext/pcre/pcrelib/COPYING index 8effa66492..8d680612c4 100644 --- a/ext/pcre/pcrelib/COPYING +++ b/ext/pcre/pcrelib/COPYING @@ -21,7 +21,7 @@ restrictions: 2. The origin of this software must not be misrepresented, either by explicit claim or by omission. In practice, this means that if you use - PCRE in software which you distribute to others, commercially or + PCRE in software that you distribute to others, commercially or otherwise, you must put a sentence like this Regular expression support is provided by the PCRE library package, @@ -34,7 +34,11 @@ restrictions: ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/ - should also be given in the documentation. + should also be given in the documentation. However, this condition is not + intended to apply to whole chains of software. If package A includes PCRE, + it must acknowledge it, but if package B is software that includes package + A, the condition is not imposed on package B (unless it uses PCRE + independently). 3. Altered versions must be plainly marked as such, and must not be misrepresented as being the original software. diff --git a/ext/pcre/pcrelib/ChangeLog b/ext/pcre/pcrelib/ChangeLog index 708b82a74b..b9123144ba 100644 --- a/ext/pcre/pcrelib/ChangeLog +++ b/ext/pcre/pcrelib/ChangeLog @@ -1,8 +1,150 @@ ChangeLog for PCRE ------------------ -Version 4.00 .... ------------------ +Version 4.3 21-May-03 +--------------------- + +1. Two instances of @WIN_PREFIX@ omitted from the Windows targets in the + Makefile. + +2. Some refactoring to improve the quality of the code: + + (i) The utf8_table... variables are now declared "const". + + (ii) The code for \cx, which used the "case flipping" table to upper case + lower case letters, now just substracts 32. This is ASCII-specific, + but the whole concept of \cx is ASCII-specific, so it seems + reasonable. + + (iii) PCRE was using its character types table to recognize decimal and + hexadecimal digits in the pattern. This is silly, because it handles + only 0-9, a-f, and A-F, but the character types table is locale- + specific, which means strange things might happen. A private + table is now used for this - though it costs 256 bytes, a table is + much faster than multiple explicit tests. Of course, the standard + character types table is still used for matching digits in subject + strings against \d. + + (iv) Strictly, the identifier ESC_t is reserved by POSIX (all identifiers + ending in _t are). So I've renamed it as ESC_tee. + +3. The first argument for regexec() in the POSIX wrapper should have been + defined as "const". + +4. Changed pcretest to use malloc() for its buffers so that they can be + Electric Fenced for debugging. + +5. There were several places in the code where, in UTF-8 mode, PCRE would try + to read one or more bytes before the start of the subject string. Often this + had no effect on PCRE's behaviour, but in some circumstances it could + provoke a segmentation fault. + +6. A lookbehind at the start of a pattern in UTF-8 mode could also cause PCRE + to try to read one or more bytes before the start of the subject string. + +7. A lookbehind in a pattern matched in non-UTF-8 mode on a PCRE compiled with + UTF-8 support could misbehave in various ways if the subject string + contained bytes with the 0x80 bit set and the 0x40 bit unset in a lookbehind + area. (PCRE was not checking for the UTF-8 mode flag, and trying to move + back over UTF-8 characters.) + + +Version 4.2 14-Apr-03 +--------------------- + +1. Typo "#if SUPPORT_UTF8" instead of "#ifdef SUPPORT_UTF8" fixed. + +2. Changes to the building process, supplied by Ronald Landheer-Cieslak + [ON_WINDOWS]: new variable, "#" on non-Windows platforms + [NOT_ON_WINDOWS]: new variable, "#" on Windows platforms + [WIN_PREFIX]: new variable, "cyg" for Cygwin + * Makefile.in: use autoconf substitution for OBJEXT, EXEEXT, BUILD_OBJEXT + and BUILD_EXEEXT + Note: automatic setting of the BUILD variables is not yet working + set CPPFLAGS and BUILD_CPPFLAGS (but don't use yet) - should be used at + compile-time but not at link-time + [LINK]: use for linking executables only + make different versions for Windows and non-Windows + [LINKLIB]: new variable, copy of UNIX-style LINK, used for linking + libraries + [LINK_FOR_BUILD]: new variable + [OBJEXT]: use throughout + [EXEEXT]: use throughout + <winshared>: new target + <wininstall>: new target + <dftables.o>: use native compiler + <dftables>: use native linker + <install>: handle Windows platform correctly + <clean>: ditto + <check>: ditto + copy DLL to top builddir before testing + + As part of these changes, -no-undefined was removed again. This was reported + to give trouble on HP-UX 11.0, so getting rid of it seems like a good idea + in any case. + +3. Some tidies to get rid of compiler warnings: + + . In the match_data structure, match_limit was an unsigned long int, whereas + match_call_count was an int. I've made them both unsigned long ints. + + . In pcretest the fact that a const uschar * doesn't automatically cast to + a void * provoked a warning. + + . Turning on some more compiler warnings threw up some "shadow" variables + and a few more missing casts. + +4. If PCRE was complied with UTF-8 support, but called without the PCRE_UTF8 + option, a class that contained a single character with a value between 128 + and 255 (e.g. /[\xFF]/) caused PCRE to crash. + +5. If PCRE was compiled with UTF-8 support, but called without the PCRE_UTF8 + option, a class that contained several characters, but with at least one + whose value was between 128 and 255 caused PCRE to crash. + + +Version 4.1 12-Mar-03 +--------------------- + +1. Compiling with gcc -pedantic found a couple of places where casts were +needed, and a string in dftables.c that was longer than standard compilers are +required to support. + +2. Compiling with Sun's compiler found a few more places where the code could +be tidied up in order to avoid warnings. + +3. The variables for cross-compiling were called HOST_CC and HOST_CFLAGS; the +first of these names is deprecated in the latest Autoconf in favour of the name +CC_FOR_BUILD, because "host" is typically used to mean the system on which the +compiled code will be run. I can't find a reference for HOST_CFLAGS, but by +analogy I have changed it to CFLAGS_FOR_BUILD. + +4. Added -no-undefined to the linking command in the Makefile, because this is +apparently helpful for Windows. To make it work, also added "-L. -lpcre" to the +linking step for the pcreposix library. + +5. PCRE was failing to diagnose the case of two named groups with the same +name. + +6. A problem with one of PCRE's optimizations was discovered. PCRE remembers a +literal character that is needed in the subject for a match, and scans along to +ensure that it is present before embarking on the full matching process. This +saves time in cases of nested unlimited repeats that are never going to match. +Problem: the scan can take a lot of time if the subject is very long (e.g. +megabytes), thus penalizing straightforward matches. It is now done only if the +amount of subject to be scanned is less than 1000 bytes. + +7. A lesser problem with the same optimization is that it was recording the +first character of an anchored pattern as "needed", thus provoking a search +right along the subject, even when the first match of the pattern was going to +fail. The "needed" character is now not set for anchored patterns, unless it +follows something in the pattern that is of non-fixed length. Thus, it still +fulfils its original purpose of finding quick non-matches in cases of nested +unlimited repeats, but isn't used for simple anchored patterns such as /^abc/. + + +Version 4.0 17-Feb-03 +--------------------- 1. If a comment in an extended regex that started immediately after a meta-item extended to the end of string, PCRE compiled incorrect data. This could lead to @@ -25,13 +167,12 @@ space and tab. 6. Perl 5.005 was a long time ago. It's time to amalgamate the tests that use its new features into the main test script, reducing the number of scripts. -7. Perl 5.8 has changed the meaning of patterns like /a(?i)b/. Earlier -versions were backward compatible, and made the (?i) apply to the whole -pattern, as if /i were given. Now it behaves more logically, and applies the -option setting only to what follows. PCRE has been changed to follow suit. -However, if it finds options settings right at the start of the pattern, it -extracts them into the global options, as before. Thus, they show up in the -info data. +7. Perl 5.8 has changed the meaning of patterns like /a(?i)b/. Earlier versions +were backward compatible, and made the (?i) apply to the whole pattern, as if +/i were given. Now it behaves more logically, and applies the option setting +only to what follows. PCRE has been changed to follow suit. However, if it +finds options settings right at the start of the pattern, it extracts them into +the global options, as before. Thus, they show up in the info data. 8. Added support for the \Q...\E escape sequence. Characters in between are treated as literals. This is slightly different from Perl in that $ and @ are @@ -44,6 +185,9 @@ interpolation. Note the following examples: \Qabc\$xyz\E abc\$xyz abc\$xyz \Qabc\E\$\Qxyz\E abc$xyz abc$xyz +For compatibility with Perl, \Q...\E sequences are recognized inside character +classes as well as outside them. + 9. Re-organized 3 code statements in pcretest to avoid "overflow in floating-point constant arithmetic" warnings from a Microsoft compiler. Added a (size_t) cast to one statement in pcretest and one in pcreposix to avoid @@ -58,11 +202,11 @@ that job. 12. Added "possessive quantifiers" ?+, *+, ++, and {,}+ which come from Sun's Java package. This provides some syntactic sugar for simple cases of what my -documentation calls "once-only subpatterns". A pattern such as x*+ is the -same as (?>x*). In other words, if what is inside (?>...) is just a single -repeated item, you can use this simplified notation. Note that only makes sense -with greedy quantifiers. Consequently, the use of the possessive quantifier -forces greediness, whatever the setting of the PCRE_UNGREEDY option. +documentation calls "once-only subpatterns". A pattern such as x*+ is the same +as (?>x*). In other words, if what is inside (?>...) is just a single repeated +item, you can use this simplified notation. Note that only makes sense with +greedy quantifiers. Consequently, the use of the possessive quantifier forces +greediness, whatever the setting of the PCRE_UNGREEDY option. 13. A change of greediness default within a pattern was not taking effect at the current level for patterns like /(b+(?U)a+)/. It did apply to parenthesized @@ -72,7 +216,9 @@ was abstracted outside. 14. PCRE now supports the \G assertion. It is true when the current matching position is at the start point of the match. This differs from \A when the starting offset is non-zero. Used with the /g option of pcretest (or similar -code), it works in the same way as it does for Perl's /g option. +code), it works in the same way as it does for Perl's /g option. If all +alternatives of a regex begin with \G, the expression is anchored to the start +match position, and the "anchored" flag is set in the compiled expression. 15. Some bugs concerning the handling of certain option changes within patterns have been fixed. These applied to options other than (?ims). For example, @@ -87,9 +233,10 @@ POSIX classes only within a class (e.g. /[[:alpha:]]/). 17. Added support for Perl's \C escape. This matches one byte, even in UTF8 mode. Unlike ".", it always matches newline, whatever the setting of PCRE_DOTALL. However, PCRE does not permit \C to appear in lookbehind -assertions. (Perl allows it, but it doesn't (in general) work because it can't +assertions. Perl allows it, but it doesn't (in general) work because it can't calculate the length of the lookbehind. At least, that's the case for Perl -5.8.0) +5.8.0 - I've been told they are going to document that it doesn't work in +future. 18. Added an error diagnosis for escapes that PCRE does not support: these are \L, \l, \N, \P, \p, \U, \u, and \X. @@ -121,7 +268,7 @@ there was a subsequent back reference to those brackets. This meant that, for example, /(.*)\d+\1/ failed to match "abc123bc". Unfortunately, it isn't possible to check for precisely this case. All we can do is abandon the optimization if .* occurs inside capturing brackets when there are any back -references whatsoever. +references whatsoever. (See below for a better fix that came later.) 26. The handling of the optimization for finding the first character of a non-anchored pattern, and for finding a character that is required later in the @@ -142,7 +289,8 @@ reaches such a point in the regex, if pcre_callout has been set, the external function is called. It is provided with data in a structure called pcre_callout_block, which is defined in pcre.h. If the function returns 0, matching continues; if it returns a non-zero value, the match at the current -point fails. However, backtracking will occur if possible. +point fails. However, backtracking will occur if possible. [This was changed +later and other features added - see item 49 below.] 29. pcretest is upgraded to test the callout functionality. It provides a callout function that displays information. By default, it shows the start of @@ -160,7 +308,7 @@ output "<stdin>" if a match was found, instead of the file name. 31. Improve the efficiency of the POSIX API to PCRE. If the number of capturing slots is less than POSIX_MALLOC_THRESHOLD, use a block on the stack to pass to pcre_exec(). This saves a malloc/free per call. The default value of -POSIX_MALLOC_THRESHOLD is 5; it can be changed by --with-posix-malloc-threshold +POSIX_MALLOC_THRESHOLD is 10; it can be changed by --with-posix-malloc-threshold when configuring. 32. The default maximum size of a compiled pattern is 64K. There have been a @@ -172,40 +320,254 @@ debugging information about compiled patterns. 33. Internal code re-arrangements: - (a) Moved the debugging function for printing out a compiled regex into - its own source file (printint.c) and used #include to pull it into - pcretest.c and, when DEBUG is defined, into pcre.c, instead of having - two separate copies. +(a) Moved the debugging function for printing out a compiled regex into + its own source file (printint.c) and used #include to pull it into + pcretest.c and, when DEBUG is defined, into pcre.c, instead of having two + separate copies. - (b) Defined the list of op-code names for debugging as a macro in - internal.h so that it is next to the definition of the opcodes. +(b) Defined the list of op-code names for debugging as a macro in + internal.h so that it is next to the definition of the opcodes. - (c) Defined a table of op-code lengths for simpler skipping along compiled - code. This is again a macro in internal.h so that it is next to the - definition of the opcodes. +(c) Defined a table of op-code lengths for simpler skipping along compiled + code. This is again a macro in internal.h so that it is next to the + definition of the opcodes. 34. Added support for recursive calls to individual subpatterns, along the - lines of Robin Houston's patch (but implemented somewhat differently). +lines of Robin Houston's patch (but implemented somewhat differently). -35. Further mods to the Makefile to help Win32. Also, added code to pcregrep - to allow it to read and process whole directories in Win32. This code was - contributed by Lionel Fourquaux; it has not been tested by me. +35. Further mods to the Makefile to help Win32. Also, added code to pcregrep to +allow it to read and process whole directories in Win32. This code was +contributed by Lionel Fourquaux; it has not been tested by me. 36. Added support for named subpatterns. The Python syntax (?P<name>...) is - used to name a group. Names consist of alphanumerics and underscores, and - must be unique. Back references use the syntax (?P=name) and recursive - calls use (?P>name) which is a PCRE extension to the Python extension. - Groups still have numbers. The function pcre_fullinfo() can be used after - compilation to extract a name/number map. There are three relevant calls: - - PCRE_INFO_NAMEENTRYSIZE yields the size of each entry in the map - PCRE_INFO_NAMECOUNT yields the number of entries - PCRE_INFO_NAMETABLE yields a pointer to the map. - - The map is a vector of fixed-size entries. The size of each entry depends - on the length of the longest name used. The first two bytes of each entry - are the group number, most significant byte first. There follows the - corresponding name, zero terminated. The names are in alphabetical order. +used to name a group. Names consist of alphanumerics and underscores, and must +be unique. Back references use the syntax (?P=name) and recursive calls use +(?P>name) which is a PCRE extension to the Python extension. Groups still have +numbers. The function pcre_fullinfo() can be used after compilation to extract +a name/number map. There are three relevant calls: + + PCRE_INFO_NAMEENTRYSIZE yields the size of each entry in the map + PCRE_INFO_NAMECOUNT yields the number of entries + PCRE_INFO_NAMETABLE yields a pointer to the map. + +The map is a vector of fixed-size entries. The size of each entry depends on +the length of the longest name used. The first two bytes of each entry are the +group number, most significant byte first. There follows the corresponding +name, zero terminated. The names are in alphabetical order. + +37. Make the maximum literal string in the compiled code 250 for the non-UTF-8 +case instead of 255. Making it the same both with and without UTF-8 support +means that the same test output works with both. + +38. There was a case of malloc(0) in the POSIX testing code in pcretest. Avoid +calling malloc() with a zero argument. + +39. Change 25 above had to resort to a heavy-handed test for the .* anchoring +optimization. I've improved things by keeping a bitmap of backreferences with +numbers 1-31 so that if .* occurs inside capturing brackets that are not in +fact referenced, the optimization can be applied. It is unlikely that a +relevant occurrence of .* (i.e. one which might indicate anchoring or forcing +the match to follow \n) will appear inside brackets with a number greater than +31, but if it does, any back reference > 31 suppresses the optimization. + +40. Added a new compile-time option PCRE_NO_AUTO_CAPTURE. This has the effect +of disabling numbered capturing parentheses. Any opening parenthesis that is +not followed by ? behaves as if it were followed by ?: but named parentheses +can still be used for capturing (and they will acquire numbers in the usual +way). + +41. Redesigned the return codes from the match() function into yes/no/error so +that errors can be passed back from deep inside the nested calls. A malloc +failure while inside a recursive subpattern call now causes the +PCRE_ERROR_NOMEMORY return instead of quietly going wrong. + +42. It is now possible to set a limit on the number of times the match() +function is called in a call to pcre_exec(). This facility makes it possible to +limit the amount of recursion and backtracking, though not in a directly +obvious way, because the match() function is used in a number of different +circumstances. The count starts from zero for each position in the subject +string (for non-anchored patterns). The default limit is, for compatibility, a +large number, namely 10 000 000. You can change this in two ways: + +(a) When configuring PCRE before making, you can use --with-match-limit=n + to set a default value for the compiled library. + +(b) For each call to pcre_exec(), you can pass a pcre_extra block in which + a different value is set. See 45 below. + +If the limit is exceeded, pcre_exec() returns PCRE_ERROR_MATCHLIMIT. + +43. Added a new function pcre_config(int, void *) to enable run-time extraction +of things that can be changed at compile time. The first argument specifies +what is wanted and the second points to where the information is to be placed. +The current list of available information is: + + PCRE_CONFIG_UTF8 + +The output is an integer that is set to one if UTF-8 support is available; +otherwise it is set to zero. + + PCRE_CONFIG_NEWLINE + +The output is an integer that it set to the value of the code that is used for +newline. It is either LF (10) or CR (13). + + PCRE_CONFIG_LINK_SIZE + +The output is an integer that contains the number of bytes used for internal +linkage in compiled expressions. The value is 2, 3, or 4. See item 32 above. + + PCRE_CONFIG_POSIX_MALLOC_THRESHOLD + +The output is an integer that contains the threshold above which the POSIX +interface uses malloc() for output vectors. See item 31 above. + + PCRE_CONFIG_MATCH_LIMIT + +The output is an unsigned integer that contains the default limit of the number +of match() calls in a pcre_exec() execution. See 42 above. + +44. pcretest has been upgraded by the addition of the -C option. This causes it +to extract all the available output from the new pcre_config() function, and to +output it. The program then exits immediately. + +45. A need has arisen to pass over additional data with calls to pcre_exec() in +order to support additional features. One way would have been to define +pcre_exec2() (for example) with extra arguments, but this would not have been +extensible, and would also have required all calls to the original function to +be mapped to the new one. Instead, I have chosen to extend the mechanism that +is used for passing in "extra" data from pcre_study(). + +The pcre_extra structure is now exposed and defined in pcre.h. It currently +contains the following fields: + + flags a bitmap indicating which of the following fields are set + study_data opaque data from pcre_study() + match_limit a way of specifying a limit on match() calls for a specific + call to pcre_exec() + callout_data data for callouts (see 49 below) + +The flag bits are also defined in pcre.h, and are + + PCRE_EXTRA_STUDY_DATA + PCRE_EXTRA_MATCH_LIMIT + PCRE_EXTRA_CALLOUT_DATA + +The pcre_study() function now returns one of these new pcre_extra blocks, with +the actual study data pointed to by the study_data field, and the +PCRE_EXTRA_STUDY_DATA flag set. This can be passed directly to pcre_exec() as +before. That is, this change is entirely upwards-compatible and requires no +change to existing code. + +If you want to pass in additional data to pcre_exec(), you can either place it +in a pcre_extra block provided by pcre_study(), or create your own pcre_extra +block. + +46. pcretest has been extended to test the PCRE_EXTRA_MATCH_LIMIT feature. If a +data string contains the escape sequence \M, pcretest calls pcre_exec() several +times with different match limits, until it finds the minimum value needed for +pcre_exec() to complete. The value is then output. This can be instructive; for +most simple matches the number is quite small, but for pathological cases it +gets very large very quickly. + +47. There's a new option for pcre_fullinfo() called PCRE_INFO_STUDYSIZE. It +returns the size of the data block pointed to by the study_data field in a +pcre_extra block, that is, the value that was passed as the argument to +pcre_malloc() when PCRE was getting memory in which to place the information +created by pcre_study(). The fourth argument should point to a size_t variable. +pcretest has been extended so that this information is shown after a successful +pcre_study() call when information about the compiled regex is being displayed. + +48. Cosmetic change to Makefile: there's no need to have / after $(DESTDIR) +because what follows is always an absolute path. (Later: it turns out that this +is more than cosmetic for MinGW, because it doesn't like empty path +components.) + +49. Some changes have been made to the callout feature (see 28 above): + +(i) A callout function now has three choices for what it returns: + + 0 => success, carry on matching + > 0 => failure at this point, but backtrack if possible + < 0 => serious error, return this value from pcre_exec() + + Negative values should normally be chosen from the set of PCRE_ERROR_xxx + values. In particular, returning PCRE_ERROR_NOMATCH forces a standard + "match failed" error. The error number PCRE_ERROR_CALLOUT is reserved for + use by callout functions. It will never be used by PCRE itself. + +(ii) The pcre_extra structure (see 45 above) has a void * field called + callout_data, with corresponding flag bit PCRE_EXTRA_CALLOUT_DATA. The + pcre_callout_block structure has a field of the same name. The contents of + the field passed in the pcre_extra structure are passed to the callout + function in the corresponding field in the callout block. This makes it + easier to use the same callout-containing regex from multiple threads. For + testing, the pcretest program has a new data escape + + \C*n pass the number n (may be negative) as callout_data + + If the callout function in pcretest receives a non-zero value as + callout_data, it returns that value. + +50. Makefile wasn't handling CFLAGS properly when compiling dftables. Also, +there were some redundant $(CFLAGS) in commands that are now specified as +$(LINK), which already includes $(CFLAGS). + +51. Extensions to UTF-8 support are listed below. These all apply when (a) PCRE +has been compiled with UTF-8 support *and* pcre_compile() has been compiled +with the PCRE_UTF8 flag. Patterns that are compiled without that flag assume +one-byte characters throughout. Note that case-insensitive matching applies +only to characters whose values are less than 256. PCRE doesn't support the +notion of cases for higher-valued characters. + +(i) A character class whose characters are all within 0-255 is handled as + a bit map, and the map is inverted for negative classes. Previously, a + character > 255 always failed to match such a class; however it should + match if the class was a negative one (e.g. [^ab]). This has been fixed. + +(ii) A negated character class with a single character < 255 is coded as + "not this character" (OP_NOT). This wasn't working properly when the test + character was multibyte, either singly or repeated. + +(iii) Repeats of multibyte characters are now handled correctly in UTF-8 + mode, for example: \x{100}{2,3}. + +(iv) The character escapes \b, \B, \d, \D, \s, \S, \w, and \W (either + singly or repeated) now correctly test multibyte characters. However, + PCRE doesn't recognize any characters with values greater than 255 as + digits, spaces, or word characters. Such characters always match \D, \S, + and \W, and never match \d, \s, or \w. + +(v) Classes may now contain characters and character ranges with values + greater than 255. For example: [ab\x{100}-\x{400}]. + +(vi) pcregrep now has a --utf-8 option (synonym -u) which makes it call + PCRE in UTF-8 mode. + +52. The info request value PCRE_INFO_FIRSTCHAR has been renamed +PCRE_INFO_FIRSTBYTE because it is a byte value. However, the old name is +retained for backwards compatibility. (Note that LASTLITERAL is also a byte +value.) + +53. The single man page has become too large. I have therefore split it up into +a number of separate man pages. These also give rise to individual HTML pages; +these are now put in a separate directory, and there is an index.html page that +lists them all. Some hyperlinking between the pages has been installed. + +54. Added convenience functions for handling named capturing parentheses. + +55. Unknown escapes inside character classes (e.g. [\M]) and escapes that +aren't interpreted therein (e.g. [\C]) are literals in Perl. This is now also +true in PCRE, except when the PCRE_EXTENDED option is set, in which case they +are faulted. + +56. Introduced HOST_CC and HOST_CFLAGS which can be set in the environment when +calling configure. These values are used when compiling the dftables.c program +which is run to generate the source of the default character tables. They +default to the values of CC and CFLAGS. If you are cross-compiling PCRE, +you will need to set these values. + +57. Updated the building process for Windows DLL, as provided by Fred Cox. Version 3.9 02-Jan-02 diff --git a/ext/pcre/pcrelib/LICENCE b/ext/pcre/pcrelib/LICENCE index 8effa66492..8d680612c4 100644 --- a/ext/pcre/pcrelib/LICENCE +++ b/ext/pcre/pcrelib/LICENCE @@ -21,7 +21,7 @@ restrictions: 2. The origin of this software must not be misrepresented, either by explicit claim or by omission. In practice, this means that if you use - PCRE in software which you distribute to others, commercially or + PCRE in software that you distribute to others, commercially or otherwise, you must put a sentence like this Regular expression support is provided by the PCRE library package, @@ -34,7 +34,11 @@ restrictions: ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/ - should also be given in the documentation. + should also be given in the documentation. However, this condition is not + intended to apply to whole chains of software. If package A includes PCRE, + it must acknowledge it, but if package B is software that includes package + A, the condition is not imposed on package B (unless it uses PCRE + independently). 3. Altered versions must be plainly marked as such, and must not be misrepresented as being the original software. diff --git a/ext/pcre/pcrelib/NEWS b/ext/pcre/pcrelib/NEWS index 27866b68a2..e620b2d727 100644 --- a/ext/pcre/pcrelib/NEWS +++ b/ext/pcre/pcrelib/NEWS @@ -1,6 +1,44 @@ News about PCRE releases ------------------------ +Release 4.0 17-Feb-03 +--------------------- + +There have been a lot of changes for the 4.0 release, adding additional +functionality and mending bugs. Below is a list of the highlights of the new +functionality. For full details of these features, please consult the +documentation. For a complete list of changes, see the ChangeLog file. + +1. Support for Perl's \Q...\E escapes. + +2. "Possessive quantifiers" ?+, *+, ++, and {,}+ which come from Sun's Java +package. They provide some syntactic sugar for simple cases of "atomic +grouping". + +3. Support for the \G assertion. It is true when the current matching position +is at the start point of the match. + +4. A new feature that provides some of the functionality that Perl provides +with (?{...}). The facility is termed a "callout". The way it is done in PCRE +is for the caller to provide an optional function, by setting pcre_callout to +its entry point. To get the function called, the regex must include (?C) at +appropriate points. + +5. Support for recursive calls to individual subpatterns. This makes it really +easy to get totally confused. + +6. Support for named subpatterns. The Python syntax (?P<name>...) is used to +name a group. + +7. Several extensions to UTF-8 support; it is now fairly complete. There is an +option for pcregrep to make it operate in UTF-8 mode. + +8. The single man page has been split into a number of separate man pages. +These also give rise to individual HTML pages which are put in a separate +directory. There is an index.html page that lists them all. Some hyperlinking +between the pages has been installed. + + Release 3.5 15-Aug-01 --------------------- diff --git a/ext/pcre/pcrelib/NON-UNIX-USE b/ext/pcre/pcrelib/NON-UNIX-USE index 8cbad88819..a000c29d41 100644 --- a/ext/pcre/pcrelib/NON-UNIX-USE +++ b/ext/pcre/pcrelib/NON-UNIX-USE @@ -1,6 +1,8 @@ Compiling PCRE on non-Unix systems ---------------------------------- +See below for comments on Cygwin or MinGW usage. + If you want to compile PCRE for a non-Unix system, note that it consists entirely of code written in Standard C, and so should compile successfully on any machine with a Standard C compiler and library, using normal compiling @@ -24,7 +26,7 @@ character tables. (4) Compile maketables.c, get.c, study.c and pcre.c and link them all together into an object library in whichever form your system keeps such -libraries. This is the pcre library (chartables.c gets included by means of an +libraries. This is the pcre library (chartables.c is included by means of an #include directive). (5) Similarly, compile pcreposix.c and link it as the pcreposix library. @@ -34,7 +36,9 @@ pcre and pcreposix libraries when linking. (7) Run pcretest on the testinput files in the testdata directory, and check that the output matches the corresponding testoutput files. You must use the --i option when checking testinput2. +-i option when checking testinput2. Note that the supplied files are in Unix +format, with just LF characters as line terminators. You may need to edit them +to change this if your system uses a different convention. If you have a system without "configure" but where you can use a Makefile, edit Makefile.in to create Makefile, substituting suitable values for the variables @@ -50,10 +54,30 @@ contributed by Paul Sokolovsky. These environments are Mingw32 linked pgrep and pcretest. If you have /bin/sh, run RunTest (three main test go ok, locale not supported). +Changes to do MinGW with autoconf 2.50 were supplied by Fred Cox +<sailorFred@yahoo.com>, who comments as follows: + + If you are using the PCRE DLL, the normal Unix style configure && make && + make check && make install should just work[*]. If you want to statically + link against the .a file, you must define PCRE_STATIC before including + pcre.h, otherwise the pcre_malloc and pcre_free exported functions will be + declared __declspec(dllimport), with hilarious results. See the configure.in + and pcretest.c for how it is done for the static test. + + Also, there will only be a libpcre.la, not a libpcreposix.la, as you + would expect from the Unix version. The single DLL includes the pcreposix + interface. + +[*] But note that the supplied test files are in Unix format, with just LF +characters as line terminators. You will have to edit them to change to CR LF +terminators. + A script for building PCRE using Borland's C++ compiler for use with VPASCAL was contributed by Alexander Tokarev. It is called makevp.bat. -These are some further comments about Win32 builds from Mark Evans: +These are some further comments about Win32 builds from Mark Evans. They +were contributed before Fred Cox's changes were made, so it is possible that +they may no longer be relevant. The documentation for Win32 builds is a bit shy. Under MSVC6 I followed their instructions to the letter, but there were still diff --git a/ext/pcre/pcrelib/README b/ext/pcre/pcrelib/README index 81cd1c6fa3..6cef77c7e8 100644 --- a/ext/pcre/pcrelib/README +++ b/ext/pcre/pcrelib/README @@ -56,11 +56,45 @@ into /source/pcre/pcre-xxx, but you want to build it in /build/pcre/pcre-xxx: cd /build/pcre/pcre-xxx /source/pcre/pcre-xxx/configure -If you want to make use of the experimential, incomplete support for UTF-8 -character strings in PCRE, you must add --enable-utf8 to the "configure" -command. Without it, the code for handling UTF-8 is not included in the -library. (Even when included, it still has to be enabled by an option at run -time.) +There are some optional features that can be included or omitted from the PCRE +library. You can read more about them in the pcrebuild man page. + +. If you want to make use of the support for UTF-8 character strings in PCRE, + you must add --enable-utf8 to the "configure" command. Without it, the code + for handling UTF-8 is not included in the library. (Even when included, it + still has to be enabled by an option at run time.) + +. You can build PCRE to recognized CR or NL as the newline character, instead + of whatever your compiler uses for "\n", by adding --newline-is-cr or + --newline-is-nl to the "configure" command, respectively. Only do this if you + really understand what you are doing. On traditional Unix-like systems, the + newline character is NL. + +. When called via the POSIX interface, PCRE uses malloc() to get additional + storage for processing capturing parentheses if there are more than 10 of + them. You can increase this threshold by setting, for example, + + --with-posix-malloc-threshold=20 + + on the "configure" command. + +. PCRE has a counter which can be set to limit the amount of resources it uses. + If the limit is exceeded during a match, the match fails. The default is ten + million. You can change the default by setting, for example, + + --with-match-limit=500000 + + on the "configure" command. This is just the default; individual calls to + pcre_exec() can supply their own value. There is discussion on the pcreapi + man page. + +. The default maximum compiled pattern size is around 64K. You can increase + this by adding --with-link-size=3 to the "configure" command. You can + increase it even more by setting --with-link-size=4, but this is unlikely + ever to be necessary. If you build PCRE with an increased link size, test 2 + (and 5 if you are using UTF-8) will fail. Part of the output of these tests + is a representation of the compiled pattern, and this changes with the link + size. The "configure" script builds five files: @@ -90,16 +124,19 @@ outputs information about where the library is installed. This command can be included in makefiles for programs that use PCRE, saving the programmer from having to remember too many details. -There is one esoteric feature that is controlled by "configure". It concerns -the character value used for "newline", and is something that you probably do -not want to change on a Unix system. The default is to use whatever value your -compiler gives to '\n'. By using --enable-newline-is-cr or ---enable-newline-is-lf you can force the value to be CR (13) or LF (10) if you -really want to. + +Cross-compiling PCRE on a Unix-like system +------------------------------------------ + +PCRE needs to compile and run an auxiliary program as part of the building +process. Obviously, if the real compilation is for some other system, it can't +use the same CC and CFLAGS values when it is doing this. For cross compilation, +therefore, you must set CC_FOR_BUILD to the local host's compiler, and you can +set flags in CFLAGS_FOR_BUILD if you need to. -Shared libraries on Unix systems --------------------------------- +Shared libraries on Unix-like systems +------------------------------------- The default distribution builds PCRE as two shared libraries and two static libraries, as long as the operating system supports shared libraries. Shared @@ -124,6 +161,19 @@ Then run "make" in the usual way. Similarly, you can use --disable-static to build only shared libraries. +Cross-compiling on a Unix-like system +------------------------------------- + +You can specify CC and CFLAGS in the normal way to the "configure" command, in +order to cross-compile PCRE for some other host. However, during the building +process, the dftables.c source file is compiled *and run* on the local host, in +order to generate the default character tables (the chartables.c file). It +therefore needs to be compiled with the local compiler, not the cross compiler. +You can do this by specifying HOST_CC (and if necessary HOST_CFLAGS) when +calling the "configure" command. If they are not specified, they default to the +values of CC and CFLAGS. + + Building on non-Unix systems ---------------------------- @@ -141,8 +191,8 @@ To test PCRE on a Unix system, run the RunTest script that is created by the configuring process. (This can also be run by "make runtest", "make check", or "make test".) For other systems, see the instruction in NON-UNIX-USE. -The script runs the pcretest test program (which is documented in the doc -directory) on each of the testinput files (in the testdata directory) in turn, +The script runs the pcretest test program (which is documented in its own man +page) on each of the testinput files (in the testdata directory) in turn, and compares the output with the contents of the corresponding testoutput file. A file called testtry is used to hold the output from pcretest. To run pcretest on just one of the test files, give its number as an argument to RunTest, for @@ -183,11 +233,11 @@ output to say why. If running this test produces instances of the error in the comparison output, it means that locale is not available on your system, despite being listed by "locale". This does not mean that PCRE is broken. -The fourth test checks the experimental, incomplete UTF-8 support. It is not -run automatically unless PCRE is built with UTF-8 support. To do this you must -set --enable-utf8 when running "configure". This file can be also fed directly -to the perltest script, provided you are running Perl 5.8 or higher. (For Perl -5.6, a small patch, commented in the script, can be be used.) +The fourth test checks the UTF-8 support. It is not run automatically unless +PCRE is built with UTF-8 support. To do this you must set --enable-utf8 when +running "configure". This file can be also fed directly to the perltest script, +provided you are running Perl 5.8 or higher. (For Perl 5.6, a small patch, +commented in the script, can be be used.) The fifth and final file tests error handling with UTF-8 encoding, and internal UTF-8 features of PCRE that are not relevant to Perl. @@ -246,6 +296,7 @@ The distribution should contain the following files: study.c ) source of pcre.c ) the functions pcreposix.c ) + printint.c ) pcre.in "source" for the header for the external API; pcre.h is built from this by "configure" pcreposix.h header for the external POSIX wrapper API @@ -269,23 +320,17 @@ The distribution should contain the following files: configure a configuring shell script (built by autoconf) configure.in the autoconf input used to build configure doc/Tech.Notes notes on the encoding - doc/pcre.3 man page source for the PCRE functions - doc/pcre.html HTML version - doc/pcre.txt plain text version - doc/pcreposix.3 man page source for the POSIX wrapper API - doc/pcreposix.html HTML version - doc/pcreposix.txt plain text version - doc/pcretest.txt documentation of test program - doc/perltest.txt documentation of Perl test program - doc/pcregrep.1 man page source for the pcregrep utility - doc/pcregrep.html HTML version - doc/pcregrep.txt plain text version + doc/*.3 man page sources for the PCRE functions + doc/*.1 man page sources for pcregrep and pcretest + doc/html/* HTML documentation + doc/pcre.txt plain text version of the man pages + doc/pcretest.txt plain text documentation of test program + doc/perltest.txt plain text documentation of Perl test program install-sh a shell script for installing files ltmain.sh file used to build a libtool script pcretest.c comprehensive test program pcredemo.c simple demonstration of coding calls to PCRE perltest Perl test program - perltest8 Perl test program for UTF-8 tests pcregrep.c source of a grep utility that uses PCRE pcre-config.in source of script which retains PCRE information testdata/testinput1 test data, compatible with Perl @@ -309,4 +354,4 @@ The distribution should contain the following files: makevp.bat Philip Hazel <ph10@cam.ac.uk> -August 2002 +February 2003 diff --git a/ext/pcre/pcrelib/chartables.c b/ext/pcre/pcrelib/chartables.c index 9055da2d2d..55e413cfca 100644 --- a/ext/pcre/pcrelib/chartables.c +++ b/ext/pcre/pcrelib/chartables.c @@ -148,7 +148,7 @@ print, punct, and cntrl. Other classes are built from combinations. */ */ 0x80,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 0- 7 */ - 0x00,0x01,0x01,0x01,0x01,0x01,0x00,0x00, /* 8- 15 */ + 0x00,0x01,0x01,0x00,0x01,0x01,0x00,0x00, /* 8- 15 */ 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 16- 23 */ 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 24- 31 */ 0x01,0x00,0x00,0x00,0x80,0x00,0x00,0x00, /* - ' */ diff --git a/ext/pcre/pcrelib/dftables.c b/ext/pcre/pcrelib/dftables.c index fe4ffcdb7a..9aa7b77e27 100644 --- a/ext/pcre/pcrelib/dftables.c +++ b/ext/pcre/pcrelib/dftables.c @@ -55,13 +55,17 @@ int main(void) int i; const unsigned char *tables = pcre_maketables(); +/* There are two printf() calls here, because gcc in pedantic mode complains +about the very long string otherwise. */ + printf( "/*************************************************\n" "* Perl-Compatible Regular Expressions *\n" "*************************************************/\n\n" "/* This file is automatically written by the dftables auxiliary \n" "program. If you edit it by hand, you might like to edit the Makefile to \n" - "prevent its ever being regenerated.\n\n" + "prevent its ever being regenerated.\n\n"); +printf( "This file is #included in the compilation of pcre.c to build the default\n" "character tables which are used when no tables are passed to the compile\n" "function. */\n\n" diff --git a/ext/pcre/pcrelib/doc/Tech.Notes b/ext/pcre/pcrelib/doc/Tech.Notes index df8f21892f..dd01932f8d 100644 --- a/ext/pcre/pcrelib/doc/Tech.Notes +++ b/ext/pcre/pcrelib/doc/Tech.Notes @@ -6,14 +6,14 @@ suggested by Martin Richards. These were not Unix-like in form, and were quite restricted in what they could do by comparison with Perl. The interesting part about the algorithm was that the amount of space required to hold the compiled form of an expression was known in advance. The code to apply an expression did -not operate by backtracking, as the Henry Spencer and Perl code does, but -instead checked all possibilities simultaneously by keeping a list of current -states and checking all of them as it advanced through the subject string. (In -the terminology of Jeffrey Friedl's book, it was a "DFA algorithm".) When the -pattern was all used up, all remaining states were possible matches, and the -one matching the longest subset of the subject string was chosen. This did not -necessarily maximize the individual wild portions of the pattern, as is -expected in Unix and Perl-style regular expressions. +not operate by backtracking, as the original Henry Spencer code and current +Perl code does, but instead checked all possibilities simultaneously by keeping +a list of current states and checking all of them as it advanced through the +subject string. (In the terminology of Jeffrey Friedl's book, it was a "DFA +algorithm".) When the pattern was all used up, all remaining states were +possible matches, and the one matching the longest subset of the subject string +was chosen. This did not necessarily maximize the individual wild portions of +the pattern, as is expected in Unix and Perl-style regular expressions. By contrast, the code originally written by Henry Spencer and subsequently heavily modified for Perl actually compiles the expression twice: once in a @@ -28,14 +28,13 @@ mentioned above), I tried at first to invent an algorithm that used an amount of store bounded by a multiple of the number of characters in the pattern, to save on compiling time. However, because of the greater complexity in Perl regular expressions, I couldn't do this. In any case, a first pass through the -pattern is needed, in order to find internal flag settings like (?i) at top -level. So PCRE works by running a very degenerate first pass to calculate a -maximum store size, and then a second pass to do the real compile - which may -use a bit less than the predicted amount of store. The idea is that this is -going to turn out faster because the first pass is degenerate and the second -pass can just store stuff straight into the vector. It does make the compiling -functions bigger, of course, but they have got quite big anyway to handle all -the Perl stuff. +pattern is needed, for a number of reasons. PCRE works by running a very +degenerate first pass to calculate a maximum store size, and then a second pass +to do the real compile - which may use a bit less than the predicted amount of +store. The idea is that this is going to turn out faster because the first pass +is degenerate and the second pass can just store stuff straight into the +vector. It does make the compiling functions bigger, of course, but they have +got quite big anyway to handle all the Perl stuff. The compiled form of a pattern is a vector of bytes, containing items of variable length. The first byte in an item is an opcode, and the length of the @@ -120,17 +119,25 @@ instances of OP_CHARS are used. Character classes ----------------- -OP_CLASS is used for a character class, provided there are at least two -characters in the class. If there is only one character, OP_CHARS is used for a -positive class, and OP_NOT for a negative one (that is, for something like -[^a]). Another set of repeating opcodes (OP_NOTSTAR etc.) are used for a -repeated, negated, single-character class. The normal ones (OP_STAR etc.) are -used for a repeated positive single-character class. +When characters less than 256 are involved, OP_CLASS is used for a character +class. If there is only one character, OP_CHARS is used for a positive class, +and OP_NOT for a negative one (that is, for something like [^a]). However, in +UTF-8 mode, this applies only to characters with values < 128, because OP_NOT +is confined to single bytes. + +Another set of repeating opcodes (OP_NOTSTAR etc.) are used for a repeated, +negated, single-character class. The normal ones (OP_STAR etc.) are used for a +repeated positive single-character class. OP_CLASS is followed by a 32-byte bit map containing a 1 bit for every character that is acceptable. The bits are counted from the least significant end of each byte. +For classes containing characters with values > 255, OP_XCLASS is used. It +optionally uses a bit map (if any characters lie within it), followed by a list +of pairs and single characters. There is a flag character than indicates +whether it's a positive or a negative class. + Back references --------------- diff --git a/ext/pcre/pcrelib/doc/pcre.3 b/ext/pcre/pcrelib/doc/pcre.3 deleted file mode 100644 index 4827c5e21d..0000000000 --- a/ext/pcre/pcrelib/doc/pcre.3 +++ /dev/null @@ -1,1993 +0,0 @@ -.TH PCRE 3 -.SH NAME -pcre - Perl-compatible regular expressions. -.SH SYNOPSIS -.B #include <pcre.h> -.PP -.SM -.br -.B pcre *pcre_compile(const char *\fIpattern\fR, int \fIoptions\fR, -.ti +5n -.B const char **\fIerrptr\fR, int *\fIerroffset\fR, -.ti +5n -.B const unsigned char *\fItableptr\fR); -.PP -.br -.B pcre_extra *pcre_study(const pcre *\fIcode\fR, int \fIoptions\fR, -.ti +5n -.B const char **\fIerrptr\fR); -.PP -.br -.B int pcre_exec(const pcre *\fIcode\fR, "const pcre_extra *\fIextra\fR," -.ti +5n -.B "const char *\fIsubject\fR," int \fIlength\fR, int \fIstartoffset\fR, -.ti +5n -.B int \fIoptions\fR, int *\fIovector\fR, int \fIovecsize\fR); -.PP -.br -.B int pcre_copy_substring(const char *\fIsubject\fR, int *\fIovector\fR, -.ti +5n -.B int \fIstringcount\fR, int \fIstringnumber\fR, char *\fIbuffer\fR, -.ti +5n -.B int \fIbuffersize\fR); -.PP -.br -.B int pcre_get_substring(const char *\fIsubject\fR, int *\fIovector\fR, -.ti +5n -.B int \fIstringcount\fR, int \fIstringnumber\fR, -.ti +5n -.B const char **\fIstringptr\fR); -.PP -.br -.B int pcre_get_substring_list(const char *\fIsubject\fR, -.ti +5n -.B int *\fIovector\fR, int \fIstringcount\fR, "const char ***\fIlistptr\fR);" -.PP -.br -.B void pcre_free_substring(const char *\fIstringptr\fR); -.PP -.br -.B void pcre_free_substring_list(const char **\fIstringptr\fR); -.PP -.br -.B const unsigned char *pcre_maketables(void); -.PP -.br -.B int pcre_fullinfo(const pcre *\fIcode\fR, "const pcre_extra *\fIextra\fR," -.ti +5n -.B int \fIwhat\fR, void *\fIwhere\fR); -.PP -.br -.B int pcre_info(const pcre *\fIcode\fR, int *\fIoptptr\fR, int -.B *\fIfirstcharptr\fR); -.PP -.br -.B char *pcre_version(void); -.PP -.br -.B void *(*pcre_malloc)(size_t); -.PP -.br -.B void (*pcre_free)(void *); - - - -.SH DESCRIPTION -The PCRE library is a set of functions that implement regular expression -pattern matching using the same syntax and semantics as Perl 5, with just a few -differences (see below). The current implementation corresponds to Perl 5.005, -with some additional features from later versions. This includes some -experimental, incomplete support for UTF-8 encoded strings. Details of exactly -what is and what is not supported are given below. - -PCRE has its own native API, which is described in this document. There is also -a set of wrapper functions that correspond to the POSIX regular expression API. -These are described in the \fBpcreposix\fR documentation. - -The native API function prototypes are defined in the header file \fBpcre.h\fR, -and on Unix systems the library itself is called \fBlibpcre.a\fR, so can be -accessed by adding \fB-lpcre\fR to the command for linking an application which -calls it. The header file defines the macros PCRE_MAJOR and PCRE_MINOR to -contain the major and minor release numbers for the library. Applications can -use these to include support for different releases. - -The functions \fBpcre_compile()\fR, \fBpcre_study()\fR, and \fBpcre_exec()\fR -are used for compiling and matching regular expressions. A sample program that -demonstrates the simplest way of using them is given in the file -\fIpcredemo.c\fR. The last section of this man page describes how to run it. - -The functions \fBpcre_copy_substring()\fR, \fBpcre_get_substring()\fR, and -\fBpcre_get_substring_list()\fR are convenience functions for extracting -captured substrings from a matched subject string; \fBpcre_free_substring()\fR -and \fBpcre_free_substring_list()\fR are also provided, to free the memory used -for extracted strings. - -The function \fBpcre_maketables()\fR is used (optionally) to build a set of -character tables in the current locale for passing to \fBpcre_compile()\fR. - -The function \fBpcre_fullinfo()\fR is used to find out information about a -compiled pattern; \fBpcre_info()\fR is an obsolete version which returns only -some of the available information, but is retained for backwards compatibility. -The function \fBpcre_version()\fR returns a pointer to a string containing the -version of PCRE and its date of release. - -The global variables \fBpcre_malloc\fR and \fBpcre_free\fR initially contain -the entry points of the standard \fBmalloc()\fR and \fBfree()\fR functions -respectively. PCRE calls the memory management functions via these variables, -so a calling program can replace them if it wishes to intercept the calls. This -should be done before calling any PCRE functions. - - -.SH MULTI-THREADING -The PCRE functions can be used in multi-threading applications, with the -proviso that the memory management functions pointed to by \fBpcre_malloc\fR -and \fBpcre_free\fR are shared by all threads. - -The compiled form of a regular expression is not altered during matching, so -the same compiled pattern can safely be used by several threads at once. - - -.SH COMPILING A PATTERN -The function \fBpcre_compile()\fR is called to compile a pattern into an -internal form. The pattern is a C string terminated by a binary zero, and -is passed in the argument \fIpattern\fR. A pointer to a single block of memory -that is obtained via \fBpcre_malloc\fR is returned. This contains the compiled -code and related data. The \fBpcre\fR type is defined for the returned block; -this is a typedef for a structure whose contents are not externally defined. It -is up to the caller to free the memory when it is no longer required. - -Although the compiled code of a PCRE regex is relocatable, that is, it does not -depend on memory location, the complete \fBpcre\fR data block is not -fully relocatable, because it contains a copy of the \fItableptr\fR argument, -which is an address (see below). - -The size of a compiled pattern is roughly proportional to the length of the -pattern string, except that each character class (other than those containing -just a single character, negated or not) requires 33 bytes, and repeat -quantifiers with a minimum greater than one or a bounded maximum cause the -relevant portions of the compiled pattern to be replicated. - -The \fIoptions\fR argument contains independent bits that affect the -compilation. It should be zero if no options are required. Some of the options, -in particular, those that are compatible with Perl, can also be set and unset -from within the pattern (see the detailed description of regular expressions -below). For these options, the contents of the \fIoptions\fR argument specifies -their initial settings at the start of compilation and execution. The -PCRE_ANCHORED option can be set at the time of matching as well as at compile -time. - -If \fIerrptr\fR is NULL, \fBpcre_compile()\fR returns NULL immediately. -Otherwise, if compilation of a pattern fails, \fBpcre_compile()\fR returns -NULL, and sets the variable pointed to by \fIerrptr\fR to point to a textual -error message. The offset from the start of the pattern to the character where -the error was discovered is placed in the variable pointed to by -\fIerroffset\fR, which must not be NULL. If it is, an immediate error is given. - -If the final argument, \fItableptr\fR, is NULL, PCRE uses a default set of -character tables which are built when it is compiled, using the default C -locale. Otherwise, \fItableptr\fR must be the result of a call to -\fBpcre_maketables()\fR. See the section on locale support below. - -This code fragment shows a typical straightforward call to \fBpcre_compile()\fR: - - pcre *re; - const char *error; - int erroffset; - re = pcre_compile( - "^A.*Z", /* the pattern */ - 0, /* default options */ - &error, /* for error message */ - &erroffset, /* for error offset */ - NULL); /* use default character tables */ - -The following option bits are defined in the header file: - - PCRE_ANCHORED - -If this bit is set, the pattern is forced to be "anchored", that is, it is -constrained to match only at the start of the string which is being searched -(the "subject string"). This effect can also be achieved by appropriate -constructs in the pattern itself, which is the only way to do it in Perl. - - PCRE_CASELESS - -If this bit is set, letters in the pattern match both upper and lower case -letters. It is equivalent to Perl's /i option. - - PCRE_DOLLAR_ENDONLY - -If this bit is set, a dollar metacharacter in the pattern matches only at the -end of the subject string. Without this option, a dollar also matches -immediately before the final character if it is a newline (but not before any -other newlines). The PCRE_DOLLAR_ENDONLY option is ignored if PCRE_MULTILINE is -set. There is no equivalent to this option in Perl. - - PCRE_DOTALL - -If this bit is set, a dot metacharater in the pattern matches all characters, -including newlines. Without it, newlines are excluded. This option is -equivalent to Perl's /s option. A negative class such as [^a] always matches a -newline character, independent of the setting of this option. - - PCRE_EXTENDED - -If this bit is set, whitespace data characters in the pattern are totally -ignored except when escaped or inside a character class, and characters between -an unescaped # outside a character class and the next newline character, -inclusive, are also ignored. This is equivalent to Perl's /x option, and makes -it possible to include comments inside complicated patterns. Note, however, -that this applies only to data characters. Whitespace characters may never -appear within special character sequences in a pattern, for example within the -sequence (?( which introduces a conditional subpattern. - - PCRE_EXTRA - -This option was invented in order to turn on additional functionality of PCRE -that is incompatible with Perl, but it is currently of very little use. When -set, any backslash in a pattern that is followed by a letter that has no -special meaning causes an error, thus reserving these combinations for future -expansion. By default, as in Perl, a backslash followed by a letter with no -special meaning is treated as a literal. There are at present no other features -controlled by this option. It can also be set by a (?X) option setting within a -pattern. - - PCRE_MULTILINE - -By default, PCRE treats the subject string as consisting of a single "line" of -characters (even if it actually contains several newlines). The "start of line" -metacharacter (^) matches only at the start of the string, while the "end of -line" metacharacter ($) matches only at the end of the string, or before a -terminating newline (unless PCRE_DOLLAR_ENDONLY is set). This is the same as -Perl. - -When PCRE_MULTILINE it is set, the "start of line" and "end of line" constructs -match immediately following or immediately before any newline in the subject -string, respectively, as well as at the very start and end. This is equivalent -to Perl's /m option. If there are no "\\n" characters in a subject string, or -no occurrences of ^ or $ in a pattern, setting PCRE_MULTILINE has no -effect. - - PCRE_UNGREEDY - -This option inverts the "greediness" of the quantifiers so that they are not -greedy by default, but become greedy if followed by "?". It is not compatible -with Perl. It can also be set by a (?U) option setting within the pattern. - - PCRE_UTF8 - -This option causes PCRE to regard both the pattern and the subject as strings -of UTF-8 characters instead of just byte strings. However, it is available only -if PCRE has been built to include UTF-8 support. If not, the use of this option -provokes an error. Support for UTF-8 is new, experimental, and incomplete. -Details of exactly what it entails are given below. - - -.SH STUDYING A PATTERN -When a pattern is going to be used several times, it is worth spending more -time analyzing it in order to speed up the time taken for matching. The -function \fBpcre_study()\fR takes a pointer to a compiled pattern as its first -argument, and returns a pointer to a \fBpcre_extra\fR block (another typedef -for a structure with hidden contents) containing additional information about -the pattern; this can be passed to \fBpcre_exec()\fR. If no additional -information is available, NULL is returned. - -The second argument contains option bits. At present, no options are defined -for \fBpcre_study()\fR, and this argument should always be zero. - -The third argument for \fBpcre_study()\fR is a pointer to an error message. If -studying succeeds (even if no data is returned), the variable it points to is -set to NULL. Otherwise it points to a textual error message. - -This is a typical call to \fBpcre_study\fR(): - - pcre_extra *pe; - pe = pcre_study( - re, /* result of pcre_compile() */ - 0, /* no options exist */ - &error); /* set to NULL or points to a message */ - -At present, studying a pattern is useful only for non-anchored patterns that do -not have a single fixed starting character. A bitmap of possible starting -characters is created. - - -.SH LOCALE SUPPORT -PCRE handles caseless matching, and determines whether characters are letters, -digits, or whatever, by reference to a set of tables. The library contains a -default set of tables which is created in the default C locale when PCRE is -compiled. This is used when the final argument of \fBpcre_compile()\fR is NULL, -and is sufficient for many applications. - -An alternative set of tables can, however, be supplied. Such tables are built -by calling the \fBpcre_maketables()\fR function, which has no arguments, in the -relevant locale. The result can then be passed to \fBpcre_compile()\fR as often -as necessary. For example, to build and use tables that are appropriate for the -French locale (where accented characters with codes greater than 128 are -treated as letters), the following code could be used: - - setlocale(LC_CTYPE, "fr"); - tables = pcre_maketables(); - re = pcre_compile(..., tables); - -The tables are built in memory that is obtained via \fBpcre_malloc\fR. The -pointer that is passed to \fBpcre_compile\fR is saved with the compiled -pattern, and the same tables are used via this pointer by \fBpcre_study()\fR -and \fBpcre_exec()\fR. Thus for any single pattern, compilation, studying and -matching all happen in the same locale, but different patterns can be compiled -in different locales. It is the caller's responsibility to ensure that the -memory containing the tables remains available for as long as it is needed. - - -.SH INFORMATION ABOUT A PATTERN -The \fBpcre_fullinfo()\fR function returns information about a compiled -pattern. It replaces the obsolete \fBpcre_info()\fR function, which is -nevertheless retained for backwards compability (and is documented below). - -The first argument for \fBpcre_fullinfo()\fR is a pointer to the compiled -pattern. The second argument is the result of \fBpcre_study()\fR, or NULL if -the pattern was not studied. The third argument specifies which piece of -information is required, while the fourth argument is a pointer to a variable -to receive the data. The yield of the function is zero for success, or one of -the following negative numbers: - - PCRE_ERROR_NULL the argument \fIcode\fR was NULL - the argument \fIwhere\fR was NULL - PCRE_ERROR_BADMAGIC the "magic number" was not found - PCRE_ERROR_BADOPTION the value of \fIwhat\fR was invalid - -Here is a typical call of \fBpcre_fullinfo()\fR, to obtain the length of the -compiled pattern: - - int rc; - unsigned long int length; - rc = pcre_fullinfo( - re, /* result of pcre_compile() */ - pe, /* result of pcre_study(), or NULL */ - PCRE_INFO_SIZE, /* what is required */ - &length); /* where to put the data */ - -The possible values for the third argument are defined in \fBpcre.h\fR, and are -as follows: - - PCRE_INFO_OPTIONS - -Return a copy of the options with which the pattern was compiled. The fourth -argument should point to an \fBunsigned long int\fR variable. These option bits -are those specified in the call to \fBpcre_compile()\fR, modified by any -top-level option settings within the pattern itself, and with the PCRE_ANCHORED -bit forcibly set if the form of the pattern implies that it can match only at -the start of a subject string. - - PCRE_INFO_SIZE - -Return the size of the compiled pattern, that is, the value that was passed as -the argument to \fBpcre_malloc()\fR when PCRE was getting memory in which to -place the compiled data. The fourth argument should point to a \fBsize_t\fR -variable. - - PCRE_INFO_CAPTURECOUNT - -Return the number of capturing subpatterns in the pattern. The fourth argument -should point to an \fbint\fR variable. - - PCRE_INFO_BACKREFMAX - -Return the number of the highest back reference in the pattern. The fourth -argument should point to an \fBint\fR variable. Zero is returned if there are -no back references. - - PCRE_INFO_FIRSTCHAR - -Return information about the first character of any matched string, for a -non-anchored pattern. If there is a fixed first character, e.g. from a pattern -such as (cat|cow|coyote), it is returned in the integer pointed to by -\fIwhere\fR. Otherwise, if either - -(a) the pattern was compiled with the PCRE_MULTILINE option, and every branch -starts with "^", or - -(b) every branch of the pattern starts with ".*" and PCRE_DOTALL is not set -(if it were set, the pattern would be anchored), - --1 is returned, indicating that the pattern matches only at the start of a -subject string or after any "\\n" within the string. Otherwise -2 is returned. -For anchored patterns, -2 is returned. - - PCRE_INFO_FIRSTTABLE - -If the pattern was studied, and this resulted in the construction of a 256-bit -table indicating a fixed set of characters for the first character in any -matching string, a pointer to the table is returned. Otherwise NULL is -returned. The fourth argument should point to an \fBunsigned char *\fR -variable. - - PCRE_INFO_LASTLITERAL - -For a non-anchored pattern, return the value of the rightmost literal character -which must exist in any matched string, other than at its start. The fourth -argument should point to an \fBint\fR variable. If there is no such character, -or if the pattern is anchored, -1 is returned. For example, for the pattern -/a\\d+z\\d+/ the returned value is 'z'. - -The \fBpcre_info()\fR function is now obsolete because its interface is too -restrictive to return all the available data about a compiled pattern. New -programs should use \fBpcre_fullinfo()\fR instead. The yield of -\fBpcre_info()\fR is the number of capturing subpatterns, or one of the -following negative numbers: - - PCRE_ERROR_NULL the argument \fIcode\fR was NULL - PCRE_ERROR_BADMAGIC the "magic number" was not found - -If the \fIoptptr\fR argument is not NULL, a copy of the options with which the -pattern was compiled is placed in the integer it points to (see -PCRE_INFO_OPTIONS above). - -If the pattern is not anchored and the \fIfirstcharptr\fR argument is not NULL, -it is used to pass back information about the first character of any matched -string (see PCRE_INFO_FIRSTCHAR above). - - -.SH MATCHING A PATTERN -The function \fBpcre_exec()\fR is called to match a subject string against a -pre-compiled pattern, which is passed in the \fIcode\fR argument. If the -pattern has been studied, the result of the study should be passed in the -\fIextra\fR argument. Otherwise this must be NULL. - -Here is an example of a simple call to \fBpcre_exec()\fR: - - int rc; - int ovector[30]; - rc = pcre_exec( - re, /* result of pcre_compile() */ - NULL, /* we didn't study the pattern */ - "some string", /* the subject string */ - 11, /* the length of the subject string */ - 0, /* start at offset 0 in the subject */ - 0, /* default options */ - ovector, /* vector for substring information */ - 30); /* number of elements in the vector */ - -The PCRE_ANCHORED option can be passed in the \fIoptions\fR argument, whose -unused bits must be zero. However, if a pattern was compiled with -PCRE_ANCHORED, or turned out to be anchored by virtue of its contents, it -cannot be made unachored at matching time. - -There are also three further options that can be set only at matching time: - - PCRE_NOTBOL - -The first character of the string is not the beginning of a line, so the -circumflex metacharacter should not match before it. Setting this without -PCRE_MULTILINE (at compile time) causes circumflex never to match. - - PCRE_NOTEOL - -The end of the string is not the end of a line, so the dollar metacharacter -should not match it nor (except in multiline mode) a newline immediately before -it. Setting this without PCRE_MULTILINE (at compile time) causes dollar never -to match. - - PCRE_NOTEMPTY - -An empty string is not considered to be a valid match if this option is set. If -there are alternatives in the pattern, they are tried. If all the alternatives -match the empty string, the entire match fails. For example, if the pattern - - a?b? - -is applied to a string not beginning with "a" or "b", it matches the empty -string at the start of the subject. With PCRE_NOTEMPTY set, this match is not -valid, so PCRE searches further into the string for occurrences of "a" or "b". - -Perl has no direct equivalent of PCRE_NOTEMPTY, but it does make a special case -of a pattern match of the empty string within its \fBsplit()\fR function, and -when using the /g modifier. It is possible to emulate Perl's behaviour after -matching a null string by first trying the match again at the same offset with -PCRE_NOTEMPTY set, and then if that fails by advancing the starting offset (see -below) and trying an ordinary match again. - -The subject string is passed as a pointer in \fIsubject\fR, a length in -\fIlength\fR, and a starting offset in \fIstartoffset\fR. Unlike the pattern -string, the subject may contain binary zero characters. When the starting -offset is zero, the search for a match starts at the beginning of the subject, -and this is by far the most common case. - -A non-zero starting offset is useful when searching for another match in the -same subject by calling \fBpcre_exec()\fR again after a previous success. -Setting \fIstartoffset\fR differs from just passing over a shortened string and -setting PCRE_NOTBOL in the case of a pattern that begins with any kind of -lookbehind. For example, consider the pattern - - \\Biss\\B - -which finds occurrences of "iss" in the middle of words. (\\B matches only if -the current position in the subject is not a word boundary.) When applied to -the string "Mississipi" the first call to \fBpcre_exec()\fR finds the first -occurrence. If \fBpcre_exec()\fR is called again with just the remainder of the -subject, namely "issipi", it does not match, because \\B is always false at the -start of the subject, which is deemed to be a word boundary. However, if -\fBpcre_exec()\fR is passed the entire string again, but with \fIstartoffset\fR -set to 4, it finds the second occurrence of "iss" because it is able to look -behind the starting point to discover that it is preceded by a letter. - -If a non-zero starting offset is passed when the pattern is anchored, one -attempt to match at the given offset is tried. This can only succeed if the -pattern does not require the match to be at the start of the subject. - -In general, a pattern matches a certain portion of the subject, and in -addition, further substrings from the subject may be picked out by parts of the -pattern. Following the usage in Jeffrey Friedl's book, this is called -"capturing" in what follows, and the phrase "capturing subpattern" is used for -a fragment of a pattern that picks out a substring. PCRE supports several other -kinds of parenthesized subpattern that do not cause substrings to be captured. - -Captured substrings are returned to the caller via a vector of integer offsets -whose address is passed in \fIovector\fR. The number of elements in the vector -is passed in \fIovecsize\fR. The first two-thirds of the vector is used to pass -back captured substrings, each substring using a pair of integers. The -remaining third of the vector is used as workspace by \fBpcre_exec()\fR while -matching capturing subpatterns, and is not available for passing back -information. The length passed in \fIovecsize\fR should always be a multiple of -three. If it is not, it is rounded down. - -When a match has been successful, information about captured substrings is -returned in pairs of integers, starting at the beginning of \fIovector\fR, and -continuing up to two-thirds of its length at the most. The first element of a -pair is set to the offset of the first character in a substring, and the second -is set to the offset of the first character after the end of a substring. The -first pair, \fIovector[0]\fR and \fIovector[1]\fR, identify the portion of the -subject string matched by the entire pattern. The next pair is used for the -first capturing subpattern, and so on. The value returned by \fBpcre_exec()\fR -is the number of pairs that have been set. If there are no capturing -subpatterns, the return value from a successful match is 1, indicating that -just the first pair of offsets has been set. - -Some convenience functions are provided for extracting the captured substrings -as separate strings. These are described in the following section. - -It is possible for an capturing subpattern number \fIn+1\fR to match some -part of the subject when subpattern \fIn\fR has not been used at all. For -example, if the string "abc" is matched against the pattern (a|(z))(bc) -subpatterns 1 and 3 are matched, but 2 is not. When this happens, both offset -values corresponding to the unused subpattern are set to -1. - -If a capturing subpattern is matched repeatedly, it is the last portion of the -string that it matched that gets returned. - -If the vector is too small to hold all the captured substrings, it is used as -far as possible (up to two-thirds of its length), and the function returns a -value of zero. In particular, if the substring offsets are not of interest, -\fBpcre_exec()\fR may be called with \fIovector\fR passed as NULL and -\fIovecsize\fR as zero. However, if the pattern contains back references and -the \fIovector\fR isn't big enough to remember the related substrings, PCRE has -to get additional memory for use during matching. Thus it is usually advisable -to supply an \fIovector\fR. - -Note that \fBpcre_info()\fR can be used to find out how many capturing -subpatterns there are in a compiled pattern. The smallest size for -\fIovector\fR that will allow for \fIn\fR captured substrings in addition to -the offsets of the substring matched by the whole pattern is (\fIn\fR+1)*3. - -If \fBpcre_exec()\fR fails, it returns a negative number. The following are -defined in the header file: - - PCRE_ERROR_NOMATCH (-1) - -The subject string did not match the pattern. - - PCRE_ERROR_NULL (-2) - -Either \fIcode\fR or \fIsubject\fR was passed as NULL, or \fIovector\fR was -NULL and \fIovecsize\fR was not zero. - - PCRE_ERROR_BADOPTION (-3) - -An unrecognized bit was set in the \fIoptions\fR argument. - - PCRE_ERROR_BADMAGIC (-4) - -PCRE stores a 4-byte "magic number" at the start of the compiled code, to catch -the case when it is passed a junk pointer. This is the error it gives when the -magic number isn't present. - - PCRE_ERROR_UNKNOWN_NODE (-5) - -While running the pattern match, an unknown item was encountered in the -compiled pattern. This error could be caused by a bug in PCRE or by overwriting -of the compiled pattern. - - PCRE_ERROR_NOMEMORY (-6) - -If a pattern contains back references, but the \fIovector\fR that is passed to -\fBpcre_exec()\fR is not big enough to remember the referenced substrings, PCRE -gets a block of memory at the start of matching to use for this purpose. If the -call via \fBpcre_malloc()\fR fails, this error is given. The memory is freed at -the end of matching. - - -.SH EXTRACTING CAPTURED SUBSTRINGS -Captured substrings can be accessed directly by using the offsets returned by -\fBpcre_exec()\fR in \fIovector\fR. For convenience, the functions -\fBpcre_copy_substring()\fR, \fBpcre_get_substring()\fR, and -\fBpcre_get_substring_list()\fR are provided for extracting captured substrings -as new, separate, zero-terminated strings. A substring that contains a binary -zero is correctly extracted and has a further zero added on the end, but the -result does not, of course, function as a C string. - -The first three arguments are the same for all three functions: \fIsubject\fR -is the subject string which has just been successfully matched, \fIovector\fR -is a pointer to the vector of integer offsets that was passed to -\fBpcre_exec()\fR, and \fIstringcount\fR is the number of substrings that -were captured by the match, including the substring that matched the entire -regular expression. This is the value returned by \fBpcre_exec\fR if it -is greater than zero. If \fBpcre_exec()\fR returned zero, indicating that it -ran out of space in \fIovector\fR, the value passed as \fIstringcount\fR should -be the size of the vector divided by three. - -The functions \fBpcre_copy_substring()\fR and \fBpcre_get_substring()\fR -extract a single substring, whose number is given as \fIstringnumber\fR. A -value of zero extracts the substring that matched the entire pattern, while -higher values extract the captured substrings. For \fBpcre_copy_substring()\fR, -the string is placed in \fIbuffer\fR, whose length is given by -\fIbuffersize\fR, while for \fBpcre_get_substring()\fR a new block of memory is -obtained via \fBpcre_malloc\fR, and its address is returned via -\fIstringptr\fR. The yield of the function is the length of the string, not -including the terminating zero, or one of - - PCRE_ERROR_NOMEMORY (-6) - -The buffer was too small for \fBpcre_copy_substring()\fR, or the attempt to get -memory failed for \fBpcre_get_substring()\fR. - - PCRE_ERROR_NOSUBSTRING (-7) - -There is no substring whose number is \fIstringnumber\fR. - -The \fBpcre_get_substring_list()\fR function extracts all available substrings -and builds a list of pointers to them. All this is done in a single block of -memory which is obtained via \fBpcre_malloc\fR. The address of the memory block -is returned via \fIlistptr\fR, which is also the start of the list of string -pointers. The end of the list is marked by a NULL pointer. The yield of the -function is zero if all went well, or - - PCRE_ERROR_NOMEMORY (-6) - -if the attempt to get the memory block failed. - -When any of these functions encounter a substring that is unset, which can -happen when capturing subpattern number \fIn+1\fR matches some part of the -subject, but subpattern \fIn\fR has not been used at all, they return an empty -string. This can be distinguished from a genuine zero-length substring by -inspecting the appropriate offset in \fIovector\fR, which is negative for unset -substrings. - -The two convenience functions \fBpcre_free_substring()\fR and -\fBpcre_free_substring_list()\fR can be used to free the memory returned by -a previous call of \fBpcre_get_substring()\fR or -\fBpcre_get_substring_list()\fR, respectively. They do nothing more than call -the function pointed to by \fBpcre_free\fR, which of course could be called -directly from a C program. However, PCRE is used in some situations where it is -linked via a special interface to another programming language which cannot use -\fBpcre_free\fR directly; it is for these cases that the functions are -provided. - - -.SH LIMITATIONS -There are some size limitations in PCRE but it is hoped that they will never in -practice be relevant. -The maximum length of a compiled pattern is 65539 (sic) bytes. -All values in repeating quantifiers must be less than 65536. -There maximum number of capturing subpatterns is 65535. -There is no limit to the number of non-capturing subpatterns, but the maximum -depth of nesting of all kinds of parenthesized subpattern, including capturing -subpatterns, assertions, and other types of subpattern, is 200. - -The maximum length of a subject string is the largest positive number that an -integer variable can hold. However, PCRE uses recursion to handle subpatterns -and indefinite repetition. This means that the available stack space may limit -the size of a subject string that can be processed by certain patterns. - - -.SH DIFFERENCES FROM PERL -The differences described here are with respect to Perl 5.005. - -1. By default, a whitespace character is any character that the C library -function \fBisspace()\fR recognizes, though it is possible to compile PCRE with -alternative character type tables. Normally \fBisspace()\fR matches space, -formfeed, newline, carriage return, horizontal tab, and vertical tab. Perl 5 -no longer includes vertical tab in its set of whitespace characters. The \\v -escape that was in the Perl documentation for a long time was never in fact -recognized. However, the character itself was treated as whitespace at least -up to 5.002. In 5.004 and 5.005 it does not match \\s. - -2. PCRE does not allow repeat quantifiers on lookahead assertions. Perl permits -them, but they do not mean what you might think. For example, (?!a){3} does -not assert that the next three characters are not "a". It just asserts that the -next character is not "a" three times. - -3. Capturing subpatterns that occur inside negative lookahead assertions are -counted, but their entries in the offsets vector are never set. Perl sets its -numerical variables from any such patterns that are matched before the -assertion fails to match something (thereby succeeding), but only if the -negative lookahead assertion contains just one branch. - -4. Though binary zero characters are supported in the subject string, they are -not allowed in a pattern string because it is passed as a normal C string, -terminated by zero. The escape sequence "\\0" can be used in the pattern to -represent a binary zero. - -5. The following Perl escape sequences are not supported: \\l, \\u, \\L, \\U, -\\E, \\Q. In fact these are implemented by Perl's general string-handling and -are not part of its pattern matching engine. - -6. The Perl \\G assertion is not supported as it is not relevant to single -pattern matches. - -7. Fairly obviously, PCRE does not support the (?{code}) and (?p{code}) -constructions. However, there is some experimental support for recursive -patterns using the non-Perl item (?R). - -8. There are at the time of writing some oddities in Perl 5.005_02 concerned -with the settings of captured strings when part of a pattern is repeated. For -example, matching "aba" against the pattern /^(a(b)?)+$/ sets $2 to the value -"b", but matching "aabbaa" against /^(aa(bb)?)+$/ leaves $2 unset. However, if -the pattern is changed to /^(aa(b(b))?)+$/ then $2 (and $3) are set. - -In Perl 5.004 $2 is set in both cases, and that is also true of PCRE. If in the -future Perl changes to a consistent state that is different, PCRE may change to -follow. - -9. Another as yet unresolved discrepancy is that in Perl 5.005_02 the pattern -/^(a)?(?(1)a|b)+$/ matches the string "a", whereas in PCRE it does not. -However, in both Perl and PCRE /^(a)?a/ matched against "a" leaves $1 unset. - -10. PCRE provides some extensions to the Perl regular expression facilities: - -(a) Although lookbehind assertions must match fixed length strings, each -alternative branch of a lookbehind assertion can match a different length of -string. Perl 5.005 requires them all to have the same length. - -(b) If PCRE_DOLLAR_ENDONLY is set and PCRE_MULTILINE is not set, the $ meta- -character matches only at the very end of the string. - -(c) If PCRE_EXTRA is set, a backslash followed by a letter with no special -meaning is faulted. - -(d) If PCRE_UNGREEDY is set, the greediness of the repetition quantifiers is -inverted, that is, by default they are not greedy, but if followed by a -question mark they are. - -(e) PCRE_ANCHORED can be used to force a pattern to be tried only at the start -of the subject. - -(f) The PCRE_NOTBOL, PCRE_NOTEOL, and PCRE_NOTEMPTY options for -\fBpcre_exec()\fR have no Perl equivalents. - -(g) The (?R) construct allows for recursive pattern matching (Perl 5.6 can do -this using the (?p{code}) construct, which PCRE cannot of course support.) - - -.SH REGULAR EXPRESSION DETAILS -The syntax and semantics of the regular expressions supported by PCRE are -described below. Regular expressions are also described in the Perl -documentation and in a number of other books, some of which have copious -examples. Jeffrey Friedl's "Mastering Regular Expressions", published by -O'Reilly (ISBN 1-56592-257), covers them in great detail. - -The description here is intended as reference documentation. The basic -operation of PCRE is on strings of bytes. However, there is the beginnings of -some support for UTF-8 character strings. To use this support you must -configure PCRE to include it, and then call \fBpcre_compile()\fR with the -PCRE_UTF8 option. How this affects the pattern matching is described in the -final section of this document. - -A regular expression is a pattern that is matched against a subject string from -left to right. Most characters stand for themselves in a pattern, and match the -corresponding characters in the subject. As a trivial example, the pattern - - The quick brown fox - -matches a portion of a subject string that is identical to itself. The power of -regular expressions comes from the ability to include alternatives and -repetitions in the pattern. These are encoded in the pattern by the use of -\fImeta-characters\fR, which do not stand for themselves but instead are -interpreted in some special way. - -There are two different sets of meta-characters: those that are recognized -anywhere in the pattern except within square brackets, and those that are -recognized in square brackets. Outside square brackets, the meta-characters are -as follows: - - \\ general escape character with several uses - ^ assert start of subject (or line, in multiline mode) - $ assert end of subject (or line, in multiline mode) - . match any character except newline (by default) - [ start character class definition - | start of alternative branch - ( start subpattern - ) end subpattern - ? extends the meaning of ( - also 0 or 1 quantifier - also quantifier minimizer - * 0 or more quantifier - + 1 or more quantifier - { start min/max quantifier - -Part of a pattern that is in square brackets is called a "character class". In -a character class the only meta-characters are: - - \\ general escape character - ^ negate the class, but only if the first character - - indicates character range - ] terminates the character class - -The following sections describe the use of each of the meta-characters. - - -.SH BACKSLASH -The backslash character has several uses. Firstly, if it is followed by a -non-alphameric character, it takes away any special meaning that character may -have. This use of backslash as an escape character applies both inside and -outside character classes. - -For example, if you want to match a "*" character, you write "\\*" in the -pattern. This applies whether or not the following character would otherwise be -interpreted as a meta-character, so it is always safe to precede a -non-alphameric with "\\" to specify that it stands for itself. In particular, -if you want to match a backslash, you write "\\\\". - -If a pattern is compiled with the PCRE_EXTENDED option, whitespace in the -pattern (other than in a character class) and characters between a "#" outside -a character class and the next newline character are ignored. An escaping -backslash can be used to include a whitespace or "#" character as part of the -pattern. - -A second use of backslash provides a way of encoding non-printing characters -in patterns in a visible manner. There is no restriction on the appearance of -non-printing characters, apart from the binary zero that terminates a pattern, -but when a pattern is being prepared by text editing, it is usually easier to -use one of the following escape sequences than the binary character it -represents: - - \\a alarm, that is, the BEL character (hex 07) - \\cx "control-x", where x is any character - \\e escape (hex 1B) - \\f formfeed (hex 0C) - \\n newline (hex 0A) - \\r carriage return (hex 0D) - \\t tab (hex 09) - \\xhh character with hex code hh - \\ddd character with octal code ddd, or backreference - -The precise effect of "\\cx" is as follows: if "x" is a lower case letter, it -is converted to upper case. Then bit 6 of the character (hex 40) is inverted. -Thus "\\cz" becomes hex 1A, but "\\c{" becomes hex 3B, while "\\c;" becomes hex -7B. - -After "\\x", up to two hexadecimal digits are read (letters can be in upper or -lower case). - -After "\\0" up to two further octal digits are read. In both cases, if there -are fewer than two digits, just those that are present are used. Thus the -sequence "\\0\\x\\07" specifies two binary zeros followed by a BEL character. -Make sure you supply two digits after the initial zero if the character that -follows is itself an octal digit. - -The handling of a backslash followed by a digit other than 0 is complicated. -Outside a character class, PCRE reads it and any following digits as a decimal -number. If the number is less than 10, or if there have been at least that many -previous capturing left parentheses in the expression, the entire sequence is -taken as a \fIback reference\fR. A description of how this works is given -later, following the discussion of parenthesized subpatterns. - -Inside a character class, or if the decimal number is greater than 9 and there -have not been that many capturing subpatterns, PCRE re-reads up to three octal -digits following the backslash, and generates a single byte from the least -significant 8 bits of the value. Any subsequent digits stand for themselves. -For example: - - \\040 is another way of writing a space - \\40 is the same, provided there are fewer than 40 - previous capturing subpatterns - \\7 is always a back reference - \\11 might be a back reference, or another way of - writing a tab - \\011 is always a tab - \\0113 is a tab followed by the character "3" - \\113 is the character with octal code 113 (since there - can be no more than 99 back references) - \\377 is a byte consisting entirely of 1 bits - \\81 is either a back reference, or a binary zero - followed by the two characters "8" and "1" - -Note that octal values of 100 or greater must not be introduced by a leading -zero, because no more than three octal digits are ever read. - -All the sequences that define a single byte value can be used both inside and -outside character classes. In addition, inside a character class, the sequence -"\\b" is interpreted as the backspace character (hex 08). Outside a character -class it has a different meaning (see below). - -The third use of backslash is for specifying generic character types: - - \\d any decimal digit - \\D any character that is not a decimal digit - \\s any whitespace character - \\S any character that is not a whitespace character - \\w any "word" character - \\W any "non-word" character - -Each pair of escape sequences partitions the complete set of characters into -two disjoint sets. Any given character matches one, and only one, of each pair. - -A "word" character is any letter or digit or the underscore character, that is, -any character which can be part of a Perl "word". The definition of letters and -digits is controlled by PCRE's character tables, and may vary if locale- -specific matching is taking place (see "Locale support" above). For example, in -the "fr" (French) locale, some character codes greater than 128 are used for -accented letters, and these are matched by \\w. - -These character type sequences can appear both inside and outside character -classes. They each match one character of the appropriate type. If the current -matching point is at the end of the subject string, all of them fail, since -there is no character to match. - -The fourth use of backslash is for certain simple assertions. An assertion -specifies a condition that has to be met at a particular point in a match, -without consuming any characters from the subject string. The use of -subpatterns for more complicated assertions is described below. The backslashed -assertions are - - \\b word boundary - \\B not a word boundary - \\A start of subject (independent of multiline mode) - \\Z end of subject or newline at end (independent of multiline mode) - \\z end of subject (independent of multiline mode) - -These assertions may not appear in character classes (but note that "\\b" has a -different meaning, namely the backspace character, inside a character class). - -A word boundary is a position in the subject string where the current character -and the previous character do not both match \\w or \\W (i.e. one matches -\\w and the other matches \\W), or the start or end of the string if the -first or last character matches \\w, respectively. - -The \\A, \\Z, and \\z assertions differ from the traditional circumflex and -dollar (described below) in that they only ever match at the very start and end -of the subject string, whatever options are set. They are not affected by the -PCRE_NOTBOL or PCRE_NOTEOL options. If the \fIstartoffset\fR argument of -\fBpcre_exec()\fR is non-zero, \\A can never match. The difference between \\Z -and \\z is that \\Z matches before a newline that is the last character of the -string as well as at the end of the string, whereas \\z matches only at the -end. - - -.SH CIRCUMFLEX AND DOLLAR -Outside a character class, in the default matching mode, the circumflex -character is an assertion which is true only if the current matching point is -at the start of the subject string. If the \fIstartoffset\fR argument of -\fBpcre_exec()\fR is non-zero, circumflex can never match. Inside a character -class, circumflex has an entirely different meaning (see below). - -Circumflex need not be the first character of the pattern if a number of -alternatives are involved, but it should be the first thing in each alternative -in which it appears if the pattern is ever to match that branch. If all -possible alternatives start with a circumflex, that is, if the pattern is -constrained to match only at the start of the subject, it is said to be an -"anchored" pattern. (There are also other constructs that can cause a pattern -to be anchored.) - -A dollar character is an assertion which is true only if the current matching -point is at the end of the subject string, or immediately before a newline -character that is the last character in the string (by default). Dollar need -not be the last character of the pattern if a number of alternatives are -involved, but it should be the last item in any branch in which it appears. -Dollar has no special meaning in a character class. - -The meaning of dollar can be changed so that it matches only at the very end of -the string, by setting the PCRE_DOLLAR_ENDONLY option at compile or matching -time. This does not affect the \\Z assertion. - -The meanings of the circumflex and dollar characters are changed if the -PCRE_MULTILINE option is set. When this is the case, they match immediately -after and immediately before an internal "\\n" character, respectively, in -addition to matching at the start and end of the subject string. For example, -the pattern /^abc$/ matches the subject string "def\\nabc" in multiline mode, -but not otherwise. Consequently, patterns that are anchored in single line mode -because all branches start with "^" are not anchored in multiline mode, and a -match for circumflex is possible when the \fIstartoffset\fR argument of -\fBpcre_exec()\fR is non-zero. The PCRE_DOLLAR_ENDONLY option is ignored if -PCRE_MULTILINE is set. - -Note that the sequences \\A, \\Z, and \\z can be used to match the start and -end of the subject in both modes, and if all branches of a pattern start with -\\A it is always anchored, whether PCRE_MULTILINE is set or not. - - -.SH FULL STOP (PERIOD, DOT) -Outside a character class, a dot in the pattern matches any one character in -the subject, including a non-printing character, but not (by default) newline. -If the PCRE_DOTALL option is set, dots match newlines as well. The handling of -dot is entirely independent of the handling of circumflex and dollar, the only -relationship being that they both involve newline characters. Dot has no -special meaning in a character class. - - -.SH SQUARE BRACKETS -An opening square bracket introduces a character class, terminated by a closing -square bracket. A closing square bracket on its own is not special. If a -closing square bracket is required as a member of the class, it should be the -first data character in the class (after an initial circumflex, if present) or -escaped with a backslash. - -A character class matches a single character in the subject; the character must -be in the set of characters defined by the class, unless the first character in -the class is a circumflex, in which case the subject character must not be in -the set defined by the class. If a circumflex is actually required as a member -of the class, ensure it is not the first character, or escape it with a -backslash. - -For example, the character class [aeiou] matches any lower case vowel, while -[^aeiou] matches any character that is not a lower case vowel. Note that a -circumflex is just a convenient notation for specifying the characters which -are in the class by enumerating those that are not. It is not an assertion: it -still consumes a character from the subject string, and fails if the current -pointer is at the end of the string. - -When caseless matching is set, any letters in a class represent both their -upper case and lower case versions, so for example, a caseless [aeiou] matches -"A" as well as "a", and a caseless [^aeiou] does not match "A", whereas a -caseful version would. - -The newline character is never treated in any special way in character classes, -whatever the setting of the PCRE_DOTALL or PCRE_MULTILINE options is. A class -such as [^a] will always match a newline. - -The minus (hyphen) character can be used to specify a range of characters in a -character class. For example, [d-m] matches any letter between d and m, -inclusive. If a minus character is required in a class, it must be escaped with -a backslash or appear in a position where it cannot be interpreted as -indicating a range, typically as the first or last character in the class. - -It is not possible to have the literal character "]" as the end character of a -range. A pattern such as [W-]46] is interpreted as a class of two characters -("W" and "-") followed by a literal string "46]", so it would match "W46]" or -"-46]". However, if the "]" is escaped with a backslash it is interpreted as -the end of range, so [W-\\]46] is interpreted as a single class containing a -range followed by two separate characters. The octal or hexadecimal -representation of "]" can also be used to end a range. - -Ranges operate in ASCII collating sequence. They can also be used for -characters specified numerically, for example [\\000-\\037]. If a range that -includes letters is used when caseless matching is set, it matches the letters -in either case. For example, [W-c] is equivalent to [][\\^_`wxyzabc], matched -caselessly, and if character tables for the "fr" locale are in use, -[\\xc8-\\xcb] matches accented E characters in both cases. - -The character types \\d, \\D, \\s, \\S, \\w, and \\W may also appear in a -character class, and add the characters that they match to the class. For -example, [\\dABCDEF] matches any hexadecimal digit. A circumflex can -conveniently be used with the upper case character types to specify a more -restricted set of characters than the matching lower case type. For example, -the class [^\\W_] matches any letter or digit, but not underscore. - -All non-alphameric characters other than \\, -, ^ (at the start) and the -terminating ] are non-special in character classes, but it does no harm if they -are escaped. - - -.SH POSIX CHARACTER CLASSES -Perl 5.6 (not yet released at the time of writing) is going to support the -POSIX notation for character classes, which uses names enclosed by [: and :] -within the enclosing square brackets. PCRE supports this notation. For example, - - [01[:alpha:]%] - -matches "0", "1", any alphabetic character, or "%". The supported class names -are - - alnum letters and digits - alpha letters - ascii character codes 0 - 127 - cntrl control characters - digit decimal digits (same as \\d) - graph printing characters, excluding space - lower lower case letters - print printing characters, including space - punct printing characters, excluding letters and digits - space white space (same as \\s) - upper upper case letters - word "word" characters (same as \\w) - xdigit hexadecimal digits - ->>>>>>>>>>>>Only WORD is perl. BLANK is GNU. - -The names "ascii" and "word" are Perl extensions. Another Perl extension is -negation, which is indicated by a ^ character after the colon. For example, - - [12[:^digit:]] - -matches "1", "2", or any non-digit. PCRE (and Perl) also recognize the POSIX -syntax [.ch.] and [=ch=] where "ch" is a "collating element", but these are not -supported, and an error is given if they are encountered. - - -.SH VERTICAL BAR -Vertical bar characters are used to separate alternative patterns. For example, -the pattern - - gilbert|sullivan - -matches either "gilbert" or "sullivan". Any number of alternatives may appear, -and an empty alternative is permitted (matching the empty string). -The matching process tries each alternative in turn, from left to right, -and the first one that succeeds is used. If the alternatives are within a -subpattern (defined below), "succeeds" means matching the rest of the main -pattern as well as the alternative in the subpattern. - - -.SH INTERNAL OPTION SETTING -The settings of PCRE_CASELESS, PCRE_MULTILINE, PCRE_DOTALL, and PCRE_EXTENDED -can be changed from within the pattern by a sequence of Perl option letters -enclosed between "(?" and ")". The option letters are - - i for PCRE_CASELESS - m for PCRE_MULTILINE - s for PCRE_DOTALL - x for PCRE_EXTENDED - -For example, (?im) sets caseless, multiline matching. It is also possible to -unset these options by preceding the letter with a hyphen, and a combined -setting and unsetting such as (?im-sx), which sets PCRE_CASELESS and -PCRE_MULTILINE while unsetting PCRE_DOTALL and PCRE_EXTENDED, is also -permitted. If a letter appears both before and after the hyphen, the option is -unset. - -The scope of these option changes depends on where in the pattern the setting -occurs. For settings that are outside any subpattern (defined below), the -effect is the same as if the options were set or unset at the start of -matching. The following patterns all behave in exactly the same way: - - (?i)abc - a(?i)bc - ab(?i)c - abc(?i) - -which in turn is the same as compiling the pattern abc with PCRE_CASELESS set. -In other words, such "top level" settings apply to the whole pattern (unless -there are other changes inside subpatterns). If there is more than one setting -of the same option at top level, the rightmost setting is used. - -If an option change occurs inside a subpattern, the effect is different. This -is a change of behaviour in Perl 5.005. An option change inside a subpattern -affects only that part of the subpattern that follows it, so - - (a(?i)b)c - -matches abc and aBc and no other strings (assuming PCRE_CASELESS is not used). -By this means, options can be made to have different settings in different -parts of the pattern. Any changes made in one alternative do carry on -into subsequent branches within the same subpattern. For example, - - (a(?i)b|c) - -matches "ab", "aB", "c", and "C", even though when matching "C" the first -branch is abandoned before the option setting. This is because the effects of -option settings happen at compile time. There would be some very weird -behaviour otherwise. - -The PCRE-specific options PCRE_UNGREEDY and PCRE_EXTRA can be changed in the -same way as the Perl-compatible options by using the characters U and X -respectively. The (?X) flag setting is special in that it must always occur -earlier in the pattern than any of the additional features it turns on, even -when it is at top level. It is best put at the start. - - -.SH SUBPATTERNS -Subpatterns are delimited by parentheses (round brackets), which can be nested. -Marking part of a pattern as a subpattern does two things: - -1. It localizes a set of alternatives. For example, the pattern - - cat(aract|erpillar|) - -matches one of the words "cat", "cataract", or "caterpillar". Without the -parentheses, it would match "cataract", "erpillar" or the empty string. - -2. It sets up the subpattern as a capturing subpattern (as defined above). -When the whole pattern matches, that portion of the subject string that matched -the subpattern is passed back to the caller via the \fIovector\fR argument of -\fBpcre_exec()\fR. Opening parentheses are counted from left to right (starting -from 1) to obtain the numbers of the capturing subpatterns. - -For example, if the string "the red king" is matched against the pattern - - the ((red|white) (king|queen)) - -the captured substrings are "red king", "red", and "king", and are numbered 1, -2, and 3, respectively. - -The fact that plain parentheses fulfil two functions is not always helpful. -There are often times when a grouping subpattern is required without a -capturing requirement. If an opening parenthesis is followed by "?:", the -subpattern does not do any capturing, and is not counted when computing the -number of any subsequent capturing subpatterns. For example, if the string "the -white queen" is matched against the pattern - - the ((?:red|white) (king|queen)) - -the captured substrings are "white queen" and "queen", and are numbered 1 and -2. The maximum number of captured substrings is 99, and the maximum number of -all subpatterns, both capturing and non-capturing, is 200. - -As a convenient shorthand, if any option settings are required at the start of -a non-capturing subpattern, the option letters may appear between the "?" and -the ":". Thus the two patterns - - (?i:saturday|sunday) - (?:(?i)saturday|sunday) - -match exactly the same set of strings. Because alternative branches are tried -from left to right, and options are not reset until the end of the subpattern -is reached, an option setting in one branch does affect subsequent branches, so -the above patterns match "SUNDAY" as well as "Saturday". - - -.SH REPETITION -Repetition is specified by quantifiers, which can follow any of the following -items: - - a single character, possibly escaped - the . metacharacter - a character class - a back reference (see next section) - a parenthesized subpattern (unless it is an assertion - see below) - -The general repetition quantifier specifies a minimum and maximum number of -permitted matches, by giving the two numbers in curly brackets (braces), -separated by a comma. The numbers must be less than 65536, and the first must -be less than or equal to the second. For example: - - z{2,4} - -matches "zz", "zzz", or "zzzz". A closing brace on its own is not a special -character. If the second number is omitted, but the comma is present, there is -no upper limit; if the second number and the comma are both omitted, the -quantifier specifies an exact number of required matches. Thus - - [aeiou]{3,} - -matches at least 3 successive vowels, but may match many more, while - - \\d{8} - -matches exactly 8 digits. An opening curly bracket that appears in a position -where a quantifier is not allowed, or one that does not match the syntax of a -quantifier, is taken as a literal character. For example, {,6} is not a -quantifier, but a literal string of four characters. - -The quantifier {0} is permitted, causing the expression to behave as if the -previous item and the quantifier were not present. - -For convenience (and historical compatibility) the three most common -quantifiers have single-character abbreviations: - - * is equivalent to {0,} - + is equivalent to {1,} - ? is equivalent to {0,1} - -It is possible to construct infinite loops by following a subpattern that can -match no characters with a quantifier that has no upper limit, for example: - - (a?)* - -Earlier versions of Perl and PCRE used to give an error at compile time for -such patterns. However, because there are cases where this can be useful, such -patterns are now accepted, but if any repetition of the subpattern does in fact -match no characters, the loop is forcibly broken. - -By default, the quantifiers are "greedy", that is, they match as much as -possible (up to the maximum number of permitted times), without causing the -rest of the pattern to fail. The classic example of where this gives problems -is in trying to match comments in C programs. These appear between the -sequences /* and */ and within the sequence, individual * and / characters may -appear. An attempt to match C comments by applying the pattern - - /\\*.*\\*/ - -to the string - - /* first command */ not comment /* second comment */ - -fails, because it matches the entire string owing to the greediness of the .* -item. - -However, if a quantifier is followed by a question mark, it ceases to be -greedy, and instead matches the minimum number of times possible, so the -pattern - - /\\*.*?\\*/ - -does the right thing with the C comments. The meaning of the various -quantifiers is not otherwise changed, just the preferred number of matches. -Do not confuse this use of question mark with its use as a quantifier in its -own right. Because it has two uses, it can sometimes appear doubled, as in - - \\d??\\d - -which matches one digit by preference, but can match two if that is the only -way the rest of the pattern matches. - -If the PCRE_UNGREEDY option is set (an option which is not available in Perl), -the quantifiers are not greedy by default, but individual ones can be made -greedy by following them with a question mark. In other words, it inverts the -default behaviour. - -When a parenthesized subpattern is quantified with a minimum repeat count that -is greater than 1 or with a limited maximum, more store is required for the -compiled pattern, in proportion to the size of the minimum or maximum. - -If a pattern starts with .* or .{0,} and the PCRE_DOTALL option (equivalent -to Perl's /s) is set, thus allowing the . to match newlines, the pattern is -implicitly anchored, because whatever follows will be tried against every -character position in the subject string, so there is no point in retrying the -overall match at any position after the first. PCRE treats such a pattern as -though it were preceded by \\A. In cases where it is known that the subject -string contains no newlines, it is worth setting PCRE_DOTALL when the pattern -begins with .* in order to obtain this optimization, or alternatively using ^ -to indicate anchoring explicitly. - -When a capturing subpattern is repeated, the value captured is the substring -that matched the final iteration. For example, after - - (tweedle[dume]{3}\\s*)+ - -has matched "tweedledum tweedledee" the value of the captured substring is -"tweedledee". However, if there are nested capturing subpatterns, the -corresponding captured values may have been set in previous iterations. For -example, after - - /(a|(b))+/ - -matches "aba" the value of the second captured substring is "b". - - -.SH BACK REFERENCES -Outside a character class, a backslash followed by a digit greater than 0 (and -possibly further digits) is a back reference to a capturing subpattern earlier -(i.e. to its left) in the pattern, provided there have been that many previous -capturing left parentheses. - -However, if the decimal number following the backslash is less than 10, it is -always taken as a back reference, and causes an error only if there are not -that many capturing left parentheses in the entire pattern. In other words, the -parentheses that are referenced need not be to the left of the reference for -numbers less than 10. See the section entitled "Backslash" above for further -details of the handling of digits following a backslash. - -A back reference matches whatever actually matched the capturing subpattern in -the current subject string, rather than anything matching the subpattern -itself. So the pattern - - (sens|respons)e and \\1ibility - -matches "sense and sensibility" and "response and responsibility", but not -"sense and responsibility". If caseful matching is in force at the time of the -back reference, the case of letters is relevant. For example, - - ((?i)rah)\\s+\\1 - -matches "rah rah" and "RAH RAH", but not "RAH rah", even though the original -capturing subpattern is matched caselessly. - -There may be more than one back reference to the same subpattern. If a -subpattern has not actually been used in a particular match, any back -references to it always fail. For example, the pattern - - (a|(bc))\\2 - -always fails if it starts to match "a" rather than "bc". Because there may be -up to 99 back references, all digits following the backslash are taken -as part of a potential back reference number. If the pattern continues with a -digit character, some delimiter must be used to terminate the back reference. -If the PCRE_EXTENDED option is set, this can be whitespace. Otherwise an empty -comment can be used. - -A back reference that occurs inside the parentheses to which it refers fails -when the subpattern is first used, so, for example, (a\\1) never matches. -However, such references can be useful inside repeated subpatterns. For -example, the pattern - - (a|b\\1)+ - -matches any number of "a"s and also "aba", "ababbaa" etc. At each iteration of -the subpattern, the back reference matches the character string corresponding -to the previous iteration. In order for this to work, the pattern must be such -that the first iteration does not need to match the back reference. This can be -done using alternation, as in the example above, or by a quantifier with a -minimum of zero. - - -.SH ASSERTIONS -An assertion is a test on the characters following or preceding the current -matching point that does not actually consume any characters. The simple -assertions coded as \\b, \\B, \\A, \\Z, \\z, ^ and $ are described above. More -complicated assertions are coded as subpatterns. There are two kinds: those -that look ahead of the current position in the subject string, and those that -look behind it. - -An assertion subpattern is matched in the normal way, except that it does not -cause the current matching position to be changed. Lookahead assertions start -with (?= for positive assertions and (?! for negative assertions. For example, - - \\w+(?=;) - -matches a word followed by a semicolon, but does not include the semicolon in -the match, and - - foo(?!bar) - -matches any occurrence of "foo" that is not followed by "bar". Note that the -apparently similar pattern - - (?!foo)bar - -does not find an occurrence of "bar" that is preceded by something other than -"foo"; it finds any occurrence of "bar" whatsoever, because the assertion -(?!foo) is always true when the next three characters are "bar". A -lookbehind assertion is needed to achieve this effect. - -Lookbehind assertions start with (?<= for positive assertions and (?<! for -negative assertions. For example, - - (?<!foo)bar - -does find an occurrence of "bar" that is not preceded by "foo". The contents of -a lookbehind assertion are restricted such that all the strings it matches must -have a fixed length. However, if there are several alternatives, they do not -all have to have the same fixed length. Thus - - (?<=bullock|donkey) - -is permitted, but - - (?<!dogs?|cats?) - -causes an error at compile time. Branches that match different length strings -are permitted only at the top level of a lookbehind assertion. This is an -extension compared with Perl 5.005, which requires all branches to match the -same length of string. An assertion such as - - (?<=ab(c|de)) - -is not permitted, because its single top-level branch can match two different -lengths, but it is acceptable if rewritten to use two top-level branches: - - (?<=abc|abde) - -The implementation of lookbehind assertions is, for each alternative, to -temporarily move the current position back by the fixed width and then try to -match. If there are insufficient characters before the current position, the -match is deemed to fail. Lookbehinds in conjunction with once-only subpatterns -can be particularly useful for matching at the ends of strings; an example is -given at the end of the section on once-only subpatterns. - -Several assertions (of any sort) may occur in succession. For example, - - (?<=\\d{3})(?<!999)foo - -matches "foo" preceded by three digits that are not "999". Notice that each of -the assertions is applied independently at the same point in the subject -string. First there is a check that the previous three characters are all -digits, and then there is a check that the same three characters are not "999". -This pattern does \fInot\fR match "foo" preceded by six characters, the first -of which are digits and the last three of which are not "999". For example, it -doesn't match "123abcfoo". A pattern to do that is - - (?<=\\d{3}...)(?<!999)foo - -This time the first assertion looks at the preceding six characters, checking -that the first three are digits, and then the second assertion checks that the -preceding three characters are not "999". - -Assertions can be nested in any combination. For example, - - (?<=(?<!foo)bar)baz - -matches an occurrence of "baz" that is preceded by "bar" which in turn is not -preceded by "foo", while - - (?<=\\d{3}(?!999)...)foo - -is another pattern which matches "foo" preceded by three digits and any three -characters that are not "999". - -Assertion subpatterns are not capturing subpatterns, and may not be repeated, -because it makes no sense to assert the same thing several times. If any kind -of assertion contains capturing subpatterns within it, these are counted for -the purposes of numbering the capturing subpatterns in the whole pattern. -However, substring capturing is carried out only for positive assertions, -because it does not make sense for negative assertions. - -Assertions count towards the maximum of 200 parenthesized subpatterns. - - -.SH ONCE-ONLY SUBPATTERNS -With both maximizing and minimizing repetition, failure of what follows -normally causes the repeated item to be re-evaluated to see if a different -number of repeats allows the rest of the pattern to match. Sometimes it is -useful to prevent this, either to change the nature of the match, or to cause -it fail earlier than it otherwise might, when the author of the pattern knows -there is no point in carrying on. - -Consider, for example, the pattern \\d+foo when applied to the subject line - - 123456bar - -After matching all 6 digits and then failing to match "foo", the normal -action of the matcher is to try again with only 5 digits matching the \\d+ -item, and then with 4, and so on, before ultimately failing. Once-only -subpatterns provide the means for specifying that once a portion of the pattern -has matched, it is not to be re-evaluated in this way, so the matcher would -give up immediately on failing to match "foo" the first time. The notation is -another kind of special parenthesis, starting with (?> as in this example: - - (?>\\d+)bar - -This kind of parenthesis "locks up" the part of the pattern it contains once -it has matched, and a failure further into the pattern is prevented from -backtracking into it. Backtracking past it to previous items, however, works as -normal. - -An alternative description is that a subpattern of this type matches the string -of characters that an identical standalone pattern would match, if anchored at -the current point in the subject string. - -Once-only subpatterns are not capturing subpatterns. Simple cases such as the -above example can be thought of as a maximizing repeat that must swallow -everything it can. So, while both \\d+ and \\d+? are prepared to adjust the -number of digits they match in order to make the rest of the pattern match, -(?>\\d+) can only match an entire sequence of digits. - -This construction can of course contain arbitrarily complicated subpatterns, -and it can be nested. - -Once-only subpatterns can be used in conjunction with lookbehind assertions to -specify efficient matching at the end of the subject string. Consider a simple -pattern such as - - abcd$ - -when applied to a long string which does not match. Because matching proceeds -from left to right, PCRE will look for each "a" in the subject and then see if -what follows matches the rest of the pattern. If the pattern is specified as - - ^.*abcd$ - -the initial .* matches the entire string at first, but when this fails (because -there is no following "a"), it backtracks to match all but the last character, -then all but the last two characters, and so on. Once again the search for "a" -covers the entire string, from right to left, so we are no better off. However, -if the pattern is written as - - ^(?>.*)(?<=abcd) - -there can be no backtracking for the .* item; it can match only the entire -string. The subsequent lookbehind assertion does a single test on the last four -characters. If it fails, the match fails immediately. For long strings, this -approach makes a significant difference to the processing time. - -When a pattern contains an unlimited repeat inside a subpattern that can itself -be repeated an unlimited number of times, the use of a once-only subpattern is -the only way to avoid some failing matches taking a very long time indeed. -The pattern - - (\\D+|<\\d+>)*[!?] - -matches an unlimited number of substrings that either consist of non-digits, or -digits enclosed in <>, followed by either ! or ?. When it matches, it runs -quickly. However, if it is applied to - - aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - -it takes a long time before reporting failure. This is because the string can -be divided between the two repeats in a large number of ways, and all have to -be tried. (The example used [!?] rather than a single character at the end, -because both PCRE and Perl have an optimization that allows for fast failure -when a single character is used. They remember the last single character that -is required for a match, and fail early if it is not present in the string.) -If the pattern is changed to - - ((?>\\D+)|<\\d+>)*[!?] - -sequences of non-digits cannot be broken, and failure happens quickly. - - -.SH CONDITIONAL SUBPATTERNS -It is possible to cause the matching process to obey a subpattern -conditionally or to choose between two alternative subpatterns, depending on -the result of an assertion, or whether a previous capturing subpattern matched -or not. The two possible forms of conditional subpattern are - - (?(condition)yes-pattern) - (?(condition)yes-pattern|no-pattern) - -If the condition is satisfied, the yes-pattern is used; otherwise the -no-pattern (if present) is used. If there are more than two alternatives in the -subpattern, a compile-time error occurs. - -There are two kinds of condition. If the text between the parentheses consists -of a sequence of digits, the condition is satisfied if the capturing subpattern -of that number has previously matched. The number must be greater than zero. -Consider the following pattern, which contains non-significant white space to -make it more readable (assume the PCRE_EXTENDED option) and to divide it into -three parts for ease of discussion: - - ( \\( )? [^()]+ (?(1) \\) ) - -The first part matches an optional opening parenthesis, and if that -character is present, sets it as the first captured substring. The second part -matches one or more characters that are not parentheses. The third part is a -conditional subpattern that tests whether the first set of parentheses matched -or not. If they did, that is, if subject started with an opening parenthesis, -the condition is true, and so the yes-pattern is executed and a closing -parenthesis is required. Otherwise, since no-pattern is not present, the -subpattern matches nothing. In other words, this pattern matches a sequence of -non-parentheses, optionally enclosed in parentheses. - -If the condition is not a sequence of digits, it must be an assertion. This may -be a positive or negative lookahead or lookbehind assertion. Consider this -pattern, again containing non-significant white space, and with the two -alternatives on the second line: - - (?(?=[^a-z]*[a-z]) - \\d{2}-[a-z]{3}-\\d{2} | \\d{2}-\\d{2}-\\d{2} ) - -The condition is a positive lookahead assertion that matches an optional -sequence of non-letters followed by a letter. In other words, it tests for the -presence of at least one letter in the subject. If a letter is found, the -subject is matched against the first alternative; otherwise it is matched -against the second. This pattern matches strings in one of the two forms -dd-aaa-dd or dd-dd-dd, where aaa are letters and dd are digits. - - -.SH COMMENTS -The sequence (?# marks the start of a comment which continues up to the next -closing parenthesis. Nested parentheses are not permitted. The characters -that make up a comment play no part in the pattern matching at all. - -If the PCRE_EXTENDED option is set, an unescaped # character outside a -character class introduces a comment that continues up to the next newline -character in the pattern. - - -.SH RECURSIVE PATTERNS -Consider the problem of matching a string in parentheses, allowing for -unlimited nested parentheses. Without the use of recursion, the best that can -be done is to use a pattern that matches up to some fixed depth of nesting. It -is not possible to handle an arbitrary nesting depth. Perl 5.6 has provided an -experimental facility that allows regular expressions to recurse (amongst other -things). It does this by interpolating Perl code in the expression at run time, -and the code can refer to the expression itself. A Perl pattern to solve the -parentheses problem can be created like this: - - $re = qr{\\( (?: (?>[^()]+) | (?p{$re}) )* \\)}x; - -The (?p{...}) item interpolates Perl code at run time, and in this case refers -recursively to the pattern in which it appears. Obviously, PCRE cannot support -the interpolation of Perl code. Instead, the special item (?R) is provided for -the specific case of recursion. This PCRE pattern solves the parentheses -problem (assume the PCRE_EXTENDED option is set so that white space is -ignored): - - \\( ( (?>[^()]+) | (?R) )* \\) - -First it matches an opening parenthesis. Then it matches any number of -substrings which can either be a sequence of non-parentheses, or a recursive -match of the pattern itself (i.e. a correctly parenthesized substring). Finally -there is a closing parenthesis. - -This particular example pattern contains nested unlimited repeats, and so the -use of a once-only subpattern for matching strings of non-parentheses is -important when applying the pattern to strings that do not match. For example, -when it is applied to - - (aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa() - -it yields "no match" quickly. However, if a once-only subpattern is not used, -the match runs for a very long time indeed because there are so many different -ways the + and * repeats can carve up the subject, and all have to be tested -before failure can be reported. - -The values set for any capturing subpatterns are those from the outermost level -of the recursion at which the subpattern value is set. If the pattern above is -matched against - - (ab(cd)ef) - -the value for the capturing parentheses is "ef", which is the last value taken -on at the top level. If additional parentheses are added, giving - - \\( ( ( (?>[^()]+) | (?R) )* ) \\) - ^ ^ - ^ ^ -the string they capture is "ab(cd)ef", the contents of the top level -parentheses. If there are more than 15 capturing parentheses in a pattern, PCRE -has to obtain extra memory to store data during a recursion, which it does by -using \fBpcre_malloc\fR, freeing it via \fBpcre_free\fR afterwards. If no -memory can be obtained, it saves data for the first 15 capturing parentheses -only, as there is no way to give an out-of-memory error from within a -recursion. - - -.SH PERFORMANCE -Certain items that may appear in patterns are more efficient than others. It is -more efficient to use a character class like [aeiou] than a set of alternatives -such as (a|e|i|o|u). In general, the simplest construction that provides the -required behaviour is usually the most efficient. Jeffrey Friedl's book -contains a lot of discussion about optimizing regular expressions for efficient -performance. - -When a pattern begins with .* and the PCRE_DOTALL option is set, the pattern is -implicitly anchored by PCRE, since it can match only at the start of a subject -string. However, if PCRE_DOTALL is not set, PCRE cannot make this optimization, -because the . metacharacter does not then match a newline, and if the subject -string contains newlines, the pattern may match from the character immediately -following one of them instead of from the very start. For example, the pattern - - (.*) second - -matches the subject "first\\nand second" (where \\n stands for a newline -character) with the first captured substring being "and". In order to do this, -PCRE has to retry the match starting after every newline in the subject. - -If you are using such a pattern with subject strings that do not contain -newlines, the best performance is obtained by setting PCRE_DOTALL, or starting -the pattern with ^.* to indicate explicit anchoring. That saves PCRE from -having to scan along the subject looking for a newline to restart at. - -Beware of patterns that contain nested indefinite repeats. These can take a -long time to run when applied to a string that does not match. Consider the -pattern fragment - - (a+)* - -This can match "aaaa" in 33 different ways, and this number increases very -rapidly as the string gets longer. (The * repeat can match 0, 1, 2, 3, or 4 -times, and for each of those cases other than 0, the + repeats can match -different numbers of times.) When the remainder of the pattern is such that the -entire match is going to fail, PCRE has in principle to try every possible -variation, and this can take an extremely long time. - -An optimization catches some of the more simple cases such as - - (a+)*b - -where a literal character follows. Before embarking on the standard matching -procedure, PCRE checks that there is a "b" later in the subject string, and if -there is not, it fails the match immediately. However, when there is no -following literal this optimization cannot be used. You can see the difference -by comparing the behaviour of - - (a+)*\\d - -with the pattern above. The former gives a failure almost instantly when -applied to a whole line of "a" characters, whereas the latter takes an -appreciable time with strings longer than about 20 characters. - - -.SH UTF-8 SUPPORT -Starting at release 3.3, PCRE has some support for character strings encoded -in the UTF-8 format. This is incomplete, and is regarded as experimental. In -order to use it, you must configure PCRE to include UTF-8 support in the code, -and, in addition, you must call \fBpcre_compile()\fR with the PCRE_UTF8 option -flag. When you do this, both the pattern and any subject strings that are -matched against it are treated as UTF-8 strings instead of just strings of -bytes, but only in the cases that are mentioned below. - -If you compile PCRE with UTF-8 support, but do not use it at run time, the -library will be a bit bigger, but the additional run time overhead is limited -to testing the PCRE_UTF8 flag in several places, so should not be very large. - -PCRE assumes that the strings it is given contain valid UTF-8 codes. It does -not diagnose invalid UTF-8 strings. If you pass invalid UTF-8 strings to PCRE, -the results are undefined. - -Running with PCRE_UTF8 set causes these changes in the way PCRE works: - -1. In a pattern, the escape sequence \\x{...}, where the contents of the braces -is a string of hexadecimal digits, is interpreted as a UTF-8 character whose -code number is the given hexadecimal number, for example: \\x{1234}. This -inserts from one to six literal bytes into the pattern, using the UTF-8 -encoding. If a non-hexadecimal digit appears between the braces, the item is -not recognized. - -2. The original hexadecimal escape sequence, \\xhh, generates a two-byte UTF-8 -character if its value is greater than 127. - -3. Repeat quantifiers are NOT correctly handled if they follow a multibyte -character. For example, \\x{100}* and \\xc3+ do not work. If you want to -repeat such characters, you must enclose them in non-capturing parentheses, -for example (?:\\x{100}), at present. - -4. The dot metacharacter matches one UTF-8 character instead of a single byte. - -5. Unlike literal UTF-8 characters, the dot metacharacter followed by a -repeat quantifier does operate correctly on UTF-8 characters instead of -single bytes. - -4. Although the \\x{...} escape is permitted in a character class, characters -whose values are greater than 255 cannot be included in a class. - -5. A class is matched against a UTF-8 character instead of just a single byte, -but it can match only characters whose values are less than 256. Characters -with greater values always fail to match a class. - -6. Repeated classes work correctly on multiple characters. - -7. Classes containing just a single character whose value is greater than 127 -(but less than 256), for example, [\\x80] or [^\\x{93}], do not work because -these are optimized into single byte matches. In the first case, of course, -the class brackets are just redundant. - -8. Lookbehind assertions move backwards in the subject by a fixed number of -characters instead of a fixed number of bytes. Simple cases have been tested -to work correctly, but there may be hidden gotchas herein. - -9. The character types such as \\d and \\w do not work correctly with UTF-8 -characters. They continue to test a single byte. - -10. Anything not explicitly mentioned here continues to work in bytes rather -than in characters. - -The following UTF-8 features of Perl 5.6 are not implemented: - -1. The escape sequence \\C to match a single byte. - -2. The use of Unicode tables and properties and escapes \\p, \\P, and \\X. - - -.SH SAMPLE PROGRAM -The code below is a simple, complete demonstration program, to get you started -with using PCRE. This code is also supplied in the file \fIpcredemo.c\fR in the -PCRE distribution. - -The program compiles the regular expression that is its first argument, and -matches it against the subject string in its second argument. No options are -set, and default character tables are used. If matching succeeds, the program -outputs the portion of the subject that matched, together with the contents of -any captured substrings. - -On a Unix system that has PCRE installed in \fI/usr/local\fR, you can compile -the demonstration program using a command like this: - - gcc -o pcredemo pcredemo.c -I/usr/local/include -L/usr/local/lib -lpcre - -Then you can run simple tests like this: - - ./pcredemo 'cat|dog' 'the cat sat on the mat' - -Note that there is a much more comprehensive test program, called -\fBpcretest\fR, which supports many more facilities for testing regular -expressions. The \fBpcredemo\fR program is provided as a simple coding example. - -On some operating systems (e.g. Solaris) you may get an error like this when -you try to run \fBpcredemo\fR: - - ld.so.1: a.out: fatal: libpcre.so.0: open failed: No such file or directory - -This is caused by the way shared library support works on those systems. You -need to add - - -R/usr/local/lib - -to the compile command to get round this problem. Here's the code: - - #include <stdio.h> - #include <string.h> - #include <pcre.h> - - #define OVECCOUNT 30 /* should be a multiple of 3 */ - - int main(int argc, char **argv) - { - pcre *re; - const char *error; - int erroffset; - int ovector[OVECCOUNT]; - int rc, i; - - if (argc != 3) - { - printf("Two arguments required: a regex and a " - "subject string\\n"); - return 1; - } - - /* Compile the regular expression in the first argument */ - - re = pcre_compile( - argv[1], /* the pattern */ - 0, /* default options */ - &error, /* for error message */ - &erroffset, /* for error offset */ - NULL); /* use default character tables */ - - /* Compilation failed: print the error message and exit */ - - if (re == NULL) - { - printf("PCRE compilation failed at offset %d: %s\\n", - erroffset, error); - return 1; - } - - /* Compilation succeeded: match the subject in the second - argument */ - - rc = pcre_exec( - re, /* the compiled pattern */ - NULL, /* we didn't study the pattern */ - argv[2], /* the subject string */ - (int)strlen(argv[2]), /* the length of the subject */ - 0, /* start at offset 0 in the subject */ - 0, /* default options */ - ovector, /* vector for substring information */ - OVECCOUNT); /* number of elements in the vector */ - - /* Matching failed: handle error cases */ - - if (rc < 0) - { - switch(rc) - { - case PCRE_ERROR_NOMATCH: printf("No match\\n"); break; - /* - Handle other special cases if you like - */ - default: printf("Matching error %d\\n", rc); break; - } - return 1; - } - - /* Match succeded */ - - printf("Match succeeded\\n"); - - /* The output vector wasn't big enough */ - - if (rc == 0) - { - rc = OVECCOUNT/3; - printf("ovector only has room for %d captured " - substrings\\n", rc - 1); - } - - /* Show substrings stored in the output vector */ - - for (i = 0; i < rc; i++) - { - char *substring_start = argv[2] + ovector[2*i]; - int substring_length = ovector[2*i+1] - ovector[2*i]; - printf("%2d: %.*s\\n", i, substring_length, - substring_start); - } - - return 0; - } - - -.SH AUTHOR -Philip Hazel <ph10@cam.ac.uk> -.br -University Computing Service, -.br -New Museums Site, -.br -Cambridge CB2 3QG, England. -.br -Phone: +44 1223 334714 - -Last updated: 15 August 2001 -.br -Copyright (c) 1997-2001 University of Cambridge. diff --git a/ext/pcre/pcrelib/doc/pcre.html b/ext/pcre/pcrelib/doc/pcre.html deleted file mode 100644 index d3d7b738d2..0000000000 --- a/ext/pcre/pcrelib/doc/pcre.html +++ /dev/null @@ -1,2672 +0,0 @@ -<HTML> -<HEAD> -<TITLE>pcre specification</TITLE> -</HEAD> -<body bgcolor="#FFFFFF" text="#00005A"> -<H1>pcre specification</H1> -This HTML document has been generated automatically from the original man page. -If there is any nonsense in it, please consult the man page in case the -conversion went wrong. -<UL> -<LI><A NAME="TOC1" HREF="#SEC1">NAME</A> -<LI><A NAME="TOC2" HREF="#SEC2">SYNOPSIS</A> -<LI><A NAME="TOC3" HREF="#SEC3">DESCRIPTION</A> -<LI><A NAME="TOC4" HREF="#SEC4">MULTI-THREADING</A> -<LI><A NAME="TOC5" HREF="#SEC5">COMPILING A PATTERN</A> -<LI><A NAME="TOC6" HREF="#SEC6">STUDYING A PATTERN</A> -<LI><A NAME="TOC7" HREF="#SEC7">LOCALE SUPPORT</A> -<LI><A NAME="TOC8" HREF="#SEC8">INFORMATION ABOUT A PATTERN</A> -<LI><A NAME="TOC9" HREF="#SEC9">MATCHING A PATTERN</A> -<LI><A NAME="TOC10" HREF="#SEC10">EXTRACTING CAPTURED SUBSTRINGS</A> -<LI><A NAME="TOC11" HREF="#SEC11">LIMITATIONS</A> -<LI><A NAME="TOC12" HREF="#SEC12">DIFFERENCES FROM PERL</A> -<LI><A NAME="TOC13" HREF="#SEC13">REGULAR EXPRESSION DETAILS</A> -<LI><A NAME="TOC14" HREF="#SEC14">BACKSLASH</A> -<LI><A NAME="TOC15" HREF="#SEC15">CIRCUMFLEX AND DOLLAR</A> -<LI><A NAME="TOC16" HREF="#SEC16">FULL STOP (PERIOD, DOT)</A> -<LI><A NAME="TOC17" HREF="#SEC17">SQUARE BRACKETS</A> -<LI><A NAME="TOC18" HREF="#SEC18">POSIX CHARACTER CLASSES</A> -<LI><A NAME="TOC19" HREF="#SEC19">VERTICAL BAR</A> -<LI><A NAME="TOC20" HREF="#SEC20">INTERNAL OPTION SETTING</A> -<LI><A NAME="TOC21" HREF="#SEC21">SUBPATTERNS</A> -<LI><A NAME="TOC22" HREF="#SEC22">REPETITION</A> -<LI><A NAME="TOC23" HREF="#SEC23">BACK REFERENCES</A> -<LI><A NAME="TOC24" HREF="#SEC24">ASSERTIONS</A> -<LI><A NAME="TOC25" HREF="#SEC25">ONCE-ONLY SUBPATTERNS</A> -<LI><A NAME="TOC26" HREF="#SEC26">CONDITIONAL SUBPATTERNS</A> -<LI><A NAME="TOC27" HREF="#SEC27">COMMENTS</A> -<LI><A NAME="TOC28" HREF="#SEC28">RECURSIVE PATTERNS</A> -<LI><A NAME="TOC29" HREF="#SEC29">PERFORMANCE</A> -<LI><A NAME="TOC30" HREF="#SEC30">UTF-8 SUPPORT</A> -<LI><A NAME="TOC31" HREF="#SEC31">SAMPLE PROGRAM</A> -<LI><A NAME="TOC32" HREF="#SEC32">AUTHOR</A> -</UL> -<LI><A NAME="SEC1" HREF="#TOC1">NAME</A> -<P> -pcre - Perl-compatible regular expressions. -</P> -<LI><A NAME="SEC2" HREF="#TOC1">SYNOPSIS</A> -<P> -<B>#include <pcre.h></B> -</P> -<P> -<B>pcre *pcre_compile(const char *<I>pattern</I>, int <I>options</I>,</B> -<B>const char **<I>errptr</I>, int *<I>erroffset</I>,</B> -<B>const unsigned char *<I>tableptr</I>);</B> -</P> -<P> -<B>pcre_extra *pcre_study(const pcre *<I>code</I>, int <I>options</I>,</B> -<B>const char **<I>errptr</I>);</B> -</P> -<P> -<B>int pcre_exec(const pcre *<I>code</I>, const pcre_extra *<I>extra</I>,</B> -<B>const char *<I>subject</I>, int <I>length</I>, int <I>startoffset</I>,</B> -<B>int <I>options</I>, int *<I>ovector</I>, int <I>ovecsize</I>);</B> -</P> -<P> -<B>int pcre_copy_substring(const char *<I>subject</I>, int *<I>ovector</I>,</B> -<B>int <I>stringcount</I>, int <I>stringnumber</I>, char *<I>buffer</I>,</B> -<B>int <I>buffersize</I>);</B> -</P> -<P> -<B>int pcre_get_substring(const char *<I>subject</I>, int *<I>ovector</I>,</B> -<B>int <I>stringcount</I>, int <I>stringnumber</I>,</B> -<B>const char **<I>stringptr</I>);</B> -</P> -<P> -<B>int pcre_get_substring_list(const char *<I>subject</I>,</B> -<B>int *<I>ovector</I>, int <I>stringcount</I>, const char ***<I>listptr</I>);</B> -</P> -<P> -<B>void pcre_free_substring(const char *<I>stringptr</I>);</B> -</P> -<P> -<B>void pcre_free_substring_list(const char **<I>stringptr</I>);</B> -</P> -<P> -<B>const unsigned char *pcre_maketables(void);</B> -</P> -<P> -<B>int pcre_fullinfo(const pcre *<I>code</I>, const pcre_extra *<I>extra</I>,</B> -<B>int <I>what</I>, void *<I>where</I>);</B> -</P> -<P> -<B>int pcre_info(const pcre *<I>code</I>, int *<I>optptr</I>, int</B> -<B>*<I>firstcharptr</I>);</B> -</P> -<P> -<B>char *pcre_version(void);</B> -</P> -<P> -<B>void *(*pcre_malloc)(size_t);</B> -</P> -<P> -<B>void (*pcre_free)(void *);</B> -</P> -<LI><A NAME="SEC3" HREF="#TOC1">DESCRIPTION</A> -<P> -The PCRE library is a set of functions that implement regular expression -pattern matching using the same syntax and semantics as Perl 5, with just a few -differences (see below). The current implementation corresponds to Perl 5.005, -with some additional features from later versions. This includes some -experimental, incomplete support for UTF-8 encoded strings. Details of exactly -what is and what is not supported are given below. -</P> -<P> -PCRE has its own native API, which is described in this document. There is also -a set of wrapper functions that correspond to the POSIX regular expression API. -These are described in the <B>pcreposix</B> documentation. -</P> -<P> -The native API function prototypes are defined in the header file <B>pcre.h</B>, -and on Unix systems the library itself is called <B>libpcre.a</B>, so can be -accessed by adding <B>-lpcre</B> to the command for linking an application which -calls it. The header file defines the macros PCRE_MAJOR and PCRE_MINOR to -contain the major and minor release numbers for the library. Applications can -use these to include support for different releases. -</P> -<P> -The functions <B>pcre_compile()</B>, <B>pcre_study()</B>, and <B>pcre_exec()</B> -are used for compiling and matching regular expressions. A sample program that -demonstrates the simplest way of using them is given in the file -<I>pcredemo.c</I>. The last section of this man page describes how to run it. -</P> -<P> -The functions <B>pcre_copy_substring()</B>, <B>pcre_get_substring()</B>, and -<B>pcre_get_substring_list()</B> are convenience functions for extracting -captured substrings from a matched subject string; <B>pcre_free_substring()</B> -and <B>pcre_free_substring_list()</B> are also provided, to free the memory used -for extracted strings. -</P> -<P> -The function <B>pcre_maketables()</B> is used (optionally) to build a set of -character tables in the current locale for passing to <B>pcre_compile()</B>. -</P> -<P> -The function <B>pcre_fullinfo()</B> is used to find out information about a -compiled pattern; <B>pcre_info()</B> is an obsolete version which returns only -some of the available information, but is retained for backwards compatibility. -The function <B>pcre_version()</B> returns a pointer to a string containing the -version of PCRE and its date of release. -</P> -<P> -The global variables <B>pcre_malloc</B> and <B>pcre_free</B> initially contain -the entry points of the standard <B>malloc()</B> and <B>free()</B> functions -respectively. PCRE calls the memory management functions via these variables, -so a calling program can replace them if it wishes to intercept the calls. This -should be done before calling any PCRE functions. -</P> -<LI><A NAME="SEC4" HREF="#TOC1">MULTI-THREADING</A> -<P> -The PCRE functions can be used in multi-threading applications, with the -proviso that the memory management functions pointed to by <B>pcre_malloc</B> -and <B>pcre_free</B> are shared by all threads. -</P> -<P> -The compiled form of a regular expression is not altered during matching, so -the same compiled pattern can safely be used by several threads at once. -</P> -<LI><A NAME="SEC5" HREF="#TOC1">COMPILING A PATTERN</A> -<P> -The function <B>pcre_compile()</B> is called to compile a pattern into an -internal form. The pattern is a C string terminated by a binary zero, and -is passed in the argument <I>pattern</I>. A pointer to a single block of memory -that is obtained via <B>pcre_malloc</B> is returned. This contains the compiled -code and related data. The <B>pcre</B> type is defined for the returned block; -this is a typedef for a structure whose contents are not externally defined. It -is up to the caller to free the memory when it is no longer required. -</P> -<P> -Although the compiled code of a PCRE regex is relocatable, that is, it does not -depend on memory location, the complete <B>pcre</B> data block is not -fully relocatable, because it contains a copy of the <I>tableptr</I> argument, -which is an address (see below). -</P> -<P> -The size of a compiled pattern is roughly proportional to the length of the -pattern string, except that each character class (other than those containing -just a single character, negated or not) requires 33 bytes, and repeat -quantifiers with a minimum greater than one or a bounded maximum cause the -relevant portions of the compiled pattern to be replicated. -</P> -<P> -The <I>options</I> argument contains independent bits that affect the -compilation. It should be zero if no options are required. Some of the options, -in particular, those that are compatible with Perl, can also be set and unset -from within the pattern (see the detailed description of regular expressions -below). For these options, the contents of the <I>options</I> argument specifies -their initial settings at the start of compilation and execution. The -PCRE_ANCHORED option can be set at the time of matching as well as at compile -time. -</P> -<P> -If <I>errptr</I> is NULL, <B>pcre_compile()</B> returns NULL immediately. -Otherwise, if compilation of a pattern fails, <B>pcre_compile()</B> returns -NULL, and sets the variable pointed to by <I>errptr</I> to point to a textual -error message. The offset from the start of the pattern to the character where -the error was discovered is placed in the variable pointed to by -<I>erroffset</I>, which must not be NULL. If it is, an immediate error is given. -</P> -<P> -If the final argument, <I>tableptr</I>, is NULL, PCRE uses a default set of -character tables which are built when it is compiled, using the default C -locale. Otherwise, <I>tableptr</I> must be the result of a call to -<B>pcre_maketables()</B>. See the section on locale support below. -</P> -<P> -This code fragment shows a typical straightforward call to <B>pcre_compile()</B>: -</P> -<P> -<PRE> - pcre *re; - const char *error; - int erroffset; - re = pcre_compile( - "^A.*Z", /* the pattern */ - 0, /* default options */ - &error, /* for error message */ - &erroffset, /* for error offset */ - NULL); /* use default character tables */ -</PRE> -</P> -<P> -The following option bits are defined in the header file: -</P> -<P> -<PRE> - PCRE_ANCHORED -</PRE> -</P> -<P> -If this bit is set, the pattern is forced to be "anchored", that is, it is -constrained to match only at the start of the string which is being searched -(the "subject string"). This effect can also be achieved by appropriate -constructs in the pattern itself, which is the only way to do it in Perl. -</P> -<P> -<PRE> - PCRE_CASELESS -</PRE> -</P> -<P> -If this bit is set, letters in the pattern match both upper and lower case -letters. It is equivalent to Perl's /i option. -</P> -<P> -<PRE> - PCRE_DOLLAR_ENDONLY -</PRE> -</P> -<P> -If this bit is set, a dollar metacharacter in the pattern matches only at the -end of the subject string. Without this option, a dollar also matches -immediately before the final character if it is a newline (but not before any -other newlines). The PCRE_DOLLAR_ENDONLY option is ignored if PCRE_MULTILINE is -set. There is no equivalent to this option in Perl. -</P> -<P> -<PRE> - PCRE_DOTALL -</PRE> -</P> -<P> -If this bit is set, a dot metacharater in the pattern matches all characters, -including newlines. Without it, newlines are excluded. This option is -equivalent to Perl's /s option. A negative class such as [^a] always matches a -newline character, independent of the setting of this option. -</P> -<P> -<PRE> - PCRE_EXTENDED -</PRE> -</P> -<P> -If this bit is set, whitespace data characters in the pattern are totally -ignored except when escaped or inside a character class, and characters between -an unescaped # outside a character class and the next newline character, -inclusive, are also ignored. This is equivalent to Perl's /x option, and makes -it possible to include comments inside complicated patterns. Note, however, -that this applies only to data characters. Whitespace characters may never -appear within special character sequences in a pattern, for example within the -sequence (?( which introduces a conditional subpattern. -</P> -<P> -<PRE> - PCRE_EXTRA -</PRE> -</P> -<P> -This option was invented in order to turn on additional functionality of PCRE -that is incompatible with Perl, but it is currently of very little use. When -set, any backslash in a pattern that is followed by a letter that has no -special meaning causes an error, thus reserving these combinations for future -expansion. By default, as in Perl, a backslash followed by a letter with no -special meaning is treated as a literal. There are at present no other features -controlled by this option. It can also be set by a (?X) option setting within a -pattern. -</P> -<P> -<PRE> - PCRE_MULTILINE -</PRE> -</P> -<P> -By default, PCRE treats the subject string as consisting of a single "line" of -characters (even if it actually contains several newlines). The "start of line" -metacharacter (^) matches only at the start of the string, while the "end of -line" metacharacter ($) matches only at the end of the string, or before a -terminating newline (unless PCRE_DOLLAR_ENDONLY is set). This is the same as -Perl. -</P> -<P> -When PCRE_MULTILINE it is set, the "start of line" and "end of line" constructs -match immediately following or immediately before any newline in the subject -string, respectively, as well as at the very start and end. This is equivalent -to Perl's /m option. If there are no "\n" characters in a subject string, or -no occurrences of ^ or $ in a pattern, setting PCRE_MULTILINE has no -effect. -</P> -<P> -<PRE> - PCRE_UNGREEDY -</PRE> -</P> -<P> -This option inverts the "greediness" of the quantifiers so that they are not -greedy by default, but become greedy if followed by "?". It is not compatible -with Perl. It can also be set by a (?U) option setting within the pattern. -</P> -<P> -<PRE> - PCRE_UTF8 -</PRE> -</P> -<P> -This option causes PCRE to regard both the pattern and the subject as strings -of UTF-8 characters instead of just byte strings. However, it is available only -if PCRE has been built to include UTF-8 support. If not, the use of this option -provokes an error. Support for UTF-8 is new, experimental, and incomplete. -Details of exactly what it entails are given below. -</P> -<LI><A NAME="SEC6" HREF="#TOC1">STUDYING A PATTERN</A> -<P> -When a pattern is going to be used several times, it is worth spending more -time analyzing it in order to speed up the time taken for matching. The -function <B>pcre_study()</B> takes a pointer to a compiled pattern as its first -argument, and returns a pointer to a <B>pcre_extra</B> block (another typedef -for a structure with hidden contents) containing additional information about -the pattern; this can be passed to <B>pcre_exec()</B>. If no additional -information is available, NULL is returned. -</P> -<P> -The second argument contains option bits. At present, no options are defined -for <B>pcre_study()</B>, and this argument should always be zero. -</P> -<P> -The third argument for <B>pcre_study()</B> is a pointer to an error message. If -studying succeeds (even if no data is returned), the variable it points to is -set to NULL. Otherwise it points to a textual error message. -</P> -<P> -This is a typical call to <B>pcre_study</B>(): -</P> -<P> -<PRE> - pcre_extra *pe; - pe = pcre_study( - re, /* result of pcre_compile() */ - 0, /* no options exist */ - &error); /* set to NULL or points to a message */ -</PRE> -</P> -<P> -At present, studying a pattern is useful only for non-anchored patterns that do -not have a single fixed starting character. A bitmap of possible starting -characters is created. -</P> -<LI><A NAME="SEC7" HREF="#TOC1">LOCALE SUPPORT</A> -<P> -PCRE handles caseless matching, and determines whether characters are letters, -digits, or whatever, by reference to a set of tables. The library contains a -default set of tables which is created in the default C locale when PCRE is -compiled. This is used when the final argument of <B>pcre_compile()</B> is NULL, -and is sufficient for many applications. -</P> -<P> -An alternative set of tables can, however, be supplied. Such tables are built -by calling the <B>pcre_maketables()</B> function, which has no arguments, in the -relevant locale. The result can then be passed to <B>pcre_compile()</B> as often -as necessary. For example, to build and use tables that are appropriate for the -French locale (where accented characters with codes greater than 128 are -treated as letters), the following code could be used: -</P> -<P> -<PRE> - setlocale(LC_CTYPE, "fr"); - tables = pcre_maketables(); - re = pcre_compile(..., tables); -</PRE> -</P> -<P> -The tables are built in memory that is obtained via <B>pcre_malloc</B>. The -pointer that is passed to <B>pcre_compile</B> is saved with the compiled -pattern, and the same tables are used via this pointer by <B>pcre_study()</B> -and <B>pcre_exec()</B>. Thus for any single pattern, compilation, studying and -matching all happen in the same locale, but different patterns can be compiled -in different locales. It is the caller's responsibility to ensure that the -memory containing the tables remains available for as long as it is needed. -</P> -<LI><A NAME="SEC8" HREF="#TOC1">INFORMATION ABOUT A PATTERN</A> -<P> -The <B>pcre_fullinfo()</B> function returns information about a compiled -pattern. It replaces the obsolete <B>pcre_info()</B> function, which is -nevertheless retained for backwards compability (and is documented below). -</P> -<P> -The first argument for <B>pcre_fullinfo()</B> is a pointer to the compiled -pattern. The second argument is the result of <B>pcre_study()</B>, or NULL if -the pattern was not studied. The third argument specifies which piece of -information is required, while the fourth argument is a pointer to a variable -to receive the data. The yield of the function is zero for success, or one of -the following negative numbers: -</P> -<P> -<PRE> - PCRE_ERROR_NULL the argument <I>code</I> was NULL - the argument <I>where</I> was NULL - PCRE_ERROR_BADMAGIC the "magic number" was not found - PCRE_ERROR_BADOPTION the value of <I>what</I> was invalid -</PRE> -</P> -<P> -Here is a typical call of <B>pcre_fullinfo()</B>, to obtain the length of the -compiled pattern: -</P> -<P> -<PRE> - int rc; - unsigned long int length; - rc = pcre_fullinfo( - re, /* result of pcre_compile() */ - pe, /* result of pcre_study(), or NULL */ - PCRE_INFO_SIZE, /* what is required */ - &length); /* where to put the data */ -</PRE> -</P> -<P> -The possible values for the third argument are defined in <B>pcre.h</B>, and are -as follows: -</P> -<P> -<PRE> - PCRE_INFO_OPTIONS -</PRE> -</P> -<P> -Return a copy of the options with which the pattern was compiled. The fourth -argument should point to an <B>unsigned long int</B> variable. These option bits -are those specified in the call to <B>pcre_compile()</B>, modified by any -top-level option settings within the pattern itself, and with the PCRE_ANCHORED -bit forcibly set if the form of the pattern implies that it can match only at -the start of a subject string. -</P> -<P> -<PRE> - PCRE_INFO_SIZE -</PRE> -</P> -<P> -Return the size of the compiled pattern, that is, the value that was passed as -the argument to <B>pcre_malloc()</B> when PCRE was getting memory in which to -place the compiled data. The fourth argument should point to a <B>size_t</B> -variable. -</P> -<P> -<PRE> - PCRE_INFO_CAPTURECOUNT -</PRE> -</P> -<P> -Return the number of capturing subpatterns in the pattern. The fourth argument -should point to an \fbint\fR variable. -</P> -<P> -<PRE> - PCRE_INFO_BACKREFMAX -</PRE> -</P> -<P> -Return the number of the highest back reference in the pattern. The fourth -argument should point to an <B>int</B> variable. Zero is returned if there are -no back references. -</P> -<P> -<PRE> - PCRE_INFO_FIRSTCHAR -</PRE> -</P> -<P> -Return information about the first character of any matched string, for a -non-anchored pattern. If there is a fixed first character, e.g. from a pattern -such as (cat|cow|coyote), it is returned in the integer pointed to by -<I>where</I>. Otherwise, if either -</P> -<P> -(a) the pattern was compiled with the PCRE_MULTILINE option, and every branch -starts with "^", or -</P> -<P> -(b) every branch of the pattern starts with ".*" and PCRE_DOTALL is not set -(if it were set, the pattern would be anchored), -</P> -<P> --1 is returned, indicating that the pattern matches only at the start of a -subject string or after any "\n" within the string. Otherwise -2 is returned. -For anchored patterns, -2 is returned. -</P> -<P> -<PRE> - PCRE_INFO_FIRSTTABLE -</PRE> -</P> -<P> -If the pattern was studied, and this resulted in the construction of a 256-bit -table indicating a fixed set of characters for the first character in any -matching string, a pointer to the table is returned. Otherwise NULL is -returned. The fourth argument should point to an <B>unsigned char *</B> -variable. -</P> -<P> -<PRE> - PCRE_INFO_LASTLITERAL -</PRE> -</P> -<P> -For a non-anchored pattern, return the value of the rightmost literal character -which must exist in any matched string, other than at its start. The fourth -argument should point to an <B>int</B> variable. If there is no such character, -or if the pattern is anchored, -1 is returned. For example, for the pattern -/a\d+z\d+/ the returned value is 'z'. -</P> -<P> -The <B>pcre_info()</B> function is now obsolete because its interface is too -restrictive to return all the available data about a compiled pattern. New -programs should use <B>pcre_fullinfo()</B> instead. The yield of -<B>pcre_info()</B> is the number of capturing subpatterns, or one of the -following negative numbers: -</P> -<P> -<PRE> - PCRE_ERROR_NULL the argument <I>code</I> was NULL - PCRE_ERROR_BADMAGIC the "magic number" was not found -</PRE> -</P> -<P> -If the <I>optptr</I> argument is not NULL, a copy of the options with which the -pattern was compiled is placed in the integer it points to (see -PCRE_INFO_OPTIONS above). -</P> -<P> -If the pattern is not anchored and the <I>firstcharptr</I> argument is not NULL, -it is used to pass back information about the first character of any matched -string (see PCRE_INFO_FIRSTCHAR above). -</P> -<LI><A NAME="SEC9" HREF="#TOC1">MATCHING A PATTERN</A> -<P> -The function <B>pcre_exec()</B> is called to match a subject string against a -pre-compiled pattern, which is passed in the <I>code</I> argument. If the -pattern has been studied, the result of the study should be passed in the -<I>extra</I> argument. Otherwise this must be NULL. -</P> -<P> -Here is an example of a simple call to <B>pcre_exec()</B>: -</P> -<P> -<PRE> - int rc; - int ovector[30]; - rc = pcre_exec( - re, /* result of pcre_compile() */ - NULL, /* we didn't study the pattern */ - "some string", /* the subject string */ - 11, /* the length of the subject string */ - 0, /* start at offset 0 in the subject */ - 0, /* default options */ - ovector, /* vector for substring information */ - 30); /* number of elements in the vector */ -</PRE> -</P> -<P> -The PCRE_ANCHORED option can be passed in the <I>options</I> argument, whose -unused bits must be zero. However, if a pattern was compiled with -PCRE_ANCHORED, or turned out to be anchored by virtue of its contents, it -cannot be made unachored at matching time. -</P> -<P> -There are also three further options that can be set only at matching time: -</P> -<P> -<PRE> - PCRE_NOTBOL -</PRE> -</P> -<P> -The first character of the string is not the beginning of a line, so the -circumflex metacharacter should not match before it. Setting this without -PCRE_MULTILINE (at compile time) causes circumflex never to match. -</P> -<P> -<PRE> - PCRE_NOTEOL -</PRE> -</P> -<P> -The end of the string is not the end of a line, so the dollar metacharacter -should not match it nor (except in multiline mode) a newline immediately before -it. Setting this without PCRE_MULTILINE (at compile time) causes dollar never -to match. -</P> -<P> -<PRE> - PCRE_NOTEMPTY -</PRE> -</P> -<P> -An empty string is not considered to be a valid match if this option is set. If -there are alternatives in the pattern, they are tried. If all the alternatives -match the empty string, the entire match fails. For example, if the pattern -</P> -<P> -<PRE> - a?b? -</PRE> -</P> -<P> -is applied to a string not beginning with "a" or "b", it matches the empty -string at the start of the subject. With PCRE_NOTEMPTY set, this match is not -valid, so PCRE searches further into the string for occurrences of "a" or "b". -</P> -<P> -Perl has no direct equivalent of PCRE_NOTEMPTY, but it does make a special case -of a pattern match of the empty string within its <B>split()</B> function, and -when using the /g modifier. It is possible to emulate Perl's behaviour after -matching a null string by first trying the match again at the same offset with -PCRE_NOTEMPTY set, and then if that fails by advancing the starting offset (see -below) and trying an ordinary match again. -</P> -<P> -The subject string is passed as a pointer in <I>subject</I>, a length in -<I>length</I>, and a starting offset in <I>startoffset</I>. Unlike the pattern -string, the subject may contain binary zero characters. When the starting -offset is zero, the search for a match starts at the beginning of the subject, -and this is by far the most common case. -</P> -<P> -A non-zero starting offset is useful when searching for another match in the -same subject by calling <B>pcre_exec()</B> again after a previous success. -Setting <I>startoffset</I> differs from just passing over a shortened string and -setting PCRE_NOTBOL in the case of a pattern that begins with any kind of -lookbehind. For example, consider the pattern -</P> -<P> -<PRE> - \Biss\B -</PRE> -</P> -<P> -which finds occurrences of "iss" in the middle of words. (\B matches only if -the current position in the subject is not a word boundary.) When applied to -the string "Mississipi" the first call to <B>pcre_exec()</B> finds the first -occurrence. If <B>pcre_exec()</B> is called again with just the remainder of the -subject, namely "issipi", it does not match, because \B is always false at the -start of the subject, which is deemed to be a word boundary. However, if -<B>pcre_exec()</B> is passed the entire string again, but with <I>startoffset</I> -set to 4, it finds the second occurrence of "iss" because it is able to look -behind the starting point to discover that it is preceded by a letter. -</P> -<P> -If a non-zero starting offset is passed when the pattern is anchored, one -attempt to match at the given offset is tried. This can only succeed if the -pattern does not require the match to be at the start of the subject. -</P> -<P> -In general, a pattern matches a certain portion of the subject, and in -addition, further substrings from the subject may be picked out by parts of the -pattern. Following the usage in Jeffrey Friedl's book, this is called -"capturing" in what follows, and the phrase "capturing subpattern" is used for -a fragment of a pattern that picks out a substring. PCRE supports several other -kinds of parenthesized subpattern that do not cause substrings to be captured. -</P> -<P> -Captured substrings are returned to the caller via a vector of integer offsets -whose address is passed in <I>ovector</I>. The number of elements in the vector -is passed in <I>ovecsize</I>. The first two-thirds of the vector is used to pass -back captured substrings, each substring using a pair of integers. The -remaining third of the vector is used as workspace by <B>pcre_exec()</B> while -matching capturing subpatterns, and is not available for passing back -information. The length passed in <I>ovecsize</I> should always be a multiple of -three. If it is not, it is rounded down. -</P> -<P> -When a match has been successful, information about captured substrings is -returned in pairs of integers, starting at the beginning of <I>ovector</I>, and -continuing up to two-thirds of its length at the most. The first element of a -pair is set to the offset of the first character in a substring, and the second -is set to the offset of the first character after the end of a substring. The -first pair, <I>ovector[0]</I> and <I>ovector[1]</I>, identify the portion of the -subject string matched by the entire pattern. The next pair is used for the -first capturing subpattern, and so on. The value returned by <B>pcre_exec()</B> -is the number of pairs that have been set. If there are no capturing -subpatterns, the return value from a successful match is 1, indicating that -just the first pair of offsets has been set. -</P> -<P> -Some convenience functions are provided for extracting the captured substrings -as separate strings. These are described in the following section. -</P> -<P> -It is possible for an capturing subpattern number <I>n+1</I> to match some -part of the subject when subpattern <I>n</I> has not been used at all. For -example, if the string "abc" is matched against the pattern (a|(z))(bc) -subpatterns 1 and 3 are matched, but 2 is not. When this happens, both offset -values corresponding to the unused subpattern are set to -1. -</P> -<P> -If a capturing subpattern is matched repeatedly, it is the last portion of the -string that it matched that gets returned. -</P> -<P> -If the vector is too small to hold all the captured substrings, it is used as -far as possible (up to two-thirds of its length), and the function returns a -value of zero. In particular, if the substring offsets are not of interest, -<B>pcre_exec()</B> may be called with <I>ovector</I> passed as NULL and -<I>ovecsize</I> as zero. However, if the pattern contains back references and -the <I>ovector</I> isn't big enough to remember the related substrings, PCRE has -to get additional memory for use during matching. Thus it is usually advisable -to supply an <I>ovector</I>. -</P> -<P> -Note that <B>pcre_info()</B> can be used to find out how many capturing -subpatterns there are in a compiled pattern. The smallest size for -<I>ovector</I> that will allow for <I>n</I> captured substrings in addition to -the offsets of the substring matched by the whole pattern is (<I>n</I>+1)*3. -</P> -<P> -If <B>pcre_exec()</B> fails, it returns a negative number. The following are -defined in the header file: -</P> -<P> -<PRE> - PCRE_ERROR_NOMATCH (-1) -</PRE> -</P> -<P> -The subject string did not match the pattern. -</P> -<P> -<PRE> - PCRE_ERROR_NULL (-2) -</PRE> -</P> -<P> -Either <I>code</I> or <I>subject</I> was passed as NULL, or <I>ovector</I> was -NULL and <I>ovecsize</I> was not zero. -</P> -<P> -<PRE> - PCRE_ERROR_BADOPTION (-3) -</PRE> -</P> -<P> -An unrecognized bit was set in the <I>options</I> argument. -</P> -<P> -<PRE> - PCRE_ERROR_BADMAGIC (-4) -</PRE> -</P> -<P> -PCRE stores a 4-byte "magic number" at the start of the compiled code, to catch -the case when it is passed a junk pointer. This is the error it gives when the -magic number isn't present. -</P> -<P> -<PRE> - PCRE_ERROR_UNKNOWN_NODE (-5) -</PRE> -</P> -<P> -While running the pattern match, an unknown item was encountered in the -compiled pattern. This error could be caused by a bug in PCRE or by overwriting -of the compiled pattern. -</P> -<P> -<PRE> - PCRE_ERROR_NOMEMORY (-6) -</PRE> -</P> -<P> -If a pattern contains back references, but the <I>ovector</I> that is passed to -<B>pcre_exec()</B> is not big enough to remember the referenced substrings, PCRE -gets a block of memory at the start of matching to use for this purpose. If the -call via <B>pcre_malloc()</B> fails, this error is given. The memory is freed at -the end of matching. -</P> -<LI><A NAME="SEC10" HREF="#TOC1">EXTRACTING CAPTURED SUBSTRINGS</A> -<P> -Captured substrings can be accessed directly by using the offsets returned by -<B>pcre_exec()</B> in <I>ovector</I>. For convenience, the functions -<B>pcre_copy_substring()</B>, <B>pcre_get_substring()</B>, and -<B>pcre_get_substring_list()</B> are provided for extracting captured substrings -as new, separate, zero-terminated strings. A substring that contains a binary -zero is correctly extracted and has a further zero added on the end, but the -result does not, of course, function as a C string. -</P> -<P> -The first three arguments are the same for all three functions: <I>subject</I> -is the subject string which has just been successfully matched, <I>ovector</I> -is a pointer to the vector of integer offsets that was passed to -<B>pcre_exec()</B>, and <I>stringcount</I> is the number of substrings that -were captured by the match, including the substring that matched the entire -regular expression. This is the value returned by <B>pcre_exec</B> if it -is greater than zero. If <B>pcre_exec()</B> returned zero, indicating that it -ran out of space in <I>ovector</I>, the value passed as <I>stringcount</I> should -be the size of the vector divided by three. -</P> -<P> -The functions <B>pcre_copy_substring()</B> and <B>pcre_get_substring()</B> -extract a single substring, whose number is given as <I>stringnumber</I>. A -value of zero extracts the substring that matched the entire pattern, while -higher values extract the captured substrings. For <B>pcre_copy_substring()</B>, -the string is placed in <I>buffer</I>, whose length is given by -<I>buffersize</I>, while for <B>pcre_get_substring()</B> a new block of memory is -obtained via <B>pcre_malloc</B>, and its address is returned via -<I>stringptr</I>. The yield of the function is the length of the string, not -including the terminating zero, or one of -</P> -<P> -<PRE> - PCRE_ERROR_NOMEMORY (-6) -</PRE> -</P> -<P> -The buffer was too small for <B>pcre_copy_substring()</B>, or the attempt to get -memory failed for <B>pcre_get_substring()</B>. -</P> -<P> -<PRE> - PCRE_ERROR_NOSUBSTRING (-7) -</PRE> -</P> -<P> -There is no substring whose number is <I>stringnumber</I>. -</P> -<P> -The <B>pcre_get_substring_list()</B> function extracts all available substrings -and builds a list of pointers to them. All this is done in a single block of -memory which is obtained via <B>pcre_malloc</B>. The address of the memory block -is returned via <I>listptr</I>, which is also the start of the list of string -pointers. The end of the list is marked by a NULL pointer. The yield of the -function is zero if all went well, or -</P> -<P> -<PRE> - PCRE_ERROR_NOMEMORY (-6) -</PRE> -</P> -<P> -if the attempt to get the memory block failed. -</P> -<P> -When any of these functions encounter a substring that is unset, which can -happen when capturing subpattern number <I>n+1</I> matches some part of the -subject, but subpattern <I>n</I> has not been used at all, they return an empty -string. This can be distinguished from a genuine zero-length substring by -inspecting the appropriate offset in <I>ovector</I>, which is negative for unset -substrings. -</P> -<P> -The two convenience functions <B>pcre_free_substring()</B> and -<B>pcre_free_substring_list()</B> can be used to free the memory returned by -a previous call of <B>pcre_get_substring()</B> or -<B>pcre_get_substring_list()</B>, respectively. They do nothing more than call -the function pointed to by <B>pcre_free</B>, which of course could be called -directly from a C program. However, PCRE is used in some situations where it is -linked via a special interface to another programming language which cannot use -<B>pcre_free</B> directly; it is for these cases that the functions are -provided. -</P> -<LI><A NAME="SEC11" HREF="#TOC1">LIMITATIONS</A> -<P> -There are some size limitations in PCRE but it is hoped that they will never in -practice be relevant. -The maximum length of a compiled pattern is 65539 (sic) bytes. -All values in repeating quantifiers must be less than 65536. -There maximum number of capturing subpatterns is 65535. -There is no limit to the number of non-capturing subpatterns, but the maximum -depth of nesting of all kinds of parenthesized subpattern, including capturing -subpatterns, assertions, and other types of subpattern, is 200. -</P> -<P> -The maximum length of a subject string is the largest positive number that an -integer variable can hold. However, PCRE uses recursion to handle subpatterns -and indefinite repetition. This means that the available stack space may limit -the size of a subject string that can be processed by certain patterns. -</P> -<LI><A NAME="SEC12" HREF="#TOC1">DIFFERENCES FROM PERL</A> -<P> -The differences described here are with respect to Perl 5.005. -</P> -<P> -1. By default, a whitespace character is any character that the C library -function <B>isspace()</B> recognizes, though it is possible to compile PCRE with -alternative character type tables. Normally <B>isspace()</B> matches space, -formfeed, newline, carriage return, horizontal tab, and vertical tab. Perl 5 -no longer includes vertical tab in its set of whitespace characters. The \v -escape that was in the Perl documentation for a long time was never in fact -recognized. However, the character itself was treated as whitespace at least -up to 5.002. In 5.004 and 5.005 it does not match \s. -</P> -<P> -2. PCRE does not allow repeat quantifiers on lookahead assertions. Perl permits -them, but they do not mean what you might think. For example, (?!a){3} does -not assert that the next three characters are not "a". It just asserts that the -next character is not "a" three times. -</P> -<P> -3. Capturing subpatterns that occur inside negative lookahead assertions are -counted, but their entries in the offsets vector are never set. Perl sets its -numerical variables from any such patterns that are matched before the -assertion fails to match something (thereby succeeding), but only if the -negative lookahead assertion contains just one branch. -</P> -<P> -4. Though binary zero characters are supported in the subject string, they are -not allowed in a pattern string because it is passed as a normal C string, -terminated by zero. The escape sequence "\0" can be used in the pattern to -represent a binary zero. -</P> -<P> -5. The following Perl escape sequences are not supported: \l, \u, \L, \U, -\E, \Q. In fact these are implemented by Perl's general string-handling and -are not part of its pattern matching engine. -</P> -<P> -6. The Perl \G assertion is not supported as it is not relevant to single -pattern matches. -</P> -<P> -7. Fairly obviously, PCRE does not support the (?{code}) and (?p{code}) -constructions. However, there is some experimental support for recursive -patterns using the non-Perl item (?R). -</P> -<P> -8. There are at the time of writing some oddities in Perl 5.005_02 concerned -with the settings of captured strings when part of a pattern is repeated. For -example, matching "aba" against the pattern /^(a(b)?)+$/ sets $2 to the value -"b", but matching "aabbaa" against /^(aa(bb)?)+$/ leaves $2 unset. However, if -the pattern is changed to /^(aa(b(b))?)+$/ then $2 (and $3) are set. -</P> -<P> -In Perl 5.004 $2 is set in both cases, and that is also true of PCRE. If in the -future Perl changes to a consistent state that is different, PCRE may change to -follow. -</P> -<P> -9. Another as yet unresolved discrepancy is that in Perl 5.005_02 the pattern -/^(a)?(?(1)a|b)+$/ matches the string "a", whereas in PCRE it does not. -However, in both Perl and PCRE /^(a)?a/ matched against "a" leaves $1 unset. -</P> -<P> -10. PCRE provides some extensions to the Perl regular expression facilities: -</P> -<P> -(a) Although lookbehind assertions must match fixed length strings, each -alternative branch of a lookbehind assertion can match a different length of -string. Perl 5.005 requires them all to have the same length. -</P> -<P> -(b) If PCRE_DOLLAR_ENDONLY is set and PCRE_MULTILINE is not set, the $ meta- -character matches only at the very end of the string. -</P> -<P> -(c) If PCRE_EXTRA is set, a backslash followed by a letter with no special -meaning is faulted. -</P> -<P> -(d) If PCRE_UNGREEDY is set, the greediness of the repetition quantifiers is -inverted, that is, by default they are not greedy, but if followed by a -question mark they are. -</P> -<P> -(e) PCRE_ANCHORED can be used to force a pattern to be tried only at the start -of the subject. -</P> -<P> -(f) The PCRE_NOTBOL, PCRE_NOTEOL, and PCRE_NOTEMPTY options for -<B>pcre_exec()</B> have no Perl equivalents. -</P> -<P> -(g) The (?R) construct allows for recursive pattern matching (Perl 5.6 can do -this using the (?p{code}) construct, which PCRE cannot of course support.) -</P> -<LI><A NAME="SEC13" HREF="#TOC1">REGULAR EXPRESSION DETAILS</A> -<P> -The syntax and semantics of the regular expressions supported by PCRE are -described below. Regular expressions are also described in the Perl -documentation and in a number of other books, some of which have copious -examples. Jeffrey Friedl's "Mastering Regular Expressions", published by -O'Reilly (ISBN 1-56592-257), covers them in great detail. -</P> -<P> -The description here is intended as reference documentation. The basic -operation of PCRE is on strings of bytes. However, there is the beginnings of -some support for UTF-8 character strings. To use this support you must -configure PCRE to include it, and then call <B>pcre_compile()</B> with the -PCRE_UTF8 option. How this affects the pattern matching is described in the -final section of this document. -</P> -<P> -A regular expression is a pattern that is matched against a subject string from -left to right. Most characters stand for themselves in a pattern, and match the -corresponding characters in the subject. As a trivial example, the pattern -</P> -<P> -<PRE> - The quick brown fox -</PRE> -</P> -<P> -matches a portion of a subject string that is identical to itself. The power of -regular expressions comes from the ability to include alternatives and -repetitions in the pattern. These are encoded in the pattern by the use of -<I>meta-characters</I>, which do not stand for themselves but instead are -interpreted in some special way. -</P> -<P> -There are two different sets of meta-characters: those that are recognized -anywhere in the pattern except within square brackets, and those that are -recognized in square brackets. Outside square brackets, the meta-characters are -as follows: -</P> -<P> -<PRE> - \ general escape character with several uses - ^ assert start of subject (or line, in multiline mode) - $ assert end of subject (or line, in multiline mode) - . match any character except newline (by default) - [ start character class definition - | start of alternative branch - ( start subpattern - ) end subpattern - ? extends the meaning of ( - also 0 or 1 quantifier - also quantifier minimizer - * 0 or more quantifier - + 1 or more quantifier - { start min/max quantifier -</PRE> -</P> -<P> -Part of a pattern that is in square brackets is called a "character class". In -a character class the only meta-characters are: -</P> -<P> -<PRE> - \ general escape character - ^ negate the class, but only if the first character - - indicates character range - ] terminates the character class -</PRE> -</P> -<P> -The following sections describe the use of each of the meta-characters. -</P> -<LI><A NAME="SEC14" HREF="#TOC1">BACKSLASH</A> -<P> -The backslash character has several uses. Firstly, if it is followed by a -non-alphameric character, it takes away any special meaning that character may -have. This use of backslash as an escape character applies both inside and -outside character classes. -</P> -<P> -For example, if you want to match a "*" character, you write "\*" in the -pattern. This applies whether or not the following character would otherwise be -interpreted as a meta-character, so it is always safe to precede a -non-alphameric with "\" to specify that it stands for itself. In particular, -if you want to match a backslash, you write "\\". -</P> -<P> -If a pattern is compiled with the PCRE_EXTENDED option, whitespace in the -pattern (other than in a character class) and characters between a "#" outside -a character class and the next newline character are ignored. An escaping -backslash can be used to include a whitespace or "#" character as part of the -pattern. -</P> -<P> -A second use of backslash provides a way of encoding non-printing characters -in patterns in a visible manner. There is no restriction on the appearance of -non-printing characters, apart from the binary zero that terminates a pattern, -but when a pattern is being prepared by text editing, it is usually easier to -use one of the following escape sequences than the binary character it -represents: -</P> -<P> -<PRE> - \a alarm, that is, the BEL character (hex 07) - \cx "control-x", where x is any character - \e escape (hex 1B) - \f formfeed (hex 0C) - \n newline (hex 0A) - \r carriage return (hex 0D) - \t tab (hex 09) - \xhh character with hex code hh - \ddd character with octal code ddd, or backreference -</PRE> -</P> -<P> -The precise effect of "\cx" is as follows: if "x" is a lower case letter, it -is converted to upper case. Then bit 6 of the character (hex 40) is inverted. -Thus "\cz" becomes hex 1A, but "\c{" becomes hex 3B, while "\c;" becomes hex -7B. -</P> -<P> -After "\x", up to two hexadecimal digits are read (letters can be in upper or -lower case). -</P> -<P> -After "\0" up to two further octal digits are read. In both cases, if there -are fewer than two digits, just those that are present are used. Thus the -sequence "\0\x\07" specifies two binary zeros followed by a BEL character. -Make sure you supply two digits after the initial zero if the character that -follows is itself an octal digit. -</P> -<P> -The handling of a backslash followed by a digit other than 0 is complicated. -Outside a character class, PCRE reads it and any following digits as a decimal -number. If the number is less than 10, or if there have been at least that many -previous capturing left parentheses in the expression, the entire sequence is -taken as a <I>back reference</I>. A description of how this works is given -later, following the discussion of parenthesized subpatterns. -</P> -<P> -Inside a character class, or if the decimal number is greater than 9 and there -have not been that many capturing subpatterns, PCRE re-reads up to three octal -digits following the backslash, and generates a single byte from the least -significant 8 bits of the value. Any subsequent digits stand for themselves. -For example: -</P> -<P> -<PRE> - \040 is another way of writing a space - \40 is the same, provided there are fewer than 40 - previous capturing subpatterns - \7 is always a back reference - \11 might be a back reference, or another way of - writing a tab - \011 is always a tab - \0113 is a tab followed by the character "3" - \113 is the character with octal code 113 (since there - can be no more than 99 back references) - \377 is a byte consisting entirely of 1 bits - \81 is either a back reference, or a binary zero - followed by the two characters "8" and "1" -</PRE> -</P> -<P> -Note that octal values of 100 or greater must not be introduced by a leading -zero, because no more than three octal digits are ever read. -</P> -<P> -All the sequences that define a single byte value can be used both inside and -outside character classes. In addition, inside a character class, the sequence -"\b" is interpreted as the backspace character (hex 08). Outside a character -class it has a different meaning (see below). -</P> -<P> -The third use of backslash is for specifying generic character types: -</P> -<P> -<PRE> - \d any decimal digit - \D any character that is not a decimal digit - \s any whitespace character - \S any character that is not a whitespace character - \w any "word" character - \W any "non-word" character -</PRE> -</P> -<P> -Each pair of escape sequences partitions the complete set of characters into -two disjoint sets. Any given character matches one, and only one, of each pair. -</P> -<P> -A "word" character is any letter or digit or the underscore character, that is, -any character which can be part of a Perl "word". The definition of letters and -digits is controlled by PCRE's character tables, and may vary if locale- -specific matching is taking place (see "Locale support" above). For example, in -the "fr" (French) locale, some character codes greater than 128 are used for -accented letters, and these are matched by \w. -</P> -<P> -These character type sequences can appear both inside and outside character -classes. They each match one character of the appropriate type. If the current -matching point is at the end of the subject string, all of them fail, since -there is no character to match. -</P> -<P> -The fourth use of backslash is for certain simple assertions. An assertion -specifies a condition that has to be met at a particular point in a match, -without consuming any characters from the subject string. The use of -subpatterns for more complicated assertions is described below. The backslashed -assertions are -</P> -<P> -<PRE> - \b word boundary - \B not a word boundary - \A start of subject (independent of multiline mode) - \Z end of subject or newline at end (independent of multiline mode) - \z end of subject (independent of multiline mode) -</PRE> -</P> -<P> -These assertions may not appear in character classes (but note that "\b" has a -different meaning, namely the backspace character, inside a character class). -</P> -<P> -A word boundary is a position in the subject string where the current character -and the previous character do not both match \w or \W (i.e. one matches -\w and the other matches \W), or the start or end of the string if the -first or last character matches \w, respectively. -</P> -<P> -The \A, \Z, and \z assertions differ from the traditional circumflex and -dollar (described below) in that they only ever match at the very start and end -of the subject string, whatever options are set. They are not affected by the -PCRE_NOTBOL or PCRE_NOTEOL options. If the <I>startoffset</I> argument of -<B>pcre_exec()</B> is non-zero, \A can never match. The difference between \Z -and \z is that \Z matches before a newline that is the last character of the -string as well as at the end of the string, whereas \z matches only at the -end. -</P> -<LI><A NAME="SEC15" HREF="#TOC1">CIRCUMFLEX AND DOLLAR</A> -<P> -Outside a character class, in the default matching mode, the circumflex -character is an assertion which is true only if the current matching point is -at the start of the subject string. If the <I>startoffset</I> argument of -<B>pcre_exec()</B> is non-zero, circumflex can never match. Inside a character -class, circumflex has an entirely different meaning (see below). -</P> -<P> -Circumflex need not be the first character of the pattern if a number of -alternatives are involved, but it should be the first thing in each alternative -in which it appears if the pattern is ever to match that branch. If all -possible alternatives start with a circumflex, that is, if the pattern is -constrained to match only at the start of the subject, it is said to be an -"anchored" pattern. (There are also other constructs that can cause a pattern -to be anchored.) -</P> -<P> -A dollar character is an assertion which is true only if the current matching -point is at the end of the subject string, or immediately before a newline -character that is the last character in the string (by default). Dollar need -not be the last character of the pattern if a number of alternatives are -involved, but it should be the last item in any branch in which it appears. -Dollar has no special meaning in a character class. -</P> -<P> -The meaning of dollar can be changed so that it matches only at the very end of -the string, by setting the PCRE_DOLLAR_ENDONLY option at compile or matching -time. This does not affect the \Z assertion. -</P> -<P> -The meanings of the circumflex and dollar characters are changed if the -PCRE_MULTILINE option is set. When this is the case, they match immediately -after and immediately before an internal "\n" character, respectively, in -addition to matching at the start and end of the subject string. For example, -the pattern /^abc$/ matches the subject string "def\nabc" in multiline mode, -but not otherwise. Consequently, patterns that are anchored in single line mode -because all branches start with "^" are not anchored in multiline mode, and a -match for circumflex is possible when the <I>startoffset</I> argument of -<B>pcre_exec()</B> is non-zero. The PCRE_DOLLAR_ENDONLY option is ignored if -PCRE_MULTILINE is set. -</P> -<P> -Note that the sequences \A, \Z, and \z can be used to match the start and -end of the subject in both modes, and if all branches of a pattern start with -\A it is always anchored, whether PCRE_MULTILINE is set or not. -</P> -<LI><A NAME="SEC16" HREF="#TOC1">FULL STOP (PERIOD, DOT)</A> -<P> -Outside a character class, a dot in the pattern matches any one character in -the subject, including a non-printing character, but not (by default) newline. -If the PCRE_DOTALL option is set, dots match newlines as well. The handling of -dot is entirely independent of the handling of circumflex and dollar, the only -relationship being that they both involve newline characters. Dot has no -special meaning in a character class. -</P> -<LI><A NAME="SEC17" HREF="#TOC1">SQUARE BRACKETS</A> -<P> -An opening square bracket introduces a character class, terminated by a closing -square bracket. A closing square bracket on its own is not special. If a -closing square bracket is required as a member of the class, it should be the -first data character in the class (after an initial circumflex, if present) or -escaped with a backslash. -</P> -<P> -A character class matches a single character in the subject; the character must -be in the set of characters defined by the class, unless the first character in -the class is a circumflex, in which case the subject character must not be in -the set defined by the class. If a circumflex is actually required as a member -of the class, ensure it is not the first character, or escape it with a -backslash. -</P> -<P> -For example, the character class [aeiou] matches any lower case vowel, while -[^aeiou] matches any character that is not a lower case vowel. Note that a -circumflex is just a convenient notation for specifying the characters which -are in the class by enumerating those that are not. It is not an assertion: it -still consumes a character from the subject string, and fails if the current -pointer is at the end of the string. -</P> -<P> -When caseless matching is set, any letters in a class represent both their -upper case and lower case versions, so for example, a caseless [aeiou] matches -"A" as well as "a", and a caseless [^aeiou] does not match "A", whereas a -caseful version would. -</P> -<P> -The newline character is never treated in any special way in character classes, -whatever the setting of the PCRE_DOTALL or PCRE_MULTILINE options is. A class -such as [^a] will always match a newline. -</P> -<P> -The minus (hyphen) character can be used to specify a range of characters in a -character class. For example, [d-m] matches any letter between d and m, -inclusive. If a minus character is required in a class, it must be escaped with -a backslash or appear in a position where it cannot be interpreted as -indicating a range, typically as the first or last character in the class. -</P> -<P> -It is not possible to have the literal character "]" as the end character of a -range. A pattern such as [W-]46] is interpreted as a class of two characters -("W" and "-") followed by a literal string "46]", so it would match "W46]" or -"-46]". However, if the "]" is escaped with a backslash it is interpreted as -the end of range, so [W-\]46] is interpreted as a single class containing a -range followed by two separate characters. The octal or hexadecimal -representation of "]" can also be used to end a range. -</P> -<P> -Ranges operate in ASCII collating sequence. They can also be used for -characters specified numerically, for example [\000-\037]. If a range that -includes letters is used when caseless matching is set, it matches the letters -in either case. For example, [W-c] is equivalent to [][\^_`wxyzabc], matched -caselessly, and if character tables for the "fr" locale are in use, -[\xc8-\xcb] matches accented E characters in both cases. -</P> -<P> -The character types \d, \D, \s, \S, \w, and \W may also appear in a -character class, and add the characters that they match to the class. For -example, [\dABCDEF] matches any hexadecimal digit. A circumflex can -conveniently be used with the upper case character types to specify a more -restricted set of characters than the matching lower case type. For example, -the class [^\W_] matches any letter or digit, but not underscore. -</P> -<P> -All non-alphameric characters other than \, -, ^ (at the start) and the -terminating ] are non-special in character classes, but it does no harm if they -are escaped. -</P> -<LI><A NAME="SEC18" HREF="#TOC1">POSIX CHARACTER CLASSES</A> -<P> -Perl 5.6 (not yet released at the time of writing) is going to support the -POSIX notation for character classes, which uses names enclosed by [: and :] -within the enclosing square brackets. PCRE supports this notation. For example, -</P> -<P> -<PRE> - [01[:alpha:]%] -</PRE> -</P> -<P> -matches "0", "1", any alphabetic character, or "%". The supported class names -are -</P> -<P> -<PRE> - alnum letters and digits - alpha letters - ascii character codes 0 - 127 - cntrl control characters - digit decimal digits (same as \d) - graph printing characters, excluding space - lower lower case letters - print printing characters, including space - punct printing characters, excluding letters and digits - space white space (same as \s) - upper upper case letters - word "word" characters (same as \w) - xdigit hexadecimal digits -</PRE> -</P> -<P> ->>>>>>>>>>>>Only WORD is perl. BLANK is GNU. -</P> -<P> -The names "ascii" and "word" are Perl extensions. Another Perl extension is -negation, which is indicated by a ^ character after the colon. For example, -</P> -<P> -<PRE> - [12[:^digit:]] -</PRE> -</P> -<P> -matches "1", "2", or any non-digit. PCRE (and Perl) also recognize the POSIX -syntax [.ch.] and [=ch=] where "ch" is a "collating element", but these are not -supported, and an error is given if they are encountered. -</P> -<LI><A NAME="SEC19" HREF="#TOC1">VERTICAL BAR</A> -<P> -Vertical bar characters are used to separate alternative patterns. For example, -the pattern -</P> -<P> -<PRE> - gilbert|sullivan -</PRE> -</P> -<P> -matches either "gilbert" or "sullivan". Any number of alternatives may appear, -and an empty alternative is permitted (matching the empty string). -The matching process tries each alternative in turn, from left to right, -and the first one that succeeds is used. If the alternatives are within a -subpattern (defined below), "succeeds" means matching the rest of the main -pattern as well as the alternative in the subpattern. -</P> -<LI><A NAME="SEC20" HREF="#TOC1">INTERNAL OPTION SETTING</A> -<P> -The settings of PCRE_CASELESS, PCRE_MULTILINE, PCRE_DOTALL, and PCRE_EXTENDED -can be changed from within the pattern by a sequence of Perl option letters -enclosed between "(?" and ")". The option letters are -</P> -<P> -<PRE> - i for PCRE_CASELESS - m for PCRE_MULTILINE - s for PCRE_DOTALL - x for PCRE_EXTENDED -</PRE> -</P> -<P> -For example, (?im) sets caseless, multiline matching. It is also possible to -unset these options by preceding the letter with a hyphen, and a combined -setting and unsetting such as (?im-sx), which sets PCRE_CASELESS and -PCRE_MULTILINE while unsetting PCRE_DOTALL and PCRE_EXTENDED, is also -permitted. If a letter appears both before and after the hyphen, the option is -unset. -</P> -<P> -The scope of these option changes depends on where in the pattern the setting -occurs. For settings that are outside any subpattern (defined below), the -effect is the same as if the options were set or unset at the start of -matching. The following patterns all behave in exactly the same way: -</P> -<P> -<PRE> - (?i)abc - a(?i)bc - ab(?i)c - abc(?i) -</PRE> -</P> -<P> -which in turn is the same as compiling the pattern abc with PCRE_CASELESS set. -In other words, such "top level" settings apply to the whole pattern (unless -there are other changes inside subpatterns). If there is more than one setting -of the same option at top level, the rightmost setting is used. -</P> -<P> -If an option change occurs inside a subpattern, the effect is different. This -is a change of behaviour in Perl 5.005. An option change inside a subpattern -affects only that part of the subpattern that follows it, so -</P> -<P> -<PRE> - (a(?i)b)c -</PRE> -</P> -<P> -matches abc and aBc and no other strings (assuming PCRE_CASELESS is not used). -By this means, options can be made to have different settings in different -parts of the pattern. Any changes made in one alternative do carry on -into subsequent branches within the same subpattern. For example, -</P> -<P> -<PRE> - (a(?i)b|c) -</PRE> -</P> -<P> -matches "ab", "aB", "c", and "C", even though when matching "C" the first -branch is abandoned before the option setting. This is because the effects of -option settings happen at compile time. There would be some very weird -behaviour otherwise. -</P> -<P> -The PCRE-specific options PCRE_UNGREEDY and PCRE_EXTRA can be changed in the -same way as the Perl-compatible options by using the characters U and X -respectively. The (?X) flag setting is special in that it must always occur -earlier in the pattern than any of the additional features it turns on, even -when it is at top level. It is best put at the start. -</P> -<LI><A NAME="SEC21" HREF="#TOC1">SUBPATTERNS</A> -<P> -Subpatterns are delimited by parentheses (round brackets), which can be nested. -Marking part of a pattern as a subpattern does two things: -</P> -<P> -1. It localizes a set of alternatives. For example, the pattern -</P> -<P> -<PRE> - cat(aract|erpillar|) -</PRE> -</P> -<P> -matches one of the words "cat", "cataract", or "caterpillar". Without the -parentheses, it would match "cataract", "erpillar" or the empty string. -</P> -<P> -2. It sets up the subpattern as a capturing subpattern (as defined above). -When the whole pattern matches, that portion of the subject string that matched -the subpattern is passed back to the caller via the <I>ovector</I> argument of -<B>pcre_exec()</B>. Opening parentheses are counted from left to right (starting -from 1) to obtain the numbers of the capturing subpatterns. -</P> -<P> -For example, if the string "the red king" is matched against the pattern -</P> -<P> -<PRE> - the ((red|white) (king|queen)) -</PRE> -</P> -<P> -the captured substrings are "red king", "red", and "king", and are numbered 1, -2, and 3, respectively. -</P> -<P> -The fact that plain parentheses fulfil two functions is not always helpful. -There are often times when a grouping subpattern is required without a -capturing requirement. If an opening parenthesis is followed by "?:", the -subpattern does not do any capturing, and is not counted when computing the -number of any subsequent capturing subpatterns. For example, if the string "the -white queen" is matched against the pattern -</P> -<P> -<PRE> - the ((?:red|white) (king|queen)) -</PRE> -</P> -<P> -the captured substrings are "white queen" and "queen", and are numbered 1 and -2. The maximum number of captured substrings is 99, and the maximum number of -all subpatterns, both capturing and non-capturing, is 200. -</P> -<P> -As a convenient shorthand, if any option settings are required at the start of -a non-capturing subpattern, the option letters may appear between the "?" and -the ":". Thus the two patterns -</P> -<P> -<PRE> - (?i:saturday|sunday) - (?:(?i)saturday|sunday) -</PRE> -</P> -<P> -match exactly the same set of strings. Because alternative branches are tried -from left to right, and options are not reset until the end of the subpattern -is reached, an option setting in one branch does affect subsequent branches, so -the above patterns match "SUNDAY" as well as "Saturday". -</P> -<LI><A NAME="SEC22" HREF="#TOC1">REPETITION</A> -<P> -Repetition is specified by quantifiers, which can follow any of the following -items: -</P> -<P> -<PRE> - a single character, possibly escaped - the . metacharacter - a character class - a back reference (see next section) - a parenthesized subpattern (unless it is an assertion - see below) -</PRE> -</P> -<P> -The general repetition quantifier specifies a minimum and maximum number of -permitted matches, by giving the two numbers in curly brackets (braces), -separated by a comma. The numbers must be less than 65536, and the first must -be less than or equal to the second. For example: -</P> -<P> -<PRE> - z{2,4} -</PRE> -</P> -<P> -matches "zz", "zzz", or "zzzz". A closing brace on its own is not a special -character. If the second number is omitted, but the comma is present, there is -no upper limit; if the second number and the comma are both omitted, the -quantifier specifies an exact number of required matches. Thus -</P> -<P> -<PRE> - [aeiou]{3,} -</PRE> -</P> -<P> -matches at least 3 successive vowels, but may match many more, while -</P> -<P> -<PRE> - \d{8} -</PRE> -</P> -<P> -matches exactly 8 digits. An opening curly bracket that appears in a position -where a quantifier is not allowed, or one that does not match the syntax of a -quantifier, is taken as a literal character. For example, {,6} is not a -quantifier, but a literal string of four characters. -</P> -<P> -The quantifier {0} is permitted, causing the expression to behave as if the -previous item and the quantifier were not present. -</P> -<P> -For convenience (and historical compatibility) the three most common -quantifiers have single-character abbreviations: -</P> -<P> -<PRE> - * is equivalent to {0,} - + is equivalent to {1,} - ? is equivalent to {0,1} -</PRE> -</P> -<P> -It is possible to construct infinite loops by following a subpattern that can -match no characters with a quantifier that has no upper limit, for example: -</P> -<P> -<PRE> - (a?)* -</PRE> -</P> -<P> -Earlier versions of Perl and PCRE used to give an error at compile time for -such patterns. However, because there are cases where this can be useful, such -patterns are now accepted, but if any repetition of the subpattern does in fact -match no characters, the loop is forcibly broken. -</P> -<P> -By default, the quantifiers are "greedy", that is, they match as much as -possible (up to the maximum number of permitted times), without causing the -rest of the pattern to fail. The classic example of where this gives problems -is in trying to match comments in C programs. These appear between the -sequences /* and */ and within the sequence, individual * and / characters may -appear. An attempt to match C comments by applying the pattern -</P> -<P> -<PRE> - /\*.*\*/ -</PRE> -</P> -<P> -to the string -</P> -<P> -<PRE> - /* first command */ not comment /* second comment */ -</PRE> -</P> -<P> -fails, because it matches the entire string owing to the greediness of the .* -item. -</P> -<P> -However, if a quantifier is followed by a question mark, it ceases to be -greedy, and instead matches the minimum number of times possible, so the -pattern -</P> -<P> -<PRE> - /\*.*?\*/ -</PRE> -</P> -<P> -does the right thing with the C comments. The meaning of the various -quantifiers is not otherwise changed, just the preferred number of matches. -Do not confuse this use of question mark with its use as a quantifier in its -own right. Because it has two uses, it can sometimes appear doubled, as in -</P> -<P> -<PRE> - \d??\d -</PRE> -</P> -<P> -which matches one digit by preference, but can match two if that is the only -way the rest of the pattern matches. -</P> -<P> -If the PCRE_UNGREEDY option is set (an option which is not available in Perl), -the quantifiers are not greedy by default, but individual ones can be made -greedy by following them with a question mark. In other words, it inverts the -default behaviour. -</P> -<P> -When a parenthesized subpattern is quantified with a minimum repeat count that -is greater than 1 or with a limited maximum, more store is required for the -compiled pattern, in proportion to the size of the minimum or maximum. -</P> -<P> -If a pattern starts with .* or .{0,} and the PCRE_DOTALL option (equivalent -to Perl's /s) is set, thus allowing the . to match newlines, the pattern is -implicitly anchored, because whatever follows will be tried against every -character position in the subject string, so there is no point in retrying the -overall match at any position after the first. PCRE treats such a pattern as -though it were preceded by \A. In cases where it is known that the subject -string contains no newlines, it is worth setting PCRE_DOTALL when the pattern -begins with .* in order to obtain this optimization, or alternatively using ^ -to indicate anchoring explicitly. -</P> -<P> -When a capturing subpattern is repeated, the value captured is the substring -that matched the final iteration. For example, after -</P> -<P> -<PRE> - (tweedle[dume]{3}\s*)+ -</PRE> -</P> -<P> -has matched "tweedledum tweedledee" the value of the captured substring is -"tweedledee". However, if there are nested capturing subpatterns, the -corresponding captured values may have been set in previous iterations. For -example, after -</P> -<P> -<PRE> - /(a|(b))+/ -</PRE> -</P> -<P> -matches "aba" the value of the second captured substring is "b". -</P> -<LI><A NAME="SEC23" HREF="#TOC1">BACK REFERENCES</A> -<P> -Outside a character class, a backslash followed by a digit greater than 0 (and -possibly further digits) is a back reference to a capturing subpattern earlier -(i.e. to its left) in the pattern, provided there have been that many previous -capturing left parentheses. -</P> -<P> -However, if the decimal number following the backslash is less than 10, it is -always taken as a back reference, and causes an error only if there are not -that many capturing left parentheses in the entire pattern. In other words, the -parentheses that are referenced need not be to the left of the reference for -numbers less than 10. See the section entitled "Backslash" above for further -details of the handling of digits following a backslash. -</P> -<P> -A back reference matches whatever actually matched the capturing subpattern in -the current subject string, rather than anything matching the subpattern -itself. So the pattern -</P> -<P> -<PRE> - (sens|respons)e and \1ibility -</PRE> -</P> -<P> -matches "sense and sensibility" and "response and responsibility", but not -"sense and responsibility". If caseful matching is in force at the time of the -back reference, the case of letters is relevant. For example, -</P> -<P> -<PRE> - ((?i)rah)\s+\1 -</PRE> -</P> -<P> -matches "rah rah" and "RAH RAH", but not "RAH rah", even though the original -capturing subpattern is matched caselessly. -</P> -<P> -There may be more than one back reference to the same subpattern. If a -subpattern has not actually been used in a particular match, any back -references to it always fail. For example, the pattern -</P> -<P> -<PRE> - (a|(bc))\2 -</PRE> -</P> -<P> -always fails if it starts to match "a" rather than "bc". Because there may be -up to 99 back references, all digits following the backslash are taken -as part of a potential back reference number. If the pattern continues with a -digit character, some delimiter must be used to terminate the back reference. -If the PCRE_EXTENDED option is set, this can be whitespace. Otherwise an empty -comment can be used. -</P> -<P> -A back reference that occurs inside the parentheses to which it refers fails -when the subpattern is first used, so, for example, (a\1) never matches. -However, such references can be useful inside repeated subpatterns. For -example, the pattern -</P> -<P> -<PRE> - (a|b\1)+ -</PRE> -</P> -<P> -matches any number of "a"s and also "aba", "ababbaa" etc. At each iteration of -the subpattern, the back reference matches the character string corresponding -to the previous iteration. In order for this to work, the pattern must be such -that the first iteration does not need to match the back reference. This can be -done using alternation, as in the example above, or by a quantifier with a -minimum of zero. -</P> -<LI><A NAME="SEC24" HREF="#TOC1">ASSERTIONS</A> -<P> -An assertion is a test on the characters following or preceding the current -matching point that does not actually consume any characters. The simple -assertions coded as \b, \B, \A, \Z, \z, ^ and $ are described above. More -complicated assertions are coded as subpatterns. There are two kinds: those -that look ahead of the current position in the subject string, and those that -look behind it. -</P> -<P> -An assertion subpattern is matched in the normal way, except that it does not -cause the current matching position to be changed. Lookahead assertions start -with (?= for positive assertions and (?! for negative assertions. For example, -</P> -<P> -<PRE> - \w+(?=;) -</PRE> -</P> -<P> -matches a word followed by a semicolon, but does not include the semicolon in -the match, and -</P> -<P> -<PRE> - foo(?!bar) -</PRE> -</P> -<P> -matches any occurrence of "foo" that is not followed by "bar". Note that the -apparently similar pattern -</P> -<P> -<PRE> - (?!foo)bar -</PRE> -</P> -<P> -does not find an occurrence of "bar" that is preceded by something other than -"foo"; it finds any occurrence of "bar" whatsoever, because the assertion -(?!foo) is always true when the next three characters are "bar". A -lookbehind assertion is needed to achieve this effect. -</P> -<P> -Lookbehind assertions start with (?<= for positive assertions and (?<! for -negative assertions. For example, -</P> -<P> -<PRE> - (?<!foo)bar -</PRE> -</P> -<P> -does find an occurrence of "bar" that is not preceded by "foo". The contents of -a lookbehind assertion are restricted such that all the strings it matches must -have a fixed length. However, if there are several alternatives, they do not -all have to have the same fixed length. Thus -</P> -<P> -<PRE> - (?<=bullock|donkey) -</PRE> -</P> -<P> -is permitted, but -</P> -<P> -<PRE> - (?<!dogs?|cats?) -</PRE> -</P> -<P> -causes an error at compile time. Branches that match different length strings -are permitted only at the top level of a lookbehind assertion. This is an -extension compared with Perl 5.005, which requires all branches to match the -same length of string. An assertion such as -</P> -<P> -<PRE> - (?<=ab(c|de)) -</PRE> -</P> -<P> -is not permitted, because its single top-level branch can match two different -lengths, but it is acceptable if rewritten to use two top-level branches: -</P> -<P> -<PRE> - (?<=abc|abde) -</PRE> -</P> -<P> -The implementation of lookbehind assertions is, for each alternative, to -temporarily move the current position back by the fixed width and then try to -match. If there are insufficient characters before the current position, the -match is deemed to fail. Lookbehinds in conjunction with once-only subpatterns -can be particularly useful for matching at the ends of strings; an example is -given at the end of the section on once-only subpatterns. -</P> -<P> -Several assertions (of any sort) may occur in succession. For example, -</P> -<P> -<PRE> - (?<=\d{3})(?<!999)foo -</PRE> -</P> -<P> -matches "foo" preceded by three digits that are not "999". Notice that each of -the assertions is applied independently at the same point in the subject -string. First there is a check that the previous three characters are all -digits, and then there is a check that the same three characters are not "999". -This pattern does <I>not</I> match "foo" preceded by six characters, the first -of which are digits and the last three of which are not "999". For example, it -doesn't match "123abcfoo". A pattern to do that is -</P> -<P> -<PRE> - (?<=\d{3}...)(?<!999)foo -</PRE> -</P> -<P> -This time the first assertion looks at the preceding six characters, checking -that the first three are digits, and then the second assertion checks that the -preceding three characters are not "999". -</P> -<P> -Assertions can be nested in any combination. For example, -</P> -<P> -<PRE> - (?<=(?<!foo)bar)baz -</PRE> -</P> -<P> -matches an occurrence of "baz" that is preceded by "bar" which in turn is not -preceded by "foo", while -</P> -<P> -<PRE> - (?<=\d{3}(?!999)...)foo -</PRE> -</P> -<P> -is another pattern which matches "foo" preceded by three digits and any three -characters that are not "999". -</P> -<P> -Assertion subpatterns are not capturing subpatterns, and may not be repeated, -because it makes no sense to assert the same thing several times. If any kind -of assertion contains capturing subpatterns within it, these are counted for -the purposes of numbering the capturing subpatterns in the whole pattern. -However, substring capturing is carried out only for positive assertions, -because it does not make sense for negative assertions. -</P> -<P> -Assertions count towards the maximum of 200 parenthesized subpatterns. -</P> -<LI><A NAME="SEC25" HREF="#TOC1">ONCE-ONLY SUBPATTERNS</A> -<P> -With both maximizing and minimizing repetition, failure of what follows -normally causes the repeated item to be re-evaluated to see if a different -number of repeats allows the rest of the pattern to match. Sometimes it is -useful to prevent this, either to change the nature of the match, or to cause -it fail earlier than it otherwise might, when the author of the pattern knows -there is no point in carrying on. -</P> -<P> -Consider, for example, the pattern \d+foo when applied to the subject line -</P> -<P> -<PRE> - 123456bar -</PRE> -</P> -<P> -After matching all 6 digits and then failing to match "foo", the normal -action of the matcher is to try again with only 5 digits matching the \d+ -item, and then with 4, and so on, before ultimately failing. Once-only -subpatterns provide the means for specifying that once a portion of the pattern -has matched, it is not to be re-evaluated in this way, so the matcher would -give up immediately on failing to match "foo" the first time. The notation is -another kind of special parenthesis, starting with (?> as in this example: -</P> -<P> -<PRE> - (?>\d+)bar -</PRE> -</P> -<P> -This kind of parenthesis "locks up" the part of the pattern it contains once -it has matched, and a failure further into the pattern is prevented from -backtracking into it. Backtracking past it to previous items, however, works as -normal. -</P> -<P> -An alternative description is that a subpattern of this type matches the string -of characters that an identical standalone pattern would match, if anchored at -the current point in the subject string. -</P> -<P> -Once-only subpatterns are not capturing subpatterns. Simple cases such as the -above example can be thought of as a maximizing repeat that must swallow -everything it can. So, while both \d+ and \d+? are prepared to adjust the -number of digits they match in order to make the rest of the pattern match, -(?>\d+) can only match an entire sequence of digits. -</P> -<P> -This construction can of course contain arbitrarily complicated subpatterns, -and it can be nested. -</P> -<P> -Once-only subpatterns can be used in conjunction with lookbehind assertions to -specify efficient matching at the end of the subject string. Consider a simple -pattern such as -</P> -<P> -<PRE> - abcd$ -</PRE> -</P> -<P> -when applied to a long string which does not match. Because matching proceeds -from left to right, PCRE will look for each "a" in the subject and then see if -what follows matches the rest of the pattern. If the pattern is specified as -</P> -<P> -<PRE> - ^.*abcd$ -</PRE> -</P> -<P> -the initial .* matches the entire string at first, but when this fails (because -there is no following "a"), it backtracks to match all but the last character, -then all but the last two characters, and so on. Once again the search for "a" -covers the entire string, from right to left, so we are no better off. However, -if the pattern is written as -</P> -<P> -<PRE> - ^(?>.*)(?<=abcd) -</PRE> -</P> -<P> -there can be no backtracking for the .* item; it can match only the entire -string. The subsequent lookbehind assertion does a single test on the last four -characters. If it fails, the match fails immediately. For long strings, this -approach makes a significant difference to the processing time. -</P> -<P> -When a pattern contains an unlimited repeat inside a subpattern that can itself -be repeated an unlimited number of times, the use of a once-only subpattern is -the only way to avoid some failing matches taking a very long time indeed. -The pattern -</P> -<P> -<PRE> - (\D+|<\d+>)*[!?] -</PRE> -</P> -<P> -matches an unlimited number of substrings that either consist of non-digits, or -digits enclosed in <>, followed by either ! or ?. When it matches, it runs -quickly. However, if it is applied to -</P> -<P> -<PRE> - aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa -</PRE> -</P> -<P> -it takes a long time before reporting failure. This is because the string can -be divided between the two repeats in a large number of ways, and all have to -be tried. (The example used [!?] rather than a single character at the end, -because both PCRE and Perl have an optimization that allows for fast failure -when a single character is used. They remember the last single character that -is required for a match, and fail early if it is not present in the string.) -If the pattern is changed to -</P> -<P> -<PRE> - ((?>\D+)|<\d+>)*[!?] -</PRE> -</P> -<P> -sequences of non-digits cannot be broken, and failure happens quickly. -</P> -<LI><A NAME="SEC26" HREF="#TOC1">CONDITIONAL SUBPATTERNS</A> -<P> -It is possible to cause the matching process to obey a subpattern -conditionally or to choose between two alternative subpatterns, depending on -the result of an assertion, or whether a previous capturing subpattern matched -or not. The two possible forms of conditional subpattern are -</P> -<P> -<PRE> - (?(condition)yes-pattern) - (?(condition)yes-pattern|no-pattern) -</PRE> -</P> -<P> -If the condition is satisfied, the yes-pattern is used; otherwise the -no-pattern (if present) is used. If there are more than two alternatives in the -subpattern, a compile-time error occurs. -</P> -<P> -There are two kinds of condition. If the text between the parentheses consists -of a sequence of digits, the condition is satisfied if the capturing subpattern -of that number has previously matched. The number must be greater than zero. -Consider the following pattern, which contains non-significant white space to -make it more readable (assume the PCRE_EXTENDED option) and to divide it into -three parts for ease of discussion: -</P> -<P> -<PRE> - ( \( )? [^()]+ (?(1) \) ) -</PRE> -</P> -<P> -The first part matches an optional opening parenthesis, and if that -character is present, sets it as the first captured substring. The second part -matches one or more characters that are not parentheses. The third part is a -conditional subpattern that tests whether the first set of parentheses matched -or not. If they did, that is, if subject started with an opening parenthesis, -the condition is true, and so the yes-pattern is executed and a closing -parenthesis is required. Otherwise, since no-pattern is not present, the -subpattern matches nothing. In other words, this pattern matches a sequence of -non-parentheses, optionally enclosed in parentheses. -</P> -<P> -If the condition is not a sequence of digits, it must be an assertion. This may -be a positive or negative lookahead or lookbehind assertion. Consider this -pattern, again containing non-significant white space, and with the two -alternatives on the second line: -</P> -<P> -<PRE> - (?(?=[^a-z]*[a-z]) - \d{2}-[a-z]{3}-\d{2} | \d{2}-\d{2}-\d{2} ) -</PRE> -</P> -<P> -The condition is a positive lookahead assertion that matches an optional -sequence of non-letters followed by a letter. In other words, it tests for the -presence of at least one letter in the subject. If a letter is found, the -subject is matched against the first alternative; otherwise it is matched -against the second. This pattern matches strings in one of the two forms -dd-aaa-dd or dd-dd-dd, where aaa are letters and dd are digits. -</P> -<LI><A NAME="SEC27" HREF="#TOC1">COMMENTS</A> -<P> -The sequence (?# marks the start of a comment which continues up to the next -closing parenthesis. Nested parentheses are not permitted. The characters -that make up a comment play no part in the pattern matching at all. -</P> -<P> -If the PCRE_EXTENDED option is set, an unescaped # character outside a -character class introduces a comment that continues up to the next newline -character in the pattern. -</P> -<LI><A NAME="SEC28" HREF="#TOC1">RECURSIVE PATTERNS</A> -<P> -Consider the problem of matching a string in parentheses, allowing for -unlimited nested parentheses. Without the use of recursion, the best that can -be done is to use a pattern that matches up to some fixed depth of nesting. It -is not possible to handle an arbitrary nesting depth. Perl 5.6 has provided an -experimental facility that allows regular expressions to recurse (amongst other -things). It does this by interpolating Perl code in the expression at run time, -and the code can refer to the expression itself. A Perl pattern to solve the -parentheses problem can be created like this: -</P> -<P> -<PRE> - $re = qr{\( (?: (?>[^()]+) | (?p{$re}) )* \)}x; -</PRE> -</P> -<P> -The (?p{...}) item interpolates Perl code at run time, and in this case refers -recursively to the pattern in which it appears. Obviously, PCRE cannot support -the interpolation of Perl code. Instead, the special item (?R) is provided for -the specific case of recursion. This PCRE pattern solves the parentheses -problem (assume the PCRE_EXTENDED option is set so that white space is -ignored): -</P> -<P> -<PRE> - \( ( (?>[^()]+) | (?R) )* \) -</PRE> -</P> -<P> -First it matches an opening parenthesis. Then it matches any number of -substrings which can either be a sequence of non-parentheses, or a recursive -match of the pattern itself (i.e. a correctly parenthesized substring). Finally -there is a closing parenthesis. -</P> -<P> -This particular example pattern contains nested unlimited repeats, and so the -use of a once-only subpattern for matching strings of non-parentheses is -important when applying the pattern to strings that do not match. For example, -when it is applied to -</P> -<P> -<PRE> - (aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa() -</PRE> -</P> -<P> -it yields "no match" quickly. However, if a once-only subpattern is not used, -the match runs for a very long time indeed because there are so many different -ways the + and * repeats can carve up the subject, and all have to be tested -before failure can be reported. -</P> -<P> -The values set for any capturing subpatterns are those from the outermost level -of the recursion at which the subpattern value is set. If the pattern above is -matched against -</P> -<P> -<PRE> - (ab(cd)ef) -</PRE> -</P> -<P> -the value for the capturing parentheses is "ef", which is the last value taken -on at the top level. If additional parentheses are added, giving -</P> -<P> -<PRE> - \( ( ( (?>[^()]+) | (?R) )* ) \) - ^ ^ - ^ ^ -</PRE> -the string they capture is "ab(cd)ef", the contents of the top level -parentheses. If there are more than 15 capturing parentheses in a pattern, PCRE -has to obtain extra memory to store data during a recursion, which it does by -using <B>pcre_malloc</B>, freeing it via <B>pcre_free</B> afterwards. If no -memory can be obtained, it saves data for the first 15 capturing parentheses -only, as there is no way to give an out-of-memory error from within a -recursion. -</P> -<LI><A NAME="SEC29" HREF="#TOC1">PERFORMANCE</A> -<P> -Certain items that may appear in patterns are more efficient than others. It is -more efficient to use a character class like [aeiou] than a set of alternatives -such as (a|e|i|o|u). In general, the simplest construction that provides the -required behaviour is usually the most efficient. Jeffrey Friedl's book -contains a lot of discussion about optimizing regular expressions for efficient -performance. -</P> -<P> -When a pattern begins with .* and the PCRE_DOTALL option is set, the pattern is -implicitly anchored by PCRE, since it can match only at the start of a subject -string. However, if PCRE_DOTALL is not set, PCRE cannot make this optimization, -because the . metacharacter does not then match a newline, and if the subject -string contains newlines, the pattern may match from the character immediately -following one of them instead of from the very start. For example, the pattern -</P> -<P> -<PRE> - (.*) second -</PRE> -</P> -<P> -matches the subject "first\nand second" (where \n stands for a newline -character) with the first captured substring being "and". In order to do this, -PCRE has to retry the match starting after every newline in the subject. -</P> -<P> -If you are using such a pattern with subject strings that do not contain -newlines, the best performance is obtained by setting PCRE_DOTALL, or starting -the pattern with ^.* to indicate explicit anchoring. That saves PCRE from -having to scan along the subject looking for a newline to restart at. -</P> -<P> -Beware of patterns that contain nested indefinite repeats. These can take a -long time to run when applied to a string that does not match. Consider the -pattern fragment -</P> -<P> -<PRE> - (a+)* -</PRE> -</P> -<P> -This can match "aaaa" in 33 different ways, and this number increases very -rapidly as the string gets longer. (The * repeat can match 0, 1, 2, 3, or 4 -times, and for each of those cases other than 0, the + repeats can match -different numbers of times.) When the remainder of the pattern is such that the -entire match is going to fail, PCRE has in principle to try every possible -variation, and this can take an extremely long time. -</P> -<P> -An optimization catches some of the more simple cases such as -</P> -<P> -<PRE> - (a+)*b -</PRE> -</P> -<P> -where a literal character follows. Before embarking on the standard matching -procedure, PCRE checks that there is a "b" later in the subject string, and if -there is not, it fails the match immediately. However, when there is no -following literal this optimization cannot be used. You can see the difference -by comparing the behaviour of -</P> -<P> -<PRE> - (a+)*\d -</PRE> -</P> -<P> -with the pattern above. The former gives a failure almost instantly when -applied to a whole line of "a" characters, whereas the latter takes an -appreciable time with strings longer than about 20 characters. -</P> -<LI><A NAME="SEC30" HREF="#TOC1">UTF-8 SUPPORT</A> -<P> -Starting at release 3.3, PCRE has some support for character strings encoded -in the UTF-8 format. This is incomplete, and is regarded as experimental. In -order to use it, you must configure PCRE to include UTF-8 support in the code, -and, in addition, you must call <B>pcre_compile()</B> with the PCRE_UTF8 option -flag. When you do this, both the pattern and any subject strings that are -matched against it are treated as UTF-8 strings instead of just strings of -bytes, but only in the cases that are mentioned below. -</P> -<P> -If you compile PCRE with UTF-8 support, but do not use it at run time, the -library will be a bit bigger, but the additional run time overhead is limited -to testing the PCRE_UTF8 flag in several places, so should not be very large. -</P> -<P> -PCRE assumes that the strings it is given contain valid UTF-8 codes. It does -not diagnose invalid UTF-8 strings. If you pass invalid UTF-8 strings to PCRE, -the results are undefined. -</P> -<P> -Running with PCRE_UTF8 set causes these changes in the way PCRE works: -</P> -<P> -1. In a pattern, the escape sequence \x{...}, where the contents of the braces -is a string of hexadecimal digits, is interpreted as a UTF-8 character whose -code number is the given hexadecimal number, for example: \x{1234}. This -inserts from one to six literal bytes into the pattern, using the UTF-8 -encoding. If a non-hexadecimal digit appears between the braces, the item is -not recognized. -</P> -<P> -2. The original hexadecimal escape sequence, \xhh, generates a two-byte UTF-8 -character if its value is greater than 127. -</P> -<P> -3. Repeat quantifiers are NOT correctly handled if they follow a multibyte -character. For example, \x{100}* and \xc3+ do not work. If you want to -repeat such characters, you must enclose them in non-capturing parentheses, -for example (?:\x{100}), at present. -</P> -<P> -4. The dot metacharacter matches one UTF-8 character instead of a single byte. -</P> -<P> -5. Unlike literal UTF-8 characters, the dot metacharacter followed by a -repeat quantifier does operate correctly on UTF-8 characters instead of -single bytes. -</P> -<P> -4. Although the \x{...} escape is permitted in a character class, characters -whose values are greater than 255 cannot be included in a class. -</P> -<P> -5. A class is matched against a UTF-8 character instead of just a single byte, -but it can match only characters whose values are less than 256. Characters -with greater values always fail to match a class. -</P> -<P> -6. Repeated classes work correctly on multiple characters. -</P> -<P> -7. Classes containing just a single character whose value is greater than 127 -(but less than 256), for example, [\x80] or [^\x{93}], do not work because -these are optimized into single byte matches. In the first case, of course, -the class brackets are just redundant. -</P> -<P> -8. Lookbehind assertions move backwards in the subject by a fixed number of -characters instead of a fixed number of bytes. Simple cases have been tested -to work correctly, but there may be hidden gotchas herein. -</P> -<P> -9. The character types such as \d and \w do not work correctly with UTF-8 -characters. They continue to test a single byte. -</P> -<P> -10. Anything not explicitly mentioned here continues to work in bytes rather -than in characters. -</P> -<P> -The following UTF-8 features of Perl 5.6 are not implemented: -</P> -<P> -1. The escape sequence \C to match a single byte. -</P> -<P> -2. The use of Unicode tables and properties and escapes \p, \P, and \X. -</P> -<LI><A NAME="SEC31" HREF="#TOC1">SAMPLE PROGRAM</A> -<P> -The code below is a simple, complete demonstration program, to get you started -with using PCRE. This code is also supplied in the file <I>pcredemo.c</I> in the -PCRE distribution. -</P> -<P> -The program compiles the regular expression that is its first argument, and -matches it against the subject string in its second argument. No options are -set, and default character tables are used. If matching succeeds, the program -outputs the portion of the subject that matched, together with the contents of -any captured substrings. -</P> -<P> -On a Unix system that has PCRE installed in <I>/usr/local</I>, you can compile -the demonstration program using a command like this: -</P> -<P> -<PRE> - gcc -o pcredemo pcredemo.c -I/usr/local/include -L/usr/local/lib -lpcre -</PRE> -</P> -<P> -Then you can run simple tests like this: -</P> -<P> -<PRE> - ./pcredemo 'cat|dog' 'the cat sat on the mat' -</PRE> -</P> -<P> -Note that there is a much more comprehensive test program, called -<B>pcretest</B>, which supports many more facilities for testing regular -expressions. The <B>pcredemo</B> program is provided as a simple coding example. -</P> -<P> -On some operating systems (e.g. Solaris) you may get an error like this when -you try to run <B>pcredemo</B>: -</P> -<P> -<PRE> - ld.so.1: a.out: fatal: libpcre.so.0: open failed: No such file or directory -</PRE> -</P> -<P> -This is caused by the way shared library support works on those systems. You -need to add -</P> -<P> -<PRE> - -R/usr/local/lib -</PRE> -</P> -<P> -to the compile command to get round this problem. Here's the code: -</P> -<P> -<PRE> - #include <stdio.h> - #include <string.h> - #include <pcre.h> -</PRE> -</P> -<P> -<PRE> - #define OVECCOUNT 30 /* should be a multiple of 3 */ -</PRE> -</P> -<P> -<PRE> - int main(int argc, char **argv) - { - pcre *re; - const char *error; - int erroffset; - int ovector[OVECCOUNT]; - int rc, i; -</PRE> -</P> -<P> -<PRE> - if (argc != 3) - { - printf("Two arguments required: a regex and a " - "subject string\n"); - return 1; - } -</PRE> -</P> -<P> -<PRE> - /* Compile the regular expression in the first argument */ -</PRE> -</P> -<P> -<PRE> - re = pcre_compile( - argv[1], /* the pattern */ - 0, /* default options */ - &error, /* for error message */ - &erroffset, /* for error offset */ - NULL); /* use default character tables */ -</PRE> -</P> -<P> -<PRE> - /* Compilation failed: print the error message and exit */ -</PRE> -</P> -<P> -<PRE> - if (re == NULL) - { - printf("PCRE compilation failed at offset %d: %s\n", - erroffset, error); - return 1; - } -</PRE> -</P> -<P> -<PRE> - /* Compilation succeeded: match the subject in the second - argument */ -</PRE> -</P> -<P> -<PRE> - rc = pcre_exec( - re, /* the compiled pattern */ - NULL, /* we didn't study the pattern */ - argv[2], /* the subject string */ - (int)strlen(argv[2]), /* the length of the subject */ - 0, /* start at offset 0 in the subject */ - 0, /* default options */ - ovector, /* vector for substring information */ - OVECCOUNT); /* number of elements in the vector */ -</PRE> -</P> -<P> -<PRE> - /* Matching failed: handle error cases */ -</PRE> -</P> -<P> -<PRE> - if (rc < 0) - { - switch(rc) - { - case PCRE_ERROR_NOMATCH: printf("No match\n"); break; - /* - Handle other special cases if you like - */ - default: printf("Matching error %d\n", rc); break; - } - return 1; - } -</PRE> -</P> -<P> -<PRE> - /* Match succeded */ -</PRE> -</P> -<P> -<PRE> - printf("Match succeeded\n"); -</PRE> -</P> -<P> -<PRE> - /* The output vector wasn't big enough */ -</PRE> -</P> -<P> -<PRE> - if (rc == 0) - { - rc = OVECCOUNT/3; - printf("ovector only has room for %d captured " - substrings\n", rc - 1); - } -</PRE> -</P> -<P> -<PRE> - /* Show substrings stored in the output vector */ -</PRE> -</P> -<P> -<PRE> - for (i = 0; i < rc; i++) - { - char *substring_start = argv[2] + ovector[2*i]; - int substring_length = ovector[2*i+1] - ovector[2*i]; - printf("%2d: %.*s\n", i, substring_length, - substring_start); - } -</PRE> -</P> -<P> -<PRE> - return 0; - } -</PRE> -</P> -<LI><A NAME="SEC32" HREF="#TOC1">AUTHOR</A> -<P> -Philip Hazel <ph10@cam.ac.uk> -<BR> -University Computing Service, -<BR> -New Museums Site, -<BR> -Cambridge CB2 3QG, England. -<BR> -Phone: +44 1223 334714 -</P> -<P> -Last updated: 15 August 2001 -<BR> -Copyright (c) 1997-2001 University of Cambridge. diff --git a/ext/pcre/pcrelib/doc/pcre.txt b/ext/pcre/pcrelib/doc/pcre.txt index 46ede59754..1ec5f2ca61 100644 --- a/ext/pcre/pcrelib/doc/pcre.txt +++ b/ext/pcre/pcrelib/doc/pcre.txt @@ -1,9 +1,303 @@ +This file contains a concatenation of the PCRE man pages, converted to plain +text format for ease of searching with a text editor, or for use on systems +that do not have a man page processor. The small individual files that give +synopses of each function in the library have not been included. There are +separate text files for the pcregrep and pcretest commands. +----------------------------------------------------------------------------- + NAME - pcre - Perl-compatible regular expressions. + PCRE - Perl-compatible regular expressions + + +DESCRIPTION + + The PCRE library is a set of functions that implement regu- + lar expression pattern matching using the same syntax and + semantics as Perl, with just a few differences. The current + implementation of PCRE (release 4.x) corresponds approxi- + mately with Perl 5.8, including support for UTF-8 encoded + strings. However, this support has to be explicitly + enabled; it is not the default. + + PCRE is written in C and released as a C library. However, a + number of people have written wrappers and interfaces of + various kinds. A C++ class is included in these contribu- + tions, which can be found in the Contrib directory at the + primary FTP site, which is: + + ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre + + Details of exactly which Perl regular expression features + are and are not supported by PCRE are given in separate + documents. See the pcrepattern and pcrecompat pages. + + Some features of PCRE can be included, excluded, or changed + when the library is built. The pcre_config() function makes + it possible for a client to discover which features are + available. Documentation about building PCRE for various + operating systems can be found in the README file in the + source distribution. + + +USER DOCUMENTATION + + The user documentation for PCRE has been split up into a + number of different sections. In the "man" format, each of + these is a separate "man page". In the HTML format, each is + a separate page, linked from the index page. In the plain + text format, all the sections are concatenated, for ease of + searching. The sections are as follows: + + pcre this document + pcreapi details of PCRE's native API + pcrebuild options for building PCRE + pcrecallout details of the callout feature + pcrecompat discussion of Perl compatibility + pcregrep description of the pcregrep command + pcrepattern syntax and semantics of supported + regular expressions + pcreperform discussion of performance issues + pcreposix the POSIX-compatible API + pcresample discussion of the sample program + pcretest the pcretest testing command + + In addition, in the "man" and HTML formats, there is a short + page for each library function, listing its arguments and + results. + + +LIMITATIONS + + There are some size limitations in PCRE but it is hoped that + they will never in practice be relevant. + + The maximum length of a compiled pattern is 65539 (sic) + bytes if PCRE is compiled with the default internal linkage + size of 2. If you want to process regular expressions that + are truly enormous, you can compile PCRE with an internal + linkage size of 3 or 4 (see the README file in the source + distribution and the pcrebuild documentation for details). + If these cases the limit is substantially larger. However, + the speed of execution will be slower. + + All values in repeating quantifiers must be less than 65536. + The maximum number of capturing subpatterns is 65535. + + There is no limit to the number of non-capturing subpat- + terns, but the maximum depth of nesting of all kinds of + parenthesized subpattern, including capturing subpatterns, + assertions, and other types of subpattern, is 200. + + The maximum length of a subject string is the largest posi- + tive number that an integer variable can hold. However, PCRE + uses recursion to handle subpatterns and indefinite repeti- + tion. This means that the available stack space may limit + the size of a subject string that can be processed by cer- + tain patterns. + + +UTF-8 SUPPORT + + Starting at release 3.3, PCRE has had some support for char- + acter strings encoded in the UTF-8 format. For release 4.0 + this has been greatly extended to cover most common require- + ments. + + In order process UTF-8 strings, you must build PCRE to + include UTF-8 support in the code, and, in addition, you + must call pcre_compile() with the PCRE_UTF8 option flag. + When you do this, both the pattern and any subject strings + that are matched against it are treated as UTF-8 strings + instead of just strings of bytes. + + If you compile PCRE with UTF-8 support, but do not use it at + run time, the library will be a bit bigger, but the addi- + tional run time overhead is limited to testing the PCRE_UTF8 + flag in several places, so should not be very large. + + The following comments apply when PCRE is running in UTF-8 + mode: + + 1. PCRE assumes that the strings it is given contain valid + UTF-8 codes. It does not diagnose invalid UTF-8 strings. If + you pass invalid UTF-8 strings to PCRE, the results are + undefined. + + 2. In a pattern, the escape sequence \x{...}, where the con- + tents of the braces is a string of hexadecimal digits, is + interpreted as a UTF-8 character whose code number is the + given hexadecimal number, for example: \x{1234}. If a non- + hexadecimal digit appears between the braces, the item is + not recognized. This escape sequence can be used either as + a literal, or within a character class. + + 3. The original hexadecimal escape sequence, \xhh, matches a + two-byte UTF-8 character if the value is greater than 127. + + 4. Repeat quantifiers apply to complete UTF-8 characters, + not to individual bytes, for example: \x{100}{3}. + + 5. The dot metacharacter matches one UTF-8 character instead + of a single byte. + + 6. The escape sequence \C can be used to match a single byte + in UTF-8 mode, but its use can lead to some strange effects. + + 7. The character escapes \b, \B, \d, \D, \s, \S, \w, and \W + correctly test characters of any code value, but the charac- + ters that PCRE recognizes as digits, spaces, or word charac- + ters remain the same set as before, all with values less + than 256. + + 8. Case-insensitive matching applies only to characters + whose values are less than 256. PCRE does not support the + notion of "case" for higher-valued characters. + + 9. PCRE does not support the use of Unicode tables and pro- + perties or the Perl escapes \p, \P, and \X. + + +AUTHOR + + Philip Hazel <ph10@cam.ac.uk> + University Computing Service, + Cambridge CB2 3QG, England. + Phone: +44 1223 334714 + +Last updated: 04 February 2003 +Copyright (c) 1997-2003 University of Cambridge. +----------------------------------------------------------------------------- + +NAME + PCRE - Perl-compatible regular expressions + + +PCRE BUILD-TIME OPTIONS + + This document describes the optional features of PCRE that + can be selected when the library is compiled. They are all + selected, or deselected, by providing options to the config- + ure script which is run before the make command. The com- + plete list of options for configure (which includes the + standard ones such as the selection of the installation + directory) can be obtained by running + + ./configure --help + + The following sections describe certain options whose names + begin with --enable or --disable. These settings specify + changes to the defaults for the configure command. Because + of the way that configure works, --enable and --disable + always come in pairs, so the complementary option always + exists as well, but as it specifies the default, it is not + described. + + +UTF-8 SUPPORT + + To build PCRE with support for UTF-8 character strings, add + + --enable-utf8 + + to the configure command. Of itself, this does not make PCRE + treat strings as UTF-8. As well as compiling PCRE with this + option, you also have have to set the PCRE_UTF8 option when + you call the pcre_compile() function. + + +CODE VALUE OF NEWLINE + + By default, PCRE treats character 10 (linefeed) as the new- + line character. This is the normal newline character on + Unix-like systems. You can compile PCRE to use character 13 + (carriage return) instead by adding + + --enable-newline-is-cr + + to the configure command. For completeness there is also a + --enable-newline-is-lf option, which explicitly specifies + linefeed as the newline character. + + +BUILDING SHARED AND STATIC LIBRARIES + + The PCRE building process uses libtool to build both shared + and static Unix libraries by default. You can suppress one + of these by adding one of + + --disable-shared + --disable-static + to the configure command, as required. -SYNOPSIS +POSIX MALLOC USAGE + + When PCRE is called through the POSIX interface (see the + pcreposix documentation), additional working storage is + required for holding the pointers to capturing substrings + because PCRE requires three integers per substring, whereas + the POSIX interface provides only two. If the number of + expected substrings is small, the wrapper function uses + space on the stack, because this is faster than using mal- + loc() for each call. The default threshold above which the + stack is no longer used is 10; it can be changed by adding a + setting such as + + --with-posix-malloc-threshold=20 + + to the configure command. + + +LIMITING PCRE RESOURCE USAGE + + Internally, PCRE has a function called match() which it + calls repeatedly (possibly recursively) when performing a + matching operation. By limiting the number of times this + function may be called, a limit can be placed on the + resources used by a single call to pcre_exec(). The limit + can be changed at run time, as described in the pcreapi + documentation. The default is 10 million, but this can be + changed by adding a setting such as + + --with-match-limit=500000 + + to the configure command. + + +HANDLING VERY LARGE PATTERNS + + Within a compiled pattern, offset values are used to point + from one part to another (for example, from an opening + parenthesis to an alternation metacharacter). By default + two-byte values are used for these offsets, leading to a + maximum size for a compiled pattern of around 64K. This is + sufficient to handle all but the most gigantic patterns. + Nevertheless, some people do want to process enormous pat- + terns, so it is possible to compile PCRE to use three-byte + or four-byte offsets by adding a setting such as + + --with-link-size=3 + + to the configure command. The value given must be 2, 3, or + 4. Using longer offsets slows down the operation of PCRE + because it has to load additional bytes when handling them. + + If you build PCRE with an increased link size, test 2 (and + test 5 if you are using UTF-8) will fail. Part of the output + of these tests is a representation of the compiled pattern, + and this changes with the link size. + +Last updated: 21 January 2003 +Copyright (c) 1997-2003 University of Cambridge. +----------------------------------------------------------------------------- + +NAME + PCRE - Perl-compatible regular expressions + + +SYNOPSIS OF PCRE API + #include <pcre.h> pcre *pcre_compile(const char *pattern, int options, @@ -17,10 +311,23 @@ SYNOPSIS const char *subject, int length, int startoffset, int options, int *ovector, int ovecsize); + int pcre_copy_named_substring(const pcre *code, + const char *subject, int *ovector, + int stringcount, const char *stringname, + char *buffer, int buffersize); + int pcre_copy_substring(const char *subject, int *ovector, int stringcount, int stringnumber, char *buffer, int buffersize); + int pcre_get_named_substring(const pcre *code, + const char *subject, int *ovector, + int stringcount, const char *stringname, + const char **stringptr); + + int pcre_get_stringnumber(const pcre *code, + const char *name); + int pcre_get_substring(const char *subject, int *ovector, int stringcount, int stringnumber, const char **stringptr); @@ -37,27 +344,21 @@ SYNOPSIS int pcre_fullinfo(const pcre *code, const pcre_extra *extra, int what, void *where); + int pcre_info(const pcre *code, int *optptr, *firstcharptr); + int pcre_config(int what, void *where); + char *pcre_version(void); void *(*pcre_malloc)(size_t); void (*pcre_free)(void *); + int (*pcre_callout)(pcre_callout_block *); - -DESCRIPTION - The PCRE library is a set of functions that implement regu- - lar expression pattern matching using the same syntax and - semantics as Perl 5, with just a few differences (see - - below). The current implementation corresponds to Perl - 5.005, with some additional features from later versions. - This includes some experimental, incomplete support for - UTF-8 encoded strings. Details of exactly what is and what - is not supported are given below. +PCRE API PCRE has its own native API, which is described in this document. There is also a set of wrapper functions that @@ -76,14 +377,20 @@ DESCRIPTION The functions pcre_compile(), pcre_study(), and pcre_exec() are used for compiling and matching regular expressions. A sample program that demonstrates the simplest way of using - them is given in the file pcredemo.c. The last section of - this man page describes how to run it. - - The functions pcre_copy_substring(), pcre_get_substring(), - and pcre_get_substring_list() are convenience functions for - extracting captured substrings from a matched subject - string; pcre_free_substring() and pcre_free_substring_list() - are also provided, to free the memory used for extracted + them is given in the file pcredemo.c. The pcresample docu- + mentation describes how to run it. + + There are convenience functions for extracting captured sub- + strings from a matched subject string. They are: + + pcre_copy_substring() + pcre_copy_named_substring() + pcre_get_substring() + pcre_get_named_substring() + pcre_get_substring_list() + + pcre_free_substring() and pcre_free_substring_list() are + also provided, to free the memory used for extracted strings. The function pcre_maketables() is used (optionally) to build @@ -104,12 +411,18 @@ DESCRIPTION replace them if it wishes to intercept the calls. This should be done before calling any PCRE functions. + The global variable pcre_callout initially contains NULL. It + can be set by the caller to a "callout" function, which PCRE + will then call at specified points during a matching opera- + tion. Details are given in the pcrecallout documentation. + +MULTITHREADING -MULTI-THREADING The PCRE functions can be used in multi-threading applica- tions, with the proviso that the memory management functions - pointed to by pcre_malloc and pcre_free are shared by all + pointed to by pcre_malloc and pcre_free, and the callout + function pointed to by pcre_callout, are shared by all threads. The compiled form of a regular expression is not altered @@ -117,8 +430,62 @@ MULTI-THREADING used by several threads at once. +CHECKING BUILD-TIME OPTIONS + + int pcre_config(int what, void *where); + + The function pcre_config() makes it possible for a PCRE + client to discover which optional features have been com- + piled into the PCRE library. The pcrebuild documentation has + more details about these optional features. + + The first argument for pcre_config() is an integer, specify- + ing which information is required; the second argument is a + pointer to a variable into which the information is placed. + The following information is available: + + PCRE_CONFIG_UTF8 + + The output is an integer that is set to one if UTF-8 support + is available; otherwise it is set to zero. + + PCRE_CONFIG_NEWLINE + + The output is an integer that is set to the value of the + code that is used for the newline character. It is either + linefeed (10) or carriage return (13), and should normally + be the standard character for your operating system. + + PCRE_CONFIG_LINK_SIZE + + The output is an integer that contains the number of bytes + used for internal linkage in compiled regular expressions. + The value is 2, 3, or 4. Larger values allow larger regular + expressions to be compiled, at the expense of slower match- + ing. The default value of 2 is sufficient for all but the + most massive patterns, since it allows the compiled pattern + to be up to 64K in size. + + PCRE_CONFIG_POSIX_MALLOC_THRESHOLD + + The output is an integer that contains the threshold above + which the POSIX interface uses malloc() for output vectors. + Further details are given in the pcreposix documentation. + + PCRE_CONFIG_MATCH_LIMIT + + The output is an integer that gives the default limit for + the number of internal matching function calls in a + pcre_exec() execution. Further details are given with + pcre_exec() below. + COMPILING A PATTERN + + pcre *pcre_compile(const char *pattern, int options, + const char **errptr, int *erroffset, + const unsigned char *tableptr); + The function pcre_compile() is called to compile a pattern into an internal form. The pattern is a C string terminated by a binary zero, and is passed in the argument pattern. A @@ -134,24 +501,16 @@ COMPILING A PATTERN pcre data block is not fully relocatable, because it con- tains a copy of the tableptr argument, which is an address (see below). - - The size of a compiled pattern is roughly proportional to - the length of the pattern string, except that each character - class (other than those containing just a single character, - negated or not) requires 33 bytes, and repeat quantifiers - with a minimum greater than one or a bounded maximum cause - the relevant portions of the compiled pattern to be repli- - cated. - The options argument contains independent bits that affect the compilation. It should be zero if no options are required. Some of the options, in particular, those that are compatible with Perl, can also be set and unset from within the pattern (see the detailed description of regular expres- - sions below). For these options, the contents of the options - argument specifies their initial settings at the start of - compilation and execution. The PCRE_ANCHORED option can be - set at the time of matching as well as at compile time. + sions in the pcrepattern documentation). For these options, + the contents of the options argument specifies their initial + settings at the start of compilation and execution. The + PCRE_ANCHORED option can be set at the time of matching as + well as at compile time. If errptr is NULL, pcre_compile() returns NULL immediately. Otherwise, if compilation of a pattern fails, pcre_compile() @@ -181,21 +540,23 @@ COMPILING A PATTERN &erroffset, /* for error offset */ NULL); /* use default character tables */ - The following option bits are defined in the header file: + The following option bits are defined: PCRE_ANCHORED If this bit is set, the pattern is forced to be "anchored", - that is, it is constrained to match only at the start of the - string which is being searched (the "subject string"). This - effect can also be achieved by appropriate constructs in the - pattern itself, which is the only way to do it in Perl. + that is, it is constrained to match only at the first match- + ing point in the string which is being searched (the "sub- + ject string"). This effect can also be achieved by appropri- + ate constructs in the pattern itself, which is the only way + to do it in Perl. PCRE_CASELESS If this bit is set, letters in the pattern match both upper and lower case letters. It is equivalent to Perl's /i - option. + option, and it can be changed within a pattern by a (?i) + option setting. PCRE_DOLLAR_ENDONLY @@ -205,28 +566,34 @@ COMPILING A PATTERN character if it is a newline (but not before any other new- lines). The PCRE_DOLLAR_ENDONLY option is ignored if PCRE_MULTILINE is set. There is no equivalent to this option - in Perl. + in Perl, and no way to set it within a pattern. PCRE_DOTALL If this bit is set, a dot metacharater in the pattern matches all characters, including newlines. Without it, new- lines are excluded. This option is equivalent to Perl's /s - option. A negative class such as [^a] always matches a new- - line character, independent of the setting of this option. + option, and it can be changed within a pattern by a (?s) + option setting. A negative class such as [^a] always matches + a newline character, independent of the setting of this + option. PCRE_EXTENDED If this bit is set, whitespace data characters in the pat- tern are totally ignored except when escaped or inside a - character class, and characters between an unescaped # out- - side a character class and the next newline character, + character class. Whitespace does not include the VT charac- + ter (code 11). In addition, characters between an unescaped + # outside a character class and the next newline character, inclusive, are also ignored. This is equivalent to Perl's /x - option, and makes it possible to include comments inside - complicated patterns. Note, however, that this applies only - to data characters. Whitespace characters may never appear + option, and it can be changed within a pattern by a (?x) + option setting. + + This option makes it possible to include comments inside + complicated patterns. Note, however, that this applies only + to data characters. Whitespace characters may never appear within special character sequences in a pattern, for example - within the sequence (?( which introduces a conditional sub- + within the sequence (?( which introduces a conditional sub- pattern. PCRE_EXTRA @@ -256,9 +623,19 @@ COMPILING A PATTERN of line" constructs match immediately following or immedi- ately before any newline in the subject string, respec- tively, as well as at the very start and end. This is - equivalent to Perl's /m option. If there are no "\n" charac- - ters in a subject string, or no occurrences of ^ or $ in a - pattern, setting PCRE_MULTILINE has no effect. + equivalent to Perl's /m option, and it can be changed within + a pattern by a (?m) option setting. If there are no "\n" + characters in a subject string, or no occurrences of ^ or $ + in a pattern, setting PCRE_MULTILINE has no effect. + + PCRE_NO_AUTO_CAPTURE + + If this option is set, it disables the use of numbered cap- + turing parentheses in the pattern. Any opening parenthesis + that is not followed by ? behaves as if it were followed by + ?: but named parentheses can still be used for capturing + (and they acquire numbers in the usual way). There is no + equivalent of this option in Perl. PCRE_UNGREEDY @@ -270,34 +647,47 @@ COMPILING A PATTERN PCRE_UTF8 This option causes PCRE to regard both the pattern and the - subject as strings of UTF-8 characters instead of just byte - strings. However, it is available only if PCRE has been - built to include UTF-8 support. If not, the use of this - option provokes an error. Support for UTF-8 is new, experi- - mental, and incomplete. Details of exactly what it entails - are given below. - + subject as strings of UTF-8 characters instead of single- + byte character strings. However, it is available only if + PCRE has been built to include UTF-8 support. If not, the + use of this option provokes an error. Details of how this + option changes the behaviour of PCRE are given in the sec- + tion on UTF-8 support in the main pcre page. STUDYING A PATTERN + + pcre_extra *pcre_study(const pcre *code, int options, + const char **errptr); + When a pattern is going to be used several times, it is worth spending more time analyzing it in order to speed up the time taken for matching. The function pcre_study() takes - a pointer to a compiled pattern as its first argument, and - returns a pointer to a pcre_extra block (another typedef for - a structure with hidden contents) containing additional - information about the pattern; this can be passed to - pcre_exec(). If no additional information is available, NULL - is returned. + a pointer to a compiled pattern as its first argument. If + studing the pattern produces additional information that + will help speed up matching, pcre_study() returns a pointer + to a pcre_extra block, in which the study_data field points + to the results of the study. + + The returned value from a pcre_study() can be passed + directly to pcre_exec(). However, the pcre_extra block also + contains other fields that can be set by the caller before + the block is passed; these are described below. If studying + the pattern does not produce any additional information, + pcre_study() returns NULL. In that circumstance, if the cal- + ling program wants to pass some of the other fields to + pcre_exec(), it must set up its own pcre_extra block. The second argument contains option bits. At present, no options are defined for pcre_study(), and this argument should always be zero. - The third argument for pcre_study() is a pointer to an error - message. If studying succeeds (even if no data is returned), - the variable it points to is set to NULL. Otherwise it - points to a textual error message. + The third argument for pcre_study() is a pointer for an + error message. If studying succeeds (even if no data is + returned), the variable it points to is set to NULL. Other- + wise it points to a textual error message. You should there- + fore test the error pointer for NULL after calling + pcre_study(), to be sure that it has run successfully. This is a typical call to pcre_study(): @@ -313,15 +703,16 @@ STUDYING A PATTERN created. - LOCALE SUPPORT + PCRE handles caseless matching, and determines whether char- acters are letters, digits, or whatever, by reference to a - set of tables. The library contains a default set of tables - which is created in the default C locale when PCRE is com- - piled. This is used when the final argument of - pcre_compile() is NULL, and is sufficient for many applica- - tions. + set of tables. When running in UTF-8 mode, this applies only + to characters with codes less than 256. The library contains + a default set of tables that is created in the default C + locale when PCRE is compiled. This is used when the final + argument of pcre_compile() is NULL, and is sufficient for + many applications. An alternative set of tables can, however, be supplied. Such tables are built by calling the pcre_maketables() function, @@ -339,7 +730,7 @@ LOCALE SUPPORT The tables are built in memory that is obtained via pcre_malloc. The pointer that is passed to pcre_compile is saved with the compiled pattern, and the same tables are - used via this pointer by pcre_study() and pcre_exec(). Thus + used via this pointer by pcre_study() and pcre_exec(). Thus, for any single pattern, compilation, studying and matching all happen in the same locale, but different patterns can be compiled in different locales. It is the caller's responsi- @@ -347,8 +738,11 @@ LOCALE SUPPORT remains available for as long as it is needed. - INFORMATION ABOUT A PATTERN + + int pcre_fullinfo(const pcre *code, const pcre_extra *extra, + int what, void *where); + The pcre_fullinfo() function returns information about a compiled pattern. It replaces the obsolete pcre_info() func- tion, which is nevertheless retained for backwards compabil- @@ -358,9 +752,9 @@ INFORMATION ABOUT A PATTERN compiled pattern. The second argument is the result of pcre_study(), or NULL if the pattern was not studied. The third argument specifies which piece of information is - required, while the fourth argument is a pointer to a vari- - able to receive the data. The yield of the function is zero - for success, or one of the following negative numbers: + required, and the fourth argument is a pointer to a variable + to receive the data. The yield of the function is zero for + success, or one of the following negative numbers: PCRE_ERROR_NULL the argument code was NULL the argument where was NULL @@ -381,40 +775,25 @@ INFORMATION ABOUT A PATTERN The possible values for the third argument are defined in pcre.h, and are as follows: - PCRE_INFO_OPTIONS - - Return a copy of the options with which the pattern was com- - piled. The fourth argument should point to an unsigned long - int variable. These option bits are those specified in the - call to pcre_compile(), modified by any top-level option - settings within the pattern itself, and with the - PCRE_ANCHORED bit forcibly set if the form of the pattern - implies that it can match only at the start of a subject - string. - - PCRE_INFO_SIZE + PCRE_INFO_BACKREFMAX - Return the size of the compiled pattern, that is, the value - that was passed as the argument to pcre_malloc() when PCRE - was getting memory in which to place the compiled data. The - fourth argument should point to a size_t variable. + Return the number of the highest back reference in the pat- + tern. The fourth argument should point to an int variable. + Zero is returned if there are no back references. PCRE_INFO_CAPTURECOUNT Return the number of capturing subpatterns in the pattern. The fourth argument should point to an int variable. - PCRE_INFO_BACKREFMAX - - Return the number of the highest back reference in the pat- - tern. The fourth argument should point to an int variable. - Zero is returned if there are no back references. + PCRE_INFO_FIRSTBYTE - PCRE_INFO_FIRSTCHAR + Return information about the first byte of any matched + string, for a non-anchored pattern. (This option used to be + called PCRE_INFO_FIRSTCHAR; the old name is still recognized + for backwards compatibility.) - Return information about the first character of any matched - string, for a non-anchored pattern. If there is a fixed - first character, e.g. from a pattern such as + If there is a fixed first byte, e.g. from a pattern such as (cat|cow|coyote), it is returned in the integer pointed to by where. Otherwise, if either @@ -426,27 +805,112 @@ INFORMATION ABOUT A PATTERN anchored), -1 is returned, indicating that the pattern matches only at - the start of a subject string or after any "\n" within the - string. Otherwise -2 is returned. For anchored patterns, -2 - is returned. + the start of a subject string or after any newline within + the string. Otherwise -2 is returned. For anchored patterns, + -2 is returned. PCRE_INFO_FIRSTTABLE If the pattern was studied, and this resulted in the con- - struction of a 256-bit table indicating a fixed set of char- - acters for the first character in any matching string, a - pointer to the table is returned. Otherwise NULL is - returned. The fourth argument should point to an unsigned - char * variable. + struction of a 256-bit table indicating a fixed set of bytes + for the first byte in any matching string, a pointer to the + table is returned. Otherwise NULL is returned. The fourth + argument should point to an unsigned char * variable. PCRE_INFO_LASTLITERAL - For a non-anchored pattern, return the value of the right- - most literal character which must exist in any matched - string, other than at its start. The fourth argument should - point to an int variable. If there is no such character, or - if the pattern is anchored, -1 is returned. For example, for - the pattern /a\d+z\d+/ the returned value is 'z'. + Return the value of the rightmost literal byte that must + exist in any matched string, other than at its start, if + such a byte has been recorded. The fourth argument should + point to an int variable. If there is no such byte, -1 is + returned. For anchored patterns, a last literal byte is + recorded only if it follows something of variable length. + For example, for the pattern /^a\d+z\d+/ the returned value + is "z", but for /^a\dz\d/ the returned value is -1. + + PCRE_INFO_NAMECOUNT + PCRE_INFO_NAMEENTRYSIZE + PCRE_INFO_NAMETABLE + + PCRE supports the use of named as well as numbered capturing + parentheses. The names are just an additional way of identi- + fying the parentheses, which still acquire a number. A + caller that wants to extract data from a named subpattern + must convert the name to a number in order to access the + correct pointers in the output vector (described with + pcre_exec() below). In order to do this, it must first use + these three values to obtain the name-to-number mapping + table for the pattern. + + The map consists of a number of fixed-size entries. + PCRE_INFO_NAMECOUNT gives the number of entries, and + PCRE_INFO_NAMEENTRYSIZE gives the size of each entry; both + of these return an int value. The entry size depends on the + length of the longest name. PCRE_INFO_NAMETABLE returns a + pointer to the first entry of the table (a pointer to char). + The first two bytes of each entry are the number of the cap- + turing parenthesis, most significant byte first. The rest of + the entry is the corresponding name, zero terminated. The + names are in alphabetical order. For example, consider the + following pattern (assume PCRE_EXTENDED is set, so white + space - including newlines - is ignored): + + (?P<date> (?P<year>(\d\d)?\d\d) - + (?P<month>\d\d) - (?P<day>\d\d) ) + + There are four named subpatterns, so the table has four + entries, and each entry in the table is eight bytes long. + The table is as follows, with non-printing bytes shows in + hex, and undefined bytes shown as ??: + + 00 01 d a t e 00 ?? + 00 05 d a y 00 ?? ?? + 00 04 m o n t h 00 + 00 02 y e a r 00 ?? + + When writing code to extract data from named subpatterns, + remember that the length of each entry may be different for + each compiled pattern. + + PCRE_INFO_OPTIONS + + Return a copy of the options with which the pattern was com- + piled. The fourth argument should point to an unsigned long + int variable. These option bits are those specified in the + call to pcre_compile(), modified by any top-level option + settings within the pattern itself. + + A pattern is automatically anchored by PCRE if all of its + top-level alternatives begin with one of the following: + + ^ unless PCRE_MULTILINE is set + \A always + \G always + .* if PCRE_DOTALL is set and there are no back + references to the subpattern in which .* appears + + For such patterns, the PCRE_ANCHORED bit is set in the + options returned by pcre_fullinfo(). + + PCRE_INFO_SIZE + + Return the size of the compiled pattern, that is, the value + that was passed as the argument to pcre_malloc() when PCRE + was getting memory in which to place the compiled data. The + fourth argument should point to a size_t variable. + + PCRE_INFO_STUDYSIZE + + Returns the size of the data block pointed to by the + study_data field in a pcre_extra block. That is, it is the + value that was passed to pcre_malloc() when PCRE was getting + memory into which to place the data created by pcre_study(). + The fourth argument should point to a size_t variable. + + +OBSOLETE INFO FUNCTION + + int pcre_info(const pcre *code, int *optptr, *firstcharptr); The pcre_info() function is now obsolete because its inter- face is too restrictive to return all the available data @@ -465,25 +929,19 @@ INFORMATION ABOUT A PATTERN If the pattern is not anchored and the firstcharptr argument is not NULL, it is used to pass back information about the first character of any matched string (see - PCRE_INFO_FIRSTCHAR above). - + PCRE_INFO_FIRSTBYTE above). MATCHING A PATTERN - The function pcre_exec() is called to match a subject string - - - - - -SunOS 5.8 Last change: 9 - + int pcre_exec(const pcre *code, const pcre_extra *extra, + const char *subject, int length, int startoffset, + int options, int *ovector, int ovecsize); + The function pcre_exec() is called to match a subject string against a pre-compiled pattern, which is passed in the code argument. If the pattern has been studied, the result of the - study should be passed in the extra argument. Otherwise this - must be NULL. + study should be passed in the extra argument. Here is an example of a simple call to pcre_exec(): @@ -499,11 +957,62 @@ SunOS 5.8 Last change: 9 ovector, /* vector for substring information */ 30); /* number of elements in the vector */ + If the extra argument is not NULL, it must point to a + pcre_extra data block. The pcre_study() function returns + such a block (when it doesn't return NULL), but you can also + create one for yourself, and pass additional information in + it. The fields in the block are as follows: + + unsigned long int flags; + void *study_data; + unsigned long int match_limit; + void *callout_data; + + The flags field is a bitmap that specifies which of the + other fields are set. The flag bits are: + + PCRE_EXTRA_STUDY_DATA + PCRE_EXTRA_MATCH_LIMIT + PCRE_EXTRA_CALLOUT_DATA + + Other flag bits should be set to zero. The study_data field + is set in the pcre_extra block that is returned by + pcre_study(), together with the appropriate flag bit. You + should not set this yourself, but you can add to the block + by setting the other fields. + + The match_limit field provides a means of preventing PCRE + from using up a vast amount of resources when running pat- + terns that are not going to match, but which have a very + large number of possibilities in their search trees. The + classic example is the use of nested unlimited repeats. + Internally, PCRE uses a function called match() which it + calls repeatedly (sometimes recursively). The limit is + imposed on the number of times this function is called dur- + ing a match, which has the effect of limiting the amount of + recursion and backtracking that can take place. For patterns + that are not anchored, the count starts from zero for each + position in the subject string. + + The default limit for the library can be set when PCRE is + built; the default default is 10 million, which handles all + but the most extreme cases. You can reduce the default by + suppling pcre_exec() with a pcre_extra block in which + match_limit is set to a smaller value, and + PCRE_EXTRA_MATCH_LIMIT is set in the flags field. If the + limit is exceeded, pcre_exec() returns + PCRE_ERROR_MATCHLIMIT. + + The pcre_callout field is used in conjunction with the "cal- + lout" feature, which is described in the pcrecallout docu- + mentation. + The PCRE_ANCHORED option can be passed in the options argu- - ment, whose unused bits must be zero. However, if a pattern - was compiled with PCRE_ANCHORED, or turned out to be - anchored by virtue of its contents, it cannot be made - unachored at matching time. + ment, whose unused bits must be zero. This limits + pcre_exec() to matching at the first matching position. How- + ever, if a pattern was compiled with PCRE_ANCHORED, or + turned out to be anchored by virtue of its contents, it can- + not be made unachored at matching time. There are also three further options that can be set only at matching time: @@ -547,13 +1056,18 @@ SunOS 5.8 Last change: 9 advancing the starting offset (see below) and trying an ordinary match again. - The subject string is passed as a pointer in subject, a - length in length, and a starting offset in startoffset. - Unlike the pattern string, the subject may contain binary - zero characters. When the starting offset is zero, the + The subject string is passed to pcre_exec() as a pointer in + subject, a length in length, and a starting offset in star- + toffset. Unlike the pattern string, the subject may contain + binary zero bytes. When the starting offset is zero, the search for a match starts at the beginning of the subject, and this is by far the most common case. + If the pattern was compiled with the PCRE_UTF8 option, the + subject must be a sequence of bytes that is a valid UTF-8 + string. If an invalid UTF-8 string is passed, PCRE's + behaviour is not defined. + A non-zero starting offset is useful when searching for another match in the same subject by calling pcre_exec() again after a previous success. Setting startoffset differs @@ -645,8 +1159,8 @@ SunOS 5.8 Last change: 9 Note that pcre_info() can be used to find out how many cap- turing subpatterns there are in a compiled pattern. The smallest size for ovector that will allow for n captured - substrings in addition to the offsets of the substring - matched by the whole pattern is (n+1)*3. + substrings, in addition to the offsets of the substring + matched by the whole pattern, is (n+1)*3. If pcre_exec() fails, it returns a negative number. The fol- lowing are defined in the header file: @@ -686,25 +1200,56 @@ SunOS 5.8 Last change: 9 pcre_malloc() fails, this error is given. The memory is freed at the end of matching. + PCRE_ERROR_NOSUBSTRING (-7) + + This error is used by the pcre_copy_substring(), + pcre_get_substring(), and pcre_get_substring_list() func- + tions (see below). It is never returned by pcre_exec(). + + PCRE_ERROR_MATCHLIMIT (-8) + + The recursion and backtracking limit, as specified by the + match_limit field in a pcre_extra structure (or defaulted) + was reached. See the description above. + + PCRE_ERROR_CALLOUT (-9) + + This error is never generated by pcre_exec() itself. It is + provided for use by callout functions that want to yield a + distinctive error code. See the pcrecallout documentation + for details. +EXTRACTING CAPTURED SUBSTRINGS BY NUMBER + + int pcre_copy_substring(const char *subject, int *ovector, + int stringcount, int stringnumber, char *buffer, + int buffersize); + + int pcre_get_substring(const char *subject, int *ovector, + int stringcount, int stringnumber, + const char **stringptr); + + int pcre_get_substring_list(const char *subject, + int *ovector, int stringcount, const char ***listptr); -EXTRACTING CAPTURED SUBSTRINGS Captured substrings can be accessed directly by using the offsets returned by pcre_exec() in ovector. For convenience, the functions pcre_copy_substring(), pcre_get_substring(), and pcre_get_substring_list() are provided for extracting captured substrings as new, separate, zero-terminated + strings. These functions identify substrings by number. The + next section describes functions for extracting named sub- strings. A substring that contains a binary zero is correctly extracted and has a further zero added on the end, - but the result does not, of course, function as a C string. - - The first three arguments are the same for all three func- - tions: subject is the subject string which has just been - successfully matched, ovector is a pointer to the vector of - integer offsets that was passed to pcre_exec(), and - stringcount is the number of substrings that were captured - by the match, including the substring that matched the + but the result is not, of course, a C string. + + The first three arguments are the same for all three of + these functions: subject is the subject string which has + just been successfully matched, ovector is a pointer to the + vector of integer offsets that was passed to pcre_exec(), + and stringcount is the number of substrings that were cap- + tured by the match, including the substring that matched the entire regular expression. This is the value returned by pcre_exec if it is greater than zero. If pcre_exec() returned zero, indicating that it ran out of space in ovec- @@ -763,49 +1308,171 @@ EXTRACTING CAPTURED SUBSTRINGS the functions are provided. +EXTRACTING CAPTURED SUBSTRINGS BY NAME -LIMITATIONS - There are some size limitations in PCRE but it is hoped that - they will never in practice be relevant. The maximum length - of a compiled pattern is 65539 (sic) bytes. All values in - repeating quantifiers must be less than 65536. There max- - imum number of capturing subpatterns is 65535. There is no - limit to the number of non-capturing subpatterns, but the - maximum depth of nesting of all kinds of parenthesized sub- - pattern, including capturing subpatterns, assertions, and - other types of subpattern, is 200. + int pcre_copy_named_substring(const pcre *code, + const char *subject, int *ovector, + int stringcount, const char *stringname, + char *buffer, int buffersize); - The maximum length of a subject string is the largest posi- - tive number that an integer variable can hold. However, PCRE - uses recursion to handle subpatterns and indefinite repeti- - tion. This means that the available stack space may limit - the size of a subject string that can be processed by cer- - tain patterns. + int pcre_get_stringnumber(const pcre *code, + const char *name); + int pcre_get_named_substring(const pcre *code, + const char *subject, int *ovector, + int stringcount, const char *stringname, + const char **stringptr); + + To extract a substring by name, you first have to find asso- + ciated number. This can be done by calling + pcre_get_stringnumber(). The first argument is the compiled + pattern, and the second is the name. For example, for this + pattern + + ab(?<xxx>\d+)... + + the number of the subpattern called "xxx" is 1. Given the + number, you can then extract the substring directly, or use + one of the functions described in the previous section. For + convenience, there are also two functions that do the whole + job. + + Most of the arguments of pcre_copy_named_substring() and + pcre_get_named_substring() are the same as those for the + functions that extract by number, and so are not re- + described here. There are just two differences. + + First, instead of a substring number, a substring name is + given. Second, there is an extra argument, given at the + start, which is a pointer to the compiled pattern. This is + needed in order to gain access to the name-to-number trans- + lation table. + + These functions call pcre_get_stringnumber(), and if it + succeeds, they then call pcre_copy_substring() or + pcre_get_substring(), as appropriate. + +Last updated: 03 February 2003 +Copyright (c) 1997-2003 University of Cambridge. +----------------------------------------------------------------------------- + +NAME + PCRE - Perl-compatible regular expressions + + +PCRE CALLOUTS + + int (*pcre_callout)(pcre_callout_block *); + + PCRE provides a feature called "callout", which is a means + of temporarily passing control to the caller of PCRE in the + middle of pattern matching. The caller of PCRE provides an + external function by putting its entry point in the global + variable pcre_callout. By default, this variable contains + NULL, which disables all calling out. + + Within a regular expression, (?C) indicates the points at + which the external function is to be called. Different cal- + lout points can be identified by putting a number less than + 256 after the letter C. The default value is zero. For + example, this pattern has two callout points: + + (?C1)9abc(?C2)def + + During matching, when PCRE reaches a callout point (and + pcre_callout is set), the external function is called. Its + only argument is a pointer to a pcre_callout block. This + contains the following variables: + + int version; + int callout_number; + int *offset_vector; + const char *subject; + int subject_length; + int start_match; + int current_position; + int capture_top; + int capture_last; + void *callout_data; + + The version field is an integer containing the version + number of the block format. The current version is zero. The + version number may change in future if additional fields are + added, but the intention is never to remove any of the + existing fields. + + The callout_number field contains the number of the callout, + as compiled into the pattern (that is, the number after ?C). + + The offset_vector field is a pointer to the vector of + offsets that was passed by the caller to pcre_exec(). The + contents can be inspected in order to extract substrings + that have been matched so far, in the same way as for + extracting substrings after a match has completed. + The subject and subject_length fields contain copies the + values that were passed to pcre_exec(). + + The start_match field contains the offset within the subject + at which the current match attempt started. If the pattern + is not anchored, the callout function may be called several + times for different starting points. + + The current_position field contains the offset within the + subject of the current match pointer. + + The capture_top field contains the number of the highest + captured substring so far. + + The capture_last field contains the number of the most + recently captured substring. + + The callout_data field contains a value that is passed to + pcre_exec() by the caller specifically so that it can be + passed back in callouts. It is passed in the pcre_callout + field of the pcre_extra data structure. If no such data was + passed, the value of callout_data in a pcre_callout block is + NULL. There is a description of the pcre_extra structure in + the pcreapi documentation. + + + +RETURN VALUES + + The callout function returns an integer. If the value is + zero, matching proceeds as normal. If the value is greater + than zero, matching fails at the current point, but back- + tracking to test other possibilities goes ahead, just as if + a lookahead assertion had failed. If the value is less than + zero, the match is abandoned, and pcre_exec() returns the + value. + + Negative values should normally be chosen from the set of + PCRE_ERROR_xxx values. In particular, PCRE_ERROR_NOMATCH + forces a standard "no match" failure. The error number + PCRE_ERROR_CALLOUT is reserved for use by callout functions; + it will never be used by PCRE itself. + +Last updated: 21 January 2003 +Copyright (c) 1997-2003 University of Cambridge. +----------------------------------------------------------------------------- + +NAME + PCRE - Perl-compatible regular expressions DIFFERENCES FROM PERL - The differences described here are with respect to Perl - 5.005. - - 1. By default, a whitespace character is any character that - the C library function isspace() recognizes, though it is - possible to compile PCRE with alternative character type - tables. Normally isspace() matches space, formfeed, newline, - carriage return, horizontal tab, and vertical tab. Perl 5 no - longer includes vertical tab in its set of whitespace char- - acters. The \v escape that was in the Perl documentation for - a long time was never in fact recognized. However, the char- - acter itself was treated as whitespace at least up to 5.002. - In 5.004 and 5.005 it does not match \s. - - 2. PCRE does not allow repeat quantifiers on lookahead + + This document describes the differences in the ways that + PCRE and Perl handle regular expressions. The differences + described here are with respect to Perl 5.8. + + 1. PCRE does not allow repeat quantifiers on lookahead assertions. Perl permits them, but they do not mean what you might think. For example, (?!a){3} does not assert that the next three characters are not "a". It just asserts that the next character is not "a" three times. - 3. Capturing subpatterns that occur inside negative looka- + 2. Capturing subpatterns that occur inside negative looka- head assertions are counted, but their entries in the offsets vector are never set. Perl sets its numerical vari- ables from any such patterns that are matched before the @@ -813,52 +1480,58 @@ DIFFERENCES FROM PERL only if the negative lookahead assertion contains just one branch. - 4. Though binary zero characters are supported in the sub- + 3. Though binary zero characters are supported in the sub- ject string, they are not allowed in a pattern string because it is passed as a normal C string, terminated by zero. The escape sequence "\0" can be used in the pattern to represent a binary zero. - 5. The following Perl escape sequences are not supported: - \l, \u, \L, \U, \E, \Q. In fact these are implemented by - Perl's general string-handling and are not part of its pat- - tern matching engine. + 4. The following Perl escape sequences are not supported: + \l, \u, \L, \U, \P, \p, and \X. In fact these are imple- + mented by Perl's general string-handling and are not part of + its pattern matching engine. If any of these are encountered + by PCRE, an error is generated. - 6. The Perl \G assertion is not supported as it is not - relevant to single pattern matches. + 5. PCRE does support the \Q...\E escape for quoting sub- + strings. Characters in between are treated as literals. This + is slightly different from Perl in that $ and @ are also + handled as literals inside the quotes. In Perl, they cause + variable interpolation (but of course PCRE does not have + variables). Note the following examples: - 7. Fairly obviously, PCRE does not support the (?{code}) and + Pattern PCRE matches Perl matches + + \Qabc$xyz\E abc$xyz abc followed by the + contents of $xyz + \Qabc\$xyz\E abc\$xyz abc\$xyz + \Qabc\E\$\Qxyz\E abc$xyz abc$xyz + + In PCRE, the \Q...\E mechanism is not recognized inside a + character class. + + 8. Fairly obviously, PCRE does not support the (?{code}) and (?p{code}) constructions. However, there is some experimen- - tal support for recursive patterns using the non-Perl item - (?R). - - 8. There are at the time of writing some oddities in Perl - 5.005_02 concerned with the settings of captured strings - when part of a pattern is repeated. For example, matching - "aba" against the pattern /^(a(b)?)+$/ sets $2 to the value - "b", but matching "aabbaa" against /^(aa(bb)?)+$/ leaves $2 - unset. However, if the pattern is changed to - /^(aa(b(b))?)+$/ then $2 (and $3) are set. - - In Perl 5.004 $2 is set in both cases, and that is also true - of PCRE. If in the future Perl changes to a consistent state - that is different, PCRE may change to follow. - - 9. Another as yet unresolved discrepancy is that in Perl - 5.005_02 the pattern /^(a)?(?(1)a|b)+$/ matches the string - "a", whereas in PCRE it does not. However, in both Perl and - PCRE /^(a)?a/ matched against "a" leaves $1 unset. + tal support for recursive patterns using the non-Perl items + (?R), (?number) and (?P>name). Also, the PCRE "callout" + feature allows an external function to be called during pat- + tern matching. + + 9. There are some differences that are concerned with the + settings of captured strings when part of a pattern is + repeated. For example, matching "aba" against the pattern + /^(a(b)?)+$/ in Perl leaves $2 unset, but in PCRE it is set + to "b". 10. PCRE provides some extensions to the Perl regular expression facilities: (a) Although lookbehind assertions must match fixed length strings, each alternative branch of a lookbehind assertion - can match a different length of string. Perl 5.005 requires - them all to have the same length. + can match a different length of string. Perl requires them + all to have the same length. (b) If PCRE_DOLLAR_ENDONLY is set and PCRE_MULTILINE is not - set, the $ meta- character matches only at the very end of + set, the $ meta-character matches only at the very end of the string. (c) If PCRE_EXTRA is set, a backslash followed by a letter @@ -869,32 +1542,52 @@ DIFFERENCES FROM PERL not greedy, but if followed by a question mark they are. (e) PCRE_ANCHORED can be used to force a pattern to be tried - only at the start of the subject. + only at the first matching position in the subject string. - (f) The PCRE_NOTBOL, PCRE_NOTEOL, and PCRE_NOTEMPTY options - for pcre_exec() have no Perl equivalents. + (f) The PCRE_NOTBOL, PCRE_NOTEOL, PCRE_NOTEMPTY, and + PCRE_NO_AUTO_CAPTURE options for pcre_exec() have no Perl + equivalents. - (g) The (?R) construct allows for recursive pattern matching - (Perl 5.6 can do this using the (?p{code}) construct, which - PCRE cannot of course support.) + (g) The (?R), (?number), and (?P>name) constructs allows for + recursive pattern matching (Perl can do this using the + (?p{code}) construct, which PCRE cannot support.) + (h) PCRE supports named capturing substrings, using the + Python syntax. + (i) PCRE supports the possessive quantifier "++" syntax, + taken from Sun's Java package. + + (j) The (R) condition, for testing recursion, is a PCRE + extension. + + (k) The callout facility is PCRE-specific. + +Last updated: 03 February 2003 +Copyright (c) 1997-2003 University of Cambridge. +----------------------------------------------------------------------------- + +NAME + PCRE - Perl-compatible regular expressions + + +PCRE REGULAR EXPRESSION DETAILS -REGULAR EXPRESSION DETAILS The syntax and semantics of the regular expressions sup- ported by PCRE are described below. Regular expressions are also described in the Perl documentation and in a number of other books, some of which have copious examples. Jeffrey Friedl's "Mastering Regular Expressions", published by - O'Reilly (ISBN 1-56592-257), covers them in great detail. + O'Reilly, covers them in great detail. The description here + is intended as reference documentation. - The description here is intended as reference documentation. The basic operation of PCRE is on strings of bytes. However, - there is the beginnings of some support for UTF-8 character - strings. To use this support you must configure PCRE to - include it, and then call pcre_compile() with the PCRE_UTF8 - option. How this affects the pattern matching is described - in the final section of this document. + there is also support for UTF-8 character strings. To use + this support you must build PCRE to include UTF-8 support, + and then call pcre_compile() with the PCRE_UTF8 option. How + this affects the pattern matching is mentioned in several + places below. There is also a summary of UTF-8 features in + the section on UTF-8 support in the main pcre page. A regular expression is a pattern that is matched against a subject string from left to right. Most characters stand for @@ -916,9 +1609,8 @@ REGULAR EXPRESSION DETAILS Outside square brackets, the meta-characters are as follows: \ general escape character with several uses - ^ assert start of subject (or line, in multiline - mode) - $ assert end of subject (or line, in multiline mode) + ^ assert start of string (or line, in multiline mode) + $ assert end of string (or line, in multiline mode) . match any character except newline (by default) [ start character class definition | start of alternative branch @@ -929,6 +1621,7 @@ REGULAR EXPRESSION DETAILS also quantifier minimizer * 0 or more quantifier + 1 or more quantifier + also "possessive quantifier" { start min/max quantifier Part of a pattern that is in square brackets is called a @@ -938,35 +1631,55 @@ REGULAR EXPRESSION DETAILS \ general escape character ^ negate the class, but only if the first character - indicates character range + [ POSIX character class (only if followed by POSIX + syntax) ] terminates the character class The following sections describe the use of each of the meta-characters. - BACKSLASH + The backslash character has several uses. Firstly, if it is followed by a non-alphameric character, it takes away any special meaning that character may have. This use of - backslash as an escape character applies both inside and outside character classes. - For example, if you want to match a "*" character, you write - "\*" in the pattern. This applies whether or not the follow- - ing character would otherwise be interpreted as a meta- - character, so it is always safe to precede a non-alphameric - with "\" to specify that it stands for itself. In particu- - lar, if you want to match a backslash, you write "\\". + For example, if you want to match a * character, you write + \* in the pattern. This escaping action applies whether or + not the following character would otherwise be interpreted + as a meta-character, so it is always safe to precede a non- + alphameric with backslash to specify that it stands for + itself. In particular, if you want to match a backslash, you + write \\. If a pattern is compiled with the PCRE_EXTENDED option, whi- tespace in the pattern (other than in a character class) and - characters between a "#" outside a character class and the + characters between a # outside a character class and the next newline character are ignored. An escaping backslash - can be used to include a whitespace or "#" character as part + can be used to include a whitespace or # character as part of the pattern. + If you want to remove the special meaning from a sequence of + characters, you can do so by putting them between \Q and \E. + This is different from Perl in that $ and @ are handled as + literals in \Q...\E sequences in PCRE, whereas in Perl, $ + and @ cause variable interpolation. Note the following exam- + ples: + + Pattern PCRE matches Perl matches + + \Qabc$xyz\E abc$xyz abc followed by the + + contents of $xyz + \Qabc\$xyz\E abc\$xyz abc\$xyz + \Qabc\E\$\Qxyz\E abc$xyz abc$xyz + + The \Q...\E sequence is recognized both inside and outside + character classes. + A second use of backslash provides a way of encoding non- printing characters in patterns in a visible manner. There is no restriction on the appearance of non-printing charac- @@ -975,30 +1688,44 @@ BACKSLASH usually easier to use one of the following escape sequences than the binary character it represents: - \a alarm, that is, the BEL character (hex 07) - \cx "control-x", where x is any character - \e escape (hex 1B) - \f formfeed (hex 0C) - \n newline (hex 0A) - \r carriage return (hex 0D) - \t tab (hex 09) - \xhh character with hex code hh - \ddd character with octal code ddd, or backreference - - The precise effect of "\cx" is as follows: if "x" is a lower + \a alarm, that is, the BEL character (hex 07) + \cx "control-x", where x is any character + \e escape (hex 1B) + \f formfeed (hex 0C) + \n newline (hex 0A) + \r carriage return (hex 0D) + \t tab (hex 09) + \ddd character with octal code ddd, or backreference + \xhh character with hex code hh + \x{hhh..} character with hex code hhh... (UTF-8 mode only) + + The precise effect of \cx is as follows: if x is a lower case letter, it is converted to upper case. Then bit 6 of - the character (hex 40) is inverted. Thus "\cz" becomes hex - 1A, but "\c{" becomes hex 3B, while "\c;" becomes hex 7B. - - After "\x", up to two hexadecimal digits are read (letters - can be in upper or lower case). - - After "\0" up to two further octal digits are read. In both + the character (hex 40) is inverted. Thus \cz becomes hex + 1A, but \c{ becomes hex 3B, while \c; becomes hex 7B. + + After \x, from zero to two hexadecimal digits are read + (letters can be in upper or lower case). In UTF-8 mode, any + number of hexadecimal digits may appear between \x{ and }, + but the value of the character code must be less than 2**31 + (that is, the maximum hexadecimal value is 7FFFFFFF). If + characters other than hexadecimal digits appear between \x{ + and }, or if there is no terminating }, this form of escape + is not recognized. Instead, the initial \x will be inter- + preted as a basic hexadecimal escape, with no following + digits, giving a byte whose value is zero. + + Characters whose value is less than 256 can be defined by + either of the two syntaxes for \x when PCRE is in UTF-8 + mode. There is no difference in the way they are handled. + For example, \xdc is exactly the same as \x{dc}. + + After \0 up to two further octal digits are read. In both cases, if there are fewer than two digits, just those that - are present are used. Thus the sequence "\0\x\07" specifies - two binary zeros followed by a BEL character. Make sure you - supply two digits after the initial zero if the character - that follows is itself an octal digit. + are present are used. Thus the sequence \0\x\07 specifies + two binary zeros followed by a BEL character (code value 7). + Make sure you supply two digits after the initial zero if + the character that follows is itself an octal digit. The handling of a backslash followed by a digit other than 0 is complicated. Outside a character class, PCRE reads it @@ -1024,9 +1751,10 @@ BACKSLASH writing a tab \011 is always a tab \0113 is a tab followed by the character "3" - \113 is the character with octal code 113 (since there - can be no more than 99 back references) - \377 is a byte consisting entirely of 1 bits + \113 might be a back reference, otherwise the + character with octal code 113 + \377 might be a back reference, otherwise + the byte consisting entirely of 1 bits \81 is either a back reference, or a binary zero followed by the two characters "8" and "1" @@ -1034,11 +1762,12 @@ BACKSLASH duced by a leading zero, because no more than three octal digits are ever read. - All the sequences that define a single byte value can be - used both inside and outside character classes. In addition, - inside a character class, the sequence "\b" is interpreted - as the backspace character (hex 08). Outside a character - class it has a different meaning (see below). + All the sequences that define a single byte value or a sin- + gle UTF-8 character (in UTF-8 mode) can be used both inside + and outside character classes. In addition, inside a charac- + ter class, the sequence \b is interpreted as the backspace + character (hex 08). Outside a character class it has a dif- + ferent meaning (see below). The third use of backslash is for specifying generic charac- ter types: @@ -1048,20 +1777,28 @@ BACKSLASH \s any whitespace character \S any character that is not a whitespace character \w any "word" character - \W any "non-word" character + W any "non-word" character Each pair of escape sequences partitions the complete set of characters into two disjoint sets. Any given character matches one, and only one, of each pair. + In UTF-8 mode, characters with values greater than 255 never + match \d, \s, or \w, and always match \D, \S, and \W. + + For compatibility with Perl, \s does not match the VT char- + acter (code 11). This makes it different from the the POSIX + "space" class. The \s characters are HT (9), LF (10), FF + (12), CR (13), and space (32). + A "word" character is any letter or digit or the underscore character, that is, any character which can be part of a Perl "word". The definition of letters and digits is con- trolled by PCRE's character tables, and may vary if locale- - specific matching is taking place (see "Locale support" - above). For example, in the "fr" (French) locale, some char- - acter codes greater than 128 are used for accented letters, - and these are matched by \w. + specific matching is taking place (see "Locale support" in + the pcreapi page). For example, in the "fr" (French) locale, + some character codes greater than 128 are used for accented + letters, and these are matched by \w. These character type sequences can appear both inside and outside character classes. They each match one character of @@ -1076,15 +1813,15 @@ BACKSLASH for more complicated assertions is described below. The backslashed assertions are - \b word boundary - \B not a word boundary - \A start of subject (independent of multiline mode) - \Z end of subject or newline at end (independent of - multiline mode) - \z end of subject (independent of multiline mode) + \b matches at a word boundary + \B matches when not at a word boundary + \A matches at start of subject + \Z matches at end of subject or before newline at end + \z matches at end of subject + \G matches at first matching position in subject These assertions may not appear in character classes (but - note that "\b" has a different meaning, namely the backspace + note that \b has a different meaning, namely the backspace character, inside a character class). A word boundary is a position in the subject string where @@ -1092,27 +1829,51 @@ BACKSLASH match \w or \W (i.e. one matches \w and the other matches \W), or the start or end of the string if the first or last character matches \w, respectively. - The \A, \Z, and \z assertions differ from the traditional circumflex and dollar (described below) in that they only ever match at the very start and end of the subject string, - whatever options are set. They are not affected by the - PCRE_NOTBOL or PCRE_NOTEOL options. If the startoffset argu- - ment of pcre_exec() is non-zero, \A can never match. The + whatever options are set. Thus, they are independent of mul- + tiline mode. + + They are not affected by the PCRE_NOTBOL or PCRE_NOTEOL + options. If the startoffset argument of pcre_exec() is non- + zero, indicating that matching is to start at a point other + than the beginning of the subject, \A can never match. The difference between \Z and \z is that \Z matches before a newline that is the last character of the string as well as at the end of the string, whereas \z matches only at the end. + The \G assertion is true only when the current matching + position is at the start point of the match, as specified by + the startoffset argument of pcre_exec(). It differs from \A + when the value of startoffset is non-zero. By calling + pcre_exec() multiple times with appropriate arguments, you + can mimic Perl's /g option, and it is in this kind of imple- + mentation where \G can be useful. + + Note, however, that PCRE's interpretation of \G, as the + start of the current match, is subtly different from Perl's, + which defines it as the end of the previous match. In Perl, + these can be different when the previously matched string + was empty. Because PCRE does just one match at a time, it + cannot reproduce this behaviour. + + If all the alternatives of a pattern begin with \G, the + expression is anchored to the starting match position, and + the "anchored" flag is set in the compiled regular expres- + sion. CIRCUMFLEX AND DOLLAR + Outside a character class, in the default matching mode, the circumflex character is an assertion which is true only if the current matching point is at the start of the subject string. If the startoffset argument of pcre_exec() is non- - zero, circumflex can never match. Inside a character class, - circumflex has an entirely different meaning (see below). + zero, circumflex can never match if the PCRE_MULTILINE + option is unset. Inside a character class, circumflex has an + entirely different meaning (see below). Circumflex need not be the first character of the pattern if a number of alternatives are involved, but it should be the @@ -1134,19 +1895,19 @@ CIRCUMFLEX AND DOLLAR The meaning of dollar can be changed so that it matches only at the very end of the string, by setting the - PCRE_DOLLAR_ENDONLY option at compile or matching time. This - does not affect the \Z assertion. + PCRE_DOLLAR_ENDONLY option at compile time. This does not + affect the \Z assertion. The meanings of the circumflex and dollar characters are changed if the PCRE_MULTILINE option is set. When this is the case, they match immediately after and immediately - before an internal "\n" character, respectively, in addition - to matching at the start and end of the subject string. For - example, the pattern /^abc$/ matches the subject string + before an internal newline character, respectively, in addi- + tion to matching at the start and end of the subject string. + For example, the pattern /^abc$/ matches the subject string "def\nabc" in multiline mode, but not otherwise. Conse- quently, patterns that are anchored in single line mode - because all branches start with "^" are not anchored in mul- - tiline mode, and a match for circumflex is possible when the + because all branches start with ^ are not anchored in multi- + line mode, and a match for circumflex is possible when the startoffset argument of pcre_exec() is non-zero. The PCRE_DOLLAR_ENDONLY option is ignored if PCRE_MULTILINE is set. @@ -1157,20 +1918,38 @@ CIRCUMFLEX AND DOLLAR whether PCRE_MULTILINE is set or not. - FULL STOP (PERIOD, DOT) + Outside a character class, a dot in the pattern matches any one character in the subject, including a non-printing char- - acter, but not (by default) newline. If the PCRE_DOTALL - option is set, dots match newlines as well. The handling of - dot is entirely independent of the handling of circumflex - and dollar, the only relationship being that they both - involve newline characters. Dot has no special meaning in a - character class. + acter, but not (by default) newline. In UTF-8 mode, a dot + matches any UTF-8 character, which might be more than one + byte long, except (by default) for newline. If the + PCRE_DOTALL option is set, dots match newlines as well. The + handling of dot is entirely independent of the handling of + circumflex and dollar, the only relationship being that they + both involve newline characters. Dot has no special meaning + in a character class. + + +MATCHING A SINGLE BYTE + + Outside a character class, the escape sequence \C matches + any one byte, both in and out of UTF-8 mode. Unlike a dot, + it always matches a newline. The feature is provided in Perl + in order to match individual bytes in UTF-8 mode. Because + it breaks up UTF-8 characters into individual bytes, what + remains in the string may be a malformed UTF-8 string. For + this reason it is best avoided. + + PCRE does not allow \C to appear in lookbehind assertions + (see below), because in UTF-8 mode it makes it impossible to + calculate the length of the lookbehind. SQUARE BRACKETS + An opening square bracket introduces a character class, ter- minated by a closing square bracket. A closing square bracket on its own is not special. If a closing square @@ -1178,13 +1957,14 @@ SQUARE BRACKETS the first data character in the class (after an initial cir- cumflex, if present) or escaped with a backslash. - A character class matches a single character in the subject; - the character must be in the set of characters defined by - the class, unless the first character in the class is a cir- - cumflex, in which case the subject character must not be in - the set defined by the class. If a circumflex is actually - required as a member of the class, ensure it is not the - first character, or escape it with a backslash. + A character class matches a single character in the subject. + In UTF-8 mode, the character may occupy more than one byte. + A matched character must be in the set of characters defined + by the class, unless the first character in the class defin- + ition is a circumflex, in which case the subject character + must not be in the set defined by the class. If a circumflex + is actually required as a member of the class, ensure it is + not the first character, or escape it with a backslash. For example, the character class [aeiou] matches any lower case vowel, while [^aeiou] matches any character that is not @@ -1195,12 +1975,16 @@ SQUARE BRACKETS string, and fails if the current pointer is at the end of the string. + In UTF-8 mode, characters with values greater than 255 can + be included in a class as a literal string of bytes, or by + using the \x{ escaping mechanism. + When caseless matching is set, any letters in a class represent both their upper case and lower case versions, so for example, a caseless [aeiou] matches "A" as well as "a", and a caseless [^aeiou] does not match "A", whereas a case- - ful version would. - + ful version would. PCRE does not support the concept of case + for characters with values greater than 255. The newline character is never treated in any special way in character classes, whatever the setting of the PCRE_DOTALL or PCRE_MULTILINE options is. A class such as [^a] will @@ -1224,14 +2008,18 @@ SQUARE BRACKETS separate characters. The octal or hexadecimal representation of "]" can also be used to end a range. - Ranges operate in ASCII collating sequence. They can also be - used for characters specified numerically, for example - [\000-\037]. If a range that includes letters is used when - caseless matching is set, it matches the letters in either - case. For example, [W-c] is equivalent to [][\^_`wxyzabc], - matched caselessly, and if character tables for the "fr" - locale are in use, [\xc8-\xcb] matches accented E characters - in both cases. + Ranges operate in the collating sequence of character + values. They can also be used for characters specified + numerically, for example [\000-\037]. In UTF-8 mode, ranges + can include characters whose values are greater than 255, + for example [\x{100}-\x{2ff}]. + + If a range that includes letters is used when caseless + matching is set, it matches the letters in either case. For + example, [W-c] is equivalent to [][\^_`wxyzabc], matched + caselessly, and if character tables for the "fr" locale are + in use, [\xc8-\xcb] matches accented E characters in both + cases. The character types \d, \D, \s, \S, \w, and \W may also appear in a character class, and add the characters that @@ -1247,12 +2035,12 @@ SQUARE BRACKETS classes, but it does no harm if they are escaped. - POSIX CHARACTER CLASSES - Perl 5.6 (not yet released at the time of writing) is going - to support the POSIX notation for character classes, which - uses names enclosed by [: and :] within the enclosing - square brackets. PCRE supports this notation. For example, + + Perl supports the POSIX notation for character classes, + which uses names enclosed by [: and :] within the enclosing + square brackets. PCRE also supports this notation. For exam- + ple, [01[:alpha:]%] @@ -1262,22 +2050,28 @@ POSIX CHARACTER CLASSES alnum letters and digits alpha letters ascii character codes 0 - 127 + blank space or tab only cntrl control characters digit decimal digits (same as \d) graph printing characters, excluding space lower lower case letters print printing characters, including space punct printing characters, excluding letters and digits - space white space (same as \s) + space white space (not quite the same as \s) upper upper case letters word "word" characters (same as \w) xdigit hexadecimal digits - >>>>>>>>>>>>Only WORD is perl. BLANK is GNU. + The "space" characters are HT (9), LF (10), VT (11), FF + (12), CR (13), and space (32). Notice that this list + includes the VT character (code 11). This makes "space" dif- + ferent to \s, which does not include VT (for Perl compati- + bility). - The names "ascii" and "word" are Perl extensions. Another - Perl extension is negation, which is indicated by a ^ char- - acter after the colon. For example, + The name "word" is a Perl extension, and "blank" is a GNU + extension from Perl 5.8. Another Perl extension is negation, + which is indicated by a ^ character after the colon. For + example, [12[:^digit:]] @@ -1286,9 +2080,12 @@ POSIX CHARACTER CLASSES "collating element", but these are not supported, and an error is given if they are encountered. + In UTF-8 mode, characters with values greater than 255 do + not match any of the POSIX character classes. VERTICAL BAR + Vertical bar characters are used to separate alternative patterns. For example, the pattern @@ -1304,12 +2101,12 @@ VERTICAL BAR subpattern. - INTERNAL OPTION SETTING - The settings of PCRE_CASELESS, PCRE_MULTILINE, PCRE_DOTALL, - and PCRE_EXTENDED can be changed from within the pattern by - a sequence of Perl option letters enclosed between "(?" and - ")". The option letters are + + The settings of the PCRE_CASELESS, PCRE_MULTILINE, + PCRE_DOTALL, and PCRE_EXTENDED options can be changed from + within the pattern by a sequence of Perl option letters + enclosed between "(?" and ")". The option letters are i for PCRE_CASELESS m for PCRE_MULTILINE @@ -1324,28 +2121,15 @@ INTERNAL OPTION SETTING If a letter appears both before and after the hyphen, the option is unset. - The scope of these option changes depends on where in the - pattern the setting occurs. For settings that are outside - any subpattern (defined below), the effect is the same as if - the options were set or unset at the start of matching. The - following patterns all behave in exactly the same way: - - (?i)abc - a(?i)bc - ab(?i)c - abc(?i) - - which in turn is the same as compiling the pattern abc with - PCRE_CASELESS set. In other words, such "top level" set- - tings apply to the whole pattern (unless there are other - changes inside subpatterns). If there is more than one set- - ting of the same option at top level, the rightmost setting - is used. - - If an option change occurs inside a subpattern, the effect - is different. This is a change of behaviour in Perl 5.005. - An option change inside a subpattern affects only that part - of the subpattern that follows it, so + When an option change occurs at top level (that is, not + inside subpattern parentheses), the change applies to the + remainder of the pattern that follows. If the change is + placed right at the start of a pattern, PCRE extracts it + into the global options (and it will therefore show up in + data extracted by the pcre_fullinfo() function). + + An option change within a subpattern affects only that part + of the current pattern that follows it, so (a(?i)b)c @@ -1372,8 +2156,8 @@ INTERNAL OPTION SETTING even when it is at top level. It is best put at the start. - SUBPATTERNS + Subpatterns are delimited by parentheses (round brackets), which can be nested. Marking part of a pattern as a subpat- tern does two things: @@ -1406,18 +2190,19 @@ SUBPATTERNS The fact that plain parentheses fulfil two functions is not always helpful. There are often times when a grouping sub- pattern is required without a capturing requirement. If an - opening parenthesis is followed by "?:", the subpattern does - not do any capturing, and is not counted when computing the - number of any subsequent capturing subpatterns. For example, - if the string "the white queen" is matched against the pat- - tern + opening parenthesis is followed by a question mark and a + colon, the subpattern does not do any capturing, and is not + counted when computing the number of any subsequent captur- + ing subpatterns. For example, if the string "the white + queen" is matched against the pattern the ((?:red|white) (king|queen)) the captured substrings are "white queen" and "queen", and - are numbered 1 and 2. The maximum number of captured sub- - strings is 99, and the maximum number of all subpatterns, - both capturing and non-capturing, is 200. + are numbered 1 and 2. The maximum number of capturing sub- + patterns is 65535, and the maximum depth of nesting of all + subpatterns, both capturing and non-capturing, is 200. + As a convenient shorthand, if any option settings are required at the start of a non-capturing subpattern, the option letters may appear between the "?" and the ":". Thus @@ -1433,17 +2218,36 @@ SUBPATTERNS the above patterns match "SUNDAY" as well as "Saturday". +NAMED SUBPATTERNS + + Identifying capturing parentheses by number is simple, but + it can be very hard to keep track of the numbers in compli- + cated regular expressions. Furthermore, if an expression is + modified, the numbers may change. To help with the diffi- + culty, PCRE supports the naming of subpatterns, something + that Perl does not provide. The Python syntax (?P<name>...) + is used. Names consist of alphanumeric characters and under- + scores, and must be unique within a pattern. + + Named capturing parentheses are still allocated numbers as + well as names. The PCRE API provides function calls for + extracting the name-to-number translation table from a com- + piled pattern. For further details see the pcreapi documen- + tation. + REPETITION + Repetition is specified by quantifiers, which can follow any of the following items: - a single character, possibly escaped + a literal data character the . metacharacter + the \C escape sequence + escapes such as \d that match single characters a character class a back reference (see next section) - a parenthesized subpattern (unless it is an assertion - - see below) + a parenthesized subpattern (unless it is an assertion) The general repetition quantifier specifies a minimum and maximum number of permitted matches, by giving the two @@ -1469,8 +2273,13 @@ REPETITION matches exactly 8 digits. An opening curly bracket that appears in a position where a quantifier is not allowed, or one that does not match the syntax of a quantifier, is taken - as a literal character. For example, {,6} is not a - quantifier, but a literal string of four characters. + as a literal character. For example, {,6} is not a quantif- + ier, but a literal string of four characters. + + In UTF-8 mode, quantifiers apply to UTF-8 characters rather + than to individual bytes. Thus, for example, \x{100}{2} + matches two UTF-8 characters, each of which is represented + by a two-byte sequence. The quantifier {0} is permitted, causing the expression to behave as if the previous item and the quantifier were not @@ -1521,8 +2330,8 @@ REPETITION does the right thing with the C comments. The meaning of the various quantifiers is not otherwise changed, just the pre- - ferred number of matches. Do not confuse this use of - question mark with its use as a quantifier in its own right. + ferred number of matches. Do not confuse this use of ques- + tion mark with its use as a quantifier in its own right. Because it has two uses, it can sometimes appear doubled, as in @@ -1541,18 +2350,31 @@ REPETITION repeat count that is greater than 1 or with a limited max- imum, more store is required for the compiled pattern, in proportion to the size of the minimum or maximum. - If a pattern starts with .* or .{0,} and the PCRE_DOTALL option (equivalent to Perl's /s) is set, thus allowing the . to match newlines, the pattern is implicitly anchored, because whatever follows will be tried against every charac- ter position in the subject string, so there is no point in retrying the overall match at any position after the first. - PCRE treats such a pattern as though it were preceded by \A. - In cases where it is known that the subject string contains - no newlines, it is worth setting PCRE_DOTALL when the pat- - tern begins with .* in order to obtain this optimization, or - alternatively using ^ to indicate anchoring explicitly. + PCRE normally treats such a pattern as though it were pre- + ceded by \A. + + In cases where it is known that the subject string contains + no newlines, it is worth setting PCRE_DOTALL in order to + obtain this optimization, or alternatively using ^ to indi- + cate anchoring explicitly. + + However, there is one situation where the optimization can- + not be used. When .* is inside capturing parentheses that + are the subject of a backreference elsewhere in the pattern, + a match at the start may fail, and a later one succeed. Con- + sider, for example: + + (.*)abc\1 + + If the subject is "xyz123abc123" the match point is the + fourth character. For this reason, such a pattern is not + implicitly anchored. When a capturing subpattern is repeated, the value captured is the substring that matched the final iteration. For exam- @@ -1572,12 +2394,110 @@ REPETITION "b". +ATOMIC GROUPING AND POSSESSIVE QUANTIFIERS + + With both maximizing and minimizing repetition, failure of + what follows normally causes the repeated item to be re- + evaluated to see if a different number of repeats allows the + rest of the pattern to match. Sometimes it is useful to + prevent this, either to change the nature of the match, or + to cause it fail earlier than it otherwise might, when the + author of the pattern knows there is no point in carrying + on. + + Consider, for example, the pattern \d+foo when applied to + the subject line + + 123456bar + + After matching all 6 digits and then failing to match "foo", + the normal action of the matcher is to try again with only 5 + digits matching the \d+ item, and then with 4, and so on, + before ultimately failing. "Atomic grouping" (a term taken + from Jeffrey Friedl's book) provides the means for specify- + ing that once a subpattern has matched, it is not to be re- + evaluated in this way. + + If we use atomic grouping for the previous example, the + matcher would give up immediately on failing to match "foo" + the first time. The notation is a kind of special + parenthesis, starting with (?> as in this example: + + (?>\d+)bar + + This kind of parenthesis "locks up" the part of the pattern + it contains once it has matched, and a failure further into + the pattern is prevented from backtracking into it. Back- + tracking past it to previous items, however, works as nor- + mal. + + An alternative description is that a subpattern of this type + matches the string of characters that an identical stan- + dalone pattern would match, if anchored at the current point + in the subject string. + + Atomic grouping subpatterns are not capturing subpatterns. + Simple cases such as the above example can be thought of as + a maximizing repeat that must swallow everything it can. So, + while both \d+ and \d+? are prepared to adjust the number of + digits they match in order to make the rest of the pattern + match, (?>\d+) can only match an entire sequence of digits. + + Atomic groups in general can of course contain arbitrarily + complicated subpatterns, and can be nested. However, when + the subpattern for an atomic group is just a single repeated + item, as in the example above, a simpler notation, called a + "possessive quantifier" can be used. This consists of an + additional + character following a quantifier. Using this + notation, the previous example can be rewritten as + + \d++bar + + Possessive quantifiers are always greedy; the setting of the + PCRE_UNGREEDY option is ignored. They are a convenient nota- + tion for the simpler forms of atomic group. However, there + is no difference in the meaning or processing of a posses- + sive quantifier and the equivalent atomic group. + + The possessive quantifier syntax is an extension to the Perl + syntax. It originates in Sun's Java package. + + When a pattern contains an unlimited repeat inside a subpat- + tern that can itself be repeated an unlimited number of + times, the use of an atomic group is the only way to avoid + some failing matches taking a very long time indeed. The + pattern + + (\D+|<\d+>)*[!?] + + matches an unlimited number of substrings that either con- + sist of non-digits, or digits enclosed in <>, followed by + either ! or ?. When it matches, it runs quickly. However, if + it is applied to + + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + + it takes a long time before reporting failure. This is + because the string can be divided between the two repeats in + a large number of ways, and all have to be tried. (The exam- + ple used [!?] rather than a single character at the end, + because both PCRE and Perl have an optimization that allows + for fast failure when a single character is used. They + remember the last single character that is required for a + match, and fail early if it is not present in the string.) + If the pattern is changed to + + ((?>\D+)|<\d+>)*[!?] + + sequences of non-digits cannot be broken, and failure hap- + pens quickly. BACK REFERENCES + Outside a character class, a backslash followed by a digit greater than 0 (and possibly further digits) is a back - reference to a capturing subpattern earlier (i.e. to its + reference to a capturing subpattern earlier (that is, to its left) in the pattern, provided there have been that many previous capturing left parentheses. @@ -1592,7 +2512,8 @@ BACK REFERENCES A back reference matches whatever actually matched the cap- turing subpattern in the current subject string, rather than - anything matching the subpattern itself. So the pattern + anything matching the subpattern itself (see "Subpatterns as + subroutines" below for a way of doing that). So the pattern (sens|respons)e and \1ibility @@ -1607,6 +2528,11 @@ BACK REFERENCES though the original capturing subpattern is matched case- lessly. + Back references to named subpatterns use the Python syntax + (?P=name). We could rewrite the above example as follows: + + (?<p1>(?i)rah)\s+(?P=p1) + There may be more than one back reference to the same sub- pattern. If a subpattern has not actually been used in a particular match, any back references to it always fail. For @@ -1615,18 +2541,19 @@ BACK REFERENCES (a|(bc))\2 always fails if it starts to match "a" rather than "bc". - Because there may be up to 99 back references, all digits - following the backslash are taken as part of a potential - back reference number. If the pattern continues with a digit - character, some delimiter must be used to terminate the back - reference. If the PCRE_EXTENDED option is set, this can be - whitespace. Otherwise an empty comment can be used. + Because there may be many capturing parentheses in a pat- + tern, all digits following the backslash are taken as part + of a potential back reference number. If the pattern contin- + ues with a digit character, some delimiter must be used to + terminate the back reference. If the PCRE_EXTENDED option is + set, this can be whitespace. Otherwise an empty comment can + be used. A back reference that occurs inside the parentheses to which it refers fails when the subpattern is first used, so, for example, (a\1) never matches. However, such references can - be useful inside repeated subpatterns. For example, the - pattern + be useful inside repeated subpatterns. For example, the pat- + tern (a|b\1)+ @@ -1639,13 +2566,13 @@ BACK REFERENCES example above, or by a quantifier with a minimum of zero. - ASSERTIONS + An assertion is a test on the characters following or preceding the current matching point that does not actually consume any characters. The simple assertions coded as \b, - \B, \A, \Z, \z, ^ and $ are described above. More compli- - cated assertions are coded as subpatterns. There are two + \B, \A, \G, \Z, \z, ^ and $ are described above. More com- + plicated assertions are coded as subpatterns. There are two kinds: those that look ahead of the current position in the subject string, and those that look behind it. @@ -1672,6 +2599,11 @@ ASSERTIONS when the next three characters are "bar". A lookbehind assertion is needed to achieve this effect. + If you want to force a matching failure at some point in a + pattern, the most convenient way to do it is with (?!) + because an empty string always matches, so an assertion that + requires there not to be an empty string must always fail. + Lookbehind assertions start with (?<= for positive asser- tions and (?<! for negative assertions. For example, @@ -1692,8 +2624,8 @@ ASSERTIONS causes an error at compile time. Branches that match dif- ferent length strings are permitted only at the top level of a lookbehind assertion. This is an extension compared with - Perl 5.005, which requires all branches to match the same - length of string. An assertion such as + Perl (at least for 5.8), which requires all branches to + match the same length of string. An assertion such as (?<=ab(c|de)) @@ -1707,10 +2639,44 @@ ASSERTIONS alternative, to temporarily move the current position back by the fixed width and then try to match. If there are insufficient characters before the current position, the - match is deemed to fail. Lookbehinds in conjunction with - once-only subpatterns can be particularly useful for match- - ing at the ends of strings; an example is given at the end - of the section on once-only subpatterns. + match is deemed to fail. + + PCRE does not allow the \C escape (which matches a single + byte in UTF-8 mode) to appear in lookbehind assertions, + because it makes it impossible to calculate the length of + the lookbehind. + + Atomic groups can be used in conjunction with lookbehind + assertions to specify efficient matching at the end of the + subject string. Consider a simple pattern such as + + abcd$ + + when applied to a long string that does not match. Because + matching proceeds from left to right, PCRE will look for + each "a" in the subject and then see if what follows matches + the rest of the pattern. If the pattern is specified as + + ^.*abcd$ + + the initial .* matches the entire string at first, but when + this fails (because there is no following "a"), it back- + tracks to match all but the last character, then all but the + last two characters, and so on. Once again the search for + "a" covers the entire string, from right to left, so we are + no better off. However, if the pattern is written as + + ^(?>.*)(?<=abcd) + + or, equivalently, + + ^.*+(?<=abcd) + + there can be no backtracking for the .* item; it can match + only the entire string. The subsequent lookbehind assertion + does a single test on the last four characters. If it fails, + the match fails immediately. For long strings, this approach + makes a significant difference to the processing time. Several assertions (of any sort) may occur in succession. For example, @@ -1755,120 +2721,9 @@ ASSERTIONS for positive assertions, because it does not make sense for negative assertions. - Assertions count towards the maximum of 200 parenthesized - subpatterns. - - - -ONCE-ONLY SUBPATTERNS - With both maximizing and minimizing repetition, failure of - what follows normally causes the repeated item to be re- - evaluated to see if a different number of repeats allows the - rest of the pattern to match. Sometimes it is useful to - prevent this, either to change the nature of the match, or - to cause it fail earlier than it otherwise might, when the - author of the pattern knows there is no point in carrying - on. - - Consider, for example, the pattern \d+foo when applied to - the subject line - - 123456bar - - After matching all 6 digits and then failing to match "foo", - the normal action of the matcher is to try again with only 5 - digits matching the \d+ item, and then with 4, and so on, - before ultimately failing. Once-only subpatterns provide the - means for specifying that once a portion of the pattern has - matched, it is not to be re-evaluated in this way, so the - matcher would give up immediately on failing to match "foo" - the first time. The notation is another kind of special - parenthesis, starting with (?> as in this example: - - (?>\d+)bar - - This kind of parenthesis "locks up" the part of the pattern - it contains once it has matched, and a failure further into - the pattern is prevented from backtracking into it. Back- - tracking past it to previous items, however, works as nor- - mal. - - An alternative description is that a subpattern of this type - matches the string of characters that an identical stan- - dalone pattern would match, if anchored at the current point - in the subject string. - - Once-only subpatterns are not capturing subpatterns. Simple - cases such as the above example can be thought of as a max- - imizing repeat that must swallow everything it can. So, - while both \d+ and \d+? are prepared to adjust the number of - digits they match in order to make the rest of the pattern - match, (?>\d+) can only match an entire sequence of digits. - - This construction can of course contain arbitrarily compli- - cated subpatterns, and it can be nested. - - Once-only subpatterns can be used in conjunction with look- - behind assertions to specify efficient matching at the end - of the subject string. Consider a simple pattern such as - - abcd$ - - when applied to a long string which does not match. Because - matching proceeds from left to right, PCRE will look for - each "a" in the subject and then see if what follows matches - the rest of the pattern. If the pattern is specified as - - ^.*abcd$ - - the initial .* matches the entire string at first, but when - this fails (because there is no following "a"), it back- - tracks to match all but the last character, then all but the - last two characters, and so on. Once again the search for - "a" covers the entire string, from right to left, so we are - no better off. However, if the pattern is written as - - ^(?>.*)(?<=abcd) - - there can be no backtracking for the .* item; it can match - only the entire string. The subsequent lookbehind assertion - does a single test on the last four characters. If it fails, - the match fails immediately. For long strings, this approach - makes a significant difference to the processing time. - - When a pattern contains an unlimited repeat inside a subpat- - tern that can itself be repeated an unlimited number of - times, the use of a once-only subpattern is the only way to - avoid some failing matches taking a very long time indeed. - The pattern - - (\D+|<\d+>)*[!?] - - matches an unlimited number of substrings that either con- - sist of non-digits, or digits enclosed in <>, followed by - either ! or ?. When it matches, it runs quickly. However, if - it is applied to - - aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - - it takes a long time before reporting failure. This is - because the string can be divided between the two repeats in - a large number of ways, and all have to be tried. (The exam- - ple used [!?] rather than a single character at the end, - because both PCRE and Perl have an optimization that allows - for fast failure when a single character is used. They - remember the last single character that is required for a - match, and fail early if it is not present in the string.) - If the pattern is changed to - - ((?>\D+)|<\d+>)*[!?] - - sequences of non-digits cannot be broken, and failure hap- - pens quickly. - - CONDITIONAL SUBPATTERNS + It is possible to cause the matching process to obey a sub- pattern conditionally or to choose between two alternative subpatterns, depending on the result of an assertion, or @@ -1883,7 +2738,7 @@ CONDITIONAL SUBPATTERNS more than two alternatives in the subpattern, a compile-time error occurs. - There are two kinds of condition. If the text between the + There are three kinds of condition. If the text between the parentheses consists of a sequence of digits, the condition is satisfied if the capturing subpattern of that number has previously matched. The number must be greater than zero. @@ -1907,11 +2762,17 @@ CONDITIONAL SUBPATTERNS matches a sequence of non-parentheses, optionally enclosed in parentheses. - If the condition is not a sequence of digits, it must be an - assertion. This may be a positive or negative lookahead or - lookbehind assertion. Consider this pattern, again contain- - ing non-significant white space, and with the two alterna- - tives on the second line: + If the condition is the string (R), it is satisfied if a + recursive call to the pattern or subpattern has been made. + At "top level", the condition is false. This is a PCRE + extension. Recursive patterns are described in the next + section. + + If the condition is not a sequence of digits or (R), it must + be an assertion. This may be a positive or negative looka- + head or lookbehind assertion. Consider this pattern, again + containing non-significant white space, and with the two + alternatives on the second line: (?(?=[^a-z]*[a-z]) \d{2}-[a-z]{3}-\d{2} | \d{2}-\d{2}-\d{2} ) @@ -1926,8 +2787,8 @@ CONDITIONAL SUBPATTERNS letters and dd are digits. - COMMENTS + The sequence (?# marks the start of a comment which contin- ues up to the next closing parenthesis. Nested parentheses are not permitted. The characters that make up a comment @@ -1938,55 +2799,83 @@ COMMENTS ues up to the next newline character in the pattern. - RECURSIVE PATTERNS + Consider the problem of matching a string in parentheses, allowing for unlimited nested parentheses. Without the use of recursion, the best that can be done is to use a pattern that matches up to some fixed depth of nesting. It is not - possible to handle an arbitrary nesting depth. Perl 5.6 has - provided an experimental facility that allows regular - expressions to recurse (amongst other things). It does this - by interpolating Perl code in the expression at run time, - and the code can refer to the expression itself. A Perl pat- - tern to solve the parentheses problem can be created like - this: + possible to handle an arbitrary nesting depth. Perl has pro- + vided an experimental facility that allows regular expres- + sions to recurse (amongst other things). It does this by + interpolating Perl code in the expression at run time, and + the code can refer to the expression itself. A Perl pattern + to solve the parentheses problem can be created like this: $re = qr{\( (?: (?>[^()]+) | (?p{$re}) )* \)}x; The (?p{...}) item interpolates Perl code at run time, and in this case refers recursively to the pattern in which it appears. Obviously, PCRE cannot support the interpolation of - Perl code. Instead, the special item (?R) is provided for - the specific case of recursion. This PCRE pattern solves the - parentheses problem (assume the PCRE_EXTENDED option is set - so that white space is ignored): + Perl code. Instead, it supports some special syntax for + recursion of the entire pattern, and also for individual + subpattern recursion. + + The special item that consists of (? followed by a number + greater than zero and a closing parenthesis is a recursive + call of the subpattern of the given number, provided that it + occurs inside that subpattern. (If not, it is a "subroutine" + call, which is described in the next section.) The special + item (?R) is a recursive call of the entire regular expres- + sion. + + For example, this PCRE pattern solves the nested parentheses + problem (assume the PCRE_EXTENDED option is set so that + white space is ignored): \( ( (?>[^()]+) | (?R) )* \) First it matches an opening parenthesis. Then it matches any number of substrings which can either be a sequence of non- parentheses, or a recursive match of the pattern itself - (i.e. a correctly parenthesized substring). Finally there is - a closing parenthesis. + (that is a correctly parenthesized substring). Finally + there is a closing parenthesis. + + If this were part of a larger pattern, you would not want to + recurse the entire pattern, so instead you could use this: + + ( \( ( (?>[^()]+) | (?1) )* \) ) + + We have put the pattern into parentheses, and caused the + recursion to refer to them instead of the whole pattern. In + a larger pattern, keeping track of parenthesis numbers can + be tricky. It may be more convenient to use named + parentheses instead. For this, PCRE uses (?P>name), which is + an extension to the Python syntax that PCRE uses for named + parentheses (Perl does not provide named parentheses). We + could rewrite the above example as follows: + + (?<pn> \( ( (?>[^()]+) | (?P>pn) )* \) ) This particular example pattern contains nested unlimited - repeats, and so the use of a once-only subpattern for match- - ing strings of non-parentheses is important when applying - the pattern to strings that do not match. For example, when - it is applied to + repeats, and so the use of atomic grouping for matching + strings of non-parentheses is important when applying the + pattern to strings that do not match. For example, when this + pattern is applied to (aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa() - it yields "no match" quickly. However, if a once-only sub- - pattern is not used, the match runs for a very long time - indeed because there are so many different ways the + and * - repeats can carve up the subject, and all have to be tested - before failure can be reported. - - The values set for any capturing subpatterns are those from - the outermost level of the recursion at which the subpattern - value is set. If the pattern above is matched against + it yields "no match" quickly. However, if atomic grouping is + not used, the match runs for a very long time indeed because + there are so many different ways the + and * repeats can + carve up the subject, and all have to be tested before + failure can be reported. + At the end of a match, the values set for any capturing sub- + patterns are those from the outermost level of the recursion + at which the subpattern value is set. If you want to obtain + intermediate values, a callout function can be used (see + below and the pcrecallout documentation). If the pattern + above is matched against (ab(cd)ef) @@ -1996,41 +2885,120 @@ RECURSIVE PATTERNS \( ( ( (?>[^()]+) | (?R) )* ) \) ^ ^ - ^ ^ the string they capture is - "ab(cd)ef", the contents of the top level parentheses. If - there are more than 15 capturing parentheses in a pattern, - PCRE has to obtain extra memory to store data during a - recursion, which it does by using pcre_malloc, freeing it - via pcre_free afterwards. If no memory can be obtained, it - saves data for the first 15 capturing parentheses only, as - there is no way to give an out-of-memory error from within a - recursion. - - - -PERFORMANCE - Certain items that may appear in patterns are more efficient - than others. It is more efficient to use a character class - like [aeiou] than a set of alternatives such as (a|e|i|o|u). - In general, the simplest construction that provides the - required behaviour is usually the most efficient. Jeffrey - Friedl's book contains a lot of discussion about optimizing - regular expressions for efficient performance. - - When a pattern begins with .* and the PCRE_DOTALL option is - set, the pattern is implicitly anchored by PCRE, since it - can match only at the start of a subject string. However, if - PCRE_DOTALL is not set, PCRE cannot make this optimization, - because the . metacharacter does not then match a newline, - and if the subject string contains newlines, the pattern may - match from the character immediately following one of them - instead of from the very start. For example, the pattern - - (.*) second + ^ ^ + + the string they capture is "ab(cd)ef", the contents of the + top level parentheses. If there are more than 15 capturing + parentheses in a pattern, PCRE has to obtain extra memory to + store data during a recursion, which it does by using + pcre_malloc, freeing it via pcre_free afterwards. If no + memory can be obtained, the match fails with the + PCRE_ERROR_NOMEMORY error. + + Do not confuse the (?R) item with the condition (R), which + tests for recursion. Consider this pattern, which matches + text in angle brackets, allowing for arbitrary nesting. Only + digits are allowed in nested brackets (that is, when recurs- + ing), whereas any characters are permitted at the outer + level. + + < (?: (?(R) \d++ | [^<>]*+) | (?R)) * > + + In this pattern, (?(R) is the start of a conditional subpat- + tern, with two different alternatives for the recursive and + non-recursive cases. The (?R) item is the actual recursive + call. + + +SUBPATTERNS AS SUBROUTINES + + If the syntax for a recursive subpattern reference (either + by number or by name) is used outside the parentheses to + which it refers, it operates like a subroutine in a program- + ming language. An earlier example pointed out that the pat- + tern + + (sens|respons)e and \1ibility + + matches "sense and sensibility" and "response and responsi- + bility", but not "sense and responsibility". If instead the + pattern + + (sens|respons)e and (?1)ibility + + is used, it does match "sense and responsibility" as well as + the other two strings. Such references must, however, follow + the subpattern to which they refer. + + +CALLOUTS + + Perl has a feature whereby using the sequence (?{...}) + causes arbitrary Perl code to be obeyed in the middle of + matching a regular expression. This makes it possible, + amongst other things, to extract different substrings that + match the same pair of parentheses when there is a repeti- + tion. + + PCRE provides a similar feature, but of course it cannot + obey arbitrary Perl code. The feature is called "callout". + The caller of PCRE provides an external function by putting + its entry point in the global variable pcre_callout. By + default, this variable contains NULL, which disables all + calling out. + + Within a regular expression, (?C) indicates the points at + which the external function is to be called. If you want to + identify different callout points, you can put a number less + than 256 after the letter C. The default value is zero. For + example, this pattern has two callout points: + + (?C1)9abc(?C2)def + + During matching, when PCRE reaches a callout point (and + pcre_callout is set), the external function is called. It is + provided with the number of the callout, and, optionally, + one item of data originally supplied by the caller of + pcre_exec(). The callout function may cause matching to + backtrack, or to fail altogether. A complete description of + the interface to the callout function is given in the pcre- + callout documentation. + +Last updated: 03 February 2003 +Copyright (c) 1997-2003 University of Cambridge. +----------------------------------------------------------------------------- + +NAME + PCRE - Perl-compatible regular expressions + + +PCRE PERFORMANCE + + Certain items that may appear in regular expression patterns + are more efficient than others. It is more efficient to use + a character class like [aeiou] than a set of alternatives + such as (a|e|i|o|u). In general, the simplest construction + that provides the required behaviour is usually the most + efficient. Jeffrey Friedl's book contains a lot of discus- + sion about optimizing regular expressions for efficient per- + formance. + + When a pattern begins with .* not in parentheses, or in + parentheses that are not the subject of a backreference, and + the PCRE_DOTALL option is set, the pattern is implicitly + anchored by PCRE, since it can match only at the start of a + subject string. However, if PCRE_DOTALL is not set, PCRE + cannot make this optimization, because the . metacharacter + does not then match a newline, and if the subject string + contains newlines, the pattern may match from the character + immediately following one of them instead of from the very + start. For example, the pattern + + .*second matches the subject "first\nand second" (where \n stands for - a newline character) with the first captured substring being - "and". In order to do this, PCRE has to retry the match + a newline character), with the match starting at the seventh + character. In order to do this, PCRE has to retry the match starting after every newline in the subject. If you are using such a pattern with subject strings that do @@ -2053,7 +3021,6 @@ PERFORMANCE that the entire match is going to fail, PCRE has in princi- ple to try every possible variation, and this can take an extremely long time. - An optimization catches some of the more simple cases such as @@ -2073,120 +3040,248 @@ PERFORMANCE whereas the latter takes an appreciable time with strings longer than about 20 characters. +Last updated: 03 February 2003 +Copyright (c) 1997-2003 University of Cambridge. +----------------------------------------------------------------------------- +NAME + PCRE - Perl-compatible regular expressions. -UTF-8 SUPPORT - Starting at release 3.3, PCRE has some support for character - strings encoded in the UTF-8 format. This is incomplete, and - is regarded as experimental. In order to use it, you must - configure PCRE to include UTF-8 support in the code, and, in - addition, you must call pcre_compile() with the PCRE_UTF8 - option flag. When you do this, both the pattern and any sub- - ject strings that are matched against it are treated as - UTF-8 strings instead of just strings of bytes, but only in - the cases that are mentioned below. - If you compile PCRE with UTF-8 support, but do not use it at - run time, the library will be a bit bigger, but the addi- - tional run time overhead is limited to testing the PCRE_UTF8 - flag in several places, so should not be very large. +SYNOPSIS OF POSIX API + #include <pcreposix.h> - PCRE assumes that the strings it is given contain valid - UTF-8 codes. It does not diagnose invalid UTF-8 strings. If - you pass invalid UTF-8 strings to PCRE, the results are - undefined. - Running with PCRE_UTF8 set causes these changes in the way - PCRE works: + int regcomp(regex_t *preg, const char *pattern, + int cflags); - 1. In a pattern, the escape sequence \x{...}, where the con- - tents of the braces is a string of hexadecimal digits, is - interpreted as a UTF-8 character whose code number is the - given hexadecimal number, for example: \x{1234}. This - inserts from one to six literal bytes into the pattern, - using the UTF-8 encoding. If a non-hexadecimal digit appears - between the braces, the item is not recognized. + int regexec(regex_t *preg, const char *string, + size_t nmatch, regmatch_t pmatch[], int eflags); - 2. The original hexadecimal escape sequence, \xhh, generates - a two-byte UTF-8 character if its value is greater than 127. + size_t regerror(int errcode, const regex_t *preg, + char *errbuf, size_t errbuf_size); - 3. Repeat quantifiers are NOT correctly handled if they fol- - low a multibyte character. For example, \x{100}* and \xc3+ - do not work. If you want to repeat such characters, you must - enclose them in non-capturing parentheses, for example - (?:\x{100}), at present. + void regfree(regex_t *preg); - 4. The dot metacharacter matches one UTF-8 character instead - of a single byte. - 5. Unlike literal UTF-8 characters, the dot metacharacter - followed by a repeat quantifier does operate correctly on - UTF-8 characters instead of single bytes. +DESCRIPTION + + This set of functions provides a POSIX-style API to the PCRE + regular expression package. See the pcreapi documentation + for a description of the native API, which contains addi- + tional functionality. + + The functions described here are just wrapper functions that + ultimately call the PCRE native API. Their prototypes are + defined in the pcreposix.h header file, and on Unix systems + the library itself is called pcreposix.a, so can be accessed + by adding -lpcreposix to the command for linking an applica- + tion which uses them. Because the POSIX functions call the + native ones, it is also necessary to add -lpcre. + + I have implemented only those option bits that can be rea- + sonably mapped to PCRE native options. In addition, the + options REG_EXTENDED and REG_NOSUB are defined with the + value zero. They have no effect, but since programs that are + written to the POSIX interface often use them, this makes it + easier to slot in PCRE as a replacement library. Other POSIX + options are not even defined. + + When PCRE is called via these functions, it is only the API + that is POSIX-like in style. The syntax and semantics of the + regular expressions themselves are still those of Perl, sub- + ject to the setting of various PCRE options, as described + below. "POSIX-like in style" means that the API approximates + to the POSIX definition; it is not fully POSIX-compatible, + and in multi-byte encoding domains it is probably even less + compatible. + + The header for these functions is supplied as pcreposix.h to + avoid any potential clash with other POSIX libraries. It + can, of course, be renamed or aliased as regex.h, which is + the "correct" name. It provides two structure types, regex_t + for compiled internal forms, and regmatch_t for returning + captured substrings. It also defines some constants whose + names start with "REG_"; these are used for setting options + and identifying error codes. + + +COMPILING A PATTERN + + The function regcomp() is called to compile a pattern into + an internal form. The pattern is a C string terminated by a + binary zero, and is passed in the argument pattern. The preg + argument is a pointer to a regex_t structure which is used + as a base for storing information about the compiled expres- + sion. + + The argument cflags is either zero, or contains one or more + of the bits defined by the following macros: + + REG_ICASE + + The PCRE_CASELESS option is set when the expression is + passed for compilation to the native function. + + REG_NEWLINE + + The PCRE_MULTILINE option is set when the expression is + passed for compilation to the native function. Note that + this does not mimic the defined POSIX behaviour for + REG_NEWLINE (see the following section). + + In the absence of these flags, no options are passed to the + native function. This means the the regex is compiled with + PCRE default semantics. In particular, the way it handles + newline characters in the subject string is the Perl way, + not the POSIX way. Note that setting PCRE_MULTILINE has only + some of the effects specified for REG_NEWLINE. It does not + affect the way newlines are matched by . (they aren't) or by + a negative class such as [^a] (they are). + + The yield of regcomp() is zero on success, and non-zero oth- + erwise. The preg structure is filled in on success, and one + member of the structure is public: re_nsub contains the + number of capturing subpatterns in the regular expression. + Various error codes are defined in the header file. + + +MATCHING NEWLINE CHARACTERS + + This area is not simple, because POSIX and Perl take dif- + ferent views of things. It is not possible to get PCRE to + obey POSIX semantics, but then PCRE was never intended to be + a POSIX engine. The following table lists the different pos- + sibilities for matching newline characters in PCRE: + + Default Change with + + . matches newline no PCRE_DOTALL + newline matches [^a] yes not changeable + $ matches \n at end yes PCRE_DOLLARENDONLY + $ matches \n in middle no PCRE_MULTILINE + ^ matches \n in middle no PCRE_MULTILINE + + This is the equivalent table for POSIX: + + Default Change with + + . matches newline yes REG_NEWLINE + newline matches [^a] yes REG_NEWLINE + $ matches \n at end no REG_NEWLINE + $ matches \n in middle no REG_NEWLINE + ^ matches \n in middle no REG_NEWLINE + + PCRE's behaviour is the same as Perl's, except that there is + no equivalent for PCRE_DOLLARENDONLY in Perl. In both PCRE + and Perl, there is no way to stop newline from matching + [^a]. + + The default POSIX newline handling can be obtained by set- + ting PCRE_DOTALL and PCRE_DOLLARENDONLY, but there is no way + to make PCRE behave exactly as for the REG_NEWLINE action. - 4. Although the \x{...} escape is permitted in a character - class, characters whose values are greater than 255 cannot - be included in a class. - 5. A class is matched against a UTF-8 character instead of - just a single byte, but it can match only characters whose - values are less than 256. Characters with greater values - always fail to match a class. +MATCHING A PATTERN + + The function regexec() is called to match a pre-compiled + pattern preg against a given string, which is terminated by + a zero byte, subject to the options in eflags. These can be: + + REG_NOTBOL + + The PCRE_NOTBOL option is set when calling the underlying + PCRE matching function. + + REG_NOTEOL + + The PCRE_NOTEOL option is set when calling the underlying + PCRE matching function. + + The portion of the string that was matched, and also any + captured substrings, are returned via the pmatch argument, + which points to an array of nmatch structures of type + regmatch_t, containing the members rm_so and rm_eo. These + contain the offset to the first character of each substring + and the offset to the first character after the end of each + substring, respectively. The 0th element of the vector + relates to the entire portion of string that was matched; + subsequent elements relate to the capturing subpatterns of + the regular expression. Unused entries in the array have + both structure members set to -1. + + A successful match yields a zero return; various error codes + are defined in the header file, of which REG_NOMATCH is the + "expected" failure code. + + +ERROR MESSAGES - 6. Repeated classes work correctly on multiple characters. + The regerror() function maps a non-zero errorcode from + either regcomp() or regexec() to a printable message. If + preg is not NULL, the error should have arisen from the use + of that structure. A message terminated by a binary zero is + placed in errbuf. The length of the message, including the + zero, is limited to errbuf_size. The yield of the function + is the size of buffer needed to hold the whole message. - 7. Classes containing just a single character whose value is - greater than 127 (but less than 256), for example, [\x80] or - [^\x{93}], do not work because these are optimized into sin- - gle byte matches. In the first case, of course, the class - brackets are just redundant. - 8. Lookbehind assertions move backwards in the subject by a - fixed number of characters instead of a fixed number of - bytes. Simple cases have been tested to work correctly, but - there may be hidden gotchas herein. +STORAGE - 9. The character types such as \d and \w do not work - correctly with UTF-8 characters. They continue to test a - single byte. - 10. Anything not explicitly mentioned here continues to work - in bytes rather than in characters. + Compiling a regular expression causes memory to be allocated + and associated with the preg structure. The function reg- + free() frees all such memory, after which preg may no longer + be used as a compiled expression. - The following UTF-8 features of Perl 5.6 are not imple- - mented: - 1. The escape sequence \C to match a single byte. +AUTHOR + + Philip Hazel <ph10@cam.ac.uk> + University Computing Service, + Cambridge CB2 3QG, England. - 2. The use of Unicode tables and properties and escapes \p, - \P, and \X. +Last updated: 03 February 2003 +Copyright (c) 1997-2003 University of Cambridge. +----------------------------------------------------------------------------- + +NAME + PCRE - Perl-compatible regular expressions +PCRE SAMPLE PROGRAM -SAMPLE PROGRAM - The code below is a simple, complete demonstration program, - to get you started with using PCRE. This code is also sup- - plied in the file pcredemo.c in the PCRE distribution. + A simple, complete demonstration program, to get you started + with using PCRE, is supplied in the file pcredemo.c in the + PCRE distribution. The program compiles the regular expression that is its first argument, and matches it against the subject string in - its second argument. No options are set, and default charac- - ter tables are used. If matching succeeds, the program out- - puts the portion of the subject that matched, together with - the contents of any captured substrings. + its second argument. No PCRE options are set, and default + character tables are used. If matching succeeds, the program + outputs the portion of the subject that matched, together + with the contents of any captured substrings. + + If the -g option is given on the command line, the program + then goes on to check for further matches of the same regu- + lar expression in the same subject string. The logic is a + little bit tricky because of the possibility of matching an + empty string. Comments in the code explain what is going on. On a Unix system that has PCRE installed in /usr/local, you can compile the demonstration program using a command like this: - gcc -o pcredemo pcredemo.c -I/usr/local/include - -L/usr/local/lib -lpcre + gcc -o pcredemo pcredemo.c -I/usr/local/include \ + -L/usr/local/lib -lpcre Then you can run simple tests like this: ./pcredemo 'cat|dog' 'the cat sat on the mat' + ./pcredemo -g 'cat|dog' 'the dog sat on the cat' Note that there is a much more comprehensive test program, called pcretest, which supports many more facilities for - testing regular expressions. The pcredemo program is pro- - vided as a simple coding example. + testing regular expressions and the PCRE library. The + pcredemo program is provided as a simple coding example. On some operating systems (e.g. Solaris) you may get an error like this when you try to run pcredemo: @@ -2199,109 +3294,9 @@ SAMPLE PROGRAM -R/usr/local/lib - to the compile command to get round this problem. Here's the - code: + to the compile command to get round this problem. - #include <stdio.h> - #include <string.h> - #include <pcre.h> +Last updated: 28 January 2003 +Copyright (c) 1997-2003 University of Cambridge. +----------------------------------------------------------------------------- - #define OVECCOUNT 30 /* should be a multiple of 3 */ - - int main(int argc, char **argv) - { - pcre *re; - const char *error; - int erroffset; - int ovector[OVECCOUNT]; - int rc, i; - - if (argc != 3) - { - printf("Two arguments required: a regex and a " - "subject string\n"); - return 1; - } - - /* Compile the regular expression in the first argument */ - - re = pcre_compile( - argv[1], /* the pattern */ - 0, /* default options */ - &error, /* for error message */ - &erroffset, /* for error offset */ - NULL); /* use default character tables */ - - /* Compilation failed: print the error message and exit */ - - if (re == NULL) - { - printf("PCRE compilation failed at offset %d: %s\n", - erroffset, error); - return 1; - } - - /* Compilation succeeded: match the subject in the second - argument */ - - rc = pcre_exec( - re, /* the compiled pattern */ - NULL, /* we didn't study the pattern */ - argv[2], /* the subject string */ - (int)strlen(argv[2]), /* the length of the subject */ - 0, /* start at offset 0 in the subject */ - 0, /* default options */ - ovector, /* vector for substring information */ - OVECCOUNT); /* number of elements in the vector */ - - /* Matching failed: handle error cases */ - - if (rc < 0) - { - switch(rc) - { - case PCRE_ERROR_NOMATCH: printf("No match\n"); break; - /* - Handle other special cases if you like - */ - default: printf("Matching error %d\n", rc); break; - } - return 1; - } - - /* Match succeded */ - - printf("Match succeeded\n"); - - /* The output vector wasn't big enough */ - - if (rc == 0) - { - rc = OVECCOUNT/3; - printf("ovector only has room for %d captured " - substrings\n", rc - 1); - } - - /* Show substrings stored in the output vector */ - - for (i = 0; i < rc; i++) - { - char *substring_start = argv[2] + ovector[2*i]; - int substring_length = ovector[2*i+1] - ovector[2*i]; - printf("%2d: %.*s\n", i, substring_length, - substring_start); - } - - return 0; - } - - - -AUTHOR - Philip Hazel <ph10@cam.ac.uk> - University Computing Service, - New Museums Site, - Cambridge CB2 3QG, England. - Phone: +44 1223 334714 - Last updated: 15 August 2001 - Copyright (c) 1997-2001 University of Cambridge. diff --git a/ext/pcre/pcrelib/doc/pcregrep.1 b/ext/pcre/pcrelib/doc/pcregrep.1 deleted file mode 100644 index b55745aca8..0000000000 --- a/ext/pcre/pcrelib/doc/pcregrep.1 +++ /dev/null @@ -1,92 +0,0 @@ -.TH PCREGREP 1 -.SH NAME -pcregrep - a grep with Perl-compatible regular expressions. -.SH SYNOPSIS -.B pcregrep [-Vcfhilnrsvx] [pattern] [file1 file2 ...] - - -.SH DESCRIPTION -\fBpcregrep\fR searches files for character patterns, in the same way as other -grep commands do, but it uses the PCRE regular expression library to support -patterns that are compatible with the regular expressions of Perl 5. See -\fBpcre(3)\fR for a full description of syntax and semantics. - -A pattern must be specified on the command line unless the \fB-f\fR option is -used (see below). - -If no files are specified, \fBpcregrep\fR reads the standard input. By default, -each line that matches the pattern is copied to the standard output, and if -there is more than one file, the file name is printed before each line of -output. However, there are options that can change how \fBpcregrep\fR behaves. - -Lines are limited to BUFSIZ characters. BUFSIZ is defined in \fB<stdio.h>\fR. -The newline character is removed from the end of each line before it is matched -against the pattern. - - -.SH OPTIONS -.TP 10 -\fB-V\fR -Write the version number of the PCRE library being used to the standard error -stream. -.TP -\fB-c\fR -Do not print individual lines; instead just print a count of the number of -lines that would otherwise have been printed. If several files are given, a -count is printed for each of them. -.TP -\fB-f\fIfilename\fR Read a number of patterns from the file, one per line, and -match all of them against each line of input. A line is output if any of the -patterns match it. When \fB-f\fR is used, no pattern is taken from the command -line; all arguments are treated as file names. There is a maximum of 100 -patterns. Trailing white space is removed, and blank lines are ignored. An -empty file contains no patterns and therefore matches nothing. -.TP -\fB-h\fR -Suppress printing of filenames when searching multiple files. -.TP -\fB-i\fR -Ignore upper/lower case distinctions during comparisons. -.TP -\fB-l\fR -Instead of printing lines from the files, just print the names of the files -containing lines that would have been printed. Each file name is printed -once, on a separate line. -.TP -\fB-n\fR -Precede each line by its line number in the file. -.TP -\fB-r\fR -If any file is a directory, recursively scan the files it contains. Without -\fB-r\fR a directory is scanned as a normal file. -.TP -\fB-s\fR -Work silently, that is, display nothing except error messages. -The exit status indicates whether any matches were found. -.TP -\fB-v\fR -Invert the sense of the match, so that lines which do \fInot\fR match the -pattern are now the ones that are found. -.TP -\fB-x\fR -Force the pattern to be anchored (it must start matching at the beginning of -the line) and in addition, require it to match the entire line. This is -equivalent to having ^ and $ characters at the start and end of each -alternative branch in the regular expression. - - -.SH SEE ALSO -\fBpcre(3)\fR, Perl 5 documentation - - -.SH DIAGNOSTICS -Exit status is 0 if any matches were found, 1 if no matches were found, and 2 -for syntax errors or inacessible files (even if matches were found). - - -.SH AUTHOR -Philip Hazel <ph10@cam.ac.uk> - -Last updated: 25 July 2002 -.br -Copyright (c) 1997-2002 University of Cambridge. diff --git a/ext/pcre/pcrelib/doc/pcregrep.html b/ext/pcre/pcrelib/doc/pcregrep.html deleted file mode 100644 index 20151d4a08..0000000000 --- a/ext/pcre/pcrelib/doc/pcregrep.html +++ /dev/null @@ -1,125 +0,0 @@ -<HTML> -<HEAD> -<TITLE>pcregrep specification</TITLE> -</HEAD> -<body bgcolor="#FFFFFF" text="#00005A"> -<H1>pcregrep specification</H1> -This HTML document has been generated automatically from the original man page. -If there is any nonsense in it, please consult the man page in case the -conversion went wrong. -<UL> -<LI><A NAME="TOC1" HREF="#SEC1">NAME</A> -<LI><A NAME="TOC2" HREF="#SEC2">SYNOPSIS</A> -<LI><A NAME="TOC3" HREF="#SEC3">DESCRIPTION</A> -<LI><A NAME="TOC4" HREF="#SEC4">OPTIONS</A> -<LI><A NAME="TOC5" HREF="#SEC5">SEE ALSO</A> -<LI><A NAME="TOC6" HREF="#SEC6">DIAGNOSTICS</A> -<LI><A NAME="TOC7" HREF="#SEC7">AUTHOR</A> -</UL> -<LI><A NAME="SEC1" HREF="#TOC1">NAME</A> -<P> -pcregrep - a grep with Perl-compatible regular expressions. -</P> -<LI><A NAME="SEC2" HREF="#TOC1">SYNOPSIS</A> -<P> -<B>pcregrep [-Vcfhilnrsvx] [pattern] [file1 file2 ...]</B> -</P> -<LI><A NAME="SEC3" HREF="#TOC1">DESCRIPTION</A> -<P> -<B>pcregrep</B> searches files for character patterns, in the same way as other -grep commands do, but it uses the PCRE regular expression library to support -patterns that are compatible with the regular expressions of Perl 5. See -<B>pcre(3)</B> for a full description of syntax and semantics. -</P> -<P> -A pattern must be specified on the command line unless the <B>-f</B> option is -used (see below). -</P> -<P> -If no files are specified, <B>pcregrep</B> reads the standard input. By default, -each line that matches the pattern is copied to the standard output, and if -there is more than one file, the file name is printed before each line of -output. However, there are options that can change how <B>pcregrep</B> behaves. -</P> -<P> -Lines are limited to BUFSIZ characters. BUFSIZ is defined in <B><stdio.h></B>. -The newline character is removed from the end of each line before it is matched -against the pattern. -</P> -<LI><A NAME="SEC4" HREF="#TOC1">OPTIONS</A> -<P> -<B>-V</B> -Write the version number of the PCRE library being used to the standard error -stream. -</P> -<P> -<B>-c</B> -Do not print individual lines; instead just print a count of the number of -lines that would otherwise have been printed. If several files are given, a -count is printed for each of them. -</P> -<P> -\fB-f<I>filename</I> Read a number of patterns from the file, one per line, and -match all of them against each line of input. A line is output if any of the -patterns match it. When <B>-f</B> is used, no pattern is taken from the command -line; all arguments are treated as file names. There is a maximum of 100 -patterns. Trailing white space is removed, and blank lines are ignored. An -empty file contains no patterns and therefore matches nothing. -</P> -<P> -<B>-h</B> -Suppress printing of filenames when searching multiple files. -</P> -<P> -<B>-i</B> -Ignore upper/lower case distinctions during comparisons. -</P> -<P> -<B>-l</B> -Instead of printing lines from the files, just print the names of the files -containing lines that would have been printed. Each file name is printed -once, on a separate line. -</P> -<P> -<B>-n</B> -Precede each line by its line number in the file. -</P> -<P> -<B>-r</B> -If any file is a directory, recursively scan the files it contains. Without -<B>-r</B> a directory is scanned as a normal file. -</P> -<P> -<B>-s</B> -Work silently, that is, display nothing except error messages. -The exit status indicates whether any matches were found. -</P> -<P> -<B>-v</B> -Invert the sense of the match, so that lines which do <I>not</I> match the -pattern are now the ones that are found. -</P> -<P> -<B>-x</B> -Force the pattern to be anchored (it must start matching at the beginning of -the line) and in addition, require it to match the entire line. This is -equivalent to having ^ and $ characters at the start and end of each -alternative branch in the regular expression. -</P> -<LI><A NAME="SEC5" HREF="#TOC1">SEE ALSO</A> -<P> -<B>pcre(3)</B>, Perl 5 documentation -</P> -<LI><A NAME="SEC6" HREF="#TOC1">DIAGNOSTICS</A> -<P> -Exit status is 0 if any matches were found, 1 if no matches were found, and 2 -for syntax errors or inacessible files (even if matches were found). -</P> -<LI><A NAME="SEC7" HREF="#TOC1">AUTHOR</A> -<P> -Philip Hazel <ph10@cam.ac.uk> -</P> -<P> -Last updated: 25 July 2002 -<BR> -Copyright (c) 1997-2002 University of Cambridge. diff --git a/ext/pcre/pcrelib/doc/pcregrep.txt b/ext/pcre/pcrelib/doc/pcregrep.txt deleted file mode 100644 index ce53f7a889..0000000000 --- a/ext/pcre/pcrelib/doc/pcregrep.txt +++ /dev/null @@ -1,109 +0,0 @@ -NAME - pcregrep - a grep with Perl-compatible regular expressions. - - - -SYNOPSIS - pcregrep [-Vcfhilnrsvx] [pattern] [file1 file2 ...] - - - -DESCRIPTION - pcregrep searches files for character patterns, in the same - way as other grep commands do, but it uses the PCRE regular - expression library to support patterns that are compatible - with the regular expressions of Perl 5. See pcre(3) for a - full description of syntax and semantics. - - A pattern must be specified on the command line unless the - -f option is used (see below). - - If no files are specified, pcregrep reads the standard - input. By default, each line that matches the pattern is - copied to the standard output, and if there is more than one - file, the file name is printed before each line of output. - However, there are options that can change how pcregrep - behaves. - - Lines are limited to BUFSIZ characters. BUFSIZ is defined in - <stdio.h>. The newline character is removed from the end of - each line before it is matched against the pattern. - - - -OPTIONS - -V Write the version number of the PCRE library being - used to the standard error stream. - - -c Do not print individual lines; instead just print - a count of the number of lines that would other- - wise have been printed. If several files are - given, a count is printed for each of them. - - - -and - - - ffilename Read a number of patterns from the file, one per line, - match all of them against each line of input. A - line is output if any of the patterns match it. - When -f is used, no pattern is taken from the com- - mand line; all arguments are treated as file - names. There is a maximum of 100 patterns. Trail- - ing white space is removed, and blank lines are - ignored. An empty file contains no patterns and - therefore matches nothing. - - -h Suppress printing of filenames when searching mul- - tiple files. - - -i Ignore upper/lower case distinctions during com- - parisons. - - -l Instead of printing lines from the files, just - print the names of the files containing lines that - would have been printed. Each file name is printed - once, on a separate line. - - -n Precede each line by its line number in the file. - - -r If any file is a directory, recursively scan the - files it contains. Without -r a directory is - scanned as a normal file. - - -s Work silently, that is, display nothing except - error messages. The exit status indicates whether - any matches were found. - - -v Invert the sense of the match, so that lines which - do not match the pattern are now the ones that are - found. - - -x Force the pattern to be anchored (it must start - matching at the beginning of the line) and in - addition, require it to match the entire line. - This is equivalent to having ^ and $ characters at - the start and end of each alternative branch in - the regular expression. - - - -SEE ALSO - pcre(3), Perl 5 documentation - - - - - -DIAGNOSTICS - Exit status is 0 if any matches were found, 1 if no matches - were found, and 2 for syntax errors or inacessible files - (even if matches were found). - - - -AUTHOR - Philip Hazel <ph10@cam.ac.uk> - - Last updated: 25 July 2002 - Copyright (c) 1997-2002 University of Cambridge. diff --git a/ext/pcre/pcrelib/doc/pcreposix.3 b/ext/pcre/pcrelib/doc/pcreposix.3 deleted file mode 100644 index 41716ead91..0000000000 --- a/ext/pcre/pcrelib/doc/pcreposix.3 +++ /dev/null @@ -1,149 +0,0 @@ -.TH PCRE 3 -.SH NAME -pcreposix - POSIX API for Perl-compatible regular expressions. -.SH SYNOPSIS -.B #include <pcreposix.h> -.PP -.SM -.br -.B int regcomp(regex_t *\fIpreg\fR, const char *\fIpattern\fR, -.ti +5n -.B int \fIcflags\fR); -.PP -.br -.B int regexec(regex_t *\fIpreg\fR, const char *\fIstring\fR, -.ti +5n -.B size_t \fInmatch\fR, regmatch_t \fIpmatch\fR[], int \fIeflags\fR); -.PP -.br -.B size_t regerror(int \fIerrcode\fR, const regex_t *\fIpreg\fR, -.ti +5n -.B char *\fIerrbuf\fR, size_t \fIerrbuf_size\fR); -.PP -.br -.B void regfree(regex_t *\fIpreg\fR); - - -.SH DESCRIPTION -This set of functions provides a POSIX-style API to the PCRE regular expression -package. See the \fBpcre\fR documentation for a description of the native API, -which contains additional functionality. - -The functions described here are just wrapper functions that ultimately call -the native API. Their prototypes are defined in the \fBpcreposix.h\fR header -file, and on Unix systems the library itself is called \fBpcreposix.a\fR, so -can be accessed by adding \fB-lpcreposix\fR to the command for linking an -application which uses them. Because the POSIX functions call the native ones, -it is also necessary to add \fR-lpcre\fR. - -I have implemented only those option bits that can be reasonably mapped to PCRE -native options. In addition, the options REG_EXTENDED and REG_NOSUB are defined -with the value zero. They have no effect, but since programs that are written -to the POSIX interface often use them, this makes it easier to slot in PCRE as -a replacement library. Other POSIX options are not even defined. - -When PCRE is called via these functions, it is only the API that is POSIX-like -in style. The syntax and semantics of the regular expressions themselves are -still those of Perl, subject to the setting of various PCRE options, as -described below. - -The header for these functions is supplied as \fBpcreposix.h\fR to avoid any -potential clash with other POSIX libraries. It can, of course, be renamed or -aliased as \fBregex.h\fR, which is the "correct" name. It provides two -structure types, \fIregex_t\fR for compiled internal forms, and -\fIregmatch_t\fR for returning captured substrings. It also defines some -constants whose names start with "REG_"; these are used for setting options and -identifying error codes. - - -.SH COMPILING A PATTERN - -The function \fBregcomp()\fR is called to compile a pattern into an -internal form. The pattern is a C string terminated by a binary zero, and -is passed in the argument \fIpattern\fR. The \fIpreg\fR argument is a pointer -to a regex_t structure which is used as a base for storing information about -the compiled expression. - -The argument \fIcflags\fR is either zero, or contains one or more of the bits -defined by the following macros: - - REG_ICASE - -The PCRE_CASELESS option is set when the expression is passed for compilation -to the native function. - - REG_NEWLINE - -The PCRE_MULTILINE option is set when the expression is passed for compilation -to the native function. - -In the absence of these flags, no options are passed to the native function. -This means the the regex is compiled with PCRE default semantics. In -particular, the way it handles newline characters in the subject string is the -Perl way, not the POSIX way. Note that setting PCRE_MULTILINE has only -\fIsome\fR of the effects specified for REG_NEWLINE. It does not affect the way -newlines are matched by . (they aren't) or a negative class such as [^a] (they -are). - -The yield of \fBregcomp()\fR is zero on success, and non-zero otherwise. The -\fIpreg\fR structure is filled in on success, and one member of the structure -is publicized: \fIre_nsub\fR contains the number of capturing subpatterns in -the regular expression. Various error codes are defined in the header file. - - -.SH MATCHING A PATTERN -The function \fBregexec()\fR is called to match a pre-compiled pattern -\fIpreg\fR against a given \fIstring\fR, which is terminated by a zero byte, -subject to the options in \fIeflags\fR. These can be: - - REG_NOTBOL - -The PCRE_NOTBOL option is set when calling the underlying PCRE matching -function. - - REG_NOTEOL - -The PCRE_NOTEOL option is set when calling the underlying PCRE matching -function. - -The portion of the string that was matched, and also any captured substrings, -are returned via the \fIpmatch\fR argument, which points to an array of -\fInmatch\fR structures of type \fIregmatch_t\fR, containing the members -\fIrm_so\fR and \fIrm_eo\fR. These contain the offset to the first character of -each substring and the offset to the first character after the end of each -substring, respectively. The 0th element of the vector relates to the entire -portion of \fIstring\fR that was matched; subsequent elements relate to the -capturing subpatterns of the regular expression. Unused entries in the array -have both structure members set to -1. - -A successful match yields a zero return; various error codes are defined in the -header file, of which REG_NOMATCH is the "expected" failure code. - - -.SH ERROR MESSAGES -The \fBregerror()\fR function maps a non-zero errorcode from either -\fBregcomp\fR or \fBregexec\fR to a printable message. If \fIpreg\fR is not -NULL, the error should have arisen from the use of that structure. A message -terminated by a binary zero is placed in \fIerrbuf\fR. The length of the -message, including the zero, is limited to \fIerrbuf_size\fR. The yield of the -function is the size of buffer needed to hold the whole message. - - -.SH STORAGE -Compiling a regular expression causes memory to be allocated and associated -with the \fIpreg\fR structure. The function \fBregfree()\fR frees all such -memory, after which \fIpreg\fR may no longer be used as a compiled expression. - - -.SH AUTHOR -Philip Hazel <ph10@cam.ac.uk> -.br -University Computing Service, -.br -New Museums Site, -.br -Cambridge CB2 3QG, England. -.br -Phone: +44 1223 334714 - -Copyright (c) 1997-2000 University of Cambridge. diff --git a/ext/pcre/pcrelib/doc/pcreposix.html b/ext/pcre/pcrelib/doc/pcreposix.html deleted file mode 100644 index 9c89478420..0000000000 --- a/ext/pcre/pcrelib/doc/pcreposix.html +++ /dev/null @@ -1,191 +0,0 @@ -<HTML> -<HEAD> -<TITLE>pcreposix specification</TITLE> -</HEAD> -<body bgcolor="#FFFFFF" text="#00005A"> -<H1>pcreposix specification</H1> -This HTML document has been generated automatically from the original man page. -If there is any nonsense in it, please consult the man page in case the -conversion went wrong. -<UL> -<LI><A NAME="TOC1" HREF="#SEC1">NAME</A> -<LI><A NAME="TOC2" HREF="#SEC2">SYNOPSIS</A> -<LI><A NAME="TOC3" HREF="#SEC3">DESCRIPTION</A> -<LI><A NAME="TOC4" HREF="#SEC4">COMPILING A PATTERN</A> -<LI><A NAME="TOC5" HREF="#SEC5">MATCHING A PATTERN</A> -<LI><A NAME="TOC6" HREF="#SEC6">ERROR MESSAGES</A> -<LI><A NAME="TOC7" HREF="#SEC7">STORAGE</A> -<LI><A NAME="TOC8" HREF="#SEC8">AUTHOR</A> -</UL> -<LI><A NAME="SEC1" HREF="#TOC1">NAME</A> -<P> -pcreposix - POSIX API for Perl-compatible regular expressions. -</P> -<LI><A NAME="SEC2" HREF="#TOC1">SYNOPSIS</A> -<P> -<B>#include <pcreposix.h></B> -</P> -<P> -<B>int regcomp(regex_t *<I>preg</I>, const char *<I>pattern</I>,</B> -<B>int <I>cflags</I>);</B> -</P> -<P> -<B>int regexec(regex_t *<I>preg</I>, const char *<I>string</I>,</B> -<B>size_t <I>nmatch</I>, regmatch_t <I>pmatch</I>[], int <I>eflags</I>);</B> -</P> -<P> -<B>size_t regerror(int <I>errcode</I>, const regex_t *<I>preg</I>,</B> -<B>char *<I>errbuf</I>, size_t <I>errbuf_size</I>);</B> -</P> -<P> -<B>void regfree(regex_t *<I>preg</I>);</B> -</P> -<LI><A NAME="SEC3" HREF="#TOC1">DESCRIPTION</A> -<P> -This set of functions provides a POSIX-style API to the PCRE regular expression -package. See the <B>pcre</B> documentation for a description of the native API, -which contains additional functionality. -</P> -<P> -The functions described here are just wrapper functions that ultimately call -the native API. Their prototypes are defined in the <B>pcreposix.h</B> header -file, and on Unix systems the library itself is called <B>pcreposix.a</B>, so -can be accessed by adding <B>-lpcreposix</B> to the command for linking an -application which uses them. Because the POSIX functions call the native ones, -it is also necessary to add \fR-lpcre\fR. -</P> -<P> -I have implemented only those option bits that can be reasonably mapped to PCRE -native options. In addition, the options REG_EXTENDED and REG_NOSUB are defined -with the value zero. They have no effect, but since programs that are written -to the POSIX interface often use them, this makes it easier to slot in PCRE as -a replacement library. Other POSIX options are not even defined. -</P> -<P> -When PCRE is called via these functions, it is only the API that is POSIX-like -in style. The syntax and semantics of the regular expressions themselves are -still those of Perl, subject to the setting of various PCRE options, as -described below. -</P> -<P> -The header for these functions is supplied as <B>pcreposix.h</B> to avoid any -potential clash with other POSIX libraries. It can, of course, be renamed or -aliased as <B>regex.h</B>, which is the "correct" name. It provides two -structure types, <I>regex_t</I> for compiled internal forms, and -<I>regmatch_t</I> for returning captured substrings. It also defines some -constants whose names start with "REG_"; these are used for setting options and -identifying error codes. -</P> -<LI><A NAME="SEC4" HREF="#TOC1">COMPILING A PATTERN</A> -<P> -The function <B>regcomp()</B> is called to compile a pattern into an -internal form. The pattern is a C string terminated by a binary zero, and -is passed in the argument <I>pattern</I>. The <I>preg</I> argument is a pointer -to a regex_t structure which is used as a base for storing information about -the compiled expression. -</P> -<P> -The argument <I>cflags</I> is either zero, or contains one or more of the bits -defined by the following macros: -</P> -<P> -<PRE> - REG_ICASE -</PRE> -</P> -<P> -The PCRE_CASELESS option is set when the expression is passed for compilation -to the native function. -</P> -<P> -<PRE> - REG_NEWLINE -</PRE> -</P> -<P> -The PCRE_MULTILINE option is set when the expression is passed for compilation -to the native function. -</P> -<P> -In the absence of these flags, no options are passed to the native function. -This means the the regex is compiled with PCRE default semantics. In -particular, the way it handles newline characters in the subject string is the -Perl way, not the POSIX way. Note that setting PCRE_MULTILINE has only -<I>some</I> of the effects specified for REG_NEWLINE. It does not affect the way -newlines are matched by . (they aren't) or a negative class such as [^a] (they -are). -</P> -<P> -The yield of <B>regcomp()</B> is zero on success, and non-zero otherwise. The -<I>preg</I> structure is filled in on success, and one member of the structure -is publicized: <I>re_nsub</I> contains the number of capturing subpatterns in -the regular expression. Various error codes are defined in the header file. -</P> -<LI><A NAME="SEC5" HREF="#TOC1">MATCHING A PATTERN</A> -<P> -The function <B>regexec()</B> is called to match a pre-compiled pattern -<I>preg</I> against a given <I>string</I>, which is terminated by a zero byte, -subject to the options in <I>eflags</I>. These can be: -</P> -<P> -<PRE> - REG_NOTBOL -</PRE> -</P> -<P> -The PCRE_NOTBOL option is set when calling the underlying PCRE matching -function. -</P> -<P> -<PRE> - REG_NOTEOL -</PRE> -</P> -<P> -The PCRE_NOTEOL option is set when calling the underlying PCRE matching -function. -</P> -<P> -The portion of the string that was matched, and also any captured substrings, -are returned via the <I>pmatch</I> argument, which points to an array of -<I>nmatch</I> structures of type <I>regmatch_t</I>, containing the members -<I>rm_so</I> and <I>rm_eo</I>. These contain the offset to the first character of -each substring and the offset to the first character after the end of each -substring, respectively. The 0th element of the vector relates to the entire -portion of <I>string</I> that was matched; subsequent elements relate to the -capturing subpatterns of the regular expression. Unused entries in the array -have both structure members set to -1. -</P> -<P> -A successful match yields a zero return; various error codes are defined in the -header file, of which REG_NOMATCH is the "expected" failure code. -</P> -<LI><A NAME="SEC6" HREF="#TOC1">ERROR MESSAGES</A> -<P> -The <B>regerror()</B> function maps a non-zero errorcode from either -<B>regcomp</B> or <B>regexec</B> to a printable message. If <I>preg</I> is not -NULL, the error should have arisen from the use of that structure. A message -terminated by a binary zero is placed in <I>errbuf</I>. The length of the -message, including the zero, is limited to <I>errbuf_size</I>. The yield of the -function is the size of buffer needed to hold the whole message. -</P> -<LI><A NAME="SEC7" HREF="#TOC1">STORAGE</A> -<P> -Compiling a regular expression causes memory to be allocated and associated -with the <I>preg</I> structure. The function <B>regfree()</B> frees all such -memory, after which <I>preg</I> may no longer be used as a compiled expression. -</P> -<LI><A NAME="SEC8" HREF="#TOC1">AUTHOR</A> -<P> -Philip Hazel <ph10@cam.ac.uk> -<BR> -University Computing Service, -<BR> -New Museums Site, -<BR> -Cambridge CB2 3QG, England. -<BR> -Phone: +44 1223 334714 -</P> -<P> -Copyright (c) 1997-2000 University of Cambridge. diff --git a/ext/pcre/pcrelib/doc/pcreposix.txt b/ext/pcre/pcrelib/doc/pcreposix.txt deleted file mode 100644 index 2d76f7cdcc..0000000000 --- a/ext/pcre/pcrelib/doc/pcreposix.txt +++ /dev/null @@ -1,159 +0,0 @@ -NAME - pcreposix - POSIX API for Perl-compatible regular expres- - sions. - - - -SYNOPSIS - #include <pcreposix.h> - - int regcomp(regex_t *preg, const char *pattern, - int cflags); - - int regexec(regex_t *preg, const char *string, - size_t nmatch, regmatch_t pmatch[], int eflags); - - size_t regerror(int errcode, const regex_t *preg, - char *errbuf, size_t errbuf_size); - - void regfree(regex_t *preg); - - - -DESCRIPTION - This set of functions provides a POSIX-style API to the PCRE - regular expression package. See the pcre documentation for a - description of the native API, which contains additional - functionality. - - The functions described here are just wrapper functions that - ultimately call the native API. Their prototypes are defined - in the pcreposix.h header file, and on Unix systems the - library itself is called pcreposix.a, so can be accessed by - adding -lpcreposix to the command for linking an application - which uses them. Because the POSIX functions call the native - ones, it is also necessary to add -lpcre. - - I have implemented only those option bits that can be rea- - sonably mapped to PCRE native options. In addition, the - options REG_EXTENDED and REG_NOSUB are defined with the - value zero. They have no effect, but since programs that are - written to the POSIX interface often use them, this makes it - easier to slot in PCRE as a replacement library. Other POSIX - options are not even defined. - - When PCRE is called via these functions, it is only the API - that is POSIX-like in style. The syntax and semantics of the - regular expressions themselves are still those of Perl, sub- - ject to the setting of various PCRE options, as described - below. - - The header for these functions is supplied as pcreposix.h to - avoid any potential clash with other POSIX libraries. It - can, of course, be renamed or aliased as regex.h, which is - the "correct" name. It provides two structure types, regex_t - for compiled internal forms, and regmatch_t for returning - captured substrings. It also defines some constants whose - names start with "REG_"; these are used for setting options - and identifying error codes. - - - -COMPILING A PATTERN - The function regcomp() is called to compile a pattern into - an internal form. The pattern is a C string terminated by a - binary zero, and is passed in the argument pattern. The preg - argument is a pointer to a regex_t structure which is used - as a base for storing information about the compiled expres- - sion. - - The argument cflags is either zero, or contains one or more - of the bits defined by the following macros: - - REG_ICASE - - The PCRE_CASELESS option is set when the expression is - passed for compilation to the native function. - - REG_NEWLINE - - The PCRE_MULTILINE option is set when the expression is - passed for compilation to the native function. - - In the absence of these flags, no options are passed to the - native function. This means the the regex is compiled with - PCRE default semantics. In particular, the way it handles - newline characters in the subject string is the Perl way, - not the POSIX way. Note that setting PCRE_MULTILINE has only - some of the effects specified for REG_NEWLINE. It does not - affect the way newlines are matched by . (they aren't) or a - negative class such as [^a] (they are). - - The yield of regcomp() is zero on success, and non-zero oth- - erwise. The preg structure is filled in on success, and one - member of the structure is publicized: re_nsub contains the - number of capturing subpatterns in the regular expression. - Various error codes are defined in the header file. - - - -MATCHING A PATTERN - The function regexec() is called to match a pre-compiled - pattern preg against a given string, which is terminated by - a zero byte, subject to the options in eflags. These can be: - - REG_NOTBOL - - The PCRE_NOTBOL option is set when calling the underlying - PCRE matching function. - - REG_NOTEOL - - The PCRE_NOTEOL option is set when calling the underlying - PCRE matching function. - - The portion of the string that was matched, and also any - captured substrings, are returned via the pmatch argument, - which points to an array of nmatch structures of type - regmatch_t, containing the members rm_so and rm_eo. These - contain the offset to the first character of each substring - and the offset to the first character after the end of each - substring, respectively. The 0th element of the vector - relates to the entire portion of string that was matched; - subsequent elements relate to the capturing subpatterns of - the regular expression. Unused entries in the array have - both structure members set to -1. - - A successful match yields a zero return; various error codes - are defined in the header file, of which REG_NOMATCH is the - "expected" failure code. - - - -ERROR MESSAGES - The regerror() function maps a non-zero errorcode from - either regcomp or regexec to a printable message. If preg is - not NULL, the error should have arisen from the use of that - structure. A message terminated by a binary zero is placed - in errbuf. The length of the message, including the zero, is - limited to errbuf_size. The yield of the function is the - size of buffer needed to hold the whole message. - - - -STORAGE - Compiling a regular expression causes memory to be allocated - and associated with the preg structure. The function reg- - free() frees all such memory, after which preg may no longer - be used as a compiled expression. - - - -AUTHOR - Philip Hazel <ph10@cam.ac.uk> - University Computing Service, - New Museums Site, - Cambridge CB2 3QG, England. - Phone: +44 1223 334714 - - Copyright (c) 1997-2000 University of Cambridge. diff --git a/ext/pcre/pcrelib/doc/pcretest.1 b/ext/pcre/pcrelib/doc/pcretest.1 deleted file mode 100644 index 02afe7a49d..0000000000 --- a/ext/pcre/pcrelib/doc/pcretest.1 +++ /dev/null @@ -1,288 +0,0 @@ -.TH PCRETEST 1 -.SH NAME -pcretest - a program for testing Perl-compatible regular expressions. -.SH SYNOPSIS -.B pcretest "[-d] [-i] [-m] [-o osize] [-p] [-t] [source] [destination]" - -\fBpcretest\fR was written as a test program for the PCRE regular expression -library itself, but it can also be used for experimenting with regular -expressions. This man page describes the features of the test program; for -details of the regular expressions themselves, see the \fBpcre\fR man page. - -.SH OPTIONS -.TP 10 -\fB-d\fR -Behave as if each regex had the \fB/D\fR modifier (see below); the internal -form is output after compilation. -.TP 10 -\fB-i\fR -Behave as if each regex had the \fB/I\fR modifier; information about the -compiled pattern is given after compilation. -.TP 10 -\fB-m\fR -Output the size of each compiled pattern after it has been compiled. This is -equivalent to adding /M to each regular expression. For compatibility with -earlier versions of pcretest, \fB-s\fR is a synonym for \fB-m\fR. -.TP 10 -\fB-o\fR \fIosize\fR -Set the number of elements in the output vector that is used when calling PCRE -to be \fIosize\fR. The default value is 45, which is enough for 14 capturing -subexpressions. The vector size can be changed for individual matching calls by -including \\O in the data line (see below). -.TP 10 -\fB-p\fR -Behave as if each regex has \fB/P\fR modifier; the POSIX wrapper API is used -to call PCRE. None of the other options has any effect when \fB-p\fR is set. -.TP 10 -\fB-t\fR -Run each compile, study, and match many times with a timer, and output -resulting time per compile or match (in milliseconds). Do not set \fB-t\fR with -\fB-m\fR, because you will then get the size output 20000 times and the timing -will be distorted. - - -.SH DESCRIPTION - -If \fBpcretest\fR is given two filename arguments, it reads from the first and -writes to the second. If it is given only one filename argument, it reads from -that file and writes to stdout. Otherwise, it reads from stdin and writes to -stdout, and prompts for each line of input, using "re>" to prompt for regular -expressions, and "data>" to prompt for data lines. - -The program handles any number of sets of input on a single input file. Each -set starts with a regular expression, and continues with any number of data -lines to be matched against the pattern. - -Each line is matched separately and independently. If you want to do -multiple-line matches, you have to use the \\n escape sequence in a single line -of input to encode the newline characters. The maximum length of data line is -30,000 characters. - -An empty line signals the end of the data lines, at which point a new regular -expression is read. The regular expressions are given enclosed in any -non-alphameric delimiters other than backslash, for example - - /(a|bc)x+yz/ - -White space before the initial delimiter is ignored. A regular expression may -be continued over several input lines, in which case the newline characters are -included within it. It is possible to include the delimiter within the pattern -by escaping it, for example - - /abc\\/def/ - -If you do so, the escape and the delimiter form part of the pattern, but since -delimiters are always non-alphameric, this does not affect its interpretation. -If the terminating delimiter is immediately followed by a backslash, for -example, - - /abc/\\ - -then a backslash is added to the end of the pattern. This is done to provide a -way of testing the error condition that arises if a pattern finishes with a -backslash, because - - /abc\\/ - -is interpreted as the first line of a pattern that starts with "abc/", causing -pcretest to read the next line as a continuation of the regular expression. - - -.SH PATTERN MODIFIERS - -The pattern may be followed by \fBi\fR, \fBm\fR, \fBs\fR, or \fBx\fR to set the -PCRE_CASELESS, PCRE_MULTILINE, PCRE_DOTALL, or PCRE_EXTENDED options, -respectively. For example: - - /caseless/i - -These modifier letters have the same effect as they do in Perl. There are -others which set PCRE options that do not correspond to anything in Perl: -\fB/A\fR, \fB/E\fR, and \fB/X\fR set PCRE_ANCHORED, PCRE_DOLLAR_ENDONLY, and -PCRE_EXTRA respectively. - -Searching for all possible matches within each subject string can be requested -by the \fB/g\fR or \fB/G\fR modifier. After finding a match, PCRE is called -again to search the remainder of the subject string. The difference between -\fB/g\fR and \fB/G\fR is that the former uses the \fIstartoffset\fR argument to -\fBpcre_exec()\fR to start searching at a new point within the entire string -(which is in effect what Perl does), whereas the latter passes over a shortened -substring. This makes a difference to the matching process if the pattern -begins with a lookbehind assertion (including \\b or \\B). - -If any call to \fBpcre_exec()\fR in a \fB/g\fR or \fB/G\fR sequence matches an -empty string, the next call is done with the PCRE_NOTEMPTY and PCRE_ANCHORED -flags set in order to search for another, non-empty, match at the same point. -If this second match fails, the start offset is advanced by one, and the normal -match is retried. This imitates the way Perl handles such cases when using the -\fB/g\fR modifier or the \fBsplit()\fR function. - -There are a number of other modifiers for controlling the way \fBpcretest\fR -operates. - -The \fB/+\fR modifier requests that as well as outputting the substring that -matched the entire pattern, pcretest should in addition output the remainder of -the subject string. This is useful for tests where the subject contains -multiple copies of the same substring. - -The \fB/L\fR modifier must be followed directly by the name of a locale, for -example, - - /pattern/Lfr - -For this reason, it must be the last modifier letter. The given locale is set, -\fBpcre_maketables()\fR is called to build a set of character tables for the -locale, and this is then passed to \fBpcre_compile()\fR when compiling the -regular expression. Without an \fB/L\fR modifier, NULL is passed as the tables -pointer; that is, \fB/L\fR applies only to the expression on which it appears. - -The \fB/I\fR modifier requests that \fBpcretest\fR output information about the -compiled expression (whether it is anchored, has a fixed first character, and -so on). It does this by calling \fBpcre_fullinfo()\fR after compiling an -expression, and outputting the information it gets back. If the pattern is -studied, the results of that are also output. - -The \fB/D\fR modifier is a PCRE debugging feature, which also assumes \fB/I\fR. -It causes the internal form of compiled regular expressions to be output after -compilation. - -The \fB/S\fR modifier causes \fBpcre_study()\fR to be called after the -expression has been compiled, and the results used when the expression is -matched. - -The \fB/M\fR modifier causes the size of memory block used to hold the compiled -pattern to be output. - -The \fB/P\fR modifier causes \fBpcretest\fR to call PCRE via the POSIX wrapper -API rather than its native API. When this is done, all other modifiers except -\fB/i\fR, \fB/m\fR, and \fB/+\fR are ignored. REG_ICASE is set if \fB/i\fR is -present, and REG_NEWLINE is set if \fB/m\fR is present. The wrapper functions -force PCRE_DOLLAR_ENDONLY always, and PCRE_DOTALL unless REG_NEWLINE is set. - -The \fB/8\fR modifier causes \fBpcretest\fR to call PCRE with the PCRE_UTF8 -option set. This turns on the (currently incomplete) support for UTF-8 -character handling in PCRE, provided that it was compiled with this support -enabled. This modifier also causes any non-printing characters in output -strings to be printed using the \\x{hh...} notation if they are valid UTF-8 -sequences. - - -.SH DATA LINES - -Before each data line is passed to \fBpcre_exec()\fR, leading and trailing -whitespace is removed, and it is then scanned for \\ escapes. The following are -recognized: - - \\a alarm (= BEL) - \\b backspace - \\e escape - \\f formfeed - \\n newline - \\r carriage return - \\t tab - \\v vertical tab - \\nnn octal character (up to 3 octal digits) - \\xhh hexadecimal character (up to 2 hex digits) - \\x{hh...} hexadecimal UTF-8 character - - \\A pass the PCRE_ANCHORED option to \fBpcre_exec()\fR - \\B pass the PCRE_NOTBOL option to \fBpcre_exec()\fR - \\Cdd call pcre_copy_substring() for substring dd - after a successful match (any decimal number - less than 32) - \\Gdd call pcre_get_substring() for substring dd - after a successful match (any decimal number - less than 32) - \\L call pcre_get_substringlist() after a - successful match - \\N pass the PCRE_NOTEMPTY option to \fBpcre_exec()\fR - \\Odd set the size of the output vector passed to - \fBpcre_exec()\fR to dd (any number of decimal - digits) - \\Z pass the PCRE_NOTEOL option to \fBpcre_exec()\fR - -When \\O is used, it may be higher or lower than the size set by the \fB-O\fR -option (or defaulted to 45); \\O applies only to the call of \fBpcre_exec()\fR -for the line in which it appears. - -A backslash followed by anything else just escapes the anything else. If the -very last character is a backslash, it is ignored. This gives a way of passing -an empty line as data, since a real empty line terminates the data input. - -If \fB/P\fR was present on the regex, causing the POSIX wrapper API to be used, -only \fB\B\fR, and \fB\Z\fR have any effect, causing REG_NOTBOL and REG_NOTEOL -to be passed to \fBregexec()\fR respectively. - -The use of \\x{hh...} to represent UTF-8 characters is not dependent on the use -of the \fB/8\fR modifier on the pattern. It is recognized always. There may be -any number of hexadecimal digits inside the braces. The result is from one to -six bytes, encoded according to the UTF-8 rules. - - -.SH OUTPUT FROM PCRETEST - -When a match succeeds, pcretest outputs the list of captured substrings that -\fBpcre_exec()\fR returns, starting with number 0 for the string that matched -the whole pattern. Here is an example of an interactive pcretest run. - - $ pcretest - PCRE version 2.06 08-Jun-1999 - - re> /^abc(\\d+)/ - data> abc123 - 0: abc123 - 1: 123 - data> xyz - No match - -If the strings contain any non-printing characters, they are output as \\0x -escapes, or as \\x{...} escapes if the \fB/8\fR modifier was present on the -pattern. If the pattern has the \fB/+\fR modifier, then the output for -substring 0 is followed by the the rest of the subject string, identified by -"0+" like this: - - re> /cat/+ - data> cataract - 0: cat - 0+ aract - -If the pattern has the \fB/g\fR or \fB/G\fR modifier, the results of successive -matching attempts are output in sequence, like this: - - re> /\\Bi(\\w\\w)/g - data> Mississippi - 0: iss - 1: ss - 0: iss - 1: ss - 0: ipp - 1: pp - -"No match" is output only if the first match attempt fails. - -If any of the sequences \fB\\C\fR, \fB\\G\fR, or \fB\\L\fR are present in a -data line that is successfully matched, the substrings extracted by the -convenience functions are output with C, G, or L after the string number -instead of a colon. This is in addition to the normal full list. The string -length (that is, the return from the extraction function) is given in -parentheses after each string for \fB\\C\fR and \fB\\G\fR. - -Note that while patterns can be continued over several lines (a plain ">" -prompt is used for continuations), data lines may not. However newlines can be -included in data by means of the \\n escape. - - -.SH AUTHOR -Philip Hazel <ph10@cam.ac.uk> -.br -University Computing Service, -.br -New Museums Site, -.br -Cambridge CB2 3QG, England. -.br -Phone: +44 1223 334714 - -Last updated: 25 August 2002 -.br -Copyright (c) 1997-2002 University of Cambridge. diff --git a/ext/pcre/pcrelib/doc/pcretest.html b/ext/pcre/pcrelib/doc/pcretest.html deleted file mode 100644 index cec1214420..0000000000 --- a/ext/pcre/pcrelib/doc/pcretest.html +++ /dev/null @@ -1,377 +0,0 @@ -<HTML> -<HEAD> -<TITLE>pcretest specification</TITLE> -</HEAD> -<body bgcolor="#FFFFFF" text="#00005A"> -<H1>pcretest specification</H1> -This HTML document has been generated automatically from the original man page. -If there is any nonsense in it, please consult the man page in case the -conversion went wrong. -<UL> -<LI><A NAME="TOC1" HREF="#SEC1">NAME</A> -<LI><A NAME="TOC2" HREF="#SEC2">SYNOPSIS</A> -<LI><A NAME="TOC3" HREF="#SEC3">OPTIONS</A> -<LI><A NAME="TOC4" HREF="#SEC4">DESCRIPTION</A> -<LI><A NAME="TOC5" HREF="#SEC5">PATTERN MODIFIERS</A> -<LI><A NAME="TOC6" HREF="#SEC6">DATA LINES</A> -<LI><A NAME="TOC7" HREF="#SEC7">OUTPUT FROM PCRETEST</A> -<LI><A NAME="TOC8" HREF="#SEC8">AUTHOR</A> -</UL> -<LI><A NAME="SEC1" HREF="#TOC1">NAME</A> -<P> -pcretest - a program for testing Perl-compatible regular expressions. -</P> -<LI><A NAME="SEC2" HREF="#TOC1">SYNOPSIS</A> -<P> -<B>pcretest [-d] [-i] [-m] [-o osize] [-p] [-t] [source] [destination]</B> -</P> -<P> -<B>pcretest</B> was written as a test program for the PCRE regular expression -library itself, but it can also be used for experimenting with regular -expressions. This man page describes the features of the test program; for -details of the regular expressions themselves, see the <B>pcre</B> man page. -</P> -<LI><A NAME="SEC3" HREF="#TOC1">OPTIONS</A> -<P> -<B>-d</B> -Behave as if each regex had the <B>/D</B> modifier (see below); the internal -form is output after compilation. -</P> -<P> -<B>-i</B> -Behave as if each regex had the <B>/I</B> modifier; information about the -compiled pattern is given after compilation. -</P> -<P> -<B>-m</B> -Output the size of each compiled pattern after it has been compiled. This is -equivalent to adding /M to each regular expression. For compatibility with -earlier versions of pcretest, <B>-s</B> is a synonym for <B>-m</B>. -</P> -<P> -<B>-o</B> <I>osize</I> -Set the number of elements in the output vector that is used when calling PCRE -to be <I>osize</I>. The default value is 45, which is enough for 14 capturing -subexpressions. The vector size can be changed for individual matching calls by -including \O in the data line (see below). -</P> -<P> -<B>-p</B> -Behave as if each regex has <B>/P</B> modifier; the POSIX wrapper API is used -to call PCRE. None of the other options has any effect when <B>-p</B> is set. -</P> -<P> -<B>-t</B> -Run each compile, study, and match many times with a timer, and output -resulting time per compile or match (in milliseconds). Do not set <B>-t</B> with -<B>-m</B>, because you will then get the size output 20000 times and the timing -will be distorted. -</P> -<LI><A NAME="SEC4" HREF="#TOC1">DESCRIPTION</A> -<P> -If <B>pcretest</B> is given two filename arguments, it reads from the first and -writes to the second. If it is given only one filename argument, it reads from -that file and writes to stdout. Otherwise, it reads from stdin and writes to -stdout, and prompts for each line of input, using "re>" to prompt for regular -expressions, and "data>" to prompt for data lines. -</P> -<P> -The program handles any number of sets of input on a single input file. Each -set starts with a regular expression, and continues with any number of data -lines to be matched against the pattern. -</P> -<P> -Each line is matched separately and independently. If you want to do -multiple-line matches, you have to use the \n escape sequence in a single line -of input to encode the newline characters. The maximum length of data line is -30,000 characters. -</P> -<P> -An empty line signals the end of the data lines, at which point a new regular -expression is read. The regular expressions are given enclosed in any -non-alphameric delimiters other than backslash, for example -</P> -<P> -<PRE> - /(a|bc)x+yz/ -</PRE> -</P> -<P> -White space before the initial delimiter is ignored. A regular expression may -be continued over several input lines, in which case the newline characters are -included within it. It is possible to include the delimiter within the pattern -by escaping it, for example -</P> -<P> -<PRE> - /abc\/def/ -</PRE> -</P> -<P> -If you do so, the escape and the delimiter form part of the pattern, but since -delimiters are always non-alphameric, this does not affect its interpretation. -If the terminating delimiter is immediately followed by a backslash, for -example, -</P> -<P> -<PRE> - /abc/\ -</PRE> -</P> -<P> -then a backslash is added to the end of the pattern. This is done to provide a -way of testing the error condition that arises if a pattern finishes with a -backslash, because -</P> -<P> -<PRE> - /abc\/ -</PRE> -</P> -<P> -is interpreted as the first line of a pattern that starts with "abc/", causing -pcretest to read the next line as a continuation of the regular expression. -</P> -<LI><A NAME="SEC5" HREF="#TOC1">PATTERN MODIFIERS</A> -<P> -The pattern may be followed by <B>i</B>, <B>m</B>, <B>s</B>, or <B>x</B> to set the -PCRE_CASELESS, PCRE_MULTILINE, PCRE_DOTALL, or PCRE_EXTENDED options, -respectively. For example: -</P> -<P> -<PRE> - /caseless/i -</PRE> -</P> -<P> -These modifier letters have the same effect as they do in Perl. There are -others which set PCRE options that do not correspond to anything in Perl: -<B>/A</B>, <B>/E</B>, and <B>/X</B> set PCRE_ANCHORED, PCRE_DOLLAR_ENDONLY, and -PCRE_EXTRA respectively. -</P> -<P> -Searching for all possible matches within each subject string can be requested -by the <B>/g</B> or <B>/G</B> modifier. After finding a match, PCRE is called -again to search the remainder of the subject string. The difference between -<B>/g</B> and <B>/G</B> is that the former uses the <I>startoffset</I> argument to -<B>pcre_exec()</B> to start searching at a new point within the entire string -(which is in effect what Perl does), whereas the latter passes over a shortened -substring. This makes a difference to the matching process if the pattern -begins with a lookbehind assertion (including \b or \B). -</P> -<P> -If any call to <B>pcre_exec()</B> in a <B>/g</B> or <B>/G</B> sequence matches an -empty string, the next call is done with the PCRE_NOTEMPTY and PCRE_ANCHORED -flags set in order to search for another, non-empty, match at the same point. -If this second match fails, the start offset is advanced by one, and the normal -match is retried. This imitates the way Perl handles such cases when using the -<B>/g</B> modifier or the <B>split()</B> function. -</P> -<P> -There are a number of other modifiers for controlling the way <B>pcretest</B> -operates. -</P> -<P> -The <B>/+</B> modifier requests that as well as outputting the substring that -matched the entire pattern, pcretest should in addition output the remainder of -the subject string. This is useful for tests where the subject contains -multiple copies of the same substring. -</P> -<P> -The <B>/L</B> modifier must be followed directly by the name of a locale, for -example, -</P> -<P> -<PRE> - /pattern/Lfr -</PRE> -</P> -<P> -For this reason, it must be the last modifier letter. The given locale is set, -<B>pcre_maketables()</B> is called to build a set of character tables for the -locale, and this is then passed to <B>pcre_compile()</B> when compiling the -regular expression. Without an <B>/L</B> modifier, NULL is passed as the tables -pointer; that is, <B>/L</B> applies only to the expression on which it appears. -</P> -<P> -The <B>/I</B> modifier requests that <B>pcretest</B> output information about the -compiled expression (whether it is anchored, has a fixed first character, and -so on). It does this by calling <B>pcre_fullinfo()</B> after compiling an -expression, and outputting the information it gets back. If the pattern is -studied, the results of that are also output. -</P> -<P> -The <B>/D</B> modifier is a PCRE debugging feature, which also assumes <B>/I</B>. -It causes the internal form of compiled regular expressions to be output after -compilation. -</P> -<P> -The <B>/S</B> modifier causes <B>pcre_study()</B> to be called after the -expression has been compiled, and the results used when the expression is -matched. -</P> -<P> -The <B>/M</B> modifier causes the size of memory block used to hold the compiled -pattern to be output. -</P> -<P> -The <B>/P</B> modifier causes <B>pcretest</B> to call PCRE via the POSIX wrapper -API rather than its native API. When this is done, all other modifiers except -<B>/i</B>, <B>/m</B>, and <B>/+</B> are ignored. REG_ICASE is set if <B>/i</B> is -present, and REG_NEWLINE is set if <B>/m</B> is present. The wrapper functions -force PCRE_DOLLAR_ENDONLY always, and PCRE_DOTALL unless REG_NEWLINE is set. -</P> -<P> -The <B>/8</B> modifier causes <B>pcretest</B> to call PCRE with the PCRE_UTF8 -option set. This turns on the (currently incomplete) support for UTF-8 -character handling in PCRE, provided that it was compiled with this support -enabled. This modifier also causes any non-printing characters in output -strings to be printed using the \x{hh...} notation if they are valid UTF-8 -sequences. -</P> -<LI><A NAME="SEC6" HREF="#TOC1">DATA LINES</A> -<P> -Before each data line is passed to <B>pcre_exec()</B>, leading and trailing -whitespace is removed, and it is then scanned for \ escapes. The following are -recognized: -</P> -<P> -<PRE> - \a alarm (= BEL) - \b backspace - \e escape - \f formfeed - \n newline - \r carriage return - \t tab - \v vertical tab - \nnn octal character (up to 3 octal digits) - \xhh hexadecimal character (up to 2 hex digits) - \x{hh...} hexadecimal UTF-8 character -</PRE> -</P> -<P> -<PRE> - \A pass the PCRE_ANCHORED option to <B>pcre_exec()</B> - \B pass the PCRE_NOTBOL option to <B>pcre_exec()</B> - \Cdd call pcre_copy_substring() for substring dd - after a successful match (any decimal number - less than 32) - \Gdd call pcre_get_substring() for substring dd - after a successful match (any decimal number - less than 32) - \L call pcre_get_substringlist() after a - successful match - \N pass the PCRE_NOTEMPTY option to <B>pcre_exec()</B> - \Odd set the size of the output vector passed to - <B>pcre_exec()</B> to dd (any number of decimal - digits) - \Z pass the PCRE_NOTEOL option to <B>pcre_exec()</B> -</PRE> -</P> -<P> -When \O is used, it may be higher or lower than the size set by the <B>-O</B> -option (or defaulted to 45); \O applies only to the call of <B>pcre_exec()</B> -for the line in which it appears. -</P> -<P> -A backslash followed by anything else just escapes the anything else. If the -very last character is a backslash, it is ignored. This gives a way of passing -an empty line as data, since a real empty line terminates the data input. -</P> -<P> -If <B>/P</B> was present on the regex, causing the POSIX wrapper API to be used, -only <B>\B</B>, and <B>\Z</B> have any effect, causing REG_NOTBOL and REG_NOTEOL -to be passed to <B>regexec()</B> respectively. -</P> -<P> -The use of \x{hh...} to represent UTF-8 characters is not dependent on the use -of the <B>/8</B> modifier on the pattern. It is recognized always. There may be -any number of hexadecimal digits inside the braces. The result is from one to -six bytes, encoded according to the UTF-8 rules. -</P> -<LI><A NAME="SEC7" HREF="#TOC1">OUTPUT FROM PCRETEST</A> -<P> -When a match succeeds, pcretest outputs the list of captured substrings that -<B>pcre_exec()</B> returns, starting with number 0 for the string that matched -the whole pattern. Here is an example of an interactive pcretest run. -</P> -<P> -<PRE> - $ pcretest - PCRE version 2.06 08-Jun-1999 -</PRE> -</P> -<P> -<PRE> - re> /^abc(\d+)/ - data> abc123 - 0: abc123 - 1: 123 - data> xyz - No match -</PRE> -</P> -<P> -If the strings contain any non-printing characters, they are output as \0x -escapes, or as \x{...} escapes if the <B>/8</B> modifier was present on the -pattern. If the pattern has the <B>/+</B> modifier, then the output for -substring 0 is followed by the the rest of the subject string, identified by -"0+" like this: -</P> -<P> -<PRE> - re> /cat/+ - data> cataract - 0: cat - 0+ aract -</PRE> -</P> -<P> -If the pattern has the <B>/g</B> or <B>/G</B> modifier, the results of successive -matching attempts are output in sequence, like this: -</P> -<P> -<PRE> - re> /\Bi(\w\w)/g - data> Mississippi - 0: iss - 1: ss - 0: iss - 1: ss - 0: ipp - 1: pp -</PRE> -</P> -<P> -"No match" is output only if the first match attempt fails. -</P> -<P> -If any of the sequences <B>\C</B>, <B>\G</B>, or <B>\L</B> are present in a -data line that is successfully matched, the substrings extracted by the -convenience functions are output with C, G, or L after the string number -instead of a colon. This is in addition to the normal full list. The string -length (that is, the return from the extraction function) is given in -parentheses after each string for <B>\C</B> and <B>\G</B>. -</P> -<P> -Note that while patterns can be continued over several lines (a plain ">" -prompt is used for continuations), data lines may not. However newlines can be -included in data by means of the \n escape. -</P> -<LI><A NAME="SEC8" HREF="#TOC1">AUTHOR</A> -<P> -Philip Hazel <ph10@cam.ac.uk> -<BR> -University Computing Service, -<BR> -New Museums Site, -<BR> -Cambridge CB2 3QG, England. -<BR> -Phone: +44 1223 334714 -</P> -<P> -Last updated: 25 August 2002 -<BR> -Copyright (c) 1997-2002 University of Cambridge. diff --git a/ext/pcre/pcrelib/doc/pcretest.txt b/ext/pcre/pcrelib/doc/pcretest.txt deleted file mode 100644 index 54e989a83c..0000000000 --- a/ext/pcre/pcrelib/doc/pcretest.txt +++ /dev/null @@ -1,329 +0,0 @@ -NAME - pcretest - a program for testing Perl-compatible regular - expressions. - - - -SYNOPSIS - pcretest [-d] [-i] [-m] [-o osize] [-p] [-t] [source] [des- - tination] - - pcretest was written as a test program for the PCRE regular - expression library itself, but it can also be used for - experimenting with regular expressions. This man page - describes the features of the test program; for details of - the regular expressions themselves, see the pcre man page. - - - -OPTIONS - -d Behave as if each regex had the /D modifier (see - below); the internal form is output after compila- - tion. - - -i Behave as if each regex had the /I modifier; - information about the compiled pattern is given - after compilation. - - -m Output the size of each compiled pattern after it - has been compiled. This is equivalent to adding /M - to each regular expression. For compatibility with - earlier versions of pcretest, -s is a synonym for - -m. - - -o osize Set the number of elements in the output vector - that is used when calling PCRE to be osize. The - default value is 45, which is enough for 14 cap- - turing subexpressions. The vector size can be - changed for individual matching calls by including - \O in the data line (see below). - - -p Behave as if each regex has /P modifier; the POSIX - wrapper API is used to call PCRE. None of the - other options has any effect when -p is set. - - -t Run each compile, study, and match many times with - a timer, and output resulting time per compile or - match (in milliseconds). Do not set -t with -m, - because you will then get the size output 20000 - times and the timing will be distorted. - - - -DESCRIPTION - If pcretest is given two filename arguments, it reads from - the first and writes to the second. If it is given only one - - - - -SunOS 5.8 Last change: 1 - - - - filename argument, it reads from that file and writes to - stdout. Otherwise, it reads from stdin and writes to stdout, - and prompts for each line of input, using "re>" to prompt - for regular expressions, and "data>" to prompt for data - lines. - - The program handles any number of sets of input on a single - input file. Each set starts with a regular expression, and - continues with any number of data lines to be matched - against the pattern. - - Each line is matched separately and independently. If you - want to do multiple-line matches, you have to use the \n - escape sequence in a single line of input to encode the new- - line characters. The maximum length of data line is 30,000 - characters. - - An empty line signals the end of the data lines, at which - point a new regular expression is read. The regular expres- - sions are given enclosed in any non-alphameric delimiters - other than backslash, for example - - /(a|bc)x+yz/ - - White space before the initial delimiter is ignored. A regu- - lar expression may be continued over several input lines, in - which case the newline characters are included within it. It - is possible to include the delimiter within the pattern by - escaping it, for example - - /abc\/def/ - - If you do so, the escape and the delimiter form part of the - pattern, but since delimiters are always non-alphameric, - this does not affect its interpretation. If the terminating - delimiter is immediately followed by a backslash, for exam- - ple, - - /abc/\ - - then a backslash is added to the end of the pattern. This is - done to provide a way of testing the error condition that - arises if a pattern finishes with a backslash, because - - /abc\/ - - is interpreted as the first line of a pattern that starts - with "abc/", causing pcretest to read the next line as a - continuation of the regular expression. - - - -PATTERN MODIFIERS - The pattern may be followed by i, m, s, or x to set the - PCRE_CASELESS, PCRE_MULTILINE, PCRE_DOTALL, or PCRE_EXTENDED - options, respectively. For example: - - /caseless/i - - These modifier letters have the same effect as they do in - Perl. There are others which set PCRE options that do not - correspond to anything in Perl: /A, /E, and /X set - PCRE_ANCHORED, PCRE_DOLLAR_ENDONLY, and PCRE_EXTRA respec- - tively. - - Searching for all possible matches within each subject - string can be requested by the /g or /G modifier. After - finding a match, PCRE is called again to search the - remainder of the subject string. The difference between /g - and /G is that the former uses the startoffset argument to - pcre_exec() to start searching at a new point within the - entire string (which is in effect what Perl does), whereas - the latter passes over a shortened substring. This makes a - difference to the matching process if the pattern begins - with a lookbehind assertion (including \b or \B). - - If any call to pcre_exec() in a /g or /G sequence matches an - empty string, the next call is done with the PCRE_NOTEMPTY - and PCRE_ANCHORED flags set in order to search for another, - non-empty, match at the same point. If this second match - fails, the start offset is advanced by one, and the normal - match is retried. This imitates the way Perl handles such - cases when using the /g modifier or the split() function. - - There are a number of other modifiers for controlling the - way pcretest operates. - - The /+ modifier requests that as well as outputting the sub- - string that matched the entire pattern, pcretest should in - addition output the remainder of the subject string. This is - useful for tests where the subject contains multiple copies - of the same substring. - - The /L modifier must be followed directly by the name of a - locale, for example, - - /pattern/Lfr - - For this reason, it must be the last modifier letter. The - given locale is set, pcre_maketables() is called to build a - set of character tables for the locale, and this is then - passed to pcre_compile() when compiling the regular expres- - sion. Without an /L modifier, NULL is passed as the tables - pointer; that is, /L applies only to the expression on which - it appears. - - The /I modifier requests that pcretest output information - about the compiled expression (whether it is anchored, has a - fixed first character, and so on). It does this by calling - pcre_fullinfo() after compiling an expression, and output- - ting the information it gets back. If the pattern is stu- - died, the results of that are also output. - - The /D modifier is a PCRE debugging feature, which also - assumes /I. It causes the internal form of compiled regular - expressions to be output after compilation. - - The /S modifier causes pcre_study() to be called after the - expression has been compiled, and the results used when the - expression is matched. - - The /M modifier causes the size of memory block used to hold - the compiled pattern to be output. - - The /P modifier causes pcretest to call PCRE via the POSIX - wrapper API rather than its native API. When this is done, - all other modifiers except /i, /m, and /+ are ignored. - REG_ICASE is set if /i is present, and REG_NEWLINE is set if - /m is present. The wrapper functions force - PCRE_DOLLAR_ENDONLY always, and PCRE_DOTALL unless - REG_NEWLINE is set. - - The /8 modifier causes pcretest to call PCRE with the - PCRE_UTF8 option set. This turns on the (currently incom- - plete) support for UTF-8 character handling in PCRE, pro- - vided that it was compiled with this support enabled. This - modifier also causes any non-printing characters in output - strings to be printed using the \x{hh...} notation if they - are valid UTF-8 sequences. - - - -DATA LINES - Before each data line is passed to pcre_exec(), leading and - trailing whitespace is removed, and it is then scanned for \ - escapes. The following are recognized: - - \a alarm (= BEL) - \b backspace - \e escape - \f formfeed - \n newline - \r carriage return - \t tab - \v vertical tab - \nnn octal character (up to 3 octal digits) - - hexadecimal character (up to 2 hex digits) - \x{hh...} hexadecimal UTF-8 character - - \A pass the PCRE_ANCHORED option to pcre_exec() - \B pass the PCRE_NOTBOL option to pcre_exec() - \Cdd call pcre_copy_substring() for substring dd - after a successful match (any decimal number - less than 32) - \Gdd call pcre_get_substring() for substring dd - after a successful match (any decimal number - less than 32) - \L call pcre_get_substringlist() after a - successful match - \N pass the PCRE_NOTEMPTY option to pcre_exec() - \Odd set the size of the output vector passed to - pcre_exec() to dd (any number of decimal - digits) - \Z pass the PCRE_NOTEOL option to pcre_exec() - - When \O is used, it may be higher or lower than the size set - by the -O option (or defaulted to 45); \O applies only to - the call of pcre_exec() for the line in which it appears. - - A backslash followed by anything else just escapes the any- - thing else. If the very last character is a backslash, it is - ignored. This gives a way of passing an empty line as data, - since a real empty line terminates the data input. - - If /P was present on the regex, causing the POSIX wrapper - API to be used, only B, and Z have any effect, causing - REG_NOTBOL and REG_NOTEOL to be passed to regexec() respec- - tively. - - The use of \x{hh...} to represent UTF-8 characters is not - dependent on the use of the /8 modifier on the pattern. It - is recognized always. There may be any number of hexadecimal - digits inside the braces. The result is from one to six - bytes, encoded according to the UTF-8 rules. - - - -OUTPUT FROM PCRETEST - When a match succeeds, pcretest outputs the list of captured - substrings that pcre_exec() returns, starting with number 0 - for the string that matched the whole pattern. Here is an - example of an interactive pcretest run. - - $ pcretest - PCRE version 2.06 08-Jun-1999 - - re> /^abc(\d+)/ - data> abc123 - - 0: abc123 - 1: 123 - data> xyz - No match - - If the strings contain any non-printing characters, they are - output as \0x escapes, or as \x{...} escapes if the /8 - modifier was present on the pattern. If the pattern has the - /+ modifier, then the output for substring 0 is followed by - the the rest of the subject string, identified by "0+" like - this: - - re> /cat/+ - data> cataract - 0: cat - 0+ aract - - If the pattern has the /g or /G modifier, the results of - successive matching attempts are output in sequence, like - this: - - re> /\Bi(\w\w)/g - data> Mississippi - 0: iss - 1: ss - 0: iss - 1: ss - 0: ipp - 1: pp - - "No match" is output only if the first match attempt fails. - - If any of the sequences \C, \G, or \L are present in a data - line that is successfully matched, the substrings extracted - by the convenience functions are output with C, G, or L - after the string number instead of a colon. This is in addi- - tion to the normal full list. The string length (that is, - the return from the extraction function) is given in - parentheses after each string for \C and \G. - - Note that while patterns can be continued over several lines - (a plain ">" prompt is used for continuations), data lines - may not. However newlines can be included in data by means - of the \n escape. - - - -AUTHOR - Philip Hazel <ph10@cam.ac.uk> - University Computing Service, - New Museums Site, - Cambridge CB2 3QG, England. - Phone: +44 1223 334714 - - Last updated: 25 August 2002 - Copyright (c) 1997-2002 University of Cambridge. diff --git a/ext/pcre/pcrelib/doc/perltest.txt b/ext/pcre/pcrelib/doc/perltest.txt deleted file mode 100644 index 9ea9d932a5..0000000000 --- a/ext/pcre/pcrelib/doc/perltest.txt +++ /dev/null @@ -1,34 +0,0 @@ -The perltest program --------------------- - -The perltest program tests Perl's regular expressions; it has the same -specification as pcretest, and so can be given identical input, except that -input patterns can be followed only by Perl's lower case modifiers and /+ (as -used by pcretest), which is recognized and handled by the program. - -The data lines are processed as Perl double-quoted strings, so if they contain -" \ $ or @ characters, these have to be escaped. For this reason, all such -characters in testinput1 and testinput3 are escaped so that they can be used -for perltest as well as for pcretest, and the special upper case modifiers such -as /A that pcretest recognizes are not used in these files. The output should -be identical, apart from the initial identifying banner. - -The perltest script can also test UTF-8 features. It works as is for Perl 5.8 -or higher. It recognizes the special modifier /8 that pcretest uses to invoke -UTF-8 functionality. The testinput5 file can be fed to perltest to run UTF-8 -tests. - -For Perl 5.6, perltest won't work unmodified for the UTF-8 tests. You need to -uncomment the "use utf8" lines that it contains. It is best to do this on a -copy of the script, because for non-UTF-8 tests, these lines should remain -commented out. - -The testinput2 and testinput4 files are not suitable for feeding to perltest, -since they do make use of the special upper case modifiers and escapes that -pcretest uses to test some features of PCRE. The first of these files also -contains malformed regular expressions, in order to check that PCRE diagnoses -them correctly. Similarly, testinput6 tests UTF-8 features that do not relate -to Perl. - -Philip Hazel <ph10@cam.ac.uk> -August 2002 diff --git a/ext/pcre/pcrelib/get.c b/ext/pcre/pcrelib/get.c index 55e736dc24..a20473cf24 100644 --- a/ext/pcre/pcrelib/get.c +++ b/ext/pcre/pcrelib/get.c @@ -9,7 +9,7 @@ the file Tech.Notes for some information on the internals. Written by: Philip Hazel <ph10@cam.ac.uk> - Copyright (c) 1997-2001 University of Cambridge + Copyright (c) 1997-2003 University of Cambridge ----------------------------------------------------------------------------- Permission is granted to anyone to use this software for any purpose on any @@ -43,6 +43,52 @@ the external pcre header. */ #include "internal.h" +/************************************************* +* Find number for named string * +*************************************************/ + +/* This function is used by the two extraction functions below, as well +as being generally available. + +Arguments: + code the compiled regex + stringname the name whose number is required + +Returns: the number of the named parentheses, or a negative number + (PCRE_ERROR_NOSUBSTRING) if not found +*/ + +int +pcre_get_stringnumber(const pcre *code, const char *stringname) +{ +int rc; +int entrysize; +int top, bot; +uschar *nametable; + +if ((rc = pcre_fullinfo(code, NULL, PCRE_INFO_NAMECOUNT, &top)) != 0) + return rc; +if (top <= 0) return PCRE_ERROR_NOSUBSTRING; + +if ((rc = pcre_fullinfo(code, NULL, PCRE_INFO_NAMEENTRYSIZE, &entrysize)) != 0) + return rc; +if ((rc = pcre_fullinfo(code, NULL, PCRE_INFO_NAMETABLE, &nametable)) != 0) + return rc; + +bot = 0; +while (top > bot) + { + int mid = (top + bot) / 2; + uschar *entry = nametable + entrysize*mid; + int c = strcmp(stringname, (char *)(entry + 2)); + if (c == 0) return (entry[0] << 8) + entry[1]; + if (c > 0) bot = mid + 1; else top = mid; + } + +return PCRE_ERROR_NOSUBSTRING; +} + + /************************************************* * Copy captured string to given buffer * @@ -89,6 +135,44 @@ return yield; /************************************************* +* Copy named captured string to given buffer * +*************************************************/ + +/* This function copies a single captured substring into a given buffer, +identifying it by name. + +Arguments: + code the compiled regex + subject the subject string that was matched + ovector pointer to the offsets table + stringcount the number of substrings that were captured + (i.e. the yield of the pcre_exec call, unless + that was zero, in which case it should be 1/3 + of the offset table size) + stringname the name of the required substring + buffer where to put the substring + size the size of the buffer + +Returns: if successful: + the length of the copied string, not including the zero + that is put on the end; can be zero + if not successful: + PCRE_ERROR_NOMEMORY (-6) buffer too small + PCRE_ERROR_NOSUBSTRING (-7) no such captured substring +*/ + +int +pcre_copy_named_substring(const pcre *code, const char *subject, int *ovector, + int stringcount, const char *stringname, char *buffer, int size) +{ +int n = pcre_get_stringnumber(code, stringname); +if (n <= 0) return n; +return pcre_copy_substring(subject, ovector, stringcount, n, buffer, size); +} + + + +/************************************************* * Copy all captured strings to new store * *************************************************/ @@ -208,6 +292,44 @@ return yield; /************************************************* +* Copy named captured string to new store * +*************************************************/ + +/* This function copies a single captured substring, identified by name, into +new store. + +Arguments: + code the compiled regex + subject the subject string that was matched + ovector pointer to the offsets table + stringcount the number of substrings that were captured + (i.e. the yield of the pcre_exec call, unless + that was zero, in which case it should be 1/3 + of the offset table size) + stringname the name of the required substring + stringptr where to put the pointer + +Returns: if successful: + the length of the copied string, not including the zero + that is put on the end; can be zero + if not successful: + PCRE_ERROR_NOMEMORY (-6) couldn't get memory + PCRE_ERROR_NOSUBSTRING (-7) no such captured substring +*/ + +int +pcre_get_named_substring(const pcre *code, const char *subject, int *ovector, + int stringcount, const char *stringname, const char **stringptr) +{ +int n = pcre_get_stringnumber(code, stringname); +if (n <= 0) return n; +return pcre_get_substring(subject, ovector, stringcount, n, stringptr); +} + + + + +/************************************************* * Free store obtained by get_substring * *************************************************/ diff --git a/ext/pcre/pcrelib/internal.h b/ext/pcre/pcrelib/internal.h index 2396668a33..7bcdeefdfb 100644 --- a/ext/pcre/pcrelib/internal.h +++ b/ext/pcre/pcrelib/internal.h @@ -53,6 +53,18 @@ On Unix systems, "configure" can be used to override this default. */ #define NEWLINE '\n' #endif +/* The value of MATCH_LIMIT determines the default number of times the match() +function can be called during a single execution of pcre_exec(). (There is a +runtime method of setting a different limit.) The limit exists in order to +catch runaway regular expressions that take for ever to determine that they do +not match. The default is set very large so that it does not accidentally catch +legitimate cases. On Unix systems, "configure" can be used to override this +default default. */ + +#ifndef MATCH_LIMIT +#define MATCH_LIMIT 10000000 +#endif + /* When compiling for use with the Virtual Pascal compiler, these functions need to have their names changed. PCRE must be compiled with the -DVPCOMPAT option on the command line. */ @@ -173,6 +185,11 @@ capturing parenthesis numbers in back references. */ #include <stdio.h> #include <stdlib.h> #include <string.h> + +#ifndef PCRE_SPY +#define PCRE_DEFINITION /* Win32 __declspec(export) trigger for .dll */ +#endif + #include "pcre.h" /* In case there is no definition of offsetof() provided - though any proper @@ -192,8 +209,8 @@ with negative values. The public options defined in pcre.h start at the least significant end. Make sure they don't overlap, though now that we have expanded to four bytes there is plenty of space. */ -#define PCRE_FIRSTSET 0x40000000 /* first_char is set */ -#define PCRE_REQCHSET 0x20000000 /* req_char is set */ +#define PCRE_FIRSTSET 0x40000000 /* first_byte is set */ +#define PCRE_REQCHSET 0x20000000 /* req_byte is set */ #define PCRE_STARTLINE 0x10000000 /* start after \n for multiline */ #define PCRE_ICHANGED 0x08000000 /* i option changes within regex */ @@ -206,7 +223,8 @@ time, run time or study time, respectively. */ #define PUBLIC_OPTIONS \ (PCRE_CASELESS|PCRE_EXTENDED|PCRE_ANCHORED|PCRE_MULTILINE| \ - PCRE_DOTALL|PCRE_DOLLAR_ENDONLY|PCRE_EXTRA|PCRE_UNGREEDY|PCRE_UTF8) + PCRE_DOTALL|PCRE_DOLLAR_ENDONLY|PCRE_EXTRA|PCRE_UNGREEDY|PCRE_UTF8| \ + PCRE_NO_AUTO_CAPTURE) #define PUBLIC_EXEC_OPTIONS \ (PCRE_ANCHORED|PCRE_NOTBOL|PCRE_NOTEOL|PCRE_NOTEMPTY) @@ -222,9 +240,11 @@ time, run time or study time, respectively. */ #define REQ_UNSET (-2) #define REQ_NONE (-1) -/* Flags added to firstchar or reqchar */ +/* Flags added to firstbyte or reqbyte; a "non-literal" item is either a +variable-length repeat, or a anything other than literal characters. */ #define REQ_CASELESS 0x0100 /* indicates caselessness */ +#define REQ_VARY 0x0200 /* reqbyte followed non-literal item */ /* Miscellaneous definitions */ @@ -253,8 +273,11 @@ ESC_n is defined as yet another macro, which is set in config.h to either \n #define ESC_r '\r' #endif -#ifndef ESC_t -#define ESC_t '\t' +/* We can't officially use ESC_t because it is a POSIX reserved identifier +(presumably because of all the others like size_t). */ + +#ifndef ESC_tee +#define ESC_tee '\t' #endif /* These are escaped items that aren't just an encoding of a particular data @@ -271,6 +294,16 @@ character, that code will have to change. */ enum { ESC_A = 1, ESC_G, ESC_B, ESC_b, ESC_D, ESC_d, ESC_S, ESC_s, ESC_W, ESC_w, ESC_dum1, ESC_C, ESC_Z, ESC_z, ESC_E, ESC_Q, ESC_REF }; +/* Flag bits and data types for the extended class (OP_XCLASS) for classes that +contain UTF-8 characters with values greater than 255. */ + +#define XCL_NOT 0x01 /* Flag: this is a negative class */ +#define XCL_MAP 0x02 /* Flag: a 32-byte map is present */ + +#define XCL_END 0 /* Marks end of individual items */ +#define XCL_SINGLE 1 /* Single item (one multibyte char) follows */ +#define XCL_RANGE 2 /* A range (two multibyte chars) follows */ + /* Opcode table: OP_BRA must be last, as all values >= it are used for brackets that extract substrings. Starting from 1 (i.e. after OP_END), the values up to @@ -343,42 +376,53 @@ enum { OP_CRRANGE, /* 53 These are different to the three seta above. */ OP_CRMINRANGE, /* 54 */ - OP_CLASS, /* 55 Match a character class */ - OP_REF, /* 56 Match a back reference */ - OP_RECURSE, /* 57 Match a numbered subpattern (possibly recursive) */ - OP_CALLOUT, /* 58 Call out to external function if provided */ + OP_CLASS, /* 55 Match a character class, chars < 256 only */ + OP_NCLASS, /* 56 Same, but the bitmap was created from a negative + class - the difference is relevant only when a UTF-8 + character > 255 is encountered. */ + + OP_XCLASS, /* 56 Extended class for handling UTF-8 chars within the + class. This does both positive and negative. */ - OP_ALT, /* 59 Start of alternation */ - OP_KET, /* 60 End of group that doesn't have an unbounded repeat */ - OP_KETRMAX, /* 61 These two must remain together and in this */ - OP_KETRMIN, /* 62 order. They are for groups the repeat for ever. */ + OP_REF, /* 57 Match a back reference */ + OP_RECURSE, /* 58 Match a numbered subpattern (possibly recursive) */ + OP_CALLOUT, /* 59 Call out to external function if provided */ + + OP_ALT, /* 60 Start of alternation */ + OP_KET, /* 61 End of group that doesn't have an unbounded repeat */ + OP_KETRMAX, /* 62 These two must remain together and in this */ + OP_KETRMIN, /* 63 order. They are for groups the repeat for ever. */ /* The assertions must come before ONCE and COND */ - OP_ASSERT, /* 63 Positive lookahead */ - OP_ASSERT_NOT, /* 64 Negative lookahead */ - OP_ASSERTBACK, /* 65 Positive lookbehind */ - OP_ASSERTBACK_NOT, /* 66 Negative lookbehind */ - OP_REVERSE, /* 67 Move pointer back - used in lookbehind assertions */ + OP_ASSERT, /* 64 Positive lookahead */ + OP_ASSERT_NOT, /* 65 Negative lookahead */ + OP_ASSERTBACK, /* 66 Positive lookbehind */ + OP_ASSERTBACK_NOT, /* 67 Negative lookbehind */ + OP_REVERSE, /* 68 Move pointer back - used in lookbehind assertions */ /* ONCE and COND must come after the assertions, with ONCE first, as there's a test for >= ONCE for a subpattern that isn't an assertion. */ - OP_ONCE, /* 68 Once matched, don't back up into the subpattern */ - OP_COND, /* 69 Conditional group */ - OP_CREF, /* 70 Used to hold an extraction string number (cond ref) */ + OP_ONCE, /* 69 Once matched, don't back up into the subpattern */ + OP_COND, /* 70 Conditional group */ + OP_CREF, /* 71 Used to hold an extraction string number (cond ref) */ - OP_BRAZERO, /* 71 These two must remain together and in this */ - OP_BRAMINZERO, /* 72 order. */ + OP_BRAZERO, /* 72 These two must remain together and in this */ + OP_BRAMINZERO, /* 73 order. */ - OP_BRANUMBER, /* 73 Used for extracting brackets whose number is greater + OP_BRANUMBER, /* 74 Used for extracting brackets whose number is greater than can fit into an opcode. */ - OP_BRA /* 74 This and greater values are used for brackets that + OP_BRA /* 75 This and greater values are used for brackets that extract substrings up to a basic limit. After that, use is made of OP_BRANUMBER. */ }; +/* WARNING: There is an implicit assumption in study.c that all opcodes are +less than 128 in value. This makes handling UTF-8 character sequences easier. +*/ + /* This macro defines textual names for all the opcodes. There are used only for debugging, in pcre.c when DEBUG is defined, and also in pcretest.c. The @@ -392,7 +436,7 @@ macro is referenced only in printint.c. */ "*", "*?", "+", "+?", "?", "??", "{", "{", "{", \ "*", "*?", "+", "+?", "?", "??", "{", "{", "{", \ "*", "*?", "+", "+?", "?", "??", "{", "{", \ - "class", "Ref", "Recurse", "Callout", \ + "class", "nclass", "xclass", "Ref", "Recurse", "Callout", \ "Alt", "Ket", "KetRmax", "KetRmin", "Assert", "Assert not", \ "AssertB", "AssertB not", "Reverse", "Once", "Cond", "Cond ref",\ "Brazero", "Braminzero", "Branumber", "Bra" @@ -401,7 +445,11 @@ macro is referenced only in printint.c. */ /* This macro defines the length of fixed length operations in the compiled regex. The lengths are used when searching for specific things, and also in the debugging printing of a compiled regex. We use a macro so that it can be -incorporated both into pcre.c and pcretest.c without being publicly exposed. */ +incorporated both into pcre.c and pcretest.c without being publicly exposed. + +As things have been extended, some of these are no longer fixed lenths, but are +minima instead. For example, the length of a single-character repeat may vary +in UTF-8 mode. The code that uses this table must know about such things. */ #define OP_LENGTHS \ 1, /* End */ \ @@ -410,18 +458,21 @@ incorporated both into pcre.c and pcretest.c without being publicly exposed. */ 2, /* Chars - the minimum length */ \ 2, /* not */ \ /* Positive single-char repeats */ \ - 2, 2, 2, 2, 2, 2, /* *, *?, +, +?, ?, ?? */ \ - 4, 4, 4, /* upto, minupto, exact */ \ + 2, 2, 2, 2, 2, 2, /* *, *?, +, +?, ?, ?? ** These are */ \ + 4, 4, 4, /* upto, minupto, exact ** minima */ \ /* Negative single-char repeats */ \ 2, 2, 2, 2, 2, 2, /* NOT *, *?, +, +?, ?, ?? */ \ 4, 4, 4, /* NOT upto, minupto, exact */ \ /* Positive type repeats */ \ 2, 2, 2, 2, 2, 2, /* Type *, *?, +, +?, ?, ?? */ \ 4, 4, 4, /* Type upto, minupto, exact */ \ - /* Multi-char class repeats */ \ + /* Character class & ref repeats */ \ 1, 1, 1, 1, 1, 1, /* *, *?, +, +?, ?, ?? */ \ 5, 5, /* CRRANGE, CRMINRANGE */ \ - 33, 3, /* CLASS, REF */ \ + 33, /* CLASS */ \ + 33, /* NCLASS */ \ + 0, /* XCLASS - variable length */ \ + 3, /* REF */ \ 1+LINK_SIZE, /* RECURSE */ \ 2, /* CALLOUT */ \ 1+LINK_SIZE, /* Alt */ \ @@ -434,7 +485,7 @@ incorporated both into pcre.c and pcretest.c without being publicly exposed. */ 1+LINK_SIZE, /* Assert behind not */ \ 1+LINK_SIZE, /* Reverse */ \ 1+LINK_SIZE, /* Once */ \ - 1, /* COND */ \ + 1+LINK_SIZE, /* COND */ \ 3, /* CREF */ \ 1, 1, /* BRAZERO, BRAMINZERO */ \ 3, /* BRANUMBER */ \ @@ -490,7 +541,7 @@ just to accommodate the POSIX wrapper. */ #define ERR30 "unknown POSIX class name" #define ERR31 "POSIX collating elements are not supported" #define ERR32 "this version of PCRE is not compiled with PCRE_UTF8 support" -#define ERR33 "characters with values > 255 are not yet supported in classes" +#define ERR33 "spare error" #define ERR34 "character value in \\x{...} sequence is too large" #define ERR35 "invalid condition (?(0)" #define ERR36 "\\C not allowed in lookbehind assertion" @@ -521,19 +572,19 @@ typedef struct real_pcre { unsigned long int options; unsigned short int top_bracket; unsigned short int top_backref; - unsigned short int first_char; - unsigned short int req_char; + unsigned short int first_byte; + unsigned short int req_byte; unsigned short int name_entry_size; /* Size of any name items; 0 => none */ unsigned short int name_count; /* Number of name items */ } real_pcre; -/* The real format of the extra block returned by pcre_study(). */ +/* The format of the block used to store data from pcre_study(). */ -typedef struct real_pcre_extra { +typedef struct pcre_study_data { + size_t size; /* Total that was malloced */ uschar options; uschar start_bits[32]; -} real_pcre_extra; - +} pcre_study_data; /* Structure for passing "static" information around between the functions doing the compiling, so that they are thread-safe. */ @@ -547,6 +598,9 @@ typedef struct compile_data { uschar *name_table; /* The name/number table */ int names_found; /* Number of entries so far */ int name_entry_size; /* Size of each entry */ + int top_backref; /* Maximum back reference */ + unsigned int backref_map; /* Bitmap of low back refs */ + int req_varyopt; /* "After variable item" flag for reqbyte */ } compile_data; /* Structure for maintaining a chain of pointers to the currently incomplete @@ -573,7 +627,8 @@ typedef struct recursion_info { doing the matching, so that they are thread-safe. */ typedef struct match_data { - int errorcode; /* As it says */ + unsigned long int match_call_count; /* As it says */ + unsigned long int match_limit;/* As it says */ int *offset_vector; /* Offset vector */ int offset_end; /* One past the end */ int offset_max; /* The maximum usable for return data */ @@ -594,6 +649,7 @@ typedef struct match_data { int capture_last; /* Most recent capture number */ int start_offset; /* The start offset value */ recursion_info *recursive; /* Linked list of recursion data */ + void *callout_data; /* To pass back to callouts */ } match_data; /* Bit definitions for entries in the pcre_ctypes table. */ diff --git a/ext/pcre/pcrelib/maketables.c b/ext/pcre/pcrelib/maketables.c index f89765214c..257fe89cec 100644 --- a/ext/pcre/pcrelib/maketables.c +++ b/ext/pcre/pcrelib/maketables.c @@ -8,7 +8,7 @@ and semantics are as close as possible to those of the Perl 5 language. Written by: Philip Hazel <ph10@cam.ac.uk> - Copyright (c) 1997-2001 University of Cambridge + Copyright (c) 1997-2003 University of Cambridge ----------------------------------------------------------------------------- Permission is granted to anyone to use this software for any purpose on any diff --git a/ext/pcre/pcrelib/pcre.c b/ext/pcre/pcrelib/pcre.c index 8c74905131..5da0f76102 100644 --- a/ext/pcre/pcrelib/pcre.c +++ b/ext/pcre/pcrelib/pcre.c @@ -32,7 +32,6 @@ restrictions: ----------------------------------------------------------------------------- */ - /* Define DEBUG to get debugging output on stdout. */ /* #define DEBUG */ @@ -69,7 +68,6 @@ compile time. */ #define BRASTACK_SIZE 200 - /* Maximum number of ints of offset to save on the stack for recursive calls. If the offset vector is bigger, malloc is used. This should be a multiple of 3, because the offset vector is always a multiple of 3 long. */ @@ -78,13 +76,17 @@ because the offset vector is always a multiple of 3 long. */ /* The number of bytes in a literal character string above which we can't add -any more is different when UTF-8 characters may be encountered. */ +any more is set at 250 in order to allow for UTF-8 characters. (In theory it +could be 255 when UTF-8 support is excluded, but that means that some of the +test output would be different, which just complicates things.) */ -#ifdef SUPPORT_UTF8 #define MAXLIT 250 -#else -#define MAXLIT 255 -#endif + + +/* The maximum remaining length of subject we are prepared to search for a +req_byte match. */ + +#define REQ_BYTE_MAX 1000 /* Table of sizes for the fixed-length opcodes. It's defined in a macro so that @@ -111,7 +113,7 @@ static const short int escapes[] = { 0, 0, -ESC_Z, '[', '\\', ']', '^', '_', /* X - _ */ '`', 7, -ESC_b, 0, -ESC_d, ESC_e, ESC_f, 0, /* ` - g */ 0, 0, 0, 0, 0, 0, ESC_n, 0, /* h - o */ - 0, 0, ESC_r, -ESC_s, ESC_t, 0, 0, -ESC_w, /* p - w */ + 0, 0, ESC_r, -ESC_s, ESC_tee, 0, 0, -ESC_w, /* p - w */ 0, 0, -ESC_z /* x - z */ }; @@ -148,6 +150,56 @@ static const int posix_class_maps[] = { cbit_xdigit,-1, -1 /* xdigit */ }; +/* Table to identify ASCII digits and hex digits. This is used when compiling +patterns. Note that the tables in chartables are dependent on the locale, and +may mark arbitrary characters as digits - but the PCRE compiling code expects +to handle only 0-9, a-z, and A-Z as digits when compiling. That is why we have +a private table here. It costs 256 bytes, but it is a lot faster than doing +character value tests (at least in some simple cases I timed), and in some +applications one wants PCRE to compile efficiently as well as match +efficiently. + +For convenience, we use the same bit definitions as in chartables: + + 0x04 decimal digit + 0x08 hexadecimal digit + +Then we can use ctype_digit and ctype_xdigit in the code. */ + +static const unsigned char digitab[] = + { + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 0- 7 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 8- 15 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 16- 23 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 24- 31 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* - ' */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* ( - / */ + 0x0c,0x0c,0x0c,0x0c,0x0c,0x0c,0x0c,0x0c, /* 0 - 7 */ + 0x0c,0x0c,0x00,0x00,0x00,0x00,0x00,0x00, /* 8 - ? */ + 0x00,0x08,0x08,0x08,0x08,0x08,0x08,0x00, /* @ - G */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* H - O */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* P - W */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* X - _ */ + 0x00,0x08,0x08,0x08,0x08,0x08,0x08,0x00, /* ` - g */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* h - o */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* p - w */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* x -127 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 128-135 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 136-143 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 144-151 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 152-159 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 160-167 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 168-175 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 176-183 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 184-191 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 192-199 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 200-207 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 208-215 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 216-223 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 224-231 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 232-239 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 240-247 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00};/* 248-255 */ /* Definition to allow mutual recursion */ @@ -170,6 +222,12 @@ typedef struct eptrblock { #define match_condassert 0x01 /* Called to check a condition assertion */ #define match_isgroup 0x02 /* Set if start of bracketed group */ +/* Non-error returns from the match() function. Error returns are externally +defined PCRE_ERROR_xxx codes, which are all negative. */ + +#define MATCH_MATCH 1 +#define MATCH_NOMATCH 0 + /************************************************* @@ -198,49 +256,86 @@ byte. The macros for character handling generate simple sequences when used in byte-mode, and more complicated ones for UTF-8 characters. */ #ifndef SUPPORT_UTF8 +#define GETCHAR(c, eptr) c = *eptr; #define GETCHARINC(c, eptr) c = *eptr++; +#define GETCHARINCTEST(c, eptr) c = *eptr++; #define GETCHARLEN(c, eptr, len) c = *eptr; #define BACKCHAR(eptr) #else /* SUPPORT_UTF8 */ -/* Get the next UTF-8 character, advancing the pointer */ +/* Get the next UTF-8 character, not advancing the pointer. This is called when +we know we are in UTF-8 mode. */ + +#define GETCHAR(c, eptr) \ + c = *eptr; \ + if ((c & 0xc0) == 0xc0) \ + { \ + int gcii; \ + int gcaa = utf8_table4[c & 0x3f]; /* Number of additional bytes */ \ + int gcss = 6*gcaa; \ + c = (c & utf8_table3[gcaa]) << gcss; \ + for (gcii = 1; gcii <= gcaa; gcii++) \ + { \ + gcss -= 6; \ + c |= (eptr[gcii] & 0x3f) << gcss; \ + } \ + } + +/* Get the next UTF-8 character, advancing the pointer. This is called when we +know we are in UTF-8 mode. */ #define GETCHARINC(c, eptr) \ c = *eptr++; \ + if ((c & 0xc0) == 0xc0) \ + { \ + int gcaa = utf8_table4[c & 0x3f]; /* Number of additional bytes */ \ + int gcss = 6*gcaa; \ + c = (c & utf8_table3[gcaa]) << gcss; \ + while (gcaa-- > 0) \ + { \ + gcss -= 6; \ + c |= (*eptr++ & 0x3f) << gcss; \ + } \ + } + +/* Get the next character, testing for UTF-8 mode, and advancing the pointer */ + +#define GETCHARINCTEST(c, eptr) \ + c = *eptr++; \ if (md->utf8 && (c & 0xc0) == 0xc0) \ { \ - int a = utf8_table4[c & 0x3f]; /* Number of additional bytes */ \ - int s = 6*a; \ - c = (c & utf8_table3[a]) << s; \ - while (a-- > 0) \ + int gcaa = utf8_table4[c & 0x3f]; /* Number of additional bytes */ \ + int gcss = 6*gcaa; \ + c = (c & utf8_table3[gcaa]) << gcss; \ + while (gcaa-- > 0) \ { \ - s -= 6; \ - c |= (*eptr++ & 0x3f) << s; \ + gcss -= 6; \ + c |= (*eptr++ & 0x3f) << gcss; \ } \ } -/* Get the next UTF-8 character, not advancing the pointer, setting length */ +/* Get the next UTF-8 character, not advancing the pointer, incrementing length +if there are extra bytes. This is called when we know we are in UTF-8 mode. */ #define GETCHARLEN(c, eptr, len) \ c = *eptr; \ - len = 1; \ - if (md->utf8 && (c & 0xc0) == 0xc0) \ + if ((c & 0xc0) == 0xc0) \ { \ - int i; \ - int a = utf8_table4[c & 0x3f]; /* Number of additional bytes */ \ - int s = 6*a; \ - c = (c & utf8_table3[a]) << s; \ - for (i = 1; i <= a; i++) \ + int gcii; \ + int gcaa = utf8_table4[c & 0x3f]; /* Number of additional bytes */ \ + int gcss = 6*gcaa; \ + c = (c & utf8_table3[gcaa]) << gcss; \ + for (gcii = 1; gcii <= gcaa; gcii++) \ { \ - s -= 6; \ - c |= (eptr[i] & 0x3f) << s; \ + gcss -= 6; \ + c |= (eptr[gcii] & 0x3f) << gcss; \ } \ - len += a; \ + len += gcaa; \ } /* If the pointer is not at the start of a character, move it back until -it is. */ +it is. Called only in UTF-8 mode. */ #define BACKCHAR(eptr) while((*eptr & 0xc0) == 0x80) eptr--; @@ -270,19 +365,20 @@ tables. */ /* These are the breakpoints for different numbers of bytes in a UTF-8 character. */ -static int utf8_table1[] = { 0x7f, 0x7ff, 0xffff, 0x1fffff, 0x3ffffff, 0x7fffffff}; +static const int utf8_table1[] = + { 0x7f, 0x7ff, 0xffff, 0x1fffff, 0x3ffffff, 0x7fffffff}; /* These are the indicator bits and the mask for the data bits to set in the first byte of a character, indexed by the number of additional bytes. */ -static int utf8_table2[] = { 0, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc}; -static int utf8_table3[] = { 0xff, 0x1f, 0x0f, 0x07, 0x03, 0x01}; +static const int utf8_table2[] = { 0, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc}; +static const int utf8_table3[] = { 0xff, 0x1f, 0x0f, 0x07, 0x03, 0x01}; /* Table of the number of extra characters, indexed by the first character masked with 0x3f. The highest number for a valid UTF-8 character is in fact 0x3d. */ -static uschar utf8_table4[] = { +static const uschar utf8_table4[] = { 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, @@ -365,7 +461,7 @@ Therefore, I haven't changed the API for pcre_info(). Arguments: external_re points to compiled code optptr where to pass back the options - first_char where to pass back the first character, + first_byte where to pass back the first character, or -1 if multiline and all branches start ^, or -2 otherwise @@ -374,14 +470,14 @@ Returns: number of capturing subpatterns */ int -pcre_info(const pcre *external_re, int *optptr, int *first_char) +pcre_info(const pcre *external_re, int *optptr, int *first_byte) { const real_pcre *re = (const real_pcre *)external_re; if (re == NULL) return PCRE_ERROR_NULL; if (re->magic_number != MAGIC_NUMBER) return PCRE_ERROR_BADMAGIC; if (optptr != NULL) *optptr = (int)(re->options & PUBLIC_OPTIONS); -if (first_char != NULL) - *first_char = ((re->options & PCRE_FIRSTSET) != 0)? re->first_char : +if (first_byte != NULL) + *first_byte = ((re->options & PCRE_FIRSTSET) != 0)? re->first_byte : ((re->options & PCRE_STARTLINE) != 0)? -1 : -2; return re->top_bracket; } @@ -397,7 +493,7 @@ that additional items can be added compatibly. Arguments: external_re points to compiled code - external_study points to study data, or NULL + extra_data points extra data, or NULL what what information is required where where to put the information @@ -405,15 +501,18 @@ Returns: 0 if data returned, negative on error */ int -pcre_fullinfo(const pcre *external_re, const pcre_extra *study_data, int what, +pcre_fullinfo(const pcre *external_re, const pcre_extra *extra_data, int what, void *where) { const real_pcre *re = (const real_pcre *)external_re; -const real_pcre_extra *study = (const real_pcre_extra *)study_data; +const pcre_study_data *study = NULL; if (re == NULL || where == NULL) return PCRE_ERROR_NULL; if (re->magic_number != MAGIC_NUMBER) return PCRE_ERROR_BADMAGIC; +if (extra_data != NULL && (extra_data->flags & PCRE_EXTRA_STUDY_DATA) != 0) + study = extra_data->study_data; + switch (what) { case PCRE_INFO_OPTIONS: @@ -424,6 +523,10 @@ switch (what) *((size_t *)where) = re->size; break; + case PCRE_INFO_STUDYSIZE: + *((size_t *)where) = (study == NULL)? 0 : study->size; + break; + case PCRE_INFO_CAPTURECOUNT: *((int *)where) = re->top_bracket; break; @@ -432,9 +535,9 @@ switch (what) *((int *)where) = re->top_backref; break; - case PCRE_INFO_FIRSTCHAR: + case PCRE_INFO_FIRSTBYTE: *((int *)where) = - ((re->options & PCRE_FIRSTSET) != 0)? re->first_char : + ((re->options & PCRE_FIRSTSET) != 0)? re->first_byte : ((re->options & PCRE_STARTLINE) != 0)? -1 : -2; break; @@ -446,7 +549,7 @@ switch (what) case PCRE_INFO_LASTLITERAL: *((int *)where) = - ((re->options & PCRE_REQCHSET) != 0)? re->req_char : -1; + ((re->options & PCRE_REQCHSET) != 0)? re->req_byte : -1; break; case PCRE_INFO_NAMEENTRYSIZE: @@ -469,6 +572,57 @@ return 0; +/************************************************* +* Return info about what features are configured * +*************************************************/ + +/* This is function which has an extensible interface so that additional items +can be added compatibly. + +Arguments: + what what information is required + where where to put the information + +Returns: 0 if data returned, negative on error +*/ + +int +pcre_config(int what, void *where) +{ +switch (what) + { + case PCRE_CONFIG_UTF8: + #ifdef SUPPORT_UTF8 + *((int *)where) = 1; + #else + *((int *)where) = 0; + #endif + break; + + case PCRE_CONFIG_NEWLINE: + *((int *)where) = NEWLINE; + break; + + case PCRE_CONFIG_LINK_SIZE: + *((int *)where) = LINK_SIZE; + break; + + case PCRE_CONFIG_POSIX_MALLOC_THRESHOLD: + *((int *)where) = POSIX_MALLOC_THRESHOLD; + break; + + case PCRE_CONFIG_MATCH_LIMIT: + *((unsigned int *)where) = MATCH_LIMIT; + break; + + default: return PCRE_ERROR_BADOPTION; + } + +return 0; +} + + + #ifdef DEBUG /************************************************* * Debugging function to print chars * @@ -583,7 +737,7 @@ else { oldptr = ptr; c -= '0'; - while ((cd->ctypes[ptr[1]] & ctype_digit) != 0) + while ((digitab[ptr[1]] & ctype_digit) != 0) c = c * 10 + *(++ptr) - '0'; if (c < 10 || c <= bracount) { @@ -609,8 +763,7 @@ else case '0': c -= '0'; - while(i++ < 2 && (cd->ctypes[ptr[1]] & ctype_digit) != 0 && - ptr[1] != '8' && ptr[1] != '9') + while(i++ < 2 && ptr[1] >= '0' && ptr[1] <= '7') c = c * 8 + *(++ptr) - '0'; c &= 255; /* Take least significant 8 bits */ break; @@ -625,12 +778,12 @@ else const uschar *pt = ptr + 2; register int count = 0; c = 0; - while ((cd->ctypes[*pt] & ctype_xdigit) != 0) + while ((digitab[*pt] & ctype_xdigit) != 0) { + int cc = *pt++; + if (cc >= 'a') cc -= 32; /* Convert to upper case */ count++; - c = c * 16 + cd->lcc[*pt] - - (((cd->ctypes[*pt] & ctype_digit) != 0)? '0' : 'W'); - pt++; + c = c * 16 + cc - ((cc < 'A')? '0' : ('A' - 10)); } if (*pt == '}') { @@ -646,11 +799,11 @@ else /* Read just a single hex char */ c = 0; - while (i++ < 2 && (cd->ctypes[ptr[1]] & ctype_xdigit) != 0) + while (i++ < 2 && (digitab[ptr[1]] & ctype_xdigit) != 0) { - ptr++; - c = c * 16 + cd->lcc[*ptr] - - (((cd->ctypes[*ptr] & ctype_digit) != 0)? '0' : 'W'); + int cc = *(++ptr); + if (cc >= 'a') cc -= 32; /* Convert to upper case */ + c = c * 16 + cc - ((cc < 'A')? '0' : ('A' - 10)); } break; @@ -664,9 +817,10 @@ else return 0; } - /* A letter is upper-cased; then the 0x40 bit is flipped */ + /* A letter is upper-cased; then the 0x40 bit is flipped. This coding + is ASCII-specific, but then the whole concept of \cx is ASCII-specific. */ - if (c >= 'a' && c <= 'z') c = cd->fcc[c]; + if (c >= 'a' && c <= 'z') c -= 32; c ^= 0x40; break; @@ -712,15 +866,16 @@ Returns: TRUE or FALSE static BOOL is_counted_repeat(const uschar *p, compile_data *cd) { -if ((cd->ctypes[*p++] & ctype_digit) == 0) return FALSE; -while ((cd->ctypes[*p] & ctype_digit) != 0) p++; +if ((digitab[*p++] && ctype_digit) == 0) return FALSE; +while ((digitab[*p] & ctype_digit) != 0) p++; if (*p == '}') return TRUE; if (*p++ != ',') return FALSE; if (*p == '}') return TRUE; -if ((cd->ctypes[*p++] & ctype_digit) == 0) return FALSE; -while ((cd->ctypes[*p] & ctype_digit) != 0) p++; +if ((digitab[*p++] && ctype_digit) == 0) return FALSE; +while ((digitab[*p] & ctype_digit) != 0) p++; + return (*p == '}'); } @@ -753,14 +908,14 @@ read_repeat_counts(const uschar *p, int *minp, int *maxp, int min = 0; int max = -1; -while ((cd->ctypes[*p] & ctype_digit) != 0) min = min * 10 + *p++ - '0'; +while ((digitab[*p] & ctype_digit) != 0) min = min * 10 + *p++ - '0'; if (*p == '}') max = min; else { if (*(++p) != '}') { max = 0; - while((cd->ctypes[*p] & ctype_digit) != 0) max = max * 10 + *p++ - '0'; + while((digitab[*p] & ctype_digit) != 0) max = max * 10 + *p++ - '0'; if (max < min) { *errorptr = ERR4; @@ -844,6 +999,7 @@ for (;;) /* Scan a pattern and compute the fixed length of subject that will match it, if the length is fixed. This is needed for dealing with backward assertions. +In UTF8 mode, the result is in characters rather than bytes. Arguments: code points to the start of the pattern (the bracket) @@ -933,15 +1089,27 @@ for (;;) case OP_CHARS: branchlength += *(++cc); #ifdef SUPPORT_UTF8 - for (d = 1; d <= *cc; d++) - if ((cc[d] & 0xc0) == 0x80) branchlength--; + if ((options & PCRE_UTF8) != 0) + for (d = 1; d <= *cc; d++) + if ((cc[d] & 0xc0) == 0x80) branchlength--; #endif cc += *cc + 1; break; - /* Handle exact repetitions */ + /* Handle exact repetitions. The count is already in characters, but we + need to skip over a multibyte character in UTF8 mode. */ case OP_EXACT: + branchlength += GET2(cc,1); + cc += 4; +#ifdef SUPPORT_UTF8 + if ((options & PCRE_UTF8) != 0) + { + while((*cc & 0x80) == 0x80) cc++; + } +#endif + break; + case OP_TYPEEXACT: branchlength += GET2(cc,1); cc += 4; @@ -967,7 +1135,14 @@ for (;;) /* Check a class for variable quantification */ +#ifdef SUPPORT_UTF8 + case OP_XCLASS: + cc += GET(cc, 1) - 33; + /* Fall through */ +#endif + case OP_CLASS: + case OP_NCLASS: cc += 33; switch (*cc) @@ -1011,14 +1186,19 @@ capturing bracket with the given number. Arguments: code points to start of expression + utf8 TRUE in UTF-8 mode number the required bracket number Returns: pointer to the opcode for the bracket, or NULL if not found */ static const uschar * -find_bracket(const uschar *code, int number) +find_bracket(const uschar *code, BOOL utf8, int number) { +#ifndef SUPPORT_UTF8 +utf8 = utf8; /* Stop pedantic compilers complaining */ +#endif + for (;;) { register int c = *code; @@ -1031,7 +1211,32 @@ for (;;) if (n == number) return (uschar *)code; code += OP_lengths[OP_BRA]; } - else code += OP_lengths[c]; + else + { + code += OP_lengths[c]; + + /* In UTF-8 mode, opcodes that are followed by a character may be followed + by a multi-byte character. The length in the table is a minimum, so we have + to scan along to skip the extra characters. All opcodes are less than 128, + so we can use relatively efficient code. */ + +#ifdef SUPPORT_UTF8 + if (utf8) switch(c) + { + case OP_EXACT: + case OP_UPTO: + case OP_MINUPTO: + case OP_STAR: + case OP_MINSTAR: + case OP_PLUS: + case OP_MINPLUS: + case OP_QUERY: + case OP_MINQUERY: + while ((*code & 0xc0) == 0x80) code++; + break; + } +#endif + } } } @@ -1050,18 +1255,21 @@ whose current branch will already have been scanned. Arguments: code points to start of search endcode points to where to stop + utf8 TRUE if in UTF8 mode Returns: TRUE if what is matched could be empty */ static BOOL -could_be_empty_branch(const uschar *code, const uschar *endcode) +could_be_empty_branch(const uschar *code, const uschar *endcode, BOOL utf8) { register int c; for (code = first_significant_code(code + 1 + LINK_SIZE, NULL, 0); code < endcode; code = first_significant_code(code + OP_lengths[c], NULL, 0)) { + const uschar *ccode; + c = *code; if (c >= OP_BRA) @@ -1074,7 +1282,7 @@ for (code = first_significant_code(code + 1 + LINK_SIZE, NULL, 0); empty_branch = FALSE; do { - if (!empty_branch && could_be_empty_branch(code, endcode)) + if (!empty_branch && could_be_empty_branch(code, endcode, utf8)) empty_branch = TRUE; code += GET(code, 1); } @@ -1084,11 +1292,23 @@ for (code = first_significant_code(code + 1 + LINK_SIZE, NULL, 0); c = *code; } - /* Check for any quantifier after a class */ - - else if (c == OP_CLASS) + else switch (c) { - const uschar *ccode = code + 33; + /* Check for quantifiers after a class */ + +#ifdef SUPPORT_UTF8 + case OP_XCLASS: + ccode = code + GET(code, 1); + goto CHECK_CLASS_REPEAT; +#endif + + case OP_CLASS: + case OP_NCLASS: + ccode = code + 33; + +#ifdef SUPPORT_UTF8 + CHECK_CLASS_REPEAT: +#endif switch (*ccode) { @@ -1108,12 +1328,10 @@ for (code = first_significant_code(code + 1 + LINK_SIZE, NULL, 0); if (GET2(ccode, 1) > 0) return FALSE; /* Minimum > 0 */ break; } - } + break; - /* Test for an opcode that must match a character. */ + /* Opcodes that must match a character */ - else switch (c) - { case OP_NOT_DIGIT: case OP_DIGIT: case OP_NOT_WHITESPACE: @@ -1142,6 +1360,20 @@ for (code = first_significant_code(code + 1 + LINK_SIZE, NULL, 0); case OP_KETRMIN: case OP_ALT: return TRUE; + + /* In UTF-8 mode, STAR, MINSTAR, QUERY, MINQUERY, UPTO, and MINUPTO may be + followed by a multibyte character */ + +#ifdef SUPPORT_UTF8 + case OP_STAR: + case OP_MINSTAR: + case OP_QUERY: + case OP_MINQUERY: + case OP_UPTO: + case OP_MINUPTO: + if (utf8) while ((code[2] & 0xc0) == 0x80) code++; + break; +#endif } } @@ -1163,16 +1395,18 @@ Arguments: code points to start of the recursion endcode points to where to stop (current RECURSE item) bcptr points to the chain of current (unclosed) branch starts + utf8 TRUE if in UTF-8 mode Returns: TRUE if what is matched could be empty */ static BOOL -could_be_empty(const uschar *code, const uschar *endcode, branch_chain *bcptr) +could_be_empty(const uschar *code, const uschar *endcode, branch_chain *bcptr, + BOOL utf8) { while (bcptr != NULL && bcptr->current >= code) { - if (!could_be_empty_branch(bcptr->current, endcode)) return FALSE; + if (!could_be_empty_branch(bcptr->current, endcode, utf8)) return FALSE; bcptr = bcptr->outer; } return TRUE; @@ -1259,8 +1493,8 @@ Arguments: code points to the pointer to the current code point ptrptr points to the current pattern pointer errorptr points to pointer to error message - firstcharptr set to initial literal character, or < 0 (REQ_UNSET, REQ_NONE) - reqcharptr set to the last literal character required, else < 0 + firstbyteptr set to initial literal character, or < 0 (REQ_UNSET, REQ_NONE) + reqbyteptr set to the last literal character required, else < 0 bcptr points to current branch chain cd contains pointers to tables etc. @@ -1270,29 +1504,38 @@ Returns: TRUE on success static BOOL compile_branch(int *optionsptr, int *brackets, uschar **codeptr, - const uschar **ptrptr, const char **errorptr, int *firstcharptr, - int *reqcharptr, branch_chain *bcptr, compile_data *cd) + const uschar **ptrptr, const char **errorptr, int *firstbyteptr, + int *reqbyteptr, branch_chain *bcptr, compile_data *cd) { int repeat_type, op_type; int repeat_min = 0, repeat_max = 0; /* To please picky compilers */ int bravalue = 0; int length; int greedy_default, greedy_non_default; -int firstchar, reqchar; -int zeroreqchar, zerofirstchar; -int req_caseopt; +int firstbyte, reqbyte; +int zeroreqbyte, zerofirstbyte; +int req_caseopt, reqvary, tempreqvary; int condcount = 0; int options = *optionsptr; register int c; register uschar *code = *codeptr; uschar *tempcode; BOOL inescq = FALSE; -BOOL groupsetfirstchar = FALSE; +BOOL groupsetfirstbyte = FALSE; const uschar *ptr = *ptrptr; const uschar *tempptr; uschar *previous = NULL; uschar class[32]; +#ifdef SUPPORT_UTF8 +BOOL class_utf8; +BOOL utf8 = (options & PCRE_UTF8) != 0; +uschar *class_utf8data; +uschar utf8_char[6]; +#else +BOOL utf8 = FALSE; +#endif + /* Set up the default and non-default settings for greediness */ greedy_default = ((options & PCRE_UNGREEDY) != 0); @@ -1300,19 +1543,19 @@ greedy_non_default = greedy_default ^ 1; /* Initialize no first char, no required char. REQ_UNSET means "no char matching encountered yet". It gets changed to REQ_NONE if we hit something that -matches a non-fixed char first char; reqchar just remains unset if we never +matches a non-fixed char first char; reqbyte just remains unset if we never find one. When we hit a repeat whose minimum is zero, we may have to adjust these values to take the zero repeat into account. This is implemented by setting them to -zerofirstchar and zeroreqchar when such a repeat is encountered. The individual +zerofirstbyte and zeroreqbyte when such a repeat is encountered. The individual item types that can be repeated set these backoff variables appropriately. */ -firstchar = reqchar = zerofirstchar = zeroreqchar = REQ_UNSET; +firstbyte = reqbyte = zerofirstbyte = zeroreqbyte = REQ_UNSET; /* The variable req_caseopt contains either the REQ_CASELESS value or zero, according to the current setting of the caseless flag. REQ_CASELESS is a bit -value > 255. It is added into the firstchar or reqchar variables to record the +value > 255. It is added into the firstbyte or reqbyte variables to record the case status of the value. */ req_caseopt = ((options & PCRE_CASELESS) != 0)? REQ_CASELESS : 0; @@ -1328,8 +1571,8 @@ for (;; ptr++) int newoptions; int recno; int skipbytes; - int subreqchar; - int subfirstchar; + int subreqbyte; + int subfirstbyte; c = *ptr; if (inescq && c != 0) goto NORMAL_CHAR; @@ -1353,8 +1596,8 @@ for (;; ptr++) case 0: case '|': case ')': - *firstcharptr = firstchar; - *reqcharptr = reqchar; + *firstbyteptr = firstbyte; + *reqbyteptr = reqbyte; *codeptr = code; *ptrptr = ptr; return TRUE; @@ -1365,7 +1608,7 @@ for (;; ptr++) case '^': if ((options & PCRE_MULTILINE) != 0) { - if (firstchar == REQ_UNSET) firstchar = REQ_NONE; + if (firstbyte == REQ_UNSET) firstbyte = REQ_NONE; } previous = NULL; *code++ = OP_CIRC; @@ -1377,24 +1620,30 @@ for (;; ptr++) break; /* There can never be a first char if '.' is first, whatever happens about - repeats. The value of reqchar doesn't change either. */ + repeats. The value of reqbyte doesn't change either. */ case '.': - if (firstchar == REQ_UNSET) firstchar = REQ_NONE; - zerofirstchar = firstchar; - zeroreqchar = reqchar; + if (firstbyte == REQ_UNSET) firstbyte = REQ_NONE; + zerofirstbyte = firstbyte; + zeroreqbyte = reqbyte; previous = code; *code++ = OP_ANY; break; - /* Character classes. These always build a 32-byte bitmap of the permitted - characters, except in the special case where there is only one character. - For negated classes, we build the map as usual, then invert it at the end. + /* Character classes. If the included characters are all < 255 in value, we + build a 32-byte bitmap of the permitted characters, except in the special + case where there is only one such character. For negated classes, we build + the map as usual, then invert it at the end. However, we use a different + opcode so that data characters > 255 can be handled correctly. + + If the class contains characters outside the 0-255 range, a different + opcode is compiled. It may optionally have a bit map for characters < 256, + but those above are are explicitly listed afterwards. A flag byte tells + whether the bitmap is present, and whether this is a negated class or not. */ case '[': previous = code; - *code++ = OP_CLASS; /* PCRE supports POSIX class stuff inside a class. Perl gives an error if they are encountered at the top level, so we'll do that too. */ @@ -1413,27 +1662,58 @@ for (;; ptr++) negate_class = TRUE; c = *(++ptr); } - else negate_class = FALSE; + else + { + negate_class = FALSE; + } - /* Keep a count of chars so that we can optimize the case of just a single - character. */ + /* Keep a count of chars with values < 256 so that we can optimize the case + of just a single character (as long as it's < 256). For higher valued UTF-8 + characters, we don't yet do any optimization. */ class_charcount = 0; class_lastchar = -1; +#ifdef SUPPORT_UTF8 + class_utf8 = FALSE; /* No chars >= 256 */ + class_utf8data = code + LINK_SIZE + 34; /* For UTF-8 items */ +#endif + /* Initialize the 32-char bit map to all zeros. We have to build the map in a temporary bit of store, in case the class contains only 1 - character, because in that case the compiled code doesn't use the + character (< 256), because in that case the compiled code doesn't use the bit map. */ memset(class, 0, 32 * sizeof(uschar)); /* Process characters until ] is reached. By writing this as a "do" it means that an initial ] is taken as a data character. The first pass - checked the overall syntax. */ + through the regex checked the overall syntax, so we don't need to be very + strict here. At the start of the loop, c contains the first byte of the + character. */ do { +#ifdef SUPPORT_UTF8 + if (utf8 && c > 127) + { /* Braces are required because the */ + GETCHARLEN(c, ptr, ptr); /* macro generates multiple statements */ + } +#endif + + /* Inside \Q...\E everything is literal except \E */ + + if (inescq) + { + if (c == '\\' && ptr[1] == 'E') + { + inescq = FALSE; + ptr++; + continue; + } + else goto LONE_SINGLE_CHARACTER; + } + /* Handle POSIX class names. Perl allows a negation extension of the form [:^name:]. A square bracket that doesn't match the syntax is treated as a literal. We also recognize the POSIX constructions @@ -1483,7 +1763,7 @@ for (;; ptr++) posix_class *= 3; for (i = 0; i < 3; i++) { - BOOL isblank = strncmp(ptr, "blank", 5) == 0; + BOOL isblank = strncmp((char *)ptr, "blank", 5) == 0; int taboffset = posix_class_maps[posix_class + i]; if (taboffset < 0) break; if (local_negate) @@ -1500,7 +1780,7 @@ for (;; ptr++) ptr = tempptr + 1; class_charcount = 10; /* Set > 1; assumes more than 1 per class */ - continue; + continue; /* End of POSIX syntax handling */ } /* Backslash may introduce a single character, or it may introduce one @@ -1509,12 +1789,23 @@ for (;; ptr++) Inside a class (and only there) it is treated as backspace. Elsewhere it marks a word boundary. Other escapes have preset maps ready to or into the one we are building. We assume they have more than one - character in them, so set class_count bigger than one. */ + character in them, so set class_charcount bigger than one. */ if (c == '\\') { c = check_escape(&ptr, errorptr, *brackets, options, TRUE, cd); - if (-c == ESC_b) c = '\b'; + if (-c == ESC_b) c = '\b'; /* \b is backslash in a class */ + + if (-c == ESC_Q) /* Handle start of quoted string */ + { + if (ptr[1] == '\\' && ptr[2] == 'E') + { + ptr += 2; /* avoid empty string */ + } + else inescq = TRUE; + continue; + } + else if (c < 0) { register const uschar *cbits = cd->cbits; @@ -1547,23 +1838,24 @@ for (;; ptr++) class[1] |= 0x08; /* Perl 5.004 onwards omits VT from \s */ continue; + /* Unrecognized escapes are faulted if PCRE is running in its + strict mode. By default, for compatibility with Perl, they are + treated as literals. */ + default: - *errorptr = ERR7; - goto FAILED; + if ((options & PCRE_EXTRA) != 0) + { + *errorptr = ERR7; + goto FAILED; + } + c = *ptr; /* The final character */ } } - /* Fall through if single character, but don't at present allow - chars > 255 in UTF-8 mode. */ + /* Fall through if we have a single character (c >= 0). This may be + > 256 in UTF-8 mode. */ -#ifdef SUPPORT_UTF8 - if (c > 255) - { - *errorptr = ERR33; - goto FAILED; - } -#endif - } + } /* End of backslash handling */ /* A single character may be followed by '-' to form a range. However, Perl does not permit ']' to be the end of the range. A '-' character @@ -1573,6 +1865,14 @@ for (;; ptr++) { int d; ptr += 2; + +#ifdef SUPPORT_UTF8 + if (utf8) + { /* Braces are required because the */ + GETCHARLEN(d, ptr, ptr); /* macro generates multiple statements */ + } + else +#endif d = *ptr; /* The second part of a range can be a single-character escape, but @@ -1584,13 +1884,6 @@ for (;; ptr++) const uschar *oldptr = ptr; d = check_escape(&ptr, errorptr, *brackets, options, TRUE, cd); -#ifdef SUPPORT_UTF8 - if (d > 255) - { - *errorptr = ERR33; - goto FAILED; - } -#endif /* \b is backslash; any other special means the '-' was literal */ if (d < 0) @@ -1598,17 +1891,45 @@ for (;; ptr++) if (d == -ESC_b) d = '\b'; else { ptr = oldptr - 2; - goto SINGLE_CHARACTER; /* A few lines below */ + goto LONE_SINGLE_CHARACTER; /* A few lines below */ } } } + /* Check that the two values are in the correct order */ + if (d < c) { *errorptr = ERR8; goto FAILED; } + /* If d is greater than 255, we can't just use the bit map, so set up + for the UTF-8 supporting class type. If we are not caseless, we can + just set up a single range. If we are caseless, the characters < 256 + are handled with a bitmap, in order to get the case-insensitive + handling. */ + +#ifdef SUPPORT_UTF8 + if (d > 255) + { + class_utf8 = TRUE; + *class_utf8data++ = XCL_RANGE; + if ((options & PCRE_CASELESS) == 0) + { + class_utf8data += ord2utf8(c, class_utf8data); + class_utf8data += ord2utf8(d, class_utf8data); + continue; /* Go get the next char in the class */ + } + class_utf8data += ord2utf8(256, class_utf8data); + class_utf8data += ord2utf8(d, class_utf8data); + d = 255; + /* Fall through */ + } +#endif + /* We use the bit map if the range is entirely < 255, or if part of it + is < 255 and matching is caseless. */ + for (; c <= d; c++) { class[c/8] |= (1 << (c&7)); @@ -1620,80 +1941,151 @@ for (;; ptr++) class_charcount++; /* in case a one-char range */ class_lastchar = c; } + continue; /* Go get the next char in the class */ } /* Handle a lone single character - we can get here for a normal non-escape char, or after \ that introduces a single character. */ - SINGLE_CHARACTER: + LONE_SINGLE_CHARACTER: + + /* Handle a multibyte character */ - class [c/8] |= (1 << (c&7)); - if ((options & PCRE_CASELESS) != 0) +#ifdef SUPPORT_UTF8 + if (utf8 && c > 255) + { + class_utf8 = TRUE; + *class_utf8data++ = XCL_SINGLE; + class_utf8data += ord2utf8(c, class_utf8data); + } + else +#endif + /* Handle a single-byte character */ { - c = cd->fcc[c]; /* flip case */ - class[c/8] |= (1 << (c&7)); + class [c/8] |= (1 << (c&7)); + if ((options & PCRE_CASELESS) != 0) + { + c = cd->fcc[c]; /* flip case */ + class[c/8] |= (1 << (c&7)); + } + class_charcount++; + class_lastchar = c; } - class_charcount++; - class_lastchar = c; } /* Loop until ']' reached; the check for end of string happens inside the loop. This "while" is the end of the "do" above. */ - while ((c = *(++ptr)) != ']'); + while ((c = *(++ptr)) != ']' || inescq); + + /* If class_charcount is 1, we saw precisely one character with a value < + 256. In UTF-8 mode, we can optimize if there were no characters >= 256 and + the one character is < 128. In non-UTF-8 mode we can always optimize. - /* If class_charcount is 1 and class_lastchar is not negative, we saw - precisely one character. This doesn't need the whole 32-byte bit map. We - turn it into a 1-character OP_CHARS if it's positive, or OP_NOT if it's - negative. In the positive case, it can cause firstchar to be set. - Otherwise, there can be no first char if this item is first, whatever - repeat count may follow. In the case of reqchar, save the previous value - for reinstating. */ + The optimization throws away the bit map. We turn the item into a + 1-character OP_CHARS if it's positive, or OP_NOT if it's negative. Note + that OP_NOT does not support multibyte characters. In the positive case, it + can cause firstbyte to be set. Otherwise, there can be no first char if + this item is first, whatever repeat count may follow. In the case of + reqbyte, save the previous value for reinstating. */ - if (class_charcount == 1 && class_lastchar >= 0) +#ifdef SUPPORT_UTF8 + if (class_charcount == 1 && + (!utf8 || + (!class_utf8 && class_lastchar < 128))) +#else + if (class_charcount == 1) +#endif { - zeroreqchar = reqchar; + zeroreqbyte = reqbyte; if (negate_class) { - if (firstchar == REQ_UNSET) firstchar = REQ_NONE; - zerofirstchar = firstchar; - code[-1] = OP_NOT; + if (firstbyte == REQ_UNSET) firstbyte = REQ_NONE; + zerofirstbyte = firstbyte; + *code++ = OP_NOT; } else { - if (firstchar == REQ_UNSET) + if (firstbyte == REQ_UNSET) { - zerofirstchar = REQ_NONE; - firstchar = class_lastchar | req_caseopt; + zerofirstbyte = REQ_NONE; + firstbyte = class_lastchar | req_caseopt; } else { - zerofirstchar = firstchar; - reqchar = class_lastchar | req_caseopt; + zerofirstbyte = firstbyte; + reqbyte = class_lastchar | req_caseopt | cd->req_varyopt; } - code[-1] = OP_CHARS; + *code++ = OP_CHARS; *code++ = 1; } *code++ = class_lastchar; + break; /* End of class handling */ + } /* End of 1-byte optimization */ + + /* Otherwise, if this is the first thing in the branch, there can be no + first char setting, whatever the repeat count. Any reqbyte setting must + remain unchanged after any kind of repeat. */ + + if (firstbyte == REQ_UNSET) firstbyte = REQ_NONE; + zerofirstbyte = firstbyte; + zeroreqbyte = reqbyte; + + /* If there are characters with values > 255, we have to compile an + extended class, with its own opcode. If there are no characters < 256, + we can omit the bitmap. */ + +#ifdef SUPPORT_UTF8 + if (class_utf8) + { + *class_utf8data++ = XCL_END; /* Marks the end of extra data */ + *code++ = OP_XCLASS; + code += LINK_SIZE; + *code = negate_class? XCL_NOT : 0; + + /* If the map is required, install it, and move on to the end of + the extra data */ + + if (class_charcount > 0) + { + *code++ |= XCL_MAP; + memcpy(code, class, 32); + code = class_utf8data; + } + + /* If the map is not required, slide down the extra data. */ + + else + { + int len = class_utf8data - (code + 33); + memmove(code + 1, code + 33, len); + code += len + 1; + } + + /* Now fill in the complete length of the item */ + + PUT(previous, 1, code - previous); + break; /* End of class handling */ } +#endif - /* Otherwise, negate the 32-byte map if necessary, and copy it into - the code vector. If this is the first thing in the branch, there can be - no first char setting, whatever the repeat count. Any reqchar setting - must remain unchanged after any kind of repeat. */ + /* If there are no characters > 255, negate the 32-byte map if necessary, + and copy it into the code vector. If this is the first thing in the branch, + there can be no first char setting, whatever the repeat count. Any reqbyte + setting must remain unchanged after any kind of repeat. */ + if (negate_class) + { + *code++ = OP_NCLASS; + for (c = 0; c < 32; c++) code[c] = ~class[c]; + } else { - if (firstchar == REQ_UNSET) firstchar = REQ_NONE; - zerofirstchar = firstchar; - zeroreqchar = reqchar; - if (negate_class) - for (c = 0; c < 32; c++) code[c] = ~class[c]; - else - memcpy(code, class, 32); - code += 32; + *code++ = OP_CLASS; + memcpy(code, class, 32); } + code += 32; break; /* Various kinds of repeat */ @@ -1727,10 +2119,14 @@ for (;; ptr++) if (repeat_min == 0) { - firstchar = zerofirstchar; /* Adjust for zero repeat */ - reqchar = zeroreqchar; /* Ditto */ + firstbyte = zerofirstbyte; /* Adjust for zero repeat */ + reqbyte = zeroreqbyte; /* Ditto */ } + /* Remember whether this is a variable length repeat */ + + reqvary = (repeat_min == repeat_max)? 0 : REQ_VARY; + op_type = 0; /* Default single-char op codes */ possessive_quantifier = FALSE; /* Default not possessive quantifier */ @@ -1775,25 +2171,54 @@ for (;; ptr++) /* If previous was a string of characters, chop off the last one and use it as the subject of the repeat. If there was only one character, we can abolish the previous item altogether. If a one-char item has a minumum of - more than one, ensure that it is set in reqchar - it might not be if a + more than one, ensure that it is set in reqbyte - it might not be if a sequence such as x{3} is the first thing in a branch because the x will - have gone into firstchar instead. */ + have gone into firstbyte instead. */ if (*previous == OP_CHARS) { - int len = previous[1]; - if (len == 1) + /* Deal with UTF-8 characters that take up more than one byte. It's + easier to write this out separately than try to macrify it. Use c to + hold the length of the character in bytes, plus 0x80 to flag that it's a + length rather than a small character. */ + +#ifdef SUPPORT_UTF8 + if (utf8 && (code[-1] & 0x80) != 0) { - c = previous[2]; - code = previous; - if (repeat_min > 1) reqchar = c | req_caseopt; + uschar *lastchar = code - 1; + while((*lastchar & 0xc0) == 0x80) lastchar--; + c = code - lastchar; /* Length of UTF-8 character */ + memcpy(utf8_char, lastchar, c); /* Save the char */ + if (lastchar == previous + 2) /* There was only one character */ + { + code = previous; /* Abolish the previous item */ + } + else + { + previous[1] -= c; /* Adjust length of previous */ + code = lastchar; /* Lost char off the end */ + tempcode = code; /* Adjust position to be moved for '+' */ + } + c |= 0x80; /* Flag c as a length */ } else +#endif + + /* Handle the case of a single byte - either with no UTF8 support, or + with UTF-8 disabled, or for a UTF-8 character < 128. */ + { - c = previous[len+1]; - previous[1]--; - code--; - tempcode = code; /* Adjust position to be moved for '+' */ + c = *(--code); + if (code == previous + 2) /* There was only one character */ + { + code = previous; /* Abolish the previous item */ + if (repeat_min > 1) reqbyte = c | req_caseopt | cd->req_varyopt; + } + else + { + previous[1]--; /* adjust length */ + tempcode = code; /* Adjust position to be moved for '+' */ + } } goto OUTPUT_SINGLE_REPEAT; /* Code shared with single character types */ @@ -1802,7 +2227,7 @@ for (;; ptr++) /* If previous was a single negated character ([^a] or similar), we use one of the special opcodes, replacing it. The code is shared with single- character repeats by setting opt_type to add a suitable offset into - repeat_type. */ + repeat_type. OP_NOT is currently used only for single-byte chars. */ else if (*previous == OP_NOT) { @@ -1868,30 +2293,58 @@ for (;; ptr++) length was 1, or add the character back onto the end of a longer string. For a character type nothing need be done; it will just get put back naturally. Note that the final character is always going to - get added below. */ + get added below, so we leave code ready for its insertion. */ else if (*previous == OP_CHARS) { - if (code == previous) code += 2; else previous[1]++; + if (code == previous) code += 2; else + + /* In UTF-8 mode, a multibyte char has its length in c, with the 0x80 + bit set as a flag. The length will always be between 2 and 6. */ + +#ifdef SUPPORT_UTF8 + if (utf8 && c >= 128) previous[1] += c & 7; else +#endif + previous[1]++; } /* For a single negated character we also have to put back the - item that got cancelled. */ + item that got cancelled. At present this applies only to single byte + characters in any mode. */ else if (*previous == OP_NOT) code++; - /* If the maximum is unlimited, insert an OP_STAR. */ + /* If the maximum is unlimited, insert an OP_STAR. Before doing so, + we have to insert the character for the previous code. In UTF-8 mode, + long characters have their length in c, with the 0x80 bit as a flag. */ if (repeat_max < 0) { +#ifdef SUPPORT_UTF8 + if (utf8 && c >= 128) + { + memcpy(code, utf8_char, c & 7); + code += c & 7; + } + else +#endif *code++ = c; *code++ = OP_STAR + repeat_type; } - /* Else insert an UPTO if the max is greater than the min. */ + /* Else insert an UPTO if the max is greater than the min, again + preceded by the character, for the previously inserted code. */ else if (repeat_max != repeat_min) { +#ifdef SUPPORT_UTF8 + if (utf8 && c >= 128) + { + memcpy(code, utf8_char, c & 7); + code += c & 7; + } + else +#endif *code++ = c; repeat_max -= repeat_min; *code++ = OP_UPTO + repeat_type; @@ -1901,13 +2354,27 @@ for (;; ptr++) /* The character or character type itself comes last in all cases. */ +#ifdef SUPPORT_UTF8 + if (utf8 && c >= 128) + { + memcpy(code, utf8_char, c & 7); + code += c & 7; + } + else +#endif + *code++ = c; } /* If previous was a character class or a back reference, we put the repeat stuff after it, but just skip the item if the repeat was {0,0}. */ - else if (*previous == OP_CLASS || *previous == OP_REF) + else if (*previous == OP_CLASS || + *previous == OP_NCLASS || +#ifdef SUPPORT_UTF8 + *previous == OP_XCLASS || +#endif + *previous == OP_REF) { if (repeat_max == 0) { @@ -2016,7 +2483,7 @@ for (;; ptr++) { if (repeat_min > 1) { - if (groupsetfirstchar && reqchar < 0) reqchar = firstchar; + if (groupsetfirstbyte && reqbyte < 0) reqbyte = firstbyte; for (i = 1; i < repeat_min; i++) { memcpy(code, previous, len); @@ -2104,10 +2571,13 @@ for (;; ptr++) PUT(tempcode, 1, len); } - /* In all case we no longer have a previous item. */ + /* In all case we no longer have a previous item. We also set the + "follows varying string" flag for subsequently encountered reqbytes if + it isn't already set and we have just passed a varying length item. */ END_REPEAT: previous = NULL; + cd->req_varyopt |= reqvary; break; @@ -2148,15 +2618,18 @@ for (;; ptr++) { code[1+LINK_SIZE] = OP_CREF; PUT2(code, 2+LINK_SIZE, CREF_RECURSE); - skipbytes += 1+LINK_SIZE; + skipbytes = 3; ptr += 3; } - /* Condition to test for a numbered subpattern match */ + /* Condition to test for a numbered subpattern match. We know that + if a digit follows ( then there will just be digits until ) because + the syntax was checked in the first pass. */ - else if ((cd->ctypes[ptr[1]] & ctype_digit) != 0) + else if ((digitab[ptr[1]] && ctype_digit) != 0) { - int condref = *(++ptr) - '0'; + int condref; /* Don't amalgamate; some compilers */ + condref = *(++ptr) - '0'; /* grumble at autoincrement in declaration */ while (*(++ptr) != ')') condref = condref*10 + *ptr - '0'; if (condref == 0) { @@ -2206,7 +2679,7 @@ for (;; ptr++) *code++ = OP_CALLOUT; { int n = 0; - while ((cd->ctypes[*(++ptr)] & ctype_digit) != 0) + while ((digitab[*(++ptr)] & ctype_digit) != 0) n = n * 10 + *ptr - '0'; if (n > 255) { @@ -2222,21 +2695,26 @@ for (;; ptr++) if (*(++ptr) == '<') /* Definition */ { int i, namelen; - const uschar *name = ++ptr; uschar *slot = cd->name_table; + const uschar *name; /* Don't amalgamate; some compilers */ + name = ++ptr; /* grumble at autoincrement in declaration */ while (*ptr++ != '>'); namelen = ptr - name - 1; for (i = 0; i < cd->names_found; i++) { - int c = strncmp(name, slot+2, namelen); - if (c == 0) + int crc = memcmp(name, slot+2, namelen); + if (crc == 0) { - *errorptr = ERR43; - goto FAILED; + if (slot[2+namelen] == 0) + { + *errorptr = ERR43; + goto FAILED; + } + crc = -1; /* Current name is substring */ } - if (c < 0) + if (crc < 0) { memmove(slot + cd->name_entry_size, slot, (cd->names_found - i) * cd->name_entry_size); @@ -2264,7 +2742,7 @@ for (;; ptr++) for (i = 0; i < cd->names_found; i++) { - if (strncmp(name, slot+2, namelen) == 0) break; + if (strncmp((char *)name, (char *)slot+2, namelen) == 0) break; slot += cd->name_entry_size; } if (i >= cd->names_found) @@ -2282,6 +2760,8 @@ for (;; ptr++) previous = code; *code++ = OP_REF; PUT2INC(code, 0, recno); + cd->backref_map |= (recno < 32)? (1 << recno) : 1; + if (recno > cd->top_backref) cd->top_backref = recno; continue; } @@ -2299,8 +2779,7 @@ for (;; ptr++) { const uschar *called; recno = 0; - - while ((cd->ctypes[*ptr] & ctype_digit) != 0) + while((digitab[*ptr] & ctype_digit) != 0) recno = recno * 10 + *ptr++ - '0'; /* Come here from code above that handles a named recursion */ @@ -2314,7 +2793,8 @@ for (;; ptr++) *code = OP_END; called = (recno == 0)? - cd->start_code : find_bracket(cd->start_code, recno); + cd->start_code : find_bracket(cd->start_code, utf8, recno); + if (called == NULL) { *errorptr = ERR15; @@ -2325,7 +2805,7 @@ for (;; ptr++) check to see if this is a left recursion that could loop for ever, and diagnose that case. */ - if (GET(called, 1) == 0 && could_be_empty(called, code, bcptr)) + if (GET(called, 1) == 0 && could_be_empty(called, code, bcptr, utf8)) { *errorptr = ERR40; goto FAILED; @@ -2385,7 +2865,7 @@ for (;; ptr++) /* Change options at this level, and pass them back for use in subsequent branches. Reset the greedy defaults and the case - value for firstchar and reqchar. */ + value for firstbyte and reqbyte. */ *optionsptr = options = newoptions; greedy_default = ((newoptions & PCRE_UNGREEDY) != 0); @@ -2406,6 +2886,14 @@ for (;; ptr++) } } + /* If PCRE_NO_AUTO_CAPTURE is set, all unadorned brackets become + non-capturing and behave like (?:...) brackets */ + + else if ((options & PCRE_NO_AUTO_CAPTURE) != 0) + { + bravalue = OP_BRA; + } + /* Else we have a referencing group; adjust the opcode. If the bracket number is greater than EXTRACT_BASIC_MAX, we set the opcode one higher, and arrange for the true number to follow later, in an OP_BRANUMBER item. */ @@ -2431,6 +2919,7 @@ for (;; ptr++) previous = (bravalue >= OP_ONCE)? code : NULL; *code = bravalue; tempcode = code; + tempreqvary = cd->req_varyopt; /* Save value before bracket */ if (!compile_regex( newoptions, /* The complete new option state */ @@ -2442,8 +2931,8 @@ for (;; ptr++) (bravalue == OP_ASSERTBACK || bravalue == OP_ASSERTBACK_NOT), /* TRUE if back assert */ skipbytes, /* Skip over OP_COND/OP_BRANUMBER */ - &subfirstchar, /* For possible first char */ - &subreqchar, /* For possible last char */ + &subfirstbyte, /* For possible first char */ + &subreqbyte, /* For possible last char */ bcptr, /* Current branch chain */ cd)) /* Tables block */ goto FAILED; @@ -2473,61 +2962,63 @@ for (;; ptr++) goto FAILED; } - /* If there is just one branch, we must not make use of its firstchar or - reqchar, because this is equivalent to an empty second branch. */ + /* If there is just one branch, we must not make use of its firstbyte or + reqbyte, because this is equivalent to an empty second branch. */ - if (condcount == 1) subfirstchar = subreqchar = REQ_NONE; + if (condcount == 1) subfirstbyte = subreqbyte = REQ_NONE; } /* Handle updating of the required and first characters. Update for normal brackets of all kinds, and conditions with two branches (see code above). If the bracket is followed by a quantifier with zero repeat, we have to - back off. Hence the definition of zeroreqchar and zerofirstchar outside the + back off. Hence the definition of zeroreqbyte and zerofirstbyte outside the main loop so that they can be accessed for the back off. */ - zeroreqchar = reqchar; - zerofirstchar = firstchar; - groupsetfirstchar = FALSE; + zeroreqbyte = reqbyte; + zerofirstbyte = firstbyte; + groupsetfirstbyte = FALSE; if (bravalue >= OP_BRA || bravalue == OP_ONCE || bravalue == OP_COND) { - /* If we have not yet set a firstchar in this branch, take it from the + /* If we have not yet set a firstbyte in this branch, take it from the subpattern, remembering that it was set here so that a repeat of more - than one can replicate it as reqchar if necessary. If the subpattern has - no firstchar, set "none" for the whole branch. In both cases, a zero - repeat forces firstchar to "none". */ + than one can replicate it as reqbyte if necessary. If the subpattern has + no firstbyte, set "none" for the whole branch. In both cases, a zero + repeat forces firstbyte to "none". */ - if (firstchar == REQ_UNSET) + if (firstbyte == REQ_UNSET) { - if (subfirstchar >= 0) + if (subfirstbyte >= 0) { - firstchar = subfirstchar; - groupsetfirstchar = TRUE; + firstbyte = subfirstbyte; + groupsetfirstbyte = TRUE; } - else firstchar = REQ_NONE; - zerofirstchar = REQ_NONE; + else firstbyte = REQ_NONE; + zerofirstbyte = REQ_NONE; } - /* If firstchar was previously set, convert the subpattern's firstchar - into reqchar if there wasn't one. */ + /* If firstbyte was previously set, convert the subpattern's firstbyte + into reqbyte if there wasn't one, using the vary flag that was in + existence beforehand. */ - else if (subfirstchar >= 0 && subreqchar < 0) subreqchar = subfirstchar; + else if (subfirstbyte >= 0 && subreqbyte < 0) + subreqbyte = subfirstbyte | tempreqvary; - /* If the subpattern set a required char (or set a first char that isn't - really the first char - see above), set it. */ + /* If the subpattern set a required byte (or set a first byte that isn't + really the first byte - see above), set it. */ - if (subreqchar >= 0) reqchar = subreqchar; + if (subreqbyte >= 0) reqbyte = subreqbyte; } - /* For a forward assertion, we take the reqchar, if set. This can be + /* For a forward assertion, we take the reqbyte, if set. This can be helpful if the pattern that follows the assertion doesn't set a different - char. For example, it's useful for /(?=abcde).+/. We can't set firstchar + char. For example, it's useful for /(?=abcde).+/. We can't set firstbyte for an assertion, however because it leads to incorrect effect for patterns - such as /(?=a)a.+/ when the "real" "a" would then become a reqchar instead - of a firstchar. This is overcome by a scan at the end if there's no - firstchar, looking for an asserted first char. */ + such as /(?=a)a.+/ when the "real" "a" would then become a reqbyte instead + of a firstbyte. This is overcome by a scan at the end if there's no + firstbyte, looking for an asserted first char. */ - else if (bravalue == OP_ASSERT && subreqchar >= 0) reqchar = subreqchar; + else if (bravalue == OP_ASSERT && subreqbyte >= 0) reqbyte = subreqbyte; /* Now update the main code pointer to the end of the group. */ @@ -2569,13 +3060,13 @@ for (;; ptr++) /* For metasequences that actually match a character, we disable the setting of a first character if it hasn't already been set. */ - if (firstchar == REQ_UNSET && -c > ESC_b && -c < ESC_Z) - firstchar = REQ_NONE; + if (firstbyte == REQ_UNSET && -c > ESC_b && -c < ESC_Z) + firstbyte = REQ_NONE; /* Set values to reset to if this is followed by a zero repeat. */ - zerofirstchar = firstchar; - zeroreqchar = reqchar; + zerofirstbyte = firstbyte; + zeroreqbyte = reqbyte; /* Back references are handled specially */ @@ -2658,7 +3149,7 @@ for (;; ptr++) two or more characters in the UTF-8 encoding. */ #ifdef SUPPORT_UTF8 - if (c > 127 && (options & PCRE_UTF8) != 0) + if (utf8 && c > 127) { uschar buffer[8]; int len = ord2utf8(c, buffer); @@ -2679,28 +3170,115 @@ for (;; ptr++) while (length < MAXLIT && (cd->ctypes[c = *(++ptr)] & ctype_meta) == 0); - /* Update the first and last character */ + /* Update the first and last requirements. These are always bytes, even in + UTF-8 mode. However, there is a special case to be considered when there + are only one or two characters. Because this gets messy in UTF-8 mode, the + code is kept separate. When we get here "length" contains the number of + bytes. */ - if (firstchar == REQ_UNSET) +#ifdef SUPPORT_UTF8 + if (utf8 && length > 1) { - if (length > 1) + uschar *t = previous + 3; /* After this code, t */ + while (t < code && (*t & 0xc0) == 0x80) t++; /* follows the 1st char */ + + /* Handle the case when there is only one multibyte character. It must + have at least two bytes because of the "length > 1" test above. */ + + if (t == code) { - zerofirstchar = firstchar = previous[2] | req_caseopt; - zeroreqchar = (length > 2)? (code[-2] | req_caseopt) : reqchar; - reqchar = code[-1] | req_caseopt; + /* If no previous first byte, set it from this character, but revert to + none on a zero repeat. */ + + if (firstbyte == REQ_UNSET) + { + zerofirstbyte = REQ_NONE; + firstbyte = previous[2]; + } + + /* Otherwise, leave the first byte value alone, and don't change it on + a zero repeat */ + + else zerofirstbyte = firstbyte; + + /* In both cases, a zero repeat resets the previous required byte */ + + zeroreqbyte = reqbyte; } + + /* Handle the case when there is more than one character. These may be + single-byte or multibyte characters */ + else { - zerofirstchar = REQ_NONE; - firstchar = code[-1] | req_caseopt; - zeroreqchar = reqchar; + t = code - 1; /* After this code, t is at the */ + while ((*t & 0xc0) == 0x80) t--; /* start of the last character */ + + /* If no previous first byte, set it from the first character, and + retain it on a zero repeat (of the last character). The required byte + is reset on a zero repeat, either to the byte before the last + character, unless this is the first byte of the string. In that case, + it reverts to its previous value. */ + + if (firstbyte == REQ_UNSET) + { + zerofirstbyte = firstbyte = previous[2] | req_caseopt; + zeroreqbyte = (t - 1 == previous + 2)? + reqbyte : t[-1] | req_caseopt | cd->req_varyopt; + } + + /* If there was a previous first byte, leave it alone, and don't change + it on a zero repeat. The required byte is reset on a zero repeat to the + byte before the last character. */ + + else + { + zerofirstbyte = firstbyte; + zeroreqbyte = t[-1] | req_caseopt | cd->req_varyopt; + } } + + /* In all cases (we know length > 1), the new required byte is the last + byte of the string. */ + + reqbyte = code[-1] | req_caseopt | cd->req_varyopt; } - else /* firstchar previously set */ + + else /* End of UTF-8 coding */ +#endif + + /* This is the code for non-UTF-8 operation, either without UTF-8 support, + or when UTF-8 is not enabled. */ + { - zerofirstchar = firstchar; - zeroreqchar = (length > 1)? (code[-2] | req_caseopt) : reqchar; - reqchar = code[-1] | req_caseopt; + /* firstbyte was not previously set; take it from this string */ + + if (firstbyte == REQ_UNSET) + { + if (length == 1) + { + zerofirstbyte = REQ_NONE; + firstbyte = previous[2] | req_caseopt; + zeroreqbyte = reqbyte; + } + else + { + zerofirstbyte = firstbyte = previous[2] | req_caseopt; + zeroreqbyte = (length > 2)? + (code[-2] | req_caseopt | cd->req_varyopt) : reqbyte; + reqbyte = code[-1] | req_caseopt | cd->req_varyopt; + } + } + + /* firstbyte was previously set */ + + else + { + zerofirstbyte = firstbyte; + zeroreqbyte = (length == 1)? reqbyte : + code[-2] | req_caseopt | cd->req_varyopt; + reqbyte = code[-1] | req_caseopt | cd->req_varyopt; + } } /* Set the length in the data vector, and advance to the next state. */ @@ -2744,8 +3322,8 @@ Argument: errorptr -> pointer to error message lookbehind TRUE if this is a lookbehind assertion skipbytes skip this many bytes at start (for OP_COND, OP_BRANUMBER) - firstcharptr place to put the first required character, or a negative number - reqcharptr place to put the last required character, or a negative number + firstbyteptr place to put the first required character, or a negative number + reqbyteptr place to put the last required character, or a negative number bcptr pointer to the chain of currently open branches cd points to the data block with tables pointers etc. @@ -2755,21 +3333,21 @@ Returns: TRUE on success static BOOL compile_regex(int options, int oldims, int *brackets, uschar **codeptr, const uschar **ptrptr, const char **errorptr, BOOL lookbehind, int skipbytes, - int *firstcharptr, int *reqcharptr, branch_chain *bcptr, compile_data *cd) + int *firstbyteptr, int *reqbyteptr, branch_chain *bcptr, compile_data *cd) { const uschar *ptr = *ptrptr; uschar *code = *codeptr; uschar *last_branch = code; uschar *start_bracket = code; uschar *reverse_count = NULL; -int firstchar, reqchar; -int branchfirstchar, branchreqchar; +int firstbyte, reqbyte; +int branchfirstbyte, branchreqbyte; branch_chain bc; bc.outer = bcptr; bc.current = code; -firstchar = reqchar = REQ_UNSET; +firstbyte = reqbyte = REQ_UNSET; /* Offset is set zero to mark that this bracket is still open */ @@ -2800,45 +3378,49 @@ for (;;) /* Now compile the branch */ if (!compile_branch(&options, brackets, &code, &ptr, errorptr, - &branchfirstchar, &branchreqchar, &bc, cd)) + &branchfirstbyte, &branchreqbyte, &bc, cd)) { *ptrptr = ptr; return FALSE; } - /* If this is the first branch, the firstchar and reqchar values for the + /* If this is the first branch, the firstbyte and reqbyte values for the branch become the values for the regex. */ if (*last_branch != OP_ALT) { - firstchar = branchfirstchar; - reqchar = branchreqchar; + firstbyte = branchfirstbyte; + reqbyte = branchreqbyte; } - /* If this is not the first branch, the first char and reqchar have to - match the values from all the previous branches. */ + /* If this is not the first branch, the first char and reqbyte have to + match the values from all the previous branches, except that if the previous + value for reqbyte didn't have REQ_VARY set, it can still match, and we set + REQ_VARY for the regex. */ else { - /* If we previously had a firstchar, but it doesn't match the new branch, - we have to abandon the firstchar for the regex, but if there was previously - no reqchar, it takes on the value of the old firstchar. */ + /* If we previously had a firstbyte, but it doesn't match the new branch, + we have to abandon the firstbyte for the regex, but if there was previously + no reqbyte, it takes on the value of the old firstbyte. */ - if (firstchar >= 0 && firstchar != branchfirstchar) + if (firstbyte >= 0 && firstbyte != branchfirstbyte) { - if (reqchar < 0) reqchar = firstchar; - firstchar = REQ_NONE; + if (reqbyte < 0) reqbyte = firstbyte; + firstbyte = REQ_NONE; } - /* If we (now or from before) have no firstchar, a firstchar from the - branch becomes a reqchar if there isn't a branch reqchar. */ + /* If we (now or from before) have no firstbyte, a firstbyte from the + branch becomes a reqbyte if there isn't a branch reqbyte. */ - if (firstchar < 0 && branchfirstchar >= 0 && branchreqchar < 0) - branchreqchar = branchfirstchar; + if (firstbyte < 0 && branchfirstbyte >= 0 && branchreqbyte < 0) + branchreqbyte = branchfirstbyte; - /* Now ensure that the reqchars match */ + /* Now ensure that the reqbytes match */ - if (reqchar != branchreqchar) reqchar = REQ_NONE; + if ((reqbyte & ~REQ_VARY) != (branchreqbyte & ~REQ_VARY)) + reqbyte = REQ_NONE; + else reqbyte |= branchreqbyte; /* To "or" REQ_VARY */ } /* If lookbehind, check that this branch matches a fixed-length string, @@ -2899,8 +3481,8 @@ for (;;) *codeptr = code; *ptrptr = ptr; - *firstcharptr = firstchar; - *reqcharptr = reqchar; + *firstbyteptr = firstbyte; + *reqbyteptr = reqbyte; return TRUE; } @@ -2931,28 +3513,37 @@ all of whose alternatives start with OP_SOD or OP_CIRC (recurse ad lib), then it's anchored. However, if this is a multiline pattern, then only OP_SOD counts, since OP_CIRC can match in the middle. +We can also consider a regex to be anchored if OP_SOM starts all its branches. +This is the code for \G, which means "match at start of match position, taking +into account the match offset". + A branch is also implicitly anchored if it starts with .* and DOTALL is set, because that will try the rest of the pattern at all possible matching points, so there is no point trying again.... er .... .... except when the .* appears inside capturing parentheses, and there is a subsequent back reference to those parentheses. We haven't enough information -to catch that case precisely. The best we can do is to detect when .* is in -capturing brackets and the highest back reference is greater than or equal to -that level. +to catch that case precisely. + +At first, the best we could do was to detect when .* was in capturing brackets +and the highest back reference was greater than or equal to that level. +However, by keeping a bitmap of the first 31 back references, we can catch some +of the more common cases more precisely. Arguments: code points to start of expression (the bracket) options points to the options setting - in_brackets TRUE if inside capturing parentheses - top_backref the highest back reference in the regex + bracket_map a bitmap of which brackets we are inside while testing; this + handles up to substring 31; after that we just have to take + the less precise approach + backref_map the back reference bitmap Returns: TRUE or FALSE */ static BOOL -is_anchored(register const uschar *code, int *options, BOOL in_brackets, - int top_backref) +is_anchored(register const uschar *code, int *options, unsigned int bracket_map, + unsigned int backref_map) { do { const uschar *scode = @@ -2963,29 +3554,32 @@ do { if (op > OP_BRA) { - if (!is_anchored(scode, options, TRUE, top_backref)) return FALSE; + int new_map; + op -= OP_BRA; + if (op > EXTRACT_BASIC_MAX) op = GET2(scode, 2+LINK_SIZE); + new_map = bracket_map | ((op < 32)? (1 << op) : 1); + if (!is_anchored(scode, options, new_map, backref_map)) return FALSE; } /* Other brackets */ else if (op == OP_BRA || op == OP_ASSERT || op == OP_ONCE || op == OP_COND) { - if (!is_anchored(scode, options, in_brackets, top_backref)) - return FALSE; + if (!is_anchored(scode, options, bracket_map, backref_map)) return FALSE; } /* .* is not anchored unless DOTALL is set and it isn't in brackets that - may be referenced. */ + are or may be referenced. */ else if ((op == OP_TYPESTAR || op == OP_TYPEMINSTAR) && (*options & PCRE_DOTALL) != 0) { - if (scode[1] != OP_ANY || (in_brackets && top_backref > 0)) return FALSE; + if (scode[1] != OP_ANY || (bracket_map & backref_map) != 0) return FALSE; } /* Check for explicit anchoring */ - else if (op != OP_SOD && + else if (op != OP_SOD && op != OP_SOM && ((*options & PCRE_MULTILINE) != 0 || op != OP_CIRC)) return FALSE; code += GET(code, 1); @@ -3009,14 +3603,17 @@ because in that case we can't make the assumption. Arguments: code points to start of expression (the bracket) - in_brackets TRUE if inside capturing parentheses - top_backref the highest back reference in the regex + bracket_map a bitmap of which brackets we are inside while testing; this + handles up to substring 31; after that we just have to take + the less precise approach + backref_map the back reference bitmap Returns: TRUE or FALSE */ static BOOL -is_startline(const uschar *code, BOOL in_brackets, int top_backref) +is_startline(const uschar *code, unsigned int bracket_map, + unsigned int backref_map) { do { const uschar *scode = first_significant_code(code + 1+LINK_SIZE, NULL, 0); @@ -3025,19 +3622,25 @@ do { /* Capturing brackets */ if (op > OP_BRA) - { if (!is_startline(scode, TRUE, top_backref)) return FALSE; } + { + int new_map; + op -= OP_BRA; + if (op > EXTRACT_BASIC_MAX) op = GET2(scode, 2+LINK_SIZE); + new_map = bracket_map | ((op < 32)? (1 << op) : 1); + if (!is_startline(scode, new_map, backref_map)) return FALSE; + } /* Other brackets */ else if (op == OP_BRA || op == OP_ASSERT || op == OP_ONCE || op == OP_COND) - { if (!is_startline(scode, in_brackets, top_backref)) return FALSE; } + { if (!is_startline(scode, bracket_map, backref_map)) return FALSE; } /* .* is not anchored unless DOTALL is set and it isn't in brackets that may be referenced. */ else if (op == OP_TYPESTAR || op == OP_TYPEMINSTAR) { - if (scode[1] != OP_ANY || (in_brackets && top_backref > 0)) return FALSE; + if (scode[1] != OP_ANY || (bracket_map & backref_map) != 0) return FALSE; } /* Check for explicit circumflex */ @@ -3149,14 +3752,18 @@ pcre_compile(const char *pattern, int options, const char **errorptr, real_pcre *re; int length = 1 + LINK_SIZE; /* For initial BRA plus length */ int runlength; -int c, firstchar, reqchar; +int c, firstbyte, reqbyte; int bracount = 0; -int top_backref = 0; int branch_extra = 0; int branch_newextra; int item_count = -1; int name_count = 0; int max_name_size = 0; +#ifdef SUPPORT_UTF8 +int lastcharlength = 0; +BOOL utf8; +BOOL class_utf8; +#endif BOOL inescq = FALSE; unsigned int brastackptr = 0; size_t size; @@ -3167,16 +3774,6 @@ compile_data compile_block; int brastack[BRASTACK_SIZE]; uschar bralenstack[BRASTACK_SIZE]; -/* Can't support UTF8 unless PCRE has been compiled to include the code. */ - -#ifndef SUPPORT_UTF8 -if ((options & PCRE_UTF8) != 0) - { - *errorptr = ERR32; - return NULL; - } -#endif - /* We can't pass back an error message if errorptr is NULL; I guess the best we can do is just return NULL. */ @@ -3192,6 +3789,18 @@ if (erroroffset == NULL) } *erroroffset = 0; +/* Can't support UTF8 unless PCRE has been compiled to include the code. */ + +#ifdef SUPPORT_UTF8 +utf8 = (options & PCRE_UTF8) != 0; +#else +if ((options & PCRE_UTF8) != 0) + { + *errorptr = ERR32; + return NULL; + } +#endif + if ((options & ~PUBLIC_OPTIONS) != 0) { *errorptr = ERR17; @@ -3206,6 +3815,14 @@ compile_block.fcc = tables + fcc_offset; compile_block.cbits = tables + cbits_offset; compile_block.ctypes = tables + ctypes_offset; +/* Maximum back reference and backref bitmap. This is updated for numeric +references during the first pass, but for named references during the actual +compile pass. The bitmap records up to 31 back references to help in deciding +whether (.*) can be treated as anchored or not. */ + +compile_block.top_backref = 0; +compile_block.backref_map = 0; + /* Reflect pattern for debugging output */ DPRINTF(("------------------------------------------------------------------\n")); @@ -3222,7 +3839,7 @@ ptr = (const uschar *)(pattern - 1); while ((c = *(++ptr)) != 0) { int min, max; - int class_charcount; + int class_optcount; int bracket_length; int duplength; @@ -3275,9 +3892,12 @@ while ((c = *(++ptr)) != 0) continue; } - /* Other escapes need one byte */ + /* Other escapes need one byte, and are of length one for repeats */ length++; +#ifdef SUPPORT_UTF8 + lastcharlength = 1; +#endif /* A back reference needs an additional 2 bytes, plus either one or 5 bytes for a repeat. We also need to keep the value of the highest @@ -3286,7 +3906,9 @@ while ((c = *(++ptr)) != 0) if (c <= -ESC_REF) { int refnum = -c - ESC_REF; - if (refnum > top_backref) top_backref = refnum; + compile_block.backref_map |= (refnum < 32)? (1 << refnum) : 1; + if (refnum > compile_block.top_backref) + compile_block.top_backref = refnum; length += 2; /* For single back reference */ if (ptr[1] == '{' && is_counted_repeat(ptr+2, &compile_block)) { @@ -3301,43 +3923,72 @@ while ((c = *(++ptr)) != 0) } continue; - case '*': /* These repeats won't be after brackets; */ - case '+': /* those are handled separately */ - case '?': - if (ptr[1] == '+') /* Handle "possessive quantifier" */ - { - length += 2 + 2*LINK_SIZE; - ptr++; - } - /* Fall through */ - case '^': /* Single-byte metacharacters */ case '.': case '$': length++; +#ifdef SUPPORT_UTF8 + lastcharlength = 1; +#endif continue; - /* This covers the cases of repeats after a single char, metachar, class, - or back reference. */ + case '*': /* These repeats won't be after brackets; */ + case '+': /* those are handled separately */ + case '?': + length++; + goto POSESSIVE; /* A few lines below */ + + /* This covers the cases of braced repeats after a single char, metachar, + class, or back reference. */ case '{': if (!is_counted_repeat(ptr+1, &compile_block)) goto NORMAL_CHAR; ptr = read_repeat_counts(ptr+1, &min, &max, errorptr, &compile_block); if (*errorptr != NULL) goto PCRE_ERROR_RETURN; + + /* These special cases just insert one extra opcode */ + if ((min == 0 && (max == 1 || max == -1)) || (min == 1 && max == -1)) length++; + + /* These cases might insert additional copies of a preceding character. */ + else { - length--; /* Uncount the original char or metachar */ - if (min == 1) length++; else if (min > 0) length += 4; - if (max > 0) length += 4; else length += 2; +#ifdef SUPPORT_UTF8 + /* In UTF-8 mode, we should find the length in lastcharlength */ + if (utf8) + { + if (min != 1) + { + length -= lastcharlength; /* Uncount the original char or metachar */ + if (min > 0) length += 3 + lastcharlength; + } + length += lastcharlength + ((max > 0)? 3 : 1); + } + else +#endif + + /* Not UTF-8 mode: all characters are one byte */ + { + if (min != 1) + { + length--; /* Uncount the original char or metachar */ + if (min > 0) length += 4; + } + + length += (max > 0)? 4 : 2; + } } - if (ptr[1] == '?') ptr++; /* Needs no extra length */ - if (ptr[1] == '+') /* Possessive quantifier */ + + if (ptr[1] == '?') ptr++; /* Needs no extra length */ + + POSESSIVE: /* Test for possessive quantifier */ + if (ptr[1] == '+') { ptr++; - length += 2 + 2*LINK_SIZE; /* Allow for atomic brackets */ + length += 2 + 2*LINK_SIZE; /* Allow for atomic brackets */ } continue; @@ -3350,25 +4001,89 @@ while ((c = *(++ptr)) != 0) length += 1 + LINK_SIZE + branch_extra; continue; - /* A character class uses 33 characters. Don't worry about character types - that aren't allowed in classes - they'll get picked up during the compile. - A character class that contains only one character uses 2 or 3 bytes, - depending on whether it is negated or not. Notice this where we can. */ + /* A character class uses 33 characters provided that all the character + values are less than 256. Otherwise, it uses a bit map for low valued + characters, and individual items for others. Don't worry about character + types that aren't allowed in classes - they'll get picked up during the + compile. A character class that contains only one single-byte character + uses 2 or 3 bytes, depending on whether it is negated or not. Notice this + where we can. (In UTF-8 mode we can do this only for chars < 128.) */ case '[': - class_charcount = 0; + class_optcount = 0; + +#ifdef SUPPORT_UTF8 + class_utf8 = FALSE; +#endif + if (*(++ptr) == '^') ptr++; /* Written as a "do" so that an initial ']' is taken as data */ if (*ptr != 0) do { + /* Inside \Q...\E everything is literal except \E */ + + if (inescq) + { + if (*ptr != '\\' || ptr[1] != 'E') goto NON_SPECIAL_CHARACTER; + inescq = FALSE; + ptr += 1; + continue; + } + + /* Outside \Q...\E, check for escapes */ + if (*ptr == '\\') { +#ifdef SUPPORT_UTF8 + int prevchar = ptr[-1]; +#endif int ch = check_escape(&ptr, errorptr, bracount, options, TRUE, &compile_block); if (*errorptr != NULL) goto PCRE_ERROR_RETURN; - if (-ch == ESC_b) class_charcount++; else class_charcount = 10; + + /* \b is backspace inside a class */ + + if (-ch == ESC_b) ch = '\b'; + + /* \Q enters quoting mode */ + + if (-ch == ESC_Q) + { + inescq = TRUE; + continue; + } + + /* Handle escapes that turn into characters */ + + if (ch >= 0) + { +#ifdef SUPPORT_UTF8 + if (utf8) + { + if (ch > 127) class_optcount = 10; /* Ensure > 1 */ + if (ch > 255) + { + uschar buffer[6]; + if (!class_utf8) + { + class_utf8 = TRUE; + length += LINK_SIZE + 1 + 1; + } + length += 1 + ord2utf8(ch, buffer); + + /* If this wide character is preceded by '-', add an extra 2 to + the length in case the previous character was < 128, because in + this case the whole range will be put into the list. */ + + if (prevchar == '-') length += 2; + } + } +#endif + class_optcount++; /* for possible optimization */ + } + else class_optcount = 10; /* \d, \s etc; make sure > 1 */ } /* Check the syntax for POSIX stuff. The bits we actually handle are @@ -3377,14 +4092,48 @@ while ((c = *(++ptr)) != 0) else if (*ptr == '[' && check_posix_syntax(ptr, &ptr, &compile_block)) { ptr++; - class_charcount = 10; /* Make sure > 1 */ + class_optcount = 10; /* Make sure > 1 */ } - /* Anything else just counts as one char */ + /* Anything else just increments the possible optimization count. If + there are wide characters, we are going to have to use an XCLASS. */ + + else + { + NON_SPECIAL_CHARACTER: + class_optcount++; + +#ifdef SUPPORT_UTF8 + if (utf8) + { + int ch; + int extra = 0; + GETCHARLEN(ch, ptr, extra); + if (ch > 127) class_optcount = 10; /* No optimization possible */ + if (ch > 255) + { + if (!class_utf8) + { + class_utf8 = TRUE; + length += LINK_SIZE + 1 + 1; + } + length += 2 + extra; + + /* If this wide character is preceded by '-', add an extra 2 to + the length in case the previous character was < 128, because in + this case the whole range will be put into the list. */ + + if (ptr[-1] == '-') length += 2; + + /* Advance to the end of this character */ - else class_charcount++; + ptr += extra; + } + } +#endif + } } - while (*(++ptr) != 0 && *ptr != ']'); /* Concludes "do" above */ + while (*(++ptr) != 0 && (inescq || *ptr != ']')); /* Concludes "do" above */ if (*ptr == 0) /* Missing terminating ']' */ { @@ -3392,9 +4141,11 @@ while ((c = *(++ptr)) != 0) goto PCRE_ERROR_RETURN; } - /* Repeats for negated single chars are handled by the general code */ + /* We can optimize when there was only one optimizable character. Repeats + for positive and negated single one-byte chars are handled by the general + code. Here, we handle repeats for the class opcodes. */ - if (class_charcount == 1) length += 3; else + if (class_optcount == 1) length += 3; else { length += 33; @@ -3466,7 +4217,7 @@ while ((c = *(++ptr)) != 0) case '5': case '6': case '7': case '8': case '9': ptr += 2; if (c != 'R') - while ((compile_block.ctypes[*(++ptr)] & ctype_digit) != 0); + while ((digitab[*(++ptr)] & ctype_digit) != 0); if (*ptr != ')') { *errorptr = ERR29; @@ -3492,7 +4243,7 @@ while ((c = *(++ptr)) != 0) case 'C': ptr += 2; - while ((compile_block.ctypes[*(++ptr)] & ctype_digit) != 0); + while ((digitab[*(++ptr)] & ctype_digit) != 0); if (*ptr != ')') { *errorptr = ERR39; @@ -3507,7 +4258,8 @@ while ((c = *(++ptr)) != 0) ptr += 3; if (*ptr == '<') { - const uschar *p = ++ptr; + const uschar *p; /* Don't amalgamate; some compilers */ + p = ++ptr; /* grumble at autoincrement in declaration */ while ((compile_block.ctypes[*ptr] & ctype_word) != 0) ptr++; if (*ptr != '>') { @@ -3558,11 +4310,11 @@ while ((c = *(++ptr)) != 0) ptr += 4; length += 3; } - else if ((compile_block.ctypes[ptr[3]] & ctype_digit) != 0) + else if ((digitab[ptr[3]] & ctype_digit) != 0) { ptr += 4; length += 3; - while ((compile_block.ctypes[*ptr] & ctype_digit) != 0) ptr++; + while ((digitab[*ptr] & ctype_digit) != 0) ptr++; if (*ptr != ')') { *errorptr = ERR26; @@ -3700,9 +4452,11 @@ while ((c = *(++ptr)) != 0) /* Extracting brackets must be counted so we can process escapes in a Perlish way. If the number exceeds EXTRACT_BASIC_MAX we are going to - need an additional 3 bytes of store per extracting bracket. */ + need an additional 3 bytes of store per extracting bracket. However, if + PCRE_NO_AUTO)CAPTURE is set, unadorned brackets become non-capturing, so we + must leave the count alone (it will aways be zero). */ - else + else if ((options & PCRE_NO_AUTO_CAPTURE) == 0) { bracount++; if (bracount > EXTRACT_BASIC_MAX) bracket_length += 3; @@ -3796,9 +4550,9 @@ while ((c = *(++ptr)) != 0) continue; /* Non-special character. For a run of such characters the length required - is the number of characters + 2, except that the maximum run length is 255. - We won't get a skipped space or a non-data escape or the start of a # - comment as the first character, so the length can't be zero. */ + is the number of characters + 2, except that the maximum run length is + MAXLIT. We won't get a skipped space or a non-data escape or the start of a + # comment as the first character, so the length can't be zero. */ NORMAL_CHAR: default: @@ -3806,6 +4560,10 @@ while ((c = *(++ptr)) != 0) runlength = 0; do { +#ifdef SUPPORT_UTF8 + lastcharlength = 1; /* Need length of last char for UTF-8 repeats */ +#endif + /* If in a \Q...\E sequence, check for end; otherwise it's a literal */ if (inescq) { @@ -3843,13 +4601,18 @@ while ((c = *(++ptr)) != 0) if (*errorptr != NULL) goto PCRE_ERROR_RETURN; if (c < 0) { ptr = saveptr; break; } + /* In UTF-8 mode, add on the number of additional bytes needed to + encode this character, and save the total length in case this is a + final char that is repeated. */ + #ifdef SUPPORT_UTF8 - if (c > 127 && (options & PCRE_UTF8) != 0) + if (utf8 && c > 127) { int i; for (i = 0; i < sizeof(utf8_table1)/sizeof(int); i++) if (c <= utf8_table1[i]) break; runlength += i; + lastcharlength += i; } #endif } @@ -3864,7 +4627,27 @@ while ((c = *(++ptr)) != 0) while (runlength < MAXLIT && (compile_block.ctypes[c = *(++ptr)] & ctype_meta) == 0); + /* If we hit a meta-character, back off to point to it */ + if (runlength < MAXLIT) ptr--; + + /* If the last char in the string is a UTF-8 multibyte character, we must + set lastcharlength correctly. If it was specified as an escape, this will + already have been done above. However, we also have to support in-line + UTF-8 characters, so check backwards from where we are. */ + +#ifdef SUPPORT_UTF8 + if (utf8) + { + const uschar *lastptr = ptr - 1; + if ((*lastptr & 0x80) != 0) + { + while((*lastptr & 0xc0) == 0x80) lastptr--; + lastcharlength = ptr - lastptr; + } + } +#endif + length += runlength; continue; } @@ -3907,6 +4690,7 @@ compile_block.name_entry_size = max_name_size + 3; compile_block.name_table = (uschar *)re + sizeof(real_pcre); codestart = compile_block.name_table + re->name_entry_size * re->name_count; compile_block.start_code = codestart; +compile_block.req_varyopt = 0; /* Set up a starting, non-extracting bracket, then compile the expression. On error, *errorptr will be set non-NULL, so we don't need to look at the result @@ -3917,9 +4701,9 @@ code = (uschar *)codestart; *code = OP_BRA; bracount = 0; (void)compile_regex(options, options & PCRE_IMS, &bracount, &code, &ptr, - errorptr, FALSE, 0, &firstchar, &reqchar, NULL, &compile_block); + errorptr, FALSE, 0, &firstbyte, &reqbyte, NULL, &compile_block); re->top_bracket = bracount; -re->top_backref = top_backref; +re->top_backref = compile_block.top_backref; /* If not reached end of pattern on success, there's an excess bracket. */ @@ -3937,7 +4721,7 @@ if (code - codestart > length) *errorptr = ERR23; /* Give an error if there's back reference to a non-existent capturing subpattern. */ -if (top_backref > re->top_bracket) *errorptr = ERR15; +if (re->top_backref > re->top_bracket) *errorptr = ERR15; /* Failed to compile, or error while post-processing */ @@ -3962,35 +4746,34 @@ start with ^. and also when all branches start with .* for non-DOTALL matches. if ((options & PCRE_ANCHORED) == 0) { int temp_options = options; - if (is_anchored(codestart, &temp_options, FALSE, top_backref)) + if (is_anchored(codestart, &temp_options, 0, compile_block.backref_map)) re->options |= PCRE_ANCHORED; else { - if (firstchar < 0) - firstchar = find_firstassertedchar(codestart, &temp_options, FALSE); - if (firstchar >= 0) /* Remove caseless flag for non-caseable chars */ + if (firstbyte < 0) + firstbyte = find_firstassertedchar(codestart, &temp_options, FALSE); + if (firstbyte >= 0) /* Remove caseless flag for non-caseable chars */ { - int ch = firstchar & 255; - re->first_char = ((firstchar & REQ_CASELESS) != 0 && - compile_block.fcc[ch] == ch)? ch : firstchar; + int ch = firstbyte & 255; + re->first_byte = ((firstbyte & REQ_CASELESS) != 0 && + compile_block.fcc[ch] == ch)? ch : firstbyte; re->options |= PCRE_FIRSTSET; } - else if (is_startline(codestart, FALSE, top_backref)) + else if (is_startline(codestart, 0, compile_block.backref_map)) re->options |= PCRE_STARTLINE; } } -/* Save the last required character if any. Remove caseless flag for -non-caseable chars. */ - -if ((re->options & PCRE_ANCHORED) != 0 && reqchar < 0 && firstchar >= 0) - reqchar = firstchar; +/* For an anchored pattern, we use the "required byte" only if it follows a +variable length item in the regex. Remove the caseless flag for non-caseable +chars. */ -if (reqchar >= 0) +if (reqbyte >= 0 && + ((re->options & PCRE_ANCHORED) == 0 || (reqbyte & REQ_VARY) != 0)) { - int ch = reqchar & 255; - re->req_char = ((reqchar & REQ_CASELESS) != 0 && - compile_block.fcc[ch] == ch)? ch : reqchar; + int ch = reqbyte & 255; + re->req_byte = ((reqbyte & REQ_CASELESS) != 0 && + compile_block.fcc[ch] == ch)? (reqbyte & ~REQ_CASELESS) : reqbyte; re->options |= PCRE_REQCHSET; } @@ -4017,16 +4800,16 @@ if (re->options != 0) if ((re->options & PCRE_FIRSTSET) != 0) { - int ch = re->first_char & 255; - char *caseless = ((re->first_char & REQ_CASELESS) == 0)? "" : " (caseless)"; + int ch = re->first_byte & 255; + char *caseless = ((re->first_byte & REQ_CASELESS) == 0)? "" : " (caseless)"; if (isprint(ch)) printf("First char = %c%s\n", ch, caseless); else printf("First char = \\x%02x%s\n", ch, caseless); } if ((re->options & PCRE_REQCHSET) != 0) { - int ch = re->req_char & 255; - char *caseless = ((re->req_char & REQ_CASELESS) == 0)? "" : " (caseless)"; + int ch = re->req_byte & 255; + char *caseless = ((re->req_byte & REQ_CASELESS) == 0)? "" : " (caseless)"; if (isprint(ch)) printf("Req char = %c%s\n", ch, caseless); else printf("Req char = \\x%02x%s\n", ch, caseless); } @@ -4104,6 +4887,63 @@ return TRUE; } +#ifdef SUPPORT_UTF8 +/************************************************* +* Match character against an XCLASS * +*************************************************/ + +/* This function is called from within the XCLASS code below, to match a +character against an extended class which might match values > 255. + +Arguments: + c the character + data points to the flag byte of the XCLASS data + +Returns: TRUE if character matches, else FALSE +*/ + +static BOOL +match_xclass(int c, const uschar *data) +{ +int t; +BOOL negated = (*data & XCL_NOT) != 0; + +/* Character values < 256 are matched against a bitmap, if one is present. If +not, we still carry on, because there may be ranges that start below 256 in the +additional data. */ + +if (c < 256) + { + if ((*data & XCL_MAP) != 0 && (data[1 + c/8] & (1 << (c&7))) != 0) + return !negated; /* char found */ + } + +/* Now match against the list of large chars or ranges that end with a large +char. First skip the bit map if present. */ + +if ((*data++ & XCL_MAP) != 0) data += 32; + +while ((t = *data++) != XCL_END) + { + int x, y; + GETCHARINC(x, data); + if (t == XCL_SINGLE) + { + if (c == x) return !negated; + } + else + { + GETCHARINC(y, data); + if (c >= x && c <= y) return !negated; + } + } + +return negated; /* char was not found */ +} +#endif + + + /************************************************* * Match from current position * @@ -4112,7 +4952,14 @@ return TRUE; /* On entry ecode points to the first opcode, and eptr to the first character in the subject string, while eptrb holds the value of eptr at the start of the last bracketed group - used for breaking infinite loops matching zero-length -strings. +strings. This function is called recursively in many circumstances. Whenever it +returns a negative (error) response, the outer incarnation must also return the +same response. + +Performance note: It might be tempting to extract commonly used fields from the +md structure (e.g. utf8, end_subject) into individual variables to improve +performance. Tests using gcc on a SPARC disproved this; in the first case, it +made performance worse. Arguments: eptr pointer in subject @@ -4126,17 +4973,23 @@ Arguments: match_condassert - this is an assertion condition match_isgroup - this is the start of a bracketed group -Returns: TRUE if matched +Returns: MATCH_MATCH if matched ) these values are >= 0 + MATCH_NOMATCH if failed to match ) + a negative PCRE_ERROR_xxx value if aborted by an error condition + (e.g. stopped by recursion limit) */ -static BOOL +static int match(register const uschar *eptr, register const uschar *ecode, int offset_top, match_data *md, unsigned long int ims, eptrblock *eptrb, int flags) { unsigned long int original_ims = ims; /* Save for resetting on ')' */ +register int rrc; eptrblock newptrb; +if (md->match_call_count++ >= md->match_limit) return PCRE_ERROR_MATCHLIMIT; + /* At the start of a bracketed group, add the current subject pointer to the stack of such pointers, to be re-instated at the end of the group when we hit the closing ket. When match() is called in other circumstances, we don't add to @@ -4203,8 +5056,8 @@ for (;;) do { - if (match(eptr, ecode + 1 + LINK_SIZE, offset_top, md, ims, eptrb, - match_isgroup)) return TRUE; + if ((rrc = match(eptr, ecode + 1 + LINK_SIZE, offset_top, md, ims, + eptrb, match_isgroup)) != MATCH_NOMATCH) return rrc; md->capture_last = save_capture_last; ecode += GET(ecode, 1); } @@ -4216,7 +5069,7 @@ for (;;) md->offset_vector[offset+1] = save_offset2; md->offset_vector[md->offset_end - number] = save_offset3; - return FALSE; + return MATCH_NOMATCH; } /* Insufficient room for saving captured contents */ @@ -4232,13 +5085,13 @@ for (;;) DPRINTF(("start bracket 0\n")); do { - if (match(eptr, ecode + 1 + LINK_SIZE, offset_top, md, ims, eptrb, - match_isgroup)) return TRUE; + if ((rrc = match(eptr, ecode + 1 + LINK_SIZE, offset_top, md, ims, eptrb, + match_isgroup)) != MATCH_NOMATCH) return rrc; ecode += GET(ecode, 1); } while (*ecode == OP_ALT); DPRINTF(("bracket 0 failed\n")); - return FALSE; + return MATCH_NOMATCH; /* Conditional group: compilation checked that there are no more than two branches. If the condition is false, skipping the first branch takes us @@ -4262,12 +5115,13 @@ for (;;) else { - if (match(eptr, ecode + 1 + LINK_SIZE, offset_top, md, ims, NULL, - match_condassert | match_isgroup)) + if ((rrc = match(eptr, ecode + 1 + LINK_SIZE, offset_top, md, ims, NULL, + match_condassert | match_isgroup)) == MATCH_MATCH) { ecode += 1 + LINK_SIZE + GET(ecode, LINK_SIZE+2); while (*ecode == OP_ALT) ecode += GET(ecode, 1); } + else if (rrc != MATCH_NOMATCH) return rrc; else ecode += GET(ecode, 1); return match(eptr, ecode + 1 + LINK_SIZE, offset_top, md, ims, eptrb, match_isgroup); @@ -4302,10 +5156,10 @@ for (;;) /* Otherwise, if PCRE_NOTEMPTY is set, fail if we have matched an empty string - backtracking will then try other alternatives, if any. */ - if (md->notempty && eptr == md->start_match) return FALSE; + if (md->notempty && eptr == md->start_match) return MATCH_NOMATCH; md->end_match_ptr = eptr; /* Record where we ended */ md->end_offset_top = offset_top; /* and how many extracts were taken */ - return TRUE; + return MATCH_MATCH; /* Change option settings */ @@ -4325,16 +5179,17 @@ for (;;) case OP_ASSERTBACK: do { - if (match(eptr, ecode + 1 + LINK_SIZE, offset_top, md, ims, NULL, - match_isgroup)) break; + if ((rrc = match(eptr, ecode + 1 + LINK_SIZE, offset_top, md, ims, NULL, + match_isgroup)) == MATCH_MATCH) break; + if (rrc != MATCH_NOMATCH) return rrc; ecode += GET(ecode, 1); } while (*ecode == OP_ALT); - if (*ecode == OP_KET) return FALSE; + if (*ecode == OP_KET) return MATCH_NOMATCH; - /* If checking an assertion for a condition, return TRUE. */ + /* If checking an assertion for a condition, return MATCH_MATCH. */ - if ((flags & match_condassert) != 0) return TRUE; + if ((flags & match_condassert) != 0) return MATCH_MATCH; /* Continue from after the assertion, updating the offsets high water mark, since extracts may have been taken during the assertion. */ @@ -4350,13 +5205,14 @@ for (;;) case OP_ASSERTBACK_NOT: do { - if (match(eptr, ecode + 1 + LINK_SIZE, offset_top, md, ims, NULL, - match_isgroup)) return FALSE; + if ((rrc = match(eptr, ecode + 1 + LINK_SIZE, offset_top, md, ims, NULL, + match_isgroup)) == MATCH_MATCH) return MATCH_NOMATCH; + if (rrc != MATCH_NOMATCH) return rrc; ecode += GET(ecode,1); } while (*ecode == OP_ALT); - if ((flags & match_condassert) != 0) return TRUE; + if ((flags & match_condassert) != 0) return MATCH_MATCH; ecode += 1 + LINK_SIZE; continue; @@ -4368,17 +5224,28 @@ for (;;) case OP_REVERSE: #ifdef SUPPORT_UTF8 - c = GET(ecode,1); - for (i = 0; i < c; i++) + if (md->utf8) { - eptr--; - BACKCHAR(eptr) + c = GET(ecode,1); + for (i = 0; i < c; i++) + { + eptr--; + if (eptr < md->start_subject) return MATCH_NOMATCH; + BACKCHAR(eptr) + } } -#else - eptr -= GET(ecode,1); + else #endif - if (eptr < md->start_subject) return FALSE; + /* No UTF-8 support, or not in UTF-8 mode: count is byte count */ + + { + eptr -= GET(ecode,1); + if (eptr < md->start_subject) return MATCH_NOMATCH; + } + + /* Skip to next op code */ + ecode += 1 + LINK_SIZE; break; @@ -4399,7 +5266,9 @@ for (;;) cb.current_position = eptr - md->start_subject; cb.capture_top = offset_top/2; cb.capture_last = md->capture_last; - if ((*pcre_callout)(&cb) != 0) return FALSE; + cb.callout_data = md->callout_data; + if ((rrc = (*pcre_callout)(&cb)) > 0) return MATCH_NOMATCH; + if (rrc < 0) return rrc; } ecode += 2; break; @@ -4457,29 +5326,9 @@ for (;;) new_recursive.offset_save = stacksave; else { - new_recursive.offset_save = (int *) - (pcre_malloc)(new_recursive.saved_max * sizeof(int)); - - /* RH: Warning: This may cause INCORRECT RESULTS if we run out of - memory here, because we won't be restoring all the stored strings - correctly. We either need proper run-time error handling or, at the - very least, some way to warn the user. Could we just spit a message to - stderr? - - PH: No, Robin, no! You must NEVER write to stderr from inside a general - library function, because you don't know anything about the state of - the file descriptor. - - RH: Returning error values would be very tedious because of the - recursion; and Philip Hazel says that longjmp() - in many ways the - obvious solution - has previously caused problems on some platforms. */ - - if (new_recursive.offset_save == NULL) - { - DPRINTF(("malloc() failed - results may be wrong\n")); - new_recursive.offset_save = stacksave; - new_recursive.saved_max = REC_STACK_SAVE_MAX; - } + new_recursive.offset_save = + (int *)(pcre_malloc)(new_recursive.saved_max * sizeof(int)); + if (new_recursive.offset_save == NULL) return PCRE_ERROR_NOMEMORY; } memcpy(new_recursive.offset_save, md->offset_vector, @@ -4493,14 +5342,15 @@ for (;;) DPRINTF(("Recursing into group %d\n", new_recursive.group_num)); do { - if (match(eptr, callpat + 1 + LINK_SIZE, offset_top, md, ims, eptrb, - match_isgroup)) + if ((rrc = match(eptr, callpat + 1 + LINK_SIZE, offset_top, md, ims, + eptrb, match_isgroup)) == MATCH_MATCH) { md->recursive = new_recursive.prev; if (new_recursive.offset_save != stacksave) (pcre_free)(new_recursive.offset_save); - return TRUE; + return MATCH_MATCH; } + else if (rrc != MATCH_NOMATCH) return rrc; md->recursive = &new_recursive; memcpy(md->offset_vector, new_recursive.offset_save, @@ -4513,9 +5363,9 @@ for (;;) md->recursive = new_recursive.prev; if (new_recursive.offset_save != stacksave) (pcre_free)(new_recursive.offset_save); - return FALSE; + return MATCH_NOMATCH; } - break; + /* Control never reaches here */ /* "Once" brackets are like assertion brackets except that after a match, the point in the subject string is not moved back. Thus there can never be @@ -4531,15 +5381,16 @@ for (;;) do { - if (match(eptr, ecode + 1 + LINK_SIZE, offset_top, md, ims, eptrb, - match_isgroup)) break; + if ((rrc = match(eptr, ecode + 1 + LINK_SIZE, offset_top, md, ims, + eptrb, match_isgroup)) == MATCH_MATCH) break; + if (rrc != MATCH_NOMATCH) return rrc; ecode += GET(ecode,1); } while (*ecode == OP_ALT); /* If hit the end of the group (which could be repeated), fail */ - if (*ecode != OP_ONCE && *ecode != OP_ALT) return FALSE; + if (*ecode != OP_ONCE && *ecode != OP_ALT) return MATCH_NOMATCH; /* Continue as from after the assertion, updating the offsets high water mark, since extracts may have been taken. */ @@ -4574,19 +5425,20 @@ for (;;) if (*ecode == OP_KETRMIN) { - if (match(eptr, ecode + 1 + LINK_SIZE, offset_top, md, ims, eptrb, 0) - || - match(eptr, prev, offset_top, md, ims, eptrb, match_isgroup)) - return TRUE; + if ((rrc = match(eptr, ecode + 1 + LINK_SIZE, offset_top, md, ims, + eptrb, 0)) != MATCH_NOMATCH) return rrc; + if ((rrc = match(eptr, prev, offset_top, md, ims, eptrb, + match_isgroup)) != MATCH_NOMATCH) return rrc; } else /* OP_KETRMAX */ { - if (match(eptr, prev, offset_top, md, ims, eptrb, match_isgroup) || - match(eptr, ecode + 1+LINK_SIZE, offset_top, md, ims, eptrb, 0)) - return TRUE; + if ((rrc = match(eptr, prev, offset_top, md, ims, eptrb, + match_isgroup)) != MATCH_NOMATCH) return rrc; + if ((rrc = match(eptr, ecode + 1+LINK_SIZE, offset_top, md, ims, eptrb, + 0)) != MATCH_NOMATCH) return rrc; } } - return FALSE; + return MATCH_NOMATCH; /* An alternation is the end of a branch; scan along to find the end of the bracketed group and go to there. */ @@ -4604,8 +5456,8 @@ for (;;) case OP_BRAZERO: { const uschar *next = ecode+1; - if (match(eptr, next, offset_top, md, ims, eptrb, match_isgroup)) - return TRUE; + if ((rrc = match(eptr, next, offset_top, md, ims, eptrb, match_isgroup)) + != MATCH_NOMATCH) return rrc; do next += GET(next,1); while (*next == OP_ALT); ecode = next + 1+LINK_SIZE; } @@ -4615,14 +5467,14 @@ for (;;) { const uschar *next = ecode+1; do next += GET(next,1); while (*next == OP_ALT); - if (match(eptr, next + 1+LINK_SIZE, offset_top, md, ims, eptrb, - match_isgroup)) return TRUE; + if ((rrc = match(eptr, next + 1+LINK_SIZE, offset_top, md, ims, eptrb, + match_isgroup)) != MATCH_NOMATCH) return rrc; ecode++; } break; /* End of a group, repeated or non-repeating. If we are at the end of - an assertion "group", stop matching and return TRUE, but record the + an assertion "group", stop matching and return MATCH_MATCH, but record the current high water mark for use by positive assertions. Do this also for the "once" (not-backup up) groups. */ @@ -4641,7 +5493,7 @@ for (;;) { md->end_match_ptr = eptr; /* For ONCE */ md->end_offset_top = offset_top; - return TRUE; + return MATCH_MATCH; } /* In all other cases except a conditional group we have to check the @@ -4721,26 +5573,29 @@ for (;;) if (*ecode == OP_KETRMIN) { - if (match(eptr, ecode + 1+LINK_SIZE, offset_top, md, ims, eptrb, 0) || - match(eptr, prev, offset_top, md, ims, eptrb, match_isgroup)) - return TRUE; + if ((rrc = match(eptr, ecode + 1+LINK_SIZE, offset_top, md, ims, eptrb, + 0)) != MATCH_NOMATCH) return rrc; + if ((rrc = match(eptr, prev, offset_top, md, ims, eptrb, + match_isgroup)) != MATCH_NOMATCH) return rrc; } else /* OP_KETRMAX */ { - if (match(eptr, prev, offset_top, md, ims, eptrb, match_isgroup) || - match(eptr, ecode + 1+LINK_SIZE, offset_top, md, ims, eptrb, 0)) - return TRUE; + if ((rrc = match(eptr, prev, offset_top, md, ims, eptrb, + match_isgroup)) != MATCH_NOMATCH) return rrc; + if ((rrc = match(eptr, ecode + 1+LINK_SIZE, offset_top, md, ims, eptrb, + 0)) != MATCH_NOMATCH) return rrc; } } - return FALSE; + return MATCH_NOMATCH; /* Start of subject unless notbol, or after internal newline if multiline */ case OP_CIRC: - if (md->notbol && eptr == md->start_subject) return FALSE; + if (md->notbol && eptr == md->start_subject) return MATCH_NOMATCH; if ((ims & PCRE_MULTILINE) != 0) { - if (eptr != md->start_subject && eptr[-1] != NEWLINE) return FALSE; + if (eptr != md->start_subject && eptr[-1] != NEWLINE) + return MATCH_NOMATCH; ecode++; break; } @@ -4749,14 +5604,14 @@ for (;;) /* Start of subject assertion */ case OP_SOD: - if (eptr != md->start_subject) return FALSE; + if (eptr != md->start_subject) return MATCH_NOMATCH; ecode++; break; /* Start of match assertion */ case OP_SOM: - if (eptr != md->start_subject + md->start_offset) return FALSE; + if (eptr != md->start_subject + md->start_offset) return MATCH_NOMATCH; ecode++; break; @@ -4766,19 +5621,21 @@ for (;;) case OP_DOLL: if ((ims & PCRE_MULTILINE) != 0) { - if (eptr < md->end_subject) { if (*eptr != NEWLINE) return FALSE; } - else { if (md->noteol) return FALSE; } + if (eptr < md->end_subject) + { if (*eptr != NEWLINE) return MATCH_NOMATCH; } + else + { if (md->noteol) return MATCH_NOMATCH; } ecode++; break; } else { - if (md->noteol) return FALSE; + if (md->noteol) return MATCH_NOMATCH; if (!md->endonly) { if (eptr < md->end_subject - 1 || - (eptr == md->end_subject - 1 && *eptr != NEWLINE)) return FALSE; - + (eptr == md->end_subject - 1 && *eptr != NEWLINE)) + return MATCH_NOMATCH; ecode++; break; } @@ -4788,7 +5645,7 @@ for (;;) /* End of subject assertion (\z) */ case OP_EOD: - if (eptr < md->end_subject) return FALSE; + if (eptr < md->end_subject) return MATCH_NOMATCH; ecode++; break; @@ -4796,7 +5653,7 @@ for (;;) case OP_EODN: if (eptr < md->end_subject - 1 || - (eptr == md->end_subject - 1 && *eptr != NEWLINE)) return FALSE; + (eptr == md->end_subject - 1 && *eptr != NEWLINE)) return MATCH_NOMATCH; ecode++; break; @@ -4805,13 +5662,45 @@ for (;;) case OP_NOT_WORD_BOUNDARY: case OP_WORD_BOUNDARY: { - BOOL prev_is_word = (eptr != md->start_subject) && - ((md->ctypes[eptr[-1]] & ctype_word) != 0); - BOOL cur_is_word = (eptr < md->end_subject) && - ((md->ctypes[*eptr] & ctype_word) != 0); + BOOL prev_is_word, cur_is_word; + + /* Find out if the previous and current characters are "word" characters. + It takes a bit more work in UTF-8 mode. Characters > 255 are assumed to + be "non-word" characters. */ + +#ifdef SUPPORT_UTF8 + if (md->utf8) + { + if (eptr == md->start_subject) prev_is_word = FALSE; else + { + const uschar *lastptr = eptr - 1; + while((*lastptr & 0xc0) == 0x80) lastptr--; + GETCHAR(c, lastptr); + prev_is_word = c < 256 && (md->ctypes[c] & ctype_word) != 0; + } + if (eptr >= md->end_subject) cur_is_word = FALSE; else + { + GETCHAR(c, eptr); + cur_is_word = c < 256 && (md->ctypes[c] & ctype_word) != 0; + } + } + else +#endif + + /* More streamlined when not in UTF-8 mode */ + + { + prev_is_word = (eptr != md->start_subject) && + ((md->ctypes[eptr[-1]] & ctype_word) != 0); + cur_is_word = (eptr < md->end_subject) && + ((md->ctypes[*eptr] & ctype_word) != 0); + } + + /* Now see if the situation is what we want */ + if ((*ecode++ == OP_WORD_BOUNDARY)? cur_is_word == prev_is_word : cur_is_word != prev_is_word) - return FALSE; + return MATCH_NOMATCH; } break; @@ -4819,8 +5708,8 @@ for (;;) case OP_ANY: if ((ims & PCRE_DOTALL) == 0 && eptr < md->end_subject && *eptr == NEWLINE) - return FALSE; - if (eptr++ >= md->end_subject) return FALSE; + return MATCH_NOMATCH; + if (eptr++ >= md->end_subject) return MATCH_NOMATCH; #ifdef SUPPORT_UTF8 if (md->utf8) while (eptr < md->end_subject && (*eptr & 0xc0) == 0x80) eptr++; @@ -4832,49 +5721,85 @@ for (;;) any byte, even newline, independent of the setting of PCRE_DOTALL. */ case OP_ANYBYTE: - if (eptr++ >= md->end_subject) return FALSE; + if (eptr++ >= md->end_subject) return MATCH_NOMATCH; ecode++; break; case OP_NOT_DIGIT: - if (eptr >= md->end_subject || - (md->ctypes[*eptr++] & ctype_digit) != 0) - return FALSE; + if (eptr >= md->end_subject) return MATCH_NOMATCH; + GETCHARINCTEST(c, eptr); + if ( +#ifdef SUPPORT_UTF8 + c < 256 && +#endif + (md->ctypes[c] & ctype_digit) != 0 + ) + return MATCH_NOMATCH; ecode++; break; case OP_DIGIT: - if (eptr >= md->end_subject || - (md->ctypes[*eptr++] & ctype_digit) == 0) - return FALSE; + if (eptr >= md->end_subject) return MATCH_NOMATCH; + GETCHARINCTEST(c, eptr); + if ( +#ifdef SUPPORT_UTF8 + c >= 256 || +#endif + (md->ctypes[c] & ctype_digit) == 0 + ) + return MATCH_NOMATCH; ecode++; break; case OP_NOT_WHITESPACE: - if (eptr >= md->end_subject || - (md->ctypes[*eptr++] & ctype_space) != 0) - return FALSE; + if (eptr >= md->end_subject) return MATCH_NOMATCH; + GETCHARINCTEST(c, eptr); + if ( +#ifdef SUPPORT_UTF8 + c < 256 && +#endif + (md->ctypes[c] & ctype_space) != 0 + ) + return MATCH_NOMATCH; ecode++; break; case OP_WHITESPACE: - if (eptr >= md->end_subject || - (md->ctypes[*eptr++] & ctype_space) == 0) - return FALSE; + if (eptr >= md->end_subject) return MATCH_NOMATCH; + GETCHARINCTEST(c, eptr); + if ( +#ifdef SUPPORT_UTF8 + c >= 256 || +#endif + (md->ctypes[c] & ctype_space) == 0 + ) + return MATCH_NOMATCH; ecode++; break; case OP_NOT_WORDCHAR: - if (eptr >= md->end_subject || - (md->ctypes[*eptr++] & ctype_word) != 0) - return FALSE; + if (eptr >= md->end_subject) return MATCH_NOMATCH; + GETCHARINCTEST(c, eptr); + if ( +#ifdef SUPPORT_UTF8 + c < 256 && +#endif + (md->ctypes[c] & ctype_word) != 0 + ) + return MATCH_NOMATCH; ecode++; break; case OP_WORDCHAR: - if (eptr >= md->end_subject || - (md->ctypes[*eptr++] & ctype_word) == 0) - return FALSE; + if (eptr >= md->end_subject) return MATCH_NOMATCH; + GETCHARINCTEST(c, eptr); + if ( +#ifdef SUPPORT_UTF8 + c >= 256 || +#endif + (md->ctypes[c] & ctype_word) == 0 + ) + return MATCH_NOMATCH; ecode++; break; @@ -4928,7 +5853,7 @@ for (;;) break; default: /* No repeat follows */ - if (!match_ref(offset, eptr, length, md, ims)) return FALSE; + if (!match_ref(offset, eptr, length, md, ims)) return MATCH_NOMATCH; eptr += length; continue; /* With the main loop */ } @@ -4944,7 +5869,7 @@ for (;;) for (i = 1; i <= min; i++) { - if (!match_ref(offset, eptr, length, md, ims)) return FALSE; + if (!match_ref(offset, eptr, length, md, ims)) return MATCH_NOMATCH; eptr += length; } @@ -4959,10 +5884,10 @@ for (;;) { for (i = min;; i++) { - if (match(eptr, ecode, offset_top, md, ims, eptrb, 0)) - return TRUE; + if ((rrc = match(eptr, ecode, offset_top, md, ims, eptrb, 0)) != + MATCH_NOMATCH) return rrc; if (i >= max || !match_ref(offset, eptr, length, md, ims)) - return FALSE; + return MATCH_NOMATCH; eptr += length; } /* Control never gets here */ @@ -4980,21 +5905,27 @@ for (;;) } while (eptr >= pp) { - if (match(eptr, ecode, offset_top, md, ims, eptrb, 0)) - return TRUE; + if ((rrc = match(eptr, ecode, offset_top, md, ims, eptrb, 0)) != + MATCH_NOMATCH) return rrc; eptr -= length; } - return FALSE; + return MATCH_NOMATCH; } } /* Control never gets here */ - /* Match a character class, possibly repeatedly. Look past the end of the - item to see if there is repeat information following. Then obey similar - code to character type repeats - written out again for speed. */ + /* Match a bit-mapped character class, possibly repeatedly. This op code is + used when all the characters in the class have values in the range 0-255. + The only difference between OP_CLASS and OP_NCLASS occurs when a data + character outside the range is encountered. + First, look past the end of the item to see if there is repeat information + following. Then obey similar code to character type repeats - written out + again for speed. */ + + case OP_NCLASS: case OP_CLASS: { const uschar *data = ecode + 1; /* Save for matching */ @@ -5031,18 +5962,184 @@ for (;;) /* First, ensure the minimum number of matches are present. */ - for (i = 1; i <= min; i++) +#ifdef SUPPORT_UTF8 + /* UTF-8 mode */ + if (md->utf8) { - if (eptr >= md->end_subject) return FALSE; - GETCHARINC(c, eptr) /* Get character; increment eptr */ + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) return MATCH_NOMATCH; + GETCHARINC(c, eptr); + if (c > 255) + { + if (op == OP_CLASS) return MATCH_NOMATCH; + } + else + { + if ((data[c/8] & (1 << (c&7))) == 0) return MATCH_NOMATCH; + } + } + } + else +#endif + /* Not UTF-8 mode */ + { + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) return MATCH_NOMATCH; + c = *eptr++; + if ((data[c/8] & (1 << (c&7))) == 0) return MATCH_NOMATCH; + } + } + + /* If max == min we can continue with the main loop without the + need to recurse. */ + + if (min == max) continue; + /* If minimizing, keep testing the rest of the expression and advancing + the pointer while it matches the class. */ + + if (minimize) + { #ifdef SUPPORT_UTF8 - /* We do not yet support class members > 255 */ - if (c > 255) return FALSE; + /* UTF-8 mode */ + if (md->utf8) + { + for (i = min;; i++) + { + if ((rrc = match(eptr, ecode, offset_top, md, ims, eptrb, 0)) != + MATCH_NOMATCH) return rrc; + if (i >= max || eptr >= md->end_subject) return MATCH_NOMATCH; + GETCHARINC(c, eptr); + if (c > 255) + { + if (op == OP_CLASS) return MATCH_NOMATCH; + } + else + { + if ((data[c/8] & (1 << (c&7))) == 0) return MATCH_NOMATCH; + } + } + } + else #endif + /* Not UTF-8 mode */ + { + for (i = min;; i++) + { + if ((rrc = match(eptr, ecode, offset_top, md, ims, eptrb, 0)) != + MATCH_NOMATCH) return rrc; + if (i >= max || eptr >= md->end_subject) return MATCH_NOMATCH; + c = *eptr++; + if ((data[c/8] & (1 << (c&7))) == 0) return MATCH_NOMATCH; + } + } + /* Control never gets here */ + } + + /* If maximizing, find the longest possible run, then work backwards. */ + + else + { + const uschar *pp = eptr; + +#ifdef SUPPORT_UTF8 + /* UTF-8 mode */ + if (md->utf8) + { + for (i = min; i < max; i++) + { + int len = 1; + if (eptr >= md->end_subject) break; + GETCHARLEN(c, eptr, len); + if (c > 255) + { + if (op == OP_CLASS) break; + } + else + { + if ((data[c/8] & (1 << (c&7))) == 0) break; + } + eptr += len; + } + for (;;) + { + if ((rrc = match(eptr, ecode, offset_top, md, ims, eptrb, 0)) != + MATCH_NOMATCH) return rrc; + if (eptr-- == pp) break; /* Stop if tried at original pos */ + BACKCHAR(eptr); + } + } + else +#endif + /* Not UTF-8 mode */ + { + for (i = min; i < max; i++) + { + if (eptr >= md->end_subject) break; + c = *eptr; + if ((data[c/8] & (1 << (c&7))) == 0) break; + eptr++; + } + while (eptr >= pp) + { + if ((rrc = match(eptr--, ecode, offset_top, md, ims, eptrb, 0)) != + MATCH_NOMATCH) return rrc; + } + } + + return MATCH_NOMATCH; + } + } + /* Control never gets here */ + + + /* Match an extended character class. This opcode is encountered only + in UTF-8 mode, because that's the only time it is compiled. */ + +#ifdef SUPPORT_UTF8 + case OP_XCLASS: + { + const uschar *data = ecode + 1 + LINK_SIZE; /* Save for matching */ + ecode += GET(ecode, 1); /* Advance past the item */ + + switch (*ecode) + { + case OP_CRSTAR: + case OP_CRMINSTAR: + case OP_CRPLUS: + case OP_CRMINPLUS: + case OP_CRQUERY: + case OP_CRMINQUERY: + c = *ecode++ - OP_CRSTAR; + minimize = (c & 1) != 0; + min = rep_min[c]; /* Pick up values from tables; */ + max = rep_max[c]; /* zero for max => infinity */ + if (max == 0) max = INT_MAX; + break; + + case OP_CRRANGE: + case OP_CRMINRANGE: + minimize = (*ecode == OP_CRMINRANGE); + min = GET2(ecode, 1); + max = GET2(ecode, 3); + if (max == 0) max = INT_MAX; + ecode += 5; + break; + + default: /* No repeat follows */ + min = max = 1; + break; + } + + /* First, ensure the minimum number of matches are present. */ - if ((data[c/8] & (1 << (c&7))) != 0) continue; - return FALSE; + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) return MATCH_NOMATCH; + GETCHARINC(c, eptr); + if (!match_xclass(c, data)) return MATCH_NOMATCH; } /* If max == min we can continue with the main loop without the @@ -5057,17 +6154,11 @@ for (;;) { for (i = min;; i++) { - if (match(eptr, ecode, offset_top, md, ims, eptrb, 0)) - return TRUE; - if (i >= max || eptr >= md->end_subject) return FALSE; - GETCHARINC(c, eptr) /* Get character; increment eptr */ - -#ifdef SUPPORT_UTF8 - /* We do not yet support class members > 255 */ - if (c > 255) return FALSE; -#endif - if ((data[c/8] & (1 << (c&7))) != 0) continue; - return FALSE; + if ((rrc = match(eptr, ecode, offset_top, md, ims, eptrb, 0)) != + MATCH_NOMATCH) return rrc; + if (i >= max || eptr >= md->end_subject) return MATCH_NOMATCH; + GETCHARINC(c, eptr); + if (!match_xclass(c, data)) return MATCH_NOMATCH; } /* Control never gets here */ } @@ -5077,33 +6168,27 @@ for (;;) else { const uschar *pp = eptr; - int len = 1; for (i = min; i < max; i++) { + int len = 1; if (eptr >= md->end_subject) break; - GETCHARLEN(c, eptr, len) /* Get character, set length if UTF-8 */ - -#ifdef SUPPORT_UTF8 - /* We do not yet support class members > 255 */ - if (c > 255) break; -#endif - if ((data[c/8] & (1 << (c&7))) == 0) break; + GETCHARLEN(c, eptr, len); + if (!match_xclass(c, data)) break; eptr += len; } - - while (eptr >= pp) + for(;;) { - if (match(eptr--, ecode, offset_top, md, ims, eptrb, 0)) - return TRUE; - -#ifdef SUPPORT_UTF8 + if ((rrc = match(eptr, ecode, offset_top, md, ims, eptrb, 0)) != + MATCH_NOMATCH) return rrc; + if (eptr-- == pp) break; /* Stop if tried at original pos */ BACKCHAR(eptr) -#endif } - return FALSE; + return MATCH_NOMATCH; } + + /* Control never gets here */ } - /* Control never gets here */ +#endif /* End of XCLASS */ /* Match a run of characters */ @@ -5125,16 +6210,16 @@ for (;;) printf("\n"); #endif - if (length > md->end_subject - eptr) return FALSE; + if (length > md->end_subject - eptr) return MATCH_NOMATCH; if ((ims & PCRE_CASELESS) != 0) { while (length-- > 0) if (md->lcc[*ecode++] != md->lcc[*eptr++]) - return FALSE; + return MATCH_NOMATCH; } else { - while (length-- > 0) if (*ecode++ != *eptr++) return FALSE; + while (length-- > 0) if (*ecode++ != *eptr++) return MATCH_NOMATCH; } } break; @@ -5171,16 +6256,84 @@ for (;;) the subject. */ REPEATCHAR: - if (min > md->end_subject - eptr) return FALSE; - c = *ecode++; +#ifdef SUPPORT_UTF8 + if (md->utf8) + { + int len = 1; + const uschar *charptr = ecode; + GETCHARLEN(c, ecode, len); + if (min * len > md->end_subject - eptr) return MATCH_NOMATCH; + ecode += len; - /* The code is duplicated for the caseless and caseful cases, for speed, - since matching characters is likely to be quite common. First, ensure the - minimum number of matches are present. If min = max, continue at the same - level without recursing. Otherwise, if minimizing, keep trying the rest of - the expression and advancing one matching character if failing, up to the - maximum. Alternatively, if maximizing, find the maximum number of - characters and work backwards. */ + /* Handle multibyte character matching specially here. There is no + support for any kind of casing for multibyte characters. */ + + if (len > 1) + { + for (i = 1; i <= min; i++) + { + if (memcmp(eptr, charptr, len) != 0) return MATCH_NOMATCH; + eptr += len; + } + + if (min == max) continue; + + if (minimize) + { + for (i = min;; i++) + { + if ((rrc = match(eptr, ecode, offset_top, md, ims, eptrb, 0)) != + MATCH_NOMATCH) return rrc; + if (i >= max || + eptr >= md->end_subject || + memcmp(eptr, charptr, len) != 0) + return MATCH_NOMATCH; + eptr += len; + } + /* Control never gets here */ + } + else + { + const uschar *pp = eptr; + for (i = min; i < max; i++) + { + if (eptr > md->end_subject - len || + memcmp(eptr, charptr, len) != 0) + break; + eptr += len; + } + while (eptr >= pp) + { + if ((rrc = match(eptr, ecode, offset_top, md, ims, eptrb, 0)) != + MATCH_NOMATCH) return rrc; + eptr -= len; + } + return MATCH_NOMATCH; + } + /* Control never gets here */ + } + + /* If the length of a UTF-8 character is 1, we fall through here, and + obey the code as for non-UTF-8 characters below, though in this case the + value of c will always be < 128. */ + } + else +#endif + + /* When not in UTF-8 mode, load a single-byte character. */ + { + if (min > md->end_subject - eptr) return MATCH_NOMATCH; + c = *ecode++; + } + + /* The value of c at this point is always less than 256, though we may or + may not be in UTF-8 mode. The code is duplicated for the caseless and + caseful cases, for speed, since matching characters is likely to be quite + common. First, ensure the minimum number of matches are present. If min = + max, continue at the same level without recursing. Otherwise, if + minimizing, keep trying the rest of the expression and advancing one + matching character if failing, up to the maximum. Alternatively, if + maximizing, find the maximum number of characters and work backwards. */ DPRINTF(("matching %c{%d,%d} against subject %.*s\n", c, min, max, max, eptr)); @@ -5189,17 +6342,17 @@ for (;;) { c = md->lcc[c]; for (i = 1; i <= min; i++) - if (c != md->lcc[*eptr++]) return FALSE; + if (c != md->lcc[*eptr++]) return MATCH_NOMATCH; if (min == max) continue; if (minimize) { for (i = min;; i++) { - if (match(eptr, ecode, offset_top, md, ims, eptrb, 0)) - return TRUE; + if ((rrc = match(eptr, ecode, offset_top, md, ims, eptrb, 0)) != + MATCH_NOMATCH) return rrc; if (i >= max || eptr >= md->end_subject || c != md->lcc[*eptr++]) - return FALSE; + return MATCH_NOMATCH; } /* Control never gets here */ } @@ -5212,26 +6365,27 @@ for (;;) eptr++; } while (eptr >= pp) - if (match(eptr--, ecode, offset_top, md, ims, eptrb, 0)) - return TRUE; - return FALSE; + if ((rrc = match(eptr--, ecode, offset_top, md, ims, eptrb, 0)) != + MATCH_NOMATCH) return rrc; + return MATCH_NOMATCH; } /* Control never gets here */ } - /* Caseful comparisons */ + /* Caseful comparisons (includes all multi-byte characters) */ else { - for (i = 1; i <= min; i++) if (c != *eptr++) return FALSE; + for (i = 1; i <= min; i++) if (c != *eptr++) return MATCH_NOMATCH; if (min == max) continue; if (minimize) { for (i = min;; i++) { - if (match(eptr, ecode, offset_top, md, ims, eptrb, 0)) - return TRUE; - if (i >= max || eptr >= md->end_subject || c != *eptr++) return FALSE; + if ((rrc = match(eptr, ecode, offset_top, md, ims, eptrb, 0)) != + MATCH_NOMATCH) return rrc; + if (i >= max || eptr >= md->end_subject || c != *eptr++) + return MATCH_NOMATCH; } /* Control never gets here */ } @@ -5244,33 +6398,40 @@ for (;;) eptr++; } while (eptr >= pp) - if (match(eptr--, ecode, offset_top, md, ims, eptrb, 0)) - return TRUE; - return FALSE; + if ((rrc = match(eptr--, ecode, offset_top, md, ims, eptrb, 0)) != + MATCH_NOMATCH) return rrc; + return MATCH_NOMATCH; } } /* Control never gets here */ - /* Match a negated single character */ + /* Match a negated single one-byte character. The character we are + checking can be multibyte. */ case OP_NOT: - if (eptr >= md->end_subject) return FALSE; + if (eptr >= md->end_subject) return MATCH_NOMATCH; ecode++; + GETCHARINCTEST(c, eptr); if ((ims & PCRE_CASELESS) != 0) { - if (md->lcc[*ecode++] == md->lcc[*eptr++]) return FALSE; +#ifdef SUPPORT_UTF8 + if (c < 256) +#endif + c = md->lcc[c]; + if (md->lcc[*ecode++] == c) return MATCH_NOMATCH; } else { - if (*ecode++ == *eptr++) return FALSE; + if (*ecode++ == c) return MATCH_NOMATCH; } break; - /* Match a negated single character repeatedly. This is almost a repeat of - the code for a repeated single character, but I haven't found a nice way of - commoning these up that doesn't require a test of the positive/negative - option for each character match. Maybe that wouldn't add very much to the - time taken, but character matching *is* what this is all about... */ + /* Match a negated single one-byte character repeatedly. This is almost a + repeat of the code for a repeated single character, but I haven't found a + nice way of commoning these up that doesn't require a test of the + positive/negative option for each character match. Maybe that wouldn't add + very much to the time taken, but character matching *is* what this is all + about... */ case OP_NOTEXACT: min = max = GET2(ecode, 1); @@ -5297,12 +6458,12 @@ for (;;) max = rep_max[c]; /* zero for max => infinity */ if (max == 0) max = INT_MAX; - /* Common code for all repeated single-character matches. We can give - up quickly if there are fewer than the minimum number of characters left in - the subject. */ + /* Common code for all repeated single-character (less than 255) matches. + We can give up quickly if there are fewer than the minimum number of + characters left in the subject. */ REPEATNOTCHAR: - if (min > md->end_subject - eptr) return FALSE; + if (min > md->end_subject - eptr) return MATCH_NOMATCH; c = *ecode++; /* The code is duplicated for the caseless and caseful cases, for speed, @@ -5319,33 +6480,108 @@ for (;;) if ((ims & PCRE_CASELESS) != 0) { c = md->lcc[c]; - for (i = 1; i <= min; i++) - if (c == md->lcc[*eptr++]) return FALSE; + +#ifdef SUPPORT_UTF8 + /* UTF-8 mode */ + if (md->utf8) + { + register int d; + for (i = 1; i <= min; i++) + { + GETCHARINC(d, eptr); + if (d < 256) d = md->lcc[d]; + if (c == d) return MATCH_NOMATCH; + } + } + else +#endif + + /* Not UTF-8 mode */ + { + for (i = 1; i <= min; i++) + if (c == md->lcc[*eptr++]) return MATCH_NOMATCH; + } + if (min == max) continue; + if (minimize) { - for (i = min;; i++) +#ifdef SUPPORT_UTF8 + /* UTF-8 mode */ + if (md->utf8) { - if (match(eptr, ecode, offset_top, md, ims, eptrb, 0)) - return TRUE; - if (i >= max || eptr >= md->end_subject || - c == md->lcc[*eptr++]) - return FALSE; + register int d; + for (i = min;; i++) + { + if ((rrc = match(eptr, ecode, offset_top, md, ims, eptrb, 0)) != + MATCH_NOMATCH) return rrc; + GETCHARINC(d, eptr); + if (d < 256) d = md->lcc[d]; + if (i >= max || eptr >= md->end_subject || c == d) + return MATCH_NOMATCH; + } + } + else +#endif + /* Not UTF-8 mode */ + { + for (i = min;; i++) + { + if ((rrc = match(eptr, ecode, offset_top, md, ims, eptrb, 0)) != + MATCH_NOMATCH) return rrc; + if (i >= max || eptr >= md->end_subject || c == md->lcc[*eptr++]) + return MATCH_NOMATCH; + } } /* Control never gets here */ } + + /* Maximize case */ + else { const uschar *pp = eptr; - for (i = min; i < max; i++) + +#ifdef SUPPORT_UTF8 + /* UTF-8 mode */ + if (md->utf8) { - if (eptr >= md->end_subject || c == md->lcc[*eptr]) break; - eptr++; + register int d; + for (i = min; i < max; i++) + { + int len = 1; + if (eptr >= md->end_subject) break; + GETCHARLEN(d, eptr, len); + if (d < 256) d = md->lcc[d]; + if (c == d) break; + eptr += len; + } + for(;;) + { + if ((rrc = match(eptr, ecode, offset_top, md, ims, eptrb, 0)) != + MATCH_NOMATCH) return rrc; + if (eptr-- == pp) break; /* Stop if tried at original pos */ + BACKCHAR(eptr); + } } - while (eptr >= pp) - if (match(eptr--, ecode, offset_top, md, ims, eptrb, 0)) - return TRUE; - return FALSE; + else +#endif + /* Not UTF-8 mode */ + { + for (i = min; i < max; i++) + { + if (eptr >= md->end_subject || c == md->lcc[*eptr]) break; + eptr++; + } + while (eptr >= pp) + { + if ((rrc = match(eptr, ecode, offset_top, md, ims, eptrb, 0)) != + MATCH_NOMATCH) return rrc; + eptr--; + } + } + + return MATCH_NOMATCH; } /* Control never gets here */ } @@ -5354,30 +6590,103 @@ for (;;) else { - for (i = 1; i <= min; i++) if (c == *eptr++) return FALSE; +#ifdef SUPPORT_UTF8 + /* UTF-8 mode */ + if (md->utf8) + { + register int d; + for (i = 1; i <= min; i++) + { + GETCHARINC(d, eptr); + if (c == d) return MATCH_NOMATCH; + } + } + else +#endif + /* Not UTF-8 mode */ + { + for (i = 1; i <= min; i++) + if (c == *eptr++) return MATCH_NOMATCH; + } + if (min == max) continue; + if (minimize) { - for (i = min;; i++) +#ifdef SUPPORT_UTF8 + /* UTF-8 mode */ + if (md->utf8) + { + register int d; + for (i = min;; i++) + { + if ((rrc = match(eptr, ecode, offset_top, md, ims, eptrb, 0)) != + MATCH_NOMATCH) return rrc; + GETCHARINC(d, eptr); + if (i >= max || eptr >= md->end_subject || c == d) + return MATCH_NOMATCH; + } + } + else +#endif + /* Not UTF-8 mode */ { - if (match(eptr, ecode, offset_top, md, ims, eptrb, 0)) - return TRUE; - if (i >= max || eptr >= md->end_subject || c == *eptr++) return FALSE; + for (i = min;; i++) + { + if ((rrc = match(eptr, ecode, offset_top, md, ims, eptrb, 0)) != + MATCH_NOMATCH) return rrc; + if (i >= max || eptr >= md->end_subject || c == *eptr++) + return MATCH_NOMATCH; + } } /* Control never gets here */ } + + /* Maximize case */ + else { const uschar *pp = eptr; - for (i = min; i < max; i++) + +#ifdef SUPPORT_UTF8 + /* UTF-8 mode */ + if (md->utf8) { - if (eptr >= md->end_subject || c == *eptr) break; - eptr++; + register int d; + for (i = min; i < max; i++) + { + int len = 1; + if (eptr >= md->end_subject) break; + GETCHARLEN(d, eptr, len); + if (c == d) break; + eptr += len; + } + for(;;) + { + if ((rrc = match(eptr, ecode, offset_top, md, ims, eptrb, 0)) != + MATCH_NOMATCH) return rrc; + if (eptr-- == pp) break; /* Stop if tried at original pos */ + BACKCHAR(eptr); + } } - while (eptr >= pp) - if (match(eptr--, ecode, offset_top, md, ims, eptrb, 0)) - return TRUE; - return FALSE; + else +#endif + /* Not UTF-8 mode */ + { + for (i = min; i < max; i++) + { + if (eptr >= md->end_subject || c == *eptr) break; + eptr++; + } + while (eptr >= pp) + { + if ((rrc = match(eptr, ecode, offset_top, md, ims, eptrb, 0)) != + MATCH_NOMATCH) return rrc; + eptr--; + } + } + + return MATCH_NOMATCH; } } /* Control never gets here */ @@ -5412,7 +6721,9 @@ for (;;) max = rep_max[c]; /* zero for max => infinity */ if (max == 0) max = INT_MAX; - /* Common code for all repeated single character type matches */ + /* Common code for all repeated single character type matches. Note that + in UTF-8 mode, '.' matches a character of any length, but for the other + character types, the valid characters are all one-byte long. */ REPEATTYPE: ctype = *ecode++; /* Code for the character type */ @@ -5420,68 +6731,142 @@ for (;;) /* First, ensure the minimum number of matches are present. Use inline code for maximizing the speed, and do the type test once at the start (i.e. keep it out of the loop). Also we can test that there are at least - the minimum number of bytes before we start, except when doing '.' in - UTF8 mode. Leave the test in in all cases; in the special case we have - to test after each character. */ + the minimum number of bytes before we start. This isn't as effective in + UTF-8 mode, but it does no harm. Separate the UTF-8 code completely as that + is tidier. */ - if (min > md->end_subject - eptr) return FALSE; - if (min > 0) switch(ctype) + if (min > md->end_subject - eptr) return MATCH_NOMATCH; + if (min > 0) { - case OP_ANY: #ifdef SUPPORT_UTF8 - if (md->utf8) + if (md->utf8) switch(ctype) { + case OP_ANY: for (i = 1; i <= min; i++) { if (eptr >= md->end_subject || (*eptr++ == NEWLINE && (ims & PCRE_DOTALL) == 0)) - return FALSE; + return MATCH_NOMATCH; + while (eptr < md->end_subject && (*eptr & 0xc0) == 0x80) eptr++; + } + break; + + case OP_ANYBYTE: + eptr += min; + break; + + case OP_NOT_DIGIT: + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) return MATCH_NOMATCH; + GETCHARINC(c, eptr); + if (c < 256 && (md->ctypes[c] & ctype_digit) != 0) + return MATCH_NOMATCH; + } + break; + + case OP_DIGIT: + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject || + *eptr >= 128 || (md->ctypes[*eptr++] & ctype_digit) == 0) + return MATCH_NOMATCH; + /* No need to skip more bytes - we know it's a 1-byte character */ + } + break; + + case OP_NOT_WHITESPACE: + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject || + (*eptr < 128 && (md->ctypes[*eptr++] & ctype_space) != 0)) + return MATCH_NOMATCH; + while (eptr < md->end_subject && (*eptr & 0xc0) == 0x80) eptr++; + } + break; + + case OP_WHITESPACE: + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject || + *eptr >= 128 || (md->ctypes[*eptr++] & ctype_space) == 0) + return MATCH_NOMATCH; + /* No need to skip more bytes - we know it's a 1-byte character */ + } + break; + + case OP_NOT_WORDCHAR: + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject || + (*eptr < 128 && (md->ctypes[*eptr++] & ctype_word) != 0)) + return MATCH_NOMATCH; while (eptr < md->end_subject && (*eptr & 0xc0) == 0x80) eptr++; } break; + + case OP_WORDCHAR: + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject || + *eptr >= 128 || (md->ctypes[*eptr++] & ctype_word) == 0) + return MATCH_NOMATCH; + /* No need to skip more bytes - we know it's a 1-byte character */ + } + break; } + else #endif - /* Non-UTF8 can be faster */ - if ((ims & PCRE_DOTALL) == 0) - { for (i = 1; i <= min; i++) if (*eptr++ == NEWLINE) return FALSE; } - else eptr += min; - break; - case OP_ANYBYTE: - eptr += min; - break; + /* Code for the non-UTF-8 case for minimum matching */ - case OP_NOT_DIGIT: - for (i = 1; i <= min; i++) - if ((md->ctypes[*eptr++] & ctype_digit) != 0) return FALSE; - break; + switch(ctype) + { + case OP_ANY: + if ((ims & PCRE_DOTALL) == 0) + { + for (i = 1; i <= min; i++) + if (*eptr++ == NEWLINE) return MATCH_NOMATCH; + } + else eptr += min; + break; - case OP_DIGIT: - for (i = 1; i <= min; i++) - if ((md->ctypes[*eptr++] & ctype_digit) == 0) return FALSE; - break; + case OP_ANYBYTE: + eptr += min; + break; - case OP_NOT_WHITESPACE: - for (i = 1; i <= min; i++) - if ((md->ctypes[*eptr++] & ctype_space) != 0) return FALSE; - break; + case OP_NOT_DIGIT: + for (i = 1; i <= min; i++) + if ((md->ctypes[*eptr++] & ctype_digit) != 0) return MATCH_NOMATCH; + break; - case OP_WHITESPACE: - for (i = 1; i <= min; i++) - if ((md->ctypes[*eptr++] & ctype_space) == 0) return FALSE; - break; + case OP_DIGIT: + for (i = 1; i <= min; i++) + if ((md->ctypes[*eptr++] & ctype_digit) == 0) return MATCH_NOMATCH; + break; - case OP_NOT_WORDCHAR: - for (i = 1; i <= min; i++) - if ((md->ctypes[*eptr++] & ctype_word) != 0) - return FALSE; - break; + case OP_NOT_WHITESPACE: + for (i = 1; i <= min; i++) + if ((md->ctypes[*eptr++] & ctype_space) != 0) return MATCH_NOMATCH; + break; - case OP_WORDCHAR: - for (i = 1; i <= min; i++) - if ((md->ctypes[*eptr++] & ctype_word) == 0) - return FALSE; - break; + case OP_WHITESPACE: + for (i = 1; i <= min; i++) + if ((md->ctypes[*eptr++] & ctype_space) == 0) return MATCH_NOMATCH; + break; + + case OP_NOT_WORDCHAR: + for (i = 1; i <= min; i++) + if ((md->ctypes[*eptr++] & ctype_word) != 0) + return MATCH_NOMATCH; + break; + + case OP_WORDCHAR: + for (i = 1; i <= min; i++) + if ((md->ctypes[*eptr++] & ctype_word) == 0) + return MATCH_NOMATCH; + break; + } } /* If min = max, continue at the same level without recursing */ @@ -5489,184 +6874,362 @@ for (;;) if (min == max) continue; /* If minimizing, we have to test the rest of the pattern before each - subsequent match. */ + subsequent match. Again, separate the UTF-8 case for speed. */ if (minimize) { - for (i = min;; i++) +#ifdef SUPPORT_UTF8 + /* UTF-8 mode */ + if (md->utf8) + { + for (i = min;; i++) + { + if ((rrc = match(eptr, ecode, offset_top, md, ims, eptrb, 0)) != + MATCH_NOMATCH) return rrc; + if (i >= max || eptr >= md->end_subject) return MATCH_NOMATCH; + + GETCHARINC(c, eptr); + switch(ctype) + { + case OP_ANY: + if ((ims & PCRE_DOTALL) == 0 && c == NEWLINE) return MATCH_NOMATCH; + break; + + case OP_ANYBYTE: + break; + + case OP_NOT_DIGIT: + if (c < 256 && (md->ctypes[c] & ctype_digit) != 0) + return MATCH_NOMATCH; + break; + + case OP_DIGIT: + if (c >= 256 || (md->ctypes[c] & ctype_digit) == 0) + return MATCH_NOMATCH; + break; + + case OP_NOT_WHITESPACE: + if (c < 256 && (md->ctypes[c] & ctype_space) != 0) + return MATCH_NOMATCH; + break; + + case OP_WHITESPACE: + if (c >= 256 || (md->ctypes[c] & ctype_space) == 0) + return MATCH_NOMATCH; + break; + + case OP_NOT_WORDCHAR: + if (c < 256 && (md->ctypes[c] & ctype_word) != 0) + return MATCH_NOMATCH; + break; + + case OP_WORDCHAR: + if (c >= 256 && (md->ctypes[c] & ctype_word) == 0) + return MATCH_NOMATCH; + break; + } + } + } + else +#endif + /* Not UTF-8 mode */ { - if (match(eptr, ecode, offset_top, md, ims, eptrb, 0)) return TRUE; - if (i >= max || eptr >= md->end_subject) return FALSE; + for (i = min;; i++) + { + if ((rrc = match(eptr, ecode, offset_top, md, ims, eptrb, 0)) != + MATCH_NOMATCH) return rrc; + if (i >= max || eptr >= md->end_subject) return MATCH_NOMATCH; + c = *eptr++; + switch(ctype) + { + case OP_ANY: + if ((ims & PCRE_DOTALL) == 0 && c == NEWLINE) return MATCH_NOMATCH; + break; + + case OP_ANYBYTE: + break; + + case OP_NOT_DIGIT: + if ((md->ctypes[c] & ctype_digit) != 0) return MATCH_NOMATCH; + break; - c = *eptr++; + case OP_DIGIT: + if ((md->ctypes[c] & ctype_digit) == 0) return MATCH_NOMATCH; + break; + + case OP_NOT_WHITESPACE: + if ((md->ctypes[c] & ctype_space) != 0) return MATCH_NOMATCH; + break; + + case OP_WHITESPACE: + if ((md->ctypes[c] & ctype_space) == 0) return MATCH_NOMATCH; + break; + + case OP_NOT_WORDCHAR: + if ((md->ctypes[c] & ctype_word) != 0) return MATCH_NOMATCH; + break; + + case OP_WORDCHAR: + if ((md->ctypes[c] & ctype_word) == 0) return MATCH_NOMATCH; + break; + } + } + } + /* Control never gets here */ + } + + /* If maximizing it is worth using inline code for speed, doing the type + test once at the start (i.e. keep it out of the loop). Again, keep the + UTF-8 stuff separate. */ + + else + { + const uschar *pp = eptr; + +#ifdef SUPPORT_UTF8 + /* UTF-8 mode */ + + if (md->utf8) + { switch(ctype) { case OP_ANY: - if ((ims & PCRE_DOTALL) == 0 && c == NEWLINE) return FALSE; -#ifdef SUPPORT_UTF8 - if (md->utf8) - while (eptr < md->end_subject && (*eptr & 0xc0) == 0x80) eptr++; -#endif + + /* Special code is required for UTF8, but when the maximum is unlimited + we don't need it, so we repeat the non-UTF8 code. This is probably + worth it, because .* is quite a common idiom. */ + + if (max < INT_MAX) + { + if ((ims & PCRE_DOTALL) == 0) + { + for (i = min; i < max; i++) + { + if (eptr >= md->end_subject || *eptr == NEWLINE) break; + eptr++; + while (eptr < md->end_subject && (*eptr & 0xc0) == 0x80) eptr++; + } + } + else + { + for (i = min; i < max; i++) + { + eptr++; + while (eptr < md->end_subject && (*eptr & 0xc0) == 0x80) eptr++; + } + } + } + + /* Handle unlimited UTF-8 repeat */ + + else + { + if ((ims & PCRE_DOTALL) == 0) + { + for (i = min; i < max; i++) + { + if (eptr >= md->end_subject || *eptr == NEWLINE) break; + eptr++; + } + break; + } + else + { + c = max - min; + if (c > md->end_subject - eptr) c = md->end_subject - eptr; + eptr += c; + } + } break; + /* The byte case is the same as non-UTF8 */ + case OP_ANYBYTE: + c = max - min; + if (c > md->end_subject - eptr) c = md->end_subject - eptr; + eptr += c; break; case OP_NOT_DIGIT: - if ((md->ctypes[c] & ctype_digit) != 0) return FALSE; + for (i = min; i < max; i++) + { + int len = 1; + if (eptr >= md->end_subject) break; + GETCHARLEN(c, eptr, len); + if (c < 256 && (md->ctypes[c] & ctype_digit) != 0) break; + eptr+= len; + } break; case OP_DIGIT: - if ((md->ctypes[c] & ctype_digit) == 0) return FALSE; + for (i = min; i < max; i++) + { + int len = 1; + if (eptr >= md->end_subject) break; + GETCHARLEN(c, eptr, len); + if (c >= 256 ||(md->ctypes[c] & ctype_digit) == 0) break; + eptr+= len; + } break; case OP_NOT_WHITESPACE: - if ((md->ctypes[c] & ctype_space) != 0) return FALSE; + for (i = min; i < max; i++) + { + int len = 1; + if (eptr >= md->end_subject) break; + GETCHARLEN(c, eptr, len); + if (c < 256 && (md->ctypes[c] & ctype_space) != 0) break; + eptr+= len; + } break; case OP_WHITESPACE: - if ((md->ctypes[c] & ctype_space) == 0) return FALSE; + for (i = min; i < max; i++) + { + int len = 1; + if (eptr >= md->end_subject) break; + GETCHARLEN(c, eptr, len); + if (c >= 256 ||(md->ctypes[c] & ctype_space) == 0) break; + eptr+= len; + } break; case OP_NOT_WORDCHAR: - if ((md->ctypes[c] & ctype_word) != 0) return FALSE; + for (i = min; i < max; i++) + { + int len = 1; + if (eptr >= md->end_subject) break; + GETCHARLEN(c, eptr, len); + if (c < 256 && (md->ctypes[c] & ctype_word) != 0) break; + eptr+= len; + } break; case OP_WORDCHAR: - if ((md->ctypes[c] & ctype_word) == 0) return FALSE; + for (i = min; i < max; i++) + { + int len = 1; + if (eptr >= md->end_subject) break; + GETCHARLEN(c, eptr, len); + if (c >= 256 || (md->ctypes[c] & ctype_word) == 0) break; + eptr+= len; + } break; } - } - /* Control never gets here */ - } - /* If maximizing it is worth using inline code for speed, doing the type - test once at the start (i.e. keep it out of the loop). */ + /* eptr is now past the end of the maximum run */ - else - { - const uschar *pp = eptr; - switch(ctype) - { - case OP_ANY: - - /* Special code is required for UTF8, but when the maximum is unlimited - we don't need it. */ + for(;;) + { + if ((rrc = match(eptr, ecode, offset_top, md, ims, eptrb, 0)) != + MATCH_NOMATCH) return rrc; + if (eptr-- == pp) break; /* Stop if tried at original pos */ + BACKCHAR(eptr); + } + } + else +#endif -#ifdef SUPPORT_UTF8 - if (md->utf8 && max < INT_MAX) + /* Not UTF-8 mode */ + { + switch(ctype) { + case OP_ANY: if ((ims & PCRE_DOTALL) == 0) { for (i = min; i < max; i++) { - if (eptr >= md->end_subject || *eptr++ == NEWLINE) break; - while (eptr < md->end_subject && (*eptr & 0xc0) == 0x80) eptr++; - } - } - else - { - for (i = min; i < max; i++) - { + if (eptr >= md->end_subject || *eptr == NEWLINE) break; eptr++; - while (eptr < md->end_subject && (*eptr & 0xc0) == 0x80) eptr++; } + break; } + /* For DOTALL case, fall through and treat as \C */ + + case OP_ANYBYTE: + c = max - min; + if (c > md->end_subject - eptr) c = md->end_subject - eptr; + eptr += c; break; - } -#endif - /* Non-UTF8 can be faster */ - if ((ims & PCRE_DOTALL) == 0) - { + + case OP_NOT_DIGIT: for (i = min; i < max; i++) { - if (eptr >= md->end_subject || *eptr == NEWLINE) break; + if (eptr >= md->end_subject || (md->ctypes[*eptr] & ctype_digit) != 0) + break; eptr++; } break; - } - /* For non-UTF8 DOTALL case, fall through and treat as \C */ - case OP_ANYBYTE: - c = max - min; - if (c > md->end_subject - eptr) c = md->end_subject - eptr; - eptr += c; - break; + case OP_DIGIT: + for (i = min; i < max; i++) + { + if (eptr >= md->end_subject || (md->ctypes[*eptr] & ctype_digit) == 0) + break; + eptr++; + } + break; - case OP_NOT_DIGIT: - for (i = min; i < max; i++) - { - if (eptr >= md->end_subject || (md->ctypes[*eptr] & ctype_digit) != 0) - break; - eptr++; - } - break; + case OP_NOT_WHITESPACE: + for (i = min; i < max; i++) + { + if (eptr >= md->end_subject || (md->ctypes[*eptr] & ctype_space) != 0) + break; + eptr++; + } + break; - case OP_DIGIT: - for (i = min; i < max; i++) - { - if (eptr >= md->end_subject || (md->ctypes[*eptr] & ctype_digit) == 0) - break; - eptr++; - } - break; + case OP_WHITESPACE: + for (i = min; i < max; i++) + { + if (eptr >= md->end_subject || (md->ctypes[*eptr] & ctype_space) == 0) + break; + eptr++; + } + break; - case OP_NOT_WHITESPACE: - for (i = min; i < max; i++) - { - if (eptr >= md->end_subject || (md->ctypes[*eptr] & ctype_space) != 0) - break; - eptr++; - } - break; + case OP_NOT_WORDCHAR: + for (i = min; i < max; i++) + { + if (eptr >= md->end_subject || (md->ctypes[*eptr] & ctype_word) != 0) + break; + eptr++; + } + break; - case OP_WHITESPACE: - for (i = min; i < max; i++) - { - if (eptr >= md->end_subject || (md->ctypes[*eptr] & ctype_space) == 0) - break; - eptr++; + case OP_WORDCHAR: + for (i = min; i < max; i++) + { + if (eptr >= md->end_subject || (md->ctypes[*eptr] & ctype_word) == 0) + break; + eptr++; + } + break; } - break; - case OP_NOT_WORDCHAR: - for (i = min; i < max; i++) - { - if (eptr >= md->end_subject || (md->ctypes[*eptr] & ctype_word) != 0) - break; - eptr++; - } - break; + /* eptr is now past the end of the maximum run */ - case OP_WORDCHAR: - for (i = min; i < max; i++) + while (eptr >= pp) { - if (eptr >= md->end_subject || (md->ctypes[*eptr] & ctype_word) == 0) - break; - eptr++; + if ((rrc = match(eptr--, ecode, offset_top, md, ims, eptrb, 0)) != + MATCH_NOMATCH) return rrc; } - break; } - while (eptr >= pp) - { - if (match(eptr--, ecode, offset_top, md, ims, eptrb, 0)) - return TRUE; -#ifdef SUPPORT_UTF8 - if (md->utf8) - while (eptr > pp && (*eptr & 0xc0) == 0x80) eptr--; -#endif - } - return FALSE; + /* Get here if we can't make it match with any permitted repetitions */ + + return MATCH_NOMATCH; } /* Control never gets here */ - /* There's been some horrible disaster. */ + /* There's been some horrible disaster. Since all codes > OP_BRA are + for capturing brackets, and there shouldn't be any gaps between 0 and + OP_BRA, arrival here can only mean there is something seriously wrong + in the code above or the OP_xxx definitions. */ default: DPRINTF(("Unknown opcode %d\n", *ecode)); - md->errorcode = PCRE_ERROR_UNKNOWN_NODE; - return FALSE; + return PCRE_ERROR_UNKNOWN_NODE; } /* Do not stick any code in here without much thought; it is assumed @@ -5690,7 +7253,7 @@ each substring: the offsets to the start and end of the substring. Arguments: external_re points to the compiled expression - external_extra points to "hints" from pcre_study() or is NULL + extra_data points to extra data or is NULL subject points to the subject string length length of subject string (may contain binary zeros) start_offset where to start in the subject string @@ -5705,40 +7268,61 @@ Returns: > 0 => success; value is the number of elements filled in */ int -pcre_exec(const pcre *external_re, const pcre_extra *external_extra, +pcre_exec(const pcre *external_re, const pcre_extra *extra_data, const char *subject, int length, int start_offset, int options, int *offsets, int offsetcount) { -int resetcount, ocount; -int first_char = -1; -int req_char = -1; -int req_char2 = -1; +int rc, resetcount, ocount; +int first_byte = -1; +int req_byte = -1; +int req_byte2 = -1; unsigned long int ims = 0; +BOOL using_temporary_offsets = FALSE; +BOOL anchored; +BOOL startline; +BOOL first_byte_caseless = FALSE; +BOOL req_byte_caseless = FALSE; match_data match_block; const uschar *start_bits = NULL; const uschar *start_match = (const uschar *)subject + start_offset; const uschar *end_subject; -const uschar *req_char_ptr = start_match - 1; +const uschar *req_byte_ptr = start_match - 1; +const pcre_study_data *study; const real_pcre *re = (const real_pcre *)external_re; -const real_pcre_extra *extra = (const real_pcre_extra *)external_extra; -const uschar *codestart = - (const uschar *)re + sizeof(real_pcre) + re->name_count * re->name_entry_size; -BOOL using_temporary_offsets = FALSE; -BOOL anchored; -BOOL startline; -BOOL first_char_caseless = FALSE; -BOOL req_char_caseless = FALSE; -if ((options & ~PUBLIC_EXEC_OPTIONS) != 0) return PCRE_ERROR_BADOPTION; +/* Plausibility checks */ +if ((options & ~PUBLIC_EXEC_OPTIONS) != 0) return PCRE_ERROR_BADOPTION; if (re == NULL || subject == NULL || (offsets == NULL && offsetcount > 0)) return PCRE_ERROR_NULL; + +/* Fish out the optional data from the extra_data structure, first setting +the default values. */ + +study = NULL; +match_block.match_limit = MATCH_LIMIT; +match_block.callout_data = NULL; + +if (extra_data != NULL) + { + register unsigned int flags = extra_data->flags; + if ((flags & PCRE_EXTRA_STUDY_DATA) != 0) + study = extra_data->study_data; + if ((flags & PCRE_EXTRA_MATCH_LIMIT) != 0) + match_block.match_limit = extra_data->match_limit; + if ((flags & PCRE_EXTRA_CALLOUT_DATA) != 0) + match_block.callout_data = extra_data->callout_data; + } + +/* Now we have re supposedly pointing to the regex */ + if (re->magic_number != MAGIC_NUMBER) return PCRE_ERROR_BADMAGIC; anchored = ((re->options | options) & PCRE_ANCHORED) != 0; startline = (re->options & PCRE_STARTLINE) != 0; -match_block.start_code = codestart; +match_block.start_code = + (const uschar *)re + sizeof(real_pcre) + re->name_count * re->name_entry_size; match_block.start_subject = (const uschar *)subject; match_block.start_offset = start_offset; match_block.end_subject = match_block.start_subject + length; @@ -5751,8 +7335,7 @@ match_block.notbol = (options & PCRE_NOTBOL) != 0; match_block.noteol = (options & PCRE_NOTEOL) != 0; match_block.notempty = (options & PCRE_NOTEMPTY) != 0; -match_block.errorcode = PCRE_ERROR_NOMATCH; /* Default error */ -match_block.recursive = NULL; /* No recursion */ +match_block.recursive = NULL; /* No recursion at top level */ match_block.lcc = re->tables + lcc_offset; match_block.ctypes = re->tables + ctypes_offset; @@ -5803,7 +7386,7 @@ if (match_block.offset_vector != NULL) while (--iptr >= iend) *iptr = -1; } -/* Set up the first character to match, if available. The first_char value is +/* Set up the first character to match, if available. The first_byte value is never set for an anchored regular expression, but the anchoring may be forced at run time, so we have to test for anchoring. The first char may be unset for an unanchored pattern, of course. If there's no first char and the pattern was @@ -5813,14 +7396,14 @@ if (!anchored) { if ((re->options & PCRE_FIRSTSET) != 0) { - first_char = re->first_char & 255; - if ((first_char_caseless = ((re->first_char & REQ_CASELESS) != 0)) == TRUE) - first_char = match_block.lcc[first_char]; + first_byte = re->first_byte & 255; + if ((first_byte_caseless = ((re->first_byte & REQ_CASELESS) != 0)) == TRUE) + first_byte = match_block.lcc[first_byte]; } else - if (!startline && extra != NULL && - (extra->options & PCRE_STUDY_MAPPED) != 0) - start_bits = extra->start_bits; + if (!startline && study != NULL && + (study->options & PCRE_STUDY_MAPPED) != 0) + start_bits = study->start_bits; } /* For anchored or unanchored matches, there may be a "last known required @@ -5828,9 +7411,9 @@ character" set. */ if ((re->options & PCRE_REQCHSET) != 0) { - req_char = re->req_char & 255; - req_char_caseless = (re->req_char & REQ_CASELESS) != 0; - req_char2 = (re->tables + fcc_offset)[req_char]; /* case flipped */ + req_byte = re->req_byte & 255; + req_byte_caseless = (re->req_byte & REQ_CASELESS) != 0; + req_byte2 = (re->tables + fcc_offset)[req_byte]; /* case flipped */ } /* Loop for handling unanchored repeated matching attempts; for anchored regexs @@ -5838,7 +7421,6 @@ the loop runs just once. */ do { - int rc; register int *iptr = match_block.offset_vector; register int *iend = iptr + resetcount; @@ -5848,14 +7430,14 @@ do /* Advance to a unique first char if possible */ - if (first_char >= 0) + if (first_byte >= 0) { - if (first_char_caseless) + if (first_byte_caseless) while (start_match < end_subject && - match_block.lcc[*start_match] != first_char) + match_block.lcc[*start_match] != first_byte) start_match++; else - while (start_match < end_subject && *start_match != first_char) + while (start_match < end_subject && *start_match != first_byte) start_match++; } @@ -5887,36 +7469,41 @@ do printf("\n"); #endif - /* If req_char is set, we know that that character must appear in the subject - for the match to succeed. If the first character is set, req_char must be + /* If req_byte is set, we know that that character must appear in the subject + for the match to succeed. If the first character is set, req_byte must be later in the subject; otherwise the test starts at the match point. This optimization can save a huge amount of backtracking in patterns with nested unlimited repeats that aren't going to match. Writing separate code for cased/caseless versions makes it go faster, as does using an autoincrement - and backing off on a match. */ + and backing off on a match. + + HOWEVER: when the subject string is very, very long, searching to its end can + take a long time, and give bad performance on quite ordinary patterns. This + showed up when somebody was matching /^C/ on a 32-megabyte string... so we + don't do this when the string is sufficiently long. */ - if (req_char >= 0) + if (req_byte >= 0 && end_subject - start_match < REQ_BYTE_MAX) { - register const uschar *p = start_match + ((first_char >= 0)? 1 : 0); + register const uschar *p = start_match + ((first_byte >= 0)? 1 : 0); /* We don't need to repeat the search if we haven't yet reached the place we found it at last time. */ - if (p > req_char_ptr) + if (p > req_byte_ptr) { - if (req_char_caseless) + if (req_byte_caseless) { while (p < end_subject) { register int pp = *p++; - if (pp == req_char || pp == req_char2) { p--; break; } + if (pp == req_byte || pp == req_byte2) { p--; break; } } } else { while (p < end_subject) { - if (*p++ == req_char) { p--; break; } + if (*p++ == req_byte) { p--; break; } } } @@ -5928,7 +7515,7 @@ do found it, so that we don't search again next time round the loop if the start hasn't passed this character yet. */ - req_char_ptr = p; + req_byte_ptr = p; } } @@ -5940,10 +7527,29 @@ do if certain parts of the pattern were not used. */ match_block.start_match = start_match; - if (!match(start_match, codestart, 2, &match_block, ims, NULL, match_isgroup)) + match_block.match_call_count = 0; + + rc = match(start_match, match_block.start_code, 2, &match_block, ims, NULL, + match_isgroup); + + if (rc == MATCH_NOMATCH) + { + start_match++; +#ifdef SUPPORT_UTF8 + if (match_block.utf8) + while((*start_match & 0xc0) == 0x80) start_match++; +#endif continue; + } + + if (rc != MATCH_MATCH) + { + DPRINTF((">>>> error: returning %d\n", rc)); + return rc; + } - /* Copy the offset information from temporary store if necessary */ + /* We have a match! Copy the offset information from temporary store if + necessary */ if (using_temporary_offsets) { @@ -5974,9 +7580,7 @@ do /* This "while" is the end of the "do" above */ -while (!anchored && - match_block.errorcode == PCRE_ERROR_NOMATCH && - start_match++ < end_subject); +while (!anchored && start_match <= end_subject); if (using_temporary_offsets) { @@ -5984,9 +7588,9 @@ if (using_temporary_offsets) (pcre_free)(match_block.offset_vector); } -DPRINTF((">>>> returning %d\n", match_block.errorcode)); +DPRINTF((">>>> returning PCRE_ERROR_NOMATCH\n")); -return match_block.errorcode; +return PCRE_ERROR_NOMATCH; } /* End of pcre.c */ diff --git a/ext/pcre/pcrelib/pcre.h b/ext/pcre/pcrelib/pcre.h index b33045c4f3..64e35332a3 100644 --- a/ext/pcre/pcrelib/pcre.h +++ b/ext/pcre/pcrelib/pcre.h @@ -12,20 +12,25 @@ make changes to pcre.in. */ #include "php_compat.h" -#define PCRE_MAJOR 3 -#define PCRE_MINOR 92 -#define PCRE_DATE 11-Sep-2002 +#define PCRE_MAJOR 4 +#define PCRE_MINOR 3 +#define PCRE_DATE 21-May-2003 /* Win32 uses DLL by default */ #ifdef _WIN32 -# ifdef PHP_EXPORTS -# define PCRE_DL_IMPORT -# else -# define PCRE_DL_IMPORT __declspec(dllimport) -# endif -#else -# define PCRE_DL_IMPORT +# ifdef PCRE_DEFINITION +# ifdef DLL_EXPORT +# define PCRE_DATA_SCOPE __declspec(dllexport) +# endif +# else +# ifndef PCRE_STATIC +# define PCRE_DATA_SCOPE __declspec(dllimport) +# endif +# endif +#endif +#ifndef PCRE_DATA_SCOPE +# define PCRE_DATA_SCOPE extern #endif /* Have to include stdlib.h in order to ensure that size_t is defined; @@ -41,20 +46,21 @@ extern "C" { /* Options */ -#define PCRE_CASELESS 0x0001 -#define PCRE_MULTILINE 0x0002 -#define PCRE_DOTALL 0x0004 -#define PCRE_EXTENDED 0x0008 -#define PCRE_ANCHORED 0x0010 -#define PCRE_DOLLAR_ENDONLY 0x0020 -#define PCRE_EXTRA 0x0040 -#define PCRE_NOTBOL 0x0080 -#define PCRE_NOTEOL 0x0100 -#define PCRE_UNGREEDY 0x0200 -#define PCRE_NOTEMPTY 0x0400 -#define PCRE_UTF8 0x0800 - -/* Exec-time and get-time error codes */ +#define PCRE_CASELESS 0x0001 +#define PCRE_MULTILINE 0x0002 +#define PCRE_DOTALL 0x0004 +#define PCRE_EXTENDED 0x0008 +#define PCRE_ANCHORED 0x0010 +#define PCRE_DOLLAR_ENDONLY 0x0020 +#define PCRE_EXTRA 0x0040 +#define PCRE_NOTBOL 0x0080 +#define PCRE_NOTEOL 0x0100 +#define PCRE_UNGREEDY 0x0200 +#define PCRE_NOTEMPTY 0x0400 +#define PCRE_UTF8 0x0800 +#define PCRE_NO_AUTO_CAPTURE 0x1000 + +/* Exec-time and get/set-time error codes */ #define PCRE_ERROR_NOMATCH (-1) #define PCRE_ERROR_NULL (-2) @@ -63,27 +69,52 @@ extern "C" { #define PCRE_ERROR_UNKNOWN_NODE (-5) #define PCRE_ERROR_NOMEMORY (-6) #define PCRE_ERROR_NOSUBSTRING (-7) +#define PCRE_ERROR_MATCHLIMIT (-8) +#define PCRE_ERROR_CALLOUT (-9) /* Never used by PCRE itself */ /* Request types for pcre_fullinfo() */ -#define PCRE_INFO_OPTIONS 0 -#define PCRE_INFO_SIZE 1 -#define PCRE_INFO_CAPTURECOUNT 2 -#define PCRE_INFO_BACKREFMAX 3 -#define PCRE_INFO_FIRSTCHAR 4 -#define PCRE_INFO_FIRSTTABLE 5 -#define PCRE_INFO_LASTLITERAL 6 -#define PCRE_INFO_NAMEENTRYSIZE 7 -#define PCRE_INFO_NAMECOUNT 8 -#define PCRE_INFO_NAMETABLE 9 +#define PCRE_INFO_OPTIONS 0 +#define PCRE_INFO_SIZE 1 +#define PCRE_INFO_CAPTURECOUNT 2 +#define PCRE_INFO_BACKREFMAX 3 +#define PCRE_INFO_FIRSTBYTE 4 +#define PCRE_INFO_FIRSTCHAR 4 /* For backwards compatibility */ +#define PCRE_INFO_FIRSTTABLE 5 +#define PCRE_INFO_LASTLITERAL 6 +#define PCRE_INFO_NAMEENTRYSIZE 7 +#define PCRE_INFO_NAMECOUNT 8 +#define PCRE_INFO_NAMETABLE 9 +#define PCRE_INFO_STUDYSIZE 10 + +/* Request types for pcre_config() */ + +#define PCRE_CONFIG_UTF8 0 +#define PCRE_CONFIG_NEWLINE 1 +#define PCRE_CONFIG_LINK_SIZE 2 +#define PCRE_CONFIG_POSIX_MALLOC_THRESHOLD 3 +#define PCRE_CONFIG_MATCH_LIMIT 4 + +/* Bit flags for the pcre_extra structure */ + +#define PCRE_EXTRA_STUDY_DATA 0x0001 +#define PCRE_EXTRA_MATCH_LIMIT 0x0002 +#define PCRE_EXTRA_CALLOUT_DATA 0x0004 /* Types */ -struct real_pcre; /* declaration; the definition is private */ -struct real_pcre_extra; /* declaration; the definition is private */ - +struct real_pcre; /* declaration; the definition is private */ typedef struct real_pcre pcre; -typedef struct real_pcre_extra pcre_extra; + +/* The structure for passing additional data to pcre_exec(). This is defined in +such as way as to be extensible. */ + +typedef struct pcre_extra { + unsigned long int flags; /* Bits for which fields are set */ + void *study_data; /* Opaque data from pcre_study() */ + unsigned long int match_limit; /* Maximum number of calls to match() */ + void *callout_data; /* Data passed back in callouts */ +} pcre_extra; /* The structure for passing out data via the pcre_callout_function. We use a structure so that new fields can be added on the end in future versions, @@ -101,19 +132,20 @@ typedef struct pcre_callout_block { int current_position; /* Where we currently are */ int capture_top; /* Max current capture */ int capture_last; /* Most recently closed capture */ + void *callout_data; /* Data passed in with the call */ /* ------------------------------------------------------------------ */ } pcre_callout_block; /* Indirection for store get and free functions. These can be set to alternative malloc/free functions if required. There is also an optional callout function that is triggered by the (?) regex item. Some magic is -required for Win32 DLL; it is null on other OS. For Virtual Pascal, these have -to be different again. */ +required for Win32 DLL; it is null on other OS. For Virtual Pascal, these +have to be different again. */ #ifndef VPCOMPAT -PCRE_DL_IMPORT extern void *(*pcre_malloc)(size_t); -PCRE_DL_IMPORT extern void (*pcre_free)(void *); -PCRE_DL_IMPORT extern int (*pcre_callout)(pcre_callout_block *); +PCRE_DATA_SCOPE void *(*pcre_malloc)(size_t); +PCRE_DATA_SCOPE void (*pcre_free)(void *); +PCRE_DATA_SCOPE int (*pcre_callout)(pcre_callout_block *); #else /* VPCOMPAT */ extern void *pcre_malloc(size_t); extern void pcre_free(void *); @@ -122,26 +154,30 @@ extern int pcre_callout(pcre_callout_block *); /* Exported PCRE functions */ -PCRE_DL_IMPORT extern pcre *pcre_compile(const char *, int, const char **, - int *, const unsigned char *); -PCRE_DL_IMPORT extern int pcre_copy_substring(const char *, int *, int, int, - char *, int); -PCRE_DL_IMPORT extern int pcre_exec(const pcre *, const pcre_extra *, - const char *, int, int, int, int *, int); -PCRE_DL_IMPORT extern void pcre_free_substring(const char *); -PCRE_DL_IMPORT extern void pcre_free_substring_list(const char **); -PCRE_DL_IMPORT extern int pcre_get_substring(const char *, int *, int, int, - const char **); -PCRE_DL_IMPORT extern int pcre_get_substring_list(const char *, int *, int, - const char ***); -PCRE_DL_IMPORT extern int pcre_info(const pcre *, int *, int *); -PCRE_DL_IMPORT extern int pcre_fullinfo(const pcre *, const pcre_extra *, int, - void *); -PCRE_DL_IMPORT extern const unsigned char *pcre_maketables(void); -PCRE_DL_IMPORT extern pcre_extra *pcre_study(const pcre *, int, const char **); -PCRE_DL_IMPORT extern const char *pcre_version(void); - -#undef PCRE_DL_IMPORT +extern pcre *pcre_compile(const char *, int, const char **, + int *, const unsigned char *); +extern int pcre_config(int, void *); +extern int pcre_copy_named_substring(const pcre *, const char *, + int *, int, const char *, char *, int); +extern int pcre_copy_substring(const char *, int *, int, int, + char *, int); +extern int pcre_exec(const pcre *, const pcre_extra *, + const char *, int, int, int, int *, int); +extern void pcre_free_substring(const char *); +extern void pcre_free_substring_list(const char **); +extern int pcre_fullinfo(const pcre *, const pcre_extra *, int, + void *); +extern int pcre_get_named_substring(const pcre *, const char *, + int *, int, const char *, const char **); +extern int pcre_get_stringnumber(const pcre *, const char *); +extern int pcre_get_substring(const char *, int *, int, int, + const char **); +extern int pcre_get_substring_list(const char *, int *, int, + const char ***); +extern int pcre_info(const pcre *, int *, int *); +extern const unsigned char *pcre_maketables(void); +extern pcre_extra *pcre_study(const pcre *, int, const char **); +extern const char *pcre_version(void); #ifdef __cplusplus } /* extern "C" */ diff --git a/ext/pcre/pcrelib/pcregrep.c b/ext/pcre/pcrelib/pcregrep.c index 87bb65ccbd..f4a59f4f41 100644 --- a/ext/pcre/pcrelib/pcregrep.c +++ b/ext/pcre/pcrelib/pcregrep.c @@ -19,7 +19,7 @@ directories. */ typedef int BOOL; -#define VERSION "2.2 10-Sep-2003" +#define VERSION "3.0 14-Jan-2003" #define MAX_PATTERN_COUNT 100 @@ -45,8 +45,8 @@ static BOOL whole_lines = FALSE; typedef struct option_item { int one_char; - char *long_name; - char *help_text; + const char *long_name; + const char *help_text; } option_item; static option_item optionlist[] = { @@ -58,6 +58,7 @@ static option_item optionlist[] = { { 'n', "line-number", "print line number with output lines" }, { 'r', "recursive", "recursively scan sub-directories" }, { 's', "no-messages", "suppress error messages" }, + { 'u', "utf-8", "use UTF-8 mode" }, { 'V', "version", "print version information and exit" }, { 'v', "invert-match", "select non-matching lines" }, { 'x', "line-regex", "force PATTERN to match only whole lines" }, @@ -84,7 +85,7 @@ recursion support". */ typedef DIR directory_type; -int +static int isdirectory(char *filename) { struct stat statbuf; @@ -93,13 +94,13 @@ if (stat(filename, &statbuf) < 0) return ((statbuf.st_mode & S_IFMT) == S_IFDIR)? '/' : 0; } -directory_type * +static directory_type * opendirectory(char *filename) { return opendir(filename); } -char * +static char * readdirectory(directory_type *dir) { for (;;) @@ -112,7 +113,7 @@ for (;;) return NULL; /* Keep compiler happy; never executed */ } -void +static void closedirectory(directory_type *dir) { closedir(dir); @@ -319,7 +320,7 @@ return rc; *************************************************/ static int -grep_or_recurse(char *filename, BOOL recurse, BOOL show_filenames, +grep_or_recurse(char *filename, BOOL dir_recurse, BOOL show_filenames, BOOL only_one_at_top) { int rc = 1; @@ -329,7 +330,7 @@ FILE *in; /* If the file is a directory and we are recursing, scan each file within it. The scanning code is localized so it can be made system-specific. */ -if ((sep = isdirectory(filename)) != 0 && recurse) +if ((sep = isdirectory(filename)) != 0 && dir_recurse) { char buffer[1024]; char *nextfile; @@ -346,7 +347,7 @@ if ((sep = isdirectory(filename)) != 0 && recurse) { int frc; sprintf(buffer, "%.512s%c%.128s", filename, sep, nextfile); - frc = grep_or_recurse(buffer, recurse, TRUE, FALSE); + frc = grep_or_recurse(buffer, dir_recurse, TRUE, FALSE); if (frc == 0 && rc == 1) rc = 0; } @@ -446,6 +447,7 @@ switch(letter) case 'n': number = TRUE; break; case 'r': recurse = TRUE; break; case 's': silent = TRUE; break; + case 'u': options |= PCRE_UTF8; break; case 'v': invert = TRUE; break; case 'x': whole_lines = TRUE; options |= PCRE_ANCHORED; break; diff --git a/ext/pcre/pcrelib/pcreposix.c b/ext/pcre/pcrelib/pcreposix.c index 0ed55bb248..49094f280d 100644 --- a/ext/pcre/pcrelib/pcreposix.c +++ b/ext/pcre/pcrelib/pcreposix.c @@ -83,7 +83,7 @@ static int eint[] = { REG_ECTYPE, /* "unknown POSIX class name" */ REG_BADPAT, /* "POSIX collating elements are not supported" */ REG_INVARG, /* "this version of PCRE is not compiled with PCRE_UTF8 support" */ - REG_BADPAT, /* "characters with values > 255 are not yet supported in classes" */ + REG_BADPAT, /* "spare error" */ REG_BADPAT, /* "character value in \x{...} sequence is too large" */ REG_BADPAT, /* "invalid condition (?(0)" */ REG_BADPAT, /* "\\C not allowed in lookbehind assertion" */ @@ -236,7 +236,7 @@ block of store on the stack, to reduce the use of malloc/free. The threshold is in a macro that can be changed at configure time. */ int -regexec(regex_t *preg, const char *string, size_t nmatch, +regexec(const regex_t *preg, const char *string, size_t nmatch, regmatch_t pmatch[], int eflags) { int rc; @@ -248,7 +248,7 @@ BOOL allocated_ovector = FALSE; if ((eflags & REG_NOTBOL) != 0) options |= PCRE_NOTBOL; if ((eflags & REG_NOTEOL) != 0) options |= PCRE_NOTEOL; -preg->re_erroffset = (size_t)(-1); /* Only has meaning after compile */ +((regex_t *)preg)->re_erroffset = (size_t)(-1); /* Only has meaning after compile */ if (nmatch > 0) { diff --git a/ext/pcre/pcrelib/pcreposix.h b/ext/pcre/pcrelib/pcreposix.h index e70af2de84..2b97bf44f5 100644 --- a/ext/pcre/pcrelib/pcreposix.h +++ b/ext/pcre/pcrelib/pcreposix.h @@ -2,7 +2,7 @@ * Perl-Compatible Regular Expressions * *************************************************/ -/* Copyright (c) 1997-2001 University of Cambridge */ +/* Copyright (c) 1997-2003 University of Cambridge */ #ifndef _PCREPOSIX_H #define _PCREPOSIX_H @@ -77,7 +77,7 @@ typedef struct { /* The functions */ extern int regcomp(regex_t *, const char *, int); -extern int regexec(regex_t *, const char *, size_t, regmatch_t *, int); +extern int regexec(const regex_t *, const char *, size_t, regmatch_t *, int); extern size_t regerror(int, const regex_t *, char *, size_t); extern void regfree(regex_t *); diff --git a/ext/pcre/pcrelib/pcretest.c b/ext/pcre/pcrelib/pcretest.c index 8977ef7f66..ad729b7cfc 100644 --- a/ext/pcre/pcrelib/pcretest.c +++ b/ext/pcre/pcrelib/pcretest.c @@ -16,6 +16,7 @@ been extended and consequently is now rather untidy in places. */ /* We need the internal info for displaying the results of pcre_study(). Also for getting the opcodes for showing compiled code. */ +#define PCRE_SPY /* For Win32 build, import data, not export */ #include "internal.h" /* It is possible to compile this test program without including support for @@ -36,6 +37,9 @@ Makefile. */ #define LOOPREPEAT 50000 +#define BUFFER_SIZE 30000 +#define DBUFFER_SIZE 1024 + static FILE *outfile; static int log_store = 0; @@ -44,18 +48,18 @@ static int callout_extra; static int callout_fail_count; static int callout_fail_id; static int first_callout; -static int utf8; +static int use_utf8; static size_t gotten_store; -static int utf8_table1[] = { +static const int utf8_table1[] = { 0x0000007f, 0x000007ff, 0x0000ffff, 0x001fffff, 0x03ffffff, 0x7fffffff}; -static int utf8_table2[] = { +static const int utf8_table2[] = { 0, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc}; -static int utf8_table3[] = { +static const int utf8_table3[] = { 0xff, 0x1f, 0x0f, 0x07, 0x03, 0x01}; @@ -154,7 +158,7 @@ Returns: > 0 => the number of bytes consumed -6 to 0 => malformed UTF-8 character at offset = (-return) */ -int +static int utf82ord(unsigned char *buffer, int *vptr) { int c = *buffer++; @@ -212,7 +216,7 @@ int yield = 0; while (length-- > 0) { - if (utf8) + if (use_utf8) { int rc = utf82ord(p, &c); @@ -259,7 +263,8 @@ return yield; *************************************************/ /* Called from PCRE as a result of the (?C) item. We print out where we are in -the match. Yield OK unless more callouts than the fail count. . */ +the match. Yield zero unless more callouts than the fail count, or the callout +data is not zero. */ static int callout(pcre_callout_block *cb) { @@ -268,7 +273,6 @@ int i, pre_start, post_start; if (callout_extra) { - int i; fprintf(f, "Callout %d: last capture = %d\n", cb->callout_number, cb->capture_last); @@ -320,6 +324,12 @@ fprintf(outfile, "\n"); first_callout = 0; +if ((int)(cb->callout_data) != 0) + { + fprintf(outfile, "Callout data = %d\n", (int)(cb->callout_data)); + return (int)(cb->callout_data); + } + return (cb->callout_number != callout_fail_id)? 0 : (++callout_count >= callout_fail_count)? 1 : 0; } @@ -380,8 +390,15 @@ int posix = 0; #endif int debug = 0; int done = 0; -unsigned char buffer[30000]; -unsigned char dbuffer[1024]; + +unsigned char *buffer; +unsigned char *dbuffer; + +/* Get buffers from malloc() so that Electric Fence will check their misuse +when I am debugging. */ + +buffer = malloc(BUFFER_SIZE); +dbuffer = malloc(DBUFFER_SIZE); /* Static so that new_malloc can use it. */ @@ -399,7 +416,8 @@ while (argc > 1 && argv[op][0] == '-') else if (strcmp(argv[op], "-i") == 0) showinfo = 1; else if (strcmp(argv[op], "-d") == 0) showinfo = debug = 1; else if (strcmp(argv[op], "-o") == 0 && argc > 2 && - ((size_offsets = get_value(argv[op+1], &endptr)), *endptr == 0)) + ((size_offsets = get_value((unsigned char *)argv[op+1], &endptr)), + *endptr == 0)) { op++; argc--; @@ -407,10 +425,28 @@ while (argc > 1 && argv[op][0] == '-') #if !defined NOPOSIX else if (strcmp(argv[op], "-p") == 0) posix = 1; #endif + else if (strcmp(argv[op], "-C") == 0) + { + int rc; + printf("PCRE version %s\n", pcre_version()); + printf("Compiled with\n"); + (void)pcre_config(PCRE_CONFIG_UTF8, &rc); + printf(" %sUTF-8 support\n", rc? "" : "No "); + (void)pcre_config(PCRE_CONFIG_NEWLINE, &rc); + printf(" Newline character is %s\n", (rc == '\r')? "CR" : "LF"); + (void)pcre_config(PCRE_CONFIG_LINK_SIZE, &rc); + printf(" Internal link size = %d\n", rc); + (void)pcre_config(PCRE_CONFIG_POSIX_MALLOC_THRESHOLD, &rc); + printf(" POSIX malloc threshold = %d\n", rc); + (void)pcre_config(PCRE_CONFIG_MATCH_LIMIT, &rc); + printf(" Default match limit = %d\n", rc); + exit(0); + } else { printf("** Unknown or malformed option %s\n", argv[op]); printf("Usage: pcretest [-d] [-i] [-o <n>] [-p] [-s] [-t] [<input> [<output>]]\n"); + printf(" -C show PCRE compile-time options and exit\n"); printf(" -d debug: show compiled code; implies -i\n" " -i show information about compiled pattern\n" " -o <n> set size of offsets vector to <n>\n"); @@ -489,10 +525,10 @@ while (!done) int do_showrest = 0; int erroroffset, len, delimiter; - utf8 = 0; + use_utf8 = 0; if (infile == stdin) printf(" re> "); - if (fgets((char *)buffer, sizeof(buffer), infile) == NULL) break; + if (fgets((char *)buffer, BUFFER_SIZE, infile) == NULL) break; if (infile != stdin) fprintf(outfile, "%s", (char *)buffer); fflush(outfile); @@ -523,7 +559,7 @@ while (!done) } if (*pp != 0) break; - len = sizeof(buffer) - (pp - buffer); + len = BUFFER_SIZE - (pp - buffer); if (len < 256) { fprintf(outfile, "** Expression too long - missing delimiter?\n"); @@ -573,6 +609,7 @@ while (!done) case 'G': do_G = 1; break; case 'I': do_showinfo = 1; break; case 'M': log_store = 1; break; + case 'N': options |= PCRE_NO_AUTO_CAPTURE; break; #if !defined NOPOSIX case 'P': do_posix = 1; break; @@ -581,7 +618,7 @@ while (!done) case 'S': do_study = 1; break; case 'U': options |= PCRE_UNGREEDY; break; case 'X': options |= PCRE_EXTRA; break; - case '8': options |= PCRE_UTF8; utf8 = 1; break; + case '8': options |= PCRE_UTF8; use_utf8 = 1; break; case 'L': ppp = pp; @@ -621,7 +658,7 @@ while (!done) if (rc != 0) { - (void)regerror(rc, &preg, (char *)buffer, sizeof(buffer)); + (void)regerror(rc, &preg, (char *)buffer, BUFFER_SIZE); fprintf(outfile, "Failed: POSIX code %d: %s\n", rc, buffer); goto SKIP_DATA; } @@ -662,7 +699,7 @@ while (!done) { for (;;) { - if (fgets((char *)buffer, sizeof(buffer), infile) == NULL) + if (fgets((char *)buffer, BUFFER_SIZE, infile) == NULL) { done = 1; goto CONTINUE; @@ -705,11 +742,11 @@ while (!done) new_info(re, NULL, PCRE_INFO_SIZE, &size); new_info(re, NULL, PCRE_INFO_CAPTURECOUNT, &count); new_info(re, NULL, PCRE_INFO_BACKREFMAX, &backrefmax); - new_info(re, NULL, PCRE_INFO_FIRSTCHAR, &first_char); + new_info(re, NULL, PCRE_INFO_FIRSTBYTE, &first_char); new_info(re, NULL, PCRE_INFO_LASTLITERAL, &need_char); new_info(re, NULL, PCRE_INFO_NAMEENTRYSIZE, &nameentrysize); new_info(re, NULL, PCRE_INFO_NAMECOUNT, &namecount); - new_info(re, NULL, PCRE_INFO_NAMETABLE, &nametable); + new_info(re, NULL, PCRE_INFO_NAMETABLE, (void *)&nametable); old_count = pcre_info(re, &old_options, &old_first_char); if (count < 0) fprintf(outfile, @@ -775,7 +812,7 @@ while (!done) else { int ch = first_char & 255; - char *caseless = ((first_char & REQ_CASELESS) == 0)? + const char *caseless = ((first_char & REQ_CASELESS) == 0)? "" : " (caseless)"; if (isprint(ch)) fprintf(outfile, "First char = \'%c\'%s\n", ch, caseless); @@ -790,9 +827,9 @@ while (!done) else { int ch = need_char & 255; - char *caseless = ((need_char & REQ_CASELESS) == 0)? + const char *caseless = ((need_char & REQ_CASELESS) == 0)? "" : " (caseless)"; - if (isprint(need_char)) + if (isprint(ch)) fprintf(outfile, "Need char = \'%c\'%s\n", ch, caseless); else fprintf(outfile, "Need char = %d%s\n", ch, caseless); @@ -826,8 +863,11 @@ while (!done) else if (do_showinfo) { + size_t size; uschar *start_bits = NULL; + new_info(re, extra, PCRE_INFO_STUDYSIZE, &size); new_info(re, extra, PCRE_INFO_FIRSTTABLE, &start_bits); + fprintf(outfile, "Study size = %d\n", size); if (start_bits == NULL) fprintf(outfile, "No starting character set\n"); else @@ -870,8 +910,11 @@ while (!done) unsigned char *bptr = dbuffer; int *use_offsets = offsets; int use_size_offsets = size_offsets; + int callout_data = 0; + int callout_data_set = 0; int count, c; int copystrings = 0; + int find_match_limit = 0; int getstrings = 0; int getlist = 0; int gmatched = 0; @@ -888,7 +931,7 @@ while (!done) callout_fail_id = -1; if (infile == stdin) printf("data> "); - if (fgets((char *)buffer, sizeof(buffer), infile) == NULL) + if (fgets((char *)buffer, BUFFER_SIZE, infile) == NULL) { done = 1; goto CONTINUE; @@ -939,11 +982,11 @@ while (!done) c = c * 16 + tolower(*pt) - ((isdigit(*pt))? '0' : 'W'); if (*pt == '}') { - unsigned char buffer[8]; + unsigned char buff8[8]; int ii, utn; - utn = ord2utf8(c, buffer); - for (ii = 0; ii < utn - 1; ii++) *q++ = buffer[ii]; - c = buffer[ii]; /* Last byte */ + utn = ord2utf8(c, buff8); + for (ii = 0; ii < utn - 1; ii++) *q++ = buff8[ii]; + c = buff8[ii]; /* Last byte */ p = pt + 1; break; } @@ -978,6 +1021,17 @@ while (!done) while(isdigit(*p)) n = n * 10 + *p++ - '0'; copystrings |= 1 << n; } + else if (isalnum(*p)) + { + uschar name[256]; + uschar *npp = name; + while (isalnum(*p)) *npp++ = *p++; + *npp = 0; + n = pcre_get_stringnumber(re, (char *)name); + if (n < 0) + fprintf(outfile, "no parentheses with name \"%s\"\n", name); + else copystrings |= 1 << n; + } else if (*p == '+') { callout_extra = 1; @@ -1002,17 +1056,45 @@ while (!done) callout_fail_count = callout_fail_count * 10 + *p++ - '0'; } } + else if (*p == '*') + { + int sign = 1; + callout_data = 0; + if (*(++p) == '-') { sign = -1; p++; } + while(isdigit(*p)) + callout_data = callout_data * 10 + *p++ - '0'; + callout_data *= sign; + callout_data_set = 1; + } continue; case 'G': - while(isdigit(*p)) n = n * 10 + *p++ - '0'; - getstrings |= 1 << n; + if (isdigit(*p)) + { + while(isdigit(*p)) n = n * 10 + *p++ - '0'; + getstrings |= 1 << n; + } + else if (isalnum(*p)) + { + uschar name[256]; + uschar *npp = name; + while (isalnum(*p)) *npp++ = *p++; + *npp = 0; + n = pcre_get_stringnumber(re, (char *)name); + if (n < 0) + fprintf(outfile, "no parentheses with name \"%s\"\n", name); + else getstrings |= 1 << n; + } continue; case 'L': getlist = 1; continue; + case 'M': + find_match_limit = 1; + continue; + case 'N': options |= PCRE_NOTEMPTY; continue; @@ -1045,14 +1127,16 @@ while (!done) len = q - dbuffer; /* Handle matching via the POSIX interface, which does not - support timing. */ + support timing or playing with the match limit or callout data. */ #if !defined NOPOSIX if (posix || do_posix) { int rc; int eflags = 0; - regmatch_t *pmatch = malloc(sizeof(regmatch_t) * use_size_offsets); + regmatch_t *pmatch = NULL; + if (use_size_offsets > 0) + pmatch = malloc(sizeof(regmatch_t) * use_size_offsets); if ((options & PCRE_NOTBOL) != 0) eflags |= REG_NOTBOL; if ((options & PCRE_NOTEOL) != 0) eflags |= REG_NOTEOL; @@ -1060,7 +1144,7 @@ while (!done) if (rc != 0) { - (void)regerror(rc, &preg, (char *)buffer, sizeof(buffer)); + (void)regerror(rc, &preg, (char *)buffer, BUFFER_SIZE); fprintf(outfile, "No match: POSIX code %d: %s\n", rc, buffer); } else @@ -1108,7 +1192,70 @@ while (!done) (double)CLOCKS_PER_SEC); } - count = pcre_exec(re, extra, (char *)bptr, len, + /* If find_match_limit is set, we want to do repeated matches with + varying limits in order to find the minimum value. */ + + if (find_match_limit) + { + int min = 0; + int mid = 64; + int max = -1; + + if (extra == NULL) + { + extra = malloc(sizeof(pcre_extra)); + extra->flags = 0; + } + extra->flags |= PCRE_EXTRA_MATCH_LIMIT; + + for (;;) + { + extra->match_limit = mid; + count = pcre_exec(re, extra, (char *)bptr, len, start_offset, + options | g_notempty, use_offsets, use_size_offsets); + if (count == PCRE_ERROR_MATCHLIMIT) + { + /* fprintf(outfile, "Testing match limit = %d\n", mid); */ + min = mid; + mid = (mid == max - 1)? max : (max > 0)? (min + max)/2 : mid*2; + } + else if (count >= 0 || count == PCRE_ERROR_NOMATCH) + { + if (mid == min + 1) + { + fprintf(outfile, "Minimum match limit = %d\n", mid); + break; + } + /* fprintf(outfile, "Testing match limit = %d\n", mid); */ + max = mid; + mid = (min + mid)/2; + } + else break; /* Some other error */ + } + + extra->flags &= ~PCRE_EXTRA_MATCH_LIMIT; + } + + /* If callout_data is set, use the interface with additional data */ + + else if (callout_data_set) + { + if (extra == NULL) + { + extra = malloc(sizeof(pcre_extra)); + extra->flags = 0; + } + extra->flags |= PCRE_EXTRA_CALLOUT_DATA; + extra->callout_data = (void *)callout_data; + count = pcre_exec(re, extra, (char *)bptr, len, start_offset, + options | g_notempty, use_offsets, use_size_offsets); + extra->flags &= ~PCRE_EXTRA_CALLOUT_DATA; + } + + /* The normal case is just to do the match once, with the default + value of match_limit. */ + + else count = pcre_exec(re, extra, (char *)bptr, len, start_offset, options | g_notempty, use_offsets, use_size_offsets); if (count == 0) diff --git a/ext/pcre/pcrelib/study.c b/ext/pcre/pcrelib/study.c index 65771b9d74..4320bd23d0 100644 --- a/ext/pcre/pcrelib/study.c +++ b/ext/pcre/pcrelib/study.c @@ -9,7 +9,7 @@ the file Tech.Notes for some information on the internals. Written by: Philip Hazel <ph10@cam.ac.uk> - Copyright (c) 1997-2003 University of Cambridge + Copyright (c) 1997-2002 University of Cambridge ----------------------------------------------------------------------------- Permission is granted to anyone to use this software for any purpose on any @@ -78,6 +78,7 @@ Arguments: code points to an expression start_bits points to a 32-byte table, initialized to 0 caseless the current state of the caseless flag + utf8 TRUE if in UTF-8 mode cd the block with char table pointers Returns: TRUE if table built, FALSE otherwise @@ -85,7 +86,7 @@ Returns: TRUE if table built, FALSE otherwise static BOOL set_start_bits(const uschar *code, uschar *start_bits, BOOL caseless, - compile_data *cd) + BOOL utf8, compile_data *cd) { register int c; @@ -109,7 +110,7 @@ do if ((int)*tcode >= OP_BRA || *tcode == OP_ASSERT) { - if (!set_start_bits(tcode, start_bits, caseless, cd)) + if (!set_start_bits(tcode, start_bits, caseless, utf8, cd)) return FALSE; try_next = FALSE; } @@ -151,7 +152,7 @@ do case OP_BRAZERO: case OP_BRAMINZERO: - if (!set_start_bits(++tcode, start_bits, caseless, cd)) + if (!set_start_bits(++tcode, start_bits, caseless, utf8, cd)) return FALSE; dummy = 1; do tcode += GET(tcode,1); while (*tcode == OP_ALT); @@ -166,6 +167,9 @@ do case OP_MINQUERY: set_bit(start_bits, tcode[1], caseless, cd); tcode += 2; +#ifdef SUPPORT_UTF8 + if (utf8) while ((*tcode & 0xc0) == 0x80) tcode++; +#endif break; /* Single-char upto sets the bit and tries the next */ @@ -174,6 +178,9 @@ do case OP_MINUPTO: set_bit(start_bits, tcode[3], caseless, cd); tcode += 4; +#ifdef SUPPORT_UTF8 + if (utf8) while ((*tcode & 0xc0) == 0x80) tcode++; +#endif break; /* At least one single char sets the bit and stops */ @@ -287,8 +294,17 @@ do tcode += 2; break; - /* Character class: set the bits and either carry on or not, - according to the repeat count. */ + /* Character class where all the information is in a bit map: set the + bits and either carry on or not, according to the repeat count. If it was + a negative class, and we are operating with UTF-8 characters, any byte + with the top-bit set is a potentially valid starter because it may start + a character with a value > 255. (This is sub-optimal in that the + character may be in the range 128-255, and those characters might be + unwanted, but that's as far as we go for the moment.) */ + + case OP_NCLASS: + if (utf8) memset(start_bits+16, 0xff, 16); + /* Fall through */ case OP_CLASS: { @@ -315,7 +331,7 @@ do break; } } - break; /* End of class handling */ + break; /* End of bitmap class handling */ } /* End of switch */ } /* End of try_next loop */ @@ -342,7 +358,8 @@ Arguments: errorptr points to where to place error messages; set NULL unless error -Returns: pointer to a pcre_extra block, +Returns: pointer to a pcre_extra block, with study_data filled in and the + appropriate flag set; NULL on error or if no optimization possible */ @@ -350,7 +367,8 @@ pcre_extra * pcre_study(const pcre *external_re, int options, const char **errorptr) { uschar start_bits[32]; -real_pcre_extra *extra; +pcre_extra *extra; +pcre_study_data *study; const real_pcre *re = (const real_pcre *)external_re; uschar *code = (uschar *)re + sizeof(real_pcre) + (re->name_count * re->name_entry_size); @@ -388,11 +406,17 @@ compile_block.ctypes = re->tables + ctypes_offset; memset(start_bits, 0, 32 * sizeof(uschar)); if (!set_start_bits(code, start_bits, (re->options & PCRE_CASELESS) != 0, - &compile_block)) return NULL; + (re->options & PCRE_UTF8) != 0, &compile_block)) return NULL; -/* Get an "extra" block and put the information therein. */ +/* Get a pcre_extra block and a pcre_study_data block. The study data is put in +the latter, which is pointed to by the former, which may also get additional +data set later by the calling program. At the moment, the size of +pcre_study_data is fixed. We nevertheless save it in a field for returning via +the pcre_fullinfo() function so that if it becomes variable in the future, we +don't have to change that code. */ -extra = (real_pcre_extra *)(pcre_malloc)(sizeof(real_pcre_extra)); +extra = (pcre_extra *)(pcre_malloc) + (sizeof(pcre_extra) + sizeof(pcre_study_data)); if (extra == NULL) { @@ -400,10 +424,15 @@ if (extra == NULL) return NULL; } -extra->options = PCRE_STUDY_MAPPED; -memcpy(extra->start_bits, start_bits, sizeof(start_bits)); +study = (pcre_study_data *)((char *)extra + sizeof(pcre_extra)); +extra->flags = PCRE_EXTRA_STUDY_DATA; +extra->study_data = study; + +study->size = sizeof(pcre_study_data); +study->options = PCRE_STUDY_MAPPED; +memcpy(study->start_bits, start_bits, sizeof(start_bits)); -return (pcre_extra *)extra; +return extra; } /* End of study.c */ |