diff options
| -rw-r--r-- | NEWS | 3 | ||||
| -rw-r--r-- | src/dfa.c | 61 | ||||
| -rw-r--r-- | tests/Makefile.am | 1 | ||||
| -rw-r--r-- | tests/word-multibyte | 23 |
4 files changed, 67 insertions, 21 deletions
@@ -21,6 +21,9 @@ GNU grep NEWS -*- outline -*- ** Bug fixes + grep no longer mishandles patterns that contain \w or \W in multibyte + locales. + grep would fail to count newlines internally when operating in non-UTF8 multibyte locales, leading it to print potentially many lines that did not match. E.g., the command, "seq 10 | env LC_ALL=zh_CN src/grep -n .." @@ -1249,6 +1249,20 @@ parse_bracket_exp (void) return CSET + charclass_index (ccl); } +#define PUSH_LEX_STATE(s) \ + do \ + { \ + char const *lexptr_saved = lexptr; \ + size_t lexleft_saved = lexleft; \ + lexptr = (s); \ + lexleft = strlen (lexptr) + +#define POP_LEX_STATE() \ + lexptr = lexptr_saved; \ + lexleft = lexleft_saved; \ + } \ + while (0) + static token lex (void) { @@ -1496,20 +1510,6 @@ lex (void) return lasttok = CSET + charclass_index (ccl); } -#define PUSH_LEX_STATE(s) \ - do \ - { \ - char const *lexptr_saved = lexptr; \ - size_t lexleft_saved = lexleft; \ - lexptr = (s); \ - lexleft = strlen (lexptr) - -#define POP_LEX_STATE() \ - lexptr = lexptr_saved; \ - lexleft = lexleft_saved; \ - } \ - while (0) - /* FIXME: see if optimizing this, as is done with ANYCHAR and add_utf8_anychar, makes sense. */ @@ -1529,14 +1529,33 @@ lex (void) case 'W': if (!backslash || (syntax_bits & RE_NO_GNU_OPS)) goto normal_char; - zeroset (ccl); - for (c2 = 0; c2 < NOTCHAR; ++c2) - if (IS_WORD_CONSTITUENT (c2)) - setbit (c2, ccl); - if (c == 'W') - notset (ccl); + + if (!dfa->multibyte) + { + zeroset (ccl); + for (c2 = 0; c2 < NOTCHAR; ++c2) + if (IS_WORD_CONSTITUENT (c2)) + setbit (c2, ccl); + if (c == 'W') + notset (ccl); + laststart = false; + return lasttok = CSET + charclass_index (ccl); + } + + /* FIXME: see if optimizing this, as is done with ANYCHAR and + add_utf8_anychar, makes sense. */ + + /* \w and \W are documented to be equivalent to [_[:alnum:]] and + [^_[:alnum:]] respectively, so tell the lexer to process those + strings, each minus its "already processed" '['. */ + PUSH_LEX_STATE (c == 'w' ? "_[:alnum:]]" : "^_[:alnum:]]"); + + lasttok = parse_bracket_exp (); + + POP_LEX_STATE (); + laststart = false; - return lasttok = CSET + charclass_index (ccl); + return lasttok; case '[': if (backslash) diff --git a/tests/Makefile.am b/tests/Makefile.am index f6f051c1..c006e582 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -114,6 +114,7 @@ TESTS = \ warn-char-classes \ word-delim-multibyte \ word-multi-file \ + word-multibyte \ yesno EXTRA_DIST = \ diff --git a/tests/word-multibyte b/tests/word-multibyte new file mode 100644 index 00000000..e067a374 --- /dev/null +++ b/tests/word-multibyte @@ -0,0 +1,23 @@ +#!/bin/sh +# This would fail for grep-2.20 +. "${srcdir=.}/init.sh"; path_prepend_ ../src + +require_en_utf8_locale_ + +printf '\xc3\xa1\n' > in || framework_failure_ +LC_ALL=en_US.UTF-8 +export LC_ALL + +fail=0 + +for LOC in en_US.UTF-8 zh_CN $LOCALE_FR_UTF8; do + out=out1-$LOC + LC_ALL=$LOC grep '\w' in >$out || fail=1 + compare in $out || fail=1 + + out=out2-$LOC + LC_ALL=$LOC grep '\W' in >$out && fail=1 + compare /dev/null $out || fail=1 +done + +Exit $fail |