diff options
| -rw-r--r-- | NEWS | 2 | ||||
| -rw-r--r-- | enc/shift_jis.c | 525 | ||||
| -rw-r--r-- | enc/windows_31j.c | 4 | ||||
| -rw-r--r-- | include/ruby/onigmo.h | 2 | ||||
| -rw-r--r-- | regexec.c | 45 | ||||
| -rw-r--r-- | regparse.c | 70 | ||||
| -rwxr-xr-x | tool/enc-unicode.rb | 21 |
7 files changed, 79 insertions, 590 deletions
@@ -99,7 +99,7 @@ with all sufficient information, see the ChangeLog file or Redmine [Feature #7688] * Regexp - * Update to Onigmo 6.1.1. + * Update to Onigmo 6.1.3-669ac9997619954c298da971fcfacccf36909d05. * Support absent operator https://github.com/k-takata/Onigmo/issues/82 * String diff --git a/enc/shift_jis.c b/enc/shift_jis.c index eacca9a5db..65fd5102de 100644 --- a/enc/shift_jis.c +++ b/enc/shift_jis.c @@ -1,9 +1,9 @@ /********************************************************************** - sjis.c - Onigmo (Oniguruma-mod) (regular expression library) + shift_jis.c - Onigmo (Oniguruma-mod) (regular expression library) **********************************************************************/ /*- - * Copyright (c) 2002-2008 K.Kosako <sndgk393 AT ybb DOT ne DOT jp> - * Copyright (c) 2011 K.Takata <kentkt AT csc DOT jp> + * Copyright (c) 2002-2009 K.Kosako <sndgk393 AT ybb DOT ne DOT jp> + * Copyright (c) 2011-2017 K.Takata <kentkt AT csc DOT jp> * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -28,524 +28,8 @@ * SUCH DAMAGE. */ -#include "regenc.h" +#include "shift_jis.h" -static const int EncLen_SJIS[] = { - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1 -}; - -static const char SJIS_CAN_BE_TRAIL_TABLE[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0 -}; - -static const OnigPairCaseFoldCodes CaseFoldMap[] = { - /* Fullwidth Alphabet */ - { 0x8260, 0x8281 }, - { 0x8261, 0x8282 }, - { 0x8262, 0x8283 }, - { 0x8263, 0x8284 }, - { 0x8264, 0x8285 }, - { 0x8265, 0x8286 }, - { 0x8266, 0x8287 }, - { 0x8267, 0x8288 }, - { 0x8268, 0x8289 }, - { 0x8269, 0x828a }, - { 0x826a, 0x828b }, - { 0x826b, 0x828c }, - { 0x826c, 0x828d }, - { 0x826d, 0x828e }, - { 0x826e, 0x828f }, - { 0x826f, 0x8290 }, - { 0x8270, 0x8291 }, - { 0x8271, 0x8292 }, - { 0x8272, 0x8293 }, - { 0x8273, 0x8294 }, - { 0x8274, 0x8295 }, - { 0x8275, 0x8296 }, - { 0x8276, 0x8297 }, - { 0x8277, 0x8298 }, - { 0x8278, 0x8299 }, - { 0x8279, 0x829a }, - - /* Greek */ - { 0x839f, 0x83bf }, - { 0x83a0, 0x83c0 }, - { 0x83a1, 0x83c1 }, - { 0x83a2, 0x83c2 }, - { 0x83a3, 0x83c3 }, - { 0x83a4, 0x83c4 }, - { 0x83a5, 0x83c5 }, - { 0x83a6, 0x83c6 }, - { 0x83a7, 0x83c7 }, - { 0x83a8, 0x83c8 }, - { 0x83a9, 0x83c9 }, - { 0x83aa, 0x83ca }, - { 0x83ab, 0x83cb }, - { 0x83ac, 0x83cc }, - { 0x83ad, 0x83cd }, - { 0x83ae, 0x83ce }, - { 0x83af, 0x83cf }, - { 0x83b0, 0x83d0 }, - { 0x83b1, 0x83d1 }, - { 0x83b2, 0x83d2 }, - { 0x83b3, 0x83d3 }, - { 0x83b4, 0x83d4 }, - { 0x83b5, 0x83d5 }, - { 0x83b6, 0x83d6 }, - - /* Cyrillic */ - { 0x8440, 0x8470 }, - { 0x8441, 0x8471 }, - { 0x8442, 0x8472 }, - { 0x8443, 0x8473 }, - { 0x8444, 0x8474 }, - { 0x8445, 0x8475 }, - { 0x8446, 0x8476 }, - { 0x8447, 0x8477 }, - { 0x8448, 0x8478 }, - { 0x8449, 0x8479 }, - { 0x844a, 0x847a }, - { 0x844b, 0x847b }, - { 0x844c, 0x847c }, - { 0x844d, 0x847d }, - { 0x844e, 0x847e }, - { 0x844f, 0x8480 }, - { 0x8450, 0x8481 }, - { 0x8451, 0x8482 }, - { 0x8452, 0x8483 }, - { 0x8453, 0x8484 }, - { 0x8454, 0x8485 }, - { 0x8455, 0x8486 }, - { 0x8456, 0x8487 }, - { 0x8457, 0x8488 }, - { 0x8458, 0x8489 }, - { 0x8459, 0x848a }, - { 0x845a, 0x848b }, - { 0x845b, 0x848c }, - { 0x845c, 0x848d }, - { 0x845d, 0x848e }, - { 0x845e, 0x848f }, - { 0x845f, 0x8490 }, - { 0x8460, 0x8491 }, -}; - -#define SJIS_ISMB_FIRST(byte) (EncLen_SJIS[byte] > 1) -#define SJIS_ISMB_TRAIL(byte) SJIS_CAN_BE_TRAIL_TABLE[(byte)] - -typedef enum { FAILURE = -2, ACCEPT = -1, S0 = 0, S1 } state_t; -#define A ACCEPT -#define F FAILURE -static const signed char trans[][0x100] = { - { /* S0 0 1 2 3 4 5 6 7 8 9 a b c d e f */ - /* 0 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, - /* 1 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, - /* 2 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, - /* 3 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, - /* 4 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, - /* 5 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, - /* 6 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, - /* 7 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, - /* 8 */ F, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - /* 9 */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - /* a */ F, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, - /* b */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, - /* c */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, - /* d */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, - /* e */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - /* f */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, F, F, F - }, - { /* S1 0 1 2 3 4 5 6 7 8 9 a b c d e f */ - /* 0 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, - /* 1 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, - /* 2 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, - /* 3 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, - /* 4 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, - /* 5 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, - /* 6 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, - /* 7 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, F, - /* 8 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, - /* 9 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, - /* a */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, - /* b */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, - /* c */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, - /* d */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, - /* e */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, - /* f */ A, A, A, A, A, A, A, A, A, A, A, A, A, F, F, F - } -}; -#undef A -#undef F - -static int -mbc_enc_len(const UChar* p, const UChar* e, OnigEncoding enc ARG_UNUSED) -{ - int firstbyte = *p++; - state_t s; - s = trans[0][firstbyte]; - if (s < 0) return s == ACCEPT ? ONIGENC_CONSTRUCT_MBCLEN_CHARFOUND(1) : - ONIGENC_CONSTRUCT_MBCLEN_INVALID(); - if (p == e) return ONIGENC_CONSTRUCT_MBCLEN_NEEDMORE(EncLen_SJIS[firstbyte]-1); - s = trans[s][*p++]; - return s == ACCEPT ? ONIGENC_CONSTRUCT_MBCLEN_CHARFOUND(2) : - ONIGENC_CONSTRUCT_MBCLEN_INVALID(); -} - -static int -code_to_mbclen(OnigCodePoint code, OnigEncoding enc ARG_UNUSED) -{ - if (code < 256) { - if (EncLen_SJIS[(int )code] == 1) - return 1; - else - return ONIGERR_INVALID_CODE_POINT_VALUE; - } - else if (code <= 0xffff) { - int low = code & 0xff; - if (! SJIS_ISMB_TRAIL(low)) - return ONIGERR_INVALID_CODE_POINT_VALUE; - return 2; - } - else - return ONIGERR_TOO_BIG_WIDE_CHAR_VALUE; -} - -static OnigCodePoint -mbc_to_code(const UChar* p, const UChar* end, OnigEncoding enc) -{ - int c, i, len; - OnigCodePoint n; - - len = mbc_enc_len(p, end, enc); - c = *p++; - n = c; - if (len == 1) return n; - - for (i = 1; i < len; i++) { - if (p >= end) break; - c = *p++; - n <<= 8; n += c; - } - return n; -} - -static int -code_to_mbc(OnigCodePoint code, UChar *buf, OnigEncoding enc) -{ - UChar *p = buf; - - if ((code & 0xff00) != 0) *p++ = (UChar )(((code >> 8) & 0xff)); - *p++ = (UChar )(code & 0xff); - -#if 0 - if (mbc_enc_len(buf, p, enc) != (p - buf)) - return REGERR_INVALID_CODE_POINT_VALUE; -#endif - return (int )(p - buf); -} - -static int -apply_all_case_fold(OnigCaseFoldType flag, - OnigApplyAllCaseFoldFunc f, void* arg, OnigEncoding enc) -{ - return onigenc_apply_all_case_fold_with_map( - numberof(CaseFoldMap), CaseFoldMap, 0, - flag, f, arg); -} - -static OnigCodePoint -get_lower_case(OnigCodePoint code) -{ - if (ONIGENC_IS_IN_RANGE(code, 0x8260, 0x8279)) { - /* Fullwidth Alphabet */ - return (OnigCodePoint )(code + 0x0021); - } - else if (ONIGENC_IS_IN_RANGE(code, 0x839f, 0x83b6)) { - /* Greek */ - return (OnigCodePoint )(code + 0x0020); - } - else if (ONIGENC_IS_IN_RANGE(code, 0x8440, 0x8460)) { - /* Cyrillic */ - int d = (code >= 0x844f) ? 1 : 0; - return (OnigCodePoint )(code + (0x0030 + d)); - } - return code; -} - -static OnigCodePoint -get_upper_case(OnigCodePoint code) -{ - if (ONIGENC_IS_IN_RANGE(code, 0x8281, 0x829a)) { - /* Fullwidth Alphabet */ - return (OnigCodePoint )(code - 0x0021); - } - else if (ONIGENC_IS_IN_RANGE(code, 0x83bf, 0x83d6)) { - /* Greek */ - return (OnigCodePoint )(code - 0x0020); - } - else if (ONIGENC_IS_IN_RANGE(code, 0x8470, 0x847e) || - ONIGENC_IS_IN_RANGE(code, 0x8480, 0x8491)) { - /* Cyrillic */ - int d = (code >= 0x8480) ? 1 : 0; - return (OnigCodePoint )(code - (0x0030 - d)); - } - return code; -} - -static int -get_case_fold_codes_by_str(OnigCaseFoldType flag, - const OnigUChar* p, const OnigUChar* end, - OnigCaseFoldCodeItem items[], OnigEncoding enc) -{ - int len; - OnigCodePoint code, code_lo, code_up; - - code = mbc_to_code(p, end, enc); - if (ONIGENC_IS_ASCII_CODE(code)) - return onigenc_ascii_get_case_fold_codes_by_str(flag, p, end, items, enc); - - len = mbc_enc_len(p, end, enc); - code_lo = get_lower_case(code); - code_up = get_upper_case(code); - - if (code != code_lo) { - items[0].byte_len = len; - items[0].code_len = 1; - items[0].code[0] = code_lo; - return 1; - } - else if (code != code_up) { - items[0].byte_len = len; - items[0].code_len = 1; - items[0].code[0] = code_up; - return 1; - } - - return 0; -} - -static int -mbc_case_fold(OnigCaseFoldType flag, - const UChar** pp, const UChar* end, UChar* lower, - OnigEncoding enc) -{ - const UChar* p = *pp; - - if (ONIGENC_IS_MBC_ASCII(p)) { - *lower = ONIGENC_ASCII_CODE_TO_LOWER_CASE(*p); - (*pp)++; - return 1; - } - else { - OnigCodePoint code; - int len; - - code = get_lower_case(mbc_to_code(p, end, enc)); - len = code_to_mbc(code, lower, enc); - (*pp) += len; - return len; /* return byte length of converted char to lower */ - } -} - -#if 0 -static int -is_mbc_ambiguous(OnigCaseFoldType flag, - const UChar** pp, const UChar* end) -{ - return onigenc_mbn_is_mbc_ambiguous(enc, flag, pp, end); - -} -#endif - -#if 0 -static int -is_code_ctype(OnigCodePoint code, unsigned int ctype) -{ - if (code < 128) - return ONIGENC_IS_ASCII_CODE_CTYPE(code, ctype); - else { - if (CTYPE_IS_WORD_GRAPH_PRINT(ctype)) { - return (code_to_mbclen(code) > 1 ? TRUE : FALSE); - } - } - - return FALSE; -} -#endif - -static UChar* -left_adjust_char_head(const UChar* start, const UChar* s, const UChar* end, OnigEncoding enc) -{ - const UChar *p; - int len; - - if (s <= start) return (UChar* )s; - p = s; - - if (SJIS_ISMB_TRAIL(*p)) { - while (p > start) { - if (! SJIS_ISMB_FIRST(*--p)) { - p++; - break; - } - } - } - len = mbc_enc_len(p, end, enc); - if (p + len > s) return (UChar* )p; - p += len; - return (UChar* )(p + ((s - p) & ~1)); -} - -static int -is_allowed_reverse_match(const UChar* s, const UChar* end, OnigEncoding enc ARG_UNUSED) -{ - const UChar c = *s; - return (SJIS_ISMB_TRAIL(c) ? FALSE : TRUE); -} - - -static const OnigCodePoint CR_Hiragana[] = { - 1, - 0x829f, 0x82f1 -}; /* CR_Hiragana */ - -static const OnigCodePoint CR_Katakana[] = { - 4, - 0x00a6, 0x00af, - 0x00b1, 0x00dd, - 0x8340, 0x837e, - 0x8380, 0x8396, -}; /* CR_Katakana */ - -#ifdef ENC_CP932 -static const OnigCodePoint CR_Han[] = { - 6, - 0x8157, 0x8157, - 0x889f, 0x9872, /* Kanji level 1 */ - 0x989f, 0x9ffc, /* Kanji level 2 */ - 0xe040, 0xeaa4, /* Kanji level 2 */ - 0xed40, 0xeeec, /* NEC-selected IBM extended characters (without symbols) */ - 0xfa5c, 0xfc4b, /* IBM extended characters (without symbols) */ -}; /* CR_Han */ -#else -static const OnigCodePoint CR_Han[] = { - 4, - 0x8157, 0x8157, - 0x889f, 0x9872, /* Kanji level 1 */ - 0x989f, 0x9ffc, /* Kanji level 2 */ - 0xe040, 0xeaa4, /* Kanji level 2 */ -}; /* CR_Han */ -#endif - -static const OnigCodePoint CR_Latin[] = { - 4, - 0x0041, 0x005a, - 0x0061, 0x007a, - 0x8260, 0x8279, - 0x8281, 0x829a, -}; /* CR_Latin */ - -static const OnigCodePoint CR_Greek[] = { - 2, - 0x839f, 0x83b6, - 0x83bf, 0x83d6, -}; /* CR_Greek */ - -static const OnigCodePoint CR_Cyrillic[] = { - 3, - 0x8440, 0x8460, - 0x8470, 0x847f, - 0x8480, 0x8491, -}; /* CR_Cyrillic */ - -#include "enc/jis/props.h" - -static int -property_name_to_ctype(OnigEncoding enc, const UChar* p, const UChar* end) -{ - const UChar *s = p, *e = end; - const struct enc_property *prop = - onig_jis_property((const char* )s, (unsigned int )(e - s)); - - if (!prop) { - return onigenc_minimum_property_name_to_ctype(enc, s, e); - } - - return (int )prop->ctype; -} - -static int -is_code_ctype(OnigCodePoint code, unsigned int ctype, OnigEncoding enc) -{ - if (ctype <= ONIGENC_MAX_STD_CTYPE) { - if (code < 128) - return ONIGENC_IS_ASCII_CODE_CTYPE(code, ctype); - else { - if (CTYPE_IS_WORD_GRAPH_PRINT(ctype)) { - return TRUE; - } - } - } - else { - ctype -= (ONIGENC_MAX_STD_CTYPE + 1); - if (ctype >= (unsigned int )PropertyListNum) - return ONIGERR_TYPE_BUG; - - return onig_is_in_code_range((UChar* )PropertyList[ctype], code); - } - - return FALSE; -} - -static int -get_ctype_code_range(OnigCtype ctype, OnigCodePoint* sb_out, - const OnigCodePoint* ranges[], OnigEncoding enc ARG_UNUSED) -{ - if (ctype <= ONIGENC_MAX_STD_CTYPE) { - return ONIG_NO_SUPPORT_CONFIG; - } - else { - *sb_out = 0x80; - - ctype -= (ONIGENC_MAX_STD_CTYPE + 1); - if (ctype >= (OnigCtype )PropertyListNum) - return ONIGERR_TYPE_BUG; - - *ranges = PropertyList[ctype]; - return 0; - } -} - -#ifndef ENC_CP932 OnigEncodingDefine(shift_jis, Shift_JIS) = { mbc_enc_len, "Shift_JIS", /* name */ @@ -581,4 +65,3 @@ OnigEncodingDefine(shift_jis, Shift_JIS) = { */ ENC_REPLICATE("MacJapanese", "Shift_JIS") ENC_ALIAS("MacJapan", "MacJapanese") -#endif diff --git a/enc/windows_31j.c b/enc/windows_31j.c index 174f8983c4..e3a468f414 100644 --- a/enc/windows_31j.c +++ b/enc/windows_31j.c @@ -1,5 +1,5 @@ /********************************************************************** - cp932.c - Onigmo (Oniguruma-mod) (regular expression library) + windows_31j.c - Onigmo (Oniguruma-mod) (regular expression library) **********************************************************************/ /*- * Copyright (c) 2002-2009 K.Kosako <sndgk393 AT ybb DOT ne DOT jp> @@ -29,7 +29,7 @@ */ #define ENC_CP932 -#include "shift_jis.c" +#include "shift_jis.h" OnigEncodingDefine(windows_31j, Windows_31J) = { mbc_enc_len, diff --git a/include/ruby/onigmo.h b/include/ruby/onigmo.h index 99afcf66fa..385f2d6a8b 100644 --- a/include/ruby/onigmo.h +++ b/include/ruby/onigmo.h @@ -39,7 +39,7 @@ extern "C" { #define ONIGMO_VERSION_MAJOR 6 #define ONIGMO_VERSION_MINOR 1 -#define ONIGMO_VERSION_TEENY 2 +#define ONIGMO_VERSION_TEENY 3 #ifndef ONIG_EXTERN # ifdef RUBY_EXTERN @@ -1811,14 +1811,9 @@ match_at(regex_t* reg, const UChar* str, const UChar* end, NEXT; CASE(OP_EXACT1) MOP_IN(OP_EXACT1); -#if 0 DATA_ENSURE(1); if (*p != *s) goto fail; p++; s++; -#endif - if (*p != *s++) goto fail; - DATA_ENSURE(0); - p++; MOP_OUT; NEXT; @@ -2164,7 +2159,7 @@ match_at(regex_t* reg, const UChar* str, const UChar* end, s += n; } MOP_OUT; - NEXT; + JUMP; CASE(OP_ANYCHAR_ML_STAR) MOP_IN(OP_ANYCHAR_ML_STAR); while (DATA_ENSURE_CHECK1) { @@ -2181,7 +2176,7 @@ match_at(regex_t* reg, const UChar* str, const UChar* end, } } MOP_OUT; - NEXT; + JUMP; CASE(OP_ANYCHAR_STAR_PEEK_NEXT) MOP_IN(OP_ANYCHAR_STAR_PEEK_NEXT); while (DATA_ENSURE_CHECK1) { @@ -2506,7 +2501,6 @@ match_at(regex_t* reg, const UChar* str, const UChar* end, CASE(OP_MEMORY_START) MOP_IN(OP_MEMORY_START); GET_MEMNUM_INC(mem, p); mem_start_stk[mem] = (OnigStackIndex )((void* )s); - mem_end_stk[mem] = INVALID_STACK_INDEX; MOP_OUT; JUMP; @@ -3918,6 +3912,8 @@ forward_search_range(regex_t* reg, const UChar* str, const UChar* end, UChar* s, } else { UChar *q = p + reg->dmin; + + if (q >= end) return 0; /* fail */ while (p < q) p += enclen(reg->enc, p, end); } } @@ -4001,18 +3997,25 @@ forward_search_range(regex_t* reg, const UChar* str, const UChar* end, UChar* s, } else { if (reg->dmax != ONIG_INFINITE_DISTANCE) { - *low = p - reg->dmax; - if (*low > s) { - *low = onigenc_get_right_adjust_char_head_with_prev(reg->enc, s, - *low, end, (const UChar** )low_prev); - if (low_prev && IS_NULL(*low_prev)) - *low_prev = onigenc_get_prev_char_head(reg->enc, - (pprev ? pprev : s), *low, end); + if (p < str + reg->dmax) { + *low = (UChar* )str; + if (low_prev) + *low_prev = onigenc_get_prev_char_head(reg->enc, str, *low, end); } else { - if (low_prev) - *low_prev = onigenc_get_prev_char_head(reg->enc, - (pprev ? pprev : str), *low, end); + *low = p - reg->dmax; + if (*low > s) { + *low = onigenc_get_right_adjust_char_head_with_prev(reg->enc, s, + *low, end, (const UChar** )low_prev); + if (low_prev && IS_NULL(*low_prev)) + *low_prev = onigenc_get_prev_char_head(reg->enc, + (pprev ? pprev : s), *low, end); + } + else { + if (low_prev) + *low_prev = onigenc_get_prev_char_head(reg->enc, + (pprev ? pprev : str), *low, end); + } } } } @@ -4540,8 +4543,10 @@ onig_scan(regex_t* reg, const UChar* str, const UChar* end, if (rs != 0) return rs; - if (region->end[0] == start - str) - start++; + if (region->end[0] == start - str) { + if (start >= end) break; + start += enclen(reg->enc, start, end); + } else start = str + region->end[0]; diff --git a/regparse.c b/regparse.c index 1e0dfd99bb..431aad92a3 100644 --- a/regparse.c +++ b/regparse.c @@ -3644,7 +3644,7 @@ fetch_token(OnigToken* tok, UChar** src, UChar* end, ScanEnv* env) if (IS_SYNTAX_OP(syn, ONIG_SYN_OP_ESC_OCTAL3)) { prev = p; num = scan_unsigned_octal_number(&p, end, (c == '0' ? 2:3), enc); - if (num < 0) return ONIGERR_TOO_BIG_NUMBER; + if (num < 0 || 0xff < num) return ONIGERR_TOO_BIG_NUMBER; if (p == prev) { /* can't read nothing. */ num = 0; /* but, it's not error */ } @@ -4440,8 +4440,8 @@ next_state_class(CClassNode* cc, CClassNode* asc_cc, static int next_state_val(CClassNode* cc, CClassNode* asc_cc, - OnigCodePoint *vs, OnigCodePoint v, - int* vs_israw, int v_israw, + OnigCodePoint *from, OnigCodePoint to, + int* from_israw, int to_israw, enum CCVALTYPE intype, enum CCVALTYPE* type, enum CCSTATE* state, ScanEnv* env) { @@ -4450,15 +4450,15 @@ next_state_val(CClassNode* cc, CClassNode* asc_cc, switch (*state) { case CCS_VALUE: if (*type == CCV_SB) { - BITSET_SET_BIT_CHKDUP(cc->bs, (int )(*vs)); + BITSET_SET_BIT_CHKDUP(cc->bs, (int )(*from)); if (IS_NOT_NULL(asc_cc)) - BITSET_SET_BIT(asc_cc->bs, (int )(*vs)); + BITSET_SET_BIT(asc_cc->bs, (int )(*from)); } else if (*type == CCV_CODE_POINT) { - r = add_code_range(&(cc->mbuf), env, *vs, *vs); + r = add_code_range(&(cc->mbuf), env, *from, *from); if (r < 0) return r; if (IS_NOT_NULL(asc_cc)) { - r = add_code_range0(&(asc_cc->mbuf), env, *vs, *vs, 0); + r = add_code_range0(&(asc_cc->mbuf), env, *from, *from, 0); if (r < 0) return r; } } @@ -4467,51 +4467,43 @@ next_state_val(CClassNode* cc, CClassNode* asc_cc, case CCS_RANGE: if (intype == *type) { if (intype == CCV_SB) { - if (*vs > 0xff || v > 0xff) + if (*from > 0xff || to > 0xff) return ONIGERR_INVALID_CODE_POINT_VALUE; - if (*vs > v) { + if (*from > to) { if (IS_SYNTAX_BV(env->syntax, ONIG_SYN_ALLOW_EMPTY_RANGE_IN_CC)) goto ccs_range_end; else return ONIGERR_EMPTY_RANGE_IN_CHAR_CLASS; } - bitset_set_range(env, cc->bs, (int )*vs, (int )v); + bitset_set_range(env, cc->bs, (int )*from, (int )to); if (IS_NOT_NULL(asc_cc)) - bitset_set_range(env, asc_cc->bs, (int )*vs, (int )v); + bitset_set_range(env, asc_cc->bs, (int )*from, (int )to); } else { - r = add_code_range(&(cc->mbuf), env, *vs, v); + r = add_code_range(&(cc->mbuf), env, *from, to); if (r < 0) return r; if (IS_NOT_NULL(asc_cc)) { - r = add_code_range0(&(asc_cc->mbuf), env, *vs, v, 0); + r = add_code_range0(&(asc_cc->mbuf), env, *from, to, 0); if (r < 0) return r; } } } else { -#if 0 - if (intype == CCV_CODE_POINT && *type == CCV_SB) { -#endif - if (*vs > v) { - if (IS_SYNTAX_BV(env->syntax, ONIG_SYN_ALLOW_EMPTY_RANGE_IN_CC)) - goto ccs_range_end; - else - return ONIGERR_EMPTY_RANGE_IN_CHAR_CLASS; - } - bitset_set_range(env, cc->bs, (int )*vs, (int )(v < 0xff ? v : 0xff)); - r = add_code_range(&(cc->mbuf), env, (OnigCodePoint )*vs, v); + if (*from > to) { + if (IS_SYNTAX_BV(env->syntax, ONIG_SYN_ALLOW_EMPTY_RANGE_IN_CC)) + goto ccs_range_end; + else + return ONIGERR_EMPTY_RANGE_IN_CHAR_CLASS; + } + bitset_set_range(env, cc->bs, (int )*from, (int )(to < 0xff ? to : 0xff)); + r = add_code_range(&(cc->mbuf), env, (OnigCodePoint )*from, to); + if (r < 0) return r; + if (IS_NOT_NULL(asc_cc)) { + bitset_set_range(env, asc_cc->bs, (int )*from, (int )(to < 0xff ? to : 0xff)); + r = add_code_range0(&(asc_cc->mbuf), env, (OnigCodePoint )*from, to, 0); if (r < 0) return r; - if (IS_NOT_NULL(asc_cc)) { - bitset_set_range(env, asc_cc->bs, (int )*vs, (int )(v < 0xff ? v : 0xff)); - r = add_code_range0(&(asc_cc->mbuf), env, (OnigCodePoint )*vs, v, 0); - if (r < 0) return r; - } -#if 0 } - else - return ONIGERR_MISMATCH_CODE_LENGTH_IN_CLASS_RANGE; -#endif } ccs_range_end: *state = CCS_COMPLETE; @@ -4526,9 +4518,9 @@ next_state_val(CClassNode* cc, CClassNode* asc_cc, break; } - *vs_israw = v_israw; - *vs = v; - *type = intype; + *from_israw = to_israw; + *from = to; + *type = intype; return 0; } @@ -4767,6 +4759,12 @@ parse_char_class(Node** np, Node** asc_np, OnigToken* tok, UChar** src, UChar* e CC_ESC_WARN(env, (UChar* )"-"); goto range_end_val; } + + if (val_type == CCV_CLASS) { + r = ONIGERR_UNMATCHED_RANGE_SPECIFIER_IN_CHAR_CLASS; + goto err; + } + state = CCS_RANGE; } else if (state == CCS_START) { diff --git a/tool/enc-unicode.rb b/tool/enc-unicode.rb index ed45a30366..d82f1dbdd1 100755 --- a/tool/enc-unicode.rb +++ b/tool/enc-unicode.rb @@ -133,14 +133,15 @@ def parse_scripts(data, categories) files = [ {:fn => 'DerivedCoreProperties.txt', :title => 'Derived Property'}, {:fn => 'Scripts.txt', :title => 'Script'}, - {:fn => 'PropList.txt', :title => 'Binary Property'} + {:fn => 'PropList.txt', :title => 'Binary Property'}, + {:fn => 'emoji-data.txt', :title => 'Emoji'} ] current = nil cps = [] names = {} files.each do |file| data_foreach(file[:fn]) do |line| - if /^# Total code points: / =~ line + if /^# Total (?:code points|elements): / =~ line data[current] = cps categories[current] = file[:title] (names[file[:title]] ||= []) << current @@ -310,13 +311,15 @@ def data_foreach(name, &block) pat = /^# #{File.basename(name).sub(/\./, '-([\\d.]+)\\.')}/ File.open(fn, 'rb') do |f| line = f.gets - unless pat =~ line - raise ArgumentError, "#{name}: no Unicode version" - end - if !$unicode_version - $unicode_version = $1 - elsif $unicode_version != $1 - raise ArgumentError, "#{name}: Unicode version mismatch: #$1" + unless /^emoji-/ =~ name + unless pat =~ line + raise ArgumentError, "#{name}: no Unicode version" + end + if !$unicode_version + $unicode_version = $1 + elsif $unicode_version != $1 + raise ArgumentError, "#{name}: Unicode version mismatch: #$1" + end end f.each(&block) end |