Add support for UTF-8 in EBCDIC environments.

git-svn-id: svn://vcs.exim.org/pcre/code/trunk@391 2f5784b3-3f2a-0410-8824-cb99058d5e15

1 files changed, 243 insertions, 121 deletions

diff --git a/pcre_tables.c b/pcre_tables.c
index 160bc5d..f1a8de4 100644
--- a/pcre_tables.c
+++ b/pcre_tables.c
@@ -6,7 +6,7 @@
 and semantics are as close as possible to those of the Perl 5 language.
 
                        Written by Philip Hazel
-           Copyright (c) 1997-2008 University of Cambridge
+           Copyright (c) 1997-2009 University of Cambridge
 
 -----------------------------------------------------------------------------
 Redistribution and use in source and binary forms, with or without
@@ -109,126 +109,248 @@ putting all the names into a single, large string and then using offsets in the
 table itself. Maintenance is more error-prone, but frequent changes to this
 data are unlikely.
 
-July 2008: There is now a script called maint/GenerateUtt.py which can be used
-to generate this data instead of maintaining it entirely by hand. */
-
-const char _pcre_utt_names[] =
-  "Any\0"
-  "Arabic\0"
-  "Armenian\0"
-  "Balinese\0"
-  "Bengali\0"
-  "Bopomofo\0"
-  "Braille\0"
-  "Buginese\0"
-  "Buhid\0"
-  "C\0"
-  "Canadian_Aboriginal\0"
-  "Carian\0"
-  "Cc\0"
-  "Cf\0"
-  "Cham\0"
-  "Cherokee\0"
-  "Cn\0"
-  "Co\0"
-  "Common\0"
-  "Coptic\0"
-  "Cs\0"
-  "Cuneiform\0"
-  "Cypriot\0"
-  "Cyrillic\0"
-  "Deseret\0"
-  "Devanagari\0"
-  "Ethiopic\0"
-  "Georgian\0"
-  "Glagolitic\0"
-  "Gothic\0"
-  "Greek\0"
-  "Gujarati\0"
-  "Gurmukhi\0"
-  "Han\0"
-  "Hangul\0"
-  "Hanunoo\0"
-  "Hebrew\0"
-  "Hiragana\0"
-  "Inherited\0"
-  "Kannada\0"
-  "Katakana\0"
-  "Kayah_Li\0"
-  "Kharoshthi\0"
-  "Khmer\0"
-  "L\0"
-  "L&\0"
-  "Lao\0"
-  "Latin\0"
-  "Lepcha\0"
-  "Limbu\0"
-  "Linear_B\0"
-  "Ll\0"
-  "Lm\0"
-  "Lo\0"
-  "Lt\0"
-  "Lu\0"
-  "Lycian\0"
-  "Lydian\0"
-  "M\0"
-  "Malayalam\0"
-  "Mc\0"
-  "Me\0"
-  "Mn\0"
-  "Mongolian\0"
-  "Myanmar\0"
-  "N\0"
-  "Nd\0"
-  "New_Tai_Lue\0"
-  "Nko\0"
-  "Nl\0"
-  "No\0"
-  "Ogham\0"
-  "Ol_Chiki\0"
-  "Old_Italic\0"
-  "Old_Persian\0"
-  "Oriya\0"
-  "Osmanya\0"
-  "P\0"
-  "Pc\0"
-  "Pd\0"
-  "Pe\0"
-  "Pf\0"
-  "Phags_Pa\0"
-  "Phoenician\0"
-  "Pi\0"
-  "Po\0"
-  "Ps\0"
-  "Rejang\0"
-  "Runic\0"
-  "S\0"
-  "Saurashtra\0"
-  "Sc\0"
-  "Shavian\0"
-  "Sinhala\0"
-  "Sk\0"
-  "Sm\0"
-  "So\0"
-  "Sundanese\0"
-  "Syloti_Nagri\0"
-  "Syriac\0"
-  "Tagalog\0"
-  "Tagbanwa\0"
-  "Tai_Le\0"
-  "Tamil\0"
-  "Telugu\0"
-  "Thaana\0"
-  "Thai\0"
-  "Tibetan\0"
-  "Tifinagh\0"
-  "Ugaritic\0"
-  "Vai\0"
-  "Yi\0"
-  "Z\0"
-  "Zl\0"
-  "Zp\0"
-  "Zs\0";
+July 2008: There is now a script called maint/GenerateUtt.py that can be used
+to generate this data instead of maintaining it entirely by hand. 
+
+The script was updated in March 2009 to generate a new EBCDIC-compliant
+version. Like all other character and string literals that are compared against
+the regular expression pattern, we must use STR_ macros instead of literal
+strings to make sure that UTF-8 support works on EBCDIC platforms. */
+
+#define STRING_Any0 STR_A STR_n STR_y "\0"
+#define STRING_Arabic0 STR_A STR_r STR_a STR_b STR_i STR_c "\0"
+#define STRING_Armenian0 STR_A STR_r STR_m STR_e STR_n STR_i STR_a STR_n "\0"
+#define STRING_Balinese0 STR_B STR_a STR_l STR_i STR_n STR_e STR_s STR_e "\0"
+#define STRING_Bengali0 STR_B STR_e STR_n STR_g STR_a STR_l STR_i "\0"
+#define STRING_Bopomofo0 STR_B STR_o STR_p STR_o STR_m STR_o STR_f STR_o "\0"
+#define STRING_Braille0 STR_B STR_r STR_a STR_i STR_l STR_l STR_e "\0"
+#define STRING_Buginese0 STR_B STR_u STR_g STR_i STR_n STR_e STR_s STR_e "\0"
+#define STRING_Buhid0 STR_B STR_u STR_h STR_i STR_d "\0"
+#define STRING_C0 STR_C "\0"
+#define STRING_Canadian_Aboriginal0 STR_C STR_a STR_n STR_a STR_d STR_i STR_a STR_n STR_UNDERSCORE STR_A STR_b STR_o STR_r STR_i STR_g STR_i STR_n STR_a STR_l "\0"
+#define STRING_Carian0 STR_C STR_a STR_r STR_i STR_a STR_n "\0"
+#define STRING_Cc0 STR_C STR_c "\0"
+#define STRING_Cf0 STR_C STR_f "\0"
+#define STRING_Cham0 STR_C STR_h STR_a STR_m "\0"
+#define STRING_Cherokee0 STR_C STR_h STR_e STR_r STR_o STR_k STR_e STR_e "\0"
+#define STRING_Cn0 STR_C STR_n "\0"
+#define STRING_Co0 STR_C STR_o "\0"
+#define STRING_Common0 STR_C STR_o STR_m STR_m STR_o STR_n "\0"
+#define STRING_Coptic0 STR_C STR_o STR_p STR_t STR_i STR_c "\0"
+#define STRING_Cs0 STR_C STR_s "\0"
+#define STRING_Cuneiform0 STR_C STR_u STR_n STR_e STR_i STR_f STR_o STR_r STR_m "\0"
+#define STRING_Cypriot0 STR_C STR_y STR_p STR_r STR_i STR_o STR_t "\0"
+#define STRING_Cyrillic0 STR_C STR_y STR_r STR_i STR_l STR_l STR_i STR_c "\0"
+#define STRING_Deseret0 STR_D STR_e STR_s STR_e STR_r STR_e STR_t "\0"
+#define STRING_Devanagari0 STR_D STR_e STR_v STR_a STR_n STR_a STR_g STR_a STR_r STR_i "\0"
+#define STRING_Ethiopic0 STR_E STR_t STR_h STR_i STR_o STR_p STR_i STR_c "\0"
+#define STRING_Georgian0 STR_G STR_e STR_o STR_r STR_g STR_i STR_a STR_n "\0"
+#define STRING_Glagolitic0 STR_G STR_l STR_a STR_g STR_o STR_l STR_i STR_t STR_i STR_c "\0"
+#define STRING_Gothic0 STR_G STR_o STR_t STR_h STR_i STR_c "\0"
+#define STRING_Greek0 STR_G STR_r STR_e STR_e STR_k "\0"
+#define STRING_Gujarati0 STR_G STR_u STR_j STR_a STR_r STR_a STR_t STR_i "\0"
+#define STRING_Gurmukhi0 STR_G STR_u STR_r STR_m STR_u STR_k STR_h STR_i "\0"
+#define STRING_Han0 STR_H STR_a STR_n "\0"
+#define STRING_Hangul0 STR_H STR_a STR_n STR_g STR_u STR_l "\0"
+#define STRING_Hanunoo0 STR_H STR_a STR_n STR_u STR_n STR_o STR_o "\0"
+#define STRING_Hebrew0 STR_H STR_e STR_b STR_r STR_e STR_w "\0"
+#define STRING_Hiragana0 STR_H STR_i STR_r STR_a STR_g STR_a STR_n STR_a "\0"
+#define STRING_Inherited0 STR_I STR_n STR_h STR_e STR_r STR_i STR_t STR_e STR_d "\0"
+#define STRING_Kannada0 STR_K STR_a STR_n STR_n STR_a STR_d STR_a "\0"
+#define STRING_Katakana0 STR_K STR_a STR_t STR_a STR_k STR_a STR_n STR_a "\0"
+#define STRING_Kayah_Li0 STR_K STR_a STR_y STR_a STR_h STR_UNDERSCORE STR_L STR_i "\0"
+#define STRING_Kharoshthi0 STR_K STR_h STR_a STR_r STR_o STR_s STR_h STR_t STR_h STR_i "\0"
+#define STRING_Khmer0 STR_K STR_h STR_m STR_e STR_r "\0"
+#define STRING_L0 STR_L "\0"
+#define STRING_L_AMPERSAND0 STR_L STR_AMPERSAND "\0"
+#define STRING_Lao0 STR_L STR_a STR_o "\0"
+#define STRING_Latin0 STR_L STR_a STR_t STR_i STR_n "\0"
+#define STRING_Lepcha0 STR_L STR_e STR_p STR_c STR_h STR_a "\0"
+#define STRING_Limbu0 STR_L STR_i STR_m STR_b STR_u "\0"
+#define STRING_Linear_B0 STR_L STR_i STR_n STR_e STR_a STR_r STR_UNDERSCORE STR_B "\0"
+#define STRING_Ll0 STR_L STR_l "\0"
+#define STRING_Lm0 STR_L STR_m "\0"
+#define STRING_Lo0 STR_L STR_o "\0"
+#define STRING_Lt0 STR_L STR_t "\0"
+#define STRING_Lu0 STR_L STR_u "\0"
+#define STRING_Lycian0 STR_L STR_y STR_c STR_i STR_a STR_n "\0"
+#define STRING_Lydian0 STR_L STR_y STR_d STR_i STR_a STR_n "\0"
+#define STRING_M0 STR_M "\0"
+#define STRING_Malayalam0 STR_M STR_a STR_l STR_a STR_y STR_a STR_l STR_a STR_m "\0"
+#define STRING_Mc0 STR_M STR_c "\0"
+#define STRING_Me0 STR_M STR_e "\0"
+#define STRING_Mn0 STR_M STR_n "\0"
+#define STRING_Mongolian0 STR_M STR_o STR_n STR_g STR_o STR_l STR_i STR_a STR_n "\0"
+#define STRING_Myanmar0 STR_M STR_y STR_a STR_n STR_m STR_a STR_r "\0"
+#define STRING_N0 STR_N "\0"
+#define STRING_Nd0 STR_N STR_d "\0"
+#define STRING_New_Tai_Lue0 STR_N STR_e STR_w STR_UNDERSCORE STR_T STR_a STR_i STR_UNDERSCORE STR_L STR_u STR_e "\0"
+#define STRING_Nko0 STR_N STR_k STR_o "\0"
+#define STRING_Nl0 STR_N STR_l "\0"
+#define STRING_No0 STR_N STR_o "\0"
+#define STRING_Ogham0 STR_O STR_g STR_h STR_a STR_m "\0"
+#define STRING_Ol_Chiki0 STR_O STR_l STR_UNDERSCORE STR_C STR_h STR_i STR_k STR_i "\0"
+#define STRING_Old_Italic0 STR_O STR_l STR_d STR_UNDERSCORE STR_I STR_t STR_a STR_l STR_i STR_c "\0"
+#define STRING_Old_Persian0 STR_O STR_l STR_d STR_UNDERSCORE STR_P STR_e STR_r STR_s STR_i STR_a STR_n "\0"
+#define STRING_Oriya0 STR_O STR_r STR_i STR_y STR_a "\0"
+#define STRING_Osmanya0 STR_O STR_s STR_m STR_a STR_n STR_y STR_a "\0"
+#define STRING_P0 STR_P "\0"
+#define STRING_Pc0 STR_P STR_c "\0"
+#define STRING_Pd0 STR_P STR_d "\0"
+#define STRING_Pe0 STR_P STR_e "\0"
+#define STRING_Pf0 STR_P STR_f "\0"
+#define STRING_Phags_Pa0 STR_P STR_h STR_a STR_g STR_s STR_UNDERSCORE STR_P STR_a "\0"
+#define STRING_Phoenician0 STR_P STR_h STR_o STR_e STR_n STR_i STR_c STR_i STR_a STR_n "\0"
+#define STRING_Pi0 STR_P STR_i "\0"
+#define STRING_Po0 STR_P STR_o "\0"
+#define STRING_Ps0 STR_P STR_s "\0"
+#define STRING_Rejang0 STR_R STR_e STR_j STR_a STR_n STR_g "\0"
+#define STRING_Runic0 STR_R STR_u STR_n STR_i STR_c "\0"
+#define STRING_S0 STR_S "\0"
+#define STRING_Saurashtra0 STR_S STR_a STR_u STR_r STR_a STR_s STR_h STR_t STR_r STR_a "\0"
+#define STRING_Sc0 STR_S STR_c "\0"
+#define STRING_Shavian0 STR_S STR_h STR_a STR_v STR_i STR_a STR_n "\0"
+#define STRING_Sinhala0 STR_S STR_i STR_n STR_h STR_a STR_l STR_a "\0"
+#define STRING_Sk0 STR_S STR_k "\0"
+#define STRING_Sm0 STR_S STR_m "\0"
+#define STRING_So0 STR_S STR_o "\0"
+#define STRING_Sundanese0 STR_S STR_u STR_n STR_d STR_a STR_n STR_e STR_s STR_e "\0"
+#define STRING_Syloti_Nagri0 STR_S STR_y STR_l STR_o STR_t STR_i STR_UNDERSCORE STR_N STR_a STR_g STR_r STR_i "\0"
+#define STRING_Syriac0 STR_S STR_y STR_r STR_i STR_a STR_c "\0"
+#define STRING_Tagalog0 STR_T STR_a STR_g STR_a STR_l STR_o STR_g "\0"
+#define STRING_Tagbanwa0 STR_T STR_a STR_g STR_b STR_a STR_n STR_w STR_a "\0"
+#define STRING_Tai_Le0 STR_T STR_a STR_i STR_UNDERSCORE STR_L STR_e "\0"
+#define STRING_Tamil0 STR_T STR_a STR_m STR_i STR_l "\0"
+#define STRING_Telugu0 STR_T STR_e STR_l STR_u STR_g STR_u "\0"
+#define STRING_Thaana0 STR_T STR_h STR_a STR_a STR_n STR_a "\0"
+#define STRING_Thai0 STR_T STR_h STR_a STR_i "\0"
+#define STRING_Tibetan0 STR_T STR_i STR_b STR_e STR_t STR_a STR_n "\0"
+#define STRING_Tifinagh0 STR_T STR_i STR_f STR_i STR_n STR_a STR_g STR_h "\0"
+#define STRING_Ugaritic0 STR_U STR_g STR_a STR_r STR_i STR_t STR_i STR_c "\0"
+#define STRING_Vai0 STR_V STR_a STR_i "\0"
+#define STRING_Yi0 STR_Y STR_i "\0"
+#define STRING_Z0 STR_Z "\0"
+#define STRING_Zl0 STR_Z STR_l "\0"
+#define STRING_Zp0 STR_Z STR_p "\0"
+#define STRING_Zs0 STR_Z STR_s "\0"
+
+const char _pcre_utt_names[] = 
+  STRING_Any0
+  STRING_Arabic0
+  STRING_Armenian0
+  STRING_Balinese0
+  STRING_Bengali0
+  STRING_Bopomofo0
+  STRING_Braille0
+  STRING_Buginese0
+  STRING_Buhid0
+  STRING_C0
+  STRING_Canadian_Aboriginal0
+  STRING_Carian0
+  STRING_Cc0
+  STRING_Cf0
+  STRING_Cham0
+  STRING_Cherokee0
+  STRING_Cn0
+  STRING_Co0
+  STRING_Common0
+  STRING_Coptic0
+  STRING_Cs0
+  STRING_Cuneiform0
+  STRING_Cypriot0
+  STRING_Cyrillic0
+  STRING_Deseret0
+  STRING_Devanagari0
+  STRING_Ethiopic0
+  STRING_Georgian0
+  STRING_Glagolitic0
+  STRING_Gothic0
+  STRING_Greek0
+  STRING_Gujarati0
+  STRING_Gurmukhi0
+  STRING_Han0
+  STRING_Hangul0
+  STRING_Hanunoo0
+  STRING_Hebrew0
+  STRING_Hiragana0
+  STRING_Inherited0
+  STRING_Kannada0
+  STRING_Katakana0
+  STRING_Kayah_Li0
+  STRING_Kharoshthi0
+  STRING_Khmer0
+  STRING_L0
+  STRING_L_AMPERSAND0
+  STRING_Lao0
+  STRING_Latin0
+  STRING_Lepcha0
+  STRING_Limbu0
+  STRING_Linear_B0
+  STRING_Ll0
+  STRING_Lm0
+  STRING_Lo0
+  STRING_Lt0
+  STRING_Lu0
+  STRING_Lycian0
+  STRING_Lydian0
+  STRING_M0
+  STRING_Malayalam0
+  STRING_Mc0
+  STRING_Me0
+  STRING_Mn0
+  STRING_Mongolian0
+  STRING_Myanmar0
+  STRING_N0
+  STRING_Nd0
+  STRING_New_Tai_Lue0
+  STRING_Nko0
+  STRING_Nl0
+  STRING_No0
+  STRING_Ogham0
+  STRING_Ol_Chiki0
+  STRING_Old_Italic0
+  STRING_Old_Persian0
+  STRING_Oriya0
+  STRING_Osmanya0
+  STRING_P0
+  STRING_Pc0
+  STRING_Pd0
+  STRING_Pe0
+  STRING_Pf0
+  STRING_Phags_Pa0
+  STRING_Phoenician0
+  STRING_Pi0
+  STRING_Po0
+  STRING_Ps0
+  STRING_Rejang0
+  STRING_Runic0
+  STRING_S0
+  STRING_Saurashtra0
+  STRING_Sc0
+  STRING_Shavian0
+  STRING_Sinhala0
+  STRING_Sk0
+  STRING_Sm0
+  STRING_So0
+  STRING_Sundanese0
+  STRING_Syloti_Nagri0
+  STRING_Syriac0
+  STRING_Tagalog0
+  STRING_Tagbanwa0
+  STRING_Tai_Le0
+  STRING_Tamil0
+  STRING_Telugu0
+  STRING_Thaana0
+  STRING_Thai0
+  STRING_Tibetan0
+  STRING_Tifinagh0
+  STRING_Ugaritic0
+  STRING_Vai0
+  STRING_Yi0
+  STRING_Z0
+  STRING_Zl0
+  STRING_Zp0
+  STRING_Zs0;
 
 const ucp_type_table _pcre_utt[] = {
   {   0, PT_ANY, 0 },