diff options
Diffstat (limited to 'utfebcdic.h')
| -rw-r--r-- | utfebcdic.h | 83 |
1 files changed, 30 insertions, 53 deletions
diff --git a/utfebcdic.h b/utfebcdic.h index 8a6176c85c..c3fe6036ee 100644 --- a/utfebcdic.h +++ b/utfebcdic.h @@ -12,13 +12,14 @@ * * To summarize, the way it works is: * To convert an EBCDIC character to UTF-EBCDIC: - * 1) convert to Unicode. The table in this file that does this is for + * 1) convert to Unicode. The table in this file that does this for * EBCDIC bytes is PL_e2a (with inverse PLa2e). The 'a' stands for * ASCIIish, meaning latin1. - * 2) convert that to a utf8-like string called I8 with variant characters - * occupying multiple bytes. This step is similar to the utf8-creating - * step from Unicode, but the details are different. There is a chart - * about the bit patterns in a comment later in this file. But + * 2) convert that to a utf8-like string called I8 (I stands for + * intermediate) with variant characters occupying multiple bytes. This + * step is similar to the utf8-creating step from Unicode, but the details + * are different. This transformation is called UTF8-Mod. There is a + * chart about the bit patterns in a comment later in this file. But * essentially here are the differences: * UTF8 I8 * invariant byte starts with 0 starts with 0 or 100 @@ -29,18 +30,19 @@ * in I8, far beyond the current Unicode standard's * max, as shown in the comment later in this file.) * 3) Use the table published in tr16 to convert each byte from step 2 into - * final UTF-EBCDIC. The table in this file is PL_utf2e, and its inverse - * is PL_e2utf. They are constructed so that all EBCDIC invariants remain - * invariant, but no others do. For example, the ordinal value of 'A' is - * 193 in EBCDIC, and also is 193 in UTF-EBCDIC. Step 1) converts it to - * 65, Step 2 leaves it at 65, and Step 3 converts it back to 193. As an - * example of how a variant character works, take LATIN SMALL LETTER Y - * WITH DIAERESIS, which is typicially 0xDF in EBCDIC. Step 1 converts it - * to the Unicode value, 0xFF. Step 2 converts that to two bytes = - * 11000111 10111111 = C7 BF, and Step 3 converts those to 0x8B 0x73. The - * table is constructed so that the first bytes of a variant will always - * have its upper bit set (at least in the encodings that Perl recognizes, - * and probably all). + * final UTF-EBCDIC. That table is reproduced in this file as PL_utf2e, + * and its inverse is PL_e2utf. They are constructed so that all EBCDIC + * invariants remain invariant, but no others do. For example, the + * ordinal value of 'A' is 193 in EBCDIC, and also is 193 in UTF-EBCDIC. + * Step 1) converts it to 65, Step 2 leaves it at 65, and Step 3 converts + * it back to 193. As an example of how a variant character works, take + * LATIN SMALL LETTER Y WITH DIAERESIS, which is typicially 0xDF in + * EBCDIC. Step 1 converts it to the Unicode value, 0xFF. Step 2 + * converts that to two bytes = 11000111 10111111 = C7 BF, and Step 3 + * converts those to 0x8B 0x73. The table is constructed so that the + * first byte of the final form of a variant will always have its upper + * bit set (at least in the encodings that Perl recognizes, and probably + * all). But note that the upper bit of some invariants is also 1. * * If you're starting from Unicode, skip step 1. For UTF-EBCDIC to straight * EBCDIC, reverse the steps. @@ -60,8 +62,8 @@ * There are actually 3 slightly different UTF-EBCDIC encodings in this file, * one for each of the code pages recognized by Perl. That means that there * are actually three different sets of tables, one for each code page. (If - * Perl is compiled on platforms using other EBCDIC code pages, it may not - * compile, or silently mistake it for one of the three.) + * Perl is compiled on platforms using another EBCDIC code page, it may not + * compile, or Perl may silently mistake it for one of the three.) * * EBCDIC characters above 0xFF are the same as Unicode in Perl's * implementation of all 3 encodings, so for those Step 1 is trivial. @@ -150,7 +152,7 @@ unsigned char PL_utf8skip[] = { * remains 'A' */ #if '^' == 95 /* if defined(__MVS__) || defined(??) (VM/ESA?) 1047 */ -EXTCONST unsigned char PL_utf2e[] = { /* UTF-8-mod to EBCDIC (IBM-1047) */ +EXTCONST unsigned char PL_utf2e[] = { /* I8 to EBCDIC (IBM-1047) */ 0x00, 0x01, 0x02, 0x03, 0x37, 0x2D, 0x2E, 0x2F, 0x16, 0x05, 0x15, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x3C, 0x3D, 0x32, 0x26, 0x18, 0x19, 0x3F, 0x27, 0x1C, 0x1D, 0x1E, 0x1F, 0x40, 0x5A, 0x7F, 0x7B, 0x5B, 0x6C, 0x50, 0x7D, 0x4D, 0x5D, 0x5C, 0x4E, 0x6B, 0x60, 0x4B, 0x61, @@ -169,7 +171,7 @@ EXTCONST unsigned char PL_utf2e[] = { /* UTF-8-mod to EBCDIC (IBM-1047) */ 0xDC, 0xDD, 0xDE, 0xDF, 0xE1, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE }; -EXTCONST unsigned char PL_e2utf[] = { /* EBCDIC (IBM-1047) to UTF-8-mod */ +EXTCONST unsigned char PL_e2utf[] = { /* EBCDIC (IBM-1047) to I8 */ 0x00, 0x01, 0x02, 0x03, 0x9C, 0x09, 0x86, 0x7F, 0x97, 0x8D, 0x8E, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x9D, 0x0A, 0x08, 0x87, 0x18, 0x19, 0x92, 0x8F, 0x1C, 0x1D, 0x1E, 0x1F, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x17, 0x1B, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x05, 0x06, 0x07, @@ -190,7 +192,7 @@ EXTCONST unsigned char PL_e2utf[] = { /* EBCDIC (IBM-1047) to UTF-8-mod */ #endif /* 1047 */ #if '^' == 106 /* if defined(_OSD_POSIX) POSIX-BC */ -unsigned char PL_utf2e[] = { /* UTF-8-mod to EBCDIC (POSIX-BC) */ +unsigned char PL_utf2e[] = { /* I8 to EBCDIC (POSIX-BC) */ 0x00, 0x01, 0x02, 0x03, 0x37, 0x2D, 0x2E, 0x2F, 0x16, 0x05, 0x15, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x3C, 0x3D, 0x32, 0x26, 0x18, 0x19, 0x3F, 0x27, 0x1C, 0x1D, 0x1E, 0x1F, 0x40, 0x5A, 0x7F, 0x7B, 0x5B, 0x6C, 0x50, 0x7D, 0x4D, 0x5D, 0x5C, 0x4E, 0x6B, 0x60, 0x4B, 0x61, @@ -209,7 +211,7 @@ unsigned char PL_utf2e[] = { /* UTF-8-mod to EBCDIC (POSIX-BC) */ 0xDC, 0xC0, 0xDE, 0xDF, 0xE1, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 0xFA, 0xDD, 0xFC, 0xE0, 0xFE }; -unsigned char PL_e2utf[] = { /* EBCDIC (POSIX-BC) to UTF-8-mod */ +unsigned char PL_e2utf[] = { /* EBCDIC (POSIX-BC) to I8 */ 0x00, 0x01, 0x02, 0x03, 0x9C, 0x09, 0x86, 0x7F, 0x97, 0x8D, 0x8E, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x9D, 0x0A, 0x08, 0x87, 0x18, 0x19, 0x92, 0x8F, 0x1C, 0x1D, 0x1E, 0x1F, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x17, 0x1B, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x05, 0x06, 0x07, @@ -230,7 +232,7 @@ unsigned char PL_e2utf[] = { /* EBCDIC (POSIX-BC) to UTF-8-mod */ #endif /* POSIX-BC */ #if '^' == 176 /* if defined(??) (OS/400?) 037 */ -unsigned char PL_utf2e[] = { /* UTF-8-mod to EBCDIC (IBM-037) */ +unsigned char PL_utf2e[] = { /* I8 to EBCDIC (IBM-037) */ 0x00, 0x01, 0x02, 0x03, 0x37, 0x2D, 0x2E, 0x2F, 0x16, 0x05, 0x25, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x3C, 0x3D, 0x32, 0x26, 0x18, 0x19, 0x3F, 0x27, 0x1C, 0x1D, 0x1E, 0x1F, 0x40, 0x5A, 0x7F, 0x7B, 0x5B, 0x6C, 0x50, 0x7D, 0x4D, 0x5D, 0x5C, 0x4E, 0x6B, 0x60, 0x4B, 0x61, @@ -249,7 +251,7 @@ unsigned char PL_utf2e[] = { /* UTF-8-mod to EBCDIC (IBM-037) */ 0xDC, 0xDD, 0xDE, 0xDF, 0xE1, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE }; -unsigned char PL_e2utf[] = { /* EBCDIC (IBM-037) to UTF-8-mod */ +unsigned char PL_e2utf[] = { /* EBCDIC (IBM-037) to I8 */ 0x00, 0x01, 0x02, 0x03, 0x9C, 0x09, 0x86, 0x7F, 0x97, 0x8D, 0x8E, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x9D, 0x85, 0x08, 0x87, 0x18, 0x19, 0x92, 0x8F, 0x1C, 0x1D, 0x1E, 0x1F, 0x80, 0x81, 0x82, 0x83, 0x84, 0x0A, 0x17, 0x1B, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x05, 0x06, 0x07, @@ -416,13 +418,10 @@ EXTCONST unsigned char PL_a2e[]; END_EXTERN_C -#define UTF8SKIP(s) PL_utf8skip[*(const U8*)(s)] - /* EBCDIC-happy ways of converting native code to UTF-8 */ /* Native to iso-8859-1 */ #define NATIVE_TO_ASCII(ch) PL_e2a[(U8)(ch)] -#define NATIVE8_TO_UNI(ch) NATIVE_TO_ASCII(ch) /* a clearer synonym */ #define ASCII_TO_NATIVE(ch) PL_a2e[(U8)(ch)] /* Transform after encoding */ #define NATIVE_TO_UTF(ch) PL_e2utf[(U8)(ch)] @@ -435,21 +434,7 @@ END_EXTERN_C #define ASCII_TO_NEED(enc,ch) ((enc) ? UTF_TO_NATIVE(ch) : ASCII_TO_NATIVE(ch)) /* - * Note: we should try and be careful never to call the isXXX_utf8() functions - * unless we're pretty sure we've seen the beginning of a UTF-EBCDIC character - * Otherwise we risk loading in the heavy-duty swash_init and swash_fetch - * routines unnecessarily. - */ - -#define isIDFIRST_lazy_if(p,c) ((IN_BYTES || (!c || UTF8_IS_INVARIANT(*p))) \ - ? isIDFIRST(*(p)) \ - : isIDFIRST_utf8((const U8*)p)) -#define isALNUM_lazy_if(p,c) ((IN_BYTES || (!c || UTF8_IS_INVARIANT(*p))) \ - ? isALNUM(*(p)) \ - : isALNUM_utf8((const U8*)p)) - -/* - The following table is adapted from tr16, it shows UTF-8-mod encoding of Unicode code points. + The following table is adapted from tr16, it shows I8 encoding of Unicode code points. Unicode Bit pattern 1st Byte 2nd Byte 3rd Byte 4th Byte 5th Byte 6th Byte 7th byte U+0000..U+007F 000000000xxxxxxx 0xxxxxxx @@ -463,7 +448,7 @@ END_EXTERN_C U+400000..U+3FFFFFF 0uvvvvvwwwwwzzzzzyyyyyxxxxx 1111110u 101vvvvv 101wwwww 101zzzzz 101yyyyy 101xxxxx U+4000000..U+7FFFFFFF 0tuuuuuvvvvvwwwwwzzzzzyyyyyxxxxx 1111111t 101uuuuu 101vvvvv 101wwwww 101zzzzz 101yyyyy 101xxxxx - Note: The UTF-8-Mod transformation is valid for UCS-4 values X'0' to + Note: The I8 transformation is valid for UCS-4 values X'0' to X'7FFFFFFF' (the full extent of ISO/IEC 10646 coding space). */ @@ -477,9 +462,7 @@ END_EXTERN_C #define UNI_IS_INVARIANT(c) ((c) < 0xA0) -/* UTF-EBCDIC sematic macros - transform back into UTF-8-Mod and then compare */ -#define NATIVE_IS_INVARIANT(c) UNI_IS_INVARIANT(NATIVE8_TO_UNI(c)) -#define UTF8_IS_INVARIANT(c) UNI_IS_INVARIANT(NATIVE_TO_UTF(c)) +/* UTF-EBCDIC semantic macros - transform back into I8 and then compare */ #define UTF8_IS_START(c) (NATIVE_TO_UTF(c) >= 0xA0 && (NATIVE_TO_UTF(c) & 0xE0) != 0xA0) #define UTF8_IS_CONTINUATION(c) ((NATIVE_TO_UTF(c) & 0xE0) == 0xA0) #define UTF8_IS_CONTINUED(c) (NATIVE_TO_UTF(c) >= 0xA0) @@ -491,12 +474,6 @@ END_EXTERN_C #define UTF_CONTINUATION_MASK ((U8)0x1f) #define UTF_ACCUMULATION_SHIFT 5 -#define UTF8_ACCUMULATE(old, new) (((old) << UTF_ACCUMULATION_SHIFT)|(NATIVE_TO_UTF(new) & UTF_CONTINUATION_MASK)) - -/* UTF-EBCDIC encode a downgradeable value */ -#define UTF8_EIGHT_BIT_HI(c) UTF_TO_NATIVE((((U8)(c))>>UTF_ACCUMULATION_SHIFT)|UTF_START_MARK(2)) -#define UTF8_EIGHT_BIT_LO(c) UTF_TO_NATIVE(((((U8)(c)))&UTF_CONTINUATION_MASK)|UTF_CONTINUATION_MARK) - /* * Local variables: * c-indentation-style: bsd |