diff options
author | Glenn Morris <rgm@gnu.org> | 2015-06-16 23:43:03 -0700 |
---|---|---|
committer | Glenn Morris <rgm@gnu.org> | 2015-06-16 23:43:03 -0700 |
commit | d67d49ceb31777cb731a56b9518bf1a1f2a94d25 (patch) | |
tree | 37aea7063e35326b069c12d9f4c16e5cf172d57f /admin | |
parent | c9e2003b44b446a2f994039d9fc73d387cf6a1df (diff) | |
download | emacs-d67d49ceb31777cb731a56b9518bf1a1f2a94d25.tar.gz |
Generate char-script-table from Unicode source. (Bug#20789)
* admin/unidata/Makefile.in (AWK): New, set by configure.
(all): Add charscript.el.
(blocks): New variable.
(charscript.el, ${unidir}/charscript.el): New targets.
(extraclean): Also remove generated charscript.el.
* admin/unidata/blocks.awk: New script.
* admin/unidata/Blocks.txt: New data file, from unicode.org.
* lisp/international/characters.el: Load charscript.
* src/Makefile.in (charscript): New variable.
(${charscript}): New target.
(${lispintdir}/characters.elc): Depend on charscript.elc.
(temacs$(EXEEXT)): Depend on charscript.
; * admin/unidata/README: Mention Blocks.txt.
; * .gitignore: Add lisp/international/charscript.el.
Diffstat (limited to 'admin')
-rw-r--r-- | admin/unidata/Blocks.txt | 298 | ||||
-rw-r--r-- | admin/unidata/Makefile.in | 16 | ||||
-rw-r--r-- | admin/unidata/README | 4 | ||||
-rwxr-xr-x | admin/unidata/blocks.awk | 225 |
4 files changed, 541 insertions, 2 deletions
diff --git a/admin/unidata/Blocks.txt b/admin/unidata/Blocks.txt new file mode 100644 index 00000000000..0a4a5807635 --- /dev/null +++ b/admin/unidata/Blocks.txt @@ -0,0 +1,298 @@ +# Blocks-8.0.0.txt +# Date: 2014-11-10, 23:04:00 GMT [KW] +# +# Unicode Character Database +# Copyright (c) 1991-2014 Unicode, Inc. +# For terms of use, see http://www.unicode.org/terms_of_use.html +# For documentation, see http://www.unicode.org/reports/tr44/ +# +# Format: +# Start Code..End Code; Block Name + +# ================================================ + +# Note: When comparing block names, casing, whitespace, hyphens, +# and underbars are ignored. +# For example, "Latin Extended-A" and "latin extended a" are equivalent. +# For more information on the comparison of property values, +# see UAX #44: http://www.unicode.org/reports/tr44/ +# +# All block ranges start with a value where (cp MOD 16) = 0, +# and end with a value where (cp MOD 16) = 15. In other words, +# the last hexadecimal digit of the start of range is ...0 +# and the last hexadecimal digit of the end of range is ...F. +# This constraint on block ranges guarantees that allocations +# are done in terms of whole columns, and that code chart display +# never involves splitting columns in the charts. +# +# All code points not explicitly listed for Block +# have the value No_Block. + +# Property: Block +# +# @missing: 0000..10FFFF; No_Block + +0000..007F; Basic Latin +0080..00FF; Latin-1 Supplement +0100..017F; Latin Extended-A +0180..024F; Latin Extended-B +0250..02AF; IPA Extensions +02B0..02FF; Spacing Modifier Letters +0300..036F; Combining Diacritical Marks +0370..03FF; Greek and Coptic +0400..04FF; Cyrillic +0500..052F; Cyrillic Supplement +0530..058F; Armenian +0590..05FF; Hebrew +0600..06FF; Arabic +0700..074F; Syriac +0750..077F; Arabic Supplement +0780..07BF; Thaana +07C0..07FF; NKo +0800..083F; Samaritan +0840..085F; Mandaic +08A0..08FF; Arabic Extended-A +0900..097F; Devanagari +0980..09FF; Bengali +0A00..0A7F; Gurmukhi +0A80..0AFF; Gujarati +0B00..0B7F; Oriya +0B80..0BFF; Tamil +0C00..0C7F; Telugu +0C80..0CFF; Kannada +0D00..0D7F; Malayalam +0D80..0DFF; Sinhala +0E00..0E7F; Thai +0E80..0EFF; Lao +0F00..0FFF; Tibetan +1000..109F; Myanmar +10A0..10FF; Georgian +1100..11FF; Hangul Jamo +1200..137F; Ethiopic +1380..139F; Ethiopic Supplement +13A0..13FF; Cherokee +1400..167F; Unified Canadian Aboriginal Syllabics +1680..169F; Ogham +16A0..16FF; Runic +1700..171F; Tagalog +1720..173F; Hanunoo +1740..175F; Buhid +1760..177F; Tagbanwa +1780..17FF; Khmer +1800..18AF; Mongolian +18B0..18FF; Unified Canadian Aboriginal Syllabics Extended +1900..194F; Limbu +1950..197F; Tai Le +1980..19DF; New Tai Lue +19E0..19FF; Khmer Symbols +1A00..1A1F; Buginese +1A20..1AAF; Tai Tham +1AB0..1AFF; Combining Diacritical Marks Extended +1B00..1B7F; Balinese +1B80..1BBF; Sundanese +1BC0..1BFF; Batak +1C00..1C4F; Lepcha +1C50..1C7F; Ol Chiki +1CC0..1CCF; Sundanese Supplement +1CD0..1CFF; Vedic Extensions +1D00..1D7F; Phonetic Extensions +1D80..1DBF; Phonetic Extensions Supplement +1DC0..1DFF; Combining Diacritical Marks Supplement +1E00..1EFF; Latin Extended Additional +1F00..1FFF; Greek Extended +2000..206F; General Punctuation +2070..209F; Superscripts and Subscripts +20A0..20CF; Currency Symbols +20D0..20FF; Combining Diacritical Marks for Symbols +2100..214F; Letterlike Symbols +2150..218F; Number Forms +2190..21FF; Arrows +2200..22FF; Mathematical Operators +2300..23FF; Miscellaneous Technical +2400..243F; Control Pictures +2440..245F; Optical Character Recognition +2460..24FF; Enclosed Alphanumerics +2500..257F; Box Drawing +2580..259F; Block Elements +25A0..25FF; Geometric Shapes +2600..26FF; Miscellaneous Symbols +2700..27BF; Dingbats +27C0..27EF; Miscellaneous Mathematical Symbols-A +27F0..27FF; Supplemental Arrows-A +2800..28FF; Braille Patterns +2900..297F; Supplemental Arrows-B +2980..29FF; Miscellaneous Mathematical Symbols-B +2A00..2AFF; Supplemental Mathematical Operators +2B00..2BFF; Miscellaneous Symbols and Arrows +2C00..2C5F; Glagolitic +2C60..2C7F; Latin Extended-C +2C80..2CFF; Coptic +2D00..2D2F; Georgian Supplement +2D30..2D7F; Tifinagh +2D80..2DDF; Ethiopic Extended +2DE0..2DFF; Cyrillic Extended-A +2E00..2E7F; Supplemental Punctuation +2E80..2EFF; CJK Radicals Supplement +2F00..2FDF; Kangxi Radicals +2FF0..2FFF; Ideographic Description Characters +3000..303F; CJK Symbols and Punctuation +3040..309F; Hiragana +30A0..30FF; Katakana +3100..312F; Bopomofo +3130..318F; Hangul Compatibility Jamo +3190..319F; Kanbun +31A0..31BF; Bopomofo Extended +31C0..31EF; CJK Strokes +31F0..31FF; Katakana Phonetic Extensions +3200..32FF; Enclosed CJK Letters and Months +3300..33FF; CJK Compatibility +3400..4DBF; CJK Unified Ideographs Extension A +4DC0..4DFF; Yijing Hexagram Symbols +4E00..9FFF; CJK Unified Ideographs +A000..A48F; Yi Syllables +A490..A4CF; Yi Radicals +A4D0..A4FF; Lisu +A500..A63F; Vai +A640..A69F; Cyrillic Extended-B +A6A0..A6FF; Bamum +A700..A71F; Modifier Tone Letters +A720..A7FF; Latin Extended-D +A800..A82F; Syloti Nagri +A830..A83F; Common Indic Number Forms +A840..A87F; Phags-pa +A880..A8DF; Saurashtra +A8E0..A8FF; Devanagari Extended +A900..A92F; Kayah Li +A930..A95F; Rejang +A960..A97F; Hangul Jamo Extended-A +A980..A9DF; Javanese +A9E0..A9FF; Myanmar Extended-B +AA00..AA5F; Cham +AA60..AA7F; Myanmar Extended-A +AA80..AADF; Tai Viet +AAE0..AAFF; Meetei Mayek Extensions +AB00..AB2F; Ethiopic Extended-A +AB30..AB6F; Latin Extended-E +AB70..ABBF; Cherokee Supplement +ABC0..ABFF; Meetei Mayek +AC00..D7AF; Hangul Syllables +D7B0..D7FF; Hangul Jamo Extended-B +D800..DB7F; High Surrogates +DB80..DBFF; High Private Use Surrogates +DC00..DFFF; Low Surrogates +E000..F8FF; Private Use Area +F900..FAFF; CJK Compatibility Ideographs +FB00..FB4F; Alphabetic Presentation Forms +FB50..FDFF; Arabic Presentation Forms-A +FE00..FE0F; Variation Selectors +FE10..FE1F; Vertical Forms +FE20..FE2F; Combining Half Marks +FE30..FE4F; CJK Compatibility Forms +FE50..FE6F; Small Form Variants +FE70..FEFF; Arabic Presentation Forms-B +FF00..FFEF; Halfwidth and Fullwidth Forms +FFF0..FFFF; Specials +10000..1007F; Linear B Syllabary +10080..100FF; Linear B Ideograms +10100..1013F; Aegean Numbers +10140..1018F; Ancient Greek Numbers +10190..101CF; Ancient Symbols +101D0..101FF; Phaistos Disc +10280..1029F; Lycian +102A0..102DF; Carian +102E0..102FF; Coptic Epact Numbers +10300..1032F; Old Italic +10330..1034F; Gothic +10350..1037F; Old Permic +10380..1039F; Ugaritic +103A0..103DF; Old Persian +10400..1044F; Deseret +10450..1047F; Shavian +10480..104AF; Osmanya +10500..1052F; Elbasan +10530..1056F; Caucasian Albanian +10600..1077F; Linear A +10800..1083F; Cypriot Syllabary +10840..1085F; Imperial Aramaic +10860..1087F; Palmyrene +10880..108AF; Nabataean +108E0..108FF; Hatran +10900..1091F; Phoenician +10920..1093F; Lydian +10980..1099F; Meroitic Hieroglyphs +109A0..109FF; Meroitic Cursive +10A00..10A5F; Kharoshthi +10A60..10A7F; Old South Arabian +10A80..10A9F; Old North Arabian +10AC0..10AFF; Manichaean +10B00..10B3F; Avestan +10B40..10B5F; Inscriptional Parthian +10B60..10B7F; Inscriptional Pahlavi +10B80..10BAF; Psalter Pahlavi +10C00..10C4F; Old Turkic +10C80..10CFF; Old Hungarian +10E60..10E7F; Rumi Numeral Symbols +11000..1107F; Brahmi +11080..110CF; Kaithi +110D0..110FF; Sora Sompeng +11100..1114F; Chakma +11150..1117F; Mahajani +11180..111DF; Sharada +111E0..111FF; Sinhala Archaic Numbers +11200..1124F; Khojki +11280..112AF; Multani +112B0..112FF; Khudawadi +11300..1137F; Grantha +11480..114DF; Tirhuta +11580..115FF; Siddham +11600..1165F; Modi +11680..116CF; Takri +11700..1173F; Ahom +118A0..118FF; Warang Citi +11AC0..11AFF; Pau Cin Hau +12000..123FF; Cuneiform +12400..1247F; Cuneiform Numbers and Punctuation +12480..1254F; Early Dynastic Cuneiform +13000..1342F; Egyptian Hieroglyphs +14400..1467F; Anatolian Hieroglyphs +16800..16A3F; Bamum Supplement +16A40..16A6F; Mro +16AD0..16AFF; Bassa Vah +16B00..16B8F; Pahawh Hmong +16F00..16F9F; Miao +1B000..1B0FF; Kana Supplement +1BC00..1BC9F; Duployan +1BCA0..1BCAF; Shorthand Format Controls +1D000..1D0FF; Byzantine Musical Symbols +1D100..1D1FF; Musical Symbols +1D200..1D24F; Ancient Greek Musical Notation +1D300..1D35F; Tai Xuan Jing Symbols +1D360..1D37F; Counting Rod Numerals +1D400..1D7FF; Mathematical Alphanumeric Symbols +1D800..1DAAF; Sutton SignWriting +1E800..1E8DF; Mende Kikakui +1EE00..1EEFF; Arabic Mathematical Alphabetic Symbols +1F000..1F02F; Mahjong Tiles +1F030..1F09F; Domino Tiles +1F0A0..1F0FF; Playing Cards +1F100..1F1FF; Enclosed Alphanumeric Supplement +1F200..1F2FF; Enclosed Ideographic Supplement +1F300..1F5FF; Miscellaneous Symbols and Pictographs +1F600..1F64F; Emoticons +1F650..1F67F; Ornamental Dingbats +1F680..1F6FF; Transport and Map Symbols +1F700..1F77F; Alchemical Symbols +1F780..1F7FF; Geometric Shapes Extended +1F800..1F8FF; Supplemental Arrows-C +1F900..1F9FF; Supplemental Symbols and Pictographs +20000..2A6DF; CJK Unified Ideographs Extension B +2A700..2B73F; CJK Unified Ideographs Extension C +2B740..2B81F; CJK Unified Ideographs Extension D +2B820..2CEAF; CJK Unified Ideographs Extension E +2F800..2FA1F; CJK Compatibility Ideographs Supplement +E0000..E007F; Tags +E0100..E01EF; Variation Selectors Supplement +F0000..FFFFF; Supplementary Private Use Area-A +100000..10FFFF; Supplementary Private Use Area-B + +# EOF diff --git a/admin/unidata/Makefile.in b/admin/unidata/Makefile.in index 6c81d32484a..954e9faacae 100644 --- a/admin/unidata/Makefile.in +++ b/admin/unidata/Makefile.in @@ -23,6 +23,8 @@ SHELL = @SHELL@ +AWK = @AWK@ + srcdir = @srcdir@ top_srcdir = @top_srcdir@ top_builddir = @top_builddir@ @@ -51,7 +53,7 @@ am__v_at_1 = .PHONY: all unifiles -all: ${top_srcdir}/src/macuvs.h unifiles +all: ${top_srcdir}/src/macuvs.h unifiles ${unidir}/charscript.el ## Specify .elc as an order-only prereq so as to not needlessly rebuild ## target just because the .elc is missing. @@ -94,6 +96,16 @@ unifiles: ${unidir}/charprop.el [ -f $(unidir)/$$f ] || exec $(MAKE) PHONY_EXTRAS=$< $<; \ done + +.PHONY: charscript.el +charscript.el: ${unidir}/charscript.el + +blocks = ${srcdir}/blocks.awk + +${unidir}/charscript.el: ${srcdir}/Blocks.txt ${blocks} + $(AM_V_GEN)$(AWK) -f ${blocks} < $< > $@ + + .PHONY: clean bootstrap-clean distclean maintainer-clean extraclean clean: @@ -110,7 +122,7 @@ maintainer-clean: distclean ## change and it slows down bootstrap (a tiny bit). ## Cf leim/ja-dic (which is much slower). extraclean: - rm -f ${top_srcdir}/src/macuvs.h + rm -f ${top_srcdir}/src/macuvs.h ${unidir}/charscript.el* ifneq (,$(wildcard $(unidir)/charprop.el)) cd $(unidir) && \ rm -f `sed -n 's/^;; FILE: //p' < charprop.el` charprop.el diff --git a/admin/unidata/README b/admin/unidata/README index e0d0e00417f..8ad0a12d728 100644 --- a/admin/unidata/README +++ b/admin/unidata/README @@ -16,3 +16,7 @@ http://www.unicode.org/ivd/data/2014-05-16/IVD_Sequences.txt UnicodeData.txt http://www.unicode.org/Public/UNIDATA/UnicodeData.txt 2014-03-10 + +Blocks.txt +http://www.unicode.org/Public/8.0.0/ucd/Blocks.txt +2014-11-10 diff --git a/admin/unidata/blocks.awk b/admin/unidata/blocks.awk new file mode 100755 index 00000000000..892ac58c927 --- /dev/null +++ b/admin/unidata/blocks.awk @@ -0,0 +1,225 @@ +#!/usr/bin/awk -f + +## Copyright (C) 2015 Free Software Foundation, Inc. + +## Author: Glenn Morris <rgm@gnu.org> + +## This file is part of GNU Emacs. + +## GNU Emacs is free software: you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation, either version 3 of the License, or +## (at your option) any later version. + +## GNU Emacs is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. + +## You should have received a copy of the GNU General Public License +## along with GNU Emacs. If not, see <http://www.gnu.org/licenses/>. + +### Commentary: + +## This script takes as input Unicode's Blocks.txt +## (http://www.unicode.org/Public/UNIDATA/Blocks.txt) +## and produces output for Emacs's lisp/international/charscript.el. + +## It lumps together all the blocks belonging to the same language. +## E.g., "Basic Latin", "Latin-1 Supplement", "Latin Extended-A", +## etc. are all lumped together under "latin". + +## The Unicode blocks actually extend past some of these ranges with +## undefined codepoints. + +## For additional details, see <http://debbugs.gnu.org/20789#11>. + +## Things to do after installing a new version of Blocks.txt: +## Check the output against the old output. +## Adjust the alias array, and the name2alias function for any new +## entries, if necessary. +## Check fix_start (and fix_end) to see if entries need adding/removing. +## Review the hard-coded splits at the end of the main body. + +### Code: + +BEGIN { + ## Hard-coded names. See name2alias for the rest. + alias["ipa extensions"] = "phonetic" + alias["letterlike symbols"] = "symbol" + alias["number forms"] = "symbol" + alias["miscellaneous technical"] = "symbol" + alias["control pictures"] = "symbol" + alias["optical character recognition"] = "symbol" + alias["enclosed alphanumerics"] = "symbol" + alias["box drawing"] = "symbol" + alias["block elements"] = "symbol" + alias["miscellaneous symbols"] = "symbol" + alias["cjk strokes"] = "cjk-misc" + alias["cjk symbols and punctuation"] = "cjk-misc" + alias["halfwidth and fullwidth forms"] = "cjk-misc" + alias["common indic number forms"] = "north-indic-number" + + tohex["a"] = 10 + tohex["b"] = 11 + tohex["c"] = 12 + tohex["d"] = 13 + tohex["e"] = 14 + tohex["f"] = 15 + + fix_start["0080"] = "00A0" + ## Define fix_end here if you need it. +} + +## From admin/charsets/. +## With gawk's --non-decimal-data switch we wouldn't need this. +function decode_hex(str , n, len, i, c) { + n = 0 + len = length(str) + for (i = 1; i <= len; i++) + { + c = substr (str, i, 1) + if (c >= "0" && c <= "9") + n = n * 16 + (c - "0") + else + n = n * 16 + tohex[tolower(c)] + } + return n +} + +function name2alias(name , w, w2) { + name = tolower(name) + if (alias[name]) return alias[name] + else if (name ~ /for symbols/) return "symbol" + else if (name ~ /latin|combining .* marks|spacing modifier|tone letters|alphabetic presentation/) return "latin" + else if (name ~ /cjk|yijing|enclosed ideograph|kangxi/) return "han" + else if (name ~ /arabic/) return "arabic" + else if (name ~ /^greek/) return "greek" + else if (name ~ /^coptic/) return "coptic" + else if (name ~ /cuneiform number/) return "cuneiform-numbers-and-punctuation" + else if (name ~ /cuneiform/) return "cuneiform" + else if (name ~ /mathematical alphanumeric symbol/) return "mathematical" + else if (name ~ /punctuation|mathematical|arrows|currency|superscript|small form variants|geometric|dingbats|enclosed|alchemical|pictograph|emoticon|transport/) return "symbol" + else if (name ~ /canadian aboriginal/) return "canadian-aboriginal" + else if (name ~ /katakana|hiragana/) return "kana" + else if (name ~ /myanmar/) return "burmese" + else if (name ~ /hangul/) return "hangul" + else if (name ~ /khmer/) return "khmer" + else if (name ~ /braille/) return "braille" + else if (name ~ /^yi /) return "yi" + else if (name ~ /surrogates|private use|variation selectors/) return 0 + else if (name ~/^(specials|tags)$/) return 0 + else if (name ~ /linear b/) return "linear-b" + else if (name ~ /aramaic/) return "aramaic" + else if (name ~ /rumi num/) return "rumi-number" + else if (name ~ /duployan|shorthand/) return "duployan-shorthand" + else if (name ~ /sutton signwriting/) return "sutton-sign-writing" + + sub(/ (extended|extensions|supplement).*/, "", name) + sub(/numbers/, "number", name) + sub(/numerals/, "numeral", name) + sub(/symbols/, "symbol", name) + sub(/forms$/, "form", name) + sub(/tiles$/, "tile", name) + sub(/^new /, "", name) + sub(/ (characters|hieroglyphs|cursive)$/, "", name) + gsub(/ /, "-", name) + + return name +} + +/^[0-9A-F]/ { + sep = index($1, "..") + len = length($1) + s = substr($1,1,sep-1) + e = substr($1,sep+2,len-sep-2) + $1 = "" + sub(/^ */, "", $0) + i++ + start[i] = fix_start[s] ? fix_start[s] : s + end[i] = fix_end[e] ? fix_end[e]: e + name[i] = $0 + + alt[i] = name2alias(name[i]) + + if (!alt[i]) + { + i-- + next + } + + ## Combine adjacent ranges with the same name. + if (alt[i] == alt[i-1] && decode_hex(start[i]) == 1 + decode_hex(end[i-1])) + { + end[i-1] = end[i] + name[i-1] = (name[i-1] ", " name[i]) + i-- + } + + ## Some hard-coded splits. + if (start[i] == "0370") + { + end[i] = "03E1" + i++ + start[i] = "03E2" + end[i] = "03EF" + alt[i] = "coptic" + i++ + start[i] = "03F0" + end[i] = "03FF" + alt[i] = "greek" + } + else if (start[i] == "FB00") + { + end[i] = "FB06" + i++ + start[i] = "FB13" + end[i] = "FB17" + alt[i] = "armenian" + i++ + start[i] = "FB1D" + end[i] = "FB4F" + alt[i] = "hebrew" + } + else if (start[i] == "FF00") + { + end[i] = "FF60" + i++ + start[i] = "FF61" + end[i] = "FF9F" + alt[i] = "kana" + i++ + start[i] = "FFA0" + end[i] = "FFDF" + alt[i] = "hangul" + i++ + start[i] = "FFE0" + end[i] = "FFEF" + alt[i] = "cjk-misc" + } +} + +END { + print ";;; charscript.el --- character script table" + print ";;; Automatically generated from admin/unidata/Blocks.txt" + print "(let (script-list)" + print " (dolist (elt '(" + + for (j=1;j<=i;j++) + { + printf(" (#x%s #x%s %s)", start[j], end[j], alt[j]) + ## Fuzz to decide whether worth printing original name as a comment. + if (name[j] && alt[j] != tolower(name[j]) && alt[j] !~ /-/) + printf(" ; %s", name[j]) + printf("\n") + } + + print " ))" + print " (set-char-table-range char-script-table" + print " (cons (car elt) (nth 1 elt)) (nth 2 elt))" + print " (or (memq (nth 2 elt) script-list)" + print " (setq script-list (cons (nth 2 elt) script-list))))" + print " (set-char-table-extra-slot char-script-table 0 (nreverse script-list)))" + print "" + print "(provide 'charscript)" +} |