1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
|
#!perl -w
use 5.015;
use strict;
use warnings;
use Unicode::UCD "prop_invlist";
require 'regen/regen_lib.pl';
# This program outputs charclass_invlists.h, which contains various inversion
# lists in the form of C arrays that are to be used as-is for inversion lists.
# Thus, the lists it contains are essentially pre-compiled, and need only a
# light-weight fast wrapper to make them usable at run-time.
# As such, this code knows about the internal structure of these lists, and
# any change made to that has to be done here as well. A random number stored
# in the headers is used to minimize the possibility of things getting
# out-of-sync, or the wrong data structure being passed. Currently that
# random number is:
my $VERSION_DATA_STRUCTURE_TYPE = 1064334010;
my $out_fh = open_new('charclass_invlists.h', '>',
{style => '*', by => $0,
from => "Unicode::UCD"});
print $out_fh "/* See the generating file for comments */\n\n";
sub output_invlist ($$) {
my $name = shift;
my $invlist = shift; # Reference to inversion list array
# Output the inversion list $invlist using the name $name for it.
# It is output in the exact internal form for inversion lists.
my $zero_or_one; # Is the last element of the header 0, or 1 ?
# If the first element is 0, it goes in the header, instead of the body
if ($invlist->[0] == 0) {
shift @$invlist;
$zero_or_one = 0;
# Add a dummy 0 at the end so that the length is constant. inversion
# lists are always stored with enough room so that if they change from
# beginning with 0, they don't have to grow.
push @$invlist, 0;
}
else {
$zero_or_one = 1;
}
print $out_fh "\nUV ${name}_invlist[] = {\n";
print $out_fh "\t", scalar @$invlist, ",\t/* Number of elements */\n";
print $out_fh "\t0,\t/* Current iteration position */\n";
print $out_fh "\t$VERSION_DATA_STRUCTURE_TYPE, /* Version and data structure type */\n";
print $out_fh "\t", $zero_or_one,
",\t/* 0 if this is the first element of the list proper;",
"\n\t\t 1 if the next element is the first */\n";
# The main body are the UVs passed in to this routine. Do the final
# element separately
for my $i (0 .. @$invlist - 1 - 1) {
print $out_fh "\t$invlist->[$i],\n";
}
# The final element does not have a trailing comma, as C can't handle it.
print $out_fh "\t$invlist->[-1]\n";
print $out_fh "};\n";
}
output_invlist("Latin1", [ 0, 256 ]);
output_invlist("AboveLatin1", [ 256 ]);
# We construct lists for all the POSIX and backslash sequence character
# classes in two forms:
# 1) ones which match only in the ASCII range
# 2) ones which match either in the Latin1 range, or the entire Unicode range
#
# These get compiled in, and hence affect the memory footprint of every Perl
# program, even those not using Unicode. To minimize the size, currently
# the Latin1 version is generated for the beyond ASCII range except for those
# lists that are quite small for the entire range, such as for \s, which is 22
# UVs long plus 4 UVs (currently) for the header.
#
# To save even more memory, the ASCII versions could be derived from the
# larger ones at runtime, saving some memory (minus the expense of the machine
# instructions to do so), but these are all small anyway, so their total is
# about 100 UVs.
#
# In the list of properties below that get generated, the L1 prefix is a fake
# property that means just the Latin1 range of the full property (whose name
# has an X prefix instead of L1).
for my $prop (qw(
ASCII
L1Cased
VertSpace
PerlSpace
XPerlSpace
PosixAlnum
L1PosixAlnum
PosixAlpha
L1PosixAlpha
PosixBlank
XPosixBlank
PosixCntrl
XPosixCntrl
PosixDigit
PosixGraph
L1PosixGraph
PosixLower
L1PosixLower
PosixPrint
L1PosixPrint
PosixPunct
L1PosixPunct
PosixSpace
XPosixSpace
PosixUpper
L1PosixUpper
PosixWord
L1PosixWord
PosixXDigit
XPosixXDigit
)
) {
# For the Latin1 properties, we change to use the eXtended version of the
# base property, then go through the result and get rid of everything not
# in Latin1 (above 255). Actually, we retain the element that crosses the
# 255/256 boundary. For example, in the Word property, there is a range
# of code points that start at U+00F8 and goes through U+02C1. Instead of
# artifically cutting that off at 256 because 256 is the first code point
# above Latin1, we let the range go to its natural ending. That gives us
# extra information with no added space taken.
my $lookup_prop = $prop;
$lookup_prop =~ s/^L1Posix/XPosix/ or $lookup_prop =~ s/^L1//;
my @invlist = prop_invlist($lookup_prop);
if ($lookup_prop ne $prop) {
for my $i (0 .. @invlist - 1 - 1) {
if ($invlist[$i] > 255) {
splice @invlist, $i+1;
last;
}
}
}
output_invlist($prop, \@invlist);
}
read_only_bottom_close_and_rename($out_fh)
|
