1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751
4752
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793
4794
4795
4796
4797
4798
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819
4820
4821
4822
4823
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877
4878
4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948
4949
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986
4987
4988
4989
4990
4991
4992
4993
4994
4995
4996
4997
4998
4999
5000
5001
5002
5003
5004
5005
5006
5007
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048
5049
5050
5051
5052
5053
5054
5055
5056
5057
5058
5059
5060
5061
5062
5063
5064
5065
5066
5067
5068
5069
5070
5071
5072
5073
5074
5075
5076
5077
5078
5079
5080
5081
5082
5083
5084
5085
5086
5087
5088
5089
5090
5091
5092
5093
5094
5095
5096
5097
5098
5099
5100
5101
5102
5103
5104
5105
5106
5107
5108
5109
5110
5111
5112
5113
5114
5115
5116
5117
5118
5119
5120
5121
5122
5123
5124
5125
5126
5127
5128
5129
5130
5131
5132
5133
5134
5135
5136
5137
5138
5139
5140
5141
5142
5143
5144
5145
5146
5147
5148
5149
5150
5151
5152
5153
5154
5155
5156
5157
5158
5159
5160
5161
5162
5163
5164
5165
5166
5167
5168
5169
5170
5171
5172
5173
5174
5175
5176
5177
5178
5179
5180
5181
5182
5183
5184
5185
5186
5187
5188
5189
5190
5191
5192
5193
5194
5195
5196
5197
5198
5199
5200
5201
5202
5203
5204
5205
5206
5207
5208
5209
5210
5211
5212
5213
5214
5215
5216
5217
5218
5219
5220
5221
5222
5223
5224
5225
5226
5227
5228
5229
5230
5231
5232
5233
5234
5235
5236
5237
5238
5239
5240
5241
5242
5243
5244
5245
5246
5247
5248
5249
5250
5251
5252
5253
5254
5255
5256
5257
5258
5259
5260
5261
5262
5263
5264
5265
5266
5267
5268
5269
5270
5271
5272
5273
5274
5275
5276
5277
5278
5279
5280
5281
5282
5283
5284
5285
5286
5287
5288
5289
5290
5291
5292
5293
5294
5295
5296
5297
5298
5299
5300
5301
5302
5303
5304
5305
5306
5307
5308
5309
5310
5311
5312
5313
5314
5315
5316
5317
5318
5319
5320
5321
5322
5323
5324
5325
5326
5327
5328
5329
5330
5331
5332
5333
5334
5335
5336
5337
5338
5339
5340
5341
5342
5343
5344
5345
5346
5347
5348
5349
5350
5351
5352
5353
5354
5355
5356
5357
5358
5359
5360
5361
5362
5363
5364
5365
5366
5367
5368
5369
5370
5371
5372
5373
5374
5375
5376
5377
5378
5379
5380
5381
5382
5383
5384
5385
5386
5387
5388
5389
5390
5391
5392
5393
5394
5395
5396
5397
5398
5399
5400
5401
5402
5403
5404
5405
5406
5407
5408
5409
5410
5411
5412
5413
5414
5415
5416
5417
5418
5419
5420
5421
5422
5423
5424
5425
5426
5427
5428
5429
5430
5431
5432
5433
5434
5435
5436
5437
5438
5439
5440
5441
5442
5443
5444
5445
5446
5447
5448
5449
5450
5451
5452
5453
5454
5455
5456
5457
5458
5459
5460
5461
5462
5463
5464
5465
5466
5467
5468
5469
5470
5471
5472
5473
5474
5475
5476
5477
5478
5479
5480
5481
5482
5483
5484
5485
5486
5487
5488
5489
5490
5491
5492
5493
5494
5495
5496
5497
5498
5499
5500
5501
5502
5503
5504
5505
5506
5507
5508
5509
5510
5511
5512
5513
5514
5515
5516
5517
5518
5519
5520
5521
5522
5523
5524
5525
5526
5527
5528
5529
5530
5531
5532
5533
5534
5535
5536
5537
5538
5539
5540
5541
5542
5543
5544
5545
5546
5547
5548
5549
5550
5551
5552
5553
5554
5555
5556
5557
5558
5559
5560
5561
5562
5563
5564
5565
5566
5567
5568
5569
5570
5571
5572
5573
5574
5575
5576
5577
5578
5579
5580
5581
5582
5583
5584
5585
5586
5587
5588
5589
5590
5591
5592
5593
5594
5595
5596
5597
5598
5599
5600
5601
5602
5603
5604
5605
5606
5607
5608
5609
5610
5611
5612
5613
5614
5615
5616
5617
5618
5619
5620
5621
5622
5623
5624
5625
5626
5627
5628
5629
5630
5631
5632
5633
5634
5635
5636
5637
5638
5639
5640
5641
5642
5643
5644
5645
5646
5647
5648
5649
5650
5651
5652
5653
5654
5655
5656
5657
5658
5659
5660
5661
5662
5663
5664
5665
5666
5667
5668
5669
5670
5671
5672
5673
5674
5675
5676
5677
5678
5679
5680
5681
5682
5683
5684
5685
5686
5687
5688
5689
5690
5691
5692
5693
5694
5695
5696
5697
5698
5699
5700
5701
5702
5703
5704
5705
5706
5707
5708
5709
5710
5711
5712
5713
5714
5715
5716
5717
5718
5719
5720
5721
5722
5723
5724
5725
5726
5727
5728
5729
5730
5731
5732
5733
5734
5735
5736
5737
5738
5739
5740
5741
5742
5743
5744
5745
5746
5747
5748
5749
5750
5751
5752
5753
5754
5755
5756
5757
5758
5759
5760
5761
5762
5763
5764
5765
5766
5767
5768
5769
5770
5771
5772
5773
5774
5775
5776
5777
5778
5779
5780
5781
5782
5783
5784
5785
5786
5787
5788
5789
5790
5791
5792
5793
5794
5795
5796
5797
5798
5799
5800
5801
5802
5803
5804
5805
5806
5807
5808
5809
5810
5811
5812
5813
5814
5815
5816
5817
5818
5819
5820
5821
5822
5823
5824
5825
5826
5827
5828
5829
5830
5831
5832
5833
5834
5835
5836
5837
5838
5839
5840
5841
5842
5843
5844
5845
5846
5847
5848
5849
5850
5851
5852
5853
5854
5855
5856
5857
5858
5859
5860
5861
5862
5863
5864
5865
5866
5867
5868
5869
5870
5871
5872
5873
5874
5875
5876
5877
5878
5879
5880
5881
5882
5883
5884
5885
5886
5887
5888
5889
5890
5891
5892
5893
5894
5895
5896
5897
5898
5899
5900
5901
5902
5903
5904
5905
5906
5907
5908
5909
5910
5911
5912
5913
5914
5915
5916
5917
5918
5919
5920
5921
5922
5923
5924
5925
5926
5927
5928
5929
5930
5931
5932
5933
5934
5935
5936
5937
5938
5939
5940
5941
5942
5943
5944
5945
5946
5947
5948
5949
5950
5951
5952
5953
5954
5955
5956
5957
5958
5959
5960
5961
5962
5963
5964
5965
5966
5967
5968
5969
5970
5971
5972
5973
5974
5975
5976
5977
5978
5979
5980
5981
5982
5983
5984
5985
5986
5987
5988
5989
5990
5991
5992
5993
5994
5995
5996
5997
5998
5999
6000
6001
6002
6003
6004
6005
6006
6007
6008
6009
6010
6011
6012
6013
6014
6015
6016
6017
6018
6019
6020
6021
6022
6023
6024
6025
6026
6027
6028
6029
6030
6031
6032
6033
6034
6035
6036
6037
6038
6039
6040
6041
6042
6043
6044
6045
6046
6047
6048
6049
6050
6051
6052
6053
6054
6055
6056
6057
6058
6059
6060
6061
6062
6063
6064
6065
6066
6067
6068
6069
6070
6071
6072
6073
6074
6075
6076
6077
6078
6079
6080
6081
6082
6083
6084
6085
6086
6087
6088
6089
6090
6091
6092
6093
6094
6095
6096
6097
6098
6099
6100
6101
6102
6103
6104
6105
6106
6107
6108
6109
6110
6111
6112
6113
6114
6115
6116
6117
6118
6119
6120
6121
6122
6123
6124
6125
6126
6127
6128
6129
6130
6131
6132
6133
6134
6135
6136
6137
6138
6139
6140
6141
6142
6143
6144
6145
6146
6147
6148
6149
6150
6151
6152
6153
6154
6155
6156
6157
6158
6159
6160
6161
6162
6163
6164
6165
6166
6167
6168
6169
6170
6171
6172
6173
6174
6175
6176
6177
6178
6179
6180
6181
6182
6183
6184
6185
6186
6187
6188
6189
6190
6191
6192
6193
6194
6195
6196
6197
6198
6199
6200
6201
6202
6203
6204
6205
6206
6207
6208
6209
6210
6211
6212
6213
6214
6215
6216
6217
6218
6219
6220
6221
6222
6223
6224
6225
6226
6227
6228
6229
6230
6231
6232
6233
6234
6235
6236
6237
6238
6239
6240
6241
6242
6243
6244
6245
6246
6247
6248
6249
6250
6251
6252
6253
6254
6255
6256
6257
6258
6259
6260
6261
6262
6263
6264
6265
6266
6267
6268
6269
6270
6271
6272
6273
6274
6275
6276
6277
6278
6279
6280
6281
6282
6283
6284
6285
6286
6287
6288
6289
6290
6291
6292
6293
6294
6295
6296
6297
6298
6299
6300
6301
6302
6303
6304
6305
6306
6307
6308
6309
6310
6311
6312
6313
6314
6315
6316
6317
6318
6319
6320
6321
6322
6323
6324
6325
6326
6327
6328
6329
6330
6331
6332
6333
6334
6335
6336
6337
6338
6339
6340
6341
6342
6343
6344
6345
6346
6347
6348
6349
6350
6351
6352
6353
6354
6355
6356
6357
6358
6359
6360
6361
6362
6363
6364
6365
6366
6367
6368
6369
6370
6371
6372
6373
6374
6375
6376
6377
6378
6379
6380
6381
6382
6383
6384
6385
6386
6387
6388
6389
6390
6391
6392
6393
6394
6395
6396
6397
6398
6399
6400
6401
6402
6403
6404
6405
6406
6407
6408
6409
6410
6411
6412
6413
6414
6415
6416
6417
6418
6419
6420
6421
6422
6423
6424
6425
6426
6427
6428
6429
6430
6431
6432
6433
6434
6435
6436
6437
6438
6439
6440
6441
6442
6443
6444
6445
6446
6447
6448
6449
6450
6451
6452
6453
6454
6455
6456
6457
6458
6459
6460
6461
6462
6463
6464
6465
6466
6467
6468
6469
6470
6471
6472
6473
6474
6475
6476
6477
6478
6479
6480
6481
6482
6483
6484
6485
6486
6487
6488
6489
6490
6491
6492
6493
6494
6495
6496
6497
6498
6499
6500
6501
6502
6503
6504
6505
6506
6507
6508
6509
6510
6511
6512
6513
6514
6515
6516
6517
6518
6519
6520
6521
6522
6523
6524
6525
6526
6527
6528
6529
6530
6531
6532
6533
6534
6535
6536
6537
6538
6539
6540
6541
6542
6543
6544
6545
6546
6547
6548
6549
6550
6551
6552
6553
6554
6555
6556
6557
6558
6559
6560
6561
6562
6563
6564
6565
6566
6567
6568
6569
6570
6571
6572
6573
6574
6575
6576
6577
6578
6579
6580
6581
6582
6583
6584
6585
6586
6587
6588
6589
6590
6591
6592
6593
6594
6595
6596
6597
6598
6599
6600
6601
6602
6603
6604
6605
6606
6607
6608
6609
6610
6611
6612
6613
6614
6615
6616
6617
6618
6619
6620
6621
6622
6623
6624
6625
6626
6627
6628
6629
6630
6631
6632
6633
6634
6635
6636
6637
6638
6639
6640
6641
6642
6643
6644
6645
6646
6647
6648
6649
6650
6651
6652
6653
6654
6655
6656
6657
6658
6659
6660
6661
6662
6663
6664
6665
6666
6667
6668
6669
6670
6671
6672
6673
6674
6675
6676
6677
6678
6679
6680
6681
6682
6683
6684
6685
6686
6687
6688
6689
6690
6691
6692
6693
6694
6695
6696
6697
6698
6699
6700
6701
6702
6703
6704
6705
6706
6707
6708
6709
6710
6711
6712
6713
6714
6715
6716
6717
6718
6719
6720
6721
6722
6723
6724
6725
6726
6727
6728
6729
6730
6731
6732
6733
6734
6735
6736
6737
6738
6739
6740
6741
6742
6743
6744
6745
6746
6747
6748
6749
6750
6751
6752
6753
6754
6755
6756
6757
6758
6759
6760
6761
6762
6763
6764
6765
6766
6767
6768
6769
6770
6771
6772
6773
6774
6775
6776
6777
6778
6779
6780
6781
6782
6783
6784
6785
6786
6787
6788
6789
6790
6791
6792
6793
6794
6795
6796
6797
6798
6799
6800
6801
6802
6803
6804
6805
6806
6807
6808
6809
6810
6811
6812
6813
6814
6815
6816
6817
6818
6819
6820
6821
6822
6823
6824
6825
6826
6827
6828
6829
6830
6831
6832
6833
6834
6835
6836
6837
6838
6839
6840
6841
6842
6843
6844
6845
6846
6847
6848
6849
6850
6851
6852
6853
6854
6855
6856
6857
6858
6859
6860
6861
6862
6863
6864
6865
6866
6867
6868
6869
6870
6871
6872
6873
6874
6875
6876
6877
6878
6879
6880
6881
6882
6883
6884
6885
6886
6887
6888
6889
6890
6891
6892
6893
6894
6895
6896
6897
6898
6899
6900
6901
6902
6903
6904
6905
6906
6907
6908
6909
6910
6911
6912
6913
6914
6915
6916
6917
6918
6919
6920
6921
6922
6923
6924
6925
6926
6927
6928
6929
6930
6931
6932
6933
6934
6935
6936
6937
6938
6939
6940
6941
6942
6943
6944
6945
6946
6947
6948
6949
6950
6951
6952
6953
6954
6955
6956
6957
6958
6959
6960
6961
6962
6963
6964
6965
6966
6967
6968
6969
6970
6971
6972
6973
6974
6975
6976
6977
6978
6979
6980
6981
6982
6983
6984
6985
6986
6987
6988
6989
6990
6991
6992
6993
6994
6995
6996
6997
6998
6999
7000
7001
7002
7003
7004
7005
7006
7007
7008
7009
7010
7011
7012
7013
7014
7015
7016
7017
7018
7019
7020
7021
7022
7023
7024
7025
7026
7027
7028
7029
7030
7031
7032
7033
7034
7035
7036
7037
7038
7039
7040
7041
7042
7043
7044
7045
7046
7047
7048
7049
7050
7051
7052
7053
7054
7055
7056
7057
7058
7059
7060
7061
7062
7063
7064
7065
7066
7067
7068
7069
7070
7071
7072
7073
7074
7075
7076
7077
7078
7079
7080
7081
7082
7083
7084
7085
7086
7087
7088
7089
7090
7091
7092
7093
7094
7095
7096
7097
7098
7099
7100
7101
7102
7103
7104
7105
7106
7107
7108
7109
7110
7111
7112
7113
7114
7115
7116
7117
7118
7119
7120
7121
7122
7123
7124
7125
7126
7127
7128
7129
7130
7131
7132
7133
7134
7135
7136
7137
7138
7139
7140
7141
7142
7143
7144
7145
7146
7147
7148
7149
7150
7151
7152
7153
7154
7155
7156
7157
7158
7159
7160
7161
7162
7163
7164
7165
7166
7167
7168
7169
7170
7171
7172
7173
7174
7175
7176
7177
7178
7179
7180
7181
7182
7183
7184
7185
7186
7187
7188
7189
7190
7191
7192
7193
7194
7195
7196
7197
7198
7199
7200
7201
7202
7203
7204
7205
7206
7207
7208
7209
7210
7211
7212
7213
7214
7215
7216
7217
7218
7219
7220
7221
7222
7223
7224
7225
7226
7227
7228
7229
7230
7231
7232
7233
7234
7235
7236
7237
7238
7239
7240
7241
7242
7243
7244
7245
7246
7247
7248
7249
7250
7251
7252
7253
7254
7255
7256
7257
7258
7259
7260
7261
7262
7263
7264
7265
7266
7267
7268
7269
7270
7271
7272
7273
7274
7275
7276
7277
7278
7279
7280
7281
7282
7283
7284
7285
7286
7287
7288
7289
7290
7291
7292
7293
7294
7295
7296
7297
7298
7299
7300
7301
7302
7303
7304
7305
7306
7307
7308
7309
7310
7311
7312
7313
7314
7315
7316
7317
7318
7319
7320
7321
7322
7323
7324
7325
7326
7327
7328
7329
7330
7331
7332
7333
7334
7335
7336
7337
7338
7339
7340
7341
7342
7343
7344
7345
7346
7347
7348
7349
7350
7351
7352
7353
7354
7355
7356
7357
7358
7359
7360
7361
7362
7363
7364
7365
7366
7367
7368
7369
7370
7371
7372
7373
7374
7375
7376
7377
7378
7379
7380
7381
7382
7383
7384
7385
7386
7387
7388
7389
7390
7391
7392
7393
7394
7395
7396
7397
7398
7399
7400
7401
7402
7403
7404
7405
7406
7407
7408
7409
7410
7411
7412
7413
7414
7415
7416
7417
7418
7419
7420
7421
7422
7423
7424
7425
7426
7427
7428
7429
7430
7431
7432
7433
7434
7435
7436
7437
7438
7439
7440
7441
7442
7443
7444
7445
7446
7447
7448
7449
7450
7451
7452
7453
7454
7455
7456
7457
7458
7459
7460
7461
7462
7463
7464
7465
7466
7467
7468
7469
7470
7471
7472
7473
7474
7475
7476
7477
7478
7479
7480
7481
7482
7483
7484
7485
7486
7487
7488
7489
7490
7491
7492
7493
7494
7495
7496
7497
7498
7499
7500
7501
7502
7503
7504
7505
7506
7507
7508
7509
7510
7511
7512
7513
7514
7515
7516
7517
7518
7519
7520
7521
7522
7523
7524
7525
7526
7527
7528
7529
7530
7531
7532
7533
7534
7535
7536
7537
7538
7539
7540
7541
7542
7543
7544
7545
7546
7547
7548
7549
7550
7551
7552
7553
7554
7555
7556
7557
7558
7559
7560
7561
7562
7563
7564
7565
7566
7567
7568
7569
7570
7571
7572
7573
7574
7575
7576
7577
7578
7579
7580
7581
7582
7583
7584
7585
7586
7587
7588
7589
7590
7591
7592
7593
7594
7595
7596
7597
7598
7599
7600
7601
7602
7603
7604
7605
7606
7607
7608
7609
7610
7611
7612
7613
7614
7615
7616
7617
7618
7619
7620
7621
7622
7623
7624
7625
7626
7627
7628
7629
7630
7631
7632
7633
7634
7635
7636
7637
7638
7639
7640
7641
7642
7643
7644
7645
7646
7647
7648
7649
7650
7651
7652
7653
7654
7655
7656
7657
7658
7659
7660
7661
7662
7663
7664
7665
7666
7667
7668
7669
7670
7671
7672
7673
7674
7675
7676
7677
7678
7679
7680
7681
7682
7683
7684
7685
7686
7687
7688
7689
7690
7691
7692
7693
7694
7695
7696
7697
7698
7699
7700
7701
7702
7703
7704
7705
7706
7707
7708
7709
7710
7711
7712
7713
7714
7715
7716
7717
7718
7719
7720
7721
7722
7723
7724
7725
7726
7727
7728
7729
7730
7731
7732
7733
7734
7735
7736
7737
7738
7739
7740
7741
7742
7743
7744
7745
7746
7747
7748
7749
7750
7751
7752
7753
7754
7755
7756
7757
7758
7759
7760
7761
7762
7763
7764
7765
7766
7767
7768
7769
7770
7771
7772
7773
7774
7775
7776
7777
7778
7779
7780
7781
7782
7783
7784
7785
7786
7787
7788
7789
7790
7791
7792
7793
7794
7795
7796
7797
7798
7799
7800
7801
7802
7803
7804
7805
7806
7807
7808
7809
7810
7811
7812
7813
7814
7815
7816
7817
7818
7819
7820
7821
7822
7823
7824
7825
7826
7827
7828
7829
7830
7831
7832
7833
7834
7835
7836
7837
7838
7839
7840
7841
7842
7843
7844
7845
7846
7847
7848
7849
7850
7851
7852
7853
7854
7855
7856
7857
7858
7859
7860
7861
7862
7863
7864
7865
7866
7867
7868
7869
7870
7871
7872
7873
7874
7875
7876
7877
7878
7879
7880
7881
7882
7883
7884
7885
7886
7887
7888
7889
7890
7891
7892
7893
7894
7895
7896
7897
7898
7899
7900
7901
7902
7903
7904
7905
7906
7907
7908
7909
7910
7911
7912
7913
7914
7915
7916
7917
7918
7919
7920
7921
7922
7923
7924
7925
7926
7927
7928
7929
7930
7931
7932
7933
7934
7935
7936
7937
7938
7939
7940
7941
7942
7943
7944
7945
7946
7947
7948
7949
7950
7951
7952
7953
7954
7955
7956
7957
7958
7959
7960
7961
7962
7963
7964
7965
7966
7967
7968
7969
7970
7971
7972
7973
7974
7975
7976
7977
7978
7979
7980
7981
7982
7983
7984
7985
7986
7987
7988
7989
7990
7991
7992
7993
7994
7995
7996
7997
7998
7999
8000
8001
8002
8003
8004
8005
8006
8007
8008
8009
8010
8011
8012
8013
8014
8015
8016
8017
8018
8019
8020
8021
8022
8023
8024
8025
8026
8027
8028
8029
8030
8031
8032
8033
8034
8035
8036
8037
8038
8039
8040
8041
8042
8043
8044
8045
8046
8047
8048
8049
8050
8051
8052
8053
8054
8055
8056
8057
8058
8059
8060
8061
8062
8063
8064
8065
8066
8067
8068
8069
8070
8071
8072
8073
8074
8075
8076
8077
8078
8079
8080
8081
8082
8083
8084
8085
8086
8087
8088
8089
8090
8091
8092
8093
8094
8095
8096
8097
8098
8099
8100
8101
8102
8103
8104
8105
8106
8107
8108
8109
8110
8111
8112
8113
8114
8115
8116
8117
8118
8119
8120
8121
8122
8123
8124
8125
8126
8127
8128
8129
8130
8131
8132
8133
8134
8135
8136
8137
8138
8139
8140
8141
8142
8143
8144
8145
8146
8147
8148
8149
8150
8151
8152
8153
8154
8155
8156
8157
8158
8159
8160
8161
8162
8163
8164
8165
8166
8167
8168
8169
8170
8171
8172
8173
8174
8175
8176
8177
8178
8179
8180
8181
8182
8183
8184
8185
8186
8187
8188
8189
8190
8191
8192
8193
8194
8195
8196
8197
8198
8199
8200
8201
8202
8203
8204
8205
8206
8207
8208
8209
8210
8211
8212
8213
8214
8215
8216
8217
8218
8219
8220
8221
8222
8223
8224
8225
8226
8227
8228
8229
8230
8231
8232
8233
8234
8235
8236
8237
8238
8239
8240
8241
8242
8243
8244
8245
8246
8247
8248
8249
8250
8251
8252
8253
8254
8255
8256
8257
8258
8259
8260
8261
8262
8263
8264
8265
8266
8267
8268
8269
8270
8271
8272
8273
8274
8275
8276
8277
8278
8279
8280
8281
8282
8283
8284
8285
8286
8287
8288
8289
8290
8291
8292
8293
8294
8295
8296
8297
8298
8299
8300
8301
8302
8303
8304
8305
8306
8307
8308
8309
8310
8311
8312
8313
8314
8315
8316
8317
8318
8319
8320
8321
8322
8323
8324
8325
8326
8327
8328
8329
8330
8331
8332
8333
8334
8335
8336
8337
8338
8339
8340
8341
8342
8343
8344
8345
8346
8347
8348
8349
8350
8351
8352
8353
8354
8355
8356
8357
8358
8359
8360
8361
8362
8363
8364
8365
8366
8367
8368
8369
8370
8371
8372
8373
8374
8375
8376
8377
8378
8379
8380
8381
8382
8383
8384
8385
8386
8387
8388
8389
8390
8391
8392
8393
8394
8395
8396
8397
8398
8399
8400
8401
8402
8403
8404
8405
8406
8407
8408
8409
8410
8411
8412
8413
8414
8415
8416
8417
8418
8419
8420
8421
8422
8423
8424
8425
8426
8427
8428
8429
8430
8431
8432
8433
8434
8435
8436
8437
8438
8439
8440
8441
8442
8443
8444
8445
8446
8447
8448
8449
8450
8451
8452
8453
8454
8455
8456
8457
8458
8459
8460
8461
8462
8463
8464
8465
8466
8467
8468
8469
8470
8471
8472
8473
8474
8475
8476
8477
8478
8479
8480
8481
8482
8483
8484
8485
8486
8487
8488
8489
8490
8491
8492
8493
8494
8495
8496
8497
8498
8499
8500
8501
8502
8503
8504
8505
8506
8507
8508
8509
8510
8511
8512
8513
8514
8515
8516
8517
8518
8519
8520
8521
8522
8523
8524
8525
8526
8527
8528
8529
8530
8531
8532
8533
8534
8535
8536
8537
8538
8539
8540
8541
8542
8543
8544
8545
8546
8547
8548
8549
8550
8551
8552
8553
8554
8555
8556
8557
8558
8559
8560
8561
8562
8563
8564
8565
8566
8567
8568
8569
8570
8571
8572
8573
8574
8575
8576
8577
8578
8579
8580
8581
8582
8583
8584
8585
8586
8587
8588
8589
8590
8591
8592
8593
8594
8595
8596
8597
8598
8599
8600
8601
8602
8603
8604
8605
8606
8607
8608
8609
8610
8611
8612
8613
8614
8615
8616
8617
8618
8619
8620
8621
8622
8623
8624
8625
8626
8627
8628
8629
8630
8631
8632
8633
8634
8635
8636
8637
8638
8639
8640
8641
8642
8643
8644
8645
8646
8647
8648
8649
8650
8651
8652
8653
8654
8655
8656
8657
8658
8659
8660
8661
8662
8663
8664
8665
8666
8667
8668
8669
8670
8671
8672
8673
8674
8675
8676
8677
8678
8679
8680
8681
8682
8683
8684
8685
8686
8687
8688
8689
8690
8691
8692
8693
8694
8695
8696
8697
8698
8699
8700
8701
8702
8703
8704
8705
8706
8707
8708
8709
8710
8711
8712
8713
8714
8715
8716
8717
8718
8719
8720
8721
8722
8723
8724
8725
8726
8727
8728
8729
8730
8731
8732
8733
8734
8735
8736
8737
8738
8739
8740
8741
8742
8743
8744
8745
8746
8747
8748
8749
8750
8751
8752
8753
8754
8755
8756
8757
8758
8759
8760
8761
8762
8763
8764
8765
8766
8767
8768
8769
8770
8771
8772
8773
8774
8775
8776
8777
8778
8779
8780
8781
8782
8783
8784
8785
8786
8787
8788
8789
8790
8791
8792
8793
8794
8795
8796
8797
8798
8799
8800
8801
8802
8803
8804
8805
8806
8807
8808
8809
8810
8811
8812
8813
8814
8815
8816
8817
8818
8819
8820
8821
8822
8823
8824
8825
8826
8827
8828
8829
8830
8831
8832
8833
8834
8835
8836
8837
8838
8839
8840
8841
8842
8843
8844
8845
8846
8847
8848
8849
8850
8851
8852
8853
8854
8855
8856
8857
8858
8859
8860
8861
8862
8863
8864
8865
8866
8867
8868
8869
8870
8871
8872
8873
8874
8875
8876
8877
8878
8879
8880
8881
8882
8883
8884
8885
8886
8887
8888
8889
8890
8891
8892
8893
8894
8895
8896
8897
8898
8899
8900
8901
8902
8903
8904
8905
8906
8907
8908
8909
8910
8911
8912
8913
8914
8915
8916
8917
8918
8919
8920
8921
8922
8923
8924
8925
8926
8927
8928
8929
8930
8931
8932
8933
8934
8935
8936
8937
8938
8939
8940
8941
8942
8943
8944
8945
8946
8947
8948
8949
8950
8951
8952
8953
8954
8955
8956
8957
8958
8959
8960
8961
8962
8963
8964
8965
8966
8967
8968
8969
8970
8971
8972
8973
8974
8975
8976
8977
8978
8979
8980
8981
8982
8983
8984
8985
8986
8987
8988
8989
8990
8991
8992
8993
8994
8995
8996
8997
8998
8999
9000
9001
9002
9003
9004
9005
9006
9007
9008
9009
9010
9011
9012
9013
9014
9015
9016
9017
9018
9019
9020
9021
9022
9023
9024
9025
9026
9027
9028
9029
9030
9031
9032
9033
9034
9035
9036
9037
9038
9039
9040
9041
9042
9043
9044
9045
9046
9047
9048
9049
9050
9051
9052
9053
9054
9055
9056
9057
9058
9059
9060
9061
9062
9063
9064
9065
9066
9067
9068
9069
9070
9071
9072
9073
9074
9075
9076
9077
9078
9079
9080
9081
9082
9083
9084
9085
9086
9087
9088
9089
9090
9091
9092
9093
9094
9095
9096
9097
9098
9099
9100
9101
9102
9103
9104
9105
9106
9107
9108
9109
9110
9111
9112
9113
9114
9115
9116
9117
9118
9119
9120
9121
9122
9123
9124
9125
9126
9127
9128
9129
9130
9131
9132
9133
9134
9135
9136
9137
9138
9139
9140
9141
9142
9143
9144
9145
9146
9147
9148
9149
9150
9151
9152
9153
9154
9155
9156
9157
9158
9159
9160
9161
9162
9163
9164
9165
9166
9167
9168
9169
9170
9171
9172
9173
9174
9175
9176
9177
9178
9179
9180
9181
9182
9183
9184
9185
9186
9187
9188
9189
9190
9191
9192
9193
9194
9195
9196
9197
9198
9199
9200
9201
9202
9203
9204
9205
9206
9207
9208
9209
9210
9211
9212
9213
9214
9215
9216
9217
9218
9219
9220
9221
9222
9223
9224
9225
9226
9227
9228
9229
9230
9231
9232
9233
9234
9235
9236
9237
9238
9239
9240
9241
9242
9243
9244
9245
9246
9247
9248
9249
9250
9251
9252
9253
9254
9255
9256
9257
9258
9259
9260
9261
9262
9263
9264
9265
9266
9267
9268
9269
9270
9271
9272
9273
9274
9275
9276
9277
9278
9279
9280
9281
9282
9283
9284
9285
9286
9287
9288
9289
9290
9291
9292
9293
9294
9295
9296
9297
9298
9299
9300
9301
9302
9303
9304
9305
9306
9307
9308
9309
9310
9311
9312
9313
9314
9315
9316
9317
9318
9319
9320
9321
9322
9323
9324
9325
9326
9327
9328
9329
9330
9331
9332
9333
9334
9335
9336
9337
9338
9339
9340
9341
9342
9343
9344
9345
9346
9347
9348
9349
9350
9351
9352
9353
9354
9355
9356
9357
9358
9359
9360
9361
9362
9363
9364
9365
9366
9367
9368
9369
9370
9371
9372
9373
9374
9375
9376
9377
9378
9379
9380
9381
9382
9383
9384
9385
9386
9387
9388
9389
9390
9391
9392
9393
9394
9395
9396
9397
9398
9399
9400
9401
9402
9403
9404
9405
9406
9407
9408
9409
9410
9411
9412
9413
9414
9415
9416
9417
9418
9419
9420
9421
9422
9423
9424
9425
9426
9427
9428
9429
9430
9431
9432
9433
9434
9435
9436
9437
9438
9439
9440
9441
9442
9443
9444
9445
9446
9447
9448
9449
9450
9451
9452
9453
9454
9455
9456
9457
9458
9459
9460
9461
9462
9463
9464
9465
9466
9467
9468
9469
9470
9471
9472
9473
9474
9475
9476
9477
9478
9479
9480
9481
9482
9483
9484
9485
9486
9487
9488
9489
9490
9491
9492
9493
9494
9495
9496
9497
9498
9499
9500
9501
9502
9503
9504
9505
9506
9507
9508
9509
9510
9511
9512
9513
9514
9515
9516
9517
9518
9519
9520
9521
9522
9523
9524
9525
9526
9527
9528
9529
9530
9531
9532
9533
9534
9535
9536
9537
9538
9539
9540
9541
9542
9543
9544
9545
9546
9547
9548
9549
9550
9551
9552
9553
9554
9555
9556
9557
9558
9559
9560
9561
9562
9563
9564
9565
9566
9567
9568
9569
9570
9571
9572
9573
9574
9575
9576
9577
9578
9579
9580
9581
9582
9583
9584
9585
9586
9587
9588
9589
9590
9591
9592
9593
9594
9595
9596
9597
9598
9599
9600
9601
9602
9603
9604
9605
9606
9607
9608
9609
9610
9611
9612
9613
9614
9615
9616
9617
9618
9619
9620
9621
9622
9623
9624
9625
9626
9627
9628
9629
9630
9631
9632
9633
9634
9635
9636
9637
9638
9639
9640
9641
9642
9643
9644
9645
9646
9647
9648
9649
9650
9651
9652
9653
9654
9655
9656
9657
9658
9659
9660
9661
9662
9663
9664
9665
9666
9667
9668
9669
9670
9671
9672
9673
9674
9675
9676
9677
9678
9679
9680
9681
9682
9683
9684
9685
9686
9687
9688
9689
9690
9691
9692
9693
9694
9695
9696
9697
9698
9699
9700
9701
9702
9703
9704
9705
9706
9707
9708
9709
9710
9711
9712
9713
9714
9715
9716
9717
9718
9719
9720
9721
9722
9723
9724
9725
9726
9727
9728
9729
9730
9731
9732
9733
9734
9735
9736
9737
9738
9739
9740
9741
9742
9743
9744
9745
9746
9747
9748
9749
9750
9751
9752
9753
9754
9755
9756
9757
9758
9759
9760
9761
9762
9763
9764
9765
9766
9767
9768
9769
9770
9771
9772
9773
9774
9775
9776
9777
9778
9779
9780
9781
9782
9783
9784
9785
9786
9787
9788
9789
9790
9791
9792
9793
9794
9795
9796
9797
9798
9799
9800
9801
9802
9803
9804
9805
9806
9807
9808
9809
9810
9811
9812
9813
9814
9815
9816
9817
9818
9819
9820
9821
9822
9823
9824
9825
9826
9827
9828
9829
9830
9831
9832
9833
9834
9835
9836
9837
9838
9839
9840
9841
9842
9843
9844
9845
9846
9847
9848
9849
9850
9851
9852
9853
9854
9855
9856
9857
9858
9859
9860
9861
9862
9863
9864
9865
9866
9867
9868
9869
9870
9871
9872
9873
9874
9875
9876
9877
9878
9879
9880
9881
9882
9883
9884
9885
9886
9887
9888
9889
9890
9891
9892
9893
9894
9895
9896
9897
9898
9899
9900
9901
9902
9903
9904
9905
9906
9907
9908
9909
9910
9911
9912
9913
9914
9915
9916
9917
9918
9919
9920
9921
9922
9923
9924
9925
9926
9927
9928
9929
9930
9931
9932
9933
9934
9935
9936
9937
9938
9939
9940
9941
9942
9943
9944
9945
9946
9947
9948
9949
9950
9951
9952
9953
9954
9955
9956
9957
9958
9959
9960
9961
9962
9963
9964
9965
9966
9967
9968
9969
9970
9971
9972
9973
9974
9975
9976
9977
9978
9979
9980
9981
9982
9983
9984
9985
9986
9987
9988
9989
9990
9991
9992
9993
9994
9995
9996
9997
9998
9999
10000
10001
10002
10003
10004
10005
10006
10007
10008
10009
10010
10011
10012
10013
10014
10015
10016
10017
10018
10019
10020
10021
10022
10023
10024
10025
10026
10027
10028
10029
10030
10031
10032
10033
10034
10035
10036
10037
10038
10039
10040
10041
10042
10043
10044
10045
10046
10047
10048
10049
10050
10051
10052
10053
10054
10055
10056
10057
10058
10059
10060
10061
10062
10063
10064
10065
10066
10067
10068
10069
10070
10071
10072
10073
10074
10075
10076
10077
10078
10079
10080
10081
10082
10083
10084
10085
10086
10087
10088
10089
10090
10091
10092
10093
10094
10095
10096
10097
10098
10099
10100
10101
10102
10103
10104
10105
10106
10107
10108
10109
10110
10111
10112
10113
10114
10115
10116
10117
10118
10119
10120
10121
10122
10123
10124
10125
10126
10127
10128
10129
10130
10131
10132
10133
10134
10135
10136
10137
10138
10139
10140
10141
10142
10143
10144
10145
10146
10147
10148
10149
10150
10151
10152
10153
10154
10155
10156
10157
10158
10159
10160
10161
10162
10163
10164
10165
10166
10167
10168
10169
10170
10171
10172
10173
10174
10175
10176
10177
10178
10179
10180
10181
10182
10183
10184
10185
10186
10187
10188
10189
10190
10191
10192
10193
10194
10195
10196
10197
10198
10199
10200
10201
10202
10203
10204
10205
10206
10207
10208
10209
10210
10211
10212
10213
10214
10215
10216
10217
10218
10219
10220
10221
10222
10223
10224
10225
10226
10227
10228
10229
10230
10231
10232
10233
10234
10235
10236
10237
10238
10239
10240
10241
10242
10243
10244
10245
10246
10247
10248
10249
10250
10251
10252
10253
10254
10255
10256
10257
10258
10259
10260
10261
10262
10263
10264
10265
10266
10267
10268
10269
10270
10271
10272
10273
10274
10275
10276
10277
10278
10279
10280
10281
10282
10283
10284
10285
10286
10287
10288
10289
10290
10291
10292
10293
10294
10295
10296
10297
10298
10299
10300
10301
10302
10303
10304
10305
10306
10307
10308
10309
10310
10311
10312
10313
10314
10315
10316
10317
10318
10319
10320
10321
10322
10323
10324
10325
10326
10327
10328
10329
10330
10331
10332
10333
10334
10335
10336
10337
10338
10339
10340
10341
10342
10343
10344
10345
10346
10347
10348
10349
10350
10351
10352
10353
10354
10355
10356
10357
10358
10359
10360
10361
10362
10363
10364
10365
10366
10367
10368
10369
10370
10371
10372
10373
10374
10375
10376
10377
10378
10379
10380
10381
10382
10383
10384
10385
10386
10387
10388
10389
10390
10391
10392
10393
10394
10395
10396
10397
10398
10399
10400
10401
10402
10403
10404
10405
10406
10407
10408
10409
10410
10411
10412
10413
10414
10415
10416
10417
10418
10419
10420
10421
10422
10423
10424
10425
10426
10427
10428
10429
10430
10431
10432
10433
10434
10435
10436
10437
10438
10439
10440
10441
10442
10443
10444
10445
10446
10447
10448
10449
10450
10451
10452
10453
10454
10455
10456
10457
10458
10459
10460
10461
10462
10463
10464
10465
10466
10467
10468
10469
10470
10471
10472
10473
10474
10475
10476
10477
10478
10479
10480
10481
10482
10483
10484
10485
10486
10487
10488
10489
10490
10491
10492
10493
10494
10495
10496
10497
10498
10499
10500
10501
10502
10503
10504
10505
10506
10507
10508
10509
10510
10511
10512
10513
10514
10515
10516
10517
10518
10519
10520
10521
10522
10523
10524
10525
10526
10527
10528
10529
10530
10531
10532
10533
10534
10535
10536
10537
10538
10539
10540
10541
10542
10543
10544
10545
10546
10547
10548
10549
10550
10551
10552
10553
10554
10555
10556
10557
10558
10559
10560
10561
10562
10563
10564
10565
10566
10567
10568
10569
10570
10571
10572
10573
10574
10575
10576
10577
10578
10579
10580
10581
10582
10583
10584
10585
10586
10587
10588
10589
10590
10591
10592
10593
10594
10595
10596
10597
10598
10599
10600
10601
10602
10603
10604
10605
10606
10607
10608
10609
10610
10611
10612
10613
10614
10615
10616
10617
10618
10619
10620
10621
10622
10623
10624
10625
10626
10627
10628
10629
10630
10631
10632
10633
10634
10635
10636
10637
10638
10639
10640
10641
10642
10643
10644
10645
10646
10647
10648
10649
10650
10651
10652
10653
10654
10655
10656
10657
10658
10659
10660
10661
10662
10663
10664
10665
10666
10667
10668
10669
10670
10671
10672
10673
10674
10675
10676
10677
10678
10679
10680
10681
10682
10683
10684
10685
10686
10687
10688
10689
10690
10691
10692
10693
10694
10695
10696
10697
10698
10699
10700
10701
10702
10703
10704
10705
10706
10707
10708
10709
10710
10711
10712
10713
10714
10715
10716
10717
10718
10719
10720
10721
10722
10723
10724
10725
10726
10727
10728
10729
10730
10731
10732
10733
10734
10735
10736
10737
10738
10739
10740
10741
10742
10743
10744
10745
10746
10747
10748
10749
10750
10751
10752
10753
10754
10755
10756
10757
10758
10759
10760
10761
10762
10763
10764
10765
10766
10767
10768
10769
10770
10771
10772
10773
10774
10775
10776
10777
10778
10779
10780
10781
10782
10783
10784
10785
10786
10787
10788
10789
10790
10791
10792
10793
10794
10795
10796
10797
10798
10799
10800
10801
10802
10803
10804
10805
10806
10807
10808
10809
10810
10811
10812
10813
10814
10815
10816
10817
10818
10819
10820
10821
10822
10823
10824
10825
10826
10827
10828
10829
10830
10831
10832
10833
10834
10835
10836
10837
10838
10839
10840
10841
10842
10843
10844
10845
10846
10847
10848
10849
10850
10851
10852
10853
10854
10855
10856
10857
10858
10859
10860
10861
10862
10863
10864
10865
10866
10867
10868
10869
10870
10871
10872
10873
10874
10875
10876
10877
10878
10879
10880
10881
10882
10883
10884
10885
10886
10887
10888
10889
10890
10891
10892
10893
10894
10895
10896
10897
10898
10899
10900
10901
10902
10903
10904
10905
10906
10907
10908
10909
10910
10911
10912
10913
10914
10915
10916
10917
10918
10919
10920
10921
10922
10923
10924
10925
10926
10927
10928
10929
10930
10931
10932
10933
10934
10935
10936
10937
10938
10939
10940
10941
10942
10943
10944
10945
10946
10947
10948
10949
10950
10951
10952
10953
10954
10955
10956
10957
10958
10959
10960
10961
10962
10963
10964
10965
10966
10967
10968
10969
10970
10971
10972
10973
10974
10975
10976
10977
10978
10979
10980
10981
10982
10983
10984
10985
10986
10987
10988
10989
10990
10991
10992
10993
10994
10995
10996
10997
10998
10999
11000
11001
11002
11003
11004
11005
11006
11007
11008
11009
11010
11011
11012
11013
11014
11015
11016
11017
11018
11019
11020
11021
11022
11023
11024
11025
11026
11027
11028
11029
11030
11031
11032
11033
11034
11035
11036
11037
11038
11039
11040
11041
11042
11043
11044
11045
11046
11047
11048
11049
11050
11051
11052
11053
11054
11055
11056
11057
11058
11059
11060
11061
11062
11063
11064
11065
11066
11067
11068
11069
11070
11071
11072
11073
11074
11075
11076
11077
11078
11079
11080
11081
11082
11083
11084
11085
11086
11087
11088
11089
11090
11091
11092
11093
11094
11095
11096
11097
11098
11099
11100
11101
11102
11103
11104
11105
11106
11107
11108
11109
11110
11111
11112
11113
11114
11115
11116
11117
11118
11119
11120
11121
11122
11123
11124
11125
11126
11127
11128
11129
11130
11131
11132
11133
11134
11135
11136
11137
11138
11139
11140
11141
11142
11143
11144
11145
11146
11147
11148
11149
11150
11151
11152
11153
11154
11155
11156
11157
11158
11159
11160
11161
11162
11163
11164
11165
11166
11167
11168
11169
11170
11171
11172
11173
11174
11175
11176
11177
11178
11179
11180
11181
11182
11183
11184
11185
11186
11187
11188
11189
11190
11191
11192
11193
11194
11195
11196
11197
11198
11199
11200
11201
11202
11203
11204
11205
11206
11207
11208
11209
11210
11211
11212
11213
11214
11215
11216
11217
11218
11219
11220
11221
11222
11223
11224
11225
11226
11227
11228
11229
11230
11231
11232
11233
11234
11235
11236
11237
11238
11239
11240
11241
11242
11243
11244
11245
11246
11247
11248
11249
11250
11251
11252
11253
11254
11255
11256
11257
11258
11259
11260
11261
11262
11263
11264
11265
11266
11267
11268
11269
11270
11271
11272
11273
11274
11275
11276
11277
11278
11279
11280
11281
11282
11283
11284
11285
11286
11287
11288
11289
11290
11291
11292
11293
11294
11295
11296
11297
11298
11299
11300
11301
11302
11303
11304
11305
11306
11307
11308
11309
11310
11311
11312
11313
11314
11315
11316
11317
11318
11319
11320
11321
11322
11323
11324
11325
11326
11327
11328
11329
11330
11331
11332
11333
11334
11335
11336
11337
11338
11339
11340
11341
11342
11343
11344
11345
11346
11347
11348
11349
11350
11351
11352
11353
11354
11355
11356
11357
11358
11359
11360
11361
11362
11363
11364
11365
11366
11367
11368
11369
11370
11371
11372
11373
11374
11375
11376
11377
11378
11379
11380
11381
11382
11383
11384
11385
11386
11387
11388
11389
11390
11391
11392
11393
11394
11395
11396
11397
11398
11399
11400
11401
11402
11403
11404
11405
11406
11407
11408
11409
11410
11411
11412
11413
11414
11415
11416
11417
11418
11419
11420
11421
11422
11423
11424
11425
11426
11427
11428
11429
11430
11431
11432
11433
11434
11435
11436
11437
11438
11439
11440
11441
11442
11443
11444
11445
11446
11447
11448
11449
11450
11451
11452
11453
11454
11455
11456
11457
11458
11459
11460
11461
11462
11463
11464
11465
11466
11467
11468
11469
11470
11471
11472
11473
11474
11475
11476
11477
11478
11479
11480
11481
11482
11483
11484
11485
11486
11487
11488
11489
11490
11491
11492
11493
11494
11495
11496
11497
11498
11499
11500
11501
11502
11503
11504
11505
11506
11507
11508
11509
11510
11511
11512
11513
11514
11515
11516
11517
11518
11519
11520
11521
11522
11523
11524
11525
11526
11527
11528
11529
11530
11531
11532
11533
11534
11535
11536
11537
11538
11539
11540
11541
11542
11543
11544
11545
11546
11547
11548
11549
11550
11551
11552
11553
11554
11555
11556
11557
11558
11559
11560
11561
11562
11563
11564
11565
11566
11567
11568
11569
11570
11571
11572
11573
11574
11575
11576
11577
11578
11579
11580
11581
11582
11583
11584
11585
11586
11587
11588
11589
11590
11591
11592
11593
11594
11595
11596
11597
11598
11599
11600
11601
11602
11603
11604
11605
11606
11607
11608
11609
11610
11611
11612
11613
11614
11615
11616
11617
11618
11619
11620
11621
11622
11623
11624
11625
11626
11627
11628
11629
11630
11631
11632
11633
11634
11635
11636
11637
11638
11639
11640
11641
11642
11643
11644
11645
11646
11647
11648
11649
11650
11651
11652
11653
11654
11655
11656
11657
11658
11659
11660
11661
11662
11663
11664
11665
11666
11667
11668
11669
11670
11671
11672
11673
11674
11675
11676
11677
11678
11679
11680
11681
11682
11683
11684
11685
11686
11687
11688
11689
11690
11691
11692
11693
11694
11695
11696
11697
11698
11699
11700
11701
11702
11703
11704
11705
11706
11707
11708
11709
11710
11711
11712
11713
11714
11715
11716
11717
11718
11719
11720
11721
11722
11723
11724
11725
11726
11727
11728
11729
11730
11731
11732
11733
11734
11735
11736
11737
11738
11739
11740
11741
11742
11743
11744
11745
11746
11747
11748
11749
11750
11751
11752
11753
11754
11755
11756
11757
11758
11759
11760
11761
11762
11763
11764
11765
11766
11767
11768
11769
11770
11771
11772
11773
11774
11775
11776
11777
11778
11779
11780
11781
11782
11783
11784
11785
11786
11787
11788
11789
11790
11791
11792
11793
11794
11795
11796
11797
11798
11799
11800
11801
11802
11803
11804
11805
11806
11807
11808
11809
11810
11811
11812
11813
11814
11815
11816
11817
11818
11819
11820
11821
11822
11823
11824
11825
11826
11827
11828
11829
11830
11831
11832
11833
11834
11835
11836
11837
11838
11839
11840
11841
11842
11843
11844
11845
11846
11847
11848
11849
11850
11851
11852
11853
11854
11855
11856
11857
11858
11859
11860
11861
11862
11863
11864
11865
11866
11867
11868
11869
11870
11871
11872
11873
11874
11875
11876
11877
11878
11879
11880
11881
11882
11883
11884
11885
11886
11887
11888
11889
11890
11891
11892
11893
11894
11895
11896
11897
11898
11899
11900
11901
11902
11903
11904
11905
11906
11907
11908
11909
11910
11911
11912
11913
11914
11915
11916
11917
11918
11919
11920
11921
11922
11923
11924
11925
11926
11927
11928
11929
11930
11931
11932
11933
11934
11935
11936
11937
11938
11939
11940
11941
11942
11943
11944
11945
11946
11947
11948
11949
11950
11951
11952
11953
11954
11955
11956
11957
11958
11959
11960
11961
11962
11963
11964
11965
11966
11967
11968
11969
11970
11971
11972
11973
11974
11975
11976
11977
11978
11979
11980
11981
11982
11983
11984
11985
11986
11987
11988
11989
11990
11991
11992
11993
11994
11995
11996
11997
11998
11999
12000
12001
12002
12003
12004
12005
12006
12007
12008
12009
12010
12011
12012
12013
12014
12015
12016
12017
12018
12019
12020
12021
12022
12023
12024
12025
12026
12027
12028
12029
12030
12031
12032
12033
12034
12035
12036
12037
12038
12039
12040
12041
12042
12043
12044
12045
12046
12047
12048
12049
12050
12051
12052
12053
12054
12055
12056
12057
12058
12059
12060
12061
12062
12063
12064
12065
12066
12067
12068
12069
12070
12071
12072
12073
12074
12075
12076
12077
12078
12079
12080
12081
12082
12083
12084
12085
12086
12087
12088
12089
12090
12091
12092
12093
12094
12095
12096
12097
12098
12099
12100
12101
12102
12103
12104
12105
12106
12107
12108
12109
12110
12111
12112
12113
12114
12115
12116
12117
12118
12119
12120
12121
12122
12123
12124
12125
12126
12127
12128
12129
12130
12131
12132
12133
12134
12135
12136
12137
12138
12139
12140
12141
12142
12143
12144
12145
12146
12147
12148
12149
12150
12151
12152
12153
12154
12155
12156
12157
12158
12159
12160
12161
12162
12163
12164
12165
12166
12167
12168
12169
12170
12171
12172
12173
12174
12175
12176
12177
12178
12179
12180
12181
12182
12183
12184
12185
12186
12187
12188
12189
12190
12191
12192
12193
12194
12195
12196
12197
12198
12199
12200
12201
12202
12203
12204
12205
12206
12207
12208
12209
12210
12211
12212
12213
12214
12215
12216
12217
12218
12219
12220
12221
12222
12223
12224
12225
12226
12227
12228
12229
12230
12231
12232
12233
12234
12235
12236
12237
12238
12239
12240
12241
12242
12243
12244
12245
12246
12247
12248
12249
12250
12251
12252
12253
12254
12255
12256
12257
12258
12259
12260
12261
12262
12263
12264
12265
12266
12267
12268
12269
12270
12271
12272
12273
12274
12275
12276
12277
12278
12279
12280
12281
12282
12283
12284
12285
12286
12287
12288
12289
12290
12291
12292
12293
12294
12295
12296
12297
12298
12299
12300
12301
12302
12303
12304
12305
12306
12307
12308
12309
12310
12311
12312
12313
12314
12315
12316
12317
12318
12319
12320
12321
12322
12323
12324
12325
12326
12327
12328
12329
12330
12331
12332
12333
12334
12335
12336
12337
12338
12339
12340
12341
12342
12343
12344
12345
12346
12347
12348
12349
12350
12351
12352
12353
12354
12355
12356
12357
12358
12359
12360
12361
12362
12363
12364
12365
12366
12367
12368
12369
12370
12371
12372
12373
12374
12375
12376
12377
12378
12379
12380
12381
12382
12383
12384
12385
12386
12387
12388
12389
12390
12391
12392
12393
12394
12395
12396
12397
12398
12399
12400
12401
12402
12403
12404
12405
12406
12407
12408
12409
12410
12411
12412
12413
12414
12415
12416
12417
12418
12419
12420
12421
12422
12423
12424
12425
12426
12427
12428
12429
12430
12431
12432
12433
12434
12435
12436
12437
12438
12439
12440
12441
12442
12443
12444
12445
12446
12447
12448
12449
12450
12451
12452
12453
12454
12455
12456
12457
12458
12459
12460
12461
12462
12463
12464
12465
12466
12467
12468
12469
12470
12471
12472
12473
12474
12475
12476
12477
12478
12479
12480
12481
12482
12483
12484
12485
12486
12487
12488
12489
12490
12491
12492
12493
12494
12495
12496
12497
12498
12499
12500
12501
12502
12503
12504
12505
12506
12507
12508
12509
12510
12511
12512
12513
12514
12515
12516
12517
12518
12519
12520
12521
12522
12523
12524
12525
12526
12527
12528
12529
12530
12531
12532
12533
12534
12535
12536
12537
12538
12539
12540
12541
12542
12543
12544
12545
12546
12547
12548
12549
12550
12551
12552
12553
12554
12555
12556
12557
12558
12559
12560
12561
12562
12563
12564
12565
12566
12567
12568
12569
12570
12571
12572
12573
12574
12575
12576
12577
12578
12579
12580
12581
12582
12583
12584
12585
12586
12587
12588
12589
12590
12591
12592
12593
12594
12595
12596
12597
12598
12599
12600
12601
12602
12603
12604
12605
12606
12607
12608
12609
12610
12611
12612
12613
12614
12615
12616
12617
12618
12619
12620
12621
12622
12623
12624
12625
12626
12627
12628
12629
12630
12631
12632
12633
12634
12635
12636
12637
12638
12639
12640
12641
12642
12643
12644
12645
12646
12647
12648
12649
12650
12651
12652
12653
12654
12655
12656
12657
12658
12659
12660
12661
12662
12663
12664
12665
12666
12667
12668
12669
12670
12671
12672
12673
12674
12675
12676
12677
12678
12679
12680
12681
12682
12683
12684
12685
12686
12687
12688
12689
12690
12691
12692
12693
12694
12695
12696
12697
12698
12699
12700
12701
12702
12703
12704
12705
12706
12707
12708
12709
12710
12711
12712
12713
12714
12715
12716
12717
12718
12719
12720
12721
12722
12723
12724
12725
12726
12727
12728
12729
12730
12731
12732
12733
12734
12735
12736
12737
12738
12739
12740
12741
12742
12743
12744
12745
12746
12747
12748
12749
12750
12751
12752
12753
12754
12755
12756
12757
12758
12759
12760
12761
12762
12763
12764
12765
12766
12767
12768
12769
12770
12771
12772
12773
12774
12775
12776
12777
12778
12779
12780
12781
12782
12783
12784
12785
12786
12787
12788
12789
12790
12791
12792
12793
12794
12795
12796
12797
12798
12799
12800
12801
12802
12803
12804
12805
12806
12807
12808
12809
12810
12811
12812
12813
12814
12815
12816
12817
12818
12819
12820
12821
12822
12823
12824
12825
12826
12827
12828
12829
12830
12831
12832
12833
12834
12835
12836
12837
12838
12839
12840
12841
12842
12843
12844
12845
12846
12847
12848
12849
12850
12851
12852
12853
12854
12855
12856
12857
12858
12859
12860
12861
12862
12863
12864
12865
12866
12867
12868
12869
12870
12871
12872
12873
12874
12875
12876
12877
12878
12879
12880
12881
12882
12883
12884
12885
12886
12887
12888
12889
12890
12891
12892
12893
12894
12895
12896
12897
12898
12899
12900
12901
12902
12903
12904
12905
12906
12907
12908
12909
12910
12911
12912
12913
12914
12915
12916
12917
12918
12919
12920
12921
12922
12923
12924
12925
12926
12927
12928
12929
12930
12931
12932
12933
12934
12935
12936
12937
12938
12939
12940
12941
12942
12943
12944
12945
12946
12947
12948
12949
12950
12951
12952
12953
12954
12955
12956
12957
12958
12959
12960
12961
12962
12963
12964
12965
12966
12967
12968
12969
12970
12971
12972
12973
12974
12975
12976
12977
12978
12979
12980
12981
12982
12983
12984
12985
12986
12987
12988
12989
12990
12991
12992
12993
12994
12995
12996
12997
12998
12999
13000
13001
13002
13003
13004
13005
13006
13007
13008
13009
13010
13011
13012
13013
13014
13015
13016
13017
13018
13019
13020
13021
13022
13023
13024
13025
13026
13027
13028
13029
13030
13031
13032
13033
13034
13035
13036
13037
13038
13039
13040
13041
13042
13043
13044
13045
13046
13047
13048
13049
13050
13051
13052
13053
13054
13055
13056
13057
13058
13059
13060
13061
13062
13063
13064
13065
13066
13067
13068
13069
13070
13071
13072
13073
13074
13075
13076
13077
13078
13079
13080
13081
13082
13083
13084
13085
13086
13087
13088
13089
13090
13091
13092
13093
13094
13095
13096
13097
13098
13099
13100
13101
13102
13103
13104
13105
13106
13107
13108
13109
13110
13111
13112
13113
13114
13115
13116
13117
13118
13119
13120
13121
13122
13123
13124
13125
13126
13127
13128
13129
13130
13131
13132
13133
13134
13135
13136
13137
13138
13139
13140
13141
13142
13143
13144
13145
13146
13147
13148
13149
13150
13151
13152
13153
13154
13155
13156
13157
13158
13159
13160
13161
13162
13163
13164
13165
13166
13167
13168
13169
13170
13171
13172
13173
13174
13175
13176
13177
13178
13179
13180
13181
13182
13183
13184
13185
13186
13187
13188
13189
13190
13191
13192
13193
13194
13195
13196
13197
13198
13199
13200
13201
13202
13203
13204
13205
13206
13207
13208
13209
13210
13211
13212
13213
13214
13215
13216
13217
13218
13219
13220
13221
13222
13223
13224
13225
13226
13227
13228
13229
13230
13231
13232
13233
13234
13235
13236
13237
13238
13239
13240
13241
13242
13243
13244
13245
13246
13247
13248
13249
13250
13251
13252
13253
13254
13255
13256
13257
13258
13259
13260
13261
13262
13263
13264
13265
13266
13267
13268
13269
13270
13271
13272
13273
13274
13275
13276
13277
13278
13279
13280
13281
13282
13283
13284
13285
13286
13287
13288
13289
13290
13291
13292
13293
13294
13295
13296
13297
13298
13299
13300
13301
13302
13303
13304
13305
13306
13307
13308
13309
13310
13311
13312
13313
13314
13315
13316
13317
13318
13319
13320
13321
13322
13323
13324
13325
13326
13327
13328
13329
13330
13331
13332
13333
13334
13335
13336
13337
13338
13339
13340
13341
13342
13343
13344
13345
13346
13347
13348
13349
13350
13351
13352
13353
13354
13355
13356
13357
13358
13359
13360
13361
13362
13363
13364
13365
13366
13367
13368
13369
13370
13371
13372
13373
13374
13375
13376
13377
13378
13379
13380
13381
13382
13383
13384
13385
13386
13387
13388
13389
13390
13391
13392
13393
13394
13395
13396
13397
13398
13399
13400
13401
13402
13403
13404
13405
13406
13407
13408
13409
13410
13411
13412
13413
13414
13415
13416
13417
13418
13419
13420
13421
13422
13423
13424
13425
13426
13427
13428
13429
13430
13431
13432
13433
13434
13435
13436
13437
13438
13439
13440
13441
13442
13443
13444
13445
13446
13447
13448
13449
13450
13451
13452
13453
13454
13455
13456
13457
13458
13459
13460
13461
13462
13463
13464
13465
13466
13467
13468
13469
13470
13471
13472
13473
13474
13475
13476
13477
13478
13479
13480
13481
13482
13483
13484
13485
13486
13487
13488
13489
13490
13491
13492
13493
13494
13495
13496
13497
13498
13499
13500
13501
13502
13503
13504
13505
13506
13507
13508
13509
13510
13511
13512
13513
13514
13515
13516
13517
13518
13519
13520
13521
13522
13523
13524
13525
13526
13527
13528
13529
13530
13531
13532
13533
13534
13535
13536
13537
13538
13539
13540
13541
13542
13543
13544
13545
13546
13547
13548
13549
13550
13551
13552
13553
13554
13555
13556
13557
13558
13559
13560
13561
13562
13563
13564
13565
13566
13567
13568
13569
13570
13571
13572
13573
13574
13575
13576
13577
13578
13579
13580
13581
13582
13583
13584
13585
13586
13587
13588
13589
13590
13591
13592
13593
13594
13595
13596
13597
13598
13599
13600
13601
13602
13603
13604
13605
13606
13607
13608
13609
13610
13611
13612
13613
13614
13615
13616
13617
13618
13619
13620
13621
13622
13623
13624
13625
13626
13627
13628
13629
13630
13631
13632
13633
13634
13635
13636
13637
13638
13639
13640
13641
13642
13643
13644
13645
13646
13647
13648
13649
13650
13651
13652
13653
13654
13655
13656
13657
13658
13659
13660
13661
13662
13663
13664
13665
13666
13667
13668
13669
13670
13671
13672
13673
13674
13675
13676
13677
13678
13679
13680
13681
13682
13683
13684
13685
13686
13687
13688
13689
13690
13691
13692
13693
13694
13695
13696
13697
13698
13699
13700
13701
13702
13703
13704
13705
13706
13707
13708
13709
13710
13711
13712
13713
13714
13715
13716
13717
13718
13719
13720
13721
13722
13723
13724
13725
13726
13727
13728
13729
13730
13731
13732
13733
13734
13735
13736
13737
13738
13739
13740
13741
13742
13743
13744
13745
13746
13747
13748
13749
13750
13751
13752
13753
13754
13755
13756
13757
13758
13759
13760
13761
13762
13763
13764
13765
13766
13767
13768
13769
13770
13771
13772
13773
13774
13775
13776
13777
13778
13779
13780
13781
13782
13783
13784
13785
13786
13787
13788
13789
13790
13791
13792
13793
13794
13795
13796
13797
13798
13799
13800
13801
13802
13803
13804
13805
13806
13807
13808
13809
13810
13811
13812
13813
13814
13815
13816
13817
13818
13819
13820
13821
13822
13823
13824
13825
13826
13827
13828
13829
13830
13831
13832
13833
13834
13835
13836
13837
13838
13839
13840
13841
13842
13843
13844
13845
13846
13847
13848
13849
13850
13851
13852
13853
13854
13855
13856
13857
13858
13859
13860
13861
13862
13863
13864
13865
13866
13867
13868
13869
13870
13871
13872
13873
13874
13875
13876
13877
13878
13879
13880
13881
13882
13883
13884
13885
13886
13887
13888
13889
13890
13891
13892
13893
13894
13895
13896
13897
13898
13899
13900
13901
13902
13903
13904
13905
13906
13907
13908
13909
13910
13911
13912
13913
13914
13915
13916
13917
13918
13919
13920
13921
13922
13923
13924
13925
13926
13927
13928
13929
13930
13931
13932
13933
13934
13935
13936
13937
13938
13939
13940
13941
13942
13943
13944
13945
13946
13947
13948
13949
13950
13951
13952
13953
13954
13955
13956
13957
13958
13959
13960
13961
13962
13963
13964
13965
13966
13967
13968
13969
13970
13971
13972
13973
13974
13975
13976
13977
13978
13979
13980
13981
13982
13983
13984
13985
13986
13987
13988
13989
13990
13991
13992
13993
13994
13995
13996
13997
13998
13999
14000
14001
14002
14003
14004
14005
14006
14007
14008
14009
14010
14011
14012
14013
14014
14015
14016
14017
14018
14019
14020
14021
14022
14023
14024
14025
14026
14027
14028
14029
14030
14031
14032
14033
14034
14035
14036
14037
14038
14039
14040
14041
14042
14043
14044
14045
14046
14047
14048
14049
14050
14051
14052
14053
14054
14055
14056
14057
14058
14059
14060
14061
14062
14063
14064
14065
14066
14067
14068
14069
14070
14071
14072
14073
14074
14075
14076
14077
14078
14079
14080
14081
14082
14083
14084
14085
14086
14087
14088
14089
14090
14091
14092
14093
14094
14095
14096
14097
14098
14099
14100
14101
14102
14103
14104
14105
14106
14107
14108
14109
14110
14111
14112
14113
14114
14115
14116
14117
14118
14119
14120
14121
14122
14123
14124
14125
14126
14127
14128
14129
14130
14131
14132
14133
14134
14135
14136
14137
14138
14139
14140
14141
14142
14143
14144
14145
14146
14147
14148
14149
14150
14151
14152
14153
14154
14155
14156
14157
14158
14159
14160
14161
14162
14163
14164
14165
14166
14167
14168
14169
14170
14171
14172
14173
14174
14175
14176
14177
14178
14179
14180
14181
14182
14183
14184
14185
14186
14187
14188
14189
14190
14191
14192
14193
14194
14195
14196
14197
14198
14199
14200
14201
14202
14203
14204
14205
14206
14207
14208
14209
14210
14211
14212
14213
14214
14215
14216
14217
14218
14219
14220
14221
14222
14223
14224
14225
14226
14227
14228
14229
14230
14231
14232
14233
14234
14235
14236
14237
14238
14239
14240
14241
14242
14243
14244
14245
14246
14247
14248
14249
14250
14251
14252
14253
14254
14255
14256
14257
14258
14259
14260
14261
14262
14263
14264
14265
14266
14267
14268
14269
14270
14271
14272
14273
14274
14275
14276
14277
14278
14279
14280
14281
14282
14283
14284
14285
14286
14287
14288
14289
14290
14291
14292
14293
14294
14295
14296
14297
14298
14299
14300
14301
14302
14303
14304
14305
14306
14307
14308
14309
14310
14311
14312
14313
14314
14315
14316
14317
14318
14319
14320
14321
14322
14323
14324
14325
14326
14327
14328
14329
14330
14331
14332
14333
14334
14335
14336
14337
14338
14339
14340
14341
14342
14343
14344
14345
14346
14347
14348
14349
14350
14351
14352
14353
14354
14355
14356
14357
14358
14359
14360
14361
14362
14363
14364
14365
14366
14367
14368
14369
14370
14371
14372
14373
14374
14375
14376
14377
14378
14379
14380
14381
14382
14383
14384
14385
14386
14387
14388
14389
14390
14391
14392
14393
14394
14395
14396
14397
14398
14399
14400
14401
14402
14403
14404
14405
14406
14407
14408
14409
14410
14411
14412
14413
14414
14415
14416
14417
14418
14419
14420
14421
14422
14423
14424
14425
14426
14427
14428
14429
14430
14431
14432
14433
14434
14435
14436
14437
14438
14439
14440
14441
14442
14443
14444
14445
14446
14447
14448
14449
14450
14451
14452
14453
14454
14455
14456
14457
14458
14459
14460
14461
14462
14463
14464
14465
14466
14467
14468
14469
14470
14471
14472
14473
14474
14475
14476
14477
14478
14479
14480
14481
14482
14483
14484
14485
14486
14487
14488
14489
14490
14491
14492
14493
14494
14495
14496
14497
14498
14499
14500
14501
14502
14503
14504
14505
14506
14507
14508
14509
14510
14511
14512
14513
14514
14515
14516
14517
14518
14519
14520
14521
14522
14523
14524
14525
14526
14527
14528
14529
14530
14531
14532
14533
14534
14535
14536
14537
14538
14539
14540
14541
14542
14543
14544
14545
14546
14547
14548
14549
14550
14551
14552
14553
14554
14555
14556
14557
14558
14559
14560
14561
14562
14563
14564
14565
14566
14567
14568
14569
14570
14571
14572
14573
14574
14575
14576
14577
14578
14579
14580
14581
14582
14583
14584
14585
14586
14587
14588
14589
14590
14591
14592
14593
14594
14595
14596
14597
14598
14599
14600
14601
14602
14603
14604
14605
14606
14607
14608
14609
14610
14611
14612
14613
14614
14615
14616
14617
14618
14619
14620
14621
14622
14623
14624
14625
14626
14627
14628
14629
14630
14631
14632
14633
14634
14635
14636
14637
14638
14639
14640
14641
14642
14643
14644
14645
14646
14647
14648
14649
14650
14651
14652
14653
14654
14655
14656
14657
14658
14659
14660
14661
14662
14663
14664
14665
14666
14667
14668
14669
14670
14671
14672
14673
14674
14675
14676
14677
14678
14679
14680
14681
14682
14683
14684
14685
14686
14687
14688
14689
14690
14691
14692
14693
14694
14695
14696
14697
14698
14699
14700
14701
14702
14703
14704
14705
14706
14707
14708
14709
14710
14711
14712
14713
14714
14715
14716
14717
14718
14719
14720
14721
14722
14723
14724
14725
14726
14727
14728
14729
14730
14731
14732
14733
14734
14735
14736
14737
14738
14739
14740
14741
14742
14743
14744
14745
14746
14747
14748
14749
14750
14751
14752
14753
14754
14755
14756
14757
14758
14759
14760
14761
14762
14763
14764
14765
14766
14767
14768
14769
14770
14771
14772
14773
14774
14775
14776
14777
14778
14779
14780
14781
14782
14783
14784
14785
14786
14787
14788
14789
14790
14791
14792
14793
14794
14795
14796
14797
14798
14799
14800
14801
14802
14803
14804
14805
14806
14807
14808
14809
14810
14811
14812
14813
14814
14815
14816
14817
14818
14819
14820
14821
14822
14823
14824
14825
14826
14827
14828
14829
14830
14831
14832
14833
14834
14835
14836
14837
14838
14839
14840
14841
14842
14843
14844
14845
14846
14847
14848
14849
14850
14851
14852
14853
14854
14855
14856
14857
14858
14859
14860
14861
14862
14863
14864
14865
14866
14867
14868
14869
14870
14871
14872
14873
14874
14875
14876
14877
14878
14879
14880
14881
14882
14883
14884
14885
14886
14887
14888
14889
14890
14891
14892
14893
14894
14895
14896
14897
14898
14899
14900
14901
14902
14903
14904
14905
14906
14907
14908
14909
14910
14911
14912
14913
14914
14915
14916
14917
14918
14919
14920
14921
14922
14923
14924
14925
14926
14927
14928
14929
14930
14931
14932
14933
14934
14935
14936
14937
14938
14939
14940
14941
14942
14943
14944
14945
14946
14947
14948
14949
14950
14951
14952
14953
14954
14955
14956
14957
14958
14959
14960
14961
14962
14963
14964
14965
14966
14967
14968
14969
14970
14971
14972
14973
14974
14975
14976
14977
14978
14979
14980
14981
14982
14983
14984
14985
14986
14987
14988
14989
14990
14991
14992
14993
14994
14995
14996
14997
14998
14999
15000
15001
15002
15003
15004
15005
15006
15007
15008
15009
15010
15011
15012
15013
15014
15015
15016
15017
15018
15019
15020
15021
15022
15023
15024
15025
15026
15027
15028
15029
15030
15031
15032
15033
15034
15035
15036
15037
15038
15039
15040
15041
15042
15043
15044
15045
15046
15047
15048
15049
15050
15051
15052
15053
15054
15055
15056
15057
15058
15059
15060
15061
15062
15063
15064
15065
15066
15067
15068
15069
15070
15071
15072
15073
15074
15075
15076
15077
15078
15079
15080
15081
15082
15083
15084
15085
15086
15087
15088
15089
15090
15091
15092
15093
15094
15095
15096
15097
15098
15099
15100
15101
15102
15103
15104
15105
15106
15107
15108
15109
15110
15111
15112
15113
15114
15115
15116
15117
15118
15119
15120
15121
15122
15123
15124
15125
15126
15127
15128
15129
15130
15131
15132
15133
15134
15135
15136
15137
15138
15139
15140
15141
15142
15143
15144
15145
15146
15147
15148
15149
15150
15151
15152
15153
15154
15155
15156
15157
15158
15159
15160
15161
15162
15163
15164
15165
15166
15167
15168
15169
15170
15171
15172
15173
15174
15175
15176
15177
15178
15179
15180
15181
15182
15183
15184
15185
15186
15187
15188
15189
15190
15191
15192
15193
15194
15195
15196
15197
15198
15199
15200
15201
15202
15203
15204
15205
15206
15207
15208
15209
15210
15211
15212
15213
15214
15215
15216
15217
15218
15219
15220
15221
15222
15223
15224
15225
15226
15227
15228
15229
15230
15231
15232
15233
15234
15235
15236
15237
15238
15239
15240
15241
15242
15243
15244
15245
15246
15247
15248
15249
15250
15251
15252
15253
15254
15255
15256
15257
15258
15259
15260
15261
15262
15263
15264
15265
15266
15267
15268
15269
15270
15271
15272
15273
15274
15275
15276
15277
15278
15279
15280
15281
15282
15283
15284
15285
15286
15287
15288
15289
15290
15291
15292
15293
15294
15295
15296
15297
15298
15299
15300
15301
15302
15303
15304
15305
15306
15307
15308
15309
15310
15311
15312
15313
15314
15315
15316
15317
15318
15319
15320
15321
15322
15323
15324
15325
15326
15327
15328
15329
15330
15331
15332
15333
15334
15335
15336
15337
15338
15339
15340
15341
15342
15343
15344
15345
15346
15347
15348
15349
15350
15351
15352
15353
15354
15355
15356
15357
15358
15359
15360
15361
15362
15363
15364
15365
15366
15367
15368
15369
15370
15371
15372
15373
15374
15375
15376
15377
15378
15379
15380
15381
15382
15383
15384
15385
15386
15387
15388
15389
15390
15391
15392
15393
15394
15395
15396
15397
15398
15399
15400
15401
15402
15403
15404
15405
15406
15407
15408
15409
15410
15411
15412
15413
15414
15415
15416
15417
15418
15419
15420
15421
15422
15423
15424
15425
15426
15427
15428
15429
15430
15431
15432
15433
15434
15435
15436
15437
15438
15439
15440
15441
15442
15443
15444
15445
15446
15447
15448
15449
15450
15451
15452
15453
15454
15455
15456
15457
15458
15459
15460
15461
15462
15463
15464
15465
15466
15467
15468
15469
15470
15471
15472
15473
15474
15475
15476
15477
15478
15479
15480
15481
15482
15483
15484
15485
15486
15487
15488
15489
15490
15491
15492
15493
15494
15495
15496
15497
15498
15499
15500
15501
15502
15503
15504
15505
15506
15507
15508
15509
15510
15511
15512
15513
15514
15515
15516
15517
15518
15519
15520
15521
15522
15523
15524
15525
15526
15527
15528
15529
15530
15531
15532
15533
15534
15535
15536
15537
15538
15539
15540
15541
15542
15543
15544
15545
15546
15547
15548
15549
15550
15551
15552
15553
15554
15555
15556
15557
15558
15559
15560
15561
15562
15563
15564
15565
15566
15567
15568
15569
15570
15571
15572
15573
15574
15575
15576
15577
15578
15579
15580
15581
15582
15583
15584
15585
15586
15587
15588
15589
15590
15591
15592
15593
15594
15595
15596
15597
15598
15599
15600
15601
15602
15603
15604
15605
15606
15607
15608
15609
15610
15611
15612
15613
15614
15615
15616
15617
15618
15619
15620
15621
15622
15623
15624
15625
15626
15627
15628
15629
15630
15631
15632
15633
15634
15635
15636
15637
15638
15639
15640
15641
15642
15643
15644
15645
15646
15647
15648
15649
15650
15651
15652
15653
15654
15655
15656
15657
15658
15659
15660
15661
15662
15663
15664
15665
15666
15667
15668
15669
15670
15671
15672
15673
15674
15675
15676
15677
15678
15679
15680
15681
15682
15683
15684
15685
15686
15687
15688
15689
15690
15691
15692
15693
15694
15695
15696
15697
15698
15699
15700
15701
15702
15703
15704
15705
15706
15707
15708
15709
15710
15711
15712
15713
15714
15715
15716
15717
15718
15719
15720
15721
15722
15723
15724
15725
15726
15727
15728
15729
15730
15731
15732
15733
15734
15735
15736
15737
15738
15739
15740
15741
15742
15743
15744
15745
15746
15747
15748
15749
15750
15751
15752
15753
15754
15755
15756
15757
15758
15759
15760
15761
15762
15763
15764
15765
15766
15767
15768
15769
15770
15771
15772
15773
15774
15775
15776
15777
15778
15779
15780
15781
15782
15783
15784
15785
15786
15787
15788
15789
15790
15791
15792
15793
15794
15795
15796
15797
15798
15799
15800
15801
15802
15803
15804
15805
15806
15807
15808
15809
15810
15811
15812
15813
15814
15815
15816
15817
15818
15819
15820
15821
15822
15823
15824
15825
15826
15827
15828
15829
15830
15831
15832
15833
15834
15835
15836
15837
15838
15839
15840
15841
15842
15843
15844
15845
15846
15847
15848
15849
15850
15851
15852
15853
15854
15855
15856
15857
15858
15859
15860
15861
15862
15863
15864
15865
15866
15867
15868
15869
15870
15871
15872
15873
15874
15875
15876
15877
15878
15879
15880
15881
15882
15883
15884
15885
15886
15887
15888
15889
15890
15891
15892
15893
15894
15895
15896
15897
15898
15899
15900
15901
15902
15903
15904
15905
15906
15907
15908
15909
15910
15911
15912
15913
15914
15915
15916
15917
15918
15919
15920
15921
15922
15923
15924
15925
15926
15927
15928
15929
15930
15931
15932
15933
15934
15935
15936
15937
15938
15939
15940
15941
15942
15943
15944
15945
15946
15947
15948
15949
15950
15951
15952
15953
15954
15955
15956
15957
15958
15959
15960
15961
15962
15963
15964
15965
15966
15967
15968
15969
15970
15971
15972
15973
15974
15975
15976
15977
15978
15979
15980
15981
15982
15983
15984
15985
15986
15987
15988
15989
15990
15991
15992
15993
15994
15995
15996
15997
15998
15999
16000
16001
16002
16003
16004
16005
16006
16007
16008
16009
16010
16011
16012
16013
16014
16015
16016
16017
16018
16019
16020
16021
16022
16023
16024
16025
16026
16027
16028
16029
16030
16031
16032
16033
16034
16035
16036
16037
16038
16039
16040
16041
16042
16043
16044
16045
16046
16047
16048
16049
16050
16051
16052
16053
16054
16055
16056
16057
16058
16059
16060
16061
16062
16063
16064
16065
16066
16067
16068
16069
16070
16071
16072
16073
16074
16075
16076
16077
16078
16079
16080
16081
16082
16083
16084
16085
16086
16087
16088
16089
16090
16091
16092
16093
16094
16095
16096
16097
16098
16099
16100
16101
16102
16103
16104
16105
16106
16107
16108
16109
16110
16111
16112
16113
16114
16115
16116
16117
16118
16119
16120
16121
16122
16123
16124
16125
16126
16127
16128
16129
16130
16131
16132
16133
16134
16135
16136
16137
16138
16139
16140
16141
16142
16143
16144
16145
16146
16147
16148
16149
16150
16151
16152
16153
16154
16155
16156
16157
16158
16159
16160
16161
16162
16163
16164
16165
16166
16167
16168
16169
16170
16171
16172
16173
16174
16175
16176
16177
16178
16179
16180
16181
16182
16183
16184
16185
16186
16187
16188
16189
16190
16191
16192
16193
16194
16195
16196
16197
16198
16199
16200
16201
16202
16203
16204
16205
16206
16207
16208
16209
16210
16211
16212
16213
16214
16215
16216
16217
16218
16219
16220
16221
16222
16223
16224
16225
16226
16227
16228
16229
16230
16231
16232
16233
16234
16235
16236
16237
16238
16239
16240
16241
16242
16243
16244
16245
16246
16247
16248
16249
16250
16251
16252
16253
16254
16255
16256
16257
16258
16259
16260
16261
16262
16263
16264
16265
16266
16267
16268
16269
16270
16271
16272
16273
16274
16275
16276
16277
16278
16279
16280
16281
16282
16283
16284
16285
16286
16287
16288
16289
16290
16291
16292
16293
16294
16295
16296
16297
16298
16299
16300
16301
16302
16303
16304
16305
16306
16307
16308
16309
16310
16311
16312
16313
16314
16315
16316
16317
16318
16319
16320
16321
16322
16323
16324
16325
16326
16327
16328
16329
16330
16331
16332
16333
16334
16335
16336
16337
16338
16339
16340
16341
16342
16343
16344
16345
16346
16347
16348
16349
16350
16351
16352
16353
16354
16355
16356
16357
16358
16359
16360
16361
16362
16363
16364
16365
16366
16367
16368
16369
16370
16371
16372
16373
16374
16375
16376
16377
16378
16379
16380
16381
16382
16383
16384
16385
16386
16387
16388
16389
16390
16391
16392
16393
16394
16395
16396
16397
16398
16399
16400
16401
16402
16403
16404
16405
16406
16407
16408
16409
16410
16411
16412
16413
16414
16415
16416
16417
16418
16419
16420
16421
16422
16423
16424
16425
16426
16427
16428
16429
16430
16431
16432
16433
16434
16435
16436
16437
16438
16439
16440
16441
16442
16443
16444
16445
16446
16447
16448
16449
16450
16451
16452
16453
16454
16455
16456
16457
16458
16459
16460
16461
16462
16463
16464
16465
16466
16467
16468
16469
16470
16471
16472
16473
16474
16475
16476
16477
16478
16479
16480
16481
16482
16483
16484
16485
16486
16487
16488
16489
16490
16491
16492
16493
16494
16495
16496
16497
16498
16499
16500
16501
16502
16503
16504
16505
16506
16507
16508
16509
16510
16511
16512
16513
16514
16515
16516
16517
16518
16519
16520
16521
16522
16523
16524
16525
16526
16527
16528
16529
16530
16531
16532
16533
16534
16535
16536
16537
16538
16539
16540
16541
16542
16543
16544
16545
16546
16547
16548
16549
16550
16551
16552
16553
16554
16555
16556
16557
16558
16559
16560
16561
16562
16563
16564
16565
16566
16567
16568
16569
16570
16571
16572
16573
16574
16575
16576
16577
16578
16579
16580
16581
16582
16583
16584
16585
16586
16587
16588
16589
16590
16591
16592
16593
16594
16595
16596
16597
16598
16599
16600
16601
16602
16603
16604
16605
16606
16607
16608
16609
16610
16611
16612
16613
16614
16615
16616
16617
16618
16619
16620
16621
16622
16623
16624
16625
16626
16627
16628
16629
16630
16631
16632
16633
16634
16635
16636
16637
16638
16639
16640
16641
16642
16643
16644
16645
16646
16647
16648
16649
16650
16651
16652
16653
16654
16655
16656
16657
16658
16659
16660
16661
16662
16663
16664
16665
16666
16667
16668
16669
16670
16671
16672
16673
16674
16675
16676
16677
16678
16679
16680
16681
16682
16683
16684
16685
16686
16687
16688
16689
16690
16691
16692
16693
16694
16695
16696
16697
16698
16699
16700
16701
16702
16703
16704
16705
16706
16707
16708
16709
16710
16711
16712
16713
16714
16715
16716
16717
16718
16719
16720
16721
16722
16723
16724
16725
16726
16727
16728
16729
16730
16731
16732
16733
16734
16735
16736
16737
16738
16739
16740
16741
16742
16743
16744
16745
16746
16747
16748
16749
16750
16751
16752
16753
16754
16755
16756
16757
16758
16759
16760
16761
16762
16763
16764
16765
16766
16767
16768
16769
16770
16771
16772
16773
16774
16775
16776
16777
16778
16779
16780
16781
16782
16783
16784
16785
16786
16787
16788
16789
16790
16791
16792
16793
16794
16795
16796
16797
16798
16799
16800
16801
16802
16803
16804
16805
16806
16807
16808
16809
16810
16811
16812
16813
16814
16815
16816
16817
16818
16819
16820
16821
16822
16823
16824
16825
16826
16827
16828
16829
16830
16831
16832
16833
16834
16835
16836
16837
16838
16839
16840
16841
16842
16843
16844
16845
16846
16847
16848
16849
16850
16851
16852
16853
16854
16855
16856
16857
16858
16859
16860
16861
16862
16863
16864
16865
16866
16867
16868
16869
16870
16871
16872
16873
16874
16875
16876
16877
16878
16879
16880
16881
16882
16883
16884
16885
16886
16887
16888
16889
16890
16891
16892
16893
16894
16895
16896
16897
16898
16899
16900
16901
16902
16903
16904
16905
16906
16907
16908
16909
16910
16911
16912
16913
16914
16915
16916
16917
16918
16919
16920
16921
16922
16923
16924
16925
16926
16927
16928
16929
16930
16931
16932
16933
16934
16935
16936
16937
16938
16939
16940
16941
16942
16943
16944
16945
16946
16947
16948
16949
16950
16951
16952
16953
16954
16955
16956
16957
16958
16959
16960
16961
16962
16963
16964
16965
16966
16967
16968
16969
16970
16971
16972
16973
16974
16975
16976
16977
16978
16979
16980
16981
16982
16983
16984
16985
16986
16987
16988
16989
16990
16991
16992
16993
16994
16995
16996
16997
16998
16999
17000
17001
17002
17003
17004
17005
17006
17007
17008
17009
17010
17011
17012
17013
17014
17015
17016
17017
17018
17019
17020
17021
17022
17023
17024
17025
17026
17027
17028
17029
17030
17031
17032
17033
17034
17035
17036
17037
17038
17039
17040
17041
17042
17043
17044
17045
17046
17047
17048
17049
17050
17051
17052
17053
17054
17055
17056
17057
17058
17059
17060
17061
17062
17063
17064
17065
17066
17067
17068
17069
17070
17071
17072
17073
17074
17075
17076
17077
17078
17079
17080
17081
17082
17083
17084
17085
17086
17087
17088
17089
17090
17091
17092
17093
17094
17095
17096
17097
17098
17099
17100
17101
17102
17103
17104
17105
17106
17107
17108
17109
17110
17111
17112
17113
17114
17115
17116
17117
17118
17119
17120
17121
17122
17123
17124
17125
17126
17127
17128
17129
17130
17131
17132
17133
17134
17135
17136
17137
17138
17139
17140
17141
17142
17143
17144
17145
17146
17147
17148
17149
17150
17151
17152
17153
17154
17155
17156
17157
17158
17159
17160
17161
17162
17163
17164
17165
17166
17167
17168
17169
17170
17171
17172
17173
17174
17175
17176
17177
17178
17179
17180
17181
17182
17183
17184
17185
17186
17187
17188
17189
17190
17191
17192
17193
17194
17195
17196
17197
17198
17199
17200
17201
17202
17203
17204
17205
17206
17207
17208
17209
17210
17211
17212
17213
17214
17215
17216
17217
17218
17219
17220
17221
17222
17223
17224
17225
17226
17227
17228
17229
17230
17231
17232
17233
17234
17235
17236
17237
17238
17239
17240
17241
17242
17243
17244
17245
17246
17247
17248
17249
17250
17251
17252
17253
17254
17255
17256
17257
17258
17259
17260
17261
17262
17263
17264
17265
17266
17267
17268
17269
17270
17271
17272
17273
17274
17275
17276
17277
17278
17279
17280
17281
17282
17283
17284
17285
17286
17287
17288
17289
17290
17291
17292
17293
17294
17295
17296
17297
17298
17299
17300
17301
17302
17303
17304
17305
17306
17307
17308
17309
17310
17311
17312
17313
17314
17315
17316
17317
17318
17319
17320
17321
17322
17323
17324
17325
17326
17327
17328
17329
17330
17331
17332
17333
17334
17335
17336
17337
17338
17339
17340
17341
17342
17343
17344
17345
17346
17347
17348
17349
17350
17351
17352
17353
17354
17355
17356
17357
17358
17359
17360
17361
17362
17363
17364
17365
17366
17367
17368
17369
17370
17371
17372
17373
17374
17375
17376
17377
17378
17379
17380
17381
17382
17383
17384
17385
17386
17387
17388
17389
17390
17391
17392
17393
17394
17395
17396
17397
17398
17399
17400
17401
17402
17403
17404
17405
17406
17407
17408
17409
17410
17411
17412
17413
17414
17415
17416
17417
17418
17419
17420
17421
17422
17423
17424
17425
17426
17427
17428
17429
17430
17431
17432
17433
17434
17435
17436
17437
17438
17439
17440
17441
17442
17443
17444
17445
17446
17447
17448
17449
17450
17451
17452
17453
17454
17455
17456
17457
17458
17459
17460
17461
17462
17463
17464
17465
17466
17467
17468
17469
17470
17471
17472
17473
17474
17475
17476
17477
17478
17479
17480
17481
17482
17483
17484
17485
17486
17487
17488
17489
17490
17491
17492
17493
17494
17495
17496
17497
17498
17499
17500
17501
17502
17503
17504
17505
17506
17507
17508
17509
17510
17511
17512
17513
17514
17515
17516
17517
17518
17519
17520
17521
17522
17523
17524
17525
17526
17527
17528
17529
17530
17531
17532
17533
17534
17535
17536
17537
17538
17539
17540
17541
17542
17543
17544
17545
17546
17547
17548
17549
17550
17551
17552
17553
17554
17555
17556
17557
17558
17559
17560
17561
17562
17563
17564
17565
17566
17567
17568
17569
17570
17571
17572
17573
17574
17575
17576
17577
17578
17579
17580
17581
17582
17583
17584
17585
17586
17587
17588
17589
17590
17591
17592
17593
17594
17595
17596
17597
17598
17599
17600
17601
17602
17603
17604
17605
17606
17607
17608
17609
17610
17611
17612
17613
17614
17615
17616
17617
17618
17619
17620
17621
17622
17623
17624
17625
17626
17627
17628
17629
17630
17631
17632
17633
17634
17635
17636
17637
17638
17639
17640
17641
17642
17643
17644
17645
17646
17647
17648
17649
17650
17651
17652
17653
17654
17655
17656
17657
17658
17659
17660
17661
17662
17663
17664
17665
17666
17667
17668
17669
17670
17671
17672
17673
17674
17675
17676
17677
17678
17679
17680
17681
17682
17683
17684
17685
17686
17687
17688
17689
17690
17691
17692
17693
17694
17695
17696
17697
17698
17699
17700
17701
17702
17703
17704
17705
17706
17707
17708
17709
17710
17711
17712
17713
17714
17715
17716
17717
17718
17719
17720
17721
17722
17723
17724
17725
17726
17727
17728
17729
17730
17731
17732
17733
17734
17735
17736
17737
17738
17739
17740
17741
17742
17743
17744
17745
17746
17747
17748
17749
17750
17751
17752
17753
17754
17755
17756
17757
17758
17759
17760
17761
17762
17763
17764
17765
17766
17767
17768
17769
17770
17771
17772
17773
17774
17775
17776
17777
17778
17779
17780
17781
17782
17783
17784
17785
17786
17787
17788
17789
17790
17791
17792
17793
17794
17795
17796
17797
17798
17799
17800
17801
17802
17803
17804
17805
17806
17807
17808
17809
17810
17811
17812
17813
17814
17815
17816
17817
17818
17819
17820
17821
17822
17823
17824
17825
17826
17827
17828
17829
17830
17831
17832
17833
17834
17835
17836
17837
17838
17839
17840
17841
17842
17843
17844
17845
17846
17847
17848
17849
17850
17851
17852
17853
17854
17855
17856
17857
17858
17859
17860
17861
17862
17863
17864
17865
17866
17867
17868
17869
17870
17871
17872
17873
17874
17875
17876
17877
17878
17879
17880
17881
17882
17883
17884
17885
17886
17887
17888
17889
17890
17891
17892
17893
17894
17895
17896
17897
17898
17899
17900
17901
17902
17903
17904
17905
17906
17907
17908
17909
17910
17911
17912
17913
17914
17915
17916
17917
17918
17919
17920
17921
17922
17923
17924
17925
17926
17927
17928
17929
17930
17931
17932
17933
17934
17935
17936
17937
17938
17939
17940
17941
17942
17943
17944
17945
17946
17947
17948
17949
17950
17951
17952
17953
17954
17955
17956
17957
17958
17959
17960
17961
17962
17963
17964
17965
17966
17967
17968
17969
17970
17971
17972
17973
17974
17975
17976
17977
17978
17979
17980
17981
17982
17983
17984
17985
17986
17987
17988
17989
17990
17991
17992
17993
17994
17995
17996
17997
17998
17999
18000
18001
18002
18003
18004
18005
18006
18007
18008
18009
18010
18011
18012
18013
18014
18015
18016
18017
18018
18019
18020
18021
18022
18023
18024
18025
18026
18027
18028
18029
18030
18031
18032
18033
18034
18035
18036
18037
18038
18039
18040
18041
18042
18043
18044
18045
18046
18047
18048
18049
18050
18051
18052
18053
18054
18055
18056
18057
18058
18059
18060
18061
18062
18063
18064
18065
18066
18067
18068
18069
18070
18071
18072
18073
18074
18075
18076
18077
18078
18079
18080
18081
18082
18083
18084
18085
18086
18087
18088
18089
18090
18091
18092
18093
18094
18095
18096
18097
18098
18099
18100
18101
18102
18103
18104
18105
18106
18107
18108
18109
18110
18111
18112
18113
18114
18115
18116
18117
18118
18119
18120
18121
18122
18123
18124
18125
18126
18127
18128
18129
18130
18131
18132
18133
18134
18135
18136
18137
18138
18139
18140
18141
18142
18143
18144
18145
18146
18147
18148
18149
18150
18151
18152
18153
18154
18155
18156
18157
18158
18159
18160
18161
18162
18163
18164
18165
18166
18167
18168
18169
18170
18171
18172
18173
18174
18175
18176
18177
18178
18179
18180
18181
18182
18183
18184
18185
18186
18187
18188
18189
18190
18191
18192
18193
18194
18195
18196
18197
18198
18199
18200
18201
18202
18203
18204
18205
18206
18207
18208
18209
18210
18211
18212
18213
18214
18215
18216
18217
18218
18219
18220
18221
18222
18223
18224
18225
18226
18227
18228
18229
18230
18231
18232
18233
18234
18235
18236
18237
18238
18239
18240
18241
18242
18243
18244
18245
18246
18247
18248
18249
18250
18251
18252
18253
18254
18255
18256
18257
18258
18259
18260
18261
18262
18263
18264
18265
18266
18267
18268
18269
18270
18271
18272
18273
18274
18275
18276
18277
18278
18279
18280
18281
18282
18283
18284
18285
18286
18287
18288
18289
18290
18291
18292
18293
18294
18295
18296
18297
18298
18299
18300
18301
18302
18303
18304
18305
18306
18307
18308
18309
18310
18311
18312
18313
18314
18315
18316
18317
18318
18319
18320
18321
18322
18323
18324
18325
18326
18327
18328
18329
18330
18331
18332
18333
18334
18335
18336
18337
18338
18339
18340
18341
18342
18343
18344
18345
18346
18347
18348
18349
18350
18351
18352
18353
18354
18355
18356
18357
18358
18359
18360
18361
18362
18363
18364
18365
18366
18367
18368
18369
18370
18371
18372
18373
18374
18375
18376
18377
18378
18379
18380
18381
18382
18383
18384
18385
18386
18387
18388
18389
18390
18391
18392
18393
18394
18395
18396
18397
18398
18399
18400
18401
18402
18403
18404
18405
18406
18407
18408
18409
18410
18411
18412
18413
18414
18415
18416
18417
18418
18419
18420
18421
18422
18423
18424
18425
18426
18427
18428
18429
18430
18431
18432
18433
18434
18435
18436
18437
18438
18439
18440
18441
18442
18443
18444
18445
18446
18447
18448
18449
18450
18451
18452
18453
18454
18455
18456
18457
18458
18459
18460
18461
18462
18463
18464
18465
18466
18467
18468
18469
18470
18471
18472
18473
18474
18475
18476
18477
18478
18479
18480
18481
18482
18483
18484
18485
18486
18487
18488
18489
18490
18491
18492
18493
18494
18495
18496
18497
18498
18499
18500
18501
18502
18503
18504
18505
18506
18507
18508
18509
18510
18511
18512
18513
18514
18515
18516
18517
18518
18519
18520
18521
18522
18523
18524
18525
18526
18527
18528
18529
18530
18531
18532
18533
18534
18535
18536
18537
18538
18539
18540
18541
18542
18543
18544
18545
18546
18547
18548
18549
18550
18551
18552
18553
18554
18555
18556
18557
18558
18559
18560
18561
18562
18563
18564
18565
18566
18567
18568
18569
18570
18571
18572
18573
18574
18575
18576
18577
18578
18579
18580
18581
18582
18583
18584
18585
18586
18587
18588
18589
18590
18591
18592
18593
18594
18595
18596
18597
18598
18599
18600
18601
18602
18603
18604
18605
18606
18607
18608
18609
18610
18611
18612
18613
18614
18615
18616
18617
18618
18619
18620
18621
18622
18623
18624
18625
18626
18627
18628
18629
18630
18631
18632
18633
18634
18635
18636
18637
18638
18639
18640
18641
18642
18643
18644
18645
18646
18647
18648
18649
18650
18651
18652
18653
18654
18655
18656
18657
18658
18659
18660
18661
18662
18663
18664
18665
18666
18667
18668
18669
18670
18671
18672
18673
18674
18675
18676
18677
18678
18679
18680
18681
18682
18683
18684
18685
18686
18687
18688
18689
18690
18691
18692
18693
18694
18695
18696
18697
18698
18699
18700
18701
18702
18703
18704
18705
18706
18707
18708
18709
18710
18711
18712
18713
18714
18715
18716
18717
18718
18719
18720
18721
18722
18723
18724
18725
18726
18727
18728
18729
18730
18731
18732
18733
18734
18735
18736
18737
18738
18739
18740
18741
18742
18743
18744
18745
18746
18747
18748
18749
18750
18751
18752
18753
18754
18755
18756
18757
18758
18759
18760
18761
18762
18763
18764
18765
18766
18767
18768
18769
18770
18771
18772
18773
18774
18775
18776
18777
18778
18779
18780
18781
18782
18783
18784
18785
18786
18787
18788
18789
18790
18791
18792
18793
18794
18795
18796
18797
18798
18799
18800
18801
18802
18803
18804
18805
18806
18807
18808
18809
18810
18811
18812
18813
18814
18815
18816
18817
18818
18819
18820
18821
18822
18823
18824
18825
18826
18827
18828
18829
18830
18831
18832
18833
18834
18835
18836
18837
18838
18839
18840
18841
18842
18843
18844
18845
18846
18847
18848
18849
18850
18851
18852
18853
18854
18855
18856
18857
18858
18859
18860
18861
18862
18863
18864
18865
18866
18867
18868
18869
18870
18871
18872
18873
18874
18875
18876
18877
18878
18879
18880
18881
18882
18883
18884
18885
18886
18887
18888
18889
18890
18891
18892
18893
18894
18895
18896
18897
18898
18899
18900
18901
18902
18903
18904
18905
18906
18907
18908
18909
18910
18911
18912
18913
18914
18915
18916
18917
18918
18919
18920
18921
18922
18923
18924
18925
18926
18927
18928
18929
18930
18931
18932
18933
18934
18935
18936
18937
18938
18939
18940
18941
18942
18943
18944
18945
18946
18947
18948
18949
18950
18951
18952
18953
18954
18955
18956
18957
18958
18959
18960
18961
18962
18963
18964
18965
18966
18967
18968
18969
18970
18971
18972
18973
18974
18975
18976
18977
18978
18979
18980
18981
18982
18983
18984
18985
18986
18987
18988
18989
18990
18991
18992
18993
18994
18995
18996
18997
18998
18999
19000
19001
19002
19003
19004
19005
19006
19007
19008
19009
19010
19011
19012
19013
19014
19015
19016
19017
19018
19019
19020
19021
19022
19023
19024
19025
19026
19027
19028
19029
19030
19031
19032
19033
19034
19035
19036
19037
19038
19039
19040
19041
19042
19043
19044
19045
19046
19047
19048
19049
19050
19051
19052
19053
19054
19055
19056
19057
19058
19059
19060
19061
19062
19063
19064
19065
19066
19067
19068
19069
19070
19071
19072
19073
19074
19075
19076
19077
19078
19079
19080
19081
19082
19083
19084
19085
19086
19087
19088
19089
19090
19091
19092
19093
19094
19095
19096
19097
19098
19099
19100
19101
19102
19103
19104
19105
19106
19107
19108
19109
19110
19111
19112
19113
19114
19115
19116
19117
19118
19119
19120
19121
19122
19123
19124
19125
19126
19127
19128
19129
19130
19131
19132
19133
19134
19135
19136
19137
19138
19139
19140
19141
19142
19143
19144
19145
19146
19147
19148
19149
19150
19151
19152
19153
19154
19155
19156
19157
19158
19159
19160
19161
19162
19163
19164
19165
19166
19167
19168
19169
19170
19171
19172
19173
19174
19175
19176
19177
19178
19179
19180
19181
19182
19183
19184
19185
19186
19187
19188
19189
19190
19191
19192
19193
19194
19195
19196
19197
19198
19199
19200
19201
19202
19203
19204
19205
19206
19207
19208
19209
19210
19211
19212
19213
19214
19215
19216
19217
19218
19219
19220
19221
19222
19223
19224
19225
19226
19227
19228
19229
19230
19231
19232
19233
19234
19235
19236
19237
19238
19239
19240
19241
19242
19243
19244
19245
19246
19247
19248
19249
19250
19251
19252
19253
19254
19255
19256
19257
19258
19259
19260
19261
19262
19263
19264
19265
19266
19267
19268
19269
19270
19271
19272
19273
19274
19275
19276
19277
19278
19279
19280
19281
19282
19283
19284
19285
19286
19287
19288
19289
19290
19291
19292
19293
19294
19295
19296
19297
19298
19299
19300
19301
19302
19303
19304
19305
19306
19307
19308
19309
19310
19311
19312
19313
19314
19315
19316
19317
19318
19319
19320
19321
19322
19323
19324
19325
19326
19327
19328
19329
19330
19331
19332
19333
19334
19335
19336
19337
19338
19339
19340
19341
19342
19343
19344
19345
19346
19347
19348
19349
19350
19351
19352
19353
19354
19355
19356
19357
19358
19359
19360
19361
19362
19363
19364
19365
19366
19367
19368
19369
19370
19371
19372
19373
19374
19375
19376
19377
19378
19379
19380
19381
19382
19383
19384
19385
19386
19387
19388
19389
19390
19391
19392
19393
19394
19395
19396
19397
19398
19399
19400
19401
19402
19403
19404
19405
19406
19407
19408
19409
19410
19411
19412
19413
19414
19415
19416
19417
19418
19419
19420
19421
19422
19423
19424
19425
19426
19427
19428
19429
19430
19431
19432
19433
19434
19435
19436
19437
19438
19439
19440
19441
19442
19443
19444
19445
19446
19447
19448
19449
19450
19451
19452
19453
19454
19455
19456
19457
19458
19459
19460
19461
19462
19463
19464
19465
19466
19467
19468
19469
19470
19471
19472
19473
19474
19475
19476
19477
19478
19479
19480
19481
19482
19483
19484
19485
19486
19487
19488
19489
19490
19491
19492
19493
19494
19495
19496
19497
19498
19499
19500
19501
19502
19503
19504
19505
19506
19507
19508
19509
19510
19511
19512
19513
19514
19515
19516
19517
19518
19519
19520
19521
19522
19523
19524
19525
19526
19527
19528
19529
19530
19531
19532
19533
19534
19535
19536
19537
19538
19539
19540
19541
19542
19543
19544
19545
19546
19547
19548
19549
19550
19551
19552
19553
19554
19555
19556
19557
19558
19559
19560
19561
19562
19563
19564
19565
19566
19567
19568
19569
19570
19571
19572
19573
19574
19575
19576
19577
19578
19579
19580
19581
19582
19583
19584
19585
19586
19587
19588
19589
19590
19591
19592
19593
19594
19595
19596
19597
19598
19599
19600
19601
19602
19603
19604
19605
19606
19607
19608
19609
19610
19611
19612
19613
19614
19615
19616
19617
19618
19619
19620
19621
19622
19623
19624
19625
19626
19627
19628
19629
19630
19631
19632
19633
19634
19635
19636
19637
19638
19639
19640
19641
19642
19643
19644
19645
19646
19647
19648
19649
19650
19651
19652
19653
19654
19655
19656
19657
19658
19659
19660
19661
19662
19663
19664
19665
19666
19667
19668
19669
19670
19671
19672
19673
19674
19675
19676
19677
19678
19679
19680
19681
19682
19683
19684
19685
19686
19687
19688
19689
19690
19691
19692
19693
19694
19695
19696
19697
19698
19699
19700
19701
19702
19703
19704
19705
19706
19707
19708
19709
19710
19711
19712
19713
19714
19715
19716
19717
19718
19719
19720
19721
19722
19723
19724
19725
19726
19727
19728
19729
19730
19731
19732
19733
19734
19735
19736
19737
19738
19739
19740
19741
19742
19743
19744
19745
19746
19747
19748
19749
19750
19751
19752
19753
19754
19755
19756
19757
19758
19759
19760
19761
19762
19763
19764
19765
19766
19767
19768
19769
19770
19771
19772
19773
19774
19775
19776
19777
19778
19779
19780
19781
19782
19783
19784
19785
19786
19787
19788
19789
19790
19791
19792
19793
19794
19795
19796
19797
19798
19799
19800
19801
19802
19803
19804
19805
19806
19807
19808
19809
19810
19811
19812
19813
19814
19815
19816
19817
19818
19819
19820
19821
19822
19823
19824
19825
19826
19827
19828
19829
19830
19831
19832
19833
19834
19835
19836
19837
19838
19839
19840
19841
19842
19843
19844
19845
19846
19847
19848
19849
19850
19851
19852
19853
19854
19855
19856
19857
19858
19859
19860
19861
19862
19863
19864
19865
19866
19867
19868
19869
19870
19871
19872
19873
19874
19875
19876
19877
19878
19879
19880
19881
19882
19883
19884
19885
19886
19887
19888
19889
19890
19891
19892
19893
19894
19895
19896
19897
19898
19899
19900
19901
19902
19903
19904
19905
19906
19907
19908
19909
19910
19911
19912
19913
19914
19915
19916
19917
19918
19919
19920
19921
19922
19923
19924
19925
19926
19927
19928
19929
19930
19931
19932
19933
19934
19935
19936
19937
19938
19939
19940
19941
19942
19943
19944
19945
19946
19947
19948
19949
19950
19951
19952
19953
19954
19955
19956
19957
19958
19959
19960
19961
19962
19963
19964
19965
19966
19967
19968
19969
19970
19971
19972
19973
19974
19975
19976
19977
19978
19979
19980
19981
19982
19983
19984
19985
19986
19987
19988
19989
19990
19991
19992
19993
19994
19995
19996
19997
19998
19999
20000
20001
20002
20003
20004
20005
20006
20007
20008
20009
20010
20011
20012
20013
20014
20015
20016
20017
20018
20019
20020
20021
20022
20023
20024
20025
20026
20027
20028
20029
20030
20031
20032
20033
20034
20035
20036
20037
20038
20039
20040
20041
20042
20043
20044
20045
20046
20047
20048
20049
20050
20051
20052
20053
20054
20055
20056
20057
20058
20059
20060
20061
20062
20063
20064
20065
20066
20067
20068
20069
20070
20071
20072
20073
20074
20075
20076
20077
20078
20079
20080
20081
20082
20083
20084
20085
20086
20087
20088
20089
20090
20091
20092
20093
20094
20095
20096
20097
20098
20099
20100
20101
20102
20103
20104
20105
20106
20107
20108
20109
20110
20111
20112
20113
20114
20115
20116
20117
20118
20119
20120
20121
20122
20123
20124
20125
20126
20127
20128
20129
20130
20131
20132
20133
20134
20135
20136
20137
20138
20139
20140
20141
20142
20143
20144
20145
20146
20147
20148
20149
20150
20151
20152
20153
20154
20155
20156
20157
20158
20159
20160
20161
20162
20163
20164
20165
20166
20167
20168
20169
20170
20171
20172
20173
20174
20175
20176
20177
20178
20179
20180
20181
20182
20183
20184
20185
20186
20187
20188
20189
20190
20191
20192
20193
20194
20195
20196
20197
20198
20199
20200
20201
20202
20203
20204
20205
20206
20207
20208
20209
20210
20211
20212
20213
20214
20215
20216
20217
20218
20219
20220
20221
20222
20223
20224
20225
20226
20227
20228
20229
20230
20231
20232
20233
20234
20235
20236
20237
20238
20239
20240
20241
20242
20243
20244
20245
20246
20247
20248
20249
20250
20251
20252
20253
20254
20255
20256
20257
20258
20259
20260
20261
20262
20263
20264
20265
20266
20267
20268
20269
20270
20271
20272
20273
20274
20275
20276
20277
20278
20279
20280
20281
20282
20283
20284
20285
20286
20287
20288
20289
20290
20291
20292
20293
20294
20295
20296
20297
20298
20299
20300
20301
20302
20303
20304
20305
20306
20307
20308
20309
20310
20311
20312
20313
20314
20315
20316
20317
20318
20319
20320
20321
20322
20323
20324
20325
20326
20327
20328
20329
20330
20331
20332
20333
20334
20335
20336
20337
20338
20339
20340
20341
20342
20343
20344
20345
20346
20347
20348
20349
20350
20351
20352
20353
20354
20355
20356
20357
20358
20359
20360
20361
20362
20363
20364
20365
20366
20367
20368
20369
20370
20371
20372
20373
20374
20375
20376
20377
20378
20379
20380
20381
20382
20383
20384
20385
20386
20387
20388
20389
20390
20391
20392
20393
20394
20395
20396
20397
20398
20399
20400
20401
20402
20403
20404
20405
20406
20407
20408
20409
20410
20411
20412
20413
20414
20415
20416
20417
20418
20419
20420
20421
20422
20423
20424
20425
20426
20427
20428
20429
20430
20431
20432
20433
20434
20435
20436
20437
20438
20439
20440
20441
20442
20443
20444
20445
20446
20447
20448
20449
20450
20451
20452
20453
20454
20455
20456
20457
20458
20459
20460
20461
20462
20463
20464
20465
20466
20467
20468
20469
20470
20471
20472
20473
20474
20475
20476
20477
20478
20479
20480
20481
20482
20483
20484
20485
20486
20487
20488
20489
20490
20491
20492
20493
20494
20495
20496
20497
20498
20499
20500
20501
20502
20503
20504
20505
20506
20507
20508
20509
20510
20511
20512
20513
20514
20515
20516
20517
20518
20519
20520
20521
20522
20523
20524
20525
20526
20527
20528
20529
20530
20531
20532
20533
20534
20535
20536
20537
20538
20539
20540
20541
20542
20543
20544
20545
20546
20547
20548
20549
20550
20551
20552
20553
20554
20555
20556
20557
20558
20559
20560
20561
20562
20563
20564
20565
20566
20567
20568
20569
20570
20571
20572
20573
20574
20575
20576
20577
20578
20579
20580
20581
20582
20583
20584
20585
20586
20587
20588
20589
20590
20591
20592
20593
20594
20595
20596
20597
20598
20599
20600
20601
20602
20603
20604
20605
20606
20607
20608
20609
20610
20611
20612
20613
20614
20615
20616
20617
20618
20619
20620
20621
20622
20623
20624
20625
20626
20627
20628
20629
20630
20631
20632
20633
20634
20635
20636
20637
20638
20639
20640
20641
20642
20643
20644
20645
20646
20647
20648
20649
20650
20651
20652
20653
20654
20655
20656
20657
20658
20659
20660
20661
20662
20663
20664
20665
20666
20667
20668
20669
20670
20671
20672
20673
20674
20675
20676
20677
20678
20679
20680
20681
20682
20683
20684
20685
20686
20687
20688
20689
20690
20691
20692
20693
20694
20695
20696
20697
20698
20699
20700
20701
20702
20703
20704
20705
20706
20707
20708
20709
20710
20711
20712
20713
20714
20715
20716
20717
20718
20719
20720
20721
20722
20723
20724
20725
20726
20727
20728
20729
20730
20731
20732
20733
20734
20735
20736
20737
20738
20739
20740
20741
20742
20743
20744
20745
20746
20747
20748
20749
20750
20751
20752
20753
20754
20755
20756
20757
20758
20759
20760
20761
20762
20763
20764
20765
20766
20767
20768
20769
20770
20771
20772
20773
20774
20775
20776
20777
20778
20779
20780
20781
20782
20783
20784
20785
20786
20787
20788
20789
20790
20791
20792
20793
20794
20795
20796
20797
20798
20799
20800
20801
20802
20803
20804
20805
20806
20807
20808
20809
20810
20811
20812
20813
20814
20815
20816
20817
20818
20819
20820
20821
20822
20823
20824
20825
20826
20827
20828
20829
20830
20831
20832
20833
20834
|
#!/usr/bin/perl -w
# !!!!!!!!!!!!!! IF YOU MODIFY THIS FILE !!!!!!!!!!!!!!!!!!!!!!!!!
# Any files created or read by this program should be listed in 'mktables.lst'
# Use -makelist to regenerate it.
# There was an attempt when this was first rewritten to make it 5.8
# compatible, but that has now been abandoned, and newer constructs are used
# as convenient.
# NOTE: this script can run quite slowly in older/slower systems.
# It can also consume a lot of memory (128 MB or more), you may need
# to raise your process resource limits (e.g. in bash, "ulimit -a"
# to inspect, and "ulimit -d ..." or "ulimit -m ..." to set)
my $start_time;
BEGIN { # Get the time the script started running; do it at compilation to
# get it as close as possible
$start_time= time;
}
require 5.010_001;
use strict;
use warnings;
use Carp;
use Config;
use File::Find;
use File::Path;
use File::Spec;
use Text::Tabs;
use re "/aa";
use feature 'state';
sub DEBUG () { 0 } # Set to 0 for production; 1 for development
my $debugging_build = $Config{"ccflags"} =~ /-DDEBUGGING/;
sub NON_ASCII_PLATFORM { ord("A") != 65 }
# When a new version of Unicode is published, unfortunately the algorithms for
# dealing with various bounds, like \b{gcb}, \b{lb} may have to be updated
# manually. The changes may or may not be backward compatible with older
# releases. The code is in regen/mk_invlist.pl and regexec.c. Make the
# changes, then come back here and set the variable below to what version the
# code is expecting. If a newer version of Unicode is being compiled than
# expected, a warning will be generated. If an older version is being
# compiled, any bounds tests that fail in the generated test file (-maketest
# option) will be marked as TODO.
my $version_of_mk_invlist_bounds = v12.1.0;
##########################################################################
#
# mktables -- create the runtime Perl Unicode files (lib/unicore/.../*.pl),
# from the Unicode database files (lib/unicore/.../*.txt), It also generates
# a pod file and .t files, depending on option parameters.
#
# The structure of this file is:
# First these introductory comments; then
# code needed for everywhere, such as debugging stuff; then
# code to handle input parameters; then
# data structures likely to be of external interest (some of which depend on
# the input parameters, so follows them; then
# more data structures and subroutine and package (class) definitions; then
# the small actual loop to process the input files and finish up; then
# a __DATA__ section, for the .t tests
#
# This program works on all releases of Unicode so far. The outputs have been
# scrutinized most intently for release 5.1. The others have been checked for
# somewhat more than just sanity. It can handle all non-provisional Unicode
# character properties in those releases.
#
# This program is mostly about Unicode character (or code point) properties.
# A property describes some attribute or quality of a code point, like if it
# is lowercase or not, its name, what version of Unicode it was first defined
# in, or what its uppercase equivalent is. Unicode deals with these disparate
# possibilities by making all properties into mappings from each code point
# into some corresponding value. In the case of it being lowercase or not,
# the mapping is either to 'Y' or 'N' (or various synonyms thereof). Each
# property maps each Unicode code point to a single value, called a "property
# value". (Some more recently defined properties, map a code point to a set
# of values.)
#
# When using a property in a regular expression, what is desired isn't the
# mapping of the code point to its property's value, but the reverse (or the
# mathematical "inverse relation"): starting with the property value, "Does a
# code point map to it?" These are written in a "compound" form:
# \p{property=value}, e.g., \p{category=punctuation}. This program generates
# files containing the lists of code points that map to each such regular
# expression property value, one file per list
#
# There is also a single form shortcut that Perl adds for many of the commonly
# used properties. This happens for all binary properties, plus script,
# general_category, and block properties.
#
# Thus the outputs of this program are files. There are map files, mostly in
# the 'To' directory; and there are list files for use in regular expression
# matching, all in subdirectories of the 'lib' directory, with each
# subdirectory being named for the property that the lists in it are for.
# Bookkeeping, test, and documentation files are also generated.
my $matches_directory = 'lib'; # Where match (\p{}) files go.
my $map_directory = 'To'; # Where map files go.
# DATA STRUCTURES
#
# The major data structures of this program are Property, of course, but also
# Table. There are two kinds of tables, very similar to each other.
# "Match_Table" is the data structure giving the list of code points that have
# a particular property value, mentioned above. There is also a "Map_Table"
# data structure which gives the property's mapping from code point to value.
# There are two structures because the match tables need to be combined in
# various ways, such as constructing unions, intersections, complements, etc.,
# and the map ones don't. And there would be problems, perhaps subtle, if
# a map table were inadvertently operated on in some of those ways.
# The use of separate classes with operations defined on one but not the other
# prevents accidentally confusing the two.
#
# At the heart of each table's data structure is a "Range_List", which is just
# an ordered list of "Ranges", plus ancillary information, and methods to
# operate on them. A Range is a compact way to store property information.
# Each range has a starting code point, an ending code point, and a value that
# is meant to apply to all the code points between the two end points,
# inclusive. For a map table, this value is the property value for those
# code points. Two such ranges could be written like this:
# 0x41 .. 0x5A, 'Upper',
# 0x61 .. 0x7A, 'Lower'
#
# Each range also has a type used as a convenience to classify the values.
# Most ranges in this program will be Type 0, or normal, but there are some
# ranges that have a non-zero type. These are used only in map tables, and
# are for mappings that don't fit into the normal scheme of things. Mappings
# that require a hash entry to communicate with utf8.c are one example;
# another example is mappings for charnames.pm to use which indicate a name
# that is algorithmically determinable from its code point (and the reverse).
# These are used to significantly compact these tables, instead of listing
# each one of the tens of thousands individually.
#
# In a match table, the value of a range is irrelevant (and hence the type as
# well, which will always be 0), and arbitrarily set to the empty string.
# Using the example above, there would be two match tables for those two
# entries, one named Upper would contain the 0x41..0x5A range, and the other
# named Lower would contain 0x61..0x7A.
#
# Actually, there are two types of range lists, "Range_Map" is the one
# associated with map tables, and "Range_List" with match tables.
# Again, this is so that methods can be defined on one and not the others so
# as to prevent operating on them in incorrect ways.
#
# Eventually, most tables are written out to files to be read by utf8_heavy.pl
# in the perl core. All tables could in theory be written, but some are
# suppressed because there is no current practical use for them. It is easy
# to change which get written by changing various lists that are near the top
# of the actual code in this file. The table data structures contain enough
# ancillary information to allow them to be treated as separate entities for
# writing, such as the path to each one's file. There is a heading in each
# map table that gives the format of its entries, and what the map is for all
# the code points missing from it. (This allows tables to be more compact.)
#
# The Property data structure contains one or more tables. All properties
# contain a map table (except the $perl property which is a
# pseudo-property containing only match tables), and any properties that
# are usable in regular expression matches also contain various matching
# tables, one for each value the property can have. A binary property can
# have two values, True and False (or Y and N, which are preferred by Unicode
# terminology). Thus each of these properties will have a map table that
# takes every code point and maps it to Y or N (but having ranges cuts the
# number of entries in that table way down), and two match tables, one
# which has a list of all the code points that map to Y, and one for all the
# code points that map to N. (For each binary property, a third table is also
# generated for the pseudo Perl property. It contains the identical code
# points as the Y table, but can be written in regular expressions, not in the
# compound form, but in a "single" form like \p{IsUppercase}.) Many
# properties are binary, but some properties have several possible values,
# some have many, and properties like Name have a different value for every
# named code point. Those will not, unless the controlling lists are changed,
# have their match tables written out. But all the ones which can be used in
# regular expression \p{} and \P{} constructs will. Prior to 5.14, generally
# a property would have either its map table or its match tables written but
# not both. Again, what gets written is controlled by lists which can easily
# be changed. Starting in 5.14, advantage was taken of this, and all the map
# tables needed to reconstruct the Unicode db are now written out, while
# suppressing the Unicode .txt files that contain the data. Our tables are
# much more compact than the .txt files, so a significant space savings was
# achieved. Also, tables are not written out that are trivially derivable
# from tables that do get written. So, there typically is no file containing
# the code points not matched by a binary property (the table for \P{} versus
# lowercase \p{}), since you just need to invert the True table to get the
# False table.
# Properties have a 'Type', like 'binary', or 'string', or 'enum' depending on
# how many match tables there are and the content of the maps. This 'Type' is
# different than a range 'Type', so don't get confused by the two concepts
# having the same name.
#
# For information about the Unicode properties, see Unicode's UAX44 document:
my $unicode_reference_url = 'http://www.unicode.org/reports/tr44/';
# As stated earlier, this program will work on any release of Unicode so far.
# Most obvious problems in earlier data have NOT been corrected except when
# necessary to make Perl or this program work reasonably, and to keep out
# potential security issues. For example, no folding information was given in
# early releases, so this program substitutes lower case instead, just so that
# a regular expression with the /i option will do something that actually
# gives the right results in many cases. There are also a couple other
# corrections for version 1.1.5, commented at the point they are made. As an
# example of corrections that weren't made (but could be) is this statement
# from DerivedAge.txt: "The supplementary private use code points and the
# non-character code points were assigned in version 2.0, but not specifically
# listed in the UCD until versions 3.0 and 3.1 respectively." (To be precise
# it was 3.0.1 not 3.0.0) More information on Unicode version glitches is
# further down in these introductory comments.
#
# This program works on all non-provisional properties as of the current
# Unicode release, though the files for some are suppressed for various
# reasons. You can change which are output by changing lists in this program.
#
# The old version of mktables emphasized the term "Fuzzy" to mean Unicode's
# loose matchings rules (from Unicode TR18):
#
# The recommended names for UCD properties and property values are in
# PropertyAliases.txt [Prop] and PropertyValueAliases.txt
# [PropValue]. There are both abbreviated names and longer, more
# descriptive names. It is strongly recommended that both names be
# recognized, and that loose matching of property names be used,
# whereby the case distinctions, whitespace, hyphens, and underbar
# are ignored.
#
# The program still allows Fuzzy to override its determination of if loose
# matching should be used, but it isn't currently used, as it is no longer
# needed; the calculations it makes are good enough.
#
# SUMMARY OF HOW IT WORKS:
#
# Process arguments
#
# A list is constructed containing each input file that is to be processed
#
# Each file on the list is processed in a loop, using the associated handler
# code for each:
# The PropertyAliases.txt and PropValueAliases.txt files are processed
# first. These files name the properties and property values.
# Objects are created of all the property and property value names
# that the rest of the input should expect, including all synonyms.
# The other input files give mappings from properties to property
# values. That is, they list code points and say what the mapping
# is under the given property. Some files give the mappings for
# just one property; and some for many. This program goes through
# each file and populates the properties and their map tables from
# them. Some properties are listed in more than one file, and
# Unicode has set up a precedence as to which has priority if there
# is a conflict. Thus the order of processing matters, and this
# program handles the conflict possibility by processing the
# overriding input files last, so that if necessary they replace
# earlier values.
# After this is all done, the program creates the property mappings not
# furnished by Unicode, but derivable from what it does give.
# The tables of code points that match each property value in each
# property that is accessible by regular expressions are created.
# The Perl-defined properties are created and populated. Many of these
# require data determined from the earlier steps
# Any Perl-defined synonyms are created, and name clashes between Perl
# and Unicode are reconciled and warned about.
# All the properties are written to files
# Any other files are written, and final warnings issued.
#
# For clarity, a number of operators have been overloaded to work on tables:
# ~ means invert (take all characters not in the set). The more
# conventional '!' is not used because of the possibility of confusing
# it with the actual boolean operation.
# + means union
# - means subtraction
# & means intersection
# The precedence of these is the order listed. Parentheses should be
# copiously used. These are not a general scheme. The operations aren't
# defined for a number of things, deliberately, to avoid getting into trouble.
# Operations are done on references and affect the underlying structures, so
# that the copy constructors for them have been overloaded to not return a new
# clone, but the input object itself.
#
# The bool operator is deliberately not overloaded to avoid confusion with
# "should it mean if the object merely exists, or also is non-empty?".
#
# WHY CERTAIN DESIGN DECISIONS WERE MADE
#
# This program needs to be able to run under miniperl. Therefore, it uses a
# minimum of other modules, and hence implements some things itself that could
# be gotten from CPAN
#
# This program uses inputs published by the Unicode Consortium. These can
# change incompatibly between releases without the Perl maintainers realizing
# it. Therefore this program is now designed to try to flag these. It looks
# at the directories where the inputs are, and flags any unrecognized files.
# It keeps track of all the properties in the files it handles, and flags any
# that it doesn't know how to handle. It also flags any input lines that
# don't match the expected syntax, among other checks.
#
# It is also designed so if a new input file matches one of the known
# templates, one hopefully just needs to add it to a list to have it
# processed.
#
# As mentioned earlier, some properties are given in more than one file. In
# particular, the files in the extracted directory are supposedly just
# reformattings of the others. But they contain information not easily
# derivable from the other files, including results for Unihan (which isn't
# usually available to this program) and for unassigned code points. They
# also have historically had errors or been incomplete. In an attempt to
# create the best possible data, this program thus processes them first to
# glean information missing from the other files; then processes those other
# files to override any errors in the extracted ones. Much of the design was
# driven by this need to store things and then possibly override them.
#
# It tries to keep fatal errors to a minimum, to generate something usable for
# testing purposes. It always looks for files that could be inputs, and will
# warn about any that it doesn't know how to handle (the -q option suppresses
# the warning).
#
# Why is there more than one type of range?
# This simplified things. There are some very specialized code points that
# have to be handled specially for output, such as Hangul syllable names.
# By creating a range type (done late in the development process), it
# allowed this to be stored with the range, and overridden by other input.
# Originally these were stored in another data structure, and it became a
# mess trying to decide if a second file that was for the same property was
# overriding the earlier one or not.
#
# Why are there two kinds of tables, match and map?
# (And there is a base class shared by the two as well.) As stated above,
# they actually are for different things. Development proceeded much more
# smoothly when I (khw) realized the distinction. Map tables are used to
# give the property value for every code point (actually every code point
# that doesn't map to a default value). Match tables are used for regular
# expression matches, and are essentially the inverse mapping. Separating
# the two allows more specialized methods, and error checks so that one
# can't just take the intersection of two map tables, for example, as that
# is nonsensical.
#
# What about 'fate' and 'status'. The concept of a table's fate was created
# late when it became clear that something more was needed. The difference
# between this and 'status' is unclean, and could be improved if someone
# wanted to spend the effort.
#
# DEBUGGING
#
# This program is written so it will run under miniperl. Occasionally changes
# will cause an error where the backtrace doesn't work well under miniperl.
# To diagnose the problem, you can instead run it under regular perl, if you
# have one compiled.
#
# There is a good trace facility. To enable it, first sub DEBUG must be set
# to return true. Then a line like
#
# local $to_trace = 1 if main::DEBUG;
#
# can be added to enable tracing in its lexical scope (plus dynamic) or until
# you insert another line:
#
# local $to_trace = 0 if main::DEBUG;
#
# To actually trace, use a line like "trace $a, @b, %c, ...;
#
# Some of the more complex subroutines already have trace statements in them.
# Permanent trace statements should be like:
#
# trace ... if main::DEBUG && $to_trace;
#
# main::stack_trace() will display what its name implies
#
# If there is just one or a few files that you're debugging, you can easily
# cause most everything else to be skipped. Change the line
#
# my $debug_skip = 0;
#
# to 1, and every file whose object is in @input_file_objects and doesn't have
# a, 'non_skip => 1,' in its constructor will be skipped. However, skipping
# Jamo.txt or UnicodeData.txt will likely cause fatal errors.
#
# To compare the output tables, it may be useful to specify the -annotate
# flag. (As of this writing, this can't be done on a clean workspace, due to
# requirements in Text::Tabs used in this option; so first run mktables
# without this option.) This option adds comment lines to each table, one for
# each non-algorithmically named character giving, currently its code point,
# name, and graphic representation if printable (and you have a font that
# knows about it). This makes it easier to see what the particular code
# points are in each output table. Non-named code points are annotated with a
# description of their status, and contiguous ones with the same description
# will be output as a range rather than individually. Algorithmically named
# characters are also output as ranges, except when there are just a few
# contiguous ones.
#
# FUTURE ISSUES
#
# The program would break if Unicode were to change its names so that
# interior white space, underscores, or dashes differences were significant
# within property and property value names.
#
# It might be easier to use the xml versions of the UCD if this program ever
# would need heavy revision, and the ability to handle old versions was not
# required.
#
# There is the potential for name collisions, in that Perl has chosen names
# that Unicode could decide it also likes. There have been such collisions in
# the past, with mostly Perl deciding to adopt the Unicode definition of the
# name. However in the 5.2 Unicode beta testing, there were a number of such
# collisions, which were withdrawn before the final release, because of Perl's
# and other's protests. These all involved new properties which began with
# 'Is'. Based on the protests, Unicode is unlikely to try that again. Also,
# many of the Perl-defined synonyms, like Any, Word, etc, are listed in a
# Unicode document, so they are unlikely to be used by Unicode for another
# purpose. However, they might try something beginning with 'In', or use any
# of the other Perl-defined properties. This program will warn you of name
# collisions, and refuse to generate tables with them, but manual intervention
# will be required in this event. One scheme that could be implemented, if
# necessary, would be to have this program generate another file, or add a
# field to mktables.lst that gives the date of first definition of a property.
# Each new release of Unicode would use that file as a basis for the next
# iteration. And the Perl synonym addition code could sort based on the age
# of the property, so older properties get priority, and newer ones that clash
# would be refused; hence existing code would not be impacted, and some other
# synonym would have to be used for the new property. This is ugly, and
# manual intervention would certainly be easier to do in the short run; lets
# hope it never comes to this.
#
# A NOTE ON UNIHAN
#
# This program can generate tables from the Unihan database. But that DB
# isn't normally available, so it is marked as optional. Prior to version
# 5.2, this database was in a single file, Unihan.txt. In 5.2 the database
# was split into 8 different files, all beginning with the letters 'Unihan'.
# If you plunk those files down into the directory mktables ($0) is in, this
# program will read them and automatically create tables for the properties
# from it that are listed in PropertyAliases.txt and PropValueAliases.txt,
# plus any you add to the @cjk_properties array and the @cjk_property_values
# array, being sure to add necessary '# @missings' lines to the latter. For
# Unicode versions earlier than 5.2, most of the Unihan properties are not
# listed at all in PropertyAliases nor PropValueAliases. This program assumes
# for these early releases that you want the properties that are specified in
# the 5.2 release.
#
# You may need to adjust the entries to suit your purposes. setup_unihan(),
# and filter_unihan_line() are the functions where this is done. This program
# already does some adjusting to make the lines look more like the rest of the
# Unicode DB; You can see what that is in filter_unihan_line()
#
# There is a bug in the 3.2 data file in which some values for the
# kPrimaryNumeric property have commas and an unexpected comment. A filter
# could be added to correct these; or for a particular installation, the
# Unihan.txt file could be edited to fix them.
#
# HOW TO ADD A FILE TO BE PROCESSED
#
# A new file from Unicode needs to have an object constructed for it in
# @input_file_objects, probably at the end or at the end of the extracted
# ones. The program should warn you if its name will clash with others on
# restrictive file systems, like DOS. If so, figure out a better name, and
# add lines to the README.perl file giving that. If the file is a character
# property, it should be in the format that Unicode has implicitly
# standardized for such files for the more recently introduced ones.
# If so, the Input_file constructor for @input_file_objects can just be the
# file name and release it first appeared in. If not, then it should be
# possible to construct an each_line_handler() to massage the line into the
# standardized form.
#
# For non-character properties, more code will be needed. You can look at
# the existing entries for clues.
#
# UNICODE VERSIONS NOTES
#
# The Unicode UCD has had a number of errors in it over the versions. And
# these remain, by policy, in the standard for that version. Therefore it is
# risky to correct them, because code may be expecting the error. So this
# program doesn't generally make changes, unless the error breaks the Perl
# core. As an example, some versions of 2.1.x Jamo.txt have the wrong value
# for U+1105, which causes real problems for the algorithms for Jamo
# calculations, so it is changed here.
#
# But it isn't so clear cut as to what to do about concepts that are
# introduced in a later release; should they extend back to earlier releases
# where the concept just didn't exist? It was easier to do this than to not,
# so that's what was done. For example, the default value for code points not
# in the files for various properties was probably undefined until changed by
# some version. No_Block for blocks is such an example. This program will
# assign No_Block even in Unicode versions that didn't have it. This has the
# benefit that code being written doesn't have to special case earlier
# versions; and the detriment that it doesn't match the Standard precisely for
# the affected versions.
#
# Here are some observations about some of the issues in early versions:
#
# Prior to version 3.0, there were 3 character decompositions. These are not
# handled by Unicode::Normalize, nor will it compile when presented a version
# that has them. However, you can trivially get it to compile by simply
# ignoring those decompositions, by changing the croak to a carp. At the time
# of this writing, the line (in dist/Unicode-Normalize/Normalize.pm or
# dist/Unicode-Normalize/mkheader) reads
#
# croak("Weird Canonical Decomposition of U+$h");
#
# Simply comment it out. It will compile, but will not know about any three
# character decompositions.
# The number of code points in \p{alpha=True} halved in 2.1.9. It turns out
# that the reason is that the CJK block starting at 4E00 was removed from
# PropList, and was not put back in until 3.1.0. The Perl extension (the
# single property name \p{alpha}) has the correct values. But the compound
# form is simply not generated until 3.1, as it can be argued that prior to
# this release, this was not an official property. The comments for
# filter_old_style_proplist() give more details.
#
# Unicode introduced the synonym Space for White_Space in 4.1. Perl has
# always had a \p{Space}. In release 3.2 only, they are not synonymous. The
# reason is that 3.2 introduced U+205F=medium math space, which was not
# classed as white space, but Perl figured out that it should have been. 4.0
# reclassified it correctly.
#
# Another change between 3.2 and 4.0 is the CCC property value ATBL. In 3.2
# this was erroneously a synonym for 202 (it should be 200). In 4.0, ATB
# became 202, and ATBL was left with no code points, as all the ones that
# mapped to 202 stayed mapped to 202. Thus if your program used the numeric
# name for the class, it would not have been affected, but if it used the
# mnemonic, it would have been.
#
# \p{Script=Hrkt} (Katakana_Or_Hiragana) came in 4.0.1. Before that, code
# points which eventually came to have this script property value, instead
# mapped to "Unknown". But in the next release all these code points were
# moved to \p{sc=common} instead.
# The tests furnished by Unicode for testing WordBreak and SentenceBreak
# generate errors in 5.0 and earlier.
#
# The default for missing code points for BidiClass is complicated. Starting
# in 3.1.1, the derived file DBidiClass.txt handles this, but this program
# tries to do the best it can for earlier releases. It is done in
# process_PropertyAliases()
#
# In version 2.1.2, the entry in UnicodeData.txt:
# 0275;LATIN SMALL LETTER BARRED O;Ll;0;L;;;;;N;;;;019F;
# should instead be
# 0275;LATIN SMALL LETTER BARRED O;Ll;0;L;;;;;N;;;019F;;019F
# Without this change, there are casing problems for this character.
#
# Search for $string_compare_versions to see how to compare changes to
# properties between Unicode versions
#
##############################################################################
my $UNDEF = ':UNDEF:'; # String to print out for undefined values in tracing
# and errors
my $MAX_LINE_WIDTH = 78;
# Debugging aid to skip most files so as to not be distracted by them when
# concentrating on the ones being debugged. Add
# non_skip => 1,
# to the constructor for those files you want processed when you set this.
# Files with a first version number of 0 are special: they are always
# processed regardless of the state of this flag. Generally, Jamo.txt and
# UnicodeData.txt must not be skipped if you want this program to not die
# before normal completion.
my $debug_skip = 0;
# Normally these are suppressed.
my $write_Unicode_deprecated_tables = 0;
# Set to 1 to enable tracing.
our $to_trace = 0;
{ # Closure for trace: debugging aid
my $print_caller = 1; # ? Include calling subroutine name
my $main_with_colon = 'main::';
my $main_colon_length = length($main_with_colon);
sub trace {
return unless $to_trace; # Do nothing if global flag not set
my @input = @_;
local $DB::trace = 0;
$DB::trace = 0; # Quiet 'used only once' message
my $line_number;
# Loop looking up the stack to get the first non-trace caller
my $caller_line;
my $caller_name;
my $i = 0;
do {
$line_number = $caller_line;
(my $pkg, my $file, $caller_line, my $caller) = caller $i++;
$caller = $main_with_colon unless defined $caller;
$caller_name = $caller;
# get rid of pkg
$caller_name =~ s/.*:://;
if (substr($caller_name, 0, $main_colon_length)
eq $main_with_colon)
{
$caller_name = substr($caller_name, $main_colon_length);
}
} until ($caller_name ne 'trace');
# If the stack was empty, we were called from the top level
$caller_name = 'main' if ($caller_name eq ""
|| $caller_name eq 'trace');
my $output = "";
#print STDERR __LINE__, ": ", join ", ", @input, "\n";
foreach my $string (@input) {
if (ref $string eq 'ARRAY' || ref $string eq 'HASH') {
$output .= simple_dumper($string);
}
else {
$string = "$string" if ref $string;
$string = $UNDEF unless defined $string;
chomp $string;
$string = '""' if $string eq "";
$output .= " " if $output ne ""
&& $string ne ""
&& substr($output, -1, 1) ne " "
&& substr($string, 0, 1) ne " ";
$output .= $string;
}
}
print STDERR sprintf "%4d: ", $line_number if defined $line_number;
print STDERR "$caller_name: " if $print_caller;
print STDERR $output, "\n";
return;
}
}
sub stack_trace() {
local $to_trace = 1 if main::DEBUG;
my $line = (caller(0))[2];
my $i = 1;
# Accumulate the stack trace
while (1) {
my ($pkg, $file, $caller_line, $caller) = caller $i++;
last unless defined $caller;
trace "called from $caller() at line $line";
$line = $caller_line;
}
}
# This is for a rarely used development feature that allows you to compare two
# versions of the Unicode standard without having to deal with changes caused
# by the code points introduced in the later version. You probably also want
# to use the -annotate option when using this. Run this program on a unicore
# containing the starting release you want to compare. Save that output
# structure. Then, switching to a unicore with the ending release, change the
# "" in the $string_compare_versions definition just below to a string
# containing a SINGLE dotted Unicode release number (e.g. "2.1") corresponding
# to the starting release. This program will then compile, but throw away all
# code points introduced after the starting release. Finally use a diff tool
# to compare the two directory structures. They include only the code points
# common to both releases, and you can see the changes caused just by the
# underlying release semantic changes. For versions earlier than 3.2, you
# must copy a version of DAge.txt into the directory.
my $string_compare_versions = DEBUG && "";
my $compare_versions = DEBUG
&& $string_compare_versions
&& pack "C*", split /\./, $string_compare_versions;
sub uniques {
# Returns non-duplicated input values. From "Perl Best Practices:
# Encapsulated Cleverness". p. 455 in first edition.
my %seen;
# Arguably this breaks encapsulation, if the goal is to permit multiple
# distinct objects to stringify to the same value, and be interchangeable.
# However, for this program, no two objects stringify identically, and all
# lists passed to this function are either objects or strings. So this
# doesn't affect correctness, but it does give a couple of percent speedup.
no overloading;
return grep { ! $seen{$_}++ } @_;
}
$0 = File::Spec->canonpath($0);
my $make_test_script = 0; # ? Should we output a test script
my $make_norm_test_script = 0; # ? Should we output a normalization test script
my $write_unchanged_files = 0; # ? Should we update the output files even if
# we don't think they have changed
my $use_directory = ""; # ? Should we chdir somewhere.
my $pod_directory; # input directory to store the pod file.
my $pod_file = 'perluniprops';
my $t_path; # Path to the .t test file
my $file_list = 'mktables.lst'; # File to store input and output file names.
# This is used to speed up the build, by not
# executing the main body of the program if
# nothing on the list has changed since the
# previous build
my $make_list = 1; # ? Should we write $file_list. Set to always
# make a list so that when the pumpking is
# preparing a release, s/he won't have to do
# special things
my $glob_list = 0; # ? Should we try to include unknown .txt files
# in the input.
my $output_range_counts = $debugging_build; # ? Should we include the number
# of code points in ranges in
# the output
my $annotate = 0; # ? Should character names be in the output
# Verbosity levels; 0 is quiet
my $NORMAL_VERBOSITY = 1;
my $PROGRESS = 2;
my $VERBOSE = 3;
my $verbosity = $NORMAL_VERBOSITY;
# Stored in mktables.lst so that if this program is called with different
# options, will regenerate even if the files otherwise look like they're
# up-to-date.
my $command_line_arguments = join " ", @ARGV;
# Process arguments
while (@ARGV) {
my $arg = shift @ARGV;
if ($arg eq '-v') {
$verbosity = $VERBOSE;
}
elsif ($arg eq '-p') {
$verbosity = $PROGRESS;
$| = 1; # Flush buffers as we go.
}
elsif ($arg eq '-q') {
$verbosity = 0;
}
elsif ($arg eq '-w') {
# update the files even if they haven't changed
$write_unchanged_files = 1;
}
elsif ($arg eq '-check') {
my $this = shift @ARGV;
my $ok = shift @ARGV;
if ($this ne $ok) {
print "Skipping as check params are not the same.\n";
exit(0);
}
}
elsif ($arg eq '-P' && defined ($pod_directory = shift)) {
-d $pod_directory or croak "Directory '$pod_directory' doesn't exist";
}
elsif ($arg eq '-maketest' || ($arg eq '-T' && defined ($t_path = shift)))
{
$make_test_script = 1;
}
elsif ($arg eq '-makenormtest')
{
$make_norm_test_script = 1;
}
elsif ($arg eq '-makelist') {
$make_list = 1;
}
elsif ($arg eq '-C' && defined ($use_directory = shift)) {
-d $use_directory or croak "Unknown directory '$use_directory'";
}
elsif ($arg eq '-L') {
# Existence not tested until have chdir'd
$file_list = shift;
}
elsif ($arg eq '-globlist') {
$glob_list = 1;
}
elsif ($arg eq '-c') {
$output_range_counts = ! $output_range_counts
}
elsif ($arg eq '-annotate') {
$annotate = 1;
$debugging_build = 1;
$output_range_counts = 1;
}
else {
my $with_c = 'with';
$with_c .= 'out' if $output_range_counts; # Complements the state
croak <<END;
usage: $0 [-c|-p|-q|-v|-w] [-C dir] [-L filelist] [ -P pod_dir ]
[ -T test_file_path ] [-globlist] [-makelist] [-maketest]
[-check A B ]
-c : Output comments $with_c number of code points in ranges
-q : Quiet Mode: Only output serious warnings.
-p : Set verbosity level to normal plus show progress.
-v : Set Verbosity level high: Show progress and non-serious
warnings
-w : Write files regardless
-C dir : Change to this directory before proceeding. All relative paths
except those specified by the -P and -T options will be done
with respect to this directory.
-P dir : Output $pod_file file to directory 'dir'.
-T path : Create a test script as 'path'; overrides -maketest
-L filelist : Use alternate 'filelist' instead of standard one
-globlist : Take as input all non-Test *.txt files in current and sub
directories
-maketest : Make test script 'TestProp.pl' in current (or -C directory),
overrides -T
-makelist : Rewrite the file list $file_list based on current setup
-annotate : Output an annotation for each character in the table files;
useful for debugging mktables, looking at diffs; but is slow
and memory intensive
-check A B : Executes $0 only if A and B are the same
END
}
}
# Stores the most-recently changed file. If none have changed, can skip the
# build
my $most_recent = (stat $0)[9]; # Do this before the chdir!
# Change directories now, because need to read 'version' early.
if ($use_directory) {
if ($pod_directory && ! File::Spec->file_name_is_absolute($pod_directory)) {
$pod_directory = File::Spec->rel2abs($pod_directory);
}
if ($t_path && ! File::Spec->file_name_is_absolute($t_path)) {
$t_path = File::Spec->rel2abs($t_path);
}
chdir $use_directory or croak "Failed to chdir to '$use_directory':$!";
if ($pod_directory && File::Spec->file_name_is_absolute($pod_directory)) {
$pod_directory = File::Spec->abs2rel($pod_directory);
}
if ($t_path && File::Spec->file_name_is_absolute($t_path)) {
$t_path = File::Spec->abs2rel($t_path);
}
}
# Get Unicode version into regular and v-string. This is done now because
# various tables below get populated based on it. These tables are populated
# here to be near the top of the file, and so easily seeable by those needing
# to modify things.
open my $VERSION, "<", "version"
or croak "$0: can't open required file 'version': $!\n";
my $string_version = <$VERSION>;
close $VERSION;
chomp $string_version;
my $v_version = pack "C*", split /\./, $string_version; # v string
my $unicode_version = ($compare_versions)
? ( "$string_compare_versions (using "
. "$string_version rules)")
: $string_version;
# The following are the complete names of properties with property values that
# are known to not match any code points in some versions of Unicode, but that
# may change in the future so they should be matchable, hence an empty file is
# generated for them.
my @tables_that_may_be_empty;
push @tables_that_may_be_empty, 'Joining_Type=Left_Joining'
if $v_version lt v6.3.0;
push @tables_that_may_be_empty, 'Script=Common' if $v_version le v4.0.1;
push @tables_that_may_be_empty, 'Title' if $v_version lt v2.0.0;
push @tables_that_may_be_empty, 'Script=Katakana_Or_Hiragana'
if $v_version ge v4.1.0;
push @tables_that_may_be_empty, 'Script_Extensions=Katakana_Or_Hiragana'
if $v_version ge v6.0.0;
push @tables_that_may_be_empty, 'Grapheme_Cluster_Break=Prepend'
if $v_version ge v6.1.0;
push @tables_that_may_be_empty, 'Canonical_Combining_Class=CCC133'
if $v_version ge v6.2.0;
# The lists below are hashes, so the key is the item in the list, and the
# value is the reason why it is in the list. This makes generation of
# documentation easier.
my %why_suppressed; # No file generated for these.
# Files aren't generated for empty extraneous properties. This is arguable.
# Extraneous properties generally come about because a property is no longer
# used in a newer version of Unicode. If we generated a file without code
# points, programs that used to work on that property will still execute
# without errors. It just won't ever match (or will always match, with \P{}).
# This means that the logic is now likely wrong. I (khw) think its better to
# find this out by getting an error message. Just move them to the table
# above to change this behavior
my %why_suppress_if_empty_warn_if_not = (
# It is the only property that has ever officially been removed from the
# Standard. The database never contained any code points for it.
'Special_Case_Condition' => 'Obsolete',
# Apparently never official, but there were code points in some versions of
# old-style PropList.txt
'Non_Break' => 'Obsolete',
);
# These would normally go in the warn table just above, but they were changed
# a long time before this program was written, so warnings about them are
# moot.
if ($v_version gt v3.2.0) {
push @tables_that_may_be_empty,
'Canonical_Combining_Class=Attached_Below_Left'
}
# Obsoleted
if ($v_version ge v11.0.0) {
push @tables_that_may_be_empty, qw(
Grapheme_Cluster_Break=E_Base
Grapheme_Cluster_Break=E_Base_GAZ
Grapheme_Cluster_Break=E_Modifier
Grapheme_Cluster_Break=Glue_After_Zwj
Word_Break=E_Base
Word_Break=E_Base_GAZ
Word_Break=E_Modifier
Word_Break=Glue_After_Zwj);
}
# Enum values for to_output_map() method in the Map_Table package. (0 is don't
# output)
my $EXTERNAL_MAP = 1;
my $INTERNAL_MAP = 2;
my $OUTPUT_ADJUSTED = 3;
# To override computed values for writing the map tables for these properties.
# The default for enum map tables is to write them out, so that the Unicode
# .txt files can be removed, but all the data to compute any property value
# for any code point is available in a more compact form.
my %global_to_output_map = (
# Needed by UCD.pm, but don't want to publicize that it exists, so won't
# get stuck supporting it if things change. Since it is a STRING
# property, it normally would be listed in the pod, but INTERNAL_MAP
# suppresses that.
Unicode_1_Name => $INTERNAL_MAP,
Present_In => 0, # Suppress, as easily computed from Age
Block => (NON_ASCII_PLATFORM) ? 1 : 0, # Suppress, as Blocks.txt is
# retained, but needed for
# non-ASCII
# Suppress, as mapping can be found instead from the
# Perl_Decomposition_Mapping file
Decomposition_Type => 0,
);
# There are several types of obsolete properties defined by Unicode. These
# must be hand-edited for every new Unicode release.
my %why_deprecated; # Generates a deprecated warning message if used.
my %why_stabilized; # Documentation only
my %why_obsolete; # Documentation only
{ # Closure
my $simple = 'Perl uses the more complete version';
my $unihan = 'Unihan properties are by default not enabled in the Perl core. Instead use CPAN: Unicode::Unihan';
my $other_properties = 'other properties';
my $contributory = "Used by Unicode internally for generating $other_properties and not intended to be used stand-alone";
my $why_no_expand = "Deprecated by Unicode. These are characters that expand to more than one character in the specified normalization form, but whether they actually take up more bytes or not depends on the encoding being used. For example, a UTF-8 encoded character may expand to a different number of bytes than a UTF-32 encoded character.";
%why_deprecated = (
'Grapheme_Link' => 'Duplicates ccc=vr (Canonical_Combining_Class=Virama)',
'Jamo_Short_Name' => $contributory,
'Line_Break=Surrogate' => 'Surrogates should never appear in well-formed text, and therefore shouldn\'t be the basis for line breaking',
'Other_Alphabetic' => $contributory,
'Other_Default_Ignorable_Code_Point' => $contributory,
'Other_Grapheme_Extend' => $contributory,
'Other_ID_Continue' => $contributory,
'Other_ID_Start' => $contributory,
'Other_Lowercase' => $contributory,
'Other_Math' => $contributory,
'Other_Uppercase' => $contributory,
'Expands_On_NFC' => $why_no_expand,
'Expands_On_NFD' => $why_no_expand,
'Expands_On_NFKC' => $why_no_expand,
'Expands_On_NFKD' => $why_no_expand,
);
%why_suppressed = (
# There is a lib/unicore/Decomposition.pl (used by Normalize.pm) which
# contains the same information, but without the algorithmically
# determinable Hangul syllables'. This file is not published, so it's
# existence is not noted in the comment.
'Decomposition_Mapping' => 'Accessible via Unicode::Normalize or prop_invmap() or charprop() in Unicode::UCD::',
# Don't suppress ISO_Comment, as otherwise special handling is needed
# to differentiate between it and gc=c, which can be written as 'isc',
# which is the same characters as ISO_Comment's short name.
'Name' => "Accessible via \\N{...} or 'use charnames;' or charprop() or prop_invmap() in Unicode::UCD::",
'Simple_Case_Folding' => "$simple. Can access this through casefold(), charprop(), or prop_invmap() in Unicode::UCD",
'Simple_Lowercase_Mapping' => "$simple. Can access this through charinfo(), charprop(), or prop_invmap() in Unicode::UCD",
'Simple_Titlecase_Mapping' => "$simple. Can access this through charinfo(), charprop(), or prop_invmap() in Unicode::UCD",
'Simple_Uppercase_Mapping' => "$simple. Can access this through charinfo(), charprop(), or prop_invmap() in Unicode::UCD",
FC_NFKC_Closure => 'Deprecated by Unicode, and supplanted in usage by NFKC_Casefold; otherwise not useful',
);
foreach my $property (
# The following are suppressed because they were made contributory
# or deprecated by Unicode before Perl ever thought about
# supporting them.
'Jamo_Short_Name',
'Grapheme_Link',
'Expands_On_NFC',
'Expands_On_NFD',
'Expands_On_NFKC',
'Expands_On_NFKD',
# The following are suppressed because they have been marked
# as deprecated for a sufficient amount of time
'Other_Alphabetic',
'Other_Default_Ignorable_Code_Point',
'Other_Grapheme_Extend',
'Other_ID_Continue',
'Other_ID_Start',
'Other_Lowercase',
'Other_Math',
'Other_Uppercase',
) {
$why_suppressed{$property} = $why_deprecated{$property};
}
# Customize the message for all the 'Other_' properties
foreach my $property (keys %why_deprecated) {
next if (my $main_property = $property) !~ s/^Other_//;
$why_deprecated{$property} =~ s/$other_properties/the $main_property property (which should be used instead)/;
}
}
if ($write_Unicode_deprecated_tables) {
foreach my $property (keys %why_suppressed) {
delete $why_suppressed{$property} if $property =~
/ ^ Other | Grapheme /x;
}
}
if ($v_version ge 4.0.0) {
$why_stabilized{'Hyphen'} = 'Use the Line_Break property instead; see www.unicode.org/reports/tr14';
if ($v_version ge 6.0.0) {
$why_deprecated{'Hyphen'} = 'Supplanted by Line_Break property values; see www.unicode.org/reports/tr14';
}
}
if ($v_version ge 5.2.0 && $v_version lt 6.0.0) {
$why_obsolete{'ISO_Comment'} = 'Code points for it have been removed';
if ($v_version ge 6.0.0) {
$why_deprecated{'ISO_Comment'} = 'No longer needed for Unicode\'s internal chart generation; otherwise not useful, and code points for it have been removed';
}
}
# Probably obsolete forever
if ($v_version ge v4.1.0) {
$why_suppressed{'Script=Katakana_Or_Hiragana'} = 'Obsolete. All code points previously matched by this have been moved to "Script=Common".';
}
if ($v_version ge v6.0.0) {
$why_suppressed{'Script=Katakana_Or_Hiragana'} .= ' Consider instead using "Script_Extensions=Katakana" or "Script_Extensions=Hiragana" (or both)';
$why_suppressed{'Script_Extensions=Katakana_Or_Hiragana'} = 'All code points that would be matched by this are matched by either "Script_Extensions=Katakana" or "Script_Extensions=Hiragana"';
}
# This program can create files for enumerated-like properties, such as
# 'Numeric_Type'. This file would be the same format as for a string
# property, with a mapping from code point to its value, so you could look up,
# for example, the script a code point is in. But no one so far wants this
# mapping, or they have found another way to get it since this is a new
# feature. So no file is generated except if it is in this list.
my @output_mapped_properties = split "\n", <<END;
END
# If you want more Unihan properties than the default, you need to add them to
# these arrays. Depending on the property type, @missing lines might have to
# be added to the second array. A sample entry would be (including the '#'):
# @missing: 0000..10FFFF; cjkAccountingNumeric; NaN
my @cjk_properties = split "\n", <<'END';
END
my @cjk_property_values = split "\n", <<'END';
END
# The input files don't list every code point. Those not listed are to be
# defaulted to some value. Below are hard-coded what those values are for
# non-binary properties as of 5.1. Starting in 5.0, there are
# machine-parsable comment lines in the files that give the defaults; so this
# list shouldn't have to be extended. The claim is that all missing entries
# for binary properties will default to 'N'. Unicode tried to change that in
# 5.2, but the beta period produced enough protest that they backed off.
#
# The defaults for the fields that appear in UnicodeData.txt in this hash must
# be in the form that it expects. The others may be synonyms.
my $CODE_POINT = '<code point>';
my %default_mapping = (
Age => "Unassigned",
# Bidi_Class => Complicated; set in code
Bidi_Mirroring_Glyph => "",
Block => 'No_Block',
Canonical_Combining_Class => 0,
Case_Folding => $CODE_POINT,
Decomposition_Mapping => $CODE_POINT,
Decomposition_Type => 'None',
East_Asian_Width => "Neutral",
FC_NFKC_Closure => $CODE_POINT,
General_Category => ($v_version le 6.3.0) ? 'Cn' : 'Unassigned',
Grapheme_Cluster_Break => 'Other',
Hangul_Syllable_Type => 'NA',
ISO_Comment => "",
Jamo_Short_Name => "",
Joining_Group => "No_Joining_Group",
# Joining_Type => Complicated; set in code
kIICore => 'N', # Is converted to binary
#Line_Break => Complicated; set in code
Lowercase_Mapping => $CODE_POINT,
Name => "",
Name_Alias => "",
NFC_QC => 'Yes',
NFD_QC => 'Yes',
NFKC_QC => 'Yes',
NFKD_QC => 'Yes',
Numeric_Type => 'None',
Numeric_Value => 'NaN',
Script => ($v_version le 4.1.0) ? 'Common' : 'Unknown',
Sentence_Break => 'Other',
Simple_Case_Folding => $CODE_POINT,
Simple_Lowercase_Mapping => $CODE_POINT,
Simple_Titlecase_Mapping => $CODE_POINT,
Simple_Uppercase_Mapping => $CODE_POINT,
Titlecase_Mapping => $CODE_POINT,
Unicode_1_Name => "",
Unicode_Radical_Stroke => "",
Uppercase_Mapping => $CODE_POINT,
Word_Break => 'Other',
);
### End of externally interesting definitions, except for @input_file_objects
my $HEADER=<<"EOF";
# !!!!!!! DO NOT EDIT THIS FILE !!!!!!!
# This file is machine-generated by $0 from the Unicode
# database, Version $unicode_version. Any changes made here will be lost!
EOF
my $INTERNAL_ONLY_HEADER = <<"EOF";
# !!!!!!! INTERNAL PERL USE ONLY !!!!!!!
# This file is for internal use by core Perl only. The format and even the
# name or existence of this file are subject to change without notice. Don't
# use it directly. Use Unicode::UCD to access the Unicode character data
# base.
EOF
my $DEVELOPMENT_ONLY=<<"EOF";
# !!!!!!! DEVELOPMENT USE ONLY !!!!!!!
# This file contains information artificially constrained to code points
# present in Unicode release $string_compare_versions.
# IT CANNOT BE RELIED ON. It is for use during development only and should
# not be used for production.
EOF
my $MAX_UNICODE_CODEPOINT_STRING = ($v_version ge v2.0.0)
? "10FFFF"
: "FFFF";
my $MAX_UNICODE_CODEPOINT = hex $MAX_UNICODE_CODEPOINT_STRING;
my $MAX_UNICODE_CODEPOINTS = $MAX_UNICODE_CODEPOINT + 1;
# We work with above-Unicode code points, up to IV_MAX, but we may want to use
# sentinels above that number. Therefore for internal use, we use a much
# smaller number, translating it to IV_MAX only for output. The exact number
# is immaterial (all above-Unicode code points are treated exactly the same),
# but the algorithm requires it to be at least
# 2 * $MAX_UNICODE_CODEPOINTS + 1
my $MAX_WORKING_CODEPOINTS= $MAX_UNICODE_CODEPOINT * 8;
my $MAX_WORKING_CODEPOINT = $MAX_WORKING_CODEPOINTS - 1;
my $MAX_WORKING_CODEPOINT_STRING = sprintf("%X", $MAX_WORKING_CODEPOINT);
my $MAX_PLATFORM_CODEPOINT = ~0 >> 1;
# Matches legal code point. 4-6 hex numbers, If there are 6, the first
# two must be 10; if there are 5, the first must not be a 0. Written this way
# to decrease backtracking. The first regex allows the code point to be at
# the end of a word, but to work properly, the word shouldn't end with a valid
# hex character. The second one won't match a code point at the end of a
# word, and doesn't have the run-on issue
my $run_on_code_point_re =
qr/ (?: 10[0-9A-F]{4} | [1-9A-F][0-9A-F]{4} | [0-9A-F]{4} ) \b/x;
my $code_point_re = qr/\b$run_on_code_point_re/;
# This matches the beginning of the line in the Unicode DB files that give the
# defaults for code points not listed (i.e., missing) in the file. The code
# depends on this ending with a semi-colon, so it can assume it is a valid
# field when the line is split() by semi-colons
my $missing_defaults_prefix = qr/^#\s+\@missing:\s+0000\.\.10FFFF\s*;/;
# Property types. Unicode has more types, but these are sufficient for our
# purposes.
my $UNKNOWN = -1; # initialized to illegal value
my $NON_STRING = 1; # Either binary or enum
my $BINARY = 2;
my $FORCED_BINARY = 3; # Not a binary property, but, besides its normal
# tables, additional true and false tables are
# generated so that false is anything matching the
# default value, and true is everything else.
my $ENUM = 4; # Include catalog
my $STRING = 5; # Anything else: string or misc
# Some input files have lines that give default values for code points not
# contained in the file. Sometimes these should be ignored.
my $NO_DEFAULTS = 0; # Must evaluate to false
my $NOT_IGNORED = 1;
my $IGNORED = 2;
# Range types. Each range has a type. Most ranges are type 0, for normal,
# and will appear in the main body of the tables in the output files, but
# there are other types of ranges as well, listed below, that are specially
# handled. There are pseudo-types as well that will never be stored as a
# type, but will affect the calculation of the type.
# 0 is for normal, non-specials
my $MULTI_CP = 1; # Sequence of more than code point
my $HANGUL_SYLLABLE = 2;
my $CP_IN_NAME = 3; # The NAME contains the code point appended to it.
my $NULL = 4; # The map is to the null string; utf8.c can't
# handle these, nor is there an accepted syntax
# for them in \p{} constructs
my $COMPUTE_NO_MULTI_CP = 5; # Pseudo-type; means that ranges that would
# otherwise be $MULTI_CP type are instead type 0
# process_generic_property_file() can accept certain overrides in its input.
# Each of these must begin AND end with $CMD_DELIM.
my $CMD_DELIM = "\a";
my $REPLACE_CMD = 'replace'; # Override the Replace
my $MAP_TYPE_CMD = 'map_type'; # Override the Type
my $NO = 0;
my $YES = 1;
# Values for the Replace argument to add_range.
# $NO # Don't replace; add only the code points not
# already present.
my $IF_NOT_EQUIVALENT = 1; # Replace only under certain conditions; details in
# the comments at the subroutine definition.
my $UNCONDITIONALLY = 2; # Replace without conditions.
my $MULTIPLE_BEFORE = 4; # Don't replace, but add a duplicate record if
# already there
my $MULTIPLE_AFTER = 5; # Don't replace, but add a duplicate record if
# already there
my $CROAK = 6; # Die with an error if is already there
# Flags to give property statuses. The phrases are to remind maintainers that
# if the flag is changed, the indefinite article referring to it in the
# documentation may need to be as well.
my $NORMAL = "";
my $DEPRECATED = 'D';
my $a_bold_deprecated = "a 'B<$DEPRECATED>'";
my $A_bold_deprecated = "A 'B<$DEPRECATED>'";
my $DISCOURAGED = 'X';
my $a_bold_discouraged = "an 'B<$DISCOURAGED>'";
my $A_bold_discouraged = "An 'B<$DISCOURAGED>'";
my $STRICTER = 'T';
my $a_bold_stricter = "a 'B<$STRICTER>'";
my $A_bold_stricter = "A 'B<$STRICTER>'";
my $STABILIZED = 'S';
my $a_bold_stabilized = "an 'B<$STABILIZED>'";
my $A_bold_stabilized = "An 'B<$STABILIZED>'";
my $OBSOLETE = 'O';
my $a_bold_obsolete = "an 'B<$OBSOLETE>'";
my $A_bold_obsolete = "An 'B<$OBSOLETE>'";
# Aliases can also have an extra status:
my $INTERNAL_ALIAS = 'P';
my %status_past_participles = (
$DISCOURAGED => 'discouraged',
$STABILIZED => 'stabilized',
$OBSOLETE => 'obsolete',
$DEPRECATED => 'deprecated',
$INTERNAL_ALIAS => 'reserved for Perl core internal use only',
);
# Table fates. These are somewhat ordered, so that fates < $MAP_PROXIED should be
# externally documented.
my $ORDINARY = 0; # The normal fate.
my $MAP_PROXIED = 1; # The map table for the property isn't written out,
# but there is a file written that can be used to
# reconstruct this table
my $INTERNAL_ONLY = 2; # The file for this table is written out, but it is
# for Perl's internal use only
my $LEGACY_ONLY = 3; # Like $INTERNAL_ONLY, but not actually used by Perl.
# Is for backwards compatibility for applications that
# read the file directly, so it's format is
# unchangeable.
my $SUPPRESSED = 4; # The file for this table is not written out, and as a
# result, we don't bother to do many computations on
# it.
my $PLACEHOLDER = 5; # Like $SUPPRESSED, but we go through all the
# computations anyway, as the values are needed for
# things to work. This happens when we have Perl
# extensions that depend on Unicode tables that
# wouldn't normally be in a given Unicode version.
# The format of the values of the tables:
my $EMPTY_FORMAT = "";
my $BINARY_FORMAT = 'b';
my $DECIMAL_FORMAT = 'd';
my $FLOAT_FORMAT = 'f';
my $INTEGER_FORMAT = 'i';
my $HEX_FORMAT = 'x';
my $RATIONAL_FORMAT = 'r';
my $STRING_FORMAT = 's';
my $ADJUST_FORMAT = 'a';
my $HEX_ADJUST_FORMAT = 'ax';
my $DECOMP_STRING_FORMAT = 'c';
my $STRING_WHITE_SPACE_LIST = 'sw';
my %map_table_formats = (
$BINARY_FORMAT => 'binary',
$DECIMAL_FORMAT => 'single decimal digit',
$FLOAT_FORMAT => 'floating point number',
$INTEGER_FORMAT => 'integer',
$HEX_FORMAT => 'non-negative hex whole number; a code point',
$RATIONAL_FORMAT => 'rational: an integer or a fraction',
$STRING_FORMAT => 'string',
$ADJUST_FORMAT => 'some entries need adjustment',
$HEX_ADJUST_FORMAT => 'mapped value in hex; some entries need adjustment',
$DECOMP_STRING_FORMAT => 'Perl\'s internal (Normalize.pm) decomposition mapping',
$STRING_WHITE_SPACE_LIST => 'string, but some elements are interpreted as a list; white space occurs only as list item separators'
);
# Unicode didn't put such derived files in a separate directory at first.
my $EXTRACTED_DIR = (-d 'extracted') ? 'extracted' : "";
my $EXTRACTED = ($EXTRACTED_DIR) ? "$EXTRACTED_DIR/" : "";
my $AUXILIARY = 'auxiliary';
# Hashes and arrays that will eventually go into Heavy.pl for the use of
# utf8_heavy.pl and into UCD.pl for the use of UCD.pm
my %loose_to_file_of; # loosely maps table names to their respective
# files
my %stricter_to_file_of; # same; but for stricter mapping.
my %loose_property_to_file_of; # Maps a loose property name to its map file
my %strict_property_to_file_of; # Same, but strict
my @inline_definitions = "V0"; # Each element gives a definition of a unique
# inversion list. When a definition is inlined,
# its value in the hash it's in (one of the two
# defined just above) will include an index into
# this array. The 0th element is initialized to
# the definition for a zero length inversion list
my %file_to_swash_name; # Maps the file name to its corresponding key name
# in the hash %utf8::SwashInfo
my %nv_floating_to_rational; # maps numeric values floating point numbers to
# their rational equivalent
my %loose_property_name_of; # Loosely maps (non_string) property names to
# standard form
my %strict_property_name_of; # Strictly maps (non_string) property names to
# standard form
my %string_property_loose_to_name; # Same, for string properties.
my %loose_defaults; # keys are of form "prop=value", where 'prop' is
# the property name in standard loose form, and
# 'value' is the default value for that property,
# also in standard loose form.
my %loose_to_standard_value; # loosely maps table names to the canonical
# alias for them
my %ambiguous_names; # keys are alias names (in standard form) that
# have more than one possible meaning.
my %combination_property; # keys are alias names (in standard form) that
# have both a map table, and a binary one that
# yields true for all non-null maps.
my %prop_aliases; # Keys are standard property name; values are each
# one's aliases
my %prop_value_aliases; # Keys of top level are standard property name;
# values are keys to another hash, Each one is
# one of the property's values, in standard form.
# The values are that prop-val's aliases.
my %skipped_files; # List of files that we skip
my %ucd_pod; # Holds entries that will go into the UCD section of the pod
# Most properties are immune to caseless matching, otherwise you would get
# nonsensical results, as properties are a function of a code point, not
# everything that is caselessly equivalent to that code point. For example,
# Changes_When_Case_Folded('s') should be false, whereas caselessly it would
# be true because 's' and 'S' are equivalent caselessly. However,
# traditionally, [:upper:] and [:lower:] are equivalent caselessly, so we
# extend that concept to those very few properties that are like this. Each
# such property will match the full range caselessly. They are hard-coded in
# the program; it's not worth trying to make it general as it's extremely
# unlikely that they will ever change.
my %caseless_equivalent_to;
# This is the range of characters that were in Release 1 of Unicode, and
# removed in Release 2 (replaced with the current Hangul syllables starting at
# U+AC00). The range was reused starting in Release 3 for other purposes.
my $FIRST_REMOVED_HANGUL_SYLLABLE = 0x3400;
my $FINAL_REMOVED_HANGUL_SYLLABLE = 0x4DFF;
# These constants names and values were taken from the Unicode standard,
# version 5.1, section 3.12. They are used in conjunction with Hangul
# syllables. The '_string' versions are so generated tables can retain the
# hex format, which is the more familiar value
my $SBase_string = "0xAC00";
my $SBase = CORE::hex $SBase_string;
my $LBase_string = "0x1100";
my $LBase = CORE::hex $LBase_string;
my $VBase_string = "0x1161";
my $VBase = CORE::hex $VBase_string;
my $TBase_string = "0x11A7";
my $TBase = CORE::hex $TBase_string;
my $SCount = 11172;
my $LCount = 19;
my $VCount = 21;
my $TCount = 28;
my $NCount = $VCount * $TCount;
# For Hangul syllables; These store the numbers from Jamo.txt in conjunction
# with the above published constants.
my %Jamo;
my %Jamo_L; # Leading consonants
my %Jamo_V; # Vowels
my %Jamo_T; # Trailing consonants
# For code points whose name contains its ordinal as a '-ABCD' suffix.
# The key is the base name of the code point, and the value is an
# array giving all the ranges that use this base name. Each range
# is actually a hash giving the 'low' and 'high' values of it.
my %names_ending_in_code_point;
my %loose_names_ending_in_code_point; # Same as above, but has blanks, dashes
# removed from the names
# Inverse mapping. The list of ranges that have these kinds of
# names. Each element contains the low, high, and base names in an
# anonymous hash.
my @code_points_ending_in_code_point;
# To hold Unicode's normalization test suite
my @normalization_tests;
# Boolean: does this Unicode version have the hangul syllables, and are we
# writing out a table for them?
my $has_hangul_syllables = 0;
# Does this Unicode version have code points whose names end in their
# respective code points, and are we writing out a table for them? 0 for no;
# otherwise points to first property that a table is needed for them, so that
# if multiple tables are needed, we don't create duplicates
my $needing_code_points_ending_in_code_point = 0;
my @backslash_X_tests; # List of tests read in for testing \X
my @LB_tests; # List of tests read in for testing \b{lb}
my @SB_tests; # List of tests read in for testing \b{sb}
my @WB_tests; # List of tests read in for testing \b{wb}
my @unhandled_properties; # Will contain a list of properties found in
# the input that we didn't process.
my @match_properties; # Properties that have match tables, to be
# listed in the pod
my @map_properties; # Properties that get map files written
my @named_sequences; # NamedSequences.txt contents.
my %potential_files; # Generated list of all .txt files in the directory
# structure so we can warn if something is being
# ignored.
my @missing_early_files; # Generated list of absent files that we need to
# proceed in compiling this early Unicode version
my @files_actually_output; # List of files we generated.
my @more_Names; # Some code point names are compound; this is used
# to store the extra components of them.
my $E_FLOAT_PRECISION = 2; # The minimum number of digits after the decimal
# point of a normalized floating point number
# needed to match before we consider it equivalent
# to a candidate rational
# These store references to certain commonly used property objects
my $age;
my $ccc;
my $gc;
my $perl;
my $block;
my $perl_charname;
my $print;
my $All;
my $Assigned; # All assigned characters in this Unicode release
my $DI; # Default_Ignorable_Code_Point property
my $NChar; # Noncharacter_Code_Point property
my $script;
my $scx; # Script_Extensions property
# Are there conflicting names because of beginning with 'In_', or 'Is_'
my $has_In_conflicts = 0;
my $has_Is_conflicts = 0;
sub internal_file_to_platform ($) {
# Convert our file paths which have '/' separators to those of the
# platform.
my $file = shift;
return undef unless defined $file;
return File::Spec->join(split '/', $file);
}
sub file_exists ($) { # platform independent '-e'. This program internally
# uses slash as a path separator.
my $file = shift;
return 0 if ! defined $file;
return -e internal_file_to_platform($file);
}
sub objaddr($) {
# Returns the address of the blessed input object.
# It doesn't check for blessedness because that would do a string eval
# every call, and the program is structured so that this is never called
# for a non-blessed object.
no overloading; # If overloaded, numifying below won't work.
# Numifying a ref gives its address.
return pack 'J', $_[0];
}
# These are used only if $annotate is true.
# The entire range of Unicode characters is examined to populate these
# after all the input has been processed. But most can be skipped, as they
# have the same descriptive phrases, such as being unassigned
my @viacode; # Contains the 1 million character names
my @age; # And their ages ("" if none)
my @printable; # boolean: And are those characters printable?
my @annotate_char_type; # Contains a type of those characters, specifically
# for the purposes of annotation.
my $annotate_ranges; # A map of ranges of code points that have the same
# name for the purposes of annotation. They map to the
# upper edge of the range, so that the end point can
# be immediately found. This is used to skip ahead to
# the end of a range, and avoid processing each
# individual code point in it.
my $unassigned_sans_noncharacters; # A Range_List of the unassigned
# characters, but excluding those which are
# also noncharacter code points
# The annotation types are an extension of the regular range types, though
# some of the latter are folded into one. Make the new types negative to
# avoid conflicting with the regular types
my $SURROGATE_TYPE = -1;
my $UNASSIGNED_TYPE = -2;
my $PRIVATE_USE_TYPE = -3;
my $NONCHARACTER_TYPE = -4;
my $CONTROL_TYPE = -5;
my $ABOVE_UNICODE_TYPE = -6;
my $UNKNOWN_TYPE = -7; # Used only if there is a bug in this program
sub populate_char_info ($) {
# Used only with the $annotate option. Populates the arrays with the
# input code point's info that are needed for outputting more detailed
# comments. If calling context wants a return, it is the end point of
# any contiguous range of characters that share essentially the same info
my $i = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
$viacode[$i] = $perl_charname->value_of($i) || "";
$age[$i] = (defined $age)
? (($age->value_of($i) =~ / ^ \d+ \. \d+ $ /x)
? $age->value_of($i)
: "")
: "";
# A character is generally printable if Unicode says it is,
# but below we make sure that most Unicode general category 'C' types
# aren't.
$printable[$i] = $print->contains($i);
# But the characters in this range were removed in v2.0 and replaced by
# different ones later. Modern fonts will be for the replacement
# characters, so suppress printing them.
if (($v_version lt v2.0
|| ($compare_versions && $compare_versions lt v2.0))
&& ( $i >= $FIRST_REMOVED_HANGUL_SYLLABLE
&& $i <= $FINAL_REMOVED_HANGUL_SYLLABLE))
{
$printable[$i] = 0;
}
$annotate_char_type[$i] = $perl_charname->type_of($i) || 0;
# Only these two regular types are treated specially for annotations
# purposes
$annotate_char_type[$i] = 0 if $annotate_char_type[$i] != $CP_IN_NAME
&& $annotate_char_type[$i] != $HANGUL_SYLLABLE;
# Give a generic name to all code points that don't have a real name.
# We output ranges, if applicable, for these. Also calculate the end
# point of the range.
my $end;
if (! $viacode[$i]) {
if ($i > $MAX_UNICODE_CODEPOINT) {
$viacode[$i] = 'Above-Unicode';
$annotate_char_type[$i] = $ABOVE_UNICODE_TYPE;
$printable[$i] = 0;
$end = $MAX_WORKING_CODEPOINT;
}
elsif ($gc-> table('Private_use')->contains($i)) {
$viacode[$i] = 'Private Use';
$annotate_char_type[$i] = $PRIVATE_USE_TYPE;
$printable[$i] = 0;
$end = $gc->table('Private_Use')->containing_range($i)->end;
}
elsif ($NChar->contains($i)) {
$viacode[$i] = 'Noncharacter';
$annotate_char_type[$i] = $NONCHARACTER_TYPE;
$printable[$i] = 0;
$end = $NChar->containing_range($i)->end;
}
elsif ($gc-> table('Control')->contains($i)) {
my $name_ref = property_ref('Name_Alias');
$name_ref = property_ref('Unicode_1_Name') if ! defined $name_ref;
$viacode[$i] = (defined $name_ref)
? $name_ref->value_of($i)
: 'Control';
$annotate_char_type[$i] = $CONTROL_TYPE;
$printable[$i] = 0;
}
elsif ($gc-> table('Unassigned')->contains($i)) {
$annotate_char_type[$i] = $UNASSIGNED_TYPE;
$printable[$i] = 0;
$viacode[$i] = 'Unassigned';
if (defined $block) { # No blocks in earliest releases
$viacode[$i] .= ', block=' . $block-> value_of($i);
$end = $gc-> table('Unassigned')->containing_range($i)->end;
# Because we name the unassigned by the blocks they are in, it
# can't go past the end of that block, and it also can't go
# past the unassigned range it is in. The special table makes
# sure that the non-characters, which are unassigned, are
# separated out.
$end = min($block->containing_range($i)->end,
$unassigned_sans_noncharacters->
containing_range($i)->end);
}
else {
$end = $i + 1;
while ($unassigned_sans_noncharacters->contains($end)) {
$end++;
}
$end--;
}
}
elsif ($perl->table('_Perl_Surrogate')->contains($i)) {
$viacode[$i] = 'Surrogate';
$annotate_char_type[$i] = $SURROGATE_TYPE;
$printable[$i] = 0;
$end = $gc->table('Surrogate')->containing_range($i)->end;
}
else {
Carp::my_carp_bug("Can't figure out how to annotate "
. sprintf("U+%04X", $i)
. ". Proceeding anyway.");
$viacode[$i] = 'UNKNOWN';
$annotate_char_type[$i] = $UNKNOWN_TYPE;
$printable[$i] = 0;
}
}
# Here, has a name, but if it's one in which the code point number is
# appended to the name, do that.
elsif ($annotate_char_type[$i] == $CP_IN_NAME) {
$viacode[$i] .= sprintf("-%04X", $i);
my $limit = $perl_charname->containing_range($i)->end;
if (defined $age) {
# Do all these as groups of the same age, instead of individually,
# because their names are so meaningless, and there are typically
# large quantities of them.
$end = $i + 1;
while ($end <= $limit && $age->value_of($end) == $age[$i]) {
$end++;
}
$end--;
}
else {
$end = $limit;
}
}
# And here, has a name, but if it's a hangul syllable one, replace it with
# the correct name from the Unicode algorithm
elsif ($annotate_char_type[$i] == $HANGUL_SYLLABLE) {
use integer;
my $SIndex = $i - $SBase;
my $L = $LBase + $SIndex / $NCount;
my $V = $VBase + ($SIndex % $NCount) / $TCount;
my $T = $TBase + $SIndex % $TCount;
$viacode[$i] = "HANGUL SYLLABLE $Jamo{$L}$Jamo{$V}";
$viacode[$i] .= $Jamo{$T} if $T != $TBase;
$end = $perl_charname->containing_range($i)->end;
}
return if ! defined wantarray;
return $i if ! defined $end; # If not a range, return the input
# Save this whole range so can find the end point quickly
$annotate_ranges->add_map($i, $end, $end);
return $end;
}
# Commented code below should work on Perl 5.8.
## This 'require' doesn't necessarily work in miniperl, and even if it does,
## the native perl version of it (which is what would operate under miniperl)
## is extremely slow, as it does a string eval every call.
#my $has_fast_scalar_util = $^X !~ /miniperl/
# && defined eval "require Scalar::Util";
#
#sub objaddr($) {
# # Returns the address of the blessed input object. Uses the XS version if
# # available. It doesn't check for blessedness because that would do a
# # string eval every call, and the program is structured so that this is
# # never called for a non-blessed object.
#
# return Scalar::Util::refaddr($_[0]) if $has_fast_scalar_util;
#
# # Check at least that is a ref.
# my $pkg = ref($_[0]) or return undef;
#
# # Change to a fake package to defeat any overloaded stringify
# bless $_[0], 'main::Fake';
#
# # Numifying a ref gives its address.
# my $addr = pack 'J', $_[0];
#
# # Return to original class
# bless $_[0], $pkg;
# return $addr;
#}
sub max ($$) {
my $a = shift;
my $b = shift;
return $a if $a >= $b;
return $b;
}
sub min ($$) {
my $a = shift;
my $b = shift;
return $a if $a <= $b;
return $b;
}
sub clarify_number ($) {
# This returns the input number with underscores inserted every 3 digits
# in large (5 digits or more) numbers. Input must be entirely digits, not
# checked.
my $number = shift;
my $pos = length($number) - 3;
return $number if $pos <= 1;
while ($pos > 0) {
substr($number, $pos, 0) = '_';
$pos -= 3;
}
return $number;
}
sub clarify_code_point_count ($) {
# This is like clarify_number(), but the input is assumed to be a count of
# code points, rather than a generic number.
my $append = "";
my $number = shift;
if ($number > $MAX_UNICODE_CODEPOINTS) {
$number -= ($MAX_WORKING_CODEPOINTS - $MAX_UNICODE_CODEPOINTS);
return "All above-Unicode code points" if $number == 0;
$append = " + all above-Unicode code points";
}
return clarify_number($number) . $append;
}
package Carp;
# These routines give a uniform treatment of messages in this program. They
# are placed in the Carp package to cause the stack trace to not include them,
# although an alternative would be to use another package and set @CARP_NOT
# for it.
our $Verbose = 1 if main::DEBUG; # Useful info when debugging
# This is a work-around suggested by Nicholas Clark to fix a problem with Carp
# and overload trying to load Scalar:Util under miniperl. See
# http://www.xray.mpe.mpg.de/mailing-lists/perl5-porters/2009-11/msg01057.html
undef $overload::VERSION;
sub my_carp {
my $message = shift || "";
my $nofold = shift || 0;
if ($message) {
$message = main::join_lines($message);
$message =~ s/^$0: *//; # Remove initial program name
$message =~ s/[.;,]+$//; # Remove certain ending punctuation
$message = "\n$0: $message;";
# Fold the message with program name, semi-colon end punctuation
# (which looks good with the message that carp appends to it), and a
# hanging indent for continuation lines.
$message = main::simple_fold($message, "", 4) unless $nofold;
$message =~ s/\n$//; # Remove the trailing nl so what carp
# appends is to the same line
}
return $message if defined wantarray; # If a caller just wants the msg
carp $message;
return;
}
sub my_carp_bug {
# This is called when it is clear that the problem is caused by a bug in
# this program.
my $message = shift;
$message =~ s/^$0: *//;
$message = my_carp("Bug in $0. Please report it by running perlbug or if that is unavailable, by sending email to perbug\@perl.org:\n$message");
carp $message;
return;
}
sub carp_too_few_args {
if (@_ != 2) {
my_carp_bug("Wrong number of arguments: to 'carp_too_few_arguments'. No action taken.");
return;
}
my $args_ref = shift;
my $count = shift;
my_carp_bug("Need at least $count arguments to "
. (caller 1)[3]
. ". Instead got: '"
. join ', ', @$args_ref
. "'. No action taken.");
return;
}
sub carp_extra_args {
my $args_ref = shift;
my_carp_bug("Too many arguments to 'carp_extra_args': (" . join(', ', @_) . "); Extras ignored.") if @_;
unless (ref $args_ref) {
my_carp_bug("Argument to 'carp_extra_args' ($args_ref) must be a ref. Not checking arguments.");
return;
}
my ($package, $file, $line) = caller;
my $subroutine = (caller 1)[3];
my $list;
if (ref $args_ref eq 'HASH') {
foreach my $key (keys %$args_ref) {
$args_ref->{$key} = $UNDEF unless defined $args_ref->{$key};
}
$list = join ', ', each %{$args_ref};
}
elsif (ref $args_ref eq 'ARRAY') {
foreach my $arg (@$args_ref) {
$arg = $UNDEF unless defined $arg;
}
$list = join ', ', @$args_ref;
}
else {
my_carp_bug("Can't cope with ref "
. ref($args_ref)
. " . argument to 'carp_extra_args'. Not checking arguments.");
return;
}
my_carp_bug("Unrecognized parameters in options: '$list' to $subroutine. Skipped.");
return;
}
package main;
{ # Closure
# This program uses the inside-out method for objects, as recommended in
# "Perl Best Practices". (This is the best solution still, since this has
# to run under miniperl.) This closure aids in generating those. There
# are two routines. setup_package() is called once per package to set
# things up, and then set_access() is called for each hash representing a
# field in the object. These routines arrange for the object to be
# properly destroyed when no longer used, and for standard accessor
# functions to be generated. If you need more complex accessors, just
# write your own and leave those accesses out of the call to set_access().
# More details below.
my %constructor_fields; # fields that are to be used in constructors; see
# below
# The values of this hash will be the package names as keys to other
# hashes containing the name of each field in the package as keys, and
# references to their respective hashes as values.
my %package_fields;
sub setup_package {
# Sets up the package, creating standard DESTROY and dump methods
# (unless already defined). The dump method is used in debugging by
# simple_dumper().
# The optional parameters are:
# a) a reference to a hash, that gets populated by later
# set_access() calls with one of the accesses being
# 'constructor'. The caller can then refer to this, but it is
# not otherwise used by these two routines.
# b) a reference to a callback routine to call during destruction
# of the object, before any fields are actually destroyed
my %args = @_;
my $constructor_ref = delete $args{'Constructor_Fields'};
my $destroy_callback = delete $args{'Destroy_Callback'};
Carp::carp_extra_args(\@_) if main::DEBUG && %args;
my %fields;
my $package = (caller)[0];
$package_fields{$package} = \%fields;
$constructor_fields{$package} = $constructor_ref;
unless ($package->can('DESTROY')) {
my $destroy_name = "${package}::DESTROY";
no strict "refs";
# Use typeglob to give the anonymous subroutine the name we want
*$destroy_name = sub {
my $self = shift;
my $addr = do { no overloading; pack 'J', $self; };
$self->$destroy_callback if $destroy_callback;
foreach my $field (keys %{$package_fields{$package}}) {
#print STDERR __LINE__, ": Destroying ", ref $self, " ", sprintf("%04X", $addr), ": ", $field, "\n";
delete $package_fields{$package}{$field}{$addr};
}
return;
}
}
unless ($package->can('dump')) {
my $dump_name = "${package}::dump";
no strict "refs";
*$dump_name = sub {
my $self = shift;
return dump_inside_out($self, $package_fields{$package}, @_);
}
}
return;
}
sub set_access {
# Arrange for the input field to be garbage collected when no longer
# needed. Also, creates standard accessor functions for the field
# based on the optional parameters-- none if none of these parameters:
# 'addable' creates an 'add_NAME()' accessor function.
# 'readable' or 'readable_array' creates a 'NAME()' accessor
# function.
# 'settable' creates a 'set_NAME()' accessor function.
# 'constructor' doesn't create an accessor function, but adds the
# field to the hash that was previously passed to
# setup_package();
# Any of the accesses can be abbreviated down, so that 'a', 'ad',
# 'add' etc. all mean 'addable'.
# The read accessor function will work on both array and scalar
# values. If another accessor in the parameter list is 'a', the read
# access assumes an array. You can also force it to be array access
# by specifying 'readable_array' instead of 'readable'
#
# A sort-of 'protected' access can be set-up by preceding the addable,
# readable or settable with some initial portion of 'protected_' (but,
# the underscore is required), like 'p_a', 'pro_set', etc. The
# "protection" is only by convention. All that happens is that the
# accessor functions' names begin with an underscore. So instead of
# calling set_foo, the call is _set_foo. (Real protection could be
# accomplished by having a new subroutine, end_package, called at the
# end of each package, and then storing the __LINE__ ranges and
# checking them on every accessor. But that is way overkill.)
# We create anonymous subroutines as the accessors and then use
# typeglobs to assign them to the proper package and name
my $name = shift; # Name of the field
my $field = shift; # Reference to the inside-out hash containing the
# field
my $package = (caller)[0];
if (! exists $package_fields{$package}) {
croak "$0: Must call 'setup_package' before 'set_access'";
}
# Stash the field so DESTROY can get it.
$package_fields{$package}{$name} = $field;
# Remaining arguments are the accessors. For each...
foreach my $access (@_) {
my $access = lc $access;
my $protected = "";
# Match the input as far as it goes.
if ($access =~ /^(p[^_]*)_/) {
$protected = $1;
if (substr('protected_', 0, length $protected)
eq $protected)
{
# Add 1 for the underscore not included in $protected
$access = substr($access, length($protected) + 1);
$protected = '_';
}
else {
$protected = "";
}
}
if (substr('addable', 0, length $access) eq $access) {
my $subname = "${package}::${protected}add_$name";
no strict "refs";
# add_ accessor. Don't add if already there, which we
# determine using 'eq' for scalars and '==' otherwise.
*$subname = sub {
use strict "refs";
return Carp::carp_too_few_args(\@_, 2) if main::DEBUG && @_ < 2;
my $self = shift;
my $value = shift;
my $addr = do { no overloading; pack 'J', $self; };
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
if (ref $value) {
return if grep { $value == $_ } @{$field->{$addr}};
}
else {
return if grep { $value eq $_ } @{$field->{$addr}};
}
push @{$field->{$addr}}, $value;
return;
}
}
elsif (substr('constructor', 0, length $access) eq $access) {
if ($protected) {
Carp::my_carp_bug("Can't set-up 'protected' constructors")
}
else {
$constructor_fields{$package}{$name} = $field;
}
}
elsif (substr('readable_array', 0, length $access) eq $access) {
# Here has read access. If one of the other parameters for
# access is array, or this one specifies array (by being more
# than just 'readable_'), then create a subroutine that
# assumes the data is an array. Otherwise just a scalar
my $subname = "${package}::${protected}$name";
if (grep { /^a/i } @_
or length($access) > length('readable_'))
{
no strict "refs";
*$subname = sub {
use strict "refs";
Carp::carp_extra_args(\@_) if main::DEBUG && @_ > 1;
my $addr = do { no overloading; pack 'J', $_[0]; };
if (ref $field->{$addr} ne 'ARRAY') {
my $type = ref $field->{$addr};
$type = 'scalar' unless $type;
Carp::my_carp_bug("Trying to read $name as an array when it is a $type. Big problems.");
return;
}
return scalar @{$field->{$addr}} unless wantarray;
# Make a copy; had problems with caller modifying the
# original otherwise
my @return = @{$field->{$addr}};
return @return;
}
}
else {
# Here not an array value, a simpler function.
no strict "refs";
*$subname = sub {
use strict "refs";
Carp::carp_extra_args(\@_) if main::DEBUG && @_ > 1;
no overloading;
return $field->{pack 'J', $_[0]};
}
}
}
elsif (substr('settable', 0, length $access) eq $access) {
my $subname = "${package}::${protected}set_$name";
no strict "refs";
*$subname = sub {
use strict "refs";
if (main::DEBUG) {
return Carp::carp_too_few_args(\@_, 2) if @_ < 2;
Carp::carp_extra_args(\@_) if @_ > 2;
}
# $self is $_[0]; $value is $_[1]
no overloading;
$field->{pack 'J', $_[0]} = $_[1];
return;
}
}
else {
Carp::my_carp_bug("Unknown accessor type $access. No accessor set.");
}
}
return;
}
}
package Input_file;
# All input files use this object, which stores various attributes about them,
# and provides for convenient, uniform handling. The run method wraps the
# processing. It handles all the bookkeeping of opening, reading, and closing
# the file, returning only significant input lines.
#
# Each object gets a handler which processes the body of the file, and is
# called by run(). All character property files must use the generic,
# default handler, which has code scrubbed to handle things you might not
# expect, including automatic EBCDIC handling. For files that don't deal with
# mapping code points to a property value, such as test files,
# PropertyAliases, PropValueAliases, and named sequences, you can override the
# handler to be a custom one. Such a handler should basically be a
# while(next_line()) {...} loop.
#
# You can also set up handlers to
# 0) call during object construction time, after everything else is done
# 1) call before the first line is read, for pre processing
# 2) call to adjust each line of the input before the main handler gets
# them. This can be automatically generated, if appropriately simple
# enough, by specifying a Properties parameter in the constructor.
# 3) call upon EOF before the main handler exits its loop
# 4) call at the end, for post processing
#
# $_ is used to store the input line, and is to be filtered by the
# each_line_handler()s. So, if the format of the line is not in the desired
# format for the main handler, these are used to do that adjusting. They can
# be stacked (by enclosing them in an [ anonymous array ] in the constructor,
# so the $_ output of one is used as the input to the next. The EOF handler
# is also stackable, but none of the others are, but could easily be changed
# to be so.
#
# Some properties are used by the Perl core but aren't defined until later
# Unicode releases. The perl interpreter would have problems working when
# compiled with an earlier Unicode version that doesn't have them, so we need
# to define them somehow for those releases. The 'Early' constructor
# parameter can be used to automatically handle this. It is essentially
# ignored if the Unicode version being compiled has a data file for this
# property. Either code to execute or a file to read can be specified.
# Details are at the %early definition.
#
# Most of the handlers can call insert_lines() or insert_adjusted_lines()
# which insert the parameters as lines to be processed before the next input
# file line is read. This allows the EOF handler(s) to flush buffers, for
# example. The difference between the two routines is that the lines inserted
# by insert_lines() are subjected to the each_line_handler()s. (So if you
# called it from such a handler, you would get infinite recursion without some
# mechanism to prevent that.) Lines inserted by insert_adjusted_lines() go
# directly to the main handler without any adjustments. If the
# post-processing handler calls any of these, there will be no effect. Some
# error checking for these conditions could be added, but it hasn't been done.
#
# carp_bad_line() should be called to warn of bad input lines, which clears $_
# to prevent further processing of the line. This routine will output the
# message as a warning once, and then keep a count of the lines that have the
# same message, and output that count at the end of the file's processing.
# This keeps the number of messages down to a manageable amount.
#
# get_missings() should be called to retrieve any @missing input lines.
# Messages will be raised if this isn't done if the options aren't to ignore
# missings.
sub trace { return main::trace(@_); }
{ # Closure
# Keep track of fields that are to be put into the constructor.
my %constructor_fields;
main::setup_package(Constructor_Fields => \%constructor_fields);
my %file; # Input file name, required
main::set_access('file', \%file, qw{ c r });
my %first_released; # Unicode version file was first released in, required
main::set_access('first_released', \%first_released, qw{ c r });
my %handler; # Subroutine to process the input file, defaults to
# 'process_generic_property_file'
main::set_access('handler', \%handler, qw{ c });
my %property;
# name of property this file is for. defaults to none, meaning not
# applicable, or is otherwise determinable, for example, from each line.
main::set_access('property', \%property, qw{ c r });
my %optional;
# This is either an unsigned number, or a list of property names. In the
# former case, if it is non-zero, it means the file is optional, so if the
# file is absent, no warning about that is output. In the latter case, it
# is a list of properties that the file (exclusively) defines. If the
# file is present, tables for those properties will be produced; if
# absent, none will, even if they are listed elsewhere (namely
# PropertyAliases.txt and PropValueAliases.txt) as being in this release,
# and no warnings will be raised about them not being available. (And no
# warning about the file itself will be raised.)
main::set_access('optional', \%optional, qw{ c readable_array } );
my %non_skip;
# This is used for debugging, to skip processing of all but a few input
# files. Add 'non_skip => 1' to the constructor for those files you want
# processed when you set the $debug_skip global.
main::set_access('non_skip', \%non_skip, 'c');
my %skip;
# This is used to skip processing of this input file (semi-) permanently.
# The value should be the reason the file is being skipped. It is used
# for files that we aren't planning to process anytime soon, but want to
# allow to be in the directory and be checked for their names not
# conflicting with any other files on a DOS 8.3 name filesystem, but to
# not otherwise be processed, and to not raise a warning about not being
# handled. In the constructor call, any value that evaluates to a numeric
# 0 or undef means don't skip. Any other value is a string giving the
# reason it is being skipped, and this will appear in generated pod.
# However, an empty string reason will suppress the pod entry.
# Internally, calls that evaluate to numeric 0 are changed into undef to
# distinguish them from an empty string call.
main::set_access('skip', \%skip, 'c', 'r');
my %each_line_handler;
# list of subroutines to look at and filter each non-comment line in the
# file. defaults to none. The subroutines are called in order, each is
# to adjust $_ for the next one, and the final one adjusts it for
# 'handler'
main::set_access('each_line_handler', \%each_line_handler, 'c');
my %retain_trailing_comments;
# This is used to not discard the comments that end data lines. This
# would be used only for files with non-typical syntax, and most code here
# assumes that comments have been stripped, so special handlers would have
# to be written. It is assumed that the code will use these in
# single-quoted contexts, and so any "'" marks in the comment will be
# prefixed by a backslash.
main::set_access('retain_trailing_comments', \%retain_trailing_comments, 'c');
my %properties; # Optional ordered list of the properties that occur in each
# meaningful line of the input file. If present, an appropriate
# each_line_handler() is automatically generated and pushed onto the stack
# of such handlers. This is useful when a file contains multiple
# properties per line, but no other special considerations are necessary.
# The special value "<ignored>" means to discard the corresponding input
# field.
# Any @missing lines in the file should also match this syntax; no such
# files exist as of 6.3. But if it happens in a future release, the code
# could be expanded to properly parse them.
main::set_access('properties', \%properties, qw{ c r });
my %has_missings_defaults;
# ? Are there lines in the file giving default values for code points
# missing from it?. Defaults to NO_DEFAULTS. Otherwise NOT_IGNORED is
# the norm, but IGNORED means it has such lines, but the handler doesn't
# use them. Having these three states allows us to catch changes to the
# UCD that this program should track. XXX This could be expanded to
# specify the syntax for such lines, like %properties above.
main::set_access('has_missings_defaults',
\%has_missings_defaults, qw{ c r });
my %construction_time_handler;
# Subroutine to call at the end of the new method. If undef, no such
# handler is called.
main::set_access('construction_time_handler',
\%construction_time_handler, qw{ c });
my %pre_handler;
# Subroutine to call before doing anything else in the file. If undef, no
# such handler is called.
main::set_access('pre_handler', \%pre_handler, qw{ c });
my %eof_handler;
# Subroutines to call upon getting an EOF on the input file, but before
# that is returned to the main handler. This is to allow buffers to be
# flushed. The handler is expected to call insert_lines() or
# insert_adjusted() with the buffered material
main::set_access('eof_handler', \%eof_handler, qw{ c });
my %post_handler;
# Subroutine to call after all the lines of the file are read in and
# processed. If undef, no such handler is called. Note that this cannot
# add lines to be processed; instead use eof_handler
main::set_access('post_handler', \%post_handler, qw{ c });
my %progress_message;
# Message to print to display progress in lieu of the standard one
main::set_access('progress_message', \%progress_message, qw{ c });
my %handle;
# cache open file handle, internal. Is undef if file hasn't been
# processed at all, empty if has;
main::set_access('handle', \%handle);
my %added_lines;
# cache of lines added virtually to the file, internal
main::set_access('added_lines', \%added_lines);
my %remapped_lines;
# cache of lines added virtually to the file, internal
main::set_access('remapped_lines', \%remapped_lines);
my %errors;
# cache of errors found, internal
main::set_access('errors', \%errors);
my %missings;
# storage of '@missing' defaults lines
main::set_access('missings', \%missings);
my %early;
# Used for properties that must be defined (for Perl's purposes) on
# versions of Unicode earlier than Unicode itself defines them. The
# parameter is an array (it would be better to be a hash, but not worth
# bothering about due to its rare use).
#
# The first element is either a code reference to call when in a release
# earlier than the Unicode file is available in, or it is an alternate
# file to use instead of the non-existent one. This file must have been
# plunked down in the same directory as mktables. Should you be compiling
# on a release that needs such a file, mktables will abort the
# compilation, and tell you where to get the necessary file(s), and what
# name(s) to use to store them as.
# In the case of specifying an alternate file, the array must contain two
# further elements:
#
# [1] is the name of the property that will be generated by this file.
# The class automatically takes the input file and excludes any code
# points in it that were not assigned in the Unicode version being
# compiled. It then uses this result to define the property in the given
# version. Since the property doesn't actually exist in the Unicode
# version being compiled, this should be a name accessible only by core
# perl. If it is the same name as the regular property, the constructor
# will mark the output table as a $PLACEHOLDER so that it doesn't actually
# get output, and so will be unusable by non-core code. Otherwise it gets
# marked as $INTERNAL_ONLY.
#
# [2] is a property value to assign (only when compiling Unicode 1.1.5) to
# the Hangul syllables in that release (which were ripped out in version
# 2) for the given property . (Hence it is ignored except when compiling
# version 1. You only get one value that applies to all of them, which
# may not be the actual reality, but probably nobody cares anyway for
# these obsolete characters.)
#
# [3] if present is the default value for the property to assign for code
# points not given in the input. If not present, the default from the
# normal property is used
#
# [-1] If there is an extra final element that is the string 'ONLY_EARLY'.
# it means to not add the name in [1] as an alias to the property name
# used for these. Normally, when compiling Unicode versions that don't
# invoke the early handling, the name is added as a synonym.
#
# Not all files can be handled in the above way, and so the code ref
# alternative is available. It can do whatever it needs to. The other
# array elements are optional in this case, and the code is free to use or
# ignore them if they are present.
#
# Internally, the constructor unshifts a 0 or 1 onto this array to
# indicate if an early alternative is actually being used or not. This
# makes for easier testing later on.
main::set_access('early', \%early, 'c');
my %only_early;
main::set_access('only_early', \%only_early, 'c');
my %required_even_in_debug_skip;
# debug_skip is used to speed up compilation during debugging by skipping
# processing files that are not needed for the task at hand. However,
# some files pretty much can never be skipped, and this is used to specify
# that this is one of them. In order to skip this file, the call to the
# constructor must be edited to comment out this parameter.
main::set_access('required_even_in_debug_skip',
\%required_even_in_debug_skip, 'c');
my %withdrawn;
# Some files get removed from the Unicode DB. This is a version object
# giving the first release without this file.
main::set_access('withdrawn', \%withdrawn, 'c');
my %in_this_release;
# Calculated value from %first_released and %withdrawn. Are we compiling
# a Unicode release which includes this file?
main::set_access('in_this_release', \%in_this_release);
sub _next_line;
sub _next_line_with_remapped_range;
sub new {
my $class = shift;
my $self = bless \do{ my $anonymous_scalar }, $class;
my $addr = do { no overloading; pack 'J', $self; };
# Set defaults
$handler{$addr} = \&main::process_generic_property_file;
$retain_trailing_comments{$addr} = 0;
$non_skip{$addr} = 0;
$skip{$addr} = undef;
$has_missings_defaults{$addr} = $NO_DEFAULTS;
$handle{$addr} = undef;
$added_lines{$addr} = [ ];
$remapped_lines{$addr} = [ ];
$each_line_handler{$addr} = [ ];
$eof_handler{$addr} = [ ];
$errors{$addr} = { };
$missings{$addr} = [ ];
$early{$addr} = [ ];
$optional{$addr} = [ ];
# Two positional parameters.
return Carp::carp_too_few_args(\@_, 2) if main::DEBUG && @_ < 2;
$file{$addr} = main::internal_file_to_platform(shift);
$first_released{$addr} = shift;
# The rest of the arguments are key => value pairs
# %constructor_fields has been set up earlier to list all possible
# ones. Either set or push, depending on how the default has been set
# up just above.
my %args = @_;
foreach my $key (keys %args) {
my $argument = $args{$key};
# Note that the fields are the lower case of the constructor keys
my $hash = $constructor_fields{lc $key};
if (! defined $hash) {
Carp::my_carp_bug("Unrecognized parameters '$key => $argument' to new() for $self. Skipped");
next;
}
if (ref $hash->{$addr} eq 'ARRAY') {
if (ref $argument eq 'ARRAY') {
foreach my $argument (@{$argument}) {
next if ! defined $argument;
push @{$hash->{$addr}}, $argument;
}
}
else {
push @{$hash->{$addr}}, $argument if defined $argument;
}
}
else {
$hash->{$addr} = $argument;
}
delete $args{$key};
};
$non_skip{$addr} = 1 if $required_even_in_debug_skip{$addr};
# Convert 0 (meaning don't skip) to undef
undef $skip{$addr} unless $skip{$addr};
# Handle the case where this file is optional
my $pod_message_for_non_existent_optional = "";
if ($optional{$addr}->@*) {
# First element is the pod message
$pod_message_for_non_existent_optional
= shift $optional{$addr}->@*;
# Convert a 0 'Optional' argument to an empty list to make later
# code more concise.
if ( $optional{$addr}->@*
&& $optional{$addr}->@* == 1
&& $optional{$addr}[0] ne ""
&& $optional{$addr}[0] !~ /\D/
&& $optional{$addr}[0] == 0)
{
$optional{$addr} = [ ];
}
else { # But if the only element doesn't evaluate to 0, make sure
# that this file is indeed considered optional below.
unshift $optional{$addr}->@*, 1;
}
}
my $progress;
my $function_instead_of_file = 0;
if ($early{$addr}->@* && $early{$addr}[-1] eq 'ONLY_EARLY') {
$only_early{$addr} = 1;
pop $early{$addr}->@*;
}
# If we are compiling a Unicode release earlier than the file became
# available, the constructor may have supplied a substitute
if ($first_released{$addr} gt $v_version && $early{$addr}->@*) {
# Yes, we have a substitute, that we will use; mark it so
unshift $early{$addr}->@*, 1;
# See the definition of %early for what the array elements mean.
# Note that we have just unshifted onto the array, so the numbers
# below are +1 of those in the %early description.
# If we have a property this defines, create a table and default
# map for it now (at essentially compile time), so that it will be
# available for the whole of run time. (We will want to add this
# name as an alias when we are using the official property name;
# but this must be deferred until run(), because at construction
# time the official names have yet to be defined.)
if ($early{$addr}[2]) {
my $fate = ($property{$addr}
&& $property{$addr} eq $early{$addr}[2])
? $PLACEHOLDER
: $INTERNAL_ONLY;
my $prop_object = Property->new($early{$addr}[2],
Fate => $fate,
Perl_Extension => 1,
);
# If not specified by the constructor, use the default mapping
# for the regular property for this substitute one.
if ($early{$addr}[4]) {
$prop_object->set_default_map($early{$addr}[4]);
}
elsif ( defined $property{$addr}
&& defined $default_mapping{$property{$addr}})
{
$prop_object
->set_default_map($default_mapping{$property{$addr}});
}
}
if (ref $early{$addr}[1] eq 'CODE') {
$function_instead_of_file = 1;
# If the first element of the array is a code ref, the others
# are optional.
$handler{$addr} = $early{$addr}[1];
$property{$addr} = $early{$addr}[2]
if defined $early{$addr}[2];
$progress = "substitute $file{$addr}";
undef $file{$addr};
}
else { # Specifying a substitute file
if (! main::file_exists($early{$addr}[1])) {
# If we don't see the substitute file, generate an error
# message giving the needed things, and add it to the list
# of such to output before actual processing happens
# (hence the user finds out all of them in one run).
# Instead of creating a general method for NameAliases,
# hard-code it here, as there is unlikely to ever be a
# second one which needs special handling.
my $string_version = ($file{$addr} eq "NameAliases.txt")
? 'at least 6.1 (the later, the better)'
: sprintf "%vd", $first_released{$addr};
push @missing_early_files, <<END;
'$file{$addr}' version $string_version should be copied to '$early{$addr}[1]'.
END
;
return;
}
$progress = $early{$addr}[1];
$progress .= ", substituting for $file{$addr}" if $file{$addr};
$file{$addr} = $early{$addr}[1];
$property{$addr} = $early{$addr}[2];
# Ignore code points not in the version being compiled
push $each_line_handler{$addr}->@*, \&_exclude_unassigned;
if ( $v_version lt v2.0 # Hanguls in this release ...
&& defined $early{$addr}[3]) # ... need special treatment
{
push $eof_handler{$addr}->@*, \&_fixup_obsolete_hanguls;
}
}
# And this substitute is valid for all releases.
$first_released{$addr} = v0;
}
else { # Normal behavior
$progress = $file{$addr};
unshift $early{$addr}->@*, 0; # No substitute
}
my $file = $file{$addr};
$progress_message{$addr} = "Processing $progress"
unless $progress_message{$addr};
# A file should be there if it is within the window of versions for
# which Unicode supplies it
if ($withdrawn{$addr} && $withdrawn{$addr} le $v_version) {
$in_this_release{$addr} = 0;
$skip{$addr} = "";
}
else {
$in_this_release{$addr} = $first_released{$addr} le $v_version;
# Check that the file for this object (possibly using a substitute
# for early releases) exists or we have a function alternative
if ( ! $function_instead_of_file
&& ! main::file_exists($file))
{
# Here there is nothing available for this release. This is
# fine if we aren't expecting anything in this release.
if (! $in_this_release{$addr}) {
$skip{$addr} = ""; # Don't remark since we expected
# nothing and got nothing
}
elsif ($optional{$addr}->@*) {
# Here the file is optional in this release; Use the
# passed in text to document this case in the pod.
$skip{$addr} = $pod_message_for_non_existent_optional;
}
elsif ( $in_this_release{$addr}
&& ! defined $skip{$addr}
&& defined $file)
{ # Doesn't exist but should.
$skip{$addr} = "'$file' not found. Possibly Big problems";
Carp::my_carp($skip{$addr});
}
}
elsif ($debug_skip && ! defined $skip{$addr} && ! $non_skip{$addr})
{
# The file exists; if not skipped for another reason, and we are
# skipping most everything during debugging builds, use that as
# the skip reason.
$skip{$addr} = '$debug_skip is on'
}
}
if ( ! $debug_skip
&& $non_skip{$addr}
&& ! $required_even_in_debug_skip{$addr}
&& $verbosity)
{
print "Warning: " . __PACKAGE__ . " constructor for $file has useless 'non_skip' in it\n";
}
# Here, we have figured out if we will be skipping this file or not.
# If so, we add any single property it defines to any passed in
# optional property list. These will be dealt with at run time.
if (defined $skip{$addr}) {
if ($property{$addr}) {
push $optional{$addr}->@*, $property{$addr};
}
} # Otherwise, are going to process the file.
elsif ($property{$addr}) {
# If the file has a property defined in the constructor for it, it
# means that the property is not listed in the file's entries. So
# add a handler (to the list of line handlers) to insert the
# property name into the lines, to provide a uniform interface to
# the final processing subroutine.
push @{$each_line_handler{$addr}}, \&_insert_property_into_line;
}
elsif ($properties{$addr}) {
# Similarly, there may be more than one property represented on
# each line, with no clue but the constructor input what those
# might be. Add a handler for each line in the input so that it
# creates a separate input line for each property in those input
# lines, thus making them suitable to handle generically.
push @{$each_line_handler{$addr}},
sub {
my $file = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my @fields = split /\s*;\s*/, $_, -1;
if (@fields - 1 > @{$properties{$addr}}) {
$file->carp_bad_line('Extra fields');
$_ = "";
return;
}
my $range = shift @fields; # 0th element is always the
# range
# The next fields in the input line correspond
# respectively to the stored properties.
for my $i (0 .. @{$properties{$addr}} - 1) {
my $property_name = $properties{$addr}[$i];
next if $property_name eq '<ignored>';
$file->insert_adjusted_lines(
"$range; $property_name; $fields[$i]");
}
$_ = "";
return;
};
}
{ # On non-ascii platforms, we use a special pre-handler
no strict;
no warnings 'once';
*next_line = (main::NON_ASCII_PLATFORM)
? *_next_line_with_remapped_range
: *_next_line;
}
&{$construction_time_handler{$addr}}($self)
if $construction_time_handler{$addr};
return $self;
}
use overload
fallback => 0,
qw("") => "_operator_stringify",
"." => \&main::_operator_dot,
".=" => \&main::_operator_dot_equal,
;
sub _operator_stringify {
my $self = shift;
return __PACKAGE__ . " object for " . $self->file;
}
sub run {
# Process the input object $self. This opens and closes the file and
# calls all the handlers for it. Currently, this can only be called
# once per file, as it destroy's the EOF handlers
# flag to make sure extracted files are processed early
state $seen_non_extracted = 0;
my $self = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $addr = do { no overloading; pack 'J', $self; };
my $file = $file{$addr};
if (! $file) {
$handle{$addr} = 'pretend_is_open';
}
else {
if ($seen_non_extracted) {
if ($file =~ /$EXTRACTED/i) # Some platforms may change the
# case of the file's name
{
Carp::my_carp_bug(main::join_lines(<<END
$file should be processed just after the 'Prop...Alias' files, and before
anything not in the $EXTRACTED_DIR directory. Proceeding, but the results may
have subtle problems
END
));
}
}
elsif ($EXTRACTED_DIR
# We only do this check for generic property files
&& $handler{$addr} == \&main::process_generic_property_file
&& $file !~ /$EXTRACTED/i)
{
# We don't set this (by the 'if' above) if we have no
# extracted directory, so if running on an early version,
# this test won't work. Not worth worrying about.
$seen_non_extracted = 1;
}
# Mark the file as having being processed, and warn if it
# isn't a file we are expecting. As we process the files,
# they are deleted from the hash, so any that remain at the
# end of the program are files that we didn't process.
my $fkey = File::Spec->rel2abs($file);
my $exists = delete $potential_files{lc($fkey)};
Carp::my_carp("Was not expecting '$file'.")
if $exists && ! $in_this_release{$addr};
# If there is special handling for compiling Unicode releases
# earlier than the first one in which Unicode defines this
# property ...
if ($early{$addr}->@* > 1) {
# Mark as processed any substitute file that would be used in
# such a release
$fkey = File::Spec->rel2abs($early{$addr}[1]);
delete $potential_files{lc($fkey)};
# As commented in the constructor code, when using the
# official property, we still have to allow the publicly
# inaccessible early name so that the core code which uses it
# will work regardless.
if ( ! $only_early{$addr}
&& ! $early{$addr}[0]
&& $early{$addr}->@* > 2)
{
my $early_property_name = $early{$addr}[2];
if ($property{$addr} ne $early_property_name) {
main::property_ref($property{$addr})
->add_alias($early_property_name);
}
}
}
# We may be skipping this file ...
if (defined $skip{$addr}) {
# If the file isn't supposed to be in this release, there is
# nothing to do
if ($in_this_release{$addr}) {
# But otherwise, we may print a message
if ($debug_skip) {
print STDERR "Skipping input file '$file'",
" because '$skip{$addr}'\n";
}
# And add it to the list of skipped files, which is later
# used to make the pod
$skipped_files{$file} = $skip{$addr};
# The 'optional' list contains properties that are also to
# be skipped along with the file. (There may also be
# digits which are just placeholders to make sure it isn't
# an empty list
foreach my $property ($optional{$addr}->@*) {
next unless $property =~ /\D/;
my $prop_object = main::property_ref($property);
next unless defined $prop_object;
$prop_object->set_fate($SUPPRESSED, $skip{$addr});
}
}
return;
}
# Here, we are going to process the file. Open it, converting the
# slashes used in this program into the proper form for the OS
my $file_handle;
if (not open $file_handle, "<", $file) {
Carp::my_carp("Can't open $file. Skipping: $!");
return;
}
$handle{$addr} = $file_handle; # Cache the open file handle
# If possible, make sure that the file is the correct version.
# (This data isn't available on early Unicode releases or in
# UnicodeData.txt.) We don't do this check if we are using a
# substitute file instead of the official one (though the code
# could be extended to do so).
if ($in_this_release{$addr}
&& ! $early{$addr}[0]
&& lc($file) ne 'unicodedata.txt')
{
if ($file !~ /^Unihan/i) {
# The non-Unihan files started getting version numbers in
# 3.2, but some files in 4.0 are unchanged from 3.2, and
# marked as 3.2. 4.0.1 is the first version where there
# are no files marked as being from less than 4.0, though
# some are marked as 4.0. In versions after that, the
# numbers are correct.
if ($v_version ge v4.0.1) {
$_ = <$file_handle>; # The version number is in the
# very first line
if ($_ !~ / - $string_version \. /x) {
chomp;
$_ =~ s/^#\s*//;
# 4.0.1 had some valid files that weren't updated.
if (! ($v_version eq v4.0.1 && $_ =~ /4\.0\.0/)) {
die Carp::my_carp("File '$file' is version "
. "'$_'. It should be "
. "version $string_version");
}
}
}
}
elsif ($v_version ge v6.0.0) { # Unihan
# Unihan files didn't get accurate version numbers until
# 6.0. The version is somewhere in the first comment
# block
while (<$file_handle>) {
if ($_ !~ /^#/) {
Carp::my_carp_bug("Could not find the expected "
. "version info in file '$file'");
last;
}
chomp;
$_ =~ s/^#\s*//;
next if $_ !~ / version: /x;
last if $_ =~ /$string_version/;
die Carp::my_carp("File '$file' is version "
. "'$_'. It should be "
. "version $string_version");
}
}
}
}
print "$progress_message{$addr}\n" if $verbosity >= $PROGRESS;
# Call any special handler for before the file.
&{$pre_handler{$addr}}($self) if $pre_handler{$addr};
# Then the main handler
&{$handler{$addr}}($self);
# Then any special post-file handler.
&{$post_handler{$addr}}($self) if $post_handler{$addr};
# If any errors have been accumulated, output the counts (as the first
# error message in each class was output when it was encountered).
if ($errors{$addr}) {
my $total = 0;
my $types = 0;
foreach my $error (keys %{$errors{$addr}}) {
$total += $errors{$addr}->{$error};
delete $errors{$addr}->{$error};
$types++;
}
if ($total > 1) {
my $message
= "A total of $total lines had errors in $file. ";
$message .= ($types == 1)
? '(Only the first one was displayed.)'
: '(Only the first of each type was displayed.)';
Carp::my_carp($message);
}
}
if (@{$missings{$addr}}) {
Carp::my_carp_bug("Handler for $file didn't look at all the \@missing lines. Generated tables likely are wrong");
}
# If a real file handle, close it.
close $handle{$addr} or Carp::my_carp("Can't close $file: $!") if
ref $handle{$addr};
$handle{$addr} = ""; # Uses empty to indicate that has already seen
# the file, as opposed to undef
return;
}
sub _next_line {
# Sets $_ to be the next logical input line, if any. Returns non-zero
# if such a line exists. 'logical' means that any lines that have
# been added via insert_lines() will be returned in $_ before the file
# is read again.
my $self = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $addr = do { no overloading; pack 'J', $self; };
# Here the file is open (or if the handle is not a ref, is an open
# 'virtual' file). Get the next line; any inserted lines get priority
# over the file itself.
my $adjusted;
LINE:
while (1) { # Loop until find non-comment, non-empty line
#local $to_trace = 1 if main::DEBUG;
my $inserted_ref = shift @{$added_lines{$addr}};
if (defined $inserted_ref) {
($adjusted, $_) = @{$inserted_ref};
trace $adjusted, $_ if main::DEBUG && $to_trace;
return 1 if $adjusted;
}
else {
last if ! ref $handle{$addr}; # Don't read unless is real file
last if ! defined ($_ = readline $handle{$addr});
}
chomp;
trace $_ if main::DEBUG && $to_trace;
# See if this line is the comment line that defines what property
# value that code points that are not listed in the file should
# have. The format or existence of these lines is not guaranteed
# by Unicode since they are comments, but the documentation says
# that this was added for machine-readability, so probably won't
# change. This works starting in Unicode Version 5.0. They look
# like:
#
# @missing: 0000..10FFFF; Not_Reordered
# @missing: 0000..10FFFF; Decomposition_Mapping; <code point>
# @missing: 0000..10FFFF; ; NaN
#
# Save the line for a later get_missings() call.
if (/$missing_defaults_prefix/) {
if ($has_missings_defaults{$addr} == $NO_DEFAULTS) {
$self->carp_bad_line("Unexpected \@missing line. Assuming no missing entries");
}
elsif ($has_missings_defaults{$addr} == $NOT_IGNORED) {
my @defaults = split /\s* ; \s*/x, $_;
# The first field is the @missing, which ends in a
# semi-colon, so can safely shift.
shift @defaults;
# Some of these lines may have empty field placeholders
# which get in the way. An example is:
# @missing: 0000..10FFFF; ; NaN
# Remove them. Process starting from the top so the
# splice doesn't affect things still to be looked at.
for (my $i = @defaults - 1; $i >= 0; $i--) {
next if $defaults[$i] ne "";
splice @defaults, $i, 1;
}
# What's left should be just the property (maybe) and the
# default. Having only one element means it doesn't have
# the property.
my $default;
my $property;
if (@defaults >= 1) {
if (@defaults == 1) {
$default = $defaults[0];
}
else {
$property = $defaults[0];
$default = $defaults[1];
}
}
if (@defaults < 1
|| @defaults > 2
|| ($default =~ /^</
&& $default !~ /^<code *point>$/i
&& $default !~ /^<none>$/i
&& $default !~ /^<script>$/i))
{
$self->carp_bad_line("Unrecognized \@missing line: $_. Assuming no missing entries");
}
else {
# If the property is missing from the line, it should
# be the one for the whole file
$property = $property{$addr} if ! defined $property;
# Change <none> to the null string, which is what it
# really means. If the default is the code point
# itself, set it to <code point>, which is what
# Unicode uses (but sometimes they've forgotten the
# space)
if ($default =~ /^<none>$/i) {
$default = "";
}
elsif ($default =~ /^<code *point>$/i) {
$default = $CODE_POINT;
}
elsif ($default =~ /^<script>$/i) {
# Special case this one. Currently is from
# ScriptExtensions.txt, and means for all unlisted
# code points, use their Script property values.
# For the code points not listed in that file, the
# default value is 'Unknown'.
$default = "Unknown";
}
# Store them as a sub-arrays with both components.
push @{$missings{$addr}}, [ $default, $property ];
}
}
# There is nothing for the caller to process on this comment
# line.
next;
}
# Unless to keep, remove comments. If to keep, ignore
# comment-only lines
if ($retain_trailing_comments{$addr}) {
next if / ^ \s* \# /x;
# But escape any single quotes (done in both the comment and
# non-comment portion; this could be a bug someday, but not
# likely)
s/'/\\'/g;
}
else {
s/#.*//;
}
# Remove trailing space, and skip this line if the result is empty
s/\s+$//;
next if /^$/;
# Call any handlers for this line, and skip further processing of
# the line if the handler sets the line to null.
foreach my $sub_ref (@{$each_line_handler{$addr}}) {
&{$sub_ref}($self);
next LINE if /^$/;
}
# Here the line is ok. return success.
return 1;
} # End of looping through lines.
# If there are EOF handlers, call each (only once) and if it generates
# more lines to process go back in the loop to handle them.
while ($eof_handler{$addr}->@*) {
&{$eof_handler{$addr}[0]}($self);
shift $eof_handler{$addr}->@*; # Currently only get one shot at it.
goto LINE if $added_lines{$addr};
}
# Return failure -- no more lines.
return 0;
}
sub _next_line_with_remapped_range {
my $self = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
# like _next_line(), but for use on non-ASCII platforms. It sets $_
# to be the next logical input line, if any. Returns non-zero if such
# a line exists. 'logical' means that any lines that have been added
# via insert_lines() will be returned in $_ before the file is read
# again.
#
# The difference from _next_line() is that this remaps the Unicode
# code points in the input to those of the native platform. Each
# input line contains a single code point, or a single contiguous
# range of them This routine splits each range into its individual
# code points and caches them. It returns the cached values,
# translated into their native equivalents, one at a time, for each
# call, before reading the next line. Since native values can only be
# a single byte wide, no translation is needed for code points above
# 0xFF, and ranges that are entirely above that number are not split.
# If an input line contains the range 254-1000, it would be split into
# three elements: 254, 255, and 256-1000. (The downstream table
# insertion code will sort and coalesce the individual code points
# into appropriate ranges.)
my $addr = do { no overloading; pack 'J', $self; };
while (1) {
# Look in cache before reading the next line. Return any cached
# value, translated
my $inserted = shift @{$remapped_lines{$addr}};
if (defined $inserted) {
trace $inserted if main::DEBUG && $to_trace;
$_ = $inserted =~ s/^ ( \d+ ) /sprintf("%04X", utf8::unicode_to_native($1))/xer;
trace $_ if main::DEBUG && $to_trace;
return 1;
}
# Get the next line.
return 0 unless _next_line($self);
# If there is a special handler for it, return the line,
# untranslated. This should happen only for files that are
# special, not being code-point related, such as property names.
return 1 if $handler{$addr}
!= \&main::process_generic_property_file;
my ($range, $property_name, $map, @remainder)
= split /\s*;\s*/, $_, -1; # -1 => retain trailing null fields
if (@remainder
|| ! defined $property_name
|| $range !~ /^ ($code_point_re) (?:\.\. ($code_point_re) )? $/x)
{
Carp::my_carp_bug("Unrecognized input line '$_'. Ignored");
}
my $low = hex $1;
my $high = (defined $2) ? hex $2 : $low;
# If the input maps the range to another code point, remap the
# target if it is between 0 and 255.
my $tail;
if (defined $map) {
$map =~ s/\b 00 ( [0-9A-F]{2} ) \b/sprintf("%04X", utf8::unicode_to_native(hex $1))/gxe;
$tail = "$property_name; $map";
$_ = "$range; $tail";
}
else {
$tail = $property_name;
}
# If entire range is above 255, just return it, unchanged (except
# any mapped-to code point, already changed above)
return 1 if $low > 255;
# Cache an entry for every code point < 255. For those in the
# range above 255, return a dummy entry for just that portion of
# the range. Note that this will be out-of-order, but that is not
# a problem.
foreach my $code_point ($low .. $high) {
if ($code_point > 255) {
$_ = sprintf "%04X..%04X; $tail", $code_point, $high;
return 1;
}
push @{$remapped_lines{$addr}}, "$code_point; $tail";
}
} # End of looping through lines.
# NOTREACHED
}
# Not currently used, not fully tested.
# sub peek {
# # Non-destructive lookahead one non-adjusted, non-comment, non-blank
# # record. Not callable from an each_line_handler(), nor does it call
# # an each_line_handler() on the line.
#
# my $self = shift;
# my $addr = do { no overloading; pack 'J', $self; };
#
# foreach my $inserted_ref (@{$added_lines{$addr}}) {
# my ($adjusted, $line) = @{$inserted_ref};
# next if $adjusted;
#
# # Remove comments and trailing space, and return a non-empty
# # resulting line
# $line =~ s/#.*//;
# $line =~ s/\s+$//;
# return $line if $line ne "";
# }
#
# return if ! ref $handle{$addr}; # Don't read unless is real file
# while (1) { # Loop until find non-comment, non-empty line
# local $to_trace = 1 if main::DEBUG;
# trace $_ if main::DEBUG && $to_trace;
# return if ! defined (my $line = readline $handle{$addr});
# chomp $line;
# push @{$added_lines{$addr}}, [ 0, $line ];
#
# $line =~ s/#.*//;
# $line =~ s/\s+$//;
# return $line if $line ne "";
# }
#
# return;
# }
sub insert_lines {
# Lines can be inserted so that it looks like they were in the input
# file at the place it was when this routine is called. See also
# insert_adjusted_lines(). Lines inserted via this routine go through
# any each_line_handler()
my $self = shift;
# Each inserted line is an array, with the first element being 0 to
# indicate that this line hasn't been adjusted, and needs to be
# processed.
no overloading;
push @{$added_lines{pack 'J', $self}}, map { [ 0, $_ ] } @_;
return;
}
sub insert_adjusted_lines {
# Lines can be inserted so that it looks like they were in the input
# file at the place it was when this routine is called. See also
# insert_lines(). Lines inserted via this routine are already fully
# adjusted, ready to be processed; each_line_handler()s handlers will
# not be called. This means this is not a completely general
# facility, as only the last each_line_handler on the stack should
# call this. It could be made more general, by passing to each of the
# line_handlers their position on the stack, which they would pass on
# to this routine, and that would replace the boolean first element in
# the anonymous array pushed here, so that the next_line routine could
# use that to call only those handlers whose index is after it on the
# stack. But this is overkill for what is needed now.
my $self = shift;
trace $_[0] if main::DEBUG && $to_trace;
# Each inserted line is an array, with the first element being 1 to
# indicate that this line has been adjusted
no overloading;
push @{$added_lines{pack 'J', $self}}, map { [ 1, $_ ] } @_;
return;
}
sub get_missings {
# Returns the stored up @missings lines' values, and clears the list.
# The values are in an array, consisting of the default in the first
# element, and the property in the 2nd. However, since these lines
# can be stacked up, the return is an array of all these arrays.
my $self = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $addr = do { no overloading; pack 'J', $self; };
# If not accepting a list return, just return the first one.
return shift @{$missings{$addr}} unless wantarray;
my @return = @{$missings{$addr}};
undef @{$missings{$addr}};
return @return;
}
sub _exclude_unassigned {
# Takes the range in $_ and excludes code points that aren't assigned
# in this release
state $skip_inserted_count = 0;
# Ignore recursive calls.
if ($skip_inserted_count) {
$skip_inserted_count--;
return;
}
# Find what code points are assigned in this release
main::calculate_Assigned() if ! defined $Assigned;
my $self = shift;
my $addr = do { no overloading; pack 'J', $self; };
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my ($range, @remainder)
= split /\s*;\s*/, $_, -1; # -1 => retain trailing null fields
# Examine the range.
if ($range =~ /^ ($code_point_re) (?:\.\. ($code_point_re) )? $/x)
{
my $low = hex $1;
my $high = (defined $2) ? hex $2 : $low;
# Split the range into subranges of just those code points in it
# that are assigned.
my @ranges = (Range_List->new(Initialize
=> Range->new($low, $high)) & $Assigned)->ranges;
# Do nothing if nothing in the original range is assigned in this
# release; handle normally if everything is in this release.
if (! @ranges) {
$_ = "";
}
elsif (@ranges != 1) {
# Here, some code points in the original range aren't in this
# release; @ranges gives the ones that are. Create fake input
# lines for each of the ranges, and set things up so that when
# this routine is called on that fake input, it will do
# nothing.
$skip_inserted_count = @ranges;
my $remainder = join ";", @remainder;
for my $range (@ranges) {
$self->insert_lines(sprintf("%04X..%04X;%s",
$range->start, $range->end, $remainder));
}
$_ = ""; # The original range is now defunct.
}
}
return;
}
sub _fixup_obsolete_hanguls {
# This is called only when compiling Unicode version 1. All Unicode
# data for subsequent releases assumes that the code points that were
# Hangul syllables in this release only are something else, so if
# using such data, we have to override it
my $self = shift;
my $addr = do { no overloading; pack 'J', $self; };
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $object = main::property_ref($property{$addr});
$object->add_map($FIRST_REMOVED_HANGUL_SYLLABLE,
$FINAL_REMOVED_HANGUL_SYLLABLE,
$early{$addr}[3], # Passed-in value for these
Replace => $UNCONDITIONALLY);
}
sub _insert_property_into_line {
# Add a property field to $_, if this file requires it.
my $self = shift;
my $addr = do { no overloading; pack 'J', $self; };
my $property = $property{$addr};
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
$_ =~ s/(;|$)/; $property$1/;
return;
}
sub carp_bad_line {
# Output consistent error messages, using either a generic one, or the
# one given by the optional parameter. To avoid gazillions of the
# same message in case the syntax of a file is way off, this routine
# only outputs the first instance of each message, incrementing a
# count so the totals can be output at the end of the file.
my $self = shift;
my $message = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $addr = do { no overloading; pack 'J', $self; };
$message = 'Unexpected line' unless $message;
# No trailing punctuation so as to fit with our addenda.
$message =~ s/[.:;,]$//;
# If haven't seen this exact message before, output it now. Otherwise
# increment the count of how many times it has occurred
unless ($errors{$addr}->{$message}) {
Carp::my_carp("$message in '$_' in "
. $file{$addr}
. " at line $.. Skipping this line;");
$errors{$addr}->{$message} = 1;
}
else {
$errors{$addr}->{$message}++;
}
# Clear the line to prevent any further (meaningful) processing of it.
$_ = "";
return;
}
} # End closure
package Multi_Default;
# Certain properties in early versions of Unicode had more than one possible
# default for code points missing from the files. In these cases, one
# default applies to everything left over after all the others are applied,
# and for each of the others, there is a description of which class of code
# points applies to it. This object helps implement this by storing the
# defaults, and for all but that final default, an eval string that generates
# the class that it applies to.
{ # Closure
main::setup_package();
my %class_defaults;
# The defaults structure for the classes
main::set_access('class_defaults', \%class_defaults);
my %other_default;
# The default that applies to everything left over.
main::set_access('other_default', \%other_default, 'r');
sub new {
# The constructor is called with default => eval pairs, terminated by
# the left-over default. e.g.
# Multi_Default->new(
# 'T' => '$gc->table("Mn") + $gc->table("Cf") - 0x200C
# - 0x200D',
# 'R' => 'some other expression that evaluates to code points',
# .
# .
# .
# 'U'));
# It is best to leave the final value be the one that matches the
# above-Unicode code points.
my $class = shift;
my $self = bless \do{my $anonymous_scalar}, $class;
my $addr = do { no overloading; pack 'J', $self; };
while (@_ > 1) {
my $default = shift;
my $eval = shift;
$class_defaults{$addr}->{$default} = $eval;
}
$other_default{$addr} = shift;
return $self;
}
sub get_next_defaults {
# Iterates and returns the next class of defaults.
my $self = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $addr = do { no overloading; pack 'J', $self; };
return each %{$class_defaults{$addr}};
}
}
package Alias;
# An alias is one of the names that a table goes by. This class defines them
# including some attributes. Everything is currently setup in the
# constructor.
{ # Closure
main::setup_package();
my %name;
main::set_access('name', \%name, 'r');
my %loose_match;
# Should this name match loosely or not.
main::set_access('loose_match', \%loose_match, 'r');
my %make_re_pod_entry;
# Some aliases should not get their own entries in the re section of the
# pod, because they are covered by a wild-card, and some we want to
# discourage use of. Binary
main::set_access('make_re_pod_entry', \%make_re_pod_entry, 'r', 's');
my %ucd;
# Is this documented to be accessible via Unicode::UCD
main::set_access('ucd', \%ucd, 'r', 's');
my %status;
# Aliases have a status, like deprecated, or even suppressed (which means
# they don't appear in documentation). Enum
main::set_access('status', \%status, 'r');
my %ok_as_filename;
# Similarly, some aliases should not be considered as usable ones for
# external use, such as file names, or we don't want documentation to
# recommend them. Boolean
main::set_access('ok_as_filename', \%ok_as_filename, 'r');
sub new {
my $class = shift;
my $self = bless \do { my $anonymous_scalar }, $class;
my $addr = do { no overloading; pack 'J', $self; };
$name{$addr} = shift;
$loose_match{$addr} = shift;
$make_re_pod_entry{$addr} = shift;
$ok_as_filename{$addr} = shift;
$status{$addr} = shift;
$ucd{$addr} = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
# Null names are never ok externally
$ok_as_filename{$addr} = 0 if $name{$addr} eq "";
return $self;
}
}
package Range;
# A range is the basic unit for storing code points, and is described in the
# comments at the beginning of the program. Each range has a starting code
# point; an ending code point (not less than the starting one); a value
# that applies to every code point in between the two end-points, inclusive;
# and an enum type that applies to the value. The type is for the user's
# convenience, and has no meaning here, except that a non-zero type is
# considered to not obey the normal Unicode rules for having standard forms.
#
# The same structure is used for both map and match tables, even though in the
# latter, the value (and hence type) is irrelevant and could be used as a
# comment. In map tables, the value is what all the code points in the range
# map to. Type 0 values have the standardized version of the value stored as
# well, so as to not have to recalculate it a lot.
sub trace { return main::trace(@_); }
{ # Closure
main::setup_package();
my %start;
main::set_access('start', \%start, 'r', 's');
my %end;
main::set_access('end', \%end, 'r', 's');
my %value;
main::set_access('value', \%value, 'r', 's');
my %type;
main::set_access('type', \%type, 'r');
my %standard_form;
# The value in internal standard form. Defined only if the type is 0.
main::set_access('standard_form', \%standard_form);
# Note that if these fields change, the dump() method should as well
sub new {
return Carp::carp_too_few_args(\@_, 3) if main::DEBUG && @_ < 3;
my $class = shift;
my $self = bless \do { my $anonymous_scalar }, $class;
my $addr = do { no overloading; pack 'J', $self; };
$start{$addr} = shift;
$end{$addr} = shift;
my %args = @_;
my $value = delete $args{'Value'}; # Can be 0
$value = "" unless defined $value;
$value{$addr} = $value;
$type{$addr} = delete $args{'Type'} || 0;
Carp::carp_extra_args(\%args) if main::DEBUG && %args;
return $self;
}
use overload
fallback => 0,
qw("") => "_operator_stringify",
"." => \&main::_operator_dot,
".=" => \&main::_operator_dot_equal,
;
sub _operator_stringify {
my $self = shift;
my $addr = do { no overloading; pack 'J', $self; };
# Output it like '0041..0065 (value)'
my $return = sprintf("%04X", $start{$addr})
. '..'
. sprintf("%04X", $end{$addr});
my $value = $value{$addr};
my $type = $type{$addr};
$return .= ' (';
$return .= "$value";
$return .= ", Type=$type" if $type != 0;
$return .= ')';
return $return;
}
sub standard_form {
# Calculate the standard form only if needed, and cache the result.
# The standard form is the value itself if the type is special.
# This represents a considerable CPU and memory saving - at the time
# of writing there are 368676 non-special objects, but the standard
# form is only requested for 22047 of them - ie about 6%.
my $self = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $addr = do { no overloading; pack 'J', $self; };
return $standard_form{$addr} if defined $standard_form{$addr};
my $value = $value{$addr};
return $value if $type{$addr};
return $standard_form{$addr} = main::standardize($value);
}
sub dump {
# Human, not machine readable. For machine readable, comment out this
# entire routine and let the standard one take effect.
my $self = shift;
my $indent = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $addr = do { no overloading; pack 'J', $self; };
my $return = $indent
. sprintf("%04X", $start{$addr})
. '..'
. sprintf("%04X", $end{$addr})
. " '$value{$addr}';";
if (! defined $standard_form{$addr}) {
$return .= "(type=$type{$addr})";
}
elsif ($standard_form{$addr} ne $value{$addr}) {
$return .= "(standard '$standard_form{$addr}')";
}
return $return;
}
} # End closure
package _Range_List_Base;
# Base class for range lists. A range list is simply an ordered list of
# ranges, so that the ranges with the lowest starting numbers are first in it.
#
# When a new range is added that is adjacent to an existing range that has the
# same value and type, it merges with it to form a larger range.
#
# Ranges generally do not overlap, except that there can be multiple entries
# of single code point ranges. This is because of NameAliases.txt.
#
# In this program, there is a standard value such that if two different
# values, have the same standard value, they are considered equivalent. This
# value was chosen so that it gives correct results on Unicode data
# There are a number of methods to manipulate range lists, and some operators
# are overloaded to handle them.
sub trace { return main::trace(@_); }
{ # Closure
our $addr;
# Max is initialized to a negative value that isn't adjacent to 0, for
# simpler tests
my $max_init = -2;
main::setup_package();
my %ranges;
# The list of ranges
main::set_access('ranges', \%ranges, 'readable_array');
my %max;
# The highest code point in the list. This was originally a method, but
# actual measurements said it was used a lot.
main::set_access('max', \%max, 'r');
my %each_range_iterator;
# Iterator position for each_range()
main::set_access('each_range_iterator', \%each_range_iterator);
my %owner_name_of;
# Name of parent this is attached to, if any. Solely for better error
# messages.
main::set_access('owner_name_of', \%owner_name_of, 'p_r');
my %_search_ranges_cache;
# A cache of the previous result from _search_ranges(), for better
# performance
main::set_access('_search_ranges_cache', \%_search_ranges_cache);
sub new {
my $class = shift;
my %args = @_;
# Optional initialization data for the range list.
my $initialize = delete $args{'Initialize'};
my $self;
# Use _union() to initialize. _union() returns an object of this
# class, which means that it will call this constructor recursively.
# But it won't have this $initialize parameter so that it won't
# infinitely loop on this.
return _union($class, $initialize, %args) if defined $initialize;
$self = bless \do { my $anonymous_scalar }, $class;
my $addr = do { no overloading; pack 'J', $self; };
# Optional parent object, only for debug info.
$owner_name_of{$addr} = delete $args{'Owner'};
$owner_name_of{$addr} = "" if ! defined $owner_name_of{$addr};
# Stringify, in case it is an object.
$owner_name_of{$addr} = "$owner_name_of{$addr}";
# This is used only for error messages, and so a colon is added
$owner_name_of{$addr} .= ": " if $owner_name_of{$addr} ne "";
Carp::carp_extra_args(\%args) if main::DEBUG && %args;
$max{$addr} = $max_init;
$_search_ranges_cache{$addr} = 0;
$ranges{$addr} = [];
return $self;
}
use overload
fallback => 0,
qw("") => "_operator_stringify",
"." => \&main::_operator_dot,
".=" => \&main::_operator_dot_equal,
;
sub _operator_stringify {
my $self = shift;
my $addr = do { no overloading; pack 'J', $self; };
return "Range_List attached to '$owner_name_of{$addr}'"
if $owner_name_of{$addr};
return "anonymous Range_List " . \$self;
}
sub _union {
# Returns the union of the input code points. It can be called as
# either a constructor or a method. If called as a method, the result
# will be a new() instance of the calling object, containing the union
# of that object with the other parameter's code points; if called as
# a constructor, the first parameter gives the class that the new object
# should be, and the second parameter gives the code points to go into
# it.
# In either case, there are two parameters looked at by this routine;
# any additional parameters are passed to the new() constructor.
#
# The code points can come in the form of some object that contains
# ranges, and has a conventionally named method to access them; or
# they can be an array of individual code points (as integers); or
# just a single code point.
#
# If they are ranges, this routine doesn't make any effort to preserve
# the range values and types of one input over the other. Therefore
# this base class should not allow _union to be called from other than
# initialization code, so as to prevent two tables from being added
# together where the range values matter. The general form of this
# routine therefore belongs in a derived class, but it was moved here
# to avoid duplication of code. The failure to overload this in this
# class keeps it safe.
#
# It does make the effort during initialization to accept tables with
# multiple values for the same code point, and to preserve the order
# of these. If there is only one input range or range set, it doesn't
# sort (as it should already be sorted to the desired order), and will
# accept multiple values per code point. Otherwise it will merge
# multiple values into a single one.
my $self;
my @args; # Arguments to pass to the constructor
my $class = shift;
# If a method call, will start the union with the object itself, and
# the class of the new object will be the same as self.
if (ref $class) {
$self = $class;
$class = ref $self;
push @args, $self;
}
# Add the other required parameter.
push @args, shift;
# Rest of parameters are passed on to the constructor
# Accumulate all records from both lists.
my @records;
my $input_count = 0;
for my $arg (@args) {
#local $to_trace = 0 if main::DEBUG;
trace "argument = $arg" if main::DEBUG && $to_trace;
if (! defined $arg) {
my $message = "";
if (defined $self) {
no overloading;
$message .= $owner_name_of{pack 'J', $self};
}
Carp::my_carp_bug($message . "Undefined argument to _union. No union done.");
return;
}
$arg = [ $arg ] if ! ref $arg;
my $type = ref $arg;
if ($type eq 'ARRAY') {
foreach my $element (@$arg) {
push @records, Range->new($element, $element);
$input_count++;
}
}
elsif ($arg->isa('Range')) {
push @records, $arg;
$input_count++;
}
elsif ($arg->can('ranges')) {
push @records, $arg->ranges;
$input_count++;
}
else {
my $message = "";
if (defined $self) {
no overloading;
$message .= $owner_name_of{pack 'J', $self};
}
Carp::my_carp_bug($message . "Cannot take the union of a $type. No union done.");
return;
}
}
# Sort with the range containing the lowest ordinal first, but if
# two ranges start at the same code point, sort with the bigger range
# of the two first, because it takes fewer cycles.
if ($input_count > 1) {
@records = sort { ($a->start <=> $b->start)
or
# if b is shorter than a, b->end will be
# less than a->end, and we want to select
# a, so want to return -1
($b->end <=> $a->end)
} @records;
}
my $new = $class->new(@_);
# Fold in records so long as they add new information.
for my $set (@records) {
my $start = $set->start;
my $end = $set->end;
my $value = $set->value;
my $type = $set->type;
if ($start > $new->max) {
$new->_add_delete('+', $start, $end, $value, Type => $type);
}
elsif ($end > $new->max) {
$new->_add_delete('+', $new->max +1, $end, $value,
Type => $type);
}
elsif ($input_count == 1) {
# Here, overlaps existing range, but is from a single input,
# so preserve the multiple values from that input.
$new->_add_delete('+', $start, $end, $value, Type => $type,
Replace => $MULTIPLE_AFTER);
}
}
return $new;
}
sub range_count { # Return the number of ranges in the range list
my $self = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
no overloading;
return scalar @{$ranges{pack 'J', $self}};
}
sub min {
# Returns the minimum code point currently in the range list, or if
# the range list is empty, 2 beyond the max possible. This is a
# method because used so rarely, that not worth saving between calls,
# and having to worry about changing it as ranges are added and
# deleted.
my $self = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $addr = do { no overloading; pack 'J', $self; };
# If the range list is empty, return a large value that isn't adjacent
# to any that could be in the range list, for simpler tests
return $MAX_WORKING_CODEPOINT + 2 unless scalar @{$ranges{$addr}};
return $ranges{$addr}->[0]->start;
}
sub contains {
# Boolean: Is argument in the range list? If so returns $i such that:
# range[$i]->end < $codepoint <= range[$i+1]->end
# which is one beyond what you want; this is so that the 0th range
# doesn't return false
my $self = shift;
my $codepoint = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $i = $self->_search_ranges($codepoint);
return 0 unless defined $i;
# The search returns $i, such that
# range[$i-1]->end < $codepoint <= range[$i]->end
# So is in the table if and only iff it is at least the start position
# of range $i.
no overloading;
return 0 if $ranges{pack 'J', $self}->[$i]->start > $codepoint;
return $i + 1;
}
sub containing_range {
# Returns the range object that contains the code point, undef if none
my $self = shift;
my $codepoint = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $i = $self->contains($codepoint);
return unless $i;
# contains() returns 1 beyond where we should look
no overloading;
return $ranges{pack 'J', $self}->[$i-1];
}
sub value_of {
# Returns the value associated with the code point, undef if none
my $self = shift;
my $codepoint = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $range = $self->containing_range($codepoint);
return unless defined $range;
return $range->value;
}
sub type_of {
# Returns the type of the range containing the code point, undef if
# the code point is not in the table
my $self = shift;
my $codepoint = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $range = $self->containing_range($codepoint);
return unless defined $range;
return $range->type;
}
sub _search_ranges {
# Find the range in the list which contains a code point, or where it
# should go if were to add it. That is, it returns $i, such that:
# range[$i-1]->end < $codepoint <= range[$i]->end
# Returns undef if no such $i is possible (e.g. at end of table), or
# if there is an error.
my $self = shift;
my $code_point = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $addr = do { no overloading; pack 'J', $self; };
return if $code_point > $max{$addr};
my $r = $ranges{$addr}; # The current list of ranges
my $range_list_size = scalar @$r;
my $i;
use integer; # want integer division
# Use the cached result as the starting guess for this one, because,
# an experiment on 5.1 showed that 90% of the time the cache was the
# same as the result on the next call (and 7% it was one less).
$i = $_search_ranges_cache{$addr};
$i = 0 if $i >= $range_list_size; # Reset if no longer valid (prob.
# from an intervening deletion
#local $to_trace = 1 if main::DEBUG;
trace "previous \$i is still valid: $i" if main::DEBUG && $to_trace && $code_point <= $r->[$i]->end && ($i == 0 || $r->[$i-1]->end < $code_point);
return $i if $code_point <= $r->[$i]->end
&& ($i == 0 || $r->[$i-1]->end < $code_point);
# Here the cache doesn't yield the correct $i. Try adding 1.
if ($i < $range_list_size - 1
&& $r->[$i]->end < $code_point &&
$code_point <= $r->[$i+1]->end)
{
$i++;
trace "next \$i is correct: $i" if main::DEBUG && $to_trace;
$_search_ranges_cache{$addr} = $i;
return $i;
}
# Here, adding 1 also didn't work. We do a binary search to
# find the correct position, starting with current $i
my $lower = 0;
my $upper = $range_list_size - 1;
while (1) {
trace "top of loop i=$i:", sprintf("%04X", $r->[$lower]->start), "[$lower] .. ", sprintf("%04X", $r->[$i]->start), "[$i] .. ", sprintf("%04X", $r->[$upper]->start), "[$upper]" if main::DEBUG && $to_trace;
if ($code_point <= $r->[$i]->end) {
# Here we have met the upper constraint. We can quit if we
# also meet the lower one.
last if $i == 0 || $r->[$i-1]->end < $code_point;
$upper = $i; # Still too high.
}
else {
# Here, $r[$i]->end < $code_point, so look higher up.
$lower = $i;
}
# Split search domain in half to try again.
my $temp = ($upper + $lower) / 2;
# No point in continuing unless $i changes for next time
# in the loop.
if ($temp == $i) {
# We can't reach the highest element because of the averaging.
# So if one below the upper edge, force it there and try one
# more time.
if ($i == $range_list_size - 2) {
trace "Forcing to upper edge" if main::DEBUG && $to_trace;
$i = $range_list_size - 1;
# Change $lower as well so if fails next time through,
# taking the average will yield the same $i, and we will
# quit with the error message just below.
$lower = $i;
next;
}
Carp::my_carp_bug("$owner_name_of{$addr}Can't find where the range ought to go. No action taken.");
return;
}
$i = $temp;
} # End of while loop
if (main::DEBUG && $to_trace) {
trace 'i-1=[', $i-1, ']', $r->[$i-1] if $i;
trace "i= [ $i ]", $r->[$i];
trace 'i+1=[', $i+1, ']', $r->[$i+1] if $i < $range_list_size - 1;
}
# Here we have found the offset. Cache it as a starting point for the
# next call.
$_search_ranges_cache{$addr} = $i;
return $i;
}
sub _add_delete {
# Add, replace or delete ranges to or from a list. The $type
# parameter gives which:
# '+' => insert or replace a range, returning a list of any changed
# ranges.
# '-' => delete a range, returning a list of any deleted ranges.
#
# The next three parameters give respectively the start, end, and
# value associated with the range. 'value' should be null unless the
# operation is '+';
#
# The range list is kept sorted so that the range with the lowest
# starting position is first in the list, and generally, adjacent
# ranges with the same values are merged into a single larger one (see
# exceptions below).
#
# There are more parameters; all are key => value pairs:
# Type gives the type of the value. It is only valid for '+'.
# All ranges have types; if this parameter is omitted, 0 is
# assumed. Ranges with type 0 are assumed to obey the
# Unicode rules for casing, etc; ranges with other types are
# not. Otherwise, the type is arbitrary, for the caller's
# convenience, and looked at only by this routine to keep
# adjacent ranges of different types from being merged into
# a single larger range, and when Replace =>
# $IF_NOT_EQUIVALENT is specified (see just below).
# Replace determines what to do if the range list already contains
# ranges which coincide with all or portions of the input
# range. It is only valid for '+':
# => $NO means that the new value is not to replace
# any existing ones, but any empty gaps of the
# range list coinciding with the input range
# will be filled in with the new value.
# => $UNCONDITIONALLY means to replace the existing values with
# this one unconditionally. However, if the
# new and old values are identical, the
# replacement is skipped to save cycles
# => $IF_NOT_EQUIVALENT means to replace the existing values
# (the default) with this one if they are not equivalent.
# Ranges are equivalent if their types are the
# same, and they are the same string; or if
# both are type 0 ranges, if their Unicode
# standard forms are identical. In this last
# case, the routine chooses the more "modern"
# one to use. This is because some of the
# older files are formatted with values that
# are, for example, ALL CAPs, whereas the
# derived files have a more modern style,
# which looks better. By looking for this
# style when the pre-existing and replacement
# standard forms are the same, we can move to
# the modern style
# => $MULTIPLE_BEFORE means that if this range duplicates an
# existing one, but has a different value,
# don't replace the existing one, but insert
# this one so that the same range can occur
# multiple times. They are stored LIFO, so
# that the final one inserted is the first one
# returned in an ordered search of the table.
# If this is an exact duplicate, including the
# value, the original will be moved to be
# first, before any other duplicate ranges
# with different values.
# => $MULTIPLE_AFTER is like $MULTIPLE_BEFORE, but is stored
# FIFO, so that this one is inserted after all
# others that currently exist. If this is an
# exact duplicate, including value, of an
# existing range, this one is discarded
# (leaving the existing one in its original,
# higher priority position
# => $CROAK Die with an error if is already there
# => anything else is the same as => $IF_NOT_EQUIVALENT
#
# "same value" means identical for non-type-0 ranges, and it means
# having the same standard forms for type-0 ranges.
return Carp::carp_too_few_args(\@_, 5) if main::DEBUG && @_ < 5;
my $self = shift;
my $operation = shift; # '+' for add/replace; '-' for delete;
my $start = shift;
my $end = shift;
my $value = shift;
my %args = @_;
$value = "" if not defined $value; # warning: $value can be "0"
my $replace = delete $args{'Replace'};
$replace = $IF_NOT_EQUIVALENT unless defined $replace;
my $type = delete $args{'Type'};
$type = 0 unless defined $type;
Carp::carp_extra_args(\%args) if main::DEBUG && %args;
my $addr = do { no overloading; pack 'J', $self; };
if ($operation ne '+' && $operation ne '-') {
Carp::my_carp_bug("$owner_name_of{$addr}First parameter to _add_delete must be '+' or '-'. No action taken.");
return;
}
unless (defined $start && defined $end) {
Carp::my_carp_bug("$owner_name_of{$addr}Undefined start and/or end to _add_delete. No action taken.");
return;
}
unless ($end >= $start) {
Carp::my_carp_bug("$owner_name_of{$addr}End of range (" . sprintf("%04X", $end) . ") must not be before start (" . sprintf("%04X", $start) . "). No action taken.");
return;
}
#local $to_trace = 1 if main::DEBUG;
if ($operation eq '-') {
if ($replace != $IF_NOT_EQUIVALENT) {
Carp::my_carp_bug("$owner_name_of{$addr}Replace => \$IF_NOT_EQUIVALENT is required when deleting a range from a range list. Assuming Replace => \$IF_NOT_EQUIVALENT.");
$replace = $IF_NOT_EQUIVALENT;
}
if ($type) {
Carp::my_carp_bug("$owner_name_of{$addr}Type => 0 is required when deleting a range from a range list. Assuming Type => 0.");
$type = 0;
}
if ($value ne "") {
Carp::my_carp_bug("$owner_name_of{$addr}Value => \"\" is required when deleting a range from a range list. Assuming Value => \"\".");
$value = "";
}
}
my $r = $ranges{$addr}; # The current list of ranges
my $range_list_size = scalar @$r; # And its size
my $max = $max{$addr}; # The current high code point in
# the list of ranges
# Do a special case requiring fewer machine cycles when the new range
# starts after the current highest point. The Unicode input data is
# structured so this is common.
if ($start > $max) {
trace "$owner_name_of{$addr} $operation", sprintf("%04X..%04X (%s) type=%d; prev max=%04X", $start, $end, $value, $type, $max) if main::DEBUG && $to_trace;
return if $operation eq '-'; # Deleting a non-existing range is a
# no-op
# If the new range doesn't logically extend the current final one
# in the range list, create a new range at the end of the range
# list. (max cleverly is initialized to a negative number not
# adjacent to 0 if the range list is empty, so even adding a range
# to an empty range list starting at 0 will have this 'if'
# succeed.)
if ($start > $max + 1 # non-adjacent means can't extend.
|| @{$r}[-1]->value ne $value # values differ, can't extend.
|| @{$r}[-1]->type != $type # types differ, can't extend.
) {
push @$r, Range->new($start, $end,
Value => $value,
Type => $type);
}
else {
# Here, the new range starts just after the current highest in
# the range list, and they have the same type and value.
# Extend the existing range to incorporate the new one.
@{$r}[-1]->set_end($end);
}
# This becomes the new maximum.
$max{$addr} = $end;
return;
}
#local $to_trace = 0 if main::DEBUG;
trace "$owner_name_of{$addr} $operation", sprintf("%04X", $start) . '..' . sprintf("%04X", $end) . " ($value) replace=$replace" if main::DEBUG && $to_trace;
# Here, the input range isn't after the whole rest of the range list.
# Most likely 'splice' will be needed. The rest of the routine finds
# the needed splice parameters, and if necessary, does the splice.
# First, find the offset parameter needed by the splice function for
# the input range. Note that the input range may span multiple
# existing ones, but we'll worry about that later. For now, just find
# the beginning. If the input range is to be inserted starting in a
# position not currently in the range list, it must (obviously) come
# just after the range below it, and just before the range above it.
# Slightly less obviously, it will occupy the position currently
# occupied by the range that is to come after it. More formally, we
# are looking for the position, $i, in the array of ranges, such that:
#
# r[$i-1]->start <= r[$i-1]->end < $start < r[$i]->start <= r[$i]->end
#
# (The ordered relationships within existing ranges are also shown in
# the equation above). However, if the start of the input range is
# within an existing range, the splice offset should point to that
# existing range's position in the list; that is $i satisfies a
# somewhat different equation, namely:
#
#r[$i-1]->start <= r[$i-1]->end < r[$i]->start <= $start <= r[$i]->end
#
# More briefly, $start can come before or after r[$i]->start, and at
# this point, we don't know which it will be. However, these
# two equations share these constraints:
#
# r[$i-1]->end < $start <= r[$i]->end
#
# And that is good enough to find $i.
my $i = $self->_search_ranges($start);
if (! defined $i) {
Carp::my_carp_bug("Searching $self for range beginning with $start unexpectedly returned undefined. Operation '$operation' not performed");
return;
}
# The search function returns $i such that:
#
# r[$i-1]->end < $start <= r[$i]->end
#
# That means that $i points to the first range in the range list
# that could possibly be affected by this operation. We still don't
# know if the start of the input range is within r[$i], or if it
# points to empty space between r[$i-1] and r[$i].
trace "[$i] is the beginning splice point. Existing range there is ", $r->[$i] if main::DEBUG && $to_trace;
# Special case the insertion of data that is not to replace any
# existing data.
if ($replace == $NO) { # If $NO, has to be operation '+'
#local $to_trace = 1 if main::DEBUG;
trace "Doesn't replace" if main::DEBUG && $to_trace;
# Here, the new range is to take effect only on those code points
# that aren't already in an existing range. This can be done by
# looking through the existing range list and finding the gaps in
# the ranges that this new range affects, and then calling this
# function recursively on each of those gaps, leaving untouched
# anything already in the list. Gather up a list of the changed
# gaps first so that changes to the internal state as new ranges
# are added won't be a problem.
my @gap_list;
# First, if the starting point of the input range is outside an
# existing one, there is a gap from there to the beginning of the
# existing range -- add a span to fill the part that this new
# range occupies
if ($start < $r->[$i]->start) {
push @gap_list, Range->new($start,
main::min($end,
$r->[$i]->start - 1),
Type => $type);
trace "gap before $r->[$i] [$i], will add", $gap_list[-1] if main::DEBUG && $to_trace;
}
# Then look through the range list for other gaps until we reach
# the highest range affected by the input one.
my $j;
for ($j = $i+1; $j < $range_list_size; $j++) {
trace "j=[$j]", $r->[$j] if main::DEBUG && $to_trace;
last if $end < $r->[$j]->start;
# If there is a gap between when this range starts and the
# previous one ends, add a span to fill it. Note that just
# because there are two ranges doesn't mean there is a
# non-zero gap between them. It could be that they have
# different values or types
if ($r->[$j-1]->end + 1 != $r->[$j]->start) {
push @gap_list,
Range->new($r->[$j-1]->end + 1,
$r->[$j]->start - 1,
Type => $type);
trace "gap between $r->[$j-1] and $r->[$j] [$j], will add: $gap_list[-1]" if main::DEBUG && $to_trace;
}
}
# Here, we have either found an existing range in the range list,
# beyond the area affected by the input one, or we fell off the
# end of the loop because the input range affects the whole rest
# of the range list. In either case, $j is 1 higher than the
# highest affected range. If $j == $i, it means that there are no
# affected ranges, that the entire insertion is in the gap between
# r[$i-1], and r[$i], which we already have taken care of before
# the loop.
# On the other hand, if there are affected ranges, it might be
# that there is a gap that needs filling after the final such
# range to the end of the input range
if ($r->[$j-1]->end < $end) {
push @gap_list, Range->new(main::max($start,
$r->[$j-1]->end + 1),
$end,
Type => $type);
trace "gap after $r->[$j-1], will add $gap_list[-1]" if main::DEBUG && $to_trace;
}
# Call recursively to fill in all the gaps.
foreach my $gap (@gap_list) {
$self->_add_delete($operation,
$gap->start,
$gap->end,
$value,
Type => $type);
}
return;
}
# Here, we have taken care of the case where $replace is $NO.
# Remember that here, r[$i-1]->end < $start <= r[$i]->end
# If inserting a multiple record, this is where it goes, before the
# first (if any) existing one if inserting LIFO. (If this is to go
# afterwards, FIFO, we below move the pointer to there.) These imply
# an insertion, and no change to any existing ranges. Note that $i
# can be -1 if this new range doesn't actually duplicate any existing,
# and comes at the beginning of the list.
if ($replace == $MULTIPLE_BEFORE || $replace == $MULTIPLE_AFTER) {
if ($start != $end) {
Carp::my_carp_bug("$owner_name_of{$addr}Can't cope with adding a multiple record when the range ($start..$end) contains more than one code point. No action taken.");
return;
}
# If the new code point is within a current range ...
if ($end >= $r->[$i]->start) {
# Don't add an exact duplicate, as it isn't really a multiple
my $existing_value = $r->[$i]->value;
my $existing_type = $r->[$i]->type;
return if $value eq $existing_value && $type eq $existing_type;
# If the multiple value is part of an existing range, we want
# to split up that range, so that only the single code point
# is affected. To do this, we first call ourselves
# recursively to delete that code point from the table, having
# preserved its current data above. Then we call ourselves
# recursively again to add the new multiple, which we know by
# the test just above is different than the current code
# point's value, so it will become a range containing a single
# code point: just itself. Finally, we add back in the
# pre-existing code point, which will again be a single code
# point range. Because 'i' likely will have changed as a
# result of these operations, we can't just continue on, but
# do this operation recursively as well. If we are inserting
# LIFO, the pre-existing code point needs to go after the new
# one, so use MULTIPLE_AFTER; and vice versa.
if ($r->[$i]->start != $r->[$i]->end) {
$self->_add_delete('-', $start, $end, "");
$self->_add_delete('+', $start, $end, $value, Type => $type);
return $self->_add_delete('+',
$start, $end,
$existing_value,
Type => $existing_type,
Replace => ($replace == $MULTIPLE_BEFORE)
? $MULTIPLE_AFTER
: $MULTIPLE_BEFORE);
}
}
# If to place this new record after, move to beyond all existing
# ones; but don't add this one if identical to any of them, as it
# isn't really a multiple. This leaves the original order, so
# that the current request is ignored. The reasoning is that the
# previous request that wanted this record to have high priority
# should have precedence.
if ($replace == $MULTIPLE_AFTER) {
while ($i < @$r && $r->[$i]->start == $start) {
return if $value eq $r->[$i]->value
&& $type eq $r->[$i]->type;
$i++;
}
}
else {
# If instead we are to place this new record before any
# existing ones, remove any identical ones that come after it.
# This changes the existing order so that the new one is
# first, as is being requested.
for (my $j = $i + 1;
$j < @$r && $r->[$j]->start == $start;
$j++)
{
if ($value eq $r->[$j]->value && $type eq $r->[$j]->type) {
splice @$r, $j, 1;
last; # There should only be one instance, so no
# need to keep looking
}
}
}
trace "Adding multiple record at $i with $start..$end, $value" if main::DEBUG && $to_trace;
my @return = splice @$r,
$i,
0,
Range->new($start,
$end,
Value => $value,
Type => $type);
if (main::DEBUG && $to_trace) {
trace "After splice:";
trace 'i-2=[', $i-2, ']', $r->[$i-2] if $i >= 2;
trace 'i-1=[', $i-1, ']', $r->[$i-1] if $i >= 1;
trace "i =[", $i, "]", $r->[$i] if $i >= 0;
trace 'i+1=[', $i+1, ']', $r->[$i+1] if $i < @$r - 1;
trace 'i+2=[', $i+2, ']', $r->[$i+2] if $i < @$r - 2;
trace 'i+3=[', $i+3, ']', $r->[$i+3] if $i < @$r - 3;
}
return @return;
}
# Here, we have taken care of $NO and $MULTIPLE_foo replaces. This
# leaves delete, insert, and replace either unconditionally or if not
# equivalent. $i still points to the first potential affected range.
# Now find the highest range affected, which will determine the length
# parameter to splice. (The input range can span multiple existing
# ones.) If this isn't a deletion, while we are looking through the
# range list, see also if this is a replacement rather than a clean
# insertion; that is if it will change the values of at least one
# existing range. Start off assuming it is an insert, until find it
# isn't.
my $clean_insert = $operation eq '+';
my $j; # This will point to the highest affected range
# For non-zero types, the standard form is the value itself;
my $standard_form = ($type) ? $value : main::standardize($value);
for ($j = $i; $j < $range_list_size; $j++) {
trace "Looking for highest affected range; the one at $j is ", $r->[$j] if main::DEBUG && $to_trace;
# If find a range that it doesn't overlap into, we can stop
# searching
last if $end < $r->[$j]->start;
# Here, overlaps the range at $j. If the values don't match,
# and so far we think this is a clean insertion, it becomes a
# non-clean insertion, i.e., a 'change' or 'replace' instead.
if ($clean_insert) {
if ($r->[$j]->standard_form ne $standard_form) {
$clean_insert = 0;
if ($replace == $CROAK) {
main::croak("The range to add "
. sprintf("%04X", $start)
. '-'
. sprintf("%04X", $end)
. " with value '$value' overlaps an existing range $r->[$j]");
}
}
else {
# Here, the two values are essentially the same. If the
# two are actually identical, replacing wouldn't change
# anything so skip it.
my $pre_existing = $r->[$j]->value;
if ($pre_existing ne $value) {
# Here the new and old standardized values are the
# same, but the non-standardized values aren't. If
# replacing unconditionally, then replace
if( $replace == $UNCONDITIONALLY) {
$clean_insert = 0;
}
else {
# Here, are replacing conditionally. Decide to
# replace or not based on which appears to look
# the "nicest". If one is mixed case and the
# other isn't, choose the mixed case one.
my $new_mixed = $value =~ /[A-Z]/
&& $value =~ /[a-z]/;
my $old_mixed = $pre_existing =~ /[A-Z]/
&& $pre_existing =~ /[a-z]/;
if ($old_mixed != $new_mixed) {
$clean_insert = 0 if $new_mixed;
if (main::DEBUG && $to_trace) {
if ($clean_insert) {
trace "Retaining $pre_existing over $value";
}
else {
trace "Replacing $pre_existing with $value";
}
}
}
else {
# Here casing wasn't different between the two.
# If one has hyphens or underscores and the
# other doesn't, choose the one with the
# punctuation.
my $new_punct = $value =~ /[-_]/;
my $old_punct = $pre_existing =~ /[-_]/;
if ($old_punct != $new_punct) {
$clean_insert = 0 if $new_punct;
if (main::DEBUG && $to_trace) {
if ($clean_insert) {
trace "Retaining $pre_existing over $value";
}
else {
trace "Replacing $pre_existing with $value";
}
}
} # else existing one is just as "good";
# retain it to save cycles.
}
}
}
}
}
} # End of loop looking for highest affected range.
# Here, $j points to one beyond the highest range that this insertion
# affects (hence to beyond the range list if that range is the final
# one in the range list).
# The splice length is all the affected ranges. Get it before
# subtracting, for efficiency, so we don't have to later add 1.
my $length = $j - $i;
$j--; # $j now points to the highest affected range.
trace "Final affected range is $j: $r->[$j]" if main::DEBUG && $to_trace;
# Here, have taken care of $NO and $MULTIPLE_foo replaces.
# $j points to the highest affected range. But it can be < $i or even
# -1. These happen only if the insertion is entirely in the gap
# between r[$i-1] and r[$i]. Here's why: j < i means that the j loop
# above exited first time through with $end < $r->[$i]->start. (And
# then we subtracted one from j) This implies also that $start <
# $r->[$i]->start, but we know from above that $r->[$i-1]->end <
# $start, so the entire input range is in the gap.
if ($j < $i) {
# Here the entire input range is in the gap before $i.
if (main::DEBUG && $to_trace) {
if ($i) {
trace "Entire range is between $r->[$i-1] and $r->[$i]";
}
else {
trace "Entire range is before $r->[$i]";
}
}
return if $operation ne '+'; # Deletion of a non-existent range is
# a no-op
}
else {
# Here part of the input range is not in the gap before $i. Thus,
# there is at least one affected one, and $j points to the highest
# such one.
# At this point, here is the situation:
# This is not an insertion of a multiple, nor of tentative ($NO)
# data.
# $i points to the first element in the current range list that
# may be affected by this operation. In fact, we know
# that the range at $i is affected because we are in
# the else branch of this 'if'
# $j points to the highest affected range.
# In other words,
# r[$i-1]->end < $start <= r[$i]->end
# And:
# r[$i-1]->end < $start <= $end < r[$j+1]->start
#
# Also:
# $clean_insert is a boolean which is set true if and only if
# this is a "clean insertion", i.e., not a change nor a
# deletion (multiple was handled above).
# We now have enough information to decide if this call is a no-op
# or not. It is a no-op if this is an insertion of already
# existing data. To be so, it must be contained entirely in one
# range.
if (main::DEBUG && $to_trace && $clean_insert
&& $start >= $r->[$i]->start
&& $end <= $r->[$i]->end)
{
trace "no-op";
}
return if $clean_insert
&& $start >= $r->[$i]->start
&& $end <= $r->[$i]->end;
}
# Here, we know that some action will have to be taken. We have
# calculated the offset and length (though adjustments may be needed)
# for the splice. Now start constructing the replacement list.
my @replacement;
my $splice_start = $i;
my $extends_below;
my $extends_above;
# See if should extend any adjacent ranges.
if ($operation eq '-') { # Don't extend deletions
$extends_below = $extends_above = 0;
}
else { # Here, should extend any adjacent ranges. See if there are
# any.
$extends_below = ($i > 0
# can't extend unless adjacent
&& $r->[$i-1]->end == $start -1
# can't extend unless are same standard value
&& $r->[$i-1]->standard_form eq $standard_form
# can't extend unless share type
&& $r->[$i-1]->type == $type);
$extends_above = ($j+1 < $range_list_size
&& $r->[$j+1]->start == $end +1
&& $r->[$j+1]->standard_form eq $standard_form
&& $r->[$j+1]->type == $type);
}
if ($extends_below && $extends_above) { # Adds to both
$splice_start--; # start replace at element below
$length += 2; # will replace on both sides
trace "Extends both below and above ranges" if main::DEBUG && $to_trace;
# The result will fill in any gap, replacing both sides, and
# create one large range.
@replacement = Range->new($r->[$i-1]->start,
$r->[$j+1]->end,
Value => $value,
Type => $type);
}
else {
# Here we know that the result won't just be the conglomeration of
# a new range with both its adjacent neighbors. But it could
# extend one of them.
if ($extends_below) {
# Here the new element adds to the one below, but not to the
# one above. If inserting, and only to that one range, can
# just change its ending to include the new one.
if ($length == 0 && $clean_insert) {
$r->[$i-1]->set_end($end);
trace "inserted range extends range to below so it is now $r->[$i-1]" if main::DEBUG && $to_trace;
return;
}
else {
trace "Changing inserted range to start at ", sprintf("%04X", $r->[$i-1]->start), " instead of ", sprintf("%04X", $start) if main::DEBUG && $to_trace;
$splice_start--; # start replace at element below
$length++; # will replace the element below
$start = $r->[$i-1]->start;
}
}
elsif ($extends_above) {
# Here the new element adds to the one above, but not below.
# Mirror the code above
if ($length == 0 && $clean_insert) {
$r->[$j+1]->set_start($start);
trace "inserted range extends range to above so it is now $r->[$j+1]" if main::DEBUG && $to_trace;
return;
}
else {
trace "Changing inserted range to end at ", sprintf("%04X", $r->[$j+1]->end), " instead of ", sprintf("%04X", $end) if main::DEBUG && $to_trace;
$length++; # will replace the element above
$end = $r->[$j+1]->end;
}
}
trace "Range at $i is $r->[$i]" if main::DEBUG && $to_trace;
# Finally, here we know there will have to be a splice.
# If the change or delete affects only the highest portion of the
# first affected range, the range will have to be split. The
# splice will remove the whole range, but will replace it by a new
# range containing just the unaffected part. So, in this case,
# add to the replacement list just this unaffected portion.
if (! $extends_below
&& $start > $r->[$i]->start && $start <= $r->[$i]->end)
{
push @replacement,
Range->new($r->[$i]->start,
$start - 1,
Value => $r->[$i]->value,
Type => $r->[$i]->type);
}
# In the case of an insert or change, but not a delete, we have to
# put in the new stuff; this comes next.
if ($operation eq '+') {
push @replacement, Range->new($start,
$end,
Value => $value,
Type => $type);
}
trace "Range at $j is $r->[$j]" if main::DEBUG && $to_trace && $j != $i;
#trace "$end >=", $r->[$j]->start, " && $end <", $r->[$j]->end if main::DEBUG && $to_trace;
# And finally, if we're changing or deleting only a portion of the
# highest affected range, it must be split, as the lowest one was.
if (! $extends_above
&& $j >= 0 # Remember that j can be -1 if before first
# current element
&& $end >= $r->[$j]->start
&& $end < $r->[$j]->end)
{
push @replacement,
Range->new($end + 1,
$r->[$j]->end,
Value => $r->[$j]->value,
Type => $r->[$j]->type);
}
}
# And do the splice, as calculated above
if (main::DEBUG && $to_trace) {
trace "replacing $length element(s) at $i with ";
foreach my $replacement (@replacement) {
trace " $replacement";
}
trace "Before splice:";
trace 'i-2=[', $i-2, ']', $r->[$i-2] if $i >= 2;
trace 'i-1=[', $i-1, ']', $r->[$i-1] if $i >= 1;
trace "i =[", $i, "]", $r->[$i];
trace 'i+1=[', $i+1, ']', $r->[$i+1] if $i < @$r - 1;
trace 'i+2=[', $i+2, ']', $r->[$i+2] if $i < @$r - 2;
}
my @return = splice @$r, $splice_start, $length, @replacement;
if (main::DEBUG && $to_trace) {
trace "After splice:";
trace 'i-2=[', $i-2, ']', $r->[$i-2] if $i >= 2;
trace 'i-1=[', $i-1, ']', $r->[$i-1] if $i >= 1;
trace "i =[", $i, "]", $r->[$i];
trace 'i+1=[', $i+1, ']', $r->[$i+1] if $i < @$r - 1;
trace 'i+2=[', $i+2, ']', $r->[$i+2] if $i < @$r - 2;
trace "removed ", @return if @return;
}
# An actual deletion could have changed the maximum in the list.
# There was no deletion if the splice didn't return something, but
# otherwise recalculate it. This is done too rarely to worry about
# performance.
if ($operation eq '-' && @return) {
if (@$r) {
$max{$addr} = $r->[-1]->end;
}
else { # Now empty
$max{$addr} = $max_init;
}
}
return @return;
}
sub reset_each_range { # reset the iterator for each_range();
my $self = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
no overloading;
undef $each_range_iterator{pack 'J', $self};
return;
}
sub each_range {
# Iterate over each range in a range list. Results are undefined if
# the range list is changed during the iteration.
my $self = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $addr = do { no overloading; pack 'J', $self; };
return if $self->is_empty;
$each_range_iterator{$addr} = -1
if ! defined $each_range_iterator{$addr};
$each_range_iterator{$addr}++;
return $ranges{$addr}->[$each_range_iterator{$addr}]
if $each_range_iterator{$addr} < @{$ranges{$addr}};
undef $each_range_iterator{$addr};
return;
}
sub count { # Returns count of code points in range list
my $self = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $addr = do { no overloading; pack 'J', $self; };
my $count = 0;
foreach my $range (@{$ranges{$addr}}) {
$count += $range->end - $range->start + 1;
}
return $count;
}
sub delete_range { # Delete a range
my $self = shift;
my $start = shift;
my $end = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
return $self->_add_delete('-', $start, $end, "");
}
sub is_empty { # Returns boolean as to if a range list is empty
my $self = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
no overloading;
return scalar @{$ranges{pack 'J', $self}} == 0;
}
sub hash {
# Quickly returns a scalar suitable for separating tables into
# buckets, i.e. it is a hash function of the contents of a table, so
# there are relatively few conflicts.
my $self = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $addr = do { no overloading; pack 'J', $self; };
# These are quickly computable. Return looks like 'min..max;count'
return $self->min . "..$max{$addr};" . scalar @{$ranges{$addr}};
}
} # End closure for _Range_List_Base
package Range_List;
use parent '-norequire', '_Range_List_Base';
# A Range_List is a range list for match tables; i.e. the range values are
# not significant. Thus a number of operations can be safely added to it,
# such as inversion, intersection. Note that union is also an unsafe
# operation when range values are cared about, and that method is in the base
# class, not here. But things are set up so that that method is callable only
# during initialization. Only in this derived class, is there an operation
# that combines two tables. A Range_Map can thus be used to initialize a
# Range_List, and its mappings will be in the list, but are not significant to
# this class.
sub trace { return main::trace(@_); }
{ # Closure
use overload
fallback => 0,
'+' => sub { my $self = shift;
my $other = shift;
return $self->_union($other)
},
'+=' => sub { my $self = shift;
my $other = shift;
my $reversed = shift;
if ($reversed) {
Carp::my_carp_bug("Bad news. Can't cope with '"
. ref($other)
. ' += '
. ref($self)
. "'. undef returned.");
return;
}
return $self->_union($other)
},
'&' => sub { my $self = shift;
my $other = shift;
return $self->_intersect($other, 0);
},
'&=' => sub { my $self = shift;
my $other = shift;
my $reversed = shift;
if ($reversed) {
Carp::my_carp_bug("Bad news. Can't cope with '"
. ref($other)
. ' &= '
. ref($self)
. "'. undef returned.");
return;
}
return $self->_intersect($other, 0);
},
'~' => "_invert",
'-' => "_subtract",
;
sub _invert {
# Returns a new Range_List that gives all code points not in $self.
my $self = shift;
my $new = Range_List->new;
# Go through each range in the table, finding the gaps between them
my $max = -1; # Set so no gap before range beginning at 0
for my $range ($self->ranges) {
my $start = $range->start;
my $end = $range->end;
# If there is a gap before this range, the inverse will contain
# that gap.
if ($start > $max + 1) {
$new->add_range($max + 1, $start - 1);
}
$max = $end;
}
# And finally, add the gap from the end of the table to the max
# possible code point
if ($max < $MAX_WORKING_CODEPOINT) {
$new->add_range($max + 1, $MAX_WORKING_CODEPOINT);
}
return $new;
}
sub _subtract {
# Returns a new Range_List with the argument deleted from it. The
# argument can be a single code point, a range, or something that has
# a range, with the _range_list() method on it returning them
my $self = shift;
my $other = shift;
my $reversed = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
if ($reversed) {
Carp::my_carp_bug("Bad news. Can't cope with '"
. ref($other)
. ' - '
. ref($self)
. "'. undef returned.");
return;
}
my $new = Range_List->new(Initialize => $self);
if (! ref $other) { # Single code point
$new->delete_range($other, $other);
}
elsif ($other->isa('Range')) {
$new->delete_range($other->start, $other->end);
}
elsif ($other->can('_range_list')) {
foreach my $range ($other->_range_list->ranges) {
$new->delete_range($range->start, $range->end);
}
}
else {
Carp::my_carp_bug("Can't cope with a "
. ref($other)
. " argument to '-'. Subtraction ignored."
);
return $self;
}
return $new;
}
sub _intersect {
# Returns either a boolean giving whether the two inputs' range lists
# intersect (overlap), or a new Range_List containing the intersection
# of the two lists. The optional final parameter being true indicates
# to do the check instead of the intersection.
my $a_object = shift;
my $b_object = shift;
my $check_if_overlapping = shift;
$check_if_overlapping = 0 unless defined $check_if_overlapping;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
if (! defined $b_object) {
my $message = "";
$message .= $a_object->_owner_name_of if defined $a_object;
Carp::my_carp_bug($message .= "Called with undefined value. Intersection not done.");
return;
}
# a & b = !(!a | !b), or in our terminology = ~ ( ~a + -b )
# Thus the intersection could be much more simply be written:
# return ~(~$a_object + ~$b_object);
# But, this is slower, and when taking the inverse of a large
# range_size_1 table, back when such tables were always stored that
# way, it became prohibitively slow, hence the code was changed to the
# below
if ($b_object->isa('Range')) {
$b_object = Range_List->new(Initialize => $b_object,
Owner => $a_object->_owner_name_of);
}
$b_object = $b_object->_range_list if $b_object->can('_range_list');
my @a_ranges = $a_object->ranges;
my @b_ranges = $b_object->ranges;
#local $to_trace = 1 if main::DEBUG;
trace "intersecting $a_object with ", scalar @a_ranges, "ranges and $b_object with", scalar @b_ranges, " ranges" if main::DEBUG && $to_trace;
# Start with the first range in each list
my $a_i = 0;
my $range_a = $a_ranges[$a_i];
my $b_i = 0;
my $range_b = $b_ranges[$b_i];
my $new = __PACKAGE__->new(Owner => $a_object->_owner_name_of)
if ! $check_if_overlapping;
# If either list is empty, there is no intersection and no overlap
if (! defined $range_a || ! defined $range_b) {
return $check_if_overlapping ? 0 : $new;
}
trace "range_a[$a_i]=$range_a; range_b[$b_i]=$range_b" if main::DEBUG && $to_trace;
# Otherwise, must calculate the intersection/overlap. Start with the
# very first code point in each list
my $a = $range_a->start;
my $b = $range_b->start;
# Loop through all the ranges of each list; in each iteration, $a and
# $b are the current code points in their respective lists
while (1) {
# If $a and $b are the same code point, ...
if ($a == $b) {
# it means the lists overlap. If just checking for overlap
# know the answer now,
return 1 if $check_if_overlapping;
# The intersection includes this code point plus anything else
# common to both current ranges.
my $start = $a;
my $end = main::min($range_a->end, $range_b->end);
if (! $check_if_overlapping) {
trace "adding intersection range ", sprintf("%04X", $start) . ".." . sprintf("%04X", $end) if main::DEBUG && $to_trace;
$new->add_range($start, $end);
}
# Skip ahead to the end of the current intersect
$a = $b = $end;
# If the current intersect ends at the end of either range (as
# it must for at least one of them), the next possible one
# will be the beginning code point in it's list's next range.
if ($a == $range_a->end) {
$range_a = $a_ranges[++$a_i];
last unless defined $range_a;
$a = $range_a->start;
}
if ($b == $range_b->end) {
$range_b = $b_ranges[++$b_i];
last unless defined $range_b;
$b = $range_b->start;
}
trace "range_a[$a_i]=$range_a; range_b[$b_i]=$range_b" if main::DEBUG && $to_trace;
}
elsif ($a < $b) {
# Not equal, but if the range containing $a encompasses $b,
# change $a to be the middle of the range where it does equal
# $b, so the next iteration will get the intersection
if ($range_a->end >= $b) {
$a = $b;
}
else {
# Here, the current range containing $a is entirely below
# $b. Go try to find a range that could contain $b.
$a_i = $a_object->_search_ranges($b);
# If no range found, quit.
last unless defined $a_i;
# The search returns $a_i, such that
# range_a[$a_i-1]->end < $b <= range_a[$a_i]->end
# Set $a to the beginning of this new range, and repeat.
$range_a = $a_ranges[$a_i];
$a = $range_a->start;
}
}
else { # Here, $b < $a.
# Mirror image code to the leg just above
if ($range_b->end >= $a) {
$b = $a;
}
else {
$b_i = $b_object->_search_ranges($a);
last unless defined $b_i;
$range_b = $b_ranges[$b_i];
$b = $range_b->start;
}
}
} # End of looping through ranges.
# Intersection fully computed, or now know that there is no overlap
return $check_if_overlapping ? 0 : $new;
}
sub overlaps {
# Returns boolean giving whether the two arguments overlap somewhere
my $self = shift;
my $other = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
return $self->_intersect($other, 1);
}
sub add_range {
# Add a range to the list.
my $self = shift;
my $start = shift;
my $end = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
return $self->_add_delete('+', $start, $end, "");
}
sub matches_identically_to {
# Return a boolean as to whether or not two Range_Lists match identical
# sets of code points.
my $self = shift;
my $other = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
# These are ordered in increasing real time to figure out (at least
# until a patch changes that and doesn't change this)
return 0 if $self->max != $other->max;
return 0 if $self->min != $other->min;
return 0 if $self->range_count != $other->range_count;
return 0 if $self->count != $other->count;
# Here they could be identical because all the tests above passed.
# The loop below is somewhat simpler since we know they have the same
# number of elements. Compare range by range, until reach the end or
# find something that differs.
my @a_ranges = $self->ranges;
my @b_ranges = $other->ranges;
for my $i (0 .. @a_ranges - 1) {
my $a = $a_ranges[$i];
my $b = $b_ranges[$i];
trace "self $a; other $b" if main::DEBUG && $to_trace;
return 0 if ! defined $b
|| $a->start != $b->start
|| $a->end != $b->end;
}
return 1;
}
sub is_code_point_usable {
# This used only for making the test script. See if the input
# proposed trial code point is one that Perl will handle. If second
# parameter is 0, it won't select some code points for various
# reasons, noted below.
my $code = shift;
my $try_hard = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
return 0 if $code < 0; # Never use a negative
# shun null. I'm (khw) not sure why this was done, but NULL would be
# the character very frequently used.
return $try_hard if $code == 0x0000;
# shun non-character code points.
return $try_hard if $code >= 0xFDD0 && $code <= 0xFDEF;
return $try_hard if ($code & 0xFFFE) == 0xFFFE; # includes FFFF
return $try_hard if $code > $MAX_UNICODE_CODEPOINT; # keep in range
return $try_hard if $code >= 0xD800 && $code <= 0xDFFF; # no surrogate
return 1;
}
sub get_valid_code_point {
# Return a code point that's part of the range list. Returns nothing
# if the table is empty or we can't find a suitable code point. This
# used only for making the test script.
my $self = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $addr = do { no overloading; pack 'J', $self; };
# On first pass, don't choose less desirable code points; if no good
# one is found, repeat, allowing a less desirable one to be selected.
for my $try_hard (0, 1) {
# Look through all the ranges for a usable code point.
for my $set (reverse $self->ranges) {
# Try the edge cases first, starting with the end point of the
# range.
my $end = $set->end;
return $end if is_code_point_usable($end, $try_hard);
$end = $MAX_UNICODE_CODEPOINT + 1 if $end > $MAX_UNICODE_CODEPOINT;
# End point didn't, work. Start at the beginning and try
# every one until find one that does work.
for my $trial ($set->start .. $end - 1) {
return $trial if is_code_point_usable($trial, $try_hard);
}
}
}
return (); # If none found, give up.
}
sub get_invalid_code_point {
# Return a code point that's not part of the table. Returns nothing
# if the table covers all code points or a suitable code point can't
# be found. This used only for making the test script.
my $self = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
# Just find a valid code point of the inverse, if any.
return Range_List->new(Initialize => ~ $self)->get_valid_code_point;
}
} # end closure for Range_List
package Range_Map;
use parent '-norequire', '_Range_List_Base';
# A Range_Map is a range list in which the range values (called maps) are
# significant, and hence shouldn't be manipulated by our other code, which
# could be ambiguous or lose things. For example, in taking the union of two
# lists, which share code points, but which have differing values, which one
# has precedence in the union?
# It turns out that these operations aren't really necessary for map tables,
# and so this class was created to make sure they aren't accidentally
# applied to them.
{ # Closure
sub add_map {
# Add a range containing a mapping value to the list
my $self = shift;
# Rest of parameters passed on
return $self->_add_delete('+', @_);
}
sub replace_map {
# Replace a range
my $self = shift;
return $self->_add_delete('+', @_, Replace => $UNCONDITIONALLY);
}
sub add_duplicate {
# Adds entry to a range list which can duplicate an existing entry
my $self = shift;
my $code_point = shift;
my $value = shift;
my %args = @_;
my $replace = delete $args{'Replace'} // $MULTIPLE_BEFORE;
Carp::carp_extra_args(\%args) if main::DEBUG && %args;
return $self->add_map($code_point, $code_point,
$value, Replace => $replace);
}
} # End of closure for package Range_Map
package _Base_Table;
# A table is the basic data structure that gets written out into a file for
# use by the Perl core. This is the abstract base class implementing the
# common elements from the derived ones. A list of the methods to be
# furnished by an implementing class is just after the constructor.
sub standardize { return main::standardize($_[0]); }
sub trace { return main::trace(@_); }
{ # Closure
main::setup_package();
my %range_list;
# Object containing the ranges of the table.
main::set_access('range_list', \%range_list, 'p_r', 'p_s');
my %full_name;
# The full table name.
main::set_access('full_name', \%full_name, 'r');
my %name;
# The table name, almost always shorter
main::set_access('name', \%name, 'r');
my %short_name;
# The shortest of all the aliases for this table, with underscores removed
main::set_access('short_name', \%short_name);
my %nominal_short_name_length;
# The length of short_name before removing underscores
main::set_access('nominal_short_name_length',
\%nominal_short_name_length);
my %complete_name;
# The complete name, including property.
main::set_access('complete_name', \%complete_name, 'r');
my %property;
# Parent property this table is attached to.
main::set_access('property', \%property, 'r');
my %aliases;
# Ordered list of alias objects of the table's name. The first ones in
# the list are output first in comments
main::set_access('aliases', \%aliases, 'readable_array');
my %comment;
# A comment associated with the table for human readers of the files
main::set_access('comment', \%comment, 's');
my %description;
# A comment giving a short description of the table's meaning for human
# readers of the files.
main::set_access('description', \%description, 'readable_array');
my %note;
# A comment giving a short note about the table for human readers of the
# files.
main::set_access('note', \%note, 'readable_array');
my %fate;
# Enum; there are a number of possibilities for what happens to this
# table: it could be normal, or suppressed, or not for external use. See
# values at definition for $SUPPRESSED.
main::set_access('fate', \%fate, 'r');
my %find_table_from_alias;
# The parent property passes this pointer to a hash which this class adds
# all its aliases to, so that the parent can quickly take an alias and
# find this table.
main::set_access('find_table_from_alias', \%find_table_from_alias, 'p_r');
my %locked;
# After this table is made equivalent to another one; we shouldn't go
# changing the contents because that could mean it's no longer equivalent
main::set_access('locked', \%locked, 'r');
my %file_path;
# This gives the final path to the file containing the table. Each
# directory in the path is an element in the array
main::set_access('file_path', \%file_path, 'readable_array');
my %status;
# What is the table's status, normal, $OBSOLETE, etc. Enum
main::set_access('status', \%status, 'r');
my %status_info;
# A comment about its being obsolete, or whatever non normal status it has
main::set_access('status_info', \%status_info, 'r');
my %caseless_equivalent;
# The table this is equivalent to under /i matching, if any.
main::set_access('caseless_equivalent', \%caseless_equivalent, 'r', 's');
my %range_size_1;
# Is the table to be output with each range only a single code point?
# This is done to avoid breaking existing code that may have come to rely
# on this behavior in previous versions of this program.)
main::set_access('range_size_1', \%range_size_1, 'r', 's');
my %perl_extension;
# A boolean set iff this table is a Perl extension to the Unicode
# standard.
main::set_access('perl_extension', \%perl_extension, 'r');
my %output_range_counts;
# A boolean set iff this table is to have comments written in the
# output file that contain the number of code points in the range.
# The constructor can override the global flag of the same name.
main::set_access('output_range_counts', \%output_range_counts, 'r');
my %write_as_invlist;
# A boolean set iff the output file for this table is to be in the form of
# an inversion list/map.
main::set_access('write_as_invlist', \%write_as_invlist, 'r');
my %format;
# The format of the entries of the table. This is calculated from the
# data in the table (or passed in the constructor). This is an enum e.g.,
# $STRING_FORMAT. It is marked protected as it should not be generally
# used to override calculations.
main::set_access('format', \%format, 'r', 'p_s');
my %has_dependency;
# A boolean that gives whether some other table in this property is
# defined as the complement of this table. This is a crude, but currently
# sufficient, mechanism to make this table not get destroyed before what
# is dependent on it is. Other dependencies could be added, so the name
# was chosen to reflect a more general situation than actually is
# currently the case.
main::set_access('has_dependency', \%has_dependency, 'r', 's');
sub new {
# All arguments are key => value pairs, which you can see below, most
# of which match fields documented above. Otherwise: Re_Pod_Entry,
# OK_as_Filename, and Fuzzy apply to the names of the table, and are
# documented in the Alias package
return Carp::carp_too_few_args(\@_, 2) if main::DEBUG && @_ < 2;
my $class = shift;
my $self = bless \do { my $anonymous_scalar }, $class;
my $addr = do { no overloading; pack 'J', $self; };
my %args = @_;
$name{$addr} = delete $args{'Name'};
$find_table_from_alias{$addr} = delete $args{'_Alias_Hash'};
$full_name{$addr} = delete $args{'Full_Name'};
my $complete_name = $complete_name{$addr}
= delete $args{'Complete_Name'};
$format{$addr} = delete $args{'Format'};
$output_range_counts{$addr} = delete $args{'Output_Range_Counts'};
$property{$addr} = delete $args{'_Property'};
$range_list{$addr} = delete $args{'_Range_List'};
$status{$addr} = delete $args{'Status'} || $NORMAL;
$status_info{$addr} = delete $args{'_Status_Info'} || "";
$range_size_1{$addr} = delete $args{'Range_Size_1'} || 0;
$caseless_equivalent{$addr} = delete $args{'Caseless_Equivalent'} || 0;
$fate{$addr} = delete $args{'Fate'} || $ORDINARY;
$write_as_invlist{$addr} = delete $args{'Write_As_Invlist'};# No default
my $ucd = delete $args{'UCD'};
my $description = delete $args{'Description'};
my $ok_as_filename = delete $args{'OK_as_Filename'};
my $loose_match = delete $args{'Fuzzy'};
my $note = delete $args{'Note'};
my $make_re_pod_entry = delete $args{'Re_Pod_Entry'};
my $perl_extension = delete $args{'Perl_Extension'};
my $suppression_reason = delete $args{'Suppression_Reason'};
# Shouldn't have any left over
Carp::carp_extra_args(\%args) if main::DEBUG && %args;
# Can't use || above because conceivably the name could be 0, and
# can't use // operator in case this program gets used in Perl 5.8
$full_name{$addr} = $name{$addr} if ! defined $full_name{$addr};
$output_range_counts{$addr} = $output_range_counts if
! defined $output_range_counts{$addr};
$aliases{$addr} = [ ];
$comment{$addr} = [ ];
$description{$addr} = [ ];
$note{$addr} = [ ];
$file_path{$addr} = [ ];
$locked{$addr} = "";
$has_dependency{$addr} = 0;
push @{$description{$addr}}, $description if $description;
push @{$note{$addr}}, $note if $note;
if ($fate{$addr} == $PLACEHOLDER) {
# A placeholder table doesn't get documented, is a perl extension,
# and quite likely will be empty
$make_re_pod_entry = 0 if ! defined $make_re_pod_entry;
$perl_extension = 1 if ! defined $perl_extension;
$ucd = 0 if ! defined $ucd;
push @tables_that_may_be_empty, $complete_name{$addr};
$self->add_comment(<<END);
This is a placeholder because it is not in Version $string_version of Unicode,
but is needed by the Perl core to work gracefully. Because it is not in this
version of Unicode, it will not be listed in $pod_file.pod
END
}
elsif (exists $why_suppressed{$complete_name}
# Don't suppress if overridden
&& ! grep { $_ eq $complete_name{$addr} }
@output_mapped_properties)
{
$fate{$addr} = $SUPPRESSED;
}
elsif ($fate{$addr} == $SUPPRESSED) {
Carp::my_carp_bug("Need reason for suppressing") unless $suppression_reason;
# Though currently unused
}
elsif ($suppression_reason) {
Carp::my_carp_bug("A reason was given for suppressing, but not suppressed");
}
# If hasn't set its status already, see if it is on one of the
# lists of properties or tables that have particular statuses; if
# not, is normal. The lists are prioritized so the most serious
# ones are checked first
if (! $status{$addr}) {
if (exists $why_deprecated{$complete_name}) {
$status{$addr} = $DEPRECATED;
}
elsif (exists $why_stabilized{$complete_name}) {
$status{$addr} = $STABILIZED;
}
elsif (exists $why_obsolete{$complete_name}) {
$status{$addr} = $OBSOLETE;
}
# Existence above doesn't necessarily mean there is a message
# associated with it. Use the most serious message.
if ($status{$addr}) {
if ($why_deprecated{$complete_name}) {
$status_info{$addr}
= $why_deprecated{$complete_name};
}
elsif ($why_stabilized{$complete_name}) {
$status_info{$addr}
= $why_stabilized{$complete_name};
}
elsif ($why_obsolete{$complete_name}) {
$status_info{$addr}
= $why_obsolete{$complete_name};
}
}
}
$perl_extension{$addr} = $perl_extension || 0;
# Don't list a property by default that is internal only
if ($fate{$addr} > $MAP_PROXIED) {
$make_re_pod_entry = 0 if ! defined $make_re_pod_entry;
$ucd = 0 if ! defined $ucd;
}
else {
$ucd = 1 if ! defined $ucd;
}
# By convention what typically gets printed only or first is what's
# first in the list, so put the full name there for good output
# clarity. Other routines rely on the full name being first on the
# list
$self->add_alias($full_name{$addr},
OK_as_Filename => $ok_as_filename,
Fuzzy => $loose_match,
Re_Pod_Entry => $make_re_pod_entry,
Status => $status{$addr},
UCD => $ucd,
);
# Then comes the other name, if meaningfully different.
if (standardize($full_name{$addr}) ne standardize($name{$addr})) {
$self->add_alias($name{$addr},
OK_as_Filename => $ok_as_filename,
Fuzzy => $loose_match,
Re_Pod_Entry => $make_re_pod_entry,
Status => $status{$addr},
UCD => $ucd,
);
}
return $self;
}
# Here are the methods that are required to be defined by any derived
# class
for my $sub (qw(
handle_special_range
append_to_body
pre_body
))
# write() knows how to write out normal ranges, but it calls
# handle_special_range() when it encounters a non-normal one.
# append_to_body() is called by it after it has handled all
# ranges to add anything after the main portion of the table.
# And finally, pre_body() is called after all this to build up
# anything that should appear before the main portion of the
# table. Doing it this way allows things in the middle to
# affect what should appear before the main portion of the
# table.
{
no strict "refs";
*$sub = sub {
Carp::my_carp_bug( __LINE__
. ": Must create method '$sub()' for "
. ref shift);
return;
}
}
use overload
fallback => 0,
"." => \&main::_operator_dot,
".=" => \&main::_operator_dot_equal,
'!=' => \&main::_operator_not_equal,
'==' => \&main::_operator_equal,
;
sub ranges {
# Returns the array of ranges associated with this table.
no overloading;
return $range_list{pack 'J', shift}->ranges;
}
sub add_alias {
# Add a synonym for this table.
return Carp::carp_too_few_args(\@_, 3) if main::DEBUG && @_ < 3;
my $self = shift;
my $name = shift; # The name to add.
my $pointer = shift; # What the alias hash should point to. For
# map tables, this is the parent property;
# for match tables, it is the table itself.
my %args = @_;
my $loose_match = delete $args{'Fuzzy'};
my $ok_as_filename = delete $args{'OK_as_Filename'};
$ok_as_filename = 1 unless defined $ok_as_filename;
# An internal name does not get documented, unless overridden by the
# input; same for making tests for it.
my $status = delete $args{'Status'} || (($name =~ /^_/)
? $INTERNAL_ALIAS
: $NORMAL);
my $make_re_pod_entry = delete $args{'Re_Pod_Entry'}
// (($status ne $INTERNAL_ALIAS)
? (($name =~ /^_/) ? $NO : $YES)
: $NO);
my $ucd = delete $args{'UCD'} // (($name =~ /^_/) ? 0 : 1);
Carp::carp_extra_args(\%args) if main::DEBUG && %args;
# Capitalize the first letter of the alias unless it is one of the CJK
# ones which specifically begins with a lower 'k'. Do this because
# Unicode has varied whether they capitalize first letters or not, and
# have later changed their minds and capitalized them, but not the
# other way around. So do it always and avoid changes from release to
# release
$name = ucfirst($name) unless $name =~ /^k[A-Z]/;
my $addr = do { no overloading; pack 'J', $self; };
# Figure out if should be loosely matched if not already specified.
if (! defined $loose_match) {
# Is a loose_match if isn't null, and doesn't begin with an
# underscore and isn't just a number
if ($name ne ""
&& substr($name, 0, 1) ne '_'
&& $name !~ qr{^[0-9_.+-/]+$})
{
$loose_match = 1;
}
else {
$loose_match = 0;
}
}
# If this alias has already been defined, do nothing.
return if defined $find_table_from_alias{$addr}->{$name};
# That includes if it is standardly equivalent to an existing alias,
# in which case, add this name to the list, so won't have to search
# for it again.
my $standard_name = main::standardize($name);
if (defined $find_table_from_alias{$addr}->{$standard_name}) {
$find_table_from_alias{$addr}->{$name}
= $find_table_from_alias{$addr}->{$standard_name};
return;
}
# Set the index hash for this alias for future quick reference.
$find_table_from_alias{$addr}->{$name} = $pointer;
$find_table_from_alias{$addr}->{$standard_name} = $pointer;
local $to_trace = 0 if main::DEBUG;
trace "adding alias $name to $pointer" if main::DEBUG && $to_trace;
trace "adding alias $standard_name to $pointer" if main::DEBUG && $to_trace;
# Put the new alias at the end of the list of aliases unless the final
# element begins with an underscore (meaning it is for internal perl
# use) or is all numeric, in which case, put the new one before that
# one. This floats any all-numeric or underscore-beginning aliases to
# the end. This is done so that they are listed last in output lists,
# to encourage the user to use a better name (either more descriptive
# or not an internal-only one) instead. This ordering is relied on
# implicitly elsewhere in this program, like in short_name()
my $list = $aliases{$addr};
my $insert_position = (@$list == 0
|| (substr($list->[-1]->name, 0, 1) ne '_'
&& $list->[-1]->name =~ /\D/))
? @$list
: @$list - 1;
splice @$list,
$insert_position,
0,
Alias->new($name, $loose_match, $make_re_pod_entry,
$ok_as_filename, $status, $ucd);
# This name may be shorter than any existing ones, so clear the cache
# of the shortest, so will have to be recalculated.
no overloading;
undef $short_name{pack 'J', $self};
return;
}
sub short_name {
# Returns a name suitable for use as the base part of a file name.
# That is, shorter wins. It can return undef if there is no suitable
# name. The name has all non-essential underscores removed.
# The optional second parameter is a reference to a scalar in which
# this routine will store the length the returned name had before the
# underscores were removed, or undef if the return is undef.
# The shortest name can change if new aliases are added. So using
# this should be deferred until after all these are added. The code
# that does that should clear this one's cache.
# Any name with alphabetics is preferred over an all numeric one, even
# if longer.
my $self = shift;
my $nominal_length_ptr = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $addr = do { no overloading; pack 'J', $self; };
# For efficiency, don't recalculate, but this means that adding new
# aliases could change what the shortest is, so the code that does
# that needs to undef this.
if (defined $short_name{$addr}) {
if ($nominal_length_ptr) {
$$nominal_length_ptr = $nominal_short_name_length{$addr};
}
return $short_name{$addr};
}
# Look at each alias
my $is_last_resort = 0;
my $deprecated_or_discouraged
= qr/ ^ (?: $DEPRECATED | $DISCOURAGED ) $/x;
foreach my $alias ($self->aliases()) {
# Don't use an alias that isn't ok to use for an external name.
next if ! $alias->ok_as_filename;
my $name = main::Standardize($alias->name);
trace $self, $name if main::DEBUG && $to_trace;
# Take the first one, or any non-deprecated non-discouraged one
# over one that is, or a shorter one that isn't numeric. This
# relies on numeric aliases always being last in the array
# returned by aliases(). Any alpha one will have precedence.
if ( ! defined $short_name{$addr}
|| ( $is_last_resort
&& $alias->status !~ $deprecated_or_discouraged)
|| ($name =~ /\D/
&& length($name) < length($short_name{$addr})))
{
# Remove interior underscores.
($short_name{$addr} = $name) =~ s/ (?<= . ) _ (?= . ) //xg;
$nominal_short_name_length{$addr} = length $name;
$is_last_resort = $alias->status =~ $deprecated_or_discouraged;
}
}
# If the short name isn't a nice one, perhaps an equivalent table has
# a better one.
if ( $self->can('children')
&& ( ! defined $short_name{$addr}
|| $short_name{$addr} eq ""
|| $short_name{$addr} eq "_"))
{
my $return;
foreach my $follower ($self->children) { # All equivalents
my $follower_name = $follower->short_name;
next unless defined $follower_name;
# Anything (except undefined) is better than underscore or
# empty
if (! defined $return || $return eq "_") {
$return = $follower_name;
next;
}
# If the new follower name isn't "_" and is shorter than the
# current best one, prefer the new one.
next if $follower_name eq "_";
next if length $follower_name > length $return;
$return = $follower_name;
}
$short_name{$addr} = $return if defined $return;
}
# If no suitable external name return undef
if (! defined $short_name{$addr}) {
$$nominal_length_ptr = undef if $nominal_length_ptr;
return;
}
# Don't allow a null short name.
if ($short_name{$addr} eq "") {
$short_name{$addr} = '_';
$nominal_short_name_length{$addr} = 1;
}
trace $self, $short_name{$addr} if main::DEBUG && $to_trace;
if ($nominal_length_ptr) {
$$nominal_length_ptr = $nominal_short_name_length{$addr};
}
return $short_name{$addr};
}
sub external_name {
# Returns the external name that this table should be known by. This
# is usually the short_name, but not if the short_name is undefined,
# in which case the external_name is arbitrarily set to the
# underscore.
my $self = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $short = $self->short_name;
return $short if defined $short;
return '_';
}
sub add_description { # Adds the parameter as a short description.
my $self = shift;
my $description = shift;
chomp $description;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
no overloading;
push @{$description{pack 'J', $self}}, $description;
return;
}
sub add_note { # Adds the parameter as a short note.
my $self = shift;
my $note = shift;
chomp $note;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
no overloading;
push @{$note{pack 'J', $self}}, $note;
return;
}
sub add_comment { # Adds the parameter as a comment.
return unless $debugging_build;
my $self = shift;
my $comment = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
chomp $comment;
no overloading;
push @{$comment{pack 'J', $self}}, $comment;
return;
}
sub comment {
# Return the current comment for this table. If called in list
# context, returns the array of comments. In scalar, returns a string
# of each element joined together with a period ending each.
my $self = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $addr = do { no overloading; pack 'J', $self; };
my @list = @{$comment{$addr}};
return @list if wantarray;
my $return = "";
foreach my $sentence (@list) {
$return .= '. ' if $return;
$return .= $sentence;
$return =~ s/\.$//;
}
$return .= '.' if $return;
return $return;
}
sub initialize {
# Initialize the table with the argument which is any valid
# initialization for range lists.
my $self = shift;
my $addr = do { no overloading; pack 'J', $self; };
my $initialization = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
# Replace the current range list with a new one of the same exact
# type.
my $class = ref $range_list{$addr};
$range_list{$addr} = $class->new(Owner => $self,
Initialize => $initialization);
return;
}
sub header {
# The header that is output for the table in the file it is written
# in.
my $self = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $return = "";
$return .= $DEVELOPMENT_ONLY if $compare_versions;
$return .= $HEADER;
return $return;
}
sub merge_single_annotation_line ($$$) {
my ($output, $annotation, $annotation_column) = @_;
# This appends an annotation comment, $annotation, to $output,
# starting in or after column $annotation_column, removing any
# pre-existing comment from $output.
$annotation =~ s/^ \s* \# \ //x;
$output =~ s/ \s* ( \# \N* )? \n //x;
$output = Text::Tabs::expand($output);
my $spaces = $annotation_column - length $output;
$spaces = 2 if $spaces < 0; # Have 2 blanks before the comment
$output = sprintf "%s%*s# %s",
$output,
$spaces,
" ",
$annotation;
return Text::Tabs::unexpand $output;
}
sub write {
# Write a representation of the table to its file. It calls several
# functions furnished by sub-classes of this abstract base class to
# handle non-normal ranges, to add stuff before the table, and at its
# end. If the table is to be written so that adjustments are
# required, this does that conversion.
my $self = shift;
my $use_adjustments = shift; # ? output in adjusted format or not
my $suppress_value = shift; # Optional, if the value associated with
# a range equals this one, don't write
# the range
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $addr = do { no overloading; pack 'J', $self; };
my $write_as_invlist = $write_as_invlist{$addr};
# Start with the header
my @HEADER = $self->header;
# Then the comments
push @HEADER, "\n", main::simple_fold($comment{$addr}, '# '), "\n"
if $comment{$addr};
# Things discovered processing the main body of the document may
# affect what gets output before it, therefore pre_body() isn't called
# until after all other processing of the table is done.
# The main body looks like a 'here' document. If there are comments,
# get rid of them when processing it.
my @OUT;
if ($annotate || $output_range_counts) {
# Use the line below in Perls that don't have /r
#push @OUT, 'return join "\n", map { s/\s*#.*//mg; $_ } split "\n", <<\'END\';' . "\n";
push @OUT, "return <<'END' =~ s/\\s*#.*//mgr;\n";
} else {
push @OUT, "return <<'END';\n";
}
if ($range_list{$addr}->is_empty) {
# This is a kludge for empty tables to silence a warning in
# utf8.c, which can't really deal with empty tables, but it can
# deal with a table that matches nothing, as the inverse of 'All'
# does.
push @OUT, "!utf8::All\n";
}
elsif ($self->name eq 'N'
# To save disk space and table cache space, avoid putting out
# binary N tables, but instead create a file which just inverts
# the Y table. Since the file will still exist and occupy a
# certain number of blocks, might as well output the whole
# thing if it all will fit in one block. The number of
# ranges below is an approximate number for that.
&& ($self->property->type == $BINARY
|| $self->property->type == $FORCED_BINARY)
# && $self->property->tables == 2 Can't do this because the
# non-binary properties, like NFDQC aren't specifiable
# by the notation
&& $range_list{$addr}->ranges > 15
&& ! $annotate) # Under --annotate, want to see everything
{
push @OUT, "!utf8::" . $self->property->name . "\n";
}
else {
my $range_size_1 = $range_size_1{$addr};
# To make it more readable, use a minimum indentation
my $comment_indent;
# These are used only in $annotate option
my $format; # e.g. $HEX_ADJUST_FORMAT
my $include_name; # ? Include the character's name in the
# annotation?
my $include_cp; # ? Include its code point
if (! $annotate) {
$comment_indent = ($self->isa('Map_Table'))
? 24
: ($write_as_invlist)
? 8
: 16;
}
else {
$format = $self->format;
# The name of the character is output only for tables that
# don't already include the name in the output.
my $property = $self->property;
$include_name =
! ($property == $perl_charname
|| $property == main::property_ref('Unicode_1_Name')
|| $property == main::property_ref('Name')
|| $property == main::property_ref('Name_Alias')
);
# Don't include the code point in the annotation where all
# lines are a single code point, so it can be easily found in
# the first column
$include_cp = ! $range_size_1;
if (! $self->isa('Map_Table')) {
$comment_indent = ($write_as_invlist) ? 8 : 16;
}
else {
$comment_indent = 16;
# There are just a few short ranges in this table, so no
# need to include the code point in the annotation.
$include_cp = 0 if $format eq $DECOMP_STRING_FORMAT;
# We're trying to get this to look good, as the whole
# point is to make human-readable tables. It is easier to
# read if almost all the annotation comments begin in the
# same column. Map tables have varying width maps, so can
# create a jagged comment appearance. This code does a
# preliminary pass through these tables looking for the
# maximum width map in each, and causing the comments to
# begin just to the right of that. However, if the
# comments begin too far to the right of most lines, it's
# hard to line them up horizontally with their real data.
# Therefore we ignore the longest outliers
my $ignore_longest_X_percent = 2; # Discard longest X%
# Each key in this hash is a width of at least one of the
# maps in the table. Its value is how many lines have
# that width.
my %widths;
# We won't space things further left than one tab stop
# after the rest of the line; initializing it to that
# number saves some work.
my $max_map_width = 8;
# Fill in the %widths hash
my $total = 0;
for my $set ($range_list{$addr}->ranges) {
my $value = $set->value;
# These range types don't appear in the main table
next if $set->type == 0
&& defined $suppress_value
&& $value eq $suppress_value;
next if $set->type == $MULTI_CP
|| $set->type == $NULL;
# Include 2 spaces before the beginning of the
# comment
my $this_width = length($value) + 2;
# Ranges of the remaining non-zero types usually
# occupy just one line (maybe occasionally two, but
# this doesn't have to be dead accurate). This is
# because these ranges are like "unassigned code
# points"
my $count = ($set->type != 0)
? 1
: $set->end - $set->start + 1;
$widths{$this_width} += $count;
$total += $count;
$max_map_width = $this_width
if $max_map_width < $this_width;
}
# If the widest map gives us less than two tab stops
# worth, just take it as-is.
if ($max_map_width > 16) {
# Otherwise go through %widths until we have included
# the desired percentage of lines in the whole table.
my $running_total = 0;
foreach my $width (sort { $a <=> $b } keys %widths)
{
$running_total += $widths{$width};
use integer;
if ($running_total * 100 / $total
>= 100 - $ignore_longest_X_percent)
{
$max_map_width = $width;
last;
}
}
}
$comment_indent += $max_map_width;
}
}
# Values for previous time through the loop. Initialize to
# something that won't be adjacent to the first iteration;
# only $previous_end matters for that.
my $previous_start;
my $previous_end = -2;
my $previous_value;
# Values for next time through the portion of the loop that splits
# the range. 0 in $next_start means there is no remaining portion
# to deal with.
my $next_start = 0;
my $next_end;
my $next_value;
my $offset = 0;
my $invlist_count = 0;
my $output_value_in_hex = $self->isa('Map_Table')
&& ($self->format eq $HEX_ADJUST_FORMAT
|| $self->to_output_map == $EXTERNAL_MAP);
# Use leading zeroes just for files whose format should not be
# changed from what it has been. Otherwise, they just take up
# space and time to process.
my $hex_format = ($self->isa('Map_Table')
&& $self->to_output_map == $EXTERNAL_MAP)
? "%04X"
: "%X";
# The values for some of these tables are stored in mktables as
# hex strings. Normally, these are just output as strings without
# change, but when we are doing adjustments, we have to operate on
# these numerically, so we convert those to decimal to do that,
# and back to hex for output
my $convert_map_to_from_hex = 0;
my $output_map_in_hex = 0;
if ($self->isa('Map_Table')) {
$convert_map_to_from_hex
= ($use_adjustments && $self->format eq $HEX_ADJUST_FORMAT)
|| ($annotate && $self->format eq $HEX_FORMAT);
$output_map_in_hex = $convert_map_to_from_hex
|| $self->format eq $HEX_FORMAT;
}
# To store any annotations about the characters.
my @annotation;
# Output each range as part of the here document.
RANGE:
for my $set ($range_list{$addr}->ranges) {
if ($set->type != 0) {
$self->handle_special_range($set);
next RANGE;
}
my $start = $set->start;
my $end = $set->end;
my $value = $set->value;
# Don't output ranges whose value is the one to suppress
next RANGE if defined $suppress_value
&& $value eq $suppress_value;
$value = CORE::hex $value if $convert_map_to_from_hex;
{ # This bare block encloses the scope where we may need to
# 'redo' to. Consider a table that is to be written out
# using single item ranges. This is given in the
# $range_size_1 boolean. To accomplish this, we split the
# range each time through the loop into two portions, the
# first item, and the rest. We handle that first item
# this time in the loop, and 'redo' to repeat the process
# for the rest of the range.
#
# We may also have to do it, with other special handling,
# if the table has adjustments. Consider the table that
# contains the lowercasing maps. mktables stores the
# ASCII range ones as 26 ranges:
# ord('A') => ord('a'), .. ord('Z') => ord('z')
# For compactness, the table that gets written has this as
# just one range
# ( ord('A') .. ord('Z') ) => ord('a')
# and the software that reads the tables is smart enough
# to "connect the dots". This change is accomplished in
# this loop by looking to see if the current iteration
# fits the paradigm of the previous iteration, and if so,
# we merge them by replacing the final output item with
# the merged data. Repeated 25 times, this gets A-Z. But
# we also have to make sure we don't screw up cases where
# we have internally stored
# ( 0x1C4 .. 0x1C6 ) => 0x1C5
# This single internal range has to be output as 3 ranges,
# which is done by splitting, like we do for $range_size_1
# tables. (There are very few of such ranges that need to
# be split, so the gain of doing the combining of other
# ranges far outweighs the splitting of these.) The
# values to use for the redo at the end of this block are
# set up just below in the scalars whose names begin with
# '$next_'.
if (($use_adjustments || $range_size_1) && $end != $start)
{
$next_start = $start + 1;
$next_end = $end;
$next_value = $value;
$end = $start;
}
if ($use_adjustments && ! $range_size_1) {
# If this range is adjacent to the previous one, and
# the values in each are integers that are also
# adjacent (differ by 1), then this range really
# extends the previous one that is already in element
# $OUT[-1]. So we pop that element, and pretend that
# the range starts with whatever it started with.
# $offset is incremented by 1 each time so that it
# gives the current offset from the first element in
# the accumulating range, and we keep in $value the
# value of that first element.
if ($start == $previous_end + 1
&& $value =~ /^ -? \d+ $/xa
&& $previous_value =~ /^ -? \d+ $/xa
&& ($value == ($previous_value + ++$offset)))
{
pop @OUT;
$start = $previous_start;
$value = $previous_value;
}
else {
$offset = 0;
if (@annotation == 1) {
$OUT[-1] = merge_single_annotation_line(
$OUT[-1], $annotation[0], $comment_indent);
}
else {
push @OUT, @annotation;
}
}
undef @annotation;
# Save the current values for the next time through
# the loop.
$previous_start = $start;
$previous_end = $end;
$previous_value = $value;
}
if ($write_as_invlist) {
if ( $previous_end > 0
&& $output_range_counts{$addr})
{
my $complement_count = $start - $previous_end - 1;
if ($complement_count > 1) {
$OUT[-1] = merge_single_annotation_line(
$OUT[-1],
"#"
. (" " x 17)
. "["
. main::clarify_code_point_count(
$complement_count)
. "] in complement\n",
$comment_indent);
}
}
# Inversion list format has a single number per line,
# the starting code point of a range that matches the
# property
push @OUT, $start, "\n";
$invlist_count++;
# Add a comment with the size of the range, if
# requested.
if ($output_range_counts{$addr}) {
$OUT[-1] = merge_single_annotation_line(
$OUT[-1],
"# ["
. main::clarify_code_point_count($end - $start + 1)
. "]\n",
$comment_indent);
}
}
elsif ($start != $end) { # If there is a range
if ($end == $MAX_WORKING_CODEPOINT) {
push @OUT, sprintf "$hex_format\t$hex_format",
$start,
$MAX_PLATFORM_CODEPOINT;
}
else {
push @OUT, sprintf "$hex_format\t$hex_format",
$start, $end;
}
if (length $value) {
if ($convert_map_to_from_hex) {
$OUT[-1] .= sprintf "\t$hex_format\n", $value;
}
else {
$OUT[-1] .= "\t$value\n";
}
}
# Add a comment with the size of the range, if
# requested.
if ($output_range_counts{$addr}) {
$OUT[-1] = merge_single_annotation_line(
$OUT[-1],
"# ["
. main::clarify_code_point_count($end - $start + 1)
. "]\n",
$comment_indent);
}
}
else { # Here to output a single code point per line.
# Use any passed in subroutine to output.
if (ref $range_size_1 eq 'CODE') {
for my $i ($start .. $end) {
push @OUT, &{$range_size_1}($i, $value);
}
}
else {
# Here, caller is ok with default output.
for (my $i = $start; $i <= $end; $i++) {
if ($convert_map_to_from_hex) {
push @OUT,
sprintf "$hex_format\t\t$hex_format\n",
$i, $value;
}
else {
push @OUT, sprintf $hex_format, $i;
$OUT[-1] .= "\t\t$value" if $value ne "";
$OUT[-1] .= "\n";
}
}
}
}
if ($annotate) {
for (my $i = $start; $i <= $end; $i++) {
my $annotation = "";
# Get character information if don't have it already
main::populate_char_info($i)
if ! defined $viacode[$i];
my $type = $annotate_char_type[$i];
# Figure out if should output the next code points
# as part of a range or not. If this is not in an
# annotation range, then won't output as a range,
# so returns $i. Otherwise use the end of the
# annotation range, but no further than the
# maximum possible end point of the loop.
my $range_end =
$range_size_1
? $start
: main::min(
$annotate_ranges->value_of($i) || $i,
$end);
# Use a range if it is a range, and either is one
# of the special annotation ranges, or the range
# is at most 3 long. This last case causes the
# algorithmically named code points to be output
# individually in spans of at most 3, as they are
# the ones whose $type is > 0.
if ($range_end != $i
&& ( $type < 0 || $range_end - $i > 2))
{
# Here is to output a range. We don't allow a
# caller-specified output format--just use the
# standard one.
my $range_name = $viacode[$i];
# For the code points which end in their hex
# value, we eliminate that from the output
# annotation, and capitalize only the first
# letter of each word.
if ($type == $CP_IN_NAME) {
my $hex = sprintf $hex_format, $i;
$range_name =~ s/-$hex$//;
my @words = split " ", $range_name;
for my $word (@words) {
$word =
ucfirst(lc($word)) if $word ne 'CJK';
}
$range_name = join " ", @words;
}
elsif ($type == $HANGUL_SYLLABLE) {
$range_name = "Hangul Syllable";
}
# If the annotation would just repeat what's
# already being output as the range, skip it.
# (When an inversion list is being written, it
# isn't a repeat, as that always is in
# decimal)
if ( $write_as_invlist
|| $i != $start
|| $range_end < $end)
{
if ($range_end < $MAX_WORKING_CODEPOINT)
{
$annotation = sprintf "%04X..%04X",
$i, $range_end;
}
else {
$annotation = sprintf "%04X..INFINITY",
$i;
}
}
else { # Indent if not displaying code points
$annotation = " " x 4;
}
if ($range_name) {
$annotation .= " $age[$i]" if $age[$i];
$annotation .= " $range_name";
}
# Include the number of code points in the
# range
my $count =
main::clarify_code_point_count($range_end - $i + 1);
$annotation .= " [$count]\n";
# Skip to the end of the range
$i = $range_end;
}
else { # Not in a range.
my $comment = "";
# When outputting the names of each character,
# use the character itself if printable
$comment .= "'" . main::display_chr($i) . "' "
if $printable[$i];
my $output_value = $value;
# Determine the annotation
if ($format eq $DECOMP_STRING_FORMAT) {
# This is very specialized, with the type
# of decomposition beginning the line
# enclosed in <...>, and the code points
# that the code point decomposes to
# separated by blanks. Create two
# strings, one of the printable
# characters, and one of their official
# names.
(my $map = $output_value)
=~ s/ \ * < .*? > \ +//x;
my $tostr = "";
my $to_name = "";
my $to_chr = "";
foreach my $to (split " ", $map) {
$to = CORE::hex $to;
$to_name .= " + " if $to_name;
$to_chr .= main::display_chr($to);
main::populate_char_info($to)
if ! defined $viacode[$to];
$to_name .= $viacode[$to];
}
$comment .=
"=> '$to_chr'; $viacode[$i] => $to_name";
}
else {
$output_value += $i - $start
if $use_adjustments
# Don't try to adjust a
# non-integer
&& $output_value !~ /[-\D]/;
if ($output_map_in_hex) {
main::populate_char_info($output_value)
if ! defined $viacode[$output_value];
$comment .= " => '"
. main::display_chr($output_value)
. "'; " if $printable[$output_value];
}
if ($include_name && $viacode[$i]) {
$comment .= " " if $comment;
$comment .= $viacode[$i];
}
if ($output_map_in_hex) {
$comment .=
" => $viacode[$output_value]"
if $viacode[$output_value];
$output_value = sprintf($hex_format,
$output_value);
}
}
if ($include_cp) {
$annotation = sprintf "%04X %s", $i, $age[$i];
if ($use_adjustments) {
$annotation .= " => $output_value";
}
}
if ($comment ne "") {
$annotation .= " " if $annotation ne "";
$annotation .= $comment;
}
$annotation .= "\n" if $annotation ne "";
}
if ($annotation ne "") {
push @annotation, (" " x $comment_indent)
. "# $annotation";
}
}
# If not adjusting, we don't have to go through the
# loop again to know that the annotation comes next
# in the output.
if (! $use_adjustments) {
if (@annotation == 1) {
$OUT[-1] = merge_single_annotation_line(
$OUT[-1], $annotation[0], $comment_indent);
}
else {
push @OUT, map { Text::Tabs::unexpand $_ }
@annotation;
}
undef @annotation;
}
}
# Add the beginning of the range that doesn't match the
# property, except if the just added match range extends
# to infinity. We do this after any annotations for the
# match range.
if ($write_as_invlist && $end < $MAX_WORKING_CODEPOINT) {
push @OUT, $end + 1, "\n";
$invlist_count++;
}
# If we split the range, set up so the next time through
# we get the remainder, and redo.
if ($next_start) {
$start = $next_start;
$end = $next_end;
$value = $next_value;
$next_start = 0;
redo;
}
}
} # End of loop through all the table's ranges
push @OUT, @annotation; # Add orphaned annotation, if any
splice @OUT, 1, 0, "V$invlist_count\n" if $invlist_count;
}
# Add anything that goes after the main body, but within the here
# document,
my $append_to_body = $self->append_to_body;
push @OUT, $append_to_body if $append_to_body;
# And finish the here document.
push @OUT, "END\n";
# Done with the main portion of the body. Can now figure out what
# should appear before it in the file.
my $pre_body = $self->pre_body;
push @HEADER, $pre_body, "\n" if $pre_body;
# All these files should have a .pl suffix added to them.
my @file_with_pl = @{$file_path{$addr}};
$file_with_pl[-1] .= '.pl';
main::write(\@file_with_pl,
$annotate, # utf8 iff annotating
\@HEADER,
\@OUT);
return;
}
sub set_status { # Set the table's status
my $self = shift;
my $status = shift; # The status enum value
my $info = shift; # Any message associated with it.
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $addr = do { no overloading; pack 'J', $self; };
$status{$addr} = $status;
$status_info{$addr} = $info;
return;
}
sub set_fate { # Set the fate of a table
my $self = shift;
my $fate = shift;
my $reason = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $addr = do { no overloading; pack 'J', $self; };
return if $fate{$addr} == $fate; # If no-op
# Can only change the ordinary fate, except if going to $MAP_PROXIED
return if $fate{$addr} != $ORDINARY && $fate != $MAP_PROXIED;
$fate{$addr} = $fate;
# Don't document anything to do with a non-normal fated table
if ($fate != $ORDINARY) {
my $put_in_pod = ($fate == $MAP_PROXIED) ? 1 : 0;
foreach my $alias ($self->aliases) {
$alias->set_ucd($put_in_pod);
# MAP_PROXIED doesn't affect the match tables
next if $fate == $MAP_PROXIED;
$alias->set_make_re_pod_entry($put_in_pod);
}
}
# Save the reason for suppression for output
if ($fate >= $SUPPRESSED) {
$reason = "" unless defined $reason;
$why_suppressed{$complete_name{$addr}} = $reason;
}
return;
}
sub lock {
# Don't allow changes to the table from now on. This stores a stack
# trace of where it was called, so that later attempts to modify it
# can immediately show where it got locked.
my $self = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $addr = do { no overloading; pack 'J', $self; };
$locked{$addr} = "";
my $line = (caller(0))[2];
my $i = 1;
# Accumulate the stack trace
while (1) {
my ($pkg, $file, $caller_line, $caller) = caller $i++;
last unless defined $caller;
$locked{$addr} .= " called from $caller() at line $line\n";
$line = $caller_line;
}
$locked{$addr} .= " called from main at line $line\n";
return;
}
sub carp_if_locked {
# Return whether a table is locked or not, and, by the way, complain
# if is locked
my $self = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $addr = do { no overloading; pack 'J', $self; };
return 0 if ! $locked{$addr};
Carp::my_carp_bug("Can't modify a locked table. Stack trace of locking:\n$locked{$addr}\n\n");
return 1;
}
sub set_file_path { # Set the final directory path for this table
my $self = shift;
# Rest of parameters passed on
no overloading;
@{$file_path{pack 'J', $self}} = @_;
return
}
# Accessors for the range list stored in this table. First for
# unconditional
for my $sub (qw(
containing_range
contains
count
each_range
hash
is_empty
matches_identically_to
max
min
range_count
reset_each_range
type_of
value_of
))
{
no strict "refs";
*$sub = sub {
use strict "refs";
my $self = shift;
return $self->_range_list->$sub(@_);
}
}
# Then for ones that should fail if locked
for my $sub (qw(
delete_range
))
{
no strict "refs";
*$sub = sub {
use strict "refs";
my $self = shift;
return if $self->carp_if_locked;
no overloading;
return $self->_range_list->$sub(@_);
}
}
} # End closure
package Map_Table;
use parent '-norequire', '_Base_Table';
# A Map Table is a table that contains the mappings from code points to
# values. There are two weird cases:
# 1) Anomalous entries are ones that aren't maps of ranges of code points, but
# are written in the table's file at the end of the table nonetheless. It
# requires specially constructed code to handle these; utf8.c can not read
# these in, so they should not go in $map_directory. As of this writing,
# the only case that these happen is for named sequences used in
# charnames.pm. But this code doesn't enforce any syntax on these, so
# something else could come along that uses it.
# 2) Specials are anything that doesn't fit syntactically into the body of the
# table. The ranges for these have a map type of non-zero. The code below
# knows about and handles each possible type. In most cases, these are
# written as part of the header.
#
# A map table deliberately can't be manipulated at will unlike match tables.
# This is because of the ambiguities having to do with what to do with
# overlapping code points. And there just isn't a need for those things;
# what one wants to do is just query, add, replace, or delete mappings, plus
# write the final result.
# However, there is a method to get the list of possible ranges that aren't in
# this table to use for defaulting missing code point mappings. And,
# map_add_or_replace_non_nulls() does allow one to add another table to this
# one, but it is clearly very specialized, and defined that the other's
# non-null values replace this one's if there is any overlap.
sub trace { return main::trace(@_); }
{ # Closure
main::setup_package();
my %default_map;
# Many input files omit some entries; this gives what the mapping for the
# missing entries should be
main::set_access('default_map', \%default_map, 'r');
my %anomalous_entries;
# Things that go in the body of the table which don't fit the normal
# scheme of things, like having a range. Not much can be done with these
# once there except to output them. This was created to handle named
# sequences.
main::set_access('anomalous_entry', \%anomalous_entries, 'a');
main::set_access('anomalous_entries', # Append singular, read plural
\%anomalous_entries,
'readable_array');
my %replacement_property;
# Certain files are unused by Perl itself, and are kept only for backwards
# compatibility for programs that used them before Unicode::UCD existed.
# These are termed legacy properties. At some point they may be removed,
# but for now mark them as legacy. If non empty, this is the name of the
# property to use instead (i.e., the modern equivalent).
main::set_access('replacement_property', \%replacement_property, 'r');
my %to_output_map;
# Enum as to whether or not to write out this map table, and how:
# 0 don't output
# $EXTERNAL_MAP means its existence is noted in the documentation, and
# it should not be removed nor its format changed. This
# is done for those files that have traditionally been
# output. Maps of legacy-only properties default to
# this.
# $INTERNAL_MAP means Perl reserves the right to do anything it wants
# with this file
# $OUTPUT_ADJUSTED means that it is an $INTERNAL_MAP, and instead of
# outputting the actual mappings as-is, we adjust things
# to create a much more compact table. Only those few
# tables where the mapping is convertible at least to an
# integer and compacting makes a big difference should
# have this. Hence, the default is to not do this
# unless the table's default mapping is to $CODE_POINT,
# and the range size is not 1.
main::set_access('to_output_map', \%to_output_map, 's');
sub new {
my $class = shift;
my $name = shift;
my %args = @_;
# Optional initialization data for the table.
my $initialize = delete $args{'Initialize'};
my $default_map = delete $args{'Default_Map'};
my $property = delete $args{'_Property'};
my $full_name = delete $args{'Full_Name'};
my $replacement_property = delete $args{'Replacement_Property'} // "";
my $to_output_map = delete $args{'To_Output_Map'};
# Rest of parameters passed on; legacy properties have several common
# other attributes
if ($replacement_property) {
$args{"Fate"} = $LEGACY_ONLY;
$args{"Range_Size_1"} = 1;
$args{"Perl_Extension"} = 1;
$args{"UCD"} = 0;
}
my $range_list = Range_Map->new(Owner => $property);
my $self = $class->SUPER::new(
Name => $name,
Complete_Name => $full_name,
Full_Name => $full_name,
_Property => $property,
_Range_List => $range_list,
Write_As_Invlist => 0,
%args);
my $addr = do { no overloading; pack 'J', $self; };
$anomalous_entries{$addr} = [];
$default_map{$addr} = $default_map;
$replacement_property{$addr} = $replacement_property;
$to_output_map = $EXTERNAL_MAP if ! defined $to_output_map
&& $replacement_property;
$to_output_map{$addr} = $to_output_map;
$self->initialize($initialize) if defined $initialize;
return $self;
}
use overload
fallback => 0,
qw("") => "_operator_stringify",
;
sub _operator_stringify {
my $self = shift;
my $name = $self->property->full_name;
$name = '""' if $name eq "";
return "Map table for Property '$name'";
}
sub add_alias {
# Add a synonym for this table (which means the property itself)
my $self = shift;
my $name = shift;
# Rest of parameters passed on.
$self->SUPER::add_alias($name, $self->property, @_);
return;
}
sub add_map {
# Add a range of code points to the list of specially-handled code
# points. $MULTI_CP is assumed if the type of special is not passed
# in.
my $self = shift;
my $lower = shift;
my $upper = shift;
my $string = shift;
my %args = @_;
my $type = delete $args{'Type'} || 0;
# Rest of parameters passed on
# Can't change the table if locked.
return if $self->carp_if_locked;
my $addr = do { no overloading; pack 'J', $self; };
$self->_range_list->add_map($lower, $upper,
$string,
@_,
Type => $type);
return;
}
sub append_to_body {
# Adds to the written HERE document of the table's body any anomalous
# entries in the table..
my $self = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $addr = do { no overloading; pack 'J', $self; };
return "" unless @{$anomalous_entries{$addr}};
return join("\n", @{$anomalous_entries{$addr}}) . "\n";
}
sub map_add_or_replace_non_nulls {
# This adds the mappings in the table $other to $self. Non-null
# mappings from $other override those in $self. It essentially merges
# the two tables, with the second having priority except for null
# mappings.
my $self = shift;
my $other = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
return if $self->carp_if_locked;
if (! $other->isa(__PACKAGE__)) {
Carp::my_carp_bug("$other should be a "
. __PACKAGE__
. ". Not a '"
. ref($other)
. "'. Not added;");
return;
}
my $addr = do { no overloading; pack 'J', $self; };
my $other_addr = do { no overloading; pack 'J', $other; };
local $to_trace = 0 if main::DEBUG;
my $self_range_list = $self->_range_list;
my $other_range_list = $other->_range_list;
foreach my $range ($other_range_list->ranges) {
my $value = $range->value;
next if $value eq "";
$self_range_list->_add_delete('+',
$range->start,
$range->end,
$value,
Type => $range->type,
Replace => $UNCONDITIONALLY);
}
return;
}
sub set_default_map {
# Define what code points that are missing from the input files should
# map to. The optional second parameter 'full_name' indicates to
# force using the full name of the map instead of its standard name.
my $self = shift;
my $map = shift;
my $use_full_name = shift // 0;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
if ($use_full_name && $use_full_name ne 'full_name') {
Carp::my_carp_bug("Second parameter to set_default_map() if"
. " present, must be 'full_name'");
}
my $addr = do { no overloading; pack 'J', $self; };
# Convert the input to the standard equivalent, if any (won't have any
# for $STRING properties)
my $standard = $self->property->table($map);
if (defined $standard) {
$map = ($use_full_name)
? $standard->full_name
: $standard->name;
}
# Warn if there already is a non-equivalent default map for this
# property. Note that a default map can be a ref, which means that
# what it actually means is delayed until later in the program, and it
# IS permissible to override it here without a message.
my $default_map = $default_map{$addr};
if (defined $default_map
&& ! ref($default_map)
&& $default_map ne $map
&& main::Standardize($map) ne $default_map)
{
my $property = $self->property;
my $map_table = $property->table($map);
my $default_table = $property->table($default_map);
if (defined $map_table
&& defined $default_table
&& $map_table != $default_table)
{
Carp::my_carp("Changing the default mapping for "
. $property
. " from $default_map to $map'");
}
}
$default_map{$addr} = $map;
# Don't also create any missing table for this map at this point,
# because if we did, it could get done before the main table add is
# done for PropValueAliases.txt; instead the caller will have to make
# sure it exists, if desired.
return;
}
sub to_output_map {
# Returns boolean: should we write this map table?
my $self = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $addr = do { no overloading; pack 'J', $self; };
# If overridden, use that
return $to_output_map{$addr} if defined $to_output_map{$addr};
my $full_name = $self->full_name;
return $global_to_output_map{$full_name}
if defined $global_to_output_map{$full_name};
# If table says to output, do so; if says to suppress it, do so.
my $fate = $self->fate;
return $INTERNAL_MAP if $fate == $INTERNAL_ONLY;
return $EXTERNAL_MAP if grep { $_ eq $full_name } @output_mapped_properties;
return 0 if $fate == $SUPPRESSED || $fate == $MAP_PROXIED;
my $type = $self->property->type;
# Don't want to output binary map tables even for debugging.
return 0 if $type == $BINARY;
# But do want to output string ones. All the ones that remain to
# be dealt with (i.e. which haven't explicitly been set to external)
# are for internal Perl use only. The default for those that map to
# $CODE_POINT and haven't been restricted to a single element range
# is to use the adjusted form.
if ($type == $STRING) {
return $INTERNAL_MAP if $self->range_size_1
|| $default_map{$addr} ne $CODE_POINT;
return $OUTPUT_ADJUSTED;
}
# Otherwise is an $ENUM, do output it, for Perl's purposes
return $INTERNAL_MAP;
}
sub inverse_list {
# Returns a Range_List that is gaps of the current table. That is,
# the inversion
my $self = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $current = Range_List->new(Initialize => $self->_range_list,
Owner => $self->property);
return ~ $current;
}
sub header {
my $self = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $return = $self->SUPER::header();
if ($self->to_output_map >= $INTERNAL_MAP) {
$return .= $INTERNAL_ONLY_HEADER;
}
else {
my $property_name = $self->property->replacement_property;
# The legacy-only properties were gotten above; but there are some
# other properties whose files are in current use that have fixed
# formats.
$property_name = $self->property->full_name unless $property_name;
$return .= <<END;
# !!!!!!! IT IS DEPRECATED TO USE THIS FILE !!!!!!!
# This file is for internal use by core Perl only. It is retained for
# backwards compatibility with applications that may have come to rely on it,
# but its format and even its name or existence are subject to change without
# notice in a future Perl version. Don't use it directly. Instead, its
# contents are now retrievable through a stable API in the Unicode::UCD
# module: Unicode::UCD::prop_invmap('$property_name') (Values for individual
# code points can be retrieved via Unicode::UCD::charprop());
END
}
return $return;
}
sub set_final_comment {
# Just before output, create the comment that heads the file
# containing this table.
return unless $debugging_build;
my $self = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
# No sense generating a comment if aren't going to write it out.
return if ! $self->to_output_map;
my $addr = do { no overloading; pack 'J', $self; };
my $property = $self->property;
# Get all the possible names for this property. Don't use any that
# aren't ok for use in a file name, etc. This is perhaps causing that
# flag to do double duty, and may have to be changed in the future to
# have our own flag for just this purpose; but it works now to exclude
# Perl generated synonyms from the lists for properties, where the
# name is always the proper Unicode one.
my @property_aliases = grep { $_->ok_as_filename } $self->aliases;
my $count = $self->count;
my $default_map = $default_map{$addr};
# The ranges that map to the default aren't output, so subtract that
# to get those actually output. A property with matching tables
# already has the information calculated.
if ($property->type != $STRING && $property->type != $FORCED_BINARY) {
$count -= $property->table($default_map)->count;
}
elsif (defined $default_map) {
# But for $STRING properties, must calculate now. Subtract the
# count from each range that maps to the default.
foreach my $range ($self->_range_list->ranges) {
if ($range->value eq $default_map) {
$count -= $range->end +1 - $range->start;
}
}
}
# Get a string version of $count with underscores in large numbers,
# for clarity.
my $string_count = main::clarify_code_point_count($count);
my $code_points = ($count == 1)
? 'single code point'
: "$string_count code points";
my $mapping;
my $these_mappings;
my $are;
if (@property_aliases <= 1) {
$mapping = 'mapping';
$these_mappings = 'this mapping';
$are = 'is'
}
else {
$mapping = 'synonymous mappings';
$these_mappings = 'these mappings';
$are = 'are'
}
my $cp;
if ($count >= $MAX_UNICODE_CODEPOINTS) {
$cp = "any code point in Unicode Version $string_version";
}
else {
my $map_to;
if ($default_map eq "") {
$map_to = 'the null string';
}
elsif ($default_map eq $CODE_POINT) {
$map_to = "itself";
}
else {
$map_to = "'$default_map'";
}
if ($count == 1) {
$cp = "the single code point";
}
else {
$cp = "one of the $code_points";
}
$cp .= " in Unicode Version $unicode_version for which the mapping is not to $map_to";
}
my $comment = "";
my $status = $self->status;
if ($status ne $NORMAL) {
my $warn = uc $status_past_participles{$status};
$comment .= <<END;
!!!!!!! $warn !!!!!!!!!!!!!!!!!!!
All property or property=value combinations contained in this file are $warn.
See $unicode_reference_url for what this means.
END
}
$comment .= "This file returns the $mapping:\n";
my $ucd_accessible_name = "";
my $has_underscore_name = 0;
my $full_name = $self->property->full_name;
for my $i (0 .. @property_aliases - 1) {
my $name = $property_aliases[$i]->name;
$has_underscore_name = 1 if $name =~ /^_/;
$comment .= sprintf("%-8s%s\n", " ", $name . '(cp)');
if ($property_aliases[$i]->ucd) {
if ($name eq $full_name) {
$ucd_accessible_name = $full_name;
}
elsif (! $ucd_accessible_name) {
$ucd_accessible_name = $name;
}
}
}
$comment .= "\nwhere 'cp' is $cp.";
if ($ucd_accessible_name) {
$comment .= " Note that $these_mappings";
if ($has_underscore_name) {
$comment .= " (except for the one(s) that begin with an underscore)";
}
$comment .= " $are accessible via the functions prop_invmap('$full_name') or charprop() in Unicode::UCD";
}
# And append any commentary already set from the actual property.
$comment .= "\n\n" . $self->comment if $self->comment;
if ($self->description) {
$comment .= "\n\n" . join " ", $self->description;
}
if ($self->note) {
$comment .= "\n\n" . join " ", $self->note;
}
$comment .= "\n";
if (! $self->perl_extension) {
$comment .= <<END;
For information about what this property really means, see:
$unicode_reference_url
END
}
if ($count) { # Format differs for empty table
$comment.= "\nThe format of the ";
if ($self->range_size_1) {
$comment.= <<END;
main body of lines of this file is: CODE_POINT\\t\\tMAPPING where CODE_POINT
is in hex; MAPPING is what CODE_POINT maps to.
END
}
else {
# There are tables which end up only having one element per
# range, but it is not worth keeping track of for making just
# this comment a little better.
$comment .= <<END;
non-comment portions of the main body of lines of this file is:
START\\tSTOP\\tMAPPING where START is the starting code point of the
range, in hex; STOP is the ending point, or if omitted, the range has just one
code point; MAPPING is what each code point between START and STOP maps to.
END
if ($self->output_range_counts) {
$comment .= <<END;
Numbers in comments in [brackets] indicate how many code points are in the
range (omitted when the range is a single code point or if the mapping is to
the null string).
END
}
}
}
$self->set_comment(main::join_lines($comment));
return;
}
my %swash_keys; # Makes sure don't duplicate swash names.
# The remaining variables are temporaries used while writing each table,
# to output special ranges.
my @multi_code_point_maps; # Map is to more than one code point.
sub handle_special_range {
# Called in the middle of write when it finds a range it doesn't know
# how to handle.
my $self = shift;
my $range = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $addr = do { no overloading; pack 'J', $self; };
my $type = $range->type;
my $low = $range->start;
my $high = $range->end;
my $map = $range->value;
# No need to output the range if it maps to the default.
return if $map eq $default_map{$addr};
my $property = $self->property;
# Switch based on the map type...
if ($type == $HANGUL_SYLLABLE) {
# These are entirely algorithmically determinable based on
# some constants furnished by Unicode; for now, just set a
# flag to indicate that have them. After everything is figured
# out, we will output the code that does the algorithm. (Don't
# output them if not needed because we are suppressing this
# property.)
$has_hangul_syllables = 1 if $property->to_output_map;
}
elsif ($type == $CP_IN_NAME) {
# Code points whose name ends in their code point are also
# algorithmically determinable, but need information about the map
# to do so. Both the map and its inverse are stored in data
# structures output in the file. They are stored in the mean time
# in global lists The lists will be written out later into Name.pm,
# which is created only if needed. In order to prevent duplicates
# in the list, only add to them for one property, should multiple
# ones need them.
if ($needing_code_points_ending_in_code_point == 0) {
$needing_code_points_ending_in_code_point = $property;
}
if ($property == $needing_code_points_ending_in_code_point) {
push @{$names_ending_in_code_point{$map}->{'low'}}, $low;
push @{$names_ending_in_code_point{$map}->{'high'}}, $high;
my $squeezed = $map =~ s/[-\s]+//gr;
push @{$loose_names_ending_in_code_point{$squeezed}->{'low'}},
$low;
push @{$loose_names_ending_in_code_point{$squeezed}->{'high'}},
$high;
push @code_points_ending_in_code_point, { low => $low,
high => $high,
name => $map
};
}
}
elsif ($range->type == $MULTI_CP || $range->type == $NULL) {
# Multi-code point maps and null string maps have an entry
# for each code point in the range. They use the same
# output format.
for my $code_point ($low .. $high) {
# The pack() below can't cope with surrogates. XXX This may
# no longer be true
if ($code_point >= 0xD800 && $code_point <= 0xDFFF) {
Carp::my_carp("Surrogate code point '$code_point' in mapping to '$map' in $self. No map created");
next;
}
# Generate the hash entries for these in the form that
# utf8.c understands.
my $tostr = "";
my $to_name = "";
my $to_chr = "";
foreach my $to (split " ", $map) {
if ($to !~ /^$code_point_re$/) {
Carp::my_carp("Illegal code point '$to' in mapping '$map' from $code_point in $self. No map created");
next;
}
$tostr .= sprintf "\\x{%s}", $to;
$to = CORE::hex $to;
if ($annotate) {
$to_name .= " + " if $to_name;
$to_chr .= main::display_chr($to);
main::populate_char_info($to)
if ! defined $viacode[$to];
$to_name .= $viacode[$to];
}
}
# The unpack yields a list of the bytes that comprise the
# UTF-8 of $code_point, which are each placed in \xZZ format
# and output in the %s to map to $tostr, so the result looks
# like:
# "\xC4\xB0" => "\x{0069}\x{0307}",
my $utf8 = sprintf(qq["%s" => "$tostr",],
join("", map { sprintf "\\x%02X", $_ }
unpack("U0C*", chr $code_point)));
# Add a comment so that a human reader can more easily
# see what's going on.
push @multi_code_point_maps,
sprintf("%-45s # U+%04X", $utf8, $code_point);
if (! $annotate) {
$multi_code_point_maps[-1] .= " => $map";
}
else {
main::populate_char_info($code_point)
if ! defined $viacode[$code_point];
$multi_code_point_maps[-1] .= " '"
. main::display_chr($code_point)
. "' => '$to_chr'; $viacode[$code_point] => $to_name";
}
}
}
else {
Carp::my_carp("Unrecognized map type '$range->type' in '$range' in $self. Not written");
}
return;
}
sub pre_body {
# Returns the string that should be output in the file before the main
# body of this table. It isn't called until the main body is
# calculated, saving a pass. The string includes some hash entries
# identifying the format of the body, and what the single value should
# be for all ranges missing from it. It also includes any code points
# which have map_types that don't go in the main table.
my $self = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $addr = do { no overloading; pack 'J', $self; };
my $name = $self->property->swash_name;
# Currently there is nothing in the pre_body unless a swash is being
# generated.
return unless defined $name;
if (defined $swash_keys{$name}) {
Carp::my_carp(main::join_lines(<<END
Already created a swash name '$name' for $swash_keys{$name}. This means that
the same name desired for $self shouldn't be used. Bad News. This must be
fixed before production use, but proceeding anyway
END
));
}
$swash_keys{$name} = "$self";
my $pre_body = "";
# Here we assume we were called after have gone through the whole
# file. If we actually generated anything for each map type, add its
# respective header and trailer
my $specials_name = "";
if (@multi_code_point_maps) {
$specials_name = "utf8::ToSpec$name";
$pre_body .= <<END;
# Some code points require special handling because their mappings are each to
# multiple code points. These do not appear in the main body, but are defined
# in the hash below.
# Each key is the string of N bytes that together make up the UTF-8 encoding
# for the code point. (i.e. the same as looking at the code point's UTF-8
# under "use bytes"). Each value is the UTF-8 of the translation, for speed.
\%$specials_name = (
END
$pre_body .= join("\n", @multi_code_point_maps) . "\n);\n";
}
my $format = $self->format;
my $return = "";
my $output_adjusted = ($self->to_output_map == $OUTPUT_ADJUSTED);
if ($output_adjusted) {
if ($specials_name) {
$return .= <<END;
# The mappings in the non-hash portion of this file must be modified to get the
# correct values by adding the code point ordinal number to each one that is
# numeric.
END
}
else {
$return .= <<END;
# The mappings must be modified to get the correct values by adding the code
# point ordinal number to each one that is numeric.
END
}
}
$return .= <<END;
# The name this swash is to be known by, with the format of the mappings in
# the main body of the table, and what all code points missing from this file
# map to.
\$utf8::SwashInfo{'To$name'}{'format'} = '$format'; # $map_table_formats{$format}
END
if ($specials_name) {
$return .= <<END;
\$utf8::SwashInfo{'To$name'}{'specials_name'} = '$specials_name'; # Name of hash of special mappings
END
}
my $default_map = $default_map{$addr};
# For $CODE_POINT default maps and using adjustments, instead the default
# becomes zero.
$return .= "\$utf8::SwashInfo{'To$name'}{'missing'} = '"
. (($output_adjusted && $default_map eq $CODE_POINT)
? "0"
: $default_map)
. "';";
if ($default_map eq $CODE_POINT) {
$return .= ' # code point maps to itself';
}
elsif ($default_map eq "") {
$return .= ' # code point maps to the null string';
}
$return .= "\n";
$return .= $pre_body;
return $return;
}
sub write {
# Write the table to the file.
my $self = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $addr = do { no overloading; pack 'J', $self; };
# Clear the temporaries
undef @multi_code_point_maps;
# Calculate the format of the table if not already done.
my $format = $self->format;
my $type = $self->property->type;
my $default_map = $self->default_map;
if (! defined $format) {
if ($type == $BINARY) {
# Don't bother checking the values, because we elsewhere
# verify that a binary table has only 2 values.
$format = $BINARY_FORMAT;
}
else {
my @ranges = $self->_range_list->ranges;
# default an empty table based on its type and default map
if (! @ranges) {
# But it turns out that the only one we can say is a
# non-string (besides binary, handled above) is when the
# table is a string and the default map is to a code point
if ($type == $STRING && $default_map eq $CODE_POINT) {
$format = $HEX_FORMAT;
}
else {
$format = $STRING_FORMAT;
}
}
else {
# Start with the most restrictive format, and as we find
# something that doesn't fit with that, change to the next
# most restrictive, and so on.
$format = $DECIMAL_FORMAT;
foreach my $range (@ranges) {
next if $range->type != 0; # Non-normal ranges don't
# affect the main body
my $map = $range->value;
if ($map ne $default_map) {
last if $format eq $STRING_FORMAT; # already at
# least
# restrictive
$format = $INTEGER_FORMAT
if $format eq $DECIMAL_FORMAT
&& $map !~ / ^ [0-9] $ /x;
$format = $FLOAT_FORMAT
if $format eq $INTEGER_FORMAT
&& $map !~ / ^ -? [0-9]+ $ /x;
$format = $RATIONAL_FORMAT
if $format eq $FLOAT_FORMAT
&& $map !~ / ^ -? [0-9]+ \. [0-9]* $ /x;
$format = $HEX_FORMAT
if ($format eq $RATIONAL_FORMAT
&& $map !~
m/ ^ -? [0-9]+ ( \/ [0-9]+ )? $ /x)
# Assume a leading zero means hex,
# even if all digits are 0-9
|| ($format eq $INTEGER_FORMAT
&& $map =~ /^0[0-9A-F]/);
$format = $STRING_FORMAT if $format eq $HEX_FORMAT
&& $map =~ /[^0-9A-F]/;
}
}
}
}
} # end of calculating format
if ($default_map eq $CODE_POINT
&& $format ne $HEX_FORMAT
&& ! defined $self->format) # manual settings are always
# considered ok
{
Carp::my_carp_bug("Expecting hex format for mapping table for $self, instead got '$format'")
}
# If the output is to be adjusted, the format of the table that gets
# output is actually 'a' or 'ax' instead of whatever it is stored
# internally as.
my $output_adjusted = ($self->to_output_map == $OUTPUT_ADJUSTED);
if ($output_adjusted) {
if ($default_map eq $CODE_POINT) {
$format = $HEX_ADJUST_FORMAT;
}
else {
$format = $ADJUST_FORMAT;
}
}
$self->_set_format($format);
return $self->SUPER::write(
$output_adjusted,
$default_map); # don't write defaulteds
}
# Accessors for the underlying list that should fail if locked.
for my $sub (qw(
add_duplicate
replace_map
))
{
no strict "refs";
*$sub = sub {
use strict "refs";
my $self = shift;
return if $self->carp_if_locked;
return $self->_range_list->$sub(@_);
}
}
} # End closure for Map_Table
package Match_Table;
use parent '-norequire', '_Base_Table';
# A Match table is one which is a list of all the code points that have
# the same property and property value, for use in \p{property=value}
# constructs in regular expressions. It adds very little data to the base
# structure, but many methods, as these lists can be combined in many ways to
# form new ones.
# There are only a few concepts added:
# 1) Equivalents and Relatedness.
# Two tables can match the identical code points, but have different names.
# This always happens when there is a perl single form extension
# \p{IsProperty} for the Unicode compound form \P{Property=True}. The two
# tables are set to be related, with the Perl extension being a child, and
# the Unicode property being the parent.
#
# It may be that two tables match the identical code points and we don't
# know if they are related or not. This happens most frequently when the
# Block and Script properties have the exact range. But note that a
# revision to Unicode could add new code points to the script, which would
# now have to be in a different block (as the block was filled, or there
# would have been 'Unknown' script code points in it and they wouldn't have
# been identical). So we can't rely on any two properties from Unicode
# always matching the same code points from release to release, and thus
# these tables are considered coincidentally equivalent--not related. When
# two tables are unrelated but equivalent, one is arbitrarily chosen as the
# 'leader', and the others are 'equivalents'. This concept is useful
# to minimize the number of tables written out. Only one file is used for
# any identical set of code points, with entries in Heavy.pl mapping all
# the involved tables to it.
#
# Related tables will always be identical; we set them up to be so. Thus
# if the Unicode one is deprecated, the Perl one will be too. Not so for
# unrelated tables. Relatedness makes generating the documentation easier.
#
# 2) Complement.
# Like equivalents, two tables may be the inverses of each other, the
# intersection between them is null, and the union is every Unicode code
# point. The two tables that occupy a binary property are necessarily like
# this. By specifying one table as the complement of another, we can avoid
# storing it on disk (using the other table and performing a fast
# transform), and some memory and calculations.
#
# 3) Conflicting. It may be that there will eventually be name clashes, with
# the same name meaning different things. For a while, there actually were
# conflicts, but they have so far been resolved by changing Perl's or
# Unicode's definitions to match the other, but when this code was written,
# it wasn't clear that that was what was going to happen. (Unicode changed
# because of protests during their beta period.) Name clashes are warned
# about during compilation, and the documentation. The generated tables
# are sane, free of name clashes, because the code suppresses the Perl
# version. But manual intervention to decide what the actual behavior
# should be may be required should this happen. The introductory comments
# have more to say about this.
#
# 4) Definition. This is a string for human consumption that specifies the
# code points that this table matches. This is used only for the generated
# pod file. It may be specified explicitly, or automatically computed.
# Only the first portion of complicated definitions is computed and
# displayed.
sub standardize { return main::standardize($_[0]); }
sub trace { return main::trace(@_); }
{ # Closure
main::setup_package();
my %leader;
# The leader table of this one; initially $self.
main::set_access('leader', \%leader, 'r');
my %equivalents;
# An array of any tables that have this one as their leader
main::set_access('equivalents', \%equivalents, 'readable_array');
my %parent;
# The parent table to this one, initially $self. This allows us to
# distinguish between equivalent tables that are related (for which this
# is set to), and those which may not be, but share the same output file
# because they match the exact same set of code points in the current
# Unicode release.
main::set_access('parent', \%parent, 'r');
my %children;
# An array of any tables that have this one as their parent
main::set_access('children', \%children, 'readable_array');
my %conflicting;
# Array of any tables that would have the same name as this one with
# a different meaning. This is used for the generated documentation.
main::set_access('conflicting', \%conflicting, 'readable_array');
my %matches_all;
# Set in the constructor for tables that are expected to match all code
# points.
main::set_access('matches_all', \%matches_all, 'r');
my %complement;
# Points to the complement that this table is expressed in terms of; 0 if
# none.
main::set_access('complement', \%complement, 'r');
my %definition;
# Human readable string of the first few ranges of code points matched by
# this table
main::set_access('definition', \%definition, 'r', 's');
sub new {
my $class = shift;
my %args = @_;
# The property for which this table is a listing of property values.
my $property = delete $args{'_Property'};
my $name = delete $args{'Name'};
my $full_name = delete $args{'Full_Name'};
$full_name = $name if ! defined $full_name;
# Optional
my $initialize = delete $args{'Initialize'};
my $matches_all = delete $args{'Matches_All'} || 0;
my $format = delete $args{'Format'};
my $definition = delete $args{'Definition'} // "";
# Rest of parameters passed on.
my $range_list = Range_List->new(Initialize => $initialize,
Owner => $property);
my $complete = $full_name;
$complete = '""' if $complete eq ""; # A null name shouldn't happen,
# but this helps debug if it
# does
# The complete name for a match table includes it's property in a
# compound form 'property=table', except if the property is the
# pseudo-property, perl, in which case it is just the single form,
# 'table' (If you change the '=' must also change the ':' in lots of
# places in this program that assume an equal sign)
$complete = $property->full_name . "=$complete" if $property != $perl;
my $self = $class->SUPER::new(%args,
Name => $name,
Complete_Name => $complete,
Full_Name => $full_name,
_Property => $property,
_Range_List => $range_list,
Format => $EMPTY_FORMAT,
Write_As_Invlist => 1,
);
my $addr = do { no overloading; pack 'J', $self; };
$conflicting{$addr} = [ ];
$equivalents{$addr} = [ ];
$children{$addr} = [ ];
$matches_all{$addr} = $matches_all;
$leader{$addr} = $self;
$parent{$addr} = $self;
$complement{$addr} = 0;
$definition{$addr} = $definition;
if (defined $format && $format ne $EMPTY_FORMAT) {
Carp::my_carp_bug("'Format' must be '$EMPTY_FORMAT' in a match table instead of '$format'. Using '$EMPTY_FORMAT'");
}
return $self;
}
# See this program's beginning comment block about overloading these.
use overload
fallback => 0,
qw("") => "_operator_stringify",
'=' => sub {
my $self = shift;
return if $self->carp_if_locked;
return $self;
},
'+' => sub {
my $self = shift;
my $other = shift;
return $self->_range_list + $other;
},
'&' => sub {
my $self = shift;
my $other = shift;
return $self->_range_list & $other;
},
'+=' => sub {
my $self = shift;
my $other = shift;
my $reversed = shift;
if ($reversed) {
Carp::my_carp_bug("Bad news. Can't cope with '"
. ref($other)
. ' += '
. ref($self)
. "'. undef returned.");
return;
}
return if $self->carp_if_locked;
my $addr = do { no overloading; pack 'J', $self; };
if (ref $other) {
# Change the range list of this table to be the
# union of the two.
$self->_set_range_list($self->_range_list
+ $other);
}
else { # $other is just a simple value
$self->add_range($other, $other);
}
return $self;
},
'&=' => sub {
my $self = shift;
my $other = shift;
my $reversed = shift;
if ($reversed) {
Carp::my_carp_bug("Bad news. Can't cope with '"
. ref($other)
. ' &= '
. ref($self)
. "'. undef returned.");
return;
}
return if $self->carp_if_locked;
$self->_set_range_list($self->_range_list & $other);
return $self;
},
'-' => sub { my $self = shift;
my $other = shift;
my $reversed = shift;
if ($reversed) {
Carp::my_carp_bug("Bad news. Can't cope with '"
. ref($other)
. ' - '
. ref($self)
. "'. undef returned.");
return;
}
return $self->_range_list - $other;
},
'~' => sub { my $self = shift;
return ~ $self->_range_list;
},
;
sub _operator_stringify {
my $self = shift;
my $name = $self->complete_name;
return "Table '$name'";
}
sub _range_list {
# Returns the range list associated with this table, which will be the
# complement's if it has one.
my $self = shift;
my $complement = $self->complement;
# In order to avoid re-complementing on each access, only do the
# complement the first time, and store the result in this table's
# range list to use henceforth. However, this wouldn't work if the
# controlling (complement) table changed after we do this, so lock it.
# Currently, the value of the complement isn't needed until after it
# is fully constructed, so this works. If this were to change, the
# each_range iteration functionality would no longer work on this
# complement.
if ($complement != 0 && $self->SUPER::_range_list->count == 0) {
$self->_set_range_list($self->SUPER::_range_list
+ ~ $complement->_range_list);
$complement->lock;
}
return $self->SUPER::_range_list;
}
sub add_alias {
# Add a synonym for this table. See the comments in the base class
my $self = shift;
my $name = shift;
# Rest of parameters passed on.
$self->SUPER::add_alias($name, $self, @_);
return;
}
sub add_conflicting {
# Add the name of some other object to the list of ones that name
# clash with this match table.
my $self = shift;
my $conflicting_name = shift; # The name of the conflicting object
my $p = shift || 'p'; # Optional, is this a \p{} or \P{} ?
my $conflicting_object = shift; # Optional, the conflicting object
# itself. This is used to
# disambiguate the text if the input
# name is identical to any of the
# aliases $self is known by.
# Sometimes the conflicting object is
# merely hypothetical, so this has to
# be an optional parameter.
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $addr = do { no overloading; pack 'J', $self; };
# Check if the conflicting name is exactly the same as any existing
# alias in this table (as long as there is a real object there to
# disambiguate with).
if (defined $conflicting_object) {
foreach my $alias ($self->aliases) {
if (standardize($alias->name) eq standardize($conflicting_name)) {
# Here, there is an exact match. This results in
# ambiguous comments, so disambiguate by changing the
# conflicting name to its object's complete equivalent.
$conflicting_name = $conflicting_object->complete_name;
last;
}
}
}
# Convert to the \p{...} final name
$conflicting_name = "\\$p" . "{$conflicting_name}";
# Only add once
return if grep { $conflicting_name eq $_ } @{$conflicting{$addr}};
push @{$conflicting{$addr}}, $conflicting_name;
return;
}
sub is_set_equivalent_to {
# Return boolean of whether or not the other object is a table of this
# type and has been marked equivalent to this one.
my $self = shift;
my $other = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
return 0 if ! defined $other; # Can happen for incomplete early
# releases
unless ($other->isa(__PACKAGE__)) {
my $ref_other = ref $other;
my $ref_self = ref $self;
Carp::my_carp_bug("Argument to 'is_set_equivalent_to' must be another $ref_self, not a '$ref_other'. $other not set equivalent to $self.");
return 0;
}
# Two tables are equivalent if they have the same leader.
no overloading;
return $leader{pack 'J', $self} == $leader{pack 'J', $other};
return;
}
sub set_equivalent_to {
# Set $self equivalent to the parameter table.
# The required Related => 'x' parameter is a boolean indicating
# whether these tables are related or not. If related, $other becomes
# the 'parent' of $self; if unrelated it becomes the 'leader'
#
# Related tables share all characteristics except names; equivalents
# not quite so many.
# If they are related, one must be a perl extension. This is because
# we can't guarantee that Unicode won't change one or the other in a
# later release even if they are identical now.
my $self = shift;
my $other = shift;
my %args = @_;
my $related = delete $args{'Related'};
Carp::carp_extra_args(\%args) if main::DEBUG && %args;
return if ! defined $other; # Keep on going; happens in some early
# Unicode releases.
if (! defined $related) {
Carp::my_carp_bug("set_equivalent_to must have 'Related => [01] parameter. Assuming $self is not related to $other");
$related = 0;
}
# If already are equivalent, no need to re-do it; if subroutine
# returns null, it found an error, also do nothing
my $are_equivalent = $self->is_set_equivalent_to($other);
return if ! defined $are_equivalent || $are_equivalent;
my $addr = do { no overloading; pack 'J', $self; };
my $current_leader = ($related) ? $parent{$addr} : $leader{$addr};
if ($related) {
if ($current_leader->perl_extension) {
if ($other->perl_extension) {
Carp::my_carp_bug("Use add_alias() to set two Perl tables '$self' and '$other', equivalent.");
return;
}
} elsif ($self->property != $other->property # Depending on
# situation, might
# be better to use
# add_alias()
# instead for same
# property
&& ! $other->perl_extension
# We allow the sc and scx properties to be marked as
# related. They are in fact related, and this allows
# the pod to show that better. This test isn't valid
# if this is an early Unicode release without the scx
# property (having that also implies the sc property
# exists, so don't have to test for no 'sc')
&& ( ! defined $scx
&& ! ( ( $self->property == $script
|| $self->property == $scx)
&& ( $self->property == $script
|| $self->property == $scx))))
{
Carp::my_carp_bug("set_equivalent_to should have 'Related => 0 for equivalencing two Unicode properties. Assuming $self is not related to $other");
$related = 0;
}
}
if (! $self->is_empty && ! $self->matches_identically_to($other)) {
Carp::my_carp_bug("$self should be empty or match identically to $other. Not setting equivalent");
return;
}
my $leader = do { no overloading; pack 'J', $current_leader; };
my $other_addr = do { no overloading; pack 'J', $other; };
# Any tables that are equivalent to or children of this table must now
# instead be equivalent to or (children) to the new leader (parent),
# still equivalent. The equivalency includes their matches_all info,
# and for related tables, their fate and status.
# All related tables are of necessity equivalent, but the converse
# isn't necessarily true
my $status = $other->status;
my $status_info = $other->status_info;
my $fate = $other->fate;
my $matches_all = $matches_all{other_addr};
my $caseless_equivalent = $other->caseless_equivalent;
foreach my $table ($current_leader, @{$equivalents{$leader}}) {
next if $table == $other;
trace "setting $other to be the leader of $table, status=$status" if main::DEBUG && $to_trace;
my $table_addr = do { no overloading; pack 'J', $table; };
$leader{$table_addr} = $other;
$matches_all{$table_addr} = $matches_all;
$self->_set_range_list($other->_range_list);
push @{$equivalents{$other_addr}}, $table;
if ($related) {
$parent{$table_addr} = $other;
push @{$children{$other_addr}}, $table;
$table->set_status($status, $status_info);
# This reason currently doesn't get exposed outside; otherwise
# would have to look up the parent's reason and use it instead.
$table->set_fate($fate, "Parent's fate");
$self->set_caseless_equivalent($caseless_equivalent);
}
}
# Now that we've declared these to be equivalent, any changes to one
# of the tables would invalidate that equivalency.
$self->lock;
$other->lock;
return;
}
sub set_complement {
# Set $self to be the complement of the parameter table. $self is
# locked, as what it contains should all come from the other table.
my $self = shift;
my $other = shift;
my %args = @_;
Carp::carp_extra_args(\%args) if main::DEBUG && %args;
if ($other->complement != 0) {
Carp::my_carp_bug("Can't set $self to be the complement of $other, which itself is the complement of " . $other->complement);
return;
}
my $addr = do { no overloading; pack 'J', $self; };
$complement{$addr} = $other;
# Be sure the other property knows we are depending on them; or the
# other table if it is one in the current property.
if ($self->property != $other->property) {
$other->property->set_has_dependency(1);
}
else {
$other->set_has_dependency(1);
}
$self->lock;
return;
}
sub add_range { # Add a range to the list for this table.
my $self = shift;
# Rest of parameters passed on
return if $self->carp_if_locked;
return $self->_range_list->add_range(@_);
}
sub header {
my $self = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
# All match tables are to be used only by the Perl core.
return $self->SUPER::header() . $INTERNAL_ONLY_HEADER;
}
sub pre_body { # Does nothing for match tables.
return
}
sub append_to_body { # Does nothing for match tables.
return
}
sub set_fate {
my $self = shift;
my $fate = shift;
my $reason = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
$self->SUPER::set_fate($fate, $reason);
# All children share this fate
foreach my $child ($self->children) {
$child->set_fate($fate, $reason);
}
return;
}
sub calculate_table_definition
{
# Returns a human-readable string showing some or all of the code
# points matched by this table. The string will include a
# bracketed-character class for all characters matched in the 00-FF
# range, and the first few ranges matched beyond that.
my $max_ranges = 6;
my $self = shift;
my $definition = $self->definition || "";
# Skip this if already have a definition.
return $definition if $definition;
my $lows_string = ""; # The string representation of the 0-FF
# characters
my $string_range = ""; # The string rep. of the above FF ranges
my $range_count = 0; # How many ranges in $string_rage
my @lows_invlist; # The inversion list of the 0-FF code points
my $first_non_control = ord(" "); # Everything below this is a
# control, on ASCII or EBCDIC
my $max_table_code_point = $self->max;
# On ASCII platforms, the range 80-FF contains no printables.
my $highest_printable = ((main::NON_ASCII_PLATFORM) ? 255 : 126);
# Look through the first few ranges matched by this table.
$self->reset_each_range; # Defensive programming
while (defined (my $range = $self->each_range())) {
my $start = $range->start;
my $end = $range->end;
# Accumulate an inversion list of the 00-FF code points
if ($start < 256 && ($start > 0 || $end < 256)) {
push @lows_invlist, $start;
push @lows_invlist, 1 + (($end < 256) ? $end : 255);
# Get next range if there are more ranges below 256
next if $end < 256 && $end < $max_table_code_point;
# If the range straddles the 255/256 boundary, we split it
# there. We already added above the low portion to the
# inversion list
$start = 256 if $end > 256;
}
# Here, @lows_invlist contains the code points below 256, and
# there is no other range, or the current one starts at or above
# 256. Generate the [char class] for the 0-255 ones.
while (@lows_invlist) {
# If this range (necessarily the first one, by the way) starts
# at 0 ...
if ($lows_invlist[0] == 0) {
# If it ends within the block of controls, that means that
# some controls are in it and some aren't. Since Unicode
# properties pretty much only know about a few of the
# controls, like \n, \t, this means that its one of them
# that isn't in the range. Complement the inversion list
# which will likely cause these to be output using their
# mnemonics, hence being clearer.
if ($lows_invlist[1] < $first_non_control) {
$lows_string .= '^';
shift @lows_invlist;
push @lows_invlist, 256;
}
elsif ($lows_invlist[1] <= $highest_printable) {
# Here, it extends into the printables block. Split
# into two ranges so that the controls are separate.
$lows_string .= sprintf "\\x00-\\x%02x",
$first_non_control - 1;
$lows_invlist[0] = $first_non_control;
}
}
# If the range completely contains the printables, don't
# individually spell out the printables.
if ( $lows_invlist[0] <= $first_non_control
&& $lows_invlist[1] > $highest_printable)
{
$lows_string .= sprintf "\\x%02x-\\x%02x",
$lows_invlist[0], $lows_invlist[1] - 1;
shift @lows_invlist;
shift @lows_invlist;
next;
}
# Here, the range may include some but not all printables.
# Look at each one individually
foreach my $ord (shift @lows_invlist .. shift(@lows_invlist) - 1) {
my $char = chr $ord;
# If there is already something in the list, an
# alphanumeric char could be the next in sequence. If so,
# we start or extend a range. That is, we could have so
# far something like 'a-c', and the next char is a 'd', so
# we change it to 'a-d'. We use native_to_unicode()
# because a-z on EBCDIC means 26 chars, and excludes the
# gap ones.
if ($lows_string ne "" && $char =~ /[[:alnum:]]/) {
my $prev = substr($lows_string, -1);
if ( $prev !~ /[[:alnum:]]/
|| utf8::native_to_unicode(ord $prev) + 1
!= utf8::native_to_unicode(ord $char))
{
# Not extending the range
$lows_string .= $char;
}
elsif ( length $lows_string > 1
&& substr($lows_string, -2, 1) eq '-')
{
# We had a sequence like '-c' and the current
# character is 'd'. Extend the range.
substr($lows_string, -1, 1) = $char;
}
else {
# We had something like 'd' and this is 'e'.
# Start a range.
$lows_string .= "-$char";
}
}
elsif ($char =~ /[[:graph:]]/) {
# We output a graphic char as-is, preceded by a
# backslash if it is a metacharacter
$lows_string .= '\\'
if $char =~ /[\\\^\$\@\%\|()\[\]\{\}\-\/"']/;
$lows_string .= $char;
} # Otherwise use mnemonic for any that have them
elsif ($char =~ /[\a]/) {
$lows_string .= '\a';
}
elsif ($char =~ /[\b]/) {
$lows_string .= '\b';
}
elsif ($char eq "\e") {
$lows_string .= '\e';
}
elsif ($char eq "\f") {
$lows_string .= '\f';
}
elsif ($char eq "\cK") {
$lows_string .= '\cK';
}
elsif ($char eq "\n") {
$lows_string .= '\n';
}
elsif ($char eq "\r") {
$lows_string .= '\r';
}
elsif ($char eq "\t") {
$lows_string .= '\t';
}
else {
# Here is a non-graphic without a mnemonic. We use \x
# notation. But if the ordinal of this is one above
# the previous, create or extend the range
my $hex_representation = sprintf("%02x", ord $char);
if ( length $lows_string >= 4
&& substr($lows_string, -4, 2) eq '\\x'
&& hex(substr($lows_string, -2)) + 1 == ord $char)
{
if ( length $lows_string >= 5
&& substr($lows_string, -5, 1) eq '-'
&& ( length $lows_string == 5
|| substr($lows_string, -6, 1) ne '\\'))
{
substr($lows_string, -2) = $hex_representation;
}
else {
$lows_string .= '-\\x' . $hex_representation;
}
}
else {
$lows_string .= '\\x' . $hex_representation;
}
}
}
}
# Done with assembling the string of all lows. If there are only
# lows in the property, are completely done.
if ($max_table_code_point < 256) {
$self->reset_each_range;
last;
}
# Otherwise, quit if reached max number of non-lows ranges. If
# there are lows, count them as one unit towards the maximum.
$range_count++;
if ($range_count > (($lows_string eq "") ? $max_ranges : $max_ranges - 1)) {
$string_range .= " ...";
$self->reset_each_range;
last;
}
# Otherwise add this range.
$string_range .= ", " if $string_range ne "";
if ($start == $end) {
$string_range .= sprintf("U+%04X", $start);
}
elsif ($end >= $MAX_WORKING_CODEPOINT) {
$string_range .= sprintf("U+%04X..infinity", $start);
}
else {
$string_range .= sprintf("U+%04X..%04X",
$start, $end);
}
}
# Done with all the ranges we're going to look at. Assemble the
# definition from the lows + non-lows.
if ($lows_string ne "" || $string_range ne "") {
if ($lows_string ne "") {
$definition .= "[$lows_string]";
$definition .= ", " if $string_range;
}
$definition .= $string_range;
}
return $definition;
}
sub write {
my $self = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
return $self->SUPER::write(0); # No adjustments
}
sub set_final_comment {
# This creates a comment for the file that is to hold the match table
# $self. It is somewhat convoluted to make the English read nicely,
# but, heh, it's just a comment.
# This should be called only with the leader match table of all the
# ones that share the same file. It lists all such tables, ordered so
# that related ones are together.
return unless $debugging_build;
my $leader = shift; # Should only be called on the leader table of
# an equivalent group
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $addr = do { no overloading; pack 'J', $leader; };
if ($leader{$addr} != $leader) {
Carp::my_carp_bug(<<END
set_final_comment() must be called on a leader table, which $leader is not.
It is equivalent to $leader{$addr}. No comment created
END
);
return;
}
# Get the number of code points matched by each of the tables in this
# file, and add underscores for clarity.
my $count = $leader->count;
my $unicode_count;
my $non_unicode_string;
if ($count > $MAX_UNICODE_CODEPOINTS) {
$unicode_count = $count - ($MAX_WORKING_CODEPOINT
- $MAX_UNICODE_CODEPOINT);
$non_unicode_string = "All above-Unicode code points match as well, and are also returned";
}
else {
$unicode_count = $count;
$non_unicode_string = "";
}
my $string_count = main::clarify_code_point_count($unicode_count);
my $loose_count = 0; # how many aliases loosely matched
my $compound_name = ""; # ? Are any names compound?, and if so, an
# example
my $properties_with_compound_names = 0; # count of these
my %flags; # The status flags used in the file
my $total_entries = 0; # number of entries written in the comment
my $matches_comment = ""; # The portion of the comment about the
# \p{}'s
my @global_comments; # List of all the tables' comments that are
# there before this routine was called.
my $has_ucd_alias = 0; # If there is an alias that is accessible via
# Unicode::UCD. If not, then don't say it is
# in the comment
# Get list of all the parent tables that are equivalent to this one
# (including itself).
my @parents = grep { $parent{main::objaddr $_} == $_ }
main::uniques($leader, @{$equivalents{$addr}});
my $has_unrelated = (@parents >= 2); # boolean, ? are there unrelated
# tables
for my $parent (@parents) {
my $property = $parent->property;
# Special case 'N' tables in properties with two match tables when
# the other is a 'Y' one. These are likely to be binary tables,
# but not necessarily. In either case, \P{} will match the
# complement of \p{}, and so if something is a synonym of \p, the
# complement of that something will be the synonym of \P. This
# would be true of any property with just two match tables, not
# just those whose values are Y and N; but that would require a
# little extra work, and there are none such so far in Unicode.
my $perl_p = 'p'; # which is it? \p{} or \P{}
my @yes_perl_synonyms; # list of any synonyms for the 'Y' table
if (scalar $property->tables == 2
&& $parent == $property->table('N')
&& defined (my $yes = $property->table('Y')))
{
my $yes_addr = do { no overloading; pack 'J', $yes; };
@yes_perl_synonyms
= grep { $_->property == $perl }
main::uniques($yes,
$parent{$yes_addr},
$parent{$yes_addr}->children);
# But these synonyms are \P{} ,not \p{}
$perl_p = 'P';
}
my @description; # Will hold the table description
my @note; # Will hold the table notes.
my @conflicting; # Will hold the table conflicts.
# Look at the parent, any yes synonyms, and all the children
my $parent_addr = do { no overloading; pack 'J', $parent; };
for my $table ($parent,
@yes_perl_synonyms,
@{$children{$parent_addr}})
{
my $table_addr = do { no overloading; pack 'J', $table; };
my $table_property = $table->property;
# Tables are separated by a blank line to create a grouping.
$matches_comment .= "\n" if $matches_comment;
# The table is named based on the property and value
# combination it is for, like script=greek. But there may be
# a number of synonyms for each side, like 'sc' for 'script',
# and 'grek' for 'greek'. Any combination of these is a valid
# name for this table. In this case, there are three more,
# 'sc=grek', 'sc=greek', and 'script='grek'. Rather than
# listing all possible combinations in the comment, we make
# sure that each synonym occurs at least once, and add
# commentary that the other combinations are possible.
# Because regular expressions don't recognize things like
# \p{jsn=}, only look at non-null right-hand-sides
my @property_aliases = grep { $_->status ne $INTERNAL_ALIAS } $table_property->aliases;
my @table_aliases = grep { $_->name ne "" } $table->aliases;
# The alias lists above are already ordered in the order we
# want to output them. To ensure that each synonym is listed,
# we must use the max of the two numbers. But if there are no
# legal synonyms (nothing in @table_aliases), then we don't
# list anything.
my $listed_combos = (@table_aliases)
? main::max(scalar @table_aliases,
scalar @property_aliases)
: 0;
trace "$listed_combos, tables=", scalar @table_aliases, "; property names=", scalar @property_aliases if main::DEBUG;
my $property_had_compound_name = 0;
for my $i (0 .. $listed_combos - 1) {
$total_entries++;
# The current alias for the property is the next one on
# the list, or if beyond the end, start over. Similarly
# for the table (\p{prop=table})
my $property_alias = $property_aliases
[$i % @property_aliases]->name;
my $table_alias_object = $table_aliases
[$i % @table_aliases];
my $table_alias = $table_alias_object->name;
my $loose_match = $table_alias_object->loose_match;
$has_ucd_alias |= $table_alias_object->ucd;
if ($table_alias !~ /\D/) { # Clarify large numbers.
$table_alias = main::clarify_number($table_alias)
}
# Add a comment for this alias combination
my $current_match_comment;
if ($table_property == $perl) {
$current_match_comment = "\\$perl_p"
. "{$table_alias}";
}
else {
$current_match_comment
= "\\p{$property_alias=$table_alias}";
$property_had_compound_name = 1;
}
# Flag any abnormal status for this table.
my $flag = $property->status
|| $table->status
|| $table_alias_object->status;
if ($flag && $flag ne $PLACEHOLDER) {
$flags{$flag} = $status_past_participles{$flag};
}
$loose_count++;
# Pretty up the comment. Note the \b; it says don't make
# this line a continuation.
$matches_comment .= sprintf("\b%-1s%-s%s\n",
$flag,
" " x 7,
$current_match_comment);
} # End of generating the entries for this table.
# Save these for output after this group of related tables.
push @description, $table->description;
push @note, $table->note;
push @conflicting, $table->conflicting;
# And this for output after all the tables.
push @global_comments, $table->comment;
# Compute an alternate compound name using the final property
# synonym and the first table synonym with a colon instead of
# the equal sign used elsewhere.
if ($property_had_compound_name) {
$properties_with_compound_names ++;
if (! $compound_name || @property_aliases > 1) {
$compound_name = $property_aliases[-1]->name
. ': '
. $table_aliases[0]->name;
}
}
} # End of looping through all children of this table
# Here have assembled in $matches_comment all the related tables
# to the current parent (preceded by the same info for all the
# previous parents). Put out information that applies to all of
# the current family.
if (@conflicting) {
# But output the conflicting information now, as it applies to
# just this table.
my $conflicting = join ", ", @conflicting;
if ($conflicting) {
$matches_comment .= <<END;
Note that contrary to what you might expect, the above is NOT the same as
END
$matches_comment .= "any of: " if @conflicting > 1;
$matches_comment .= "$conflicting\n";
}
}
if (@description) {
$matches_comment .= "\n Meaning: "
. join('; ', @description)
. "\n";
}
if (@note) {
$matches_comment .= "\n Note: "
. join("\n ", @note)
. "\n";
}
} # End of looping through all tables
$matches_comment .= "\n$non_unicode_string\n" if $non_unicode_string;
my $code_points;
my $match;
my $any_of_these;
if ($unicode_count == 1) {
$match = 'matches';
$code_points = 'single code point';
}
else {
$match = 'match';
$code_points = "$string_count code points";
}
my $synonyms;
my $entries;
if ($total_entries == 1) {
$synonyms = "";
$entries = 'entry';
$any_of_these = 'this'
}
else {
$synonyms = " any of the following regular expression constructs";
$entries = 'entries';
$any_of_these = 'any of these'
}
my $comment = "";
if ($has_ucd_alias) {
$comment .= "Use Unicode::UCD::prop_invlist() to access the contents of this file.\n\n";
}
if ($has_unrelated) {
$comment .= <<END;
This file is for tables that are not necessarily related: To conserve
resources, every table that matches the identical set of code points in this
version of Unicode uses this file. Each one is listed in a separate group
below. It could be that the tables will match the same set of code points in
other Unicode releases, or it could be purely coincidence that they happen to
be the same in Unicode $unicode_version, and hence may not in other versions.
END
}
if (%flags) {
foreach my $flag (sort keys %flags) {
$comment .= <<END;
'$flag' below means that this form is $flags{$flag}.
END
if ($flag eq $INTERNAL_ALIAS) {
$comment .= "DO NOT USE!!!";
}
else {
$comment .= "Consult $pod_file.pod";
}
$comment .= "\n";
}
$comment .= "\n";
}
if ($total_entries == 0) {
Carp::my_carp("No regular expression construct can match $leader, as all names for it are the null string. Creating file anyway.");
$comment .= <<END;
This file returns the $code_points in Unicode Version
$unicode_version for
$leader, but it is inaccessible through Perl regular expressions, as
"\\p{prop=}" is not recognized.
END
} else {
$comment .= <<END;
This file returns the $code_points in Unicode Version
$unicode_version that
$match$synonyms:
$matches_comment
$pod_file.pod should be consulted for the syntax rules for $any_of_these,
including if adding or subtracting white space, underscore, and hyphen
characters matters or doesn't matter, and other permissible syntactic
variants. Upper/lower case distinctions never matter.
END
}
if ($compound_name) {
$comment .= <<END;
A colon can be substituted for the equals sign, and
END
if ($properties_with_compound_names > 1) {
$comment .= <<END;
within each group above,
END
}
$compound_name = sprintf("%-8s\\p{%s}", " ", $compound_name);
# Note the \b below, it says don't make that line a continuation.
$comment .= <<END;
anything to the left of the equals (or colon) can be combined with anything to
the right. Thus, for example,
$compound_name
\bis also valid.
END
}
# And append any comment(s) from the actual tables. They are all
# gathered here, so may not read all that well.
if (@global_comments) {
$comment .= "\n" . join("\n\n", @global_comments) . "\n";
}
if ($count) { # The format differs if no code points, and needs no
# explanation in that case
if ($leader->write_as_invlist) {
$comment.= <<END;
The first data line of this file begins with the letter V to indicate it is in
inversion list format. The number following the V gives the number of lines
remaining. Each of those remaining lines is a single number representing the
starting code point of a range which goes up to but not including the number
on the next line; The 0th, 2nd, 4th... ranges are for code points that match
the property; the 1st, 3rd, 5th... are ranges of code points that don't match
the property. The final line's range extends to the platform's infinity.
END
}
else {
$comment.= <<END;
The format of the lines of this file is:
START\\tSTOP\\twhere START is the starting code point of the range, in hex;
STOP is the ending point, or if omitted, the range has just one code point.
END
}
if ($leader->output_range_counts) {
$comment .= <<END;
Numbers in comments in [brackets] indicate how many code points are in the
range.
END
}
}
$leader->set_comment(main::join_lines($comment));
return;
}
# Accessors for the underlying list
for my $sub (qw(
get_valid_code_point
get_invalid_code_point
))
{
no strict "refs";
*$sub = sub {
use strict "refs";
my $self = shift;
return $self->_range_list->$sub(@_);
}
}
} # End closure for Match_Table
package Property;
# The Property class represents a Unicode property, or the $perl
# pseudo-property. It contains a map table initialized empty at construction
# time, and for properties accessible through regular expressions, various
# match tables, created through the add_match_table() method, and referenced
# by the table('NAME') or tables() methods, the latter returning a list of all
# of the match tables. Otherwise table operations implicitly are for the map
# table.
#
# Most of the data in the property is actually about its map table, so it
# mostly just uses that table's accessors for most methods. The two could
# have been combined into one object, but for clarity because of their
# differing semantics, they have been kept separate. It could be argued that
# the 'file' and 'directory' fields should be kept with the map table.
#
# Each property has a type. This can be set in the constructor, or in the
# set_type accessor, but mostly it is figured out by the data. Every property
# starts with unknown type, overridden by a parameter to the constructor, or
# as match tables are added, or ranges added to the map table, the data is
# inspected, and the type changed. After the table is mostly or entirely
# filled, compute_type() should be called to finalize they analysis.
#
# There are very few operations defined. One can safely remove a range from
# the map table, and property_add_or_replace_non_nulls() adds the maps from another
# table to this one, replacing any in the intersection of the two.
sub standardize { return main::standardize($_[0]); }
sub trace { return main::trace(@_) if main::DEBUG && $to_trace }
{ # Closure
# This hash will contain as keys, all the aliases of all properties, and
# as values, pointers to their respective property objects. This allows
# quick look-up of a property from any of its names.
my %alias_to_property_of;
sub dump_alias_to_property_of {
# For debugging
print "\n", main::simple_dumper (\%alias_to_property_of), "\n";
return;
}
sub property_ref {
# This is a package subroutine, not called as a method.
# If the single parameter is a literal '*' it returns a list of all
# defined properties.
# Otherwise, the single parameter is a name, and it returns a pointer
# to the corresponding property object, or undef if none.
#
# Properties can have several different names. The 'standard' form of
# each of them is stored in %alias_to_property_of as they are defined.
# But it's possible that this subroutine will be called with some
# variant, so if the initial lookup fails, it is repeated with the
# standardized form of the input name. If found, besides returning the
# result, the input name is added to the list so future calls won't
# have to do the conversion again.
my $name = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
if (! defined $name) {
Carp::my_carp_bug("Undefined input property. No action taken.");
return;
}
return main::uniques(values %alias_to_property_of) if $name eq '*';
# Return cached result if have it.
my $result = $alias_to_property_of{$name};
return $result if defined $result;
# Convert the input to standard form.
my $standard_name = standardize($name);
$result = $alias_to_property_of{$standard_name};
return unless defined $result; # Don't cache undefs
# Cache the result before returning it.
$alias_to_property_of{$name} = $result;
return $result;
}
main::setup_package();
my %map;
# A pointer to the map table object for this property
main::set_access('map', \%map);
my %full_name;
# The property's full name. This is a duplicate of the copy kept in the
# map table, but is needed because stringify needs it during
# construction of the map table, and then would have a chicken before egg
# problem.
main::set_access('full_name', \%full_name, 'r');
my %table_ref;
# This hash will contain as keys, all the aliases of any match tables
# attached to this property, and as values, the pointers to their
# respective tables. This allows quick look-up of a table from any of its
# names.
main::set_access('table_ref', \%table_ref);
my %type;
# The type of the property, $ENUM, $BINARY, etc
main::set_access('type', \%type, 'r');
my %file;
# The filename where the map table will go (if actually written).
# Normally defaulted, but can be overridden.
main::set_access('file', \%file, 'r', 's');
my %directory;
# The directory where the map table will go (if actually written).
# Normally defaulted, but can be overridden.
main::set_access('directory', \%directory, 's');
my %pseudo_map_type;
# This is used to affect the calculation of the map types for all the
# ranges in the table. It should be set to one of the values that signify
# to alter the calculation.
main::set_access('pseudo_map_type', \%pseudo_map_type, 'r');
my %has_only_code_point_maps;
# A boolean used to help in computing the type of data in the map table.
main::set_access('has_only_code_point_maps', \%has_only_code_point_maps);
my %unique_maps;
# A list of the first few distinct mappings this property has. This is
# used to disambiguate between binary and enum property types, so don't
# have to keep more than three.
main::set_access('unique_maps', \%unique_maps);
my %pre_declared_maps;
# A boolean that gives whether the input data should declare all the
# tables used, or not. If the former, unknown ones raise a warning.
main::set_access('pre_declared_maps',
\%pre_declared_maps, 'r', 's');
my %has_dependency;
# A boolean that gives whether some table somewhere is defined as the
# complement of a table in this property. This is a crude, but currently
# sufficient, mechanism to make this property not get destroyed before
# what is dependent on it is. Other dependencies could be added, so the
# name was chosen to reflect a more general situation than actually is
# currently the case.
main::set_access('has_dependency', \%has_dependency, 'r', 's');
sub new {
# The only required parameter is the positionally first, name. All
# other parameters are key => value pairs. See the documentation just
# above for the meanings of the ones not passed directly on to the map
# table constructor.
my $class = shift;
my $name = shift || "";
my $self = property_ref($name);
if (defined $self) {
my $options_string = join ", ", @_;
$options_string = ". Ignoring options $options_string" if $options_string;
Carp::my_carp("$self is already in use. Using existing one$options_string;");
return $self;
}
my %args = @_;
$self = bless \do { my $anonymous_scalar }, $class;
my $addr = do { no overloading; pack 'J', $self; };
$directory{$addr} = delete $args{'Directory'};
$file{$addr} = delete $args{'File'};
$full_name{$addr} = delete $args{'Full_Name'} || $name;
$type{$addr} = delete $args{'Type'} || $UNKNOWN;
$pseudo_map_type{$addr} = delete $args{'Map_Type'};
$pre_declared_maps{$addr} = delete $args{'Pre_Declared_Maps'}
# Starting in this release, property
# values should be defined for all
# properties, except those overriding this
// $v_version ge v5.1.0;
# Rest of parameters passed on.
$has_only_code_point_maps{$addr} = 1;
$table_ref{$addr} = { };
$unique_maps{$addr} = { };
$has_dependency{$addr} = 0;
$map{$addr} = Map_Table->new($name,
Full_Name => $full_name{$addr},
_Alias_Hash => \%alias_to_property_of,
_Property => $self,
%args);
return $self;
}
# See this program's beginning comment block about overloading the copy
# constructor. Few operations are defined on properties, but a couple are
# useful. It is safe to take the inverse of a property, and to remove a
# single code point from it.
use overload
fallback => 0,
qw("") => "_operator_stringify",
"." => \&main::_operator_dot,
".=" => \&main::_operator_dot_equal,
'==' => \&main::_operator_equal,
'!=' => \&main::_operator_not_equal,
'=' => sub { return shift },
'-=' => "_minus_and_equal",
;
sub _operator_stringify {
return "Property '" . shift->full_name . "'";
}
sub _minus_and_equal {
# Remove a single code point from the map table of a property.
my $self = shift;
my $other = shift;
my $reversed = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
if (ref $other) {
Carp::my_carp_bug("Bad news. Can't cope with a "
. ref($other)
. " argument to '-='. Subtraction ignored.");
return $self;
}
elsif ($reversed) { # Shouldn't happen in a -=, but just in case
Carp::my_carp_bug("Bad news. Can't cope with subtracting a "
. ref $self
. " from a non-object. undef returned.");
return;
}
else {
no overloading;
$map{pack 'J', $self}->delete_range($other, $other);
}
return $self;
}
sub add_match_table {
# Add a new match table for this property, with name given by the
# parameter. It returns a pointer to the table.
my $self = shift;
my $name = shift;
my %args = @_;
my $addr = do { no overloading; pack 'J', $self; };
my $table = $table_ref{$addr}{$name};
my $standard_name = main::standardize($name);
if (defined $table
|| (defined ($table = $table_ref{$addr}{$standard_name})))
{
Carp::my_carp("Table '$name' in $self is already in use. Using existing one");
$table_ref{$addr}{$name} = $table;
return $table;
}
else {
# See if this is a perl extension, if not passed in.
my $perl_extension = delete $args{'Perl_Extension'};
$perl_extension
= $self->perl_extension if ! defined $perl_extension;
my $fate;
my $suppression_reason = "";
if ($self->name =~ /^_/) {
$fate = $SUPPRESSED;
$suppression_reason = "Parent property is internal only";
}
elsif ($self->fate >= $SUPPRESSED) {
$fate = $self->fate;
$suppression_reason = $why_suppressed{$self->complete_name};
}
elsif ($name =~ /^_/) {
$fate = $INTERNAL_ONLY;
}
$table = Match_Table->new(
Name => $name,
Perl_Extension => $perl_extension,
_Alias_Hash => $table_ref{$addr},
_Property => $self,
Fate => $fate,
Suppression_Reason => $suppression_reason,
Status => $self->status,
_Status_Info => $self->status_info,
%args);
return unless defined $table;
}
# Save the names for quick look up
$table_ref{$addr}{$standard_name} = $table;
$table_ref{$addr}{$name} = $table;
# Perhaps we can figure out the type of this property based on the
# fact of adding this match table. First, string properties don't
# have match tables; second, a binary property can't have 3 match
# tables
if ($type{$addr} == $UNKNOWN) {
$type{$addr} = $NON_STRING;
}
elsif ($type{$addr} == $STRING) {
Carp::my_carp("$self Added a match table '$name' to a string property '$self'. Changed it to a non-string property. Bad News.");
$type{$addr} = $NON_STRING;
}
elsif ($type{$addr} != $ENUM && $type{$addr} != $FORCED_BINARY) {
if (scalar main::uniques(values %{$table_ref{$addr}}) > 2) {
if ($type{$addr} == $BINARY) {
Carp::my_carp("$self now has more than 2 tables (with the addition of '$name'), and so is no longer binary. Changing its type to 'enum'. Bad News.");
}
$type{$addr} = $ENUM;
}
}
return $table;
}
sub delete_match_table {
# Delete the table referred to by $2 from the property $1.
my $self = shift;
my $table_to_remove = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $addr = do { no overloading; pack 'J', $self; };
# Remove all names that refer to it.
foreach my $key (keys %{$table_ref{$addr}}) {
delete $table_ref{$addr}{$key}
if $table_ref{$addr}{$key} == $table_to_remove;
}
$table_to_remove->DESTROY;
return;
}
sub table {
# Return a pointer to the match table (with name given by the
# parameter) associated with this property; undef if none.
my $self = shift;
my $name = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $addr = do { no overloading; pack 'J', $self; };
return $table_ref{$addr}{$name} if defined $table_ref{$addr}{$name};
# If quick look-up failed, try again using the standard form of the
# input name. If that succeeds, cache the result before returning so
# won't have to standardize this input name again.
my $standard_name = main::standardize($name);
return unless defined $table_ref{$addr}{$standard_name};
$table_ref{$addr}{$name} = $table_ref{$addr}{$standard_name};
return $table_ref{$addr}{$name};
}
sub tables {
# Return a list of pointers to all the match tables attached to this
# property
no overloading;
return main::uniques(values %{$table_ref{pack 'J', shift}});
}
sub directory {
# Returns the directory the map table for this property should be
# output in. If a specific directory has been specified, that has
# priority; 'undef' is returned if the type isn't defined;
# or $map_directory for everything else.
my $addr = do { no overloading; pack 'J', shift; };
return $directory{$addr} if defined $directory{$addr};
return undef if $type{$addr} == $UNKNOWN;
return $map_directory;
}
sub swash_name {
# Return the name that is used to both:
# 1) Name the file that the map table is written to.
# 2) The name of swash related stuff inside that file.
# The reason for this is that the Perl core historically has used
# certain names that aren't the same as the Unicode property names.
# To continue using these, $file is hard-coded in this file for those,
# but otherwise the standard name is used. This is different from the
# external_name, so that the rest of the files, like in lib can use
# the standard name always, without regard to historical precedent.
my $self = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $addr = do { no overloading; pack 'J', $self; };
# Swash names are used only on either
# 1) legacy-only properties, because the formats for these are
# unchangeable, and they have had these lines in them; or
# 2) regular or internal-only map tables
# 3) otherwise there should be no access to the
# property map table from other parts of Perl.
return if $map{$addr}->fate != $ORDINARY
&& $map{$addr}->fate != $LEGACY_ONLY
&& ! ($map{$addr}->name =~ /^_/
&& $map{$addr}->fate == $INTERNAL_ONLY);
return $file{$addr} if defined $file{$addr};
return $map{$addr}->external_name;
}
sub to_create_match_tables {
# Returns a boolean as to whether or not match tables should be
# created for this property.
my $self = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
# The whole point of this pseudo property is match tables.
return 1 if $self == $perl;
my $addr = do { no overloading; pack 'J', $self; };
# Don't generate tables of code points that match the property values
# of a string property. Such a list would most likely have many
# property values, each with just one or very few code points mapping
# to it.
return 0 if $type{$addr} == $STRING;
# Otherwise, do.
return 1;
}
sub property_add_or_replace_non_nulls {
# This adds the mappings in the property $other to $self. Non-null
# mappings from $other override those in $self. It essentially merges
# the two properties, with the second having priority except for null
# mappings.
my $self = shift;
my $other = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
if (! $other->isa(__PACKAGE__)) {
Carp::my_carp_bug("$other should be a "
. __PACKAGE__
. ". Not a '"
. ref($other)
. "'. Not added;");
return;
}
no overloading;
return $map{pack 'J', $self}->map_add_or_replace_non_nulls($map{pack 'J', $other});
}
sub set_proxy_for {
# Certain tables are not generally written out to files, but
# Unicode::UCD has the intelligence to know that the file for $self
# can be used to reconstruct those tables. This routine just changes
# things so that UCD pod entries for those suppressed tables are
# generated, so the fact that a proxy is used is invisible to the
# user.
my $self = shift;
foreach my $property_name (@_) {
my $ref = property_ref($property_name);
next if $ref->to_output_map;
$ref->set_fate($MAP_PROXIED);
}
}
sub set_type {
# Set the type of the property. Mostly this is figured out by the
# data in the table. But this is used to set it explicitly. The
# reason it is not a standard accessor is that when setting a binary
# property, we need to make sure that all the true/false aliases are
# present, as they were omitted in early Unicode releases.
my $self = shift;
my $type = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
if ($type != $ENUM
&& $type != $BINARY
&& $type != $FORCED_BINARY
&& $type != $STRING)
{
Carp::my_carp("Unrecognized type '$type'. Type not set");
return;
}
{ no overloading; $type{pack 'J', $self} = $type; }
return if $type != $BINARY && $type != $FORCED_BINARY;
my $yes = $self->table('Y');
$yes = $self->table('Yes') if ! defined $yes;
$yes = $self->add_match_table('Y', Full_Name => 'Yes')
if ! defined $yes;
# Add aliases in order wanted, duplicates will be ignored. We use a
# binary property present in all releases for its ordered lists of
# true/false aliases. Note, that could run into problems in
# outputting things in that we don't distinguish between the name and
# full name of these. Hopefully, if the table was already created
# before this code is executed, it was done with these set properly.
my $bm = property_ref("Bidi_Mirrored");
foreach my $alias ($bm->table("Y")->aliases) {
$yes->add_alias($alias->name);
}
my $no = $self->table('N');
$no = $self->table('No') if ! defined $no;
$no = $self->add_match_table('N', Full_Name => 'No') if ! defined $no;
foreach my $alias ($bm->table("N")->aliases) {
$no->add_alias($alias->name);
}
return;
}
sub add_map {
# Add a map to the property's map table. This also keeps
# track of the maps so that the property type can be determined from
# its data.
my $self = shift;
my $start = shift; # First code point in range
my $end = shift; # Final code point in range
my $map = shift; # What the range maps to.
# Rest of parameters passed on.
my $addr = do { no overloading; pack 'J', $self; };
# If haven't the type of the property, gather information to figure it
# out.
if ($type{$addr} == $UNKNOWN) {
# If the map contains an interior blank or dash, or most other
# nonword characters, it will be a string property. This
# heuristic may actually miss some string properties. If so, they
# may need to have explicit set_types called for them. This
# happens in the Unihan properties.
if ($map =~ / (?<= . ) [ -] (?= . ) /x
|| $map =~ / [^\w.\/\ -] /x)
{
$self->set_type($STRING);
# $unique_maps is used for disambiguating between ENUM and
# BINARY later; since we know the property is not going to be
# one of those, no point in keeping the data around
undef $unique_maps{$addr};
}
else {
# Not necessarily a string. The final decision has to be
# deferred until all the data are in. We keep track of if all
# the values are code points for that eventual decision.
$has_only_code_point_maps{$addr} &=
$map =~ / ^ $code_point_re $/x;
# For the purposes of disambiguating between binary and other
# enumerations at the end, we keep track of the first three
# distinct property values. Once we get to three, we know
# it's not going to be binary, so no need to track more.
if (scalar keys %{$unique_maps{$addr}} < 3) {
$unique_maps{$addr}{main::standardize($map)} = 1;
}
}
}
# Add the mapping by calling our map table's method
return $map{$addr}->add_map($start, $end, $map, @_);
}
sub compute_type {
# Compute the type of the property: $ENUM, $STRING, or $BINARY. This
# should be called after the property is mostly filled with its maps.
# We have been keeping track of what the property values have been,
# and now have the necessary information to figure out the type.
my $self = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $addr = do { no overloading; pack 'J', $self; };
my $type = $type{$addr};
# If already have figured these out, no need to do so again, but we do
# a double check on ENUMS to make sure that a string property hasn't
# improperly been classified as an ENUM, so continue on with those.
return if $type == $STRING
|| $type == $BINARY
|| $type == $FORCED_BINARY;
# If every map is to a code point, is a string property.
if ($type == $UNKNOWN
&& ($has_only_code_point_maps{$addr}
|| (defined $map{$addr}->default_map
&& $map{$addr}->default_map eq "")))
{
$self->set_type($STRING);
}
else {
# Otherwise, it is to some sort of enumeration. (The case where
# it is a Unicode miscellaneous property, and treated like a
# string in this program is handled in add_map()). Distinguish
# between binary and some other enumeration type. Of course, if
# there are more than two values, it's not binary. But more
# subtle is the test that the default mapping is defined means it
# isn't binary. This in fact may change in the future if Unicode
# changes the way its data is structured. But so far, no binary
# properties ever have @missing lines for them, so the default map
# isn't defined for them. The few properties that are two-valued
# and aren't considered binary have the default map defined
# starting in Unicode 5.0, when the @missing lines appeared; and
# this program has special code to put in a default map for them
# for earlier than 5.0 releases.
if ($type == $ENUM
|| scalar keys %{$unique_maps{$addr}} > 2
|| defined $self->default_map)
{
my $tables = $self->tables;
my $count = $self->count;
if ($verbosity && $tables > 500 && $tables/$count > .1) {
Carp::my_carp_bug("It appears that $self should be a \$STRING property, not an \$ENUM because it has too many match tables: $tables\n");
}
$self->set_type($ENUM);
}
else {
$self->set_type($BINARY);
}
}
undef $unique_maps{$addr}; # Garbage collect
return;
}
sub set_fate {
my $self = shift;
my $fate = shift;
my $reason = shift; # Ignored unless suppressing
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $addr = do { no overloading; pack 'J', $self; };
if ($fate >= $SUPPRESSED) {
$why_suppressed{$self->complete_name} = $reason;
}
# Each table shares the property's fate, except that MAP_PROXIED
# doesn't affect match tables
$map{$addr}->set_fate($fate, $reason);
if ($fate != $MAP_PROXIED) {
foreach my $table ($map{$addr}, $self->tables) {
$table->set_fate($fate, $reason);
}
}
return;
}
# Most of the accessors for a property actually apply to its map table.
# Setup up accessor functions for those, referring to %map
for my $sub (qw(
add_alias
add_anomalous_entry
add_comment
add_conflicting
add_description
add_duplicate
add_note
aliases
comment
complete_name
containing_range
count
default_map
definition
delete_range
description
each_range
external_name
fate
file_path
format
initialize
inverse_list
is_empty
replacement_property
name
note
perl_extension
property
range_count
ranges
range_size_1
replace_map
reset_each_range
set_comment
set_default_map
set_file_path
set_final_comment
_set_format
set_range_size_1
set_status
set_to_output_map
short_name
status
status_info
to_output_map
type_of
value_of
write
))
# 'property' above is for symmetry, so that one can take
# the property of a property and get itself, and so don't
# have to distinguish between properties and tables in
# calling code
{
no strict "refs";
*$sub = sub {
use strict "refs";
my $self = shift;
no overloading;
return $map{pack 'J', $self}->$sub(@_);
}
}
} # End closure
package main;
sub display_chr {
# Converts an ordinal printable character value to a displayable string,
# using a dotted circle to hold combining characters.
my $ord = shift;
my $chr = chr $ord;
return $chr if $ccc->table(0)->contains($ord);
return "\x{25CC}$chr";
}
sub join_lines($) {
# Returns lines of the input joined together, so that they can be folded
# properly.
# This causes continuation lines to be joined together into one long line
# for folding. A continuation line is any line that doesn't begin with a
# space or "\b" (the latter is stripped from the output). This is so
# lines can be be in a HERE document so as to fit nicely in the terminal
# width, but be joined together in one long line, and then folded with
# indents, '#' prefixes, etc, properly handled.
# A blank separates the joined lines except if there is a break; an extra
# blank is inserted after a period ending a line.
# Initialize the return with the first line.
my ($return, @lines) = split "\n", shift;
# If the first line is null, it was an empty line, add the \n back in
$return = "\n" if $return eq "";
# Now join the remainder of the physical lines.
for my $line (@lines) {
# An empty line means wanted a blank line, so add two \n's to get that
# effect, and go to the next line.
if (length $line == 0) {
$return .= "\n\n";
next;
}
# Look at the last character of what we have so far.
my $previous_char = substr($return, -1, 1);
# And at the next char to be output.
my $next_char = substr($line, 0, 1);
if ($previous_char ne "\n") {
# Here didn't end wth a nl. If the next char a blank or \b, it
# means that here there is a break anyway. So add a nl to the
# output.
if ($next_char eq " " || $next_char eq "\b") {
$previous_char = "\n";
$return .= $previous_char;
}
# Add an extra space after periods.
$return .= " " if $previous_char eq '.';
}
# Here $previous_char is still the latest character to be output. If
# it isn't a nl, it means that the next line is to be a continuation
# line, with a blank inserted between them.
$return .= " " if $previous_char ne "\n";
# Get rid of any \b
substr($line, 0, 1) = "" if $next_char eq "\b";
# And append this next line.
$return .= $line;
}
return $return;
}
sub simple_fold($;$$$) {
# Returns a string of the input (string or an array of strings) folded
# into multiple-lines each of no more than $MAX_LINE_WIDTH characters plus
# a \n
# This is tailored for the kind of text written by this program,
# especially the pod file, which can have very long names with
# underscores in the middle, or words like AbcDefgHij.... We allow
# breaking in the middle of such constructs if the line won't fit
# otherwise. The break in such cases will come either just after an
# underscore, or just before one of the Capital letters.
local $to_trace = 0 if main::DEBUG;
my $line = shift;
my $prefix = shift; # Optional string to prepend to each output
# line
$prefix = "" unless defined $prefix;
my $hanging_indent = shift; # Optional number of spaces to indent
# continuation lines
$hanging_indent = 0 unless $hanging_indent;
my $right_margin = shift; # Optional number of spaces to narrow the
# total width by.
$right_margin = 0 unless defined $right_margin;
# Call carp with the 'nofold' option to avoid it from trying to call us
# recursively
Carp::carp_extra_args(\@_, 'nofold') if main::DEBUG && @_;
# The space available doesn't include what's automatically prepended
# to each line, or what's reserved on the right.
my $max = $MAX_LINE_WIDTH - length($prefix) - $right_margin;
# XXX Instead of using the 'nofold' perhaps better to look up the stack
if (DEBUG && $hanging_indent >= $max) {
Carp::my_carp("Too large a hanging indent ($hanging_indent); must be < $max. Using 0", 'nofold');
$hanging_indent = 0;
}
# First, split into the current physical lines.
my @line;
if (ref $line) { # Better be an array, because not bothering to
# test
foreach my $line (@{$line}) {
push @line, split /\n/, $line;
}
}
else {
@line = split /\n/, $line;
}
#local $to_trace = 1 if main::DEBUG;
trace "", join(" ", @line), "\n" if main::DEBUG && $to_trace;
# Look at each current physical line.
for (my $i = 0; $i < @line; $i++) {
Carp::my_carp("Tabs don't work well.", 'nofold') if $line[$i] =~ /\t/;
#local $to_trace = 1 if main::DEBUG;
trace "i=$i: $line[$i]\n" if main::DEBUG && $to_trace;
# Remove prefix, because will be added back anyway, don't want
# doubled prefix
$line[$i] =~ s/^$prefix//;
# Remove trailing space
$line[$i] =~ s/\s+\Z//;
# If the line is too long, fold it.
if (length $line[$i] > $max) {
my $remainder;
# Here needs to fold. Save the leading space in the line for
# later.
$line[$i] =~ /^ ( \s* )/x;
my $leading_space = $1;
trace "line length", length $line[$i], "; lead length", length($leading_space) if main::DEBUG && $to_trace;
# If character at final permissible position is white space,
# fold there, which will delete that white space
if (substr($line[$i], $max - 1, 1) =~ /\s/) {
$remainder = substr($line[$i], $max);
$line[$i] = substr($line[$i], 0, $max - 1);
}
else {
# Otherwise fold at an acceptable break char closest to
# the max length. Look at just the maximal initial
# segment of the line
my $segment = substr($line[$i], 0, $max - 1);
if ($segment =~
/^ ( .{$hanging_indent} # Don't look before the
# indent.
\ * # Don't look in leading
# blanks past the indent
[^ ] .* # Find the right-most
(?: # acceptable break:
[ \s = ] # space or equal
| - (?! [.0-9] ) # or non-unary minus.
) # $1 includes the character
)/x)
{
# Split into the initial part that fits, and remaining
# part of the input
$remainder = substr($line[$i], length $1);
$line[$i] = $1;
trace $line[$i] if DEBUG && $to_trace;
trace $remainder if DEBUG && $to_trace;
}
# If didn't find a good breaking spot, see if there is a
# not-so-good breaking spot. These are just after
# underscores or where the case changes from lower to
# upper. Use \a as a soft hyphen, but give up
# and don't break the line if there is actually a \a
# already in the input. We use an ascii character for the
# soft-hyphen to avoid any attempt by miniperl to try to
# access the files that this program is creating.
elsif ($segment !~ /\a/
&& ($segment =~ s/_/_\a/g
|| $segment =~ s/ ( [a-z] ) (?= [A-Z] )/$1\a/xg))
{
# Here were able to find at least one place to insert
# our substitute soft hyphen. Find the right-most one
# and replace it by a real hyphen.
trace $segment if DEBUG && $to_trace;
substr($segment,
rindex($segment, "\a"),
1) = '-';
# Then remove the soft hyphen substitutes.
$segment =~ s/\a//g;
trace $segment if DEBUG && $to_trace;
# And split into the initial part that fits, and
# remainder of the line
my $pos = rindex($segment, '-');
$remainder = substr($line[$i], $pos);
trace $remainder if DEBUG && $to_trace;
$line[$i] = substr($segment, 0, $pos + 1);
}
}
# Here we know if we can fold or not. If we can, $remainder
# is what remains to be processed in the next iteration.
if (defined $remainder) {
trace "folded='$line[$i]'" if main::DEBUG && $to_trace;
# Insert the folded remainder of the line as a new element
# of the array. (It may still be too long, but we will
# deal with that next time through the loop.) Omit any
# leading space in the remainder.
$remainder =~ s/^\s+//;
trace "remainder='$remainder'" if main::DEBUG && $to_trace;
# But then indent by whichever is larger of:
# 1) the leading space on the input line;
# 2) the hanging indent.
# This preserves indentation in the original line.
my $lead = ($leading_space)
? length $leading_space
: $hanging_indent;
$lead = max($lead, $hanging_indent);
splice @line, $i+1, 0, (" " x $lead) . $remainder;
}
}
# Ready to output the line. Get rid of any trailing space
# And prefix by the required $prefix passed in.
$line[$i] =~ s/\s+$//;
$line[$i] = "$prefix$line[$i]\n";
} # End of looping through all the lines.
return join "", @line;
}
sub property_ref { # Returns a reference to a property object.
return Property::property_ref(@_);
}
sub force_unlink ($) {
my $filename = shift;
return unless file_exists($filename);
return if CORE::unlink($filename);
# We might need write permission
chmod 0777, $filename;
CORE::unlink($filename) or Carp::my_carp("Couldn't unlink $filename. Proceeding anyway: $!");
return;
}
sub write ($$@) {
# Given a filename and references to arrays of lines, write the lines of
# each array to the file
# Filename can be given as an arrayref of directory names
return Carp::carp_too_few_args(\@_, 3) if main::DEBUG && @_ < 3;
my $file = shift;
my $use_utf8 = shift;
# Get into a single string if an array, and get rid of, in Unix terms, any
# leading '.'
$file= File::Spec->join(@$file) if ref $file eq 'ARRAY';
$file = File::Spec->canonpath($file);
# If has directories, make sure that they all exist
(undef, my $directories, undef) = File::Spec->splitpath($file);
File::Path::mkpath($directories) if $directories && ! -d $directories;
push @files_actually_output, $file;
force_unlink ($file);
my $OUT;
if (not open $OUT, ">", $file) {
Carp::my_carp("can't open $file for output. Skipping this file: $!");
return;
}
binmode $OUT, ":utf8" if $use_utf8;
while (defined (my $lines_ref = shift)) {
unless (@$lines_ref) {
Carp::my_carp("An array of lines for writing to file '$file' is empty; writing it anyway;");
}
print $OUT @$lines_ref or die Carp::my_carp("write to '$file' failed: $!");
}
close $OUT or die Carp::my_carp("close '$file' failed: $!");
print "$file written.\n" if $verbosity >= $VERBOSE;
return;
}
sub Standardize($) {
# This converts the input name string into a standardized equivalent to
# use internally.
my $name = shift;
unless (defined $name) {
Carp::my_carp_bug("Standardize() called with undef. Returning undef.");
return;
}
# Remove any leading or trailing white space
$name =~ s/^\s+//g;
$name =~ s/\s+$//g;
# Convert interior white space and hyphens into underscores.
$name =~ s/ (?<= .) [ -]+ (.) /_$1/xg;
# Capitalize the letter following an underscore, and convert a sequence of
# multiple underscores to a single one
$name =~ s/ (?<= .) _+ (.) /_\u$1/xg;
# And capitalize the first letter, but not for the special cjk ones.
$name = ucfirst($name) unless $name =~ /^k[A-Z]/;
return $name;
}
sub standardize ($) {
# Returns a lower-cased standardized name, without underscores. This form
# is chosen so that it can distinguish between any real versus superficial
# Unicode name differences. It relies on the fact that Unicode doesn't
# have interior underscores, white space, nor dashes in any
# stricter-matched name. It should not be used on Unicode code point
# names (the Name property), as they mostly, but not always follow these
# rules.
my $name = Standardize(shift);
return if !defined $name;
$name =~ s/ (?<= .) _ (?= . ) //xg;
return lc $name;
}
sub utf8_heavy_name ($$) {
# Returns the name that utf8_heavy.pl will use to find a table. XXX
# perhaps this function should be placed somewhere, like Heavy.pl so that
# utf8_heavy can use it directly without duplicating code that can get
# out-of sync.
my $table = shift;
my $alias = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $property = $table->property;
$property = ($property == $perl)
? "" # 'perl' is never explicitly stated
: standardize($property->name) . '=';
if ($alias->loose_match) {
return $property . standardize($alias->name);
}
else {
return lc ($property . $alias->name);
}
return;
}
{ # Closure
my $indent_increment = " " x (($debugging_build) ? 2 : 0);
%main::already_output = ();
$main::simple_dumper_nesting = 0;
sub simple_dumper {
# Like Simple Data::Dumper. Good enough for our needs. We can't use
# the real thing as we have to run under miniperl.
# It is designed so that on input it is at the beginning of a line,
# and the final thing output in any call is a trailing ",\n".
my $item = shift;
my $indent = shift;
$indent = "" if ! $debugging_build || ! defined $indent;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
# nesting level is localized, so that as the call stack pops, it goes
# back to the prior value.
local $main::simple_dumper_nesting = $main::simple_dumper_nesting;
local %main::already_output = %main::already_output;
$main::simple_dumper_nesting++;
#print STDERR __LINE__, ": $main::simple_dumper_nesting: $indent$item\n";
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
# Determine the indent for recursive calls.
my $next_indent = $indent . $indent_increment;
my $output;
if (! ref $item) {
# Dump of scalar: just output it in quotes if not a number. To do
# so we must escape certain characters, and therefore need to
# operate on a copy to avoid changing the original
my $copy = $item;
$copy = $UNDEF unless defined $copy;
# Quote non-integers (integers also have optional leading '-')
if ($copy eq "" || $copy !~ /^ -? \d+ $/x) {
# Escape apostrophe and backslash
$copy =~ s/ ( ['\\] ) /\\$1/xg;
$copy = "'$copy'";
}
$output = "$indent$copy,\n";
}
else {
# Keep track of cycles in the input, and refuse to infinitely loop
my $addr = do { no overloading; pack 'J', $item; };
if (defined $main::already_output{$addr}) {
return "${indent}ALREADY OUTPUT: $item\n";
}
$main::already_output{$addr} = $item;
if (ref $item eq 'ARRAY') {
my $using_brackets;
$output = $indent;
if ($main::simple_dumper_nesting > 1) {
$output .= '[';
$using_brackets = 1;
}
else {
$using_brackets = 0;
}
# If the array is empty, put the closing bracket on the same
# line. Otherwise, recursively add each array element
if (@$item == 0) {
$output .= " ";
}
else {
$output .= "\n";
for (my $i = 0; $i < @$item; $i++) {
# Indent array elements one level
$output .= &simple_dumper($item->[$i], $next_indent);
next if ! $debugging_build;
$output =~ s/\n$//; # Remove any trailing nl so
$output .= " # [$i]\n"; # as to add a comment giving
# the array index
}
$output .= $indent; # Indent closing ']' to orig level
}
$output .= ']' if $using_brackets;
$output .= ",\n";
}
elsif (ref $item eq 'HASH') {
my $is_first_line;
my $using_braces;
my $body_indent;
# No surrounding braces at top level
$output .= $indent;
if ($main::simple_dumper_nesting > 1) {
$output .= "{\n";
$is_first_line = 0;
$body_indent = $next_indent;
$next_indent .= $indent_increment;
$using_braces = 1;
}
else {
$is_first_line = 1;
$body_indent = $indent;
$using_braces = 0;
}
# Output hashes sorted alphabetically instead of apparently
# random. Use caseless alphabetic sort
foreach my $key (sort { lc $a cmp lc $b } keys %$item)
{
if ($is_first_line) {
$is_first_line = 0;
}
else {
$output .= "$body_indent";
}
# The key must be a scalar, but this recursive call quotes
# it
$output .= &simple_dumper($key);
# And change the trailing comma and nl to the hash fat
# comma for clarity, and so the value can be on the same
# line
$output =~ s/,\n$/ => /;
# Recursively call to get the value's dump.
my $next = &simple_dumper($item->{$key}, $next_indent);
# If the value is all on one line, remove its indent, so
# will follow the => immediately. If it takes more than
# one line, start it on a new line.
if ($next !~ /\n.*\n/) {
$next =~ s/^ *//;
}
else {
$output .= "\n";
}
$output .= $next;
}
$output .= "$indent},\n" if $using_braces;
}
elsif (ref $item eq 'CODE' || ref $item eq 'GLOB') {
$output = $indent . ref($item) . "\n";
# XXX see if blessed
}
elsif ($item->can('dump')) {
# By convention in this program, objects furnish a 'dump'
# method. Since not doing any output at this level, just pass
# on the input indent
$output = $item->dump($indent);
}
else {
Carp::my_carp("Can't cope with dumping a " . ref($item) . ". Skipping.");
}
}
return $output;
}
}
sub dump_inside_out {
# Dump inside-out hashes in an object's state by converting them to a
# regular hash and then calling simple_dumper on that.
my $object = shift;
my $fields_ref = shift;
my $addr = do { no overloading; pack 'J', $object; };
my %hash;
foreach my $key (keys %$fields_ref) {
$hash{$key} = $fields_ref->{$key}{$addr};
}
return simple_dumper(\%hash, @_);
}
sub _operator_dot {
# Overloaded '.' method that is common to all packages. It uses the
# package's stringify method.
my $self = shift;
my $other = shift;
my $reversed = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
$other = "" unless defined $other;
foreach my $which (\$self, \$other) {
next unless ref $$which;
if ($$which->can('_operator_stringify')) {
$$which = $$which->_operator_stringify;
}
else {
my $ref = ref $$which;
my $addr = do { no overloading; pack 'J', $$which; };
$$which = "$ref ($addr)";
}
}
return ($reversed)
? "$other$self"
: "$self$other";
}
sub _operator_dot_equal {
# Overloaded '.=' method that is common to all packages.
my $self = shift;
my $other = shift;
my $reversed = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
$other = "" unless defined $other;
if ($reversed) {
return $other .= "$self";
}
else {
return "$self" . "$other";
}
}
sub _operator_equal {
# Generic overloaded '==' routine. To be equal, they must be the exact
# same object
my $self = shift;
my $other = shift;
return 0 unless defined $other;
return 0 unless ref $other;
no overloading;
return $self == $other;
}
sub _operator_not_equal {
my $self = shift;
my $other = shift;
return ! _operator_equal($self, $other);
}
sub substitute_PropertyAliases($) {
# Deal with early releases that don't have the crucial PropertyAliases.txt
# file.
my $file_object = shift;
$file_object->insert_lines(get_old_property_aliases());
process_PropertyAliases($file_object);
}
sub process_PropertyAliases($) {
# This reads in the PropertyAliases.txt file, which contains almost all
# the character properties in Unicode and their equivalent aliases:
# scf ; Simple_Case_Folding ; sfc
#
# Field 0 is the preferred short name for the property.
# Field 1 is the full name.
# Any succeeding ones are other accepted names.
my $file= shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
# Add any cjk properties that may have been defined.
$file->insert_lines(@cjk_properties);
while ($file->next_line) {
my @data = split /\s*;\s*/;
my $full = $data[1];
# This line is defective in early Perls. The property in Unihan.txt
# is kRSUnicode.
if ($full eq 'Unicode_Radical_Stroke' && @data < 3) {
push @data, qw(cjkRSUnicode kRSUnicode);
}
my $this = Property->new($data[0], Full_Name => $full);
$this->set_fate($SUPPRESSED, $why_suppressed{$full})
if $why_suppressed{$full};
# Start looking for more aliases after these two.
for my $i (2 .. @data - 1) {
$this->add_alias($data[$i]);
}
}
my $scf = property_ref("Simple_Case_Folding");
$scf->add_alias("scf");
$scf->add_alias("sfc");
return;
}
sub finish_property_setup {
# Finishes setting up after PropertyAliases.
my $file = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
# This entry was missing from this file in earlier Unicode versions
if (-e 'Jamo.txt' && ! defined property_ref('JSN')) {
Property->new('JSN', Full_Name => 'Jamo_Short_Name');
}
# These are used so much, that we set globals for them.
$gc = property_ref('General_Category');
$block = property_ref('Block');
$script = property_ref('Script');
$age = property_ref('Age');
# Perl adds this alias.
$gc->add_alias('Category');
# Unicode::Normalize expects this file with this name and directory.
$ccc = property_ref('Canonical_Combining_Class');
if (defined $ccc) {
$ccc->set_file('CombiningClass');
$ccc->set_directory(File::Spec->curdir());
}
# These two properties aren't actually used in the core, but unfortunately
# the names just above that are in the core interfere with these, so
# choose different names. These aren't a problem unless the map tables
# for these files get written out.
my $lowercase = property_ref('Lowercase');
$lowercase->set_file('IsLower') if defined $lowercase;
my $uppercase = property_ref('Uppercase');
$uppercase->set_file('IsUpper') if defined $uppercase;
# Set up the hard-coded default mappings, but only on properties defined
# for this release
foreach my $property (keys %default_mapping) {
my $property_object = property_ref($property);
next if ! defined $property_object;
my $default_map = $default_mapping{$property};
$property_object->set_default_map($default_map);
# A map of <code point> implies the property is string.
if ($property_object->type == $UNKNOWN
&& $default_map eq $CODE_POINT)
{
$property_object->set_type($STRING);
}
}
# The following use the Multi_Default class to create objects for
# defaults.
# Bidi class has a complicated default, but the derived file takes care of
# the complications, leaving just 'L'.
if (file_exists("${EXTRACTED}DBidiClass.txt")) {
property_ref('Bidi_Class')->set_default_map('L');
}
else {
my $default;
# The derived file was introduced in 3.1.1. The values below are
# taken from table 3-8, TUS 3.0
my $default_R =
'my $default = Range_List->new;
$default->add_range(0x0590, 0x05FF);
$default->add_range(0xFB1D, 0xFB4F);'
;
# The defaults apply only to unassigned characters
$default_R .= '$gc->table("Unassigned") & $default;';
if ($v_version lt v3.0.0) {
$default = Multi_Default->new(R => $default_R, 'L');
}
else {
# AL apparently not introduced until 3.0: TUS 2.x references are
# not on-line to check it out
my $default_AL =
'my $default = Range_List->new;
$default->add_range(0x0600, 0x07BF);
$default->add_range(0xFB50, 0xFDFF);
$default->add_range(0xFE70, 0xFEFF);'
;
# Non-character code points introduced in this release; aren't AL
if ($v_version ge 3.1.0) {
$default_AL .= '$default->delete_range(0xFDD0, 0xFDEF);';
}
$default_AL .= '$gc->table("Unassigned") & $default';
$default = Multi_Default->new(AL => $default_AL,
R => $default_R,
'L');
}
property_ref('Bidi_Class')->set_default_map($default);
}
# Joining type has a complicated default, but the derived file takes care
# of the complications, leaving just 'U' (or Non_Joining), except the file
# is bad in 3.1.0
if (file_exists("${EXTRACTED}DJoinType.txt") || -e 'ArabicShaping.txt') {
if (file_exists("${EXTRACTED}DJoinType.txt") && $v_version ne 3.1.0) {
property_ref('Joining_Type')->set_default_map('Non_Joining');
}
else {
# Otherwise, there are not one, but two possibilities for the
# missing defaults: T and U.
# The missing defaults that evaluate to T are given by:
# T = Mn + Cf - ZWNJ - ZWJ
# where Mn and Cf are the general category values. In other words,
# any non-spacing mark or any format control character, except
# U+200C ZERO WIDTH NON-JOINER (joining type U) and U+200D ZERO
# WIDTH JOINER (joining type C).
my $default = Multi_Default->new(
'T' => '$gc->table("Mn") + $gc->table("Cf") - 0x200C - 0x200D',
'Non_Joining');
property_ref('Joining_Type')->set_default_map($default);
}
}
# Line break has a complicated default in early releases. It is 'Unknown'
# for non-assigned code points; 'AL' for assigned.
if (file_exists("${EXTRACTED}DLineBreak.txt") || -e 'LineBreak.txt') {
my $lb = property_ref('Line_Break');
if (file_exists("${EXTRACTED}DLineBreak.txt")) {
$lb->set_default_map('Unknown');
}
else {
my $default = Multi_Default->new('AL' => '~ $gc->table("Cn")',
'Unknown',
);
$lb->set_default_map($default);
}
}
# For backwards compatibility with applications that may read the mapping
# file directly (it was documented in 5.12 and 5.14 as being thusly
# usable), keep it from being adjusted. (range_size_1 is
# used to force the traditional format.)
if (defined (my $nfkc_cf = property_ref('NFKC_Casefold'))) {
$nfkc_cf->set_to_output_map($EXTERNAL_MAP);
$nfkc_cf->set_range_size_1(1);
}
if (defined (my $bmg = property_ref('Bidi_Mirroring_Glyph'))) {
$bmg->set_to_output_map($EXTERNAL_MAP);
$bmg->set_range_size_1(1);
}
property_ref('Numeric_Value')->set_to_output_map($OUTPUT_ADJUSTED);
return;
}
sub get_old_property_aliases() {
# Returns what would be in PropertyAliases.txt if it existed in very old
# versions of Unicode. It was derived from the one in 3.2, and pared
# down based on the data that was actually in the older releases.
# An attempt was made to use the existence of files to mean inclusion or
# not of various aliases, but if this was not sufficient, using version
# numbers was resorted to.
my @return;
# These are to be used in all versions (though some are constructed by
# this program if missing)
push @return, split /\n/, <<'END';
bc ; Bidi_Class
Bidi_M ; Bidi_Mirrored
cf ; Case_Folding
ccc ; Canonical_Combining_Class
dm ; Decomposition_Mapping
dt ; Decomposition_Type
gc ; General_Category
isc ; ISO_Comment
lc ; Lowercase_Mapping
na ; Name
na1 ; Unicode_1_Name
nt ; Numeric_Type
nv ; Numeric_Value
scf ; Simple_Case_Folding
slc ; Simple_Lowercase_Mapping
stc ; Simple_Titlecase_Mapping
suc ; Simple_Uppercase_Mapping
tc ; Titlecase_Mapping
uc ; Uppercase_Mapping
END
if (-e 'Blocks.txt') {
push @return, "blk ; Block\n";
}
if (-e 'ArabicShaping.txt') {
push @return, split /\n/, <<'END';
jg ; Joining_Group
jt ; Joining_Type
END
}
if (-e 'PropList.txt') {
# This first set is in the original old-style proplist.
push @return, split /\n/, <<'END';
Bidi_C ; Bidi_Control
Dash ; Dash
Dia ; Diacritic
Ext ; Extender
Hex ; Hex_Digit
Hyphen ; Hyphen
IDC ; ID_Continue
Ideo ; Ideographic
Join_C ; Join_Control
Math ; Math
QMark ; Quotation_Mark
Term ; Terminal_Punctuation
WSpace ; White_Space
END
# The next sets were added later
if ($v_version ge v3.0.0) {
push @return, split /\n/, <<'END';
Upper ; Uppercase
Lower ; Lowercase
END
}
if ($v_version ge v3.0.1) {
push @return, split /\n/, <<'END';
NChar ; Noncharacter_Code_Point
END
}
# The next sets were added in the new-style
if ($v_version ge v3.1.0) {
push @return, split /\n/, <<'END';
OAlpha ; Other_Alphabetic
OLower ; Other_Lowercase
OMath ; Other_Math
OUpper ; Other_Uppercase
END
}
if ($v_version ge v3.1.1) {
push @return, "AHex ; ASCII_Hex_Digit\n";
}
}
if (-e 'EastAsianWidth.txt') {
push @return, "ea ; East_Asian_Width\n";
}
if (-e 'CompositionExclusions.txt') {
push @return, "CE ; Composition_Exclusion\n";
}
if (-e 'LineBreak.txt') {
push @return, "lb ; Line_Break\n";
}
if (-e 'BidiMirroring.txt') {
push @return, "bmg ; Bidi_Mirroring_Glyph\n";
}
if (-e 'Scripts.txt') {
push @return, "sc ; Script\n";
}
if (-e 'DNormalizationProps.txt') {
push @return, split /\n/, <<'END';
Comp_Ex ; Full_Composition_Exclusion
FC_NFKC ; FC_NFKC_Closure
NFC_QC ; NFC_Quick_Check
NFD_QC ; NFD_Quick_Check
NFKC_QC ; NFKC_Quick_Check
NFKD_QC ; NFKD_Quick_Check
XO_NFC ; Expands_On_NFC
XO_NFD ; Expands_On_NFD
XO_NFKC ; Expands_On_NFKC
XO_NFKD ; Expands_On_NFKD
END
}
if (-e 'DCoreProperties.txt') {
push @return, split /\n/, <<'END';
Alpha ; Alphabetic
IDS ; ID_Start
XIDC ; XID_Continue
XIDS ; XID_Start
END
# These can also appear in some versions of PropList.txt
push @return, "Lower ; Lowercase\n"
unless grep { $_ =~ /^Lower\b/} @return;
push @return, "Upper ; Uppercase\n"
unless grep { $_ =~ /^Upper\b/} @return;
}
# This flag requires the DAge.txt file to be copied into the directory.
if (DEBUG && $compare_versions) {
push @return, 'age ; Age';
}
return @return;
}
sub substitute_PropValueAliases($) {
# Deal with early releases that don't have the crucial
# PropValueAliases.txt file.
my $file_object = shift;
$file_object->insert_lines(get_old_property_value_aliases());
process_PropValueAliases($file_object);
}
sub process_PropValueAliases {
# This file contains values that properties look like:
# bc ; AL ; Arabic_Letter
# blk; n/a ; Greek_And_Coptic ; Greek
#
# Field 0 is the property.
# Field 1 is the short name of a property value or 'n/a' if no
# short name exists;
# Field 2 is the full property value name;
# Any other fields are more synonyms for the property value.
# Purely numeric property values are omitted from the file; as are some
# others, fewer and fewer in later releases
# Entries for the ccc property have an extra field before the
# abbreviation:
# ccc; 0; NR ; Not_Reordered
# It is the numeric value that the names are synonyms for.
# There are comment entries for values missing from this file:
# # @missing: 0000..10FFFF; ISO_Comment; <none>
# # @missing: 0000..10FFFF; Lowercase_Mapping; <code point>
my $file= shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
if ($v_version lt 4.0.0) {
$file->insert_lines(split /\n/, <<'END'
Hangul_Syllable_Type; L ; Leading_Jamo
Hangul_Syllable_Type; LV ; LV_Syllable
Hangul_Syllable_Type; LVT ; LVT_Syllable
Hangul_Syllable_Type; NA ; Not_Applicable
Hangul_Syllable_Type; T ; Trailing_Jamo
Hangul_Syllable_Type; V ; Vowel_Jamo
END
);
}
if ($v_version lt 4.1.0) {
$file->insert_lines(split /\n/, <<'END'
_Perl_GCB; CN ; Control
_Perl_GCB; CR ; CR
_Perl_GCB; EX ; Extend
_Perl_GCB; L ; L
_Perl_GCB; LF ; LF
_Perl_GCB; LV ; LV
_Perl_GCB; LVT ; LVT
_Perl_GCB; T ; T
_Perl_GCB; V ; V
_Perl_GCB; XX ; Other
END
);
}
# Add any explicit cjk values
$file->insert_lines(@cjk_property_values);
# This line is used only for testing the code that checks for name
# conflicts. There is a script Inherited, and when this line is executed
# it causes there to be a name conflict with the 'Inherited' that this
# program generates for this block property value
#$file->insert_lines('blk; n/a; Herited');
# Process each line of the file ...
while ($file->next_line) {
# Fix typo in input file
s/CCC133/CCC132/g if $v_version eq v6.1.0;
my ($property, @data) = split /\s*;\s*/;
# The ccc property has an extra field at the beginning, which is the
# numeric value. Move it to be after the other two, mnemonic, fields,
# so that those will be used as the property value's names, and the
# number will be an extra alias. (Rightmost splice removes field 1-2,
# returning them in a slice; left splice inserts that before anything,
# thus shifting the former field 0 to after them.)
splice (@data, 0, 0, splice(@data, 1, 2)) if $property eq 'ccc';
if ($v_version le v5.0.0 && $property eq 'blk' && $data[1] =~ /-/) {
my $new_style = $data[1] =~ s/-/_/gr;
splice @data, 1, 0, $new_style;
}
# Field 0 is a short name unless "n/a"; field 1 is the full name. If
# there is no short name, use the full one in element 1
if ($data[0] eq "n/a") {
$data[0] = $data[1];
}
elsif ($data[0] ne $data[1]
&& standardize($data[0]) eq standardize($data[1])
&& $data[1] !~ /[[:upper:]]/)
{
# Also, there is a bug in the file in which "n/a" is omitted, and
# the two fields are identical except for case, and the full name
# is all lower case. Copy the "short" name unto the full one to
# give it some upper case.
$data[1] = $data[0];
}
# Earlier releases had the pseudo property 'qc' that should expand to
# the ones that replace it below.
if ($property eq 'qc') {
if (lc $data[0] eq 'y') {
$file->insert_lines('NFC_QC; Y ; Yes',
'NFD_QC; Y ; Yes',
'NFKC_QC; Y ; Yes',
'NFKD_QC; Y ; Yes',
);
}
elsif (lc $data[0] eq 'n') {
$file->insert_lines('NFC_QC; N ; No',
'NFD_QC; N ; No',
'NFKC_QC; N ; No',
'NFKD_QC; N ; No',
);
}
elsif (lc $data[0] eq 'm') {
$file->insert_lines('NFC_QC; M ; Maybe',
'NFKC_QC; M ; Maybe',
);
}
else {
$file->carp_bad_line("qc followed by unexpected '$data[0]");
}
next;
}
# The first field is the short name, 2nd is the full one.
my $property_object = property_ref($property);
my $table = $property_object->add_match_table($data[0],
Full_Name => $data[1]);
# Start looking for more aliases after these two.
for my $i (2 .. @data - 1) {
$table->add_alias($data[$i]);
}
} # End of looping through the file
# As noted in the comments early in the program, it generates tables for
# the default values for all releases, even those for which the concept
# didn't exist at the time. Here we add those if missing.
if (defined $age && ! defined $age->table('Unassigned')) {
$age->add_match_table('Unassigned');
}
$block->add_match_table('No_Block') if -e 'Blocks.txt'
&& ! defined $block->table('No_Block');
# Now set the default mappings of the properties from the file. This is
# done after the loop because a number of properties have only @missings
# entries in the file, and may not show up until the end.
my @defaults = $file->get_missings;
foreach my $default_ref (@defaults) {
my $default = $default_ref->[0];
my $property = property_ref($default_ref->[1]);
$property->set_default_map($default);
}
return;
}
sub get_old_property_value_aliases () {
# Returns what would be in PropValueAliases.txt if it existed in very old
# versions of Unicode. It was derived from the one in 3.2, and pared
# down. An attempt was made to use the existence of files to mean
# inclusion or not of various aliases, but if this was not sufficient,
# using version numbers was resorted to.
my @return = split /\n/, <<'END';
bc ; AN ; Arabic_Number
bc ; B ; Paragraph_Separator
bc ; CS ; Common_Separator
bc ; EN ; European_Number
bc ; ES ; European_Separator
bc ; ET ; European_Terminator
bc ; L ; Left_To_Right
bc ; ON ; Other_Neutral
bc ; R ; Right_To_Left
bc ; WS ; White_Space
Bidi_M; N; No; F; False
Bidi_M; Y; Yes; T; True
# The standard combining classes are very much different in v1, so only use
# ones that look right (not checked thoroughly)
ccc; 0; NR ; Not_Reordered
ccc; 1; OV ; Overlay
ccc; 7; NK ; Nukta
ccc; 8; KV ; Kana_Voicing
ccc; 9; VR ; Virama
ccc; 202; ATBL ; Attached_Below_Left
ccc; 216; ATAR ; Attached_Above_Right
ccc; 218; BL ; Below_Left
ccc; 220; B ; Below
ccc; 222; BR ; Below_Right
ccc; 224; L ; Left
ccc; 228; AL ; Above_Left
ccc; 230; A ; Above
ccc; 232; AR ; Above_Right
ccc; 234; DA ; Double_Above
dt ; can ; canonical
dt ; enc ; circle
dt ; fin ; final
dt ; font ; font
dt ; fra ; fraction
dt ; init ; initial
dt ; iso ; isolated
dt ; med ; medial
dt ; n/a ; none
dt ; nb ; noBreak
dt ; sqr ; square
dt ; sub ; sub
dt ; sup ; super
gc ; C ; Other # Cc | Cf | Cn | Co | Cs
gc ; Cc ; Control
gc ; Cn ; Unassigned
gc ; Co ; Private_Use
gc ; L ; Letter # Ll | Lm | Lo | Lt | Lu
gc ; LC ; Cased_Letter # Ll | Lt | Lu
gc ; Ll ; Lowercase_Letter
gc ; Lm ; Modifier_Letter
gc ; Lo ; Other_Letter
gc ; Lu ; Uppercase_Letter
gc ; M ; Mark # Mc | Me | Mn
gc ; Mc ; Spacing_Mark
gc ; Mn ; Nonspacing_Mark
gc ; N ; Number # Nd | Nl | No
gc ; Nd ; Decimal_Number
gc ; No ; Other_Number
gc ; P ; Punctuation # Pc | Pd | Pe | Pf | Pi | Po | Ps
gc ; Pd ; Dash_Punctuation
gc ; Pe ; Close_Punctuation
gc ; Po ; Other_Punctuation
gc ; Ps ; Open_Punctuation
gc ; S ; Symbol # Sc | Sk | Sm | So
gc ; Sc ; Currency_Symbol
gc ; Sm ; Math_Symbol
gc ; So ; Other_Symbol
gc ; Z ; Separator # Zl | Zp | Zs
gc ; Zl ; Line_Separator
gc ; Zp ; Paragraph_Separator
gc ; Zs ; Space_Separator
nt ; de ; Decimal
nt ; di ; Digit
nt ; n/a ; None
nt ; nu ; Numeric
END
if (-e 'ArabicShaping.txt') {
push @return, split /\n/, <<'END';
jg ; n/a ; AIN
jg ; n/a ; ALEF
jg ; n/a ; DAL
jg ; n/a ; GAF
jg ; n/a ; LAM
jg ; n/a ; MEEM
jg ; n/a ; NO_JOINING_GROUP
jg ; n/a ; NOON
jg ; n/a ; QAF
jg ; n/a ; SAD
jg ; n/a ; SEEN
jg ; n/a ; TAH
jg ; n/a ; WAW
jt ; C ; Join_Causing
jt ; D ; Dual_Joining
jt ; L ; Left_Joining
jt ; R ; Right_Joining
jt ; U ; Non_Joining
jt ; T ; Transparent
END
if ($v_version ge v3.0.0) {
push @return, split /\n/, <<'END';
jg ; n/a ; ALAPH
jg ; n/a ; BEH
jg ; n/a ; BETH
jg ; n/a ; DALATH_RISH
jg ; n/a ; E
jg ; n/a ; FEH
jg ; n/a ; FINAL_SEMKATH
jg ; n/a ; GAMAL
jg ; n/a ; HAH
jg ; n/a ; HAMZA_ON_HEH_GOAL
jg ; n/a ; HE
jg ; n/a ; HEH
jg ; n/a ; HEH_GOAL
jg ; n/a ; HETH
jg ; n/a ; KAF
jg ; n/a ; KAPH
jg ; n/a ; KNOTTED_HEH
jg ; n/a ; LAMADH
jg ; n/a ; MIM
jg ; n/a ; NUN
jg ; n/a ; PE
jg ; n/a ; QAPH
jg ; n/a ; REH
jg ; n/a ; REVERSED_PE
jg ; n/a ; SADHE
jg ; n/a ; SEMKATH
jg ; n/a ; SHIN
jg ; n/a ; SWASH_KAF
jg ; n/a ; TAW
jg ; n/a ; TEH_MARBUTA
jg ; n/a ; TETH
jg ; n/a ; YEH
jg ; n/a ; YEH_BARREE
jg ; n/a ; YEH_WITH_TAIL
jg ; n/a ; YUDH
jg ; n/a ; YUDH_HE
jg ; n/a ; ZAIN
END
}
}
if (-e 'EastAsianWidth.txt') {
push @return, split /\n/, <<'END';
ea ; A ; Ambiguous
ea ; F ; Fullwidth
ea ; H ; Halfwidth
ea ; N ; Neutral
ea ; Na ; Narrow
ea ; W ; Wide
END
}
if (-e 'LineBreak.txt' || -e 'LBsubst.txt') {
my @lb = split /\n/, <<'END';
lb ; AI ; Ambiguous
lb ; AL ; Alphabetic
lb ; B2 ; Break_Both
lb ; BA ; Break_After
lb ; BB ; Break_Before
lb ; BK ; Mandatory_Break
lb ; CB ; Contingent_Break
lb ; CL ; Close_Punctuation
lb ; CM ; Combining_Mark
lb ; CR ; Carriage_Return
lb ; EX ; Exclamation
lb ; GL ; Glue
lb ; HY ; Hyphen
lb ; ID ; Ideographic
lb ; IN ; Inseperable
lb ; IS ; Infix_Numeric
lb ; LF ; Line_Feed
lb ; NS ; Nonstarter
lb ; NU ; Numeric
lb ; OP ; Open_Punctuation
lb ; PO ; Postfix_Numeric
lb ; PR ; Prefix_Numeric
lb ; QU ; Quotation
lb ; SA ; Complex_Context
lb ; SG ; Surrogate
lb ; SP ; Space
lb ; SY ; Break_Symbols
lb ; XX ; Unknown
lb ; ZW ; ZWSpace
END
# If this Unicode version predates the lb property, we use our
# substitute one
if (-e 'LBsubst.txt') {
$_ = s/^lb/_Perl_LB/r for @lb;
}
push @return, @lb;
}
if (-e 'DNormalizationProps.txt') {
push @return, split /\n/, <<'END';
qc ; M ; Maybe
qc ; N ; No
qc ; Y ; Yes
END
}
if (-e 'Scripts.txt') {
push @return, split /\n/, <<'END';
sc ; Arab ; Arabic
sc ; Armn ; Armenian
sc ; Beng ; Bengali
sc ; Bopo ; Bopomofo
sc ; Cans ; Canadian_Aboriginal
sc ; Cher ; Cherokee
sc ; Cyrl ; Cyrillic
sc ; Deva ; Devanagari
sc ; Dsrt ; Deseret
sc ; Ethi ; Ethiopic
sc ; Geor ; Georgian
sc ; Goth ; Gothic
sc ; Grek ; Greek
sc ; Gujr ; Gujarati
sc ; Guru ; Gurmukhi
sc ; Hang ; Hangul
sc ; Hani ; Han
sc ; Hebr ; Hebrew
sc ; Hira ; Hiragana
sc ; Ital ; Old_Italic
sc ; Kana ; Katakana
sc ; Khmr ; Khmer
sc ; Knda ; Kannada
sc ; Laoo ; Lao
sc ; Latn ; Latin
sc ; Mlym ; Malayalam
sc ; Mong ; Mongolian
sc ; Mymr ; Myanmar
sc ; Ogam ; Ogham
sc ; Orya ; Oriya
sc ; Qaai ; Inherited
sc ; Runr ; Runic
sc ; Sinh ; Sinhala
sc ; Syrc ; Syriac
sc ; Taml ; Tamil
sc ; Telu ; Telugu
sc ; Thaa ; Thaana
sc ; Thai ; Thai
sc ; Tibt ; Tibetan
sc ; Yiii ; Yi
sc ; Zyyy ; Common
END
}
if ($v_version ge v2.0.0) {
push @return, split /\n/, <<'END';
dt ; com ; compat
dt ; nar ; narrow
dt ; sml ; small
dt ; vert ; vertical
dt ; wide ; wide
gc ; Cf ; Format
gc ; Cs ; Surrogate
gc ; Lt ; Titlecase_Letter
gc ; Me ; Enclosing_Mark
gc ; Nl ; Letter_Number
gc ; Pc ; Connector_Punctuation
gc ; Sk ; Modifier_Symbol
END
}
if ($v_version ge v2.1.2) {
push @return, "bc ; S ; Segment_Separator\n";
}
if ($v_version ge v2.1.5) {
push @return, split /\n/, <<'END';
gc ; Pf ; Final_Punctuation
gc ; Pi ; Initial_Punctuation
END
}
if ($v_version ge v2.1.8) {
push @return, "ccc; 240; IS ; Iota_Subscript\n";
}
if ($v_version ge v3.0.0) {
push @return, split /\n/, <<'END';
bc ; AL ; Arabic_Letter
bc ; BN ; Boundary_Neutral
bc ; LRE ; Left_To_Right_Embedding
bc ; LRO ; Left_To_Right_Override
bc ; NSM ; Nonspacing_Mark
bc ; PDF ; Pop_Directional_Format
bc ; RLE ; Right_To_Left_Embedding
bc ; RLO ; Right_To_Left_Override
ccc; 233; DB ; Double_Below
END
}
if ($v_version ge v3.1.0) {
push @return, "ccc; 226; R ; Right\n";
}
return @return;
}
sub process_NormalizationsTest {
# Each line looks like:
# source code point; NFC; NFD; NFKC; NFKD
# e.g.
# 1E0A;1E0A;0044 0307;1E0A;0044 0307;
my $file= shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
# Process each line of the file ...
while ($file->next_line) {
next if /^@/;
my ($c1, $c2, $c3, $c4, $c5) = split /\s*;\s*/;
foreach my $var (\$c1, \$c2, \$c3, \$c4, \$c5) {
$$var = pack "U0U*", map { hex } split " ", $$var;
$$var =~ s/(\\)/$1$1/g;
}
push @normalization_tests,
"Test_N(q$c1, q$c2, q$c3, q$c4, q$c5);\n";
} # End of looping through the file
}
sub output_perl_charnames_line ($$) {
# Output the entries in Perl_charnames specially, using 5 digits instead
# of four. This makes the entries a constant length, and simplifies
# charnames.pm which this table is for. Unicode can have 6 digit
# ordinals, but they are all private use or noncharacters which do not
# have names, so won't be in this table.
return sprintf "%05X\t%s\n", $_[0], $_[1];
}
{ # Closure
# These are constants to the $property_info hash in this subroutine, to
# avoid using a quoted-string which might have a typo.
my $TYPE = 'type';
my $DEFAULT_MAP = 'default_map';
my $DEFAULT_TABLE = 'default_table';
my $PSEUDO_MAP_TYPE = 'pseudo_map_type';
my $MISSINGS = 'missings';
sub process_generic_property_file {
# This processes a file containing property mappings and puts them
# into internal map tables. It should be used to handle any property
# files that have mappings from a code point or range thereof to
# something else. This means almost all the UCD .txt files.
# each_line_handlers() should be set to adjust the lines of these
# files, if necessary, to what this routine understands:
#
# 0374 ; NFD_QC; N
# 003C..003E ; Math
#
# the fields are: "codepoint-range ; property; map"
#
# meaning the codepoints in the range all have the value 'map' under
# 'property'.
# Beginning and trailing white space in each field are not significant.
# Note there is not a trailing semi-colon in the above. A trailing
# semi-colon means the map is a null-string. An omitted map, as
# opposed to a null-string, is assumed to be 'Y', based on Unicode
# table syntax. (This could have been hidden from this routine by
# doing it in the $file object, but that would require parsing of the
# line there, so would have to parse it twice, or change the interface
# to pass this an array. So not done.)
#
# The map field may begin with a sequence of commands that apply to
# this range. Each such command begins and ends with $CMD_DELIM.
# These are used to indicate, for example, that the mapping for a
# range has a non-default type.
#
# This loops through the file, calling its next_line() method, and
# then taking the map and adding it to the property's table.
# Complications arise because any number of properties can be in the
# file, in any order, interspersed in any way. The first time a
# property is seen, it gets information about that property and
# caches it for quick retrieval later. It also normalizes the maps
# so that only one of many synonyms is stored. The Unicode input
# files do use some multiple synonyms.
my $file = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my %property_info; # To keep track of what properties
# have already had entries in the
# current file, and info about each,
# so don't have to recompute.
my $property_name; # property currently being worked on
my $property_type; # and its type
my $previous_property_name = ""; # name from last time through loop
my $property_object; # pointer to the current property's
# object
my $property_addr; # the address of that object
my $default_map; # the string that code points missing
# from the file map to
my $default_table; # For non-string properties, a
# reference to the match table that
# will contain the list of code
# points that map to $default_map.
# Get the next real non-comment line
LINE:
while ($file->next_line) {
# Default replacement type; means that if parts of the range have
# already been stored in our tables, the new map overrides them if
# they differ more than cosmetically
my $replace = $IF_NOT_EQUIVALENT;
my $map_type; # Default type for the map of this range
#local $to_trace = 1 if main::DEBUG;
trace $_ if main::DEBUG && $to_trace;
# Split the line into components
my ($range, $property_name, $map, @remainder)
= split /\s*;\s*/, $_, -1; # -1 => retain trailing null fields
# If more or less on the line than we are expecting, warn and skip
# the line
if (@remainder) {
$file->carp_bad_line('Extra fields');
next LINE;
}
elsif ( ! defined $property_name) {
$file->carp_bad_line('Missing property');
next LINE;
}
# Examine the range.
if ($range !~ /^ ($code_point_re) (?:\.\. ($code_point_re) )? $/x)
{
$file->carp_bad_line("Range '$range' not of the form 'CP1' or 'CP1..CP2' (where CP1,2 are code points in hex)");
next LINE;
}
my $low = hex $1;
my $high = (defined $2) ? hex $2 : $low;
# If changing to a new property, get the things constant per
# property
if ($previous_property_name ne $property_name) {
$property_object = property_ref($property_name);
if (! defined $property_object) {
$file->carp_bad_line("Unexpected property '$property_name'. Skipped");
next LINE;
}
{ no overloading; $property_addr = pack 'J', $property_object; }
# Defer changing names until have a line that is acceptable
# (the 'next' statement above means is unacceptable)
$previous_property_name = $property_name;
# If not the first time for this property, retrieve info about
# it from the cache
if (defined ($property_info{$property_addr}{$TYPE})) {
$property_type = $property_info{$property_addr}{$TYPE};
$default_map = $property_info{$property_addr}{$DEFAULT_MAP};
$map_type
= $property_info{$property_addr}{$PSEUDO_MAP_TYPE};
$default_table
= $property_info{$property_addr}{$DEFAULT_TABLE};
}
else {
# Here, is the first time for this property. Set up the
# cache.
$property_type = $property_info{$property_addr}{$TYPE}
= $property_object->type;
$map_type
= $property_info{$property_addr}{$PSEUDO_MAP_TYPE}
= $property_object->pseudo_map_type;
# The Unicode files are set up so that if the map is not
# defined, it is a binary property
if (! defined $map && $property_type != $BINARY) {
if ($property_type != $UNKNOWN
&& $property_type != $NON_STRING)
{
$file->carp_bad_line("No mapping defined on a non-binary property. Using 'Y' for the map");
}
else {
$property_object->set_type($BINARY);
$property_type
= $property_info{$property_addr}{$TYPE}
= $BINARY;
}
}
# Get any @missings default for this property. This
# should precede the first entry for the property in the
# input file, and is located in a comment that has been
# stored by the Input_file class until we access it here.
# It's possible that there is more than one such line
# waiting for us; collect them all, and parse
my @missings_list = $file->get_missings
if $file->has_missings_defaults;
foreach my $default_ref (@missings_list) {
my $default = $default_ref->[0];
my $addr = do { no overloading; pack 'J', property_ref($default_ref->[1]); };
# For string properties, the default is just what the
# file says, but non-string properties should already
# have set up a table for the default property value;
# use the table for these, so can resolve synonyms
# later to a single standard one.
if ($property_type == $STRING
|| $property_type == $UNKNOWN)
{
$property_info{$addr}{$MISSINGS} = $default;
}
else {
$property_info{$addr}{$MISSINGS}
= $property_object->table($default);
}
}
# Finished storing all the @missings defaults in the input
# file so far. Get the one for the current property.
my $missings = $property_info{$property_addr}{$MISSINGS};
# But we likely have separately stored what the default
# should be. (This is to accommodate versions of the
# standard where the @missings lines are absent or
# incomplete.) Hopefully the two will match. But check
# it out.
$default_map = $property_object->default_map;
# If the map is a ref, it means that the default won't be
# processed until later, so undef it, so next few lines
# will redefine it to something that nothing will match
undef $default_map if ref $default_map;
# Create a $default_map if don't have one; maybe a dummy
# that won't match anything.
if (! defined $default_map) {
# Use any @missings line in the file.
if (defined $missings) {
if (ref $missings) {
$default_map = $missings->full_name;
$default_table = $missings;
}
else {
$default_map = $missings;
}
# And store it with the property for outside use.
$property_object->set_default_map($default_map);
}
else {
# Neither an @missings nor a default map. Create
# a dummy one, so won't have to test definedness
# in the main loop.
$default_map = '_Perl This will never be in a file
from Unicode';
}
}
# Here, we have $default_map defined, possibly in terms of
# $missings, but maybe not, and possibly is a dummy one.
if (defined $missings) {
# Make sure there is no conflict between the two.
# $missings has priority.
if (ref $missings) {
$default_table
= $property_object->table($default_map);
if (! defined $default_table
|| $default_table != $missings)
{
if (! defined $default_table) {
$default_table = $UNDEF;
}
$file->carp_bad_line(<<END
The \@missings line for $property_name in $file says that missings default to
$missings, but we expect it to be $default_table. $missings used.
END
);
$default_table = $missings;
$default_map = $missings->full_name;
}
$property_info{$property_addr}{$DEFAULT_TABLE}
= $default_table;
}
elsif ($default_map ne $missings) {
$file->carp_bad_line(<<END
The \@missings line for $property_name in $file says that missings default to
$missings, but we expect it to be $default_map. $missings used.
END
);
$default_map = $missings;
}
}
$property_info{$property_addr}{$DEFAULT_MAP}
= $default_map;
# If haven't done so already, find the table corresponding
# to this map for non-string properties.
if (! defined $default_table
&& $property_type != $STRING
&& $property_type != $UNKNOWN)
{
$default_table = $property_info{$property_addr}
{$DEFAULT_TABLE}
= $property_object->table($default_map);
}
} # End of is first time for this property
} # End of switching properties.
# Ready to process the line.
# The Unicode files are set up so that if the map is not defined,
# it is a binary property with value 'Y'
if (! defined $map) {
$map = 'Y';
}
else {
# If the map begins with a special command to us (enclosed in
# delimiters), extract the command(s).
while ($map =~ s/ ^ $CMD_DELIM (.*?) $CMD_DELIM //x) {
my $command = $1;
if ($command =~ / ^ $REPLACE_CMD= (.*) /x) {
$replace = $1;
}
elsif ($command =~ / ^ $MAP_TYPE_CMD= (.*) /x) {
$map_type = $1;
}
else {
$file->carp_bad_line("Unknown command line: '$1'");
next LINE;
}
}
}
if ($default_map eq $CODE_POINT && $map =~ / ^ $code_point_re $/x)
{
# Here, we have a map to a particular code point, and the
# default map is to a code point itself. If the range
# includes the particular code point, change that portion of
# the range to the default. This makes sure that in the final
# table only the non-defaults are listed.
my $decimal_map = hex $map;
if ($low <= $decimal_map && $decimal_map <= $high) {
# If the range includes stuff before or after the map
# we're changing, split it and process the split-off parts
# later.
if ($low < $decimal_map) {
$file->insert_adjusted_lines(
sprintf("%04X..%04X; %s; %s",
$low,
$decimal_map - 1,
$property_name,
$map));
}
if ($high > $decimal_map) {
$file->insert_adjusted_lines(
sprintf("%04X..%04X; %s; %s",
$decimal_map + 1,
$high,
$property_name,
$map));
}
$low = $high = $decimal_map;
$map = $CODE_POINT;
}
}
# If we can tell that this is a synonym for the default map, use
# the default one instead.
if ($property_type != $STRING
&& $property_type != $UNKNOWN)
{
my $table = $property_object->table($map);
if (defined $table && $table == $default_table) {
$map = $default_map;
}
}
# And figure out the map type if not known.
if (! defined $map_type || $map_type == $COMPUTE_NO_MULTI_CP) {
if ($map eq "") { # Nulls are always $NULL map type
$map_type = $NULL;
} # Otherwise, non-strings, and those that don't allow
# $MULTI_CP, and those that aren't multiple code points are
# 0
elsif
(($property_type != $STRING && $property_type != $UNKNOWN)
|| (defined $map_type && $map_type == $COMPUTE_NO_MULTI_CP)
|| $map !~ /^ $code_point_re ( \ $code_point_re )+ $ /x)
{
$map_type = 0;
}
else {
$map_type = $MULTI_CP;
}
}
$property_object->add_map($low, $high,
$map,
Type => $map_type,
Replace => $replace);
} # End of loop through file's lines
return;
}
}
{ # Closure for UnicodeData.txt handling
# This file was the first one in the UCD; its design leads to some
# awkwardness in processing. Here is a sample line:
# 0041;LATIN CAPITAL LETTER A;Lu;0;L;;;;;N;;;;0061;
# The fields in order are:
my $i = 0; # The code point is in field 0, and is shifted off.
my $CHARNAME = $i++; # character name (e.g. "LATIN CAPITAL LETTER A")
my $CATEGORY = $i++; # category (e.g. "Lu")
my $CCC = $i++; # Canonical combining class (e.g. "230")
my $BIDI = $i++; # directional class (e.g. "L")
my $PERL_DECOMPOSITION = $i++; # decomposition mapping
my $PERL_DECIMAL_DIGIT = $i++; # decimal digit value
my $NUMERIC_TYPE_OTHER_DIGIT = $i++; # digit value, like a superscript
# Dual-use in this program; see below
my $NUMERIC = $i++; # numeric value
my $MIRRORED = $i++; # ? mirrored
my $UNICODE_1_NAME = $i++; # name in Unicode 1.0
my $COMMENT = $i++; # iso comment
my $UPPER = $i++; # simple uppercase mapping
my $LOWER = $i++; # simple lowercase mapping
my $TITLE = $i++; # simple titlecase mapping
my $input_field_count = $i;
# This routine in addition outputs these extra fields:
my $DECOMP_TYPE = $i++; # Decomposition type
# These fields are modifications of ones above, and are usually
# suppressed; they must come last, as for speed, the loop upper bound is
# normally set to ignore them
my $NAME = $i++; # This is the strict name field, not the one that
# charnames uses.
my $DECOMP_MAP = $i++; # Strict decomposition mapping; not the one used
# by Unicode::Normalize
my $last_field = $i - 1;
# All these are read into an array for each line, with the indices defined
# above. The empty fields in the example line above indicate that the
# value is defaulted. The handler called for each line of the input
# changes these to their defaults.
# Here are the official names of the properties, in a parallel array:
my @field_names;
$field_names[$BIDI] = 'Bidi_Class';
$field_names[$CATEGORY] = 'General_Category';
$field_names[$CCC] = 'Canonical_Combining_Class';
$field_names[$CHARNAME] = 'Perl_Charnames';
$field_names[$COMMENT] = 'ISO_Comment';
$field_names[$DECOMP_MAP] = 'Decomposition_Mapping';
$field_names[$DECOMP_TYPE] = 'Decomposition_Type';
$field_names[$LOWER] = 'Lowercase_Mapping';
$field_names[$MIRRORED] = 'Bidi_Mirrored';
$field_names[$NAME] = 'Name';
$field_names[$NUMERIC] = 'Numeric_Value';
$field_names[$NUMERIC_TYPE_OTHER_DIGIT] = 'Numeric_Type';
$field_names[$PERL_DECIMAL_DIGIT] = 'Perl_Decimal_Digit';
$field_names[$PERL_DECOMPOSITION] = 'Perl_Decomposition_Mapping';
$field_names[$TITLE] = 'Titlecase_Mapping';
$field_names[$UNICODE_1_NAME] = 'Unicode_1_Name';
$field_names[$UPPER] = 'Uppercase_Mapping';
# Some of these need a little more explanation:
# The $PERL_DECIMAL_DIGIT field does not lead to an official Unicode
# property, but is used in calculating the Numeric_Type. Perl however,
# creates a file from this field, so a Perl property is created from it.
# Similarly, the Other_Digit field is used only for calculating the
# Numeric_Type, and so it can be safely re-used as the place to store
# the value for Numeric_Type; hence it is referred to as
# $NUMERIC_TYPE_OTHER_DIGIT.
# The input field named $PERL_DECOMPOSITION is a combination of both the
# decomposition mapping and its type. Perl creates a file containing
# exactly this field, so it is used for that. The two properties are
# separated into two extra output fields, $DECOMP_MAP and $DECOMP_TYPE.
# $DECOMP_MAP is usually suppressed (unless the lists are changed to
# output it), as Perl doesn't use it directly.
# The input field named here $CHARNAME is used to construct the
# Perl_Charnames property, which is a combination of the Name property
# (which the input field contains), and the Unicode_1_Name property, and
# others from other files. Since, the strict Name property is not used
# by Perl, this field is used for the table that Perl does use. The
# strict Name property table is usually suppressed (unless the lists are
# changed to output it), so it is accumulated in a separate field,
# $NAME, which to save time is discarded unless the table is actually to
# be output
# This file is processed like most in this program. Control is passed to
# process_generic_property_file() which calls filter_UnicodeData_line()
# for each input line. This filter converts the input into line(s) that
# process_generic_property_file() understands. There is also a setup
# routine called before any of the file is processed, and a handler for
# EOF processing, all in this closure.
# A huge speed-up occurred at the cost of some added complexity when these
# routines were altered to buffer the outputs into ranges. Almost all the
# lines of the input file apply to just one code point, and for most
# properties, the map for the next code point up is the same as the
# current one. So instead of creating a line for each property for each
# input line, filter_UnicodeData_line() remembers what the previous map
# of a property was, and doesn't generate a line to pass on until it has
# to, as when the map changes; and that passed-on line encompasses the
# whole contiguous range of code points that have the same map for that
# property. This means a slight amount of extra setup, and having to
# flush these buffers on EOF, testing if the maps have changed, plus
# remembering state information in the closure. But it means a lot less
# real time in not having to change the data base for each property on
# each line.
# Another complication is that there are already a few ranges designated
# in the input. There are two lines for each, with the same maps except
# the code point and name on each line. This was actually the hardest
# thing to design around. The code points in those ranges may actually
# have real maps not given by these two lines. These maps will either
# be algorithmically determinable, or be in the extracted files furnished
# with the UCD. In the event of conflicts between these extracted files,
# and this one, Unicode says that this one prevails. But it shouldn't
# prevail for conflicts that occur in these ranges. The data from the
# extracted files prevails in those cases. So, this program is structured
# so that those files are processed first, storing maps. Then the other
# files are processed, generally overwriting what the extracted files
# stored. But just the range lines in this input file are processed
# without overwriting. This is accomplished by adding a special string to
# the lines output to tell process_generic_property_file() to turn off the
# overwriting for just this one line.
# A similar mechanism is used to tell it that the map is of a non-default
# type.
sub setup_UnicodeData { # Called before any lines of the input are read
my $file = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
# Create a new property specially located that is a combination of
# various Name properties: Name, Unicode_1_Name, Named Sequences, and
# _Perl_Name_Alias properties. (The final one duplicates elements of the
# first, and starting in v6.1, is the same as the 'Name_Alias
# property.) A comment for the new property will later be constructed
# based on the actual properties present and used
$perl_charname = Property->new('Perl_Charnames',
Default_Map => "",
Directory => File::Spec->curdir(),
File => 'Name',
Fate => $INTERNAL_ONLY,
Perl_Extension => 1,
Range_Size_1 => \&output_perl_charnames_line,
Type => $STRING,
);
$perl_charname->set_proxy_for('Name');
my $Perl_decomp = Property->new('Perl_Decomposition_Mapping',
Directory => File::Spec->curdir(),
File => 'Decomposition',
Format => $DECOMP_STRING_FORMAT,
Fate => $INTERNAL_ONLY,
Perl_Extension => 1,
Default_Map => $CODE_POINT,
# normalize.pm can't cope with these
Output_Range_Counts => 0,
# This is a specially formatted table
# explicitly for normalize.pm, which
# is expecting a particular format,
# which means that mappings containing
# multiple code points are in the main
# body of the table
Map_Type => $COMPUTE_NO_MULTI_CP,
Type => $STRING,
To_Output_Map => $INTERNAL_MAP,
);
$Perl_decomp->set_proxy_for('Decomposition_Mapping', 'Decomposition_Type');
$Perl_decomp->add_comment(join_lines(<<END
This mapping is a combination of the Unicode 'Decomposition_Type' and
'Decomposition_Mapping' properties, formatted for use by normalize.pm. It is
identical to the official Unicode 'Decomposition_Mapping' property except for
two things:
1) It omits the algorithmically determinable Hangul syllable decompositions,
which normalize.pm handles algorithmically.
2) It contains the decomposition type as well. Non-canonical decompositions
begin with a word in angle brackets, like <super>, which denotes the
compatible decomposition type. If the map does not begin with the <angle
brackets>, the decomposition is canonical.
END
));
my $Decimal_Digit = Property->new("Perl_Decimal_Digit",
Default_Map => "",
Perl_Extension => 1,
Directory => $map_directory,
Type => $STRING,
To_Output_Map => $OUTPUT_ADJUSTED,
);
$Decimal_Digit->add_comment(join_lines(<<END
This file gives the mapping of all code points which represent a single
decimal digit [0-9] to their respective digits, but it has ranges of 10 code
points, and the mapping of each non-initial element of each range is actually
not to "0", but to the offset that element has from its corresponding DIGIT 0.
These code points are those that have Numeric_Type=Decimal; not special
things, like subscripts nor Roman numerals.
END
));
# These properties are not used for generating anything else, and are
# usually not output. By making them last in the list, we can just
# change the high end of the loop downwards to avoid the work of
# generating a table(s) that is/are just going to get thrown away.
if (! property_ref('Decomposition_Mapping')->to_output_map
&& ! property_ref('Name')->to_output_map)
{
$last_field = min($NAME, $DECOMP_MAP) - 1;
} elsif (property_ref('Decomposition_Mapping')->to_output_map) {
$last_field = $DECOMP_MAP;
} elsif (property_ref('Name')->to_output_map) {
$last_field = $NAME;
}
return;
}
my $first_time = 1; # ? Is this the first line of the file
my $in_range = 0; # ? Are we in one of the file's ranges
my $previous_cp; # hex code point of previous line
my $decimal_previous_cp = -1; # And its decimal equivalent
my @start; # For each field, the current starting
# code point in hex for the range
# being accumulated.
my @fields; # The input fields;
my @previous_fields; # And those from the previous call
sub filter_UnicodeData_line {
# Handle a single input line from UnicodeData.txt; see comments above
# Conceptually this takes a single line from the file containing N
# properties, and converts it into N lines with one property per line,
# which is what the final handler expects. But there are
# complications due to the quirkiness of the input file, and to save
# time, it accumulates ranges where the property values don't change
# and only emits lines when necessary. This is about an order of
# magnitude fewer lines emitted.
my $file = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
# $_ contains the input line.
# -1 in split means retain trailing null fields
(my $cp, @fields) = split /\s*;\s*/, $_, -1;
#local $to_trace = 1 if main::DEBUG;
trace $cp, @fields , $input_field_count if main::DEBUG && $to_trace;
if (@fields > $input_field_count) {
$file->carp_bad_line('Extra fields');
$_ = "";
return;
}
my $decimal_cp = hex $cp;
# We have to output all the buffered ranges when the next code point
# is not exactly one after the previous one, which means there is a
# gap in the ranges.
my $force_output = ($decimal_cp != $decimal_previous_cp + 1);
# The decomposition mapping field requires special handling. It looks
# like either:
#
# <compat> 0032 0020
# 0041 0300
#
# The decomposition type is enclosed in <brackets>; if missing, it
# means the type is canonical. There are two decomposition mapping
# tables: the one for use by Perl's normalize.pm has a special format
# which is this field intact; the other, for general use is of
# standard format. In either case we have to find the decomposition
# type. Empty fields have None as their type, and map to the code
# point itself
if ($fields[$PERL_DECOMPOSITION] eq "") {
$fields[$DECOMP_TYPE] = 'None';
$fields[$DECOMP_MAP] = $fields[$PERL_DECOMPOSITION] = $CODE_POINT;
}
else {
($fields[$DECOMP_TYPE], my $map) = $fields[$PERL_DECOMPOSITION]
=~ / < ( .+? ) > \s* ( .+ ) /x;
if (! defined $fields[$DECOMP_TYPE]) {
$fields[$DECOMP_TYPE] = 'Canonical';
$fields[$DECOMP_MAP] = $fields[$PERL_DECOMPOSITION];
}
else {
$fields[$DECOMP_MAP] = $map;
}
}
# The 3 numeric fields also require special handling. The 2 digit
# fields must be either empty or match the number field. This means
# that if it is empty, they must be as well, and the numeric type is
# None, and the numeric value is 'Nan'.
# The decimal digit field must be empty or match the other digit
# field. If the decimal digit field is non-empty, the code point is
# a decimal digit, and the other two fields will have the same value.
# If it is empty, but the other digit field is non-empty, the code
# point is an 'other digit', and the number field will have the same
# value as the other digit field. If the other digit field is empty,
# but the number field is non-empty, the code point is a generic
# numeric type.
if ($fields[$NUMERIC] eq "") {
if ($fields[$PERL_DECIMAL_DIGIT] ne ""
|| $fields[$NUMERIC_TYPE_OTHER_DIGIT] ne ""
) {
$file->carp_bad_line("Numeric values inconsistent. Trying to process anyway");
}
$fields[$NUMERIC_TYPE_OTHER_DIGIT] = 'None';
$fields[$NUMERIC] = 'NaN';
}
else {
$file->carp_bad_line("'$fields[$NUMERIC]' should be a whole or rational number. Processing as if it were") if $fields[$NUMERIC] !~ qr{ ^ -? \d+ ( / \d+ )? $ }x;
if ($fields[$PERL_DECIMAL_DIGIT] ne "") {
$file->carp_bad_line("$fields[$PERL_DECIMAL_DIGIT] should equal $fields[$NUMERIC]. Processing anyway") if $fields[$PERL_DECIMAL_DIGIT] != $fields[$NUMERIC];
$file->carp_bad_line("$fields[$PERL_DECIMAL_DIGIT] should be empty since the general category ($fields[$CATEGORY]) isn't 'Nd'. Processing as Decimal") if $fields[$CATEGORY] ne "Nd";
$fields[$NUMERIC_TYPE_OTHER_DIGIT] = 'Decimal';
}
elsif ($fields[$NUMERIC_TYPE_OTHER_DIGIT] ne "") {
$file->carp_bad_line("$fields[$NUMERIC_TYPE_OTHER_DIGIT] should equal $fields[$NUMERIC]. Processing anyway") if $fields[$NUMERIC_TYPE_OTHER_DIGIT] != $fields[$NUMERIC];
$fields[$NUMERIC_TYPE_OTHER_DIGIT] = 'Digit';
}
else {
$fields[$NUMERIC_TYPE_OTHER_DIGIT] = 'Numeric';
# Rationals require extra effort.
if ($fields[$NUMERIC] =~ qr{/}) {
reduce_fraction(\$fields[$NUMERIC]);
register_fraction($fields[$NUMERIC])
}
}
}
# For the properties that have empty fields in the file, and which
# mean something different from empty, change them to that default.
# Certain fields just haven't been empty so far in any Unicode
# version, so don't look at those, namely $MIRRORED, $BIDI, $CCC,
# $CATEGORY. This leaves just the two fields, and so we hard-code in
# the defaults; which are very unlikely to ever change.
$fields[$UPPER] = $CODE_POINT if $fields[$UPPER] eq "";
$fields[$LOWER] = $CODE_POINT if $fields[$LOWER] eq "";
# UAX44 says that if title is empty, it is the same as whatever upper
# is,
$fields[$TITLE] = $fields[$UPPER] if $fields[$TITLE] eq "";
# There are a few pairs of lines like:
# AC00;<Hangul Syllable, First>;Lo;0;L;;;;;N;;;;;
# D7A3;<Hangul Syllable, Last>;Lo;0;L;;;;;N;;;;;
# that define ranges. These should be processed after the fields are
# adjusted above, as they may override some of them; but mostly what
# is left is to possibly adjust the $CHARNAME field. The names of all the
# paired lines start with a '<', but this is also true of '<control>,
# which isn't one of these special ones.
if ($fields[$CHARNAME] eq '<control>') {
# Some code points in this file have the pseudo-name
# '<control>', but the official name for such ones is the null
# string.
$fields[$NAME] = $fields[$CHARNAME] = "";
# We had better not be in between range lines.
if ($in_range) {
$file->carp_bad_line("Expecting a closing range line, not a $fields[$CHARNAME]'. Trying anyway");
$in_range = 0;
}
}
elsif (substr($fields[$CHARNAME], 0, 1) ne '<') {
# Here is a non-range line. We had better not be in between range
# lines.
if ($in_range) {
$file->carp_bad_line("Expecting a closing range line, not a $fields[$CHARNAME]'. Trying anyway");
$in_range = 0;
}
if ($fields[$CHARNAME] =~ s/- $cp $//x) {
# These are code points whose names end in their code points,
# which means the names are algorithmically derivable from the
# code points. To shorten the output Name file, the algorithm
# for deriving these is placed in the file instead of each
# code point, so they have map type $CP_IN_NAME
$fields[$CHARNAME] = $CMD_DELIM
. $MAP_TYPE_CMD
. '='
. $CP_IN_NAME
. $CMD_DELIM
. $fields[$CHARNAME];
}
$fields[$NAME] = $fields[$CHARNAME];
}
elsif ($fields[$CHARNAME] =~ /^<(.+), First>$/) {
$fields[$CHARNAME] = $fields[$NAME] = $1;
# Here we are at the beginning of a range pair.
if ($in_range) {
$file->carp_bad_line("Expecting a closing range line, not a beginning one, $fields[$CHARNAME]'. Trying anyway");
}
$in_range = 1;
# Because the properties in the range do not overwrite any already
# in the db, we must flush the buffers of what's already there, so
# they get handled in the normal scheme.
$force_output = 1;
}
elsif ($fields[$CHARNAME] !~ s/^<(.+), Last>$/$1/) {
$file->carp_bad_line("Unexpected name starting with '<' $fields[$CHARNAME]. Ignoring this line.");
$_ = "";
return;
}
else { # Here, we are at the last line of a range pair.
if (! $in_range) {
$file->carp_bad_line("Unexpected end of range $fields[$CHARNAME] when not in one. Ignoring this line.");
$_ = "";
return;
}
$in_range = 0;
$fields[$NAME] = $fields[$CHARNAME];
# Check that the input is valid: that the closing of the range is
# the same as the beginning.
foreach my $i (0 .. $last_field) {
next if $fields[$i] eq $previous_fields[$i];
$file->carp_bad_line("Expecting '$fields[$i]' to be the same as '$previous_fields[$i]'. Bad News. Trying anyway");
}
# The processing differs depending on the type of range,
# determined by its $CHARNAME
if ($fields[$CHARNAME] =~ /^Hangul Syllable/) {
# Check that the data looks right.
if ($decimal_previous_cp != $SBase) {
$file->carp_bad_line("Unexpected Hangul syllable start = $previous_cp. Bad News. Results will be wrong");
}
if ($decimal_cp != $SBase + $SCount - 1) {
$file->carp_bad_line("Unexpected Hangul syllable end = $cp. Bad News. Results will be wrong");
}
# The Hangul syllable range has a somewhat complicated name
# generation algorithm. Each code point in it has a canonical
# decomposition also computable by an algorithm. The
# perl decomposition map table built from these is used only
# by normalize.pm, which has the algorithm built in it, so the
# decomposition maps are not needed, and are large, so are
# omitted from it. If the full decomposition map table is to
# be output, the decompositions are generated for it, in the
# EOF handling code for this input file.
$previous_fields[$DECOMP_TYPE] = 'Canonical';
# This range is stored in our internal structure with its
# own map type, different from all others.
$previous_fields[$CHARNAME] = $previous_fields[$NAME]
= $CMD_DELIM
. $MAP_TYPE_CMD
. '='
. $HANGUL_SYLLABLE
. $CMD_DELIM
. $fields[$CHARNAME];
}
elsif ($fields[$CATEGORY] eq 'Lo') { # Is a letter
# All the CJK ranges like this have the name given as a
# special case in the next code line. And for the others, we
# hope that Unicode continues to use the correct name in
# future releases, so we don't have to make further special
# cases.
my $name = ($fields[$CHARNAME] =~ /^CJK/)
? 'CJK UNIFIED IDEOGRAPH'
: uc $fields[$CHARNAME];
# The name for these contains the code point itself, and all
# are defined to have the same base name, regardless of what
# is in the file. They are stored in our internal structure
# with a map type of $CP_IN_NAME
$previous_fields[$CHARNAME] = $previous_fields[$NAME]
= $CMD_DELIM
. $MAP_TYPE_CMD
. '='
. $CP_IN_NAME
. $CMD_DELIM
. $name;
}
elsif ($fields[$CATEGORY] eq 'Co'
|| $fields[$CATEGORY] eq 'Cs')
{
# The names of all the code points in these ranges are set to
# null, as there are no names for the private use and
# surrogate code points.
$previous_fields[$CHARNAME] = $previous_fields[$NAME] = "";
}
else {
$file->carp_bad_line("Unexpected code point range $fields[$CHARNAME] because category is $fields[$CATEGORY]. Attempting to process it.");
}
# The first line of the range caused everything else to be output,
# and then its values were stored as the beginning values for the
# next set of ranges, which this one ends. Now, for each value,
# add a command to tell the handler that these values should not
# replace any existing ones in our database.
foreach my $i (0 .. $last_field) {
$previous_fields[$i] = $CMD_DELIM
. $REPLACE_CMD
. '='
. $NO
. $CMD_DELIM
. $previous_fields[$i];
}
# And change things so it looks like the entire range has been
# gone through with this being the final part of it. Adding the
# command above to each field will cause this range to be flushed
# during the next iteration, as it guaranteed that the stored
# field won't match whatever value the next one has.
$previous_cp = $cp;
$decimal_previous_cp = $decimal_cp;
# We are now set up for the next iteration; so skip the remaining
# code in this subroutine that does the same thing, but doesn't
# know about these ranges.
$_ = "";
return;
}
# On the very first line, we fake it so the code below thinks there is
# nothing to output, and initialize so that when it does get output it
# uses the first line's values for the lowest part of the range.
# (One could avoid this by using peek(), but then one would need to
# know the adjustments done above and do the same ones in the setup
# routine; not worth it)
if ($first_time) {
$first_time = 0;
@previous_fields = @fields;
@start = ($cp) x scalar @fields;
$decimal_previous_cp = $decimal_cp - 1;
}
# For each field, output the stored up ranges that this code point
# doesn't fit in. Earlier we figured out if all ranges should be
# terminated because of changing the replace or map type styles, or if
# there is a gap between this new code point and the previous one, and
# that is stored in $force_output. But even if those aren't true, we
# need to output the range if this new code point's value for the
# given property doesn't match the stored range's.
#local $to_trace = 1 if main::DEBUG;
foreach my $i (0 .. $last_field) {
my $field = $fields[$i];
if ($force_output || $field ne $previous_fields[$i]) {
# Flush the buffer of stored values.
$file->insert_adjusted_lines("$start[$i]..$previous_cp; $field_names[$i]; $previous_fields[$i]");
# Start a new range with this code point and its value
$start[$i] = $cp;
$previous_fields[$i] = $field;
}
}
# Set the values for the next time.
$previous_cp = $cp;
$decimal_previous_cp = $decimal_cp;
# The input line has generated whatever adjusted lines are needed, and
# should not be looked at further.
$_ = "";
return;
}
sub EOF_UnicodeData {
# Called upon EOF to flush the buffers, and create the Hangul
# decomposition mappings if needed.
my $file = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
# Flush the buffers.
foreach my $i (0 .. $last_field) {
$file->insert_adjusted_lines("$start[$i]..$previous_cp; $field_names[$i]; $previous_fields[$i]");
}
if (-e 'Jamo.txt') {
# The algorithm is published by Unicode, based on values in
# Jamo.txt, (which should have been processed before this
# subroutine), and the results left in %Jamo
unless (%Jamo) {
Carp::my_carp_bug("Jamo.txt should be processed before Unicode.txt. Hangul syllables not generated.");
return;
}
# If the full decomposition map table is being output, insert
# into it the Hangul syllable mappings. This is to avoid having
# to publish a subroutine in it to compute them. (which would
# essentially be this code.) This uses the algorithm published by
# Unicode. (No hangul syllables in version 1)
if ($v_version ge v2.0.0
&& property_ref('Decomposition_Mapping')->to_output_map) {
for (my $S = $SBase; $S < $SBase + $SCount; $S++) {
use integer;
my $SIndex = $S - $SBase;
my $L = $LBase + $SIndex / $NCount;
my $V = $VBase + ($SIndex % $NCount) / $TCount;
my $T = $TBase + $SIndex % $TCount;
trace "L=$L, V=$V, T=$T" if main::DEBUG && $to_trace;
my $decomposition = sprintf("%04X %04X", $L, $V);
$decomposition .= sprintf(" %04X", $T) if $T != $TBase;
$file->insert_adjusted_lines(
sprintf("%04X; Decomposition_Mapping; %s",
$S,
$decomposition));
}
}
}
return;
}
sub filter_v1_ucd {
# Fix UCD lines in version 1. This is probably overkill, but this
# fixes some glaring errors in Version 1 UnicodeData.txt. That file:
# 1) had many Hangul (U+3400 - U+4DFF) code points that were later
# removed. This program retains them
# 2) didn't include ranges, which it should have, and which are now
# added in @corrected_lines below. It was hand populated by
# taking the data from Version 2, verified by analyzing
# DAge.txt.
# 3) There is a syntax error in the entry for U+09F8 which could
# cause problems for utf8_heavy, and so is changed. It's
# numeric value was simply a minus sign, without any number.
# (Eventually Unicode changed the code point to non-numeric.)
# 4) The decomposition types often don't match later versions
# exactly, and the whole syntax of that field is different; so
# the syntax is changed as well as the types to their later
# terminology. Otherwise normalize.pm would be very unhappy
# 5) Many ccc classes are different. These are left intact.
# 6) U+FF10..U+FF19 are missing their numeric values in all three
# fields. These are unchanged because it doesn't really cause
# problems for Perl.
# 7) A number of code points, such as controls, don't have their
# Unicode Version 1 Names in this file. These are added.
# 8) A number of Symbols were marked as Lm. This changes those in
# the Latin1 range, so that regexes work.
# 9) The odd characters U+03DB .. U+03E1 weren't encoded but are
# referred to by their lc equivalents. Not fixed.
my @corrected_lines = split /\n/, <<'END';
4E00;<CJK Ideograph, First>;Lo;0;L;;;;;N;;;;;
9FA5;<CJK Ideograph, Last>;Lo;0;L;;;;;N;;;;;
E000;<Private Use, First>;Co;0;L;;;;;N;;;;;
F8FF;<Private Use, Last>;Co;0;L;;;;;N;;;;;
F900;<CJK Compatibility Ideograph, First>;Lo;0;L;;;;;N;;;;;
FA2D;<CJK Compatibility Ideograph, Last>;Lo;0;L;;;;;N;;;;;
END
my $file = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
#local $to_trace = 1 if main::DEBUG;
trace $_ if main::DEBUG && $to_trace;
# -1 => retain trailing null fields
my ($code_point, @fields) = split /\s*;\s*/, $_, -1;
# At the first place that is wrong in the input, insert all the
# corrections, replacing the wrong line.
if ($code_point eq '4E00') {
my @copy = @corrected_lines;
$_ = shift @copy;
($code_point, @fields) = split /\s*;\s*/, $_, -1;
$file->insert_lines(@copy);
}
elsif ($code_point =~ /^00/ && $fields[$CATEGORY] eq 'Lm') {
# There are no Lm characters in Latin1; these should be 'Sk', but
# there isn't that in V1.
$fields[$CATEGORY] = 'So';
}
if ($fields[$NUMERIC] eq '-') {
$fields[$NUMERIC] = '-1'; # This is what 2.0 made it.
}
if ($fields[$PERL_DECOMPOSITION] ne "") {
# Several entries have this change to superscript 2 or 3 in the
# middle. Convert these to the modern version, which is to use
# the actual U+00B2 and U+00B3 (the superscript forms) instead.
# So 'HHHH HHHH <+sup> 0033 <-sup> HHHH' becomes
# 'HHHH HHHH 00B3 HHHH'.
# It turns out that all of these that don't have another
# decomposition defined at the beginning of the line have the
# <square> decomposition in later releases.
if ($code_point ne '00B2' && $code_point ne '00B3') {
if ($fields[$PERL_DECOMPOSITION]
=~ s/<\+sup> 003([23]) <-sup>/00B$1/)
{
if (substr($fields[$PERL_DECOMPOSITION], 0, 1) ne '<') {
$fields[$PERL_DECOMPOSITION] = '<square> '
. $fields[$PERL_DECOMPOSITION];
}
}
}
# If is like '<+circled> 0052 <-circled>', convert to
# '<circled> 0052'
$fields[$PERL_DECOMPOSITION] =~
s/ < \+ ( .*? ) > \s* (.*?) \s* <-\1> /<$1> $2/xg;
# Convert '<join> HHHH HHHH <join>' to '<medial> HHHH HHHH', etc.
$fields[$PERL_DECOMPOSITION] =~
s/ <join> \s* (.*?) \s* <no-join> /<final> $1/x
or $fields[$PERL_DECOMPOSITION] =~
s/ <join> \s* (.*?) \s* <join> /<medial> $1/x
or $fields[$PERL_DECOMPOSITION] =~
s/ <no-join> \s* (.*?) \s* <join> /<initial> $1/x
or $fields[$PERL_DECOMPOSITION] =~
s/ <no-join> \s* (.*?) \s* <no-join> /<isolated> $1/x;
# Convert '<break> HHHH HHHH <break>' to '<break> HHHH', etc.
$fields[$PERL_DECOMPOSITION] =~
s/ <(break|no-break)> \s* (.*?) \s* <\1> /<$1> $2/x;
# Change names to modern form.
$fields[$PERL_DECOMPOSITION] =~ s/<font variant>/<font>/g;
$fields[$PERL_DECOMPOSITION] =~ s/<no-break>/<noBreak>/g;
$fields[$PERL_DECOMPOSITION] =~ s/<circled>/<circle>/g;
$fields[$PERL_DECOMPOSITION] =~ s/<break>/<fraction>/g;
# One entry has weird braces
$fields[$PERL_DECOMPOSITION] =~ s/[{}]//g;
# One entry at U+2116 has an extra <sup>
$fields[$PERL_DECOMPOSITION] =~ s/( < .*? > .* ) < .*? > \ * /$1/x;
}
$_ = join ';', $code_point, @fields;
trace $_ if main::DEBUG && $to_trace;
return;
}
sub filter_bad_Nd_ucd {
# Early versions specified a value in the decimal digit field even
# though the code point wasn't a decimal digit. Clear the field in
# that situation, so that the main code doesn't think it is a decimal
# digit.
my ($code_point, @fields) = split /\s*;\s*/, $_, -1;
if ($fields[$PERL_DECIMAL_DIGIT] ne "" && $fields[$CATEGORY] ne 'Nd') {
$fields[$PERL_DECIMAL_DIGIT] = "";
$_ = join ';', $code_point, @fields;
}
return;
}
my @U1_control_names = split /\n/, <<'END';
NULL
START OF HEADING
START OF TEXT
END OF TEXT
END OF TRANSMISSION
ENQUIRY
ACKNOWLEDGE
BELL
BACKSPACE
HORIZONTAL TABULATION
LINE FEED
VERTICAL TABULATION
FORM FEED
CARRIAGE RETURN
SHIFT OUT
SHIFT IN
DATA LINK ESCAPE
DEVICE CONTROL ONE
DEVICE CONTROL TWO
DEVICE CONTROL THREE
DEVICE CONTROL FOUR
NEGATIVE ACKNOWLEDGE
SYNCHRONOUS IDLE
END OF TRANSMISSION BLOCK
CANCEL
END OF MEDIUM
SUBSTITUTE
ESCAPE
FILE SEPARATOR
GROUP SEPARATOR
RECORD SEPARATOR
UNIT SEPARATOR
DELETE
BREAK PERMITTED HERE
NO BREAK HERE
INDEX
NEXT LINE
START OF SELECTED AREA
END OF SELECTED AREA
CHARACTER TABULATION SET
CHARACTER TABULATION WITH JUSTIFICATION
LINE TABULATION SET
PARTIAL LINE DOWN
PARTIAL LINE UP
REVERSE LINE FEED
SINGLE SHIFT TWO
SINGLE SHIFT THREE
DEVICE CONTROL STRING
PRIVATE USE ONE
PRIVATE USE TWO
SET TRANSMIT STATE
CANCEL CHARACTER
MESSAGE WAITING
START OF GUARDED AREA
END OF GUARDED AREA
START OF STRING
SINGLE CHARACTER INTRODUCER
CONTROL SEQUENCE INTRODUCER
STRING TERMINATOR
OPERATING SYSTEM COMMAND
PRIVACY MESSAGE
APPLICATION PROGRAM COMMAND
END
sub filter_early_U1_names {
# Very early versions did not have the Unicode_1_name field specified.
# They differed in which ones were present; make sure a U1 name
# exists, so that Unicode::UCD::charinfo will work
my ($code_point, @fields) = split /\s*;\s*/, $_, -1;
# @U1_control names above are entirely positional, so we pull them out
# in the exact order required, with gaps for the ones that don't have
# names.
if ($code_point =~ /^00[01]/
|| $code_point eq '007F'
|| $code_point =~ /^008[2-9A-F]/
|| $code_point =~ /^009[0-8A-F]/)
{
my $u1_name = shift @U1_control_names;
$fields[$UNICODE_1_NAME] = $u1_name unless $fields[$UNICODE_1_NAME];
$_ = join ';', $code_point, @fields;
}
return;
}
sub filter_v2_1_5_ucd {
# A dozen entries in this 2.1.5 file had the mirrored and numeric
# columns swapped; These all had mirrored be 'N'. So if the numeric
# column appears to be N, swap it back.
my ($code_point, @fields) = split /\s*;\s*/, $_, -1;
if ($fields[$NUMERIC] eq 'N') {
$fields[$NUMERIC] = $fields[$MIRRORED];
$fields[$MIRRORED] = 'N';
$_ = join ';', $code_point, @fields;
}
return;
}
sub filter_v6_ucd {
# Unicode 6.0 co-opted the name BELL for U+1F514, but until 5.17,
# it wasn't accepted, to allow for some deprecation cycles. This
# function is not called after 5.16
return if $_ !~ /^(?:0007|1F514|070F);/;
my ($code_point, @fields) = split /\s*;\s*/, $_, -1;
if ($code_point eq '0007') {
$fields[$CHARNAME] = "";
}
elsif ($code_point eq '070F') { # Unicode Corrigendum #8; see
# http://www.unicode.org/versions/corrigendum8.html
$fields[$BIDI] = "AL";
}
elsif ($^V lt v5.18.0) { # For 5.18 will convert to use Unicode's name
$fields[$CHARNAME] = "";
}
$_ = join ';', $code_point, @fields;
return;
}
} # End closure for UnicodeData
sub process_GCB_test {
my $file = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
while ($file->next_line) {
push @backslash_X_tests, $_;
}
return;
}
sub process_LB_test {
my $file = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
while ($file->next_line) {
push @LB_tests, $_;
}
return;
}
sub process_SB_test {
my $file = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
while ($file->next_line) {
push @SB_tests, $_;
}
return;
}
sub process_WB_test {
my $file = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
while ($file->next_line) {
push @WB_tests, $_;
}
return;
}
sub process_NamedSequences {
# NamedSequences.txt entries are just added to an array. Because these
# don't look like the other tables, they have their own handler.
# An example:
# LATIN CAPITAL LETTER A WITH MACRON AND GRAVE;0100 0300
#
# This just adds the sequence to an array for later handling
my $file = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
while ($file->next_line) {
my ($name, $sequence, @remainder) = split /\s*;\s*/, $_, -1;
if (@remainder) {
$file->carp_bad_line(
"Doesn't look like 'KHMER VOWEL SIGN OM;17BB 17C6'");
next;
}
# Code points below 0x0100 need to be converted to native
$sequence =~ s/\b 00 ( [0-9A-F]{2} ) \b/
sprintf("%04X", utf8::unicode_to_native(hex $1))/gxe
if NON_ASCII_PLATFORM;
# Note single \t in keeping with special output format of
# Perl_charnames. But it turns out that the code points don't have to
# be 5 digits long, like the rest, based on the internal workings of
# charnames.pm. This could be easily changed for consistency.
push @named_sequences, "$sequence\t$name";
}
return;
}
{ # Closure
my $first_range;
sub filter_early_ea_lb {
# Fixes early EastAsianWidth.txt and LineBreak.txt files. These had a
# third field be the name of the code point, which can be ignored in
# most cases. But it can be meaningful if it marks a range:
# 33FE;W;IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY THIRTY-ONE
# 3400;W;<CJK Ideograph Extension A, First>
#
# We need to see the First in the example above to know it's a range.
# They did not use the later range syntaxes. This routine changes it
# to use the modern syntax.
# $1 is the Input_file object.
my @fields = split /\s*;\s*/;
if ($fields[2] =~ /^<.*, First>/) {
$first_range = $fields[0];
$_ = "";
}
elsif ($fields[2] =~ /^<.*, Last>/) {
$_ = $_ = "$first_range..$fields[0]; $fields[1]";
}
else {
undef $first_range;
$_ = "$fields[0]; $fields[1]";
}
return;
}
}
sub filter_substitute_lb {
# Used on Unicodes that predate the LB property, where there is a
# substitute file. This just does the regular ea_lb handling for such
# files, and then substitutes the long property value name for the short
# one that comes with the file. (The other break files have the long
# names in them, so this is the odd one out.) The reason for doing this
# kludge is that regen/mk_invlists.pl is expecting the long name. This
# also fixes the typo 'Inseperable' that leads to problems.
filter_early_ea_lb;
return unless $_;
my @fields = split /\s*;\s*/;
$fields[1] = property_ref('_Perl_LB')->table($fields[1])->full_name;
$fields[1] = 'Inseparable' if lc $fields[1] eq 'inseperable';
$_ = join '; ', @fields;
}
sub filter_old_style_arabic_shaping {
# Early versions used a different term for the later one.
my @fields = split /\s*;\s*/;
$fields[3] =~ s/<no shaping>/No_Joining_Group/;
$fields[3] =~ s/\s+/_/g; # Change spaces to underscores
$_ = join ';', @fields;
return;
}
{ # Closure
my $lc; # Table for lowercase mapping
my $tc;
my $uc;
my %special_casing_code_points;
sub setup_special_casing {
# SpecialCasing.txt contains the non-simple case change mappings. The
# simple ones are in UnicodeData.txt, which should already have been
# read in to the full property data structures, so as to initialize
# these with the simple ones. Then the SpecialCasing.txt entries
# add or overwrite the ones which have different full mappings.
# This routine sees if the simple mappings are to be output, and if
# so, copies what has already been put into the full mapping tables,
# while they still contain only the simple mappings.
# The reason it is done this way is that the simple mappings are
# probably not going to be output, so it saves work to initialize the
# full tables with the simple mappings, and then overwrite those
# relatively few entries in them that have different full mappings,
# and thus skip the simple mapping tables altogether.
my $file= shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
$lc = property_ref('lc');
$tc = property_ref('tc');
$uc = property_ref('uc');
# For each of the case change mappings...
foreach my $full_casing_table ($lc, $tc, $uc) {
my $full_casing_name = $full_casing_table->name;
my $full_casing_full_name = $full_casing_table->full_name;
unless (defined $full_casing_table
&& ! $full_casing_table->is_empty)
{
Carp::my_carp_bug("Need to process UnicodeData before SpecialCasing. Only special casing will be generated.");
}
# Create a table in the old-style format and with the original
# file name for backwards compatibility with applications that
# read it directly. The new tables contain both the simple and
# full maps, and the old are missing simple maps when there is a
# conflicting full one. Probably it would have been ok to add
# those to the legacy version, as was already done in 5.14 to the
# case folding one, but this was not done, out of an abundance of
# caution. The tables are set up here before we deal with the
# full maps so that as we handle those, we can override the simple
# maps for them in the legacy table, and merely add them in the
# new-style one.
my $legacy = Property->new("Legacy_" . $full_casing_full_name,
File => $full_casing_full_name
=~ s/case_Mapping//r,
Format => $HEX_FORMAT,
Default_Map => $CODE_POINT,
Initialize => $full_casing_table,
Replacement_Property => $full_casing_full_name,
);
$full_casing_table->add_comment(join_lines( <<END
This file includes both the simple and full case changing maps. The simple
ones are in the main body of the table below, and the full ones adding to or
overriding them are in the hash.
END
));
# The simple version's name in each mapping merely has an 's' in
# front of the full one's
my $simple_name = 's' . $full_casing_name;
my $simple = property_ref($simple_name);
$simple->initialize($full_casing_table) if $simple->to_output_map();
}
return;
}
sub filter_2_1_8_special_casing_line {
# This version had duplicate entries in this file. Delete all but the
# first one
my @fields = split /\s*;\s*/, $_, -1; # -1 => retain trailing null
# fields
if (exists $special_casing_code_points{$fields[0]}) {
$_ = "";
return;
}
$special_casing_code_points{$fields[0]} = 1;
filter_special_casing_line(@_);
}
sub filter_special_casing_line {
# Change the format of $_ from SpecialCasing.txt into something that
# the generic handler understands. Each input line contains three
# case mappings. This will generate three lines to pass to the
# generic handler for each of those.
# The input syntax (after stripping comments and trailing white space
# is like one of the following (with the final two being entries that
# we ignore):
# 00DF; 00DF; 0053 0073; 0053 0053; # LATIN SMALL LETTER SHARP S
# 03A3; 03C2; 03A3; 03A3; Final_Sigma;
# 0307; ; 0307; 0307; tr After_I; # COMBINING DOT ABOVE
# Note the trailing semi-colon, unlike many of the input files. That
# means that there will be an extra null field generated by the split
my $file = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my @fields = split /\s*;\s*/, $_, -1; # -1 => retain trailing null
# fields
# field #4 is when this mapping is conditional. If any of these get
# implemented, it would be by hard-coding in the casing functions in
# the Perl core, not through tables. But if there is a new condition
# we don't know about, output a warning. We know about all the
# conditions through 6.0
if ($fields[4] ne "") {
my @conditions = split ' ', $fields[4];
if ($conditions[0] ne 'tr' # We know that these languages have
# conditions, and some are multiple
&& $conditions[0] ne 'az'
&& $conditions[0] ne 'lt'
# And, we know about a single condition Final_Sigma, but
# nothing else.
&& ($v_version gt v5.2.0
&& (@conditions > 1 || $conditions[0] ne 'Final_Sigma')))
{
$file->carp_bad_line("Unknown condition '$fields[4]'. You should inspect it and either add code to handle it, or add to list of those that are to ignore");
}
elsif ($conditions[0] ne 'Final_Sigma') {
# Don't print out a message for Final_Sigma, because we
# have hard-coded handling for it. (But the standard
# could change what the rule should be, but it wouldn't
# show up here anyway.
print "# SKIPPING Special Casing: $_\n"
if $verbosity >= $VERBOSE;
}
$_ = "";
return;
}
elsif (@fields > 6 || (@fields == 6 && $fields[5] ne "" )) {
$file->carp_bad_line('Extra fields');
$_ = "";
return;
}
my $decimal_code_point = hex $fields[0];
# Loop to handle each of the three mappings in the input line, in
# order, with $i indicating the current field number.
my $i = 0;
for my $object ($lc, $tc, $uc) {
$i++; # First time through, $i = 0 ... 3rd time = 3
my $value = $object->value_of($decimal_code_point);
$value = ($value eq $CODE_POINT)
? $decimal_code_point
: hex $value;
# If this isn't a multi-character mapping, it should already have
# been read in.
if ($fields[$i] !~ / /) {
if ($value != hex $fields[$i]) {
Carp::my_carp("Bad news. UnicodeData.txt thinks "
. $object->name
. "(0x$fields[0]) is $value"
. " and SpecialCasing.txt thinks it is "
. hex($fields[$i])
. ". Good luck. Retaining UnicodeData value, and proceeding anyway.");
}
}
else {
# The mapping goes into both the legacy table, in which it
# replaces the simple one...
$file->insert_adjusted_lines("$fields[0]; Legacy_"
. $object->full_name
. "; $fields[$i]");
# ... and the regular table, in which it is additional,
# beyond the simple mapping.
$file->insert_adjusted_lines("$fields[0]; "
. $object->name
. "; "
. $CMD_DELIM
. "$REPLACE_CMD=$MULTIPLE_BEFORE"
. $CMD_DELIM
. $fields[$i]);
}
}
# Everything has been handled by the insert_adjusted_lines()
$_ = "";
return;
}
}
sub filter_old_style_case_folding {
# This transforms $_ containing the case folding style of 3.0.1, to 3.1
# and later style. Different letters were used in the earlier.
my $file = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my @fields = split /\s*;\s*/;
if ($fields[1] eq 'L') {
$fields[1] = 'C'; # L => C always
}
elsif ($fields[1] eq 'E') {
if ($fields[2] =~ / /) { # E => C if one code point; F otherwise
$fields[1] = 'F'
}
else {
$fields[1] = 'C'
}
}
else {
$file->carp_bad_line("Expecting L or E in second field");
$_ = "";
return;
}
$_ = join("; ", @fields) . ';';
return;
}
{ # Closure for case folding
# Create the map for simple only if are going to output it, for otherwise
# it takes no part in anything we do.
my $to_output_simple;
sub setup_case_folding($) {
# Read in the case foldings in CaseFolding.txt. This handles both
# simple and full case folding.
$to_output_simple
= property_ref('Simple_Case_Folding')->to_output_map;
if (! $to_output_simple) {
property_ref('Case_Folding')->set_proxy_for('Simple_Case_Folding');
}
# If we ever wanted to show that these tables were combined, a new
# property method could be created, like set_combined_props()
property_ref('Case_Folding')->add_comment(join_lines( <<END
This file includes both the simple and full case folding maps. The simple
ones are in the main body of the table below, and the full ones adding to or
overriding them are in the hash.
END
));
return;
}
sub filter_case_folding_line {
# Called for each line in CaseFolding.txt
# Input lines look like:
# 0041; C; 0061; # LATIN CAPITAL LETTER A
# 00DF; F; 0073 0073; # LATIN SMALL LETTER SHARP S
# 1E9E; S; 00DF; # LATIN CAPITAL LETTER SHARP S
#
# 'C' means that folding is the same for both simple and full
# 'F' that it is only for full folding
# 'S' that it is only for simple folding
# 'T' is locale-dependent, and ignored
# 'I' is a type of 'F' used in some early releases.
# Note the trailing semi-colon, unlike many of the input files. That
# means that there will be an extra null field generated by the split
# below, which we ignore and hence is not an error.
my $file = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my ($range, $type, $map, @remainder) = split /\s*;\s*/, $_, -1;
if (@remainder > 1 || (@remainder == 1 && $remainder[0] ne "" )) {
$file->carp_bad_line('Extra fields');
$_ = "";
return;
}
if ($type =~ / ^ [IT] $/x) { # Skip Turkic case folding, is locale dependent
$_ = "";
return;
}
# C: complete, F: full, or I: dotted uppercase I -> dotless lowercase
# I are all full foldings; S is single-char. For S, there is always
# an F entry, so we must allow multiple values for the same code
# point. Fortunately this table doesn't need further manipulation
# which would preclude using multiple-values. The S is now included
# so that _swash_inversion_hash() is able to construct closures
# without having to worry about F mappings.
if ($type eq 'C' || $type eq 'F' || $type eq 'I' || $type eq 'S') {
$_ = "$range; Case_Folding; "
. "$CMD_DELIM$REPLACE_CMD=$MULTIPLE_BEFORE$CMD_DELIM$map";
}
else {
$_ = "";
$file->carp_bad_line('Expecting C F I S or T in second field');
}
# C and S are simple foldings, but simple case folding is not needed
# unless we explicitly want its map table output.
if ($to_output_simple && $type eq 'C' || $type eq 'S') {
$file->insert_adjusted_lines("$range; Simple_Case_Folding; $map");
}
return;
}
} # End case fold closure
sub filter_jamo_line {
# Filter Jamo.txt lines. This routine mainly is used to populate hashes
# from this file that is used in generating the Name property for Jamo
# code points. But, it also is used to convert early versions' syntax
# into the modern form. Here are two examples:
# 1100; G # HANGUL CHOSEONG KIYEOK # Modern syntax
# U+1100; G; HANGUL CHOSEONG KIYEOK # 2.0 syntax
#
# The input is $_, the output is $_ filtered.
my @fields = split /\s*;\s*/, $_, -1; # -1 => retain trailing null fields
# Let the caller handle unexpected input. In earlier versions, there was
# a third field which is supposed to be a comment, but did not have a '#'
# before it.
return if @fields > (($v_version gt v3.0.0) ? 2 : 3);
$fields[0] =~ s/^U\+//; # Also, early versions had this extraneous
# beginning.
# Some 2.1 versions had this wrong. Causes havoc with the algorithm.
$fields[1] = 'R' if $fields[0] eq '1105';
# Add to structure so can generate Names from it.
my $cp = hex $fields[0];
my $short_name = $fields[1];
$Jamo{$cp} = $short_name;
if ($cp <= $LBase + $LCount) {
$Jamo_L{$short_name} = $cp - $LBase;
}
elsif ($cp <= $VBase + $VCount) {
$Jamo_V{$short_name} = $cp - $VBase;
}
elsif ($cp <= $TBase + $TCount) {
$Jamo_T{$short_name} = $cp - $TBase;
}
else {
Carp::my_carp_bug("Unexpected Jamo code point in $_");
}
# Reassemble using just the first two fields to look like a typical
# property file line
$_ = "$fields[0]; $fields[1]";
return;
}
sub register_fraction($) {
# This registers the input rational number so that it can be passed on to
# utf8_heavy.pl, both in rational and floating forms.
my $rational = shift;
my $floating = eval $rational;
my @floats = sprintf "%.*e", $E_FLOAT_PRECISION, $floating;
# See if the denominator is a power of 2.
$rational =~ m!.*/(.*)!;
my $denominator = $1;
if (defined $denominator && (($denominator & ($denominator - 1)) == 0)) {
# Here the denominator is a power of 2. This means it has an exact
# representation in binary, so rounding could go either way. It turns
# out that Windows doesn't necessarily round towards even, so output
# an extra entry. This happens when the final digit we output is even
# and the next digits would be 50* to the precision of the machine.
my $extra_digit_float = sprintf "%e", $floating;
my $q = $E_FLOAT_PRECISION - 1;
if ($extra_digit_float =~ / ( .* \. \d{$q} )
( [02468] ) 5 0* ( e .*)
/ix)
{
push @floats, $1 . ($2 + 1) . $3;
}
}
foreach my $float (@floats) {
# Strip off any leading zeros beyond 2 digits to make it C99
# compliant. (Windows has 3 digit exponents, contrary to C99)
$float =~ s/ ( .* e [-+] ) 0* ( \d{2,}? ) /$1$2/x;
if ( defined $nv_floating_to_rational{$float}
&& $nv_floating_to_rational{$float} ne $rational)
{
die Carp::my_carp_bug("Both '$rational' and"
. " '$nv_floating_to_rational{$float}' evaluate to"
. " the same floating point number."
. " \$E_FLOAT_PRECISION must be increased");
}
$nv_floating_to_rational{$float} = $rational;
}
return;
}
sub gcd($$) { # Greatest-common-divisor; from
# http://en.wikipedia.org/wiki/Euclidean_algorithm
my ($a, $b) = @_;
use integer;
while ($b != 0) {
my $temp = $b;
$b = $a % $b;
$a = $temp;
}
return $a;
}
sub reduce_fraction($) {
my $fraction_ref = shift;
# Reduce a fraction to lowest terms. The Unicode data may be reducible,
# hence this is needed. The argument is a reference to the
# string denoting the fraction, which must be of the form:
if ($$fraction_ref !~ / ^ (-?) (\d+) \/ (\d+) $ /ax) {
Carp::my_carp_bug("Non-fraction input '$$fraction_ref'. Unchanged");
return;
}
my $sign = $1;
my $numerator = $2;
my $denominator = $3;
use integer;
# Find greatest common divisor
my $gcd = gcd($numerator, $denominator);
# And reduce using the gcd.
if ($gcd != 1) {
$numerator /= $gcd;
$denominator /= $gcd;
$$fraction_ref = "$sign$numerator/$denominator";
}
return;
}
sub filter_numeric_value_line {
# DNumValues contains lines of a different syntax than the typical
# property file:
# 0F33 ; -0.5 ; ; -1/2 # No TIBETAN DIGIT HALF ZERO
#
# This routine transforms $_ containing the anomalous syntax to the
# typical, by filtering out the extra columns, and convert early version
# decimal numbers to strings that look like rational numbers.
my $file = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
# Starting in 5.1, there is a rational field. Just use that, omitting the
# extra columns. Otherwise convert the decimal number in the second field
# to a rational, and omit extraneous columns.
my @fields = split /\s*;\s*/, $_, -1;
my $rational;
if ($v_version ge v5.1.0) {
if (@fields != 4) {
$file->carp_bad_line('Not 4 semi-colon separated fields');
$_ = "";
return;
}
reduce_fraction(\$fields[3]) if $fields[3] =~ qr{/};
$rational = $fields[3];
$_ = join '; ', @fields[ 0, 3 ];
}
else {
# Here, is an older Unicode file, which has decimal numbers instead of
# rationals in it. Use the fraction to calculate the denominator and
# convert to rational.
if (@fields != 2 && @fields != 3) {
$file->carp_bad_line('Not 2 or 3 semi-colon separated fields');
$_ = "";
return;
}
my $codepoints = $fields[0];
my $decimal = $fields[1];
if ($decimal =~ s/\.0+$//) {
# Anything ending with a decimal followed by nothing but 0's is an
# integer
$_ = "$codepoints; $decimal";
$rational = $decimal;
}
else {
my $denominator;
if ($decimal =~ /\.50*$/) {
$denominator = 2;
}
# Here have the hardcoded repeating decimals in the fraction, and
# the denominator they imply. There were only a few denominators
# in the older Unicode versions of this file which this code
# handles, so it is easy to convert them.
# The 4 is because of a round-off error in the Unicode 3.2 files
elsif ($decimal =~ /\.33*[34]$/ || $decimal =~ /\.6+7$/) {
$denominator = 3;
}
elsif ($decimal =~ /\.[27]50*$/) {
$denominator = 4;
}
elsif ($decimal =~ /\.[2468]0*$/) {
$denominator = 5;
}
elsif ($decimal =~ /\.16+7$/ || $decimal =~ /\.83+$/) {
$denominator = 6;
}
elsif ($decimal =~ /\.(12|37|62|87)50*$/) {
$denominator = 8;
}
if ($denominator) {
my $sign = ($decimal < 0) ? "-" : "";
my $numerator = int((abs($decimal) * $denominator) + .5);
$rational = "$sign$numerator/$denominator";
$_ = "$codepoints; $rational";
}
else {
$file->carp_bad_line("Can't cope with number '$decimal'.");
$_ = "";
return;
}
}
}
register_fraction($rational) if $rational =~ qr{/};
return;
}
{ # Closure
my %unihan_properties;
sub construct_unihan {
my $file_object = shift;
return unless file_exists($file_object->file);
if ($v_version lt v4.0.0) {
push @cjk_properties, 'URS ; Unicode_Radical_Stroke';
push @cjk_property_values, split "\n", <<'END';
# @missing: 0000..10FFFF; Unicode_Radical_Stroke; <none>
END
}
if ($v_version ge v3.0.0) {
push @cjk_properties, split "\n", <<'END';
cjkIRG_GSource; kIRG_GSource
cjkIRG_JSource; kIRG_JSource
cjkIRG_KSource; kIRG_KSource
cjkIRG_TSource; kIRG_TSource
cjkIRG_VSource; kIRG_VSource
END
push @cjk_property_values, split "\n", <<'END';
# @missing: 0000..10FFFF; cjkIRG_GSource; <none>
# @missing: 0000..10FFFF; cjkIRG_JSource; <none>
# @missing: 0000..10FFFF; cjkIRG_KSource; <none>
# @missing: 0000..10FFFF; cjkIRG_TSource; <none>
# @missing: 0000..10FFFF; cjkIRG_VSource; <none>
END
}
if ($v_version ge v3.1.0) {
push @cjk_properties, 'cjkIRG_HSource; kIRG_HSource';
push @cjk_property_values, '# @missing: 0000..10FFFF; cjkIRG_HSource; <none>';
}
if ($v_version ge v3.1.1) {
push @cjk_properties, 'cjkIRG_KPSource; kIRG_KPSource';
push @cjk_property_values, '# @missing: 0000..10FFFF; cjkIRG_KPSource; <none>';
}
if ($v_version ge v3.2.0) {
push @cjk_properties, split "\n", <<'END';
cjkAccountingNumeric; kAccountingNumeric
cjkCompatibilityVariant; kCompatibilityVariant
cjkOtherNumeric; kOtherNumeric
cjkPrimaryNumeric; kPrimaryNumeric
END
push @cjk_property_values, split "\n", <<'END';
# @missing: 0000..10FFFF; cjkAccountingNumeric; NaN
# @missing: 0000..10FFFF; cjkCompatibilityVariant; <code point>
# @missing: 0000..10FFFF; cjkOtherNumeric; NaN
# @missing: 0000..10FFFF; cjkPrimaryNumeric; NaN
END
}
if ($v_version gt v4.0.0) {
push @cjk_properties, 'cjkIRG_USource; kIRG_USource';
push @cjk_property_values, '# @missing: 0000..10FFFF; cjkIRG_USource; <none>';
}
if ($v_version ge v4.1.0) {
push @cjk_properties, 'cjkIICore ; kIICore';
push @cjk_property_values, '# @missing: 0000..10FFFF; cjkIICore; <none>';
}
}
sub setup_unihan {
# Do any special setup for Unihan properties.
# This property gives the wrong computed type, so override.
my $usource = property_ref('kIRG_USource');
$usource->set_type($STRING) if defined $usource;
# This property is to be considered binary (it says so in
# http://www.unicode.org/reports/tr38/)
my $iicore = property_ref('kIICore');
if (defined $iicore) {
$iicore->set_type($FORCED_BINARY);
$iicore->table("Y")->add_note("Matches any code point which has a non-null value for this property; see unicode.org UAX #38.");
# Unicode doesn't include the maps for this property, so don't
# warn that they are missing.
$iicore->set_pre_declared_maps(0);
$iicore->add_comment(join_lines( <<END
This property contains string values, but any non-empty ones are considered to
be 'core', so Perl creates tables for both: 1) its string values, plus 2)
tables so that \\p{kIICore} matches any code point which has a non-empty
value for this property.
END
));
}
return;
}
sub filter_unihan_line {
# Change unihan db lines to look like the others in the db. Here is
# an input sample:
# U+341C kCangjie IEKN
# Tabs are used instead of semi-colons to separate fields; therefore
# they may have semi-colons embedded in them. Change these to periods
# so won't screw up the rest of the code.
s/;/./g;
# Remove lines that don't look like ones we accept.
if ($_ !~ /^ [^\t]* \t ( [^\t]* ) /x) {
$_ = "";
return;
}
# Extract the property, and save a reference to its object.
my $property = $1;
if (! exists $unihan_properties{$property}) {
$unihan_properties{$property} = property_ref($property);
}
# Don't do anything unless the property is one we're handling, which
# we determine by seeing if there is an object defined for it or not
if (! defined $unihan_properties{$property}) {
$_ = "";
return;
}
# Convert the tab separators to our standard semi-colons, and convert
# the U+HHHH notation to the rest of the standard's HHHH
s/\t/;/g;
s/\b U \+ (?= $code_point_re )//xg;
#local $to_trace = 1 if main::DEBUG;
trace $_ if main::DEBUG && $to_trace;
return;
}
}
sub filter_blocks_lines {
# In the Blocks.txt file, the names of the blocks don't quite match the
# names given in PropertyValueAliases.txt, so this changes them so they
# do match: Blanks and hyphens are changed into underscores. Also makes
# early release versions look like later ones
#
# $_ is transformed to the correct value.
my $file = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
if ($v_version lt v3.2.0) {
if (/FEFF.*Specials/) { # Bug in old versions: line wrongly inserted
$_ = "";
return;
}
# Old versions used a different syntax to mark the range.
$_ =~ s/;\s+/../ if $v_version lt v3.1.0;
}
my @fields = split /\s*;\s*/, $_, -1;
if (@fields != 2) {
$file->carp_bad_line("Expecting exactly two fields");
$_ = "";
return;
}
# Change hyphens and blanks in the block name field only
$fields[1] =~ s/[ -]/_/g;
$fields[1] =~ s/_ ( [a-z] ) /_\u$1/xg; # Capitalize first letter of word
$_ = join("; ", @fields);
return;
}
{ # Closure
my $current_property;
sub filter_old_style_proplist {
# PropList.txt has been in Unicode since version 2.0. Until 3.1, it
# was in a completely different syntax. Ken Whistler of Unicode says
# that it was something he used as an aid for his own purposes, but
# was never an official part of the standard. Many of the properties
# in it were incorporated into the later PropList.txt, but some were
# not. This program uses this early file to generate property tables
# that are otherwise not accessible in the early UCD's. It does this
# for the ones that eventually became official, and don't appear to be
# too different in their contents from the later official version, and
# throws away the rest. It could be argued that the ones it generates
# were probably not really official at that time, so should be
# ignored. You can easily modify things to skip all of them by
# changing this function to just set $_ to "", and return; and to skip
# certain of them by by simply removing their declarations from
# get_old_property_aliases().
#
# Here is a list of all the ones that are thrown away:
# Alphabetic The definitions for this are very
# defective, so better to not mislead
# people into thinking it works.
# Instead the Perl extension of the
# same name is constructed from first
# principles.
# Bidi=* duplicates UnicodeData.txt
# Combining never made into official property;
# is \P{ccc=0}
# Composite never made into official property.
# Currency Symbol duplicates UnicodeData.txt: gc=sc
# Decimal Digit duplicates UnicodeData.txt: gc=nd
# Delimiter never made into official property;
# removed in 3.0.1
# Format Control never made into official property;
# similar to gc=cf
# High Surrogate duplicates Blocks.txt
# Ignorable Control never made into official property;
# similar to di=y
# ISO Control duplicates UnicodeData.txt: gc=cc
# Left of Pair never made into official property;
# Line Separator duplicates UnicodeData.txt: gc=zl
# Low Surrogate duplicates Blocks.txt
# Non-break was actually listed as a property
# in 3.2, but without any code
# points. Unicode denies that this
# was ever an official property
# Non-spacing duplicate UnicodeData.txt: gc=mn
# Numeric duplicates UnicodeData.txt: gc=cc
# Paired Punctuation never made into official property;
# appears to be gc=ps + gc=pe
# Paragraph Separator duplicates UnicodeData.txt: gc=cc
# Private Use duplicates UnicodeData.txt: gc=co
# Private Use High Surrogate duplicates Blocks.txt
# Punctuation duplicates UnicodeData.txt: gc=p
# Space different definition than eventual
# one.
# Titlecase duplicates UnicodeData.txt: gc=lt
# Unassigned Code Value duplicates UnicodeData.txt: gc=cn
# Zero-width never made into official property;
# subset of gc=cf
# Most of the properties have the same names in this file as in later
# versions, but a couple do not.
#
# This subroutine filters $_, converting it from the old style into
# the new style. Here's a sample of the old-style
#
# *******************************************
#
# Property dump for: 0x100000A0 (Join Control)
#
# 200C..200D (2 chars)
#
# In the example, the property is "Join Control". It is kept in this
# closure between calls to the subroutine. The numbers beginning with
# 0x were internal to Ken's program that generated this file.
# If this line contains the property name, extract it.
if (/^Property dump for: [^(]*\((.*)\)/) {
$_ = $1;
# Convert white space to underscores.
s/ /_/g;
# Convert the few properties that don't have the same name as
# their modern counterparts
s/Identifier_Part/ID_Continue/
or s/Not_a_Character/NChar/;
# If the name matches an existing property, use it.
if (defined property_ref($_)) {
trace "new property=", $_ if main::DEBUG && $to_trace;
$current_property = $_;
}
else { # Otherwise discard it
trace "rejected property=", $_ if main::DEBUG && $to_trace;
undef $current_property;
}
$_ = ""; # The property is saved for the next lines of the
# file, but this defining line is of no further use,
# so clear it so that the caller won't process it
# further.
}
elsif (! defined $current_property || $_ !~ /^$code_point_re/) {
# Here, the input line isn't a header defining a property for the
# following section, and either we aren't in such a section, or
# the line doesn't look like one that defines the code points in
# such a section. Ignore this line.
$_ = "";
}
else {
# Here, we have a line defining the code points for the current
# stashed property. Anything starting with the first blank is
# extraneous. Otherwise, it should look like a normal range to
# the caller. Append the property name so that it looks just like
# a modern PropList entry.
$_ =~ s/\s.*//;
$_ .= "; $current_property";
}
trace $_ if main::DEBUG && $to_trace;
return;
}
} # End closure for old style proplist
sub filter_old_style_normalization_lines {
# For early releases of Unicode, the lines were like:
# 74..2A76 ; NFKD_NO
# For later releases this became:
# 74..2A76 ; NFKD_QC; N
# Filter $_ to look like those in later releases.
# Similarly for MAYBEs
s/ _NO \b /_QC; N/x || s/ _MAYBE \b /_QC; M/x;
# Also, the property FC_NFKC was abbreviated to FNC
s/FNC/FC_NFKC/;
return;
}
sub setup_script_extensions {
# The Script_Extensions property starts out with a clone of the Script
# property.
$scx = property_ref("Script_Extensions");
return unless defined $scx;
$scx->_set_format($STRING_WHITE_SPACE_LIST);
$scx->initialize($script);
$scx->set_default_map($script->default_map);
$scx->set_pre_declared_maps(0); # PropValueAliases doesn't list these
$scx->add_comment(join_lines( <<END
The values for code points that appear in one script are just the same as for
the 'Script' property. Likewise the values for those that appear in many
scripts are either 'Common' or 'Inherited', same as with 'Script'. But the
values of code points that appear in a few scripts are a space separated list
of those scripts.
END
));
# Initialize scx's tables and the aliases for them to be the same as sc's
foreach my $table ($script->tables) {
my $scx_table = $scx->add_match_table($table->name,
Full_Name => $table->full_name);
foreach my $alias ($table->aliases) {
$scx_table->add_alias($alias->name);
}
}
}
sub filter_script_extensions_line {
# The Scripts file comes with the full name for the scripts; the
# ScriptExtensions, with the short name. The final mapping file is a
# combination of these, and without adjustment, would have inconsistent
# entries. This filters the latter file to convert to full names.
# Entries look like this:
# 064B..0655 ; Arab Syrc # Mn [11] ARABIC FATHATAN..ARABIC HAMZA BELOW
my @fields = split /\s*;\s*/;
# This script was erroneously omitted in this Unicode version.
$fields[1] .= ' Takr' if $v_version eq v6.1.0 && $fields[0] =~ /^0964/;
my @full_names;
foreach my $short_name (split " ", $fields[1]) {
push @full_names, $script->table($short_name)->full_name;
}
$fields[1] = join " ", @full_names;
$_ = join "; ", @fields;
return;
}
sub setup_emojidata {
my $prop_ref = Property->new('XPG',
Full_Name => 'Extended_Pictographic',
);
$prop_ref->set_fate($PLACEHOLDER,
"Not part of the Unicode Character Database");
}
sub filter_emojidata_line {
# We only are interested in this single property from this non-UCD data
# file, and we turn it into a Perl property, so that it isn't accessible
# to the users
$_ = "" unless /\bExtended_Pictographic\b/;
return;
}
sub generate_hst {
# Populates the Hangul Syllable Type property from first principles
my $file= shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
# These few ranges are hard-coded in.
$file->insert_lines(split /\n/, <<'END'
1100..1159 ; L
115F ; L
1160..11A2 ; V
11A8..11F9 ; T
END
);
# The Hangul syllables in version 1 are at different code points than
# those that came along starting in version 2, and have different names;
# they comprise about 60% of the code points of the later version.
# From my (khw) research on them (see <558493EB.4000807@att.net>), the
# initial set is a subset of the later version, with different English
# transliterations. I did not see an easy mapping between them. The
# later set includes essentially all possibilities, even ones that aren't
# in modern use (if they ever were), and over 96% of the new ones are type
# LVT. Mathematically, the early set must also contain a preponderance of
# LVT values. In lieu of doing nothing, we just set them all to LVT, and
# expect that this will be right most of the time, which is better than
# not being right at all.
if ($v_version lt v2.0.0) {
my $property = property_ref($file->property);
$file->insert_lines(sprintf("%04X..%04X; LVT\n",
$FIRST_REMOVED_HANGUL_SYLLABLE,
$FINAL_REMOVED_HANGUL_SYLLABLE));
push @tables_that_may_be_empty, $property->table('LV')->complete_name;
return;
}
# The algorithmically derived syllables are almost all LVT ones, so
# initialize the whole range with that.
$file->insert_lines(sprintf "%04X..%04X; LVT\n",
$SBase, $SBase + $SCount -1);
# Those ones that aren't LVT are LV, and they occur at intervals of
# $TCount code points, starting with the first code point, at $SBase.
for (my $i = $SBase; $i < $SBase + $SCount; $i += $TCount) {
$file->insert_lines(sprintf "%04X..%04X; LV\n", $i, $i);
}
return;
}
sub generate_GCB {
# Populates the Grapheme Cluster Break property from first principles
my $file= shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
# All these definitions are from
# http://www.unicode.org/reports/tr29/tr29-3.html with confirmation
# from http://www.unicode.org/reports/tr29/tr29-4.html
foreach my $range ($gc->ranges) {
# Extend includes gc=Me and gc=Mn, while Control includes gc=Cc
# and gc=Cf
if ($range->value =~ / ^ M [en] $ /x) {
$file->insert_lines(sprintf "%04X..%04X; Extend",
$range->start, $range->end);
}
elsif ($range->value =~ / ^ C [cf] $ /x) {
$file->insert_lines(sprintf "%04X..%04X; Control",
$range->start, $range->end);
}
}
$file->insert_lines("2028; Control"); # Line Separator
$file->insert_lines("2029; Control"); # Paragraph Separator
$file->insert_lines("000D; CR");
$file->insert_lines("000A; LF");
# Also from http://www.unicode.org/reports/tr29/tr29-3.html.
foreach my $code_point ( qw{
09BE 09D7 0B3E 0B57 0BBE 0BD7 0CC2 0CD5 0CD6
0D3E 0D57 0DCF 0DDF FF9E FF9F 1D165 1D16E 1D16F
}
) {
my $category = $gc->value_of(hex $code_point);
next if ! defined $category || $category eq 'Cn'; # But not if
# unassigned in this
# release
$file->insert_lines("$code_point; Extend");
}
my $hst = property_ref('Hangul_Syllable_Type');
if ($hst->count > 0) {
foreach my $range ($hst->ranges) {
$file->insert_lines(sprintf "%04X..%04X; %s",
$range->start, $range->end, $range->value);
}
}
else {
generate_hst($file);
}
main::process_generic_property_file($file);
}
sub fixup_early_perl_name_alias {
# Different versions of Unicode have varying support for the name synonyms
# below. Just include everything. As of 6.1, all these are correct in
# the Unicode-supplied file.
my $file= shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
# ALERT did not come along until 6.0, at which point it became preferred
# over BELL. By inserting it last in early releases, BELL is preferred
# over it; and vice-vers in 6.0
my $type_for_bell = ($v_version lt v6.0.0)
? 'correction'
: 'alternate';
$file->insert_lines(split /\n/, <<END
0007;BELL; $type_for_bell
000A;LINE FEED (LF);alternate
000C;FORM FEED (FF);alternate
000D;CARRIAGE RETURN (CR);alternate
0085;NEXT LINE (NEL);alternate
END
);
# One might think that the the 'Unicode_1_Name' field, could work for most
# of the above names, but sadly that field varies depending on the
# release. Version 1.1.5 had no names for any of the controls; Version
# 2.0 introduced names for the C0 controls, and 3.0 introduced C1 names.
# 3.0.1 removed the name INDEX; and 3.2 changed some names:
# changed to parenthesized versions like "NEXT LINE" to
# "NEXT LINE (NEL)";
# changed PARTIAL LINE DOWN to PARTIAL LINE FORWARD
# changed PARTIAL LINE UP to PARTIAL LINE BACKWARD;;
# changed e.g. FILE SEPARATOR to INFORMATION SEPARATOR FOUR
#
# All these are present in the 6.1 NameAliases.txt
return;
}
sub filter_later_version_name_alias_line {
# This file has an extra entry per line for the alias type. This is
# handled by creating a compound entry: "$alias: $type"; First, split
# the line into components.
my ($range, $alias, $type, @remainder)
= split /\s*;\s*/, $_, -1; # -1 => retain trailing null fields
# This file contains multiple entries for some components, so tell the
# downstream code to allow this in our internal tables; the
# $MULTIPLE_AFTER preserves the input ordering.
$_ = join ";", $range, $CMD_DELIM
. $REPLACE_CMD
. '='
. $MULTIPLE_AFTER
. $CMD_DELIM
. "$alias: $type",
@remainder;
return;
}
sub filter_early_version_name_alias_line {
# Early versions did not have the trailing alias type field; implicitly it
# was 'correction'.
$_ .= "; correction";
filter_later_version_name_alias_line;
return;
}
sub filter_all_caps_script_names {
# Some early Unicode releases had the script names in all CAPS. This
# converts them to just the first letter of each word being capital.
my ($range, $script, @remainder)
= split /\s*;\s*/, $_, -1; # -1 => retain trailing null fields
my @words = split /[_-]/, $script;
for my $word (@words) {
$word =
ucfirst(lc($word)) if $word ne 'CJK';
}
$script = join "_", @words;
$_ = join ";", $range, $script, @remainder;
}
sub finish_Unicode() {
# This routine should be called after all the Unicode files have been read
# in. It:
# 1) Creates properties that are missing from the version of Unicode being
# compiled, and which, for whatever reason, are needed for the Perl
# core to function properly. These are minimally populated as
# necessary.
# 2) Adds the mappings for code points missing from the files which have
# defaults specified for them.
# 3) At this this point all mappings are known, so it computes the type of
# each property whose type hasn't been determined yet.
# 4) Calculates all the regular expression match tables based on the
# mappings.
# 5) Calculates and adds the tables which are defined by Unicode, but
# which aren't derived by them, and certain derived tables that Perl
# uses.
# Folding information was introduced later into Unicode data. To get
# Perl's case ignore (/i) to work at all in releases that don't have
# folding, use the best available alternative, which is lower casing.
my $fold = property_ref('Case_Folding');
if ($fold->is_empty) {
$fold->initialize(property_ref('Lowercase_Mapping'));
$fold->add_note(join_lines(<<END
WARNING: This table uses lower case as a substitute for missing fold
information
END
));
}
# Multiple-character mapping was introduced later into Unicode data, so it
# is by default the simple version. If to output the simple versions and
# not present, just use the regular (which in these Unicode versions is
# the simple as well).
foreach my $map (qw { Uppercase_Mapping
Lowercase_Mapping
Titlecase_Mapping
Case_Folding
} )
{
my $comment = <<END;
Note that although the Perl core uses this file, it has the standard values
for code points from U+0000 to U+00FF compiled in, so changing this table will
not change the core's behavior with respect to these code points. Use
Unicode::Casing to override this table.
END
if ($map eq 'Case_Folding') {
$comment .= <<END;
(/i regex matching is not overridable except by using a custom regex engine)
END
}
property_ref($map)->add_comment(join_lines($comment));
my $simple = property_ref("Simple_$map");
next if ! $simple->is_empty;
if ($simple->to_output_map) {
$simple->initialize(property_ref($map));
}
else {
property_ref($map)->set_proxy_for($simple->name);
}
}
# For each property, fill in any missing mappings, and calculate the re
# match tables. If a property has more than one missing mapping, the
# default is a reference to a data structure, and may require data from
# other properties to resolve. The sort is used to cause these to be
# processed last, after all the other properties have been calculated.
# (Fortunately, the missing properties so far don't depend on each other.)
foreach my $property
(sort { (defined $a->default_map && ref $a->default_map) ? 1 : -1 }
property_ref('*'))
{
# $perl has been defined, but isn't one of the Unicode properties that
# need to be finished up.
next if $property == $perl;
# Nor do we need to do anything with properties that aren't going to
# be output.
next if $property->fate == $SUPPRESSED;
# Handle the properties that have more than one possible default
if (ref $property->default_map) {
my $default_map = $property->default_map;
# These properties have stored in the default_map:
# One or more of:
# 1) A default map which applies to all code points in a
# certain class
# 2) an expression which will evaluate to the list of code
# points in that class
# And
# 3) the default map which applies to every other missing code
# point.
#
# Go through each list.
while (my ($default, $eval) = $default_map->get_next_defaults) {
# Get the class list, and intersect it with all the so-far
# unspecified code points yielding all the code points
# in the class that haven't been specified.
my $list = eval $eval;
if ($@) {
Carp::my_carp("Can't set some defaults for missing code points for $property because eval '$eval' failed with '$@'");
last;
}
# Narrow down the list to just those code points we don't have
# maps for yet.
$list = $list & $property->inverse_list;
# Add mappings to the property for each code point in the list
foreach my $range ($list->ranges) {
$property->add_map($range->start, $range->end, $default,
Replace => $CROAK);
}
}
# All remaining code points have the other mapping. Set that up
# so the normal single-default mapping code will work on them
$property->set_default_map($default_map->other_default);
# And fall through to do that
}
# We should have enough data now to compute the type of the property.
my $property_name = $property->name;
$property->compute_type;
my $property_type = $property->type;
next if ! $property->to_create_match_tables;
# Here want to create match tables for this property
# The Unicode db always (so far, and they claim into the future) have
# the default for missing entries in binary properties be 'N' (unless
# there is a '@missing' line that specifies otherwise)
if (! defined $property->default_map) {
if ($property_type == $BINARY) {
$property->set_default_map('N');
}
elsif ($property_type == $ENUM) {
Carp::my_carp("Property '$property_name doesn't have a default mapping. Using a fake one");
$property->set_default_map('XXX This makes sure there is a default map');
}
}
# Add any remaining code points to the mapping, using the default for
# missing code points.
my $default_table;
my $default_map = $property->default_map;
if ($property_type == $FORCED_BINARY) {
# A forced binary property creates a 'Y' table that matches all
# non-default values. The actual string values are also written out
# as a map table. (The default value will almost certainly be the
# empty string, so the pod glosses over the distinction, and just
# talks about empty vs non-empty.)
my $yes = $property->table("Y");
foreach my $range ($property->ranges) {
next if $range->value eq $default_map;
$yes->add_range($range->start, $range->end);
}
$property->table("N")->set_complement($yes);
}
else {
if (defined $default_map) {
# Make sure there is a match table for the default
if (! defined ($default_table = $property->table($default_map)))
{
$default_table = $property->add_match_table($default_map);
}
# And, if the property is binary, the default table will just
# be the complement of the other table.
if ($property_type == $BINARY) {
my $non_default_table;
# Find the non-default table.
for my $table ($property->tables) {
if ($table == $default_table) {
if ($v_version le v5.0.0) {
$table->add_alias($_) for qw(N No F False);
}
next;
} elsif ($v_version le v5.0.0) {
$table->add_alias($_) for qw(Y Yes T True);
}
$non_default_table = $table;
}
$default_table->set_complement($non_default_table);
}
else {
# This fills in any missing values with the default. It's
# not necessary to do this with binary properties, as the
# default is defined completely in terms of the Y table.
$property->add_map(0, $MAX_WORKING_CODEPOINT,
$default_map, Replace => $NO);
}
}
# Have all we need to populate the match tables.
my $maps_should_be_defined = $property->pre_declared_maps;
foreach my $range ($property->ranges) {
my $map = $range->value;
my $table = $property->table($map);
if (! defined $table) {
# Integral and rational property values are not
# necessarily defined in PropValueAliases, but whether all
# the other ones should be depends on the property.
if ($maps_should_be_defined
&& $map !~ /^ -? \d+ ( \/ \d+ )? $/x)
{
Carp::my_carp("Table '$property_name=$map' should "
. "have been defined. Defining it now.")
}
$table = $property->add_match_table($map);
}
next if $table->complement != 0; # Don't need to populate these
$table->add_range($range->start, $range->end);
}
}
# For Perl 5.6 compatibility, all properties matchable in regexes can
# have an optional 'Is_' prefix. This is now done in utf8_heavy.pl.
# But warn if this creates a conflict with a (new) Unicode property
# name, although it appears that Unicode has made a decision never to
# begin a property name with 'Is_', so this shouldn't happen.
foreach my $alias ($property->aliases) {
my $Is_name = 'Is_' . $alias->name;
if (defined (my $pre_existing = property_ref($Is_name))) {
Carp::my_carp(<<END
There is already an alias named $Is_name (from " . $pre_existing . "), so
creating one for $property won't work. This is bad news. If it is not too
late, get Unicode to back off. Otherwise go back to the old scheme (findable
from the git blame log for this area of the code that suppressed individual
aliases that conflict with the new Unicode names. Proceeding anyway.
END
);
}
} # End of loop through aliases for this property
} # End of loop through all Unicode properties.
# Fill in the mappings that Unicode doesn't completely furnish. First the
# single letter major general categories. If Unicode were to start
# delivering the values, this would be redundant, but better that than to
# try to figure out if should skip and not get it right. Ths could happen
# if a new major category were to be introduced, and the hard-coded test
# wouldn't know about it.
# This routine depends on the standard names for the general categories
# being what it thinks they are, like 'Cn'. The major categories are the
# union of all the general category tables which have the same first
# letters. eg. L = Lu + Lt + Ll + Lo + Lm
foreach my $minor_table ($gc->tables) {
my $minor_name = $minor_table->name;
next if length $minor_name == 1;
if (length $minor_name != 2) {
Carp::my_carp_bug("Unexpected general category '$minor_name'. Skipped.");
next;
}
my $major_name = uc(substr($minor_name, 0, 1));
my $major_table = $gc->table($major_name);
$major_table += $minor_table;
}
# LC is Ll, Lu, and Lt. (used to be L& or L_, but PropValueAliases.txt
# defines it as LC)
my $LC = $gc->table('LC');
$LC->add_alias('L_', Status => $DISCOURAGED); # For backwards...
$LC->add_alias('L&', Status => $DISCOURAGED); # compatibility.
if ($LC->is_empty) { # Assume if not empty that Unicode has started to
# deliver the correct values in it
$LC->initialize($gc->table('Ll') + $gc->table('Lu'));
# Lt not in release 1.
if (defined $gc->table('Lt')) {
$LC += $gc->table('Lt');
$gc->table('Lt')->set_caseless_equivalent($LC);
}
}
$LC->add_description('[\p{Ll}\p{Lu}\p{Lt}]');
$gc->table('Ll')->set_caseless_equivalent($LC);
$gc->table('Lu')->set_caseless_equivalent($LC);
# Create digit and case fold tables with the original file names for
# backwards compatibility with applications that read them directly.
my $Digit = Property->new("Legacy_Perl_Decimal_Digit",
Default_Map => "",
File => 'Digit', # Trad. location
Directory => $map_directory,
Type => $STRING,
Replacement_Property => "Perl_Decimal_Digit",
Initialize => property_ref('Perl_Decimal_Digit'),
);
$Digit->add_comment(join_lines(<<END
This file gives the mapping of all code points which represent a single
decimal digit [0-9] to their respective digits. For example, the code point
U+0031 (an ASCII '1') is mapped to a numeric 1. These code points are those
that have Numeric_Type=Decimal; not special things, like subscripts nor Roman
numerals.
END
));
# Make sure this assumption in perl core code is valid in this Unicode
# release, with known exceptions
foreach my $range (property_ref('Numeric-Type')->table('Decimal')->ranges) {
next if $range->end - $range->start == 9;
next if $range->start == 0x1D7CE; # This whole range was added in 3.1
next if $range->end == 0x19DA && $v_version eq v5.2.0;
next if $range->end - $range->start < 9 && $v_version le 4.0.0;
Carp::my_carp("Range $range unexpectedly doesn't contain 10"
. " decimal digits. Code in regcomp.c assumes it does,"
. " and will have to be fixed. Proceeding anyway.");
}
Property->new('Legacy_Case_Folding',
File => "Fold",
Directory => $map_directory,
Default_Map => $CODE_POINT,
Type => $STRING,
Replacement_Property => "Case_Folding",
Format => $HEX_FORMAT,
Initialize => property_ref('cf'),
);
# The Script_Extensions property started out as a clone of the Script
# property. But processing its data file caused some elements to be
# replaced with different data. (These elements were for the Common and
# Inherited properties.) This data is a qw() list of all the scripts that
# the code points in the given range are in. An example line is:
# 060C ; Arab Syrc Thaa # Po ARABIC COMMA
#
# The code above has created a new match table named "Arab Syrc Thaa"
# which contains 060C. (The cloned table started out with this code point
# mapping to "Common".) Now we add 060C to each of the Arab, Syrc, and
# Thaa match tables. Then we delete the now spurious "Arab Syrc Thaa"
# match table. This is repeated for all these tables and ranges. The map
# data is retained in the map table for reference, but the spurious match
# tables are deleted.
if (defined $scx) {
foreach my $table ($scx->tables) {
next unless $table->name =~ /\s/; # All the new and only the new
# tables have a space in their
# names
my @scripts = split /\s+/, $table->name;
foreach my $script (@scripts) {
my $script_table = $scx->table($script);
$script_table += $table;
}
$scx->delete_match_table($table);
}
# Mark the scx table as the parent of the corresponding sc table for
# those which are identical. This causes the pod for the script table
# to refer to the corresponding scx one.
#
# This has to be in a separate loop from above, so as to wait until
# the tables are stabilized before checking for equivalency.
if (defined $pod_directory) {
foreach my $table ($scx->tables) {
my $plain_sc_equiv = $script->table($table->name);
if ($table->matches_identically_to($plain_sc_equiv)) {
$plain_sc_equiv->set_equivalent_to($table, Related => 1);
}
}
}
}
return;
}
sub pre_3_dot_1_Nl () {
# Return a range list for gc=nl for Unicode versions prior to 3.1, which
# is when Unicode's became fully usable. These code points were
# determined by inspection and experimentation. gc=nl is important for
# certain Perl-extension properties that should be available in all
# releases.
my $Nl = Range_List->new();
if (defined (my $official = $gc->table('Nl'))) {
$Nl += $official;
}
else {
$Nl->add_range(0x2160, 0x2182);
$Nl->add_range(0x3007, 0x3007);
$Nl->add_range(0x3021, 0x3029);
}
$Nl->add_range(0xFE20, 0xFE23);
$Nl->add_range(0x16EE, 0x16F0) if $v_version ge v3.0.0; # 3.0 was when
# these were added
return $Nl;
}
sub calculate_Assigned() { # Set $Assigned to the gc != Cn code points; may be
# called before the Cn's are completely filled.
# Works on Unicodes earlier than ones that
# explicitly specify Cn.
return if defined $Assigned;
if (! defined $gc || $gc->is_empty()) {
Carp::my_carp_bug("calculate_Assigned() called before $gc is populated");
}
$Assigned = $perl->add_match_table('Assigned',
Description => "All assigned code points",
);
while (defined (my $range = $gc->each_range())) {
my $standard_value = standardize($range->value);
next if $standard_value eq 'cn' || $standard_value eq 'unassigned';
$Assigned->add_range($range->start, $range->end);
}
}
sub calculate_DI() { # Set $DI to a Range_List equivalent to the
# Default_Ignorable_Code_Point property. Works on
# Unicodes earlier than ones that explicitly specify
# DI.
return if defined $DI;
if (defined (my $di = property_ref('Default_Ignorable_Code_Point'))) {
$DI = $di->table('Y');
}
else {
$DI = Range_List->new(Initialize => [ 0x180B .. 0x180D,
0x2060 .. 0x206F,
0xFE00 .. 0xFE0F,
0xFFF0 .. 0xFFFB,
]);
if ($v_version ge v2.0) {
$DI += $gc->table('Cf')
+ $gc->table('Cs');
# These are above the Unicode version 1 max
$DI->add_range(0xE0000, 0xE0FFF);
}
$DI += $gc->table('Cc')
- ord("\t")
- utf8::unicode_to_native(0x0A) # LINE FEED
- utf8::unicode_to_native(0x0B) # VERTICAL TAB
- ord("\f")
- utf8::unicode_to_native(0x0D) # CARRIAGE RETURN
- utf8::unicode_to_native(0x85); # NEL
}
}
sub calculate_NChar() { # Create a Perl extension match table which is the
# same as the Noncharacter_Code_Point property, and
# set $NChar to point to it. Works on Unicodes
# earlier than ones that explicitly specify NChar
return if defined $NChar;
$NChar = $perl->add_match_table('_Perl_Nchar',
Perl_Extension => 1,
Fate => $INTERNAL_ONLY);
if (defined (my $off_nchar = property_ref('NChar'))) {
$NChar->initialize($off_nchar->table('Y'));
}
else {
$NChar->initialize([ 0xFFFE .. 0xFFFF ]);
if ($v_version ge v2.0) { # First release with these nchars
for (my $i = 0x1FFFE; $i <= 0x10FFFE; $i += 0x10000) {
$NChar += [ $i .. $i+1 ];
}
}
}
}
sub handle_compare_versions () {
# This fixes things up for the $compare_versions capability, where we
# compare Unicode version X with version Y (with Y > X), and we are
# running it on the Unicode Data for version Y.
#
# It works by calculating the code points whose meaning has been specified
# after release X, by using the Age property. The complement of this set
# is the set of code points whose meaning is unchanged between the
# releases. This is the set the program restricts itself to. It includes
# everything whose meaning has been specified by the time version X came
# along, plus those still unassigned by the time of version Y. (We will
# continue to use the word 'assigned' to mean 'meaning has been
# specified', as it's shorter and is accurate in all cases except the
# Noncharacter code points.)
#
# This function is run after all the properties specified by Unicode have
# been calculated for release Y. This makes sure we get all the nuances
# of Y's rules. (It is done before the Perl extensions are calculated, as
# those are based entirely on the Unicode ones.) But doing it after the
# Unicode table calculations means we have to fix up the Unicode tables.
# We do this by subtracting the code points that have been assigned since
# X (which is actually done by ANDing each table of assigned code points
# with the set of unchanged code points). Most Unicode properties are of
# the form such that all unassigned code points have a default, grab-bag,
# property value which is changed when the code point gets assigned. For
# these, we just remove the changed code points from the table for the
# latter property value, and add them back in to the grab-bag one. A few
# other properties are not entirely of this form and have values for some
# or all unassigned code points that are not the grab-bag one. These have
# to be handled specially, and are hard-coded in to this routine based on
# manual inspection of the Unicode character database. A list of the
# outlier code points is made for each of these properties, and those
# outliers are excluded from adding and removing from tables.
#
# Note that there are glitches when comparing against Unicode 1.1, as some
# Hangul syllables in it were later ripped out and eventually replaced
# with other things.
print "Fixing up for version comparison\n" if $verbosity >= $PROGRESS;
my $after_first_version = "All matching code points were added after "
. "Unicode $string_compare_versions";
# Calculate the delta as those code points that have been newly assigned
# since the first compare version.
my $delta = Range_List->new();
foreach my $table ($age->tables) {
use version;
next if $table == $age->table('Unassigned');
next if version->parse($table->name)
le version->parse($string_compare_versions);
$delta += $table;
}
if ($delta->is_empty) {
die ("No changes; perhaps you need a 'DAge.txt' file?");
}
my $unchanged = ~ $delta;
calculate_Assigned() if ! defined $Assigned;
$Assigned &= $unchanged;
# $Assigned now contains the code points that were assigned as of Unicode
# version X.
# A block is all or nothing. If nothing is assigned in it, it all goes
# back to the No_Block pool; but if even one code point is assigned, the
# block is retained.
my $no_block = $block->table('No_Block');
foreach my $this_block ($block->tables) {
next if $this_block == $no_block
|| ! ($this_block & $Assigned)->is_empty;
$this_block->set_fate($SUPPRESSED, $after_first_version);
foreach my $range ($this_block->ranges) {
$block->replace_map($range->start, $range->end, 'No_Block')
}
$no_block += $this_block;
}
my @special_delta_properties; # List of properties that have to be
# handled specially.
my %restricted_delta; # Keys are the entries in
# @special_delta_properties; values
# are the range list of the code points
# that behave normally when they get
# assigned.
# In the next three properties, the Default Ignorable code points are
# outliers.
calculate_DI();
$DI &= $unchanged;
push @special_delta_properties, property_ref('_Perl_GCB');
$restricted_delta{$special_delta_properties[-1]} = ~ $DI;
if (defined (my $cwnfkcc = property_ref('Changes_When_NFKC_Casefolded')))
{
push @special_delta_properties, $cwnfkcc;
$restricted_delta{$special_delta_properties[-1]} = ~ $DI;
}
calculate_NChar(); # Non-character code points
$NChar &= $unchanged;
# This may have to be updated from time-to-time to get the most accurate
# results.
my $default_BC_non_LtoR = Range_List->new(Initialize =>
# These came from the comments in v8.0 DBidiClass.txt
[ # AL
0x0600 .. 0x07BF,
0x08A0 .. 0x08FF,
0xFB50 .. 0xFDCF,
0xFDF0 .. 0xFDFF,
0xFE70 .. 0xFEFF,
0x1EE00 .. 0x1EEFF,
# R
0x0590 .. 0x05FF,
0x07C0 .. 0x089F,
0xFB1D .. 0xFB4F,
0x10800 .. 0x10FFF,
0x1E800 .. 0x1EDFF,
0x1EF00 .. 0x1EFFF,
# ET
0x20A0 .. 0x20CF,
]
);
$default_BC_non_LtoR += $DI + $NChar;
push @special_delta_properties, property_ref('BidiClass');
$restricted_delta{$special_delta_properties[-1]} = ~ $default_BC_non_LtoR;
if (defined (my $eaw = property_ref('East_Asian_Width'))) {
my $default_EA_width_W = Range_List->new(Initialize =>
# From comments in v8.0 EastAsianWidth.txt
[
0x3400 .. 0x4DBF,
0x4E00 .. 0x9FFF,
0xF900 .. 0xFAFF,
0x20000 .. 0x2A6DF,
0x2A700 .. 0x2B73F,
0x2B740 .. 0x2B81F,
0x2B820 .. 0x2CEAF,
0x2F800 .. 0x2FA1F,
0x20000 .. 0x2FFFD,
0x30000 .. 0x3FFFD,
]
);
push @special_delta_properties, $eaw;
$restricted_delta{$special_delta_properties[-1]}
= ~ $default_EA_width_W;
# Line break came along in the same release as East_Asian_Width, and
# the non-grab-bag default set is a superset of the EAW one.
if (defined (my $lb = property_ref('Line_Break'))) {
my $default_LB_non_XX = Range_List->new(Initialize =>
# From comments in v8.0 LineBreak.txt
[ 0x20A0 .. 0x20CF ]);
$default_LB_non_XX += $default_EA_width_W;
push @special_delta_properties, $lb;
$restricted_delta{$special_delta_properties[-1]}
= ~ $default_LB_non_XX;
}
}
# Go through every property, skipping those we've already worked on, those
# that are immutable, and the perl ones that will be calculated after this
# routine has done its fixup.
foreach my $property (property_ref('*')) {
next if $property == $perl # Done later in the program
|| $property == $block # Done just above
|| $property == $DI # Done just above
|| $property == $NChar # Done just above
# The next two are invariant across Unicode versions
|| $property == property_ref('Pattern_Syntax')
|| $property == property_ref('Pattern_White_Space');
# Find the grab-bag value.
my $default_map = $property->default_map;
if (! $property->to_create_match_tables) {
# Here there aren't any match tables. So far, all such properties
# have a default map, and don't require special handling. Just
# change each newly assigned code point back to the default map,
# as if they were unassigned.
foreach my $range ($delta->ranges) {
$property->add_map($range->start,
$range->end,
$default_map,
Replace => $UNCONDITIONALLY);
}
}
else { # Here there are match tables. Find the one (if any) for the
# grab-bag value that unassigned code points go to.
my $default_table;
if (defined $default_map) {
$default_table = $property->table($default_map);
}
# If some code points don't go back to the the grab-bag when they
# are considered unassigned, exclude them from the list that does
# that.
my $this_delta = $delta;
my $this_unchanged = $unchanged;
if (grep { $_ == $property } @special_delta_properties) {
$this_delta = $delta & $restricted_delta{$property};
$this_unchanged = ~ $this_delta;
}
# Fix up each match table for this property.
foreach my $table ($property->tables) {
if (defined $default_table && $table == $default_table) {
# The code points assigned after release X (the ones we
# are excluding in this routine) go back on to the default
# (grab-bag) table. However, some of these tables don't
# actually exist, but are specified solely by the other
# tables. (In a binary property, we don't need to
# actually have an 'N' table, as it's just the complement
# of the 'Y' table.) Such tables will be locked, so just
# skip those.
$table += $this_delta unless $table->locked;
}
else {
# Here the table is not for the default value. We need to
# subtract the code points we are ignoring for this
# comparison (the deltas) from it. But if the table
# started out with nothing, no need to exclude anything,
# and want to skip it here anyway, so it gets listed
# properly in the pod.
next if $table->is_empty;
# Save the deltas for later, before we do the subtraction
my $deltas = $table & $this_delta;
$table &= $this_unchanged;
# Suppress the table if the subtraction left it with
# nothing in it
if ($table->is_empty) {
if ($property->type == $BINARY) {
push @tables_that_may_be_empty, $table->complete_name;
}
else {
$table->set_fate($SUPPRESSED, $after_first_version);
}
}
# Now we add the removed code points to the property's
# map, as they should now map to the grab-bag default
# property (which they did in the first comparison
# version). But we don't have to do this if the map is
# only for internal use.
if (defined $default_map && $property->to_output_map) {
# The gc property has pseudo property values whose names
# have length 1. These are the union of all the
# property values whose name is longer than 1 and
# whose first letter is all the same. The replacement
# is done once for the longer-named tables.
next if $property == $gc && length $table->name == 1;
foreach my $range ($deltas->ranges) {
$property->add_map($range->start,
$range->end,
$default_map,
Replace => $UNCONDITIONALLY);
}
}
}
}
}
}
# The above code doesn't work on 'gc=C', as it is a superset of the default
# ('Cn') table. It's easiest to just special case it here.
my $C = $gc->table('C');
$C += $gc->table('Cn');
return;
}
sub compile_perl() {
# Create perl-defined tables. Almost all are part of the pseudo-property
# named 'perl' internally to this program. Many of these are recommended
# in UTS#18 "Unicode Regular Expressions", and their derivations are based
# on those found there.
# Almost all of these are equivalent to some Unicode property.
# A number of these properties have equivalents restricted to the ASCII
# range, with their names prefaced by 'Posix', to signify that these match
# what the Posix standard says they should match. A couple are
# effectively this, but the name doesn't have 'Posix' in it because there
# just isn't any Posix equivalent. 'XPosix' are the Posix tables extended
# to the full Unicode range, by our guesses as to what is appropriate.
# 'All' is all code points. As an error check, instead of just setting it
# to be that, construct it to be the union of all the major categories
$All = $perl->add_match_table('All',
Description
=> "All code points, including those above Unicode. Same as qr/./s",
Matches_All => 1);
foreach my $major_table ($gc->tables) {
# Major categories are the ones with single letter names.
next if length($major_table->name) != 1;
$All += $major_table;
}
if ($All->max != $MAX_WORKING_CODEPOINT) {
Carp::my_carp_bug("Generated highest code point ("
. sprintf("%X", $All->max)
. ") doesn't match expected value $MAX_WORKING_CODEPOINT_STRING.")
}
if ($All->range_count != 1 || $All->min != 0) {
Carp::my_carp_bug("Generated table 'All' doesn't match all code points.")
}
my $Any = $perl->add_match_table('Any',
Description => "All Unicode code points");
$Any->add_range(0, $MAX_UNICODE_CODEPOINT);
$Any->add_alias('Unicode');
calculate_Assigned();
my $ASCII = $perl->add_match_table('ASCII');
if (defined $block) { # This is equivalent to the block if have it.
my $Unicode_ASCII = $block->table('Basic_Latin');
if (defined $Unicode_ASCII && ! $Unicode_ASCII->is_empty) {
$ASCII->set_equivalent_to($Unicode_ASCII, Related => 1);
}
}
# Very early releases didn't have blocks, so initialize ASCII ourselves if
# necessary
if ($ASCII->is_empty) {
if (! NON_ASCII_PLATFORM) {
$ASCII->add_range(0, 127);
}
else {
for my $i (0 .. 127) {
$ASCII->add_range(utf8::unicode_to_native($i),
utf8::unicode_to_native($i));
}
}
}
# Get the best available case definitions. Early Unicode versions didn't
# have Uppercase and Lowercase defined, so use the general category
# instead for them, modified by hard-coding in the code points each is
# missing.
my $Lower = $perl->add_match_table('XPosixLower');
my $Unicode_Lower = property_ref('Lowercase');
if (defined $Unicode_Lower && ! $Unicode_Lower->is_empty) {
$Lower->set_equivalent_to($Unicode_Lower->table('Y'), Related => 1);
}
else {
$Lower += $gc->table('Lowercase_Letter');
# There are quite a few code points in Lower, that aren't in gc=lc,
# and not all are in all releases.
my $temp = Range_List->new(Initialize => [
utf8::unicode_to_native(0xAA),
utf8::unicode_to_native(0xBA),
0x02B0 .. 0x02B8,
0x02C0 .. 0x02C1,
0x02E0 .. 0x02E4,
0x0345,
0x037A,
0x1D2C .. 0x1D6A,
0x1D78,
0x1D9B .. 0x1DBF,
0x2071,
0x207F,
0x2090 .. 0x209C,
0x2170 .. 0x217F,
0x24D0 .. 0x24E9,
0x2C7C .. 0x2C7D,
0xA770,
0xA7F8 .. 0xA7F9,
]);
$Lower += $temp & $Assigned;
}
my $Posix_Lower = $perl->add_match_table("PosixLower",
Initialize => $Lower & $ASCII,
);
my $Upper = $perl->add_match_table("XPosixUpper");
my $Unicode_Upper = property_ref('Uppercase');
if (defined $Unicode_Upper && ! $Unicode_Upper->is_empty) {
$Upper->set_equivalent_to($Unicode_Upper->table('Y'), Related => 1);
}
else {
# Unlike Lower, there are only two ranges in Upper that aren't in
# gc=Lu, and all code points were assigned in all releases.
$Upper += $gc->table('Uppercase_Letter');
$Upper->add_range(0x2160, 0x216F); # Uppercase Roman numerals
$Upper->add_range(0x24B6, 0x24CF); # Circled Latin upper case letters
}
my $Posix_Upper = $perl->add_match_table("PosixUpper",
Initialize => $Upper & $ASCII,
);
# Earliest releases didn't have title case. Initialize it to empty if not
# otherwise present
my $Title = $perl->add_match_table('Title', Full_Name => 'Titlecase',
Description => '(= \p{Gc=Lt})');
my $lt = $gc->table('Lt');
# Earlier versions of mktables had this related to $lt since they have
# identical code points, but their caseless equivalents are not the same,
# one being 'Cased' and the other being 'LC', and so now must be kept as
# separate entities.
if (defined $lt) {
$Title += $lt;
}
else {
push @tables_that_may_be_empty, $Title->complete_name;
}
my $Unicode_Cased = property_ref('Cased');
if (defined $Unicode_Cased) {
my $yes = $Unicode_Cased->table('Y');
my $no = $Unicode_Cased->table('N');
$Title->set_caseless_equivalent($yes);
if (defined $Unicode_Upper) {
$Unicode_Upper->table('Y')->set_caseless_equivalent($yes);
$Unicode_Upper->table('N')->set_caseless_equivalent($no);
}
$Upper->set_caseless_equivalent($yes);
if (defined $Unicode_Lower) {
$Unicode_Lower->table('Y')->set_caseless_equivalent($yes);
$Unicode_Lower->table('N')->set_caseless_equivalent($no);
}
$Lower->set_caseless_equivalent($yes);
}
else {
# If this Unicode version doesn't have Cased, set up the Perl
# extension from first principles. From Unicode 5.1: Definition D120:
# A character C is defined to be cased if and only if C has the
# Lowercase or Uppercase property or has a General_Category value of
# Titlecase_Letter.
my $cased = $perl->add_match_table('Cased',
Initialize => $Lower + $Upper + $Title,
Description => 'Uppercase or Lowercase or Titlecase',
);
# $notcased is purely for the caseless equivalents below
my $notcased = $perl->add_match_table('_Not_Cased',
Initialize => ~ $cased,
Fate => $INTERNAL_ONLY,
Description => 'All not-cased code points');
$Title->set_caseless_equivalent($cased);
if (defined $Unicode_Upper) {
$Unicode_Upper->table('Y')->set_caseless_equivalent($cased);
$Unicode_Upper->table('N')->set_caseless_equivalent($notcased);
}
$Upper->set_caseless_equivalent($cased);
if (defined $Unicode_Lower) {
$Unicode_Lower->table('Y')->set_caseless_equivalent($cased);
$Unicode_Lower->table('N')->set_caseless_equivalent($notcased);
}
$Lower->set_caseless_equivalent($cased);
}
# The remaining perl defined tables are mostly based on Unicode TR 18,
# "Annex C: Compatibility Properties". All of these have two versions,
# one whose name generally begins with Posix that is posix-compliant, and
# one that matches Unicode characters beyond the Posix, ASCII range
my $Alpha = $perl->add_match_table('XPosixAlpha');
# Alphabetic was not present in early releases
my $Alphabetic = property_ref('Alphabetic');
if (defined $Alphabetic && ! $Alphabetic->is_empty) {
$Alpha->set_equivalent_to($Alphabetic->table('Y'), Related => 1);
}
else {
# The Alphabetic property doesn't exist for early releases, so
# generate it. The actual definition, in 5.2 terms is:
#
# gc=L + gc=Nl + Other_Alphabetic
#
# Other_Alphabetic is also not defined in these early releases, but it
# contains one gc=So range plus most of gc=Mn and gc=Mc, so we add
# those last two as well, then subtract the relatively few of them that
# shouldn't have been added. (The gc=So range is the circled capital
# Latin characters. Early releases mistakenly didn't also include the
# lower-case versions of these characters, and so we don't either, to
# maintain consistency with those releases that first had this
# property.
$Alpha->initialize($gc->table('Letter')
+ pre_3_dot_1_Nl()
+ $gc->table('Mn')
+ $gc->table('Mc')
);
$Alpha->add_range(0x24D0, 0x24E9); # gc=So
foreach my $range ( [ 0x0300, 0x0344 ],
[ 0x0346, 0x034E ],
[ 0x0360, 0x0362 ],
[ 0x0483, 0x0486 ],
[ 0x0591, 0x05AF ],
[ 0x06DF, 0x06E0 ],
[ 0x06EA, 0x06EC ],
[ 0x0740, 0x074A ],
0x093C,
0x094D,
[ 0x0951, 0x0954 ],
0x09BC,
0x09CD,
0x0A3C,
0x0A4D,
0x0ABC,
0x0ACD,
0x0B3C,
0x0B4D,
0x0BCD,
0x0C4D,
0x0CCD,
0x0D4D,
0x0DCA,
[ 0x0E47, 0x0E4C ],
0x0E4E,
[ 0x0EC8, 0x0ECC ],
[ 0x0F18, 0x0F19 ],
0x0F35,
0x0F37,
0x0F39,
[ 0x0F3E, 0x0F3F ],
[ 0x0F82, 0x0F84 ],
[ 0x0F86, 0x0F87 ],
0x0FC6,
0x1037,
0x1039,
[ 0x17C9, 0x17D3 ],
[ 0x20D0, 0x20DC ],
0x20E1,
[ 0x302A, 0x302F ],
[ 0x3099, 0x309A ],
[ 0xFE20, 0xFE23 ],
[ 0x1D165, 0x1D169 ],
[ 0x1D16D, 0x1D172 ],
[ 0x1D17B, 0x1D182 ],
[ 0x1D185, 0x1D18B ],
[ 0x1D1AA, 0x1D1AD ],
) {
if (ref $range) {
$Alpha->delete_range($range->[0], $range->[1]);
}
else {
$Alpha->delete_range($range, $range);
}
}
$Alpha->add_description('Alphabetic');
$Alpha->add_alias('Alphabetic');
}
my $Posix_Alpha = $perl->add_match_table("PosixAlpha",
Initialize => $Alpha & $ASCII,
);
$Posix_Upper->set_caseless_equivalent($Posix_Alpha);
$Posix_Lower->set_caseless_equivalent($Posix_Alpha);
my $Alnum = $perl->add_match_table('Alnum', Full_Name => 'XPosixAlnum',
Description => 'Alphabetic and (decimal) Numeric',
Initialize => $Alpha + $gc->table('Decimal_Number'),
);
$perl->add_match_table("PosixAlnum",
Initialize => $Alnum & $ASCII,
);
my $Word = $perl->add_match_table('Word', Full_Name => 'XPosixWord',
Description => '\w, including beyond ASCII;'
. ' = \p{Alnum} + \pM + \p{Pc}'
. ' + \p{Join_Control}',
Initialize => $Alnum + $gc->table('Mark'),
);
my $Pc = $gc->table('Connector_Punctuation'); # 'Pc' Not in release 1
if (defined $Pc) {
$Word += $Pc;
}
else {
$Word += ord('_'); # Make sure this is a $Word
}
my $JC = property_ref('Join_Control'); # Wasn't in release 1
if (defined $JC) {
$Word += $JC->table('Y');
}
else {
$Word += 0x200C + 0x200D;
}
# This is a Perl extension, so the name doesn't begin with Posix.
my $PerlWord = $perl->add_match_table('PosixWord',
Description => '\w, restricted to ASCII',
Initialize => $Word & $ASCII,
);
$PerlWord->add_alias('PerlWord');
my $Blank = $perl->add_match_table('Blank', Full_Name => 'XPosixBlank',
Description => '\h, Horizontal white space',
# 200B is Zero Width Space which is for line
# break control, and was listed as
# Space_Separator in early releases
Initialize => $gc->table('Space_Separator')
+ ord("\t")
- 0x200B, # ZWSP
);
$Blank->add_alias('HorizSpace'); # Another name for it.
$perl->add_match_table("PosixBlank",
Initialize => $Blank & $ASCII,
);
my $VertSpace = $perl->add_match_table('VertSpace',
Description => '\v',
Initialize =>
$gc->table('Line_Separator')
+ $gc->table('Paragraph_Separator')
+ utf8::unicode_to_native(0x0A) # LINE FEED
+ utf8::unicode_to_native(0x0B) # VERTICAL TAB
+ ord("\f")
+ utf8::unicode_to_native(0x0D) # CARRIAGE RETURN
+ utf8::unicode_to_native(0x85) # NEL
);
# No Posix equivalent for vertical space
my $Space = $perl->add_match_table('XPosixSpace',
Description => '\s including beyond ASCII and vertical tab',
Initialize => $Blank + $VertSpace,
);
$Space->add_alias('XPerlSpace'); # Pre-existing synonyms
$Space->add_alias('SpacePerl');
$Space->add_alias('Space') if $v_version lt v4.1.0;
my $Posix_space = $perl->add_match_table("PosixSpace",
Initialize => $Space & $ASCII,
);
$Posix_space->add_alias('PerlSpace'); # A pre-existing synonym
my $Cntrl = $perl->add_match_table('Cntrl', Full_Name => 'XPosixCntrl',
Description => 'Control characters');
$Cntrl->set_equivalent_to($gc->table('Cc'), Related => 1);
$perl->add_match_table("PosixCntrl",
Description => "ASCII control characters",
Definition => "ACK, BEL, BS, CAN, CR, DC1, DC2,"
. " DC3, DC4, DEL, DLE, ENQ, EOM,"
. " EOT, ESC, ETB, ETX, FF, FS, GS,"
. " HT, LF, NAK, NUL, RS, SI, SO,"
. " SOH, STX, SUB, SYN, US, VT",
Initialize => $Cntrl & $ASCII,
);
my $perl_surrogate = $perl->add_match_table('_Perl_Surrogate');
my $Cs = $gc->table('Cs');
if (defined $Cs && ! $Cs->is_empty) {
$perl_surrogate += $Cs;
}
else {
push @tables_that_may_be_empty, '_Perl_Surrogate';
}
# $controls is a temporary used to construct Graph.
my $controls = Range_List->new(Initialize => $gc->table('Unassigned')
+ $gc->table('Control')
+ $perl_surrogate);
# Graph is ~space & ~(Cc|Cs|Cn) = ~(space + $controls)
my $Graph = $perl->add_match_table('Graph', Full_Name => 'XPosixGraph',
Description => 'Characters that are graphical',
Initialize => ~ ($Space + $controls),
);
$perl->add_match_table("PosixGraph",
Initialize => $Graph & $ASCII,
);
$print = $perl->add_match_table('Print', Full_Name => 'XPosixPrint',
Description => 'Characters that are graphical plus space characters (but no controls)',
Initialize => $Blank + $Graph - $gc->table('Control'),
);
$perl->add_match_table("PosixPrint",
Initialize => $print & $ASCII,
);
my $Punct = $perl->add_match_table('Punct');
$Punct->set_equivalent_to($gc->table('Punctuation'), Related => 1);
# \p{punct} doesn't include the symbols, which posix does
my $XPosixPunct = $perl->add_match_table('XPosixPunct',
Description => '\p{Punct} + ASCII-range \p{Symbol}',
Initialize => $gc->table('Punctuation')
+ ($ASCII & $gc->table('Symbol')),
Perl_Extension => 1
);
$perl->add_match_table('PosixPunct', Perl_Extension => 1,
Initialize => $ASCII & $XPosixPunct,
);
my $Digit = $perl->add_match_table('Digit', Full_Name => 'XPosixDigit',
Description => '[0-9] + all other decimal digits');
$Digit->set_equivalent_to($gc->table('Decimal_Number'), Related => 1);
my $PosixDigit = $perl->add_match_table("PosixDigit",
Initialize => $Digit & $ASCII,
);
# Hex_Digit was not present in first release
my $Xdigit = $perl->add_match_table('XDigit', Full_Name => 'XPosixXDigit');
my $Hex = property_ref('Hex_Digit');
if (defined $Hex && ! $Hex->is_empty) {
$Xdigit->set_equivalent_to($Hex->table('Y'), Related => 1);
}
else {
$Xdigit->initialize([ ord('0') .. ord('9'),
ord('A') .. ord('F'),
ord('a') .. ord('f'),
0xFF10..0xFF19, 0xFF21..0xFF26, 0xFF41..0xFF46]);
}
# AHex was not present in early releases
my $PosixXDigit = $perl->add_match_table('PosixXDigit');
my $AHex = property_ref('ASCII_Hex_Digit');
if (defined $AHex && ! $AHex->is_empty) {
$PosixXDigit->set_equivalent_to($AHex->table('Y'), Related => 1);
}
else {
$PosixXDigit->initialize($Xdigit & $ASCII);
$PosixXDigit->add_alias('AHex');
$PosixXDigit->add_alias('Ascii_Hex_Digit');
}
my $any_folds = $perl->add_match_table("_Perl_Any_Folds",
Description => "Code points that particpate in some fold",
);
my $loc_problem_folds = $perl->add_match_table(
"_Perl_Problematic_Locale_Folds",
Description =>
"Code points that are in some way problematic under locale",
);
# This allows regexec.c to skip some work when appropriate. Some of the
# entries in _Perl_Problematic_Locale_Folds are multi-character folds,
my $loc_problem_folds_start = $perl->add_match_table(
"_Perl_Problematic_Locale_Foldeds_Start",
Description =>
"The first character of every sequence in _Perl_Problematic_Locale_Folds",
);
my $cf = property_ref('Case_Folding');
# Every character 0-255 is problematic because what each folds to depends
# on the current locale
$loc_problem_folds->add_range(0, 255);
$loc_problem_folds->add_range(0x130, 0x131); # These are problematic in
# Turkic locales
$loc_problem_folds_start += $loc_problem_folds;
# Also problematic are anything these fold to outside the range. Likely
# forever the only thing folded to by these outside the 0-255 range is the
# GREEK SMALL MU (from the MICRO SIGN), but it's easy to make the code
# completely general, which should catch any unexpected changes or errors.
# We look at each code point 0-255, and add its fold (including each part
# of a multi-char fold) to the list. See commit message
# 31f05a37c4e9c37a7263491f2fc0237d836e1a80 for a more complete description
# of the MU issue.
foreach my $range ($loc_problem_folds->ranges) {
foreach my $code_point ($range->start .. $range->end) {
my $fold_range = $cf->containing_range($code_point);
next unless defined $fold_range;
# Skip if folds to itself
next if $fold_range->value eq $CODE_POINT;
my @hex_folds = split " ", $fold_range->value;
my $start_cp = $hex_folds[0];
next if $start_cp eq $CODE_POINT;
$start_cp = hex $start_cp;
foreach my $i (0 .. @hex_folds - 1) {
my $cp = $hex_folds[$i];
next if $cp eq $CODE_POINT;
$cp = hex $cp;
next unless $cp > 255; # Already have the < 256 ones
$loc_problem_folds->add_range($cp, $cp);
$loc_problem_folds_start->add_range($start_cp, $start_cp);
}
}
}
my $folds_to_multi_char = $perl->add_match_table(
"_Perl_Folds_To_Multi_Char",
Description =>
"Code points whose fold is a string of more than one character",
);
my $in_multi_fold = $perl->add_match_table(
"_Perl_Is_In_Multi_Char_Fold",
Description =>
"Code points that are in some multiple character fold",
);
my $non_final_fold = $perl->add_match_table(
"_Perl_Non_Final_Folds",
Description => "Code points that are in some multiple character fold, but not in the final position",
);
if ($v_version lt v3.0.1) {
push @tables_that_may_be_empty, '_Perl_Folds_To_Multi_Char',
'_Perl_Is_In_Multi_Char_Fold',
'_Perl_Non_Final_Folds';
}
# Look through all the known folds to populate these tables.
foreach my $range ($cf->ranges) {
next if $range->value eq $CODE_POINT;
my $start = $range->start;
my $end = $range->end;
$any_folds->add_range($start, $end);
my @hex_folds = split " ", $range->value;
if (@hex_folds > 1) { # Is multi-char fold
$folds_to_multi_char->add_range($start, $end);
}
my $found_locale_problematic = 0;
# Look at each of the folded-to characters...
foreach my $i (0 .. @hex_folds - 1) {
my $cp = hex $hex_folds[$i];
$any_folds->add_range($cp, $cp);
# The fold is problematic if any of the folded-to characters is
# already considered problematic.
if ($loc_problem_folds->contains($cp)) {
$loc_problem_folds->add_range($start, $end);
$found_locale_problematic = 1;
}
if (@hex_folds > 1) {
$in_multi_fold->add_range($cp, $cp);
next if $i < @hex_folds - 1;
$non_final_fold->add_range($cp, $cp);
}
}
# If this is a problematic fold, add to the start chars the
# folding-from characters and first folded-to character.
if ($found_locale_problematic) {
$loc_problem_folds_start->add_range($start, $end);
my $cp = hex $hex_folds[0];
$loc_problem_folds_start->add_range($cp, $cp);
}
}
my $dt = property_ref('Decomposition_Type');
$dt->add_match_table('Non_Canon', Full_Name => 'Non_Canonical',
Initialize => ~ ($dt->table('None') + $dt->table('Canonical')),
Perl_Extension => 1,
Note => 'Union of all non-canonical decompositions',
);
# For backward compatibility, Perl has its own definition for IDStart.
# It is regular XID_Start plus the underscore, but all characters must be
# Word characters as well
my $XID_Start = property_ref('XID_Start');
my $perl_xids = $perl->add_match_table('_Perl_IDStart',
Perl_Extension => 1,
Fate => $INTERNAL_ONLY,
Initialize => ord('_')
);
if (defined $XID_Start
|| defined ($XID_Start = property_ref('ID_Start')))
{
$perl_xids += $XID_Start->table('Y');
}
else {
# For Unicode versions that don't have the property, construct our own
# from first principles. The actual definition is:
# Letters
# + letter numbers (Nl)
# - Pattern_Syntax
# - Pattern_White_Space
# + stability extensions
# - NKFC modifications
#
# What we do in the code below is to include the identical code points
# that are in the first release that had Unicode's version of this
# property, essentially extrapolating backwards. There were no
# stability extensions until v4.1, so none are included; likewise in
# no Unicode version so far do subtracting PatSyn and PatWS make any
# difference, so those also are ignored.
$perl_xids += $gc->table('Letter') + pre_3_dot_1_Nl();
# We do subtract the NFKC modifications that are in the first version
# that had this property. We don't bother to test if they are in the
# version in question, because if they aren't, the operation is a
# no-op. The NKFC modifications are discussed in
# http://www.unicode.org/reports/tr31/#NFKC_Modifications
foreach my $range ( 0x037A,
0x0E33,
0x0EB3,
[ 0xFC5E, 0xFC63 ],
[ 0xFDFA, 0xFE70 ],
[ 0xFE72, 0xFE76 ],
0xFE78,
0xFE7A,
0xFE7C,
0xFE7E,
[ 0xFF9E, 0xFF9F ],
) {
if (ref $range) {
$perl_xids->delete_range($range->[0], $range->[1]);
}
else {
$perl_xids->delete_range($range, $range);
}
}
}
$perl_xids &= $Word;
my $perl_xidc = $perl->add_match_table('_Perl_IDCont',
Perl_Extension => 1,
Fate => $INTERNAL_ONLY);
my $XIDC = property_ref('XID_Continue');
if (defined $XIDC
|| defined ($XIDC = property_ref('ID_Continue')))
{
$perl_xidc += $XIDC->table('Y');
}
else {
# Similarly, we construct our own XIDC if necessary for early Unicode
# versions. The definition is:
# everything in XIDS
# + Gc=Mn
# + Gc=Mc
# + Gc=Nd
# + Gc=Pc
# - Pattern_Syntax
# - Pattern_White_Space
# + stability extensions
# - NFKC modifications
#
# The same thing applies to this as with XIDS for the PatSyn, PatWS,
# and stability extensions. There is a somewhat different set of NFKC
# mods to remove (and add in this case). The ones below make this
# have identical code points as in the first release that defined it.
$perl_xidc += $perl_xids
+ $gc->table('L')
+ $gc->table('Mn')
+ $gc->table('Mc')
+ $gc->table('Nd')
+ utf8::unicode_to_native(0xB7)
;
if (defined (my $pc = $gc->table('Pc'))) {
$perl_xidc += $pc;
}
else { # 1.1.5 didn't have Pc, but these should have been in it
$perl_xidc += 0xFF3F;
$perl_xidc->add_range(0x203F, 0x2040);
$perl_xidc->add_range(0xFE33, 0xFE34);
$perl_xidc->add_range(0xFE4D, 0xFE4F);
}
# Subtract the NFKC mods
foreach my $range ( 0x037A,
[ 0xFC5E, 0xFC63 ],
[ 0xFDFA, 0xFE1F ],
0xFE70,
[ 0xFE72, 0xFE76 ],
0xFE78,
0xFE7A,
0xFE7C,
0xFE7E,
) {
if (ref $range) {
$perl_xidc->delete_range($range->[0], $range->[1]);
}
else {
$perl_xidc->delete_range($range, $range);
}
}
}
$perl_xidc &= $Word;
my $charname_begin = $perl->add_match_table('_Perl_Charname_Begin',
Perl_Extension => 1,
Fate => $INTERNAL_ONLY,
Initialize => $gc->table('Letter') & $Alpha & $perl_xids,
);
my $charname_continue = $perl->add_match_table('_Perl_Charname_Continue',
Perl_Extension => 1,
Fate => $INTERNAL_ONLY,
Initialize => $perl_xidc
+ ord(" ")
+ ord("(")
+ ord(")")
+ ord("-")
);
my @composition = ('Name', 'Unicode_1_Name', '_Perl_Name_Alias');
if (@named_sequences) {
push @composition, 'Named_Sequence';
foreach my $sequence (@named_sequences) {
$perl_charname->add_anomalous_entry($sequence);
}
}
my $alias_sentence = "";
my %abbreviations;
my $alias = property_ref('_Perl_Name_Alias');
$perl_charname->set_proxy_for('_Perl_Name_Alias');
# Add each entry in _Perl_Name_Alias to Perl_Charnames. Where these go
# with respect to any existing entry depends on the entry type.
# Corrections go before said entry, as they should be returned in
# preference over the existing entry. (A correction to a correction
# should be later in the _Perl_Name_Alias table, so it will correctly
# precede the erroneous correction in Perl_Charnames.)
#
# Abbreviations go after everything else, so they are saved temporarily in
# a hash for later.
#
# Everything else is added added afterwards, which preserves the input
# ordering
foreach my $range ($alias->ranges) {
next if $range->value eq "";
my $code_point = $range->start;
if ($code_point != $range->end) {
Carp::my_carp_bug("Bad News. Expecting only one code point in the range $range. Just to keep going, using only the first code point;");
}
my ($value, $type) = split ': ', $range->value;
my $replace_type;
if ($type eq 'correction') {
$replace_type = $MULTIPLE_BEFORE;
}
elsif ($type eq 'abbreviation') {
# Save for later
$abbreviations{$value} = $code_point;
next;
}
else {
$replace_type = $MULTIPLE_AFTER;
}
# Actually add; before or after current entry(ies) as determined
# above.
$perl_charname->add_duplicate($code_point, $value, Replace => $replace_type);
}
$alias_sentence = <<END;
The _Perl_Name_Alias property adds duplicate code point entries that are
alternatives to the original name. If an addition is a corrected
name, it will be physically first in the table. The original (less correct,
but still valid) name will be next; then any alternatives, in no particular
order; and finally any abbreviations, again in no particular order.
END
# Now add the Unicode_1 names for the controls. The Unicode_1 names had
# precedence before 6.1, including the awful ones like "LINE FEED (LF)",
# so should be first in the file; the other names have precedence starting
# in 6.1,
my $before_or_after = ($v_version lt v6.1.0)
? $MULTIPLE_BEFORE
: $MULTIPLE_AFTER;
foreach my $range (property_ref('Unicode_1_Name')->ranges) {
my $code_point = $range->start;
my $unicode_1_value = $range->value;
next if $unicode_1_value eq ""; # Skip if name doesn't exist.
if ($code_point != $range->end) {
Carp::my_carp_bug("Bad News. Expecting only one code point in the range $range. Just to keep going, using only the first code point;");
}
# To handle EBCDIC, we don't hard code in the code points of the
# controls; instead realizing that all of them are below 256.
last if $code_point > 255;
# We only add in the controls.
next if $gc->value_of($code_point) ne 'Cc';
# We reject this Unicode1 name for later Perls, as it is used for
# another code point
next if $unicode_1_value eq 'BELL' && $^V ge v5.17.0;
# This won't add an exact duplicate.
$perl_charname->add_duplicate($code_point, $unicode_1_value,
Replace => $before_or_after);
}
# Now that have everything added, add in abbreviations after
# everything else. Sort so results don't change between runs of this
# program
foreach my $value (sort keys %abbreviations) {
$perl_charname->add_duplicate($abbreviations{$value}, $value,
Replace => $MULTIPLE_AFTER);
}
my $comment;
if (@composition <= 2) { # Always at least 2
$comment = join " and ", @composition;
}
else {
$comment = join ", ", @composition[0 .. scalar @composition - 2];
$comment .= ", and $composition[-1]";
}
$perl_charname->add_comment(join_lines( <<END
This file is for charnames.pm. It is the union of the $comment properties.
Unicode_1_Name entries are used only for nameless code points in the Name
property.
$alias_sentence
This file doesn't include the algorithmically determinable names. For those,
use 'unicore/Name.pm'
END
));
property_ref('Name')->add_comment(join_lines( <<END
This file doesn't include the algorithmically determinable names. For those,
use 'unicore/Name.pm'
END
));
# Construct the Present_In property from the Age property.
if (-e 'DAge.txt' && defined $age) {
my $default_map = $age->default_map;
my $in = Property->new('In',
Default_Map => $default_map,
Full_Name => "Present_In",
Perl_Extension => 1,
Type => $ENUM,
Initialize => $age,
);
$in->add_comment(join_lines(<<END
THIS FILE SHOULD NOT BE USED FOR ANY PURPOSE. The values in this file are the
same as for $age, and not for what $in really means. This is because anything
defined in a given release should have multiple values: that release and all
higher ones. But only one value per code point can be represented in a table
like this.
END
));
# The Age tables are named like 1.5, 2.0, 2.1, .... Sort so that the
# lowest numbered (earliest) come first, with the non-numeric one
# last.
my ($first_age, @rest_ages) = sort { ($a->name !~ /^[\d.]*$/)
? 1
: ($b->name !~ /^[\d.]*$/)
? -1
: $a->name <=> $b->name
} $age->tables;
# The Present_In property is the cumulative age properties. The first
# one hence is identical to the first age one.
my $previous_in = $in->add_match_table($first_age->name);
$previous_in->set_equivalent_to($first_age, Related => 1);
my $description_start = "Code point's usage introduced in version ";
$first_age->add_description($description_start . $first_age->name);
# To construct the accumulated values, for each of the age tables
# starting with the 2nd earliest, merge the earliest with it, to get
# all those code points existing in the 2nd earliest. Repeat merging
# the new 2nd earliest with the 3rd earliest to get all those existing
# in the 3rd earliest, and so on.
foreach my $current_age (@rest_ages) {
next if $current_age->name !~ /^[\d.]*$/; # Skip the non-numeric
my $current_in = $in->add_match_table(
$current_age->name,
Initialize => $current_age + $previous_in,
Description => $description_start
. $current_age->name
. ' or earlier',
);
foreach my $alias ($current_age->aliases) {
$current_in->add_alias($alias->name);
}
$previous_in = $current_in;
# Add clarifying material for the corresponding age file. This is
# in part because of the confusing and contradictory information
# given in the Standard's documentation itself, as of 5.2.
$current_age->add_description(
"Code point's usage was introduced in version "
. $current_age->name);
$current_age->add_note("See also $in");
}
# And finally the code points whose usages have yet to be decided are
# the same in both properties. Note that permanently unassigned code
# points actually have their usage assigned (as being permanently
# unassigned), so that these tables are not the same as gc=cn.
my $unassigned = $in->add_match_table($default_map);
my $age_default = $age->table($default_map);
$age_default->add_description(<<END
Code point's usage has not been assigned in any Unicode release thus far.
END
);
$unassigned->set_equivalent_to($age_default, Related => 1);
}
my $patws = $perl->add_match_table('_Perl_PatWS',
Perl_Extension => 1,
Fate => $INTERNAL_ONLY);
if (defined (my $off_patws = property_ref('Pattern_White_Space'))) {
$patws->initialize($off_patws->table('Y'));
}
else {
$patws->initialize([ ord("\t"),
ord("\n"),
utf8::unicode_to_native(0x0B), # VT
ord("\f"),
ord("\r"),
ord(" "),
utf8::unicode_to_native(0x85), # NEL
0x200E..0x200F, # Left, Right marks
0x2028..0x2029 # Line, Paragraph seps
] );
}
# See L<perlfunc/quotemeta>
my $quotemeta = $perl->add_match_table('_Perl_Quotemeta',
Perl_Extension => 1,
Fate => $INTERNAL_ONLY,
# Initialize to what's common in
# all Unicode releases.
Initialize =>
$gc->table('Control')
+ $Space
+ $patws
+ ((~ $Word) & $ASCII)
);
if (defined (my $patsyn = property_ref('Pattern_Syntax'))) {
$quotemeta += $patsyn->table('Y');
}
else {
$quotemeta += ((~ $Word) & Range->new(0, 255))
- utf8::unicode_to_native(0xA8)
- utf8::unicode_to_native(0xAF)
- utf8::unicode_to_native(0xB2)
- utf8::unicode_to_native(0xB3)
- utf8::unicode_to_native(0xB4)
- utf8::unicode_to_native(0xB7)
- utf8::unicode_to_native(0xB8)
- utf8::unicode_to_native(0xB9)
- utf8::unicode_to_native(0xBC)
- utf8::unicode_to_native(0xBD)
- utf8::unicode_to_native(0xBE);
$quotemeta += [ # These are above-Latin1 patsyn; hence should be the
# same in all releases
0x2010 .. 0x2027,
0x2030 .. 0x203E,
0x2041 .. 0x2053,
0x2055 .. 0x205E,
0x2190 .. 0x245F,
0x2500 .. 0x2775,
0x2794 .. 0x2BFF,
0x2E00 .. 0x2E7F,
0x3001 .. 0x3003,
0x3008 .. 0x3020,
0x3030 .. 0x3030,
0xFD3E .. 0xFD3F,
0xFE45 .. 0xFE46
];
}
if (defined (my $di = property_ref('Default_Ignorable_Code_Point'))) {
$quotemeta += $di->table('Y')
}
else {
if ($v_version ge v2.0) {
$quotemeta += $gc->table('Cf')
+ $gc->table('Cs');
# These are above the Unicode version 1 max
$quotemeta->add_range(0xE0000, 0xE0FFF);
}
$quotemeta += $gc->table('Cc')
- $Space;
my $temp = Range_List->new(Initialize => [ 0x180B .. 0x180D,
0x2060 .. 0x206F,
0xFE00 .. 0xFE0F,
0xFFF0 .. 0xFFFB,
]);
$temp->add_range(0xE0000, 0xE0FFF) if $v_version ge v2.0;
$quotemeta += $temp;
}
calculate_DI();
$quotemeta += $DI;
calculate_NChar();
# Finished creating all the perl properties. All non-internal non-string
# ones have a synonym of 'Is_' prefixed. (Internal properties begin with
# an underscore.) These do not get a separate entry in the pod file
foreach my $table ($perl->tables) {
foreach my $alias ($table->aliases) {
next if $alias->name =~ /^_/;
$table->add_alias('Is_' . $alias->name,
Re_Pod_Entry => 0,
UCD => 0,
Status => $alias->status,
OK_as_Filename => 0);
}
}
# Perl tailors the WordBreak property so that \b{wb} doesn't split
# adjacent spaces into separate words. Unicode 11.0 moved in that
# direction, but left TAB, FIGURE SPACE (U+2007), and (ironically) NO
# BREAK SPACE as breaking, so we retained the original Perl customization.
# To do this, in the Perl copy of WB, simply replace the mappings of
# horizontal space characters that otherwise would map to the default or
# the 11.0 'WSegSpace' to instead map to our tailoring.
my $perl_wb = property_ref('_Perl_WB');
my $default = $perl_wb->default_map;
for my $range ($Blank->ranges) {
for my $i ($range->start .. $range->end) {
my $value = $perl_wb->value_of($i);
next unless $value eq $default || $value eq 'WSegSpace';
$perl_wb->add_map($i, $i, 'Perl_Tailored_HSpace',
Replace => $UNCONDITIONALLY);
}
}
# Also starting in Unicode 11.0, rules for some of the boundary types are
# based on a non-UCD property (which we have read in if it exists).
# Recall that these boundary properties partition the code points into
# equivalence classes (represented as enums).
#
# The loop below goes through each code point that matches the non-UCD
# property, and for each current equivalence class containing such a code
# point, splits it so that those that are in both are now in a newly
# created equivalence class whose name is a combination of the property
# and the old class name, leaving unchanged everything that doesn't match
# the non-UCD property.
my $pictographic_emoji = property_ref('XPG');
if (defined $pictographic_emoji) {
foreach my $base_property (property_ref('GCB'),
property_ref('WB'))
{
my $property = property_ref('_Perl_' . $base_property->name);
foreach my $range ($pictographic_emoji->table('Y')->ranges) {
foreach my $i ($range->start .. $range->end) {
my $current = $property->value_of($i);
$current = $property->table($current)->short_name;
$property->add_map($i, $i, 'XPG_' . $current,
Replace => $UNCONDITIONALLY);
}
}
}
}
# Create a version of the LineBreak property with the mappings that are
# omitted in the default algorithm remapped to what
# http://www.unicode.org/reports/tr14 says they should be.
#
# Original Resolved General_Category
# AI, SG, XX AL Any
# SA CM Only Mn or Mc
# SA AL Any except Mn and Mc
# CJ NS Any
#
# All property values are also written out in their long form, as
# regen/mk_invlist.pl expects that. This also fixes occurrences of the
# typo in early Unicode versions: 'inseperable'.
my $perl_lb = property_ref('_Perl_LB');
if (! defined $perl_lb) {
$perl_lb = Property->new('_Perl_LB',
Fate => $INTERNAL_ONLY,
Perl_Extension => 1,
Directory => $map_directory,
Type => $STRING);
my $lb = property_ref('Line_Break');
# Populate from $lb, but use full name and fix typo.
foreach my $range ($lb->ranges) {
my $full_name = $lb->table($range->value)->full_name;
$full_name = 'Inseparable'
if standardize($full_name) eq 'inseperable';
$perl_lb->add_map($range->start, $range->end, $full_name);
}
}
$perl_lb->set_default_map('Alphabetic', 'full_name'); # XX -> AL
for my $range ($perl_lb->ranges) {
my $value = standardize($range->value);
if ( $value eq standardize('Unknown')
|| $value eq standardize('Ambiguous')
|| $value eq standardize('Surrogate'))
{
$perl_lb->add_map($range->start, $range->end, 'Alphabetic',
Replace => $UNCONDITIONALLY);
}
elsif ($value eq standardize('Conditional_Japanese_Starter')) {
$perl_lb->add_map($range->start, $range->end, 'Nonstarter',
Replace => $UNCONDITIONALLY);
}
elsif ($value eq standardize('Complex_Context')) {
for my $i ($range->start .. $range->end) {
my $gc_val = $gc->value_of($i);
if ($gc_val eq 'Mn' || $gc_val eq 'Mc') {
$perl_lb->add_map($i, $i, 'Combining_Mark',
Replace => $UNCONDITIONALLY);
}
else {
$perl_lb->add_map($i, $i, 'Alphabetic',
Replace => $UNCONDITIONALLY);
}
}
}
}
# This property is a modification of the scx property
my $perl_scx = Property->new('_Perl_SCX',
Fate => $INTERNAL_ONLY,
Perl_Extension => 1,
Directory => $map_directory,
Type => $ENUM);
my $source;
# Use scx if available; otherwise sc; if neither is there (a very old
# Unicode version, just say that everything is 'Common'
if (defined $scx) {
$source = $scx;
$perl_scx->set_default_map('Unknown');
}
elsif (defined $script) {
$source = $script;
# Early versions of 'sc', had everything be 'Common'
if (defined $script->table('Unknown')) {
$perl_scx->set_default_map('Unknown');
}
else {
$perl_scx->set_default_map('Common');
}
} else {
$perl_scx->add_match_table('Common');
$perl_scx->add_map(0, $MAX_UNICODE_CODEPOINT, 'Common');
$perl_scx->add_match_table('Unknown');
$perl_scx->set_default_map('Unknown');
}
$perl_scx->_set_format($STRING_WHITE_SPACE_LIST);
$perl_scx->set_pre_declared_maps(0); # PropValueAliases doesn't list these
if (defined $source) {
$perl_scx->initialize($source);
# UTS 39 says that the scx property should be modified for these
# countries where certain mixed scripts are commonly used.
for my $range ($perl_scx->ranges) {
my $value = $range->value;
my $changed = $value =~ s/ ( \b Han i? \b ) /$1 Hanb Jpan Kore/xi;
$changed |= $value =~ s/ ( \b Hira (gana)? \b ) /$1 Jpan/xi;
$changed |= $value =~ s/ ( \b Kata (kana)? \b ) /$1 Jpan/xi;
$changed |= $value =~ s{ ( \b Katakana_or_Hiragana \b ) }
{$1 Katakana Hiragana Jpan}xi;
$changed |= $value =~ s/ ( \b Hang (ul)? \b ) /$1 Kore/xi;
$changed |= $value =~ s/ ( \b Bopo (mofo)? \b ) /$1 Hanb/xi;
if ($changed) {
$value = join " ", uniques split " ", $value;
$range->set_value($value)
}
}
foreach my $table ($source->tables) {
my $scx_table = $perl_scx->add_match_table($table->name,
Full_Name => $table->full_name);
foreach my $alias ($table->aliases) {
$scx_table->add_alias($alias->name);
}
}
}
# Here done with all the basic stuff. Ready to populate the information
# about each character if annotating them.
if ($annotate) {
# See comments at its declaration
$annotate_ranges = Range_Map->new;
# This separates out the non-characters from the other unassigneds, so
# can give different annotations for each.
$unassigned_sans_noncharacters = Range_List->new(
Initialize => $gc->table('Unassigned'));
$unassigned_sans_noncharacters &= (~ $NChar);
for (my $i = 0; $i <= $MAX_UNICODE_CODEPOINT + 1; $i++ ) {
$i = populate_char_info($i); # Note sets $i so may cause skips
}
}
return;
}
sub add_perl_synonyms() {
# A number of Unicode tables have Perl synonyms that are expressed in
# the single-form, \p{name}. These are:
# All the binary property Y tables, so that \p{Name=Y} gets \p{Name} and
# \p{Is_Name} as synonyms
# \p{Script_Extensions=Value} gets \p{Value}, \p{Is_Value} as synonyms
# \p{General_Category=Value} gets \p{Value}, \p{Is_Value} as synonyms
# \p{Block=Value} gets \p{In_Value} as a synonym, and, if there is no
# conflict, \p{Value} and \p{Is_Value} as well
#
# This routine generates these synonyms, warning of any unexpected
# conflicts.
# Construct the list of tables to get synonyms for. Start with all the
# binary and the General_Category ones.
my @tables = grep { $_->type == $BINARY || $_->type == $FORCED_BINARY }
property_ref('*');
push @tables, $gc->tables;
# If the version of Unicode includes the Script Extensions (preferably),
# or Script property, add its tables
if (defined $scx) {
push @tables, $scx->tables;
}
else {
push @tables, $script->tables if defined $script;
}
# The Block tables are kept separate because they are treated differently.
# And the earliest versions of Unicode didn't include them, so add only if
# there are some.
my @blocks;
push @blocks, $block->tables if defined $block;
# Here, have the lists of tables constructed. Process blocks last so that
# if there are name collisions with them, blocks have lowest priority.
# Should there ever be other collisions, manual intervention would be
# required. See the comments at the beginning of the program for a
# possible way to handle those semi-automatically.
foreach my $table (@tables, @blocks) {
# For non-binary properties, the synonym is just the name of the
# table, like Greek, but for binary properties the synonym is the name
# of the property, and means the code points in its 'Y' table.
my $nominal = $table;
my $nominal_property = $nominal->property;
my $actual;
if (! $nominal->isa('Property')) {
$actual = $table;
}
else {
# Here is a binary property. Use the 'Y' table. Verify that is
# there
my $yes = $nominal->table('Y');
unless (defined $yes) { # Must be defined, but is permissible to
# be empty.
Carp::my_carp_bug("Undefined $nominal, 'Y'. Skipping.");
next;
}
$actual = $yes;
}
foreach my $alias ($nominal->aliases) {
# Attempt to create a table in the perl directory for the
# candidate table, using whatever aliases in it that don't
# conflict. Also add non-conflicting aliases for all these
# prefixed by 'Is_' (and/or 'In_' for Block property tables)
PREFIX:
foreach my $prefix ("", 'Is_', 'In_') {
# Only Block properties can have added 'In_' aliases.
next if $prefix eq 'In_' and $nominal_property != $block;
my $proposed_name = $prefix . $alias->name;
# No Is_Is, In_In, nor combinations thereof
trace "$proposed_name is a no-no" if main::DEBUG && $to_trace && $proposed_name =~ /^ I [ns] _I [ns] _/x;
next if $proposed_name =~ /^ I [ns] _I [ns] _/x;
trace "Seeing if can add alias or table: 'perl=$proposed_name' based on $nominal" if main::DEBUG && $to_trace;
# Get a reference to any existing table in the perl
# directory with the desired name.
my $pre_existing = $perl->table($proposed_name);
if (! defined $pre_existing) {
# No name collision, so OK to add the perl synonym.
my $make_re_pod_entry;
my $ok_as_filename;
my $status = $alias->status;
if ($nominal_property == $block) {
# For block properties, only the compound form is
# preferred for external use; the others are
# discouraged. The pod file contains wild cards for
# the 'In' and 'Is' forms so no entries for those; and
# we don't want people using the name without any
# prefix, so discourage that.
if ($prefix eq "") {
$make_re_pod_entry = 1;
$status = $status || $DISCOURAGED;
$ok_as_filename = 0;
}
elsif ($prefix eq 'In_') {
$make_re_pod_entry = 0;
$status = $status || $DISCOURAGED;
$ok_as_filename = 1;
}
else {
$make_re_pod_entry = 0;
$status = $status || $DISCOURAGED;
$ok_as_filename = 0;
}
}
elsif ($prefix ne "") {
# The 'Is' prefix is handled in the pod by a wild
# card, and we won't use it for an external name
$make_re_pod_entry = 0;
$status = $status || $NORMAL;
$ok_as_filename = 0;
}
else {
# Here, is an empty prefix, non block. This gets its
# own pod entry and can be used for an external name.
$make_re_pod_entry = 1;
$status = $status || $NORMAL;
$ok_as_filename = 1;
}
# Here, there isn't a perl pre-existing table with the
# name. Look through the list of equivalents of this
# table to see if one is a perl table.
foreach my $equivalent ($actual->leader->equivalents) {
next if $equivalent->property != $perl;
# Here, have found a table for $perl. Add this alias
# to it, and are done with this prefix.
$equivalent->add_alias($proposed_name,
Re_Pod_Entry => $make_re_pod_entry,
# Currently don't output these in the
# ucd pod, as are strongly discouraged
# from being used
UCD => 0,
Status => $status,
OK_as_Filename => $ok_as_filename);
trace "adding alias perl=$proposed_name to $equivalent" if main::DEBUG && $to_trace;
next PREFIX;
}
# Here, $perl doesn't already have a table that is a
# synonym for this property, add one.
my $added_table = $perl->add_match_table($proposed_name,
Re_Pod_Entry => $make_re_pod_entry,
# See UCD comment just above
UCD => 0,
Status => $status,
OK_as_Filename => $ok_as_filename);
# And it will be related to the actual table, since it is
# based on it.
$added_table->set_equivalent_to($actual, Related => 1);
trace "added ", $perl->table($proposed_name) if main::DEBUG && $to_trace;
next;
} # End of no pre-existing.
# Here, there is a pre-existing table that has the proposed
# name. We could be in trouble, but not if this is just a
# synonym for another table that we have already made a child
# of the pre-existing one.
if ($pre_existing->is_set_equivalent_to($actual)) {
trace "$pre_existing is already equivalent to $actual; adding alias perl=$proposed_name to it" if main::DEBUG && $to_trace;
$pre_existing->add_alias($proposed_name);
next;
}
# Here, there is a name collision, but it still could be OK if
# the tables match the identical set of code points, in which
# case, we can combine the names. Compare each table's code
# point list to see if they are identical.
trace "Potential name conflict with $pre_existing having ", $pre_existing->count, " code points" if main::DEBUG && $to_trace;
if ($pre_existing->matches_identically_to($actual)) {
# Here, they do match identically. Not a real conflict.
# Make the perl version a child of the Unicode one, except
# in the non-obvious case of where the perl name is
# already a synonym of another Unicode property. (This is
# excluded by the test for it being its own parent.) The
# reason for this exclusion is that then the two Unicode
# properties become related; and we don't really know if
# they are or not. We generate documentation based on
# relatedness, and this would be misleading. Code
# later executed in the process will cause the tables to
# be represented by a single file anyway, without making
# it look in the pod like they are necessarily related.
if ($pre_existing->parent == $pre_existing
&& ($pre_existing->property == $perl
|| $actual->property == $perl))
{
trace "Setting $pre_existing equivalent to $actual since one is \$perl, and match identical sets" if main::DEBUG && $to_trace;
$pre_existing->set_equivalent_to($actual, Related => 1);
}
elsif (main::DEBUG && $to_trace) {
trace "$pre_existing is equivalent to $actual since match identical sets, but not setting them equivalent, to preserve the separateness of the perl aliases";
trace $pre_existing->parent;
}
next PREFIX;
}
# Here they didn't match identically, there is a real conflict
# between our new name and a pre-existing property.
$actual->add_conflicting($proposed_name, 'p', $pre_existing);
$pre_existing->add_conflicting($nominal->full_name,
'p',
$actual);
# Don't output a warning for aliases for the block
# properties (unless they start with 'In_') as it is
# expected that there will be conflicts and the block
# form loses.
if ($verbosity >= $NORMAL_VERBOSITY
&& ($actual->property != $block || $prefix eq 'In_'))
{
print simple_fold(join_lines(<<END
There is already an alias named $proposed_name (from $pre_existing),
so not creating this alias for $actual
END
), "", 4);
}
# Keep track for documentation purposes.
$has_In_conflicts++ if $prefix eq 'In_';
$has_Is_conflicts++ if $prefix eq 'Is_';
}
}
}
# There are some properties which have No and Yes (and N and Y) as
# property values, but aren't binary, and could possibly be confused with
# binary ones. So create caveats for them. There are tables that are
# named 'No', and tables that are named 'N', but confusion is not likely
# unless they are the same table. For example, N meaning Number or
# Neutral is not likely to cause confusion, so don't add caveats to things
# like them.
foreach my $property (grep { $_->type != $BINARY
&& $_->type != $FORCED_BINARY }
property_ref('*'))
{
my $yes = $property->table('Yes');
if (defined $yes) {
my $y = $property->table('Y');
if (defined $y && $yes == $y) {
foreach my $alias ($property->aliases) {
$yes->add_conflicting($alias->name);
}
}
}
my $no = $property->table('No');
if (defined $no) {
my $n = $property->table('N');
if (defined $n && $no == $n) {
foreach my $alias ($property->aliases) {
$no->add_conflicting($alias->name, 'P');
}
}
}
}
return;
}
sub register_file_for_name($$$) {
# Given info about a table and a datafile that it should be associated
# with, register that association
my $table = shift;
my $directory_ref = shift; # Array of the directory path for the file
my $file = shift; # The file name in the final directory.
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
trace "table=$table, file=$file, directory=@$directory_ref, fate=", $table->fate if main::DEBUG && $to_trace;
if ($table->isa('Property')) {
$table->set_file_path(@$directory_ref, $file);
push @map_properties, $table;
# No swash means don't do the rest of this.
return if $table->fate != $ORDINARY
&& ! ($table->name =~ /^_/ && $table->fate == $INTERNAL_ONLY);
# Get the path to the file
my @path = $table->file_path;
# Use just the file name if no subdirectory.
shift @path if $path[0] eq File::Spec->curdir();
my $file = join '/', @path;
# Create a hash entry for utf8_heavy to get the file that stores this
# property's map table
foreach my $alias ($table->aliases) {
my $name = $alias->name;
if ($name =~ /^_/) {
$strict_property_to_file_of{lc $name} = $file;
}
else {
$loose_property_to_file_of{standardize($name)} = $file;
}
}
# And a way for utf8_heavy to find the proper key in the SwashInfo
# hash for this property.
$file_to_swash_name{$file} = "To" . $table->swash_name;
return;
}
# Do all of the work for all equivalent tables when called with the leader
# table, so skip if isn't the leader.
return if $table->leader != $table;
# If this is a complement of another file, use that other file instead,
# with a ! prepended to it.
my $complement;
if (($complement = $table->complement) != 0) {
my @directories = $complement->file_path;
# This assumes that the 0th element is something like 'lib',
# the 1th element the property name (in its own directory), like
# 'AHex', and the 2th element the file like 'Y' which will have a .pl
# appended to it later.
$directories[1] =~ s/^/!/;
$file = pop @directories;
$directory_ref =\@directories;
}
# Join all the file path components together, using slashes.
my $full_filename = join('/', @$directory_ref, $file);
# All go in the same subdirectory of unicore, or the special
# pseudo-directory '#'
if ($directory_ref->[0] !~ / ^ $matches_directory | \# $ /x) {
Carp::my_carp("Unexpected directory in "
. join('/', @{$directory_ref}, $file));
}
# For this table and all its equivalents ...
foreach my $table ($table, $table->equivalents) {
# Associate it with its file internally. Don't include the
# $matches_directory first component
$table->set_file_path(@$directory_ref, $file);
# No swash means don't do the rest of this.
next if $table->isa('Map_Table') && $table->fate != $ORDINARY;
my $sub_filename = join('/', $directory_ref->[1, -1], $file);
my $property = $table->property;
my $property_name = ($property == $perl)
? "" # 'perl' is never explicitly stated
: standardize($property->name) . '=';
my $is_default = 0; # Is this table the default one for the property?
# To calculate $is_default, we find if this table is the same as the
# default one for the property. But this is complicated by the
# possibility that there is a master table for this one, and the
# information is stored there instead of here.
my $parent = $table->parent;
my $leader_prop = $parent->property;
my $default_map = $leader_prop->default_map;
if (defined $default_map) {
my $default_table = $leader_prop->table($default_map);
$is_default = 1 if defined $default_table && $parent == $default_table;
}
# Calculate the loose name for this table. Mostly it's just its name,
# standardized. But in the case of Perl tables that are single-form
# equivalents to Unicode properties, it is the latter's name.
my $loose_table_name =
($property != $perl || $leader_prop == $perl)
? standardize($table->name)
: standardize($parent->name);
my $deprecated = ($table->status eq $DEPRECATED)
? $table->status_info
: "";
my $caseless_equivalent = $table->caseless_equivalent;
# And for each of the table's aliases... This inner loop eventually
# goes through all aliases in the UCD that we generate regex match
# files for
foreach my $alias ($table->aliases) {
my $standard = utf8_heavy_name($table, $alias);
# Generate an entry in either the loose or strict hashes, which
# will translate the property and alias names combination into the
# file where the table for them is stored.
if ($alias->loose_match) {
if (exists $loose_to_file_of{$standard}) {
Carp::my_carp("Can't change file registered to $loose_to_file_of{$standard} to '$sub_filename'.");
}
else {
$loose_to_file_of{$standard} = $sub_filename;
}
}
else {
if (exists $stricter_to_file_of{$standard}) {
Carp::my_carp("Can't change file registered to $stricter_to_file_of{$standard} to '$sub_filename'.");
}
else {
$stricter_to_file_of{$standard} = $sub_filename;
# Tightly coupled with how utf8_heavy.pl works, for a
# floating point number that is a whole number, get rid of
# the trailing decimal point and 0's, so that utf8_heavy
# will work. Also note that this assumes that such a
# number is matched strictly; so if that were to change,
# this would be wrong.
if ((my $integer_name = $alias->name)
=~ s/^ ( -? \d+ ) \.0+ $ /$1/x)
{
$stricter_to_file_of{$property_name . $integer_name}
= $sub_filename;
}
}
}
# For Unicode::UCD, create a mapping of the prop=value to the
# canonical =value for that property.
if ($standard =~ /=/) {
# This could happen if a strict name mapped into an existing
# loose name. In that event, the strict names would have to
# be moved to a new hash.
if (exists($loose_to_standard_value{$standard})) {
Carp::my_carp_bug("'$standard' conflicts with a pre-existing use. Bad News. Continuing anyway");
}
$loose_to_standard_value{$standard} = $loose_table_name;
}
# Keep a list of the deprecated properties and their filenames
if ($deprecated && $complement == 0) {
$utf8::why_deprecated{$sub_filename} = $deprecated;
}
# And a substitute table, if any, for case-insensitive matching
if ($caseless_equivalent != 0) {
$caseless_equivalent_to{$standard} = $caseless_equivalent;
}
# Add to defaults list if the table this alias belongs to is the
# default one
$loose_defaults{$standard} = 1 if $is_default;
}
}
return;
}
{ # Closure
my %base_names; # Names already used for avoiding DOS 8.3 filesystem
# conflicts
my %full_dir_name_of; # Full length names of directories used.
sub construct_filename($$$) {
# Return a file name for a table, based on the table name, but perhaps
# changed to get rid of non-portable characters in it, and to make
# sure that it is unique on a file system that allows the names before
# any period to be at most 8 characters (DOS). While we're at it
# check and complain if there are any directory conflicts.
my $name = shift; # The name to start with
my $mutable = shift; # Boolean: can it be changed? If no, but
# yet it must be to work properly, a warning
# is given
my $directories_ref = shift; # A reference to an array containing the
# path to the file, with each element one path
# component. This is used because the same
# name can be used in different directories.
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $warn = ! defined wantarray; # If true, then if the name is
# changed, a warning is issued as well.
if (! defined $name) {
Carp::my_carp("Undefined name in directory "
. File::Spec->join(@$directories_ref)
. ". '_' used");
return '_';
}
# Make sure that no directory names conflict with each other. Look at
# each directory in the input file's path. If it is already in use,
# assume it is correct, and is merely being re-used, but if we
# truncate it to 8 characters, and find that there are two directories
# that are the same for the first 8 characters, but differ after that,
# then that is a problem.
foreach my $directory (@$directories_ref) {
my $short_dir = substr($directory, 0, 8);
if (defined $full_dir_name_of{$short_dir}) {
next if $full_dir_name_of{$short_dir} eq $directory;
Carp::my_carp("$directory conflicts with $full_dir_name_of{$short_dir}. Bad News. Continuing anyway");
}
else {
$full_dir_name_of{$short_dir} = $directory;
}
}
my $path = join '/', @$directories_ref;
$path .= '/' if $path;
# Remove interior underscores.
(my $filename = $name) =~ s/ (?<=.) _ (?=.) //xg;
# Convert the dot in floating point numbers to an underscore
$filename =~ s/\./_/ if $filename =~ / ^ \d+ \. \d+ $ /x;
my $suffix = "";
# Extract any suffix, delete any non-word character, and truncate to 3
# after the dot
if ($filename =~ m/ ( .*? ) ( \. .* ) /x) {
$filename = $1;
$suffix = $2;
$suffix =~ s/\W+//g;
substr($suffix, 4) = "" if length($suffix) > 4;
}
# Change any non-word character outside the suffix into an underscore,
# and truncate to 8.
$filename =~ s/\W+/_/g; # eg., "L&" -> "L_"
substr($filename, 8) = "" if length($filename) > 8;
# Make sure the basename doesn't conflict with something we
# might have already written. If we have, say,
# InGreekExtended1
# InGreekExtended2
# they become
# InGreekE
# InGreek2
my $warned = 0;
while (my $num = $base_names{$path}{lc "$filename$suffix"}++) {
$num++; # so basenames with numbers start with '2', which
# just looks more natural.
# Want to append $num, but if it'll make the basename longer
# than 8 characters, pre-truncate $filename so that the result
# is acceptable.
my $delta = length($filename) + length($num) - 8;
if ($delta > 0) {
substr($filename, -$delta) = $num;
}
else {
$filename .= $num;
}
if ($warn && ! $warned) {
$warned = 1;
Carp::my_carp("'$path$name' conflicts with another name on a filesystem with 8 significant characters (like DOS). Proceeding anyway.");
}
}
return $filename if $mutable;
# If not changeable, must return the input name, but warn if needed to
# change it beyond shortening it.
if ($name ne $filename
&& substr($name, 0, length($filename)) ne $filename) {
Carp::my_carp("'$path$name' had to be changed into '$filename'. Bad News. Proceeding anyway.");
}
return $name;
}
}
# The pod file contains a very large table. Many of the lines in that table
# would exceed a typical output window's size, and so need to be wrapped with
# a hanging indent to make them look good. The pod language is really
# insufficient here. There is no general construct to do that in pod, so it
# is done here by beginning each such line with a space to cause the result to
# be output without formatting, and doing all the formatting here. This leads
# to the result that if the eventual display window is too narrow it won't
# look good, and if the window is too wide, no advantage is taken of that
# extra width. A further complication is that the output may be indented by
# the formatter so that there is less space than expected. What I (khw) have
# done is to assume that that indent is a particular number of spaces based on
# what it is in my Linux system; people can always resize their windows if
# necessary, but this is obviously less than desirable, but the best that can
# be expected.
my $automatic_pod_indent = 8;
# Try to format so that uses fewest lines, but few long left column entries
# slide into the right column. An experiment on 5.1 data yielded the
# following percentages that didn't cut into the other side along with the
# associated first-column widths
# 69% = 24
# 80% not too bad except for a few blocks
# 90% = 33; # , cuts 353/3053 lines from 37 = 12%
# 95% = 37;
my $indent_info_column = 27; # 75% of lines didn't have overlap
my $FILLER = 3; # Length of initial boiler-plate columns in a pod line
# The 3 is because of:
# 1 for the leading space to tell the pod formatter to
# output as-is
# 1 for the flag
# 1 for the space between the flag and the main data
sub format_pod_line ($$$;$$) {
# Take a pod line and return it, formatted properly
my $first_column_width = shift;
my $entry = shift; # Contents of left column
my $info = shift; # Contents of right column
my $status = shift || ""; # Any flag
my $loose_match = shift; # Boolean.
$loose_match = 1 unless defined $loose_match;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
my $flags = "";
$flags .= $STRICTER if ! $loose_match;
$flags .= $status if $status;
# There is a blank in the left column to cause the pod formatter to
# output the line as-is.
return sprintf " %-*s%-*s %s\n",
# The first * in the format is replaced by this, the -1 is
# to account for the leading blank. There isn't a
# hard-coded blank after this to separate the flags from
# the rest of the line, so that in the unlikely event that
# multiple flags are shown on the same line, they both
# will get displayed at the expense of that separation,
# but since they are left justified, a blank will be
# inserted in the normal case.
$FILLER - 1,
$flags,
# The other * in the format is replaced by this number to
# cause the first main column to right fill with blanks.
# The -1 is for the guaranteed blank following it.
$first_column_width - $FILLER - 1,
$entry,
$info;
}
my @zero_match_tables; # List of tables that have no matches in this release
sub make_re_pod_entries($) {
# This generates the entries for the pod file for a given table.
# Also done at this time are any children tables. The output looks like:
# \p{Common} \p{Script=Common} (Short: \p{Zyyy}) (5178)
my $input_table = shift; # Table the entry is for
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
# Generate parent and all its children at the same time.
return if $input_table->parent != $input_table;
my $property = $input_table->property;
my $type = $property->type;
my $full_name = $property->full_name;
my $count = $input_table->count;
my $unicode_count;
my $non_unicode_string;
if ($count > $MAX_UNICODE_CODEPOINTS) {
$unicode_count = $count - ($MAX_WORKING_CODEPOINT
- $MAX_UNICODE_CODEPOINT);
$non_unicode_string = " plus all above-Unicode code points";
}
else {
$unicode_count = $count;
$non_unicode_string = "";
}
my $string_count = clarify_number($unicode_count) . $non_unicode_string;
my $definition = $input_table->calculate_table_definition;
if ($definition) {
# Save the definition for later use.
$input_table->set_definition($definition);
$definition = ": $definition";
}
my $status = $input_table->status;
my $status_info = $input_table->status_info;
my $caseless_equivalent = $input_table->caseless_equivalent;
# Don't mention a placeholder equivalent as it isn't to be listed in the
# pod
$caseless_equivalent = 0 if $caseless_equivalent != 0
&& $caseless_equivalent->fate > $ORDINARY;
my $entry_for_first_table; # The entry for the first table output.
# Almost certainly, it is the parent.
# For each related table (including itself), we will generate a pod entry
# for each name each table goes by
foreach my $table ($input_table, $input_table->children) {
# utf8_heavy.pl cannot deal with null string property values, so skip
# any tables that have no non-null names.
next if ! grep { $_->name ne "" } $table->aliases;
# First, gather all the info that applies to this table as a whole.
push @zero_match_tables, $table if $count == 0
# Don't mention special tables
# as being zero length
&& $table->fate == $ORDINARY;
my $table_property = $table->property;
# The short name has all the underscores removed, while the full name
# retains them. Later, we decide whether to output a short synonym
# for the full one, we need to compare apples to apples, so we use the
# short name's length including underscores.
my $table_property_short_name_length;
my $table_property_short_name
= $table_property->short_name(\$table_property_short_name_length);
my $table_property_full_name = $table_property->full_name;
# Get how much savings there is in the short name over the full one
# (delta will always be <= 0)
my $table_property_short_delta = $table_property_short_name_length
- length($table_property_full_name);
my @table_description = $table->description;
my @table_note = $table->note;
# Generate an entry for each alias in this table.
my $entry_for_first_alias; # saves the first one encountered.
foreach my $alias ($table->aliases) {
# Skip if not to go in pod.
next unless $alias->make_re_pod_entry;
# Start gathering all the components for the entry
my $name = $alias->name;
# Skip if name is empty, as can't be accessed by regexes.
next if $name eq "";
my $entry; # Holds the left column, may include extras
my $entry_ref; # To refer to the left column's contents from
# another entry; has no extras
# First the left column of the pod entry. Tables for the $perl
# property always use the single form.
if ($table_property == $perl) {
$entry = "\\p{$name}";
$entry .= " \\p$name" if length $name == 1; # Show non-braced
# form too
$entry_ref = "\\p{$name}";
}
else { # Compound form.
# Only generate one entry for all the aliases that mean true
# or false in binary properties. Append a '*' to indicate
# some are missing. (The heading comment notes this.)
my $rhs;
if ($type == $BINARY) {
next if $name ne 'N' && $name ne 'Y';
$rhs = "$name*";
}
elsif ($type != $FORCED_BINARY) {
$rhs = $name;
}
else {
# Forced binary properties require special handling. It
# has two sets of tables, one set is true/false; and the
# other set is everything else. Entries are generated for
# each set. Use the Bidi_Mirrored property (which appears
# in all Unicode versions) to get a list of the aliases
# for the true/false tables. Of these, only output the N
# and Y ones, the same as, a regular binary property. And
# output all the rest, same as a non-binary property.
my $bm = property_ref("Bidi_Mirrored");
if ($name eq 'N' || $name eq 'Y') {
$rhs = "$name*";
} elsif (grep { $name eq $_->name } $bm->table("Y")->aliases,
$bm->table("N")->aliases)
{
next;
}
else {
$rhs = $name;
}
}
# Colon-space is used to give a little more space to be easier
# to read;
$entry = "\\p{"
. $table_property_full_name
. ": $rhs}";
# But for the reference to this entry, which will go in the
# right column, where space is at a premium, use equals
# without a space
$entry_ref = "\\p{" . $table_property_full_name . "=$name}";
}
# Then the right (info) column. This is stored as components of
# an array for the moment, then joined into a string later. For
# non-internal only properties, begin the info with the entry for
# the first table we encountered (if any), as things are ordered
# so that that one is the most descriptive. This leads to the
# info column of an entry being a more descriptive version of the
# name column
my @info;
if ($name =~ /^_/) {
push @info,
'(For internal use by Perl, not necessarily stable)';
}
elsif ($entry_for_first_alias) {
push @info, $entry_for_first_alias;
}
# If this entry is equivalent to another, add that to the info,
# using the first such table we encountered
if ($entry_for_first_table) {
if (@info) {
push @info, "(= $entry_for_first_table)";
}
else {
push @info, $entry_for_first_table;
}
}
# If the name is a large integer, add an equivalent with an
# exponent for better readability
if ($name =~ /^[+-]?[\d]+$/ && $name >= 10_000) {
push @info, sprintf "(= %.1e)", $name
}
my $parenthesized = "";
if (! $entry_for_first_alias) {
# This is the first alias for the current table. The alias
# array is ordered so that this is the fullest, most
# descriptive alias, so it gets the fullest info. The other
# aliases are mostly merely pointers to this one, using the
# information already added above.
# Display any status message, but only on the parent table
if ($status && ! $entry_for_first_table) {
push @info, $status_info;
}
# Put out any descriptive info
if (@table_description || @table_note) {
push @info, join "; ", @table_description, @table_note;
}
# Look to see if there is a shorter name we can point people
# at
my $standard_name = standardize($name);
my $short_name;
my $proposed_short = $table->short_name;
if (defined $proposed_short) {
my $standard_short = standardize($proposed_short);
# If the short name is shorter than the standard one, or
# even if it's not, but the combination of it and its
# short property name (as in \p{prop=short} ($perl doesn't
# have this form)) saves at least two characters, then,
# cause it to be listed as a shorter synonym.
if (length $standard_short < length $standard_name
|| ($table_property != $perl
&& (length($standard_short)
- length($standard_name)
+ $table_property_short_delta) # (<= 0)
< -2))
{
$short_name = $proposed_short;
if ($table_property != $perl) {
$short_name = $table_property_short_name
. "=$short_name";
}
$short_name = "\\p{$short_name}";
}
}
# And if this is a compound form name, see if there is a
# single form equivalent
my $single_form;
if ($table_property != $perl && $table_property != $block) {
# Special case the binary N tables, so that will print
# \P{single}, but use the Y table values to populate
# 'single', as we haven't likewise populated the N table.
# For forced binary tables, we can't just look at the N
# table, but must see if this table is equivalent to the N
# one, as there are two equivalent beasts in these
# properties.
my $test_table;
my $p;
if ( ($type == $BINARY
&& $input_table == $property->table('No'))
|| ($type == $FORCED_BINARY
&& $property->table('No')->
is_set_equivalent_to($input_table)))
{
$test_table = $property->table('Yes');
$p = 'P';
}
else {
$test_table = $input_table;
$p = 'p';
}
# Look for a single form amongst all the children.
foreach my $table ($test_table->children) {
next if $table->property != $perl;
my $proposed_name = $table->short_name;
next if ! defined $proposed_name;
# Don't mention internal-only properties as a possible
# single form synonym
next if substr($proposed_name, 0, 1) eq '_';
$proposed_name = "\\$p\{$proposed_name}";
if (! defined $single_form
|| length($proposed_name) < length $single_form)
{
$single_form = $proposed_name;
# The goal here is to find a single form; not the
# shortest possible one. We've already found a
# short name. So, stop at the first single form
# found, which is likely to be closer to the
# original.
last;
}
}
}
# Output both short and single in the same parenthesized
# expression, but with only one of 'Single', 'Short' if there
# are both items.
if ($short_name || $single_form || $table->conflicting) {
$parenthesized .= "Short: $short_name" if $short_name;
if ($short_name && $single_form) {
$parenthesized .= ', ';
}
elsif ($single_form) {
$parenthesized .= 'Single: ';
}
$parenthesized .= $single_form if $single_form;
}
}
if ($caseless_equivalent != 0) {
$parenthesized .= '; ' if $parenthesized ne "";
$parenthesized .= "/i= " . $caseless_equivalent->complete_name;
}
# Warn if this property isn't the same as one that a
# semi-casual user might expect. The other components of this
# parenthesized structure are calculated only for the first entry
# for this table, but the conflicting is deemed important enough
# to go on every entry.
my $conflicting = join " NOR ", $table->conflicting;
if ($conflicting) {
$parenthesized .= '; ' if $parenthesized ne "";
$parenthesized .= "NOT $conflicting";
}
push @info, "($parenthesized)" if $parenthesized;
if ($name =~ /_$/ && $alias->loose_match) {
push @info, "Note the trailing '_' matters in spite of loose matching rules.";
}
if ($table_property != $perl && $table->perl_extension) {
push @info, '(Perl extension)';
}
my $definition = $table->definition // "";
$definition = "" if $entry_for_first_alias;
$definition = ": $definition" if $definition;
push @info, "($string_count$definition)";
# Now, we have both the entry and info so add them to the
# list of all the properties.
push @match_properties,
format_pod_line($indent_info_column,
$entry,
join( " ", @info),
$alias->status,
$alias->loose_match);
$entry_for_first_alias = $entry_ref unless $entry_for_first_alias;
} # End of looping through the aliases for this table.
if (! $entry_for_first_table) {
$entry_for_first_table = $entry_for_first_alias;
}
} # End of looping through all the related tables
return;
}
sub make_ucd_table_pod_entries {
my $table = shift;
# Generate the entries for the UCD section of the pod for $table. This
# also calculates if names are ambiguous, so has to be called even if the
# pod is not being output
my $short_name = $table->name;
my $standard_short_name = standardize($short_name);
my $full_name = $table->full_name;
my $standard_full_name = standardize($full_name);
my $full_info = ""; # Text of info column for full-name entries
my $other_info = ""; # Text of info column for short-name entries
my $short_info = ""; # Text of info column for other entries
my $meaning = ""; # Synonym of this table
my $property = ($table->isa('Property'))
? $table
: $table->parent->property;
my $perl_extension = $table->perl_extension;
my $is_perl_extension_match_table_but_not_dollar_perl
= $property != $perl
&& $perl_extension
&& $property != $table;
# Get the more official name for for perl extensions that aren't
# stand-alone properties
if ($is_perl_extension_match_table_but_not_dollar_perl) {
if ($property->type == $BINARY) {
$meaning = $property->full_name;
}
else {
$meaning = $table->parent->complete_name;
}
}
# There are three types of info column. One for the short name, one for
# the full name, and one for everything else. They mostly are the same,
# so initialize in the same loop.
foreach my $info_ref (\$full_info, \$short_info, \$other_info) {
if ($info_ref != \$full_info) {
# The non-full name columns include the full name
$$info_ref .= $full_name;
}
if ($is_perl_extension_match_table_but_not_dollar_perl) {
# Add the synonymous name for the non-full name entries; and to
# the full-name entry if it adds extra information
if ( standardize($meaning) ne $standard_full_name
|| $info_ref == \$other_info
|| $info_ref == \$short_info)
{
my $parenthesized = $info_ref != \$full_info;
$$info_ref .= " " if $$info_ref && $parenthesized;
$$info_ref .= "(=" if $parenthesized;
$$info_ref .= "$meaning";
$$info_ref .= ")" if $parenthesized;
$$info_ref .= ".";
}
}
# And the full-name entry includes the short name, if shorter
if ($info_ref == \$full_info
&& length $standard_short_name < length $standard_full_name)
{
$full_info =~ s/\.\Z//;
$full_info .= " " if $full_info;
$full_info .= "(Short: $short_name)";
}
if ($table->perl_extension) {
$$info_ref =~ s/\.\Z//;
$$info_ref .= ". " if $$info_ref;
$$info_ref .= "(Perl extension)";
}
}
my $definition;
my $definition_table;
my $type = $table->property->type;
if ($type == $BINARY || $type == $FORCED_BINARY) {
$definition_table = $table->property->table('Y');
}
elsif ($table->isa('Match_Table')) {
$definition_table = $table;
}
$definition = $definition_table->calculate_table_definition
if defined $definition_table
&& $definition_table != 0;
# Add any extra annotations to the full name entry
foreach my $more_info ($table->description,
$definition,
$table->note,
$table->status_info)
{
next unless $more_info;
$full_info =~ s/\.\Z//;
$full_info .= ". " if $full_info;
$full_info .= $more_info;
}
if ($table->property->type == $FORCED_BINARY) {
if ($full_info) {
$full_info =~ s/\.\Z//;
$full_info .= ". ";
}
$full_info .= "This is a combination property which has both:"
. " 1) a map to various string values; and"
. " 2) a map to boolean Y/N, where 'Y' means the"
. " string value is non-empty. Add the prefix 'is'"
. " to the prop_invmap() call to get the latter";
}
# These keep track if have created full and short name pod entries for the
# property
my $done_full = 0;
my $done_short = 0;
# Every possible name is kept track of, even those that aren't going to be
# output. This way we can be sure to find the ambiguities.
foreach my $alias ($table->aliases) {
my $name = $alias->name;
my $standard = standardize($name);
my $info;
my $output_this = $alias->ucd;
# If the full and short names are the same, we want to output the full
# one's entry, so it has priority.
if ($standard eq $standard_full_name) {
next if $done_full;
$done_full = 1;
$info = $full_info;
}
elsif ($standard eq $standard_short_name) {
next if $done_short;
$done_short = 1;
next if $standard_short_name eq $standard_full_name;
$info = $short_info;
}
else {
$info = $other_info;
}
$combination_property{$standard} = 1
if $table->property->type == $FORCED_BINARY;
# Here, we have set up the two columns for this entry. But if an
# entry already exists for this name, we have to decide which one
# we're going to later output.
if (exists $ucd_pod{$standard}) {
# If the two entries refer to the same property, it's not going to
# be ambiguous. (Likely it's because the names when standardized
# are the same.) But that means if they are different properties,
# there is ambiguity.
if ($ucd_pod{$standard}->{'property'} != $property) {
# Here, we have an ambiguity. This code assumes that one is
# scheduled to be output and one not and that one is a perl
# extension (which is not to be output) and the other isn't.
# If those assumptions are wrong, things have to be rethought.
if ($ucd_pod{$standard}{'output_this'} == $output_this
|| $ucd_pod{$standard}{'perl_extension'} == $perl_extension
|| $output_this == $perl_extension)
{
Carp::my_carp("Bad news. $property and $ucd_pod{$standard}->{'property'} have unexpected output status and perl-extension combinations. Proceeding anyway.");
}
# We modify the info column of the one being output to
# indicate the ambiguity. Set $which to point to that one's
# info.
my $which;
if ($ucd_pod{$standard}{'output_this'}) {
$which = \$ucd_pod{$standard}->{'info'};
}
else {
$which = \$info;
$meaning = $ucd_pod{$standard}{'meaning'};
}
chomp $$which;
$$which =~ s/\.\Z//;
$$which .= "; NOT '$standard' meaning '$meaning'";
$ambiguous_names{$standard} = 1;
}
# Use the non-perl-extension variant
next unless $ucd_pod{$standard}{'perl_extension'};
}
# Store enough information about this entry that we can later look for
# ambiguities, and output it properly.
$ucd_pod{$standard} = { 'name' => $name,
'info' => $info,
'meaning' => $meaning,
'output_this' => $output_this,
'perl_extension' => $perl_extension,
'property' => $property,
'status' => $alias->status,
};
} # End of looping through all this table's aliases
return;
}
sub pod_alphanumeric_sort {
# Sort pod entries alphanumerically.
# The first few character columns are filler, plus the '\p{'; and get rid
# of all the trailing stuff, starting with the trailing '}', so as to sort
# on just 'Name=Value'
(my $a = lc $a) =~ s/^ .*? \{ //x;
$a =~ s/}.*//;
(my $b = lc $b) =~ s/^ .*? \{ //x;
$b =~ s/}.*//;
# Determine if the two operands are both internal only or both not.
# Character 0 should be a '\'; 1 should be a p; 2 should be '{', so 3
# should be the underscore that begins internal only
my $a_is_internal = (substr($a, 0, 1) eq '_');
my $b_is_internal = (substr($b, 0, 1) eq '_');
# Sort so the internals come last in the table instead of first (which the
# leading underscore would otherwise indicate).
if ($a_is_internal != $b_is_internal) {
return 1 if $a_is_internal;
return -1
}
# Determine if the two operands are compound or not, and if so if are
# "numeric" property values or not, like \p{Age: 3.0}. But there are also
# things like \p{Canonical_Combining_Class: CCC133} and \p{Age: V10_0},
# all of which this considers numeric, and for sorting, looks just at the
# numeric parts. It can also be a rational like \p{Numeric Value=-1/2}.
my $split_re = qr/
^ ( [^:=]+ ) # $1 is undef if not a compound form, otherwise is the
# property name
[:=] \s* # The syntax for the compound form
(?: # followed by ...
( # $2 gets defined if what follows is a "numeric"
# expression, which is ...
( -? \d+ (?: [.\/] \d+)? # An integer, float, or rational
# number, optionally signed
| [[:alpha:]]{2,} \d+ $ ) # or something like CCC131. Either
# of these go into $3
| ( V \d+ _ \d+ ) # or a Unicode's Age property version
# number, into $4
)
| .* $ # If not "numeric", accept anything so that $1 gets
# defined if it is any compound form
) /ix;
my ($a_initial, $a_numeric, $a_number, $a_version) = ($a =~ $split_re);
my ($b_initial, $b_numeric, $b_number, $b_version) = ($b =~ $split_re);
# Sort alphabeticlly on the whole property name if either operand isn't
# compound, or they differ.
return $a cmp $b if ! defined $a_initial
|| ! defined $b_initial
|| $a_initial ne $b_initial;
if (! defined $a_numeric) {
# If neither is numeric, use alpha sort
return $a cmp $b if ! defined $b_numeric;
return 1; # Sort numeric ahead of alpha
}
# Here $a is numeric
return -1 if ! defined $b_numeric; # Numeric sorts before alpha
# Here they are both numeric in the same property.
# Convert version numbers into regular numbers
if (defined $a_version) {
($a_number = $a_version) =~ s/^V//i;
$a_number =~ s/_/./;
}
else { # Otherwise get rid of the, e.g., CCC in CCC9 */
$a_number =~ s/ ^ [[:alpha:]]+ //x;
}
if (defined $b_version) {
($b_number = $b_version) =~ s/^V//i;
$b_number =~ s/_/./;
}
else {
$b_number =~ s/ ^ [[:alpha:]]+ //x;
}
# Convert rationals to floating for the comparison.
$a_number = eval $a_number if $a_number =~ qr{/};
$b_number = eval $b_number if $b_number =~ qr{/};
return $a_number <=> $b_number || $a cmp $b;
}
sub make_pod () {
# Create the .pod file. This generates the various subsections and then
# combines them in one big HERE document.
my $Is_flags_text = "If an entry has flag(s) at its beginning, like \"$DEPRECATED\", the \"Is_\" form has the same flag(s)";
return unless defined $pod_directory;
print "Making pod file\n" if $verbosity >= $PROGRESS;
my $exception_message =
'(Any exceptions are individually noted beginning with the word NOT.)';
my @block_warning;
if (-e 'Blocks.txt') {
# Add the line: '\p{In_*} \p{Block: *}', with the warning message
# if the global $has_In_conflicts indicates we have them.
push @match_properties, format_pod_line($indent_info_column,
'\p{In_*}',
'\p{Block: *}'
. (($has_In_conflicts)
? " $exception_message"
: ""),
$DISCOURAGED);
@block_warning = << "END";
In particular, matches in the Block property have single forms
defined by Perl that begin with C<"In_">, C<"Is_>, or even with no prefix at
all, Like all B<DISCOURAGED> forms, these are not stable. For example,
C<\\p{Block=Deseret}> can currently be written as C<\\p{In_Deseret}>,
C<\\p{Is_Deseret}>, or C<\\p{Deseret}>. But, a new Unicode version may
come along that would force Perl to change the meaning of one or more of
these, and your program would no longer be correct. Currently there are no
such conflicts with the form that begins C<"In_">, but there are many with the
other two shortcuts, and Unicode continues to define new properties that begin
with C<"In">, so it's quite possible that a conflict will occur in the future.
The compound form is guaranteed to not become obsolete, and its meaning is
clearer anyway. See L<perlunicode/"Blocks"> for more information about this.
END
}
my $text = $Is_flags_text;
$text = "$exception_message $text" if $has_Is_conflicts;
# And the 'Is_ line';
push @match_properties, format_pod_line($indent_info_column,
'\p{Is_*}',
"\\p{*} $text");
# Sort the properties array for output. It is sorted alphabetically
# except numerically for numeric properties, and only output unique lines.
@match_properties = sort pod_alphanumeric_sort uniques @match_properties;
my $formatted_properties = simple_fold(\@match_properties,
"",
# indent succeeding lines by two extra
# which looks better
$indent_info_column + 2,
# shorten the line length by how much
# the formatter indents, so the folded
# line will fit in the space
# presumably available
$automatic_pod_indent);
# Add column headings, indented to be a little more centered, but not
# exactly
$formatted_properties = format_pod_line($indent_info_column,
' NAME',
' INFO')
. "\n"
. $formatted_properties;
# Generate pod documentation lines for the tables that match nothing
my $zero_matches = "";
if (@zero_match_tables) {
@zero_match_tables = uniques(@zero_match_tables);
$zero_matches = join "\n\n",
map { $_ = '=item \p{' . $_->complete_name . "}" }
sort { $a->complete_name cmp $b->complete_name }
@zero_match_tables;
$zero_matches = <<END;
=head2 Legal C<\\p{}> and C<\\P{}> constructs that match no characters
Unicode has some property-value pairs that currently don't match anything.
This happens generally either because they are obsolete, or they exist for
symmetry with other forms, but no language has yet been encoded that uses
them. In this version of Unicode, the following match zero code points:
=over 4
$zero_matches
=back
END
}
# Generate list of properties that we don't accept, grouped by the reasons
# why. This is so only put out the 'why' once, and then list all the
# properties that have that reason under it.
my %why_list; # The keys are the reasons; the values are lists of
# properties that have the key as their reason
# For each property, add it to the list that are suppressed for its reason
# The sort will cause the alphabetically first properties to be added to
# each list first, so each list will be sorted.
foreach my $property (sort keys %why_suppressed) {
next unless $why_suppressed{$property};
push @{$why_list{$why_suppressed{$property}}}, $property;
}
# For each reason (sorted by the first property that has that reason)...
my @bad_re_properties;
foreach my $why (sort { $why_list{$a}->[0] cmp $why_list{$b}->[0] }
keys %why_list)
{
# Add to the output, all the properties that have that reason.
my $has_item = 0; # Flag if actually output anything.
foreach my $name (@{$why_list{$why}}) {
# Split compound names into $property and $table components
my $property = $name;
my $table;
if ($property =~ / (.*) = (.*) /x) {
$property = $1;
$table = $2;
}
# This release of Unicode may not have a property that is
# suppressed, so don't reference a non-existent one.
$property = property_ref($property);
next if ! defined $property;
# And since this list is only for match tables, don't list the
# ones that don't have match tables.
next if ! $property->to_create_match_tables;
# Find any abbreviation, and turn it into a compound name if this
# is a property=value pair.
my $short_name = $property->name;
$short_name .= '=' . $property->table($table)->name if $table;
# Start with an empty line.
push @bad_re_properties, "\n\n" unless $has_item;
# And add the property as an item for the reason.
push @bad_re_properties, "\n=item I<$name> ($short_name)\n";
$has_item = 1;
}
# And add the reason under the list of properties, if such a list
# actually got generated. Note that the header got added
# unconditionally before. But pod ignores extra blank lines, so no
# harm.
push @bad_re_properties, "\n$why\n" if $has_item;
} # End of looping through each reason.
if (! @bad_re_properties) {
push @bad_re_properties,
"*** This installation accepts ALL non-Unihan properties ***";
}
else {
# Add =over only if non-empty to avoid an empty =over/=back section,
# which is considered bad form.
unshift @bad_re_properties, "\n=over 4\n";
push @bad_re_properties, "\n=back\n";
}
# Similarly, generate a list of files that we don't use, grouped by the
# reasons why (Don't output if the reason is empty). First, create a hash
# whose keys are the reasons, and whose values are anonymous arrays of all
# the files that share that reason.
my %grouped_by_reason;
foreach my $file (keys %skipped_files) {
next unless $skipped_files{$file};
push @{$grouped_by_reason{$skipped_files{$file}}}, $file;
}
# Then, sort each group.
foreach my $group (keys %grouped_by_reason) {
@{$grouped_by_reason{$group}} = sort { lc $a cmp lc $b }
@{$grouped_by_reason{$group}} ;
}
# Finally, create the output text. For each reason (sorted by the
# alphabetically first file that has that reason)...
my @unused_files;
foreach my $reason (sort { lc $grouped_by_reason{$a}->[0]
cmp lc $grouped_by_reason{$b}->[0]
}
keys %grouped_by_reason)
{
# Add all the files that have that reason to the output. Start
# with an empty line.
push @unused_files, "\n\n";
push @unused_files, map { "\n=item F<$_> \n" }
@{$grouped_by_reason{$reason}};
# And add the reason under the list of files
push @unused_files, "\n$reason\n";
}
# Similarly, create the output text for the UCD section of the pod
my @ucd_pod;
foreach my $key (keys %ucd_pod) {
next unless $ucd_pod{$key}->{'output_this'};
push @ucd_pod, format_pod_line($indent_info_column,
$ucd_pod{$key}->{'name'},
$ucd_pod{$key}->{'info'},
$ucd_pod{$key}->{'status'},
);
}
# Sort alphabetically, and fold for output
@ucd_pod = sort { lc substr($a, 2) cmp lc substr($b, 2) } @ucd_pod;
my $ucd_pod = simple_fold(\@ucd_pod,
' ',
$indent_info_column,
$automatic_pod_indent);
$ucd_pod = format_pod_line($indent_info_column, 'NAME', ' INFO')
. "\n"
. $ucd_pod;
my $space_hex = sprintf("%02x", ord " ");
local $" = "";
# Everything is ready to assemble.
my @OUT = << "END";
=begin comment
$HEADER
To change this file, edit $0 instead.
=end comment
=head1 NAME
$pod_file - Index of Unicode Version $unicode_version character properties in Perl
=head1 DESCRIPTION
This document provides information about the portion of the Unicode database
that deals with character properties, that is the portion that is defined on
single code points. (L</Other information in the Unicode data base>
below briefly mentions other data that Unicode provides.)
Perl can provide access to all non-provisional Unicode character properties,
though not all are enabled by default. The omitted ones are the Unihan
properties (accessible via the CPAN module L<Unicode::Unihan>) and certain
deprecated or Unicode-internal properties. (An installation may choose to
recompile Perl's tables to change this. See L</Unicode character
properties that are NOT accepted by Perl>.)
For most purposes, access to Unicode properties from the Perl core is through
regular expression matches, as described in the next section.
For some special purposes, and to access the properties that are not suitable
for regular expression matching, all the Unicode character properties that
Perl handles are accessible via the standard L<Unicode::UCD> module, as
described in the section L</Properties accessible through Unicode::UCD>.
Perl also provides some additional extensions and short-cut synonyms
for Unicode properties.
This document merely lists all available properties and does not attempt to
explain what each property really means. There is a brief description of each
Perl extension; see L<perlunicode/Other Properties> for more information on
these. There is some detail about Blocks, Scripts, General_Category,
and Bidi_Class in L<perlunicode>, but to find out about the intricacies of the
official Unicode properties, refer to the Unicode standard. A good starting
place is L<$unicode_reference_url>.
Note that you can define your own properties; see
L<perlunicode/"User-Defined Character Properties">.
=head1 Properties accessible through C<\\p{}> and C<\\P{}>
The Perl regular expression C<\\p{}> and C<\\P{}> constructs give access to
most of the Unicode character properties. The table below shows all these
constructs, both single and compound forms.
B<Compound forms> consist of two components, separated by an equals sign or a
colon. The first component is the property name, and the second component is
the particular value of the property to match against, for example,
C<\\p{Script_Extensions: Greek}> and C<\\p{Script_Extensions=Greek}> both mean
to match characters whose Script_Extensions property value is Greek.
(C<Script_Extensions> is an improved version of the C<Script> property.)
B<Single forms>, like C<\\p{Greek}>, are mostly Perl-defined shortcuts for
their equivalent compound forms. The table shows these equivalences. (In our
example, C<\\p{Greek}> is a just a shortcut for
C<\\p{Script_Extensions=Greek}>). There are also a few Perl-defined single
forms that are not shortcuts for a compound form. One such is C<\\p{Word}>.
These are also listed in the table.
In parsing these constructs, Perl always ignores Upper/lower case differences
everywhere within the {braces}. Thus C<\\p{Greek}> means the same thing as
C<\\p{greek}>. But note that changing the case of the C<"p"> or C<"P"> before
the left brace completely changes the meaning of the construct, from "match"
(for C<\\p{}>) to "doesn't match" (for C<\\P{}>). Casing in this document is
for improved legibility.
Also, white space, hyphens, and underscores are normally ignored
everywhere between the {braces}, and hence can be freely added or removed
even if the C</x> modifier hasn't been specified on the regular expression.
But in the table below $a_bold_stricter at the beginning of an entry
means that tighter (stricter) rules are used for that entry:
=over 4
=over 4
=item Single form (C<\\p{name}>) tighter rules:
White space, hyphens, and underscores ARE significant
except for:
=over 4
=item * white space adjacent to a non-word character
=item * underscores separating digits in numbers
=back
That means, for example, that you can freely add or remove white space
adjacent to (but within) the braces without affecting the meaning.
=item Compound form (C<\\p{name=value}> or C<\\p{name:value}>) tighter rules:
The tighter rules given above for the single form apply to everything to the
right of the colon or equals; the looser rules still apply to everything to
the left.
That means, for example, that you can freely add or remove white space
adjacent to (but within) the braces and the colon or equal sign.
=back
=back
Some properties are considered obsolete by Unicode, but still available.
There are several varieties of obsolescence:
=over 4
=over 4
=item Stabilized
A property may be stabilized. Such a determination does not indicate
that the property should or should not be used; instead it is a declaration
that the property will not be maintained nor extended for newly encoded
characters. Such properties are marked with $a_bold_stabilized in the
table.
=item Deprecated
A property may be deprecated, perhaps because its original intent
has been replaced by another property, or because its specification was
somehow defective. This means that its use is strongly
discouraged, so much so that a warning will be issued if used, unless the
regular expression is in the scope of a C<S<no warnings 'deprecated'>>
statement. $A_bold_deprecated flags each such entry in the table, and
the entry there for the longest, most descriptive version of the property will
give the reason it is deprecated, and perhaps advice. Perl may issue such a
warning, even for properties that aren't officially deprecated by Unicode,
when there used to be characters or code points that were matched by them, but
no longer. This is to warn you that your program may not work like it did on
earlier Unicode releases.
A deprecated property may be made unavailable in a future Perl version, so it
is best to move away from them.
A deprecated property may also be stabilized, but this fact is not shown.
=item Obsolete
Properties marked with $a_bold_obsolete in the table are considered (plain)
obsolete. Generally this designation is given to properties that Unicode once
used for internal purposes (but not any longer).
=item Discouraged
This is not actually a Unicode-specified obsolescence, but applies to certain
Perl extensions that are present for backwards compatibility, but are
discouraged from being used. These are not obsolete, but their meanings are
not stable. Future Unicode versions could force any of these extensions to be
removed without warning, replaced by another property with the same name that
means something different. $A_bold_discouraged flags each such entry in the
table. Use the equivalent shown instead.
@block_warning
=back
=back
The table below has two columns. The left column contains the C<\\p{}>
constructs to look up, possibly preceded by the flags mentioned above; and
the right column contains information about them, like a description, or
synonyms. The table shows both the single and compound forms for each
property that has them. If the left column is a short name for a property,
the right column will give its longer, more descriptive name; and if the left
column is the longest name, the right column will show any equivalent shortest
name, in both single and compound forms if applicable.
If braces are not needed to specify a property (e.g., C<\\pL>), the left
column contains both forms, with and without braces.
The right column will also caution you if a property means something different
than what might normally be expected.
All single forms are Perl extensions; a few compound forms are as well, and
are noted as such.
Numbers in (parentheses) indicate the total number of Unicode code points
matched by the property. For the entries that give the longest, most
descriptive version of the property, the count is followed by a list of some
of the code points matched by it. The list includes all the matched
characters in the 0-255 range, enclosed in the familiar [brackets] the same as
a regular expression bracketed character class. Following that, the next few
higher matching ranges are also given. To avoid visual ambiguity, the SPACE
character is represented as C<\\x$space_hex>.
For emphasis, those properties that match no code points at all are listed as
well in a separate section following the table.
Most properties match the same code points regardless of whether C<"/i">
case-insensitive matching is specified or not. But a few properties are
affected. These are shown with the notation S<C<(/i= I<other_property>)>>
in the second column. Under case-insensitive matching they match the
same code pode points as the property I<other_property>.
There is no description given for most non-Perl defined properties (See
L<$unicode_reference_url> for that).
For compactness, 'B<*>' is used as a wildcard instead of showing all possible
combinations. For example, entries like:
\\p{Gc: *} \\p{General_Category: *}
mean that 'Gc' is a synonym for 'General_Category', and anything that is valid
for the latter is also valid for the former. Similarly,
\\p{Is_*} \\p{*}
means that if and only if, for example, C<\\p{Foo}> exists, then
C<\\p{Is_Foo}> and C<\\p{IsFoo}> are also valid and all mean the same thing.
And similarly, C<\\p{Foo=Bar}> means the same as C<\\p{Is_Foo=Bar}> and
C<\\p{IsFoo=Bar}>. "*" here is restricted to something not beginning with an
underscore.
Also, in binary properties, 'Yes', 'T', and 'True' are all synonyms for 'Y'.
And 'No', 'F', and 'False' are all synonyms for 'N'. The table shows 'Y*' and
'N*' to indicate this, and doesn't have separate entries for the other
possibilities. Note that not all properties which have values 'Yes' and 'No'
are binary, and they have all their values spelled out without using this wild
card, and a C<NOT> clause in their description that highlights their not being
binary. These also require the compound form to match them, whereas true
binary properties have both single and compound forms available.
Note that all non-essential underscores are removed in the display of the
short names below.
B<Legend summary:>
=over 4
=item Z<>B<*> is a wild-card
=item B<(\\d+)> in the info column gives the number of Unicode code points matched
by this property.
=item B<$DEPRECATED> means this is deprecated.
=item B<$OBSOLETE> means this is obsolete.
=item B<$STABILIZED> means this is stabilized.
=item B<$STRICTER> means tighter (stricter) name matching applies.
=item B<$DISCOURAGED> means use of this form is discouraged, and may not be
stable.
=back
$formatted_properties
$zero_matches
=head1 Properties accessible through Unicode::UCD
The value of any Unicode (not including Perl extensions) character
property mentioned above for any single code point is available through
L<Unicode::UCD/charprop()>. L<Unicode::UCD/charprops_all()> returns the
values of all the Unicode properties for a given code point.
Besides these, all the Unicode character properties mentioned above
(except for those marked as for internal use by Perl) are also
accessible by L<Unicode::UCD/prop_invlist()>.
Due to their nature, not all Unicode character properties are suitable for
regular expression matches, nor C<prop_invlist()>. The remaining
non-provisional, non-internal ones are accessible via
L<Unicode::UCD/prop_invmap()> (except for those that this Perl installation
hasn't included; see L<below for which those are|/Unicode character properties
that are NOT accepted by Perl>).
For compatibility with other parts of Perl, all the single forms given in the
table in the L<section above|/Properties accessible through \\p{} and \\P{}>
are recognized. BUT, there are some ambiguities between some Perl extensions
and the Unicode properties, all of which are silently resolved in favor of the
official Unicode property. To avoid surprises, you should only use
C<prop_invmap()> for forms listed in the table below, which omits the
non-recommended ones. The affected forms are the Perl single form equivalents
of Unicode properties, such as C<\\p{sc}> being a single-form equivalent of
C<\\p{gc=sc}>, which is treated by C<prop_invmap()> as the C<Script> property,
whose short name is C<sc>. The table indicates the current ambiguities in the
INFO column, beginning with the word C<"NOT">.
The standard Unicode properties listed below are documented in
L<$unicode_reference_url>; Perl_Decimal_Digit is documented in
L<Unicode::UCD/prop_invmap()>. The other Perl extensions are in
L<perlunicode/Other Properties>;
The first column in the table is a name for the property; the second column is
an alternative name, if any, plus possibly some annotations. The alternative
name is the property's full name, unless that would simply repeat the first
column, in which case the second column indicates the property's short name
(if different). The annotations are given only in the entry for the full
name. The annotations for binary properties include a list of the first few
ranges that the property matches. To avoid any ambiguity, the SPACE character
is represented as C<\\x$space_hex>.
If a property is obsolete, etc, the entry will be flagged with the same
characters used in the table in the L<section above|/Properties accessible
through \\p{} and \\P{}>, like B<$DEPRECATED> or B<$STABILIZED>.
$ucd_pod
=head1 Properties accessible through other means
Certain properties are accessible also via core function calls. These are:
Lowercase_Mapping lc() and lcfirst()
Titlecase_Mapping ucfirst()
Uppercase_Mapping uc()
Also, Case_Folding is accessible through the C</i> modifier in regular
expressions, the C<\\F> transliteration escape, and the C<L<fc|perlfunc/fc>>
operator.
And, the Name and Name_Aliases properties are accessible through the C<\\N{}>
interpolation in double-quoted strings and regular expressions; and functions
C<charnames::viacode()>, C<charnames::vianame()>, and
C<charnames::string_vianame()> (which require a C<use charnames ();> to be
specified.
Finally, most properties related to decomposition are accessible via
L<Unicode::Normalize>.
=head1 Unicode character properties that are NOT accepted by Perl
Perl will generate an error for a few character properties in Unicode when
used in a regular expression. The non-Unihan ones are listed below, with the
reasons they are not accepted, perhaps with work-arounds. The short names for
the properties are listed enclosed in (parentheses).
As described after the list, an installation can change the defaults and choose
to accept any of these. The list is machine generated based on the
choices made for the installation that generated this document.
@bad_re_properties
An installation can choose to allow any of these to be matched by downloading
the Unicode database from L<http://www.unicode.org/Public/> to
C<\$Config{privlib}>/F<unicore/> in the Perl source tree, changing the
controlling lists contained in the program
C<\$Config{privlib}>/F<unicore/mktables> and then re-compiling and installing.
(C<\%Config> is available from the Config module).
Also, perl can be recompiled to operate on an earlier version of the Unicode
standard. Further information is at
C<\$Config{privlib}>/F<unicore/README.perl>.
=head1 Other information in the Unicode data base
The Unicode data base is delivered in two different formats. The XML version
is valid for more modern Unicode releases. The other version is a collection
of files. The two are intended to give equivalent information. Perl uses the
older form; this allows you to recompile Perl to use early Unicode releases.
The only non-character property that Perl currently supports is Named
Sequences, in which a sequence of code points
is given a name and generally treated as a single entity. (Perl supports
these via the C<\\N{...}> double-quotish construct,
L<charnames/charnames::string_vianame(name)>, and L<Unicode::UCD/namedseq()>.
Below is a list of the files in the Unicode data base that Perl doesn't
currently use, along with very brief descriptions of their purposes.
Some of the names of the files have been shortened from those that Unicode
uses, in order to allow them to be distinguishable from similarly named files
on file systems for which only the first 8 characters of a name are
significant.
=over 4
@unused_files
=back
=head1 SEE ALSO
L<$unicode_reference_url>
L<perlrecharclass>
L<perlunicode>
END
# And write it. The 0 means no utf8.
main::write([ $pod_directory, "$pod_file.pod" ], 0, \@OUT);
return;
}
sub make_Heavy () {
# Create and write Heavy.pl, which passes info about the tables to
# utf8_heavy.pl
# Stringify structures for output
my $loose_property_name_of
= simple_dumper(\%loose_property_name_of, ' ' x 4);
chomp $loose_property_name_of;
my $strict_property_name_of
= simple_dumper(\%strict_property_name_of, ' ' x 4);
chomp $strict_property_name_of;
my $stricter_to_file_of = simple_dumper(\%stricter_to_file_of, ' ' x 4);
chomp $stricter_to_file_of;
my $inline_definitions = simple_dumper(\@inline_definitions, " " x 4);
chomp $inline_definitions;
my $loose_to_file_of = simple_dumper(\%loose_to_file_of, ' ' x 4);
chomp $loose_to_file_of;
my $nv_floating_to_rational
= simple_dumper(\%nv_floating_to_rational, ' ' x 4);
chomp $nv_floating_to_rational;
my $why_deprecated = simple_dumper(\%utf8::why_deprecated, ' ' x 4);
chomp $why_deprecated;
# We set the key to the file when we associated files with tables, but we
# couldn't do the same for the value then, as we might not have the file
# for the alternate table figured out at that time.
foreach my $cased (keys %caseless_equivalent_to) {
my @path = $caseless_equivalent_to{$cased}->file_path;
my $path;
if ($path[0] eq "#") { # Pseudo-directory '#'
$path = join '/', @path;
}
else { # Gets rid of lib/
$path = join '/', @path[1, -1];
}
$caseless_equivalent_to{$cased} = $path;
}
my $caseless_equivalent_to
= simple_dumper(\%caseless_equivalent_to, ' ' x 4);
chomp $caseless_equivalent_to;
my $loose_property_to_file_of
= simple_dumper(\%loose_property_to_file_of, ' ' x 4);
chomp $loose_property_to_file_of;
my $strict_property_to_file_of
= simple_dumper(\%strict_property_to_file_of, ' ' x 4);
chomp $strict_property_to_file_of;
my $file_to_swash_name = simple_dumper(\%file_to_swash_name, ' ' x 4);
chomp $file_to_swash_name;
my @heavy = <<END;
$HEADER
$INTERNAL_ONLY_HEADER
# This file is for the use of utf8_heavy.pl and Unicode::UCD
# Maps Unicode (not Perl single-form extensions) property names in loose
# standard form to their corresponding standard names
\%utf8::loose_property_name_of = (
$loose_property_name_of
);
# Same, but strict names
\%utf8::strict_property_name_of = (
$strict_property_name_of
);
# Gives the definitions (in the form of inversion lists) for those properties
# whose definitions aren't kept in files
\@utf8::inline_definitions = (
$inline_definitions
);
# Maps property, table to file for those using stricter matching. For paths
# whose directory is '#', the file is in the form of a numeric index into
# \@inline_definitions
\%utf8::stricter_to_file_of = (
$stricter_to_file_of
);
# Maps property, table to file for those using loose matching. For paths
# whose directory is '#', the file is in the form of a numeric index into
# \@inline_definitions
\%utf8::loose_to_file_of = (
$loose_to_file_of
);
# Maps floating point to fractional form
\%utf8::nv_floating_to_rational = (
$nv_floating_to_rational
);
# If a %e floating point number doesn't have this number of digits in it after
# the decimal point to get this close to a fraction, it isn't considered to be
# that fraction even if all the digits it does have match.
\$utf8::e_precision = $E_FLOAT_PRECISION;
# Deprecated tables to generate a warning for. The key is the file containing
# the table, so as to avoid duplication, as many property names can map to the
# file, but we only need one entry for all of them.
\%utf8::why_deprecated = (
$why_deprecated
);
# A few properties have different behavior under /i matching. This maps
# those to substitute files to use under /i.
\%utf8::caseless_equivalent = (
$caseless_equivalent_to
);
# Property names to mapping files
\%utf8::loose_property_to_file_of = (
$loose_property_to_file_of
);
# Property names to mapping files
\%utf8::strict_property_to_file_of = (
$strict_property_to_file_of
);
# Files to the swash names within them.
\%utf8::file_to_swash_name = (
$file_to_swash_name
);
1;
END
main::write("Heavy.pl", 0, \@heavy); # The 0 means no utf8.
return;
}
sub make_Name_pm () {
# Create and write Name.pm, which contains subroutines and data to use in
# conjunction with Name.pl
# Maybe there's nothing to do.
return unless $has_hangul_syllables || @code_points_ending_in_code_point;
my @name = <<END;
$HEADER
$INTERNAL_ONLY_HEADER
END
# Convert these structures to output format.
my $code_points_ending_in_code_point =
main::simple_dumper(\@code_points_ending_in_code_point,
' ' x 8);
my $names = main::simple_dumper(\%names_ending_in_code_point,
' ' x 8);
my $loose_names = main::simple_dumper(\%loose_names_ending_in_code_point,
' ' x 8);
# Do the same with the Hangul names,
my $jamo;
my $jamo_l;
my $jamo_v;
my $jamo_t;
my $jamo_re;
if ($has_hangul_syllables) {
# Construct a regular expression of all the possible
# combinations of the Hangul syllables.
my @L_re; # Leading consonants
for my $i ($LBase .. $LBase + $LCount - 1) {
push @L_re, $Jamo{$i}
}
my @V_re; # Middle vowels
for my $i ($VBase .. $VBase + $VCount - 1) {
push @V_re, $Jamo{$i}
}
my @T_re; # Trailing consonants
for my $i ($TBase + 1 .. $TBase + $TCount - 1) {
push @T_re, $Jamo{$i}
}
# The whole re is made up of the L V T combination.
$jamo_re = '('
. join ('|', sort @L_re)
. ')('
. join ('|', sort @V_re)
. ')('
. join ('|', sort @T_re)
. ')?';
# These hashes needed by the algorithm were generated
# during reading of the Jamo.txt file
$jamo = main::simple_dumper(\%Jamo, ' ' x 8);
$jamo_l = main::simple_dumper(\%Jamo_L, ' ' x 8);
$jamo_v = main::simple_dumper(\%Jamo_V, ' ' x 8);
$jamo_t = main::simple_dumper(\%Jamo_T, ' ' x 8);
}
push @name, <<END;
package charnames;
# This module contains machine-generated tables and code for the
# algorithmically-determinable Unicode character names. The following
# routines can be used to translate between name and code point and vice versa
{ # Closure
# Matches legal code point. 4-6 hex numbers, If there are 6, the first
# two must be 10; if there are 5, the first must not be a 0. Written this
# way to decrease backtracking. The first regex allows the code point to
# be at the end of a word, but to work properly, the word shouldn't end
# with a valid hex character. The second one won't match a code point at
# the end of a word, and doesn't have the run-on issue
my \$run_on_code_point_re = qr/$run_on_code_point_re/;
my \$code_point_re = qr/$code_point_re/;
# In the following hash, the keys are the bases of names which include
# the code point in the name, like CJK UNIFIED IDEOGRAPH-4E01. The value
# of each key is another hash which is used to get the low and high ends
# for each range of code points that apply to the name.
my %names_ending_in_code_point = (
$names
);
# The following hash is a copy of the previous one, except is for loose
# matching, so each name has blanks and dashes squeezed out
my %loose_names_ending_in_code_point = (
$loose_names
);
# And the following array gives the inverse mapping from code points to
# names. Lowest code points are first
my \@code_points_ending_in_code_point = (
$code_points_ending_in_code_point
);
END
# Earlier releases didn't have Jamos. No sense outputting
# them unless will be used.
if ($has_hangul_syllables) {
push @name, <<END;
# Convert from code point to Jamo short name for use in composing Hangul
# syllable names
my %Jamo = (
$jamo
);
# Leading consonant (can be null)
my %Jamo_L = (
$jamo_l
);
# Vowel
my %Jamo_V = (
$jamo_v
);
# Optional trailing consonant
my %Jamo_T = (
$jamo_t
);
# Computed re that splits up a Hangul name into LVT or LV syllables
my \$syllable_re = qr/$jamo_re/;
my \$HANGUL_SYLLABLE = "HANGUL SYLLABLE ";
my \$loose_HANGUL_SYLLABLE = "HANGULSYLLABLE";
# These constants names and values were taken from the Unicode standard,
# version 5.1, section 3.12. They are used in conjunction with Hangul
# syllables
my \$SBase = $SBase_string;
my \$LBase = $LBase_string;
my \$VBase = $VBase_string;
my \$TBase = $TBase_string;
my \$SCount = $SCount;
my \$LCount = $LCount;
my \$VCount = $VCount;
my \$TCount = $TCount;
my \$NCount = \$VCount * \$TCount;
END
} # End of has Jamos
push @name, << 'END';
sub name_to_code_point_special {
my ($name, $loose) = @_;
# Returns undef if not one of the specially handled names; otherwise
# returns the code point equivalent to the input name
# $loose is non-zero if to use loose matching, 'name' in that case
# must be input as upper case with all blanks and dashes squeezed out.
END
if ($has_hangul_syllables) {
push @name, << 'END';
if ((! $loose && $name =~ s/$HANGUL_SYLLABLE//)
|| ($loose && $name =~ s/$loose_HANGUL_SYLLABLE//))
{
return if $name !~ qr/^$syllable_re$/;
my $L = $Jamo_L{$1};
my $V = $Jamo_V{$2};
my $T = (defined $3) ? $Jamo_T{$3} : 0;
return ($L * $VCount + $V) * $TCount + $T + $SBase;
}
END
}
push @name, << 'END';
# Name must end in 'code_point' for this to handle.
return if (($loose && $name !~ /^ (.*?) ($run_on_code_point_re) $/x)
|| (! $loose && $name !~ /^ (.*) ($code_point_re) $/x));
my $base = $1;
my $code_point = CORE::hex $2;
my $names_ref;
if ($loose) {
$names_ref = \%loose_names_ending_in_code_point;
}
else {
return if $base !~ s/-$//;
$names_ref = \%names_ending_in_code_point;
}
# Name must be one of the ones which has the code point in it.
return if ! $names_ref->{$base};
# Look through the list of ranges that apply to this name to see if
# the code point is in one of them.
for (my $i = 0; $i < scalar @{$names_ref->{$base}{'low'}}; $i++) {
return if $names_ref->{$base}{'low'}->[$i] > $code_point;
next if $names_ref->{$base}{'high'}->[$i] < $code_point;
# Here, the code point is in the range.
return $code_point;
}
# Here, looked like the name had a code point number in it, but
# did not match one of the valid ones.
return;
}
sub code_point_to_name_special {
my $code_point = shift;
# Returns the name of a code point if algorithmically determinable;
# undef if not
END
if ($has_hangul_syllables) {
push @name, << 'END';
# If in the Hangul range, calculate the name based on Unicode's
# algorithm
if ($code_point >= $SBase && $code_point <= $SBase + $SCount -1) {
use integer;
my $SIndex = $code_point - $SBase;
my $L = $LBase + $SIndex / $NCount;
my $V = $VBase + ($SIndex % $NCount) / $TCount;
my $T = $TBase + $SIndex % $TCount;
$name = "$HANGUL_SYLLABLE$Jamo{$L}$Jamo{$V}";
$name .= $Jamo{$T} if $T != $TBase;
return $name;
}
END
}
push @name, << 'END';
# Look through list of these code points for one in range.
foreach my $hash (@code_points_ending_in_code_point) {
return if $code_point < $hash->{'low'};
if ($code_point <= $hash->{'high'}) {
return sprintf("%s-%04X", $hash->{'name'}, $code_point);
}
}
return; # None found
}
} # End closure
1;
END
main::write("Name.pm", 0, \@name); # The 0 means no utf8.
return;
}
sub make_UCD () {
# Create and write UCD.pl, which passes info about the tables to
# Unicode::UCD
# Create a mapping from each alias of Perl single-form extensions to all
# its equivalent aliases, for quick look-up.
my %perlprop_to_aliases;
foreach my $table ($perl->tables) {
# First create the list of the aliases of each extension
my @aliases_list; # List of legal aliases for this extension
my $table_name = $table->name;
my $standard_table_name = standardize($table_name);
my $table_full_name = $table->full_name;
my $standard_table_full_name = standardize($table_full_name);
# Make sure that the list has both the short and full names
push @aliases_list, $table_name, $table_full_name;
my $found_ucd = 0; # ? Did we actually get an alias that should be
# output for this table
# Go through all the aliases (including the two just added), and add
# any new unique ones to the list
foreach my $alias ($table->aliases) {
# Skip non-legal names
next unless $alias->ok_as_filename;
next unless $alias->ucd;
$found_ucd = 1; # have at least one legal name
my $name = $alias->name;
my $standard = standardize($name);
# Don't repeat a name that is equivalent to one already on the
# list
next if $standard eq $standard_table_name;
next if $standard eq $standard_table_full_name;
push @aliases_list, $name;
}
# If there were no legal names, don't output anything.
next unless $found_ucd;
# To conserve memory in the program reading these in, omit full names
# that are identical to the short name, when those are the only two
# aliases for the property.
if (@aliases_list == 2 && $aliases_list[0] eq $aliases_list[1]) {
pop @aliases_list;
}
# Here, @aliases_list is the list of all the aliases that this
# extension legally has. Now can create a map to it from each legal
# standardized alias
foreach my $alias ($table->aliases) {
next unless $alias->ucd;
next unless $alias->ok_as_filename;
push @{$perlprop_to_aliases{standardize($alias->name)}},
uniques @aliases_list;
}
}
# Make a list of all combinations of properties/values that are suppressed.
my @suppressed;
if (! $debug_skip) { # This tends to fail in this debug mode
foreach my $property_name (keys %why_suppressed) {
# Just the value
my $value_name = $1 if $property_name =~ s/ = ( .* ) //x;
# The hash may contain properties not in this release of Unicode
next unless defined (my $property = property_ref($property_name));
# Find all combinations
foreach my $prop_alias ($property->aliases) {
my $prop_alias_name = standardize($prop_alias->name);
# If no =value, there's just one combination possible for this
if (! $value_name) {
# The property may be suppressed, but there may be a proxy
# for it, so it shouldn't be listed as suppressed
next if $prop_alias->ucd;
push @suppressed, $prop_alias_name;
}
else { # Otherwise
foreach my $value_alias
($property->table($value_name)->aliases)
{
next if $value_alias->ucd;
push @suppressed, "$prop_alias_name="
. standardize($value_alias->name);
}
}
}
}
}
@suppressed = sort @suppressed; # So doesn't change between runs of this
# program
# Convert the structure below (designed for Name.pm) to a form that UCD
# wants, so it doesn't have to modify it at all; i.e. so that it includes
# an element for the Hangul syllables in the appropriate place, and
# otherwise changes the name to include the "-<code point>" suffix.
my @algorithm_names;
my $done_hangul = $v_version lt v2.0.0; # Hanguls as we know them came
# along in this version
# Copy it linearly.
for my $i (0 .. @code_points_ending_in_code_point - 1) {
# Insert the hanguls in the correct place.
if (! $done_hangul
&& $code_points_ending_in_code_point[$i]->{'low'} > $SBase)
{
$done_hangul = 1;
push @algorithm_names, { low => $SBase,
high => $SBase + $SCount - 1,
name => '<hangul syllable>',
};
}
# Copy the current entry, modified.
push @algorithm_names, {
low => $code_points_ending_in_code_point[$i]->{'low'},
high => $code_points_ending_in_code_point[$i]->{'high'},
name =>
"$code_points_ending_in_code_point[$i]->{'name'}-<code point>",
};
}
# Serialize these structures for output.
my $loose_to_standard_value
= simple_dumper(\%loose_to_standard_value, ' ' x 4);
chomp $loose_to_standard_value;
my $string_property_loose_to_name
= simple_dumper(\%string_property_loose_to_name, ' ' x 4);
chomp $string_property_loose_to_name;
my $perlprop_to_aliases = simple_dumper(\%perlprop_to_aliases, ' ' x 4);
chomp $perlprop_to_aliases;
my $prop_aliases = simple_dumper(\%prop_aliases, ' ' x 4);
chomp $prop_aliases;
my $prop_value_aliases = simple_dumper(\%prop_value_aliases, ' ' x 4);
chomp $prop_value_aliases;
my $suppressed = (@suppressed) ? simple_dumper(\@suppressed, ' ' x 4) : "";
chomp $suppressed;
my $algorithm_names = simple_dumper(\@algorithm_names, ' ' x 4);
chomp $algorithm_names;
my $ambiguous_names = simple_dumper(\%ambiguous_names, ' ' x 4);
chomp $ambiguous_names;
my $combination_property = simple_dumper(\%combination_property, ' ' x 4);
chomp $combination_property;
my $loose_defaults = simple_dumper(\%loose_defaults, ' ' x 4);
chomp $loose_defaults;
my @ucd = <<END;
$HEADER
$INTERNAL_ONLY_HEADER
# This file is for the use of Unicode::UCD
# Highest legal Unicode code point
\$Unicode::UCD::MAX_UNICODE_CODEPOINT = 0x$MAX_UNICODE_CODEPOINT_STRING;
# Hangul syllables
\$Unicode::UCD::HANGUL_BEGIN = $SBase_string;
\$Unicode::UCD::HANGUL_COUNT = $SCount;
# Keys are all the possible "prop=value" combinations, in loose form; values
# are the standard loose name for the 'value' part of the key
\%Unicode::UCD::loose_to_standard_value = (
$loose_to_standard_value
);
# String property loose names to standard loose name
\%Unicode::UCD::string_property_loose_to_name = (
$string_property_loose_to_name
);
# Keys are Perl extensions in loose form; values are each one's list of
# aliases
\%Unicode::UCD::loose_perlprop_to_name = (
$perlprop_to_aliases
);
# Keys are standard property name; values are each one's aliases
\%Unicode::UCD::prop_aliases = (
$prop_aliases
);
# Keys of top level are standard property name; values are keys to another
# hash, Each one is one of the property's values, in standard form. The
# values are that prop-val's aliases. If only one specified, the short and
# long alias are identical.
\%Unicode::UCD::prop_value_aliases = (
$prop_value_aliases
);
# Ordered (by code point ordinal) list of the ranges of code points whose
# names are algorithmically determined. Each range entry is an anonymous hash
# of the start and end points and a template for the names within it.
\@Unicode::UCD::algorithmic_named_code_points = (
$algorithm_names
);
# The properties that as-is have two meanings, and which must be disambiguated
\%Unicode::UCD::ambiguous_names = (
$ambiguous_names
);
# Keys are the prop-val combinations which are the default values for the
# given property, expressed in standard loose form
\%Unicode::UCD::loose_defaults = (
$loose_defaults
);
# The properties that are combinations, in that they have both a map table and
# a match table. This is actually for UCD.t, so it knows how to test for
# these.
\%Unicode::UCD::combination_property = (
$combination_property
);
# All combinations of names that are suppressed.
# This is actually for UCD.t, so it knows which properties shouldn't have
# entries. If it got any bigger, would probably want to put it in its own
# file to use memory only when it was needed, in testing.
\@Unicode::UCD::suppressed_properties = (
$suppressed
);
1;
END
main::write("UCD.pl", 0, \@ucd); # The 0 means no utf8.
return;
}
sub write_all_tables() {
# Write out all the tables generated by this program to files, as well as
# the supporting data structures, pod file, and .t file.
my @writables; # List of tables that actually get written
my %match_tables_to_write; # Used to collapse identical match tables
# into one file. Each key is a hash function
# result to partition tables into buckets.
# Each value is an array of the tables that
# fit in the bucket.
# For each property ...
# (sort so that if there is an immutable file name, it has precedence, so
# some other property can't come in and take over its file name. (We
# don't care if both defined, as they had better be different anyway.)
# The property named 'Perl' needs to be first (it doesn't have any
# immutable file name) because empty properties are defined in terms of
# its table named 'All' under the -annotate option.) We also sort by
# the property's name. This is just for repeatability of the outputs
# between runs of this program, but does not affect correctness.
PROPERTY:
foreach my $property ($perl,
sort { return -1 if defined $a->file;
return 1 if defined $b->file;
return $a->name cmp $b->name;
} grep { $_ != $perl } property_ref('*'))
{
my $type = $property->type;
# And for each table for that property, starting with the mapping
# table for it ...
TABLE:
foreach my $table($property,
# and all the match tables for it (if any), sorted so
# the ones with the shortest associated file name come
# first. The length sorting prevents problems of a
# longer file taking a name that might have to be used
# by a shorter one. The alphabetic sorting prevents
# differences between releases
sort { my $ext_a = $a->external_name;
return 1 if ! defined $ext_a;
my $ext_b = $b->external_name;
return -1 if ! defined $ext_b;
# But return the non-complement table before
# the complement one, as the latter is defined
# in terms of the former, and needs to have
# the information for the former available.
return 1 if $a->complement != 0;
return -1 if $b->complement != 0;
# Similarly, return a subservient table after
# a leader
return 1 if $a->leader != $a;
return -1 if $b->leader != $b;
my $cmp = length $ext_a <=> length $ext_b;
# Return result if lengths not equal
return $cmp if $cmp;
# Alphabetic if lengths equal
return $ext_a cmp $ext_b
} $property->tables
)
{
# Here we have a table associated with a property. It could be
# the map table (done first for each property), or one of the
# other tables. Determine which type.
my $is_property = $table->isa('Property');
my $name = $table->name;
my $complete_name = $table->complete_name;
# See if should suppress the table if is empty, but warn if it
# contains something.
my $suppress_if_empty_warn_if_not
= $why_suppress_if_empty_warn_if_not{$complete_name} || 0;
# Calculate if this table should have any code points associated
# with it or not.
my $expected_empty =
# $perl should be empty
($is_property && ($table == $perl))
# Match tables in properties we skipped populating should be
# empty
|| (! $is_property && ! $property->to_create_match_tables)
# Tables and properties that are expected to have no code
# points should be empty
|| $suppress_if_empty_warn_if_not
;
# Set a boolean if this table is the complement of an empty binary
# table
my $is_complement_of_empty_binary =
$type == $BINARY &&
(($table == $property->table('Y')
&& $property->table('N')->is_empty)
|| ($table == $property->table('N')
&& $property->table('Y')->is_empty));
if ($table->is_empty) {
if ($suppress_if_empty_warn_if_not) {
$table->set_fate($SUPPRESSED,
$suppress_if_empty_warn_if_not);
}
# Suppress (by skipping them) expected empty tables.
next TABLE if $expected_empty;
# And setup to later output a warning for those that aren't
# known to be allowed to be empty. Don't do the warning if
# this table is a child of another one to avoid duplicating
# the warning that should come from the parent one.
if (($table == $property || $table->parent == $table)
&& $table->fate != $SUPPRESSED
&& $table->fate != $MAP_PROXIED
&& ! grep { $complete_name =~ /^$_$/ }
@tables_that_may_be_empty)
{
push @unhandled_properties, "$table";
}
# The old way of expressing an empty match list was to
# complement the list that matches everything. The new way is
# to create an empty inversion list, but this doesn't work for
# annotating, so use the old way then.
$table->set_complement($All) if $annotate
&& $table != $property;
}
elsif ($expected_empty) {
my $because = "";
if ($suppress_if_empty_warn_if_not) {
$because = " because $suppress_if_empty_warn_if_not";
}
Carp::my_carp("Not expecting property $table$because. Generating file for it anyway.");
}
# Some tables should match everything
my $expected_full =
($table->fate == $SUPPRESSED)
? 0
: ($is_property)
? # All these types of map tables will be full because
# they will have been populated with defaults
($type == $ENUM)
: # A match table should match everything if its method
# shows it should
($table->matches_all
# The complement of an empty binary table will match
# everything
|| $is_complement_of_empty_binary
)
;
my $count = $table->count;
if ($expected_full) {
if ($count != $MAX_WORKING_CODEPOINTS) {
Carp::my_carp("$table matches only "
. clarify_number($count)
. " Unicode code points but should match "
. clarify_number($MAX_WORKING_CODEPOINTS)
. " (off by "
. clarify_number(abs($MAX_WORKING_CODEPOINTS - $count))
. "). Proceeding anyway.");
}
# Here is expected to be full. If it is because it is the
# complement of an (empty) binary table that is to be
# suppressed, then suppress this one as well.
if ($is_complement_of_empty_binary) {
my $opposing_name = ($name eq 'Y') ? 'N' : 'Y';
my $opposing = $property->table($opposing_name);
my $opposing_status = $opposing->status;
if ($opposing_status) {
$table->set_status($opposing_status,
$opposing->status_info);
}
}
}
elsif ($count == $MAX_UNICODE_CODEPOINTS
&& $name ne "Any"
&& ($table == $property || $table->leader == $table)
&& $table->property->status ne $NORMAL)
{
Carp::my_carp("$table unexpectedly matches all Unicode code points. Proceeding anyway.");
}
if ($table->fate >= $SUPPRESSED) {
if (! $is_property) {
my @children = $table->children;
foreach my $child (@children) {
if ($child->fate < $SUPPRESSED) {
Carp::my_carp_bug("'$table' is suppressed and has a child '$child' which isn't");
}
}
}
next TABLE;
}
if (! $is_property) {
make_ucd_table_pod_entries($table) if $table->property == $perl;
# Several things need to be done just once for each related
# group of match tables. Do them on the parent.
if ($table->parent == $table) {
# Add an entry in the pod file for the table; it also does
# the children.
make_re_pod_entries($table) if defined $pod_directory;
# See if the table matches identical code points with
# something that has already been processed and is ready
# for output. In that case, no need to have two files
# with the same code points in them. We use the table's
# hash() method to store these in buckets, so that it is
# quite likely that if two tables are in the same bucket
# they will be identical, so don't have to compare tables
# frequently. The tables have to have the same status to
# share a file, so add this to the bucket hash. (The
# reason for this latter is that Heavy.pl associates a
# status with a file.) We don't check tables that are
# inverses of others, as it would lead to some coding
# complications, and checking all the regular ones should
# find everything.
if ($table->complement == 0) {
my $hash = $table->hash . ';' . $table->status;
# Look at each table that is in the same bucket as
# this one would be.
foreach my $comparison
(@{$match_tables_to_write{$hash}})
{
# If the table doesn't point back to this one, we
# see if it matches identically
if ( $comparison->leader != $table
&& $table->matches_identically_to($comparison))
{
$table->set_equivalent_to($comparison,
Related => 0);
next TABLE;
}
}
# Here, not equivalent, add this table to the bucket.
push @{$match_tables_to_write{$hash}}, $table;
}
}
}
else {
# Here is the property itself.
# Don't write out or make references to the $perl property
next if $table == $perl;
make_ucd_table_pod_entries($table);
# There is a mapping stored of the various synonyms to the
# standardized name of the property for utf8_heavy.pl.
# Also, the pod file contains entries of the form:
# \p{alias: *} \p{full: *}
# rather than show every possible combination of things.
my @property_aliases = $property->aliases;
my $full_property_name = $property->full_name;
my $property_name = $property->name;
my $standard_property_name = standardize($property_name);
my $standard_property_full_name
= standardize($full_property_name);
# We also create for Unicode::UCD a list of aliases for
# the property. The list starts with the property name;
# then its full name. Legacy properties are not listed in
# Unicode::UCD.
my @property_list;
my @standard_list;
if ( $property->fate <= $MAP_PROXIED) {
@property_list = ($property_name, $full_property_name);
@standard_list = ($standard_property_name,
$standard_property_full_name);
}
# For each synonym ...
for my $i (0 .. @property_aliases - 1) {
my $alias = $property_aliases[$i];
my $alias_name = $alias->name;
my $alias_standard = standardize($alias_name);
# Add other aliases to the list of property aliases
if ($property->fate <= $MAP_PROXIED
&& ! grep { $alias_standard eq $_ } @standard_list)
{
push @property_list, $alias_name;
push @standard_list, $alias_standard;
}
# For utf8_heavy, set the mapping of the alias to the
# property
if ($type == $STRING) {
if ($property->fate <= $MAP_PROXIED) {
$string_property_loose_to_name{$alias_standard}
= $standard_property_name;
}
}
else {
my $hash_ref = ($alias_standard =~ /^_/)
? \%strict_property_name_of
: \%loose_property_name_of;
if (exists $hash_ref->{$alias_standard}) {
Carp::my_carp("There already is a property with the same standard name as $alias_name: $hash_ref->{$alias_standard}. Old name is retained");
}
else {
$hash_ref->{$alias_standard}
= $standard_property_name;
}
# Now for the re pod entry for this alias. Skip if not
# outputting a pod; skip the first one, which is the
# full name so won't have an entry like: '\p{full: *}
# \p{full: *}', and skip if don't want an entry for
# this one.
next if $i == 0
|| ! defined $pod_directory
|| ! $alias->make_re_pod_entry;
my $rhs = "\\p{$full_property_name: *}";
if ($property != $perl && $table->perl_extension) {
$rhs .= ' (Perl extension)';
}
push @match_properties,
format_pod_line($indent_info_column,
'\p{' . $alias->name . ': *}',
$rhs,
$alias->status);
}
}
# The list of all possible names is attached to each alias, so
# lookup is easy
if (@property_list) {
push @{$prop_aliases{$standard_list[0]}}, @property_list;
}
if ($property->fate <= $MAP_PROXIED) {
# Similarly, we create for Unicode::UCD a list of
# property-value aliases.
# Look at each table in the property...
foreach my $table ($property->tables) {
my @values_list;
my $table_full_name = $table->full_name;
my $standard_table_full_name
= standardize($table_full_name);
my $table_name = $table->name;
my $standard_table_name = standardize($table_name);
# The list starts with the table name and its full
# name.
push @values_list, $table_name, $table_full_name;
# We add to the table each unique alias that isn't
# discouraged from use.
foreach my $alias ($table->aliases) {
next if $alias->status
&& $alias->status eq $DISCOURAGED;
my $name = $alias->name;
my $standard = standardize($name);
next if $standard eq $standard_table_name;
next if $standard eq $standard_table_full_name;
push @values_list, $name;
}
# Here @values_list is a list of all the aliases for
# the table. That is, all the property-values given
# by this table. By agreement with Unicode::UCD,
# if the name and full name are identical, and there
# are no other names, drop the duplicate entry to save
# memory.
if (@values_list == 2
&& $values_list[0] eq $values_list[1])
{
pop @values_list
}
# To save memory, unlike the similar list for property
# aliases above, only the standard forms have the list.
# This forces an extra step of converting from input
# name to standard name, but the savings are
# considerable. (There is only marginal savings if we
# did this with the property aliases.)
push @{$prop_value_aliases{$standard_property_name}{$standard_table_name}}, @values_list;
}
}
# Don't write out a mapping file if not desired.
next if ! $property->to_output_map;
}
# Here, we know we want to write out the table, but don't do it
# yet because there may be other tables that come along and will
# want to share the file, and the file's comments will change to
# mention them. So save for later.
push @writables, $table;
} # End of looping through the property and all its tables.
} # End of looping through all properties.
# Now have all the tables that will have files written for them. Do it.
foreach my $table (@writables) {
my @directory;
my $filename;
my $property = $table->property;
my $is_property = ($table == $property);
# For very short tables, instead of writing them out to actual files,
# we in-line their inversion list definitions into Heavy.pl. The
# definition replaces the file name, and the special pseudo-directory
# '#' is used to signal this. This significantly cuts down the number
# of files written at little extra cost to the hashes in Heavy.pl.
# And it means, no run-time files to read to get the definitions.
if (! $is_property
&& ! $annotate # For annotation, we want to explicitly show
# everything, so keep in files
&& $table->ranges <= 3)
{
my @ranges = $table->ranges;
my $count = @ranges;
if ($count == 0) { # 0th index reserved for 0-length lists
$filename = 0;
}
elsif ($table->leader != $table) {
# Here, is a table that is equivalent to another; code
# in register_file_for_name() causes its leader's definition
# to be used
next;
}
else { # No equivalent table so far.
# Build up its definition range-by-range.
my $definition = "";
while (defined (my $range = shift @ranges)) {
my $end = $range->end;
if ($end < $MAX_WORKING_CODEPOINT) {
$count++;
$end = "\n" . ($end + 1);
}
else { # Extends to infinity, hence no 'end'
$end = "";
}
$definition .= "\n" . $range->start . $end;
}
$definition = "V$count" . $definition;
$filename = @inline_definitions;
push @inline_definitions, $definition;
}
@directory = "#";
register_file_for_name($table, \@directory, $filename);
next;
}
if (! $is_property) {
# Match tables for the property go in lib/$subdirectory, which is
# the property's name. Don't use the standard file name for this,
# as may get an unfamiliar alias
@directory = ($matches_directory, $property->external_name);
}
else {
@directory = $table->directory;
$filename = $table->file;
}
# Use specified filename if available, or default to property's
# shortest name. We need an 8.3 safe filename (which means "an 8
# safe" filename, since after the dot is only 'pl', which is < 3)
# The 2nd parameter is if the filename shouldn't be changed, and
# it shouldn't iff there is a hard-coded name for this table.
$filename = construct_filename(
$filename || $table->external_name,
! $filename, # mutable if no filename
\@directory);
register_file_for_name($table, \@directory, $filename);
# Only need to write one file when shared by more than one
# property
next if ! $is_property
&& ($table->leader != $table || $table->complement != 0);
# Construct a nice comment to add to the file
$table->set_final_comment;
$table->write;
}
# Write out the pod file
make_pod;
# And Heavy.pl, Name.pm, UCD.pl
make_Heavy;
make_Name_pm;
make_UCD;
make_property_test_script() if $make_test_script;
make_normalization_test_script() if $make_norm_test_script;
return;
}
my @white_space_separators = ( # This used only for making the test script.
"",
' ',
"\t",
' '
);
sub generate_separator($) {
# This used only for making the test script. It generates the colon or
# equal separator between the property and property value, with random
# white space surrounding the separator
my $lhs = shift;
return "" if $lhs eq ""; # No separator if there's only one (the r) side
# Choose space before and after randomly
my $spaces_before =$white_space_separators[rand(@white_space_separators)];
my $spaces_after = $white_space_separators[rand(@white_space_separators)];
# And return the whole complex, half the time using a colon, half the
# equals
return $spaces_before
. (rand() < 0.5) ? '=' : ':'
. $spaces_after;
}
sub generate_tests($$$$$) {
# This used only for making the test script. It generates test cases that
# are expected to compile successfully in perl. Note that the LHS and
# RHS are assumed to already be as randomized as the caller wants.
my $lhs = shift; # The property: what's to the left of the colon
# or equals separator
my $rhs = shift; # The property value; what's to the right
my $valid_code = shift; # A code point that's known to be in the
# table given by LHS=RHS; undef if table is
# empty
my $invalid_code = shift; # A code point known to not be in the table;
# undef if the table is all code points
my $warning = shift;
# Get the colon or equal
my $separator = generate_separator($lhs);
# The whole 'property=value'
my $name = "$lhs$separator$rhs";
my @output;
# Create a complete set of tests, with complements.
if (defined $valid_code) {
push @output, <<"EOC"
Expect(1, $valid_code, '\\p{$name}', $warning);
Expect(0, $valid_code, '\\p{^$name}', $warning);
Expect(0, $valid_code, '\\P{$name}', $warning);
Expect(1, $valid_code, '\\P{^$name}', $warning);
EOC
}
if (defined $invalid_code) {
push @output, <<"EOC"
Expect(0, $invalid_code, '\\p{$name}', $warning);
Expect(1, $invalid_code, '\\p{^$name}', $warning);
Expect(1, $invalid_code, '\\P{$name}', $warning);
Expect(0, $invalid_code, '\\P{^$name}', $warning);
EOC
}
return @output;
}
sub generate_wildcard_tests($$$$$) {
# This used only for making the test script. It generates wildcardl
# matching test cases that are expected to compile successfully in perl.
my $lhs = shift; # The property: what's to the left of the
# or equals separator
my $rhs = shift; # The property value; what's to the right
my $valid_code = shift; # A code point that's known to be in the
# table given by LHS=RHS; undef if table is
# empty
my $invalid_code = shift; # A code point known to not be in the table;
# undef if the table is all code points
my $warning = shift;
return if $lhs eq "";
return if $lhs =~ / ^ Is_ /x; # These are not currently supported
# Generate a standardized pattern, with colon being the delimitter
my $wildcard = "$lhs=:\\A$rhs\\z:";
my @output;
push @output, "Expect(1, $valid_code, '\\p{$wildcard}', $warning);"
if defined $valid_code;
push @output, "Expect(0, $invalid_code, '\\p{$wildcard}', $warning);"
if defined $invalid_code;
return @output;
}
sub generate_error($$$) {
# This used only for making the test script. It generates test cases that
# are expected to not only not match, but to be syntax or similar errors
my $lhs = shift; # The property: what's to the left of the
# colon or equals separator
my $rhs = shift; # The property value; what's to the right
my $already_in_error = shift; # Boolean; if true it's known that the
# unmodified LHS and RHS will cause an error.
# This routine should not force another one
# Get the colon or equal
my $separator = generate_separator($lhs);
# Since this is an error only, don't bother to randomly decide whether to
# put the error on the left or right side; and assume that the RHS is
# loosely matched, again for convenience rather than rigor.
$rhs = randomize_loose_name($rhs, 'ERROR') unless $already_in_error;
my $property = $lhs . $separator . $rhs;
return <<"EOC";
Error('\\p{$property}');
Error('\\P{$property}');
EOC
}
# These are used only for making the test script
# XXX Maybe should also have a bad strict seps, which includes underscore.
my @good_loose_seps = (
" ",
"-",
"\t",
"",
"_",
);
my @bad_loose_seps = (
"/a/",
':=',
);
sub randomize_stricter_name {
# This used only for making the test script. Take the input name and
# return a randomized, but valid version of it under the stricter matching
# rules.
my $name = shift;
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
# If the name looks like a number (integer, floating, or rational), do
# some extra work
if ($name =~ qr{ ^ ( -? ) (\d+ ( ( [./] ) \d+ )? ) $ }x) {
my $sign = $1;
my $number = $2;
my $separator = $3;
# If there isn't a sign, part of the time add a plus
# Note: Not testing having any denominator having a minus sign
if (! $sign) {
$sign = '+' if rand() <= .3;
}
# And add 0 or more leading zeros.
$name = $sign . ('0' x int rand(10)) . $number;
if (defined $separator) {
my $extra_zeros = '0' x int rand(10);
if ($separator eq '.') {
# Similarly, add 0 or more trailing zeros after a decimal
# point
$name .= $extra_zeros;
}
else {
# Or, leading zeros before the denominator
$name =~ s,/,/$extra_zeros,;
}
}
}
# For legibility of the test, only change the case of whole sections at a
# time. To do this, first split into sections. The split returns the
# delimiters
my @sections;
for my $section (split / ( [ - + \s _ . ]+ ) /x, $name) {
trace $section if main::DEBUG && $to_trace;
if (length $section > 1 && $section !~ /\D/) {
# If the section is a sequence of digits, about half the time
# randomly add underscores between some of them.
if (rand() > .5) {
# Figure out how many underscores to add. max is 1 less than
# the number of digits. (But add 1 at the end to make sure
# result isn't 0, and compensate earlier by subtracting 2
# instead of 1)
my $num_underscores = int rand(length($section) - 2) + 1;
# And add them evenly throughout, for convenience, not rigor
use integer;
my $spacing = (length($section) - 1)/ $num_underscores;
my $temp = $section;
$section = "";
for my $i (1 .. $num_underscores) {
$section .= substr($temp, 0, $spacing, "") . '_';
}
$section .= $temp;
}
push @sections, $section;
}
else {
# Here not a sequence of digits. Change the case of the section
# randomly
my $switch = int rand(4);
if ($switch == 0) {
push @sections, uc $section;
}
elsif ($switch == 1) {
push @sections, lc $section;
}
elsif ($switch == 2) {
push @sections, ucfirst $section;
}
else {
push @sections, $section;
}
}
}
trace "returning", join "", @sections if main::DEBUG && $to_trace;
return join "", @sections;
}
sub randomize_loose_name($;$) {
# This used only for making the test script
my $name = shift;
my $want_error = shift; # if true, make an error
Carp::carp_extra_args(\@_) if main::DEBUG && @_;
$name = randomize_stricter_name($name);
my @parts;
push @parts, $good_loose_seps[rand(@good_loose_seps)];
# Preserve trailing ones for the sake of not stripping the underscore from
# 'L_'
for my $part (split /[-\s_]+ (?= . )/, $name) {
if (@parts) {
if ($want_error and rand() < 0.3) {
push @parts, $bad_loose_seps[rand(@bad_loose_seps)];
$want_error = 0;
}
else {
push @parts, $good_loose_seps[rand(@good_loose_seps)];
}
}
push @parts, $part;
}
my $new = join("", @parts);
trace "$name => $new" if main::DEBUG && $to_trace;
if ($want_error) {
if (rand() >= 0.5) {
$new .= $bad_loose_seps[rand(@bad_loose_seps)];
}
else {
$new = $bad_loose_seps[rand(@bad_loose_seps)] . $new;
}
}
return $new;
}
# Used to make sure don't generate duplicate test cases.
my %test_generated;
sub make_property_test_script() {
# This used only for making the test script
# this written directly -- it's huge.
print "Making test script\n" if $verbosity >= $PROGRESS;
# This uses randomness to test different possibilities without testing all
# possibilities. To ensure repeatability, set the seed to 0. But if
# tests are added, it will perturb all later ones in the .t file
srand 0;
$t_path = 'TestProp.pl' unless defined $t_path; # the traditional name
# Create a list of what the %f representation is for each rational number.
# This will be used below.
my @valid_base_floats = '0.0';
foreach my $e_representation (keys %nv_floating_to_rational) {
push @valid_base_floats,
eval $nv_floating_to_rational{$e_representation};
}
# It doesn't matter whether the elements of this array contain single lines
# or multiple lines. main::write doesn't count the lines.
my @output;
push @output, <<'EOF_CODE';
Error('\p{Script=InGreek}'); # Bug #69018
Test_GCB("1100 $nobreak 1161"); # Bug #70940
Expect(0, 0x2028, '\p{Print}', ""); # Bug # 71722
Expect(0, 0x2029, '\p{Print}', ""); # Bug # 71722
Expect(1, 0xFF10, '\p{XDigit}', ""); # Bug # 71726
Error('\p{InKana}'); # 'Kana' is not a block so InKana shouldn't compile
# Make sure this gets tested; it was not part of the official test suite at
# the time this was added. Note that this is as it would appear in the
# official suite, and gets modified to check for the perl tailoring by
# Test_WB()
Test_WB("$breakable 0020 $breakable 0020 $breakable 0308 $breakable");
Test_LB("$nobreak 200B $nobreak 0020 $nobreak 0020 $breakable 2060 $breakable");
EOF_CODE
# Sort these so get results in same order on different runs of this
# program
foreach my $property (sort { $a->has_dependency <=> $b->has_dependency
or
lc $a->name cmp lc $b->name
} property_ref('*'))
{
# Non-binary properties should not match \p{}; Test all for that.
if ($property->type != $BINARY && $property->type != $FORCED_BINARY) {
my @property_aliases = grep { $_->status ne $INTERNAL_ALIAS }
$property->aliases;
foreach my $property_alias ($property->aliases) {
my $name = standardize($property_alias->name);
# But some names are ambiguous, meaning a binary property with
# the same name when used in \p{}, and a different
# (non-binary) property in other contexts.
next if grep { $name eq $_ } keys %ambiguous_names;
push @output, <<"EOF_CODE";
Error('\\p{$name}');
Error('\\P{$name}');
EOF_CODE
}
}
foreach my $table (sort { $a->has_dependency <=> $b->has_dependency
or
lc $a->name cmp lc $b->name
} $property->tables)
{
# Find code points that match, and don't match this table.
my $valid = $table->get_valid_code_point;
my $invalid = $table->get_invalid_code_point;
my $warning = ($table->status eq $DEPRECATED)
? "'deprecated'"
: '""';
# Test each possible combination of the property's aliases with
# the table's. If this gets to be too many, could do what is done
# in the set_final_comment() for Tables
my @table_aliases = grep { $_->status ne $INTERNAL_ALIAS } $table->aliases;
next unless @table_aliases;
my @property_aliases = grep { $_->status ne $INTERNAL_ALIAS } $table->property->aliases;
next unless @property_aliases;
# Every property can be optionally be prefixed by 'Is_', so test
# that those work, by creating such a new alias for each
# pre-existing one.
push @property_aliases, map { Alias->new("Is_" . $_->name,
$_->loose_match,
$_->make_re_pod_entry,
$_->ok_as_filename,
$_->status,
$_->ucd,
)
} @property_aliases;
my $max = max(scalar @table_aliases, scalar @property_aliases);
for my $j (0 .. $max - 1) {
# The current alias for property is the next one on the list,
# or if beyond the end, start over. Similarly for table
my $property_name
= $property_aliases[$j % @property_aliases]->name;
$property_name = "" if $table->property == $perl;
my $table_alias = $table_aliases[$j % @table_aliases];
my $table_name = $table_alias->name;
my $loose_match = $table_alias->loose_match;
# If the table doesn't have a file, any test for it is
# already guaranteed to be in error
my $already_error = ! $table->file_path;
# A table that begins with these could actually be a
# user-defined property, so won't be compile time errors, as
# the definitions of those can be deferred until runtime
next if $already_error && $table_name =~ / ^ I[ns] /x;
# Generate error cases for this alias.
push @output, generate_error($property_name,
$table_name,
$already_error);
# If the table is guaranteed to always generate an error,
# quit now without generating success cases.
next if $already_error;
# Now for the success cases. First, wildcard matching, as it
# shouldn't have any randomization.
if ($table_alias->status eq $NORMAL) {
push @output, generate_wildcard_tests($property_name,
$table_name,
$valid,
$invalid,
$warning,
);
}
my $random;
if ($loose_match) {
# For loose matching, create an extra test case for the
# standard name.
my $standard = standardize($table_name);
# $test_name should be a unique combination for each test
# case; used just to avoid duplicate tests
my $test_name = "$property_name=$standard";
# Don't output duplicate test cases.
if (! exists $test_generated{$test_name}) {
$test_generated{$test_name} = 1;
push @output, generate_tests($property_name,
$standard,
$valid,
$invalid,
$warning,
);
if ($table_alias->status eq $NORMAL) {
push @output, generate_wildcard_tests(
$property_name,
$standard,
$valid,
$invalid,
$warning,
);
}
}
$random = randomize_loose_name($table_name)
}
else { # Stricter match
$random = randomize_stricter_name($table_name);
}
# Now for the main test case for this alias.
my $test_name = "$property_name=$random";
if (! exists $test_generated{$test_name}) {
$test_generated{$test_name} = 1;
push @output, generate_tests($property_name,
$random,
$valid,
$invalid,
$warning,
);
if ($property->name eq 'nv') {
if ($table_name !~ qr{/}) {
push @output, generate_tests($property_name,
sprintf("%.15e", $table_name),
$valid,
$invalid,
$warning,
);
}
else {
# If the name is a rational number, add tests for a
# non-reduced form, and for a floating point equivalent.
# 60 is a number divisible by a bunch of things
my ($numerator, $denominator) = $table_name
=~ m! (.+) / (.+) !x;
$numerator *= 60;
$denominator *= 60;
push @output, generate_tests($property_name,
"$numerator/$denominator",
$valid,
$invalid,
$warning,
);
# Calculate the float, and the %e representation
my $float = eval $table_name;
my $e_representation = sprintf("%.*e",
$E_FLOAT_PRECISION, $float);
# Parse that
my ($non_zeros, $zeros, $exponent_sign, $exponent)
= $e_representation
=~ / -? [1-9] \. (\d*?) (0*) e ([+-]) (\d+) /x;
my $min_e_precision;
my $min_f_precision;
if ($exponent_sign eq '+' && $exponent != 0) {
Carp::my_carp_bug("Not yet equipped to handle"
. " positive exponents");
return;
}
else {
# We're trying to find the minimum precision that
# is needed to indicate this particular rational
# for the given $E_FLOAT_PRECISION. For %e, any
# trailing zeros, like 1.500e-02 aren't needed, so
# the correct value is how many non-trailing zeros
# there are after the decimal point.
$min_e_precision = length $non_zeros;
# For %f, like .01500, we want at least
# $E_FLOAT_PRECISION digits, but any trailing
# zeros aren't needed, so we can subtract the
# length of those. But we also need to include
# the zeros after the decimal point, but before
# the first significant digit.
$min_f_precision = $E_FLOAT_PRECISION
+ $exponent
- length $zeros;
}
# Make tests for each possible precision from 1 to
# just past the worst case.
my $upper_limit = ($min_e_precision > $min_f_precision)
? $min_e_precision
: $min_f_precision;
for my $i (1 .. $upper_limit + 1) {
for my $format ("e", "f") {
my $this_table
= sprintf("%.*$format", $i, $float);
# If we don't have enough precision digits,
# make a fail test; otherwise a pass test.
my $pass = ($format eq "e")
? $i >= $min_e_precision
: $i >= $min_f_precision;
if ($pass) {
push @output, generate_tests($property_name,
$this_table,
$valid,
$invalid,
$warning,
);
}
elsif ( $format eq "e"
# Here we would fail, but in the %f
# case, the representation at this
# precision could actually be a
# valid one for some other rational
|| ! grep { $this_table
=~ / ^ $_ 0* $ /x }
@valid_base_floats)
{
push @output,
generate_error($property_name,
$this_table,
1 # 1 => already an
# error
);
}
}
}
}
}
}
}
$table->DESTROY();
}
$property->DESTROY();
}
# Make any test of the boundary (break) properties TODO if the code
# doesn't match the version being compiled
my $TODO_FAILING_BREAKS = ($version_of_mk_invlist_bounds ne $v_version)
? "\nsub TODO_FAILING_BREAKS { 1 }\n"
: "\nsub TODO_FAILING_BREAKS { 0 }\n";
@output= map {
map s/^/ /mgr,
map "$_;\n",
split /;\n/, $_
} @output;
# Cause there to be 'if' statements to only execute a portion of this
# long-running test each time, so that we can have a bunch of .t's running
# in parallel
my $chunks = 10 # Number of test files
- 1 # For GCB & SB
- 1 # For WB
- 4; # LB split into this many files
my @output_chunked;
my $chunk_count=0;
my $chunk_size= int(@output / $chunks) + 1;
while (@output) {
$chunk_count++;
my @chunk= splice @output, 0, $chunk_size;
push @output_chunked,
"if (!\$::TESTCHUNK or \$::TESTCHUNK == $chunk_count) {\n",
@chunk,
"}\n";
}
$chunk_count++;
push @output_chunked,
"if (!\$::TESTCHUNK or \$::TESTCHUNK == $chunk_count) {\n",
(map {" Test_GCB('$_');\n"} @backslash_X_tests),
(map {" Test_SB('$_');\n"} @SB_tests),
"}\n";
$chunk_size= int(@LB_tests / 4) + 1;
@LB_tests = map {" Test_LB('$_');\n"} @LB_tests;
while (@LB_tests) {
$chunk_count++;
my @chunk= splice @LB_tests, 0, $chunk_size;
push @output_chunked,
"if (!\$::TESTCHUNK or \$::TESTCHUNK == $chunk_count) {\n",
@chunk,
"}\n";
}
$chunk_count++;
push @output_chunked,
"if (!\$::TESTCHUNK or \$::TESTCHUNK == $chunk_count) {\n",
(map {" Test_WB('$_');\n"} @WB_tests),
"}\n";
&write($t_path,
0, # Not utf8;
[$HEADER,
$TODO_FAILING_BREAKS,
<DATA>,
@output_chunked,
"Finished();\n",
]);
return;
}
sub make_normalization_test_script() {
print "Making normalization test script\n" if $verbosity >= $PROGRESS;
my $n_path = 'TestNorm.pl';
unshift @normalization_tests, <<'END';
use utf8;
use Test::More;
sub ord_string { # Convert packed ords to printable string
use charnames ();
return "'" . join("", map { '\N{' . charnames::viacode($_) . '}' }
unpack "U*", shift) . "'";
#return "'" . join(" ", map { sprintf "%04X", $_ } unpack "U*", shift) . "'";
}
sub Test_N {
my ($source, $nfc, $nfd, $nfkc, $nfkd) = @_;
my $display_source = ord_string($source);
my $display_nfc = ord_string($nfc);
my $display_nfd = ord_string($nfd);
my $display_nfkc = ord_string($nfkc);
my $display_nfkd = ord_string($nfkd);
use Unicode::Normalize;
# NFC
# nfc == toNFC(source) == toNFC(nfc) == toNFC(nfd)
# nfkc == toNFC(nfkc) == toNFC(nfkd)
#
# NFD
# nfd == toNFD(source) == toNFD(nfc) == toNFD(nfd)
# nfkd == toNFD(nfkc) == toNFD(nfkd)
#
# NFKC
# nfkc == toNFKC(source) == toNFKC(nfc) == toNFKC(nfd) ==
# toNFKC(nfkc) == toNFKC(nfkd)
#
# NFKD
# nfkd == toNFKD(source) == toNFKD(nfc) == toNFKD(nfd) ==
# toNFKD(nfkc) == toNFKD(nfkd)
is(NFC($source), $nfc, "NFC($display_source) eq $display_nfc");
is(NFC($nfc), $nfc, "NFC($display_nfc) eq $display_nfc");
is(NFC($nfd), $nfc, "NFC($display_nfd) eq $display_nfc");
is(NFC($nfkc), $nfkc, "NFC($display_nfkc) eq $display_nfkc");
is(NFC($nfkd), $nfkc, "NFC($display_nfkd) eq $display_nfkc");
is(NFD($source), $nfd, "NFD($display_source) eq $display_nfd");
is(NFD($nfc), $nfd, "NFD($display_nfc) eq $display_nfd");
is(NFD($nfd), $nfd, "NFD($display_nfd) eq $display_nfd");
is(NFD($nfkc), $nfkd, "NFD($display_nfkc) eq $display_nfkd");
is(NFD($nfkd), $nfkd, "NFD($display_nfkd) eq $display_nfkd");
is(NFKC($source), $nfkc, "NFKC($display_source) eq $display_nfkc");
is(NFKC($nfc), $nfkc, "NFKC($display_nfc) eq $display_nfkc");
is(NFKC($nfd), $nfkc, "NFKC($display_nfd) eq $display_nfkc");
is(NFKC($nfkc), $nfkc, "NFKC($display_nfkc) eq $display_nfkc");
is(NFKC($nfkd), $nfkc, "NFKC($display_nfkd) eq $display_nfkc");
is(NFKD($source), $nfkd, "NFKD($display_source) eq $display_nfkd");
is(NFKD($nfc), $nfkd, "NFKD($display_nfc) eq $display_nfkd");
is(NFKD($nfd), $nfkd, "NFKD($display_nfd) eq $display_nfkd");
is(NFKD($nfkc), $nfkd, "NFKD($display_nfkc) eq $display_nfkd");
is(NFKD($nfkd), $nfkd, "NFKD($display_nfkd) eq $display_nfkd");
}
END
&write($n_path,
1, # Is utf8;
[
@normalization_tests,
'done_testing();'
]);
return;
}
# Skip reasons, so will be exact same text and hence the files with each
# reason will get grouped together in perluniprops.
my $Documentation = "Documentation";
my $Indic_Skip
= "Provisional; for the analysis and processing of Indic scripts";
my $Validation = "Validation Tests";
my $Validation_Documentation = "Documentation of validation Tests";
# This is a list of the input files and how to handle them. The files are
# processed in their order in this list. Some reordering is possible if
# desired, but the PropertyAliases and PropValueAliases files should be first,
# and the extracted before the others (as data in an extracted file can be
# over-ridden by the non-extracted. Some other files depend on data derived
# from an earlier file, like UnicodeData requires data from Jamo, and the case
# changing and folding requires data from Unicode. Mostly, it is safest to
# order by first version releases in (except the Jamo).
#
# The version strings allow the program to know whether to expect a file or
# not, but if a file exists in the directory, it will be processed, even if it
# is in a version earlier than expected, so you can copy files from a later
# release into an earlier release's directory.
my @input_file_objects = (
Input_file->new('PropertyAliases.txt', v3.2,
Handler => \&process_PropertyAliases,
Early => [ \&substitute_PropertyAliases ],
Required_Even_in_Debug_Skip => 1,
),
Input_file->new(undef, v0, # No file associated with this
Progress_Message => 'Finishing property setup',
Handler => \&finish_property_setup,
),
Input_file->new('PropValueAliases.txt', v3.2,
Handler => \&process_PropValueAliases,
Early => [ \&substitute_PropValueAliases ],
Has_Missings_Defaults => $NOT_IGNORED,
Required_Even_in_Debug_Skip => 1,
),
Input_file->new("${EXTRACTED}DGeneralCategory.txt", v3.1.0,
Property => 'General_Category',
),
Input_file->new("${EXTRACTED}DCombiningClass.txt", v3.1.0,
Property => 'Canonical_Combining_Class',
Has_Missings_Defaults => $NOT_IGNORED,
),
Input_file->new("${EXTRACTED}DNumType.txt", v3.1.0,
Property => 'Numeric_Type',
Has_Missings_Defaults => $NOT_IGNORED,
),
Input_file->new("${EXTRACTED}DEastAsianWidth.txt", v3.1.0,
Property => 'East_Asian_Width',
Has_Missings_Defaults => $NOT_IGNORED,
),
Input_file->new("${EXTRACTED}DLineBreak.txt", v3.1.0,
Property => 'Line_Break',
Has_Missings_Defaults => $NOT_IGNORED,
),
Input_file->new("${EXTRACTED}DBidiClass.txt", v3.1.1,
Property => 'Bidi_Class',
Has_Missings_Defaults => $NOT_IGNORED,
),
Input_file->new("${EXTRACTED}DDecompositionType.txt", v3.1.0,
Property => 'Decomposition_Type',
Has_Missings_Defaults => $NOT_IGNORED,
),
Input_file->new("${EXTRACTED}DBinaryProperties.txt", v3.1.0),
Input_file->new("${EXTRACTED}DNumValues.txt", v3.1.0,
Property => 'Numeric_Value',
Each_Line_Handler => \&filter_numeric_value_line,
Has_Missings_Defaults => $NOT_IGNORED,
),
Input_file->new("${EXTRACTED}DJoinGroup.txt", v3.1.0,
Property => 'Joining_Group',
Has_Missings_Defaults => $NOT_IGNORED,
),
Input_file->new("${EXTRACTED}DJoinType.txt", v3.1.0,
Property => 'Joining_Type',
Has_Missings_Defaults => $NOT_IGNORED,
),
Input_file->new("${EXTRACTED}DName.txt", v10.0.0,
Skip => 'This file adds no new information not already'
. ' present in other files',
# And it's unnecessary programmer work to handle this new
# format. Previous Derived files actually had bug fixes
# in them that were useful, but that should not be the
# case here.
),
Input_file->new('Jamo.txt', v2.0.0,
Property => 'Jamo_Short_Name',
Each_Line_Handler => \&filter_jamo_line,
),
Input_file->new('UnicodeData.txt', v1.1.5,
Pre_Handler => \&setup_UnicodeData,
# We clean up this file for some early versions.
Each_Line_Handler => [ (($v_version lt v2.0.0 )
? \&filter_v1_ucd
: ($v_version eq v2.1.5)
? \&filter_v2_1_5_ucd
# And for 5.14 Perls with 6.0,
# have to also make changes
: ($v_version ge v6.0.0
&& $^V lt v5.17.0)
? \&filter_v6_ucd
: undef),
# Early versions did not have the
# proper Unicode_1 names for the
# controls
(($v_version lt v3.0.0)
? \&filter_early_U1_names
: undef),
# Early versions did not correctly
# use the later method for giving
# decimal digit values
(($v_version le v3.2.0)
? \&filter_bad_Nd_ucd
: undef),
# And the main filter
\&filter_UnicodeData_line,
],
EOF_Handler => \&EOF_UnicodeData,
),
Input_file->new('CJKXREF.TXT', v1.1.5,
Withdrawn => v2.0.0,
Skip => 'Gives the mapping of CJK code points '
. 'between Unicode and various other standards',
),
Input_file->new('ArabicShaping.txt', v2.0.0,
Each_Line_Handler =>
($v_version lt 4.1.0)
? \&filter_old_style_arabic_shaping
: undef,
# The first field after the range is a "schematic name"
# not used by Perl
Properties => [ '<ignored>', 'Joining_Type', 'Joining_Group' ],
Has_Missings_Defaults => $NOT_IGNORED,
),
Input_file->new('Blocks.txt', v2.0.0,
Property => 'Block',
Has_Missings_Defaults => $NOT_IGNORED,
Each_Line_Handler => \&filter_blocks_lines
),
Input_file->new('Index.txt', v2.0.0,
Skip => 'Alphabetical index of Unicode characters',
),
Input_file->new('NamesList.txt', v2.0.0,
Skip => 'Annotated list of characters',
),
Input_file->new('PropList.txt', v2.0.0,
Each_Line_Handler => (($v_version lt v3.1.0)
? \&filter_old_style_proplist
: undef),
),
Input_file->new('Props.txt', v2.0.0,
Withdrawn => v3.0.0,
Skip => 'A subset of F<PropList.txt> (which is used instead)',
),
Input_file->new('ReadMe.txt', v2.0.0,
Skip => $Documentation,
),
Input_file->new('Unihan.txt', v2.0.0,
Withdrawn => v5.2.0,
Construction_Time_Handler => \&construct_unihan,
Pre_Handler => \&setup_unihan,
Optional => [ "",
'Unicode_Radical_Stroke'
],
Each_Line_Handler => \&filter_unihan_line,
),
Input_file->new('SpecialCasing.txt', v2.1.8,
Each_Line_Handler => ($v_version eq 2.1.8)
? \&filter_2_1_8_special_casing_line
: \&filter_special_casing_line,
Pre_Handler => \&setup_special_casing,
Has_Missings_Defaults => $IGNORED,
),
Input_file->new(
'LineBreak.txt', v3.0.0,
Has_Missings_Defaults => $NOT_IGNORED,
Property => 'Line_Break',
# Early versions had problematic syntax
Each_Line_Handler => ($v_version ge v3.1.0)
? undef
: ($v_version lt v3.0.0)
? \&filter_substitute_lb
: \&filter_early_ea_lb,
# Must use long names for property values see comments at
# sub filter_substitute_lb
Early => [ "LBsubst.txt", '_Perl_LB', 'Alphabetic',
'Alphabetic', # default to this because XX ->
# AL
# Don't use _Perl_LB as a synonym for
# Line_Break in later perls, as it is tailored
# and isn't the same as Line_Break
'ONLY_EARLY' ],
),
Input_file->new('EastAsianWidth.txt', v3.0.0,
Property => 'East_Asian_Width',
Has_Missings_Defaults => $NOT_IGNORED,
# Early versions had problematic syntax
Each_Line_Handler => (($v_version lt v3.1.0)
? \&filter_early_ea_lb
: undef),
),
Input_file->new('CompositionExclusions.txt', v3.0.0,
Property => 'Composition_Exclusion',
),
Input_file->new('UnicodeData.html', v3.0.0,
Withdrawn => v4.0.1,
Skip => $Documentation,
),
Input_file->new('BidiMirroring.txt', v3.0.1,
Property => 'Bidi_Mirroring_Glyph',
Has_Missings_Defaults => ($v_version lt v6.2.0)
? $NO_DEFAULTS
# Is <none> which doesn't mean
# anything to us, we will use the
# null string
: $IGNORED,
),
Input_file->new('NamesList.html', v3.0.0,
Skip => 'Describes the format and contents of '
. 'F<NamesList.txt>',
),
Input_file->new('UnicodeCharacterDatabase.html', v3.0.0,
Withdrawn => v5.1,
Skip => $Documentation,
),
Input_file->new('CaseFolding.txt', v3.0.1,
Pre_Handler => \&setup_case_folding,
Each_Line_Handler =>
[ ($v_version lt v3.1.0)
? \&filter_old_style_case_folding
: undef,
\&filter_case_folding_line
],
Has_Missings_Defaults => $IGNORED,
),
Input_file->new("NormTest.txt", v3.0.1,
Handler => \&process_NormalizationsTest,
Skip => ($make_norm_test_script) ? 0 : $Validation,
),
Input_file->new('DCoreProperties.txt', v3.1.0,
# 5.2 changed this file
Has_Missings_Defaults => (($v_version ge v5.2.0)
? $NOT_IGNORED
: $NO_DEFAULTS),
),
Input_file->new('DProperties.html', v3.1.0,
Withdrawn => v3.2.0,
Skip => $Documentation,
),
Input_file->new('PropList.html', v3.1.0,
Withdrawn => v5.1,
Skip => $Documentation,
),
Input_file->new('Scripts.txt', v3.1.0,
Property => 'Script',
Each_Line_Handler => (($v_version le v4.0.0)
? \&filter_all_caps_script_names
: undef),
Has_Missings_Defaults => $NOT_IGNORED,
),
Input_file->new('DNormalizationProps.txt', v3.1.0,
Has_Missings_Defaults => $NOT_IGNORED,
Each_Line_Handler => (($v_version lt v4.0.1)
? \&filter_old_style_normalization_lines
: undef),
),
Input_file->new('DerivedProperties.html', v3.1.1,
Withdrawn => v5.1,
Skip => $Documentation,
),
Input_file->new('DAge.txt', v3.2.0,
Has_Missings_Defaults => $NOT_IGNORED,
Property => 'Age'
),
Input_file->new('HangulSyllableType.txt', v4.0,
Has_Missings_Defaults => $NOT_IGNORED,
Early => [ \&generate_hst, 'Hangul_Syllable_Type' ],
Property => 'Hangul_Syllable_Type'
),
Input_file->new('NormalizationCorrections.txt', v3.2.0,
# This documents the cumulative fixes to erroneous
# normalizations in earlier Unicode versions. Its main
# purpose is so that someone running on an earlier
# version can use this file to override what got
# published in that earlier release. It would be easy
# for mktables to handle this file. But all the
# corrections in it should already be in the other files
# for the release it is. To get it to actually mean
# something useful, someone would have to be using an
# earlier Unicode release, and copy it into the directory
# for that release and recompile. So far there has been
# no demand to do that, so this hasn't been implemented.
Skip => 'Documentation of corrections already '
. 'incorporated into the Unicode data base',
),
Input_file->new('StandardizedVariants.html', v3.2.0,
Skip => 'Obsoleted as of Unicode 9.0, but previously '
. 'provided a visual display of the standard '
. 'variant sequences derived from '
. 'F<StandardizedVariants.txt>.',
# I don't know why the html came earlier than the
# .txt, but both are skipped anyway, so it doesn't
# matter.
),
Input_file->new('StandardizedVariants.txt', v4.0.0,
Skip => 'Certain glyph variations for character display '
. 'are standardized. This lists the non-Unihan '
. 'ones; the Unihan ones are also not used by '
. 'Perl, and are in a separate Unicode data base '
. 'L<http://www.unicode.org/ivd>',
),
Input_file->new('UCD.html', v4.0.0,
Withdrawn => v5.2,
Skip => $Documentation,
),
Input_file->new("$AUXILIARY/WordBreakProperty.txt", v4.1.0,
Early => [ "WBsubst.txt", '_Perl_WB', 'ALetter' ],
Property => 'Word_Break',
Has_Missings_Defaults => $NOT_IGNORED,
),
Input_file->new("$AUXILIARY/GraphemeBreakProperty.txt", v4.1,
Early => [ \&generate_GCB, '_Perl_GCB' ],
Property => 'Grapheme_Cluster_Break',
Has_Missings_Defaults => $NOT_IGNORED,
),
Input_file->new("$AUXILIARY/GCBTest.txt", v4.1.0,
Handler => \&process_GCB_test,
retain_trailing_comments => 1,
),
Input_file->new("$AUXILIARY/GraphemeBreakTest.html", v4.1.0,
Skip => $Validation_Documentation,
),
Input_file->new("$AUXILIARY/SBTest.txt", v4.1.0,
Handler => \&process_SB_test,
retain_trailing_comments => 1,
),
Input_file->new("$AUXILIARY/SentenceBreakTest.html", v4.1.0,
Skip => $Validation_Documentation,
),
Input_file->new("$AUXILIARY/WBTest.txt", v4.1.0,
Handler => \&process_WB_test,
retain_trailing_comments => 1,
),
Input_file->new("$AUXILIARY/WordBreakTest.html", v4.1.0,
Skip => $Validation_Documentation,
),
Input_file->new("$AUXILIARY/SentenceBreakProperty.txt", v4.1.0,
Property => 'Sentence_Break',
Early => [ "SBsubst.txt", '_Perl_SB', 'OLetter' ],
Has_Missings_Defaults => $NOT_IGNORED,
),
Input_file->new('NamedSequences.txt', v4.1.0,
Handler => \&process_NamedSequences
),
Input_file->new('Unihan.html', v4.1.0,
Withdrawn => v5.2,
Skip => $Documentation,
),
Input_file->new('NameAliases.txt', v5.0,
Property => 'Name_Alias',
Each_Line_Handler => ($v_version le v6.0.0)
? \&filter_early_version_name_alias_line
: \&filter_later_version_name_alias_line,
),
# NameAliases.txt came along in v5.0. The above constructor handles
# this. But until 6.1, it was lacking some information needed by core
# perl. The constructor below handles that. It is either a kludge or
# clever, depending on your point of view. The 'Withdrawn' parameter
# indicates not to use it at all starting in 6.1 (so the above
# constructor applies), and the 'v6.1' parameter indicates to use the
# Early parameter before 6.1. Therefore 'Early" is always used,
# yielding the internal-only property '_Perl_Name_Alias', which it
# gets from a NameAliases.txt from 6.1 or later stored in
# N_Asubst.txt. In combination with the above constructor,
# 'Name_Alias' is publicly accessible starting with v5.0, and the
# better 6.1 version is accessible to perl core in all releases.
Input_file->new("NameAliases.txt", v6.1,
Withdrawn => v6.1,
Early => [ "N_Asubst.txt", '_Perl_Name_Alias', "" ],
Property => 'Name_Alias',
EOF_Handler => \&fixup_early_perl_name_alias,
Each_Line_Handler =>
\&filter_later_version_name_alias_line,
),
Input_file->new('NamedSqProv.txt', v5.0.0,
Skip => 'Named sequences proposed for inclusion in a '
. 'later version of the Unicode Standard; if you '
. 'need them now, you can append this file to '
. 'F<NamedSequences.txt> and recompile perl',
),
Input_file->new("$AUXILIARY/LBTest.txt", v5.1.0,
Handler => \&process_LB_test,
retain_trailing_comments => 1,
),
Input_file->new("$AUXILIARY/LineBreakTest.html", v5.1.0,
Skip => $Validation_Documentation,
),
Input_file->new("BidiTest.txt", v5.2.0,
Skip => $Validation,
),
Input_file->new('UnihanIndicesDictionary.txt', v5.2.0,
Optional => "",
Each_Line_Handler => \&filter_unihan_line,
),
Input_file->new('UnihanDataDictionaryLike.txt', v5.2.0,
Optional => "",
Each_Line_Handler => \&filter_unihan_line,
),
Input_file->new('UnihanIRGSources.txt', v5.2.0,
Optional => [ "",
'kCompatibilityVariant',
'kIICore',
'kIRG_GSource',
'kIRG_HSource',
'kIRG_JSource',
'kIRG_KPSource',
'kIRG_MSource',
'kIRG_KSource',
'kIRG_TSource',
'kIRG_USource',
'kIRG_VSource',
],
Pre_Handler => \&setup_unihan,
Each_Line_Handler => \&filter_unihan_line,
),
Input_file->new('UnihanNumericValues.txt', v5.2.0,
Optional => [ "",
'kAccountingNumeric',
'kOtherNumeric',
'kPrimaryNumeric',
],
Each_Line_Handler => \&filter_unihan_line,
),
Input_file->new('UnihanOtherMappings.txt', v5.2.0,
Optional => "",
Each_Line_Handler => \&filter_unihan_line,
),
Input_file->new('UnihanRadicalStrokeCounts.txt', v5.2.0,
Optional => [ "",
'Unicode_Radical_Stroke'
],
Each_Line_Handler => \&filter_unihan_line,
),
Input_file->new('UnihanReadings.txt', v5.2.0,
Optional => "",
Each_Line_Handler => \&filter_unihan_line,
),
Input_file->new('UnihanVariants.txt', v5.2.0,
Optional => "",
Each_Line_Handler => \&filter_unihan_line,
),
Input_file->new('CJKRadicals.txt', v5.2.0,
Skip => 'Maps the kRSUnicode property values to '
. 'corresponding code points',
),
Input_file->new('EmojiSources.txt', v6.0.0,
Skip => 'Maps certain Unicode code points to their '
. 'legacy Japanese cell-phone values',
),
# This file is actually not usable as-is until 6.1.0, because the property
# is provisional, so its name is missing from PropertyAliases.txt until
# that release, so that further work would have to be done to get it to
# work properly
Input_file->new('ScriptExtensions.txt', v6.0.0,
Property => 'Script_Extensions',
Early => [ sub {} ], # Doesn't do anything but ensures
# that this isn't skipped for early
# versions
Pre_Handler => \&setup_script_extensions,
Each_Line_Handler => \&filter_script_extensions_line,
Has_Missings_Defaults => (($v_version le v6.0.0)
? $NO_DEFAULTS
: $IGNORED),
),
# These two Indic files are actually not usable as-is until 6.1.0,
# because they are provisional, so their property values are missing from
# PropValueAliases.txt until that release, so that further work would have
# to be done to get them to work properly.
Input_file->new('IndicMatraCategory.txt', v6.0.0,
Withdrawn => v8.0.0,
Property => 'Indic_Matra_Category',
Has_Missings_Defaults => $NOT_IGNORED,
Skip => $Indic_Skip,
),
Input_file->new('IndicSyllabicCategory.txt', v6.0.0,
Property => 'Indic_Syllabic_Category',
Has_Missings_Defaults => $NOT_IGNORED,
Skip => (($v_version lt v8.0.0)
? $Indic_Skip
: 0),
),
Input_file->new('USourceData.txt', v6.2.0,
Skip => 'Documentation of status and cross reference of '
. 'proposals for encoding by Unicode of Unihan '
. 'characters',
),
Input_file->new('USourceGlyphs.pdf', v6.2.0,
Skip => 'Pictures of the characters in F<USourceData.txt>',
),
Input_file->new('BidiBrackets.txt', v6.3.0,
Properties => [ 'Bidi_Paired_Bracket',
'Bidi_Paired_Bracket_Type'
],
Has_Missings_Defaults => $NO_DEFAULTS,
),
Input_file->new("BidiCharacterTest.txt", v6.3.0,
Skip => $Validation,
),
Input_file->new('IndicPositionalCategory.txt', v8.0.0,
Property => 'Indic_Positional_Category',
Has_Missings_Defaults => $NOT_IGNORED,
),
Input_file->new('TangutSources.txt', v9.0.0,
Skip => 'Specifies source mappings for Tangut ideographs'
. ' and components. This data file also includes'
. ' informative radical-stroke values that are used'
. ' internally by Unicode',
),
Input_file->new('VerticalOrientation.txt', v10.0.0,
Property => 'Vertical_Orientation',
Has_Missings_Defaults => $NOT_IGNORED,
),
Input_file->new('NushuSources.txt', v10.0.0,
Skip => 'Specifies source material for Nushu characters',
),
Input_file->new('EquivalentUnifiedIdeograph.txt', v11.0.0,
Property => 'Equivalent_Unified_Ideograph',
Has_Missings_Defaults => $NOT_IGNORED,
),
Input_file->new('EmojiData.txt', v11.0.0,
# Is in UAX #51 and not the UCD, so must be updated
# separately, and the first line edited to indicate the
# UCD release we're pretending it to be in. The UTC says
# this is a transitional state.
Pre_Handler => \&setup_emojidata,
Has_Missings_Defaults => $NOT_IGNORED,
Each_Line_Handler => \&filter_emojidata_line,
),
);
# End of all the preliminaries.
# Do it...
if (@missing_early_files) {
print simple_fold(join_lines(<<END
The compilation cannot be completed because one or more required input files,
listed below, are missing. This is because you are compiling Unicode version
$unicode_version, which predates the existence of these file(s). To fully
function, perl needs the data that these files would have contained if they
had been in this release. To work around this, create copies of later
versions of the missing files in the directory containing '$0'. (Perl will
make the necessary adjustments to the data to compensate for it not being the
same version as is being compiled.) The files are available from unicode.org,
via either ftp or http. If using http, they will be under
www.unicode.org/versions/. Below are listed the source file name of each
missing file, the Unicode version to copy it from, and the name to store it
as. (Note that the listed source file name may not be exactly the one that
Unicode calls it. If you don't find it, you can look it up in 'README.perl'
to get the correct name.)
END
));
print simple_fold(join_lines("\n$_")) for @missing_early_files;
exit 2;
}
if ($compare_versions) {
Carp::my_carp(<<END
Warning. \$compare_versions is set. Output is not suitable for production
END
);
}
# Put into %potential_files a list of all the files in the directory structure
# that could be inputs to this program
File::Find::find({
wanted=>sub {
return unless / \. ( txt | htm l? ) $ /xi; # Some platforms change the
# name's case
my $full = lc(File::Spec->rel2abs($_));
$potential_files{$full} = 1;
return;
}
}, File::Spec->curdir());
my @mktables_list_output_files;
my $old_start_time = 0;
my $old_options = "";
if (! -e $file_list) {
print "'$file_list' doesn't exist, so forcing rebuild.\n" if $verbosity >= $VERBOSE;
$write_unchanged_files = 1;
} elsif ($write_unchanged_files) {
print "Not checking file list '$file_list'.\n" if $verbosity >= $VERBOSE;
}
else {
print "Reading file list '$file_list'\n" if $verbosity >= $VERBOSE;
my $file_handle;
if (! open $file_handle, "<", $file_list) {
Carp::my_carp("Failed to open '$file_list'; turning on -globlist option instead: $!");
$glob_list = 1;
}
else {
my @input;
# Read and parse mktables.lst, placing the results from the first part
# into @input, and the second part into @mktables_list_output_files
for my $list ( \@input, \@mktables_list_output_files ) {
while (<$file_handle>) {
s/^ \s+ | \s+ $//xg;
if (/^ \s* \# \s* Autogenerated\ starting\ on\ (\d+)/x) {
$old_start_time = $1;
next;
}
if (/^ \s* \# \s* From\ options\ (.+) /x) {
$old_options = $1;
next;
}
next if /^ \s* (?: \# .* )? $/x;
last if /^ =+ $/x;
my ( $file ) = split /\t/;
push @$list, $file;
}
@$list = uniques(@$list);
next;
}
# Look through all the input files
foreach my $input (@input) {
next if $input eq 'version'; # Already have checked this.
# Ignore if doesn't exist. The checking about whether we care or
# not is done via the Input_file object.
next if ! file_exists($input);
# The paths are stored with relative names, and with '/' as the
# delimiter; convert to absolute on this machine
my $full = lc(File::Spec->rel2abs(internal_file_to_platform($input)));
$potential_files{lc $full} = 1;
}
}
close $file_handle;
}
if ($glob_list) {
# Here wants to process all .txt files in the directory structure.
# Convert them to full path names. They are stored in the platform's
# relative style
my @known_files;
foreach my $object (@input_file_objects) {
my $file = $object->file;
next unless defined $file;
push @known_files, File::Spec->rel2abs($file);
}
my @unknown_input_files;
foreach my $file (keys %potential_files) { # The keys are stored in lc
next if grep { $file eq lc($_) } @known_files;
# Here, the file is unknown to us. Get relative path name
$file = File::Spec->abs2rel($file);
push @unknown_input_files, $file;
# What will happen is we create a data structure for it, and add it to
# the list of input files to process. First get the subdirectories
# into an array
my (undef, $directories, undef) = File::Spec->splitpath($file);
$directories =~ s;/$;;; # Can have extraneous trailing '/'
my @directories = File::Spec->splitdir($directories);
# If the file isn't extracted (meaning none of the directories is the
# extracted one), just add it to the end of the list of inputs.
if (! grep { $EXTRACTED_DIR eq $_ } @directories) {
push @input_file_objects, Input_file->new($file, v0);
}
else {
# Here, the file is extracted. It needs to go ahead of most other
# processing. Search for the first input file that isn't a
# special required property (that is, find one whose first_release
# is non-0), and isn't extracted. Also, the Age property file is
# processed before the extracted ones, just in case
# $compare_versions is set.
for (my $i = 0; $i < @input_file_objects; $i++) {
if ($input_file_objects[$i]->first_released ne v0
&& lc($input_file_objects[$i]->file) ne 'dage.txt'
&& $input_file_objects[$i]->file !~ /$EXTRACTED_DIR/i)
{
splice @input_file_objects, $i, 0,
Input_file->new($file, v0);
last;
}
}
}
}
if (@unknown_input_files) {
print STDERR simple_fold(join_lines(<<END
The following files are unknown as to how to handle. Assuming they are
typical property files. You'll know by later error messages if it worked or
not:
END
) . " " . join(", ", @unknown_input_files) . "\n\n");
}
} # End of looking through directory structure for more .txt files.
# Create the list of input files from the objects we have defined, plus
# version
my @input_files = qw(version Makefile);
foreach my $object (@input_file_objects) {
my $file = $object->file;
next if ! defined $file; # Not all objects have files
next if defined $object->skip;;
push @input_files, $file;
}
if ( $verbosity >= $VERBOSE ) {
print "Expecting ".scalar( @input_files )." input files. ",
"Checking ".scalar( @mktables_list_output_files )." output files.\n";
}
# We set $most_recent to be the most recently changed input file, including
# this program itself (done much earlier in this file)
foreach my $in (@input_files) {
next unless -e $in; # Keep going even if missing a file
my $mod_time = (stat $in)[9];
$most_recent = $mod_time if $mod_time > $most_recent;
# See that the input files have distinct names, to warn someone if they
# are adding a new one
if ($make_list) {
my ($volume, $directories, $file ) = File::Spec->splitpath($in);
$directories =~ s;/$;;; # Can have extraneous trailing '/'
my @directories = File::Spec->splitdir($directories);
construct_filename($file, 'mutable', \@directories);
}
}
# We use 'Makefile' just to see if it has changed since the last time we
# rebuilt. Now discard it.
@input_files = grep { $_ ne 'Makefile' } @input_files;
my $rebuild = $write_unchanged_files # Rebuild: if unconditional rebuild
|| ! scalar @mktables_list_output_files # or if no outputs known
|| $old_start_time < $most_recent # or out-of-date
|| $old_options ne $command_line_arguments; # or with different
# options
# Now we check to see if any output files are older than youngest, if
# they are, we need to continue on, otherwise we can presumably bail.
if (! $rebuild) {
foreach my $out (@mktables_list_output_files) {
if ( ! file_exists($out)) {
print "'$out' is missing.\n" if $verbosity >= $VERBOSE;
$rebuild = 1;
last;
}
#local $to_trace = 1 if main::DEBUG;
trace $most_recent, (stat $out)[9] if main::DEBUG && $to_trace;
if ( (stat $out)[9] <= $most_recent ) {
#trace "$out: most recent mod time: ", (stat $out)[9], ", youngest: $most_recent\n" if main::DEBUG && $to_trace;
print "'$out' is too old.\n" if $verbosity >= $VERBOSE;
$rebuild = 1;
last;
}
}
}
if (! $rebuild) {
print "$0: Files seem to be ok, not bothering to rebuild. Add '-w' option to force build\n";
exit(0);
}
print "$0: Must rebuild tables.\n" if $verbosity >= $VERBOSE;
# Ready to do the major processing. First create the perl pseudo-property.
$perl = Property->new('perl', Type => $NON_STRING, Perl_Extension => 1);
# Process each input file
foreach my $file (@input_file_objects) {
$file->run;
}
# Finish the table generation.
print "Finishing processing Unicode properties\n" if $verbosity >= $PROGRESS;
finish_Unicode();
# For the very specialized case of comparing two Unicode versions...
if (DEBUG && $compare_versions) {
handle_compare_versions();
}
print "Compiling Perl properties\n" if $verbosity >= $PROGRESS;
compile_perl();
print "Creating Perl synonyms\n" if $verbosity >= $PROGRESS;
add_perl_synonyms();
print "Writing tables\n" if $verbosity >= $PROGRESS;
write_all_tables();
# Write mktables.lst
if ( $file_list and $make_list ) {
print "Updating '$file_list'\n" if $verbosity >= $PROGRESS;
foreach my $file (@input_files, @files_actually_output) {
my (undef, $directories, $basefile) = File::Spec->splitpath($file);
my @directories = grep length, File::Spec->splitdir($directories);
$file = join '/', @directories, $basefile;
}
my $ofh;
if (! open $ofh,">",$file_list) {
Carp::my_carp("Can't write to '$file_list'. Skipping: $!");
return
}
else {
my $localtime = localtime $start_time;
print $ofh <<"END";
#
# $file_list -- File list for $0.
#
# Autogenerated starting on $start_time ($localtime)
# From options $command_line_arguments
#
# - First section is input files
# ($0 itself is not listed but is automatically considered an input)
# - Section separator is /^=+\$/
# - Second section is a list of output files.
# - Lines matching /^\\s*#/ are treated as comments
# which along with blank lines are ignored.
#
# Input files:
END
print $ofh "$_\n" for sort(@input_files);
print $ofh "\n=================================\n# Output files:\n\n";
print $ofh "$_\n" for sort @files_actually_output;
print $ofh "\n# ",scalar(@input_files)," input files\n",
"# ",scalar(@files_actually_output)+1," output files\n\n",
"# End list\n";
close $ofh
or Carp::my_carp("Failed to close $ofh: $!");
print "Filelist has ",scalar(@input_files)," input files and ",
scalar(@files_actually_output)+1," output files\n"
if $verbosity >= $VERBOSE;
}
}
# Output these warnings unless -q explicitly specified.
if ($verbosity >= $NORMAL_VERBOSITY && ! $debug_skip) {
if (@unhandled_properties) {
print "\nProperties and tables that unexpectedly have no code points\n";
foreach my $property (sort @unhandled_properties) {
print $property, "\n";
}
}
if (%potential_files) {
print "\nInput files that are not considered:\n";
foreach my $file (sort keys %potential_files) {
print File::Spec->abs2rel($file), "\n";
}
}
print "\nAll done\n" if $verbosity >= $VERBOSE;
}
if ($version_of_mk_invlist_bounds lt $v_version) {
Carp::my_carp("WARNING: \\b{} algorithms (regen/mk_invlist.pl) need"
. " to be checked and possibly updated to Unicode"
. " $string_version. Failing tests will be marked TODO");
}
exit(0);
# TRAILING CODE IS USED BY make_property_test_script()
__DATA__
use strict;
use warnings;
no warnings 'experimental::uniprop_wildcards';
# Test qr/\X/ and the \p{} regular expression constructs. This file is
# constructed by mktables from the tables it generates, so if mktables is
# buggy, this won't necessarily catch those bugs. Tests are generated for all
# feasible properties; a few aren't currently feasible; see
# is_code_point_usable() in mktables for details.
# Standard test packages are not used because this manipulates SIG_WARN. It
# exits 0 if every non-skipped test succeeded; -1 if any failed.
my $Tests = 0;
my $Fails = 0;
# loc_tools.pl requires this function to be defined
sub ok($@) {
my ($pass, @msg) = @_;
print "not " unless $pass;
print "ok ";
print ++$Tests;
print " - ", join "", @msg if @msg;
print "\n";
}
sub Expect($$$$) {
my $expected = shift;
my $ord = shift;
my $regex = shift;
my $warning_type = shift; # Type of warning message, like 'deprecated'
# or empty if none
my $line = (caller)[2];
# Convert the code point to hex form
my $string = sprintf "\"\\x{%04X}\"", $ord;
my @tests = "";
# The first time through, use all warnings. If the input should generate
# a warning, add another time through with them turned off
push @tests, "no warnings '$warning_type';" if $warning_type;
foreach my $no_warnings (@tests) {
# Store any warning messages instead of outputting them
local $SIG{__WARN__} = $SIG{__WARN__};
my $warning_message;
$SIG{__WARN__} = sub { $warning_message = $_[0] };
$Tests++;
# A string eval is needed because of the 'no warnings'.
# Assumes no parentheses in the regular expression
my $result = eval "$no_warnings
my \$RegObj = qr($regex);
$string =~ \$RegObj ? 1 : 0";
if (not defined $result) {
print "not ok $Tests - couldn't compile /$regex/; line $line: $@\n";
$Fails++;
}
elsif ($result ^ $expected) {
print "not ok $Tests - expected $expected but got $result for $string =~ qr/$regex/; line $line\n";
$Fails++;
}
elsif ($warning_message) {
if (! $warning_type || ($warning_type && $no_warnings)) {
print "not ok $Tests - for qr/$regex/ did not expect warning message '$warning_message'; line $line\n";
$Fails++;
}
else {
print "ok $Tests - expected and got a warning message for qr/$regex/; line $line\n";
}
}
elsif ($warning_type && ! $no_warnings) {
print "not ok $Tests - for qr/$regex/ expected a $warning_type warning message, but got none; line $line\n";
$Fails++;
}
else {
print "ok $Tests - got $result for $string =~ qr/$regex/; line $line\n";
}
}
return;
}
sub Error($) {
my $regex = shift;
$Tests++;
if (eval { 'x' =~ qr/$regex/; 1 }) {
$Fails++;
my $line = (caller)[2];
print "not ok $Tests - re compiled ok, but expected error for qr/$regex/; line $line: $@\n";
}
else {
my $line = (caller)[2];
print "ok $Tests - got and expected error for qr/$regex/; line $line\n";
}
return;
}
# Break test files (e.g. GCBTest.txt) character that break allowed here
my $breakable_utf8 = my $breakable = chr(utf8::unicode_to_native(0xF7));
utf8::upgrade($breakable_utf8);
# Break test files (e.g. GCBTest.txt) character that indicates can't break
# here
my $nobreak_utf8 = my $nobreak = chr(utf8::unicode_to_native(0xD7));
utf8::upgrade($nobreak_utf8);
my $are_ctype_locales_available;
my $utf8_locale;
chdir 't' if -d 't';
eval { require "./loc_tools.pl" };
if (defined &locales_enabled) {
$are_ctype_locales_available = locales_enabled('LC_CTYPE');
if ($are_ctype_locales_available) {
$utf8_locale = &find_utf8_ctype_locale;
}
}
# Eval'd so can run on versions earlier than the property is available in
my $WB_Extend_or_Format_re = eval 'qr/[\p{WB=Extend}\p{WB=Format}\p{WB=ZWJ}]/';
if (! defined $WB_Extend_or_Format_re) {
$WB_Extend_or_Format_re = eval 'qr/[\p{WB=Extend}\p{WB=Format}]/';
}
sub _test_break($$) {
# Test various break property matches. The 2nd parameter gives the
# property name. The input is a line from auxiliary/*Test.txt for the
# given property. Each such line is a sequence of Unicode (not native)
# code points given by their hex numbers, separated by the two characters
# defined just before this subroutine that indicate that either there can
# or cannot be a break between the adjacent code points. All these are
# tested.
#
# For the gcb property extra tests are made. if there isn't a break, that
# means the sequence forms an extended grapheme cluster, which means that
# \X should match the whole thing. If there is a break, \X should stop
# there. This is all converted by this routine into a match: $string =~
# /(\X)/, Each \X should match the next cluster; and that is what is
# checked.
my $template = shift;
my $break_type = shift;
my $line = (caller 1)[2]; # Line number
my $comment = "";
if ($template =~ / ( .*? ) \s* \# (.*) /x) {
$template = $1;
$comment = $2;
# Replace leading spaces with a single one.
$comment =~ s/ ^ \s* / # /x;
}
# The line contains characters above the ASCII range, but in Latin1. It
# may or may not be in utf8, and if it is, it may or may not know it. So,
# convert these characters to 8 bits. If knows is in utf8, simply
# downgrade.
if (utf8::is_utf8($template)) {
utf8::downgrade($template);
} else {
# Otherwise, if it is in utf8, but doesn't know it, the next lines
# convert the two problematic characters to their 8-bit equivalents.
# If it isn't in utf8, they don't harm anything.
use bytes;
$template =~ s/$nobreak_utf8/$nobreak/g;
$template =~ s/$breakable_utf8/$breakable/g;
}
# Perl customizes wb. So change the official tests accordingly
if ($break_type eq 'wb' && $WB_Extend_or_Format_re) {
# Split into elements that alternate between code point and
# break/no-break
my @line = split / +/, $template;
# Look at each code point and its following one
for (my $i = 1; $i < @line - 1 - 1; $i+=2) {
# The customization only involves changing some breaks to
# non-breaks.
next if $line[$i+1] =~ /$nobreak/;
my $lhs = chr utf8::unicode_to_native(hex $line[$i]);
my $rhs = chr utf8::unicode_to_native(hex $line[$i+2]);
# And it only affects adjacent space characters.
next if $lhs !~ /\s/u;
# But, we want to make sure to test spaces followed by a Extend
# or Format.
next if $rhs !~ /\s|$WB_Extend_or_Format_re/;
# To test the customization, add some white-space before this to
# create a span. The $lhs white space may or may not be bound to
# that span, and also with the $rhs. If the $rhs is a binding
# character, the $lhs is bound to it and not to the span, unless
# $lhs is vertical space. In all other cases, the $lhs is bound
# to the span. If the $rhs is white space, it is bound to the
# $lhs
my $bound;
my $span;
if ($rhs =~ /$WB_Extend_or_Format_re/) {
if ($lhs =~ /\v/) {
$bound = $breakable;
$span = $nobreak;
}
else {
$bound = $nobreak;
$span = $breakable;
}
}
else {
$span = $nobreak;
$bound = $nobreak;
}
splice @line, $i, 0, ( '0020', $nobreak, '0020', $span);
$i += 4;
$line[$i+1] = $bound;
}
$template = join " ", @line;
}
# The input is just the break/no-break symbols and sequences of Unicode
# code points as hex digits separated by spaces for legibility. e.g.:
# ÷ 0020 × 0308 ÷ 0020 ÷
# Convert to native \x format
$template =~ s/ \s* ( [[:xdigit:]]+ ) \s* /sprintf("\\x{%02X}", utf8::unicode_to_native(hex $1))/gex;
$template =~ s/ \s* //gx; # Probably the line above removed all spaces;
# but be sure
# Make a copy of the input with the symbols replaced by \b{} and \B{} as
# appropriate
my $break_pattern = $template =~ s/ $breakable /\\b{$break_type}/grx;
$break_pattern =~ s/ $nobreak /\\B{$break_type}/gx;
my $display_string = $template =~ s/[$breakable$nobreak]//gr;
my $string = eval "\"$display_string\"";
# The remaining massaging of the input is for the \X tests. Get rid of
# the leading and trailing breakables
$template =~ s/^ \s* $breakable \s* //x;
$template =~ s/ \s* $breakable \s* $ //x;
# Delete no-breaks
$template =~ s/ \s* $nobreak \s* //xg;
# Split the input into segments that are breakable between them.
my @should_display = split /\s*$breakable\s*/, $template;
my @should_match = map { eval "\"$_\"" } @should_display;
# If a string can be represented in both non-ut8 and utf8, test both cases
my $display_upgrade = "";
UPGRADE:
for my $to_upgrade (0 .. 1) {
if ($to_upgrade) {
# If already in utf8, would just be a repeat
next UPGRADE if utf8::is_utf8($string);
utf8::upgrade($string);
$display_upgrade = " (utf8-upgraded)";
}
my @modifiers = qw(a aa d u i);
if ($are_ctype_locales_available) {
push @modifiers, "l$utf8_locale" if defined $utf8_locale;
# The /l modifier has C after it to indicate the locale to try
push @modifiers, "lC";
}
# Test for each of the regex modifiers.
for my $modifier (@modifiers) {
my $display_locale = "";
# For /l, set the locale to what it says to.
if ($modifier =~ / ^ l (.*) /x) {
my $locale = $1;
$display_locale = "(locale = $locale)";
POSIX::setlocale(&POSIX::LC_CTYPE, $locale);
$modifier = 'l';
}
no warnings qw(locale regexp surrogate);
my $pattern = "(?$modifier:$break_pattern)";
# Actually do the test
my $matched_text;
my $matched = $string =~ qr/$pattern/;
if ($matched) {
$matched_text = "matched";
}
else {
$matched_text = "failed to match";
print "not ";
if (TODO_FAILING_BREAKS) {
$comment = " # $comment" unless $comment =~ / ^ \s* \# /x;
$comment =~ s/#/# TODO/;
}
}
print "ok ", ++$Tests, " - \"$display_string\" $matched_text /$pattern/$display_upgrade; line $line $display_locale$comment\n";
# Only print the comment on the first use of this line
$comment = "";
# Repeat with the first \B{} in the pattern. This makes sure the
# code in regexec.c:find_byclass() for \B gets executed
if ($pattern =~ / ( .*? : ) .* ( \\B\{ .* ) /x) {
my $B_pattern = "$1$2";
$matched = $string =~ qr/$B_pattern/;
print "not " unless $matched;
$matched_text = ($matched) ? "matched" : "failed to match";
print "ok ", ++$Tests, " - \"$display_string\" $matched_text /$B_pattern/$display_upgrade; line $line $display_locale";
print " # TODO" if TODO_FAILING_BREAKS && ! $matched;
print "\n";
}
}
next if $break_type ne 'gcb';
# Finally, do the \X match.
my @matches = $string =~ /(\X)/g;
# Look through each matched cluster to verify that it matches what we
# expect.
my $min = (@matches < @should_match) ? @matches : @should_match;
for my $i (0 .. $min - 1) {
$Tests++;
if ($matches[$i] eq $should_match[$i]) {
print "ok $Tests - ";
if ($i == 0) {
print "In \"$display_string\" =~ /(\\X)/g, \\X #1";
} else {
print "And \\X #", $i + 1,
}
print " correctly matched $should_display[$i]; line $line\n";
} else {
$matches[$i] = join("", map { sprintf "\\x{%04X}", ord $_ }
split "", $matches[$i]);
print "not ok $Tests -";
print " # TODO" if TODO_FAILING_BREAKS;
print " In \"$display_string\" =~ /(\\X)/g, \\X #",
$i + 1,
" should have matched $should_display[$i]",
" but instead matched $matches[$i]",
". Abandoning rest of line $line\n";
next UPGRADE;
}
}
# And the number of matches should equal the number of expected matches.
$Tests++;
if (@matches == @should_match) {
print "ok $Tests - Nothing was left over; line $line\n";
} else {
print "not ok $Tests - There were ", scalar @should_match, " \\X matches expected, but got ", scalar @matches, " instead; line $line";
print " # TODO" if TODO_FAILING_BREAKS;
print "\n";
}
}
return;
}
sub Test_GCB($) {
_test_break(shift, 'gcb');
}
sub Test_LB($) {
_test_break(shift, 'lb');
}
sub Test_SB($) {
_test_break(shift, 'sb');
}
sub Test_WB($) {
_test_break(shift, 'wb');
}
sub Finished() {
print "1..$Tests\n";
exit($Fails ? -1 : 0);
}
|