path: root/regen/mk_invlists.pl

#!perl -w
use 5.015;
use strict;
use warnings;
use Unicode::UCD qw(prop_aliases
                    prop_values
                    prop_value_aliases
                    prop_invlist
                    prop_invmap search_invlist
                   );
require 'regen/regen_lib.pl';
require 'regen/charset_translations.pl';

# This program outputs charclass_invlists.h, which contains various inversion
# lists in the form of C arrays that are to be used as-is for inversion lists.
# Thus, the lists it contains are essentially pre-compiled, and need only a
# light-weight fast wrapper to make them usable at run-time.

# As such, this code knows about the internal structure of these lists, and
# any change made to that has to be done here as well.  A random number stored
# in the headers is used to minimize the possibility of things getting
# out-of-sync, or the wrong data structure being passed.  Currently that
# random number is:

# charclass_invlists.h now also has a partial implementation of inversion
# maps; enough to generate tables for the line break properties, such as GCB

my $VERSION_DATA_STRUCTURE_TYPE = 148565664;

# integer or float
my $numeric_re = qr/ ^ -? \d+ (:? \. \d+ )? $ /ax;

# Matches valid C language enum names: begins with ASCII alphabetic, then any
# ASCII \w
my $enum_name_re = qr / ^ [[:alpha:]] \w* $ /ax;

my $out_fh = open_new('charclass_invlists.h', '>',
		      {style => '*', by => $0,
                      from => "Unicode::UCD"});

my $in_file_pound_if = 0;

print $out_fh "/* See the generating file for comments */\n\n";

# The symbols generated by this program are all currently defined only in a
# single dot c each.  The code knows where most of them go, but this hash
# gives overrides for the exceptions to the typical place
my %exceptions_to_where_to_define =
                        ( NonL1_Perl_Non_Final_Folds => 'PERL_IN_REGCOMP_C',
                          AboveLatin1                => 'PERL_IN_REGCOMP_C',
                          Latin1                     => 'PERL_IN_REGCOMP_C',
                          UpperLatin1                => 'PERL_IN_REGCOMP_C',
                          _Perl_Any_Folds            => 'PERL_IN_REGCOMP_C',
                          _Perl_Folds_To_Multi_Char  => 'PERL_IN_REGCOMP_C',
                          _Perl_IDCont               => 'PERL_IN_UTF8_C',
                          _Perl_IDStart              => 'PERL_IN_UTF8_C',
                        );

# This hash contains the properties with enums that have hard-coded references
# to them in C code.  Its only use is to make sure that if perl is compiled
# with an older Unicode data set, that all the enum values the code is
# expecting will still be in the enum typedef.  Thus the code doesn't have to
# change.  The Unicode version won't have any code points that have these enum
# values, so the code that handles them will not get exercised.  This is far
# better than having to #ifdef things.
my %hard_coded_enums =
 ( gcb => [
            'Control',
            'CR',
            'Extend',
            'L',
            'LF',
            'LV',
            'LVT',
            'Other',
            'Prepend',
            'Regional_Indicator',
            'SpacingMark',
            'T',
            'V',
        ],
    lb => [
            'Alphabetic',
            'Break_After',
            'Break_Before',
            'Break_Both',
            'Break_Symbols',
            'Carriage_Return',
            'Close_Parenthesis',
            'Close_Punctuation',
            'Combining_Mark',
            'Contingent_Break',
            'Exclamation',
            'Glue',
            'H2',
            'H3',
            'Hebrew_Letter',
            'Hyphen',
            'Ideographic',
            'Infix_Numeric',
            'Inseparable',
            'JL',
            'JT',
            'JV',
            'Line_Feed',
            'Mandatory_Break',
            'Next_Line',
            'Nonstarter',
            'Numeric',
            'Open_Punctuation',
            'Postfix_Numeric',
            'Prefix_Numeric',
            'Quotation',
            'Regional_Indicator',
            'Space',
            'Word_Joiner',
            'ZWSpace',
        ],
   sb  => [
            'ATerm',
            'Close',
            'CR',
            'Extend',
            'Format',
            'LF',
            'Lower',
            'Numeric',
            'OLetter',
            'Other',
            'SContinue',
            'Sep',
            'Sp',
            'STerm',
            'Upper',
        ],
   wb  => [
            'ALetter',
            'CR',
            'Double_Quote',
            'Extend',
            'ExtendNumLet',
            'Format',
            'Hebrew_Letter',
            'Katakana',
            'LF',
            'MidLetter',
            'MidNum',
            'MidNumLet',
            'Newline',
            'Numeric',
            'Other',
            'Perl_Tailored_HSpace',
            'Regional_Indicator',
            'Single_Quote',
        ],
);

my %lb_enums;
my @lb_short_enums;

my @a2n;

sub uniques {
    # Returns non-duplicated input values.  From "Perl Best Practices:
    # Encapsulated Cleverness".  p. 455 in first edition.

    my %seen;
    return grep { ! $seen{$_}++ } @_;
}

sub a2n($) {
    my $cp = shift;

    # Returns the input Unicode code point translated to native.

    return $cp if $cp !~ $numeric_re || $cp > 255;
    return $a2n[$cp];
}

sub end_file_pound_if {
    if ($in_file_pound_if) {
        print $out_fh "\n#endif\t/* $in_file_pound_if */\n";
        $in_file_pound_if = 0;
    }
}

sub switch_pound_if ($$) {
    my $name = shift;
    my $new_pound_if = shift;

    # Switch to new #if given by the 2nd argument.  If there is an override
    # for this, it instead switches to that.  The 1st argument is the
    # static's name, used to look up the overrides

    if (exists $exceptions_to_where_to_define{$name}) {
        $new_pound_if = $exceptions_to_where_to_define{$name};
    }

    # Exit current #if if the new one is different from the old
    if ($in_file_pound_if
        && $in_file_pound_if !~ /$new_pound_if/)
    {
        end_file_pound_if;
    }

    # Enter new #if, if not already in it.
    if (! $in_file_pound_if) {
        $in_file_pound_if = "defined($new_pound_if)";
        print $out_fh "\n#if $in_file_pound_if\n";
    }
}

sub output_invlist ($$;$) {
    my $name = shift;
    my $invlist = shift;     # Reference to inversion list array
    my $charset = shift // "";  # name of character set for comment

    die "No inversion list for $name" unless defined $invlist
                                             && ref $invlist eq 'ARRAY';

    # Output the inversion list $invlist using the name $name for it.
    # It is output in the exact internal form for inversion lists.

    # Is the last element of the header 0, or 1 ?
    my $zero_or_one = 0;
    if (@$invlist && $invlist->[0] != 0) {
        unshift @$invlist, 0;
        $zero_or_one = 1;
    }
    my $count = @$invlist;

    switch_pound_if ($name, 'PERL_IN_PERL_C');

    print $out_fh "\nstatic const UV ${name}_invlist[] = {";
    print $out_fh " /* for $charset */" if $charset;
    print $out_fh "\n";

    print $out_fh "\t$count,\t/* Number of elements */\n";
    print $out_fh "\t$VERSION_DATA_STRUCTURE_TYPE, /* Version and data structure type */\n";
    print $out_fh "\t", $zero_or_one,
                  ",\t/* 0 if the list starts at 0;",
                  "\n\t\t   1 if it starts at the element beyond 0 */\n";

    # The main body are the UVs passed in to this routine.  Do the final
    # element separately
    for my $i (0 .. @$invlist - 1) {
        printf $out_fh "\t0x%X", $invlist->[$i];
        print $out_fh "," if $i < @$invlist - 1;
        print $out_fh "\n";
    }

    print $out_fh "};\n";
}

sub output_invmap ($$$$$$$) {
    my $name = shift;
    my $invmap = shift;     # Reference to inversion map array
    my $prop_name = shift;
    my $input_format = shift;   # The inversion map's format
    my $default = shift;        # The property value for code points who
                                # otherwise don't have a value specified.
    my $extra_enums = shift;    # comma-separated list of our additions to the
                                # property's standard possible values
    my $charset = shift // "";  # name of character set for comment

    # Output the inversion map $invmap for property $prop_name, but use $name
    # as the actual data structure's name.

    my $count = @$invmap;

    my $output_format;
    my $declaration_type;
    my %enums;
    my $name_prefix;

    if ($input_format eq 's') {
        my $orig_prop_name = $prop_name;
        $prop_name = (prop_aliases($prop_name))[1] // $prop_name =~ s/^_Perl_//r; # Get full name
        my $short_name = (prop_aliases($prop_name))[0] // $prop_name;
            my @enums;
            if ($orig_prop_name eq $prop_name) {
                @enums = prop_values($prop_name);
            }
            else {
                @enums = uniques(@$invmap);
            }
            if (! @enums) {
                die "Only enum properties are currently handled; '$prop_name' isn't one";
            }
            else {

                # Convert short names to long
                @enums = map { (prop_value_aliases($prop_name, $_))[1] } @enums;

                my @expected_enums = @{$hard_coded_enums{lc $short_name}};
                die 'You need to update %hard_coded_enums to reflect new entries in this Unicode version'
                    if @expected_enums < @enums;

                # Remove the enums found in the input from the ones we expect
                for (my $i = @expected_enums - 1; $i >= 0; $i--) {
                    splice(@expected_enums, $i, 1)
                                if grep { $expected_enums[$i] eq $_ } @enums;
                }

                # The ones remaining must be because we're using an older
                # Unicode version.  Add them to the list.
                push @enums, @expected_enums;

                # Add in the extra values coded into this program, and sort.
                @enums = sort @enums;

                # The internal enums comes last.
                push @enums, split /,/, $extra_enums if $extra_enums ne "";

                # Assign a value to each element of the enum.  The default
                # value always gets 0; the others are arbitrarily assigned.
                my $enum_val = 0;
                my $canonical_default = prop_value_aliases($prop_name, $default);
                $default = $canonical_default if defined $canonical_default;
                $enums{$default} = $enum_val++;
                for my $enum (@enums) {
                    $enums{$enum} = $enum_val++ unless exists $enums{$enum};
                }

                # Calculate the enum values for property _Perl_LB because we
                # output a special table for that
                if ($name eq '_Perl_LB' && ! %lb_enums) {
                    while (my ($enum, $value) = each %enums) {
                        my ($short) = prop_value_aliases('LB', $enum);
                        $short = substr(lc $enum, 0, 2) unless defined $short;
                        $lb_enums{$short} = $value;
                        @lb_short_enums[$value] = $short;
                    }
                }
            }

            # Inversion map stuff is currently used only by regexec
            switch_pound_if($name, 'PERL_IN_REGEXEC_C');
        {

            # The short names tend to be two lower case letters, but it looks
            # better for those if they are upper. XXX
            $short_name = uc($short_name) if length($short_name) < 3
                                             || substr($short_name, 0, 1) =~ /[[:lower:]]/;
            $name_prefix = "${short_name}_";
            my $enum_count = keys %enums;
            print $out_fh "\n#define ${name_prefix}ENUM_COUNT ", scalar keys %enums, "\n";

            print $out_fh "\ntypedef enum {\n";
            my @enum_list;
            foreach my $enum (keys %enums) {
                $enum_list[$enums{$enum}] = $enum;
            }
            foreach my $i (0 .. @enum_list - 1) {
                my $name = $enum_list[$i];
                print $out_fh  "\t${name_prefix}$name = $i";
                print $out_fh "," if $i < $enum_count - 1;
                print $out_fh "\n";
            }
            $declaration_type = "${name_prefix}enum";
            print $out_fh "} $declaration_type;\n";

            $output_format = "${name_prefix}%s";
        }
    }
    else {
        die "'$input_format' invmap() format for '$prop_name' unimplemented";
    }

    die "No inversion map for $prop_name" unless defined $invmap
                                             && ref $invmap eq 'ARRAY'
                                             && $count;

    print $out_fh "\nstatic const $declaration_type ${name}_invmap[] = {";
    print $out_fh " /* for $charset */" if $charset;
    print $out_fh "\n";

    # The main body are the scalars passed in to this routine.
    for my $i (0 .. $count - 1) {
        my $element = $invmap->[$i];
        my $full_element_name = prop_value_aliases($prop_name, $element);
        $element = $full_element_name if defined $full_element_name;
        $element = $name_prefix . $element;
        print $out_fh "\t$element";
        print $out_fh "," if $i < $count - 1;
        print $out_fh  "\n";
    }
    print $out_fh "};\n";
}

sub mk_invlist_from_sorted_cp_list {

    # Returns an inversion list constructed from the sorted input array of
    # code points

    my $list_ref = shift;

    return unless @$list_ref;

    # Initialize to just the first element
    my @invlist = ( $list_ref->[0], $list_ref->[0] + 1);

    # For each succeeding element, if it extends the previous range, adjust
    # up, otherwise add it.
    for my $i (1 .. @$list_ref - 1) {
        if ($invlist[-1] == $list_ref->[$i]) {
            $invlist[-1]++;
        }
        else {
            push @invlist, $list_ref->[$i], $list_ref->[$i] + 1;
        }
    }
    return @invlist;
}

# Read in the Case Folding rules, and construct arrays of code points for the
# properties we need.
my ($cp_ref, $folds_ref, $format) = prop_invmap("Case_Folding");
die "Could not find inversion map for Case_Folding" unless defined $format;
die "Incorrect format '$format' for Case_Folding inversion map"
                                                    unless $format eq 'al'
                                                           || $format eq 'a';
my @has_multi_char_fold;
my @is_non_final_fold;

for my $i (0 .. @$folds_ref - 1) {
    next unless ref $folds_ref->[$i];   # Skip single-char folds
    push @has_multi_char_fold, $cp_ref->[$i];

    # Add to the non-finals list each code point that is in a non-final
    # position
    for my $j (0 .. @{$folds_ref->[$i]} - 2) {
        push @is_non_final_fold, $folds_ref->[$i][$j]
                unless grep { $folds_ref->[$i][$j] == $_ } @is_non_final_fold;
    }
}

sub _Perl_Non_Final_Folds {
    @is_non_final_fold = sort { $a <=> $b } @is_non_final_fold;
    return mk_invlist_from_sorted_cp_list(\@is_non_final_fold);
}

sub prop_name_for_cmp ($) { # Sort helper
    my $name = shift;

    # Returns the input lowercased, with non-alphas removed, as well as
    # everything starting with a comma

    $name =~ s/,.*//;
    $name =~ s/[[:^alpha:]]//g;
    return lc $name;
}

sub UpperLatin1 {
    return mk_invlist_from_sorted_cp_list([ 128 .. 255 ]);
}

sub output_LB_table() {

    # Create and output the enums, #defines, and pair table for use in
    # determining Line Breaks.  This uses the default line break algorithm,
    # given in http://www.unicode.org/reports/tr14/, but tailored by example 7
    # in that page, as the Unicode-furnished tests assume that tailoring.

    switch_pound_if('LB_table', 'PERL_IN_REGEXEC_C');

    # The result is really just true or false.  But we follow along with tr14,
    # creating a rule which is false for something like X SP* X.  That gets
    # encoding 2.  The rest of the actions are synthetic ones that indicate
    # some context handling is required.  These each are added to the
    # underlying 0, 1, or 2, instead of replacing them, so that the underlying
    # value can be retrieved.  Actually only rules from 7 through 18 (which
    # are the ones where space matter) are possible to have 2 added to them.
    # The others below add just 0 or 1.  It might be possible for one
    # synthetic rule to be added to another, yielding a larger value.  This
    # doesn't happen in the Unicode 8.0 rule set, and as you can see from the
    # names of the middle grouping below, it is impossible for that to occur
    # for them because they all start with mutually exclusive classes.  That
    # the final rule can't be added to any of the others isn't obvious from
    # its name, so it is assigned a power of 2 higher than the others can get
    # to so any addition would preserve all data.  (And the code will reach an
    # assert(0) on debugging builds should this happen.)
    my %lb_actions = (
        LB_NOBREAK                      => 0,
        LB_BREAKABLE                    => 1,
        LB_NOBREAK_EVEN_WITH_SP_BETWEEN => 2,

        LB_CM_foo                       => 3,   # Rule 9
        LB_SP_foo                       => 6,   # Rule 18
        LB_PR_or_PO_then_OP_or_HY       => 9,   # Rule 25
        LB_SY_or_IS_then_various        => 11,  # Rule 25
        LB_HY_or_BA_then_foo            => 13,  # Rule 21

        LB_various_then_PO_or_PR        => (1<<4),  # Rule 25
    );

    # Output the #define list, sorted by numeric value
    my @defines;
    while (my ($enum, $value) = each %lb_actions) {
        $defines[$value] = $enum;
    }

    print $out_fh "\n";

    foreach my $i (0 .. @defines - 1) {
        next unless defined $defines[$i];
        print $out_fh "#define $defines[$i]\t$i\n";
    }

    # Construct the LB pair table.  This is based on the rules in
    # http://www.unicode.org/reports/tr14/, but modified as those rules are
    # designed for someone taking a string of text and sequentially going
    # through it to find the break opportunities, whereas, Perl requires
    # determining if a given random spot is a break opportunity, without
    # knowing all the entire string before it.
    #
    # The table is constructed in reverse order of the rules, to make the
    # lower-numbered, higher priority ones override the later ones, as the
    # algorithm stops at the earliest matching rule

    my @lb_table;
    my $table_size = @lb_short_enums;

    # LB31. Break everywhere else
    for my $i (0 .. $table_size - 1) {
        for my $j (0 .. $table_size - 1) {
            $lb_table[$i][$j] = $lb_actions{'LB_BREAKABLE'};
        }
    }

    # LB30a. Don't break between Regional Indicators
    $lb_table[$lb_enums{'RI'}][$lb_enums{'RI'}] = $lb_actions{'LB_NOBREAK'};

    # LB30 Do not break between letters, numbers, or ordinary symbols and
    # opening or closing parentheses.
    # (AL | HL | NU) × OP
    $lb_table[$lb_enums{'AL'}][$lb_enums{'OP'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'HL'}][$lb_enums{'OP'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'NU'}][$lb_enums{'OP'}] = $lb_actions{'LB_NOBREAK'};

    # CP × (AL | HL | NU)
    $lb_table[$lb_enums{'CP'}][$lb_enums{'AL'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'CP'}][$lb_enums{'HL'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'CP'}][$lb_enums{'NU'}] = $lb_actions{'LB_NOBREAK'};

    # LB29 Do not break between numeric punctuation and alphabetics (“e.g.”).
    # IS × (AL | HL)
    $lb_table[$lb_enums{'IS'}][$lb_enums{'AL'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'IS'}][$lb_enums{'HL'}] = $lb_actions{'LB_NOBREAK'};

    # LB28 Do not break between alphabetics (“at”).
    # (AL | HL) × (AL | HL)
    $lb_table[$lb_enums{'AL'}][$lb_enums{'AL'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'HL'}][$lb_enums{'AL'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'AL'}][$lb_enums{'HL'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'HL'}][$lb_enums{'HL'}] = $lb_actions{'LB_NOBREAK'};

    # LB27 Treat a Korean Syllable Block the same as ID.
    # (JL | JV | JT | H2 | H3) × IN
    $lb_table[$lb_enums{'JL'}][$lb_enums{'IN'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'JV'}][$lb_enums{'IN'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'JT'}][$lb_enums{'IN'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'H2'}][$lb_enums{'IN'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'H3'}][$lb_enums{'IN'}] = $lb_actions{'LB_NOBREAK'};

    # (JL | JV | JT | H2 | H3) × PO
    $lb_table[$lb_enums{'JL'}][$lb_enums{'PO'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'JV'}][$lb_enums{'PO'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'JT'}][$lb_enums{'PO'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'H2'}][$lb_enums{'PO'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'H3'}][$lb_enums{'PO'}] = $lb_actions{'LB_NOBREAK'};

    # PR × (JL | JV | JT | H2 | H3)
    $lb_table[$lb_enums{'PR'}][$lb_enums{'JL'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'PR'}][$lb_enums{'JV'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'PR'}][$lb_enums{'JT'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'PR'}][$lb_enums{'H2'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'PR'}][$lb_enums{'H3'}] = $lb_actions{'LB_NOBREAK'};

    # LB26 Do not break a Korean syllable.
    # JL × (JL | JV | H2 | H3)
    $lb_table[$lb_enums{'JL'}][$lb_enums{'JL'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'JL'}][$lb_enums{'JV'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'JL'}][$lb_enums{'H2'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'JL'}][$lb_enums{'H3'}] = $lb_actions{'LB_NOBREAK'};

    # (JV | H2) × (JV | JT)
    $lb_table[$lb_enums{'JV'}][$lb_enums{'JV'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'H2'}][$lb_enums{'JV'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'JV'}][$lb_enums{'JT'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'H2'}][$lb_enums{'JT'}] = $lb_actions{'LB_NOBREAK'};

    # (JT | H3) × JT
    $lb_table[$lb_enums{'JT'}][$lb_enums{'JT'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'H3'}][$lb_enums{'JT'}] = $lb_actions{'LB_NOBREAK'};

    # LB25 Do not break between the following pairs of classes relevant to
    # numbers, as tailored by example 7 in
    # http://www.unicode.org/reports/tr14/#Examples
    # We follow that tailoring because Unicode's test cases expect it
    # (PR | PO) × ( OP | HY )? NU
    $lb_table[$lb_enums{'PR'}][$lb_enums{'NU'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'PO'}][$lb_enums{'NU'}] = $lb_actions{'LB_NOBREAK'};

        # Given that (OP | HY )? is optional, we have to test for it in code.
        # We add in the action (instead of overriding) for this, so that in
        # the code we can recover the underlying break value.
    $lb_table[$lb_enums{'PR'}][$lb_enums{'OP'}]
                                    += $lb_actions{'LB_PR_or_PO_then_OP_or_HY'};
    $lb_table[$lb_enums{'PO'}][$lb_enums{'OP'}]
                                    += $lb_actions{'LB_PR_or_PO_then_OP_or_HY'};
    $lb_table[$lb_enums{'PR'}][$lb_enums{'HY'}]
                                    += $lb_actions{'LB_PR_or_PO_then_OP_or_HY'};
    $lb_table[$lb_enums{'PO'}][$lb_enums{'HY'}]
                                    += $lb_actions{'LB_PR_or_PO_then_OP_or_HY'};

    # ( OP | HY ) × NU
    $lb_table[$lb_enums{'OP'}][$lb_enums{'NU'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'HY'}][$lb_enums{'NU'}] = $lb_actions{'LB_NOBREAK'};

    # NU (NU | SY | IS)* × (NU | SY | IS | CL | CP )
    # which can be rewritten as:
    # NU (SY | IS)* × (NU | SY | IS | CL | CP )
    $lb_table[$lb_enums{'NU'}][$lb_enums{'NU'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'NU'}][$lb_enums{'SY'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'NU'}][$lb_enums{'IS'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'NU'}][$lb_enums{'CL'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'NU'}][$lb_enums{'CP'}] = $lb_actions{'LB_NOBREAK'};

        # Like earlier where we have to test in code, we add in the action so
        # that we can recover the underlying values.  This is done in rules
        # below, as well.  The code assumes that we haven't added 2 actions.
        # Shoul a later Unicode release break that assumption, then tests
        # should start failing.
    $lb_table[$lb_enums{'SY'}][$lb_enums{'NU'}]
                                    += $lb_actions{'LB_SY_or_IS_then_various'};
    $lb_table[$lb_enums{'SY'}][$lb_enums{'SY'}]
                                    += $lb_actions{'LB_SY_or_IS_then_various'};
    $lb_table[$lb_enums{'SY'}][$lb_enums{'IS'}]
                                    += $lb_actions{'LB_SY_or_IS_then_various'};
    $lb_table[$lb_enums{'SY'}][$lb_enums{'CL'}]
                                    += $lb_actions{'LB_SY_or_IS_then_various'};
    $lb_table[$lb_enums{'SY'}][$lb_enums{'CP'}]
                                    += $lb_actions{'LB_SY_or_IS_then_various'};
    $lb_table[$lb_enums{'IS'}][$lb_enums{'NU'}]
                                    += $lb_actions{'LB_SY_or_IS_then_various'};
    $lb_table[$lb_enums{'IS'}][$lb_enums{'SY'}]
                                    += $lb_actions{'LB_SY_or_IS_then_various'};
    $lb_table[$lb_enums{'IS'}][$lb_enums{'IS'}]
                                    += $lb_actions{'LB_SY_or_IS_then_various'};
    $lb_table[$lb_enums{'IS'}][$lb_enums{'CL'}]
                                    += $lb_actions{'LB_SY_or_IS_then_various'};
    $lb_table[$lb_enums{'IS'}][$lb_enums{'CP'}]
                                    += $lb_actions{'LB_SY_or_IS_then_various'};

    # NU (NU | SY | IS)* (CL | CP)? × (PO | PR)
    # which can be rewritten as:
    # NU (SY | IS)* (CL | CP)? × (PO | PR)
    $lb_table[$lb_enums{'NU'}][$lb_enums{'PO'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'NU'}][$lb_enums{'PR'}] = $lb_actions{'LB_NOBREAK'};

    $lb_table[$lb_enums{'CP'}][$lb_enums{'PO'}]
                                    += $lb_actions{'LB_various_then_PO_or_PR'};
    $lb_table[$lb_enums{'CL'}][$lb_enums{'PO'}]
                                    += $lb_actions{'LB_various_then_PO_or_PR'};
    $lb_table[$lb_enums{'IS'}][$lb_enums{'PO'}]
                                    += $lb_actions{'LB_various_then_PO_or_PR'};
    $lb_table[$lb_enums{'SY'}][$lb_enums{'PO'}]
                                    += $lb_actions{'LB_various_then_PO_or_PR'};

    $lb_table[$lb_enums{'CP'}][$lb_enums{'PR'}]
                                    += $lb_actions{'LB_various_then_PO_or_PR'};
    $lb_table[$lb_enums{'CL'}][$lb_enums{'PR'}]
                                    += $lb_actions{'LB_various_then_PO_or_PR'};
    $lb_table[$lb_enums{'IS'}][$lb_enums{'PR'}]
                                    += $lb_actions{'LB_various_then_PO_or_PR'};
    $lb_table[$lb_enums{'SY'}][$lb_enums{'PR'}]
                                    += $lb_actions{'LB_various_then_PO_or_PR'};

    # LB24 Do not break between prefix and letters or ideographs.
    # PR × ID
    $lb_table[$lb_enums{'PR'}][$lb_enums{'ID'}] = $lb_actions{'LB_NOBREAK'};

    # PR × (AL | HL)
    $lb_table[$lb_enums{'PR'}][$lb_enums{'AL'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'PR'}][$lb_enums{'HL'}] = $lb_actions{'LB_NOBREAK'};

    # PO × (AL | HL)
    $lb_table[$lb_enums{'PO'}][$lb_enums{'AL'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'PO'}][$lb_enums{'HL'}] = $lb_actions{'LB_NOBREAK'};

    # LB23 Do not break within ‘a9’, ‘3a’, or ‘H%’.
    # ID × PO
    $lb_table[$lb_enums{'ID'}][$lb_enums{'PO'}] = $lb_actions{'LB_NOBREAK'};

    # (AL | HL) × NU
    $lb_table[$lb_enums{'AL'}][$lb_enums{'NU'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'HL'}][$lb_enums{'NU'}] = $lb_actions{'LB_NOBREAK'};

    # NU × (AL | HL)
    $lb_table[$lb_enums{'NU'}][$lb_enums{'AL'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'NU'}][$lb_enums{'HL'}] = $lb_actions{'LB_NOBREAK'};

    # LB22 Do not break between two ellipses, or between letters, numbers or
    # exclamations and ellipsis.
    # (AL | HL) × IN
    $lb_table[$lb_enums{'AL'}][$lb_enums{'IN'}] = $lb_actions{'LB_NOBREAK'};
    $lb_table[$lb_enums{'HL'}][$lb_enums{'IN'}] = $lb_actions{'LB_NOBREAK'};

    # EX × IN
    $lb_table[$lb_enums{'EX'}][$lb_enums{'IN'}] = $lb_actions{'LB_NOBREAK'};

    # ID × IN
    $lb_table[$lb_enums{'ID'}][$lb_enums{'IN'}] = $lb_actions{'LB_NOBREAK'};

    # IN × IN
    $lb_table[$lb_enums{'IN'}][$lb_enums{'IN'}] = $lb_actions{'LB_NOBREAK'};

    # NU × IN
    $lb_table[$lb_enums{'NU'}][$lb_enums{'IN'}] = $lb_actions{'LB_NOBREAK'};

    # LB21b Don’t break between Solidus and Hebrew letters.
    # SY × HL
    $lb_table[$lb_enums{'SY'}][$lb_enums{'HL'}] = $lb_actions{'LB_NOBREAK'};

    # LB21a Don't break after Hebrew + Hyphen.
    # HL (HY | BA) ×
    for my $i (0 .. @lb_table - 1) {
        $lb_table[$lb_enums{'HY'}][$i] += $lb_actions{'LB_HY_or_BA_then_foo'};
        $lb_table[$lb_enums{'BA'}][$i] += $lb_actions{'LB_HY_or_BA_then_foo'};
    }

    # LB21 Do not break before hyphen-minus, other hyphens, fixed-width
    # spaces, small kana, and other non-starters, or after acute accents.
    # × BA
    # × HY
    # × NS
    # BB ×
    for my $i (0 .. @lb_table - 1) {
        $lb_table[$i][$lb_enums{'BA'}] = $lb_actions{'LB_NOBREAK'};
        $lb_table[$i][$lb_enums{'HY'}] = $lb_actions{'LB_NOBREAK'};
        $lb_table[$i][$lb_enums{'NS'}] = $lb_actions{'LB_NOBREAK'};
        $lb_table[$lb_enums{'BB'}][$i] = $lb_actions{'LB_NOBREAK'};
    }

    # LB20 Break before and after unresolved CB.
    # ÷ CB
    # CB ÷
    # Conditional breaks should be resolved external to the line breaking
    # rules. However, the default action is to treat unresolved CB as breaking
    # before and after.
    for my $i (0 .. @lb_table - 1) {
        $lb_table[$i][$lb_enums{'CB'}] = $lb_actions{'LB_BREAKABLE'};
        $lb_table[$lb_enums{'CB'}][$i] = $lb_actions{'LB_BREAKABLE'};
    }

    # LB19 Do not break before or after quotation marks, such as ‘ ” ’.
    # × QU
    # QU ×
    for my $i (0 .. @lb_table - 1) {
        $lb_table[$i][$lb_enums{'QU'}] = $lb_actions{'LB_NOBREAK'};
        $lb_table[$lb_enums{'QU'}][$i] = $lb_actions{'LB_NOBREAK'};
    }

    # LB18 Break after spaces
    # SP ÷
    for my $i (0 .. @lb_table - 1) {
        $lb_table[$lb_enums{'SP'}][$i] = $lb_actions{'LB_BREAKABLE'};
    }

    # LB17 Do not break within ‘——’, even with intervening spaces.
    # B2 SP* × B2
    $lb_table[$lb_enums{'B2'}][$lb_enums{'B2'}]
                           = $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'};

    # LB16 Do not break between closing punctuation and a nonstarter even with
    # intervening spaces.
    # (CL | CP) SP* × NS
    $lb_table[$lb_enums{'CL'}][$lb_enums{'NS'}]
                            = $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'};
    $lb_table[$lb_enums{'CP'}][$lb_enums{'NS'}]
                            = $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'};


    # LB15 Do not break within ‘”[’, even with intervening spaces.
    # QU SP* × OP
    $lb_table[$lb_enums{'QU'}][$lb_enums{'OP'}]
                            = $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'};

    # LB14 Do not break after ‘[’, even after spaces.
    # OP SP* ×
    for my $i (0 .. @lb_table - 1) {
        $lb_table[$lb_enums{'OP'}][$i]
                            = $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'};
    }

    # LB13 Do not break before ‘]’ or ‘!’ or ‘;’ or ‘/’, even after spaces, as
    # tailored by example 7 in http://www.unicode.org/reports/tr14/#Examples
    # [^NU] × CL
    # [^NU] × CP
    # × EX
    # [^NU] × IS
    # [^NU] × SY
    for my $i (0 .. @lb_table - 1) {
        $lb_table[$i][$lb_enums{'EX'}]
                            = $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'};

        next if $i == $lb_enums{'NU'};

        $lb_table[$i][$lb_enums{'CL'}]
                            = $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'};
        $lb_table[$i][$lb_enums{'CP'}]
                            = $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'};
        $lb_table[$i][$lb_enums{'IS'}]
                            = $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'};
        $lb_table[$i][$lb_enums{'SY'}]
                            = $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'};
    }

    # LB12a Do not break before NBSP and related characters, except after
    # spaces and hyphens.
    # [^SP BA HY] × GL
    for my $i (0 .. @lb_table - 1) {
        next if    $i == $lb_enums{'SP'}
                || $i == $lb_enums{'BA'}
                || $i == $lb_enums{'HY'};

        # We don't break, but if a property above has said don't break even
        # with space between, don't override that (also in the next few rules)
        next if $lb_table[$i][$lb_enums{'GL'}]
                            == $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'};
        $lb_table[$i][$lb_enums{'GL'}] = $lb_actions{'LB_NOBREAK'};
    }

    # LB12 Do not break after NBSP and related characters.
    # GL ×
    for my $i (0 .. @lb_table - 1) {
        next if $lb_table[$lb_enums{'GL'}][$i]
                            == $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'};
        $lb_table[$lb_enums{'GL'}][$i] = $lb_actions{'LB_NOBREAK'};
    }

    # LB11 Do not break before or after Word joiner and related characters.
    # × WJ
    # WJ ×
    for my $i (0 .. @lb_table - 1) {
        if ($lb_table[$i][$lb_enums{'WJ'}]
                        != $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'})
        {
            $lb_table[$i][$lb_enums{'WJ'}] = $lb_actions{'LB_NOBREAK'};
        }
        if ($lb_table[$lb_enums{'WJ'}][$i]
                        != $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'})
        {
            $lb_table[$lb_enums{'WJ'}][$i] = $lb_actions{'LB_NOBREAK'};
        }
    }

    # Special case this here to avoid having to do a special case in the code,
    # by making this the same as other things with a SP in front of them that
    # don't break, we avoid an extra test
    $lb_table[$lb_enums{'SP'}][$lb_enums{'WJ'}]
                            = $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'};

    # LB9 and LB10 are done in the same loop
    #
    # LB9 Do not break a combining character sequence; treat it as if it has
    # the line breaking class of the base character in all of the
    # higher-numbered rules.
    # Treat X CM* as if it were X.
    # where X is any line break class except BK, CR, LF, NL, SP, or ZW.

    # LB10 Treat any remaining combining mark as AL.  This catches the case
    # where a CM is the first character on the line or follows SP, BK, CR, LF,
    # NL, or ZW.
    for my $i (0 .. @lb_table - 1) {

        # When the CM is the first in the pair, we don't know without looking
        # behind whether the CM is going to inherit from an earlier character,
        # or not.  So have to figure this out in the code
        $lb_table[$lb_enums{'CM'}][$i] = $lb_actions{'LB_CM_foo'};

        if (   $i == $lb_enums{'BK'}
            || $i == $lb_enums{'ed'}
            || $i == $lb_enums{'CR'}
            || $i == $lb_enums{'LF'}
            || $i == $lb_enums{'NL'}
            || $i == $lb_enums{'SP'}
            || $i == $lb_enums{'ZW'})
        {
            # For these classes, a following CM doesn't combine, and should do
            # whatever 'AL' would do.
            $lb_table[$i][$lb_enums{'CM'}] = $lb_table[$i][$lb_enums{'AL'}];
        }
        else {
            # For these classes, the CM combines, so doesn't break, inheriting
            # the type of nobreak from the master character.
            if ($lb_table[$i][$lb_enums{'CM'}]
                            != $lb_actions{'LB_NOBREAK_EVEN_WITH_SP_BETWEEN'})
            {
                $lb_table[$i][$lb_enums{'CM'}] = $lb_actions{'LB_NOBREAK'};
            }
        }
    }

    # LB8 Break before any character following a zero-width space, even if one
    # or more spaces intervene.
    # ZW SP* ÷
    for my $i (0 .. @lb_table - 1) {
        $lb_table[$lb_enums{'ZW'}][$i] = $lb_actions{'LB_BREAKABLE'};
    }

    # Because of LB8-10, we need to look at context for "SP x", and this must
    # be done in the code.  So override the existing rules for that, by adding
    # a constant to get new rules that tell the code it needs to look at
    # context.  By adding this action instead of replacing the existing one,
    # we can get back to the original rule if necessary.
    for my $i (0 .. @lb_table - 1) {
        $lb_table[$lb_enums{'SP'}][$i] += $lb_actions{'LB_SP_foo'};
    }

    # LB7 Do not break before spaces or zero width space.
    # × SP
    # × ZW
    for my $i (0 .. @lb_table - 1) {
        $lb_table[$i][$lb_enums{'SP'}] = $lb_actions{'LB_NOBREAK'};
        $lb_table[$i][$lb_enums{'ZW'}] = $lb_actions{'LB_NOBREAK'};
    }

    # LB6 Do not break before hard line breaks.
    # × ( BK | CR | LF | NL )
    for my $i (0 .. @lb_table - 1) {
        $lb_table[$i][$lb_enums{'BK'}] = $lb_actions{'LB_NOBREAK'};
        $lb_table[$i][$lb_enums{'CR'}] = $lb_actions{'LB_NOBREAK'};
        $lb_table[$i][$lb_enums{'LF'}] = $lb_actions{'LB_NOBREAK'};
        $lb_table[$i][$lb_enums{'NL'}] = $lb_actions{'LB_NOBREAK'};
    }

    # LB5 Treat CR followed by LF, as well as CR, LF, and NL as hard line breaks.
    # CR × LF
    # CR !
    # LF !
    # NL !
    for my $i (0 .. @lb_table - 1) {
        $lb_table[$lb_enums{'CR'}][$i] = $lb_actions{'LB_BREAKABLE'};
        $lb_table[$lb_enums{'LF'}][$i] = $lb_actions{'LB_BREAKABLE'};
        $lb_table[$lb_enums{'NL'}][$i] = $lb_actions{'LB_BREAKABLE'};
    }
    $lb_table[$lb_enums{'CR'}][$lb_enums{'LF'}] = $lb_actions{'LB_NOBREAK'};

    # LB4 Always break after hard line breaks.
    # BK !
    for my $i (0 .. @lb_table - 1) {
        $lb_table[$lb_enums{'BK'}][$i] = $lb_actions{'LB_BREAKABLE'};
    }

    # LB2 Never break at the start of text.
    # sot ×
    # LB3 Always break at the end of text.
    # ! eot
    # but these are reversed in the loop below, so that won't break if there
    # is no text
    for my $i (0 .. @lb_table - 1) {
        $lb_table[$i][$lb_enums{'ed'}] = $lb_actions{'LB_BREAKABLE'};
        $lb_table[$lb_enums{'ed'}][$i] = $lb_actions{'LB_NOBREAK'};
    }

    # LB1 Assign a line breaking class to each code point of the input.
    # Resolve AI, CB, CJ, SA, SG, and XX into other line breaking classes
    # depending on criteria outside the scope of this algorithm.
    #
    # In the absence of such criteria all characters with a specific
    # combination of original class and General_Category property value are
    # resolved as follows:
    # Original 	   Resolved  General_Category
    # AI, SG, XX      AL      Any
    # SA              CM      Only Mn or Mc
    # SA              AL      Any except Mn and Mc
    # CJ              NS      Any
    #
    # This is done in mktables, so we never see any of the remapped-from
    # classes.

    print $out_fh "\nstatic const U8 LB_table[$table_size][$table_size] = {\n";
    print $out_fh "\n/* 'ed' stands for 'edge' */\n";
    print $out_fh "/*      ";
    for my $i (0 .. @lb_table - 1) {
        print $out_fh "  $lb_short_enums[$i]";
    }
    print $out_fh " */\n";

    for my $i (0 .. @lb_table - 1) {
        print $out_fh "/* $lb_short_enums[$i] */ ";
        for my $j (0 .. @lb_table - 1) {
            printf $out_fh "%2d", $lb_table[$i][$j];
            print $out_fh "," if $i < @lb_table - 1 || $j < @lb_table - 1;
            print $out_fh " " if $j < @lb_table - 1;
        }
        print $out_fh "\n";
    }

    print $out_fh "};\n";

    end_file_pound_if;
}

output_invlist("Latin1", [ 0, 256 ]);
output_invlist("AboveLatin1", [ 256 ]);

end_file_pound_if;

# We construct lists for all the POSIX and backslash sequence character
# classes in two forms:
#   1) ones which match only in the ASCII range
#   2) ones which match either in the Latin1 range, or the entire Unicode range
#
# These get compiled in, and hence affect the memory footprint of every Perl
# program, even those not using Unicode.  To minimize the size, currently
# the Latin1 version is generated for the beyond ASCII range except for those
# lists that are quite small for the entire range, such as for \s, which is 22
# UVs long plus 4 UVs (currently) for the header.
#
# To save even more memory, the ASCII versions could be derived from the
# larger ones at runtime, saving some memory (minus the expense of the machine
# instructions to do so), but these are all small anyway, so their total is
# about 100 UVs.
#
# In the list of properties below that get generated, the L1 prefix is a fake
# property that means just the Latin1 range of the full property (whose name
# has an X prefix instead of L1).
#
# An initial & means to use the subroutine from this file instead of an
# official inversion list.

for my $charset (get_supported_code_pages()) {
    print $out_fh "\n" . get_conditional_compile_line_start($charset);

    @a2n = @{get_a2n($charset)};
    no warnings 'qw';
                         # Ignore non-alpha in sort
    for my $prop (sort { prop_name_for_cmp($a) cmp prop_name_for_cmp($b) } qw(
                             ASCII
                             Cased
                             VertSpace
                             XPerlSpace
                             XPosixAlnum
                             XPosixAlpha
                             XPosixBlank
                             XPosixCntrl
                             XPosixDigit
                             XPosixGraph
                             XPosixLower
                             XPosixPrint
                             XPosixPunct
                             XPosixSpace
                             XPosixUpper
                             XPosixWord
                             XPosixXDigit
                             _Perl_Any_Folds
                             &NonL1_Perl_Non_Final_Folds
                             _Perl_Folds_To_Multi_Char
                             &UpperLatin1
                             _Perl_IDStart
                             _Perl_IDCont
                             _Perl_GCB,EDGE
                             _Perl_LB,EDGE
                             _Perl_SB,EDGE
                             _Perl_WB,EDGE,UNKNOWN
                           )
    ) {

        # For the Latin1 properties, we change to use the eXtended version of the
        # base property, then go through the result and get rid of everything not
        # in Latin1 (above 255).  Actually, we retain the element for the range
        # that crosses the 255/256 boundary if it is one that matches the
        # property.  For example, in the Word property, there is a range of code
        # points that start at U+00F8 and goes through U+02C1.  Instead of
        # artificially cutting that off at 256 because 256 is the first code point
        # above Latin1, we let the range go to its natural ending.  That gives us
        # extra information with no added space taken.  But if the range that
        # crosses the boundary is one that doesn't match the property, we don't
        # start a new range above 255, as that could be construed as going to
        # infinity.  For example, the Upper property doesn't include the character
        # at 255, but does include the one at 256.  We don't include the 256 one.
        my $prop_name = $prop;
        my $is_local_sub = $prop_name =~ s/^&//;
        my $extra_enums = "";
        $extra_enums = $1 if $prop_name =~ s/, ( .* ) //x;
        my $lookup_prop = $prop_name;
        my $l1_only = ($lookup_prop =~ s/^L1Posix/XPosix/
                       or $lookup_prop =~ s/^L1//);
        my $nonl1_only = 0;
        $nonl1_only = $lookup_prop =~ s/^NonL1// unless $l1_only;
        ($lookup_prop, my $has_suffixes) = $lookup_prop =~ / (.*) ( , .* )? /x;

        my @invlist;
        my @invmap;
        my $map_format;
        my $map_default;
        my $maps_to_code_point;
        my $to_adjust;
        if ($is_local_sub) {
            @invlist = eval $lookup_prop;
        }
        else {
            @invlist = prop_invlist($lookup_prop, '_perl_core_internal_ok');
            if (! @invlist) {

                # If couldn't find a non-empty inversion list, see if it is
                # instead an inversion map
                my ($list_ref, $map_ref, $format, $default)
                          = prop_invmap($lookup_prop, '_perl_core_internal_ok');
                if (! $list_ref) {
                    # An empty return here could mean an unknown property, or
                    # merely that the original inversion list is empty.  Call
                    # in scalar context to differentiate
                    my $count = prop_invlist($lookup_prop,
                                             '_perl_core_internal_ok');
                    die "Could not find inversion list for '$lookup_prop'"
                                                          unless defined $count;
                }
                else {
                    @invlist = @$list_ref;
                    @invmap = @$map_ref;
                    $map_format = $format;
                    $map_default = $default;
                    $maps_to_code_point = $map_format =~ /x/;
                    $to_adjust = $map_format =~ /a/;
                }
            }
        }


        # Short-circuit an empty inversion list.
        if (! @invlist) {
            output_invlist($prop_name, \@invlist, $charset);
            next;
        }

        # Re-order the Unicode code points to native ones for this platform.
        # This is only needed for code points below 256, because native code
        # points are only in that range.  For inversion maps of properties
        # where the mappings are adjusted (format =~ /a/), this reordering
        # could mess up the adjustment pattern that was in the input, so that
        # has to be dealt with.
        #
        # And inversion maps that map to code points need to eventually have
        # all those code points remapped to native, and it's better to do that
        # here, going through the whole list not just those below 256.  This
        # is because some inversion maps have adjustments (format =~ /a/)
        # which may be affected by the reordering.  This code needs to be done
        # both for when we are translating the inversion lists for < 256, and
        # for the inversion maps for everything.  By doing both in this loop,
        # we can share that code.
        #
        # So, we go through everything for an inversion map to code points;
        # otherwise, we can skip any remapping at all if we are going to
        # output only the above-Latin1 values, or if the range spans the whole
        # of 0..256, as the remap will also include all of 0..256  (256 not
        # 255 because a re-ordering could cause 256 to need to be in the same
        # range as 255.)
        if ((@invmap && $maps_to_code_point)
            || (! $nonl1_only || ($invlist[0] < 256
                                  && ! ($invlist[0] == 0 && $invlist[1] > 256))))
        {

            if (! @invmap) {    # Straight inversion list
            # Look at all the ranges that start before 257.
            my @latin1_list;
            while (@invlist) {
                last if $invlist[0] > 256;
                my $upper = @invlist > 1
                            ? $invlist[1] - 1      # In range

                              # To infinity.  You may want to stop much much
                              # earlier; going this high may expose perl
                              # deficiencies with very large numbers.
                            : $Unicode::UCD::MAX_CP;
                for my $j ($invlist[0] .. $upper) {
                    push @latin1_list, a2n($j);
                }

                shift @invlist; # Shift off the range that's in the list
                shift @invlist; # Shift off the range not in the list
            }

            # Here @invlist contains all the ranges in the original that start
            # at code points above 256, and @latin1_list contains all the
            # native code points for ranges that start with a Unicode code
            # point below 257.  We sort the latter and convert it to inversion
            # list format.  Then simply prepend it to the list of the higher
            # code points.
            @latin1_list = sort { $a <=> $b } @latin1_list;
            @latin1_list = mk_invlist_from_sorted_cp_list(\@latin1_list);
            unshift @invlist, @latin1_list;
            }
            else {  # Is an inversion map

                # This is a similar procedure as plain inversion list, but has
                # multiple buckets.  A plain inversion list just has two
                # buckets, 1) 'in' the list; and 2) 'not' in the list, and we
                # pretty much can ignore the 2nd bucket, as it is completely
                # defined by the 1st.  But here, what we do is create buckets
                # which contain the code points that map to each, translated
                # to native and turned into an inversion list.  Thus each
                # bucket is an inversion list of native code points that map
                # to it or don't map to it.  We use these to create an
                # inversion map for the whole property.

                # As mentioned earlier, we use this procedure to not just
                # remap the inversion list to native values, but also the maps
                # of code points to native ones.  In the latter case we have
                # to look at the whole of the inversion map (or at least to
                # above Unicode; as the maps of code points above that should
                # all be to the default).
                my $upper_limit = ($maps_to_code_point) ? 0x10FFFF : 256;

                my %mapped_lists;   # A hash whose keys are the buckets.
                while (@invlist) {
                    last if $invlist[0] > $upper_limit;

                    # This shouldn't actually happen, as prop_invmap() returns
                    # an extra element at the end that is beyond $upper_limit
                    die "inversion map that extends to infinity is unimplemented" unless @invlist > 1;

                    my $bucket;

                    # A hash key can't be a ref (we are only expecting arrays
                    # of scalars here), so convert any such to a string that
                    # will be converted back later (using a vertical tab as
                    # the separator).  Even if the mapping is to code points,
                    # we don't translate to native here because the code
                    # output_map() calls to output these arrays assumes the
                    # input is Unicode, not native.
                    if (ref $invmap[0]) {
                        $bucket = join "\cK", @{$invmap[0]};
                    }
                    elsif ($maps_to_code_point && $invmap[0] =~ $numeric_re) {

                        # Do convert to native for maps to single code points.
                        # There are some properties that have a few outlier
                        # maps that aren't code points, so the above test
                        # skips those.
                        $bucket = a2n($invmap[0]);
                    } else {
                        $bucket = $invmap[0];
                    }

                    # We now have the bucket that all code points in the range
                    # map to, though possibly they need to be adjusted.  Go
                    # through the range and put each translated code point in
                    # it into its bucket.
                    my $base_map = $invmap[0];
                    for my $j ($invlist[0] .. $invlist[1] - 1) {
                        if ($to_adjust
                               # The 1st code point doesn't need adjusting
                            && $j > $invlist[0]

                               # Skip any non-numeric maps: these are outliers
                               # that aren't code points.
                            && $base_map =~ $numeric_re

                               #  'ne' because the default can be a string
                            && $base_map ne $map_default)
                        {
                            # We adjust, by incrementing each the bucket and
                            # the map.  For code point maps, translate to
                            # native
                            $base_map++;
                            $bucket = ($maps_to_code_point)
                                      ? a2n($base_map)
                                      : $base_map;
                        }

                        # Add the native code point to the bucket for the
                        # current map
                        push @{$mapped_lists{$bucket}}, a2n($j);
                    } # End of loop through all code points in the range

                    # Get ready for the next range
                    shift @invlist;
                    shift @invmap;
                } # End of loop through all ranges in the map.

                # Here, @invlist and @invmap retain all the ranges from the
                # originals that start with code points above $upper_limit.
                # Each bucket in %mapped_lists contains all the code points
                # that map to that bucket.  If the bucket is for a map to a
                # single code point is a single code point, the bucket has
                # been converted to native.  If something else (including
                # multiple code points), no conversion is done.
                #
                # Now we recreate the inversion map into %xlated, but this
                # time for the native character set.
                my %xlated;
                foreach my $bucket (keys %mapped_lists) {

                    # Sort and convert this bucket to an inversion list.  The
                    # result will be that ranges that start with even-numbered
                    # indexes will be for code points that map to this bucket;
                    # odd ones map to some other bucket, and are discarded
                    # below.
                    @{$mapped_lists{$bucket}}
                                    = sort{ $a <=> $b} @{$mapped_lists{$bucket}};
                    @{$mapped_lists{$bucket}}
                     = mk_invlist_from_sorted_cp_list(\@{$mapped_lists{$bucket}});

                    # Add each even-numbered range in the bucket to %xlated;
                    # so that the keys of %xlated become the range start code
                    # points, and the values are their corresponding maps.
                    while (@{$mapped_lists{$bucket}}) {
                        my $range_start = $mapped_lists{$bucket}->[0];
                        if ($bucket =~ /\cK/) {
                            @{$xlated{$range_start}} = split /\cK/, $bucket;
                        }
                        else {
                            $xlated{$range_start} = $bucket;
                        }
                        shift @{$mapped_lists{$bucket}}; # Discard odd ranges
                        shift @{$mapped_lists{$bucket}}; # Get ready for next
                                                         # iteration
                    }
                } # End of loop through all the buckets.

                # Here %xlated's keys are the range starts of all the code
                # points in the inversion map.  Construct an inversion list
                # from them.
                my @new_invlist = sort { $a <=> $b } keys %xlated;

                # If the list is adjusted, we want to munge this list so that
                # we only have one entry for where consecutive code points map
                # to consecutive values.  We just skip the subsequent entries
                # where this is the case.
                if ($to_adjust) {
                    my @temp;
                    for my $i (0 .. @new_invlist - 1) {
                        next if $i > 0
                                && $new_invlist[$i-1] + 1 == $new_invlist[$i]
                                && $xlated{$new_invlist[$i-1]} =~ $numeric_re
                                && $xlated{$new_invlist[$i]} =~ $numeric_re
                                && $xlated{$new_invlist[$i-1]} + 1 == $xlated{$new_invlist[$i]};
                        push @temp, $new_invlist[$i];
                    }
                    @new_invlist = @temp;
                }

                # The inversion map comes from %xlated's values.  We can
                # unshift each onto the front of the untouched portion, in
                # reverse order of the portion we did process.
                foreach my $start (reverse @new_invlist) {
                    unshift @invmap, $xlated{$start};
                }

                # Finally prepend the inversion list we have just constructed to the
                # one that contains anything we didn't process.
                unshift @invlist, @new_invlist;
            }
        }

        # prop_invmap() returns an extra final entry, which we can now
        # discard.
        if (@invmap) {
            pop @invlist;
            pop @invmap;
        }

        if ($l1_only) {
            die "Unimplemented to do a Latin-1 only inversion map" if @invmap;
            for my $i (0 .. @invlist - 1 - 1) {
                if ($invlist[$i] > 255) {

                    # In an inversion list, even-numbered elements give the code
                    # points that begin ranges that match the property;
                    # odd-numbered give ones that begin ranges that don't match.
                    # If $i is odd, we are at the first code point above 255 that
                    # doesn't match, which means the range it is ending does
                    # match, and crosses the 255/256 boundary.  We want to include
                    # this ending point, so increment $i, so the splice below
                    # includes it.  Conversely, if $i is even, it is the first
                    # code point above 255 that matches, which means there was no
                    # matching range that crossed the boundary, and we don't want
                    # to include this code point, so splice before it.
                    $i++ if $i % 2 != 0;

                    # Remove everything past this.
                    splice @invlist, $i;
                    splice @invmap, $i if @invmap;
                    last;
                }
            }
        }
        elsif ($nonl1_only) {
            my $found_nonl1 = 0;
            for my $i (0 .. @invlist - 1 - 1) {
                next if $invlist[$i] < 256;

                # Here, we have the first element in the array that indicates an
                # element above Latin1.  Get rid of all previous ones.
                splice @invlist, 0, $i;
                splice @invmap, 0, $i if @invmap;

                # If this one's index is not divisible by 2, it means that this
                # element is inverting away from being in the list, which means
                # all code points from 256 to this one are in this list (or
                # map to the default for inversion maps)
                if ($i % 2 != 0) {
                    unshift @invlist, 256;
                    unshift @invmap, $map_default if @invmap;
                }
                $found_nonl1 = 1;
                last;
            }
            die "No non-Latin1 code points in $lookup_prop" unless $found_nonl1;
        }

        output_invlist($prop_name, \@invlist, $charset);
        output_invmap($prop_name, \@invmap, $lookup_prop, $map_format, $map_default, $extra_enums, $charset) if @invmap;
    }
    end_file_pound_if;
    print $out_fh "\n" . get_conditional_compile_line_end();
}

output_LB_table();

my $sources_list = "lib/unicore/mktables.lst";
my @sources = ($0, qw(lib/unicore/mktables
                      lib/Unicode/UCD.pm
                      regen/charset_translations.pl
                      ));
{
    # Depend on mktables’ own sources.  It’s a shorter list of files than
    # those that Unicode::UCD uses.
    if (! open my $mktables_list, $sources_list) {

          # This should force a rebuild once $sources_list exists
          push @sources, $sources_list;
    }
    else {
        while(<$mktables_list>) {
            last if /===/;
            chomp;
            push @sources, "lib/unicore/$_" if /^[^#]/;
        }
    }
}

read_only_bottom_close_and_rename($out_fh, \@sources);