diff options
Diffstat (limited to 'lib/stdlib')
154 files changed, 21457 insertions, 4926 deletions
diff --git a/lib/stdlib/Makefile b/lib/stdlib/Makefile index f1db3c77e3..f1a72cd6b9 100644 --- a/lib/stdlib/Makefile +++ b/lib/stdlib/Makefile @@ -39,6 +39,7 @@ include $(ERL_TOP)/make/otp_subdir.mk DIA_PLT_APPS=compiler crypto TEST_NEEDS_RELEASE=true +NO_TEST_TARGET:=1 # Avoid warning about ignoring old recipe for target 'test' include $(ERL_TOP)/make/app_targets.mk # Enable feature maybe_expr in runtime when running tests. diff --git a/lib/stdlib/doc/src/Makefile b/lib/stdlib/doc/src/Makefile index 8cd2ceb53c..d13fa47064 100644 --- a/lib/stdlib/doc/src/Makefile +++ b/lib/stdlib/doc/src/Makefile @@ -33,6 +33,7 @@ APPLICATION=stdlib XML_APPLICATION_FILES = ref_man.xml XML_REF3_FILES = \ + argparse.xml \ array.xml \ base64.xml \ beam_lib.xml \ @@ -43,6 +44,7 @@ XML_REF3_FILES = \ dict.xml \ digraph.xml \ digraph_utils.xml \ + edlin_expand.xml \ epp.xml \ erl_anno.xml \ erl_error.xmlsrc \ diff --git a/lib/stdlib/doc/src/argparse.xml b/lib/stdlib/doc/src/argparse.xml new file mode 100644 index 0000000000..20e1f3a721 --- /dev/null +++ b/lib/stdlib/doc/src/argparse.xml @@ -0,0 +1,739 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE erlref SYSTEM "erlref.dtd"> + +<!-- %ExternalCopyright% --> + +<erlref> + <header> + <copyright> + <year>2020</year><year>2023</year> + <holder>Maxim Fedorov</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + </legalnotice> + + <title>argparse</title> + <prepared>maximfca@gmail.com</prepared> + <responsible></responsible> + <docno></docno> + <approved></approved> + <checked></checked> + <date></date> + <rev>A</rev> + <file>argparse.xml</file> + </header> + <module since="OTP 26.0">argparse</module> + <modulesummary>Command line arguments parser.</modulesummary> + <description> + + <p>This module implements command line parser. Parser operates with + <em>commands</em> and <em>arguments</em> represented as a tree. Commands + are branches, and arguments are leaves of the tree. Parser always starts with the + root command, named after <c>progname</c> (the name of the program which started Erlang). + </p> + + <p> + A <seetype marker="#command"><c>command specification</c></seetype> may contain handler + definition for each command, and a number argument specifications. When parser is + successful, <c>argparse</c> calls the matching handler, passing arguments extracted + from the command line. Arguments can be positional (occupying specific position in + the command line), and optional, residing anywhere but prefixed with a specified + character. + </p> + + <p> + <c>argparse</c> automatically generates help and usage messages. It will also issue + errors when users give the program invalid arguments. + </p> + + </description> + + <section> + <title>Quick start</title> + + <p><c>argparse</c> is designed to work with <seecom marker="erts:escript"><c>escript</c></seecom>. + The example below is a fully functioning Erlang program accepting two command line + arguments and printing their product.</p> + + <code> +#!/usr/bin/env escript + +main(Args) -> + argparse:run(Args, cli(), #{progname => mul}). + +cli() -> + #{ + arguments => [ + #{name => left, type => integer}, + #{name => right, type => integer} + ], + handler => + fun (#{left := Left, right := Right}) -> + io:format("~b~n", [Left * Right]) + end + }. + </code> + + <p>Running this script with no arguments results in an error, accompanied + by the usage information.</p> + + <p> + The <c>cli</c> function defines a single command with embedded handler + accepting a map. Keys of the map are argument names as defined by + the <c>argument</c> field of the command, <c>left</c> and <c>right</c> + in the example. Values are taken from the command line, and converted + into integers, as requested by the type specification. Both arguments + in the example above are required (and therefore defined as positional). + </p> + </section> + + <section> + <title>Command hierarchy</title> + + <p>A command may contain nested commands, forming a hierarchy. Arguments + defined at the upper level command are automatically added to all nested + commands. Nested commands example (assuming <c>progname</c> is <c>nested</c>): + </p> + + <code> +cli() -> + #{ + %% top level argument applicable to all commands + arguments => [#{name => top}], + commands => #{ + "first" => #{ + %% argument applicable to "first" command and + %% all commands nested into "first" + arguments => [#{name => mid}], + commands => #{ + "second" => #{ + %% argument only applicable for "second" command + arguments => [#{name => bottom}], + handler => fun (A) -> io:format("~p~n", [A]) end + } + } + } + } + }. + </code> + + <p>In the example above, a 3-level hierarchy is defined. First is the script + itself (<c>nested</c>), accepting the only argument <c>top</c>. Since it + has no associated handler, <seemfa marker="#run/3">run/3</seemfa> will + not accept user input omitting nested command selection. For this example, + user has to supply 5 arguments in the command line, two being command + names, and another 3 - required positional arguments:</p> + + <code> +./nested.erl one first second two three +#{top => "one",mid => "two",bottom => "three"} + </code> + + <p>Commands have preference over positional argument values. In the example + above, commands and positional arguments are interleaving, and <c>argparse</c> + matches command name first.</p> + + </section> + + <section> + <title>Arguments</title> + <p><c>argparse</c> supports positional and optional arguments. Optional arguments, + or options for short, must be prefixed with a special character (<c>-</c> is the default + on all operating systems). Both options and positional arguments have 1 or more associated + values. See <seetype marker="#argument"><c>argument specification</c></seetype> to + find more details about supported combinations.</p> + + <p>In the user input, short options may be concatenated with their values. Long + options support values separated by <c>=</c>. Consider this definition:</p> + + <code> +cli() -> + #{ + arguments => [ + #{name => long, long => "-long"}, + #{name => short, short => $s} + ], + handler => fun (Args) -> io:format("~p~n", [Args]) end + }. + </code> + + <p>Running <c>./args --long=VALUE</c> prints <c>#{long => "VALUE"}</c>, running + <c>./args -sVALUE</c> prints <c>#{short => "VALUE"}</c></p> + + <p><c>argparse</c> supports boolean flags concatenation: it is possible to shorten + <c>-r -f -v</c> to <c>-rfv</c>.</p> + + <p>Shortened option names are not supported: it is not possible to use <c>--my-argum</c> + instead of <c>--my-argument-name</c> even when such option can be unambiguously found.</p> + </section> + + <datatypes> + <datatype> + <name name="arg_type"/> + <desc> + <p>Defines type conversion applied to the string retrieved from the user input. + If the conversion is successful, resulting value is validated using optional + <c>Choices</c>, or minimums and maximums (for integer and floating point values + only). Strings and binary values may be validated using regular expressions. + It's possible to define custom type conversion function, accepting a string + and returning Erlang term. If this function raises error with <c>badarg</c> + reason, argument is treated as invalid. + </p> + </desc> + </datatype> + + <datatype> + <name name="argument_help"/> + <desc> + <p>User-defined help template to print in the command usage. First element of + a tuple must be a string. It is printed as a part of the usage header. Second + element of the tuple can be either a string printed as-is, a list + containing strings, <c>type</c> and <c>default</c> atoms, or a user-defined + function that must return a string.</p> + </desc> + </datatype> + + <datatype> + <name name="argument_name"/> + <desc> + <p>Argument name is used to populate argument map.</p> + </desc> + </datatype> + + <datatype> + <name name="argument"/> + <desc> + <p>Argument specification. Defines a single named argument that is returned + in the <seetype marker="#arg_map"><c>argument map</c></seetype>. The only + required field is <c>name</c>, all other fields have defaults.</p> + <p>If either of the <c>short</c> or <c>long</c> fields is specified, the + argument is treated as optional. Optional arguments do not have specific + order and may appear anywhere in the command line. Positional arguments + are ordered the same way as they appear in the arguments list of the command + specification.</p> + <p>By default, all positional arguments must be present in the command line. + The parser will return an error otherwise. Options, however, may be omitted, + in which case resulting argument map will either contain the default value, + or not have the key at all.</p> + <taglist> + <tag><c>name</c></tag> + <item> + <p>Sets the argument name in the parsed argument map. If <c>help</c> is not defined, + name is also used to generate the default usage message. + </p> + </item> + <tag><c>short</c></tag> + <item> + <p>Defines a short (single character) form of an optional argument.</p> + <code> +%% Define a command accepting argument named myarg, with short form $a: +1> Cmd = #{arguments => [#{name => myarg, short => $a}]}. +%% Parse command line "-a str": +2> {ok, ArgMap, _, _} = argparse:parse(["-a", "str"], Cmd), ArgMap. + +#{myarg => "str"} + +%% Option value can be concatenated with the switch: "-astr" +3> {ok, ArgMap, _, _} = argparse:parse(["-astr"], Cmd), ArgMap. + +#{myarg => "str"} + </code> + <p>By default all options expect a single value following the option switch. + The only exception is an option of a boolean type.</p> + </item> + <tag><c>long</c></tag> + <item> + <p>Defines a long form of an optional argument.</p> + <code> +1> Cmd = #{arguments => [#{name => myarg, long => "name"}]}. +%% Parse command line "-name Erlang": +2> {ok, ArgMap, _, _} = argparse:parse(["-name", "Erlang"], Cmd), ArgMap. + +#{myarg => "Erlang"} +%% Or use "=" to separate the switch and the value: +3> {ok, ArgMap, _, _} = argparse:parse(["-name=Erlang"], Cmd), ArgMap. + +#{myarg => "Erlang"} + </code> + <p>If neither <c>short</c> not <c>long</c> is defined, the + argument is treated as positional.</p> + </item> + <tag><c>required</c></tag> + <item> + <p>Forces the parser to expect the argument to be present in the + command line. By default, all positional argument are required, + and all options are not.</p> + </item> + <tag><c>default</c></tag> + <item> + <p>Specifies the default value to put in the parsed argument map + if the value is not supplied in the command line.</p> + <code> +1> argparse:parse([], #{arguments => [#{name => myarg, short => $m}]}). + +{ok,#{}, ... +2> argparse:parse([], #{arguments => [#{name => myarg, short => $m, default => "def"}]}). + +{ok,#{myarg => "def"}, ... + </code> + </item> + <tag><c>type</c></tag> + <item> + <p>Defines type conversion and validation routine. The default is <c>string</c>, + assuming no conversion.</p> + </item> + <tag><c>nargs</c></tag> + <item> + <p>Defines the number of following arguments to consume from the command line. + By default, the parser consumes the next argument and converts it into an + Erlang term according to the specified type. + </p> + <taglist> + <tag><c>pos_integer()</c></tag> + <item><p> Consume exactly this number of positional arguments, fail if there + is not enough. Value in the argument map contains a list of exactly this + length. Example, defining a positional argument expecting 3 integer values:</p> + <code> +1> Cmd = #{arguments => [#{name => ints, type => integer, nargs => 3}]}, +argparse:parse(["1", "2", "3"], Cmd). + +{ok, #{ints => [1, 2, 3]}, ... + </code> + <p>Another example defining an option accepted as <c>-env</c> and + expecting two string arguments:</p> + <code> +1> Cmd = #{arguments => [#{name => env, long => "env", nargs => 2}]}, +argparse:parse(["-env", "key", "value"], Cmd). + +{ok, #{env => ["key", "value"]}, ... + </code> + </item> + <tag><c>list</c></tag> + <item> + <p>Consume all following arguments until hitting the next option (starting + with an option prefix). May result in an empty list added to the arguments + map.</p> + <code> +1> Cmd = #{arguments => [ + #{name => nodes, long => "nodes", nargs => list}, + #{name => verbose, short => $v, type => boolean} +]}, +argparse:parse(["-nodes", "one", "two", "-v"], Cmd). + +{ok, #{nodes => ["one", "two"], verbose => true}, ... + </code> + </item> + <tag><c>nonempty_list</c></tag> + <item> + <p>Same as <c>list</c>, but expects at least one argument. Returns an error + if the following command line argument is an option switch (starting with the + prefix).</p> + </item> + <tag><c>'maybe'</c></tag> + <item> + <p>Consumes the next argument from the command line, if it does not start + with an option prefix. Otherwise, adds a default value to the arguments + map.</p> + <code> +1> Cmd = #{arguments => [ + #{name => level, short => $l, nargs => 'maybe', default => "error"}, + #{name => verbose, short => $v, type => boolean} +]}, +argparse:parse(["-l", "info", "-v"], Cmd). + +{ok,#{level => "info",verbose => true}, ... + +%% When "info" is omitted, argument maps receives the default "error" +2> argparse:parse(["-l", "-v"], Cmd). + +{ok,#{level => "error",verbose => true}, ... + </code> + </item> + <tag><c>{'maybe', term()}</c></tag> + <item> + <p>Consumes the next argument from the command line, if it does not start + with an option prefix. Otherwise, adds a specified Erlang term to the + arguments map.</p> + </item> + <tag><c>all</c></tag> + <item> + <p>Fold all remaining command line arguments into a list, ignoring + any option prefixes or switches. Useful for proxying arguments + into another command line utility.</p> + <code> +1> Cmd = #{arguments => [ + #{name => verbose, short => $v, type => boolean}, + #{name => raw, long => "-", nargs => all} +]}, +argparse:parse(["-v", "--", "-kernel", "arg", "opt"], Cmd). + +{ok,#{raw => ["-kernel","arg","opt"],verbose => true}, ... + </code> + </item> + </taglist> + </item> + <tag><c>action</c></tag> + <item> + <p>Defines an action to take when the argument is found in the command line. The + default action is <c>store</c>.</p> + <taglist> + <tag><c>store</c></tag> + <item><p> + Store the value in the arguments map. Overwrites the value previously written. + </p> + <code> +1> Cmd = #{arguments => [#{name => str, short => $s}]}, +argparse:parse(["-s", "one", "-s", "two"], Cmd). + +{ok, #{str => "two"}, ... + </code> + </item> + <tag><c>{store, term()}</c></tag> + <item><p> + Stores the specified term instead of reading the value from the command line. + </p> + <code> +1> Cmd = #{arguments => [#{name => str, short => $s, action => {store, "two"}}]}, +argparse:parse(["-s"], Cmd). + +{ok, #{str => "two"}, ... + </code> + </item> + <tag><c>append</c></tag> + <item><p> + Appends the repeating occurrences of the argument instead of overwriting. + </p> + <code> +1> Cmd = #{arguments => [#{name => node, short => $n, action => append}]}, +argparse:parse(["-n", "one", "-n", "two", "-n", "three"], Cmd). + +{ok, #{node => ["one", "two", "three"]}, ... + +%% Always produces a list - even if there is one occurrence +2> argparse:parse(["-n", "one"], Cmd). + +{ok, #{node => ["one"]}, ... + </code> + </item> + <tag><c>{append, term()}</c></tag> + <item><p> + Same as <c>append</c>, but instead of consuming the argument from the + command line, appends a provided <c>term()</c>. + </p></item> + <tag><c>count</c></tag> + <item><p> + Puts a counter as a value in the arguments map. Useful for implementing + verbosity option: + </p> + <code> +1> Cmd = #{arguments => [#{name => verbose, short => $v, action => count}]}, +argparse:parse(["-v"], Cmd). + +{ok, #{verbose => 1}, ... + +2> argparse:parse(["-vvvv"], Cmd). + +{ok, #{verbose => 4}, ... + </code> + </item> + <tag><c>extend</c></tag> + <item><p> + Works as <c>append</c>, but flattens the resulting list. + Valid only for <c>nargs</c> set to <c>list</c>, <c>nonempty_list</c>, + <c>all</c> or <c>pos_integer()</c>. + </p> + <code> +1> Cmd = #{arguments => [#{name => duet, short => $d, nargs => 2, action => extend}]}, +argparse:parse(["-d", "a", "b", "-d", "c", "d"], Cmd). + +{ok, #{duet => ["a", "b", "c", "d"]}, ... + +%% 'append' would result in {ok, #{duet => [["a", "b"],["c", "d"]]}, + </code> + </item> + </taglist> + </item> + <tag><c>help</c></tag> + <item> + <p>Specifies help/usage text for the argument. <c>argparse</c> provides automatic + generation based on the argument name, type and default value, but for better + usability it is recommended to have a proper description. Setting this field + to <c>hidden</c> suppresses usage output for this argument.</p> + </item> + </taglist> + </desc> + </datatype> + + <datatype> + <name name="arg_map"/> + <desc> + <p>Arguments map is the map of argument names to the values extracted from the + command line. It is passed to the matching command handler. + If an argument is omitted, but has the default value is specified, + it is added to the map. When no default value specified, and argument is not + present in the command line, corresponding key is not present in the resulting + map.</p> + </desc> + </datatype> + + <datatype> + <name name="handler"/> + <desc> + <p>Command handler specification. Called by <seemfa marker="#run/3"><c>run/3</c> + </seemfa> upon successful parser return.</p> + <taglist> + <tag><c>fun((arg_map()) -> term())</c></tag> + <item><p> + Function accepting <seetype marker="#arg_map"><c>argument map</c></seetype>. + See the basic example in the <seeerl marker="#quick-start">Quick Start</seeerl> + section. + </p></item> + <tag><c>{Module :: module(), Function :: atom()}</c></tag> + <item><p> + Function named <c>Function</c>, exported from <c>Module</c>, accepting + <seetype marker="#arg_map"><c>argument map</c></seetype>. + </p></item> + <tag><c>{fun(() -> term()), Default :: term()}</c></tag> + <item><p> + Function accepting as many arguments as there are in the <c>arguments</c> + list for this command. Arguments missing from the parsed map are replaced + with the <c>Default</c>. Convenient way to expose existing functions. + </p> + <code> +1> Cmd = #{arguments => [ + #{name => x, type => float}, + #{name => y, type => float, short => $p}], + handler => {fun math:pow/2, 1}}, +argparse:run(["2", "-p", "3"], Cmd, #{}). + +8.0 + +%% default term 1 is passed to math:pow/2 +2> argparse:run(["2"], Cmd, #{}). + +2.0 + </code> + </item> + <tag><c>{Module :: module(), Function :: atom(), Default :: term()}</c></tag> + <item><p>Function named <c>Function</c>, exported from <c>Module</c>, accepting + as many arguments as defined for this command. Arguments missing from the parsed + map are replaced with the <c>Default</c>. Effectively, just a different syntax + to the same functionality as demonstrated in the code above.</p></item> + </taglist> + </desc> + </datatype> + + <datatype> + <name name="command_help"/> + <desc> + <p>User-defined help template. Use this option to mix custom and predefined usage text. + Help template may contain unicode strings, and following atoms:</p> + <taglist> + <tag>usage</tag> + <item><p> + Formatted command line usage text, e.g. <c>rm [-rf] <directory></c>. + </p></item> + <tag>commands</tag> + <item><p> + Expanded list of sub-commands. + </p></item> + <tag>arguments</tag> + <item><p> + Detailed description of positional arguments. + </p></item> + <tag>options</tag> + <item><p> + Detailed description of optional arguments. + </p></item> + </taglist> + </desc> + </datatype> + + <datatype> + <name name="command"/> + <desc> + <p>Command specification. May contain nested commands, forming a hierarchy.</p> + <taglist> + <tag><c>commands</c></tag> + <item><p> + Maps of nested commands. Keys must be strings, matching command line input. + Basic utilities do not need to specify any nested commands. + </p> + </item> + <tag><c>arguments</c></tag> + <item><p> + List of arguments accepted by this command, and all nested commands in the + hierarchy. + </p></item> + <tag><c>help</c></tag> + <item><p> + Specifies help/usage text for this command. Pass <c>hidden</c> to remove + this command from the usage output. + </p></item> + <tag><c>handler</c></tag> + <item><p> + Specifies a callback function to call by <seemfa marker="#run/3">run/3</seemfa> + when the parser is successful. + </p></item> + </taglist> + </desc> + </datatype> + + <datatype> + <name name="cmd_path"/> + <desc> + <p>Path to the nested command. First element is always the <c>progname</c>, + subsequent elements are nested command names.</p> + </desc> + </datatype> + + <datatype> + <name name="parser_error"/> + <desc> + <p>Returned from <seemfa marker="#parse/3"><c>parse/2,3</c></seemfa> when the + user input cannot be parsed according to the command specification.</p> + <p>First element is the path to the command that was considered when the + parser detected an error. Second element, <c>Expected</c>, is the argument + specification that caused an error. It could be <c>undefined</c>, meaning + that <c>Actual</c> argument had no corresponding specification in the + arguments list for the current command. </p> + <p>When <c>Actual</c> is set to <c>undefined</c>, it means that a required + argument is missing from the command line. If both <c>Expected</c> and + <c>Actual</c> have values, it means validation error.</p> + <p>Use <seemfa marker="#format_error/1"><c>format_error/1</c></seemfa> to + generate a human-readable error description, unless there is a need to + provide localised error messages.</p> + </desc> + </datatype> + + <datatype> + <name name="parser_options"/> + <desc> + <p>Options changing parser behaviour.</p> + <taglist> + <tag><c>prefixes</c></tag> + <item><p> + Changes the option prefix (the default is <c>-</c>). + </p></item> + <tag><c>default</c></tag> + <item><p> + Specifies the default value for all optional arguments. When + this field is set, resulting argument map will contain all + argument names. Useful for easy pattern matching on the + argument map in the handler function. + </p></item> + <tag><c>progname</c></tag> + <item><p> + Specifies the program (root command) name. Returned as the + first element of the command path, and printed in help/usage + text. It is recommended to have this value set, otherwise the + default one is determined with <c>init:get_argument(progname)</c> + and is often set to <c>erl</c> instead of the actual script name. + </p></item> + <tag><c>command</c></tag> + <item><p> + Specifies the path to the nested command for + <seemfa marker="#help/2"><c>help/2</c></seemfa>. Useful to + limit output for complex utilities with multiple commands, + and used by the default error handling logic. + </p></item> + <tag><c>columns</c></tag> + <item><p> + Specifies the help/usage text width (characters) for + <seemfa marker="#help/2"><c>help/2</c></seemfa>. Default value + is 80. + </p></item> + </taglist> + </desc> + </datatype> + + <datatype> + <name name="parse_result"/> + <desc> + <p>Returned from <seemfa marker="#parse/3"><c>parse/2,3</c></seemfa>. Contains + arguments extracted from the command line, path to the nested command (if any), + and a (potentially nested) command specification that was considered when + the parser finished successfully. It is expected that the command contains + a handler definition, that will be called passing the argument map.</p> + </desc> + </datatype> + + </datatypes> + + <funcs> + + <func> + <name name="format_error" arity="1" since="OTP 26.0"/> + <fsummary>Generates human-readable text for parser errors.</fsummary> + <desc> + <p>Generates human-readable text for + <seetype marker="#parser_error"><c>parser error</c></seetype>. Does + not include help/usage information, and does not provide localisation. + </p> + </desc> + </func> + + <func> + <name name="help" arity="1" since="OTP 26.0"/> + <name name="help" arity="2" since="OTP 26.0"/> + <fsummary>Generates help/usage information text.</fsummary> + <desc> + <p>Generates help/usage information text for the command + supplied, or any nested command when <c>command</c> + option is specified. Does not provide localisaton. + Expects <c>progname</c> to be set, otherwise defaults to + return value of <c>init:get_argument(progname)</c>.</p> + </desc> + </func> + + <func> + <name name="parse" arity="2" since="OTP 26.0"/> + <name name="parse" arity="3" since="OTP 26.0"/> + <fsummary>Parses command line arguments according to the command specification.</fsummary> + <desc> + <p>Parses command line arguments according to the command specification. + Raises an exception if the command specification is not valid. Use + <seemfa marker="erl_error#format_exception/3"><c>erl_error:format_exception/3,4</c> + </seemfa> to see a friendlier message. Invalid command line input + does not raise an exception, but makes <c>parse/2,3</c> to return a tuple + <seetype marker="#parser_error"><c>{error, parser_error()}</c></seetype>. + </p> + <p>This function does not call command handler.</p> + </desc> + </func> + + <func> + <name name="run" arity="3" since="OTP 26.0"/> + <fsummary>Parses command line arguments and calls the matching command handler.</fsummary> + <desc> + <p>Parses command line arguments and calls the matching command handler. + Prints human-readable error, help/usage information for the discovered + command, and halts the emulator with code 1 if there is any error in the + command specification or user-provided command line input. + </p> + <warning> + <p>This function is designed to work as an entry point to a standalone + <seecom marker="erts:escript"><c>escript</c></seecom>. Therefore, it halts + the emulator for any error detected. Do not use this function through + remote procedure call, or it may result in an unexpected shutdown of a remote + node.</p> + </warning> + </desc> + </func> + + </funcs> + +</erlref> + diff --git a/lib/stdlib/doc/src/base64.xml b/lib/stdlib/doc/src/base64.xml index bb45927c3f..a4ab294336 100644 --- a/lib/stdlib/doc/src/base64.xml +++ b/lib/stdlib/doc/src/base64.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>2007</year><year>2021</year> + <year>2007</year><year>2023</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -40,7 +40,27 @@ <datatypes> <datatype> <name name="base64_alphabet"/> - <desc><p>Base 64 Encoding alphabet, see <url href="https://www.ietf.org/rfc/rfc4648.txt">RFC 4648</url>.</p> + <desc><p>Base 64 Encoding alphabet, see + <url href="https://datatracker.ietf.org/doc/html/rfc4648">RFC 4648</url>.</p> + </desc> + </datatype> + <datatype> + <name name="base64_mode"/> + <desc> + <p>Selector for the Base 64 Encoding alphabet used for + <seemfa marker="#encode/2">encoding</seemfa> and + <seemfa marker="#decode/2">decoding</seemfa>. + See <url href="https://datatracker.ietf.org/doc/html/rfc4648">RFC 4648</url> + Sections <url href="https://datatracker.ietf.org/doc/html/rfc4648#section-4">4</url> + and <url href="https://datatracker.ietf.org/doc/html/rfc4648#section-5">5</url>.</p> + </desc> + </datatype> + <datatype> + <name name="options" /> + <desc> + <p>Customises the behaviour of the encode and decode functions. + Default value if omitted entirely or partially is + <c>#{mode => standard, padding => true}</c>.</p> </desc> </datatype> <datatype> @@ -67,15 +87,62 @@ <name name="mime_decode" arity="1" since=""/> <name name="mime_decode_to_string" arity="1" since=""/> <fsummary>Decode a base64 encoded string to data.</fsummary> - <type variable="Base64" name_i="1"/> + <type variable="Base64"/> <type variable="Data" name_i="1"/> <type variable="DataString" name_i="2"/> <desc> - <p>Decodes a base64-encoded string to plain ASCII. See - <url href="https://www.ietf.org/html/rfc4648">RFC 4648</url>.</p> + <p>Decodes a base64 string encoded using the standard alphabet according + to <url href="https://datatracker.ietf.org/doc/html/rfc4648#section-4">RFC 4648 + Section 4</url> to plain ASCII.</p> <p><c>mime_decode/1</c> and <c>mime_decode_to_string/1</c> strip away illegal characters, while <c>decode/1</c> and <c>decode_to_string/1</c> only strip away whitespace characters.</p> + <p>Checks the correct number of <c>=</c> padding characters + at the end of the encoded string.</p> + </desc> + </func> + + <func> + <name name="decode" arity="2" since="OTP @OTP-18247@"/> + <name name="decode_to_string" arity="2" since="OTP @OTP-18247@"/> + <name name="mime_decode" arity="2" since="OTP @OTP-18247@"/> + <name name="mime_decode_to_string" arity="2" since="OTP @OTP-18247@"/> + <fsummary>Decode a base64 encoded string to data.</fsummary> + <type variable="Base64"/> + <type variable="Options" name_i="1"/> + <type variable="Data" name_i="1"/> + <type variable="DataString" name_i="2"/> + <desc> + <p>Decodes a base64 string encoded using the alphabet indicated by the + <c>mode</c> option to plain ASCII.</p> + <p><c>mime_decode/2</c> and <c>mime_decode_to_string/2</c> strip away + illegal characters, while <c>decode/2</c> and + <c>decode_to_string/2</c> only strip away whitespace characters.</p> + <p>The <c>mode</c> option can be one of the following:</p> + <taglist> + <tag><c>standard</c></tag> + <item>Default. Decode the given string using the standard base64 alphabet according + to <url href="https://datatracker.ietf.org/doc/html/rfc4648#section-4">RFC 4648 + Section 4</url>, that is <c>"+"</c> and <c>"/"</c> are representing bytes <c>62</c> + and <c>63</c> respectively, while <c>"-"</c> and <c>"_"</c> are illegal + characters.</item> + <tag><c>urlsafe</c></tag> + <item>Decode the given string using the alternative "URL and Filename safe" base64 + alphabet according to + <url href="https://datatracker.ietf.org/doc/html/rfc4648#section-5">RFC 4648 + Section 5</url>, that is <c>"-"</c> and <c>"_"</c> are representing bytes <c>62</c> + and <c>63</c> respectively, while <c>"+"</c> and <c>"/"</c> are illegal + characters.</item> + </taglist> + <p>The <c>padding</c> option can be one of the following:</p> + <taglist> + <tag><c>true</c></tag> + <item>Default. Checks the correct number of <c>=</c> padding characters + at the end of the encoded string.</item> + <tag><c>false</c></tag> + <item>Accepts an encoded string with missing <c>=</c> padding characters + at the end.</item> + </taglist> </desc> </func> @@ -87,10 +154,46 @@ <type variable="Base64" name_i="1"/> <type variable="Base64String"/> <desc> - <p>Encodes a plain ASCII string into base64. The result is 33% larger - than the data.</p> + <p>Encodes a plain ASCII string into base64 using the standard alphabet + according to <url href="https://datatracker.ietf.org/doc/html/rfc4648#section-4">RFC 4648 + Section 4</url>. The result is 33% larger than the data.</p> + <p>Always appends correct number of <c>=</c> padding characters + to the encoded string.</p> + </desc> + </func> + + <func> + <name name="encode" arity="2" since="OTP @OTP-18247@"/> + <name name="encode_to_string" arity="2" since="OTP @OTP-18247@"/> + <fsummary>Encode data into base64.</fsummary> + <type variable="Data"/> + <type variable="Options"/> + <type variable="Base64" name_i="1"/> + <type variable="Base64String"/> + <desc> + <p>Encodes a plain ASCII string into base64 using the alphabet indicated by + the <c>mode</c> option. The result is 33% larger than the data.</p> + <p>The <c>mode</c> option can be one of the following:</p> + <taglist> + <tag><c>standard</c></tag> + <item>Default. Encode the given string using the standard base64 alphabet according + to <url href="https://datatracker.ietf.org/doc/html/rfc4648#section-4">RFC 4648 + Section 4</url>.</item> + <tag><c>urlsafe</c></tag> + <item>Encode the given string using the alternative "URL and Filename safe" base64 + alphabet according to + <url href="https://datatracker.ietf.org/doc/html/rfc4648#section-5">RFC 4648 + Section 5</url>.</item> + </taglist> + <p>The <c>padding</c> option can be one of the following:</p> + <taglist> + <tag><c>true</c></tag> + <item>Default. Appends correct number of <c>=</c> padding characters + to the encoded string.</item> + <tag><c>false</c></tag> + <item>Skips appending <c>=</c> padding characters to the encoded string.</item> + </taglist> </desc> </func> </funcs> </erlref> - diff --git a/lib/stdlib/doc/src/binary.xml b/lib/stdlib/doc/src/binary.xml index 220caaaaee..07f55ed30e 100644 --- a/lib/stdlib/doc/src/binary.xml +++ b/lib/stdlib/doc/src/binary.xml @@ -5,7 +5,7 @@ <header> <copyright> <year>2009</year> - <year>2021</year> + <year>2023</year> <holder>Ericsson AB, All Rights Reserved</holder> </copyright> <legalnotice> @@ -244,15 +244,23 @@ <func> <name name="encode_hex" arity="1" since="OTP 24.0"/> - <fsummary>Encodes a binary into a hex encoded binary.</fsummary> + <name name="encode_hex" arity="2" since="OTP @OTP-18354@"/> + <fsummary>Encodes a binary into a hex encoded binary with specified case</fsummary> <desc> - <p>Encodes a binary into a hex encoded binary.</p> + <p>Encodes a binary into a hex encoded binary using the specified case for the hexadecimal digits "a" to "f".</p> + <p>The default case is <c>uppercase</c>.</p> + <p><em>Example:</em></p> - <p><em>Example:</em></p> - - <code> + <code> 1> binary:encode_hex(<<"f">>). -<<"66">></code> +<<"66">> +2> binary:encode_hex(<<"/">>). +<<"2F">> +3> binary:encode_hex(<<"/">>, lowercase). +<<"2f">> +4> binary:encode_hex(<<"/">>, uppercase). +<<"2F">> + </code> </desc> </func> diff --git a/lib/stdlib/doc/src/edlin_expand.xml b/lib/stdlib/doc/src/edlin_expand.xml new file mode 100644 index 0000000000..62e634c1a5 --- /dev/null +++ b/lib/stdlib/doc/src/edlin_expand.xml @@ -0,0 +1,117 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE erlref SYSTEM "erlref.dtd"> + +<erlref> + <header> + <copyright> + <year>1996</year><year>2023</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + </legalnotice> + + <title>edlin_expand</title> + <prepared></prepared> + <docno></docno> + <date></date> + <rev></rev> + </header> + <module since="OTP @OTP-14835@">edlin_expand</module> + <modulesummary>Shell expansion and formatting of expansion suggestions.</modulesummary> + <description> + <p>This module provides an expand_fun for the erlang shell + <seemfa marker="#expand/1"><c>expand/1,2</c></seemfa>. + It is possible to override this expand_fun + <seemfa marker="io#setopts/1"><c>io:setopts/1,2</c></seemfa>.</p> + </description> + <funcs> + <func> + <name name="expand" arity="1" since="OTP @OTP-14835@"/> + <name name="expand" arity="2" since="OTP @OTP-14835@"/> + <fsummary>Standard expanion function for the erl shell.</fsummary> + <desc> + <p>The standard expansion function is able to expand strings to + valid erlang terms. This includes module names:</p> + <pre> +1> erla +modules +erlang: + </pre> + <p>function names:</p> + <pre> +1> is_ato +functions +is_atom( +2> erlang:is_ato +functions +is_atom( + </pre> +<p> + function types: +</p> +<pre> +1> erlang:is_atom( +typespecs +erlang:is_atom(Term) +any() +</pre> +<p> + and automatically add , or closing parenthesis when no other + valid expansion is possible. The expand function also completes: + shell bindings, record names, record fields and map keys. +</p> +<p> + As seen below, function headers are grouped together if they've got the same + expansion suggestion, in this case all had the same suggestions, that is '}'. + There is also limited support for filtering out function typespecs that that does + not match the types on the terms on the prompt. Only 4 suggestions are shown below + but there exists plenty more typespecs for <c>erlang:system_info</c>. + </p> +<pre> +1> erlang:system_info({allocator, my_allocator +typespecs +erlang:system_info(wordsize | {wordsize, ...} | {wordsize, ...}) +erlang:system_info({allocator, ...}) +erlang:system_info({allocator_sizes, ...}) +erlang:system_info({cpu_topology, ...}) +} +</pre> + <p>The return type of <c>expand</c> function specifies either a list of <c>Element</c> + tuples or a list of <c>Section</c> maps. The section concept was introduced to enable + more formatting options for the expansion results. For example, the shell expansion has + support to highlight text and hide suggestions. + There are also a <c>{highlight, Text}</c> that highlights all occurances of + <c>Text</c> in the title, and a <c>highlight_all</c> for simplicity which + highlights the whole title, as can be seen above for <c>functions</c> and <c>typespecs</c>.</p> + + <p>By setting the <c>{hide, result}</c> or <c>{hide, title}</c> options you may hide + suggestions. Sometimes the title isn't useful and just produces text noise, in the example + above the <c>any()</c> result is part of a section with title <c>Types</c>. Hiding results + is currently not in use, but the idea is that a section can be selected in the expand area + and all the other section entries should be collapsed. + </p> + + <p>Its possible to set a custom separator between the title and the results. This can be + done with <c>{separator, Separator}</c>. + By default its set to be <c>\n</c>, some results display a <c>type_name() :: </c> + followed by all types that define <c>type_name()</c>. + </p> + + <p>The <c>{ending, Text}</c> ElementOption just appends Text to the <c>Element</c>. + </p> + </desc> + </func> + </funcs> +</erlref> diff --git a/lib/stdlib/doc/src/epp.xml b/lib/stdlib/doc/src/epp.xml index 11130b0065..45c0f241d2 100644 --- a/lib/stdlib/doc/src/epp.xml +++ b/lib/stdlib/doc/src/epp.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>1996</year><year>2022</year> + <year>1996</year><year>2023</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -142,6 +142,10 @@ <p>The option <c>location</c> is forwarded to the Erlang token scanner, see <seemfa marker="erl_scan#tokens/3"><c>erl_scan:tokens/3,4</c></seemfa>.</p> + <p>The <c>{compiler_internal,term()}</c> option is forwarded + to the Erlang token scanner, see + <seeerl marker="erl_scan#compiler_interal"> + <c>{compiler_internal,term()}</c></seeerl>.</p> </desc> </func> @@ -193,6 +197,10 @@ <p>The option <c>location</c> is forwarded to the Erlang token scanner, see <seemfa marker="erl_scan#tokens/3"><c>erl_scan:tokens/3,4</c></seemfa>.</p> + <p>The <c>{compiler_internal,term()}</c> option is forwarded + to the Erlang token scanner, see + <seeerl marker="erl_scan#compiler_interal"> + <c>{compiler_internal,term()}</c></seeerl>.</p> </desc> </func> diff --git a/lib/stdlib/doc/src/erl_scan.xml b/lib/stdlib/doc/src/erl_scan.xml index 960ff9d019..1f0a4b2cb8 100644 --- a/lib/stdlib/doc/src/erl_scan.xml +++ b/lib/stdlib/doc/src/erl_scan.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>1996</year><year>2022</year> + <year>1996</year><year>2023</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -237,6 +237,22 @@ If neither are present the text will not be saved in the token annotation.</p> </item> + <tag><marker id="compiler_interal"/> + <c>{compiler_internal, term()}</c> + </tag> + <item><p>Pass compiler-internal options to the scanner. The + set of internal options understood by the scanner should + be considered experimental and can thus be changed at any time + without prior warning.</p> + <p>The following options are currently understood:</p> + <taglist> + <tag><c>ssa_checks</c></tag> + <item> + <p>Tokenizes source code annotations used for encoding + tests on the BEAM SSA code produced by the compiler.</p> + </item> + </taglist> + </item> </taglist> </desc> </func> diff --git a/lib/stdlib/doc/src/ets.xml b/lib/stdlib/doc/src/ets.xml index ce116c50be..2aa05a599a 100644 --- a/lib/stdlib/doc/src/ets.xml +++ b/lib/stdlib/doc/src/ets.xml @@ -999,6 +999,32 @@ Error: fun containing local Erlang function calls </func> <func> + <name name="lookup_element" arity="4" since="OTP @OTP-18279@"/> + <fsummary>Return the <c>Pos</c>:th element of all objects with a + specified key in an ETS table, or <c>Default</c> if there is no such object.</fsummary> + <desc> + <p>For a table <c><anno>Table</anno></c> of type <c>set</c> or + <c>ordered_set</c>, the function returns the + <c><anno>Pos</anno></c>:th + element of the object with key <c><anno>Key</anno></c>.</p> + <p>For tables of type <c>bag</c> or <c>duplicate_bag</c>, + the functions returns a list with the <c><anno>Pos</anno></c>:th + element of every object with key <c><anno>Key</anno></c>.</p> + <p>If no object with key <c><anno>Key</anno></c> exists, the + function returns <c><anno>Default</anno></c>.</p> + <p>If <c><anno>Pos</anno></c> is larger than the size of + any tuple with a matching key, the function exits with + reason <c>badarg</c>.</p> + <p>The difference between <c>set</c>, <c>bag</c>, and + <c>duplicate_bag</c> on one hand, and <c>ordered_set</c> on + the other, regarding the fact that <c>ordered_set</c> + view keys as equal when they <em>compare equal</em> + whereas the other table types regard them equal only when + they <em>match</em>, holds for <c>lookup_element/4</c>.</p> + </desc> + </func> + + <func> <name name="match" arity="1" since=""/> <fsummary>Continues matching objects in an ETS table.</fsummary> <desc> @@ -1239,8 +1265,8 @@ ets:select(Table, MatchSpec),</code> means that to an <c>ordered_set</c> table, <c>integer()</c> <c>1</c> and <c>float()</c> <c>1.0</c> are regarded as equal. This also means that the - key used to lookup an element not necessarily - <em>matches</em> the key in the returned elements, if + key used to lookup an element does not necessarily + <em>match</em> the key in the returned elements, if <c>float()</c>'s and <c>integer()</c>'s are mixed in keys of a table.</p> </item> @@ -1361,7 +1387,7 @@ ets:select(Table, MatchSpec),</code> </note> <marker id="new_2_read_concurrency"></marker> </item> - <tag><c>{read_concurrency,boolean()}</c></tag> + <tag since="OTP R14B"><c>{read_concurrency,boolean()}</c></tag> <item> <p>Performance tuning. Defaults to <c>false</c>. When set to <c>true</c>, the table is optimized for concurrent read @@ -1386,7 +1412,7 @@ ets:select(Table, MatchSpec),</code> read bursts and large concurrent write bursts are common.</p> <marker id="new_2_decentralized_counters"></marker> </item> - <tag><c>{decentralized_counters,boolean()}</c></tag> + <tag since="OTP 23.0"><c>{decentralized_counters,boolean()}</c></tag> <item> <p> Performance tuning. Defaults to <c>true</c> for all @@ -1421,7 +1447,7 @@ ets:select(Table, MatchSpec),</code> </p> <marker id="new_2_compressed"></marker> </item> - <tag><c>compressed</c></tag> + <tag since="OTP R14B01"><c>compressed</c></tag> <item> <p>If this option is present, the table data is stored in a more compact format to consume less memory. However, it will make diff --git a/lib/stdlib/doc/src/gb_sets.xml b/lib/stdlib/doc/src/gb_sets.xml index 3477c2c90e..283c3f9198 100644 --- a/lib/stdlib/doc/src/gb_sets.xml +++ b/lib/stdlib/doc/src/gb_sets.xml @@ -68,48 +68,10 @@ <section> <title>Compatibility</title> - <p>The following functions in this module also exist and provides - the same functionality in the - <seeerl marker="sets"><c>sets(3)</c></seeerl> and - <seeerl marker="ordsets"><c>ordsets(3)</c></seeerl> - modules. That is, by only changing the module name for each call, - you can try out different set representations.</p> - <list type="bulleted"> - <item><seemfa marker="#add_element/2"><c>add_element/2</c></seemfa> - </item> - <item><seemfa marker="#del_element/2"><c>del_element/2</c></seemfa> - </item> - <item><seemfa marker="#filter/2"><c>filter/2</c></seemfa> - </item> - <item><seemfa marker="#fold/3"><c>fold/3</c></seemfa> - </item> - <item><seemfa marker="#from_list/1"><c>from_list/1</c></seemfa> - </item> - <item><seemfa marker="#intersection/1"><c>intersection/1</c></seemfa> - </item> - <item><seemfa marker="#intersection/2"><c>intersection/2</c></seemfa> - </item> - <item><seemfa marker="#is_element/2"><c>is_element/2</c></seemfa> - </item> - <item><seemfa marker="#is_empty/1"><c>is_empty/1</c></seemfa> - </item> - <item><seemfa marker="#is_set/1"><c>is_set/1</c></seemfa> - </item> - <item><seemfa marker="#is_subset/2"><c>is_subset/2</c></seemfa> - </item> - <item><seemfa marker="#new/0"><c>new/0</c></seemfa> - </item> - <item><seemfa marker="#size/1"><c>size/1</c></seemfa> - </item> - <item><seemfa marker="#subtract/2"><c>subtract/2</c></seemfa> - </item> - <item><seemfa marker="#to_list/1"><c>to_list/1</c></seemfa> - </item> - <item><seemfa marker="#union/1"><c>union/1</c></seemfa> - </item> - <item><seemfa marker="#union/2"><c>union/2</c></seemfa> - </item> - </list> + <p>See the <seeerl marker="sets#compatibility">Compatibility Section + in the <c>sets(3)</c> module</seeerl> for information about + the compatibility of the different implementations of sets in the + Standard Library.</p> </section> <datatypes> diff --git a/lib/stdlib/doc/src/gen_event.xml b/lib/stdlib/doc/src/gen_event.xml index 26a7f5646b..25d02d83f9 100644 --- a/lib/stdlib/doc/src/gen_event.xml +++ b/lib/stdlib/doc/src/gen_event.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>1996</year><year>2022</year> + <year>1996</year><year>2023</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -113,6 +113,15 @@ gen_event:stop -----> Module:terminate/2 <p>Unless otherwise stated, all functions in this module fail if the specified event manager does not exist or if bad arguments are specified.</p> + + <note><p> + For some important information about distributed signals, see the + <seeguide marker="system/reference_manual:processes#blocking-signaling-over-distribution"> + <i>Blocking Signaling Over Distribution</i></seeguide> section in the + <i>Processes</i> chapter of the <i>Erlang Reference Manual</i>. + Blocking signaling can, for example, cause call timeouts in + <c>gen_event</c> to be significantly delayed. + </p></note> </description> <datatypes> diff --git a/lib/stdlib/doc/src/gen_server.xml b/lib/stdlib/doc/src/gen_server.xml index 4948418b3d..ad1deed965 100644 --- a/lib/stdlib/doc/src/gen_server.xml +++ b/lib/stdlib/doc/src/gen_server.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>1996</year><year>2022</year> + <year>1996</year><year>2023</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -107,6 +107,15 @@ gen_server:abcast -----> Module:handle_cast/2 Processes</seeguide> in the Reference Manual for details regarding error handling using exit signals.</p> + <note><p> + For some important information about distributed signals, see the + <seeguide marker="system/reference_manual:processes#blocking-signaling-over-distribution"> + <i>Blocking Signaling Over Distribution</i></seeguide> section in the + <i>Processes</i> chapter of the <i>Erlang Reference Manual</i>. + Blocking signaling can, for example, cause call timeouts in + <c>gen_server</c> to be significantly delayed. + </p></note> + </description> @@ -1916,6 +1925,7 @@ format_status(Status) -> <v> | {ok,State,hibernate}</v> <v> | {ok,State,{continue,Continue}}</v> <v> | {stop,Reason}</v> + <v> | {error,Reason}</v> <v> | ignore</v> <v> State = term()</v> <v> @@ -1960,14 +1970,22 @@ format_status(Status) -> </item> <tag> <c>{stop,Reason}</c><br/> + <c>{error,Reason}</c><br/> <c>ignore</c> </tag> <item> <p> Initialization failed. - An exit signal with this <c>Reason</c> - (or with reason <c>normal</c> if <c>ignore</c> is returned) - is sent to linked processes and ports, + An exit signal with reason</p> + <taglist> + <tag>stop:</tag> + <item><c>Reason</c></item> + <tag>error:</tag> + <item><c>normal</c></item> + <tag>ignore:</tag> + <item><c>normal</c></item> + </taglist> + <p>is sent to linked processes and ports, notably to the process starting the gen_server when <seemfa marker="#start_link/3"> <c>start_link/3,4</c> diff --git a/lib/stdlib/doc/src/gen_statem.xml b/lib/stdlib/doc/src/gen_statem.xml index 9dd1c6f270..319fafee07 100644 --- a/lib/stdlib/doc/src/gen_statem.xml +++ b/lib/stdlib/doc/src/gen_statem.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>2016</year><year>2022</year> + <year>2016</year><year>2023</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -381,6 +381,14 @@ erlang:'!' -----> Module:StateName/3 Processes</seeguide> in the Reference Manual for details regarding error handling using exit signals. </p> + <note><p> + For some important information about distributed signals, see the + <seeguide marker="system/reference_manual:processes#blocking-signaling-over-distribution"> + <i>Blocking Signaling Over Distribution</i></seeguide> section in the + <i>Processes</i> chapter of the <i>Erlang Reference Manual</i>. + Blocking signaling can, for example, cause call timeouts in + <c>gen_statem</c> to be significantly delayed. + </p></note> </description> <section> @@ -1494,7 +1502,8 @@ handle_event(_, _, State, Data) -> </p> <p> For an unsuccesful initialization, - <c>{stop,<anno>Reason</anno>}</c> + <c>{stop, <anno>Reason</anno>}</c>, + <c>{error, <anno>Reason</anno>}</c> or <c>ignore</c> should be used; see <seemfa marker="#start_link/3"><c>start_link/3,4</c></seemfa>. </p> @@ -1773,43 +1782,17 @@ handle_event(_, _, State, Data) -> which is the default. If no reply is received within the specified time, the function call fails. </p> - <note> - <p> - For <c><anno>Timeout</anno> < infinity</c>, - to avoid getting a late reply in the caller's - inbox if the caller should catch exceptions, - this function spawns a proxy process that - does the call. A late reply gets delivered to the - dead proxy process, hence gets discarded. This is - less efficient than using - <c><anno>Timeout</anno> == infinity</c>. - </p> - </note> <p> - <c><anno>Timeout</anno></c> can also be a tuple - <c>{clean_timeout,<anno>T</anno>}</c> or - <c>{dirty_timeout,<anno>T</anno>}</c>, where - <c><anno>T</anno></c> is the time-out time. - <c>{clean_timeout,<anno>T</anno>}</c> works like - just <c>T</c> described in the note above - and uses a proxy process - while <c>{dirty_timeout,<anno>T</anno>}</c> - bypasses the proxy process which is more lightweight. + Previous issue with late replies that could occur when having + network issues or using <c>dirty_timeout</c> is now prevented + by use of + <seeguide marker="system/reference_manual:processes#process-aliases"><i>process + aliases</i></seeguide>. <c>{clean_timeout, <anno>T</anno>}</c> + and <c>{dirty_timeout, <anno>T</anno>}</c> therefore no longer + serves any purpose and will work the same as + <c><anno>Timeout</anno></c> while all of them also being + equally efficient. </p> - <note> - <p> - If you combine catching exceptions from this function - with <c>{dirty_timeout,<anno>T</anno>}</c> - to avoid that the calling process dies when the call - times out, you will have to be prepared to handle - a late reply. Note that there is an odd chance - to get a late reply even with - <c>{dirty_timeout,infinity}</c> or <c>infinity</c> - for example in the event of network problems. - So why not just let the calling process die - by not catching the exception? - </p> - </note> <p> The call can also fail, for example, if the <c>gen_statem</c> dies before or during this function call. @@ -2495,9 +2478,10 @@ handle_event(_, _, State, Data) -> <p> If <c>Module:init/1</c> fails with <c>Reason</c>, this function returns - <seetype marker="#start_ret"><c>{error,Reason}</c></seetype>. + <seetype marker="#start_ret"><c>{error, Reason}</c></seetype>. If <c>Module:init/1</c> returns - <seetype marker="#start_ret"><c>{stop,Reason}</c></seetype> + <seetype marker="#start_ret"><c>{stop, Reason}</c></seetype>, + <seetype marker="#start_ret"><c>{shutdown, Reason}</c></seetype> or <seetype marker="#start_ret"><c>ignore</c></seetype>, the process is terminated and this function @@ -2510,6 +2494,9 @@ handle_event(_, _, State, Data) -> <c>Module:init/1</c> returns <c>ignore</c>) is set to linked processes and ports, including the process calling <c>start_link/3,4</c>. </p> + <p>The difference between returning <c>{stop, Reason}</c> and + <c>{error, Reason}</c> (from <c>Module:init/1</c>) is that + <c>error</c> results in a graceful ("silent") termination. </p> </desc> </func> diff --git a/lib/stdlib/doc/src/io.xml b/lib/stdlib/doc/src/io.xml index a400d2af23..a1c467ca1c 100644 --- a/lib/stdlib/doc/src/io.xml +++ b/lib/stdlib/doc/src/io.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>1996</year><year>2021</year> + <year>1996</year><year>2023</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -83,6 +83,9 @@ <datatype> <name name="opt_pair"/> </datatype> + <datatype> + <name name="get_opt_pair"/> + </datatype> <datatype> <name name="expand_fun"/> </datatype> @@ -174,9 +177,38 @@ ok</pre> <item> <p><c>Mod</c> is the control sequence modifier. This is one or more characters that change the interpretation of - <c>Data</c>. The current modifiers are <c>t</c>, for Unicode - translation, and <c>l</c>, for stopping <c>p</c> and <c>P</c> - from detecting printable characters.</p> + <c>Data</c>.</p> + <p>The current modifiers are:</p> + <taglist> + <tag><c>t</c></tag> + <item> + <p>For Unicode translation.</p> + </item> + <tag><c>l</c></tag> + <item> + <p>For stopping <c>p</c> and <c>P</c> from detecting + printable characters.</p> + </item> + <tag><c>k</c></tag> + <item> + <p>For use with <c>p</c>, <c>P</c>, <c>w</c>, and <c>W</c> + to format maps in map-key <c>ordered</c> order (see + <seetype marker="maps#iterator_order">maps:iterator_order()</seetype>).</p> + </item> + <tag><c>K</c></tag> + <item> + <p>Similar to <c>k</c>, for formatting maps in map-key order, + but takes an extra argument that specifies the + <seetype marker="maps#iterator_order">maps:iterator_order()</seetype>.</p> + <p>For example:</p> + <pre> +> <input>M = #{ a => 1, b => 2 }.</input> +#{a => 1,b => 2} +> <input><![CDATA[io:format("~Kp~n", [reversed, M]).]]></input> +#{b => 2,a => 1} +ok</pre> + </item> + </taglist> </item> </list> <p>If <c>F</c>, <c>P</c>, or <c>Pad</c> is a <c>*</c> character, @@ -779,9 +811,14 @@ enter><input>:</input> <input>alan</input> <input>:</input> <input>joe</in [{expand_fun,#Fun<group.0.120017273>}, {echo,true}, {binary,false}, - {encoding,unicode}]</pre> + {encoding,unicode}, + {terminal,true}]</pre> <p>This example is, as can be seen, run in an environment where the terminal supports Unicode input and output.</p> + <p>The <c>terminal</c> option is read only and indicates whether + the output stream is a terminal or not. + See <seemfa marker="#setopts/1"><c>setopts/1</c></seemfa> for a description + of the other options.</p> </desc> </func> @@ -1135,13 +1172,18 @@ enter><input>1.0er.</input> <seemfa marker="#get_line/1"><c>get_line/1,2</c></seemfa>.</p> <p>The function is called with the current line, up to the cursor, as a reversed string. It is to return a - three-tuple: <c>{yes|no, string(), [string(), ...]}</c>. The + three-tuple: <c>{yes|no, string(), list()}</c>. The first element gives a beep if <c>no</c>, otherwise the expansion is silent; the second is a string that will be entered at the cursor position; the third is a list of possible expansions. If this list is not empty, - it is printed and the current input line is written - once again.</p> + it is printed below the current input line. + The list of possible expansions can be formatted in + different ways to make more advanced expansion suggestions + more readable to the user, see + <seemfa marker="edlin_expand#expand/2"> + <c>edlin_expand:expand/2</c></seemfa> for + documentation of that.</p> <p>Trivial example (beep on anything except empty line, which is expanded to <c>"quit"</c>):</p> <code type="none"> diff --git a/lib/stdlib/doc/src/io_lib.xml b/lib/stdlib/doc/src/io_lib.xml index 3b7aea529e..f882e632bd 100644 --- a/lib/stdlib/doc/src/io_lib.xml +++ b/lib/stdlib/doc/src/io_lib.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>1996</year><year>2020</year> + <year>1996</year><year>2023</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -92,6 +92,10 @@ <item><p><c>strings</c> is set to <c>false</c> if modifier <c>l</c> is present.</p> </item> + <item><p><c>maps_order</c> is set to <c>undefined</c> by default, + <c>ordered</c> if modifier <c>k</c> is present, or <c>reversed</c> + or <c>CmpFun</c> if modifier <c>K</c> is present.</p> + </item> </list> </desc> </datatype> diff --git a/lib/stdlib/doc/src/io_protocol.xml b/lib/stdlib/doc/src/io_protocol.xml index 67352543ec..e47ace0228 100644 --- a/lib/stdlib/doc/src/io_protocol.xml +++ b/lib/stdlib/doc/src/io_protocol.xml @@ -5,7 +5,7 @@ <header> <copyright> <year>1999</year> - <year>2021</year> + <year>2023</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -449,10 +449,10 @@ ok <c>columns</c>.</item> </list> - <p>The I/O server is to send the <c>Reply</c> as:</p> + <p>The I/O server is to send one of the following as <c>Reply</c>:</p> <pre> -{ok, N} +N {error, Error}</pre> <list type="bulleted"> diff --git a/lib/stdlib/doc/src/lists.xml b/lib/stdlib/doc/src/lists.xml index d2d9870aee..1a14654821 100644 --- a/lib/stdlib/doc/src/lists.xml +++ b/lib/stdlib/doc/src/lists.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>1996</year><year>2022</year> + <year>1996</year><year>2023</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -176,43 +176,33 @@ <func> <name name="enumerate" arity="1" since="OTP 25.0"/> + <name name="enumerate" arity="2" since="OTP 25.0"/> + <name name="enumerate" arity="3" since="OTP @OTP-18495@"/> <fsummary>Annotates elements with their index.</fsummary> <desc> <p>Returns <c><anno>List1</anno></c> with each element - <c>H</c> replaced by a tuple of form <c>{I, H}</c> where - <c>I</c> is the position of <c>H</c> in - <c><anno>List1</anno></c>. The enumeration starts with 1 and - increases by 1 in each step.</p> - <p>That is, <c>enumerate/1</c> behaves as if it had been defined as follows:</p> + <c>H</c> replaced by a tuple of form <c>{I, H}</c> where + <c>I</c> is the position of <c>H</c> in + <c><anno>List1</anno></c>. The enumeration starts with + <c><anno>Index</anno></c> and increases by <c><anno>Step</anno></c> + in each step.</p> + <p>That is, <c>enumerate/3</c> behaves as if it had been defined as follows:</p> <code type="erl"> -enumerate(List) -> - {List1, _ } = lists:mapfoldl(fun(T, Acc) -> {{Acc, T}, Acc+1} end, 1, List), +enumerate(I, S, List) -> + {List1, _ } = lists:mapfoldl(fun(T, Acc) -> {{Acc, T}, Acc+S} end, I, List), List1.</code> - <p><em>Example:</em></p> + <p>The default values for <c><anno>Index</anno></c> and + <c><anno>Step</anno></c> are both <c>1</c>.</p> + <p><em>Examples:</em></p> <pre> > <input>lists:enumerate([a,b,c]).</input> [{1,a},{2,b},{3,c}]</pre> - </desc> - </func> - - <func> - <name name="enumerate" arity="2" since="OTP 25.0"/> - <fsummary>Annotates elements with their index.</fsummary> - <desc> - <p>Returns <c><anno>List1</anno></c> with each element - <c>H</c> replaced by a tuple of form <c>{I, H}</c> where - <c>I</c> is the position of <c>H</c> in - <c><anno>List1</anno></c>. The enumeration starts with - <c><anno>Index</anno></c> and increases by 1 in each step.</p> - <p>That is, <c>enumerate/2</c> behaves as if it had been defined as follows:</p> - <code type="erl"> -enumerate(I, List) -> - {List1, _ } = lists:mapfoldl(fun(T, Acc) -> {{Acc, T}, Acc+1} end, I, List), - List1.</code> - <p><em>Example:</em></p> <pre> > <input>lists:enumerate(10, [a,b,c]).</input> [{10,a},{11,b},{12,c}]</pre> + <pre> +> <input>lists:enumerate(0, -2, [a,b,c]).</input> +[{0,a},{-2,b},{-4,c}]</pre> </desc> </func> @@ -1069,35 +1059,69 @@ splitwith(Pred, List) -> <func> <name name="zip" arity="2" since=""/> + <name name="zip" arity="3" since="OTP @OTP-18318@"/> <fsummary>Zip two lists into a list of two-tuples.</fsummary> <desc> - <p>"Zips" two lists of equal length into one list of two-tuples, + <p>"Zips" two lists into one list of two-tuples, where the first element of each tuple is taken from the first list and the second element is taken from the corresponding element in the second list.</p> + <p>The <c><anno>How</anno></c> parameter specifies the behavior + if the given lists are of different lengths.</p> + <taglist> + <tag><c>fail</c></tag> + <item>The call will fail if the given lists are not of equal + length. This is the default.</item> + <tag><c>trim</c></tag> + <item>Surplus elements from the longer list will be ignored. + <p><em>Examples:</em></p> + <pre> +> <input>lists:zip([a, b], [1, 2, 3], trim).</input> +[{a,1},{b,2}] +> <input>lists:zip([a, b, c], [1, 2], trim).</input> +[{a,1},{b,2}]</pre> + </item> + <tag><c>{pad, Defaults}</c></tag> + <item>The shorter list will be padded to the length of the + longer list, using the respective elements from the given + <c>Defaults</c> tuple. + <p><em>Examples:</em></p> + <pre> +> <input>lists:zip([a, b], [1, 2, 3], {pad, {x, 0}}).</input> +[{a,1},{b,2},{x,3}] +> <input>lists:zip([a, b, c], [1, 2], {pad, {x, 0}}).</input> +[{a,1},{b,2},{c,0}]</pre> + </item> + </taglist> </desc> </func> <func> <name name="zip3" arity="3" since=""/> + <name name="zip3" arity="4" since="OTP @OTP-18318@"/> <fsummary>Zip three lists into a list of three-tuples.</fsummary> <desc> - <p>"Zips" three lists of equal length into one list of + <p>"Zips" three lists into one list of three-tuples, where the first element of each tuple is taken from the first list, the second element is taken from the corresponding element in the second list, and the third element is taken from the corresponding element in the third list.</p> + <p>For a description of the <c><anno>How</anno></c> parameter, see + <seemfa marker="#zip/3"><c>zip/3</c></seemfa>.</p> </desc> </func> <func> <name name="zipwith" arity="3" since=""/> + <name name="zipwith" arity="4" since="OTP @OTP-18318@"/> <fsummary>Zip two lists into one list according to a fun.</fsummary> <desc> - <p>Combines the elements of two lists of equal length into one list. + <p>Combines the elements of two lists into one list. For each pair <c><anno>X</anno>, <anno>Y</anno></c> of list elements from the two lists, the element in the result list is <c><anno>Combine</anno>(<anno>X</anno>, <anno>Y</anno>)</c>.</p> + <p>For a description of the <c><anno>How</anno></c> parameter, see + <seemfa marker="#zip/3"><c>zip/3</c></seemfa>.</p> <p><c>zipwith(fun(X, Y) -> {X,Y} end, List1, List2)</c> is equivalent to <c>zip(List1, List2)</c>.</p> <p><em>Example:</em></p> @@ -1109,13 +1133,16 @@ splitwith(Pred, List) -> <func> <name name="zipwith3" arity="4" since=""/> + <name name="zipwith3" arity="5" since="OTP @OTP-18318@"/> <fsummary>Zip three lists into one list according to a fun.</fsummary> <desc> - <p>Combines the elements of three lists of equal length into one + <p>Combines the elements of three lists into one list. For each triple <c><anno>X</anno>, <anno>Y</anno>, <anno>Z</anno></c> of list elements from the three lists, the element in the result list is <c><anno>Combine</anno>(<anno>X</anno>, <anno>Y</anno>, <anno>Z</anno>)</c>.</p> + <p>For a description of the <c><anno>How</anno></c> parameter, see + <seemfa marker="#zip/3"><c>zip/3</c></seemfa>.</p> <p><c>zipwith3(fun(X, Y, Z) -> {X,Y,Z} end, List1, List2, List3)</c> is equivalent to <c>zip3(List1, List2, List3)</c>.</p> <p><em>Examples:</em></p> diff --git a/lib/stdlib/doc/src/maps.xml b/lib/stdlib/doc/src/maps.xml index 203eeccaf3..8a2c573920 100644 --- a/lib/stdlib/doc/src/maps.xml +++ b/lib/stdlib/doc/src/maps.xml @@ -42,7 +42,8 @@ <desc> <p>An iterator representing the associations in a map with keys of type <c><anno>Key</anno></c> and values of type <c><anno>Value</anno></c>.</p> - <p>Created using <seemfa marker="#iterator/1"><c>maps:iterator/1</c></seemfa>.</p> + <p>Created using <seemfa marker="#iterator/1"><c>maps:iterator/1</c></seemfa> or + <seemfa marker="#iterator/2"><c>maps:iterator/2</c></seemfa>.</p> <p>Consumed by:</p> <list type="bulleted"> <item><seemfa marker="#next/1"><c>maps:next/1</c></seemfa></item> @@ -51,6 +52,7 @@ <item><seemfa marker="#fold/3"><c>maps:fold/3</c></seemfa></item> <item><seemfa marker="#foreach/2"><c>maps:foreach/2</c></seemfa></item> <item><seemfa marker="#map/2"><c>maps:map/2</c></seemfa></item> + <item><seemfa marker="#to_list/1"><c>maps:to_list/1</c></seemfa></item> </list> </desc> </datatype> @@ -58,6 +60,25 @@ <datatype> <name name="iterator" n_vars="0"/> </datatype> + + <datatype> + <name name="iterator_order" n_vars="1"/> + <desc> + <p>Key-based iterator order option that can be one of <c>undefined</c> + (default for <seemfa marker="#iterator/1"><c>maps:iterator/1</c></seemfa>), + <c>ordered</c> (sorted in map-key order), <c>reversed</c>, + or a custom sorting function.</p> + <p>Used by <seemfa marker="#iterator/2"><c>maps:iterator/2</c></seemfa>.</p> + <p>The + <seeguide + marker="system/reference_manual:expressions#term-comparisons"> + Expressions section</seeguide> contains descriptions of how terms are ordered.</p> + </desc> + </datatype> + + <datatype> + <name name="iterator_order" n_vars="0"/> + </datatype> </datatypes> <funcs> @@ -92,7 +113,7 @@ returns <c>true</c>, the association is copied to the result map. If it returns <c>false</c>, the association is not copied. If it returns <c>{true, NewValue}</c>, the value for <c><anno>Key</anno></c> is - replaced with <c>NewValue</c>at this position is replaced in the + replaced with <c>NewValue</c> at this position is replaced in the result map.</p> <p>The call fails with a <c>{badmap,Map}</c> exception if <c><anno>MapOrIter</anno></c> is not a map or valid iterator, @@ -237,15 +258,19 @@ val1 <func> <name name="groups_from_list" arity="2" since="OTP 25.0"/> - <fsummary>Splits the list into groups using a function as discriminator.</fsummary> - <desc> - <p>The result is a map where each key is given by <anno>Fun</anno> - and each value is a list of elements. The order of elements within - each list is preserved from the list.</p> + <fsummary>Partitions a list into groups using a function as discriminator.</fsummary> + <desc> + <p>Partitions the given <c><anno>List</anno></c> into a map of groups.</p> + <p>The result is a map where each key is given by <c><anno>KeyFun</anno></c> + and each value is a list of elements from the given <c><anno>List</anno></c> + for which <c><anno>KeyFun</anno></c> returned the same key.</p> + <p>The order of elements within each group list is preserved from the original + list.</p> <p><em>Examples:</em></p> <pre> -> <input>maps:groups_from_list(fun(X) -> X rem 2 end, [1,2,3]).</input> -#{0 => [2], 1 => [1, 3]} +> <input>EvenOdd = fun(X) -> case X rem 2 of 0 -> even; 1 -> odd end end,</input> +<input>maps:groups_from_list(EvenOdd, [1, 2, 3]).</input> +#{even => [2], odd => [1, 3]} > <input>maps:groups_from_list(fun erlang:length/1, ["ant", "buffalo", "cat", "dingo"]).</input> #{3 => ["ant", "cat"], 5 => ["dingo"], 7 => ["buffalo"]}</pre> </desc> @@ -253,18 +278,26 @@ val1 <func> <name name="groups_from_list" arity="3" since="OTP 25.0"/> - <fsummary>Splits the list into groups using a function as discriminator.</fsummary> - <desc> - <p>The result is a map where each key is given by - <anno>Fun</anno> and each value is a list of elements given by - the <anno>ValueFun</anno>. The order of elements within each - list is preserved from the list.</p> + <fsummary>Partitions a list into groups using a function as discriminator.</fsummary> + <desc> + <p>Partitions the given <c><anno>List</anno></c> into a map of groups.</p> + <p>The result is a map where each key is given by <c><anno>KeyFun</anno></c> + and each value is a list of elements from the given <c><anno>List</anno></c>, + mapped via <c><anno>ValueFun</anno></c>, for which <c><anno>KeyFun</anno></c> + returned the same key.</p> + <p>The order of elements within each group list is preserved from the + original list.</p> <p><em>Examples:</em></p> <pre> -> <input>maps:groups_from_list(fun(X) -> X rem 2 end, fun(X) -> X*X end, [1,2,3]).</input> -#{0 => [4], 1 => [1, 9]} -> <input>maps:groups_from_list(fun erlang:length/1, fun lists:reverse/1, ["ant", "buffalo", "cat", "dingo"]).</input> -#{3 => ["tna","tac"],5 => ["ognid"],7 => ["olaffub"]}</pre> +> <input>EvenOdd = fun(X) -> case X rem 2 of 0 -> even; 1 -> odd end end,</input> +> <input>Square = fun(X) -> X * X end,</input> +> <input>maps:groups_from_list(EvenOdd, Square, [1, 2, 3]).</input> +#{even => [4], odd => [1, 9]} +> <input>maps:groups_from_list(</input> +<input> fun erlang:length/1,</input> +<input> fun lists:reverse/1,</input> +<input> ["ant", "buffalo", "cat", "dingo"]).</input> +#{3 => ["tna", "tac"],5 => ["ognid"],7 => ["olaffub"]}</pre> </desc> </func> @@ -362,6 +395,59 @@ none</code> </func> <func> + <name name="iterator" arity="2" since="OTP 26.0"/> + <fsummary>Create a map iterator.</fsummary> + <desc> + <p>Returns a map iterator <c><anno>Iterator</anno></c> that can + be used by <seemfa marker="#next/1"><c>maps:next/1</c></seemfa> + to traverse the key-value associations in a map sorted by key using + the given <c><anno>Order</anno></c>.</p> + <p>The call fails with a <c>{badmap,Map}</c> exception if + <c><anno>Map</anno></c> is not a map or if <c><anno>Order</anno></c> + is invalid.</p> + <p><em>Example (when </em><c><anno>Order</anno></c><em> is </em><c>ordered</c><em>):</em></p> + <code type="none"><![CDATA[ +> M = #{ a => 1, b => 2 }. +#{a => 1,b => 2} +> OrdI = maps:iterator(M, ordered), ok. +ok +> {K1, V1, OrdI2} = maps:next(OrdI), {K1, V1}. +{a,1} +> {K2, V2, OrdI3} = maps:next(OrdI2),{K2, V2}. +{b,2} +> maps:next(OrdI3). +none + ]]></code> + <p><em>Example (when </em><c><anno>Order</anno></c><em> is </em><c>reversed</c><em>):</em></p> + <code type="none"><![CDATA[ +> M = #{ a => 1, b => 2 }. +#{a => 1,b => 2} +> RevI = maps:iterator(M, reversed), ok. +ok +> {K2, V2, RevI2} = maps:next(RevI), {K2, V2}. +{b,2} +> {K1, V1, RevI3} = maps:next(RevI2),{K1, V1}. +{a,1} +> maps:next(RevI3). +none + ]]></code> + <p><em>Example (when </em><c><anno>Order</anno></c><em> is an arithmetic sorting function):</em></p> + <code type="none"><![CDATA[ +> M = #{ -1 => a, -1.0 => b, 0 => c, 0.0 => d }. +#{-1 => a,0 => c,-1.0 => b,0.0 => d} +> ArithOrdI = maps:iterator(M, fun(A, B) -> A =< B end), ok. +ok +> maps:to_list(ArithOrdI). +[{-1,a},{-1.0,b},{0,c},{0.0,d}] +> ArithRevI = maps:iterator(M, fun(A, B) -> B < A end), ok. +ok +> maps:to_list(ArithRevI). +[{0.0,d},{0,c},{-1.0,b},{-1,a}] + ]]></code> + </desc> + </func> + + <func> <name name="keys" arity="1" since="OTP 17.0"/> <fsummary></fsummary> <desc> @@ -575,15 +661,24 @@ error</code> <fsummary></fsummary> <desc> <p>Returns a list of pairs representing the key-value associations of - <c><anno>Map</anno></c>, where the pairs + <c><anno>MapOrIterator</anno></c>, where the pairs <c>[{K1,V1}, ..., {Kn,Vn}]</c> are returned in arbitrary order.</p> <p>The call fails with a <c>{badmap,Map}</c> exception if - <c><anno>Map</anno></c> is not a map.</p> + <c><anno>MapOrIterator</anno></c> is not a map or an iterator obtained + by a call to <seemfa marker="#iterator/1">iterator/1</seemfa> or + <seemfa marker="#iterator/2">iterator/2</seemfa>.</p> <p><em>Example:</em></p> <code type="none"> > Map = #{42 => value_three,1337 => "value two","a" => 1}, maps:to_list(Map). [{42,value_three},{1337,"value two"},{"a",1}]</code> + <p><em>Example (using </em><seemfa marker="#iterator/2">iterator/2</seemfa><em>):</em></p> +<code type="none"><![CDATA[ +> Map = #{ z => 1, y => 2, x => 3 }. +#{x => 3,y => 2,z => 1} +> maps:to_list(maps:iterator(Map, ordered)). +[{x,3},{y,2},{z,1}] + ]]></code> </desc> </func> diff --git a/lib/stdlib/doc/src/math.xml b/lib/stdlib/doc/src/math.xml index 59a05a4e5a..69df237496 100644 --- a/lib/stdlib/doc/src/math.xml +++ b/lib/stdlib/doc/src/math.xml @@ -5,7 +5,7 @@ <header> <copyright> <year>1996</year> - <year>2020</year> + <year>2023</year> <holder>Ericsson AB, All Rights Reserved</holder> </copyright> <legalnotice> @@ -102,9 +102,20 @@ erf(X) = 2/sqrt(pi)*integral from 0 to X of exp(-t*t) dt.</pre> <func> <name name="pi" arity="0" since=""/> - <fsummary>A useful number.</fsummary> + <fsummary>Ratio of the circumference of a circle to its diameter.</fsummary> <desc> - <p>A useful number.</p> + <p>Ratio of the circumference of a circle to its diameter.</p> + <p>Floating point approximation of mathematical constant pi.</p> + </desc> + </func> + + <func> + <name name="tau" arity="0" since="OTP @OTP-18361@"/> + <fsummary>Ratio of the circumference of a circle to its radius.</fsummary> + <desc> + <p>Ratio of the circumference of a circle to its radius.</p> + <p>This constant is equivalent to a full turn when described in radians.</p> + <p>The same as <c>2 * pi()</c>.</p> </desc> </func> diff --git a/lib/stdlib/doc/src/orddict.xml b/lib/stdlib/doc/src/orddict.xml index 796cb42ede..46370358e6 100644 --- a/lib/stdlib/doc/src/orddict.xml +++ b/lib/stdlib/doc/src/orddict.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>2000</year><year>2020</year> + <year>2000</year><year>2023</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -38,7 +38,7 @@ <p>This module provides a <c>Key</c>-<c>Value</c> dictionary. An <c>orddict</c> is a representation of a dictionary, where a list of pairs is used to store the keys and values. The list is - ordered after the keys in the <em>Erlang term order</em>.</p> + ordered after the keys in the <seeguide marker="system/reference_manual:expressions#term-comparisons">Erlang term order</seeguide>.</p> <p>This module provides the same interface as the <seeerl marker="dict"><c>dict(3)</c></seeerl> module @@ -69,6 +69,24 @@ generated if the initial value associated with <c><anno>Key</anno></c> is not a list of values.</p> <p>See also section <seeerl marker="#notes">Notes</seeerl>.</p> + <p><em>Example 1:</em></p> + <pre> +1> <input>OrdDict1 = orddict:from_list([{x, []}]).</input> +[{x,[]}] +2> <input>OrdDict2 = orddict:append(x, 1, OrdDict1).</input> +[{x,[1]}] +3> <input>OrdDict3 = orddict:append(x, 2, OrdDict2).</input> +[{x,[1,2]}] +4> <input>orddict:append(y, 3, OrdDict3).</input> +[{x,[1,2]},{y,[3]}]</pre> + <p><em>Example 2:</em></p> + <pre> +1> <input>OrdDict1 = orddict:from_list([{a, no_list}]).</input> +[{a,no_list}] +2> <input>orddict:append(a, 1, OrdDict1).</input> +** exception error: bad argument + in operator ++/2 + called as no_list ++ [1]</pre> </desc> </func> @@ -81,6 +99,14 @@ An exception is generated if the initial value associated with <c><anno>Key</anno></c> is not a list of values.</p> <p>See also section <seeerl marker="#notes">Notes</seeerl>.</p> + <p><em>Example:</em></p> + <pre> +1> <input>OrdDict1 = orddict:from_list([{x, []}]).</input> +[{x,[]}] +2> <input>OrdDict2 = orddict:append_list(x, [1,2], OrdDict1).</input> +[{x,[1,2]}] +3> <input>OrdDict3 = orddict:append_list(y, [3,4], OrdDict2).</input> +[{x,[1,2]},{y,[3,4]}]</pre> </desc> </func> @@ -89,6 +115,12 @@ <fsummary>Erase a key from a dictionary.</fsummary> <desc> <p>Erases all items with a specified key from a dictionary.</p> + <p><em>Example:</em></p> + <pre> +1> <input>OrdDict1 = orddict:from_list([{a, 1}, {b, 2}]).</input> +[{a,1},{b,2}] +2> <input>orddict:erase(a, OrdDict1).</input> +[{b,2}]</pre> </desc> </func> @@ -101,6 +133,14 @@ the <c><anno>Key</anno></c> is present in the dictionary. An exception is generated if <c><anno>Key</anno></c> is not in the dictionary.</p> <p>See also section <seeerl marker="#notes">Notes</seeerl>.</p> + <p><em>Example:</em></p> +<pre> +1> <input>OrdDict1 = orddict:from_list([{a, 1}, {b, 2}]).</input> +[{a,1},{b,2}] +2> <input>orddict:fetch(a, OrdDict1).</input> +1 +3> <input>orddict:fetch(missing, OrdDict1).</input> +** exception error: no function clause matching orddict:fetch(missing,[])</pre> </desc> </func> @@ -109,6 +149,12 @@ <fsummary>Return all keys in a dictionary.</fsummary> <desc> <p>Returns a list of all keys in a dictionary.</p> + <p><em>Example:</em></p> +<pre> +1> <input>OrdDict1 = orddict:from_list([{a, 1}, {b, 2}]).</input> +[{a,1},{b,2}] +2> <input>orddict:fetch_keys(OrdDict1).</input> +[a,b]</pre> </desc> </func> @@ -118,6 +164,14 @@ <desc> <p>This function returns value from dictionary and new dictionary without this value. Returns <c>error</c> if the key is not present in the dictionary.</p> + <p><em>Example:</em></p> +<pre> +1> <input>OrdDict1 = orddict:from_list([{a, 1}, {b, 2}]).</input> +[{a,1},{b,2}] +2> <input>orddict:take(a, OrdDict1).</input> +{1, [{b,2}]} +3> <input>orddict:take(missing, OrdDict1).</input> +error</pre> </desc> </func> @@ -129,6 +183,12 @@ in <c><anno>Orddict1</anno></c> for which <c><anno>Pred</anno>(<anno>Key</anno>, <anno>Value</anno>)</c> is <c>true</c>.</p> + <p><em>Example:</em></p> +<pre> +1> <input>OrdDict1 = orddict:from_list([{a, 1}, {b, 2}]).</input> +[{a,1},{b,2}] +2> <input>orddict:filter(fun (K, V) -> V > 1 end, OrdDict1).</input> +[{b,2}]</pre> </desc> </func> @@ -141,6 +201,14 @@ the value associated with <c><anno>Key</anno></c>, or <c>error</c> if the key is not present in the dictionary.</p> <p>See also section <seeerl marker="#notes">Notes</seeerl>.</p> + <p><em>Example:</em></p> +<pre> +1> <input>OrdDict1 = orddict:from_list([{a, 1}, {b, 2}]).</input> +[{a,1},{b,2}] +2> <input>orddict:find(a, OrdDict1).</input> +{ok,1} +3> <input>orddict:find(c, OrdDict1).</input> +error</pre> </desc> </func> @@ -153,6 +221,12 @@ (short for accumulator). <c><anno>Fun</anno></c> must return a new accumulator that is passed to the next call. <c><anno>Acc0</anno></c> is returned if the list is empty.</p> + <p><em>Example:</em></p> +<pre> +1> <input>OrdDict1 = orddict:from_list([{a, 1}, {b, 2}]).</input> +[{a,1},{b,2}] +2> <input>orddict:fold(fun (K, V, Acc) -> [{K, V+100} | Acc] end, [], OrdDict1).</input> +[{b,102},{a,101}]</pre> </desc> </func> @@ -188,7 +262,13 @@ <fsummary>Map a function over a dictionary.</fsummary> <desc> <p>Calls <c><anno>Fun</anno></c> on successive keys and values of - <c><anno>Orddict1</anno></c> tvo return a new value for each key.</p> + <c><anno>Orddict1</anno></c> to return a new value for each key.</p> + <p><em>Example:</em></p> +<pre> +1> <input>OrdDict1 = orddict:from_list([{a, 1}, {b, 2}]).</input> +[{a,1},{b,2}] +2> <input>orddict:map(fun (_K, V) -> V + 100 end, OrdDict1).</input> +[{a,101},{b,102}]</pre> </desc> </func> @@ -208,6 +288,14 @@ merge(Fun, D1, D2) -> fold(fun (K, V1, D) -> update(K, fun (V2) -> Fun(K, V1, V2) end, V1, D) end, D2, D1).</code> + <p><em>Example:</em></p> +<pre> +1> <input>OrdDict1 = orddict:from_list([{a, 1}, {b, 2}]).</input> +[{a,1},{b,2}] +2> <input>OrdDict2 = orddict:from_list([{b, 7}, {c, 8}]).</input> +[{b,7},{c,8}] +3> <input>orddict:merge(fun (K, V1, V2) -> V1 * V2 end, OrdDict1, OrdDict2).</input> +[{a, 1},{b, 14},{c,8}]</pre> </desc> </func> @@ -236,6 +324,14 @@ merge(Fun, D1, D2) -> dictionary. If the <c><anno>Key</anno></c> already exists in <c><anno>Orddict1</anno></c>, the associated value is replaced by <c><anno>Value</anno></c>.</p> + <p><em>Example:</em></p> +<pre> +1> <input>OrdDict1 = orddict:from_list([{a, 1}, {b, 2}]).</input> +[{a,1},{b,2}] +2> <input>orddict:store(a, 99, OrdDict1).</input> +[{a,99},{b,2}] +3> <input>orddict:store(c, 100, OrdDict1).</input> +[{a,1},{b,2},{c,99}]</pre> </desc> </func> @@ -254,6 +350,12 @@ merge(Fun, D1, D2) -> <p>Updates a value in a dictionary by calling <c><anno>Fun</anno></c> on the value to get a new value. An exception is generated if <c><anno>Key</anno></c> is not present in the dictionary.</p> + <p><em>Example:</em></p> +<pre> +1> <input>OrdDict1 = orddict:from_list([{a, 1}, {b, 2}]).</input> +[{a,1},{b,2}] +2> <input>orddict:update(a, fun (V) -> V1 + 100 end, OrdDict1).</input> +[{a, 101}, {b, 102}]</pre> </desc> </func> @@ -269,6 +371,18 @@ merge(Fun, D1, D2) -> <code type="none"> append(Key, Val, D) -> update(Key, fun (Old) -> Old ++ [Val] end, [Val], D).</code> + <p><em>Example 1:</em></p> +<pre> +1> <input>OrdDict1 = orddict:from_list([{a, 1}, {b, 2}]).</input> +[{a,1},{b,2}] +2> <input>orddict:update(c, fun (V) -> V1 + 100 end, 99, OrdDict1).</input> +[{a,1},{b,2},{c,99}]</pre> + <p><em>Example 2:</em></p> +<pre> +1> <input>OrdDict1 = orddict:from_list([{a, 1}, {b, 2}]).</input> +[{a,1},{b,2}] +2> <input>orddict:update(a, fun (V) -> V1 + 100 end, 99, OrdDict1).</input> +[{a,101},{b,2}]</pre> </desc> </func> diff --git a/lib/stdlib/doc/src/ordsets.xml b/lib/stdlib/doc/src/ordsets.xml index 35127dcf95..7b02d13ab3 100644 --- a/lib/stdlib/doc/src/ordsets.xml +++ b/lib/stdlib/doc/src/ordsets.xml @@ -48,6 +48,11 @@ that while <c>sets</c> considers two elements as different if they do not match (<c>=:=</c>), this module considers two elements as different if and only if they do not compare equal (<c>==</c>).</p> + + <p>See the <seeerl marker="sets#compatibility">Compatibility Section + in the <c>sets(3)</c> module</seeerl> for more information about + the compatibility of the different implementations of sets in the + Standard Library.</p> </description> <datatypes> diff --git a/lib/stdlib/doc/src/peer.xml b/lib/stdlib/doc/src/peer.xml index d8ac800605..71bf71e75e 100644 --- a/lib/stdlib/doc/src/peer.xml +++ b/lib/stdlib/doc/src/peer.xml @@ -99,60 +99,60 @@ of the same test suite running in parallel</item> </list> <code type="erl"> - -module(my_SUITE). - -behaviour(ct_suite). - -export([all/0, groups/0]). - -export([basic/1, args/1, named/1, restart_node/1, multi_node/1]). - - -include_lib("common_test/include/ct.hrl"). - - groups() -> - [{quick, [parallel], - [basic, args, named, restart_node, multi_node]}]. - - all() -> - [{group, quick}]. - - basic(Config) when is_list(Config) -> - {ok, Peer, _Node} = ?CT_PEER(), - peer:stop(Peer). - - args(Config) when is_list(Config) -> - %% specify additional arguments to the new node - {ok, Peer, _Node} = ?CT_PEER(["-emu_flavor", "smp"]), - peer:stop(Peer). - - named(Config) when is_list(Config) -> - %% pass test case name down to function starting nodes - Peer = start_node_impl(named_test), - peer:stop(Peer). - - start_node_impl(ActualTestCase) -> - {ok, Peer, Node} = ?CT_PEER(#{name => ?CT_PEER_NAME(ActualTestCase)}), - %% extra setup needed for multiple test cases - ok = rpc:call(Node, application, set_env, [kernel, key, value]), - Peer. - - restart_node(Config) when is_list(Config) -> - Name = ?CT_PEER_NAME(), - {ok, Peer, Node} = ?CT_PEER(#{name => Name}), - peer:stop(Peer), - %% restart the node with the same name as before - {ok, Peer2, Node} = ?CT_PEER(#{name => Name, args => ["+fnl"]}), - peer:stop(Peer2). +-module(my_SUITE). +-behaviour(ct_suite). +-export([all/0, groups/0]). +-export([basic/1, args/1, named/1, restart_node/1, multi_node/1]). + +-include_lib("common_test/include/ct.hrl"). + +groups() -> + [{quick, [parallel], + [basic, args, named, restart_node, multi_node]}]. + +all() -> + [{group, quick}]. + +basic(Config) when is_list(Config) -> + {ok, Peer, _Node} = ?CT_PEER(), + peer:stop(Peer). + +args(Config) when is_list(Config) -> + %% specify additional arguments to the new node + {ok, Peer, _Node} = ?CT_PEER(["-emu_flavor", "smp"]), + peer:stop(Peer). + +named(Config) when is_list(Config) -> + %% pass test case name down to function starting nodes + Peer = start_node_impl(named_test), + peer:stop(Peer). + +start_node_impl(ActualTestCase) -> + {ok, Peer, Node} = ?CT_PEER(#{name => ?CT_PEER_NAME(ActualTestCase)}), + %% extra setup needed for multiple test cases + ok = rpc:call(Node, application, set_env, [kernel, key, value]), + Peer. + +restart_node(Config) when is_list(Config) -> + Name = ?CT_PEER_NAME(), + {ok, Peer, Node} = ?CT_PEER(#{name => Name}), + peer:stop(Peer), + %% restart the node with the same name as before + {ok, Peer2, Node} = ?CT_PEER(#{name => Name, args => ["+fnl"]}), + peer:stop(Peer2). </code> <p> The next example demonstrates how to start multiple nodes concurrently: </p> <code type="erl"> - multi_node(Config) when is_list(Config) -> - Peers = [?CT_PEER(#{wait_boot => {self(), tag}}) - || _ <- lists:seq(1, 4)], - %% wait for all nodes to complete boot process, get their names: - _Nodes = [receive {tag, {started, Node, Peer}} -> Node end - || {ok, Peer} <- Peers], - [peer:stop(Peer) || {ok, Peer} <- Peers]. +multi_node(Config) when is_list(Config) -> + Peers = [?CT_PEER(#{wait_boot => {self(), tag}}) + || _ <- lists:seq(1, 4)], + %% wait for all nodes to complete boot process, get their names: + _Nodes = [receive {tag, {started, Node, Peer}} -> Node end + || {ok, Peer} <- Peers], + [peer:stop(Peer) || {ok, Peer} <- Peers]. </code> <p> @@ -161,9 +161,9 @@ prompt. </p> <code type="erl"> - Ssh = os:find_executable("ssh"), - peer:start_link(#{exec => {Ssh, ["another_host", "erl"]}, - connection => standard_io}), +Ssh = os:find_executable("ssh"), +peer:start_link(#{exec => {Ssh, ["another_host", "erl"]}, + connection => standard_io}), </code> <p> @@ -172,76 +172,76 @@ running inside containers form an Erlang cluster. </p> <code type="erl"> - docker(Config) when is_list(Config) -> - Docker = os:find_executable("docker"), - PrivDir = proplists:get_value(priv_dir, Config), - build_release(PrivDir), - build_image(PrivDir), - - %% start two Docker containers - {ok, Peer, Node} = peer:start_link(#{name => lambda, - connection => standard_io, - exec => {Docker, ["run", "-h", "one", "-i", "lambda"]}}), - {ok, Peer2, Node2} = peer:start_link(#{name => lambda, - connection => standard_io, - exec => {Docker, ["run", "-h", "two", "-i", "lambda"]}}), - - %% find IP address of the second node using alternative connection RPC - {ok, Ips} = peer:call(Peer2, inet, getifaddrs, []), - {"eth0", Eth0} = lists:keyfind("eth0", 1, Ips), - {addr, Ip} = lists:keyfind(addr, 1, Eth0), - - %% make first node to discover second one - ok = peer:call(Peer, inet_db, set_lookup, [[file]]), - ok = peer:call(Peer, inet_db, add_host, [Ip, ["two"]]), - - %% join a cluster - true = peer:call(Peer, net_kernel, connect_node, [Node2]), - %% verify that second peer node has only the first node visible - [Node] = peer:call(Peer2, erlang, nodes, []), - - %% stop peers, causing containers to also stop - peer:stop(Peer2), - peer:stop(Peer). - - build_release(Dir) -> - %% load sasl.app file, otherwise application:get_key will fail - application:load(sasl), - %% create *.rel - release file - RelFile = filename:join(Dir, "lambda.rel"), - Release = {release, {"lambda", "1.0.0"}, - {erts, erlang:system_info(version)}, - [{App, begin {ok, Vsn} = application:get_key(App, vsn), Vsn end} - || App <- [kernel, stdlib, sasl]]}, - ok = file:write_file(RelFile, list_to_binary(lists:flatten( - io_lib:format("~tp.", [Release])))), - RelFileNoExt = filename:join(Dir, "lambda"), - - %% create boot script - {ok, systools_make, []} = systools:make_script(RelFileNoExt, - [silent, {outdir, Dir}]), - %% package release into *.tar.gz - ok = systools:make_tar(RelFileNoExt, [{erts, code:root_dir()}]). - - build_image(Dir) -> - %% Create Dockerfile example, working only for Ubuntu 20.04 - %% Expose port 4445, and make Erlang distribution to listen - %% on this port, and connect to it without EPMD - %% Set cookie on both nodes to be the same. - BuildScript = filename:join(Dir, "Dockerfile"), - Dockerfile = - "FROM ubuntu:20.04 as runner\n" - "EXPOSE 4445\n" - "WORKDIR /opt/lambda\n" - "COPY lambda.tar.gz /tmp\n" - "RUN tar -zxvf /tmp/lambda.tar.gz -C /opt/lambda\n" - "ENTRYPOINT [\"/opt/lambda/erts-" ++ erlang:system_info(version) ++ - "/bin/dyn_erl\", \"-boot\", \"/opt/lambda/releases/1.0.0/start\"," - " \"-kernel\", \"inet_dist_listen_min\", \"4445\"," - " \"-erl_epmd_port\", \"4445\"," - " \"-setcookie\", \"secret\"]\n", - ok = file:write_file(BuildScript, Dockerfile), - os:cmd("docker build -t lambda " ++ Dir). +docker(Config) when is_list(Config) -> + Docker = os:find_executable("docker"), + PrivDir = proplists:get_value(priv_dir, Config), + build_release(PrivDir), + build_image(PrivDir), + + %% start two Docker containers + {ok, Peer, Node} = peer:start_link(#{name => lambda, + connection => standard_io, + exec => {Docker, ["run", "-h", "one", "-i", "lambda"]}}), + {ok, Peer2, Node2} = peer:start_link(#{name => lambda, + connection => standard_io, + exec => {Docker, ["run", "-h", "two", "-i", "lambda"]}}), + + %% find IP address of the second node using alternative connection RPC + {ok, Ips} = peer:call(Peer2, inet, getifaddrs, []), + {"eth0", Eth0} = lists:keyfind("eth0", 1, Ips), + {addr, Ip} = lists:keyfind(addr, 1, Eth0), + + %% make first node to discover second one + ok = peer:call(Peer, inet_db, set_lookup, [[file]]), + ok = peer:call(Peer, inet_db, add_host, [Ip, ["two"]]), + + %% join a cluster + true = peer:call(Peer, net_kernel, connect_node, [Node2]), + %% verify that second peer node has only the first node visible + [Node] = peer:call(Peer2, erlang, nodes, []), + + %% stop peers, causing containers to also stop + peer:stop(Peer2), + peer:stop(Peer). + +build_release(Dir) -> + %% load sasl.app file, otherwise application:get_key will fail + application:load(sasl), + %% create *.rel - release file + RelFile = filename:join(Dir, "lambda.rel"), + Release = {release, {"lambda", "1.0.0"}, + {erts, erlang:system_info(version)}, + [{App, begin {ok, Vsn} = application:get_key(App, vsn), Vsn end} + || App <- [kernel, stdlib, sasl]]}, + ok = file:write_file(RelFile, list_to_binary(lists:flatten( + io_lib:format("~tp.", [Release])))), + RelFileNoExt = filename:join(Dir, "lambda"), + + %% create boot script + {ok, systools_make, []} = systools:make_script(RelFileNoExt, + [silent, {outdir, Dir}]), + %% package release into *.tar.gz + ok = systools:make_tar(RelFileNoExt, [{erts, code:root_dir()}]). + +build_image(Dir) -> + %% Create Dockerfile example, working only for Ubuntu 20.04 + %% Expose port 4445, and make Erlang distribution to listen + %% on this port, and connect to it without EPMD + %% Set cookie on both nodes to be the same. + BuildScript = filename:join(Dir, "Dockerfile"), + Dockerfile = + "FROM ubuntu:20.04 as runner\n" + "EXPOSE 4445\n" + "WORKDIR /opt/lambda\n" + "COPY lambda.tar.gz /tmp\n" + "RUN tar -zxvf /tmp/lambda.tar.gz -C /opt/lambda\n" + "ENTRYPOINT [\"/opt/lambda/erts-" ++ erlang:system_info(version) ++ + "/bin/dyn_erl\", \"-boot\", \"/opt/lambda/releases/1.0.0/start\"," + " \"-kernel\", \"inet_dist_listen_min\", \"4445\"," + " \"-erl_epmd_port\", \"4445\"," + " \"-setcookie\", \"secret\"]\n", + ok = file:write_file(BuildScript, Dockerfile), + os:cmd("docker build -t lambda " ++ Dir). </code> </section> @@ -270,13 +270,6 @@ is, <c>peer</c> follows compatibility behaviour and uses the origin node name. </p> </item> - <tag><c>host</c></tag> - <item> - <p> - Enforces a specific host name. Can be used to override the default - behaviour and start "node@localhost" instead of "node@realhostname". - </p> - </item> <tag><c>longnames</c></tag> <item> <p> @@ -285,6 +278,13 @@ short names is the default. </p> </item> + <tag><c>host</c></tag> + <item> + <p> + Enforces a specific host name. Can be used to override the default + behaviour and start "node@localhost" instead of "node@realhostname". + </p> + </item> <tag><c>peer_down</c></tag> <item> <p> @@ -296,6 +296,11 @@ the controlling process to exit abnormally. </p> </item> + <tag><c>connection</c></tag> + <item> + <p>Alternative connection specification. See the + <seetype marker="#connection"><c>connection</c> datatype</seetype>.</p> + </item> <tag><c>exec</c></tag> <item> <p> @@ -303,16 +308,32 @@ default bash. </p> </item> - <tag><c>connection</c></tag> + <tag><c>detached</c></tag> <item> - <p>Alternative connection specification. See the - <seetype marker="#connection"><c>connection</c> datatype</seetype>.</p> + <p>Defines whether to pass the <c>-detached</c> flag to the started peer. + This option cannot be set to <c>false</c> using the standard_io alternative + connection type. Default is <c>true</c>. + </p> </item> <tag><c>args</c></tag> <item> <p>Extra command line arguments to append to the "erl" command. Arguments are passed as is, no escaping or quoting is needed or accepted.</p> </item> + <tag><c>post_process_args</c></tag> + <item> + <p>Allows the user to change the arguments passed to <c>exec</c> before the + peer is started. This can for example be useful when the <c>exec</c> program + wants the arguments to "erl" as a single argument. Example: + </p> + <code type="erl"> +peer:start(#{ name => peer:random_name(), + exec => {os:find_executable("bash"),["-c","erl"]}, + post_process_args => + fun(["-c"|Args]) -> ["-c", lists:flatten(lists:join($\s, Args))] end + }). + </code> + </item> <tag><c>env</c></tag> <item> <p> diff --git a/lib/stdlib/doc/src/proc_lib.xml b/lib/stdlib/doc/src/proc_lib.xml index aa649a280a..a064c8341e 100644 --- a/lib/stdlib/doc/src/proc_lib.xml +++ b/lib/stdlib/doc/src/proc_lib.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>1996</year><year>2020</year> + <year>1996</year><year>2023</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -178,16 +178,34 @@ <name name="init_ack" arity="2" since=""/> <fsummary>Used by a process when it has started.</fsummary> <desc> - <p>This function must be used by a process that has been started by - a <seemfa marker="#start/3"><c>start[_link]/3,4,5</c></seemfa> + <p> + This function must only be used by a process + that has been started by a + <seemfa marker="#start/3"><c>start[_link|_monitor]/3,4,5</c></seemfa> function. It tells <c><anno>Parent</anno></c> that the process has - initialized itself, has started, or has failed to initialize - itself.</p> + initialized itself and started. + </p> <p>Function <c>init_ack/1</c> uses the parent value previously stored by the start function used.</p> - <p>If this function is not called, the start function - returns an error tuple (if a link and/or a time-out is used) or - hang otherwise.</p> + <p> + If neither this function nor + <seemfa marker="#init_fail/2"><c>init_fail/2,3</c></seemfa> + is called by the started process, the start function + returns an error tuple when the started process exits, + or when the start function time-out (if used) has passed, + see <seemfa marker="#start/3"><c>start/3,4,5</c></seemfa>. + </p> + <warning> + <p> + Do not use this function to return an error indicating + that the process start failed. When doing so + the start function can return before the failing + process has exited, which may block VM resources + required for a new start attempt to succeed. Use + <seemfa marker="#init_fail/2"><c>init_fail/2,3</c></seemfa> + for that purpose. + </p> + </warning> <p>The following example illustrates how this function and <c>proc_lib:start_link/3</c> are used:</p> <code type="none"> @@ -212,6 +230,76 @@ init(Parent) -> </func> <func> + <name since="OTP 26.0">init_fail(Ret, Exception) -> no_return()</name> + <name since="OTP 26.0">init_fail(Parent, Ret, Exception) -> no_return()</name> + <fsummary>Used by a process that fails to start.</fsummary> + <type> + <v>Parent = <seetype marker="erts:erlang#pid">pid()</seetype></v> + <v>Ret = <seetype marker="erts:erlang#term">term()</seetype></v> + <v>Exception = {Class, Reason} | {Class, Reason, Stacktrace}</v> + </type> + <desc> + <p> + This function must only be used by a process + that has been started by a + <seemfa marker="#start/3"><c>start[_link|_monitor]/3,4,5</c></seemfa> + function. It tells <c>Parent</c> that the process has failed to + initialize, and immediately raises an exception + according to <c>Exception</c>. + The start function then returns <c>Ret</c>. + </p> + <p> + See + <seemfa marker="erts:erlang#raise/3"><c>erlang:raise/3</c></seemfa> + for a description of <c>Class</c>, <c>Reason</c> + and <c>Stacktrace</c>. + </p> + <p> + Function <c>init_fail/2</c> uses the parent value + previously stored by the start function used. + </p> + <warning> + <p> + Do not consider catching the exception from this function. + That would defeat its purpose. A process started by a + <seemfa marker="#start/3"><c>start[_link|_monitor]/3,4,5</c></seemfa> + function should end in a value (that will be ignored) + or an exception that will be handled by this module. + See <seeerl marker="#description">Description</seeerl>. + </p> + </warning> + <p> + If neither this function nor + <seemfa marker="#init_ack/1"><c>init_ack/1,2</c></seemfa> + is called by the started process, the start function + returns an error tuple when the started process exits, + or when the start function time-out (if used) has passed, + see <seemfa marker="#start/3"><c>start/3,4,5</c></seemfa>. + </p> + <p>The following example illustrates how this function and + <c>proc_lib:start_link/3</c> can be used:</p> + <code type="none"> +-module(my_proc). +-export([start_link/0]). +-export([init/1]). + +start_link() -> + proc_lib:start_link(my_proc, init, [self()]). + +init(Parent) -> + case do_initialization() of + ok -> + proc_lib:init_ack(Parent, {ok, self()}); + {error, Reason} = Error -> + proc_lib:init_fail(Parent, Error, {exit, normal}) + end, + loop(). + +...</code> + </desc> + </func> + + <func> <name name="initial_call" arity="1" since=""/> <fsummary>Extract the initial call of a <c>proc_lib</c>spawned process. </fsummary> @@ -304,17 +392,42 @@ init(Parent) -> <name name="start" arity="5" since=""/> <fsummary>Start a new process synchronously.</fsummary> <desc> - <p>Starts a new process synchronously. Spawns the process and - waits for it to start. When the process has started, it - <em>must</em> call + <p> + Starts a new process synchronously. Spawns the process and + waits for it to start. + </p> + <p> + To indicate a succesful start, + the started process <em>must</em> call <seemfa marker="#init_ack/2"><c>init_ack(Parent, Ret)</c></seemfa> - or <seemfa marker="#init_ack/1"><c>init_ack(Ret)</c></seemfa>, - where <c>Parent</c> is the process that evaluates this - function. At this time, <c>Ret</c> is returned.</p> + where <c>Parent</c> is the process that evaluates this function, + or <seemfa marker="#init_ack/1"><c>init_ack(Ret)</c></seemfa>. + <c>Ret</c> is then returned by this function. + </p> + <p> + If the process fails to start, it <em>must</em> fail; + preferably by calling + <seemfa marker="#init_fail/3"> + <c>init_fail(Parent, Ret, Exception)</c> + </seemfa> + where <c>Parent</c> is the process that evaluates this function, + or <seemfa marker="#init_fail/2"><c>init_fail(Ret, Exception)</c></seemfa>. + <c>Ret</c> is then returned by this function, + and the started process fails with <c>Exception</c>. + </p> + <p> + If the process instead fails before calling + <c>init_ack/1,2</c> or <c>init_fail/2,3</c>, + this function returns <c>{error, Reason}</c> + where <c>Reason</c> depends a bit on the exception + just like for a process link <c>{'EXIT',Pid,Reason}</c> + message. + </p> <p>If <c><anno>Time</anno></c> is specified as an integer, this function waits for <c><anno>Time</anno></c> milliseconds for the - new process to call <c>init_ack</c>, or <c>Ret = {error, timeout}</c> - will be returned, and the process is killed.</p> + new process to call <c>init_ack/1,2</c> or <c>init_fail/2,3</c>, + otherwise the process gets killed + and <c>Ret = {error, timeout}</c> is returned.</p> <p>Argument <c><anno>SpawnOpts</anno></c>, if specified, is passed as the last argument to the <seemfa marker="erts:erlang#spawn_opt/2"> <c>spawn_opt/2,3,4,5</c></seemfa> BIF.</p> @@ -322,6 +435,11 @@ init(Parent) -> <p>Using spawn option <c>monitor</c> is not allowed. It causes the function to fail with reason <c>badarg</c>.</p> + <p> + Using spawn option <c>link</c> will set a link to + the spawned process, just like + <seemfa marker="#start_link/3">start_link/3,4,5</seemfa>. + </p> </note> </desc> </func> @@ -335,22 +453,31 @@ init(Parent) -> <p> Starts a new process synchronously. Spawns the process and waits for it to start. A link is atomically set on the - newly spawned process. When the process has started, it - <em>must</em> call - <seemfa marker="#init_ack/2"><c>init_ack(Parent, Ret)</c></seemfa> - or <seemfa marker="#init_ack/1"><c>init_ack(Ret)</c></seemfa>, - where <c>Parent</c> is the process that evaluates this - function. At this time, <c>Ret</c> is returned.</p> - <p>If <c><anno>Time</anno></c> is specified as an integer, this - function waits for <c><anno>Time</anno></c> milliseconds for the - new process to call <c>init_ack</c>, or <c>Ret = {error, timeout}</c> - will be returned, and the process is killed.</p> - <p>If the process crashes before it has called <c>init_ack/1,2</c>, - <c>Ret = {error, <anno>Reason</anno>}</c> will be returned if - the calling process traps exits.</p> - <p>Argument <c><anno>SpawnOpts</anno></c>, if specified, is passed - as the last argument to the <seemfa marker="erts:erlang#spawn_opt/2"> - <c>spawn_opt/2,3,4,5</c></seemfa> BIF.</p> + newly spawned process. + </p> + <note> + <p> + If the started process gets killed or crashes with a reason + that is not `normal`, the process link will kill the calling + process so this function does not return, + unless the calling process traps exits. + For example, if this function times out it will kill + the spawned process, and then the link might kill + the calling process. + </p> + </note> + <p> + Besides setting a link on the spawned process + this function behaves like + <seemfa marker="#start/3">start/3,4,5</seemfa>. + </p> + <p> + When the calling process traps exits; + if this function returns due to the spawned process exiting + (any error return), this function receives (consumes) + the <c>'EXIT'</c> message, also when this function times out + and kills the spawned process. + </p> <note> <p>Using spawn option <c>monitor</c> is not allowed. It causes the function to fail with reason @@ -368,34 +495,36 @@ init(Parent) -> <p> Starts a new process synchronously. Spawns the process and waits for it to start. A monitor is atomically set on the - newly spawned process. When the process has started, it - <em>must</em> call - <seemfa marker="#init_ack/2"><c>init_ack(Parent, Ret)</c></seemfa> - or <seemfa marker="#init_ack/1"><c>init_ack(Ret)</c></seemfa>, - where <c>Parent</c> is the process that evaluates this - function. At this time, <c>Ret</c> is returned.</p> - <p>If <c><anno>Time</anno></c> is specified as an integer, this - function waits for <c><anno>Time</anno></c> milliseconds for the - new process to call <c>init_ack</c>, or <c>Ret = {error, timeout}</c> - will be returned, and the process is killed.</p> + newly spawned process. + </p> + <p> + Besides setting a monitor on the spawned process + this function behaves like + <seemfa marker="#start/3">start/3,4,5</seemfa>. + </p> <p> The return value is <c>{Ret, Mon}</c> where <c>Ret</c> corresponds - to the <c>Ret</c> argument in the call to <c>init_ack()</c>, and - <c>Mon</c> is the monitor reference of the monitor that has been - set up. + to the <c>Ret</c> argument in the call to <c>init_ack/1,2</c> + or <c>init_fail/2,3</c>, and <c>Mon</c> is the monitor reference + of the monitor that has been set up. </p> <p> - A <c>'DOWN'</c> message will be delivered to the caller if - this function returns, and the spawned process terminates. This is - true also in the case when the operation times out. + If this function returns due to the spawned process exiting, + that is returns any error value, + a <c>'DOWN'</c> message will be delivered to the calling process, + also when this function times out and kills the spawned process. </p> - <p>Argument <c><anno>SpawnOpts</anno></c>, if specified, is passed - as the last argument to the <seemfa marker="erts:erlang#spawn_opt/2"> - <c>spawn_opt/2,3,4,5</c></seemfa> BIF.</p> <note> - <p>Using spawn option <c>monitor</c> is not + <p> + Using spawn option <c>monitor</c> is not allowed. It causes the function to fail with reason - <c>badarg</c>.</p> + <c>badarg</c>. + </p> + <p> + Using spawn option <c>link</c> will set a link to + the spawned process, just like + <seemfa marker="#start_link/3">start_link/3,4,5</seemfa>. + </p> </note> </desc> </func> diff --git a/lib/stdlib/doc/src/queue.xml b/lib/stdlib/doc/src/queue.xml index 2e6f424a84..480ca89963 100644 --- a/lib/stdlib/doc/src/queue.xml +++ b/lib/stdlib/doc/src/queue.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>1996</year><year>2022</year> + <year>1996</year><year>2023</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -44,7 +44,6 @@ not lists. Improper lists cause internal crashes. An index out of range for a queue also causes a failure with reason <c>badarg</c>.</p> - <p>Some functions, where noted, fail with reason <c>empty</c> for an empty queue.</p> @@ -89,7 +88,7 @@ are reverse operations on the queue.</p> <p>This module has three sets of interface functions: the - "Original API", the "Extended API", and the "Okasaki API".</p> + <em>"Original API"</em>, the <em>"Extended API"</em>, and the <em>"Okasaki API"</em>.</p> <p>The "Original API" and the "Extended API" both use the mental picture of a waiting line of items. Both @@ -133,6 +132,13 @@ <p>Returns <c>true</c> if <c><anno>Pred</anno>(<anno>Item</anno>)</c> returns <c>true</c> for all items <c><anno>Item</anno></c> in <c><anno>Q</anno></c>, otherwise <c>false</c>.</p> + <p><em>Example:</em></p> + <pre> +1> <input>Queue = queue:from_list([1,2,3,4,5]).</input> +2> <input>queue:all(fun (E) -> E > 3 end, Queue).</input> +false +3> <input>queue:all(fun (E) -> E > 0 end, Queue).</input> +true</pre> </desc> </func> @@ -144,6 +150,13 @@ <p>Returns <c>true</c> if <c><anno>Pred</anno>(<anno>Item</anno>)</c> returns <c>true</c> for at least one item <c><anno>Item</anno></c> in <c><anno>Q</anno></c>, otherwise <c>false</c>.</p> + <p><em>Example:</em></p> + <pre> +1> <input>Queue = queue:from_list([1,2,3,4,5]).</input> +2> <input>queue:any(fun (E) -> E > 10 end, Queue).</input> +false +3> <input>queue:any(fun (E) -> E > 3 end, Queue).</input> +true</pre> </desc> </func> @@ -154,6 +167,12 @@ <p>Returns a copy of <c><anno>Q1</anno></c> where the first item matching <c><anno>Item</anno></c> is deleted, if there is such an item.</p> + <p><em>Example:</em></p> + <pre> +1> <input>Queue = queue:from_list([1,2,3,4,5]).</input> +2> <input>Queue1 = queue:delete(3, Queue).</input> +3> <input>queue:member(3, Queue1).</input> +false</pre> </desc> </func> @@ -164,6 +183,12 @@ <p>Returns a copy of <c><anno>Q1</anno></c> where the last item matching <c><anno>Item</anno></c> is deleted, if there is such an item.</p> + <p><em>Example:</em></p> + <pre> +1> <input>Queue = queue:from_list([1,2,3,4,3,5]).</input> +2> <input>Queue1 = queue:delete_r(3, Queue).</input> +3> <input>queue:to_list(Queue1).</input> +[1,2,3,4,5]</pre> </desc> </func> @@ -175,6 +200,12 @@ <p>Returns a copy of <c><anno>Q1</anno></c> where the first item for which <c><anno>Pred</anno></c> returns <c>true</c> is deleted, if there is such an item.</p> + <p><em>Example:</em></p> + <pre> +1> <input>Queue = queue:from_list([100,1,2,3,4,5]).</input> +2> <input>Queue1 = queue:delete_with(fun (E) -> E > 0, Queue).</input> +3> <input>queue:to_list(Queue1).</input> +[1,2,3,4,5]</pre> </desc> </func> @@ -186,6 +217,12 @@ <p>Returns a copy of <c><anno>Q1</anno></c> where the last item for which <c><anno>Pred</anno></c> returns <c>true</c> is deleted, if there is such an item.</p> + <p><em>Example:</em></p> + <pre> +1> <input>Queue = queue:from_list([1,2,3,4,5,100]).</input> +2> <input>Queue1 = queue:delete_with(fun (E) -> E > 10, Queue).</input> +3> <input>queue:to_list(Queue1).</input> +[1,2,3,4,5]</pre> </desc> </func> @@ -201,12 +238,28 @@ <c><anno>Item</anno></c> is not copied. If it returns a list, the list elements are inserted instead of <c>Item</c> in the result queue.</p> + <p><em>Example 1:</em></p> + <pre> +1> <input>Queue = queue:from_list([1,2,3,4,5]).</input> +{[5,4,3],[1,2]} +2> <input>Queue1 = queue:filter(fun (E) -> E > 2 end, Queue).</input> +{[5],[3,4]} +3> <input>queue:to_list(Queue1).</input> +[3,4,5]</pre> <p>So, <c><anno>Fun</anno>(<anno>Item</anno>)</c> returning <c>[<anno>Item</anno>]</c> is thereby semantically equivalent to returning <c>true</c>, just as returning <c>[]</c> is semantically equivalent to returning <c>false</c>. But returning a list builds more garbage than returning an atom.</p> + <p><em>Example 2:</em></p> + <pre> +1> <input>Queue = queue:from_list([1,2,3,4,5]).</input> +{[5,4,3],[1,2]} +2> <input>Queue1 = queue:filter(fun (E) -> [E, E+1] end, Queue).</input> +{[6,5,5,4,4,3],[1,2,2,3]} +3> <input>queue:to_list(Queue1).</input> +[1,2,2,3,3,4,4,5,5,6]</pre> </desc> </func> @@ -222,6 +275,18 @@ <c><anno>Item</anno></c> is not copied. If it returns <c>{true, NewItem}</c>, the queue element at this position is replaced with <c>NewItem</c> in the result queue.</p> + <p><em>Example 1:</em></p> + <pre> +1> <input>Queue = queue:from_list([1,2,3,4,5]).</input> +{[5,4,3],[1,2]} +2> <input>Queue1 = queue:filtermap(fun (E) -> E > 2 end, Queue).</input> +{[5],[3,4]} +3> <input>queue:to_list(Queue1).</input> +[3,4,5] +4> <input>Queue1 = queue:filtermap(fun (E) -> {true, E+100} end, Queue).</input> +{"ihg","ef"} +5> <input>queue:to_list(Queue1).</input> +"efghi</pre> </desc> </func> @@ -239,9 +304,9 @@ empty.</p> <p><em>Example:</em></p> <pre> -> <input>queue:fold(fun(X, Sum) -> X + Sum end, 0, queue:from_list([1,2,3,4,5])).</input> +1> <input>queue:fold(fun(X, Sum) -> X + Sum end, 0, queue:from_list([1,2,3,4,5])).</input> 15 -> <input>queue:fold(fun(X, Prod) -> X * Prod end, 1, queue:from_list([1,2,3,4,5])).</input> +2> <input>queue:fold(fun(X, Prod) -> X * Prod end, 1, queue:from_list([1,2,3,4,5])).</input> 120</pre> </desc> </func> @@ -263,6 +328,14 @@ <p>Inserts <c><anno>Item</anno></c> at the rear of queue <c><anno>Q1</anno></c>. Returns the resulting queue <c><anno>Q2</anno></c>.</p> + <p><em>Example:</em></p> + <pre> +1> <input>Queue = queue:from_list([1,2,3,4,5]).</input> +{[5,4,3],[1,2]} +2> <input>Queue1 = queue:in(100, Queue).</input> +{[100,5,4,3],[1,2]} +3> <input>queue:to_list(Queue1).</input> +[1,2,3,4,5,100]</pre> </desc> </func> @@ -273,6 +346,14 @@ <p>Inserts <c><anno>Item</anno></c> at the front of queue <c><anno>Q1</anno></c>. Returns the resulting queue <c><anno>Q2</anno></c>.</p> + <p><em>Example:</em></p> + <pre> +1> <input>Queue = queue:from_list([1,2,3,4,5]).</input> +{[5,4,3],[1,2]} +2> <input>Queue1 = queue:in_r(100, Queue).</input> +{[5,4,3],[100,1,2]} +3> <input>queue:to_list(Queue1).</input> +[100,1,2,3,4,5]</pre> </desc> </func> @@ -306,6 +387,14 @@ <p>Returns a queue <c><anno>Q3</anno></c> that is the result of joining <c><anno>Q1</anno></c> and <c><anno>Q2</anno></c> with <c><anno>Q1</anno></c> in front of <c><anno>Q2</anno></c>.</p> + <p><em>Example:</em></p> + <pre> +1> <input>Queue1 = queue:from_list([1,3]).</input> +{[3],[1]} +2> <input>Queue2 = queue:from_list([2,4]).</input> +{[4],[2]} +3> <input>queue:to_list(queue:join(Queue1, Queue2)).</input> +[1,3,2,4]</pre> </desc> </func> @@ -344,6 +433,14 @@ <c><anno>Q2</anno></c> is the resulting queue. If <c><anno>Q1</anno></c> is empty, tuple <c>{empty, <anno>Q1</anno>}</c> is returned.</p> + <p><em>Example:</em></p> + <pre> +1> <input>Queue = queue:from_list([1,2,3,4,5]).</input> +{[5,4,3],[1,2]} +2> <input>{{value, 1=Item}, Queue1} = queue:out(Queue).</input> +{{value,1},{[5,4,3],[2]}} +3> <input>queue:to_list(Queue1).</input> +[2,3,4,5]</pre> </desc> </func> @@ -356,6 +453,14 @@ where <c><anno>Item</anno></c> is the item removed and <c><anno>Q2</anno></c> is the new queue. If <c><anno>Q1</anno></c> is empty, tuple <c>{empty, <anno>Q1</anno>}</c> is returned.</p> + <p><em>Example:</em></p> + <pre> +1> <input>Queue = queue:from_list([1,2,3,4,5]).</input> +{[5,4,3],[1,2]} +2> <input>{{value, 5=Item}, Queue1} = queue:out_r(Queue).</input> +{{value,5},{[4,3],[1,2]}} +3> <input>queue:to_list(Queue1).</input> +[1,2,3,4]</pre> </desc> </func> @@ -384,6 +489,12 @@ <desc> <p>Returns a list of the items in the queue in the same order; the front item of the queue becomes the head of the list.</p> + <p><em>Example:</em></p> + <pre> +1> <input>Queue = queue:from_list([1,2,3,4,5]).</input> +{[5,4,3],[1,2]} +2> <input>List == queue:to_list(Queue).</input> +true</pre> </desc> </func> </funcs> @@ -401,6 +512,14 @@ <p>Returns a queue <c><anno>Q2</anno></c> that is the result of removing the front item from <c><anno>Q1</anno></c>.</p> <p>Fails with reason <c>empty</c> if <c><anno>Q1</anno></c> is empty.</p> + <p><em>Example:</em></p> + <pre> +1> <input>Queue = queue:from_list([1,2,3,4,5]).</input> +{[5,4,3],[1,2]} +2> <input>Queue = queue:drop(Queue).</input> +{[5,4,3],[2]} +3> <input>queue:to_list(Queue1).</input> +[2,3,4,5]</pre> </desc> </func> @@ -411,6 +530,14 @@ <p>Returns a queue <c><anno>Q2</anno></c> that is the result of removing the rear item from <c><anno>Q1</anno></c>.</p> <p>Fails with reason <c>empty</c> if <c><anno>Q1</anno></c> is empty.</p> + <p><em>Example:</em></p> + <pre> +1> <input>Queue = queue:from_list([1,2,3,4,5]).</input> +{[5,4,3],[1,2]} +2> <input>Queue = queue:drop_r(Queue).</input> +{[4,3],[1,2]} +3> <input>queue:to_list(Queue1).</input> +[1,2,3,4]</pre> </desc> </func> @@ -421,6 +548,12 @@ <p>Returns <c><anno>Item</anno></c> at the front of queue <c><anno>Q</anno></c>.</p> <p>Fails with reason <c>empty</c> if <c><anno>Q</anno></c> is empty.</p> + <p><em>Example 1:</em></p> + <pre> +1> <input>Queue = queue:from_list([1,2,3,4,5]).</input> +{[5,4,3],[1,2]} +2> <input>1 == queue:get(Queue).</input> +true</pre> </desc> </func> @@ -431,6 +564,12 @@ <p>Returns <c><anno>Item</anno></c> at the rear of queue <c><anno>Q</anno></c>.</p> <p>Fails with reason <c>empty</c> if <c><anno>Q</anno></c> is empty.</p> + <p><em>Example 1:</em></p> + <pre> +1> <input>Queue = queue:from_list([1,2,3,4,5]).</input> +{[5,4,3],[1,2]} +2> <input>5 == queue:get_r(Queue).</input> +true</pre> </desc> </func> @@ -441,6 +580,14 @@ <p>Returns tuple <c>{value, <anno>Item</anno>}</c>, where <c><anno>Item</anno></c> is the front item of <c><anno>Q</anno></c>, or <c>empty</c> if <c><anno>Q</anno></c> is empty.</p> + <p><em>Example 1:</em></p> + <pre> +1> <input>queue:peek(queue:new()).</input> +empty +2> <input>Queue = queue:from_list([1,2,3,4,5]).</input> +{[5,4,3],[1,2]} +3> <input>queue:peek(Queue).</input> +{value, 1}</pre> </desc> </func> @@ -451,12 +598,18 @@ <p>Returns tuple <c>{value, <anno>Item</anno>}</c>, where <c><anno>Item</anno></c> is the rear item of <c><anno>Q</anno></c>, or <c>empty</c> if <c><anno>Q</anno></c> is empty.</p> + <p><em>Example 1:</em></p> + <pre> +1> <input>queue:peek_r(queue:new()).</input> +empty +2> <input>Queue = queue:from_list([1,2,3,4,5]).</input> +{[5,4,3],[1,2]} +3> <input>queue:peek_r(Queue).</input> +{value, 5}</pre> </desc> </func> </funcs> - - <funcs> <fsdescription> <title>Okasaki API</title> @@ -468,6 +621,12 @@ <p>Inserts <c><anno>Item</anno></c> at the head of queue <c><anno>Q1</anno></c>. Returns the new queue <c><anno>Q2</anno></c>.</p> + <p><em>Example:</em></p> + <pre> +1> <input>Queue = queue:cons(0, queue:from_list([1,2,3])).</input> +{[3,2],[0,1]} +2> <input>queue:to_list(Queue).</input> +[0,1,2,3]</pre> </desc> </func> @@ -477,6 +636,10 @@ <desc> <p>Returns the tail item of queue <c><anno>Q</anno></c>.</p> <p>Fails with reason <c>empty</c> if <c><anno>Q</anno></c> is empty.</p> + <p><em>Example 1:</em></p> + <pre> +1> <input>queue:daeh(queue:from_list([1,2,3])).</input> +3</pre> </desc> </func> @@ -487,6 +650,10 @@ <p>Returns <c><anno>Item</anno></c> from the head of queue <c><anno>Q</anno></c>.</p> <p>Fails with reason <c>empty</c> if <c><anno>Q</anno></c> is empty.</p> + <p><em>Example 1:</em></p> + <pre> +1> <input>queue:head(queue:from_list([1,2,3])).</input> +1</pre> </desc> </func> @@ -497,6 +664,12 @@ <p>Returns a queue <c><anno>Q2</anno></c> that is the result of removing the tail item from <c><anno>Q1</anno></c>.</p> <p>Fails with reason <c>empty</c> if <c><anno>Q1</anno></c> is empty.</p> + <p><em>Example:</em></p> + <pre> +1> <input>Queue = queue:init(queue:from_list([1,2,3])).</input> +{[2],[1]} +2> <input>queue:to_list(Queue).</input> +[1,2]</pre> </desc> </func> @@ -517,6 +690,10 @@ <desc> <p>Returns the tail item of queue <c><anno>Q</anno></c>.</p> <p>Fails with reason <c>empty</c> if <c><anno>Q</anno></c> is empty.</p> + <p><em>Example:</em></p> + <pre> +1> <input>queue:last(queue:from_list([1,2,3])).</input> +3</pre> </desc> </func> @@ -527,6 +704,12 @@ <p>Returns a queue <c><anno>Q2</anno></c> that is the result of removing the tail item from <c><anno>Q1</anno></c>.</p> <p>Fails with reason <c>empty</c> if <c><anno>Q1</anno></c> is empty.</p> + <p><em>Example:</em></p> + <pre> +1> <input>Queue = queue:liat(queue:from_list([1,2,3])).</input> +{[2],[1]} +2> <input>queue:to_list(Queue).</input> +[1,2]</pre> </desc> </func> @@ -537,6 +720,12 @@ <p>Inserts <c><anno>Item</anno></c> as the tail item of queue <c><anno>Q1</anno></c>. Returns the new queue <c><anno>Q2</anno></c>.</p> + <p><em>Example:</em></p> + <pre> +1> <input>Queue = queue:snoc(queue:from_list([1,2,3]), 4).</input> +{[4,3,2],[1]} +2> <input>queue:to_list(Queue).</input> +[1,2,3,4]</pre> </desc> </func> diff --git a/lib/stdlib/doc/src/rand.xml b/lib/stdlib/doc/src/rand.xml index 471a23f6b9..fb8075cb32 100644 --- a/lib/stdlib/doc/src/rand.xml +++ b/lib/stdlib/doc/src/rand.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>2015</year><year>2022</year> + <year>2015</year><year>2023</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -78,7 +78,7 @@ </p> <taglist> - <tag><c>exsss</c></tag> + <tag since="OTP 22.0"><c>exsss</c></tag> <item> <p>Xorshift116**, 58 bits precision and period of 2^116-1</p> <p>Jump function: equivalent to 2^64 calls</p> @@ -104,7 +104,7 @@ thanks to its statistical qualities. </p> </item> - <tag><c>exro928ss</c></tag> + <tag since="OTP 22.0"><c>exro928ss</c></tag> <item> <p>Xoroshiro928**, 58 bits precision and a period of 2^928-1</p> <p>Jump function: equivalent to 2^512 calls</p> @@ -127,17 +127,17 @@ the 58 bit adaption. </p> </item> - <tag><c>exrop</c></tag> + <tag since="OTP 20.0"><c>exrop</c></tag> <item> <p>Xoroshiro116+, 58 bits precision and period of 2^116-1</p> <p>Jump function: equivalent to 2^64 calls</p> </item> - <tag><c>exs1024s</c></tag> + <tag since="OTP 20.0"><c>exs1024s</c></tag> <item> <p>Xorshift1024*, 64 bits precision and a period of 2^1024-1</p> <p>Jump function: equivalent to 2^512 calls</p> </item> - <tag><c>exsp</c></tag> + <tag since="OTP 20.0"><c>exsp</c></tag> <item> <p>Xorshift116+, 58 bits precision and period of 2^116-1</p> <p>Jump function: equivalent to 2^64 calls</p> @@ -668,7 +668,7 @@ end.</pre> <seemfa marker="#uniform/0"><c>uniform/0</c></seemfa> because all bits in the mantissa are random. This property, in combination with the fact that exactly zero - is never returned is useful for algoritms doing for example + is never returned is useful for algorithms doing for example <c>1.0 / <anno>X</anno></c> or <c>math:log(<anno>X</anno>)</c>. </p> </note> @@ -751,7 +751,7 @@ end.</pre> <seemfa marker="#uniform_s/1"><c>uniform_s/1</c></seemfa> because all bits in the mantissa are random. This property, in combination with the fact that exactly zero - is never returned is useful for algoritms doing for example + is never returned is useful for algorithms doing for example <c>1.0 / <anno>X</anno></c> or <c>math:log(<anno>X</anno>)</c>. </p> </note> diff --git a/lib/stdlib/doc/src/re.xml b/lib/stdlib/doc/src/re.xml index e16ef12f16..d18b976d65 100644 --- a/lib/stdlib/doc/src/re.xml +++ b/lib/stdlib/doc/src/re.xml @@ -5,7 +5,7 @@ <header> <copyright> <year>2007</year> - <year>2021</year> + <year>2023</year> <holder>Ericsson AB, All Rights Reserved</holder> </copyright> <legalnotice> @@ -75,6 +75,9 @@ <datatype> <name name="compile_option"/> </datatype> + <datatype> + <name name="replace_fun"/> + </datatype> </datatypes> <funcs> @@ -363,7 +366,7 @@ elements with Replacement.</fsummary> <desc> <p>Replaces the matched part of the <c><anno>Subject</anno></c> string - with the contents of <c><anno>Replacement</anno></c>.</p> + with <c><anno>Replacement</anno></c>.</p> <p>The permissible options are the same as for <seemfa marker="#run/3"><c>run/3</c></seemfa>, except that option<c> capture</c> is not allowed. Instead a <c>{return, @@ -378,8 +381,8 @@ <c>unicode</c> compilation option is specified to this function, both the regular expression and <c><anno>Subject</anno></c> are to specified as valid Unicode <c>charlist()</c>s.</p> - <p>The replacement string can contain the special character - <c>&</c>, which inserts the whole matching expression in the + <p>If the replacement is given as a string, it can contain the special + character <c>&</c>, which inserts the whole matching expression in the result, and the special sequence <c>\</c>N (where N is an integer > 0), <c>\g</c>N, or <c>\g{</c>N<c>}</c>, resulting in the subexpression number N, is inserted in the result. If no subexpression with that @@ -401,6 +404,35 @@ re:replace("abcd","c","[\\&]",[{return,list}]).</code> <p>gives</p> <code> "ab[&]d"</code> + <p>If the replacement is given as a fun, it will be called with the + whole matching expression as the first argument and a list of subexpression + matches in the order in which they appear in the regular expression. + The returned value will be inserted in the result.</p> + <p><em>Example:</em></p> + <code> +re:replace("abcd", ".(.)", fun(Whole, [<<C>>]) -> <<$#, Whole/binary, $-, (C - $a + $A), $#>> end, [{return, list}]).</code> + <p>gives</p> + <code> +"#ab-B#cd"</code> + <note> + <p>Non-matching optional subexpressions will not be included in the list + of subexpression matches if they are the last subexpressions in the + regular expression.</p> + <p><em>Example:</em></p> + <p>The regular expression <c>"(a)(b)?(c)?"</c> ("a", optionally followed + by "b", optionally followed by "c") will create the following subexpression + lists:</p> + <list> + <item><c>[<<"a">>, <<"b">>, <<"c">>]</c> + when applied to the string <c>"abc"</c></item> + <item><c>[<<"a">>, <<>>, <<"c">>]</c> + when applied to the string <c>"acx"</c></item> + <item><c>[<<"a">>, <<"b">>]</c> + when applied to the string <c>"abx"</c></item> + <item><c>[<<"a">>]</c> + when applied to the string <c>"axx"</c></item> + </list> + </note> <p>As with <c>run/3</c>, compilation errors raise the <c>badarg</c> exception. <seemfa marker="#compile/2"><c>compile/2</c></seemfa> can be used to get more information about the error.</p> @@ -972,7 +1004,7 @@ re:run("ABCabcdABC",".*(?<FOO>abcd).*",[{capture,['FOO']}]).</code> <p>Here the empty binary (<c><<>></c>) represents the unassigned subpattern. In the <c>binary</c> case, some information about the matching is therefore lost, as - <c><<>></c> can + <c><<>></c> can also be an empty string captured.</p> <p>If differentiation between empty matches and non-existing subpatterns is necessary, use the <c>type</c> <c>index</c> and do diff --git a/lib/stdlib/doc/src/ref_man.xml b/lib/stdlib/doc/src/ref_man.xml index e63c455ec8..04990db408 100644 --- a/lib/stdlib/doc/src/ref_man.xml +++ b/lib/stdlib/doc/src/ref_man.xml @@ -32,6 +32,7 @@ <description> </description> <xi:include href="stdlib_app.xml"/> + <xi:include href="argparse.xml"/> <xi:include href="array.xml"/> <xi:include href="assert_hrl.xml"/> <xi:include href="base64.xml"/> @@ -43,6 +44,7 @@ <xi:include href="dict.xml"/> <xi:include href="digraph.xml"/> <xi:include href="digraph_utils.xml"/> + <xi:include href="edlin_expand.xml"/> <xi:include href="epp.xml"/> <xi:include href="erl_anno.xml"/> <xi:include href="erl_error.xml"/> diff --git a/lib/stdlib/doc/src/sets.xml b/lib/stdlib/doc/src/sets.xml index 53b64a3ac0..b3899c440f 100644 --- a/lib/stdlib/doc/src/sets.xml +++ b/lib/stdlib/doc/src/sets.xml @@ -69,6 +69,71 @@ </description> + <section> + <title>Compatibility</title> + <p>The following functions in this module also exist and provide + the same functionality in the + <seeerl marker="gb_sets"><c>gb_sets(3)</c></seeerl> and + <seeerl marker="ordsets"><c>ordsets(3)</c></seeerl> + modules. That is, by only changing the module name for each call, + you can try out different set representations.</p> + <list type="bulleted"> + <item><seemfa marker="#add_element/2"><c>add_element/2</c></seemfa> + </item> + <item><seemfa marker="#del_element/2"><c>del_element/2</c></seemfa> + </item> + <item><seemfa marker="#filter/2"><c>filter/2</c></seemfa> + </item> + <item><seemfa marker="#fold/3"><c>fold/3</c></seemfa> + </item> + <item><seemfa marker="#from_list/1"><c>from_list/1</c></seemfa> + </item> + <item><seemfa marker="#intersection/1"><c>intersection/1</c></seemfa> + </item> + <item><seemfa marker="#intersection/2"><c>intersection/2</c></seemfa> + </item> + <item><seemfa marker="#is_element/2"><c>is_element/2</c></seemfa> + </item> + <item><seemfa marker="#is_empty/1"><c>is_empty/1</c></seemfa> + </item> + <item><seemfa marker="#is_set/1"><c>is_set/1</c></seemfa> + </item> + <item><seemfa marker="#is_subset/2"><c>is_subset/2</c></seemfa> + </item> + <item><seemfa marker="#new/0"><c>new/0</c></seemfa> + </item> + <item><seemfa marker="#size/1"><c>size/1</c></seemfa> + </item> + <item><seemfa marker="#subtract/2"><c>subtract/2</c></seemfa> + </item> + <item><seemfa marker="#to_list/1"><c>to_list/1</c></seemfa> + </item> + <item><seemfa marker="#union/1"><c>union/1</c></seemfa> + </item> + <item><seemfa marker="#union/2"><c>union/2</c></seemfa> + </item> + </list> + <note> + <p> + While the three set implementations offer the same <em>functionality</em> + with respect to the aforementioned functions, their overall <em>behavior</em> + may differ. As mentioned, this module considers elements as different if + and only if they do not match (<c>=:=</c>), while both + <seeerl marker="ordsets"><c>ordsets</c></seeerl> and + <seeerl marker="gb_sets"><c>gb_sets</c></seeerl> consider elements as + different if and only if they do not compare equal (<c>==</c>). + </p> + <p><em>Example:</em></p> + <pre> +1> <input>sets:is_element(1.0, sets:from_list([1])).</input> +false +2> <input>ordsets:is_element(1.0, ordsets:from_list([1])).</input> +true +2> <input>gb_sets:is_element(1.0, gb_sets:from_list([1])).</input> +true</pre> + </note> + </section> + <datatypes> <datatype> <name name="set" n_vars="1"/> diff --git a/lib/stdlib/doc/src/shell.xml b/lib/stdlib/doc/src/shell.xml index 928d2686b6..ba6dc771e9 100644 --- a/lib/stdlib/doc/src/shell.xml +++ b/lib/stdlib/doc/src/shell.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>1996</year><year>2022</year> + <year>1996</year><year>2023</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -957,6 +957,85 @@ q - quit erlang </func> <func> + <name name="start_interactive" arity="0" since="OTP @OTP-17932@"/> + <fsummary>Start the interactive shell</fsummary> + <desc> + <p>Starts the interactive shell if it has not already been started. + It can be used to programatically start the shell from an escript + or when erl is started with the -noinput or -noshell flags.</p> + </desc> + </func> + + <func> + <name name="start_interactive" arity="1" clause_i="1" since="OTP @OTP-17932@"/> + <name name="start_interactive" arity="1" clause_i="2" since="OTP @OTP-17932@"/> + <name name="start_interactive" arity="1" clause_i="3" since="OTP @OTP-17932@"/> + <fsummary>Start the interactive shell</fsummary> + <desc> + <p>Starts the interactive shell if it has not already been started. + It can be used to programatically start the shell from an + <seecom marker="erts:escript"><c>escript</c></seecom> or when + <seecom marker="erts:erl"><c>erl</c></seecom> is started with the + <seecom marker="erts:erl#noinput"><c>-noinput</c></seecom> or + <seecom marker="erts:erl#noshell"><c>-noshell</c></seecom> flags. + The following options are allowed:</p> + <taglist> + <tag>noshell</tag> + <item> + <p>Starts the interactive shell as if <seecom marker="erts:erl#noshell"> + <c>-noshell</c></seecom> was given to <seecom marker="erts:erl"><c>erl</c></seecom>. + This is only useful when erl is started with + <seecom marker="erts:erl#noinput"><c>-noinput</c></seecom> and the + system want to read input data. + </p> + </item> + <tag><seetype marker="erts:erlang#mfa">mfa()</seetype></tag> + <item> + <p>Starts the interactive shell using + <seetype marker="erts:erlang#mfa"><c>mfa()</c></seetype> + as the default shell.</p> + </item> + <tag>{<seetype marker="erts:erlang#node">node()</seetype>, + <seetype marker="erts:erlang#mfa">mfa()</seetype>}</tag> + <item> + <p>Starts the interactive shell using + <seetype marker="erts:erlang#mfa"><c>mfa()</c></seetype> on + <seetype marker="erts:erlang#node"><c>node()</c></seetype> as the default shell.</p> + </item> + <tag>{remote, <seetype marker="erts:erlang#string"><c>string()</c></seetype>}</tag> + <item> + <p>Starts the interactive shell using as if + <seecom marker="erts:erl#remsh"><c>-remsh</c></seecom> + was given to <seecom marker="erts:erl"><c>erl</c></seecom>.</p> + </item> + <tag>{remote, + <seetype marker="erts:erlang#string"><c>string()</c></seetype>, + <seetype marker="erts:erlang#mfa"><c>mfa()</c></seetype>}</tag> + <item> + <p>Starts the interactive shell using as if + <seecom marker="erts:erl#remsh"><c>-remsh</c></seecom> + was given to <seecom marker="erts:erl"><c>erl</c></seecom> + but with an alternative shell implementation.</p> + </item> + </taglist> + <p>On error this function will return:</p> + <taglist> + <tag>already_started</tag> + <item>if an interactive shell is already started.</item> + <tag>noconnection</tag> + <item>if a remote shell was requested but it could not be connected to.</item> + <tag>badfile | nofile | on_load_failure</tag> + <item>if a remote shell was requested with a custom + <seetype marker="erts:erlang#mfa">mfa()</seetype>, + but the module could not be loaded. See <seeerl marker="kernel:code#error_reasons"> + Error Reasons for Code-Loading Functions</seeerl> + for a description of the error reasons. + </item> + </taglist> + </desc> + </func> + + <func> <name name="start_restricted" arity="1" since=""/> <fsummary>Exit a normal shell and starts a restricted shell.</fsummary> <desc> @@ -994,6 +1073,17 @@ q - quit erlang string syntax.</p> </desc> </func> + + <func> + <name name="whereis" arity="0" since="OTP @OTP-17932@"/> + <fsummary>Return the current shell process.</fsummary> + <desc> + <p>Returns the current shell process on the node where the + calling process' group_leader is located. If that node + has no shell this function will return undefined.</p> + </desc> + </func> + </funcs> </erlref> diff --git a/lib/stdlib/doc/src/specs.xml b/lib/stdlib/doc/src/specs.xml index 9fdd0d1c21..fc19db4bf3 100644 --- a/lib/stdlib/doc/src/specs.xml +++ b/lib/stdlib/doc/src/specs.xml @@ -1,5 +1,6 @@ <?xml version="1.0" encoding="utf-8" ?> <specs xmlns:xi="http://www.w3.org/2001/XInclude"> + <xi:include href="../specs/specs_argparse.xml"/> <xi:include href="../specs/specs_array.xml"/> <xi:include href="../specs/specs_base64.xml"/> <xi:include href="../specs/specs_beam_lib.xml"/> @@ -10,6 +11,7 @@ <xi:include href="../specs/specs_dict.xml"/> <xi:include href="../specs/specs_digraph.xml"/> <xi:include href="../specs/specs_digraph_utils.xml"/> + <xi:include href="../specs/specs_edlin_expand.xml"/> <xi:include href="../specs/specs_epp.xml"/> <xi:include href="../specs/specs_erl_anno.xml"/> <xi:include href="../specs/specs_erl_error.xml"/> diff --git a/lib/stdlib/doc/src/stdlib_app.xml b/lib/stdlib/doc/src/stdlib_app.xml index 243853140b..e0f50292d6 100644 --- a/lib/stdlib/doc/src/stdlib_app.xml +++ b/lib/stdlib/doc/src/stdlib_app.xml @@ -57,6 +57,11 @@ <p>Can be used to set the exception handling of the evaluator process of Erlang shell.</p> </item> + <tag><marker id="shell_expand_location"/><c>shell_expand_location = above | below</c></tag> + <item> + <p>Sets where the tab expansion text should appear in the shell. + The default is <c>below</c>.</p> + </item> <tag><marker id="shell_history_length"/><c>shell_history_length = integer() >= 0</c></tag> <item> <p>Can be used to determine how many commands are saved by the Erlang @@ -76,6 +81,29 @@ <p>Can be used to determine how many results are saved by the Erlang shell.</p> </item> + <tag><marker id="shell_session_slogan"/><c>shell_session_slogan = string() | fun() -> string())</c></tag> + <item> + <p>The slogan printed when starting an Erlang shell. Example: </p> + <code type="erl"> +$ erl -stdlib shell_session_slogan '"Test slogan"' +Erlang/OTP 26 [DEVELOPMENT] [erts-13.0.2] [source] [64-bit] [smp:8:8] [ds:8:8:10] [async-threads:1] [jit:ns] + +Test slogan +1> + </code> + </item> + <tag><marker id="shell_slogan"/><c>shell_slogan = string() | fun(() -> string())</c></tag> + <item> + <p>The slogan printed when starting the Erlang shell subsystem. Example: </p> + <code type="erl"> +$ erl -stdlib shell_slogan '"Test slogan"' +Test slogan +Eshell V13.0.2 (abort with ^G) +1> + </code> + <p>The default is the return value of <seeerl marker="erts:erlang#system_info_system_version"> + <c>erlang:system_info(system_version)</c></seeerl>.</p> + </item> <tag><marker id="shell_strings"/><c>shell_strings = boolean()</c></tag> <item> <p>Can be used to determine how the Erlang shell outputs lists of diff --git a/lib/stdlib/doc/src/string.xml b/lib/stdlib/doc/src/string.xml index 3c8e7250df..5176f6de60 100644 --- a/lib/stdlib/doc/src/string.xml +++ b/lib/stdlib/doc/src/string.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>1996</year><year>2021</year> + <year>1996</year><year>2023</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -69,7 +69,7 @@ <seemfa marker="#find/3"><c>find/3</c></seemfa>, <seemfa marker="#replace/3"><c>replace/3</c></seemfa>, <seemfa marker="#split/2"><c>split/2</c></seemfa>, - <seemfa marker="#lexemes/2"><c>split/2</c></seemfa> and + <seemfa marker="#split/3"><c>split/3</c></seemfa> and <seemfa marker="#trim/3"><c>trim/3</c></seemfa>. </p> <p> diff --git a/lib/stdlib/doc/src/timer.xml b/lib/stdlib/doc/src/timer.xml index 11279ff410..3fceddbec6 100644 --- a/lib/stdlib/doc/src/timer.xml +++ b/lib/stdlib/doc/src/timer.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>1996</year><year>2021</year> + <year>1996</year><year>2023</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -71,10 +71,10 @@ <funcs> <func> <name name="apply_after" arity="4" since=""/> - <fsummary>Apply <c>Module:Function(Arguments)</c> after a specified - <c>Time</c>.</fsummary> + <fsummary>Spawn a process evaluating <c>Module:Function(Arguments)</c> + after a specified <c>Time</c>.</fsummary> <desc> - <p>Evaluates <c>apply(<anno>Module</anno>, <anno>Function</anno>, + <p>Evaluates <c>spawn(<anno>Module</anno>, <anno>Function</anno>, <anno>Arguments</anno>)</c> after <c><anno>Time</anno></c> milliseconds.</p> <p>Returns <c>{ok, <anno>TRef</anno>}</c> or @@ -84,12 +84,53 @@ <func> <name name="apply_interval" arity="4" since=""/> - <fsummary>Evaluate <c>Module:Function(Arguments)</c> repeatedly at - intervals of <c>Time</c>.</fsummary> + <fsummary>Spawn a process evaluating <c>Module:Function(Arguments)</c> + repeatedly at intervals of <c>Time</c>.</fsummary> <desc> - <p>Evaluates <c>apply(<anno>Module</anno>, <anno>Function</anno>, + <p>Evaluates <c>spawn(<anno>Module</anno>, <anno>Function</anno>, <anno>Arguments</anno>)</c> repeatedly at intervals of - <c><anno>Time</anno></c>.</p> + <c><anno>Time</anno></c>, irrespective of whether a previously + spawned process has finished or not.</p> + <warning> + <p>If the execution time of the spawned process is, on average, + greater than the given <c><anno>Time</anno></c>, multiple such + processes will run at the same time. With long execution times, + short intervals, and many interval timers running, this may even + lead to exceeding the number of allowed processes. As an extreme + example, consider + <c>[timer:apply_interval(1, timer, sleep, [1000]) || _ <- lists:seq(1, 1000)]</c>, + that is, 1,000 interval timers executing a process that takes 1s + to complete, started in intervals of 1ms, which would result in + 1,000,000 processes running at the same time, far more than a node + started with default settings allows (see the + <seeguide marker="system/efficiency_guide:advanced#system-limits">System + Limits section in the Effiency Guide</seeguide>).</p> + </warning> + <p>Returns <c>{ok, <anno>TRef</anno>}</c> or + <c>{error, <anno>Reason</anno>}</c>.</p> + </desc> + </func> + + <func> + <name name="apply_repeatedly" arity="4" since="OTP @OTP-18236@"/> + <fsummary>Spawn a process evaluating <c>Module:Function(Arguments)</c> + repeatedly at intervals of <c>Time</c>.</fsummary> + <desc> + <p>Evaluates <c>spawn(<anno>Module</anno>, <anno>Function</anno>, + <anno>Arguments</anno>)</c> repeatedly at intervals of + <c><anno>Time</anno></c>, waiting for the spawned process to + finish before starting the next.</p> + <p>If the execution time of the spawned process is greater than the + given <c><anno>Time</anno></c>, the next process is spawned immediately + after the one currently running has finished. Assuming that execution + times of the spawned processes performing the applies on average + are smaller than <c><anno>Time</anno></c>, the amount of applies + made over a large amount of time will be the same even if some + individual execution times are larger than + <c><anno>Time</anno></c>. The system will try to catch up as soon + as possible. For example, if one apply takes + <c>2.5*<anno>Time</anno></c>, the following two applies will be + made immediately one after the other in sequence.</p> <p>Returns <c>{ok, <anno>TRef</anno>}</c> or <c>{error, <anno>Reason</anno>}</c>.</p> </desc> @@ -278,14 +319,37 @@ <func> <name name="tc" arity="1" since="OTP R14B03"/> - <name name="tc" arity="2" since="OTP R14B"/> - <name name="tc" arity="3" since=""/> + <name name="tc" arity="2" clause_i="1" since="OTP R14B"/> + <name name="tc" arity="3" clause_i="1" since=""/> + <fsummary>Measure the real time it takes to evaluate <c>Fun</c>.</fsummary> + <desc> + <taglist> + <tag><c>tc/3</c></tag> + <item> + <p>Calls function <c>timer:tc(Module, Function, Arguments, microsecond)</c>.</p> + </item> + <tag><c>tc/2</c></tag> + <item> + <p>Calls function <c>timer:tc(Fun, Arguments, microsecond)</c>.</p> + </item> + <tag><c>tc/1</c></tag> + <item> + <p>Calls function <c>timer:tc(Fun, microsecond)</c>.</p> + </item> + </taglist> + </desc> + </func> + + <func> + <name name="tc" arity="2" clause_i="2" since="OTP @OTP-18355@"/> + <name name="tc" arity="3" clause_i="2" since="OTP @OTP-18355@"/> + <name name="tc" arity="4" since="OTP @OTP-18355@"/> <fsummary>Measure the real time it takes to evaluate <c>apply(Module, Function, Arguments)</c> or <c>apply(Fun, Arguments)</c>.</fsummary> - <type_desc variable="Time">In microseconds</type_desc> + <type_desc variable="Time">In the specified <c>TimeUnit</c></type_desc> <desc> <taglist> - <tag><c>tc/3</c></tag> + <tag><c>tc/4</c></tag> <item> <p>Evaluates <c>apply(<anno>Module</anno>, <anno>Function</anno>, <anno>Arguments</anno>)</c> and measures the elapsed real time as @@ -293,18 +357,18 @@ <c>erlang:monotonic_time/0</c></seemfa>.</p> <p>Returns <c>{<anno>Time</anno>, <anno>Value</anno>}</c>, where <c><anno>Time</anno></c> is the elapsed real time in - <em>microseconds</em>, and <c><anno>Value</anno></c> is what is + the specified <c>TimeUnit</c>, and <c><anno>Value</anno></c> is what is returned from the apply.</p> </item> - <tag><c>tc/2</c></tag> + <tag><c>tc/3</c></tag> <item> <p>Evaluates <c>apply(<anno>Fun</anno>, <anno>Arguments</anno>)</c>. - Otherwise the same as <c>tc/3</c>.</p> + Otherwise the same as <c>tc/4</c>.</p> </item> - <tag><c>tc/1</c></tag> + <tag><c>tc/2</c></tag> <item> <p>Evaluates <c><anno>Fun</anno>()</c>. Otherwise the same as - <c>tc/2</c>.</p> + <c>tc/3</c>.</p> </item> </taglist> </desc> diff --git a/lib/stdlib/doc/src/unicode.xml b/lib/stdlib/doc/src/unicode.xml index 1f0133de67..5008f13075 100644 --- a/lib/stdlib/doc/src/unicode.xml +++ b/lib/stdlib/doc/src/unicode.xml @@ -5,7 +5,7 @@ <header> <copyright> <year>1996</year> - <year>2020</year> + <year>2023</year> <holder>Ericsson AB, All Rights Reserved</holder> </copyright> <legalnotice> @@ -37,7 +37,7 @@ representations. It converts between ISO Latin-1 characters and Unicode characters, but it can also convert between different Unicode encodings (like UTF-8, UTF-16, and UTF-32).</p> - <p>The default Unicode encoding in Erlang is in binaries UTF-8, which is also + <p>The default Unicode encoding in Erlang binaries is UTF-8, which is also the format in which built-in functions and libraries in OTP expect to find binary Unicode data. In lists, Unicode data is encoded as integers, each integer representing one character and encoded simply as the Unicode code diff --git a/lib/stdlib/doc/src/zip.xml b/lib/stdlib/doc/src/zip.xml index a056621ca1..6ba9f4c212 100644 --- a/lib/stdlib/doc/src/zip.xml +++ b/lib/stdlib/doc/src/zip.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>2006</year><year>2020</year> + <year>2006</year><year>2023</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -507,6 +507,14 @@ </func> <func> + <name name="zip_get_crc32" arity="2" since="OTP @OTP-18159@"/> + <fsummary>Extracts a crc32 checksum from an open archive.</fsummary> + <desc> + <p>Extracts one crc32 checksum from an open archive.</p> + </desc> + </func> + + <func> <name name="zip_list_dir" arity="1" since=""/> <fsummary>Return a table of files in open zip archive.</fsummary> <desc> diff --git a/lib/stdlib/examples/erl_id_trans.erl b/lib/stdlib/examples/erl_id_trans.erl index d9d1154515..792f944d36 100644 --- a/lib/stdlib/examples/erl_id_trans.erl +++ b/lib/stdlib/examples/erl_id_trans.erl @@ -409,13 +409,17 @@ expr({cons,Anno,H0,T0}) -> T1 = expr(T0), %They see the same variables {cons,Anno,H1,T1}; expr({lc,Anno,E0,Qs0}) -> - Qs1 = lc_bc_quals(Qs0), + Qs1 = comprehension_quals(Qs0), E1 = expr(E0), {lc,Anno,E1,Qs1}; expr({bc,Anno,E0,Qs0}) -> - Qs1 = lc_bc_quals(Qs0), + Qs1 = comprehension_quals(Qs0), E1 = expr(E0), {bc,Anno,E1,Qs1}; +expr({mc,Anno,E0,Qs0}) -> + Qs1 = comprehension_quals(Qs0), + E1 = expr(E0), + {mc,Anno,E1,Qs1}; expr({tuple,Anno,Es0}) -> Es1 = expr_list(Es0), {tuple,Anno,Es1}; @@ -570,21 +574,25 @@ icr_clauses([C0|Cs]) -> [C1|icr_clauses(Cs)]; icr_clauses([]) -> []. -%% -type lc_bc_quals([Qualifier]) -> [Qualifier]. +%% -type comprehension_quals([Qualifier]) -> [Qualifier]. %% Allow filters to be both guard tests and general expressions. -lc_bc_quals([{generate,Anno,P0,E0}|Qs]) -> +comprehension_quals([{generate,Anno,P0,E0}|Qs]) -> + E1 = expr(E0), + P1 = pattern(P0), + [{generate,Anno,P1,E1}|comprehension_quals(Qs)]; +comprehension_quals([{b_generate,Anno,P0,E0}|Qs]) -> E1 = expr(E0), P1 = pattern(P0), - [{generate,Anno,P1,E1}|lc_bc_quals(Qs)]; -lc_bc_quals([{b_generate,Anno,P0,E0}|Qs]) -> + [{b_generate,Anno,P1,E1}|comprehension_quals(Qs)]; +comprehension_quals([{m_generate,Anno,P0,E0}|Qs]) -> E1 = expr(E0), P1 = pattern(P0), - [{b_generate,Anno,P1,E1}|lc_bc_quals(Qs)]; -lc_bc_quals([E0|Qs]) -> + [{m_generate,Anno,P1,E1}|comprehension_quals(Qs)]; +comprehension_quals([E0|Qs]) -> E1 = expr(E0), - [E1|lc_bc_quals(Qs)]; -lc_bc_quals([]) -> []. + [E1|comprehension_quals(Qs)]; +comprehension_quals([]) -> []. %% -type fun_clauses([Clause]) -> [Clause]. diff --git a/lib/stdlib/src/Makefile b/lib/stdlib/src/Makefile index 761d6c4c28..abdb665b09 100644 --- a/lib/stdlib/src/Makefile +++ b/lib/stdlib/src/Makefile @@ -42,6 +42,7 @@ RELSYSDIR = $(RELEASE_PATH)/lib/stdlib-$(VSN) # ---------------------------------------------------- MODULES= \ array \ + argparse \ base64 \ beam_lib \ binary \ @@ -56,7 +57,9 @@ MODULES= \ digraph \ digraph_utils \ edlin \ + edlin_context \ edlin_expand \ + edlin_type_suggestion \ epp \ erl_abstract_code \ erl_anno \ @@ -196,7 +199,8 @@ primary_bootstrap_compiler: \ $(BOOTSTRAP_COMPILER)/ebin/erl_parse.beam \ $(BOOTSTRAP_COMPILER)/ebin/erl_lint.beam \ $(BOOTSTRAP_COMPILER)/ebin/io.beam \ - $(BOOTSTRAP_COMPILER)/ebin/otp_internal.beam + $(BOOTSTRAP_COMPILER)/ebin/otp_internal.beam \ + $(BOOTSTRAP_COMPILER)/ebin/erl_internal.beam $(BOOTSTRAP_COMPILER)/ebin/erl_parse.beam: erl_parse.yrl diff --git a/lib/stdlib/src/argparse.erl b/lib/stdlib/src/argparse.erl new file mode 100644 index 0000000000..a5fdd8d3d9 --- /dev/null +++ b/lib/stdlib/src/argparse.erl @@ -0,0 +1,1357 @@ +%% +%% +%% Copyright Maxim Fedorov +%% +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(argparse). +-author("maximfca@gmail.com"). + +%% API Exports +-export([ + run/3, + parse/2, parse/3, + help/1, help/2, + format_error/1 +]). + +%% Internal exports for validation and error reporting. +-export([validate/1, validate/2, format_error/2]). + +%%-------------------------------------------------------------------- +%% API + +-type arg_type() :: + boolean | + float | + {float, Choice :: [float()]} | + {float, [{min, float()} | {max, float()}]} | + integer | + {integer, Choices :: [integer()]} | + {integer, [{min, integer()} | {max, integer()}]} | + string | + {string, Choices :: [string()]} | + {string, Re :: string()} | + {string, Re :: string(), ReOptions :: [term()]} | + binary | + {binary, Choices :: [binary()]} | + {binary, Re :: binary()} | + {binary, Re :: binary(), ReOptions :: [term()]} | + atom | + {atom, Choices :: [atom()]} | + {atom, unsafe} | + {custom, fun((string()) -> term())}. +%% Built-in types include basic validation abilities +%% String and binary validation may use regex match (ignoring captured value). +%% For float, integer, string, binary and atom type, it is possible to specify +%% available choices instead of regex/min/max. + +-type argument_help() :: { + unicode:chardata(), %% short form, printed in command usage, e.g. "[--dir <dirname>]", developer is + %% responsible for proper formatting (e.g. adding <>, dots... and so on) + [unicode:chardata() | type | default] | fun(() -> unicode:chardata()) +}. +%% Help template definition for argument. Short and long forms exist for every argument. +%% Short form is printed together with command definition, e.g. "usage: rm [--force]", +%% while long description is printed in detailed section below: "--force forcefully remove". + +-type argument_name() :: atom() | string() | binary(). + +-type argument() :: #{ + %% Argument name, and a destination to store value too + %% It is allowed to have several arguments named the same, setting or appending to the same variable. + name := argument_name(), + + %% short, single-character variant of command line option, omitting dash (example: $b, meaning -b), + %% when present, the argument is considered optional + short => char(), + + %% long command line option, omitting first dash (example: "kernel" means "-kernel" in the command line) + %% long command always wins over short abbreviation (e.g. -kernel is considered before -k -e -r -n -e -l) + %% when present, the argument is considered optional + long => string(), + + %% makes parser to return an error if the argument is not present in the command line + required => boolean(), + + %% default value, produced if the argument is not present in the command line + %% parser also accepts a global default + default => term(), + + %% parameter type (string by default) + type => arg_type(), + + %% action to take when argument is matched + action => store | %% default: store argument consumed (last stored wins) + {store, term()} | %% does not consume argument, stores term() instead + append | %% appends consumed argument to a list + {append, term()} | %% does not consume an argument, appends term() to a list + count | %% does not consume argument, bumps counter + extend, %% uses when nargs is list/nonempty_list/all - appends every element to the list + + %% how many positional arguments to consume + nargs => + pos_integer() | %% consume exactly this amount, e.g. '-kernel key value' #{long => "-kernel", args => 2} + %% returns #{kernel => ["key", "value"]} + 'maybe' | %% if the next argument is positional, consume it, otherwise produce default + {'maybe', term()} | %% if the next argument is positional, consume it, otherwise produce term() + list | %% consume zero or more positional arguments, until next optional + nonempty_list | %% consume at least one positional argument, until next optional + all, %% fold remaining command line into this argument + + %% help string printed in usage, hidden help is not printed at all + help => hidden | unicode:chardata() | argument_help() +}. +%% Command line argument specification. +%% Argument can be optional - starting with - (dash), and positional. + +-type arg_map() :: #{argument_name() => term()}. +%% Arguments map: argument name to a term, produced by parser. Supplied to the command handler + +-type handler() :: + optional | %% valid for commands with sub-commands, suppresses parser error when no + %% sub-command is selected + fun((arg_map()) -> term()) | %% handler accepting arg_map + {module(), Fn :: atom()} | %% handler, accepting arg_map, Fn exported from module() + {fun(() -> term()), term()} | %% handler, positional form (term() is supplied for omitted args) + {module(), atom(), term()}. %% handler, positional form, exported from module() +%% Command handler. May produce some output. Can accept a map, or be +%% arbitrary mfa() for handlers accepting positional list. +%% Special value 'optional' may be used to suppress an error that +%% otherwise raised when command contains sub-commands, but arguments +%% supplied via command line do not select any. + +-type command_help() :: [unicode:chardata() | usage | commands | arguments | options]. +%% Template for the command help/usage message. + +%% Command descriptor +-type command() :: #{ + %% Sub-commands are arranged into maps. Command name must not start with <em>prefix</em>. + commands => #{string() => command()}, + %% accepted arguments list. Order is important! + arguments => [argument()], + %% help line + help => hidden | unicode:chardata() | command_help(), + %% recommended handler function + handler => handler() +}. + +-type cmd_path() :: [string()]. +%% Command path, for nested commands + +-export_type([arg_type/0, argument_help/0, argument/0, + command/0, handler/0, cmd_path/0, arg_map/0]). + +-type parser_error() :: {Path :: cmd_path(), + Expected :: argument() | undefined, + Actual :: string() | undefined, + Details :: unicode:chardata()}. +%% Returned from `parse/2,3' when command spec is valid, but the command line +%% cannot be parsed using the spec. +%% When `Expected' is undefined, but `Actual' is not, it means that the input contains +%% an unexpected argument which cannot be parsed according to command spec. +%% When `Expected' is an argument, and `Actual' is undefined, it means that a mandatory +%% argument is not provided in the command line. +%% When both `Expected' and `Actual' are defined, it means that the supplied argument +%% is failing validation. +%% When both are `undefined', there is some logical issue (e.g. a sub-command is required, +%% but was not selected). + +-type parser_options() :: #{ + %% allowed prefixes (default is [$-]). + prefixes => [char()], + %% default value for all missing optional arguments + default => term(), + %% root command name (program name) + progname => string() | atom(), + %% considered by `help/2' only + command => cmd_path(), %% command to print the help for + columns => pos_integer() %% viewport width, in characters +}. +%% Parser options + +-type parse_result() :: + {ok, arg_map(), Path :: cmd_path(), command()} | + {error, parser_error()}. +%% Parser result: argument map, path leading to successfully +%% matching command (contains only ["progname"] if there were +%% no subcommands matched), and a matching command. + +%% @equiv validate(Command, #{}) +-spec validate(command()) -> Progname :: string(). +validate(Command) -> + validate(Command, #{}). + +%% @doc Validate command specification, taking Options into account. +%% Raises an error if the command specification is invalid. +-spec validate(command(), parser_options()) -> Progname :: string(). +validate(Command, Options) -> + Prog = executable(Options), + is_list(Prog) orelse erlang:error(badarg, [Command, Options], + [{error_info, #{cause => #{2 => <<"progname is not valid">>}}}]), + Prefixes = maps:from_list([{P, true} || P <- maps:get(prefixes, Options, [$-])]), + _ = validate_command([{Prog, Command}], Prefixes), + Prog. + +%% @equiv parse(Args, Command, #{}) +-spec parse(Args :: [string()], command()) -> parse_result(). +parse(Args, Command) -> + parse(Args, Command, #{}). + +%% @doc Parses supplied arguments according to expected command specification. +%% @param Args command line arguments (e.g. `init:get_plain_arguments()') +%% @returns argument map, or argument map with deepest matched command +%% definition. +-spec parse(Args :: [string()], command(), Options :: parser_options()) -> parse_result(). +parse(Args, Command, Options) -> + Prog = validate(Command, Options), + %% use maps and not sets v2, because sets:is_element/2 cannot be used in guards (unlike is_map_key) + Prefixes = maps:from_list([{P, true} || P <- maps:get(prefixes, Options, [$-])]), + try + parse_impl(Args, merge_arguments(Prog, Command, init_parser(Prefixes, Command, Options))) + catch + %% Parser error may happen at any depth, and bubbling the error is really + %% cumbersome. Use exceptions and catch it before returning from `parse/2,3' instead. + throw:Reason -> + {error, Reason} + end. + +%% @equiv help(Command, #{}) +-spec help(command()) -> string(). +help(Command) -> + help(Command, #{}). + +%% @doc Returns help for Command formatted according to Options specified +-spec help(command(), parser_options()) -> unicode:chardata(). +help(Command, Options) -> + Prog = validate(Command, Options), + format_help({Prog, Command}, Options). + +%% @doc +-spec run(Args :: [string()], command(), parser_options()) -> term(). +run(Args, Command, Options) -> + try parse(Args, Command, Options) of + {ok, ArgMap, Path, SubCmd} -> + handle(Command, ArgMap, tl(Path), SubCmd); + {error, Reason} -> + io:format("error: ~ts~n", [argparse:format_error(Reason)]), + io:format("~ts", [argparse:help(Command, Options#{command => tl(element(1, Reason))})]), + erlang:halt(1) + catch + error:Reason:Stack -> + io:format(erl_error:format_exception(error, Reason, Stack)), + erlang:halt(1) + end. + +%% @doc Basic formatter for the parser error reason. +-spec format_error(Reason :: parser_error()) -> unicode:chardata(). +format_error({Path, undefined, undefined, Details}) -> + io_lib:format("~ts: ~ts", [format_path(Path), Details]); +format_error({Path, undefined, Actual, Details}) -> + io_lib:format("~ts: unknown argument: ~ts~ts", [format_path(Path), Actual, Details]); +format_error({Path, #{name := Name}, undefined, Details}) -> + io_lib:format("~ts: required argument missing: ~ts~ts", [format_path(Path), Name, Details]); +format_error({Path, #{name := Name}, Value, Details}) -> + io_lib:format("~ts: invalid argument for ~ts: ~ts ~ts", [format_path(Path), Name, Value, Details]). + +-type validator_error() :: + {?MODULE, command | argument, cmd_path(), Field :: atom(), Detail :: unicode:chardata()}. + +%% @doc Transforms exception thrown by `validate/1,2' according to EEP54. +%% Use `erl_error:format_exception/3,4' to get the shell-like output. +-spec format_error(Reason :: validator_error(), erlang:stacktrace()) -> map(). +format_error({?MODULE, command, Path, Field, Reason}, [{_M, _F, [Cmd], Info} | _]) -> + #{cause := Cause} = proplists:get_value(error_info, Info, #{}), + Cause#{general => <<"command specification is invalid">>, 1 => io_lib:format("~tp", [Cmd]), + reason => io_lib:format("command \"~ts\": invalid field '~ts', reason: ~ts", [format_path(Path), Field, Reason])}; +format_error({?MODULE, argument, Path, Field, Reason}, [{_M, _F, [Arg], Info} | _]) -> + #{cause := Cause} = proplists:get_value(error_info, Info, #{}), + ArgName = maps:get(name, Arg, ""), + Cause#{general => "argument specification is invalid", 1 => io_lib:format("~tp", [Arg]), + reason => io_lib:format("command \"~ts\", argument '~ts', invalid field '~ts': ~ts", + [format_path(Path), ArgName, Field, Reason])}. + +%%-------------------------------------------------------------------- +%% Parser implementation + +%% Parser state (not available via API) +-record(eos, { + %% prefix character map, by default, only - + prefixes :: #{char() => true}, + %% argument map to be returned + argmap = #{} :: arg_map(), + %% sub-commands, in reversed orders, allowing to recover the path taken + commands = [] :: cmd_path(), + %% command being matched + current :: command(), + %% unmatched positional arguments, in the expected match order + pos = [] :: [argument()], + %% expected optional arguments, mapping between short/long form and an argument + short = #{} :: #{integer() => argument()}, + long = #{} :: #{string() => argument()}, + %% flag, whether there are no options that can be confused with negative numbers + no_digits = true :: boolean(), + %% global default for not required arguments + default :: error | {ok, term()} +}). + +init_parser(Prefixes, Cmd, Options) -> + #eos{prefixes = Prefixes, current = Cmd, default = maps:find(default, Options)}. + +%% Optional or positional argument? +-define(IS_OPTION(Arg), is_map_key(short, Arg) orelse is_map_key(long, Arg)). + +%% helper function to match either a long form of "--arg=value", or just "--arg" +match_long(Arg, LongOpts) -> + case maps:find(Arg, LongOpts) of + {ok, Option} -> + {ok, Option}; + error -> + %% see if there is '=' equals sign in the Arg + case string:split(Arg, "=") of + [MaybeLong, Value] -> + case maps:find(MaybeLong, LongOpts) of + {ok, Option} -> + {ok, Option, Value}; + error -> + nomatch + end; + _ -> + nomatch + end + end. + +%% parse_impl implements entire internal parse logic. + +%% Clause: option starting with any prefix +%% No separate clause for single-character short form, because there could be a single-character +%% long form taking precedence. +parse_impl([[Prefix | Name] | Tail], #eos{prefixes = Pref} = Eos) when is_map_key(Prefix, Pref) -> + %% match "long" option from the list of currently known + case match_long(Name, Eos#eos.long) of + {ok, Option} -> + consume(Tail, Option, Eos); + {ok, Option, Value} -> + consume([Value | Tail], Option, Eos); + nomatch -> + %% try to match single-character flag + case Name of + [Flag] when is_map_key(Flag, Eos#eos.short) -> + %% found a flag + consume(Tail, maps:get(Flag, Eos#eos.short), Eos); + [Flag | Rest] when is_map_key(Flag, Eos#eos.short) -> + %% can be a combination of flags, or flag with value, + %% but can never be a negative integer, because otherwise + %% it will be reflected in no_digits + case abbreviated(Name, [], Eos#eos.short) of + false -> + %% short option with Rest being an argument + consume([Rest | Tail], maps:get(Flag, Eos#eos.short), Eos); + Expanded -> + %% expand multiple flags into actual list, adding prefix + parse_impl([[Prefix,E] || E <- Expanded] ++ Tail, Eos) + end; + MaybeNegative when Prefix =:= $-, Eos#eos.no_digits -> + case is_digits(MaybeNegative) of + true -> + %% found a negative number + parse_positional([Prefix|Name], Tail, Eos); + false -> + catch_all_positional([[Prefix|Name] | Tail], Eos) + end; + _Unknown -> + catch_all_positional([[Prefix|Name] | Tail], Eos) + end + end; + +%% Arguments not starting with Prefix: attempt to match sub-command, if available +parse_impl([Positional | Tail], #eos{current = #{commands := SubCommands}} = Eos) -> + case maps:find(Positional, SubCommands) of + error -> + %% sub-command not found, try positional argument + parse_positional(Positional, Tail, Eos); + {ok, SubCmd} -> + %% found matching sub-command with arguments, descend into it + parse_impl(Tail, merge_arguments(Positional, SubCmd, Eos)) + end; + +%% Clause for arguments that don't have sub-commands (therefore check for +%% positional argument). +parse_impl([Positional | Tail], Eos) -> + parse_positional(Positional, Tail, Eos); + +%% Entire command line has been matched, go over missing arguments, +%% add defaults etc +parse_impl([], #eos{argmap = ArgMap0, commands = Commands, current = Current, pos = Pos, default = Def} = Eos) -> + %% error if stopped at sub-command with no handler + map_size(maps:get(commands, Current, #{})) >0 andalso + (not is_map_key(handler, Current)) andalso + throw({Commands, undefined, undefined, <<"subcommand expected">>}), + + %% go over remaining positional, verify they are all not required + ArgMap1 = fold_args_map(Commands, true, ArgMap0, Pos, Def), + %% go over optionals, and either raise an error, or set default + ArgMap2 = fold_args_map(Commands, false, ArgMap1, maps:values(Eos#eos.short), Def), + ArgMap3 = fold_args_map(Commands, false, ArgMap2, maps:values(Eos#eos.long), Def), + + %% return argument map, command path taken, and the deepest + %% last command matched (usually it contains a handler to run) + {ok, ArgMap3, Eos#eos.commands, Eos#eos.current}. + +%% Generate error for missing required argument, and supply defaults for +%% missing optional arguments that have defaults. +fold_args_map(Commands, Req, ArgMap, Args, GlobalDefault) -> + lists:foldl( + fun (#{name := Name}, Acc) when is_map_key(Name, Acc) -> + %% argument present + Acc; + (#{required := true} = Opt, _Acc) -> + %% missing, and required explicitly + throw({Commands, Opt, undefined, <<>>}); + (#{name := Name, required := false, default := Default}, Acc) -> + %% explicitly not required argument with default + Acc#{Name => Default}; + (#{name := Name, required := false}, Acc) -> + %% explicitly not required with no local default, try global one + try_global_default(Name, Acc, GlobalDefault); + (#{name := Name, default := Default}, Acc) when Req =:= true -> + %% positional argument with default + Acc#{Name => Default}; + (Opt, _Acc) when Req =:= true -> + %% missing, for positional argument, implicitly required + throw({Commands, Opt, undefined, <<>>}); + (#{name := Name, default := Default}, Acc) -> + %% missing, optional, and there is a default + Acc#{Name => Default}; + (#{name := Name}, Acc) -> + %% missing, optional, no local default, try global default + try_global_default(Name, Acc, GlobalDefault) + end, ArgMap, Args). + +try_global_default(_Name, Acc, error) -> + Acc; +try_global_default(Name, Acc, {ok, Term}) -> + Acc#{Name => Term}. + +%%-------------------------------------------------------------------- +%% argument consumption (nargs) handling + +catch_all_positional(Tail, #eos{pos = [#{nargs := all} = Opt]} = Eos) -> + action([], Tail, Opt#{type => {list, maps:get(type, Opt, string)}}, Eos); +%% it is possible that some positional arguments are not required, +%% and therefore it is possible to catch all skipping those +catch_all_positional(Tail, #eos{argmap = Args, pos = [#{name := Name, default := Default, required := false} | Pos]} = Eos) -> + catch_all_positional(Tail, Eos#eos{argmap = Args#{Name => Default}, pos = Pos}); +%% same as above, but no default specified +catch_all_positional(Tail, #eos{pos = [#{required := false} | Pos]} = Eos) -> + catch_all_positional(Tail, Eos#eos{pos = Pos}); +catch_all_positional([Arg | _Tail], #eos{commands = Commands}) -> + throw({Commands, undefined, Arg, <<>>}). + +parse_positional(Arg, _Tail, #eos{pos = [], commands = Commands}) -> + throw({Commands, undefined, Arg, <<>>}); +parse_positional(Arg, Tail, #eos{pos = Pos} = Eos) -> + %% positional argument itself is a value + consume([Arg | Tail], hd(Pos), Eos). + +%% Adds CmdName to path, and includes any arguments found there +merge_arguments(CmdName, #{arguments := Args} = SubCmd, Eos) -> + add_args(Args, Eos#eos{current = SubCmd, commands = Eos#eos.commands ++ [CmdName]}); +merge_arguments(CmdName, SubCmd, Eos) -> + Eos#eos{current = SubCmd, commands = Eos#eos.commands ++ [CmdName]}. + +%% adds arguments into current set of discovered pos/opts +add_args([], Eos) -> + Eos; +add_args([#{short := S, long := L} = Option | Tail], #eos{short = Short, long = Long} = Eos) -> + %% remember if this option can be confused with negative number + NoDigits = no_digits(Eos#eos.no_digits, Eos#eos.prefixes, S, L), + add_args(Tail, Eos#eos{short = Short#{S => Option}, long = Long#{L => Option}, no_digits = NoDigits}); +add_args([#{short := S} = Option | Tail], #eos{short = Short} = Eos) -> + %% remember if this option can be confused with negative number + NoDigits = no_digits(Eos#eos.no_digits, Eos#eos.prefixes, S, 0), + add_args(Tail, Eos#eos{short = Short#{S => Option}, no_digits = NoDigits}); +add_args([#{long := L} = Option | Tail], #eos{long = Long} = Eos) -> + %% remember if this option can be confused with negative number + NoDigits = no_digits(Eos#eos.no_digits, Eos#eos.prefixes, 0, L), + add_args(Tail, Eos#eos{long = Long#{L => Option}, no_digits = NoDigits}); +add_args([PosOpt | Tail], #eos{pos = Pos} = Eos) -> + add_args(Tail, Eos#eos{pos = Pos ++ [PosOpt]}). + +%% If no_digits is still true, try to find out whether it should turn false, +%% because added options look like negative numbers, and prefixes include - +no_digits(false, _, _, _) -> + false; +no_digits(true, Prefixes, _, _) when not is_map_key($-, Prefixes) -> + true; +no_digits(true, _, Short, _) when Short >= $0, Short =< $9 -> + false; +no_digits(true, _, _, Long) -> + not is_digits(Long). + +%%-------------------------------------------------------------------- +%% additional functions for optional arguments processing + +%% Returns true when option (!) description passed requires a positional argument, +%% hence cannot be treated as a flag. +requires_argument(#{nargs := {'maybe', _Term}}) -> + false; +requires_argument(#{nargs := 'maybe'}) -> + false; +requires_argument(#{nargs := _Any}) -> + true; +requires_argument(Opt) -> + case maps:get(action, Opt, store) of + store -> + maps:get(type, Opt, string) =/= boolean; + append -> + maps:get(type, Opt, string) =/= boolean; + _ -> + false + end. + +%% Attempts to find if passed list of flags can be expanded +abbreviated([Last], Acc, AllShort) when is_map_key(Last, AllShort) -> + lists:reverse([Last | Acc]); +abbreviated([_], _Acc, _Eos) -> + false; +abbreviated([Flag | Tail], Acc, AllShort) -> + case maps:find(Flag, AllShort) of + error -> + false; + {ok, Opt} -> + case requires_argument(Opt) of + true -> + false; + false -> + abbreviated(Tail, [Flag | Acc], AllShort) + end + end. + +%%-------------------------------------------------------------------- +%% argument consumption (nargs) handling + +%% consume predefined amount (none of which can be an option?) +consume(Tail, #{nargs := Count} = Opt, Eos) when is_integer(Count) -> + {Consumed, Remain} = split_to_option(Tail, Count, Eos, []), + length(Consumed) < Count andalso + throw({Eos#eos.commands, Opt, Tail, + io_lib:format("expected ~b, found ~b argument(s)", [Count, length(Consumed)])}), + action(Remain, Consumed, Opt#{type => {list, maps:get(type, Opt, string)}}, Eos); + +%% handle 'reminder' by just dumping everything in +consume(Tail, #{nargs := all} = Opt, Eos) -> + action([], Tail, Opt#{type => {list, maps:get(type, Opt, string)}}, Eos); + +%% require at least one argument +consume(Tail, #{nargs := nonempty_list} = Opt, Eos) -> + {Consumed, Remains} = split_to_option(Tail, -1, Eos, []), + Consumed =:= [] andalso throw({Eos#eos.commands, Opt, Tail, <<"expected argument">>}), + action(Remains, Consumed, Opt#{type => {list, maps:get(type, Opt, string)}}, Eos); + +%% consume all until next option +consume(Tail, #{nargs := list} = Opt, Eos) -> + {Consumed, Remains} = split_to_option(Tail, -1, Eos, []), + action(Remains, Consumed, Opt#{type => {list, maps:get(type, Opt, string)}}, Eos); + +%% maybe consume one, maybe not... +%% special cases for 'boolean maybe', only consume 'true' and 'false' +consume(["true" | Tail], #{type := boolean} = Opt, Eos) -> + action(Tail, true, Opt#{type => raw}, Eos); +consume(["false" | Tail], #{type := boolean} = Opt, Eos) -> + action(Tail, false, Opt#{type => raw}, Eos); +consume(Tail, #{type := boolean} = Opt, Eos) -> + %% neither true nor false means 'undefined' (with the default for boolean being true) + action(Tail, undefined, Opt, Eos); + +%% maybe behaviour, as '?' +consume(Tail, #{nargs := 'maybe'} = Opt, Eos) -> + case split_to_option(Tail, 1, Eos, []) of + {[], _} -> + %% no argument given, produce default argument (if not present, + %% then produce default value of the specified type) + action(Tail, default(Opt), Opt#{type => raw}, Eos); + {[Consumed], Remains} -> + action(Remains, Consumed, Opt, Eos) + end; + +%% maybe consume one, maybe not... +consume(Tail, #{nargs := {'maybe', Const}} = Opt, Eos) -> + case split_to_option(Tail, 1, Eos, []) of + {[], _} -> + action(Tail, Const, Opt, Eos); + {[Consumed], Remains} -> + action(Remains, Consumed, Opt, Eos) + end; + +%% default case, which depends on action +consume(Tail, #{action := count} = Opt, Eos) -> + action(Tail, undefined, Opt, Eos); + +%% for {store, ...} and {append, ...} don't take argument out +consume(Tail, #{action := {Act, _Const}} = Opt, Eos) when Act =:= store; Act =:= append -> + action(Tail, undefined, Opt, Eos); + +%% optional: ensure not to consume another option start +consume([[Prefix | _] = ArgValue | Tail], Opt, Eos) when ?IS_OPTION(Opt), is_map_key(Prefix, Eos#eos.prefixes) -> + case Eos#eos.no_digits andalso is_digits(ArgValue) of + true -> + action(Tail, ArgValue, Opt, Eos); + false -> + throw({Eos#eos.commands, Opt, undefined, <<"expected argument">>}) + end; + +consume([ArgValue | Tail], Opt, Eos) -> + action(Tail, ArgValue, Opt, Eos); + +%% we can only be here if it's optional argument, but there is no value supplied, +%% and type is not 'boolean' - this is an error! +consume([], Opt, Eos) -> + throw({Eos#eos.commands, Opt, undefined, <<"expected argument">>}). + +%% no more arguments for consumption, but last optional may still be action-ed +%%consume([], Current, Opt, Eos) -> +%% action([], Current, undefined, Opt, Eos). + +%% smart split: ignore arguments that can be parsed as negative numbers, +%% unless there are arguments that look like negative numbers +split_to_option([], _, _Eos, Acc) -> + {lists:reverse(Acc), []}; +split_to_option(Tail, 0, _Eos, Acc) -> + {lists:reverse(Acc), Tail}; +split_to_option([[Prefix | _] = MaybeNumber | Tail] = All, Left, + #eos{no_digits = true, prefixes = Prefixes} = Eos, Acc) when is_map_key(Prefix, Prefixes) -> + case is_digits(MaybeNumber) of + true -> + split_to_option(Tail, Left - 1, Eos, [MaybeNumber | Acc]); + false -> + {lists:reverse(Acc), All} + end; +split_to_option([[Prefix | _] | _] = All, _Left, + #eos{no_digits = false, prefixes = Prefixes}, Acc) when is_map_key(Prefix, Prefixes) -> + {lists:reverse(Acc), All}; +split_to_option([Head | Tail], Left, Opts, Acc) -> + split_to_option(Tail, Left - 1, Opts, [Head | Acc]). + +%%-------------------------------------------------------------------- +%% Action handling + +action(Tail, ArgValue, #{name := ArgName, action := store} = Opt, #eos{argmap = ArgMap} = Eos) -> + Value = convert_type(maps:get(type, Opt, string), ArgValue, Opt, Eos), + continue_parser(Tail, Opt, Eos#eos{argmap = ArgMap#{ArgName => Value}}); + +action(Tail, undefined, #{name := ArgName, action := {store, Value}} = Opt, #eos{argmap = ArgMap} = Eos) -> + continue_parser(Tail, Opt, Eos#eos{argmap = ArgMap#{ArgName => Value}}); + +action(Tail, ArgValue, #{name := ArgName, action := append} = Opt, #eos{argmap = ArgMap} = Eos) -> + Value = convert_type(maps:get(type, Opt, string), ArgValue, Opt, Eos), + continue_parser(Tail, Opt, Eos#eos{argmap = ArgMap#{ArgName => maps:get(ArgName, ArgMap, []) ++ [Value]}}); + +action(Tail, undefined, #{name := ArgName, action := {append, Value}} = Opt, #eos{argmap = ArgMap} = Eos) -> + continue_parser(Tail, Opt, Eos#eos{argmap = ArgMap#{ArgName => maps:get(ArgName, ArgMap, []) ++ [Value]}}); + +action(Tail, ArgValue, #{name := ArgName, action := extend} = Opt, #eos{argmap = ArgMap} = Eos) -> + Value = convert_type(maps:get(type, Opt, string), ArgValue, Opt, Eos), + Extended = maps:get(ArgName, ArgMap, []) ++ Value, + continue_parser(Tail, Opt, Eos#eos{argmap = ArgMap#{ArgName => Extended}}); + +action(Tail, _, #{name := ArgName, action := count} = Opt, #eos{argmap = ArgMap} = Eos) -> + continue_parser(Tail, Opt, Eos#eos{argmap = ArgMap#{ArgName => maps:get(ArgName, ArgMap, 0) + 1}}); + +%% default action is `store' (important to sync the code with the first clause above) +action(Tail, ArgValue, #{name := ArgName} = Opt, #eos{argmap = ArgMap} = Eos) -> + Value = convert_type(maps:get(type, Opt, string), ArgValue, Opt, Eos), + continue_parser(Tail, Opt, Eos#eos{argmap = ArgMap#{ArgName => Value}}). + +%% pop last positional, unless nargs is list/nonempty_list +continue_parser(Tail, Opt, Eos) when ?IS_OPTION(Opt) -> + parse_impl(Tail, Eos); +continue_parser(Tail, #{nargs := List}, Eos) when List =:= list; List =:= nonempty_list -> + parse_impl(Tail, Eos); +continue_parser(Tail, _Opt, Eos) -> + parse_impl(Tail, Eos#eos{pos = tl(Eos#eos.pos)}). + +%%-------------------------------------------------------------------- +%% Type conversion + +%% Handle "list" variant for nargs returning list +convert_type({list, Type}, Arg, Opt, Eos) -> + [convert_type(Type, Var, Opt, Eos) || Var <- Arg]; + +%% raw - no conversion applied (most likely default) +convert_type(raw, Arg, _Opt, _Eos) -> + Arg; + +%% Handle actual types +convert_type(string, Arg, _Opt, _Eos) -> + Arg; +convert_type({string, Choices}, Arg, Opt, Eos) when is_list(Choices), is_list(hd(Choices)) -> + lists:member(Arg, Choices) orelse + throw({Eos#eos.commands, Opt, Arg, <<"is not one of the choices">>}), + Arg; +convert_type({string, Re}, Arg, Opt, Eos) -> + case re:run(Arg, Re) of + {match, _X} -> Arg; + _ -> throw({Eos#eos.commands, Opt, Arg, <<"does not match">>}) + end; +convert_type({string, Re, ReOpt}, Arg, Opt, Eos) -> + case re:run(Arg, Re, ReOpt) of + match -> Arg; + {match, _} -> Arg; + _ -> throw({Eos#eos.commands, Opt, Arg, <<"does not match">>}) + end; +convert_type(integer, Arg, Opt, Eos) -> + get_int(Arg, Opt, Eos); +convert_type({integer, Opts}, Arg, Opt, Eos) -> + minimax(get_int(Arg, Opt, Eos), Opts, Eos, Opt, Arg); +convert_type(boolean, "true", _Opt, _Eos) -> + true; +convert_type(boolean, undefined, _Opt, _Eos) -> + true; +convert_type(boolean, "false", _Opt, _Eos) -> + false; +convert_type(boolean, Arg, Opt, Eos) -> + throw({Eos#eos.commands, Opt, Arg, <<"is not a boolean">>}); +convert_type(binary, Arg, _Opt, _Eos) -> + unicode:characters_to_binary(Arg); +convert_type({binary, Choices}, Arg, Opt, Eos) when is_list(Choices), is_binary(hd(Choices)) -> + Conv = unicode:characters_to_binary(Arg), + lists:member(Conv, Choices) orelse + throw({Eos#eos.commands, Opt, Arg, <<"is not one of the choices">>}), + Conv; +convert_type({binary, Re}, Arg, Opt, Eos) -> + case re:run(Arg, Re) of + {match, _X} -> unicode:characters_to_binary(Arg); + _ -> throw({Eos#eos.commands, Opt, Arg, <<"does not match">>}) + end; +convert_type({binary, Re, ReOpt}, Arg, Opt, Eos) -> + case re:run(Arg, Re, ReOpt) of + match -> unicode:characters_to_binary(Arg); + {match, _} -> unicode:characters_to_binary(Arg); + _ -> throw({Eos#eos.commands, Opt, Arg, <<"does not match">>}) + end; +convert_type(float, Arg, Opt, Eos) -> + get_float(Arg, Opt, Eos); +convert_type({float, Opts}, Arg, Opt, Eos) -> + minimax(get_float(Arg, Opt, Eos), Opts, Eos, Opt, Arg); +convert_type(atom, Arg, Opt, Eos) -> + try list_to_existing_atom(Arg) + catch error:badarg -> + throw({Eos#eos.commands, Opt, Arg, <<"is not an existing atom">>}) + end; +convert_type({atom, unsafe}, Arg, _Opt, _Eos) -> + list_to_atom(Arg); +convert_type({atom, Choices}, Arg, Opt, Eos) -> + try + Atom = list_to_existing_atom(Arg), + lists:member(Atom, Choices) orelse throw({Eos#eos.commands, Opt, Arg, <<"is not one of the choices">>}), + Atom + catch error:badarg -> + throw({Eos#eos.commands, Opt, Arg, <<"is not an existing atom">>}) + end; +convert_type({custom, Fun}, Arg, Opt, Eos) -> + try Fun(Arg) + catch error:badarg -> + throw({Eos#eos.commands, Opt, Arg, <<"failed faildation">>}) + end. + +%% Given Var, and list of {min, X}, {max, Y}, ensure that +%% value falls within defined limits. +minimax(Var, [], _Eos, _Opt, _Orig) -> + Var; +minimax(Var, [{min, Min} | _], Eos, Opt, Orig) when Var < Min -> + throw({Eos#eos.commands, Opt, Orig, <<"is less than accepted minimum">>}); +minimax(Var, [{max, Max} | _], Eos, Opt, Orig) when Var > Max -> + throw({Eos#eos.commands, Opt, Orig, <<"is greater than accepted maximum">>}); +minimax(Var, [Num | Tail], Eos, Opt, Orig) when is_number(Num) -> + lists:member(Var, [Num|Tail]) orelse + throw({Eos#eos.commands, Opt, Orig, <<"is not one of the choices">>}), + Var; +minimax(Var, [_ | Tail], Eos, Opt, Orig) -> + minimax(Var, Tail, Eos, Opt, Orig). + +%% returns integer from string, or errors out with debugging info +get_int(Arg, Opt, Eos) -> + case string:to_integer(Arg) of + {Int, []} -> + Int; + _ -> + throw({Eos#eos.commands, Opt, Arg, <<"is not an integer">>}) + end. + +%% returns float from string, that is floating-point, or integer +get_float(Arg, Opt, Eos) -> + case string:to_float(Arg) of + {Float, []} -> + Float; + _ -> + %% possibly in disguise + case string:to_integer(Arg) of + {Int, []} -> + Int; + _ -> + throw({Eos#eos.commands, Opt, Arg, <<"is not a number">>}) + end + end. + +%% Returns 'true' if String can be converted to a number +is_digits(String) -> + case string:to_integer(String) of + {_Int, []} -> + true; + {_, _} -> + case string:to_float(String) of + {_Float, []} -> + true; + {_, _} -> + false + end + end. + +%% 'maybe' nargs for an option that does not have default set still have +%% to produce something, let's call it hardcoded default. +default(#{default := Default}) -> + Default; +default(#{type := boolean}) -> + true; +default(#{type := integer}) -> + 0; +default(#{type := float}) -> + 0.0; +default(#{type := string}) -> + ""; +default(#{type := binary}) -> + <<"">>; +default(#{type := atom}) -> + undefined; +%% no type given, consider it 'undefined' atom +default(_) -> + undefined. + +%% command path is now in direct order +format_path(Commands) -> + lists:join(" ", Commands). + +%%-------------------------------------------------------------------- +%% Validation and preprocessing +%% Theoretically, Dialyzer should do that too. +%% Practically, so many people ignore Dialyzer and then spend hours +%% trying to understand why things don't work, that is makes sense +%% to provide a mini-Dialyzer here. + +%% to simplify throwing errors with the right reason +-define (INVALID(Kind, Entity, Path, Field, Text), + erlang:error({?MODULE, Kind, clean_path(Path), Field, Text}, [Entity], [{error_info, #{cause => #{}}}])). + +executable(#{progname := Prog}) when is_atom(Prog) -> + atom_to_list(Prog); +executable(#{progname := Prog}) when is_binary(Prog) -> + binary_to_list(Prog); +executable(#{progname := Prog}) -> + Prog; +executable(_) -> + {ok, [[Prog]]} = init:get_argument(progname), + Prog. + +%% Recursive command validator +validate_command([{Name, Cmd} | _] = Path, Prefixes) -> + (is_list(Name) andalso (not is_map_key(hd(Name), Prefixes))) orelse + ?INVALID(command, Cmd, tl(Path), commands, + <<"command name must be a string not starting with option prefix">>), + is_map(Cmd) orelse + ?INVALID(command, Cmd, Path, commands, <<"expected command()">>), + is_valid_command_help(maps:get(help, Cmd, [])) orelse + ?INVALID(command, Cmd, Path, help, <<"must be a printable unicode list, or a command help template">>), + is_map(maps:get(commands, Cmd, #{})) orelse + ?INVALID(command, Cmd, Path, commands, <<"expected map of #{string() => command()}">>), + case maps:get(handler, Cmd, optional) of + optional -> ok; + {Mod, ModFun} when is_atom(Mod), is_atom(ModFun) -> ok; %% map form + {Mod, ModFun, _} when is_atom(Mod), is_atom(ModFun) -> ok; %% positional form + {Fun, _} when is_function(Fun) -> ok; %% positional form + Fun when is_function(Fun, 1) -> ok; + _ -> ?INVALID(command, Cmd, Path, handler, <<"handler must be a valid callback, or an atom 'optional'">>) + end, + Cmd1 = + case maps:find(arguments, Cmd) of + error -> + Cmd; + {ok, Opts} when not is_list(Opts) -> + ?INVALID(command, Cmd, Path, arguments, <<"expected a list, [argument()]">>); + {ok, Opts} -> + Cmd#{arguments => [validate_option(Path, Opt) || Opt <- Opts]} + end, + %% collect all short & long option identifiers - to figure out any conflicts + lists:foldl( + fun ({_, #{arguments := Opts}}, Acc) -> + lists:foldl( + fun (#{short := Short, name := OName} = Arg, {AllS, AllL}) -> + is_map_key(Short, AllS) andalso + ?INVALID(argument, Arg, Path, short, + "short conflicting with previously defined short for " + ++ atom_to_list(maps:get(Short, AllS))), + {AllS#{Short => OName}, AllL}; + (#{long := Long, name := OName} = Arg, {AllS, AllL}) -> + is_map_key(Long, AllL) andalso + ?INVALID(argument, Arg, Path, long, + "long conflicting with previously defined long for " + ++ atom_to_list(maps:get(Long, AllL))), + {AllS, AllL#{Long => OName}}; + (_, AccIn) -> + AccIn + end, Acc, Opts); + (_, Acc) -> + Acc + end, {#{}, #{}}, Path), + %% verify all sub-commands + case maps:find(commands, Cmd1) of + error -> + {Name, Cmd1}; + {ok, Sub} -> + {Name, Cmd1#{commands => maps:map( + fun (K, V) -> + {K, Updated} = validate_command([{K, V} | Path], Prefixes), + Updated + end, Sub)}} + end. + +%% validates option spec +validate_option(Path, #{name := Name} = Arg) when is_atom(Name); is_list(Name); is_binary(Name) -> + %% verify specific arguments + %% help: string, 'hidden', or a tuple of {string(), ...} + is_valid_option_help(maps:get(help, Arg, [])) orelse + ?INVALID(argument, Arg, Path, help, <<"must be a string or valid help template">>), + io_lib:printable_unicode_list(maps:get(long, Arg, [])) orelse + ?INVALID(argument, Arg, Path, long, <<"must be a printable string">>), + is_boolean(maps:get(required, Arg, true)) orelse + ?INVALID(argument, Arg, Path, required, <<"must be a boolean">>), + io_lib:printable_unicode_list([maps:get(short, Arg, $a)]) orelse + ?INVALID(argument, Arg, Path, short, <<"must be a printable character">>), + Opt1 = maybe_validate(action, Arg, fun validate_action/3, Path), + Opt2 = maybe_validate(type, Opt1, fun validate_type/3, Path), + maybe_validate(nargs, Opt2, fun validate_args/3, Path); +validate_option(Path, Arg) -> + ?INVALID(argument, Arg, Path, name, <<"argument must be a map containing 'name' field">>). + +maybe_validate(Key, Map, Fun, Path) when is_map_key(Key, Map) -> + maps:put(Key, Fun(maps:get(Key, Map), Path, Map), Map); +maybe_validate(_Key, Map, _Fun, _Path) -> + Map. + +%% validate action field +validate_action(store, _Path, _Opt) -> + store; +validate_action({store, Term}, _Path, _Opt) -> + {store, Term}; +validate_action(append, _Path, _Opt) -> + append; +validate_action({append, Term}, _Path, _Opt) -> + {append, Term}; +validate_action(count, _Path, _Opt) -> + count; +validate_action(extend, _Path, #{nargs := Nargs}) when + Nargs =:= list; Nargs =:= nonempty_list; Nargs =:= all; is_integer(Nargs) -> + extend; +validate_action(extend, _Path, #{type := {custom, _}}) -> + extend; +validate_action(extend, Path, Arg) -> + ?INVALID(argument, Arg, Path, action, <<"extend action works only with lists">>); +validate_action(_Action, Path, Arg) -> + ?INVALID(argument, Arg, Path, action, <<"unsupported">>). + +%% validate type field +validate_type(Simple, _Path, _Opt) when Simple =:= boolean; Simple =:= integer; Simple =:= float; + Simple =:= string; Simple =:= binary; Simple =:= atom; Simple =:= {atom, unsafe} -> + Simple; +validate_type({custom, Fun}, _Path, _Opt) when is_function(Fun, 1) -> + {custom, Fun}; +validate_type({float, Opts}, Path, Arg) -> + [?INVALID(argument, Arg, Path, type, <<"invalid validator">>) + || {Kind, Val} <- Opts, (Kind =/= min andalso Kind =/= max) orelse (not is_float(Val))], + {float, Opts}; +validate_type({integer, Opts}, Path, Arg) -> + [?INVALID(argument, Arg, Path, type, <<"invalid validator">>) + || {Kind, Val} <- Opts, (Kind =/= min andalso Kind =/= max) orelse (not is_integer(Val))], + {integer, Opts}; +validate_type({atom, Choices} = Valid, Path, Arg) when is_list(Choices) -> + [?INVALID(argument, Arg, Path, type, <<"unsupported">>) || C <- Choices, not is_atom(C)], + Valid; +validate_type({string, Re} = Valid, _Path, _Opt) when is_list(Re) -> + Valid; +validate_type({string, Re, L} = Valid, _Path, _Opt) when is_list(Re), is_list(L) -> + Valid; +validate_type({binary, Re} = Valid, _Path, _Opt) when is_binary(Re) -> + Valid; +validate_type({binary, Choices} = Valid, _Path, _Opt) when is_list(Choices), is_binary(hd(Choices)) -> + Valid; +validate_type({binary, Re, L} = Valid, _Path, _Opt) when is_binary(Re), is_list(L) -> + Valid; +validate_type(_Type, Path, Arg) -> + ?INVALID(argument, Arg, Path, type, <<"unsupported">>). + +validate_args(N, _Path, _Opt) when is_integer(N), N >= 1 -> N; +validate_args(Simple, _Path, _Opt) when Simple =:= all; Simple =:= list; Simple =:= 'maybe'; Simple =:= nonempty_list -> + Simple; +validate_args({'maybe', Term}, _Path, _Opt) -> {'maybe', Term}; +validate_args(_Nargs, Path, Arg) -> + ?INVALID(argument, Arg, Path, nargs, <<"unsupported">>). + +%% used to throw an error - strips command component out of path +clean_path(Path) -> + {Cmds, _} = lists:unzip(Path), + lists:reverse(Cmds). + +is_valid_option_help(hidden) -> + true; +is_valid_option_help(Help) when is_list(Help); is_binary(Help) -> + true; +is_valid_option_help({Short, Desc}) when is_list(Short) orelse is_binary(Short), is_list(Desc) -> + %% verify that Desc is a list of string/type/default + lists:all(fun(type) -> true; + (default) -> true; + (S) when is_list(S); is_binary(S) -> true; + (_) -> false + end, Desc); +is_valid_option_help({Short, Desc}) when is_list(Short) orelse is_binary(Short), is_function(Desc, 0) -> + true; +is_valid_option_help(_) -> + false. + +is_valid_command_help(hidden) -> + true; +is_valid_command_help(Help) when is_binary(Help) -> + true; +is_valid_command_help(Help) when is_list(Help) -> + %% allow printable lists + case io_lib:printable_unicode_list(Help) of + true -> + true; + false -> + %% ... or a command help template + lists:all( + fun (Atom) when Atom =:= usage; Atom =:= commands; Atom =:= arguments; Atom =:= options -> true; + (Bin) when is_binary(Bin) -> true; + (Str) -> io_lib:printable_unicode_list(Str) + end, Help) + end; +is_valid_command_help(_) -> + false. + +%%-------------------------------------------------------------------- +%% Built-in Help formatter + +format_help({ProgName, Root}, Format) -> + Prefix = hd(maps:get(prefixes, Format, [$-])), + Nested = maps:get(command, Format, []), + %% descent into commands collecting all options on the way + {_CmdName, Cmd, AllArgs} = collect_options(ProgName, Root, Nested, []), + %% split arguments into Flags, Options, Positional, and create help lines + {_, Longest, Flags, Opts, Args, OptL, PosL} = lists:foldl(fun format_opt_help/2, + {Prefix, 0, "", [], [], [], []}, AllArgs), + %% collect and format sub-commands + Immediate = maps:get(commands, Cmd, #{}), + {Long, Subs} = maps:fold( + fun (_Name, #{help := hidden}, {Long, SubAcc}) -> + {Long, SubAcc}; + (Name, Sub, {Long, SubAcc}) -> + Help = maps:get(help, Sub, ""), + {max(Long, string:length(Name)), [{Name, Help}|SubAcc]} + end, {Longest, []}, maps:iterator(Immediate, ordered)), + %% format sub-commands + ShortCmd0 = + case map_size(Immediate) of + 0 -> + []; + Small when Small < 4 -> + Keys = lists:sort(maps:keys(Immediate)), + ["{" ++ lists:append(lists:join("|", Keys)) ++ "}"]; + _Largs -> + ["<command>"] + end, + %% was it nested command? + ShortCmd = if Nested =:= [] -> ShortCmd0; true -> [lists:append(lists:join(" ", Nested)) | ShortCmd0] end, + %% format flags + FlagsForm = if Flags =:= [] -> []; + true -> [unicode:characters_to_list(io_lib:format("[~tc~ts]", [Prefix, Flags]))] + end, + %% format extended view + %% usage line has hardcoded format for now + Usage = [ProgName, ShortCmd, FlagsForm, Opts, Args], + %% format usage according to help template + Template0 = maps:get(help, Root, ""), + %% when there is no help defined for the command, or help is a string, + %% use the default format (original argparse behaviour) + Template = + case Template0 =:= "" orelse io_lib:printable_unicode_list(Template0) of + true -> + %% classic/compatibility format + NL = [io_lib:nl()], + Template1 = ["Usage:" ++ NL, usage, NL], + Template2 = maybe_add("~n", Template0, Template0 ++ NL, Template1), + Template3 = maybe_add("~nSubcommands:~n", Subs, commands, Template2), + Template4 = maybe_add("~nArguments:~n", PosL, arguments, Template3), + maybe_add("~nOptional arguments:~n", OptL, options, Template4); + false -> + Template0 + end, + + %% produce formatted output, taking viewport width into account + Parts = #{usage => Usage, commands => {Long, Subs}, + arguments => {Longest, PosL}, options => {Longest, OptL}}, + Width = maps:get(columns, Format, 80), %% might also use io:columns() here + lists:append([format_width(maps:find(Part, Parts), Part, Width) || Part <- Template]). + +%% collects options on the Path, and returns found Command +collect_options(CmdName, Command, [], Args) -> + {CmdName, Command, maps:get(arguments, Command, []) ++ Args}; +collect_options(CmdName, Command, [Cmd|Tail], Args) -> + Sub = maps:get(commands, Command), + SubCmd = maps:get(Cmd, Sub), + collect_options(CmdName ++ " " ++ Cmd, SubCmd, Tail, maps:get(arguments, Command, []) ++ Args). + +%% conditionally adds text and empty lines +maybe_add(_ToAdd, [], _Element, Template) -> + Template; +maybe_add(ToAdd, _List, Element, Template) -> + Template ++ [io_lib:format(ToAdd, []), Element]. + +format_width(error, Part, Width) -> + wrap_text(Part, 0, Width); +format_width({ok, [ProgName, ShortCmd, FlagsForm, Opts, Args]}, usage, Width) -> + %% make every separate command/option to be a "word", and then + %% wordwrap it indented by the ProgName length + 3 + Words = ShortCmd ++ FlagsForm ++ Opts ++ Args, + if Words =:= [] -> io_lib:format(" ~ts", [ProgName]); + true -> + Indent = string:length(ProgName), + Wrapped = wordwrap(Words, Width - Indent, 0, [], []), + Pad = lists:append(lists:duplicate(Indent + 3, " ")), + ArgLines = lists:join([io_lib:nl() | Pad], Wrapped), + io_lib:format(" ~ts~ts", [ProgName, ArgLines]) + end; +format_width({ok, {Len, Texts}}, _Part, Width) -> + SubFormat = io_lib:format(" ~~-~bts ~~ts~n", [Len]), + [io_lib:format(SubFormat, [N, wrap_text(D, Len + 3, Width)]) || {N, D} <- lists:reverse(Texts)]. + +wrap_text(Text, Indent, Width) -> + %% split text into separate lines (paragraphs) + NL = io_lib:nl(), + Lines = string:split(Text, NL, all), + %% wordwrap every paragraph + Paragraphs = lists:append([wrap_line(L, Width, Indent) || L <- Lines]), + Pad = lists:append(lists:duplicate(Indent, " ")), + lists:join([NL | Pad], Paragraphs). + +wrap_line([], _Width, _Indent) -> + [[]]; +wrap_line(Line, Width, Indent) -> + [First | Tail] = string:split(Line, " ", all), + wordwrap(Tail, Width - Indent, string:length(First), First, []). + +wordwrap([], _Max, _Len, [], Lines) -> + lists:reverse(Lines); +wordwrap([], _Max, _Len, Line, Lines) -> + lists:reverse([Line | Lines]); +wordwrap([Word | Tail], Max, Len, Line, Lines) -> + WordLen = string:length(Word), + case Len + 1 + WordLen > Max of + true -> + wordwrap(Tail, Max, WordLen, Word, [Line | Lines]); + false -> + wordwrap(Tail, Max, WordLen + 1 + Len, [Line, <<" ">>, Word], Lines) + end. + +%% create help line for every option, collecting together all flags, short options, +%% long options, and positional arguments + +%% format optional argument +format_opt_help(#{help := hidden}, Acc) -> + Acc; +format_opt_help(Opt, {Prefix, Longest, Flags, Opts, Args, OptL, PosL}) when ?IS_OPTION(Opt) -> + Desc = format_description(Opt), + %% does it need an argument? look for nargs and action + RequiresArg = requires_argument(Opt), + %% long form always added to Opts + NonOption = maps:get(required, Opt, false) =:= true, + {Name0, MaybeOpt0} = + case maps:find(long, Opt) of + error -> + {"", []}; + {ok, Long} when NonOption, RequiresArg -> + FN = [Prefix | Long], + {FN, [format_required(true, [FN, " "], Opt)]}; + {ok, Long} when RequiresArg -> + FN = [Prefix | Long], + {FN, [format_required(false, [FN, " "], Opt)]}; + {ok, Long} when NonOption -> + FN = [Prefix | Long], + {FN, [FN]}; + {ok, Long} -> + FN = [Prefix | Long], + {FN, [io_lib:format("[~ts]", [FN])]} + end, + %% short may go to flags, or Opts + {Name, MaybeFlag, MaybeOpt1} = + case maps:find(short, Opt) of + error -> + {Name0, [], MaybeOpt0}; + {ok, Short} when RequiresArg -> + SN = [Prefix, Short], + {maybe_concat(SN, Name0), [], + [format_required(NonOption, [SN, " "], Opt) | MaybeOpt0]}; + {ok, Short} -> + {maybe_concat([Prefix, Short], Name0), [Short], MaybeOpt0} + end, + %% apply override for non-default usage (in form of {Quick, Advanced} tuple + MaybeOpt2 = + case maps:find(help, Opt) of + {ok, {Str, _}} -> + [Str]; + _ -> + MaybeOpt1 + end, + %% name length, capped at 24 + NameLen = string:length(Name), + Capped = min(24, NameLen), + {Prefix, max(Capped, Longest), Flags ++ MaybeFlag, Opts ++ MaybeOpt2, Args, [{Name, Desc} | OptL], PosL}; + +%% format positional argument +format_opt_help(#{name := Name} = Opt, {Prefix, Longest, Flags, Opts, Args, OptL, PosL}) -> + Desc = format_description(Opt), + %% positional, hence required + LName = io_lib:format("~ts", [Name]), + LPos = case maps:find(help, Opt) of + {ok, {Str, _}} -> + Str; + _ -> + format_required(maps:get(required, Opt, true), "", Opt) + end, + {Prefix, max(Longest, string:length(LName)), Flags, Opts, Args ++ [LPos], OptL, [{LName, Desc} | PosL]}. + +%% custom format +format_description(#{help := {_Short, Fun}}) when is_function(Fun, 0) -> + Fun(); +format_description(#{help := {_Short, Desc}} = Opt) -> + lists:map( + fun (type) -> + format_type(Opt); + (default) -> + format_default(Opt); + (String) -> + String + end, Desc + ); +%% default format: "desc", "desc (type)", "desc (default)", "desc (type, default)" +format_description(#{name := Name} = Opt) -> + NameStr = maps:get(help, Opt, io_lib:format("~ts", [Name])), + case {NameStr, format_type(Opt), format_default(Opt)} of + {"", "", Type} -> Type; + {"", Default, ""} -> Default; + {Desc, "", ""} -> Desc; + {Desc, "", Default} -> [Desc, " (", Default, ")"]; + {Desc, Type, ""} -> [Desc, " (", Type, ")"]; + {"", Type, Default} -> [Type, ", ", Default]; + {Desc, Type, Default} -> [Desc, " (", Type, ", ", Default, ")"] + end. + +%% option formatting helpers +maybe_concat(No, []) -> No; +maybe_concat(No, L) -> [No, ", ", L]. + +format_required(true, Extra, #{name := Name} = Opt) -> + io_lib:format("~ts<~ts>~ts", [Extra, Name, format_nargs(Opt)]); +format_required(false, Extra, #{name := Name} = Opt) -> + io_lib:format("[~ts<~ts>~ts]", [Extra, Name, format_nargs(Opt)]). + +format_nargs(#{nargs := Dots}) when Dots =:= list; Dots =:= all; Dots =:= nonempty_list -> + "..."; +format_nargs(_) -> + "". + +format_type(#{type := {integer, Choices}}) when is_list(Choices), is_integer(hd(Choices)) -> + io_lib:format("choice: ~s", [lists:join(", ", [integer_to_list(C) || C <- Choices])]); +format_type(#{type := {float, Choices}}) when is_list(Choices), is_number(hd(Choices)) -> + io_lib:format("choice: ~s", [lists:join(", ", [io_lib:format("~g", [C]) || C <- Choices])]); +format_type(#{type := {Num, Valid}}) when Num =:= integer; Num =:= float -> + case {proplists:get_value(min, Valid), proplists:get_value(max, Valid)} of + {undefined, undefined} -> + io_lib:format("~s", [format_type(#{type => Num})]); + {Min, undefined} -> + io_lib:format("~s >= ~tp", [format_type(#{type => Num}), Min]); + {undefined, Max} -> + io_lib:format("~s <= ~tp", [format_type(#{type => Num}), Max]); + {Min, Max} -> + io_lib:format("~tp <= ~s <= ~tp", [Min, format_type(#{type => Num}), Max]) + end; +format_type(#{type := {string, Re, _}}) when is_list(Re), not is_list(hd(Re)) -> + io_lib:format("string re: ~ts", [Re]); +format_type(#{type := {string, Re}}) when is_list(Re), not is_list(hd(Re)) -> + io_lib:format("string re: ~ts", [Re]); +format_type(#{type := {binary, Re}}) when is_binary(Re) -> + io_lib:format("binary re: ~ts", [Re]); +format_type(#{type := {binary, Re, _}}) when is_binary(Re) -> + io_lib:format("binary re: ~ts", [Re]); +format_type(#{type := {StrBin, Choices}}) when StrBin =:= string orelse StrBin =:= binary, is_list(Choices) -> + io_lib:format("choice: ~ts", [lists:join(", ", Choices)]); +format_type(#{type := atom}) -> + "existing atom"; +format_type(#{type := {atom, unsafe}}) -> + "atom"; +format_type(#{type := {atom, Choices}}) -> + io_lib:format("choice: ~ts", [lists:join(", ", [atom_to_list(C) || C <- Choices])]); +format_type(#{type := boolean}) -> + ""; +format_type(#{type := integer}) -> + "int"; +format_type(#{type := Type}) when is_atom(Type) -> + io_lib:format("~ts", [Type]); +format_type(_Opt) -> + "". + +format_default(#{default := Def}) when is_list(Def); is_binary(Def); is_atom(Def) -> + io_lib:format("~ts", [Def]); +format_default(#{default := Def}) -> + io_lib:format("~tp", [Def]); +format_default(_) -> + "". + +%%-------------------------------------------------------------------- +%% Basic handler execution +handle(CmdMap, ArgMap, Path, #{handler := {Mod, ModFun, Default}}) -> + ArgList = arg_map_to_arg_list(CmdMap, Path, ArgMap, Default), + %% if argument count may not match, better error can be produced + erlang:apply(Mod, ModFun, ArgList); +handle(_CmdMap, ArgMap, _Path, #{handler := {Mod, ModFun}}) when is_atom(Mod), is_atom(ModFun) -> + Mod:ModFun(ArgMap); +handle(CmdMap, ArgMap, Path, #{handler := {Fun, Default}}) when is_function(Fun) -> + ArgList = arg_map_to_arg_list(CmdMap, Path, ArgMap, Default), + %% if argument count may not match, better error can be produced + erlang:apply(Fun, ArgList); +handle(_CmdMap, ArgMap, _Path, #{handler := Handler}) when is_function(Handler, 1) -> + Handler(ArgMap). + +%% Given command map, path to reach a specific command, and a parsed argument +%% map, returns a list of arguments (effectively used to transform map-based +%% callback handler into positional). +arg_map_to_arg_list(Command, Path, ArgMap, Default) -> + AllArgs = collect_arguments(Command, Path, []), + [maps:get(Arg, ArgMap, Default) || #{name := Arg} <- AllArgs]. + +%% recursively descend into Path, ignoring arguments with duplicate names +collect_arguments(Command, [], Acc) -> + Acc ++ maps:get(arguments, Command, []); +collect_arguments(Command, [H|Tail], Acc) -> + Args = maps:get(arguments, Command, []), + Next = maps:get(H, maps:get(commands, Command, H)), + collect_arguments(Next, Tail, Acc ++ Args). diff --git a/lib/stdlib/src/array.erl b/lib/stdlib/src/array.erl index 1504326c61..03dedabd55 100644 --- a/lib/stdlib/src/array.erl +++ b/lib/stdlib/src/array.erl @@ -462,7 +462,7 @@ fix_test_() -> -spec relax(Array :: array(Type)) -> array(Type). -relax(#array{size = N}=A) -> +relax(#array{size = N}=A) when is_integer(N), N >= 0 -> A#array{max = find_max(N-1, ?LEAFSIZE)}. @@ -489,7 +489,9 @@ relax_test_() -> array(Type). resize(Size, #array{size = N, max = M, elements = E}=A) - when is_integer(Size), Size >= 0 -> + when is_integer(Size), Size >= 0, + is_integer(N), N >= 0, + is_integer(M), M >= 0 -> if Size > N -> {E1, M1} = grow(Size-1, E, if M > 0 -> M; @@ -570,7 +572,7 @@ resize_test_() -> -spec set(I :: array_indx(), Value :: Type, Array :: array(Type)) -> array(Type). set(I, Value, #array{size = N, max = M, default = D, elements = E}=A) - when is_integer(I), I >= 0 -> + when is_integer(I), I >= 0, is_integer(N), is_integer(M) -> if I < N -> A#array{elements = set_1(I, E, Value, D)}; I < M -> @@ -599,7 +601,7 @@ set_1(I, E, X, _D) -> %% Enlarging the array upwards to accommodate an index `I' -grow(I, E, _M) when is_integer(E) -> +grow(I, E, _M) when is_integer(I), is_integer(E) -> M1 = find_max(I, E), {M1, M1}; grow(I, E, M) -> @@ -633,7 +635,7 @@ expand(I, _S, X, D) -> -spec get(I :: array_indx(), Array :: array(Type)) -> Value :: Type. get(I, #array{size = N, max = M, elements = E, default = D}) - when is_integer(I), I >= 0 -> + when is_integer(I), I >= 0, is_integer(N), is_integer(M) -> if I < N -> get_1(I, E, D); M > 0 -> @@ -673,7 +675,7 @@ get_1(I, E, _D) -> -spec reset(I :: array_indx(), Array :: array(Type)) -> array(Type). reset(I, #array{size = N, max = M, default = D, elements = E}=A) - when is_integer(I), I >= 0 -> + when is_integer(I), I >= 0, is_integer(N), is_integer(M) -> if I < N -> try A#array{elements = reset_1(I, E, D)} catch throw:default -> A @@ -760,7 +762,7 @@ set_get_test_() -> to_list(#array{size = 0}) -> []; -to_list(#array{size = N, elements = E, default = D}) -> +to_list(#array{size = N, elements = E, default = D}) when is_integer(N) -> to_list_1(E, D, N - 1); to_list(_) -> erlang:error(badarg). @@ -833,7 +835,7 @@ to_list_test_() -> sparse_to_list(#array{size = 0}) -> []; -sparse_to_list(#array{size = N, elements = E, default = D}) -> +sparse_to_list(#array{size = N, elements = E, default = D}) when is_integer(N) -> sparse_to_list_1(E, D, N - 1); sparse_to_list(_) -> erlang:error(badarg). @@ -1011,7 +1013,7 @@ from_list_test_() -> to_orddict(#array{size = 0}) -> []; -to_orddict(#array{size = N, elements = E, default = D}) -> +to_orddict(#array{size = N, elements = E, default = D}) when is_integer(N) -> I = N - 1, to_orddict_1(E, I, D, I); to_orddict(_) -> @@ -1030,7 +1032,7 @@ to_orddict_1(E, R, D, I) when is_integer(E) -> to_orddict_1(E, R, _D, I) -> push_tuple_pairs(I+1, R, E, []). -to_orddict_2(E=?NODEPATTERN(S), R, D, L) -> +to_orddict_2(E=?NODEPATTERN(S), R, D, L) when is_integer(S) -> to_orddict_3(?NODESIZE, R, D, L, E, S); to_orddict_2(E, R, D, L) when is_integer(E) -> push_pairs(E, R, D, L); @@ -1103,7 +1105,8 @@ to_orddict_test_() -> sparse_to_orddict(#array{size = 0}) -> []; -sparse_to_orddict(#array{size = N, elements = E, default = D}) -> +sparse_to_orddict(#array{size = N, elements = E, default = D}) + when is_integer(N) -> I = N - 1, sparse_to_orddict_1(E, I, D, I); sparse_to_orddict(_) -> @@ -1122,7 +1125,7 @@ sparse_to_orddict_1(E, _R, _D, _I) when is_integer(E) -> sparse_to_orddict_1(E, R, D, I) -> sparse_push_tuple_pairs(I+1, R, D, E, []). -sparse_to_orddict_2(E=?NODEPATTERN(S), R, D, L) -> +sparse_to_orddict_2(E=?NODEPATTERN(S), R, D, L) when is_integer(S) -> sparse_to_orddict_3(?NODESIZE, R, D, L, E, S); sparse_to_orddict_2(E, _R, _D, L) when is_integer(E) -> L; @@ -1223,7 +1226,7 @@ from_orddict_0([], N, _Max, _D, Es) -> end; from_orddict_0(Xs=[{Ix1, _}|_], Ix, Max0, D, Es0) - when Ix1 > Max0, is_integer(Ix1) -> + when is_integer(Ix1), Ix1 > Max0 -> %% We have a hole larger than a leaf Hole = Ix1-Ix, Step = Hole - (Hole rem ?LEAFSIZE), @@ -1393,7 +1396,7 @@ from_orddict_test_() -> Function :: fun((Index :: array_indx(), Type1) -> Type2). map(Function, Array=#array{size = N, elements = E, default = D}) - when is_function(Function, 2) -> + when is_function(Function, 2), is_integer(N) -> if N > 0 -> A = Array#array{elements = []}, % kill reference, for GC A#array{elements = map_1(N-1, E, 0, Function, D)}; @@ -1485,7 +1488,7 @@ map_test_() -> Function :: fun((Index :: array_indx(), Type1) -> Type2). sparse_map(Function, Array=#array{size = N, elements = E, default = D}) - when is_function(Function, 2) -> + when is_function(Function, 2), is_integer(N) -> if N > 0 -> A = Array#array{elements = []}, % kill reference, for GC A#array{elements = sparse_map_1(N-1, E, 0, Function, D)}; @@ -1581,7 +1584,7 @@ sparse_map_test_() -> Function :: fun((Index :: array_indx(), Value :: Type, Acc :: A) -> B). foldl(Function, A, #array{size = N, elements = E, default = D}) - when is_function(Function, 3) -> + when is_function(Function, 3), is_integer(N) -> if N > 0 -> foldl_1(N-1, E, A, 0, Function, D); true -> @@ -1653,7 +1656,7 @@ foldl_test_() -> Function :: fun((Index :: array_indx(), Value :: Type, Acc :: A) -> B). sparse_foldl(Function, A, #array{size = N, elements = E, default = D}) - when is_function(Function, 3) -> + when is_function(Function, 3), is_integer(N) -> if N > 0 -> sparse_foldl_1(N-1, E, A, 0, Function, D); true -> @@ -1730,7 +1733,7 @@ sparse_foldl_test_() -> Function :: fun((Index :: array_indx(), Value :: Type, Acc :: A) -> B). foldr(Function, A, #array{size = N, elements = E, default = D}) - when is_function(Function, 3) -> + when is_function(Function, 3), is_integer(N) -> if N > 0 -> I = N - 1, foldr_1(I, E, I, A, Function, D); @@ -1808,7 +1811,7 @@ foldr_test_() -> Function :: fun((Index :: array_indx(), Value :: Type, Acc :: A) -> B). sparse_foldr(Function, A, #array{size = N, elements = E, default = D}) - when is_function(Function, 3) -> + when is_function(Function, 3), is_integer(N) -> if N > 0 -> I = N - 1, sparse_foldr_1(I, E, I, A, Function, D); @@ -1862,7 +1865,7 @@ sparse_size(A) -> try sparse_foldr(F, [], A) of [] -> 0 catch - {value, I} -> + {value, I} when is_integer(I) -> I + 1 end. diff --git a/lib/stdlib/src/base64.erl b/lib/stdlib/src/base64.erl index be4d9d42b4..62bcd0d24f 100644 --- a/lib/stdlib/src/base64.erl +++ b/lib/stdlib/src/base64.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2021. All Rights Reserved. +%% Copyright Ericsson AB 2007-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -21,12 +21,22 @@ -module(base64). --export([encode/1, decode/1, mime_decode/1, - encode_to_string/1, decode_to_string/1, mime_decode_to_string/1]). - +-export([encode/1, encode/2, + decode/1, decode/2, + mime_decode/1, mime_decode/2, + encode_to_string/1, encode_to_string/2, + decode_to_string/1, decode_to_string/2, + mime_decode_to_string/1, mime_decode_to_string/2, + format_error/2]). %% RFC 4648: Base 64 Encoding alphabet --type base64_alphabet() :: $A..$Z | $a..$z | $0..$9 | $+ | $/ | $=. +-type base64_alphabet() :: $A..$Z | $a..$z | $0..$9 | $+ | $/ | $- | $_ | $=. + +%% Selector for the Base 64 alphabet, `standard' for RFC 4648 +%% Section 4, `urlsafe' for RFC 4648 Section 5. +-type base64_mode() :: 'standard' | 'urlsafe'. + +-type options() :: #{padding => boolean(), mode => base64_mode()}. %% The following type is a subtype of string() for return values %% of encoding functions. @@ -40,67 +50,126 @@ Data :: byte_string() | binary(), Base64String :: base64_string(). -encode_to_string(Bin) when is_binary(Bin) -> - encode_to_string(binary_to_list(Bin)); -encode_to_string(List) when is_list(List) -> - encode_list_to_string(List). +encode_to_string(Data) -> + encode_to_string(Data, #{}). + +-spec encode_to_string(Data, Options) -> Base64String when + Data :: byte_string() | binary(), + Options :: options(), + Base64String :: base64_string(). + +encode_to_string(Bin, Options) when is_binary(Bin), is_map(Options) -> + encode_to_string(binary_to_list(Bin), Options); +encode_to_string(List, Options) when is_list(List), is_map(Options) -> + encode_list_to_string(get_encoding_offset(Options), get_padding(Options), List). -spec encode(Data) -> Base64 when Data :: byte_string() | binary(), Base64 :: base64_binary(). -encode(Bin) when is_binary(Bin) -> - encode_binary(Bin, <<>>); -encode(List) when is_list(List) -> - encode_list(List, <<>>). +encode(Data) -> + encode(Data, #{}). + +-spec encode(Data, Options) -> Base64 when + Data :: byte_string() | binary(), + Options :: options(), + Base64 :: base64_binary(). -encode_list_to_string([]) -> +encode(Bin, Options) when is_binary(Bin), is_map(Options) -> + encode_binary(get_encoding_offset(Options), get_padding(Options), Bin, <<>>); +encode(List, Options) when is_list(List) -> + encode_list(get_encoding_offset(Options), get_padding(Options), List, <<>>). + +encode_list_to_string(_ModeOffset, _Padding, []) -> []; -encode_list_to_string([B1]) -> - [b64e(B1 bsr 2), - b64e((B1 band 3) bsl 4), $=, $=]; -encode_list_to_string([B1,B2]) -> - [b64e(B1 bsr 2), - b64e(((B1 band 3) bsl 4) bor (B2 bsr 4)), - b64e((B2 band 15) bsl 2), $=]; -encode_list_to_string([B1,B2,B3|Ls]) -> +encode_list_to_string(ModeOffset, Padding, [B1]) -> + [b64e(B1 bsr 2, ModeOffset), + b64e((B1 band 3) bsl 4, ModeOffset) | + case Padding of + true -> "=="; + false -> "" + end]; +encode_list_to_string(ModeOffset, Padding, [B1,B2]) -> + [b64e(B1 bsr 2, ModeOffset), + b64e(((B1 band 3) bsl 4) bor (B2 bsr 4), ModeOffset), + b64e((B2 band 15) bsl 2, ModeOffset) | + case Padding of + true -> "="; + false -> "" + end]; +encode_list_to_string(ModeOffset, Padding, [B1,B2,B3|Ls]) -> BB = (B1 bsl 16) bor (B2 bsl 8) bor B3, - [b64e(BB bsr 18), - b64e((BB bsr 12) band 63), - b64e((BB bsr 6) band 63), - b64e(BB band 63) | encode_list_to_string(Ls)]. - -encode_binary(<<>>, A) -> + [b64e(BB bsr 18, ModeOffset), + b64e((BB bsr 12) band 63, ModeOffset), + b64e((BB bsr 6) band 63, ModeOffset), + b64e(BB band 63, ModeOffset) | encode_list_to_string(ModeOffset, Padding, Ls)]. + +encode_binary(ModeOffset, Padding, <<B1:6, B2:6, B3:6, B4:6, B5:6, B6:6, B7:6, B8:6, Ls/bits>>, A) -> + encode_binary(ModeOffset, + Padding, + Ls, + <<A/bits, + (b64e(B1, ModeOffset)):8, + (b64e(B2, ModeOffset)):8, + (b64e(B3, ModeOffset)):8, + (b64e(B4, ModeOffset)):8, + (b64e(B5, ModeOffset)):8, + (b64e(B6, ModeOffset)):8, + (b64e(B7, ModeOffset)):8, + (b64e(B8, ModeOffset)):8>>); +encode_binary(_ModeOffset, _Padding, <<>>, A) -> A; -encode_binary(<<B1:8>>, A) -> - <<A/bits,(b64e(B1 bsr 2)):8,(b64e((B1 band 3) bsl 4)):8,$=:8,$=:8>>; -encode_binary(<<B1:8, B2:8>>, A) -> - <<A/bits,(b64e(B1 bsr 2)):8, - (b64e(((B1 band 3) bsl 4) bor (B2 bsr 4))):8, - (b64e((B2 band 15) bsl 2)):8, $=:8>>; -encode_binary(<<B1:8, B2:8, B3:8, Ls/bits>>, A) -> - BB = (B1 bsl 16) bor (B2 bsl 8) bor B3, - encode_binary(Ls, - <<A/bits,(b64e(BB bsr 18)):8, - (b64e((BB bsr 12) band 63)):8, - (b64e((BB bsr 6) band 63)):8, - (b64e(BB band 63)):8>>). +encode_binary(ModeOffset, Padding, <<B1:6, B2:6, B3:6, B4:6, Ls/bits>>, A) -> + encode_binary(ModeOffset, + Padding, + Ls, + <<A/bits, + (b64e(B1, ModeOffset)):8, + (b64e(B2, ModeOffset)):8, + (b64e(B3, ModeOffset)):8, + (b64e(B4, ModeOffset)):8>>); +encode_binary(ModeOffset, Padding, <<B1:6, B2:2>>, A) -> + E1 = b64e(B1, ModeOffset), + E2 = b64e(B2 bsl 4, ModeOffset), + case Padding of + true -> <<A/bits,E1,E2,$=,$=>>; + _ -> <<A/bits,E1,E2>> + end; +encode_binary(ModeOffset, Padding, <<B1:6, B2:6, B3:4>>, A) -> + E1 = b64e(B1, ModeOffset), + E2 = b64e(B2, ModeOffset), + E3 = b64e(B3 bsl 2, ModeOffset), + case Padding of + true -> <<A/bits,E1,E2,E3,$=>>; + _ -> <<A/bits,E1,E2,E3>> + end. -encode_list([], A) -> +encode_list(_ModeOffset, _Padding, [], A) -> A; -encode_list([B1], A) -> - <<A/bits,(b64e(B1 bsr 2)):8,(b64e((B1 band 3) bsl 4)):8,$=:8,$=:8>>; -encode_list([B1,B2], A) -> - <<A/bits,(b64e(B1 bsr 2)):8, - (b64e(((B1 band 3) bsl 4) bor (B2 bsr 4))):8, - (b64e((B2 band 15) bsl 2)):8, $=:8>>; -encode_list([B1,B2,B3|Ls], A) -> +encode_list(ModeOffset, Padding, [B1], A) -> + E1 = b64e(B1 bsr 2, ModeOffset), + E2 = b64e((B1 band 3) bsl 4, ModeOffset), + case Padding of + true -> <<A/bits,E1,E2,$=,$=>>; + false -> <<A/bits,E1,E2>> + end; +encode_list(ModeOffset, Padding, [B1,B2], A) -> + E1 = b64e(B1 bsr 2, ModeOffset), + E2 = b64e(((B1 band 3) bsl 4) bor (B2 bsr 4), ModeOffset), + E3 = b64e((B2 band 15) bsl 2, ModeOffset), + case Padding of + true -> <<A/bits,E1,E2,E3,$=>>; + false -> <<A/bits,E1,E2,E3>> + end; +encode_list(ModeOffset, Padding, [B1,B2,B3|Ls], A) -> BB = (B1 bsl 16) bor (B2 bsl 8) bor B3, - encode_list(Ls, - <<A/bits,(b64e(BB bsr 18)):8, - (b64e((BB bsr 12) band 63)):8, - (b64e((BB bsr 6) band 63)):8, - (b64e(BB band 63)):8>>). + encode_list(ModeOffset, + Padding, + Ls, + <<A/bits,(b64e(BB bsr 18, ModeOffset)):8, + (b64e((BB bsr 12) band 63, ModeOffset)):8, + (b64e((BB bsr 6) band 63, ModeOffset)):8, + (b64e(BB band 63, ModeOffset)):8>>). %% mime_decode strips away all characters not Base64 before %% converting, whereas decode crashes if an illegal character is found @@ -109,19 +178,35 @@ encode_list([B1,B2,B3|Ls], A) -> Base64 :: base64_string() | base64_binary(), Data :: binary(). -decode(Bin) when is_binary(Bin) -> - decode_binary(Bin, <<>>); -decode(List) when is_list(List) -> - decode_list(List, <<>>). +decode(Base64) -> + decode(Base64, #{}). + +-spec decode(Base64, Options) -> Data when + Base64 :: base64_string() | base64_binary(), + Options :: options(), + Data :: binary(). + +decode(Bin, Options) when is_binary(Bin) -> + decode_binary(get_decoding_offset(Options), get_padding(Options), Bin, <<>>); +decode(List, Options) when is_list(List) -> + decode_list(get_decoding_offset(Options), get_padding(Options), List, <<>>). -spec mime_decode(Base64) -> Data when Base64 :: base64_string() | base64_binary(), Data :: binary(). -mime_decode(Bin) when is_binary(Bin) -> - mime_decode_binary(Bin, <<>>); -mime_decode(List) when is_list(List) -> - mime_decode_list(List, <<>>). +mime_decode(Base64) -> + mime_decode(Base64, #{}). + +-spec mime_decode(Base64, Options) -> Data when + Base64 :: base64_string() | base64_binary(), + Options :: options(), + Data :: binary(). + +mime_decode(Bin, Options) when is_binary(Bin) -> + mime_decode_binary(get_decoding_offset(Options), get_padding(Options), Bin, <<>>); +mime_decode(List, Options) when is_list(List) -> + mime_decode_list(get_decoding_offset(Options), get_padding(Options), List, <<>>). %% mime_decode_to_string strips away all characters not Base64 before %% converting, whereas decode_to_string crashes if an illegal @@ -131,324 +216,439 @@ mime_decode(List) when is_list(List) -> Base64 :: base64_string() | base64_binary(), DataString :: byte_string(). -decode_to_string(Bin) when is_binary(Bin) -> - decode_to_string(binary_to_list(Bin)); -decode_to_string(List) when is_list(List) -> - decode_list_to_string(List). +decode_to_string(Base64) -> + decode_to_string(Base64, #{}). + +-spec decode_to_string(Base64, Options) -> DataString when + Base64 :: base64_string() | base64_binary(), + Options :: options(), + DataString :: byte_string(). + +decode_to_string(Bin, Options) when is_binary(Bin) -> + decode_to_string(binary_to_list(Bin), Options); +decode_to_string(List, Options) when is_list(List) -> + decode_list_to_string(get_decoding_offset(Options), get_padding(Options), List). -spec mime_decode_to_string(Base64) -> DataString when Base64 :: base64_string() | base64_binary(), DataString :: byte_string(). -mime_decode_to_string(Bin) when is_binary(Bin) -> - mime_decode_to_string(binary_to_list(Bin)); -mime_decode_to_string(List) when is_list(List) -> - mime_decode_list_to_string(List). +mime_decode_to_string(Base64) -> + mime_decode_to_string(Base64, #{}). + +-spec mime_decode_to_string(Base64, Options) -> DataString when + Base64 :: base64_string() | base64_binary(), + Options :: options(), + DataString :: byte_string(). + +mime_decode_to_string(Bin, Options) when is_binary(Bin) -> + mime_decode_to_string(binary_to_list(Bin), Options); +mime_decode_to_string(List, Options) when is_list(List) -> + mime_decode_list_to_string(get_decoding_offset(Options), get_padding(Options), List). %% Skipping pad character if not at end of string. Also liberal about %% excess padding and skipping of other illegal (non-base64 alphabet) %% characters. See section 3.3 of RFC4648 -mime_decode_list([0 | Cs], A) -> - mime_decode_list(Cs, A); -mime_decode_list([C1 | Cs], A) -> - case b64d(C1) of - B1 when is_integer(B1) -> mime_decode_list(Cs, A, B1); - _ -> mime_decode_list(Cs, A) % eq is padding +mime_decode_list(ModeOffset, Padding, [C1 | Cs], A) -> + case b64d(C1, ModeOffset) of + B1 when is_integer(B1) -> mime_decode_list(ModeOffset, Padding, Cs, A, B1); + _ -> mime_decode_list(ModeOffset, Padding, Cs, A) % eq is padding end; -mime_decode_list([], A) -> +mime_decode_list(_ModeOffset, _Padding, [], A) -> A. -mime_decode_list([0 | Cs], A, B1) -> - mime_decode_list(Cs, A, B1); -mime_decode_list([C2 | Cs], A, B1) -> - case b64d(C2) of +mime_decode_list(ModeOffset, Padding, [C2 | Cs], A, B1) -> + case b64d(C2, ModeOffset) of B2 when is_integer(B2) -> - mime_decode_list(Cs, A, B1, B2); - _ -> mime_decode_list(Cs, A, B1) % eq is padding + mime_decode_list(ModeOffset, Padding, Cs, A, B1, B2); + _ -> mime_decode_list(ModeOffset, Padding, Cs, A, B1) % eq is padding end. -mime_decode_list([0 | Cs], A, B1, B2) -> - mime_decode_list(Cs, A, B1, B2); -mime_decode_list([C3 | Cs], A, B1, B2) -> - case b64d(C3) of +mime_decode_list(ModeOffset, Padding, [C3 | Cs], A, B1, B2) -> + case b64d(C3, ModeOffset) of B3 when is_integer(B3) -> - mime_decode_list(Cs, A, B1, B2, B3); + mime_decode_list(ModeOffset, Padding, Cs, A, B1, B2, B3); eq=B3 -> - mime_decode_list_after_eq(Cs, A, B1, B2, B3); - _ -> mime_decode_list(Cs, A, B1, B2) + mime_decode_list_after_eq(ModeOffset, Padding, Cs, A, B1, B2, B3); + _ -> mime_decode_list(ModeOffset, Padding, Cs, A, B1, B2) + end; +mime_decode_list(ModeOffset, Padding, [], A, B1, B2) -> + case Padding of + true -> missing_padding_error(); + false -> mime_decode_list_after_eq(ModeOffset, Padding, [], A, B1, B2, eq) end. -mime_decode_list([0 | Cs], A, B1, B2, B3) -> - mime_decode_list(Cs, A, B1, B2, B3); -mime_decode_list([C4 | Cs], A, B1, B2, B3) -> - case b64d(C4) of +mime_decode_list(ModeOffset, Padding, [C4 | Cs], A, B1, B2, B3) -> + case b64d(C4, ModeOffset) of B4 when is_integer(B4) -> - mime_decode_list(Cs, <<A/bits,B1:6,B2:6,B3:6,B4:6>>); + mime_decode_list(ModeOffset, Padding, Cs, <<A/bits,B1:6,B2:6,B3:6,B4:6>>); eq -> - mime_decode_list_after_eq(Cs, A, B1, B2, B3); - _ -> mime_decode_list(Cs, A, B1, B2, B3) + mime_decode_list_after_eq(ModeOffset, Padding, Cs, A, B1, B2, B3); + _ -> mime_decode_list(ModeOffset, Padding, Cs, A, B1, B2, B3) + end; +mime_decode_list(ModeOffset, Padding, [], A, B1, B2, B3) -> + case Padding of + true -> missing_padding_error(); + false -> mime_decode_list_after_eq(ModeOffset, Padding, [], A, B1, B2, B3) end. -mime_decode_list_after_eq([0 | Cs], A, B1, B2, B3) -> - mime_decode_list_after_eq(Cs, A, B1, B2, B3); -mime_decode_list_after_eq([C | Cs], A, B1, B2, B3) -> - case b64d(C) of +mime_decode_list_after_eq(ModeOffset, Padding, [C | Cs], A, B1, B2, B3) -> + case b64d(C, ModeOffset) of B when is_integer(B) -> %% More valid data, skip the eq as invalid case B3 of - eq -> mime_decode_list(Cs, A, B1, B2, B); - _ -> mime_decode_list(Cs, <<A/bits,B1:6,B2:6,B3:6,B:6>>) + eq -> mime_decode_list(ModeOffset, Padding, Cs, A, B1, B2, B); + _ -> mime_decode_list(ModeOffset, Padding, Cs, <<A/bits,B1:6,B2:6,B3:6,B:6>>) end; - _ -> mime_decode_list_after_eq(Cs, A, B1, B2, B3) + _ -> mime_decode_list_after_eq(ModeOffset, Padding, Cs, A, B1, B2, B3) end; -mime_decode_list_after_eq([], A, B1, B2, eq) -> +mime_decode_list_after_eq(_ModeOffset, _Padding, [], A, B1, B2, eq) -> <<A/bits,B1:6,(B2 bsr 4):2>>; -mime_decode_list_after_eq([], A, B1, B2, B3) -> +mime_decode_list_after_eq(_ModeOffset, _Padding, [], A, B1, B2, B3) -> <<A/bits,B1:6,B2:6,(B3 bsr 2):4>>. -mime_decode_binary(<<0:8, Cs/bits>>, A) -> - mime_decode_binary(Cs, A); -mime_decode_binary(<<C1:8, Cs/bits>>, A) -> - case b64d(C1) of - B1 when is_integer(B1) -> mime_decode_binary(Cs, A, B1); - _ -> mime_decode_binary(Cs, A) % eq is padding +mime_decode_binary(ModeOffset, Padding, <<C1:8, Cs/bits>>, A) -> + case b64d(C1, ModeOffset) of + B1 when is_integer(B1) -> mime_decode_binary(ModeOffset, Padding, Cs, A, B1); + _ -> mime_decode_binary(ModeOffset, Padding, Cs, A) % eq is padding end; -mime_decode_binary(<<>>, A) -> +mime_decode_binary(_ModeOffset, _Padding, <<>>, A) -> A. -mime_decode_binary(<<0:8, Cs/bits>>, A, B1) -> - mime_decode_binary(Cs, A, B1); -mime_decode_binary(<<C2:8, Cs/bits>>, A, B1) -> - case b64d(C2) of +mime_decode_binary(ModeOffset, Padding, <<C2:8, Cs/bits>>, A, B1) -> + case b64d(C2, ModeOffset) of B2 when is_integer(B2) -> - mime_decode_binary(Cs, A, B1, B2); - _ -> mime_decode_binary(Cs, A, B1) % eq is padding + mime_decode_binary(ModeOffset, Padding, Cs, A, B1, B2); + _ -> mime_decode_binary(ModeOffset, Padding, Cs, A, B1) % eq is padding end. -mime_decode_binary(<<0:8, Cs/bits>>, A, B1, B2) -> - mime_decode_binary(Cs, A, B1, B2); -mime_decode_binary(<<C3:8, Cs/bits>>, A, B1, B2) -> - case b64d(C3) of +mime_decode_binary(ModeOffset, Padding, <<C3:8, Cs/bits>>, A, B1, B2) -> + case b64d(C3, ModeOffset) of B3 when is_integer(B3) -> - mime_decode_binary(Cs, A, B1, B2, B3); + mime_decode_binary(ModeOffset, Padding, Cs, A, B1, B2, B3); eq=B3 -> - mime_decode_binary_after_eq(Cs, A, B1, B2, B3); - _ -> mime_decode_binary(Cs, A, B1, B2) + mime_decode_binary_after_eq(ModeOffset, Padding, Cs, A, B1, B2, B3); + _ -> mime_decode_binary(ModeOffset, Padding, Cs, A, B1, B2) + end; +mime_decode_binary(ModeOffset, Padding, <<Cs/bits>>, A, B1, B2) -> + case Padding of + true -> missing_padding_error(); + false -> mime_decode_binary_after_eq(ModeOffset, Padding, Cs, A, B1, B2, eq) end. -mime_decode_binary(<<0:8, Cs/bits>>, A, B1, B2, B3) -> - mime_decode_binary(Cs, A, B1, B2, B3); -mime_decode_binary(<<C4:8, Cs/bits>>, A, B1, B2, B3) -> - case b64d(C4) of +mime_decode_binary(ModeOffset, Padding, <<C4:8, Cs/bits>>, A, B1, B2, B3) -> + case b64d(C4, ModeOffset) of B4 when is_integer(B4) -> - mime_decode_binary(Cs, <<A/bits,B1:6,B2:6,B3:6,B4:6>>); + mime_decode_binary(ModeOffset, Padding, Cs, <<A/bits,B1:6,B2:6,B3:6,B4:6>>); eq -> - mime_decode_binary_after_eq(Cs, A, B1, B2, B3); - _ -> mime_decode_binary(Cs, A, B1, B2, B3) + mime_decode_binary_after_eq(ModeOffset, Padding, Cs, A, B1, B2, B3); + _ -> mime_decode_binary(ModeOffset, Padding, Cs, A, B1, B2, B3) + end; +mime_decode_binary(ModeOffset, Padding, <<Cs/bits>>, A, B1, B2, B3) -> + case Padding of + true -> missing_padding_error(); + false -> mime_decode_binary_after_eq(ModeOffset, Padding, Cs, A, B1, B2, B3) end. -mime_decode_binary_after_eq(<<0:8, Cs/bits>>, A, B1, B2, B3) -> - mime_decode_binary_after_eq(Cs, A, B1, B2, B3); -mime_decode_binary_after_eq(<<C:8, Cs/bits>>, A, B1, B2, B3) -> - case b64d(C) of +mime_decode_binary_after_eq(ModeOffset, Padding, <<C:8, Cs/bits>>, A, B1, B2, B3) -> + case b64d(C, ModeOffset) of B when is_integer(B) -> %% More valid data, skip the eq as invalid case B3 of - eq -> mime_decode_binary(Cs, A, B1, B2, B); - _ -> mime_decode_binary(Cs, <<A/bits,B1:6,B2:6,B3:6,B:6>>) + eq -> mime_decode_binary(ModeOffset, Padding, Cs, A, B1, B2, B); + _ -> mime_decode_binary(ModeOffset, Padding, Cs, <<A/bits,B1:6,B2:6,B3:6,B:6>>) end; - _ -> mime_decode_binary_after_eq(Cs, A, B1, B2, B3) + _ -> mime_decode_binary_after_eq(ModeOffset, Padding, Cs, A, B1, B2, B3) end; -mime_decode_binary_after_eq(<<>>, A, B1, B2, eq) -> +mime_decode_binary_after_eq(_ModeOffset, _Padding, <<>>, A, B1, B2, eq) -> <<A/bits,B1:6,(B2 bsr 4):2>>; -mime_decode_binary_after_eq(<<>>, A, B1, B2, B3) -> +mime_decode_binary_after_eq(_ModeOffset, _Padding, <<>>, A, B1, B2, B3) -> <<A/bits,B1:6,B2:6,(B3 bsr 2):4>>. -mime_decode_list_to_string([0 | Cs]) -> - mime_decode_list_to_string(Cs); -mime_decode_list_to_string([C1 | Cs]) -> - case b64d(C1) of - B1 when is_integer(B1) -> mime_decode_list_to_string(Cs, B1); - _ -> mime_decode_list_to_string(Cs) % eq is padding +mime_decode_list_to_string(ModeOffset, Padding, [C1 | Cs]) -> + case b64d(C1, ModeOffset) of + B1 when is_integer(B1) -> mime_decode_list_to_string(ModeOffset, Padding, Cs, B1); + _ -> mime_decode_list_to_string(ModeOffset, Padding, Cs) % eq is padding end; -mime_decode_list_to_string([]) -> +mime_decode_list_to_string(_ModeOffset, _Padding, []) -> []. -mime_decode_list_to_string([0 | Cs], B1) -> - mime_decode_list_to_string(Cs, B1); -mime_decode_list_to_string([C2 | Cs], B1) -> - case b64d(C2) of +mime_decode_list_to_string(ModeOffset, Padding, [C2 | Cs], B1) -> + case b64d(C2, ModeOffset) of B2 when is_integer(B2) -> - mime_decode_list_to_string(Cs, B1, B2); - _ -> mime_decode_list_to_string(Cs, B1) % eq is padding + mime_decode_list_to_string(ModeOffset, Padding, Cs, B1, B2); + _ -> mime_decode_list_to_string(ModeOffset, Padding, Cs, B1) % eq is padding end. -mime_decode_list_to_string([0 | Cs], B1, B2) -> - mime_decode_list_to_string(Cs, B1, B2); -mime_decode_list_to_string([C3 | Cs], B1, B2) -> - case b64d(C3) of +mime_decode_list_to_string(ModeOffset, Padding, [C3 | Cs], B1, B2) -> + case b64d(C3, ModeOffset) of B3 when is_integer(B3) -> - mime_decode_list_to_string(Cs, B1, B2, B3); - eq=B3 -> mime_decode_list_to_string_after_eq(Cs, B1, B2, B3); - _ -> mime_decode_list_to_string(Cs, B1, B2) + mime_decode_list_to_string(ModeOffset, Padding, Cs, B1, B2, B3); + eq=B3 -> mime_decode_list_to_string_after_eq(ModeOffset, Padding, Cs, B1, B2, B3); + _ -> mime_decode_list_to_string(ModeOffset, Padding, Cs, B1, B2) + end; +mime_decode_list_to_string(ModeOffset, Padding, [], B1, B2) -> + case Padding of + true -> missing_padding_error(); + false -> mime_decode_list_to_string_after_eq(ModeOffset, Padding, [], B1, B2, eq) end. -mime_decode_list_to_string([0 | Cs], B1, B2, B3) -> - mime_decode_list_to_string(Cs, B1, B2, B3); -mime_decode_list_to_string([C4 | Cs], B1, B2, B3) -> - case b64d(C4) of +mime_decode_list_to_string(ModeOffset, Padding, [C4 | Cs], B1, B2, B3) -> + case b64d(C4, ModeOffset) of B4 when is_integer(B4) -> Bits4x6 = (B1 bsl 18) bor (B2 bsl 12) bor (B3 bsl 6) bor B4, Octet1 = Bits4x6 bsr 16, Octet2 = (Bits4x6 bsr 8) band 16#ff, Octet3 = Bits4x6 band 16#ff, - [Octet1, Octet2, Octet3 | mime_decode_list_to_string(Cs)]; + [Octet1, Octet2, Octet3 | mime_decode_list_to_string(ModeOffset, Padding, Cs)]; eq -> - mime_decode_list_to_string_after_eq(Cs, B1, B2, B3); - _ -> mime_decode_list_to_string(Cs, B1, B2, B3) + mime_decode_list_to_string_after_eq(ModeOffset, Padding, Cs, B1, B2, B3); + _ -> mime_decode_list_to_string(ModeOffset, Padding, Cs, B1, B2, B3) + end; +mime_decode_list_to_string(ModeOffset, Padding, [], B1, B2, B3) -> + case Padding of + true -> missing_padding_error(); + false -> mime_decode_list_to_string_after_eq(ModeOffset, Padding, [], B1, B2, B3) end. -mime_decode_list_to_string_after_eq([0 | Cs], B1, B2, B3) -> - mime_decode_list_to_string_after_eq(Cs, B1, B2, B3); -mime_decode_list_to_string_after_eq([C | Cs], B1, B2, B3) -> - case b64d(C) of +mime_decode_list_to_string_after_eq(ModeOffset, Padding, [C | Cs], B1, B2, B3) -> + case b64d(C, ModeOffset) of B when is_integer(B) -> %% More valid data, skip the eq as invalid case B3 of - eq -> mime_decode_list_to_string(Cs, B1, B2, B); + eq -> mime_decode_list_to_string(ModeOffset, Padding, Cs, B1, B2, B); _ -> Bits4x6 = (B1 bsl 18) bor (B2 bsl 12) bor (B3 bsl 6) bor B, Octet1 = Bits4x6 bsr 16, Octet2 = (Bits4x6 bsr 8) band 16#ff, Octet3 = Bits4x6 band 16#ff, - [Octet1, Octet2, Octet3 | mime_decode_list_to_string(Cs)] + [Octet1, Octet2, Octet3 | mime_decode_list_to_string(ModeOffset, Padding, Cs)] end; - _ -> mime_decode_list_to_string_after_eq(Cs, B1, B2, B3) + _ -> mime_decode_list_to_string_after_eq(ModeOffset, Padding, Cs, B1, B2, B3) end; -mime_decode_list_to_string_after_eq([], B1, B2, eq) -> +mime_decode_list_to_string_after_eq(_ModeOffset, _Padding, [], B1, B2, eq) -> binary_to_list(<<B1:6,(B2 bsr 4):2>>); -mime_decode_list_to_string_after_eq([], B1, B2, B3) -> +mime_decode_list_to_string_after_eq(_ModeOffset, _Padding, [], B1, B2, B3) -> binary_to_list(<<B1:6,B2:6,(B3 bsr 2):4>>). -decode_list([C1 | Cs], A) -> - case b64d(C1) of - ws -> decode_list(Cs, A); - B1 -> decode_list(Cs, A, B1) +decode_list(ModeOffset, Padding, [C1 | Cs], A) -> + case b64d(C1, ModeOffset) of + ws -> decode_list(ModeOffset, Padding, Cs, A); + B1 -> decode_list(ModeOffset, Padding, Cs, A, B1) end; -decode_list([], A) -> +decode_list(_ModeOffset, _Padding, [], A) -> A. -decode_list([C2 | Cs], A, B1) -> - case b64d(C2) of - ws -> decode_list(Cs, A, B1); - B2 -> decode_list(Cs, A, B1, B2) +decode_list(ModeOffset, Padding, [C2 | Cs], A, B1) -> + case b64d(C2, ModeOffset) of + ws -> decode_list(ModeOffset, Padding, Cs, A, B1); + B2 -> decode_list(ModeOffset, Padding, Cs, A, B1, B2) end. -decode_list([C3 | Cs], A, B1, B2) -> - case b64d(C3) of - ws -> decode_list(Cs, A, B1, B2); - B3 -> decode_list(Cs, A, B1, B2, B3) +decode_list(ModeOffset, Padding, [C3 | Cs], A, B1, B2) -> + case b64d(C3, ModeOffset) of + ws -> decode_list(ModeOffset, Padding, Cs, A, B1, B2); + B3 -> decode_list(ModeOffset, Padding, Cs, A, B1, B2, B3) + end; +decode_list(ModeOffset, Padding, [], A, B1, B2) -> + case Padding of + true -> missing_padding_error(); + false -> decode_list(ModeOffset, Padding, [], A, B1, B2, eq) end. -decode_list([C4 | Cs], A, B1, B2, B3) -> - case b64d(C4) of - ws -> decode_list(Cs, A, B1, B2, B3); - eq when B3 =:= eq -> only_ws(Cs, <<A/bits,B1:6,(B2 bsr 4):2>>); - eq -> only_ws(Cs, <<A/bits,B1:6,B2:6,(B3 bsr 2):4>>); - B4 -> decode_list(Cs, <<A/bits,B1:6,B2:6,B3:6,B4:6>>) +decode_list(ModeOffset, Padding, [C4 | Cs], A, B1, B2, B3) -> + case b64d(C4, ModeOffset) of + ws -> decode_list(ModeOffset, Padding, Cs, A, B1, B2, B3); + eq when B3 =:= eq -> only_ws(ModeOffset, Padding, Cs, <<A/bits,B1:6,(B2 bsr 4):2>>); + eq -> only_ws(ModeOffset, Padding, Cs, <<A/bits,B1:6,B2:6,(B3 bsr 2):4>>); + B4 -> decode_list(ModeOffset, Padding, Cs, <<A/bits,B1:6,B2:6,B3:6,B4:6>>) + end; +decode_list(_ModeOffset, Padding, [], A, B1, B2, B3) -> + case Padding of + true -> missing_padding_error(); + false when B3 == eq -> <<A/bits,B1:6,(B2 bsr 4):2>>; + false -> <<A/bits,B1:6,B2:6,(B3 bsr 2):4>> end. -decode_binary(<<C1:8, Cs/bits>>, A) -> - case b64d(C1) of - ws -> decode_binary(Cs, A); - B1 -> decode_binary(Cs, A, B1) +decode_binary(ModeOffset, Padding, <<C1:8, C2:8, C3:8, C4:8, Cs/bits>>, A) -> + case {b64d(C1, ModeOffset), b64d(C2, ModeOffset), b64d(C3, ModeOffset), b64d(C4, ModeOffset)} of + {B1, B2, B3, B4} when is_integer(B1), is_integer(B2), + is_integer(B3), is_integer(B4) -> + decode_binary(ModeOffset, Padding, Cs, <<A/bits,B1:6,B2:6,B3:6,B4:6>>); + {B1, B2, B3, B4} -> + dec_bin(ModeOffset, Padding, Cs, B1, B2, B3, B4, A) end; -decode_binary(<<>>, A) -> - A. +decode_binary(_ModeOffset, _Padding, <<>>, A) -> + A; +decode_binary(ModeOffset, Padding, <<C1:8, Cs/bits>>, A) -> + case b64d(C1, ModeOffset) of + ws -> decode_binary(ModeOffset, Padding, Cs, A); + B1 -> decode_binary(ModeOffset, Padding, Cs, A, B1) + end. -decode_binary(<<C2:8, Cs/bits>>, A, B1) -> - case b64d(C2) of - ws -> decode_binary(Cs, A, B1); - B2 -> decode_binary(Cs, A, B1, B2) +dec_bin(ModeOffset, Padding, Cs, ws, B2, B3, B4, A) -> + dec_bin(ModeOffset, Padding, Cs, B2, B3, B4, A); +dec_bin(ModeOffset, Padding, Cs, B1, ws, B3, B4, A) -> + dec_bin(ModeOffset, Padding, Cs, B1, B3, B4, A); +dec_bin(ModeOffset, Padding, Cs, B1, B2, ws, B4, A) -> + dec_bin(ModeOffset, Padding, Cs, B1, B2, B4, A); +dec_bin(ModeOffset, Padding, Cs, B1, B2, B3, B4, A) -> + case B4 of + ws -> decode_binary(ModeOffset, Padding, Cs, A, B1, B2, B3); + eq when B3 =:= eq -> only_ws_binary(ModeOffset, Padding, Cs, <<A/bits,B1:6,(B2 bsr 4):2>>); + eq -> only_ws_binary(ModeOffset, Padding, Cs, <<A/bits,B1:6,B2:6,(B3 bsr 2):4>>); + B4 -> decode_binary(ModeOffset, Padding, Cs, <<A/bits,B1:6,B2:6,B3:6,B4:6>>) end. -decode_binary(<<C3:8, Cs/bits>>, A, B1, B2) -> - case b64d(C3) of - ws -> decode_binary(Cs, A, B1, B2); - B3 -> decode_binary(Cs, A, B1, B2, B3) +dec_bin(ModeOffset, Padding, Cs, ws, B2, B3, A) -> + dec_bin(ModeOffset, Padding, Cs, B2, B3, A); +dec_bin(ModeOffset, Padding, Cs, B1, ws, B3, A) -> + dec_bin(ModeOffset, Padding, Cs, B1, B3, A); +dec_bin(ModeOffset, Padding, Cs, B1, B2, ws, A) -> + dec_bin(ModeOffset, Padding, Cs, B1, B2, A); +dec_bin(ModeOffset, Padding, Cs, B1, B2, B3, A) -> + decode_binary(ModeOffset, Padding, Cs, A, B1, B2, B3). + +dec_bin(ModeOffset, Padding, Cs, ws, B2, A) -> + dec_bin(ModeOffset, Padding, Cs, B2, A); +dec_bin(ModeOffset, Padding, Cs, B1, ws, A) -> + dec_bin(ModeOffset, Padding, Cs, B1, A); +dec_bin(ModeOffset, Padding, Cs, B1, B2, A) -> + decode_binary(ModeOffset, Padding, Cs, A, B1, B2). + +dec_bin(ModeOffset, Padding, Cs, ws, A) -> + decode_binary(ModeOffset, Padding, Cs, A); +dec_bin(ModeOffset, Padding, Cs, B1, A) -> + decode_binary(ModeOffset, Padding,Cs, A, B1). + +decode_binary(ModeOffset, Padding, <<C2:8, Cs/bits>>, A, B1) -> + case b64d(C2, ModeOffset) of + ws -> decode_binary(ModeOffset, Padding, Cs, A, B1); + B2 -> decode_binary(ModeOffset, Padding, Cs, A, B1, B2) end. -decode_binary(<<C4:8, Cs/bits>>, A, B1, B2, B3) -> - case b64d(C4) of - ws -> decode_binary(Cs, A, B1, B2, B3); - eq when B3 =:= eq -> only_ws_binary(Cs, <<A/bits,B1:6,(B2 bsr 4):2>>); - eq -> only_ws_binary(Cs, <<A/bits,B1:6,B2:6,(B3 bsr 2):4>>); - B4 -> decode_binary(Cs, <<A/bits,B1:6,B2:6,B3:6,B4:6>>) +decode_binary(ModeOffset, Padding, <<C3:8, Cs/bits>>, A, B1, B2) -> + case b64d(C3, ModeOffset) of + ws -> decode_binary(ModeOffset, Padding, Cs, A, B1, B2); + B3 -> decode_binary(ModeOffset, Padding, Cs, A, B1, B2, B3) + end; +decode_binary(ModeOffset, Padding, <<Cs/bits>>, A, B1, B2) -> + case Padding of + true -> missing_padding_error(); + false -> decode_binary(ModeOffset, Padding, Cs, A, B1, B2, eq) + end. + +decode_binary(ModeOffset, Padding, <<C4:8, Cs/bits>>, A, B1, B2, B3) -> + case b64d(C4, ModeOffset) of + ws -> decode_binary(ModeOffset, Padding, Cs, A, B1, B2, B3); + eq when B3 =:= eq -> only_ws_binary(ModeOffset, Padding, Cs, <<A/bits,B1:6,(B2 bsr 4):2>>); + eq -> only_ws_binary(ModeOffset, Padding, Cs, <<A/bits,B1:6,B2:6,(B3 bsr 2):4>>); + B4 -> decode_binary(ModeOffset, Padding, Cs, <<A/bits,B1:6,B2:6,B3:6,B4:6>>) + end; +decode_binary(_ModeOffset, Padding, <<>>, A, B1, B2, B3) -> + case Padding of + true -> missing_padding_error(); + false when B3 =:= eq -> <<A/bits,B1:6,(B2 bsr 4):2>>; + false -> <<A/bits,B1:6,B2:6,(B3 bsr 2):4>> end. -only_ws_binary(<<>>, A) -> +only_ws_binary(_ModeOffset, _Padding, <<>>, A) -> A; -only_ws_binary(<<C:8, Cs/bits>>, A) -> - case b64d(C) of - ws -> only_ws_binary(Cs, A) +only_ws_binary(ModeOffset, Padding, <<C:8, Cs/bits>>, A) -> + case b64d(C, ModeOffset) of + ws -> only_ws_binary(ModeOffset, Padding, Cs, A) end. -decode_list_to_string([C1 | Cs]) -> - case b64d(C1) of - ws -> decode_list_to_string(Cs); - B1 -> decode_list_to_string(Cs, B1) +decode_list_to_string(ModeOffset, Padding, [C1 | Cs]) -> + case b64d(C1, ModeOffset) of + ws -> decode_list_to_string(ModeOffset, Padding, Cs); + B1 -> decode_list_to_string(ModeOffset, Padding, Cs, B1) end; -decode_list_to_string([]) -> +decode_list_to_string(_ModeOffset, _Padding, []) -> []. -decode_list_to_string([C2 | Cs], B1) -> - case b64d(C2) of - ws -> decode_list_to_string(Cs, B1); - B2 -> decode_list_to_string(Cs, B1, B2) +decode_list_to_string(ModeOffset, Padding, [C2 | Cs], B1) -> + case b64d(C2, ModeOffset) of + ws -> decode_list_to_string(ModeOffset, Padding, Cs, B1); + B2 -> decode_list_to_string(ModeOffset, Padding, Cs, B1, B2) end. -decode_list_to_string([C3 | Cs], B1, B2) -> - case b64d(C3) of - ws -> decode_list_to_string(Cs, B1, B2); - B3 -> decode_list_to_string(Cs, B1, B2, B3) +decode_list_to_string(ModeOffset, Padding, [C3 | Cs], B1, B2) -> + case b64d(C3, ModeOffset) of + ws -> decode_list_to_string(ModeOffset, Padding, Cs, B1, B2); + B3 -> decode_list_to_string(ModeOffset, Padding, Cs, B1, B2, B3) + end; +decode_list_to_string(ModeOffset, Padding, [], B1, B2) -> + case Padding of + true -> missing_padding_error(); + false -> decode_list_to_string(ModeOffset, Padding, [], B1, B2, eq) end. -decode_list_to_string([C4 | Cs], B1, B2, B3) -> - case b64d(C4) of +decode_list_to_string(ModeOffset, Padding, [C4 | Cs], B1, B2, B3) -> + case b64d(C4, ModeOffset) of ws -> - decode_list_to_string(Cs, B1, B2, B3); + decode_list_to_string(ModeOffset, Padding, Cs, B1, B2, B3); eq when B3 =:= eq -> - only_ws(Cs, binary_to_list(<<B1:6,(B2 bsr 4):2>>)); + only_ws(ModeOffset, Padding, Cs, binary_to_list(<<B1:6,(B2 bsr 4):2>>)); eq -> - only_ws(Cs, binary_to_list(<<B1:6,B2:6,(B3 bsr 2):4>>)); + only_ws(ModeOffset, Padding, Cs, binary_to_list(<<B1:6,B2:6,(B3 bsr 2):4>>)); B4 -> Bits4x6 = (B1 bsl 18) bor (B2 bsl 12) bor (B3 bsl 6) bor B4, Octet1 = Bits4x6 bsr 16, Octet2 = (Bits4x6 bsr 8) band 16#ff, Octet3 = Bits4x6 band 16#ff, - [Octet1, Octet2, Octet3 | decode_list_to_string(Cs)] + [Octet1, Octet2, Octet3 | decode_list_to_string(ModeOffset, Padding, Cs)] + end; +decode_list_to_string(_ModeOffset, Padding, [], B1, B2, B3) -> + case Padding of + true -> missing_padding_error(); + false when B3 =:= eq -> binary_to_list(<<B1:6,(B2 bsr 4):2>>); + false -> binary_to_list(<<B1:6,B2:6,(B3 bsr 2):4>>) end. -only_ws([], A) -> +only_ws(_ModeOffset, _Padding, [], A) -> A; -only_ws([C | Cs], A) -> - case b64d(C) of - ws -> only_ws(Cs, A) +only_ws(ModeOffset, Padding, [C | Cs], A) -> + case b64d(C, ModeOffset) of + ws -> only_ws(ModeOffset, Padding, Cs, A) end. %%%======================================================================== +%%% Error handling functions +%%%======================================================================== + +% always inlined for useful stacktraces when called in tail position +-compile({inline, missing_padding_error/0}). +missing_padding_error() -> + error(missing_padding, none, [{error_info, #{}}]). + +format_error(missing_padding, _) -> + #{general => "data to decode is missing final = padding characters, if this is intended, use the `padding => false` option"}; +format_error(_, _) -> + #{}. + +%%%======================================================================== %%% Internal functions %%%======================================================================== -%% accessors --compile({inline, [{b64d, 1}]}). -%% One-based decode map. -b64d(X) -> - element(X, - {bad,bad,bad,bad,bad,bad,bad,bad,ws,ws,bad,bad,ws,bad,bad, %1-15 +%% accessors + +get_padding(#{padding := Bool}) when is_boolean(Bool) -> Bool; +get_padding(#{}) -> true. + +get_decoding_offset(#{mode := standard}) -> 1; +get_decoding_offset(#{mode := urlsafe}) -> 257; +get_decoding_offset(#{}) -> 1. + +-compile({inline, [{b64d, 2}]}). +b64d(X, Off) -> + element(X + Off, + { + %% standard base64 alphabet (RFC 4648 Section 4) + bad,bad,bad,bad,bad,bad,bad,bad,bad,ws,ws,bad,bad,ws,bad,bad, %0-15 bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, %16-31 ws,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,62,bad,bad,bad,63, %32-47 - 52,53,54,55,56,57,58,59,60,61,bad,bad,bad,eq,bad,bad, %48-63 + 52,53,54,55,56,57,58,59,60,61,bad,bad,bad,eq,bad,bad, %48-61 bad,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14, 15,16,17,18,19,20,21,22,23,24,25,bad,bad,bad,bad,bad, bad,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40, @@ -460,13 +660,44 @@ b64d(X) -> bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, + bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, + + %% alternative base64url alphabet (RFC 4648 Section 5) + bad,bad,bad,bad,bad,bad,bad,bad,bad,ws,ws,bad,bad,ws,bad,bad, %0-15 + bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, %16-31 + ws,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,62,bad,bad, %32-47 + 52,53,54,55,56,57,58,59,60,61,bad,bad,bad,eq,bad,bad, %48-61 + bad,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14, + 15,16,17,18,19,20,21,22,23,24,25,bad,bad,bad,bad,63, + bad,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40, + 41,42,43,44,45,46,47,48,49,50,51,bad,bad,bad,bad,bad, + bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, + bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, + bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, + bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, + bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, + bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, + bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad, bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad,bad}). --compile({inline, [{b64e, 1}]}). -b64e(X) -> - element(X+1, - {$A, $B, $C, $D, $E, $F, $G, $H, $I, $J, $K, $L, $M, $N, +get_encoding_offset(#{mode := standard}) -> 1; +get_encoding_offset(#{mode := urlsafe}) -> 65; +get_encoding_offset(#{}) -> 1. + +-compile({inline, [{b64e, 2}]}). +b64e(X, Off) -> + element(X + Off, + { + %% standard base64 alphabet (RFC 4648 Section 4) + $A, $B, $C, $D, $E, $F, $G, $H, $I, $J, $K, $L, $M, $N, + $O, $P, $Q, $R, $S, $T, $U, $V, $W, $X, $Y, $Z, + $a, $b, $c, $d, $e, $f, $g, $h, $i, $j, $k, $l, $m, $n, + $o, $p, $q, $r, $s, $t, $u, $v, $w, $x, $y, $z, + $0, $1, $2, $3, $4, $5, $6, $7, $8, $9, $+, $/, + + %% alternative base64url alphabet (RFC 4648 Section 5) + $A, $B, $C, $D, $E, $F, $G, $H, $I, $J, $K, $L, $M, $N, $O, $P, $Q, $R, $S, $T, $U, $V, $W, $X, $Y, $Z, $a, $b, $c, $d, $e, $f, $g, $h, $i, $j, $k, $l, $m, $n, $o, $p, $q, $r, $s, $t, $u, $v, $w, $x, $y, $z, - $0, $1, $2, $3, $4, $5, $6, $7, $8, $9, $+, $/}). + $0, $1, $2, $3, $4, $5, $6, $7, $8, $9, $-, $_}). diff --git a/lib/stdlib/src/beam_lib.erl b/lib/stdlib/src/beam_lib.erl index 5eed3c06b6..14c6d76430 100644 --- a/lib/stdlib/src/beam_lib.erl +++ b/lib/stdlib/src/beam_lib.erl @@ -51,6 +51,7 @@ -export([make_crypto_key/2, get_crypto_key/1]). %Utilities used by compiler -export_type([attrib_entry/0, compinfo_entry/0, labeled_entry/0, label/0]). +-export_type([chunkid/0]). -export_type([chnk_rsn/0]). -import(lists, [append/1, delete/2, foreach/2, keysort/2, @@ -155,7 +156,7 @@ chunks(File, Chunks, Options) -> catch Error -> Error end. -spec all_chunks(beam()) -> - {'ok', 'beam_lib', [{chunkid(), dataB()}]} | {'error', 'beam_lib', info_rsn()}. + {'ok', module(), [{chunkid(), dataB()}]} | {'error', 'beam_lib', info_rsn()}. all_chunks(File) -> read_all_chunks(File). @@ -831,8 +832,7 @@ symbol(_, AT, I1, I2, _I3, _Cnt) -> {atm(AT, I1), I2}. atm(AT, N) -> - [{_N, S}] = ets:lookup(AT, N), - S. + ets:lookup_element(AT, N, 2). %% AT is updated. ensure_atoms({empty, AT}, Cs) -> diff --git a/lib/stdlib/src/binary.erl b/lib/stdlib/src/binary.erl index f3e2f54215..e587cfe98d 100644 --- a/lib/stdlib/src/binary.erl +++ b/lib/stdlib/src/binary.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2021. All Rights Reserved. +%% Copyright Ericsson AB 2010-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -20,8 +20,8 @@ -module(binary). %% %% Implemented in this module: --export([replace/3,replace/4, - encode_hex/1, decode_hex/1]). +-export([replace/3, replace/4, + encode_hex/1, encode_hex/2, decode_hex/1]). -export_type([cp/0]). @@ -365,127 +365,112 @@ get_opts_replace(_,_) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Hex encoding functions %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% --define(HEX(X), (hex(X)):16). --compile({inline,[hex/1]}). +-compile({inline, [hex/2]}). -spec encode_hex(Bin) -> Bin2 when Bin :: binary(), Bin2 :: <<_:_*16>>. -encode_hex(Data) when byte_size(Data) rem 8 =:= 0 -> - << <<?HEX(A),?HEX(B),?HEX(C),?HEX(D),?HEX(E),?HEX(F),?HEX(G),?HEX(H)>> || <<A,B,C,D,E,F,G,H>> <= Data >>; -encode_hex(Data) when byte_size(Data) rem 7 =:= 0 -> - << <<?HEX(A),?HEX(B),?HEX(C),?HEX(D),?HEX(E),?HEX(F),?HEX(G)>> || <<A,B,C,D,E,F,G>> <= Data >>; -encode_hex(Data) when byte_size(Data) rem 6 =:= 0 -> - << <<?HEX(A),?HEX(B),?HEX(C),?HEX(D),?HEX(E),?HEX(F)>> || <<A,B,C,D,E,F>> <= Data >>; -encode_hex(Data) when byte_size(Data) rem 5 =:= 0 -> - << <<?HEX(A),?HEX(B),?HEX(C),?HEX(D),?HEX(E)>> || <<A,B,C,D,E>> <= Data >>; -encode_hex(Data) when byte_size(Data) rem 4 =:= 0 -> - << <<?HEX(A),?HEX(B),?HEX(C),?HEX(D)>> || <<A,B,C,D>> <= Data >>; -encode_hex(Data) when byte_size(Data) rem 3 =:= 0 -> - << <<?HEX(A),?HEX(B),?HEX(C)>> || <<A,B,C>> <= Data >>; -encode_hex(Data) when byte_size(Data) rem 2 =:= 0 -> - << <<?HEX(A),?HEX(B)>> || <<A,B>> <= Data >>; -encode_hex(Data) when is_binary(Data) -> - << <<?HEX(N)>> || <<N>> <= Data >>; +encode_hex(Bin) when is_binary(Bin) -> + encode_hex(Bin, uppercase); encode_hex(Bin) -> - badarg_with_info([Bin]). + error_with_info(badarg, [Bin]). -hex(X) -> +-spec encode_hex(Bin, Case) -> Bin2 when + Bin :: binary(), + Case :: lowercase | uppercase, + Bin2 :: <<_:_*16>>. +encode_hex(Bin, uppercase) when is_binary(Bin) -> + encode_hex1(Bin, 1); +encode_hex(Bin, lowercase) when is_binary(Bin) -> + encode_hex1(Bin, 257); +encode_hex(Bin, Case) -> + error_with_info(badarg, [Bin, Case]). + +encode_hex1(Data, Offset) -> + <<First:(bit_size(Data) div 64)/binary-unit:64, Rest/binary>> = Data, + Hex = << <<(hex(A, Offset)):16, (hex(B, Offset)):16, (hex(C, Offset)):16, (hex(D, Offset)):16, + (hex(E, Offset)):16, (hex(F, Offset)):16, (hex(G, Offset)):16, (hex(H, Offset)):16>> || + <<A,B,C,D,E,F,G,H>> <= First >>, + encode_hex2(Rest, Offset, Hex). + +encode_hex2(<<A,Data/binary>>, Offset, Acc) -> + encode_hex2(Data, Offset, <<Acc/binary, (hex(A, Offset)):16>>); +encode_hex2(<<>>, _Offset, Acc) -> + Acc. + +hex(X, Offset) -> element( - X+1, {16#3030, 16#3031, 16#3032, 16#3033, 16#3034, 16#3035, 16#3036, 16#3037, 16#3038, 16#3039, 16#3041, 16#3042, 16#3043, 16#3044, 16#3045, 16#3046, - 16#3130, 16#3131, 16#3132, 16#3133, 16#3134, 16#3135, 16#3136, 16#3137, 16#3138, 16#3139, 16#3141, 16#3142, 16#3143, 16#3144, 16#3145, 16#3146, - 16#3230, 16#3231, 16#3232, 16#3233, 16#3234, 16#3235, 16#3236, 16#3237, 16#3238, 16#3239, 16#3241, 16#3242, 16#3243, 16#3244, 16#3245, 16#3246, - 16#3330, 16#3331, 16#3332, 16#3333, 16#3334, 16#3335, 16#3336, 16#3337, 16#3338, 16#3339, 16#3341, 16#3342, 16#3343, 16#3344, 16#3345, 16#3346, - 16#3430, 16#3431, 16#3432, 16#3433, 16#3434, 16#3435, 16#3436, 16#3437, 16#3438, 16#3439, 16#3441, 16#3442, 16#3443, 16#3444, 16#3445, 16#3446, - 16#3530, 16#3531, 16#3532, 16#3533, 16#3534, 16#3535, 16#3536, 16#3537, 16#3538, 16#3539, 16#3541, 16#3542, 16#3543, 16#3544, 16#3545, 16#3546, - 16#3630, 16#3631, 16#3632, 16#3633, 16#3634, 16#3635, 16#3636, 16#3637, 16#3638, 16#3639, 16#3641, 16#3642, 16#3643, 16#3644, 16#3645, 16#3646, - 16#3730, 16#3731, 16#3732, 16#3733, 16#3734, 16#3735, 16#3736, 16#3737, 16#3738, 16#3739, 16#3741, 16#3742, 16#3743, 16#3744, 16#3745, 16#3746, - 16#3830, 16#3831, 16#3832, 16#3833, 16#3834, 16#3835, 16#3836, 16#3837, 16#3838, 16#3839, 16#3841, 16#3842, 16#3843, 16#3844, 16#3845, 16#3846, - 16#3930, 16#3931, 16#3932, 16#3933, 16#3934, 16#3935, 16#3936, 16#3937, 16#3938, 16#3939, 16#3941, 16#3942, 16#3943, 16#3944, 16#3945, 16#3946, - 16#4130, 16#4131, 16#4132, 16#4133, 16#4134, 16#4135, 16#4136, 16#4137, 16#4138, 16#4139, 16#4141, 16#4142, 16#4143, 16#4144, 16#4145, 16#4146, - 16#4230, 16#4231, 16#4232, 16#4233, 16#4234, 16#4235, 16#4236, 16#4237, 16#4238, 16#4239, 16#4241, 16#4242, 16#4243, 16#4244, 16#4245, 16#4246, - 16#4330, 16#4331, 16#4332, 16#4333, 16#4334, 16#4335, 16#4336, 16#4337, 16#4338, 16#4339, 16#4341, 16#4342, 16#4343, 16#4344, 16#4345, 16#4346, - 16#4430, 16#4431, 16#4432, 16#4433, 16#4434, 16#4435, 16#4436, 16#4437, 16#4438, 16#4439, 16#4441, 16#4442, 16#4443, 16#4444, 16#4445, 16#4446, - 16#4530, 16#4531, 16#4532, 16#4533, 16#4534, 16#4535, 16#4536, 16#4537, 16#4538, 16#4539, 16#4541, 16#4542, 16#4543, 16#4544, 16#4545, 16#4546, - 16#4630, 16#4631, 16#4632, 16#4633, 16#4634, 16#4635, 16#4636, 16#4637, 16#4638, 16#4639, 16#4641, 16#4642, 16#4643, 16#4644, 16#4645, 16#4646}). - + X + Offset, { + %% Used for Uppercase + 16#3030, 16#3031, 16#3032, 16#3033, 16#3034, 16#3035, 16#3036, 16#3037, 16#3038, 16#3039, 16#3041, 16#3042, 16#3043, 16#3044, 16#3045, 16#3046, + 16#3130, 16#3131, 16#3132, 16#3133, 16#3134, 16#3135, 16#3136, 16#3137, 16#3138, 16#3139, 16#3141, 16#3142, 16#3143, 16#3144, 16#3145, 16#3146, + 16#3230, 16#3231, 16#3232, 16#3233, 16#3234, 16#3235, 16#3236, 16#3237, 16#3238, 16#3239, 16#3241, 16#3242, 16#3243, 16#3244, 16#3245, 16#3246, + 16#3330, 16#3331, 16#3332, 16#3333, 16#3334, 16#3335, 16#3336, 16#3337, 16#3338, 16#3339, 16#3341, 16#3342, 16#3343, 16#3344, 16#3345, 16#3346, + 16#3430, 16#3431, 16#3432, 16#3433, 16#3434, 16#3435, 16#3436, 16#3437, 16#3438, 16#3439, 16#3441, 16#3442, 16#3443, 16#3444, 16#3445, 16#3446, + 16#3530, 16#3531, 16#3532, 16#3533, 16#3534, 16#3535, 16#3536, 16#3537, 16#3538, 16#3539, 16#3541, 16#3542, 16#3543, 16#3544, 16#3545, 16#3546, + 16#3630, 16#3631, 16#3632, 16#3633, 16#3634, 16#3635, 16#3636, 16#3637, 16#3638, 16#3639, 16#3641, 16#3642, 16#3643, 16#3644, 16#3645, 16#3646, + 16#3730, 16#3731, 16#3732, 16#3733, 16#3734, 16#3735, 16#3736, 16#3737, 16#3738, 16#3739, 16#3741, 16#3742, 16#3743, 16#3744, 16#3745, 16#3746, + 16#3830, 16#3831, 16#3832, 16#3833, 16#3834, 16#3835, 16#3836, 16#3837, 16#3838, 16#3839, 16#3841, 16#3842, 16#3843, 16#3844, 16#3845, 16#3846, + 16#3930, 16#3931, 16#3932, 16#3933, 16#3934, 16#3935, 16#3936, 16#3937, 16#3938, 16#3939, 16#3941, 16#3942, 16#3943, 16#3944, 16#3945, 16#3946, + 16#4130, 16#4131, 16#4132, 16#4133, 16#4134, 16#4135, 16#4136, 16#4137, 16#4138, 16#4139, 16#4141, 16#4142, 16#4143, 16#4144, 16#4145, 16#4146, + 16#4230, 16#4231, 16#4232, 16#4233, 16#4234, 16#4235, 16#4236, 16#4237, 16#4238, 16#4239, 16#4241, 16#4242, 16#4243, 16#4244, 16#4245, 16#4246, + 16#4330, 16#4331, 16#4332, 16#4333, 16#4334, 16#4335, 16#4336, 16#4337, 16#4338, 16#4339, 16#4341, 16#4342, 16#4343, 16#4344, 16#4345, 16#4346, + 16#4430, 16#4431, 16#4432, 16#4433, 16#4434, 16#4435, 16#4436, 16#4437, 16#4438, 16#4439, 16#4441, 16#4442, 16#4443, 16#4444, 16#4445, 16#4446, + 16#4530, 16#4531, 16#4532, 16#4533, 16#4534, 16#4535, 16#4536, 16#4537, 16#4538, 16#4539, 16#4541, 16#4542, 16#4543, 16#4544, 16#4545, 16#4546, + 16#4630, 16#4631, 16#4632, 16#4633, 16#4634, 16#4635, 16#4636, 16#4637, 16#4638, 16#4639, 16#4641, 16#4642, 16#4643, 16#4644, 16#4645, 16#4646, + %% Used for Lowercase + 16#3030, 16#3031, 16#3032, 16#3033, 16#3034, 16#3035, 16#3036, 16#3037, 16#3038, 16#3039, 16#3061, 16#3062, 16#3063, 16#3064, 16#3065, 16#3066, + 16#3130, 16#3131, 16#3132, 16#3133, 16#3134, 16#3135, 16#3136, 16#3137, 16#3138, 16#3139, 16#3161, 16#3162, 16#3163, 16#3164, 16#3165, 16#3166, + 16#3230, 16#3231, 16#3232, 16#3233, 16#3234, 16#3235, 16#3236, 16#3237, 16#3238, 16#3239, 16#3261, 16#3262, 16#3263, 16#3264, 16#3265, 16#3266, + 16#3330, 16#3331, 16#3332, 16#3333, 16#3334, 16#3335, 16#3336, 16#3337, 16#3338, 16#3339, 16#3361, 16#3362, 16#3363, 16#3364, 16#3365, 16#3366, + 16#3430, 16#3431, 16#3432, 16#3433, 16#3434, 16#3435, 16#3436, 16#3437, 16#3438, 16#3439, 16#3461, 16#3462, 16#3463, 16#3464, 16#3465, 16#3466, + 16#3530, 16#3531, 16#3532, 16#3533, 16#3534, 16#3535, 16#3536, 16#3537, 16#3538, 16#3539, 16#3561, 16#3562, 16#3563, 16#3564, 16#3565, 16#3566, + 16#3630, 16#3631, 16#3632, 16#3633, 16#3634, 16#3635, 16#3636, 16#3637, 16#3638, 16#3639, 16#3661, 16#3662, 16#3663, 16#3664, 16#3665, 16#3666, + 16#3730, 16#3731, 16#3732, 16#3733, 16#3734, 16#3735, 16#3736, 16#3737, 16#3738, 16#3739, 16#3761, 16#3762, 16#3763, 16#3764, 16#3765, 16#3766, + 16#3830, 16#3831, 16#3832, 16#3833, 16#3834, 16#3835, 16#3836, 16#3837, 16#3838, 16#3839, 16#3861, 16#3862, 16#3863, 16#3864, 16#3865, 16#3866, + 16#3930, 16#3931, 16#3932, 16#3933, 16#3934, 16#3935, 16#3936, 16#3937, 16#3938, 16#3939, 16#3961, 16#3962, 16#3963, 16#3964, 16#3965, 16#3966, + 16#6130, 16#6131, 16#6132, 16#6133, 16#6134, 16#6135, 16#6136, 16#6137, 16#6138, 16#6139, 16#6161, 16#6162, 16#6163, 16#6164, 16#6165, 16#6166, + 16#6230, 16#6231, 16#6232, 16#6233, 16#6234, 16#6235, 16#6236, 16#6237, 16#6238, 16#6239, 16#6261, 16#6262, 16#6263, 16#6264, 16#6265, 16#6266, + 16#6330, 16#6331, 16#6332, 16#6333, 16#6334, 16#6335, 16#6336, 16#6337, 16#6338, 16#6339, 16#6361, 16#6362, 16#6363, 16#6364, 16#6365, 16#6366, + 16#6430, 16#6431, 16#6432, 16#6433, 16#6434, 16#6435, 16#6436, 16#6437, 16#6438, 16#6439, 16#6461, 16#6462, 16#6463, 16#6464, 16#6465, 16#6466, + 16#6530, 16#6531, 16#6532, 16#6533, 16#6534, 16#6535, 16#6536, 16#6537, 16#6538, 16#6539, 16#6561, 16#6562, 16#6563, 16#6564, 16#6565, 16#6566, + 16#6630, 16#6631, 16#6632, 16#6633, 16#6634, 16#6635, 16#6636, 16#6637, 16#6638, 16#6639, 16#6661, 16#6662, 16#6663, 16#6664, 16#6665, 16#6666}). + +-compile({inline, [unhex/1]}). -spec decode_hex(Bin) -> Bin2 when Bin :: <<_:_*16>>, Bin2 :: binary(). -decode_hex(Bin) when byte_size(Bin) rem 2 =:= 0 -> - << <<(unhex(Int))>> || <<Int:16>> <= Bin >>; -decode_hex(Bin) -> - badarg_with_info([Bin]). - -%% This function pattern-matches on the hexadecimal representation of a pair of characters -%% for example, 16#3030 is matching on the integers <<48, 48>>, which is ascii for <<"00">> -unhex(16#3030) -> 0; unhex(16#3031) -> 1; unhex(16#3032) -> 2; unhex(16#3033) -> 3; unhex(16#3034) -> 4; unhex(16#3035) -> 5; unhex(16#3036) -> 6; unhex(16#3037) -> 7; unhex(16#3038) -> 8; unhex(16#3039) -> 9; -unhex(16#3041) -> 10; unhex(16#3042) -> 11; unhex(16#3043) -> 12; unhex(16#3044) -> 13; unhex(16#3045) -> 14; unhex(16#3046) -> 15; -unhex(16#3061) -> 10; unhex(16#3062) -> 11; unhex(16#3063) -> 12; unhex(16#3064) -> 13; unhex(16#3065) -> 14; unhex(16#3066) -> 15; -unhex(16#3130) -> 16; unhex(16#3131) -> 17; unhex(16#3132) -> 18; unhex(16#3133) -> 19; unhex(16#3134) -> 20; unhex(16#3135) -> 21; unhex(16#3136) -> 22; unhex(16#3137) -> 23; unhex(16#3138) -> 24; unhex(16#3139) -> 25; -unhex(16#3141) -> 26; unhex(16#3142) -> 27; unhex(16#3143) -> 28; unhex(16#3144) -> 29; unhex(16#3145) -> 30; unhex(16#3146) -> 31; -unhex(16#3161) -> 26; unhex(16#3162) -> 27; unhex(16#3163) -> 28; unhex(16#3164) -> 29; unhex(16#3165) -> 30; unhex(16#3166) -> 31; -unhex(16#3230) -> 32; unhex(16#3231) -> 33; unhex(16#3232) -> 34; unhex(16#3233) -> 35; unhex(16#3234) -> 36; unhex(16#3235) -> 37; unhex(16#3236) -> 38; unhex(16#3237) -> 39; unhex(16#3238) -> 40; unhex(16#3239) -> 41; -unhex(16#3241) -> 42; unhex(16#3242) -> 43; unhex(16#3243) -> 44; unhex(16#3244) -> 45; unhex(16#3245) -> 46; unhex(16#3246) -> 47; -unhex(16#3261) -> 42; unhex(16#3262) -> 43; unhex(16#3263) -> 44; unhex(16#3264) -> 45; unhex(16#3265) -> 46; unhex(16#3266) -> 47; -unhex(16#3330) -> 48; unhex(16#3331) -> 49; unhex(16#3332) -> 50; unhex(16#3333) -> 51; unhex(16#3334) -> 52; unhex(16#3335) -> 53; unhex(16#3336) -> 54; unhex(16#3337) -> 55; unhex(16#3338) -> 56; unhex(16#3339) -> 57; -unhex(16#3341) -> 58; unhex(16#3342) -> 59; unhex(16#3343) -> 60; unhex(16#3344) -> 61; unhex(16#3345) -> 62; unhex(16#3346) -> 63; -unhex(16#3361) -> 58; unhex(16#3362) -> 59; unhex(16#3363) -> 60; unhex(16#3364) -> 61; unhex(16#3365) -> 62; unhex(16#3366) -> 63; -unhex(16#3430) -> 64; unhex(16#3431) -> 65; unhex(16#3432) -> 66; unhex(16#3433) -> 67; unhex(16#3434) -> 68; unhex(16#3435) -> 69; unhex(16#3436) -> 70; unhex(16#3437) -> 71; unhex(16#3438) -> 72; unhex(16#3439) -> 73; -unhex(16#3441) -> 74; unhex(16#3442) -> 75; unhex(16#3443) -> 76; unhex(16#3444) -> 77; unhex(16#3445) -> 78; unhex(16#3446) -> 79; -unhex(16#3461) -> 74; unhex(16#3462) -> 75; unhex(16#3463) -> 76; unhex(16#3464) -> 77; unhex(16#3465) -> 78; unhex(16#3466) -> 79; -unhex(16#3530) -> 80; unhex(16#3531) -> 81; unhex(16#3532) -> 82; unhex(16#3533) -> 83; unhex(16#3534) -> 84; unhex(16#3535) -> 85; unhex(16#3536) -> 86; unhex(16#3537) -> 87; unhex(16#3538) -> 88; unhex(16#3539) -> 89; -unhex(16#3541) -> 90; unhex(16#3542) -> 91; unhex(16#3543) -> 92; unhex(16#3544) -> 93; unhex(16#3545) -> 94; unhex(16#3546) -> 95; -unhex(16#3561) -> 90; unhex(16#3562) -> 91; unhex(16#3563) -> 92; unhex(16#3564) -> 93; unhex(16#3565) -> 94; unhex(16#3566) -> 95; -unhex(16#3630) -> 96; unhex(16#3631) -> 97; unhex(16#3632) -> 98; unhex(16#3633) -> 99; unhex(16#3634) -> 100; unhex(16#3635) -> 101; unhex(16#3636) -> 102; unhex(16#3637) -> 103; unhex(16#3638) -> 104; unhex(16#3639) -> 105; -unhex(16#3641) -> 106; unhex(16#3642) -> 107; unhex(16#3643) -> 108; unhex(16#3644) -> 109; unhex(16#3645) -> 110; unhex(16#3646) -> 111; -unhex(16#3661) -> 106; unhex(16#3662) -> 107; unhex(16#3663) -> 108; unhex(16#3664) -> 109; unhex(16#3665) -> 110; unhex(16#3666) -> 111; -unhex(16#3730) -> 112; unhex(16#3731) -> 113; unhex(16#3732) -> 114; unhex(16#3733) -> 115; unhex(16#3734) -> 116; unhex(16#3735) -> 117; unhex(16#3736) -> 118; unhex(16#3737) -> 119; unhex(16#3738) -> 120; unhex(16#3739) -> 121; -unhex(16#3741) -> 122; unhex(16#3742) -> 123; unhex(16#3743) -> 124; unhex(16#3744) -> 125; unhex(16#3745) -> 126; unhex(16#3746) -> 127; -unhex(16#3761) -> 122; unhex(16#3762) -> 123; unhex(16#3763) -> 124; unhex(16#3764) -> 125; unhex(16#3765) -> 126; unhex(16#3766) -> 127; -unhex(16#3830) -> 128; unhex(16#3831) -> 129; unhex(16#3832) -> 130; unhex(16#3833) -> 131; unhex(16#3834) -> 132; unhex(16#3835) -> 133; unhex(16#3836) -> 134; unhex(16#3837) -> 135; unhex(16#3838) -> 136; unhex(16#3839) -> 137; -unhex(16#3841) -> 138; unhex(16#3842) -> 139; unhex(16#3843) -> 140; unhex(16#3844) -> 141; unhex(16#3845) -> 142; unhex(16#3846) -> 143; -unhex(16#3861) -> 138; unhex(16#3862) -> 139; unhex(16#3863) -> 140; unhex(16#3864) -> 141; unhex(16#3865) -> 142; unhex(16#3866) -> 143; -unhex(16#3930) -> 144; unhex(16#3931) -> 145; unhex(16#3932) -> 146; unhex(16#3933) -> 147; unhex(16#3934) -> 148; unhex(16#3935) -> 149; unhex(16#3936) -> 150; unhex(16#3937) -> 151; unhex(16#3938) -> 152; unhex(16#3939) -> 153; -unhex(16#3941) -> 154; unhex(16#3942) -> 155; unhex(16#3943) -> 156; unhex(16#3944) -> 157; unhex(16#3945) -> 158; unhex(16#3946) -> 159; -unhex(16#3961) -> 154; unhex(16#3962) -> 155; unhex(16#3963) -> 156; unhex(16#3964) -> 157; unhex(16#3965) -> 158; unhex(16#3966) -> 159; -unhex(16#4130) -> 160; unhex(16#4131) -> 161; unhex(16#4132) -> 162; unhex(16#4133) -> 163; unhex(16#4134) -> 164; unhex(16#4135) -> 165; unhex(16#4136) -> 166; unhex(16#4137) -> 167; unhex(16#4138) -> 168; unhex(16#4139) -> 169; -unhex(16#4141) -> 170; unhex(16#4142) -> 171; unhex(16#4143) -> 172; unhex(16#4144) -> 173; unhex(16#4145) -> 174; unhex(16#4146) -> 175; -unhex(16#4161) -> 170; unhex(16#4162) -> 171; unhex(16#4163) -> 172; unhex(16#4164) -> 173; unhex(16#4165) -> 174; unhex(16#4166) -> 175; -unhex(16#4230) -> 176; unhex(16#4231) -> 177; unhex(16#4232) -> 178; unhex(16#4233) -> 179; unhex(16#4234) -> 180; unhex(16#4235) -> 181; unhex(16#4236) -> 182; unhex(16#4237) -> 183; unhex(16#4238) -> 184; unhex(16#4239) -> 185; -unhex(16#4241) -> 186; unhex(16#4242) -> 187; unhex(16#4243) -> 188; unhex(16#4244) -> 189; unhex(16#4245) -> 190; unhex(16#4246) -> 191; -unhex(16#4261) -> 186; unhex(16#4262) -> 187; unhex(16#4263) -> 188; unhex(16#4264) -> 189; unhex(16#4265) -> 190; unhex(16#4266) -> 191; -unhex(16#4330) -> 192; unhex(16#4331) -> 193; unhex(16#4332) -> 194; unhex(16#4333) -> 195; unhex(16#4334) -> 196; unhex(16#4335) -> 197; unhex(16#4336) -> 198; unhex(16#4337) -> 199; unhex(16#4338) -> 200; unhex(16#4339) -> 201; -unhex(16#4341) -> 202; unhex(16#4342) -> 203; unhex(16#4343) -> 204; unhex(16#4344) -> 205; unhex(16#4345) -> 206; unhex(16#4346) -> 207; -unhex(16#4361) -> 202; unhex(16#4362) -> 203; unhex(16#4363) -> 204; unhex(16#4364) -> 205; unhex(16#4365) -> 206; unhex(16#4366) -> 207; -unhex(16#4430) -> 208; unhex(16#4431) -> 209; unhex(16#4432) -> 210; unhex(16#4433) -> 211; unhex(16#4434) -> 212; unhex(16#4435) -> 213; unhex(16#4436) -> 214; unhex(16#4437) -> 215; unhex(16#4438) -> 216; unhex(16#4439) -> 217; -unhex(16#4441) -> 218; unhex(16#4442) -> 219; unhex(16#4443) -> 220; unhex(16#4444) -> 221; unhex(16#4445) -> 222; unhex(16#4446) -> 223; -unhex(16#4461) -> 218; unhex(16#4462) -> 219; unhex(16#4463) -> 220; unhex(16#4464) -> 221; unhex(16#4465) -> 222; unhex(16#4466) -> 223; -unhex(16#4530) -> 224; unhex(16#4531) -> 225; unhex(16#4532) -> 226; unhex(16#4533) -> 227; unhex(16#4534) -> 228; unhex(16#4535) -> 229; unhex(16#4536) -> 230; unhex(16#4537) -> 231; unhex(16#4538) -> 232; unhex(16#4539) -> 233; -unhex(16#4541) -> 234; unhex(16#4542) -> 235; unhex(16#4543) -> 236; unhex(16#4544) -> 237; unhex(16#4545) -> 238; unhex(16#4546) -> 239; -unhex(16#4561) -> 234; unhex(16#4562) -> 235; unhex(16#4563) -> 236; unhex(16#4564) -> 237; unhex(16#4565) -> 238; unhex(16#4566) -> 239; -unhex(16#4630) -> 240; unhex(16#4631) -> 241; unhex(16#4632) -> 242; unhex(16#4633) -> 243; unhex(16#4634) -> 244; unhex(16#4635) -> 245; unhex(16#4636) -> 246; unhex(16#4637) -> 247; unhex(16#4638) -> 248; unhex(16#4639) -> 249; -unhex(16#4641) -> 250; unhex(16#4642) -> 251; unhex(16#4643) -> 252; unhex(16#4644) -> 253; unhex(16#4645) -> 254; unhex(16#4646) -> 255; -unhex(16#4661) -> 250; unhex(16#4662) -> 251; unhex(16#4663) -> 252; unhex(16#4664) -> 253; unhex(16#4665) -> 254; unhex(16#4666) -> 255; -unhex(16#6130) -> 160; unhex(16#6131) -> 161; unhex(16#6132) -> 162; unhex(16#6133) -> 163; unhex(16#6134) -> 164; unhex(16#6135) -> 165; unhex(16#6136) -> 166; unhex(16#6137) -> 167; unhex(16#6138) -> 168; unhex(16#6139) -> 169; -unhex(16#6141) -> 170; unhex(16#6142) -> 171; unhex(16#6143) -> 172; unhex(16#6144) -> 173; unhex(16#6145) -> 174; unhex(16#6146) -> 175; -unhex(16#6161) -> 170; unhex(16#6162) -> 171; unhex(16#6163) -> 172; unhex(16#6164) -> 173; unhex(16#6165) -> 174; unhex(16#6166) -> 175; -unhex(16#6230) -> 176; unhex(16#6231) -> 177; unhex(16#6232) -> 178; unhex(16#6233) -> 179; unhex(16#6234) -> 180; unhex(16#6235) -> 181; unhex(16#6236) -> 182; unhex(16#6237) -> 183; unhex(16#6238) -> 184; unhex(16#6239) -> 185; -unhex(16#6241) -> 186; unhex(16#6242) -> 187; unhex(16#6243) -> 188; unhex(16#6244) -> 189; unhex(16#6245) -> 190; unhex(16#6246) -> 191; -unhex(16#6261) -> 186; unhex(16#6262) -> 187; unhex(16#6263) -> 188; unhex(16#6264) -> 189; unhex(16#6265) -> 190; unhex(16#6266) -> 191; -unhex(16#6330) -> 192; unhex(16#6331) -> 193; unhex(16#6332) -> 194; unhex(16#6333) -> 195; unhex(16#6334) -> 196; unhex(16#6335) -> 197; unhex(16#6336) -> 198; unhex(16#6337) -> 199; unhex(16#6338) -> 200; unhex(16#6339) -> 201; -unhex(16#6341) -> 202; unhex(16#6342) -> 203; unhex(16#6343) -> 204; unhex(16#6344) -> 205; unhex(16#6345) -> 206; unhex(16#6346) -> 207; -unhex(16#6361) -> 202; unhex(16#6362) -> 203; unhex(16#6363) -> 204; unhex(16#6364) -> 205; unhex(16#6365) -> 206; unhex(16#6366) -> 207; -unhex(16#6430) -> 208; unhex(16#6431) -> 209; unhex(16#6432) -> 210; unhex(16#6433) -> 211; unhex(16#6434) -> 212; unhex(16#6435) -> 213; unhex(16#6436) -> 214; unhex(16#6437) -> 215; unhex(16#6438) -> 216; unhex(16#6439) -> 217; -unhex(16#6441) -> 218; unhex(16#6442) -> 219; unhex(16#6443) -> 220; unhex(16#6444) -> 221; unhex(16#6445) -> 222; unhex(16#6446) -> 223; -unhex(16#6461) -> 218; unhex(16#6462) -> 219; unhex(16#6463) -> 220; unhex(16#6464) -> 221; unhex(16#6465) -> 222; unhex(16#6466) -> 223; -unhex(16#6530) -> 224; unhex(16#6531) -> 225; unhex(16#6532) -> 226; unhex(16#6533) -> 227; unhex(16#6534) -> 228; unhex(16#6535) -> 229; unhex(16#6536) -> 230; unhex(16#6537) -> 231; unhex(16#6538) -> 232; unhex(16#6539) -> 233; -unhex(16#6541) -> 234; unhex(16#6542) -> 235; unhex(16#6543) -> 236; unhex(16#6544) -> 237; unhex(16#6545) -> 238; unhex(16#6546) -> 239; -unhex(16#6561) -> 234; unhex(16#6562) -> 235; unhex(16#6563) -> 236; unhex(16#6564) -> 237; unhex(16#6565) -> 238; unhex(16#6566) -> 239; -unhex(16#6630) -> 240; unhex(16#6631) -> 241; unhex(16#6632) -> 242; unhex(16#6633) -> 243; unhex(16#6634) -> 244; unhex(16#6635) -> 245; unhex(16#6636) -> 246; unhex(16#6637) -> 247; unhex(16#6638) -> 248; unhex(16#6639) -> 249; -unhex(16#6641) -> 250; unhex(16#6642) -> 251; unhex(16#6643) -> 252; unhex(16#6644) -> 253; unhex(16#6645) -> 254; unhex(16#6646) -> 255; -unhex(16#6661) -> 250; unhex(16#6662) -> 251; unhex(16#6663) -> 252; unhex(16#6664) -> 253; unhex(16#6665) -> 254; unhex(16#6666) -> 255; -unhex(Char) -> - badarg_with_info([<<Char:16>>]). +decode_hex(Data) when byte_size(Data) rem 2 =:= 0 -> + try + decode_hex1(Data) + catch + error:badarg -> + badarg_with_info([Data]) + end; +decode_hex(Data) -> + badarg_with_info([Data]). + +decode_hex1(Data) -> + <<First:(byte_size(Data) div 8)/binary-unit:64, Rest/binary>> = Data, + Bin = << <<(unhex(A)):4, (unhex(B)):4, (unhex(C)):4, (unhex(D)):4, + (unhex(E)):4, (unhex(F)):4, (unhex(G)):4, (unhex(H)):4>> || + <<A,B,C,D,E,F,G,H>> <= First >>, + decode_hex2(Rest, Bin). + +decode_hex2(<<A,Data/binary>>, Acc) -> + decode_hex2(Data, <<Acc/binary-unit:4, (unhex(A)):4>>); +decode_hex2(<<>>, Acc) -> + Acc. + +unhex(X) -> + element(X, + {nonono, no, no, no, no, no, no, no, no, no, no, no, no, no, no, %1 + no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, %16 + no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, %32 + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, no, no, no, no, no, no, %48 + no, 10, 11, 12, 13, 14, 15, no, no, no, no, no, no, no, no, no, %64 + no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, %80 + no, 10, 11, 12, 13, 14, 15, no, no, no, no, no, no, no, no, no %96 + }). badarg_with_cause(Args, Cause) -> erlang:error(badarg, Args, [{error_info, #{module => erl_stdlib_errors, diff --git a/lib/stdlib/src/dets.erl b/lib/stdlib/src/dets.erl index 640ad7a81c..133d209ae1 100644 --- a/lib/stdlib/src/dets.erl +++ b/lib/stdlib/src/dets.erl @@ -93,7 +93,8 @@ tab_name/0]). -compile({inline, [{einval,2},{badarg,2},{undefined,1}, - {badarg_exit,2},{lookup_reply,2}]}). + {badarg_exit,2},{lookup_reply,2}, + {pidof,1},{resp,2}]}). -include_lib("kernel/include/file.hrl"). @@ -1237,16 +1238,25 @@ treq(Tab, R) -> req(Proc, R) -> Ref = erlang:monitor(process, Proc), - Proc ! ?DETS_CALL(self(), R), + Proc ! ?DETS_CALL({self(), Ref}, R), receive {'DOWN', Ref, process, Proc, _Info} -> badarg; - {Proc, Reply} -> + {Ref, Reply} -> erlang:demonitor(Ref, [flush]), Reply end. %% Inlined. +pidof({Pid, _Tag}) -> + Pid. + +%% Inlined. +resp({Pid, Tag} = _From, Message) -> + Pid ! {Tag, Message}, + ok. + +%% Inlined. einval({error, {file_error, _, einval}}, A) -> erlang:error(badarg, A); einval({error, {file_error, _, badarg}}, A) -> @@ -1398,7 +1408,7 @@ apply_op(Op, From, Head, N) -> true -> err({error, incompatible_arguments}) end, - From ! {self(), Res}, + resp(From, Res), ok; auto_save -> case Head#head.update_mode of @@ -1419,7 +1429,7 @@ apply_op(Op, From, Head, N) -> {0, Head} end; close -> - From ! {self(), fclose(Head)}, + resp(From, fclose(Head)), _NewHead = unlink_fixing_procs(Head), ?PROFILE(ep:done()), exit(normal); @@ -1427,25 +1437,25 @@ apply_op(Op, From, Head, N) -> %% Used from dets_server when Pid has closed the table, %% but the table is still opened by some process. NewHead = remove_fix(Head, Pid, close), - From ! {self(), status(NewHead)}, + resp(From, status(NewHead)), NewHead; {corrupt, Reason} -> {H2, Error} = dets_utils:corrupt_reason(Head, Reason), - From ! {self(), Error}, + resp(From, Error), H2; {delayed_write, WrTime} -> delayed_write(Head, WrTime); info -> {H2, Res} = finfo(Head), - From ! {self(), Res}, + resp(From, Res), H2; {info, Tag} -> {H2, Res} = finfo(Head, Tag), - From ! {self(), Res}, + resp(From, Res), H2; {is_compatible_bchunk_format, Term} -> Res = test_bchunk_format(Head, Term), - From ! {self(), Res}, + resp(From, Res), ok; {internal_open, Ref, Args} -> do_internal_open(Head#head.parent, Head#head.server, From, @@ -1462,27 +1472,27 @@ apply_op(Op, From, Head, N) -> end; {set_verbose, What} -> set_verbose(What), - From ! {self(), ok}, + resp(From, ok), ok; {where, Object} -> {H2, Res} = where_is_object(Head, Object), - From ! {self(), Res}, + resp(From, Res), H2; _Message when element(1, Head#head.update_mode) =:= error -> - From ! {self(), status(Head)}, + resp(From, status(Head)), ok; %% The following messages assume that the status of the table is OK. {bchunk_init, Tab} -> {H2, Res} = do_bchunk_init(Head, Tab), - From ! {self(), Res}, + resp(From, Res), H2; {bchunk, State} -> {H2, Res} = do_bchunk(Head, State), - From ! {self(), Res}, + resp(From, Res), H2; delete_all_objects -> {H2, Res} = fdelete_all_objects(Head), - From ! {self(), Res}, + resp(From, Res), erlang:garbage_collect(), {0, H2}; {delete_key, _Keys} when Head#head.update_mode =:= dirty -> @@ -1492,16 +1502,16 @@ apply_op(Op, From, Head, N) -> true -> stream_op(Op, From, [], Head, N); false -> - From ! {self(), badarg}, + resp(From, badarg), ok end; first -> {H2, Res} = ffirst(Head), - From ! {self(), Res}, + resp(From, Res), H2; {initialize, InitFun, Format, MinNoSlots} -> {H2, Res} = finit(Head, InitFun, Format, MinNoSlots), - From ! {self(), Res}, + resp(From, Res), erlang:garbage_collect(), H2; {insert, Objs} when Head#head.update_mode =:= dirty -> @@ -1509,12 +1519,12 @@ apply_op(Op, From, Head, N) -> true -> stream_op(Op, From, [], Head, N); false -> - From ! {self(), badarg}, + resp(From, badarg), ok end; {insert_new, Objs} when Head#head.update_mode =:= dirty -> {H2, Res} = finsert_new(Head, Objs), - From ! {self(), Res}, + resp(From, Res), {N + 1, H2}; {lookup_keys, _Keys} -> stream_op(Op, From, [], Head, N); @@ -1523,48 +1533,48 @@ apply_op(Op, From, Head, N) -> H2 = case Res of {cont,_} -> H1; _ when Safe =:= no_safe-> H1; - _ when Safe =:= safe -> do_safe_fixtable(H1, From, false) + _ when Safe =:= safe -> do_safe_fixtable(H1, pidof(From), false) end, - From ! {self(), Res}, + resp(From, Res), H2; {match, MP, Spec, NObjs, Safe} -> {H2, Res} = fmatch(Head, MP, Spec, NObjs, Safe, From), - From ! {self(), Res}, + resp(From, Res), H2; {member, _Key} = Op -> stream_op(Op, From, [], Head, N); {next, Key} -> {H2, Res} = fnext(Head, Key), - From ! {self(), Res}, + resp(From, Res), H2; {match_delete, State} when Head#head.update_mode =:= dirty -> {H1, Res} = fmatch_delete(Head, State), H2 = case Res of {cont,_S,_N} -> H1; - _ -> do_safe_fixtable(H1, From, false) + _ -> do_safe_fixtable(H1, pidof(From), false) end, - From ! {self(), Res}, + resp(From, Res), {N + 1, H2}; {match_delete_init, MP, Spec} when Head#head.update_mode =:= dirty -> {H2, Res} = fmatch_delete_init(Head, MP, Spec, From), - From ! {self(), Res}, + resp(From, Res), {N + 1, H2}; {safe_fixtable, Bool} -> - NewHead = do_safe_fixtable(Head, From, Bool), - From ! {self(), ok}, + NewHead = do_safe_fixtable(Head, pidof(From), Bool), + resp(From, ok), NewHead; {slot, Slot} -> {H2, Res} = fslot(Head, Slot), - From ! {self(), Res}, + resp(From, Res), H2; sync -> {NewHead, Res} = perform_save(Head, true), - From ! {self(), Res}, + resp(From, Res), erlang:garbage_collect(), {0, NewHead}; {update_counter, Key, Incr} when Head#head.update_mode =:= dirty -> {NewHead, Res} = do_update_counter(Head, Key, Incr), - From ! {self(), Res}, + resp(From, Res), {N + 1, NewHead}; WriteOp when Head#head.update_mode =:= new_dirty -> H2 = Head#head{update_mode = dirty}, @@ -1577,12 +1587,12 @@ apply_op(Op, From, Head, N) -> H2 = Head#head{update_mode = dirty}, apply_op(WriteOp, From, H2, 0); {NewHead, Error} when is_record(NewHead, head) -> - From ! {self(), Error}, + resp(From, Error), NewHead end; WriteOp when is_tuple(WriteOp), Head#head.access =:= read -> Reason = {access_mode, Head#head.filename}, - From ! {self(), err({error, Reason})}, + resp(From, err({error, Reason})), ok end. @@ -1603,7 +1613,7 @@ bug_found(Name, Op, Bad, Stacktrace, From) -> end, if From =/= self() -> - From ! {self(), {error, {dets_bug, Name, Op, Bad}}}, + resp(From, {error, {dets_bug, Name, Op, Bad}}), ok; true -> % auto_save | may_grow | {delayed_write, _} ok @@ -1613,10 +1623,10 @@ do_internal_open(Parent, Server, From, Ref, Args) -> ?PROFILE(ep:do()), case do_open_file(Args, Parent, Server, Ref) of {ok, Head} -> - From ! {self(), ok}, + resp(From, ok), Head; Error -> - From ! {self(), Error}, + resp(From, Error), exit(normal) end. @@ -1698,7 +1708,7 @@ stream_end1(Pids, Next, N, C, Head, PwriteList) -> stream_end2(Pids, Pids, Next, N, C, Head1, PR). stream_end2([Pid | Pids], Ps, Next, N, C, Head, Reply) -> - Pid ! {self(), Reply}, + resp(Pid, Reply), stream_end2(Pids, Ps, Next, N+1, C, Head, Reply); stream_end2([], Ps, no_more, N, C, Head, _Reply) -> penalty(Head, Ps, C), @@ -1710,7 +1720,7 @@ penalty(H, _Ps, _C) when H#head.fixed =:= false -> ok; penalty(_H, _Ps, [{{lookup,_Pids},_Keys}]) -> ok; -penalty(#head{fixed = {_,[{Pid,_}]}}, [Pid], _C) -> +penalty(#head{fixed = {_,[{Pid, _}]}}, [{Pid, _Tag} = _From], _C) -> ok; penalty(_H, _Ps, _C) -> timer:sleep(1). @@ -1729,9 +1739,9 @@ lookup_replies(P, O, [{P2,O2} | L]) -> %% If a list of Pid then op was {member, Key}. Inlined. lookup_reply([P], O) -> - P ! {self(), O =/= []}; + resp(P, O =/= []); lookup_reply(P, O) -> - P ! {self(), O}. + resp(P, O). %%----------------------------------------------------------------- %% Callback functions for system messages handling. @@ -2253,7 +2263,7 @@ fmatch(Head, MP, Spec, N, Safe, From) -> {Head1, []} -> NewHead = case Safe of - safe -> do_safe_fixtable(Head1, From, true); + safe -> do_safe_fixtable(Head1, pidof(From), true); no_safe -> Head1 end, C0 = init_scan(NewHead, N), @@ -2370,7 +2380,7 @@ do_fmatch_delete_var_keys(Head, _MP, ?PATTERN_TO_TRUE_MATCH_SPEC('_'), _From) Reply end; do_fmatch_delete_var_keys(Head, MP, _Spec, From) -> - Head1 = do_safe_fixtable(Head, From, true), + Head1 = do_safe_fixtable(Head, pidof(From), true), {NewHead, []} = write_cache(Head1), C0 = init_scan(NewHead, default), {NewHead, {cont, C0#dets_cont{match_program = MP}, 0}}. diff --git a/lib/stdlib/src/edlin.erl b/lib/stdlib/src/edlin.erl index 6078c5e67b..b015479b9f 100644 --- a/lib/stdlib/src/edlin.erl +++ b/lib/stdlib/src/edlin.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2019. All Rights Reserved. +%% Copyright Ericsson AB 1996-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -91,8 +91,8 @@ edit([C|Cs], P, Line, {blink,_}, [_|Rs]) -> %Remove blink here edit([C|Cs], P, Line, none, Rs); edit([C|Cs], P, {Bef,Aft}, Prefix, Rs0) -> case key_map(C, Prefix) of - meta -> - edit(Cs, P, {Bef,Aft}, meta, Rs0); + meta -> + edit(Cs, P, {Bef,Aft}, meta, Rs0); meta_o -> edit(Cs, P, {Bef,Aft}, meta_o, Rs0); meta_csi -> @@ -101,56 +101,44 @@ edit([C|Cs], P, {Bef,Aft}, Prefix, Rs0) -> edit(Cs, P, {Bef,Aft}, meta_meta, Rs0); {csi, _} = Csi -> edit(Cs, P, {Bef,Aft}, Csi, Rs0); - meta_left_sq_bracket -> - edit(Cs, P, {Bef,Aft}, meta_left_sq_bracket, Rs0); - search_meta -> - edit(Cs, P, {Bef,Aft}, search_meta, Rs0); - search_meta_left_sq_bracket -> - edit(Cs, P, {Bef,Aft}, search_meta_left_sq_bracket, Rs0); - ctlx -> - edit(Cs, P, {Bef,Aft}, ctlx, Rs0); - new_line -> - {done, get_line(Bef, Aft ++ "\n"), Cs, - reverse(Rs0, [{move_rel,cp_len(Aft)},{put_chars,unicode,"\n"}])}; - redraw_line -> - Rs1 = erase(P, Bef, Aft, Rs0), - Rs = redraw(P, Bef, Aft, Rs1), - edit(Cs, P, {Bef,Aft}, none, Rs); + meta_left_sq_bracket -> + edit(Cs, P, {Bef,Aft}, meta_left_sq_bracket, Rs0); + search_meta -> + edit(Cs, P, {Bef,Aft}, search_meta, Rs0); + search_meta_left_sq_bracket -> + edit(Cs, P, {Bef,Aft}, search_meta_left_sq_bracket, Rs0); + ctlx -> + edit(Cs, P, {Bef,Aft}, ctlx, Rs0); + new_line -> + {done, get_line(Bef, Aft ++ "\n"), Cs, + reverse(Rs0, [{move_rel,cp_len(Aft)},{put_chars,unicode,"\n"}])}; + redraw_line -> + Rs1 = erase(P, Bef, Aft, Rs0), + Rs = redraw(P, Bef, Aft, Rs1), + edit(Cs, P, {Bef,Aft}, none, Rs); + clear -> + Rs = redraw(P, Bef, Aft, [clear | Rs0]), + edit(Cs, P, {Bef,Aft}, none, Rs); tab_expand -> {expand, Bef, Cs, - {line, P, {Bef, Aft}, none}, + {line, P, {Bef, Aft}, tab_expand}, reverse(Rs0)}; - -%% tab -> -%% %% Always redraw the line since expand/1 might have printed -%% %% possible expansions. -%% case expand(Bef) of -%% {yes,Str} -> -%% edit([redraw_line| -%% (Str ++ Cs)], P, {Bef,Aft}, none, Rs0); -%% no -> -%% %% don't beep if there's only whitespace before -%% %% us - user may have pasted in a lot of indented stuff. -%% case whitespace_only(Bef) of -%% false -> -%% edit([redraw_line|Cs], P, {Bef,Aft}, none, -%% [beep|Rs0]); -%% true -> -%% edit([redraw_line|Cs], P, {Bef,Aft}, none, [Rs0]) -%% end -%% end; - {undefined,C} -> - {undefined,{none,Prefix,C},Cs,{line,P,{Bef,Aft},none}, + tab_expand_full -> + {expand_full, Bef, Cs, + {line, P, {Bef, Aft}, tab_expand}, reverse(Rs0)}; - Op -> - case do_op(Op, Bef, Aft, Rs0) of - {blink,N,Line,Rs} -> - edit(Cs, P, Line, {blink,N}, Rs); - {Line, Rs, Mode} -> % allow custom modes from do_op - edit(Cs, P, Line, Mode, Rs); - {Line,Rs} -> - edit(Cs, P, Line, none, Rs) - end + {undefined,C} -> + {undefined,{none,Prefix,C},Cs,{line,P,{Bef,Aft},none}, + reverse(Rs0)}; + Op -> + case do_op(Op, Bef, Aft, Rs0) of + {blink,N,Line,Rs} -> + edit(Cs, P, Line, {blink,N}, Rs); + {Line, Rs, Mode} -> % allow custom modes from do_op + edit(Cs, P, Line, Mode, Rs); + {Line,Rs} -> + edit(Cs, P, Line, none, Rs) + end end; edit([], P, L, {blink,N}, Rs) -> {blink,{line,P,L,{blink,N}},reverse(Rs)}; @@ -183,7 +171,7 @@ prefix_arg(N) -> N. %% key_map(Char, Prefix) %% Map a character and a prefix to an action. -key_map(A, _) when is_atom(A) -> A; % so we can push keywords +key_map(A, _) when is_atom(A) -> A; % so we can push keywords key_map($\^A, none) -> beginning_of_line; key_map($\^B, none) -> backward_char; key_map($\^D, none) -> forward_delete_char; @@ -191,9 +179,11 @@ key_map($\^E, none) -> end_of_line; key_map($\^F, none) -> forward_char; key_map($\^H, none) -> backward_delete_char; key_map($\t, none) -> tab_expand; -key_map($\^L, none) -> redraw_line; -key_map($\n, none) -> new_line; +key_map($\t, tab_expand) -> tab_expand_full; +key_map(C, tab_expand) -> key_map(C, none); key_map($\^K, none) -> kill_line; +key_map($\^L, none) -> clear; +key_map($\n, none) -> new_line; key_map($\r, none) -> new_line; key_map($\^T, none) -> transpose_char; key_map($\^U, none) -> ctlu; @@ -214,11 +204,13 @@ key_map($], Prefix) when Prefix =/= meta, key_map($B, meta) -> backward_word; key_map($D, meta) -> kill_word; key_map($F, meta) -> forward_word; +key_map($L, meta) -> redraw_line; key_map($T, meta) -> transpose_word; key_map($Y, meta) -> yank_pop; key_map($b, meta) -> backward_word; key_map($d, meta) -> kill_word; key_map($f, meta) -> forward_word; +key_map($l, meta) -> redraw_line; key_map($t, meta) -> transpose_word; key_map($y, meta) -> yank_pop; key_map($O, meta) -> meta_o; @@ -320,9 +312,9 @@ do_op({search, backward_delete_char}, [_|Bef], Aft, Rs) -> {{Bef,NAft}, [{insert_chars, unicode, NAft}, {delete_chars,-Offset}|Rs], search}; -do_op({search, backward_delete_char}, [], _Aft, Rs) -> - Aft="': ", - {{[],Aft}, Rs, search}; +do_op({search, backward_delete_char}, [], Aft, Rs) -> + NAft="': ", + {{[],NAft}, [{insert_chars, unicode, NAft}, {delete_chars,-cp_len(Aft)}|Rs], search}; do_op({search, skip_up}, Bef, Aft, Rs) -> Offset= cp_len(Aft), NAft = "': ", @@ -621,148 +613,3 @@ cp_len(Str) -> cp_len([GC|R], Len) -> cp_len(R, Len + gc_len(GC)); cp_len([], Len) -> Len. - -%% %% expand(CurrentBefore) -> -%% %% {yes,Expansion} | no -%% %% Try to expand the word before as either a module name or a function -%% %% name. We can handle white space around the seperating ':' but the -%% %% function name must be on the same line. CurrentBefore is reversed -%% %% and over_word/3 reverses the characters it finds. In certain cases -%% %% possible expansions are printed. - -%% expand(Bef0) -> -%% {Bef1,Word,_} = over_word(Bef0, [], 0), -%% case over_white(Bef1, [], 0) of -%% {[$:|Bef2],_White,_Nwh} -> -%% {Bef3,_White1,_Nwh1} = over_white(Bef2, [], 0), -%% {_,Mod,_Nm} = over_word(Bef3, [], 0), -%% expand_function_name(Mod, Word); -%% {_,_,_} -> -%% expand_module_name(Word) -%% end. - -%% expand_module_name(Prefix) -> -%% match(Prefix, code:all_loaded(), ":"). - -%% expand_function_name(ModStr, FuncPrefix) -> -%% Mod = list_to_atom(ModStr), -%% case erlang:module_loaded(Mod) of -%% true -> -%% L = apply(Mod, module_info, []), -%% case lists:keyfind(exports, 1, L) of -%% {_, Exports} -> -%% match(FuncPrefix, Exports, "("); -%% _ -> -%% no -%% end; -%% false -> -%% no -%% end. - -%% match(Prefix, Alts, Extra) -> -%% Matches = match1(Prefix, Alts), -%% case longest_common_head([N || {N,_} <- Matches]) of -%% {partial, []} -> -%% print_matches(Matches), -%% no; -%% {partial, Str} -> -%% case lists:nthtail(length(Prefix), Str) of -%% [] -> -%% print_matches(Matches), -%% {yes, []}; -%% Remain -> -%% {yes, Remain} -%% end; -%% {complete, Str} -> -%% {yes, lists:nthtail(length(Prefix), Str) ++ Extra}; -%% no -> -%% no -%% end. - -%% %% Print the list of names L in multiple columns. -%% print_matches(L) -> -%% io:nl(), -%% col_print(lists:sort(L)), -%% ok. - -%% col_print([]) -> ok; -%% col_print(L) -> col_print(L, field_width(L), 0). - -%% col_print(X, Width, Len) when Width + Len > 79 -> -%% io:nl(), -%% col_print(X, Width, 0); -%% col_print([{H0,A}|T], Width, Len) -> -%% H = if -%% %% If the second element is an integer, we assume it's an -%% %% arity, and meant to be printed. -%% integer(A) -> -%% H0 ++ "/" ++ integer_to_list(A); -%% true -> -%% H0 -%% end, -%% io:format("~-*s",[Width,H]), -%% col_print(T, Width, Len+Width); -%% col_print([], _, _) -> -%% io:nl(). - -%% field_width([{H,_}|T]) -> field_width(T, length(H)). - -%% field_width([{H,_}|T], W) -> -%% case length(H) of -%% L when L > W -> field_width(T, L); -%% _ -> field_width(T, W) -%% end; -%% field_width([], W) when W < 40 -> -%% W + 4; -%% field_width([], _) -> -%% 40. - -%% match1(Prefix, Alts) -> -%% match1(Prefix, Alts, []). - -%% match1(Prefix, [{H,A}|T], L) -> -%% case prefix(Prefix, Str = atom_to_list(H)) of -%% true -> -%% match1(Prefix, T, [{Str,A}|L]); -%% false -> -%% match1(Prefix, T, L) -%% end; -%% match1(_, [], L) -> -%% L. - -%% longest_common_head([]) -> -%% no; -%% longest_common_head(LL) -> -%% longest_common_head(LL, []). - -%% longest_common_head([[]|_], L) -> -%% {partial, reverse(L)}; -%% longest_common_head(LL, L) -> -%% case same_head(LL) of -%% true -> -%% [[H|_]|_] = LL, -%% LL1 = all_tails(LL), -%% case all_nil(LL1) of -%% false -> -%% longest_common_head(LL1, [H|L]); -%% true -> -%% {complete, reverse([H|L])} -%% end; -%% false -> -%% {partial, reverse(L)} -%% end. - -%% same_head([[H|_]|T1]) -> same_head(H, T1). - -%% same_head(H, [[H|_]|T]) -> same_head(H, T); -%% same_head(_, []) -> true; -%% same_head(_, _) -> false. - -%% all_tails(LL) -> all_tails(LL, []). - -%% all_tails([[_|T]|T1], L) -> all_tails(T1, [T|L]); -%% all_tails([], L) -> L. - -%% all_nil([]) -> true; -%% all_nil([[] | Rest]) -> all_nil(Rest); -%% all_nil(_) -> false. diff --git a/lib/stdlib/src/edlin_context.erl b/lib/stdlib/src/edlin_context.erl new file mode 100644 index 0000000000..589bac4a02 --- /dev/null +++ b/lib/stdlib/src/edlin_context.erl @@ -0,0 +1,649 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2005-2023. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(edlin_context). +%% description +%% +-export([get_context/1, get_context/2, odd_quotes/2]). +%% The context record is a structure that helps with viewing a nested expression +%% Typically we do not know the types of a tuple or a map in isolation, but if +%% we make use of the type information available for a function or a record +%% then we are able to extract more information. +%% A nesting can be either be a tuple a list or a map, more types of nestings could +%% be supported in the future. But it is for these types that we have type information. +%% We do not need to care about nested records or nested functions since the type information +%% is available for the deepest nested one. +%% The context record stores the top most nesting so far, while the nesting field contains +%% a list of nestings we found, the last element being the deepest nesting. +-type nesting() :: {'tuple', [{atom(), string()}], {atom(), string()} | []} + | {'list', [{atom(), string()}], {atom(), string()} | []} + | {'map', [string()], string(), [{atom(), string()}], {atom(), string()} | []}. +-record(context,{ + arguments = [] :: [any()], + fields = [] :: [string()], %% The Field or Keys in this nesting + parameter_count = 0 :: non_neg_integer(), %% Number of parameters in this nesting + current_field = [] :: string(), %% field of a record or key of a map field=<tab> %% should be the last element in this context + nestings = [] :: [nesting()]}). +%% get_context - basically get the context to be able to deduce how we should complete the word +%% If the word is empty, then we do not want to complete with anything if we just closed +%% a bracket, ended a quote (user has to enter , or . themselves) +%% but maybe we can help to add ',',},] depending on context in the future +-spec get_context(Line) -> Context when + Line :: string(), %% The whole line in reverse excluding Word + Mod :: string(), + Fun :: string(), + Args :: list(any()), + Unfinished :: any(), + Binding :: string(), %% map variable + Keys :: [string()], %% list of keys in the map MapBind + Record :: string(), + Fields :: [string()], %% list of keys in the record + FieldToComplete :: string(), + Nesting :: [nesting()] | [], + Context :: {string} %% cursor is inside a string "_ + | {binding} %% cursor is inside a Binding statement + | {term} %% cursor is at a free position, where any value is expected + | {term, Args, Unfinished} + | {fun_} %% cursor is in a fun statement (either a NewVar, '(' or mfa() is expected) + | {fun_, Mod} %% cursor is in a fun mod: statement (mfa) + | {fun_, Mod, Fun} %% cursor is in a fun mod:fun statement + | {new_fun, Unfinished} + | {function} + | {function, Mod, Fun, Args, Unfinished, Nesting} + | {map, Binding, Keys} + | {map_or_record} + | {record} + | {record, Record, Fields, FieldToComplete, Args, Unfinished, Nesting} + | {error, integer()}. +get_context(Line) -> + {Bef0, Word} = edlin_expand:over_word(Line), + case {{Bef0, Word}, odd_quotes($", Bef0)} of + {_, true} -> {string}; + {{[$#|_], []}, _} -> {map_or_record}; + {{_Bef1, Word}, _} -> + case is_binding(Word) of + true -> {binding}; + false -> get_context(Bef0, Word) + end + end. +get_context(">-" ++ _, L) when is_list(L) -> {term}; +get_context([$?|_], _) -> + {macro}; +get_context(Bef0, Word) when is_list(Word) -> + get_context(lists:reverse(Word) ++ Bef0, #context{}); +get_context([], #context{arguments = Args, parameter_count = Count, nestings = Nestings} = _CR) -> + case Count+1 == length(Args) of + true -> {term, lists:droplast(Args), lists:last(Args)}; + _ -> + %% Nestings will not end up as an argument + case Nestings of + [] -> case Count of + 0 when length(Args) > 0 -> {term, lists:droplast(Args), lists:last(Args)}; + _ -> {term, Args, []} + end; + [{list, Args1, Arg}] -> {term, Args1, Arg}; + [{tuple, Args1, Arg}] -> {term, Args1, Arg}; + [{map, _, _, Args1, Arg}] -> {term, Args1, Arg} + end + end; +get_context([$(|Bef], CR) -> + %% We have an unclosed opening parenthesis + %% Check if we have a function call + %% We can deduce the minimum arity based on how many terms we trimmed + %% We can check the type of the following Term and suggest those in special cases + %% shell_default and erlang are imported, make sure we can do expansion for those + {Bef1, Fun} = edlin_expand:over_word(Bef), + case Fun of + [] -> {term}; % parenthesis + _ -> + {_, Mod} = over_module(Bef1, Fun), + case Mod of + "shell" -> {term}; + "shell_default" -> {term}; + _ -> + case CR#context.parameter_count+1 == length(CR#context.arguments) of + true -> + %% N arguments N-1 commas, this means that we have an argument + %% still being worked on. + {function, Mod, Fun, lists:droplast(CR#context.arguments), + lists:last(CR#context.arguments),CR#context.nestings}; + _ -> + {function, Mod, Fun, CR#context.arguments, + [], CR#context.nestings} + end + end + end; +get_context([${|Bef], #context{ fields=Fields, + current_field=FieldToComplete, + arguments = Arguments, + parameter_count = Count, + nestings=Nestings}) -> + {Args, Unfinished} = case Count+1 == length(Arguments) of + true -> {lists:droplast(Arguments), lists:last(Arguments)}; + _ -> {Arguments, []} + end, + case edlin_expand:over_word(Bef) of + {[$#|Bef1], []} -> %% Map + {Bef2, Map} = edlin_expand:over_word(Bef1), + case Map of + [] -> get_context(Bef2, #context{ + %% We finished a nesting lets reset and read the next nesting + nestings = [{'map', Fields, FieldToComplete, Args, Unfinished}|Nestings]}); + _ -> {map, Map, Fields} + end; + {_, []} -> + get_context(Bef, #context{ + %% We finished a nesting lets reset and read the next nesting + nestings = [{'tuple', Args, Unfinished}|Nestings]}); + {[$#|_Bef3], Record} -> %% Record + {record, Record, Fields, FieldToComplete, Args, Unfinished, Nestings} + end; +get_context([$[|Bef1], #context{arguments = Arguments, parameter_count = Count, nestings=Nestings}) -> + {Args, Unfinished} = case Count+1 == length(Arguments) of + true -> {lists:droplast(Arguments), lists:last(Arguments)}; + _ -> {Arguments, []} + end, + get_context(Bef1, #context{ + %% We finished a nesting lets reset and read the next nesting + nestings = [{'list', Args, Unfinished}|Nestings]}); +get_context([$,|Bef1], #context{parameter_count=Count}=CR) -> + get_context(Bef1, CR#context{ + parameter_count = Count+1}); +get_context([$>,$=|Bef1], #context{ parameter_count=Count, + fields=Fields}=CR) -> + {Bef2, Field}=edlin_expand:over_word(Bef1), + case Count of + 0 -> %% If count is 0, then we know its a value we may want to complete. + get_context(Bef2, CR#context{fields = [Field|Fields], + current_field = Field}); + _ -> get_context(Bef2, CR#context{fields = [Field|Fields]}) + end; +get_context([$=,$:|Bef1], #context{ parameter_count=Count, + fields=Fields}=CR) -> + {Bef2, Field}=edlin_expand:over_word(Bef1), + case Count of + 0 -> %% If count is 0, then we know its a value we may want to complete. + get_context(Bef2, CR#context{fields = [Field|Fields], + current_field = Field}); + _ -> get_context(Bef2, CR#context{fields = [Field|Fields]}) + end; +get_context([$=|Bef1], #context{ + parameter_count=Count, + fields=Fields}=CR) -> + {Bef2, Field}=edlin_expand:over_word(Bef1), + % if we are here, its always going to be + case Count of + 0 -> %%[$=|_], + get_context(Bef2, CR#context{fields = [Field|Fields], + current_field = Field}); + _ -> get_context(Bef2, CR#context{fields = [Field|Fields]}) + end; +get_context([$.|Bef2], CR) -> + Arguments = CR#context.arguments, + Count = CR#context.parameter_count, + {Args, Unfinished} = case Count+1 == length(Arguments) of + true -> {lists:droplast(Arguments), lists:last(Arguments)}; + _ -> {Arguments, []} + end, + case edlin_expand:over_word(Bef2) of + {[$#|_Bef3], Record} -> %% Record + {record, Record, CR#context.fields, CR#context.current_field, Args, Unfinished, CR#context.nestings}; + _ -> {'end'} + end; +get_context([$:|Bef2], _) -> + %% look backwards to see if its a fun + {Bef3, Mod} = edlin_expand:over_word(Bef2), + case edlin_expand:over_word(Bef3) of + {_, "fun"} -> {fun_, Mod}; + _ -> {function} + end; +get_context([$/|Bef1], _) -> + {Bef2, Fun} = edlin_expand:over_word(Bef1), + {_, Mod} = over_module(Bef2, Fun), + {fun_, Mod, Fun}; +get_context([$>,$-|_Bef2], #context{arguments = Args} = CR) -> + %% Inside a function + case CR#context.parameter_count+1 == length(Args) of + true -> + {term, lists:droplast(Args), lists:last(Args)}; + _ -> {term, Args, []} + end; +get_context("nehw " ++ _Bef2, #context{arguments = Args} = CR) -> + %% Inside a guard + case CR#context.parameter_count+1 == length(Args) of + true -> + {term, lists:droplast(Args), lists:last(Args)}; + _ -> {term, Args, []} + end; +get_context([$\ |Bef],CR) -> get_context(Bef, CR); %% matching space here simplifies the other clauses +get_context(Bef0, #context{arguments=Args, parameter_count=Count} = CR) -> + case over_to_opening(Bef0) of + {_,[]} -> {term}; + {error, _}=E -> E; + {record} -> {record}; + {fun_} -> {fun_}; + {new_fun, _}=F -> F; + {Bef1, {fun_, Str}=Arg} -> + case Count of + 0 -> + [_, Mod, Fun| _] = string:tokens(Str, " :/"), + {fun_, Mod, Fun}; + _ -> get_context(Bef1, CR#context{arguments=[Arg|Args]}) + end; + {Bef1, Arg} -> get_context(Bef1, CR#context{arguments=[Arg|Args]}) + end. + +read_operator(Bef) -> + read_operator1(Bef). + +operator_string() -> "-=><:+*/|&^~". +read_operator1([$\ |Bef]) -> read_operator1(Bef); +read_operator1("mer " ++ Bef) -> {Bef, "rem"}; +read_operator1("osladna " ++ Bef) -> {Bef, "andalso"}; +read_operator1("dna " ++ Bef) -> {Bef, "and"}; +read_operator1("eslero " ++ Bef) -> {Bef, "orelse"}; +read_operator1("ro " ++ Bef) -> {Bef, "or"}; +read_operator1([$>,$>,$>|Bef1]) -> {[$>,$>|Bef1], [$>]}; %% comparison with a binary or typo +read_operator1([$>,$>|Bef1]) -> {[$>|Bef1], [$>]}; %% comparison with a pid, or binary +read_operator1([$>,$-, C|Bef1]=Bef) -> + case lists:member(C, operator_string()) of + true -> {Bef1, [C,$-,$>]}; + false -> {Bef, []} + end; +read_operator1([$>,$=, C|Bef1]=Bef) -> + case lists:member(C, operator_string()) of + true -> {Bef1, [C,$=,$>]}; + false -> {Bef, []} + end; +read_operator1([$=,$:, C|Bef1]=Bef) -> + case lists:member(C, operator_string()) of + true -> {Bef1, [C,$:,$=]}; + false -> {Bef, []} + end; +read_operator1([$:|_]=Bef) -> {Bef, []}; %% this operator does not count +read_operator1([Op1,Op2,Op3|Bef])-> + case {lists:member(Op1, operator_string()), + lists:member(Op2, operator_string()), + lists:member(Op3, operator_string())} of + {true, true, true} -> {Bef, [Op3, Op2, Op1]}; + {true, true, false} -> {[Op3|Bef], [Op2, Op1]}; + {true, false, _} -> {[Op2,Op3|Bef], [Op1]}; + _ -> {[Op1,Op2,Op3|Bef], []} + end; +read_operator1([Op1, Op2]) -> + case {lists:member(Op1, operator_string()), + lists:member(Op2, operator_string())} of + {true, true} -> {[], [Op2, Op1]}; + {true, false} -> {[Op2], [Op1]}; + _ -> {[Op1,Op2], []} + end; +read_operator1([Op1]) -> + case lists:member(Op1, operator_string()) of + true -> {[], [Op1]}; + _ -> {[Op1], []} + end; +read_operator1(Bef) -> {Bef, []}. + +read_opening_char("nehw "++Bef) -> + {Bef, "when"}; +read_opening_char([OC|Bef]) when OC =:= $(; OC =:= $[; OC =:= ${; OC =:= $,; OC =:= $. -> + {Bef, [OC]}; +read_opening_char([$>,$-|_]=Bef) -> + case read_operator(Bef) of + {_, []} -> {Bef, "->"}; + _ -> {Bef, []} + end; +read_opening_char([$\ |Bef]) -> read_opening_char(Bef); +read_opening_char(Bef) -> {Bef, []}. + +over_to_opening(Bef) -> try + over_to_opening1(Bef,#{args => []}) + catch + throw:E -> E + end. +over_to_opening1([], #{'args' := Args}) -> + over_to_opening_return([], Args); +over_to_opening1(Bef, Acc = #{args := Args}) -> + case edlin_expand:over_word(Bef) of + {_, []} -> %% removed spaces + case read_opening_char(Bef) of + {Bef1, []} -> %% not an opening + case extract_argument2(Bef1) of + {stop} -> over_to_opening_return(Bef1, Args); + {Bef2, []} -> over_to_opening_return(Bef2, Args); + {Bef2, Arg} -> over_to_opening1(Bef2, Acc#{args => [Arg | Args]}) + end; + {_Bef1, _Opening} -> over_to_opening_return(Bef, Args) + end; + _ -> case extract_argument2(Bef) of + {stop} -> over_to_opening_return(Bef, Args); + {Bef2, []} -> over_to_opening_return(Bef2, Args); + {Bef2, Arg} -> over_to_opening1(Bef2, Acc#{args => [Arg | Args]}) + end + end. +over_to_opening_return(Bef, Args) -> + case Args of + [] -> {Bef, []}; + [Arg] -> {Bef, Arg}; + [{operator, "-"}, {integer, I}] -> {Bef, {integer, "-" ++ I}}; + [{operator, "-"}, {float, F}] -> {Bef, {float, "-" ++ F}}; + [{atom, "fun"}, {atom, _}] -> throw({fun_}); + _ -> + case look_for_non_operator_separator(Args) of + true -> {Bef, {operation, lists:flatten(lists:join(" ", lists:map(fun({_, Arg}) -> Arg end, Args)))}}; + false -> {error, length(Bef)} + end + end. +look_for_non_operator_separator([{string, _},{string, _}=A|Args]) -> + look_for_non_operator_separator([A|Args]); +look_for_non_operator_separator([{operator, _}, {operator, _}|_]) -> false; + +look_for_non_operator_separator([_, {operator, _}=B|Args]) -> + look_for_non_operator_separator([B|Args]); +look_for_non_operator_separator([{operator, _}, B|Args]) -> + look_for_non_operator_separator([B|Args]); +look_for_non_operator_separator([_]) -> true; +look_for_non_operator_separator(_) -> false. + +over_map_record_or_tuple(Bef0) -> + case over_to_opening_paren($},Bef0) of + {_, []} -> %% no matching { + throw({error, length(Bef0)}); + {Bef3, Clause} -> + {Bef4, MaybeRecord} = edlin_expand:over_word(Bef3), + case MaybeRecord of + [] -> case Bef4 of + [$#|Bef5] -> %% Map + {Bef6, _Var} = edlin_expand:over_word(Bef5), + {Bef6, {map, _Var++"#"++Clause}}; + _ -> %% Tuple + {Bef4, {tuple, Clause}} + end; + _Record -> %% Record + [$#|Bef5] = Bef4, + {Bef6, _Var} = edlin_expand:over_word(Bef5), + {Bef6, {record, _Var++"#"++_Record++Clause}} + end + end. +over_pid_port_or_ref(Bef2) -> + %% Extracts argument or part of an operation + %% Consume Pid, Ref, FunRef or Binary + case over_to_opening_paren($>,Bef2) of + {_, []} -> %% no matching <, maybe a '>' operator + throw({soft_error, length(Bef2)}); + {Bef3, Clause} -> + case Bef3 of + "feR#" ++ Bef4 -> + {Bef4, {ref, "#Ref" ++ Clause}}; + "nuF#" ++ Bef4 -> + {Bef4, {'funref', "#Fun" ++ Clause}}; + "troP#" ++ Bef4 -> + {Bef4, {port, "#Port" ++ Clause}}; + _ -> case edlin_expand:over_word(Bef3) of + {Bef3, []} -> + case Bef2 of + [$>|_] -> %% binary + {Bef3, {binary, Clause}}; + _ -> %% pid + %% match <Num.Num.Num> + {Bef3, {pid, Clause}} + end; + _ -> + throw({error, length(Bef3)}) + end + end + end. +over_list(Bef2) -> + case over_to_opening_paren($],Bef2) of + {_, []} -> %% no matching [ + throw({error, length(Bef2)}); + {Bef3, Clause} -> + {Bef3, {list, Clause}} + end. +over_parenthesis_or_call(Bef2) -> + case over_to_opening_paren($),Bef2) of + {_, []} -> %% no matching ( + throw({error, length(Bef2)}); + {Bef3, Clause} -> + {Bef4, Fun} = edlin_expand:over_word(Bef3), + {Bef5, ModFun} = case Bef4 of + [$:|Bef41] -> + {Bef42, Mod} = edlin_expand:over_word(Bef41), + {Bef42, Mod++[$:|Fun]}; + _ -> {Bef4, Fun} + end, + case ModFun of + [] -> {Bef5, {parenthesis, Clause}}; + "fun" -> throw({new_fun, Clause}); + _ -> {Bef5, {call, ModFun++Clause}} + end + end. +over_keyword_or_fun(Bef1) -> + case over_keyword_expression(Bef1) of + {Bef2, KeywordExpression} -> {Bef2, {keyword, KeywordExpression ++ " end"}}; + _ -> throw({error, length(Bef1)}) + end. +extract_argument2([$>|Bef0]=Bef)-> + case read_operator(Bef) of + {[$>|_]=Bef1, ">"=Operator} -> + try over_pid_port_or_ref(Bef0) + catch + %% not a pid, port, ref or binary + throw:{error, _}=E -> throw(E); + throw:{soft_error, _Col} -> {Bef1, {operator, Operator}} + end; + {Bef1, ">"=Operator} -> + try over_pid_port_or_ref(Bef1) + catch + %% not a pid, port or ref + throw:{error, _}=E -> throw(E); + throw:{soft_error, _Col} -> {Bef1, {operator, Operator}} + end; + {_Bef1, []} -> {stop}; + {Bef1, Operator} -> {Bef1, {operator, Operator}} + end; +extract_argument2(Bef0) -> + case read_operator(Bef0) of + {[$}|Bef1], []} -> over_map_record_or_tuple(Bef1); + {[$)|Bef1], []} -> over_parenthesis_or_call(Bef1); + {[$]|Bef1], []} -> over_list(Bef1); + {[$"|Bef2], []} -> {Bef3, _Quote} = over_to_opening_quote($", Bef2), + {Bef3, {string, _Quote}}; + {"dne "++Bef1, []} -> over_keyword_or_fun(Bef1); + {[$=,$:|_], []} -> {stop}; + {[$:|_], []} -> {stop}; + {"nehw" ++ _Bef1,[]} -> {stop}; + {_, []} -> extract_argument(Bef0); + {Bef1, Operator} -> + {Bef1, {operator, Operator}} + end. + +extract_argument(Bef0) -> + %% TODO: We probably need to be able to extract Terms with operators... + case edlin_expand:over_word(Bef0) of + {_Bef1, []} -> + case read_char(_Bef1) of + {_, []} -> {_Bef1, []}; + {Bef2, Char} -> {Bef2, {char, Char}} + end; + {Bef2, Var} -> + try list_to_integer(Var) of + _ -> %% there is an integer + case over_fun_function(Bef0) of + {Bef3, "fun " ++ _ModFunArr} -> {Bef3, {fun_, "fun "++_ModFunArr}}; + _ -> case over_number(Bef0) of + {Bef3, []} -> {Bef3, []}; %% how to deal with operators + {Bef3, Number} -> {Bef3, Number} + + end + end + catch + _:_ -> + case is_binding(Var) of + true -> {Bef2,{var, Var}}; + false -> case Bef2 of + [$#|_] -> throw({record}); + _ -> {Bef2, {atom, Var}} + end + end + end + end. +over_number(Bef) -> + case edlin_expand:over_word(Bef) of + {_, []} -> {Bef, []}; + {Bef2, Var} -> + try list_to_integer(Var) of + _ -> + {Bef6, {NumberType, Number}}=Res = case edlin_expand:over_word(Bef2) of + {[$.|Bef3],[]} -> %% float + {Bef4, Integer} = edlin_expand:over_word(Bef3), + {Bef4, {float, Integer ++ "." ++ Var}}; + {[$#|Bef3],[]} -> %% integer base + {Bef4, Base} = edlin_expand:over_word(Bef3), + {Bef4, {integer, Base ++ "#" ++ Var}}; + _ -> + {Bef2, {integer, Var}} + %% otherwise its an operation that can be very complicated we should read everything up to the closest CC + %% and return an {operation, Clause} + end, + case edlin_expand:over_word(Bef6) of + {[$-|Bef5], []} -> + case read_opening_char(Bef5) of + {_, []} -> Res; + _ -> {Bef5, {NumberType, "-" ++Number}} + end; + _ -> Res + end + catch + _:_ -> {Bef, []} + end + end. +read_char([C,$$|Line]) -> + {Line, [$$,C]}; +read_char([$$|Line]) -> + {Line, "$ "}; +read_char(Line) -> + {Line, []}. + +over_fun_function(Bef) -> + over_fun_function(Bef, []). +over_fun_function(Bef, Acc) -> + case edlin_expand:over_word(Bef) of + {[$/|Bef1], Arity} -> over_fun_function(Bef1, [$/|Arity]++Acc); + {[$:|Bef1], Fun} -> over_fun_function(Bef1, [$:|Fun]++Acc); + {" nuf"++Bef1, ModOrFun} -> over_fun_function(Bef1, "fun "++ModOrFun ++ Acc); + _ -> {Bef,Acc} + end. + +%% Extracts everything within the quote +over_to_opening_quote(Q, Bef) when Q == $'; Q == $" -> + over_to_opening_quote([Q], Bef, [Q]); +over_to_opening_quote(_, Bef) -> {Bef, []}. +over_to_opening_quote([], Bef, Word) -> {Bef, Word}; +over_to_opening_quote([Q|Stack], [Q|Bef], Word) -> + over_to_opening_quote(Stack, Bef, [Q| Word]); +over_to_opening_quote([Q|Stack], [Q,EC|Bef], Word) when EC=:=$\\; EC=:=$$ -> + over_to_opening_quote([Q|Stack], Bef, [EC,Q| Word]); +over_to_opening_quote([Stack], [C|Bef], Word) -> + over_to_opening_quote([Stack], Bef, [C| Word]); +over_to_opening_quote(_,_,Word) -> {lists:reverse(Word), []}. + +matching_paren($(,$)) -> true; +matching_paren($[,$]) -> true; +matching_paren(${,$}) -> true; +matching_paren($<,$>) -> true; +matching_paren(_,_) -> false. + +%% Extracts everything within the brackets +%% Recursively extracts nested bracket expressions. +over_to_opening_paren(CC, Bef) when CC == $); CC == $]; + CC == $}; CC == $> -> + over_to_opening_paren([CC], Bef, [CC]); +over_to_opening_paren(_, Bef) -> {Bef, []}. %% Not a closing parenthesis +over_to_opening_paren([], Bef, Word) -> {Bef, Word}; +over_to_opening_paren(_, [], Word) -> {lists:reverse(Word), []}; %% Not a closing parenthesis +over_to_opening_paren([CC|Stack], [CC,$$|Bef], Word) -> + over_to_opening_paren([CC|Stack], Bef, [$$,CC|Word]); +over_to_opening_paren([CC|Stack], [OC|Bef], Word) when OC==$(; OC==$[; OC==${; OC==$< -> + case matching_paren(OC, CC) of + true -> over_to_opening_paren(Stack, Bef, [OC|Word]); + false -> over_to_opening_paren([CC|Stack], Bef, [OC|Word]) + end; +over_to_opening_paren([CC|Stack], [CC|Bef], Word) -> %% Nested parenthesis of same type + over_to_opening_paren([CC,CC|Stack], Bef, [CC|Word]); +over_to_opening_paren(Stack, [Q,NEC|Bef], Word) when Q == $"; Q == $', NEC /= $$, NEC /= $\\ -> + %% Consume the whole quoted text, it may contain parenthesis which + %% would have confused us. + {Bef1, QuotedWord} = over_to_opening_quote(Q, Bef), + over_to_opening_paren(Stack, Bef1, QuotedWord ++ Word); +over_to_opening_paren(CC, [C|Bef], Word) -> over_to_opening_paren(CC, Bef, [C|Word]). + +%% Extract a whole keyword expression +%% Keyword<code>end +%% Function expects a string of erlang code in reverse, and extracts everything +%% including a keyword being one of if, fun, case, maybe, receiver (need to add all here) +%% Recursively extracts nested keyword expressions +%% Note: In the future we could autocomplete case expressions by looking at the +%% return type of the expression. +over_keyword_expression(Bef) -> + over_keyword_expression(Bef, []). +over_keyword_expression("dne"++Bef, Expr)-> + %% Nested expression + {Bef1, KWE}=over_keyword_expression(Bef), + over_keyword_expression(Bef1, KWE++"end"++Expr); +over_keyword_expression("fi"++Bef, Expr) -> {Bef, "if" ++ Expr}; +over_keyword_expression("nuf"++Bef, Expr) -> {Bef, "fun" ++ Expr}; +over_keyword_expression("yrt"++Bef, Expr) -> {Bef, "try" ++ Expr}; +over_keyword_expression("esac"++Bef, Expr) -> {Bef, "case" ++ Expr}; +over_keyword_expression("hctac"++Bef, Expr) -> + case over_keyword_expression(Bef, []) of + {Bef1, "try" ++ Expr1} -> {Bef1, "try" ++ Expr1 ++ "catch" ++ Expr}; + _ -> {Bef, "catch" ++ Expr} + end; +over_keyword_expression("nigeb"++Bef, Expr) -> {Bef, "begin" ++ Expr}; +over_keyword_expression("ebyam"++Bef, Expr) -> {Bef, "maybe" ++ Expr}; +over_keyword_expression("eviecer"++Bef, Expr) -> {Bef, "receive" ++ Expr}; +over_keyword_expression([], _) -> {no, [], []}; +over_keyword_expression([C|Bef], Expr) -> over_keyword_expression(Bef, [C|Expr]). + +odd_quotes(Q, [Q,C|Line], Acc) when C == $\\; C == $$ -> + odd_quotes(Q, Line, Acc); +odd_quotes(Q, [Q|Line], Acc) -> + odd_quotes(Q, Line, Acc+1); +odd_quotes(Q, [_|Line], Acc) -> + odd_quotes(Q, Line, Acc); +odd_quotes(_, [], Acc) -> Acc band 1 == 1. +odd_quotes(Q, Line) -> + odd_quotes(Q, Line, 0). + +over_module(Bef, Fun)-> + case edlin_expand:over_word(Bef) of + {[$:|Bef1], _} -> + edlin_expand:over_word(Bef1); + {[], _} -> {Bef, edlin_expand:shell_default_or_bif(Fun)}; + _ -> {Bef, edlin_expand:bif(Fun)} + end. + +%% Check that the given string starts with a capital letter, or an underscore +%% followed by an alphanumeric grapheme. +is_binding(Word) -> + Normalized = unicode:characters_to_nfc_list(Word), + nomatch =/= re:run(Normalized, + "^[_[:upper:]][[:alpha:]]*$", + [unicode, ucp]). diff --git a/lib/stdlib/src/edlin_expand.erl b/lib/stdlib/src/edlin_expand.erl index bc3de13750..9823534e9b 100644 --- a/lib/stdlib/src/edlin_expand.erl +++ b/lib/stdlib/src/edlin_expand.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2021. All Rights Reserved. +%% Copyright Ericsson AB 2005-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -18,47 +18,784 @@ %% %CopyrightEnd% %% -module(edlin_expand). +%% a default expand function for edlin, expanding modules, functions +%% filepaths, variable binding, record names, function parameter values, +%% record fields and map keys and record field values. +-include_lib("kernel/include/eep48.hrl"). +-export([expand/1, expand/2, expand/3, format_matches/2, number_matches/1, get_exports/1, + shell_default_or_bif/1, bif/1, over_word/1]). +-export([is_type/3, match_arguments1/3]). +-record(shell_state,{ + bindings = [], + records = [], + functions = [] + }). -%% a default expand function for edlin, expanding modules and functions +-spec expand(Bef0) -> {Res, Completion, Matches} when + Bef0 :: string(), %% a line of erlang expressions in reverse + Res :: 'yes' | 'no', + Completion :: string(), + Matches :: [Element] | [Section], + Element :: {string(), [ElementOption]}, + ElementOption :: {ending, string()}, + Section :: #{title:=string(), elems:=Matches, options:=SectionOption}, + SectionOption :: {highlight_all} %% highlight the whole title + | {highlight, string()} %% highlight this part of the title + | {highlight_param, integer()} %% highlight this parameter + | {hide, title} %% hide the title + | {hide, result} %% hide the results + | {separator, string()}. %% specify another separator between title and result +expand(Bef0) -> + expand(Bef0, [{legacy_output, true}]). + +-spec expand(Bef0, Opts) -> {Res, Completion, Matches} when + Bef0 :: string(), %% a line of erlang expressions in reverse + Opts :: [Option], + Option :: {legacy_output, boolean()}, + Res :: 'yes' | 'no', + Completion :: string(), + Matches :: [Element] | [Section], + Element :: {string(), [ElementOption]}, + ElementOption :: {ending, string()}, + Section :: #{title:=string(), elems:=Matches, options:=SectionOption}, + SectionOption :: {highlight_all} %% highlight the whole title + | {highlight, string()} %% highlight this part of the title + | {highlight_param, integer()} %% highlight this parameter + | {hide, title} %% hide the title + | {hide, result} %% hide the results + | {separator, string()}. %% specify another separator between title and result +expand(Bef0, Opts) -> + ShellState = try + shell:get_state() + catch + _:_ -> + %% Running on a shell that does not support get_state() + #shell_state{bindings=[],records=[],functions=[]} + end, + expand(Bef0, Opts, ShellState). + +%% Only used for testing +expand(Bef0, Opts, #shell_state{bindings = Bs, records = RT, functions = FT}) -> + LegacyOutput = proplists:get_value(legacy_output, Opts, false), + {_Bef1, Word} = over_word(Bef0), + {Res, Expansion, Matches} = case edlin_context:get_context(Bef0) of + + {string} -> expand_string(Bef0); + + {binding} -> expand_binding(Word, Bs); + + {term} -> expand_module_function(Bef0, FT); + {term, _, {_, Unfinished}} -> expand_module_function(lists:reverse(Unfinished), FT); + {error, _Column} -> + {no, [], []}; + {function} -> expand_module_function(Bef0, FT); + {fun_} -> expand_module_function(Bef0, FT); + + {fun_, Mod} -> expand_function_name(Mod, Word, "/", FT); + + %% Complete with arity in a 'fun mod:fun/' expression + {fun_, Mod, Fun} -> + Arities = [integer_to_list(A) || A <- get_arities(Mod, Fun)], + match(Word, Arities, ""); + {new_fun, _ArgsString} -> {no, [], []}; + %% Suggest type of function parameter + %% Complete an unfinished list, tuple or map using type of function parameter + {function, Mod, Fun, Args, Unfinished, Nesting} -> + Mod2 = case Mod of + "user_defined" -> "shell_default"; + _ -> Mod + end, + FunExpansion = expand_function_type(Mod2, Fun, Args, Unfinished, Nesting, FT), + case Word of + [] -> FunExpansion; + _ -> + ModuleOrBifs = expand_helper(FT, module, Word, ":"), + Functions = case Args =/= [] andalso lists:last(Args) of + {atom, MaybeMod} -> expand_function_name(MaybeMod, Word, "", FT); + _ -> {no, [], []} + end, + fold_results([FunExpansion] ++ ModuleOrBifs ++ [Functions]) + end; + + %% Complete an unfinished key or suggest valid keys of a map binding + {map, Binding, Keys} -> expand_map(Word, Bs, Binding, Keys); + + {map_or_record} -> + {[$#|Bef2], _} = over_word(Bef0), + {_, Var} = over_word(Bef2), + case Bs of + [] -> expand_record(Word, RT); + _ -> + case proplists:get_value(list_to_atom(Var), Bs) of + undefined -> + expand_record(Word, RT); + Map when is_map(Map) -> {yes, "{", []}; + RecordTuple when is_tuple(RecordTuple), tuple_size(RecordTuple) > 0 -> + Atom = erlang:element(1, RecordTuple), + case (is_atom(Atom) andalso lists:keysearch(Atom, 1, RT)) of + {value, {Atom, _}} -> match(Word, [Atom], "{"); + _ -> {no, [], []} + end; + _ -> {no, [], []} + end + end; + + {record} -> expand_record(Word, RT); + + {record, Record, Fields, FieldToComplete, Args, Unfinished, Nestings} -> + RecordExpansion = expand_record_fields(FieldToComplete, Unfinished, Record, Fields, RT, Args, Nestings, FT), + case Word of + [] -> RecordExpansion; + _ -> + ModuleOrBifs = expand_helper(FT, module,Word,":"), + fold_results([RecordExpansion] ++ ModuleOrBifs) + end; + _ -> {no, [], []} + + end, + Matches1 = case {Res,number_matches(Matches)} of + {yes, 1} -> []; + _ -> Matches + end, + case LegacyOutput of + true -> {Res, Expansion, to_legacy_format(Matches1)}; + false -> {Res, Expansion, Matches1} + end. +expand_map(_, [], _, _) -> + {no, [], []}; +expand_map(Word, Bs, Binding, Keys) -> + case proplists:get_value(list_to_atom(Binding), Bs) of + Map when is_map(Map) -> + K1 = sets:from_list(maps:keys(Map)), + K2 = sets:subtract(K1, sets:from_list([list_to_atom(K) || K <- Keys])), + match(Word, sets:to_list(K2), "=>"); + _ -> {no, [], []} + end. + +over_word(Bef) -> + {Bef1,_,_} = over_white(Bef, [], 0), + {Bef2, Word, _} = edlin:over_word(Bef1, [], 0), + {Bef2, Word}. + + +expand_binding(Prefix, Bindings) -> + Alts = [strip_quotes(K) || {K,_} <- Bindings], + case match(Prefix, Alts, "") of + {_Res,_Expansion,[]}=M -> M; + {Res, Expansion, Matches} -> {Res,Expansion,[#{title=>"bindings", elems=>Matches, options=>[highlight_all]}]} + end. + +expand_record(Prefix, RT) -> + Alts = [Name || {Name, _} <- RT], + case match(Prefix, Alts, "{") of + {_Res,_Expansion,[]}=M -> M; + {Res, Expansion, Matches} -> {Res,Expansion,[#{title=>"records", elems=>Matches, options=>[highlight_all]}]} + end. + +expand_record_fields(FieldToComplete, Word, Record, Fields, RT, _Args, Nestings, FT) -> + Record2 = list_to_atom(Record), + FieldSet2 = sets:from_list([list_to_atom(F) || F <- Fields]), + FieldToComplete2 = list_to_atom(FieldToComplete), + Word1 = case Word of + {_, Word2} -> Word2; + [] -> [] + end, + case [RecordSpec || {Record3, RecordSpec} <- RT, Record2 =:= Record3] of + [RecordType|_] -> + case sets:is_element(FieldToComplete2, FieldSet2) of + true -> + expand_record_field_content(FieldToComplete2, RecordType, Word1, Nestings, FT); + false -> + expand_record_field_name(Record2, FieldSet2, RecordType, Word1) + end; + _ -> + {no, [], []} + end. + +expand_record_field_name(Record, Fields, RecordType, Word) -> + RecordFieldsList = extract_record_fields(Record, RecordType), + RecordFieldsSet = sets:from_list(RecordFieldsList), + RecordFields = sets:subtract(RecordFieldsSet, Fields), + Alts = sets:to_list(RecordFields), + case match(Word, Alts, "=") of + {_Res,_Expansion,[]}=M -> M; + {Res, Expansion, Matches} -> {Res,Expansion,[#{title=>"fields", elems=>Matches, options=>[highlight_all]}]} + end. --export([expand/1, format_matches/1]). +expand_record_field_content(Field, + {attribute, _, record, + {_Record, FieldTypes}}, Word, Nestings, FT) -> + FieldTypesFiltered = [Type1 || {typed_record_field, {record_field, _, {_,_, F}}, Type1} <- FieldTypes, F == Field] ++ + [Type1 || {typed_record_field, {record_field, _, {_,_, F}, _}, Type1} <- FieldTypes, F == Field], + case FieldTypesFiltered of + [] -> {no, [], []}; + [Type] -> + T = edlin_type_suggestion:type_tree(erlang, Type, Nestings, FT), + Types = edlin_type_suggestion:get_types([], T, Nestings), + case Nestings of + [] -> + Atoms = edlin_type_suggestion:get_atoms([], T, Nestings), + case {Word, match(Word, Atoms, ", ")} of + {[],{_Res,_Expansion,_}} -> {_Res, _Expansion, [#{title=>"types", elems=>Types, options=>[{hide, title}]}]}; + {_,{_Res,_Expansion,[]}=M} -> M; + {_,{Res,Expansion,Matches}} -> {Res, Expansion, [#{title=>"matches", elems=>Matches, options=>[highlight_all]}]} + end; + _ -> + expand_nesting_content(T, [], Nestings, #{title=>"types", elems=>Types, options=>[{hide, title}]}) + end + end. + +%% Check that the actual type on previous arguments +%% matches with the expected types +%% Since we are not doing any evaluations at this point we +%% don't know if a parenthesis, keyword, var, call or fun returns +%% a value with the wrong type. +match_arguments({function, {{parameters, Ps}, _}, Cs}, As) -> + match_arguments1(Ps, Cs, As); +match_arguments({{parameters, Ps}, _}, As) -> + match_arguments1(Ps, [], As). +match_arguments1(_,_,[]) -> true; +%% Just assume that it will evaluate to the correct type. +match_arguments1([_|Ps], Cs, [{parenthesis, _}|As]) -> + match_arguments1(Ps, Cs, As); +match_arguments1([_|Ps], Cs, [{operation, _}|As]) -> + match_arguments1(Ps, Cs, As); +match_arguments1([_|Ps], Cs, [{keyword, _}|As]) -> + match_arguments1(Ps, Cs, As); +match_arguments1([_|Ps], Cs, [{var, _}|As]) -> + match_arguments1(Ps, Cs, As); +match_arguments1([_|Ps], Cs, [{call, _}|As]) -> + match_arguments1(Ps, Cs, As); +match_arguments1([_|Ps], Cs, [{fun_, _}|As]) -> + match_arguments1(Ps, Cs, As); +match_arguments1([P|Ps], Cs, [{atom, [$'|_]=String}|As]) -> + case edlin_context:odd_quotes($', lists:reverse(String)) of + true -> false; % we know that the atom is unfinished, and thus cannot match any valid atom + _ -> case is_type(P, Cs, String) of + true -> match_arguments1(Ps, Cs, As); + false -> false + end + end; +match_arguments1([P|Ps], Cs, [{_, String}|As]) -> + case is_type(P, Cs, String) of + true -> match_arguments1(Ps, Cs, As); + false -> false + end. + +is_type(Type, Cs, String) -> + {ok, A, _} = erl_scan:string(String++"."), + Types = [T || T <- edlin_type_suggestion:get_types(Cs, Type, [], [no_print]) ], + try + {ok, Term} = erl_parse:parse_term(A), + case Term of + Atom when is_atom(Atom) -> + Atoms = edlin_type_suggestion:get_atoms(Cs, Type, []), + lists:member(to_list(Atom), Atoms) orelse + lists:member(atom_to_list(Atom), Atoms) orelse + find_type(Types, [atom, node, module, 'fun']); + Tuple when is_tuple(Tuple) -> find_type(Types, [tuple]); + Map when is_map(Map) -> find_type(Types, [map]); + Binary when is_binary(Binary) -> find_type(Types, [binary]); + Float when is_float(Float) -> find_type(Types, [float]); + Integer when is_integer(Integer) -> check_integer_type(Types, Integer); + List when is_list(List), length(List) > 0 -> + find_type(Types, [list, string, nonempty_list,maybe_improper_list, nonempty_improper_list]); + List when is_list(List) -> find_type(Types, [list, string, maybe_improper_list]) + end + catch + _:_ -> + %% Types not possible to deduce with erl_parse + % If string contains variables, erl_parse:parse_term will fail, but we + % consider them valid sooo.. lets replace them with the atom var + B = [(fun({var, Anno, _}) -> {atom, Anno, var}; (Token) -> Token end)(X) || X <- A], + try + {ok, Term2} = erl_parse:parse_term(B), + case Term2 of + Tuple2 when is_tuple(Tuple2) -> find_type(Types, [tuple]); + Map2 when is_map(Map2) -> find_type(Types, [map]); + Binary2 when is_binary(Binary2) -> find_type(Types, [binary]); + List2 when is_list(List2), length(List2) > 0 -> + find_type(Types, [list, string, nonempty_list,maybe_improper_list, nonempty_improper_list]); + List2 when is_list(List2) -> find_type(Types, [list, string, maybe_improper_list]) + end + catch + _:_ -> + case A of + [{'#',_},{var,_,'Port'},{'<',_},{float,_,_},{'>',_},{dot,_}] -> find_type(Types, [port]); + [{'#',_},{var,_,'Ref'},{'<',_},{float,_,_},{'.',_},{float,_,_},{'>',_},{dot,_}] -> find_type(Types, [reference]); + [{'fun',_},{'(',_} | _] -> find_type(Types, [parameters, function, 'fun']); + [{'#',_},{var,_,'Fun'},{'<',_},{atom,_,erl_eval},{'.',_},{float,_,_},{'>',_}] -> find_type(Types, [parameters, function, 'fun']); + [{'<', _}, {float, _, _}, {'.', _}, {integer, _, _}, {'>', _}, {dot, _}] -> find_type(Types, [pid]); + [{'#', _}, {atom, _, RecordName},{'{', _}| _] -> find_type(Types, [{record, RecordName}]); + _ -> false + end + end + end. + +find_type([],_) -> false; +find_type([any|_], _) -> true; % If we find any then every type is valid +find_type([{type, any, []}|_], _) -> true; +find_type([{{parameters, _},_}|Types], ValidTypes) -> + case lists:member(parameters, ValidTypes) of + true -> true; + false -> find_type(Types, ValidTypes) + end; +find_type([{record, _}=Type|Types], ValidTypes) -> + case lists:member(Type, ValidTypes) of + true -> true; + false -> find_type(Types, ValidTypes) + end; +find_type([{Type, _}|Types], ValidTypes) -> + case lists:member(Type, ValidTypes) of + true -> true; + false -> find_type(Types, ValidTypes) + end; +find_type([{type, Type, _}|Types], ValidTypes) -> + case lists:member(Type, ValidTypes) of + true -> true; + false -> find_type(Types, ValidTypes) + end; +find_type([{type, Type, _, any}|Types], ValidTypes) -> + case lists:member(Type, ValidTypes) of + true -> true; + false -> find_type(Types, ValidTypes) + end; +find_type([_|Types], ValidTypes) -> find_type(Types, ValidTypes). + +in_range(_, []) -> false; +in_range(Integer, [{type, range, [{integer, Start}, {integer, End}]}|_]) when Start =< Integer, Integer =< End -> true; +in_range(Integer, [_|Types]) -> in_range(Integer, Types). + +check_integer_type(Types, Integer) when Integer == 0 -> find_type(Types, [integer, non_neg_integer, arity]) orelse in_range(Integer, Types); +check_integer_type(Types, Integer) when Integer < 0 -> find_type(Types, [integer, neg_integer]) orelse in_range(Integer, Types); +check_integer_type(Types, Integer) when Integer > 0 -> find_type(Types, [integer, non_neg_integer, pos_integer]) orelse in_range(Integer, Types). --import(lists, [reverse/1, prefix/2]). +add_to_last_nesting(Term, Nesting) -> + Last = lists:last(Nesting), + List = lists:droplast(Nesting), + case Last of + {tuple, Args, U} -> + List ++ [{tuple, Args ++ [Term], U}]; + {list, Args, U} -> + List ++ [{list, Args ++ [Term], U}]; + {map, F, Fs, Args, U} -> + List ++ [{map, F, Fs, Args ++ [Term], U}] + end. + +close_nesting(Nesting) -> + Last = lists:last(Nesting), + case Last of + {tuple, _Args, _} -> + "}"; + {list, _Args, _} -> + "]"; + {map, _F, _Fs, _Args, _} -> + "}" + end. +expand_function_parameter_type(Mod, MFA, FunType, Args, Unfinished, Nestings, FT) -> + TypeTree = edlin_type_suggestion:type_tree(Mod, FunType, Nestings, FT), + + {Parameters, Constraints1} = case TypeTree of + {function, {{parameters, Parameters1},_}, Constraints} -> + {Parameters1, Constraints}; + {{parameters, Parameters1},_}=_F -> + {Parameters1, []} + end, + case match_arguments(TypeTree, Args) of + false -> {no, [], []}; + true when Parameters == [] -> {yes, ")", [#{title=>MFA, elems=>[")"], options=>[]}]}; + true -> + Parameter = lists:nth(length(Args)+1, Parameters), + {T, _Name} = case Parameter of + Atom when is_atom(Atom) -> {Atom, atom_to_list(Atom)}; + {var, Name1}=T1 -> {T1, atom_to_list(Name1)}; + {ann_type, {var, Name1}, T1} -> {T1, atom_to_list(Name1)}; + T1 -> {T1, edlin_type_suggestion:print_type(T1, [], [{first_only, true}])} + end, + Ts = edlin_type_suggestion:get_types(Constraints1, T, Nestings), + Types = case Ts of + [] -> []; + _ -> + SectionTypes = [S || #{}=S <- Ts], + Types1 = case [E || {_, _}=E<-Ts] of + [] -> SectionTypes; + Elems -> + case SectionTypes of + [] -> Elems; + ST -> [#{title=>"simple types", elems=>Elems, options=>[{hide, title}]}|ST] + end + end, + + [#{title=>"types", elems=>(Types1), options=>[{hide, title}]}] + end, + case Nestings of + [] -> %% Expand function type + case Unfinished of + [] -> + case T of + Atom1 when is_atom(Atom1) -> + CC = case length(Args)+1 < length(Parameters) of + true -> ", "; + false ->")" + end, + {Res, Expansion, Matches} = match([], [Atom1], CC), + case Matches of + [] -> {no, [], []}; + _ -> {Res, Expansion, [#{title=>MFA, elems=>[], options=>[{highlight_param, length(Args)+1}]}]} + end; + _ when Types == [] -> + {no, [], []}; + _ -> + {no, [], [#{title=>MFA, elems=>Types, options=>[{highlight_param, length(Args)+1}]}]} + end; + {_, Word} when is_atom(T) -> + CC = case length(Args)+1 < length(Parameters) of + true -> ", "; + false ->")" + end, + {Res, Expansion, Matches} = match(Word, [T], CC), + case Matches of + [] -> {no, [], []}; + _ -> {Res, Expansion, [#{title=>MFA, elems=>[], options=>[{highlight_param, length(Args)+1}]}]} + end; + {_, Word} -> + {Res, Expansion, Matches} = begin + CC = case length(Args)+1 < length(Parameters) of + true -> ", "; + false ->")" + end, + Atoms1 = edlin_type_suggestion:get_atoms(Constraints1, T, Nestings), + {Res1, Expansion1, Matches1} = match(Word, Atoms1, CC), + case Matches1 of + [] -> + case match_arguments(TypeTree, Args ++ [Unfinished]) of + false -> {Res1, Expansion1, Matches1}; + true -> + {yes, CC, [{CC, []}]} + end; + _ -> + {Res1, Expansion1, Matches1} + end + end, + Match1 = case Matches of + [] -> []; + _ -> Atoms = [#{title=>"atoms", elems=>Matches, options=>[{hide, title}]}], + [#{title=>MFA, elems=>Atoms, options=>[{highlight_param, length(Args)+1}]}] + end, + {Res, Expansion,Match1} + end; + _ -> %% Expand last nesting types + expand_nesting_content(T, Constraints1, Nestings, #{title=>MFA, elems=>Types, options=>[{highlight_param, length(Args)+1}]}) + end + end. +expand_nesting_content(T, Constraints, Nestings, Section) -> + {NestingType, UnfinishedNestingArg, NestingArgs} = case lists:last(Nestings) of + {tuple, NestingArgs1, Unfinished1} -> {tuple, Unfinished1, NestingArgs1}; + {list, NestingArgs1, Unfinished1} -> {list, Unfinished1, NestingArgs1}; + {map, _, _, NestingArgs1, Unfinished1} -> {map, Unfinished1, NestingArgs1} + end, + %% in the case of + %% erlang:system_info({allocator, ) + %% we have a tuple nesting with an atom + %% this should give us "allocator" in the nestingsargs, and empty unfinished part + %% but we also know that we have a nesting, if we expect something other than a tuple, we shouldnt print that function + %% lets call it NestingType + %% now when that is fixed, how do we filter {allocator_sizes, ...} and others + Types = [Ts || Ts <- edlin_type_suggestion:get_types(Constraints, T, lists:droplast(Nestings), [no_print]) ], + case UnfinishedNestingArg of + [] -> + case find_type(Types, [NestingType]) of + true -> + %% if we know had a tuple, {allocator_sizes, } will be allowed + %% probably get_arity will return none + Nestings2 = add_to_last_nesting({var, "Var"}, Nestings), + NestingArities = edlin_type_suggestion:get_arity(Constraints, T, Nestings2), + + fold_results([begin + case NestingArity of + none -> {no, [], []}; + _ -> {no, [], [Section]} + end + end || NestingArity <- NestingArities]); + false -> {no, [], []} + end; + {_, Word} -> + Atoms1 = edlin_type_suggestion:get_atoms(Constraints, T, Nestings), + {Res1, Expansion1, Matches1} = match(Word, Atoms1, ""), + {Res, Expansion, Matches} = case Matches1 of + [] -> + Nestings2 = add_to_last_nesting(UnfinishedNestingArg, Nestings), + NestingArities = edlin_type_suggestion:get_arity(Constraints, T, Nestings2), + fold_results([begin + case NestingArity of + none -> {no, [], []}; + _ when NestingType =:= tuple -> + CC = case length(NestingArgs)+1 < NestingArity of + true -> ", "; + false -> close_nesting(Nestings) + end, + {yes, CC, [{CC, []}]}; + _ when NestingType =:= list -> + {no, [], [{", ", []}, {"]", []}]}; + _ when NestingType =:= map -> + {no, [], [{", ",[]},{"}", []}]}; + _ -> + {no, [], []} + end + end || NestingArity <- NestingArities]); + [{Word2,_}] -> + Nestings2 = add_to_last_nesting({atom, Word2}, Nestings), + NestingArities = edlin_type_suggestion:get_arity(Constraints, T, Nestings2), + fold_results([begin + case NestingArity of + none -> {no, [], []}; + _ when NestingType =:= tuple -> + CC = case length(NestingArgs)+1 < NestingArity of + true -> ", "; + false -> close_nesting(Nestings) + end, + {yes, Expansion1++CC, [{Word2, [{ending, CC}]}]}; + _ -> + {Res1, Expansion1, Matches1} + end + end || NestingArity <- NestingArities]); + _ -> {Res1, Expansion1, Matches1} + end, + Match1 = case Matches of + [] -> []; + _ -> Atoms = [#{title=>"atoms", elems=>Matches, options=>[{hide, title}]}], + [Section#{elems:=Atoms}] + end, + {Res, Expansion, Match1} + end. + +extract_record_fields(Record, {attribute,_,record,{Record, Fields}})-> + [X || X <- [extract_record_field(F) || F <- Fields], X /= []]; +extract_record_fields(_, _)-> error. +extract_record_field({typed_record_field, {_, _,{atom, _, Field}},_})-> + Field; +extract_record_field({typed_record_field, {_, _,{atom, _, Field}, _},_})-> + Field; +extract_record_field({record_field, _,{atom, _, Field},_})-> + Field; +extract_record_field({record_field, _,{atom, _, Field}})-> + Field; +extract_record_field(_) -> []. + +fold_results([]) -> {no, [], []}; +fold_results([R|Results]) -> + lists:foldl(fun fold_completion_result/2, R, Results). + +fold_completion_result({yes, Cmp1, Matches1}, {yes, Cmp2, Matches2}) -> + {_, Cmp} = longest_common_head([Cmp1,Cmp2]), + case Cmp of + [] -> {no, [], ordsets:union([Matches1,Matches2])}; + _ -> {yes, Cmp, ordsets:union([Matches1,Matches2])} + end; +fold_completion_result({yes, Cmp, Matches}, {no, [], []}) -> + {yes, Cmp, Matches}; +fold_completion_result({no, [], []},{yes, Cmp, Matches}) -> + {yes, Cmp, Matches}; +fold_completion_result({_, _, Matches1}, {_, [], Matches2}) -> + {no, [], ordsets:union([Matches1,Matches2])}; +fold_completion_result(A, B) -> + fold_completion_result(B,A). + +expand_function_type(ModStr, FunStr, Args, Unfinished, Nestings, FT) -> + Mod = list_to_atom(ModStr), + Fun = list_to_atom(FunStr), + MinArity = if Unfinished =:= [], length(Args) =:= 0 -> 0; + true -> length(Args)+1 + end, + case [A || A <- get_arities(ModStr, FunStr, FT), A >= MinArity] of + [] -> {no, [], []}; + Arities -> + {Res, Expansion, Matches} = fold_results([begin + FunTypes = edlin_type_suggestion:get_function_type(Mod, Fun, Arity, FT), + case FunTypes of + [] -> MFA = print_function_head(ModStr, FunStr, Arity), + case Unfinished of + [] -> {no, [], [#{title=>MFA, elems=>[], options=>[]}]}; + _ -> {no, [], []} + end; + _ -> + fold_results([begin + MFA = print_function_head(ModStr, FunStr, FunType, FT), + expand_function_parameter_type(Mod, MFA, FunType, Args, Unfinished, Nestings, FT) + end || FunType <- FunTypes]) + end + end || Arity <- Arities]), + case Matches of + [] -> {Res, Expansion, Matches}; + _ -> {Res, Expansion, [#{title=>"typespecs", elems=>Matches, options=>[highlight_all]}]} + end + end. -%% expand(CurrentBefore) -> -%% {yes, Expansion, Matches} | {no, Matches} +%% Behaves like zsh +%% filters all files starting with . unless Word starts with . +%% outputs / on end of folders +expand_filepath(PathPrefix, Word) -> + Path = case PathPrefix of + [$/|_] -> PathPrefix; + _ -> + {ok, Cwd} = file:get_cwd(), + Cwd ++ "/" ++ PathPrefix + end, + ShowHidden = case Word of + "." ++ _ -> true; + _ -> false + end, + Entries = case file:list_dir(Path) of + {ok, E} -> lists:map( + fun(X)-> + case filelib:is_dir(Path ++ "/" ++ X) of + true -> X ++ "/"; + false -> X + end + end, [".."|E]); + _ -> [] + end, + EntriesFiltered = [File || File <- Entries, + case File of + [$.|_] -> ShowHidden; + _ -> true + end], + case match(Word, EntriesFiltered, []) of + {yes, Cmp, [Match]} -> + case filelib:is_dir(Path ++ "/" ++ Word ++ Cmp) of + true -> {yes, Cmp, [Match]}; + false -> {yes, Cmp ++ "\"", [Match]} + end; + X -> X + end. + +shell(Fun) -> + case shell:local_func(list_to_atom(Fun)) of + true -> "shell"; + false -> "user_defined" + end. + +shell_default_or_bif(Fun) -> + case lists:member(list_to_atom(Fun), [E || {E,_}<-get_exports(shell_default)]) of + true -> "shell_default"; + _ -> bif(Fun) + end. +bif(Fun) -> + case lists:member(list_to_atom(Fun), [E || {E,A}<-get_exports(erlang), erl_internal:bif(E,A)]) of + true -> "erlang"; + _ -> shell(Fun) + end. + +expand_string(Bef0) -> + case over_filepath(Bef0, []) of + {_, Filepath} -> + {Path, File} = split_at_last_slash(Filepath), + expand_filepath(Path, File); + _ -> {no, [], []} + end. +%% Extract a whole filepath +%% Stops as soon as we hit a double quote (") +%% and returns everything it found before stopping. +%% assumes the string is not a filepath if it contains unescaped spaces +over_filepath([],_) -> none; +over_filepath([$", $\\|Bef1], Filepath) -> over_filepath(Bef1, [$" | Filepath]); +over_filepath([$"|Bef1], Filepath) -> {Bef1, Filepath}; +over_filepath([$\ ,$\\|Bef1], Filepath) -> over_filepath(Bef1, [$\ |Filepath]); +over_filepath([$\ |_], _) -> none; +over_filepath([C|Bef1], Filepath) -> + over_filepath(Bef1, [C|Filepath]). +split_at_last_slash(Filepath) -> + {File, Path} = lists:splitwith(fun(X)->X/=$/ end, lists:reverse(Filepath)), + {lists:reverse(Path), lists:reverse(File)}. + +print_function_head(ModStr, FunStr, Arity) -> + lists:flatten(ModStr ++ ":" ++ FunStr ++ "/" ++ integer_to_list(Arity)). +print_function_head(ModStr, FunStr, FunType, FT) -> + lists:flatten(print_function_head_from_type(ModStr, FunStr, FunType, FT)). + +print_function_head1(Mod, Fun, Par, _Ret) -> + Mod++":"++Fun++"("++lists:join(", ", + [case P of + Atom when is_atom(Atom) -> atom_to_list(Atom); + {var, V} -> atom_to_list(V); + {ann_type, {var, V}, _T} -> atom_to_list(V); + T -> edlin_type_suggestion:print_type(T, [], [{first_only, true}]) + end || {_N,P} <- lists:enumerate(Par)])++")". +print_function_head_from_type(Mod, Fun, FunType, FT) -> + case edlin_type_suggestion:type_tree(list_to_atom(Mod), FunType, [], FT) of + {function, {{parameters, Parameters},{return, Return}}, _} -> + print_function_head1(Mod, Fun, Parameters, Return); + {{parameters, Parameters},{return, Return}} -> + print_function_head1(Mod, Fun, Parameters, Return) + end. + +%% expand_module_function(CurrentBefore, FT) -> {yes, Expansion, Matches} | {no, [], Matches} %% Try to expand the word before as either a module name or a function %% name. We can handle white space around the seperating ':' but the %% function name must be on the same line. CurrentBefore is reversed %% and over_word/3 reverses the characters it finds. In certain cases -%% possible expansions are printed. +%% possible expansions are printed.ยดยดยด %% -%% The function also handles expansion with "h(" for module and functions. -expand(Bef0) -> +%% The function also handles expansion with "h(" and "ht("" for module and functions. +expand_module_function(Bef0, FT) -> {Bef1,Word,_} = edlin:over_word(Bef0, [], 0), case over_white(Bef1, [], 0) of {[$,|Bef2],_White,_Nwh} -> - {Bef3,_White1,_Nwh1} = over_white(Bef2, [], 0), - {Bef4,Mod,_Nm} = edlin:over_word(Bef3, [], 0), + {Bef3,_White1,_Nwh1} = over_white(Bef2, [], 0), + {Bef4,Mod,_Nm} = edlin:over_word(Bef3, [], 0), case expand_function(Bef4) of help -> - expand_function_name(Mod, Word, ","); + expand_function_name(Mod, Word, ", ", FT); + help_type -> + expand_type_name(Mod, Word, ", "); _ -> - expand_module_name(Word, ",") + fold_results(expand_helper(FT, module, Word, ":")) end; {[$:|Bef2],_White,_Nwh} -> - {Bef3,_White1,_Nwh1} = over_white(Bef2, [], 0), - {_,Mod,_Nm} = edlin:over_word(Bef3, [], 0), - expand_function_name(Mod, Word, "("); - {_,_,_} -> + {Bef3,_White1,_Nwh1} = over_white(Bef2, [], 0), + {_,Mod,_Nm} = edlin:over_word(Bef3, [], 0), + expand_function_name(Mod, Word, "(", FT); + {[CC, N_Esc|_], _White, _Nwh} when (CC =:= $] orelse CC =:= $) orelse CC =:= $> orelse CC =:= $} + orelse CC =:= $" orelse CC =:= $'), + N_Esc =/= $$, N_Esc =/= $- -> + {no, [], []}; + {[], _, _} -> + case Word of + [] -> {no, [], []}; %fold_results([expand_shell_default(Word), expand_user_defined_functions(FT, Word)]); + _ -> fold_results(expand_helper(FT, all, Word, ":")) + end; + {_,_,_} -> + case Word of + [] -> {no, [], []}; + _ -> + TypeOfExpand = expand_function(Bef1), CompleteChar - = case expand_function(Bef1) of - help -> ","; + = case TypeOfExpand of + help -> ", "; + help_type -> ", "; _ -> ":" end, - expand_module_name(Word, CompleteChar) + fold_results(expand_helper(FT, TypeOfExpand, Word, CompleteChar)) + end end. - +expand_keyword(Word) -> + Keywords = ["begin", "case", "of", "receive", "after", "maybe", "try", "catch", "throw", "if", "fun", "when", "end"], + {Res, Expansion, Matches} = match(Word, Keywords, ""), + case Matches of + [] -> {no, [], []}; + [{Word, _}] -> {no, [], []}; %% exact match + _ -> {Res,Expansion,[#{title=>"keywords", elems=>Matches, options=>[highlight_all]}]} + end. +expand_helper(_, help, Word, CompleteChar) -> + [expand_module_name(Word, CompleteChar)]; +expand_helper(_, help_type, Word, CompleteChar) -> + [expand_module_name(Word, CompleteChar)]; +expand_helper(FT, all, Word, CompleteChar) -> + [expand_module_name(Word, CompleteChar), expand_bifs(Word), expand_shell_default(Word), + expand_user_defined_functions(FT, Word), expand_keyword(Word)]; +expand_helper(FT, _, Word, CompleteChar) -> + [expand_module_name(Word, CompleteChar), expand_bifs(Word), + expand_user_defined_functions(FT, Word), expand_keyword(Word)]. expand_function("("++Str) -> case edlin:over_word(Str, [], 0) of {_,"h",_} -> @@ -71,68 +808,157 @@ expand_function("("++Str) -> expand_function(_) -> module. +expand_bifs(Prefix) -> + Alts = [EA || {E,A}=EA <- get_exports(erlang), erl_internal:bif(E,A)], + CC = "(", + case match(Prefix, Alts, CC) of + {_Res,_Expansion,[]}=M -> M; + {Res,Expansion, Matches} -> {Res,Expansion,[#{title=>"bifs", elems=>Matches, options=>[highlight_all]}]} + end. + +expand_shell_default(Prefix) -> + Alts = get_exports(shell_default) ++ shell:local_func(), + CC = "(", + case match(Prefix, Alts, CC) of + {_Res,_Expansion,[]}=M -> M; + {Res,Expansion, Matches} -> {Res,Expansion,[#{title=>"commands",elems=>Matches, options=>[highlight_all]}]} + end. + +expand_user_defined_functions(FT, Prefix) -> + Alts = [{Name, Arity}||{{function, {_, Name, Arity}}, _} <- FT], + CC = "(", + case match(Prefix, Alts, CC) of + {_Res,_Expansion,[]}=M -> M; + {Res,Expansion, Matches} -> {Res,Expansion,[#{title=>"user_defined", elems=>Matches, options=>[highlight_all]}]} + end. + expand_module_name("",_) -> {no, [], []}; -expand_module_name(Prefix,CompleteChar) -> - match(Prefix, [{list_to_atom(M),P} || {M,P,_} <- code:all_available()], CompleteChar). +expand_module_name(Prefix,CC) -> + Alts = [{list_to_atom(M),""} || {M,_,_} <- code:all_available()], + case match(Prefix, Alts, CC) of + {_Res,_Expansion,[]}=M -> M; + {Res,Expansion, Matches} -> {Res,Expansion,[#{title=>"modules", elems=>Matches, options=>[highlight_all]}]} + end. -expand_function_name(ModStr, FuncPrefix, CompleteChar) -> +get_arities("shell_default"=ModStr, FuncStr, FT) -> + case [A || {{function, {_, Fun, A}}, _} <- FT, Fun =:= list_to_atom(FuncStr)] of + [] -> get_arities(ModStr, FuncStr); + Arities -> Arities + end; +get_arities(ModStr, FuncStr, _) -> + get_arities(ModStr, FuncStr). +get_arities(ModStr, FuncStr) -> case to_atom(ModStr) of - {ok, Mod} -> - Exports = - case erlang:module_loaded(Mod) of - true -> - Mod:module_info(exports); - false -> - case beam_lib:chunks(code:which(Mod), [exports]) of - {ok, {Mod, [{exports,E}]}} -> - E; - _ -> - {no, [], []} - end - end, - case Exports of + {ok, Mod} -> + Exports = get_exports(Mod), + lists:sort( + [A || {H, A} <- Exports, string:equal(FuncStr, flat_write(H))]); + error -> + {no, [], []} + end. + +get_exports(Mod) -> + case erlang:module_loaded(Mod) of + true -> + Mod:module_info(exports); + false -> + case beam_lib:chunks(code:which(Mod), [exports]) of + {ok, {Mod, [{exports,E}]}} -> + E; + _ -> + [] + end + end. + +expand_function_name(ModStr, FuncPrefix, CompleteChar, FT) -> + case to_atom(ModStr) of + {ok, Mod} -> + Extra = case Mod of + shell_default -> [{Name, Arity}||{{function, {_, Name, Arity}}, _} <- FT]; + _ -> [] + end, + Exports = get_exports(Mod) ++ Extra, + {Res, Expansion, Matches}=Result = match(FuncPrefix, Exports, CompleteChar), + case Matches of + [] -> Result; + _ -> {Res, Expansion, [#{title=>"functions", elems=>Matches, options=>[highlight_all]}]} + end; + error -> + {no, [], []} + end. + +get_module_types(Mod) -> + case code:get_doc(Mod, #{sources => [debug_info]}) of + {ok, #docs_v1{ docs = Docs } } -> + [{T, A} || {{type, T, A},_Anno,_Sig,_Doc,_Meta} <- Docs]; + _ -> {no, [], []} + end. + +expand_type_name(ModStr, TypePrefix, CompleteChar) -> + case to_atom(ModStr) of + {ok, Mod} -> + case get_module_types(Mod) of {no, [], []} -> {no, [], []}; - Exports -> - match(FuncPrefix, Exports, CompleteChar) + Types -> + {Res, Expansion, Matches}=Result = match(TypePrefix, Types, CompleteChar), + case Matches of + [] -> Result; + _ -> {Res, Expansion, [#{title=>"types", elems=>Matches, options=>[highlight_all]}]} + end end; - error -> - {no, [], []} + error -> + {no, [], []} end. -%% if it's a quoted atom, atom_to_list/1 will do the wrong thing. to_atom(Str) -> case erl_scan:string(Str) of - {ok, [{atom,_,A}], _} -> - {ok, A}; - _ -> - error + {ok, [{atom,_,A}], _} -> + {ok, A}; + _ -> + error end. +to_list(Atom) -> + io_lib:write_atom(Atom). + +strip_quotes(Atom) -> + [C || C<-atom_to_list(Atom), C/=$']. + +match_preprocess_alt({_,_}=Alt) -> Alt; +match_preprocess_alt(X) -> {X, ""}. + match(Prefix, Alts, Extra0) -> + Alts2 = [match_preprocess_alt(A) || A <- Alts], Len = string:length(Prefix), Matches = lists:sort( - [{S, A} || {H, A} <- Alts, - prefix(Prefix, S=flat_write(H))]), + [{S, A} || {H, A} <- Alts2, + lists:prefix(Prefix, S=flat_write(H))]), + Matches2 = lists:usort( + case Extra0 of + [] -> [{S,[]} || {S,_} <- Matches]; + _ -> [{S,[{ending, Extra0}]} || {S,_} <- Matches] + end), case longest_common_head([N || {N, _} <- Matches]) of - {partial, []} -> - {no, [], Matches}; % format_matches(Matches)}; - {partial, Str} -> + {partial, []} -> + {no, [], Matches2}; + {partial, Str} -> case string:slice(Str, Len) of - [] -> - {yes, [], Matches}; % format_matches(Matches)}; - Remain -> - {yes, Remain, []} - end; - {complete, Str} -> - Extra = case {Extra0,Matches} of - {"(",[{Str,0}]} -> "()"; - {_,_} -> Extra0 - end, - {yes, string:slice(Str, Len) ++ Extra, []}; - no -> - {no, [], []} + [] -> + {yes, [], Matches2}; + Remain -> + {yes, Remain, Matches2} + end; + {complete, Str} -> + Extra = case {Extra0,Matches} of + {"/",[{Str,N}]} when is_integer(N) -> "/"++integer_to_list(N); + {"(",[{Str,0}]} -> "()"; + {_,_} -> Extra0 + end, + {yes, string:slice(Str, Len) ++ Extra, ordsets:from_list(Matches2)}; + no -> + {no, [], []} end. flat_write(T) when is_atom(T) -> @@ -140,79 +966,198 @@ flat_write(T) when is_atom(T) -> flat_write(S) -> S. -%% Return the list of names L in multiple columns. -format_matches(L) -> - {S1, Dots} = format_col(lists:sort(L), []), - S = case Dots of - true -> - {_, Prefix} = longest_common_head(vals(L)), - PrefixLen = string:length(Prefix), - case PrefixLen =< 3 of - true -> S1; % Do not replace the prefix with "...". - false -> - LeadingDotsL = leading_dots(L, PrefixLen), - {S2, _} = format_col(lists:sort(LeadingDotsL), []), - S2 - end; - false -> S1 +special_sort1([C|A], B) when C == ${ ; C == $. ; C == $# -> + special_sort1(A, B); +special_sort1(A, [C|B]) when C == ${ ; C == $. ; C == $# -> + special_sort1(A,B); +special_sort1(A,B) -> + string:lowercase(A) =< string:lowercase(B). +special_sort(#{title:=A}, #{title:=B}) -> + special_sort1(A,B); +%% Sections and elemts should not be in the same list +special_sort(#{}, {}) -> + error; +special_sort({}, #{}) -> + error; +special_sort({A,_},{B,_}) -> + special_sort1(A,B); +special_sort(A,B) -> + special_sort1(A,B). + +to_legacy_format([]) -> []; +to_legacy_format([#{title:=Title}|Rest]) when Title =:= "commands"; Title =:= "bifs" -> + to_legacy_format(Rest); +to_legacy_format([#{title:=Title, elems:=Elems}|Rest]) + when Title =:= "modules"; Title =:= "functions"; Title =:= "bindings"; + Title =:= "user_defined", Title =:= "records"; Title =:= "fields"; + Title =:= "types"; Title =:= "atoms"; Title =:= "matches"; + Title =:= "keywords"; Title =:= "typespecs" -> + Elems1 = to_legacy_format(Elems), + Elems1 ++ to_legacy_format(Rest); +to_legacy_format([#{title:=Title, elems:=_Elems}|Rest]) -> + [Title] ++ to_legacy_format(Rest); +to_legacy_format([{Val, _}|Rest]) -> + [{Val, ""}] ++ to_legacy_format(Rest). + +format_matches([], _LineWidth) -> []; +format_matches([#{}|_]=FF, LineWidth) -> + %% Group function head that have the exact same Type suggestion + Groups = maps:groups_from_list( + fun(#{title:=Title, elems:=T, options:=Opts}) -> + Separator = proplists:get_value(separator, Opts, "\n"), + case lists:last(string:split(Title++Separator, "\n", all)) of + [] -> format_section_matches(T, LineWidth); + Chars -> %% we have chars that compete with the results on the first line + Len = length(Chars), + format_section_matches(T, LineWidth, Len) + end + end, + fun(F) -> + format_title(F, LineWidth) + end, FF), + S = lists:flatten( + [lists:join("", F)++Matches || + {Matches, F}<-lists:sort(fun({_,A},{_,B}) -> A =< B end, maps:to_list(Groups))]), + lists:flatten(string:trim(S, trailing)++"\n"); +format_matches(Elems, LineWidth) -> + S = format_section_matches1(Elems, LineWidth, 0), + lists:flatten(string:trim(S, trailing)++"\n"). +format_title(#{title:=MFA, options:=Options}, _LineWidth) -> + case proplists:get_value(hide, Options) of + title -> ""; + _ -> + Separator = proplists:get_value(separator, Options, "\n"), + HighlightAll = proplists:is_defined(highlight_all, Options), + case HighlightAll of + true -> "\033[;1;4m"++MFA++"\033[0m"++Separator; + _ -> + HighlightParam = proplists:get_value(highlight_param, Options, false), + + MFA2 = case HighlightParam of + false -> MFA; + _ -> + PreviousParams = HighlightParam -1, + TuplePattern = "(?:\\{[^\\}]+\\})", + AtomVarPattern = "(?:\\w+)", + TypePattern="(?:(?:"++AtomVarPattern++":)?(?:"++AtomVarPattern++"\\(\\))(?:\\s[><=]+\\s\\d+)?)", + SimplePatterns = "(?:"++TuplePattern++"|"++TypePattern++"|"++AtomVarPattern++")", + UnionPattern = "(?:"++SimplePatterns++"(?:\\s\\|\\s"++SimplePatterns++")*)", + FunPattern="(?:fun\\(\\(" ++ UnionPattern ++ "\\)\\s*->\\s*" ++ UnionPattern ++ "\\))", + ArgPattern3 = "(?:"++FunPattern++"|"++UnionPattern++")", + PrevArgs="(?:"++ArgPattern3++",\\s){"++integer_to_list(PreviousParams) ++ "}", + FunctionHeadStart="^([^\\(]+\\("++PrevArgs++")", %% \\1 + + HighlightArg="("++ArgPattern3++")", %\\2 + NextArgs="(?:,\\s"++ArgPattern3++")*", + FunctionHeadEnd="("++NextArgs++"\\)(?:.*))$", % \\3 + + re:replace(MFA, + FunctionHeadStart ++ HighlightArg ++ FunctionHeadEnd, + "\\1\033[;1;4m\\2\033[0m\\3", + [global, {return, list}, unicode]) + end, + Highlight = proplists:get_value(highlight, Options, false), + case Highlight of + false -> MFA2; + _ -> re:replace(MFA2, "(\\Q"++Highlight++"\\E)", "\033[;1;4m\\1\033[0m", [global, {return, list}, unicode]) + end ++ Separator + end + end; +format_title(_Elems, _LineWidth) -> + %% not a section, old interface + %% output empty list + "". + +format_section_matches(LS, LineWidth) -> format_section_matches(LS, LineWidth, 0). +format_section_matches([], _, _) -> "\n"; +format_section_matches([#{}|_]=FF, LineWidth, Acc) -> + Groups = maps:groups_from_list( + fun(#{title:=Title, elems:=T, options:=Opts}) -> + Separator = proplists:get_value(separator, Opts, "\n"), + case lists:last(string:split(Title++Separator, "\n", trailing)) of + [] -> format_section_matches(T, LineWidth); + Chars -> %% we have chars that compete with the results on the first line + Len = string:length(Chars), + format_section_matches(T, LineWidth, Len+Acc) + end end, - ["\n" | S]. + fun(F) -> + format_title(F, LineWidth) + end, FF), + lists:flatten( + [lists:join("", F)++Matches || + {Matches, F}<-lists:sort(fun({_,A},{_,B}) -> A =< B end, maps:to_list(Groups))]); +format_section_matches(Elems, LineWidth, Acc) -> + format_section_matches1(Elems, LineWidth, Acc). -format_col([], _) -> []; -format_col(L, Acc) -> - LL = 79, - format_col(L, field_width(L, LL), 0, Acc, LL, false). +format_section_matches1([], _, _) -> []; +format_section_matches1(LS, LineWidth, Len) -> + L = lists:usort(fun special_sort/2, ordsets:to_list(LS)), + Opt = case Len == 0 of + true -> []; + false -> [{title, Len}] + end, + S1 = format_col(Opt ++ L, field_width(Opt ++ L, LineWidth), Len, [], LineWidth, Opt), + S2 = lists:map( + fun(Line) -> + case string:length(Line) of + Len1 when Len1 > LineWidth -> + string:sub_string(Line, 1, LineWidth-4) ++ "...\n"; + _ -> Line + end + end,S1), + lists:flatten(string:trim(S2, trailing)++"\n"). -format_col(X, Width, Len, Acc, LL, Dots) when Width + Len > LL -> - format_col(X, Width, 0, ["\n" | Acc], LL, Dots); -format_col([A|T], Width, Len, Acc0, LL, Dots) -> +format_col(X, Width, Len, Acc, LL, Opt) when Width + Len > LL -> + format_col(X, Width, 0, ["\n" | Acc], LL, Opt); +format_col([{title,TitleLen}|T], Width, Len, Acc0, LL, Opt) -> + Acc = [io_lib:format("~-*ts", [Width-TitleLen, ""])|Acc0], + format_col(T, Width, Len+Width, Acc, LL, Opt); +format_col([A|T], Width, Len, Acc0, LL, _Opt) -> {H0, R} = format_val(A), - Hmax = LL - length(R), - {H, NewDots} = + Hmax = LL - string:length(R), + {H, _} = case string:length(H0) > Hmax of true -> {io_lib:format("~-*ts", [Hmax - 3, H0]) ++ "...", true}; - false -> {H0, Dots} + false -> {H0, false} end, Acc = [io_lib:format("~-*ts", [Width, H ++ R]) | Acc0], - format_col(T, Width, Len+Width, Acc, LL, NewDots); -format_col([], _, _, Acc, _LL, Dots) -> - {lists:reverse(Acc, "\n"), Dots}. + format_col(T, Width, Len+Width, Acc, LL, []); +format_col([], _, _, Acc, _LL, _Opt) -> + lists:reverse(Acc). +format_val({H, L}) when is_list(L) -> + {H, proplists:get_value(ending, L, "")}; format_val({H, I}) when is_integer(I) -> - %% If it's a tuple {string(), integer()}, we assume it's an - %% arity, and meant to be printed. - {H, "/" ++ integer_to_list(I)}; + {H, "/"++integer_to_list(I)}; format_val({H, _}) -> {H, ""}; format_val(H) -> {H, ""}. field_width(L, LL) -> field_width(L, 0, LL). - -field_width([{H,_}|T], W, LL) -> - case string:length(H) of +field_width([{title, Len}|T], W, LL) -> + case Len of L when L > W -> field_width(T, L, LL); _ -> field_width(T, W, LL) end; field_width([H|T], W, LL) -> - case string:length(H) of + {H1, Ending} = format_val(H), + case string:length(H1++Ending) of L when L > W -> field_width(T, L, LL); _ -> field_width(T, W, LL) end; -field_width([], W, LL) when W < LL - 3 -> +field_width([], W, LL) when W < LL -> W + 4; field_width([], _, LL) -> LL. -vals([]) -> []; -vals([{S, _}|L]) -> [S|vals(L)]; -vals([S|L]) -> [S|vals(L)]. - -leading_dots([], _Len) -> []; -leading_dots([{H, I}|L], Len) -> - [{"..." ++ string:slice(H, Len), I}|leading_dots(L, Len)]; -leading_dots([H|L], Len) -> - ["..." ++ string:slice(H, Len)|leading_dots(L, Len)]. +number_matches([#{ elems := Matches }|T]) -> + number_matches(Matches) + number_matches(T); +number_matches([_|T]) -> + 1 + number_matches(T); +number_matches([]) -> + 0. %% Strings are handled naively, but it should be OK here. longest_common_head([]) -> @@ -221,24 +1166,23 @@ longest_common_head(LL) -> longest_common_head(LL, []). longest_common_head([[]|_], L) -> - {partial, reverse(L)}; + {partial, lists:reverse(L)}; longest_common_head(LL, L) -> case same_head(LL) of - true -> - [[H|_]|_] = LL, - LL1 = all_tails(LL), - case all_nil(LL1) of - false -> - longest_common_head(LL1, [H|L]); - true -> - {complete, reverse([H|L])} - end; - false -> - {partial, reverse(L)} + true -> + [[H|_]|_] = LL, + LL1 = all_tails(LL), + case all_nil(LL1) of + false -> + longest_common_head(LL1, [H|L]); + true -> + {complete, lists:reverse([H|L])} + end; + false -> + {partial, lists:reverse(L)} end. same_head([[H|_]|T1]) -> same_head(H, T1). - same_head(H, [[H|_]|T]) -> same_head(H, T); same_head(_, []) -> true; same_head(_, _) -> false. diff --git a/lib/stdlib/src/edlin_type_suggestion.erl b/lib/stdlib/src/edlin_type_suggestion.erl new file mode 100644 index 0000000000..5b148ab998 --- /dev/null +++ b/lib/stdlib/src/edlin_type_suggestion.erl @@ -0,0 +1,487 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2005-2023. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(edlin_type_suggestion). +-include_lib("kernel/include/eep48.hrl"). +-export([type_tree/4, get_arity/3, get_atoms/3, get_types/3, get_types/4, get_function_type/4, print_type/3]). + + +%% type_tree/4 returns a unwrapped and trimmed type specification containing +%% all the valid 'types' that are valid in each function parameter of a function or +%% each record field of a record. +%% +%% Translates the spec AST to a structure that resembles the AST but trimmed of unneeded data. +%% User types and remote types are fetched and embedded in the structure depending on requested +%% level of unnestling. +%% Unions are flattened. +%% Visited is used to prevent infinite loops when looking up a recursive / cyclic type +%% FT is a map of type {{type, Type}, {attribute,_,type,{_,TypeAST,_}}}) +type_tree(Mod, FunType, Nestings, FT) -> + %% TODO when FT is updated we would be getting incorrect results because of cache + %% we should probably store a "dirty bit" in the table to make this aware that + %% a new result should be calculated. Preferably only types that depend on the table + %% would be invalidated, but it may turn advanced. + %% TODO look this over to make sure we don't do unnecessary work, + case get({type_traverser, Mod, FunType, Nestings}) of + undefined -> Res = type_traverser_cache(Mod, FunType, #{}, length(Nestings)+1, FT), + put({type_traverser, Mod, FunType, Nestings}, Res), + Res; + Res -> Res + end. +type_traverser_cache(Mod, T, Visited, Level, FT) -> + case get({Mod, T, Level}) of + undefined -> + Res = type_traverser(Mod, T, Visited, Level, FT), + put({Mod, T, Level}, Res), + Res; + Res -> Res + end. + +type_traverser(Mod, {type, _, bounded_fun, [Fun, Constraints]}, Visited, Level, FT) -> + Cl = [type_traverser(Mod,X,Visited, Level, FT) || X <- Constraints], + F = type_traverser(Mod, Fun, Visited, Level, FT), + {function, F, Cl}; +type_traverser(Mod, {type, _, 'fun', [Product, Return]}, Visited, Level, FT) -> + P = type_traverser(Mod, Product, Visited, Level, FT), + R = type_traverser(Mod, Return, Visited, Level, FT), + {P, {return, R}}; +type_traverser(Mod, {type, _, product, Childs}, Visited, Level, FT) -> + Cl = [type_traverser(Mod, X, Visited, Level, FT) || X <- Childs], + {parameters, Cl}; +type_traverser(Mod, {type, _, constraint, [{atom, _, is_subtype}, [Type1, Type2]]}, Visited, Level, FT) -> + {constraint, type_traverser(Mod, Type1, Visited, Level, FT), type_traverser(Mod, Type2, Visited, Level, FT)}; +type_traverser(_, {var, _, Name}, _Visited, _Level, _FT) -> + {var, Name}; +type_traverser(_Mod,{type, _, map, any}, _Visited, _Level, _FT) -> + {type, map, []}; +type_traverser(Mod, {type, _, map, Params}, Visited, Level, FT) -> + {map, [type_traverser(Mod, X, Visited, Level-1, FT) || X <- Params]}; +type_traverser(Mod, {type, _, map_field_exact, [Type1, Type2]}, Visited, Level, FT) -> + {map_field_exact, type_traverser(Mod,Type1, Visited, Level, FT), type_traverser(Mod,Type2, Visited, Level, FT)}; +type_traverser(Mod, {type, _, map_field_assoc, [Type1, Type2]}, Visited, Level, FT) -> + {map_field_assoc, type_traverser(Mod,Type1, Visited, Level, FT), type_traverser(Mod,Type2, Visited, Level, FT)}; +type_traverser(_Mod, {atom, _, Atom}, _Visited, _Level, _FT) when is_atom(Atom) -> + Atom; +type_traverser(Mod, {op, _, Op, Type}, Visited, Level, FT) -> + {op, Op, type_traverser(Mod, Type, Visited, Level, FT)}; +type_traverser(Mod, {op, _, Op, Type1, Type2}, Visited, Level, FT) -> + {op, Op, type_traverser(Mod, Type1, Visited, Level, FT), type_traverser(Mod, Type2, Visited, Level, FT)}; +type_traverser(_Mod, {integer, _, Int}, _Visited, _Level, _FT) -> + {integer, Int}; +type_traverser(Mod, {type, _, list, [ChildType]}, Visited, Level, FT) -> + {list, type_traverser(Mod, ChildType, Visited, Level-1, FT)}; +type_traverser(_Mod, {type, _, tuple, any}, _Visited, _Level, _FT) -> + {type, tuple, []}; +type_traverser(Mod, {type, _, tuple, ChildTypes}, Visited, Level, FT) -> + {tuple, [type_traverser(Mod, X, Visited, Level-1, FT) || X <- ChildTypes]}; +type_traverser(Mod, {type, _, union, ChildTypes}, Visited, Level, FT) -> + Childs = [type_traverser(Mod, X, Visited, Level, FT) || X <- ChildTypes], + ChildsFiltered = [X || X <- Childs, X/=undefined], + {UnionChilds, NonUnionChilds} = lists:partition( + fun(X) -> + case X of + {union, _} -> true; + _ -> false + end + end, ChildsFiltered), + ChildsFlattened = lists:flatten([T || {union, T} <- UnionChilds]) ++ NonUnionChilds, + {union, ChildsFlattened}; +type_traverser(Mod, {ann_type,_,[T1,T2]}, Visited, Level, FT) -> + {ann_type, type_traverser(Mod, T1, Visited, Level, FT), type_traverser(Mod, T2, Visited, Level, FT)}; +type_traverser(Mod, {user_type,_,Name,Params}=T, Visited, Level, FT) when 0 >= Level -> + %% when we have level 0, do not traverse the type further, just print it + case maps:is_key(strip_anno(T), Visited) of + false -> + {type, Mod, Name, [type_traverser(Mod, P, Visited#{ strip_anno(T) => true }, 0, FT) || P <- Params]}; + true -> {type, Mod, Name, []} + end; +type_traverser(_, {remote_type,_,[{_,_,Mod},{_,_,Name}, Params]}=T, Visited, Level, FT) when 0 >= Level -> + case maps:is_key(strip_anno(T), Visited) of + false -> + {type, Mod, Name, [type_traverser(Mod, P, Visited#{ strip_anno(T) => true }, 0, FT) || P <- Params]}; + true -> {type, Mod, Name, []} + end; +type_traverser(Mod, {user_type,_,Name,Params}=T, Visited, 1=Level, FT) -> + case maps:is_key(strip_anno(T), Visited) of + false -> + case get({strip_anno(T), 1}) of + undefined -> + Res = case lookup_type(Mod, Name, length(Params), FT) of + hidden -> {type, Mod, Name, [type_traverser(Mod, P, Visited#{ strip_anno(T) => true }, Level, FT) || P <- Params]}; + Type -> {user_type, Mod, Name, [type_traverser(Mod, P, Visited#{ strip_anno(T) => true }, Level, FT) || P <- Params], type_traverser(Mod, Type, Visited#{ strip_anno(T) => true }, Level, FT)} + end, + put({strip_anno(T), 1}, Res), + Res; + Res -> Res + end; + true -> {type, Mod, Name, []} + end; +type_traverser(_, {remote_type,_,[{_,_,Mod},{_,_,Name}, Params]}=T, Visited, 1=Level, FT) -> + case maps:is_key(strip_anno(T), Visited) of + false -> + case get({strip_anno(T), 1}) of + undefined -> + Res = case lookup_type(Mod, Name, length(Params), FT) of + hidden -> {type, Mod, Name, [type_traverser(Mod, P, Visited#{ strip_anno(T) => true }, Level, FT) || P <- Params]}; + Type -> {user_type, Mod, Name, [type_traverser(Mod, P, Visited#{ strip_anno(T) => true }, Level, FT) || P <- Params], type_traverser(Mod, Type, Visited#{ strip_anno(T) => true }, Level, FT)} + end, + put({strip_anno(T), 1}, Res), + Res; + Res -> Res + end; + true -> {type, Mod, Name, []} + end; +type_traverser(Mod, {user_type,_,Name,Params}=T, Visited, Level, FT) -> + case maps:is_key(strip_anno(T), Visited) of + false -> + case get({strip_anno(T), Level}) of + undefined -> + Res = case lookup_type(Mod, Name, length(Params), FT) of + hidden -> {type, Mod, Name, [type_traverser(Mod, P, Visited#{ strip_anno(T) => true }, Level, FT) || P <- Params]}; + Type -> type_traverser(Mod, Type, Visited#{ strip_anno(T) => true }, Level, FT) + end, + put({strip_anno(T), Level}, Res), + Res; + Res -> Res + end; + true -> {type, Mod, Name, []} + end; +type_traverser(_, {remote_type, _, [{_,_,Mod},{_,_,Name}, Params]}=T, Visited, Level, FT) -> + case maps:is_key(strip_anno(T), Visited) of + false -> + case get({strip_anno(T), Level}) of + undefined -> + Res = case lookup_type(Mod, Name, length(Params), FT) of + hidden -> {type, Mod, Name, [type_traverser(Mod, P, Visited#{ strip_anno(T) => true }, Level, FT) || P <- Params]}; + Type -> type_traverser(Mod, Type, Visited#{ strip_anno(T) => true }, Level, FT) + end, + put({strip_anno(T), Level}, Res), + Res; + Res -> Res + end; + true -> {type, Mod, Name, []} + end; +type_traverser(_, {type, _, record, [{atom, _, Record}]}, _Visited, _Level, _FT) -> + {record, Record}; +type_traverser(_, {type, _, Name, any}, _, _, _) -> + {type, Name, []}; +type_traverser(_, {type, _, term}, _, _, _) -> + {type, any, []}; +type_traverser(_, {type, _, Name}, _, _, _) -> + {type, Name, []}; +type_traverser(_, {type, _, term, _}, _, _, _) -> + {type, any, []}; +type_traverser(_, {type, _, Name, Params}=T, Visited, Level, FT) -> + case maps:is_key(strip_anno(T), Visited) of + false -> + case get({strip_anno(T), 1}) of + undefined -> + Res = case lookup_type(erlang, Name, length(Params), FT) of + hidden -> {type, Name, [type_traverser(erlang, P, Visited#{ strip_anno(T) => true }, Level, FT) || P <- Params]}; + Type -> type_traverser(erlang, Type, Visited#{ strip_anno(T) => true}, Level, FT) + end, + put({strip_anno(T), 1}, Res), + Res; + Res -> Res + end; + true -> {type, Name, []} + end. + +strip_anno({A, _, B}) -> {A, B}; +strip_anno({A, _, B, C}) -> {A, B, C}. + +simplified_type(erlang, binary, 0) -> {type, undefined, binary, []}; +simplified_type(erlang, char, 0) -> {type, undefined, char, []}; +simplified_type(erlang, iolist, 0) -> {type, undefined, iolist, []}; +simplified_type(erlang, string, 0) -> {type, undefined, string, []}; +simplified_type(unicode, chardata, 0) -> {type, erlang, string, []}; +simplified_type(file, filename_all, 0) -> {type, erlang, string, []}; +simplified_type(file, filename, 0) -> {type, erlang, string, []}; +simplified_type(file, name_all, 0) -> {type, erlang, string, []}; +simplified_type(file, name, 0) -> {type, erlang, string, []}; +simplified_type(_Module, _TypeName, _Arity) -> none. + +lookup_type(Mod, Type, Arity, FT) -> + case simplified_type(Mod, Type, Arity) of + none -> + case code:get_doc(Mod, #{sources => [debug_info]}) of + {ok, #docs_v1{ docs = Docs } } -> + FnFunctions = + lists:filter(fun({{type, T, A},_Anno,_Sig,_Doc,_Meta}) -> + T =:= Type andalso A =:= Arity; + (_) -> + false + end, Docs), + case FnFunctions of + [] -> + case [TypeAST || {{type, Type2}, {attribute,_,type,{_,TypeAST,_}}} <- FT, Type2 =:= Type] of + [] -> hidden; %% can be an opaque type or missing type + [SingleTypeAST] -> SingleTypeAST + end; + [{_,_,_,_,#{signature := [{attribute,_,type,{_,TypeAST,_}}]}}] -> TypeAST + end; + _ -> + case [TypeAST || {{type, Type2}, {attribute,_,type,{_,TypeAST,_}}} <- FT, Type2 =:= Type] of + [] -> hidden; %% can be an opaque type or missing type + [SingleTypeAST] -> SingleTypeAST + end + end; + T -> T + end. +get_function_type(Mod, Fun, Arity, FT) -> + case code:get_doc(Mod, #{sources => [debug_info]}) of + {ok, #docs_v1{ docs = Docs } } -> + R = lists:flatten([FunTypes || + {{function, F, A},_Anno,_Sig,_Doc, #{ signature := [{attribute,_,spec,{_,FunTypes}}]}} <- Docs, + F =:= Fun, A =:= Arity]), + case {Mod, R} of + {shell_default, []} -> + lists:flatten([FunTypes || + {{function_type, {shell_default, F, A}},{attribute,_,spec,{_,FunTypes}}} <- FT, + F =:= Fun, A =:= Arity]); + _ -> R + end; + _ when Mod =:= shell_default -> + lists:flatten([FunTypes || {{function_type, {shell_default, F, A}},{attribute,_,spec,{_,FunTypes}}} <- FT, + F =:= Fun, A =:= Arity]); + _ -> [] + end. +get_arity(Constraints, Type, Nestings) -> + case get_arity1(Type, Constraints, Nestings) of + List when is_list(List) -> List; + Val -> [Val] + end. +get_arity1({var, _Var}=C, Constraints, Nestings) -> + case get_constraint(C, Constraints) of + {constraint, _, T} -> get_arity1(T, Constraints, Nestings); + _ -> none + end; +get_arity1({list, _T}, _Constraints, [{'list', _, _}]) -> + 99; %% Can be higher, but probably do not need completion for that +get_arity1({list, T}, Constraints, [{'list', _, _}|Nestings]) -> + get_arity1(T, Constraints, Nestings); +get_arity1({tuple, LT}, Constraints, [{'tuple', Args, _}]) when length(LT) >= length(Args) -> + case edlin_expand:match_arguments1(LT, Constraints, Args) of + true -> length(LT); + false -> + none + end; +get_arity1({tuple, LT}, Constraints, [{'tuple', Args, _}|Nestings]) when length(LT) >= length(Args)+1 -> + case edlin_expand:match_arguments1(LT, Constraints, Args) of + true -> get_arity1(lists:nth(length(Args)+1, LT), Constraints, Nestings); + false -> none + end; +get_arity1({map, Types}, _Constraints, [{'map', _Keys, [], _, _}]) -> + length(Types); +get_arity1({map, Types}, _Constraints, [{'map', _Keys, _Key, _, _}]) -> + length(Types); +get_arity1({map, Types}, Constraints, [{'map', Keys, [], _, _}|Nestings]) -> + lists:flatten([get_arity1(T, Constraints, Nestings) || {_, Key, _}=T <- Types, not lists:member(atom_to_list(Key), Keys)]); +get_arity1({map, Types}, Constraints, [{'map', _Keys, Key, _, _}|Nestings]) -> + case [V || {_, K, V} <- Types, K =:= list_to_atom(Key)] of + [] -> none; + [Type] -> get_arity1(Type, Constraints, Nestings) + end; +get_arity1({map_field_assoc, K, _V}, C, Nestings) -> + get_arity1(K, C, Nestings); +get_arity1({map_field_exact, K, _V}, C, Nestings) -> + get_arity1(K, C, Nestings); +get_arity1({union, Types}, Constraints, Nestings) -> + Arities = [get_arity1(T, Constraints, Nestings) || T <- Types], + [X || X <- lists:flatten(Arities), X/=none]; +get_arity1({ann_type, _Var, Type}, Constraints, Nestings) -> + get_arity1(Type, Constraints, Nestings); +get_arity1({user_type, _, _, _, Type}, Constraints, Nestings) -> + get_arity1(Type, Constraints, Nestings); +get_arity1(_, _, _) -> + none. + +%% get_atoms returns the valid atoms in the current context as a list +get_atoms(Constraints, Type, Nestings) -> + case get_atoms1(Type, Constraints, Nestings) of + List when is_list(List) -> [io_lib:write_atom(Atom) || Atom <- List]; + Atom when is_atom(Atom) -> [io_lib:write_atom(Atom)] + end. +get_atoms1({var, _Var}=C, Constraints, Nestings) -> + case get_constraint(C, Constraints) of + {constraint, _, T} -> get_atoms1(T, Constraints, Nestings); + _ -> [] + end; +get_atoms1({list, T}, Constraints, [{'list', _, _}|Nestings]) -> + get_atoms1(T, Constraints, Nestings); +get_atoms1({tuple, LT}, Constraints, [{'tuple', Args, _}|Nestings]) when length(LT) >= length(Args)+1 -> + case edlin_expand:match_arguments1(LT, Constraints, Args) of + true -> get_atoms1(lists:nth(length(Args)+1, LT), Constraints, Nestings); + false -> [] + end; +get_atoms1({map, Types}, Constraints, [{'map', Keys, [], _, _}|Nestings]) -> + lists:flatten([get_atoms1(T, Constraints, Nestings) || {_, Key, _}=T <- Types, not lists:member(atom_to_list(Key), Keys)]); +get_atoms1({map, Types}, Constraints, [{'map', _Keys, Key, _, _}|Nestings]) -> + case [V || {_, K, V} <- Types, K =:= list_to_atom(Key)] of + [] -> []; + [Type] -> get_atoms1(Type, Constraints, Nestings) + end; +get_atoms1({map_field_assoc, K, _V}, C, Nestings) -> + get_atoms1(K, C, Nestings); +get_atoms1({map_field_exact, K, _V}, C, Nestings) -> + get_atoms1(K, C, Nestings); +get_atoms1( {union, Types}, Constraints, Nestings) -> + Atoms = [get_atoms1(T, Constraints, Nestings) || T <- Types], + [X || X <- lists:flatten(Atoms), X/=[]]; +get_atoms1(Atom, _Constraints, []) when is_atom(Atom) -> + Atom; +get_atoms1({user_type, _, _, _, Type}, Constraints, Nestings) -> + get_atoms1(Type, Constraints, Nestings); +get_atoms1(_, _, _) -> + []. + +get_types(Constraints, T, Nestings) -> + get_types(Constraints, T, Nestings,[]). +get_types(Constraints, T, Nestings, Options) -> + MaxUserTypeExpansions = 1, + case get_types1(T, Constraints, Nestings, MaxUserTypeExpansions, Options) of + [] -> []; + [_|_]=Types -> [Type || Type <- Types, Type /= []]; + Type -> [Type] + end. +get_types1({var, _Var}=C, Constraints, Nestings, MaxUserTypeExpansions, Options) -> + case get_constraint(C, Constraints) of + {constraint, _, T} -> get_types1(T, Constraints, Nestings, MaxUserTypeExpansions, Options); + _ -> [] + end; +get_types1({union, Types}, Cs, Nestings, MaxUserTypeExpansions, Options) -> + lists:flatten([get_types1(T, Cs, Nestings, MaxUserTypeExpansions, Options) || T <- Types]); + +get_types1({list, T}, Cs, [{list, _Args, _}|Nestings], MaxUserTypeExpansions, Options) -> + get_types1(T, Cs, Nestings, MaxUserTypeExpansions, Options); +get_types1({tuple, LT}, Cs, [{tuple, Args, _}|Nestings], MaxUserTypeExpansions, Options) when length(LT) >= length(Args)+1 -> + case edlin_expand:match_arguments1(LT, Cs, Args) of + true -> get_types1(lists:nth(length(Args)+1, LT), Cs, Nestings, MaxUserTypeExpansions, Options); + false -> [] + end; +get_types1({'map', Types}, Cs, [{'map', Keys, [], _Args, _}|Nestings], MaxUserTypeExpansions, Options) -> + lists:flatten([get_types1(T, Cs, Nestings, MaxUserTypeExpansions, Options) || {_, Key, _}=T <- Types, not lists:member(atom_to_list(Key), Keys)]); +get_types1({'map', Types}, Cs, [{'map', _, Key, _Args, _}|Nestings], MaxUserTypeExpansions, Options) -> + case [V || {_, K, V} <- Types, K =:= list_to_atom(Key)] of + [] -> []; + [Type] -> get_types1(Type, Cs, Nestings, MaxUserTypeExpansions, Options) + end; +get_types1({user_type, _Mod, _Name, _Params, Type}, Cs, Nestings, MaxUserTypeExpansions, [no_print]=Options) when MaxUserTypeExpansions > 0 -> + lists:flatten([get_types1(Type, Cs, Nestings, MaxUserTypeExpansions-1, Options)]); +get_types1({user_type, _, _, _, Type}, Cs, Nestings, 0, [no_print]=Options) -> + get_types1(Type, Cs, Nestings, 0, Options); +get_types1({ann_type, _Var, T}, Cs, Nestings, MaxUserTypeExpansions, [no_print]) -> + get_types1(T, Cs, Nestings, MaxUserTypeExpansions, [no_print]); +get_types1({ann_type, _Var, _T}=Type, Cs, [], _MaxUserTypeExpansions, []) -> + {print_type(Type, Cs), ""}; +get_types1({ann_type, _Var, T}, Cs, Nestings, MaxUserTypeExpansions, []) -> + get_types1(T, Cs, Nestings, MaxUserTypeExpansions, []); +get_types1(Type, _Cs, [], _, [no_print]) -> + Type; +get_types1({user_type, Mod, Name, Params, Type}, Cs, Nestings, MaxUserTypeExpansions, []) when MaxUserTypeExpansions > 0 -> + Title = print_type({type, Mod, Name, Params}, Cs, []), + Elems = lists:flatten([get_types1(Type, Cs, Nestings, MaxUserTypeExpansions-1, [])]), + #{title=>Title, elems=>Elems, options=>[{separator, " :: "}, {highlight_all}]}; +get_types1({user_type, _, _, _, Type}, Cs, Nestings, 0, []) -> + get_types1(Type, Cs, Nestings, 0, []); +get_types1(Type, Cs, [],_, []) -> + {print_type(Type, Cs), ""}; +get_types1(_, _, _, _, _) -> []. + +get_constraint(Type, Constraints) -> + case [ X || {constraint, T, _}=X <- Constraints, T == Type] of + [C|_] -> C; + [] -> [] + end. + +print_type(Type, Constraints) -> + lists:flatten(print_type(Type, Constraints, [], [])). +print_type(Type, Constraints, Options) -> + lists:flatten(print_type(Type, Constraints, [], Options)). +print_type({var, Name}=Var, Constraints, Visited, Options) -> + case lists:member(Var, Visited) of + true -> atom_to_list(Name); + false -> + case get_constraint(Var, Constraints) of + {constraint, _, T2} -> print_type(T2, Constraints, [Var| Visited], Options); + _ -> atom_to_list(Name) + end + end; +print_type(Atom, _Cs, _V, _) when is_atom(Atom) -> io_lib:write_atom(Atom); +print_type({{parameters, Ps}, {return, R}}, Cs, V, Options) -> + "fun(("++lists:join(", ", [print_type(X, Cs, V, Options) || X <- Ps]) ++ ") -> " ++ print_type(R, Cs, V, Options) ++ ")"; +print_type({list, Type}, Cs, V, Options)-> + "[" ++ print_type(Type, Cs, V, Options) ++ "]"; +print_type({tuple, Types}, Cs, V, Options) when is_list(Types) -> + Types1 = [print_type(X, Cs, V, Options) || X <- Types], + case Types1 of + [] -> "{}"; + _ -> "{"++ lists:nth(1, Types1) ++ ", ...}" + end; +print_type({ann_type, Var, Type}, Cs, V, Options) -> + print_type(Var, Cs, V, Options) ++ " :: " ++ print_type(Type, Cs, V, Options); +print_type({map, Types}, Cs, V, Options) -> + Types1 = [print_type(X, Cs, V, Options) || X <- Types], + "#{"++lists:join(", ", Types1) ++ "}"; +print_type({map_field_assoc, Type1, Type2}, Cs, V, Options) -> + print_type(Type1, Cs, V, Options) ++ "=>" ++ print_type(Type2, Cs, V, Options); +print_type({map_field_exact, Type1, Type2}, Cs, V, Options) -> + print_type(Type1, Cs, V, Options) ++ ":=" ++ print_type(Type2, Cs, V, Options); +print_type({integer, Int}, _Cs, _V, _) -> + integer_to_list(Int); +print_type({op, Op, Type}, Cs, V, Options) -> + "op ("++atom_to_list(Op)++" "++print_type(Type, Cs, V, Options)++")"; +print_type({op, Op, Type1, Type2}, Cs, V, Options) -> + "op ("++print_type(Type1, Cs, V, Options)++" "++atom_to_list(Op)++" "++print_type(Type2, Cs, V, Options)++")"; +print_type({record, Record}, _Cs, _V, _) -> + "#" ++ atom_to_list(Record); +print_type({type, range, [{integer, Int1},{integer, Int2}]}, _Cs, _V, _) -> + integer_to_list(Int1) ++ ".." ++ integer_to_list(Int2); +print_type({type, non_neg_integer, []}, _Cs, _V, _) -> + "integer() >= 0"; +print_type({type, neg_integer, []}, _Cs, _V, _) -> + "integer() < 0"; +print_type({type, pos_integer, []}, _Cs, _V, _) -> + "integer() > 0"; +print_type({type, Name, []}, _Cs, _V, _) -> + atom_to_list(Name)++"()"; +print_type({type, Name, Params}, _Cs, _V, _) -> + atom_to_list(Name) ++ "(" ++ lists:join(", ",[ extract_param(P) || P <- Params]) ++ ")"; +print_type({union, Types}, Cs, V, Options) -> + lists:join(" | ", [print_type(X, Cs, V, Options) || X <- Types]); +print_type({type, Mod, Name, Params}, _Cs, _V, _) -> + atom_to_list(Mod) ++ ":" ++ atom_to_list(Name) ++ + "(" ++ lists:join(", ", [extract_param(P) || P <- Params]) ++ ")"; +print_type({user_type, Mod, Name, Params, Type}, Cs, V, Options) -> + First = proplists:get_value(first_only, Options, false), + case First of + true -> print_type({type, Mod, Name, Params}, Cs, V, Options); + _ -> print_type({type, Mod, Name, Params}, Cs, V, Options) ++ " :: " ++ print_type(Type, Cs, V, Options) + end; +print_type(_,_,_,_) -> atom_to_list(unknown). + + +extract_param({var, Var}) -> + atom_to_list(Var); +extract_param({integer, Value}) -> + io_lib:format("~p",[Value]); +extract_param({type, Type,_}) -> + io_lib:format("~p", [Type]); +extract_param(T)-> + print_type(T, []). diff --git a/lib/stdlib/src/epp.erl b/lib/stdlib/src/epp.erl index bdb0bc64a2..1f7f614b00 100644 --- a/lib/stdlib/src/epp.erl +++ b/lib/stdlib/src/epp.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2022. All Rights Reserved. +%% Copyright Ericsson AB 1996-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -125,7 +125,8 @@ open(Name, Path, Pdm) -> {'name',FileName :: file:name()} | {'location',StartLocation :: erl_anno:location()} | {'fd',FileDescriptor :: file:io_device()} | - 'extra'], + 'extra' | + {'compiler_internal', [term()]}], Epp :: epp_handle(), Extra :: [{'encoding', source_encoding() | 'none'}], ErrorDescriptor :: term(). @@ -309,7 +310,8 @@ parse_file(Ifile, Path, Predefs) -> {'location',StartLocation :: erl_anno:location()} | {'reserved_word_fun', Fun :: fun((atom()) -> boolean())} | {'features', [Feature :: atom()]} | - 'extra'], + 'extra' | + {'compiler_internal', [term()]}], Form :: erl_parse:abstract_form() | {'error', ErrorInfo} | {'eof',Location}, @@ -608,6 +610,8 @@ init_server(Pid, FileName, Options, St0) -> SourceName = proplists:get_value(source_name, Options, FileName), Pdm = proplists:get_value(macros, Options, []), Features = proplists:get_value(features, Options, []), + Internal = proplists:get_value(compiler_internal, Options, []), + ParseChecks = proplists:get_bool(ssa_checks, Internal), Ms0 = predef_macros(SourceName, Features), case user_predef(Pdm, Ms0) of {ok,Ms1} -> @@ -631,7 +635,11 @@ init_server(Pid, FileName, Options, St0) -> default_encoding=DefEncoding, erl_scan_opts = [{text_fun, keep_ftr_keywords()}, - {reserved_word_fun, ResWordFun}], + {reserved_word_fun, ResWordFun}] + ++ if ParseChecks -> + [{compiler_internal,[ssa_checks]}]; + true -> [] + end, features = Features, else_reserved = ResWordFun('else'), deterministic = Deterministic}, @@ -750,7 +758,7 @@ wait_request(St) -> wait_request(St); {epp_request,From,macro_defs} -> %% Return the old format to avoid any incompability issues. - Defs = [{{atom,K},V} || {K,V} <- maps:to_list(St#epp.macs)], + Defs = [{{atom,K},V} || K := V <- St#epp.macs], epp_reply(From, Defs), wait_request(St); {epp_request,From,close} -> diff --git a/lib/stdlib/src/erl_eval.erl b/lib/stdlib/src/erl_eval.erl index 987ba0cf0a..f88cba1ba3 100644 --- a/lib/stdlib/src/erl_eval.erl +++ b/lib/stdlib/src/erl_eval.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2021. All Rights Reserved. +%% Copyright Ericsson AB 1996-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -286,6 +286,8 @@ expr({lc,_,E,Qs}, Bs, Lf, Ef, RBs, FUVs) -> eval_lc(E, Qs, Bs, Lf, Ef, RBs, FUVs); expr({bc,_,E,Qs}, Bs, Lf, Ef, RBs, FUVs) -> eval_bc(E, Qs, Bs, Lf, Ef, RBs, FUVs); +expr({mc,_,E,Qs}, Bs, Lf, Ef, RBs, FUVs) -> + eval_mc(E, Qs, Bs, Lf, Ef, RBs, FUVs); expr({tuple,_,Es}, Bs0, Lf, Ef, RBs, FUVs) -> {Vs,Bs} = expr_list(Es, Bs0, Lf, Ef, FUVs), ret_expr(list_to_tuple(Vs), Bs, RBs); @@ -760,17 +762,15 @@ do_apply(F, _Anno, FunOrModFun, Args) when is_function(F, 2) -> eval_lc(E, Qs, Bs, Lf, Ef, RBs, FUVs) -> ret_expr(lists:reverse(eval_lc1(E, Qs, Bs, Lf, Ef, FUVs, [])), Bs, RBs). -eval_lc1(E, [{generate,Anno,P,L0}|Qs], Bs0, Lf, Ef, FUVs, Acc0) -> - {value,L1,_Bs1} = expr(L0, Bs0, Lf, Ef, none, FUVs), - CompFun = fun(Bs, Acc) -> eval_lc1(E, Qs, Bs, Lf, Ef, FUVs, Acc) end, - eval_generate(L1, P, Anno, Bs0, Lf, Ef, CompFun, Acc0); -eval_lc1(E, [{b_generate,Anno,P,L0}|Qs], Bs0, Lf, Ef, FUVs, Acc0) -> - {value,Bin,_Bs1} = expr(L0, Bs0, Lf, Ef, none, FUVs), - CompFun = fun(Bs, Acc) -> eval_lc1(E, Qs, Bs, Lf, Ef, FUVs, Acc) end, - eval_b_generate(Bin, P, Anno, Bs0, Lf, Ef, CompFun, Acc0); -eval_lc1(E, [F|Qs], Bs0, Lf, Ef, FUVs, Acc) -> - CompFun = fun(Bs) -> eval_lc1(E, Qs, Bs, Lf, Ef, FUVs, Acc) end, - eval_filter(F, Bs0, Lf, Ef, CompFun, FUVs, Acc); +eval_lc1(E, [Q|Qs], Bs0, Lf, Ef, FUVs, Acc0) -> + case is_generator(Q) of + true -> + CF = fun(Bs, Acc) -> eval_lc1(E, Qs, Bs, Lf, Ef, FUVs, Acc) end, + eval_generator(Q, Bs0, Lf, Ef, FUVs, Acc0, CF); + false -> + CF = fun(Bs) -> eval_lc1(E, Qs, Bs, Lf, Ef, FUVs, Acc0) end, + eval_filter(Q, Bs0, Lf, Ef, CF, FUVs, Acc0) + end; eval_lc1(E, [], Bs, Lf, Ef, FUVs, Acc) -> {value,V,_} = expr(E, Bs, Lf, Ef, none, FUVs), [V|Acc]. @@ -782,21 +782,66 @@ eval_lc1(E, [], Bs, Lf, Ef, FUVs, Acc) -> eval_bc(E, Qs, Bs, Lf, Ef, RBs, FUVs) -> ret_expr(eval_bc1(E, Qs, Bs, Lf, Ef, FUVs, <<>>), Bs, RBs). -eval_bc1(E, [{b_generate,Anno,P,L0}|Qs], Bs0, Lf, Ef, FUVs, Acc0) -> - {value,Bin,_Bs1} = expr(L0, Bs0, Lf, Ef, none, FUVs), - CompFun = fun(Bs, Acc) -> eval_bc1(E, Qs, Bs, Lf, Ef, FUVs, Acc) end, - eval_b_generate(Bin, P, Anno, Bs0, Lf, Ef, CompFun, Acc0); -eval_bc1(E, [{generate,Anno,P,L0}|Qs], Bs0, Lf, Ef, FUVs, Acc0) -> - {value,List,_Bs1} = expr(L0, Bs0, Lf, Ef, none, FUVs), - CompFun = fun(Bs, Acc) -> eval_bc1(E, Qs, Bs, Lf, Ef, FUVs, Acc) end, - eval_generate(List, P, Anno, Bs0, Lf, Ef, CompFun, Acc0); -eval_bc1(E, [F|Qs], Bs0, Lf, Ef, FUVs, Acc) -> - CompFun = fun(Bs) -> eval_bc1(E, Qs, Bs, Lf, Ef, FUVs, Acc) end, - eval_filter(F, Bs0, Lf, Ef, CompFun, FUVs, Acc); +eval_bc1(E, [Q|Qs], Bs0, Lf, Ef, FUVs, Acc0) -> + case is_generator(Q) of + true -> + CF = fun(Bs, Acc) -> eval_bc1(E, Qs, Bs, Lf, Ef, FUVs, Acc) end, + eval_generator(Q, Bs0, Lf, Ef, FUVs, Acc0, CF); + false -> + CF = fun(Bs) -> eval_bc1(E, Qs, Bs, Lf, Ef, FUVs, Acc0) end, + eval_filter(Q, Bs0, Lf, Ef, CF, FUVs, Acc0) + end; eval_bc1(E, [], Bs, Lf, Ef, FUVs, Acc) -> {value,V,_} = expr(E, Bs, Lf, Ef, none, FUVs), <<Acc/bitstring,V/bitstring>>. +%% eval_mc(Expr, [Qualifier], Bindings, LocalFunctionHandler, +%% ExternalFuncHandler, RetBindings) -> +%% {value,Value,Bindings} | Value + +eval_mc(E, Qs, Bs, Lf, Ef, RBs, FUVs) -> + L = eval_mc1(E, Qs, Bs, Lf, Ef, FUVs, []), + Map = maps:from_list(L), + ret_expr(Map, Bs, RBs). + +eval_mc1(E, [Q|Qs], Bs0, Lf, Ef, FUVs, Acc0) -> + case is_generator(Q) of + true -> + CF = fun(Bs, Acc) -> eval_mc1(E, Qs, Bs, Lf, Ef, FUVs, Acc) end, + eval_generator(Q, Bs0, Lf, Ef, FUVs, Acc0, CF); + false -> + CF = fun(Bs) -> eval_mc1(E, Qs, Bs, Lf, Ef, FUVs, Acc0) end, + eval_filter(Q, Bs0, Lf, Ef, CF, FUVs, Acc0) + end; +eval_mc1({map_field_assoc,Lfa,K0,V0}, [], Bs, Lf, Ef, FUVs, Acc) -> + {value,KV,_} = expr({tuple,Lfa,[K0,V0]}, Bs, Lf, Ef, none, FUVs), + [KV|Acc]. + +eval_generator({generate,Anno,P,L0}, Bs0, Lf, Ef, FUVs, Acc0, CompFun) -> + {value,L1,_Bs1} = expr(L0, Bs0, Lf, Ef, none, FUVs), + eval_generate(L1, P, Anno, Bs0, Lf, Ef, CompFun, Acc0); +eval_generator({b_generate,Anno,P,Bin0}, Bs0, Lf, Ef, FUVs, Acc0, CompFun) -> + {value,Bin,_Bs1} = expr(Bin0, Bs0, Lf, Ef, none, FUVs), + eval_b_generate(Bin, P, Anno, Bs0, Lf, Ef, CompFun, Acc0); +eval_generator({m_generate,Anno,P,Map0}, Bs0, Lf, Ef, FUVs, Acc0, CompFun) -> + {map_field_exact,_,K,V} = P, + {value,Map,_Bs1} = expr(Map0, Bs0, Lf, Ef, none, FUVs), + Iter = case is_map(Map) of + true -> + maps:iterator(Map); + false -> + %% Validate iterator. + try maps:foreach(fun(_, _) -> ok end, Map) of + _ -> + Map + catch + _:_ -> + apply_error({bad_generator,Map}, ?STACKTRACE, + Anno, Bs0, Ef, none) + end + end, + eval_m_generate(Iter, {tuple,Anno,[K,V]}, Anno, Bs0, Lf, Ef, CompFun, Acc0). + eval_generate([V|Rest], P, Anno, Bs0, Lf, Ef, CompFun, Acc) -> case match(P, V, Anno, new_bindings(Bs0), Bs0, Ef) of {match,Bsn} -> @@ -828,6 +873,21 @@ eval_b_generate(<<_/bitstring>>=Bin, P, Anno, Bs0, Lf, Ef, CompFun, Acc) -> eval_b_generate(Term, _P, Anno, Bs0, _Lf, Ef, _CompFun, _Acc) -> apply_error({bad_generator,Term}, ?STACKTRACE, Anno, Bs0, Ef, none). +eval_m_generate(Iter0, P, Anno, Bs0, Lf, Ef, CompFun, Acc0) -> + case maps:next(Iter0) of + {K,V,Iter} -> + case match(P, {K,V}, Anno, new_bindings(Bs0), Bs0, Ef) of + {match,Bsn} -> + Bs2 = add_bindings(Bsn, Bs0), + Acc = CompFun(Bs2, Acc0), + eval_m_generate(Iter, P, Anno, Bs0, Lf, Ef, CompFun, Acc); + nomatch -> + eval_m_generate(Iter, P, Anno, Bs0, Lf, Ef, CompFun, Acc0) + end; + none -> + Acc0 + end. + eval_filter(F, Bs0, Lf, Ef, CompFun, FUVs, Acc) -> case erl_lint:is_guard_test(F) of true -> @@ -844,6 +904,11 @@ eval_filter(F, Bs0, Lf, Ef, CompFun, FUVs, Acc) -> end end. +is_generator({generate,_,_,_}) -> true; +is_generator({b_generate,_,_,_}) -> true; +is_generator({m_generate,_,_,_}) -> true; +is_generator(_) -> false. + %% eval_map_fields([Field], Bindings, LocalFunctionHandler, %% ExternalFuncHandler) -> %% {[{map_assoc | map_exact,Key,Value}],Bindings} diff --git a/lib/stdlib/src/erl_expand_records.erl b/lib/stdlib/src/erl_expand_records.erl index 7715f7d458..8cd78e597d 100644 --- a/lib/stdlib/src/erl_expand_records.erl +++ b/lib/stdlib/src/erl_expand_records.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2022. All Rights Reserved. +%% Copyright Ericsson AB 2005-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -55,8 +55,7 @@ compiler_options(Forms) -> lists:flatten([C || {attribute,_,compile,C} <- Forms]). init_calltype(Forms) -> - Locals = [{{Name,Arity},local} || {function,_,Name,Arity,_} <- Forms], - Ctype = maps:from_list(Locals), + Ctype = #{{Name,Arity} => local || {function,_,Name,Arity,_} <- Forms}, init_calltype_imports(Forms, Ctype). init_calltype_imports([{attribute,_,import,{Mod,Fs}}|T], Ctype0) -> @@ -289,6 +288,10 @@ expr({bc,Anno,E0,Qs0}, St0) -> {Qs1,St1} = lc_tq(Anno, Qs0, St0), {E1,St2} = expr(E0, St1), {{bc,Anno,E1,Qs1},St2}; +expr({mc,Anno,E0,Qs0}, St0) -> + {Qs1,St1} = lc_tq(Anno, Qs0, St0), + {E1,St2} = expr(E0, St1), + {{mc,Anno,E1,Qs1},St2}; expr({tuple,Anno,Es0}, St0) -> {Es1,St1} = expr_list(Es0, St0), {{tuple,Anno,Es1},St1}; @@ -442,7 +445,9 @@ expr({op,Anno,Op,L0,R0}, St0) when Op =:= 'andalso'; expr({op,Anno,Op,L0,R0}, St0) -> {L,St1} = expr(L0, St0), {R,St2} = expr(R0, St1), - {{op,Anno,Op,L,R},St2}. + {{op,Anno,Op,L,R},St2}; +expr(E={ssa_check_when,_,_,_,_,_}, St) -> + {E, St}. expr_list([E0 | Es0], St0) -> {E,St1} = expr(E0, St0), @@ -513,6 +518,11 @@ lc_tq(Anno, [{b_generate,AnnoG,P0,G0} | Qs0], St0) -> {P1,St2} = pattern(P0, St1), {Qs1,St3} = lc_tq(Anno, Qs0, St2), {[{b_generate,AnnoG,P1,G1} | Qs1],St3}; +lc_tq(Anno, [{m_generate,AnnoG,P0,G0} | Qs0], St0) -> + {G1,St1} = expr(G0, St0), + {P1,St2} = pattern(P0, St1), + {Qs1,St3} = lc_tq(Anno, Qs0, St2), + {[{m_generate,AnnoG,P1,G1} | Qs1],St3}; lc_tq(Anno, [F0 | Qs0], #exprec{calltype=Calltype,raw_records=Records}=St0) -> %% Allow record/2 and expand out as guard test. IsOverriden = fun(FA) -> @@ -694,24 +704,20 @@ record_wildcard_init([]) -> none. record_update(R, Name, Fs, Us0, St0) -> Anno = element(2, R), {Pre,Us,St1} = record_exprs(Us0, St0), - Nf = length(Fs), %# of record fields - Nu = length(Us), %# of update fields - Nc = Nf - Nu, %# of copy fields %% We need a new variable for the record expression %% to guarantee that it is only evaluated once. {Var,St2} = new_var(Anno, St1), + %% Honor the `strict_record_updates` option needed by `dialyzer`, otherwise + %% expand everything to chains of `setelement/3` as that's far more + %% efficient in the JIT. StrictUpdates = strict_record_updates(St2#exprec.compile), - - %% Try to be intelligent about which method of updating record to use. {Update,St} = if - Nu =:= 0 -> - record_match(Var, Name, Anno, Fs, Us, St2); - Nu =< Nc, not StrictUpdates -> %Few fields updated + not StrictUpdates, Us =/= [] -> {record_setel(Var, Name, Fs, Us), St2}; - true -> %The wide area inbetween + true -> record_match(Var, Name, Anno, Fs, Us, St2) end, {{block,Anno,Pre ++ [{match,Anno,Var,R},Update]},St}. diff --git a/lib/stdlib/src/erl_features.erl b/lib/stdlib/src/erl_features.erl index 625a9d7952..44056a27b0 100644 --- a/lib/stdlib/src/erl_features.erl +++ b/lib/stdlib/src/erl_features.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2021-2022. All Rights Reserved. +%% Copyright Ericsson AB 2021-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ %% %CopyrightEnd% %% -module(erl_features). +-feature(maybe_expr, enable). -export([all/0, configurable/0, @@ -25,7 +26,6 @@ short/1, long/1, enabled/0, - load_allowed/1, keywords/0, keywords/1, keyword_fun/2, @@ -371,7 +371,7 @@ init_features() -> end, FOps = lists:filtermap(F, FeatureOps), {Features, _, _} = collect_features(FOps), - {Enabled, Keywords} = + {Enabled0, Keywords} = lists:foldl(fun(Ftr, {Ftrs, Keys}) -> case lists:member(Ftr, Ftrs) of true -> @@ -385,6 +385,7 @@ init_features() -> Features), %% Save state + Enabled = lists:uniq(Enabled0), enabled_features(Enabled), set_keywords(Keywords), persistent_term:put({?MODULE, init_done}, true), @@ -423,32 +424,6 @@ keywords() -> set_keywords(Words) -> persistent_term:put({?MODULE, keywords}, Words). -%% Check that any features used in the module are enabled in the -%% runtime system. If not, return -%% {not_allowed, <list of not enabled features>}. --spec load_allowed(binary()) -> ok | {not_allowed, [feature()]}. -load_allowed(Binary) -> - case erts_internal:beamfile_chunk(Binary, "Meta") of - undefined -> - ok; - Meta -> - MetaData = erlang:binary_to_term(Meta), - case proplists:get_value(enabled_features, MetaData) of - undefined -> - ok; - Used -> - Enabled = enabled(), - case lists:filter(fun(UFtr) -> - not lists:member(UFtr, Enabled) - end, - Used) of - [] -> ok; - NotEnabled -> - {not_allowed, NotEnabled} - end - end - end. - %% Return features used by module or beam file -spec used(module() | file:filename()) -> [feature()]. used(Module) when is_atom(Module) -> diff --git a/lib/stdlib/src/erl_internal.erl b/lib/stdlib/src/erl_internal.erl index 92651084c7..8f0b1e43ee 100644 --- a/lib/stdlib/src/erl_internal.erl +++ b/lib/stdlib/src/erl_internal.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2022. All Rights Reserved. +%% Copyright Ericsson AB 1998-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -78,6 +78,8 @@ guard_bif(is_map_key, 2) -> true; guard_bif(length, 1) -> true; guard_bif(map_size, 1) -> true; guard_bif(map_get, 2) -> true; +guard_bif(max, 2) -> true; +guard_bif(min, 2) -> true; guard_bif(node, 0) -> true; guard_bif(node, 1) -> true; guard_bif(round, 1) -> true; @@ -564,6 +566,7 @@ is_type(bool, 0) -> true; is_type(boolean, 0) -> true; is_type(byte, 0) -> true; is_type(char, 0) -> true; +is_type(dynamic, 0) -> true; is_type(float, 0) -> true; is_type(function, 0) -> true; is_type(identifier, 0) -> true; diff --git a/lib/stdlib/src/erl_lint.erl b/lib/stdlib/src/erl_lint.erl index b9b58d6576..55e53cad3d 100644 --- a/lib/stdlib/src/erl_lint.erl +++ b/lib/stdlib/src/erl_lint.erl @@ -2,7 +2,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2022. All Rights Reserved. +%% Copyright Ericsson AB 1996-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ %% Do necessary checking of Erlang code. -module(erl_lint). +-feature(maybe_expr, enable). -export([module/1,module/2,module/3,format_error/1]). -export([exprs/2,exprs_opt/3,used_vars/2]). % Used from erl_eval.erl. @@ -314,8 +315,6 @@ format_error({too_many_arguments,Arity}) -> %% --- patterns and guards --- format_error(illegal_pattern) -> "illegal pattern"; format_error(illegal_map_key) -> "illegal map key in pattern"; -format_error(illegal_bin_pattern) -> - "binary patterns cannot be matched in parallel using '='"; format_error(illegal_expr) -> "illegal expression"; format_error({illegal_guard_local_call, {F,A}}) -> io_lib:format("call to local/imported function ~tw/~w is illegal in guard", @@ -436,12 +435,8 @@ format_error({undefined_type, {TypeName, Arity}}) -> io_lib:format("type ~tw~s undefined", [TypeName, gen_type_paren(Arity)]); format_error({unused_type, {TypeName, Arity}}) -> io_lib:format("type ~tw~s is unused", [TypeName, gen_type_paren(Arity)]); -format_error({new_builtin_type, {TypeName, Arity}}) -> - io_lib:format("type ~w~s is a new builtin type; " - "its (re)definition is allowed only until the next release", - [TypeName, gen_type_paren(Arity)]); -format_error({builtin_type, {TypeName, Arity}}) -> - io_lib:format("type ~w~s is a builtin type; it cannot be redefined", +format_error({redefine_builtin_type, {TypeName, Arity}}) -> + io_lib:format("local redefinition of built-in type: ~w~s", [TypeName, gen_type_paren(Arity)]); format_error({renamed_type, OldName, NewName}) -> io_lib:format("type ~w() is now called ~w(); " @@ -489,9 +484,6 @@ format_error({deprecated_builtin_type, {Name, Arity}, format_error({not_exported_opaque, {TypeName, Arity}}) -> io_lib:format("opaque type ~tw~s is not exported", [TypeName, gen_type_paren(Arity)]); -format_error({underspecified_opaque, {TypeName, Arity}}) -> - io_lib:format("opaque type ~tw~s is underspecified and therefore meaningless", - [TypeName, gen_type_paren(Arity)]); format_error({bad_dialyzer_attribute,Term}) -> io_lib:format("badly formed dialyzer attribute: ~tw", [Term]); format_error({bad_dialyzer_option,Term}) -> @@ -674,13 +666,19 @@ start(File, Opts) -> true, Opts)}, {keyword_warning, bool_option(warn_keywords, nowarn_keywords, - false, Opts)} + false, Opts)}, + {redefined_builtin_type, + bool_option(warn_redefined_builtin_type, nowarn_redefined_builtin_type, + true, Opts)}, + {singleton_typevar, + bool_option(warn_singleton_typevar, nowarn_singleton_typevar, + true, Opts)} ], Enabled1 = [Category || {Category,true} <- Enabled0], Enabled = ordsets:from_list(Enabled1), Calls = case ordsets:is_element(unused_function, Enabled) of true -> - maps:from_list([{{module_info,1},pseudolocals()}]); + #{{module_info,1} => pseudolocals()}; false -> undefined end, @@ -1416,7 +1414,7 @@ check_unused_records(Forms, St0) -> maps:remove(Used, Recs) end, St1#lint.records, UsedRecords), Unused = [{Name,Anno} || - {Name,{Anno,_Fields}} <- maps:to_list(URecs), + Name := {Anno,_Fields} <- URecs, element(1, loc(Anno, St1)) =:= FirstFile], foldl(fun ({N,Anno}, St) -> add_warning(Anno, {unused_record, N}, St) @@ -1749,10 +1747,9 @@ pattern({op,_Anno,'++',{string,_Ai,_S},R}, Vt, Old, St) -> pattern({match,_Anno,Pat1,Pat2}, Vt0, Old, St0) -> {Lvt, Lnew, St1} = pattern(Pat1, Vt0, Old, St0), {Rvt, Rnew, St2} = pattern(Pat2, Vt0, Old, St1), - St3 = reject_invalid_alias(Pat1, Pat2, Vt0, St2), - {Vt1, St4} = vtmerge_pat(Lvt, Rvt, St3), - {New, St5} = vtmerge_pat(Lnew, Rnew, St4), - {Vt1, New, St5}; + {Vt1, St3} = vtmerge_pat(Lvt, Rvt, St2), + {New, St4} = vtmerge_pat(Lnew, Rnew, St3), + {Vt1, New, St4}; %% Catch legal constant expressions, including unary +,-. pattern(Pat, _Vt, _Old, St) -> case is_pattern_expr(Pat) of @@ -1779,101 +1776,6 @@ check_multi_field_init(Fs, Anno, Fields, St) -> false -> St end. -%% reject_invalid_alias(Pat, Expr, Vt, St) -> St' -%% Reject aliases for binary patterns at the top level. -%% Reject aliases for maps patterns at the top level. -%% The variables table (Vt) are for maps checkking. - -reject_invalid_alias_expr({bin,_,_}=P, {match,_,P0,E}, Vt, St0) -> - St = reject_invalid_alias(P, P0, Vt, St0), - reject_invalid_alias_expr(P, E, Vt, St); -reject_invalid_alias_expr({map,_,_}=P, {match,_,P0,E}, Vt, St0) -> - St = reject_invalid_alias(P, P0, Vt, St0), - reject_invalid_alias_expr(P, E, Vt, St); -reject_invalid_alias_expr({match,_,_,_}=P, {match,_,P0,E}, Vt, St0) -> - St = reject_invalid_alias(P, P0, Vt, St0), - reject_invalid_alias_expr(P, E, Vt, St); -reject_invalid_alias_expr(_, _, _, St) -> St. - - - -%% reject_invalid_alias(Pat1, Pat2, St) -> St' -%% Aliases of binary patterns, such as <<A:8>> = <<B:4,C:4>> or even -%% <<A:8>> = <<A:8>>, are not allowed. Traverse the patterns in parallel -%% and generate an error if any binary aliases are found. -%% We generate an error even if is obvious that the overall pattern can't -%% possibly match, for instance, {a,<<A:8>>,c}={x,<<A:8>>} WILL generate an -%% error. -%% Maps should reject unbound variables here. - -reject_invalid_alias({bin,Anno,_}, {bin,_,_}, _, St) -> - add_error(Anno, illegal_bin_pattern, St); -reject_invalid_alias({map,_Anno,Ps1}, {map,_,Ps2}, Vt, St0) -> - Fun = fun ({map_field_exact,_,{var,A,K},_V}, Sti) -> - case is_var_bound(K,Vt) of - true -> - Sti; - false -> - add_error(A, {unbound_var,K}, Sti) - end; - ({map_field_exact,_A,_K,_V}, Sti) -> - Sti - end, - foldl(Fun, foldl(Fun, St0, Ps1), Ps2); -reject_invalid_alias({cons,_,H1,T1}, {cons,_,H2,T2}, Vt, St0) -> - St = reject_invalid_alias(H1, H2, Vt, St0), - reject_invalid_alias(T1, T2, Vt, St); -reject_invalid_alias({tuple,_,Es1}, {tuple,_,Es2}, Vt, St) -> - reject_invalid_alias_list(Es1, Es2, Vt, St); -reject_invalid_alias({record,_,Name1,Pfs1}, {record,_,Name2,Pfs2}, Vt, - #lint{records=Recs}=St) -> - case Recs of - #{Name1 := {_Anno1,Fields1}, Name2 := {_Anno2,Fields2}} -> - reject_invalid_alias_rec(Pfs1, Pfs2, Fields1, Fields2, Vt, St); - #{} -> - %% One or more non-existing records. (An error messages has - %% already been generated, so we are done here.) - St - end; -reject_invalid_alias({match,_,P1,P2}, P, Vt, St0) -> - St = reject_invalid_alias(P1, P, Vt, St0), - reject_invalid_alias(P2, P, Vt, St); -reject_invalid_alias(P, {match,_,_,_}=M, Vt, St) -> - reject_invalid_alias(M, P, Vt, St); -reject_invalid_alias(_P1, _P2, _Vt, St) -> St. - -reject_invalid_alias_list([E1|Es1], [E2|Es2], Vt, St0) -> - St = reject_invalid_alias(E1, E2, Vt, St0), - reject_invalid_alias_list(Es1, Es2, Vt, St); -reject_invalid_alias_list(_, _, _, St) -> St. - -reject_invalid_alias_rec(PfsA0, PfsB0, FieldsA0, FieldsB0, Vt, St) -> - %% We treat records as if they have been converted to tuples. - PfsA1 = rbia_field_vars(PfsA0), - PfsB1 = rbia_field_vars(PfsB0), - FieldsA1 = rbia_fields(lists:reverse(FieldsA0), 0, []), - FieldsB1 = rbia_fields(lists:reverse(FieldsB0), 0, []), - FieldsA = sofs:relation(FieldsA1), - PfsA = sofs:relation(PfsA1), - A = sofs:join(FieldsA, 1, PfsA, 1), - FieldsB = sofs:relation(FieldsB1), - PfsB = sofs:relation(PfsB1), - B = sofs:join(FieldsB, 1, PfsB, 1), - C = sofs:join(A, 2, B, 2), - D = sofs:projection({external,fun({_,_,P1,_,P2}) -> {P1,P2} end}, C), - E = sofs:to_external(D), - {Ps1,Ps2} = lists:unzip(E), - reject_invalid_alias_list(Ps1, Ps2, Vt, St). - -rbia_field_vars(Fs) -> - [{Name,Pat} || {record_field,_,{atom,_,Name},Pat} <- Fs]. - -rbia_fields([{record_field,_,{atom,_,Name},_}|Fs], I, Acc) -> - rbia_fields(Fs, I+1, [{Name,I}|Acc]); -rbia_fields([_|Fs], I, Acc) -> - rbia_fields(Fs, I+1, Acc); -rbia_fields([], _, Acc) -> Acc. - %% is_pattern_expr(Expression) -> boolean(). %% Test if a general expression is a valid pattern expression. @@ -1897,14 +1799,14 @@ is_pattern_expr_1({integer,_Anno,_I}) -> true; is_pattern_expr_1({float,_Anno,_F}) -> true; is_pattern_expr_1({atom,_Anno,_A}) -> true; is_pattern_expr_1({tuple,_Anno,Es}) -> - all(fun is_pattern_expr/1, Es); + all(fun is_pattern_expr_1/1, Es); is_pattern_expr_1({nil,_Anno}) -> true; is_pattern_expr_1({cons,_Anno,H,T}) -> is_pattern_expr_1(H) andalso is_pattern_expr_1(T); is_pattern_expr_1({op,_Anno,Op,A}) -> erl_internal:arith_op(Op, 1) andalso is_pattern_expr_1(A); is_pattern_expr_1({op,_Anno,Op,A1,A2}) -> - erl_internal:arith_op(Op, 2) andalso all(fun is_pattern_expr/1, [A1,A2]); + erl_internal:arith_op(Op, 2) andalso all(fun is_pattern_expr_1/1, [A1,A2]); is_pattern_expr_1(_Other) -> false. pattern_map(Ps, Vt0, Old, St0) -> @@ -2084,7 +1986,8 @@ bit_size_check(Anno, all, #bittype{type=Type}, St) -> binary -> {all,St}; _ -> {unknown,add_error(Anno, illegal_bitsize, St)} end; -bit_size_check(Anno, Size, #bittype{type=Type,unit=Unit}, St) -> +bit_size_check(Anno, Size, #bittype{type=Type,unit=Unit}, St) + when is_integer(Size), is_integer(Unit) -> Sz = Unit * Size, %Total number of bits! St2 = elemtype_check(Anno, Type, Sz, St), {Sz,St2}. @@ -2445,6 +2348,8 @@ expr({lc,_Anno,E,Qs}, Vt, St) -> handle_comprehension(E, Qs, Vt, St); expr({bc,_Anno,E,Qs}, Vt, St) -> handle_comprehension(E, Qs, Vt, St); +expr({mc,_Anno,E,Qs}, Vt, St) -> + handle_comprehension(E, Qs, Vt, St); expr({tuple,_Anno,Es}, Vt, St) -> expr_list(Es, Vt, St); expr({map,_Anno,Es}, Vt, St) -> @@ -2632,8 +2537,7 @@ expr({'catch',Anno,E}, Vt, St0) -> {vtupdate(vtunsafe({'catch',Anno}, Evt, Vt), Evt),St}; expr({match,_Anno,P,E}, Vt, St0) -> {Evt,St1} = expr(E, Vt, St0), - {Pvt,Pnew,St2} = pattern(P, vtupdate(Evt, Vt), St1), - St = reject_invalid_alias_expr(P, E, Vt, St2), + {Pvt,Pnew,St} = pattern(P, vtupdate(Evt, Vt), St1), {vtupdate(Pnew, vtmerge(Evt, Pvt)),St}; expr({maybe_match,Anno,P,E}, Vt, St0) -> expr({match,Anno,P,E}, Vt, St0); @@ -2665,7 +2569,10 @@ expr({op,_Anno,_Op,L,R}, Vt, St) -> expr_list([L,R], Vt, St); %They see the same variables %% The following are not allowed to occur anywhere! expr({remote,_Anno,M,_F}, _Vt, St) -> - {[],add_error(erl_parse:first_anno(M), illegal_expr, St)}. + {[],add_error(erl_parse:first_anno(M), illegal_expr, St)}; +expr({ssa_check_when,_Anno,_WantedResult,_Args,_Tag,_Exprs}, _Vt, St) -> + {[], St}. + %% expr_list(Expressions, Variables, State) -> %% {UsedVarTable,State} @@ -2986,104 +2893,130 @@ type_def(Attr, Anno, TypeName, ProtoType, Args, St0) -> not member(no_auto_import_types, St0#lint.compile) of true -> case is_obsolete_builtin_type(TypePair) of - true -> StoreType(St0); + true -> + StoreType(St0); false -> - case is_newly_introduced_builtin_type(TypePair) of - %% allow some types just for bootstrapping - true -> - Warn = {new_builtin_type, TypePair}, - St1 = add_warning(Anno, Warn, St0), - StoreType(St1); - false -> - add_error(Anno, {builtin_type, TypePair}, St0) - end + %% Starting from OTP 26, redefining built-in types + %% is allowed. + St1 = StoreType(St0), + warn_redefined_builtin_type(Anno, TypePair, St1) end; false -> case is_map_key(TypePair, TypeDefs) of true -> add_error(Anno, {redefine_type, TypePair}, St0); false -> - St1 = case - Attr =:= opaque andalso - is_underspecified(ProtoType, Arity) - of - true -> - Warn = {underspecified_opaque, TypePair}, - add_warning(Anno, Warn, St0); - false -> St0 - end, - StoreType(St1) + StoreType(St0) end end. -is_underspecified({type,_,term,[]}, 0) -> true; -is_underspecified({type,_,any,[]}, 0) -> true; -is_underspecified(_ProtType, _Arity) -> false. +warn_redefined_builtin_type(Anno, TypePair, #lint{compile=Opts}=St) -> + case is_warn_enabled(redefined_builtin_type, St) of + true -> + NoWarn = [Type || + {nowarn_redefined_builtin_type, Type0} <- Opts, + Type <- lists:flatten([Type0])], + case lists:member(TypePair, NoWarn) of + true -> + St; + false -> + Warn = {redefine_builtin_type, TypePair}, + add_warning(Anno, Warn, St) + end; + false -> + St + end. check_type(Types, St) -> - {SeenVars, St1} = check_type(Types, maps:new(), St), + {SeenVars, St1} = check_type_1(Types, maps:new(), St), maps:fold(fun(Var, {seen_once, Anno}, AccSt) -> case atom_to_list(Var) of "_"++_ -> AccSt; _ -> add_error(Anno, {singleton_typevar, Var}, AccSt) end; + (Var, {seen_once_union, Anno}, AccSt) -> + case is_warn_enabled(singleton_typevar, AccSt) of + true -> + case atom_to_list(Var) of + "_"++_ -> AccSt; + _ -> add_warning(Anno, {singleton_typevar, Var}, AccSt) + end; + false -> + AccSt + end; (_Var, seen_multiple, AccSt) -> AccSt end, St1, SeenVars). -check_type({ann_type, _A, [_Var, Type]}, SeenVars, St) -> - check_type(Type, SeenVars, St); -check_type({remote_type, A, [{atom, _, Mod}, {atom, _, Name}, Args]}, +check_type_1({type, Anno, TypeName, Args}=Type, SeenVars, #lint{types=Types}=St) -> + TypePair = {TypeName, + if + is_list(Args) -> length(Args); + true -> 0 + end}, + case is_map_key(TypePair, Types) of + true -> + check_type_2(Type, SeenVars, used_type(TypePair, Anno, St)); + false -> + check_type_2(Type, SeenVars, St) + end; +check_type_1(Types, SeenVars, St) -> + check_type_2(Types, SeenVars, St). + +check_type_2({ann_type, _A, [_Var, Type]}, SeenVars, St) -> + check_type_1(Type, SeenVars, St); +check_type_2({remote_type, A, [{atom, _, Mod}, {atom, _, Name}, Args]}, SeenVars, St00) -> St0 = check_module_name(Mod, A, St00), St = deprecated_type(A, Mod, Name, Args, St0), CurrentMod = St#lint.module, case Mod =:= CurrentMod of - true -> check_type({user_type, A, Name, Args}, SeenVars, St); + true -> check_type_2({user_type, A, Name, Args}, SeenVars, St); false -> lists:foldl(fun(T, {AccSeenVars, AccSt}) -> - check_type(T, AccSeenVars, AccSt) + check_type_1(T, AccSeenVars, AccSt) end, {SeenVars, St}, Args) end; -check_type({integer, _A, _}, SeenVars, St) -> {SeenVars, St}; -check_type({atom, _A, _}, SeenVars, St) -> {SeenVars, St}; -check_type({var, _A, '_'}, SeenVars, St) -> {SeenVars, St}; -check_type({var, A, Name}, SeenVars, St) -> +check_type_2({integer, _A, _}, SeenVars, St) -> {SeenVars, St}; +check_type_2({atom, _A, _}, SeenVars, St) -> {SeenVars, St}; +check_type_2({var, _A, '_'}, SeenVars, St) -> {SeenVars, St}; +check_type_2({var, A, Name}, SeenVars, St) -> NewSeenVars = case maps:find(Name, SeenVars) of {ok, {seen_once, _}} -> maps:put(Name, seen_multiple, SeenVars); + {ok, {seen_once_union, _}} -> maps:put(Name, seen_multiple, SeenVars); {ok, seen_multiple} -> SeenVars; error -> maps:put(Name, {seen_once, A}, SeenVars) end, {NewSeenVars, St}; -check_type({type, A, bool, []}, SeenVars, St) -> +check_type_2({type, A, bool, []}, SeenVars, St) -> {SeenVars, add_warning(A, {renamed_type, bool, boolean}, St)}; -check_type({type, A, 'fun', [Dom, Range]}, SeenVars, St) -> +check_type_2({type, A, 'fun', [Dom, Range]}, SeenVars, St) -> St1 = case Dom of {type, _, product, _} -> St; {type, _, any} -> St; _ -> add_error(A, {type_syntax, 'fun'}, St) end, - check_type({type, nowarn(), product, [Dom, Range]}, SeenVars, St1); -check_type({type, A, range, [From, To]}, SeenVars, St) -> + check_type_2({type, nowarn(), product, [Dom, Range]}, SeenVars, St1); +check_type_2({type, A, range, [From, To]}, SeenVars, St) -> St1 = case {erl_eval:partial_eval(From), erl_eval:partial_eval(To)} of {{integer, _, X}, {integer, _, Y}} when X < Y -> St; _ -> add_error(A, {type_syntax, range}, St) end, {SeenVars, St1}; -check_type({type, _A, map, any}, SeenVars, St) -> +check_type_2({type, _A, map, any}, SeenVars, St) -> {SeenVars, St}; -check_type({type, _A, map, Pairs}, SeenVars, St) -> +check_type_2({type, _A, map, Pairs}, SeenVars, St) -> lists:foldl(fun(Pair, {AccSeenVars, AccSt}) -> - check_type(Pair, AccSeenVars, AccSt) + check_type_2(Pair, AccSeenVars, AccSt) end, {SeenVars, St}, Pairs); -check_type({type, _A, map_field_assoc, [Dom, Range]}, SeenVars, St) -> - check_type({type, nowarn(), product, [Dom, Range]}, SeenVars, St); -check_type({type, _A, tuple, any}, SeenVars, St) -> {SeenVars, St}; -check_type({type, _A, any}, SeenVars, St) -> {SeenVars, St}; -check_type({type, A, binary, [Base, Unit]}, SeenVars, St) -> +check_type_2({type, _A, map_field_assoc, [Dom, Range]}, SeenVars, St) -> + check_type_2({type, nowarn(), product, [Dom, Range]}, SeenVars, St); +check_type_2({type, _A, tuple, any}, SeenVars, St) -> {SeenVars, St}; +check_type_2({type, _A, any}, SeenVars, St) -> {SeenVars, St}; +check_type_2({type, A, binary, [Base, Unit]}, SeenVars, St) -> St1 = case {erl_eval:partial_eval(Base), erl_eval:partial_eval(Unit)} of {{integer, _, BaseVal}, @@ -3091,20 +3024,43 @@ check_type({type, A, binary, [Base, Unit]}, SeenVars, St) -> _ -> add_error(A, {type_syntax, binary}, St) end, {SeenVars, St1}; -check_type({type, A, record, [Name|Fields]}, SeenVars, St) -> +check_type_2({type, A, record, [Name|Fields]}, SeenVars, St) -> case Name of {atom, _, Atom} -> St1 = used_record(Atom, St), check_record_types(A, Atom, Fields, SeenVars, St1); _ -> {SeenVars, add_error(A, {type_syntax, record}, St)} end; -check_type({type, _A, Tag, Args}, SeenVars, St) when Tag =:= product; - Tag =:= union; - Tag =:= tuple -> +check_type_2({type, _A, Tag, Args}=_F, SeenVars, St) when Tag =:= product; + Tag =:= tuple -> lists:foldl(fun(T, {AccSeenVars, AccSt}) -> - check_type(T, AccSeenVars, AccSt) - end, {SeenVars, St}, Args); -check_type({type, Anno, TypeName, Args}, SeenVars, St) -> + check_type_1(T, AccSeenVars, AccSt) + end, {SeenVars, St}, Args); +check_type_2({type, _A, union, Args}=_F, SeenVars0, St) -> + lists:foldl(fun(T, {AccSeenVars0, AccSt}) -> + {SeenVars1, St0} = check_type_1(T, SeenVars0, AccSt), + AccSeenVars = maps:merge_with( + fun (K, {seen_once, Anno}, {seen_once, _}) -> + case SeenVars0 of + #{K := _} -> + %% Unused outside of this union. + {seen_once, Anno}; + #{} -> + {seen_once_union, Anno} + end; + (_K, {seen_once, Anno}, {seen_once_union, _}) -> + {seen_once_union, Anno}; + (_K, {seen_once_union, _}=R, {seen_once, _}) -> R; + (_K, {seen_once_union, _}=R, {seen_once_union, _}) -> R; + (_K, {seen_once_union, _}, Else) -> Else; + (_K, {seen_once, _}, Else) -> Else; + (_K, Else, {seen_once_union, _}) -> Else; + (_K, Else, {seen_once, _}) -> Else; + (_K, Else1, _Else2) -> Else1 + end, AccSeenVars0, SeenVars1), + {AccSeenVars, St0} + end, {SeenVars0, St}, Args); +check_type_2({type, Anno, TypeName, Args}, SeenVars, St) -> #lint{module = Module, types=Types} = St, Arity = length(Args), TypePair = {TypeName, Arity}, @@ -3120,20 +3076,26 @@ check_type({type, Anno, TypeName, Args}, SeenVars, St) -> Tag = deprecated_builtin_type, W = {Tag, TypePair, Replacement, Rel}, add_warning(Anno, W, St) - end; - _ -> St - end, - check_type({type, nowarn(), product, Args}, SeenVars, St1); -check_type({user_type, A, TypeName, Args}, SeenVars, St) -> + end; + _ -> + case is_default_type(TypePair) of + true -> + used_type(TypePair, Anno, St); + false -> + St + end + end, + check_type_2({type, nowarn(), product, Args}, SeenVars, St1); +check_type_2({user_type, A, TypeName, Args}, SeenVars, St) -> Arity = length(Args), TypePair = {TypeName, Arity}, St1 = used_type(TypePair, A, St), lists:foldl(fun(T, {AccSeenVars, AccSt}) -> - check_type(T, AccSeenVars, AccSt) + check_type_1(T, AccSeenVars, AccSt) end, {SeenVars, St1}, Args); -check_type([{typed_record_field,Field,_T}|_], SeenVars, St) -> +check_type_2([{typed_record_field,Field,_T}|_], SeenVars, St) -> {SeenVars, add_error(element(2, Field), old_abstract_code, St)}; -check_type(I, SeenVars, St) -> +check_type_2(I, SeenVars, St) -> case erl_eval:partial_eval(I) of {integer,_A,_Integer} -> {SeenVars, St}; _Other -> @@ -3168,7 +3130,7 @@ check_record_types([{type, _, field_type, [{atom, Anno, FName}, Type]}|Left], false -> St1 end, %% Check Type - {NewSeenVars, St3} = check_type(Type, SeenVars, St2), + {NewSeenVars, St3} = check_type_2(Type, SeenVars, St2), NewSeenFields = ordsets:add_element(FName, SeenFields), check_record_types(Left, Name, DefFields, NewSeenVars, St3, NewSeenFields); check_record_types([], _Name, _DefFields, SeenVars, St, _SeenFields) -> @@ -3184,8 +3146,6 @@ used_type(TypePair, Anno, #lint{usage = Usage, file = File} = St) -> is_default_type({Name, NumberOfTypeVariables}) -> erl_internal:is_type(Name, NumberOfTypeVariables). -is_newly_introduced_builtin_type({Name, _}) when is_atom(Name) -> false. - is_obsolete_builtin_type(TypePair) -> obsolete_builtin_type(TypePair) =/= no. @@ -3406,7 +3366,7 @@ check_unused_types_1(Forms, #lint{types=Ts}=St) -> reached_types(#lint{usage = Usage}) -> Es = [{From, {type, To}} || - {To, UsedTs} <- maps:to_list(Usage#usage.used_types), + To := UsedTs <- Usage#usage.used_types, #used_type{at = From} <- UsedTs], Initial = initially_reached_types(Es), G = sofs:family_to_digraph(sofs:rel2fam(sofs:relation(Es))), @@ -3480,12 +3440,12 @@ is_function_dialyzer_option(Option) -> is_module_dialyzer_option(Option) -> lists:member(Option, [no_return,no_unused,no_improper_lists,no_fun_app, - no_match,no_opaque,no_fail_call,no_contracts, + no_match,no_opaque,no_fail_call,no_contracts,no_unknown, no_behaviours,no_undefined_callbacks,unmatched_returns, error_handling,race_conditions,no_missing_calls, specdiffs,overspecs,underspecs,unknown, no_underspecs,extra_return,no_extra_return, - missing_return,no_missing_return + missing_return,no_missing_return,overlapping_contract ]). %% try_catch_clauses(Scs, Ccs, In, ImportVarTable, State) -> @@ -3612,7 +3572,7 @@ icrt_export([], _, _, _, Acc) -> handle_comprehension(E, Qs, Vt0, St0) -> {Vt1, Uvt, St1} = lc_quals(Qs, Vt0, St0), - {Evt,St2} = expr(E, Vt1, St1), + {Evt,St2} = comprehension_expr(E, Vt1, St1), Vt2 = vtupdate(Evt, Vt1), %% Shadowed global variables. {_,St3} = check_old_unused_vars(Vt2, Uvt, St2), @@ -3629,6 +3589,11 @@ handle_comprehension(E, Qs, Vt0, St0) -> Vt = vt_no_unsafe(vt_no_unused(Vt4)), {Vt, St}. +comprehension_expr({map_field_assoc,_,K,V}, Vt0, St0) -> + expr_list([K,V], Vt0, St0); +comprehension_expr(E, Vt, St) -> + expr(E, Vt, St). + %% lc_quals(Qualifiers, ImportVarTable, State) -> %% {VarTable,ShadowedVarTable,State} %% Test list comprehension qualifiers, return all variables. Allow @@ -3652,6 +3617,9 @@ lc_quals([{b_generate,_Anno,P,E} | Qs], Vt0, Uvt0, St0) -> St1 = handle_bitstring_gen_pat(P,St0), {Vt,Uvt,St} = handle_generator(P,E,Vt0,Uvt0,St1), lc_quals(Qs, Vt, Uvt, St); +lc_quals([{m_generate,_Anno,P,E} | Qs], Vt0, Uvt0, St0) -> + {Vt,Uvt,St} = handle_generator(P,E,Vt0,Uvt0,St0), + lc_quals(Qs, Vt, Uvt, St); lc_quals([F|Qs], Vt, Uvt, St0) -> Info = is_guard_test2_info(St0), {Fvt,St1} = case is_guard_test2(F, Info) of @@ -3673,7 +3641,7 @@ handle_generator(P,E,Vt,Uvt,St0) -> %% Forget variables local to E immediately. Vt1 = vtupdate(vtold(Evt, Vt), Vt), {_, St2} = check_unused_vars(Evt, Vt, St1), - {Pvt,Pnew,St3} = pattern(P, Vt1, [], St2), + {Pvt,Pnew,St3} = comprehension_pattern(P, Vt1, St2), %% Have to keep fresh variables separated from used variables somehow %% in order to handle for example X = foo(), [X || <<X:X>> <- bar()]. %% 1 2 2 1 @@ -3685,6 +3653,11 @@ handle_generator(P,E,Vt,Uvt,St0) -> Vt3 = vtupdate(vtsubtract(Vt2, Pnew), Pnew), {Vt3,NUvt,St5}. +comprehension_pattern({map_field_exact,_,K,V}, Vt, St) -> + pattern_list([K,V], Vt, [], St); +comprehension_pattern(P, Vt, St) -> + pattern(P, Vt, [], St). + handle_bitstring_gen_pat({bin,_,Segments=[_|_]},St) -> case lists:last(Segments) of {bin_element,Anno,_,default,Flags} when is_list(Flags) -> @@ -3957,14 +3930,6 @@ warn_unused_vars(U, Vt, St0) -> UVt = map(fun ({V,{State,_,As}}) -> {V,{State,used,As}} end, U), {vtmerge(Vt, UVt), St1}. - -is_var_bound(V, Vt) -> - case orddict:find(V, Vt) of - {ok,{bound,_Usage,_}} -> true; - _ -> false - end. - - %% vtupdate(UpdVarTable, VarTable) -> VarTable. %% Add the variables in the updated vartable to VarTable. The variables %% will be updated with their property in UpdVarTable. The state of @@ -4252,19 +4217,18 @@ keyword_warning(Anno, Atom, St) -> %% Add warning for bad calls to io:fwrite/format functions. format_function(DefAnno, M, F, As, St) -> - case is_format_function(M, F) of - true -> - case St#lint.warn_format of - Lev when Lev > 0 -> - case check_format_1(As) of - {warn,Level,Fmt,Fas} when Level =< Lev -> - add_warning(DefAnno, {format_error,{Fmt,Fas}}, St); - {warn,Level,Anno,Fmt,Fas} when Level =< Lev -> - add_warning(Anno, {format_error,{Fmt,Fas}}, St); - _ -> St - end; - _Lev -> St - end; + maybe + true ?= is_format_function(M, F), + Lev = St#lint.warn_format, + true ?= Lev > 0, + case check_format_1(As) of + {warn,Level,Fmt,Fas} when Level =< Lev -> + add_warning(DefAnno, {format_error,{Fmt,Fas}}, St); + {warn,Level,Anno,Fmt,Fas} when Level =< Lev -> + add_warning(Anno, {format_error,{Fmt,Fas}}, St); + _ -> St + end + else false -> St end. @@ -4383,9 +4347,11 @@ extract_sequences(Fmt, Need0) -> end end. -extract_sequence(1, [$-,C|Fmt], Need) when C >= $0, C =< $9 -> +extract_sequence(1, [$-,C|Fmt], Need) + when is_integer(C), C >= $0, C =< $9 -> extract_sequence_digits(1, Fmt, Need); -extract_sequence(1, [C|Fmt], Need) when C >= $0, C =< $9 -> +extract_sequence(1, [C|Fmt], Need) + when is_integer(C), C >= $0, C =< $9 -> extract_sequence_digits(1, Fmt, Need); extract_sequence(1, [$-,$*|Fmt], Need) -> extract_sequence(2, Fmt, [int|Need]); @@ -4394,7 +4360,8 @@ extract_sequence(1, [$*|Fmt], Need) -> extract_sequence(1, Fmt, Need) -> extract_sequence(2, Fmt, Need); -extract_sequence(2, [$.,C|Fmt], Need) when C >= $0, C =< $9 -> +extract_sequence(2, [$.,C|Fmt], Need) + when is_integer(C), C >= $0, C =< $9 -> extract_sequence_digits(2, Fmt, Need); extract_sequence(2, [$.,$*|Fmt], Need) -> extract_sequence(3, Fmt, [int|Need]); @@ -4410,36 +4377,23 @@ extract_sequence(3, [$.,_|Fmt], Need) -> extract_sequence(3, Fmt, Need) -> extract_sequence(4, Fmt, Need); -extract_sequence(4, [$t, $l | Fmt], Need) -> - extract_sequence(4, [$l, $t | Fmt], Need); -extract_sequence(4, [$t, $c | Fmt], Need) -> - extract_sequence(5, [$c|Fmt], Need); -extract_sequence(4, [$t, $s | Fmt], Need) -> - extract_sequence(5, [$s|Fmt], Need); -extract_sequence(4, [$t, $p | Fmt], Need) -> - extract_sequence(5, [$p|Fmt], Need); -extract_sequence(4, [$t, $P | Fmt], Need) -> - extract_sequence(5, [$P|Fmt], Need); -extract_sequence(4, [$t, $w | Fmt], Need) -> - extract_sequence(5, [$w|Fmt], Need); -extract_sequence(4, [$t, $W | Fmt], Need) -> - extract_sequence(5, [$W|Fmt], Need); -extract_sequence(4, [$t, C | _Fmt], _Need) -> - {error,"invalid control ~t" ++ [C]}; -extract_sequence(4, [$l, $p | Fmt], Need) -> - extract_sequence(5, [$p|Fmt], Need); -extract_sequence(4, [$l, $t, $p | Fmt], Need) -> - extract_sequence(5, [$p|Fmt], Need); -extract_sequence(4, [$l, $P | Fmt], Need) -> - extract_sequence(5, [$P|Fmt], Need); -extract_sequence(4, [$l, $t, $P | Fmt], Need) -> - extract_sequence(5, [$P|Fmt], Need); -extract_sequence(4, [$l, $t, C | _Fmt], _Need) -> - {error,"invalid control ~lt" ++ [C]}; -extract_sequence(4, [$l, C | _Fmt], _Need) -> - {error,"invalid control ~l" ++ [C]}; -extract_sequence(4, Fmt, Need) -> - extract_sequence(5, Fmt, Need); +extract_sequence(4, Fmt0, Need) -> + case extract_modifiers(Fmt0, []) of + {error, _} = Error -> + Error; + {[C|Fmt], Modifiers} -> + maybe + ok ?= check_modifiers(C, Modifiers), + case ordsets:is_element($K, Modifiers) of + true -> + extract_sequence(5, [C|Fmt], ['fun'|Need]); + false -> + extract_sequence(5, [C|Fmt], Need) + end + end; + {[], _} -> + extract_sequence(5, [], Need) + end; extract_sequence(5, [C|Fmt], Need0) -> case control_type(C, Need0) of @@ -4448,11 +4402,56 @@ extract_sequence(5, [C|Fmt], Need0) -> end; extract_sequence(_, [], _Need) -> {error,"truncated"}. -extract_sequence_digits(Fld, [C|Fmt], Need) when C >= $0, C =< $9 -> +extract_sequence_digits(Fld, [C|Fmt], Need) + when is_integer(C), C >= $0, C =< $9 -> extract_sequence_digits(Fld, Fmt, Need); extract_sequence_digits(Fld, Fmt, Need) -> extract_sequence(Fld+1, Fmt, Need). +extract_modifiers([C|Fmt], Modifiers0) -> + case is_modifier(C) of + true -> + case ordsets:add_element(C, Modifiers0) of + Modifiers0 -> + {error, "repeated modifier " ++ [C]}; + Modifiers -> + extract_modifiers(Fmt, Modifiers) + end; + false -> + {[C|Fmt], Modifiers0} + end; +extract_modifiers([], Modifiers) -> + {[], Modifiers}. + +check_modifiers(C, Modifiers) -> + maybe + ok ?= check_modifiers_1("l", Modifiers, C, "Pp"), + ok ?= check_modifiers_1("lt", Modifiers, C, "cPpsWw"), + ok ?= check_modifiers_1("Kk", Modifiers, C, "PpWw") + end. + +check_modifiers_1(M, Modifiers, C, Cs) -> + case ordsets:intersection(ordsets:from_list(M), Modifiers) of + [_]=Mod -> + case lists:member(C, Cs) of + true -> + ok; + false -> + {error, "invalid modifier/control combination ~" ++ + Mod ++ [C]} + end; + [] -> + ok; + [_,_]=M -> + {error, "conflicting modifiers ~" ++ M ++ [C]} + end. + +is_modifier($k) -> true; +is_modifier($K) -> true; +is_modifier($l) -> true; +is_modifier($t) -> true; +is_modifier(_) -> false. + control_type($~, Need) -> Need; control_type($c, Need) -> [int|Need]; control_type($f, Need) -> [float|Need]; diff --git a/lib/stdlib/src/erl_parse.yrl b/lib/stdlib/src/erl_parse.yrl index 505a6b1af8..bef858cdc5 100644 --- a/lib/stdlib/src/erl_parse.yrl +++ b/lib/stdlib/src/erl_parse.yrl @@ -31,6 +31,7 @@ pat_expr pat_expr_max map_pat_expr record_pat_expr pat_argument_list pat_exprs list tail list_comprehension lc_expr lc_exprs +map_comprehension binary_comprehension tuple record_expr record_tuple record_field record_fields @@ -49,7 +50,31 @@ type_sig type_sigs type_guard type_guards fun_type binary_type type_spec spec_fun typed_exprs typed_record_fields field_types field_type map_pair_types map_pair_type bin_base_type bin_unit_type -maybe_expr maybe_match_exprs maybe_match. +maybe_expr maybe_match_exprs maybe_match +clause_body_exprs +ssa_check_anno +ssa_check_anno_clause +ssa_check_anno_clauses +ssa_check_args +ssa_check_binary_lit +ssa_check_binary_lit_bytes_ls +ssa_check_binary_lit_rest +ssa_check_clause_args +ssa_check_clause_args_ls +ssa_check_expr +ssa_check_exprs +ssa_check_fun_ref +ssa_check_list_lit +ssa_check_list_lit_ls +ssa_check_map_key +ssa_check_map_key_element +ssa_check_map_key_elements +ssa_check_map_key_list +ssa_check_map_key_tuple_elements +ssa_check_pat +ssa_check_pats +ssa_check_when_clause +ssa_check_when_clauses. Terminals char integer float atom string var @@ -67,7 +92,8 @@ char integer float atom string var '!' '=' '::' '..' '...' '?=' 'spec' 'callback' % helper -dot. +dot +'%ssa%'. Expect 0. @@ -86,6 +112,7 @@ Left 500 mult_op. Unary 600 prefix_op. Nonassoc 700 '#'. Nonassoc 800 ':'. +Nonassoc 900 clause_body_exprs. %% Types @@ -225,8 +252,7 @@ clause_args -> pat_argument_list : element(1, '$1'). clause_guard -> 'when' guard : '$2'. clause_guard -> '$empty' : []. -clause_body -> '->' exprs: '$2'. - +clause_body -> '->' clause_body_exprs: '$2'. expr -> 'catch' expr : {'catch',?anno('$1'),'$2'}. expr -> expr '=' expr : {match,first_anno('$1'),'$1','$3'}. @@ -251,6 +277,7 @@ expr_max -> atomic : '$1'. expr_max -> list : '$1'. expr_max -> binary : '$1'. expr_max -> list_comprehension : '$1'. +expr_max -> map_comprehension : '$1'. expr_max -> binary_comprehension : '$1'. expr_max -> tuple : '$1'. expr_max -> '(' expr ')' : '$2'. @@ -324,12 +351,15 @@ bit_size_expr -> expr_max : '$1'. list_comprehension -> '[' expr '||' lc_exprs ']' : {lc,?anno('$1'),'$2','$4'}. +map_comprehension -> '#' '{' map_field_assoc '||' lc_exprs '}' : + {mc,?anno('$1'),'$3','$5'}. binary_comprehension -> '<<' expr_max '||' lc_exprs '>>' : {bc,?anno('$1'),'$2','$4'}. lc_exprs -> lc_expr : ['$1']. lc_exprs -> lc_expr ',' lc_exprs : ['$1'|'$3']. lc_expr -> expr : '$1'. +lc_expr -> map_field_exact '<-' expr : {m_generate,?anno('$2'),'$1','$3'}. lc_expr -> expr '<-' expr : {generate,?anno('$2'),'$1','$3'}. lc_expr -> binary '<=' expr : {b_generate,?anno('$2'),'$1','$3'}. @@ -500,6 +530,9 @@ pat_argument_list -> '(' pat_exprs ')' : {'$2',?anno('$1')}. exprs -> expr : ['$1']. exprs -> expr ',' exprs : ['$1' | '$3']. +clause_body_exprs -> ssa_check_when_clauses exprs : '$1' ++ '$2'. +clause_body_exprs -> exprs : '$1'. + pat_exprs -> pat_expr : ['$1']. pat_exprs -> pat_expr ',' pat_exprs : ['$1' | '$3']. @@ -549,6 +582,132 @@ comp_op -> '>' : '$1'. comp_op -> '=:=' : '$1'. comp_op -> '=/=' : '$1'. +ssa_check_when_clauses -> ssa_check_when_clause : ['$1']. +ssa_check_when_clauses -> ssa_check_when_clause ssa_check_when_clauses : + ['$1'|'$2']. + +ssa_check_when_clause -> '%ssa%' atom ssa_check_clause_args_ls 'when' atom '->' + ssa_check_exprs '.' : + {ssa_check_when, ?anno('$1'), '$2', '$3', '$5', '$7'}. + +ssa_check_when_clause -> '%ssa%' ssa_check_clause_args_ls 'when' atom '->' + ssa_check_exprs '.' : + {ssa_check_when, ?anno('$1'), {atom,?anno('$1'),pass}, '$2', '$4', '$6'}. + +ssa_check_exprs -> ssa_check_expr : [add_anno_check('$1', [])]. +ssa_check_exprs -> ssa_check_expr ssa_check_anno : [add_anno_check('$1', '$2')]. +ssa_check_exprs -> ssa_check_expr ',' ssa_check_exprs : + [add_anno_check('$1', [])|'$3']. +ssa_check_exprs -> ssa_check_expr ssa_check_anno ',' ssa_check_exprs : + [add_anno_check('$1', '$2')|'$4']. + +ssa_check_anno -> '{' ssa_check_anno_clauses '}' : '$2'. + +ssa_check_anno_clauses -> ssa_check_anno_clause : ['$1']. +ssa_check_anno_clauses -> ssa_check_anno_clause ',' ssa_check_anno_clauses : + ['$1'|'$3']. + +ssa_check_anno_clause -> atom '=>' ssa_check_pat : {term, '$1', '$3'}. + +ssa_check_expr -> var '=' atom ssa_check_args : + {check_expr, ?anno('$1'), [set, '$1', '$3'|'$4']}. +ssa_check_expr -> atom ssa_check_args : + {check_expr, ?anno('$1'), [none, '$1'|'$2']}. +ssa_check_expr -> var '=' atom ':' atom ssa_check_args : + {check_expr, ?anno('$1'), [set, '$1', {'$3', '$5'}|'$6']}. +ssa_check_expr -> atom integer : + {check_expr, ?anno('$1'), build_ssa_check_label('$1', '$2')}. +ssa_check_expr -> atom var : + {check_expr, ?anno('$1'), build_ssa_check_label('$1', '$2')}. + +ssa_check_clause_args_ls -> '(' ')' : []. +ssa_check_clause_args_ls -> '(' ssa_check_clause_args ')' : '$2'. +ssa_check_clause_args_ls -> '(' '...' ')' : ['$2']. + +ssa_check_clause_args -> var : ['$1']. +ssa_check_clause_args -> var ',' ssa_check_clause_args : ['$1'|'$3']. +ssa_check_clause_args -> var ',' '...' : ['$1', '$3']. + +ssa_check_args -> '(' ')' : {[], ?anno('$1')}. +ssa_check_args -> '(' ssa_check_pats ')' : '$2'. +ssa_check_args -> '(' '...' ')' : ['$2']. + +ssa_check_pats -> ssa_check_pat : ['$1']. +ssa_check_pats -> ssa_check_pat ',' ssa_check_pats : ['$1'|'$3']. +ssa_check_pats -> ssa_check_pat ',' '...' : ['$1', '$3']. + +ssa_check_pat -> var : '$1'. +ssa_check_pat -> atom : '$1'. +ssa_check_pat -> integer : '$1'. +ssa_check_pat -> float : '$1'. +ssa_check_pat -> float '(' float ')': {float_epsilon, '$1', '$3'}. +ssa_check_pat -> ssa_check_fun_ref : '$1'. +ssa_check_pat -> '{' '}' : {tuple, ?anno('$1'), []}. +ssa_check_pat -> '{' ssa_check_pats '}' : {tuple, ?anno('$1'), '$2'}. +ssa_check_pat -> '{' '...' '}' : {tuple, ?anno('$1'), ['$2']}. +ssa_check_pat -> ssa_check_binary_lit : '$1'. +ssa_check_pat -> ssa_check_list_lit : '$1'. +ssa_check_pat -> '#' '{' '}' : {map, ?anno('$1'), []}. +ssa_check_pat -> '#' '{' ssa_check_map_key_elements '}' : {map, ?anno('$1'), '$3'}. + +ssa_check_fun_ref -> 'fun' atom '/' integer : {local_fun, '$2', '$4'}. +ssa_check_fun_ref -> 'fun' atom ':' atom '/' integer : {external_fun, '$2', '$4', '$6'}. + +ssa_check_binary_lit -> '<<' '>>' : {binary, ?anno('$1'), []}. +ssa_check_binary_lit -> '<<' ssa_check_binary_lit_bytes_ls '>>' : + {binary, ?anno('$1'), '$2'}. +ssa_check_binary_lit -> '<<' ssa_check_binary_lit_rest '>>' : + {binary, ?anno('$1'), ['$2']}. + +ssa_check_binary_lit_bytes_ls -> integer : ['$1']. +ssa_check_binary_lit_bytes_ls -> integer ',' ssa_check_binary_lit_bytes_ls : + ['$1'|'$3']. +ssa_check_binary_lit_bytes_ls -> integer ',' ssa_check_binary_lit_rest : + ['$1', '$3']. + +ssa_check_binary_lit_rest -> integer ':' integer : {'$1', '$3'}. + +ssa_check_list_lit -> '[' ']' : {list, ?anno('$1'), []}. +ssa_check_list_lit -> '[' ssa_check_list_lit_ls ']' : + {list, ?anno('$1'), '$2'}. + +ssa_check_list_lit_ls -> ssa_check_pat : ['$1']. +ssa_check_list_lit_ls -> ssa_check_pat ',' ssa_check_list_lit_ls : ['$1'|'$3']. +ssa_check_list_lit_ls -> ssa_check_pat ',' '...' : ['$1', '$3']. +ssa_check_list_lit_ls -> ssa_check_pat '|' ssa_check_pat : ['$1'|'$3']. + +ssa_check_map_key -> atom : '$1'. +ssa_check_map_key -> integer : '$1'. +ssa_check_map_key -> float : '$1'. +ssa_check_map_key -> '{' ssa_check_map_key_tuple_elements '}' : + {tuple, ?anno('$1'), '$2'}. +ssa_check_map_key -> '{' '}' : {tuple, ?anno('$1'), []}. +ssa_check_map_key -> ssa_check_binary_lit : '$1'. +ssa_check_map_key -> '[' ssa_check_map_key_list ']' : + {list, ?anno('$1'), '$2'}. +ssa_check_map_key -> '[' ']' : {list, ?anno('$1'), []}. +ssa_check_map_key -> '#' '{' '}' : {map, ?anno('$1'), []}. +ssa_check_map_key -> '#' '{' ssa_check_map_key_elements '}' : '$3'. + +ssa_check_map_key_list -> ssa_check_map_key : ['$1']. +ssa_check_map_key_list -> ssa_check_map_key ',' ssa_check_map_key_list : + ['$1'|'$3']. +ssa_check_map_key_list -> ssa_check_map_key '|' ssa_check_map_key : + ['$1'|'$3']. + +ssa_check_map_key_elements -> ssa_check_map_key_element : ['$1']. +ssa_check_map_key_elements -> ssa_check_map_key_element ',' ssa_check_map_key_elements : + ['$1'|'$3']. + +ssa_check_map_key_element -> ssa_check_map_key '=>' ssa_check_map_key: + {'$1', '$3'}. +%% ssa_check_map_key_element -> ssa_check_map_key '::' top_type: +%% {type, '$1', '$3'}. + +ssa_check_map_key_tuple_elements -> ssa_check_map_key : ['$1']. +ssa_check_map_key_tuple_elements -> ssa_check_map_key ',' ssa_check_map_key_tuple_elements: + ['$1'|'$3']. + Header "%% This file was automatically generated from the file \"erl_parse.yrl\"." "%%" @@ -679,6 +838,7 @@ Erlang code. | af_local_call() | af_remote_call() | af_list_comprehension() + | af_map_comprehension() | af_binary_comprehension() | af_block() | af_if() @@ -714,6 +874,9 @@ Erlang code. -type af_list_comprehension() :: {'lc', anno(), af_template(), af_qualifier_seq()}. +-type af_map_comprehension() :: + {'mc', anno(), af_assoc(abstract_expr()), af_qualifier_seq()}. + -type af_binary_comprehension() :: {'bc', anno(), af_template(), af_qualifier_seq()}. @@ -724,6 +887,7 @@ Erlang code. -type af_qualifier() :: af_generator() | af_filter(). -type af_generator() :: {'generate', anno(), af_pattern(), abstract_expr()} + | {'m_generate', anno(), af_assoc_exact(af_pattern()), abstract_expr()} | {'b_generate', anno(), af_pattern(), abstract_expr()}. -type af_filter() :: abstract_expr(). @@ -1529,16 +1693,16 @@ abstract_list([H|T], String, A, E) -> abstract_list(T, [H|String], A, E); false -> AbstrList = {cons,A,abstract(H, A, E),abstract(T, A, E)}, - not_string(String, AbstrList, A, E) + not_string(String, AbstrList, A) end; abstract_list([], String, A, _E) -> {string, A, lists:reverse(String)}; abstract_list(T, String, A, E) -> - not_string(String, abstract(T, A, E), A, E). + not_string(String, abstract(T, A, E), A). -not_string([C|T], Result, A, E) -> - not_string(T, {cons, A, {integer, A, C}, Result}, A, E); -not_string([], Result, _A, _E) -> +not_string([C|T], Result, A) -> + not_string(T, {cons, A, {integer, A, C}, Result}, A); +not_string([], Result, _A) -> Result. abstract_tuple_list([H|T], A, E) -> @@ -1841,4 +2005,12 @@ modify_anno1([H|T], Ac, Mf) -> modify_anno1([], Ac, _Mf) -> {[],Ac}; modify_anno1(E, Ac, _Mf) when not is_tuple(E), not is_list(E) -> {E,Ac}. +build_ssa_check_label({atom,_,label}, Lbl) -> + [label, Lbl]; +build_ssa_check_label({atom,L,_}, _) -> + return_error(L, "expected 'label'"). + +add_anno_check({check_expr,Loc,Args}, AnnoCheck) -> + {check_expr,Loc,Args,AnnoCheck}. + %% vim: ft=erlang diff --git a/lib/stdlib/src/erl_posix_msg.erl b/lib/stdlib/src/erl_posix_msg.erl index e86ba81170..2a6676aede 100644 --- a/lib/stdlib/src/erl_posix_msg.erl +++ b/lib/stdlib/src/erl_posix_msg.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2021. All Rights Reserved. +%% Copyright Ericsson AB 1997-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -172,4 +172,4 @@ message_1(exfull) -> <<"message tables full">>; message_1(nxdomain) -> <<"non-existing domain">>; message_1(exbadport) -> <<"inet_drv bad port state">>; message_1(exbadseq) -> <<"inet_drv bad request sequence">>; -message_1(_) -> <<"unknown POSIX error">>. +message_1(Other) -> <<"unknown POSIX error: ", (atom_to_binary(Other))/binary>>. diff --git a/lib/stdlib/src/erl_pp.erl b/lib/stdlib/src/erl_pp.erl index 191aa75698..50ff87643c 100644 --- a/lib/stdlib/src/erl_pp.erl +++ b/lib/stdlib/src/erl_pp.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2021. All Rights Reserved. +%% Copyright Ericsson AB 1996-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -580,12 +580,13 @@ lexpr({cons,_,H,T}, _, Opts) -> lexpr({lc,_,E,Qs}, _Prec, Opts) -> Lcl = {list,[{step,[lexpr(E, Opts),leaf(" ||")],lc_quals(Qs, Opts)}]}, {list,[{seq,$[,[],[[]],[{force_nl,leaf(" "),[Lcl]}]},$]]}; - %% {list,[{step,$[,Lcl},$]]}; lexpr({bc,_,E,Qs}, _Prec, Opts) -> P = max_prec(), Lcl = {list,[{step,[lexpr(E, P, Opts),leaf(" ||")],lc_quals(Qs, Opts)}]}, {list,[{seq,'<<',[],[[]],[{force_nl,leaf(" "),[Lcl]}]},'>>']}; - %% {list,[{step,'<<',Lcl},'>>']}; +lexpr({mc,_,E,Qs}, _Prec, Opts) -> + Lcl = {list,[{step,[map_field(E, Opts),leaf(" ||")],lc_quals(Qs, Opts)}]}, + {list,[{seq,'#{',[],[[]],[{force_nl,leaf(" "),[Lcl]}]},$}]}; lexpr({tuple,_,Elts}, _, Opts) -> tuple(Elts, Opts); lexpr({record_index, _, Name, F}, Prec, Opts) -> @@ -956,6 +957,9 @@ clauses(Type, Opts, Cs) -> lc_quals(Qs, Opts) -> {prefer_nl,[$,],lexprs(Qs, fun lc_qual/2, Opts)}. +lc_qual({m_generate,_,Pat,E}, Opts) -> + Pl = map_field(Pat, Opts), + {list,[{step,[Pl,leaf(" <-")],lexpr(E, 0, Opts)}]}; lc_qual({b_generate,_,Pat,E}, Opts) -> Pl = lexpr(Pat, 0, Opts), {list,[{step,[Pl,leaf(" <=")],lexpr(E, 0, Opts)}]}; @@ -1367,7 +1371,7 @@ wordtable() -> L = [begin {leaf,Sz,S} = leaf(W), {S,Sz} end || W <- [" ->"," =","<<",">>","[]","after","begin","case","catch", "end","fun","if","of","receive","try","when"," ::","..", - " |","maybe","else"]], + " |","maybe","else","#{"]], list_to_tuple(L). word(' ->', WT) -> element(1, WT); @@ -1390,7 +1394,8 @@ word(' ::', WT) -> element(17, WT); word('..', WT) -> element(18, WT); word(' |', WT) -> element(19, WT); word('maybe', WT) -> element(20, WT); -word('else', WT) -> element(21, WT). +word('else', WT) -> element(21, WT); +word('#{', WT) -> element(22, WT). %% Make up an unique variable name for Name that won't clash with any %% name in Used. We first try by converting the name to uppercase and diff --git a/lib/stdlib/src/erl_scan.erl b/lib/stdlib/src/erl_scan.erl index f2e9d2d7b9..b7975c6ed2 100644 --- a/lib/stdlib/src/erl_scan.erl +++ b/lib/stdlib/src/erl_scan.erl @@ -93,7 +93,7 @@ -type text_fun() :: fun((atom(), string()) -> boolean()). -type option() :: 'return' | 'return_white_spaces' | 'return_comments' | 'text' | {'reserved_word_fun', resword_fun()} - | {'text_fun', text_fun()}. + | {'text_fun', text_fun()} | {'compiler_internal', [term()]}. -type options() :: option() | [option()]. -type symbol() :: atom() | float() | integer() | string(). -type token() :: {category(), Anno :: erl_anno:anno(), symbol()} @@ -108,7 +108,11 @@ text_fun = fun(_, _) -> false end :: text_fun(), ws = false :: boolean(), comment = false :: boolean(), - has_fun = false :: boolean()}). + has_fun = false :: boolean(), + %% True if requested to parse %ssa%-check comments + checks = false :: boolean(), + %% True if we're scanning inside a %ssa%-check comment + in_check = false :: boolean()}). %%---------------------------------------------------------------------------- @@ -264,15 +268,15 @@ string_thing(_) -> "string". -define(WHITE_SPACE(C), is_integer(C) andalso (C >= $\000 andalso C =< $\s orelse C >= $\200 andalso C =< $\240)). --define(DIGIT(C), C >= $0 andalso C =< $9). --define(CHAR(C), is_integer(C), C >= 0). +-define(DIGIT(C), (is_integer(C) andalso $0 =< C andalso C =< $9)). +-define(CHAR(C), (is_integer(C) andalso 0 =< C andalso C < 16#110000)). -define(UNICODE(C), - is_integer(C) andalso + (is_integer(C) andalso (C >= 0 andalso C < 16#D800 orelse C > 16#DFFF andalso C < 16#FFFE orelse - C > 16#FFFF andalso C =< 16#10FFFF)). + C > 16#FFFF andalso C =< 16#10FFFF))). --define(UNI255(C), C >= 0, C =< 16#ff). +-define(UNI255(C), (is_integer(C) andalso 0 =< C andalso C =< 16#ff)). options(Opts0) when is_list(Opts0) -> Opts = lists:foldr(fun expand_opt/2, [], Opts0), @@ -287,6 +291,8 @@ options(Opts0) when is_list(Opts0) -> WS = proplists:get_bool(return_white_spaces, Opts), Txt = proplists:get_bool(text, Opts), TxtFunOpt = proplists:get_value(text_fun, Opts, none), + Internal = proplists:get_value(compiler_internal, Opts, []), + Checks = proplists:get_bool(ssa_checks, Internal), DefTxtFun = fun(_, _) -> Txt end, {HasFun, TxtFun} = if @@ -298,7 +304,8 @@ options(Opts0) when is_list(Opts0) -> comment = Comment, ws = WS, text_fun = TxtFun, - has_fun = HasFun}; + has_fun = HasFun, + checks = Checks}; options(Opt) -> options([Opt]). @@ -330,8 +337,8 @@ expand_opt(O, Os) -> tokens1(Cs, St, Line, Col, Toks, Fun, Any) when ?STRING(Cs); Cs =:= eof -> case Fun(Cs, St, Line, Col, Toks, Any) of - {more,{Cs0,Ncol,Ntoks,Nline,Nany,Nfun}} -> - {more,{erl_scan_continuation,Cs0,Ncol,Ntoks,Nline,St,Nany,Nfun}}; + {more,{Cs0,Nst,Ncol,Ntoks,Nline,Nany,Nfun}} -> + {more,{erl_scan_continuation,Cs0,Ncol,Ntoks,Nline,Nst,Nany,Nfun}}; {ok,Toks0,eof,Nline,Ncol} -> Res = case Toks0 of [] -> @@ -348,8 +355,8 @@ tokens1(Cs, St, Line, Col, Toks, Fun, Any) when ?STRING(Cs); Cs =:= eof -> string1(Cs, St, Line, Col, Toks) -> case scan1(Cs, St, Line, Col, Toks) of - {more,{Cs0,Ncol,Ntoks,Nline,Any,Fun}} -> - case Fun(Cs0++eof, St, Nline, Ncol, Ntoks, Any) of + {more,{Cs0,Nst,Ncol,Ntoks,Nline,Any,Fun}} -> + case Fun(Cs0++eof, Nst, Nline, Ncol, Ntoks, Any) of {ok,Toks1,_Rest,Line2,Col2} -> {ok,lists:reverse(Toks1),location(Line2, Col2)}; {{error,_,_}=Error,_Rest} -> @@ -363,7 +370,7 @@ string1(Cs, St, Line, Col, Toks) -> Error end. -scan(Cs, St, Line, Col, Toks, _) -> +scan(Cs, #erl_scan{}=St, Line, Col, Toks, _) -> scan1(Cs, St, Line, Col, Toks). scan1([$\s|Cs], St, Line, Col, Toks) when St#erl_scan.ws -> @@ -374,10 +381,6 @@ scan1([$\n|Cs], St, Line, Col, Toks) when St#erl_scan.ws -> scan_newline(Cs, St, Line, Col, Toks); scan1([$\n|Cs], St, Line, Col, Toks) -> skip_white_space(Cs, St, Line+1, new_column(Col, 1), Toks, 0); -scan1([C|Cs], St, Line, Col, Toks) when C >= $A, C =< $Z -> - scan_variable(Cs, St, Line, Col, Toks, [C]); -scan1([C|Cs], St, Line, Col, Toks) when C >= $a, C =< $z -> - scan_atom(Cs, St, Line, Col, Toks, [C]); %% Optimization: some very common punctuation characters: scan1([$,|Cs], St, Line, Col, Toks) -> tok2(Cs, St, Line, Col, Toks, ",", ',', 1); @@ -397,21 +400,29 @@ scan1([$;|Cs], St, Line, Col, Toks) -> tok2(Cs, St, Line, Col, Toks, ";", ';', 1); scan1([$_=C|Cs], St, Line, Col, Toks) -> scan_variable(Cs, St, Line, Col, Toks, [C]); -%% More punctuation characters below. +scan1([$\%=C|Cs], St, Line, Col, Toks) when St#erl_scan.checks -> + scan_check(Cs, St, Line, Col, Toks, [C]); scan1([$\%|Cs], St, Line, Col, Toks) when not St#erl_scan.comment -> skip_comment(Cs, St, Line, Col, Toks, 1); scan1([$\%=C|Cs], St, Line, Col, Toks) -> scan_comment(Cs, St, Line, Col, Toks, [C]); +%% More punctuation characters below. +scan1([C|_], _St, _Line, _Col0, _Toks) when not ?CHAR(C) -> + error({not_character,C}); +scan1([C|Cs], St, Line, Col, Toks) when C >= $A, C =< $Z -> + scan_variable(Cs, St, Line, Col, Toks, [C]); +scan1([C|Cs], St, Line, Col, Toks) when C >= $a, C =< $z -> + scan_atom(Cs, St, Line, Col, Toks, [C]); scan1([C|Cs], St, Line, Col, Toks) when ?DIGIT(C) -> scan_number(Cs, St, Line, Col, Toks, [C], no_underscore); scan1("..."++Cs, St, Line, Col, Toks) -> tok2(Cs, St, Line, Col, Toks, "...", '...', 3); -scan1(".."=Cs, _St, Line, Col, Toks) -> - {more,{Cs,Col,Toks,Line,[],fun scan/6}}; +scan1(".."=Cs, St, Line, Col, Toks) -> + {more,{Cs,St,Col,Toks,Line,[],fun scan/6}}; scan1(".."++Cs, St, Line, Col, Toks) -> tok2(Cs, St, Line, Col, Toks, "..", '..', 2); -scan1("."=Cs, _St, Line, Col, Toks) -> - {more,{Cs,Col,Toks,Line,[],fun scan/6}}; +scan1("."=Cs, St, Line, Col, Toks) -> + {more,{Cs,St,Col,Toks,Line,[],fun scan/6}}; scan1([$.=C|Cs], St, Line, Col, Toks) -> scan_dot(Cs, St, Line, Col, Toks, [C]); scan1([$"|Cs], St, Line, Col, Toks) -> %" Emacs @@ -443,8 +454,8 @@ scan1([C|Cs], St, Line, Col, Toks) when ?WHITE_SPACE(C) -> %% ?= for the maybe ... else ... end construct scan1("?="++Cs, St, Line, Col, Toks) -> tok2(Cs, St, Line, Col, Toks, "?=", '?=', 2); -scan1("?"=Cs, _St, Line, Col, Toks) -> - {more,{Cs,Col,Toks,Line,[],fun scan/6}}; +scan1("?"=Cs, St, Line, Col, Toks) -> + {more,{Cs,St,Col,Toks,Line,[],fun scan/6}}; %% << <- <= scan1("<<"++Cs, St, Line, Col, Toks) -> tok2(Cs, St, Line, Col, Toks, "<<", '<<', 2); @@ -452,62 +463,62 @@ scan1("<-"++Cs, St, Line, Col, Toks) -> tok2(Cs, St, Line, Col, Toks, "<-", '<-', 2); scan1("<="++Cs, St, Line, Col, Toks) -> tok2(Cs, St, Line, Col, Toks, "<=", '<=', 2); -scan1("<"=Cs, _St, Line, Col, Toks) -> - {more,{Cs,Col,Toks,Line,[],fun scan/6}}; +scan1("<"=Cs, St, Line, Col, Toks) -> + {more,{Cs,St,Col,Toks,Line,[],fun scan/6}}; %% >> >= scan1(">>"++Cs, St, Line, Col, Toks) -> tok2(Cs, St, Line, Col, Toks, ">>", '>>', 2); scan1(">="++Cs, St, Line, Col, Toks) -> tok2(Cs, St, Line, Col, Toks, ">=", '>=', 2); -scan1(">"=Cs, _St, Line, Col, Toks) -> - {more,{Cs,Col,Toks,Line,[],fun scan/6}}; +scan1(">"=Cs, St, Line, Col, Toks) -> + {more,{Cs,St,Col,Toks,Line,[],fun scan/6}}; %% -> -- scan1("->"++Cs, St, Line, Col, Toks) -> tok2(Cs, St, Line, Col, Toks, "->", '->', 2); scan1("--"++Cs, St, Line, Col, Toks) -> tok2(Cs, St, Line, Col, Toks, "--", '--', 2); -scan1("-"=Cs, _St, Line, Col, Toks) -> - {more,{Cs,Col,Toks,Line,[],fun scan/6}}; +scan1("-"=Cs, St, Line, Col, Toks) -> + {more,{Cs,St,Col,Toks,Line,[],fun scan/6}}; %% ++ scan1("++"++Cs, St, Line, Col, Toks) -> tok2(Cs, St, Line, Col, Toks, "++", '++', 2); -scan1("+"=Cs, _St, Line, Col, Toks) -> - {more,{Cs,Col,Toks,Line,[],fun scan/6}}; +scan1("+"=Cs, St, Line, Col, Toks) -> + {more,{Cs,St,Col,Toks,Line,[],fun scan/6}}; %% =:= =/= =< == => scan1("=:="++Cs, St, Line, Col, Toks) -> tok2(Cs, St, Line, Col, Toks, "=:=", '=:=', 3); -scan1("=:"=Cs, _St, Line, Col, Toks) -> - {more,{Cs,Col,Toks,Line,[],fun scan/6}}; +scan1("=:"=Cs, St, Line, Col, Toks) -> + {more,{Cs,St,Col,Toks,Line,[],fun scan/6}}; scan1("=/="++Cs, St, Line, Col, Toks) -> tok2(Cs, St, Line, Col, Toks, "=/=", '=/=', 3); -scan1("=/"=Cs, _St, Line, Col, Toks) -> - {more,{Cs,Col,Toks,Line,[],fun scan/6}}; +scan1("=/"=Cs, St, Line, Col, Toks) -> + {more,{Cs,St,Col,Toks,Line,[],fun scan/6}}; scan1("=<"++Cs, St, Line, Col, Toks) -> tok2(Cs, St, Line, Col, Toks, "=<", '=<', 2); scan1("=>"++Cs, St, Line, Col, Toks) -> tok2(Cs, St, Line, Col, Toks, "=>", '=>', 2); scan1("=="++Cs, St, Line, Col, Toks) -> tok2(Cs, St, Line, Col, Toks, "==", '==', 2); -scan1("="=Cs, _St, Line, Col, Toks) -> - {more,{Cs,Col,Toks,Line,[],fun scan/6}}; +scan1("="=Cs, St, Line, Col, Toks) -> + {more,{Cs,St,Col,Toks,Line,[],fun scan/6}}; %% /= scan1("/="++Cs, St, Line, Col, Toks) -> tok2(Cs, St, Line, Col, Toks, "/=", '/=', 2); -scan1("/"=Cs, _St, Line, Col, Toks) -> - {more,{Cs,Col,Toks,Line,[],fun scan/6}}; +scan1("/"=Cs, St, Line, Col, Toks) -> + {more,{Cs,St,Col,Toks,Line,[],fun scan/6}}; %% || scan1("||"++Cs, St, Line, Col, Toks) -> tok2(Cs, St, Line, Col, Toks, "||", '||', 2); -scan1("|"=Cs, _St, Line, Col, Toks) -> - {more,{Cs,Col,Toks,Line,[],fun scan/6}}; +scan1("|"=Cs, St, Line, Col, Toks) -> + {more,{Cs,St,Col,Toks,Line,[],fun scan/6}}; %% := scan1(":="++Cs, St, Line, Col, Toks) -> tok2(Cs, St, Line, Col, Toks, ":=", ':=', 2); %% :: for typed records scan1("::"++Cs, St, Line, Col, Toks) -> tok2(Cs, St, Line, Col, Toks, "::", '::', 2); -scan1(":"=Cs, _St, Line, Col, Toks) -> - {more,{Cs,Col,Toks,Line,[],fun scan/6}}; +scan1(":"=Cs, St, Line, Col, Toks) -> + {more,{Cs,St,Col,Toks,Line,[],fun scan/6}}; %% Optimization: punctuation characters less than 127: scan1([$=|Cs], St, Line, Col, Toks) -> tok2(Cs, St, Line, Col, Toks, "=", '=', 1); @@ -552,44 +563,54 @@ scan1([C|Cs], St, Line, Col, Toks) when ?UNI255(C) -> scan1([C|Cs], _St, Line, Col, _Toks) when ?CHAR(C) -> Ncol = incr_column(Col, 1), scan_error({illegal,character}, Line, Col, Line, Ncol, Cs); -scan1([]=Cs, _St, Line, Col, Toks) -> - {more,{Cs,Col,Toks,Line,[],fun scan/6}}; +scan1([]=Cs, St, Line, Col, Toks) -> + {more,{Cs,St,Col,Toks,Line,[],fun scan/6}}; scan1(eof=Cs, _St, Line, Col, Toks) -> {ok,Toks,Cs,Line,Col}. +scan_atom_fun(Cs, #erl_scan{}=St, Line, Col, Toks, Ncs) -> + scan_atom(Cs, St, Line, Col, Toks, Ncs). + scan_atom(Cs0, St, Line, Col, Toks, Ncs0) -> case scan_name(Cs0, Ncs0) of {more,Ncs} -> - {more,{[],Col,Toks,Line,Ncs,fun scan_atom/6}}; + {more,{[],St,Col,Toks,Line,Ncs,fun scan_atom_fun/6}}; {Wcs,Cs} -> - case catch list_to_atom(Wcs) of - Name when is_atom(Name) -> + try list_to_atom(Wcs) of + Name -> case (St#erl_scan.resword_fun)(Name) of true -> tok2(Cs, St, Line, Col, Toks, Wcs, Name); false -> tok3(Cs, St, Line, Col, Toks, atom, Wcs, Name) - end; - _Error -> + end + catch + _:_ -> Ncol = incr_column(Col, length(Wcs)), scan_error({illegal,atom}, Line, Col, Line, Ncol, Cs) end end. +scan_variable_fun(Cs, #erl_scan{}=St, Line, Col, Toks, Ncs) -> + scan_variable(Cs, St, Line, Col, Toks, Ncs). + scan_variable(Cs0, St, Line, Col, Toks, Ncs0) -> case scan_name(Cs0, Ncs0) of {more,Ncs} -> - {more,{[],Col,Toks,Line,Ncs,fun scan_variable/6}}; + {more,{[],St,Col,Toks,Line,Ncs,fun scan_variable_fun/6}}; {Wcs,Cs} -> - case catch list_to_atom(Wcs) of - Name when is_atom(Name) -> - tok3(Cs, St, Line, Col, Toks, var, Wcs, Name); - _Error -> + try list_to_atom(Wcs) of + Name -> + tok3(Cs, St, Line, Col, Toks, var, Wcs, Name) + catch + _:_ -> Ncol = incr_column(Col, length(Wcs)), scan_error({illegal,var}, Line, Col, Line, Ncol, Cs) end end. +scan_name([C|_]=Cs, Ncs) when not ?CHAR(C) -> + {lists:reverse(Ncs),Cs}; scan_name([C|Cs], Ncs) when C >= $a, C =< $z -> scan_name(Cs, [C|Ncs]); scan_name([C|Cs], Ncs) when C >= $A, C =< $Z -> @@ -616,6 +637,9 @@ scan_name(Cs, Ncs) -> false -> [] end). +scan_dot([C|_]=Cs, St, Line, Col, Toks, Ncs) + when St#erl_scan.in_check, C =/= $. -> + tok2(Cs, St#erl_scan{in_check=false}, Line, Col, Toks, Ncs, '.', 1); scan_dot([$%|_]=Cs, St, Line, Col, Toks, Ncs) -> Anno = anno(Line, Col, St, ?STR(dot, St, Ncs)), {ok,[{dot,Anno}|Toks],Cs,Line,incr_column(Col, 1)}; @@ -658,25 +682,36 @@ scan_newline([$\r|Cs], St, Line, Col, Toks) -> newline_end(Cs, St, Line, Col, Toks, 2, "\n\r"); scan_newline([$\f|Cs], St, Line, Col, Toks) -> newline_end(Cs, St, Line, Col, Toks, 2, "\n\f"); -scan_newline([], _St, Line, Col, Toks) -> - {more,{[$\n],Col,Toks,Line,[],fun scan/6}}; +scan_newline([], St, Line, Col, Toks) -> + {more,{[$\n],St,Col,Toks,Line,[],fun scan/6}}; scan_newline(Cs, St, Line, Col, Toks) -> scan_nl_white_space(Cs, St, Line, Col, Toks, "\n"). +scan_nl_spcs_fun(Cs, #erl_scan{}=St, Line, Col, Toks, N) + when is_integer(N) -> + scan_nl_spcs(Cs, St, Line, Col, Toks, N). + scan_nl_spcs([$\s|Cs], St, Line, Col, Toks, N) when N < 17 -> scan_nl_spcs(Cs, St, Line, Col, Toks, N+1); -scan_nl_spcs([]=Cs, _St, Line, Col, Toks, N) -> - {more,{Cs,Col,Toks,Line,N,fun scan_nl_spcs/6}}; +scan_nl_spcs([]=Cs, St, Line, Col, Toks, N) -> + {more,{Cs,St,Col,Toks,Line,N,fun scan_nl_spcs_fun/6}}; scan_nl_spcs(Cs, St, Line, Col, Toks, N) -> newline_end(Cs, St, Line, Col, Toks, N, nl_spcs(N)). +scan_nl_tabs_fun(Cs, #erl_scan{}=St, Line, Col, Toks, N) + when is_integer(N) -> + scan_nl_tabs(Cs, St, Line, Col, Toks, N). + scan_nl_tabs([$\t|Cs], St, Line, Col, Toks, N) when N < 11 -> scan_nl_tabs(Cs, St, Line, Col, Toks, N+1); -scan_nl_tabs([]=Cs, _St, Line, Col, Toks, N) -> - {more,{Cs,Col,Toks,Line,N,fun scan_nl_tabs/6}}; +scan_nl_tabs([]=Cs, St, Line, Col, Toks, N) -> + {more,{Cs,St,Col,Toks,Line,N,fun scan_nl_tabs_fun/6}}; scan_nl_tabs(Cs, St, Line, Col, Toks, N) -> newline_end(Cs, St, Line, Col, Toks, N, nl_tabs(N)). +scan_nl_white_space_fun(Cs, #erl_scan{}=St, Line, Col, Toks, Ncs) -> + scan_nl_white_space(Cs, St, Line, Col, Toks, Ncs). + %% Note: returning {more,Cont} is meaningless here; one could just as %% well return several tokens. But since tokens() scans up to a full %% stop anyway, nothing is gained by not collecting all white spaces. @@ -689,10 +724,11 @@ scan_nl_white_space([$\n|Cs], St, Line, Col, Toks, Ncs0) -> Anno = anno(Line, Col, St, ?STR(white_space, St, Ncs)), Token = {white_space,Anno,Ncs}, scan_newline(Cs, St, Line+1, new_column(Col, length(Ncs)), [Token|Toks]); -scan_nl_white_space([C|Cs], St, Line, Col, Toks, Ncs) when ?WHITE_SPACE(C) -> +scan_nl_white_space([C|Cs], St, Line, Col, Toks, Ncs) + when ?WHITE_SPACE(C) -> scan_nl_white_space(Cs, St, Line, Col, Toks, [C|Ncs]); -scan_nl_white_space([]=Cs, _St, Line, Col, Toks, Ncs) -> - {more,{Cs,Col,Toks,Line,Ncs,fun scan_nl_white_space/6}}; +scan_nl_white_space([]=Cs, St, Line, Col, Toks, Ncs) -> + {more,{Cs,St,Col,Toks,Line,Ncs,fun scan_nl_white_space_fun/6}}; scan_nl_white_space(Cs, #erl_scan{has_fun = false}=St, Line, no_col=Col, Toks, Ncs) -> Anno = anno(Line), @@ -706,40 +742,54 @@ scan_nl_white_space(Cs, St, Line, Col, Toks, Ncs0) -> newline_end(Cs, #erl_scan{has_fun = false}=St, Line, no_col=Col, Toks, _N, Ncs) -> scan1(Cs, St, Line+1, Col, [{white_space,anno(Line),Ncs}|Toks]); -newline_end(Cs, St, Line, Col, Toks, N, Ncs) -> +newline_end(Cs, #erl_scan{}=St, Line, Col, Toks, N, Ncs) -> Anno = anno(Line, Col, St, ?STR(white_space, St, Ncs)), scan1(Cs, St, Line+1, new_column(Col, N), [{white_space,Anno,Ncs}|Toks]). +scan_spcs_fun(Cs, #erl_scan{}=St, Line, Col, Toks, N) + when is_integer(N), N >= 1 -> + scan_spcs(Cs, St, Line, Col, Toks, N). + scan_spcs([$\s|Cs], St, Line, Col, Toks, N) when N < 16 -> scan_spcs(Cs, St, Line, Col, Toks, N+1); -scan_spcs([]=Cs, _St, Line, Col, Toks, N) -> - {more,{Cs,Col,Toks,Line,N,fun scan_spcs/6}}; +scan_spcs([]=Cs, St, Line, Col, Toks, N) -> + {more,{Cs,St,Col,Toks,Line,N,fun scan_spcs_fun/6}}; scan_spcs(Cs, St, Line, Col, Toks, N) -> white_space_end(Cs, St, Line, Col, Toks, N, spcs(N)). +scan_tabs_fun(Cs, #erl_scan{}=St, Line, Col, Toks, N) + when is_integer(N), N >= 1 -> + scan_tabs(Cs, St, Line, Col, Toks, N). + scan_tabs([$\t|Cs], St, Line, Col, Toks, N) when N < 10 -> scan_tabs(Cs, St, Line, Col, Toks, N+1); -scan_tabs([]=Cs, _St, Line, Col, Toks, N) -> - {more,{Cs,Col,Toks,Line,N,fun scan_tabs/6}}; +scan_tabs([]=Cs, St, Line, Col, Toks, N) -> + {more,{Cs,St,Col,Toks,Line,N,fun scan_tabs_fun/6}}; scan_tabs(Cs, St, Line, Col, Toks, N) -> white_space_end(Cs, St, Line, Col, Toks, N, tabs(N)). +skip_white_space_fun(Cs, #erl_scan{}=St, Line, Col, Toks, N) -> + skip_white_space(Cs, St, Line, Col, Toks, N). + skip_white_space([$\n|Cs], St, Line, Col, Toks, _N) -> skip_white_space(Cs, St, Line+1, new_column(Col, 1), Toks, 0); skip_white_space([C|Cs], St, Line, Col, Toks, N) when ?WHITE_SPACE(C) -> skip_white_space(Cs, St, Line, Col, Toks, N+1); -skip_white_space([]=Cs, _St, Line, Col, Toks, N) -> - {more,{Cs,Col,Toks,Line,N,fun skip_white_space/6}}; +skip_white_space([]=Cs, St, Line, Col, Toks, N) -> + {more,{Cs,St,Col,Toks,Line,N,fun skip_white_space_fun/6}}; skip_white_space(Cs, St, Line, Col, Toks, N) -> scan1(Cs, St, Line, incr_column(Col, N), Toks). +scan_white_space_fun(Cs, #erl_scan{}=St, Line, Col, Toks, Ncs) -> + scan_white_space(Cs, St, Line, Col, Toks, Ncs). + %% Maybe \t and \s should break the loop. scan_white_space([$\n|_]=Cs, St, Line, Col, Toks, Ncs) -> white_space_end(Cs, St, Line, Col, Toks, length(Ncs), lists:reverse(Ncs)); scan_white_space([C|Cs], St, Line, Col, Toks, Ncs) when ?WHITE_SPACE(C) -> scan_white_space(Cs, St, Line, Col, Toks, [C|Ncs]); -scan_white_space([]=Cs, _St, Line, Col, Toks, Ncs) -> - {more,{Cs,Col,Toks,Line,Ncs,fun scan_white_space/6}}; +scan_white_space([]=Cs, St, Line, Col, Toks, Ncs) -> + {more,{Cs,St,Col,Toks,Line,Ncs,fun scan_white_space_fun/6}}; scan_white_space(Cs, St, Line, Col, Toks, Ncs) -> white_space_end(Cs, St, Line, Col, Toks, length(Ncs), lists:reverse(Ncs)). @@ -751,7 +801,7 @@ white_space_end(Cs, St, Line, Col, Toks, N, Ncs) -> scan_char([$\\|Cs]=Cs0, St, Line, Col, Toks) -> case scan_escape(Cs, incr_column(Col, 2)) of more -> - {more,{[$$|Cs0],Col,Toks,Line,[],fun scan/6}}; + {more,{[$$|Cs0],St,Col,Toks,Line,[],fun scan/6}}; {error,Ncs,Error,Ncol} -> scan_error(Error, Line, Col, Line, Ncol, Ncs); {eof,Ncol} -> @@ -773,16 +823,16 @@ scan_char([C|Cs], St, Line, Col, Toks) when ?UNICODE(C) -> scan1(Cs, St, Line, incr_column(Col, 2), [{char,Anno,C}|Toks]); scan_char([C|_Cs], _St, Line, Col, _Toks) when ?CHAR(C) -> scan_error({illegal,character}, Line, Col, Line, incr_column(Col, 1), eof); -scan_char([], _St, Line, Col, Toks) -> - {more,{[$$],Col,Toks,Line,[],fun scan/6}}; +scan_char([], St, Line, Col, Toks) -> + {more,{[$$],St,Col,Toks,Line,[],fun scan/6}}; scan_char(eof, _St, Line, Col, _Toks) -> scan_error(char, Line, Col, Line, incr_column(Col, 1), eof). -scan_string(Cs, St, Line, Col, Toks, {Wcs,Str,Line0,Col0}) -> +scan_string(Cs, #erl_scan{}=St, Line, Col, Toks, {Wcs,Str,Line0,Col0}) -> case scan_string0(Cs, St, Line, Col, $\", Str, Wcs) of %" {more,Ncs,Nline,Ncol,Nstr,Nwcs} -> State = {Nwcs,Nstr,Line0,Col0}, - {more,{Ncs,Ncol,Toks,Nline,State,fun scan_string/6}}; + {more,{Ncs,St,Ncol,Toks,Nline,State,fun scan_string/6}}; {char_error,Ncs,Error,Nline,Ncol,EndCol} -> scan_error(Error, Nline, Ncol, Nline, EndCol, Ncs); {error,Nline,Ncol,Nwcs,Ncs} -> @@ -793,22 +843,23 @@ scan_string(Cs, St, Line, Col, Toks, {Wcs,Str,Line0,Col0}) -> scan1(Ncs, St, Nline, Ncol, [{string,Anno,Nwcs}|Toks]) end. -scan_qatom(Cs, St, Line, Col, Toks, {Wcs,Str,Line0,Col0}) -> +scan_qatom(Cs, #erl_scan{}=St, Line, Col, Toks, {Wcs,Str,Line0,Col0}) -> case scan_string0(Cs, St, Line, Col, $\', Str, Wcs) of %' {more,Ncs,Nline,Ncol,Nstr,Nwcs} -> State = {Nwcs,Nstr,Line0,Col0}, - {more,{Ncs,Ncol,Toks,Nline,State,fun scan_qatom/6}}; + {more,{Ncs,St,Ncol,Toks,Nline,State,fun scan_qatom/6}}; {char_error,Ncs,Error,Nline,Ncol,EndCol} -> scan_error(Error, Nline, Ncol, Nline, EndCol, Ncs); {error,Nline,Ncol,Nwcs,Ncs} -> Estr = string:slice(Nwcs, 0, 16), % Expanded escape chars. scan_error({string,$\',Estr}, Line0, Col0, Nline, Ncol, Ncs); %' - {Ncs,Nline,Ncol,Nstr,Nwcs} -> - case catch list_to_atom(Nwcs) of + {Ncs,Nline,Ncol,Nstr,Nwcs} -> + try list_to_atom(Nwcs) of A when is_atom(A) -> Anno = anno(Line0, Col0, St, ?STR(atom, St, Nstr)), - scan1(Ncs, St, Nline, Ncol, [{atom,Anno,A}|Toks]); - _ -> + scan1(Ncs, St, Nline, Ncol, [{atom,Anno,A}|Toks]) + catch + _:_ -> scan_error({illegal,atom}, Line0, Col0, Nline, Ncol, Ncs) end end. @@ -884,10 +935,11 @@ scan_string1([]=Cs, Line, Col, _Q, Str, Wcs) -> scan_string1(eof, Line, Col, _Q, _Str, Wcs) -> {error,Line,Col,lists:reverse(Wcs),eof}. --define(OCT(C), C >= $0, C =< $7). --define(HEX(C), C >= $0 andalso C =< $9 orelse - C >= $A andalso C =< $F orelse - C >= $a andalso C =< $f). +-define(OCT(C), (is_integer(C) andalso $0 =< C andalso C =< $7)). +-define(HEX(C), (is_integer(C) andalso + (C >= $0 andalso C =< $9 orelse + C >= $A andalso C =< $F orelse + C >= $a andalso C =< $f))). %% \<1-3> octal digits scan_escape([O1,O2,O3|Cs], Col) when ?OCT(O1), ?OCT(O2), ?OCT(O3) -> @@ -917,12 +969,14 @@ scan_escape([$x,H1], _Col) when ?HEX(H1) -> more; scan_escape([$x|Cs], Col) -> {error,Cs,{illegal,character},incr_column(Col, 1)}; -%% \^X -> CTL-X -scan_escape([$^=C0,$\n=C|Cs], Col) -> - {nl,C,[C0,C],Cs,new_column(Col, 1)}; +%% \^X -> Control-X scan_escape([$^=C0,C|Cs], Col) when ?CHAR(C) -> - Val = C band 31, - {Val,[C0,C],Cs,incr_column(Col, 2)}; + case caret_char_code(C) of + error -> + {error,[C|Cs],{illegal,character},incr_column(Col, 1)}; + Code -> + {Code,[C0,C],Cs,incr_column(Col, 2)} + end; scan_escape([$^], _Col) -> more; scan_escape([$^|eof], Col) -> @@ -939,26 +993,31 @@ scan_escape([], _Col) -> scan_escape(eof, Col) -> {eof,Col}. -scan_hex([C|Cs], no_col=Col, Wcs) when ?HEX(C) -> - scan_hex(Cs, Col, [C|Wcs]); scan_hex([C|Cs], Col, Wcs) when ?HEX(C) -> - scan_hex(Cs, Col+1, [C|Wcs]); + scan_hex(Cs, incr_column(Col, 1), [C|Wcs]); scan_hex(Cs, Col, Wcs) -> - scan_esc_end(Cs, Col, Wcs, 16, "x{"). + scan_hex_end(Cs, Col, Wcs, "x{"). -scan_esc_end([$}|Cs], Col, Wcs0, B, Str0) -> +scan_hex_end([$}|Cs], Col, [], _Str) -> + %% Empty escape sequence. + {error,Cs,{illegal,character},incr_column(Col, 1)}; +scan_hex_end([$}|Cs], Col, Wcs0, Str0) -> Wcs = lists:reverse(Wcs0), - case catch erlang:list_to_integer(Wcs, B) of + try list_to_integer(Wcs, 16) of Val when ?UNICODE(Val) -> {Val,Str0++Wcs++[$}],Cs,incr_column(Col, 1)}; - _ -> + _Val -> + {error,Cs,{illegal,character},incr_column(Col, 1)} + catch + error:system_limit -> + %% Extremely unlikely to occur in practice. {error,Cs,{illegal,character},incr_column(Col, 1)} end; -scan_esc_end([], _Col, _Wcs, _B, _Str0) -> +scan_hex_end([], _Col, _Wcs, _Str0) -> more; -scan_esc_end(eof, Col, _Wcs, _B, _Str0) -> +scan_hex_end(eof, Col, _Wcs, _Str0) -> {eof,Col}; -scan_esc_end(Cs, Col, _Wcs, _B, _Str0) -> +scan_hex_end(Cs, Col, _Wcs, _Str0) -> {error,Cs,{illegal,character},Col}. escape_char($n) -> $\n; % \n = LF @@ -972,7 +1031,11 @@ escape_char($s) -> $\s; % \s = SPC escape_char($d) -> $\d; % \d = DEL escape_char(C) -> C. -scan_number(Cs, St, Line, Col, Toks, {Ncs, Us}) -> +caret_char_code($?) -> 16#7f; +caret_char_code(C) when $@ =< C, C =< $_; $a =< C, C =< $z -> C band 16#1f; +caret_char_code(_) -> error. + +scan_number(Cs, #erl_scan{}=St, Line, Col, Toks, {Ncs, Us}) -> scan_number(Cs, St, Line, Col, Toks, Ncs, Us). scan_number([C|Cs], St, Line, Col, Toks, Ncs, Us) when ?DIGIT(C) -> @@ -980,30 +1043,36 @@ scan_number([C|Cs], St, Line, Col, Toks, Ncs, Us) when ?DIGIT(C) -> scan_number([$_,Next|Cs], St, Line, Col, Toks, [Prev|_]=Ncs, _Us) when ?DIGIT(Next) andalso ?DIGIT(Prev) -> scan_number(Cs, St, Line, Col, Toks, [Next,$_|Ncs], with_underscore); -scan_number([$_]=Cs, _St, Line, Col, Toks, Ncs, Us) -> - {more,{Cs,Col,Toks,Line,{Ncs,Us},fun scan_number/6}}; +scan_number([$_]=Cs, St, Line, Col, Toks, Ncs, Us) -> + {more,{Cs,St,Col,Toks,Line,{Ncs,Us},fun scan_number/6}}; scan_number([$.,C|Cs], St, Line, Col, Toks, Ncs, Us) when ?DIGIT(C) -> scan_fraction(Cs, St, Line, Col, Toks, [C,$.|Ncs], Us); -scan_number([$.]=Cs, _St, Line, Col, Toks, Ncs, Us) -> - {more,{Cs,Col,Toks,Line,{Ncs,Us},fun scan_number/6}}; +scan_number([$.]=Cs, St, Line, Col, Toks, Ncs, Us) -> + {more,{Cs,St,Col,Toks,Line,{Ncs,Us},fun scan_number/6}}; scan_number([$#|Cs]=Cs0, St, Line, Col, Toks, Ncs0, Us) -> Ncs = lists:reverse(Ncs0), - case catch list_to_integer(remove_digit_separators(Ncs, Us)) of - B when B >= 2, B =< 1+$Z-$A+10 -> + try list_to_integer(remove_digit_separators(Ncs, Us)) of + B when is_integer(B), 2 =< B, B =< 1+$Z-$A+10 -> Bcs = Ncs++[$#], scan_based_int(Cs, St, Line, Col, Toks, B, [], Bcs, no_underscore); - B -> + B when is_integer(B) -> Len = length(Ncs), scan_error({base,B}, Line, Col, Line, incr_column(Col, Len), Cs0) + catch + error:system_limit -> + %% Extremely unlikely to occur in practice. + scan_error({illegal,base}, Line, Col, Line, Col, Cs0) end; -scan_number([]=Cs, _St, Line, Col, Toks, Ncs, Us) -> - {more,{Cs,Col,Toks,Line,{Ncs,Us},fun scan_number/6}}; +scan_number([]=Cs, St, Line, Col, Toks, Ncs, Us) -> + {more,{Cs,St,Col,Toks,Line,{Ncs,Us},fun scan_number/6}}; scan_number(Cs, St, Line, Col, Toks, Ncs0, Us) -> Ncs = lists:reverse(Ncs0), - case catch list_to_integer(remove_digit_separators(Ncs, Us)) of - N when is_integer(N) -> - tok3(Cs, St, Line, Col, Toks, integer, Ncs, N); - _ -> + try list_to_integer(remove_digit_separators(Ncs, Us), 10) of + N -> + tok3(Cs, St, Line, Col, Toks, integer, Ncs, N) + catch + error:system_limit -> + %% Extremely unlikely to occur in practice. Ncol = incr_column(Col, length(Ncs)), scan_error({illegal,integer}, Line, Col, Line, Ncol, Cs) end. @@ -1014,11 +1083,14 @@ remove_digit_separators(Number, with_underscore) -> [C || C <- Number, C =/= $_]. -define(BASED_DIGIT(C, B), - ((?DIGIT(C) andalso C < $0 + B) - orelse (C >= $A andalso B > 10 andalso C < $A + B - 10) - orelse (C >= $a andalso B > 10 andalso C < $a + B - 10))). - -scan_based_int(Cs, St, Line, Col, Toks, {B,NCs,BCs,Us}) -> + (is_integer(C) + andalso + ((?DIGIT(C) andalso C < $0 + B) + orelse (C >= $A andalso B > 10 andalso C < $A + B - 10) + orelse (C >= $a andalso B > 10 andalso C < $a + B - 10)))). + +scan_based_int(Cs, #erl_scan{}=St, Line, Col, Toks, {B,NCs,BCs,Us}) + when is_integer(B), 2 =< B, B =< 1+$Z-$A+10 -> scan_based_int(Cs, St, Line, Col, Toks, B, NCs, BCs, Us). scan_based_int([C|Cs], St, Line, Col, Toks, B, Ncs, Bcs, Us) when @@ -1028,22 +1100,29 @@ scan_based_int([$_,Next|Cs], St, Line, Col, Toks, B, [Prev|_]=Ncs, Bcs, _Us) when ?BASED_DIGIT(Next, B) andalso ?BASED_DIGIT(Prev, B) -> scan_based_int(Cs, St, Line, Col, Toks, B, [Next,$_|Ncs], Bcs, with_underscore); -scan_based_int([$_]=Cs, _St, Line, Col, Toks, B, NCs, BCs, Us) -> - {more,{Cs,Col,Toks,Line,{B,NCs,BCs,Us},fun scan_based_int/6}}; -scan_based_int([]=Cs, _St, Line, Col, Toks, B, NCs, BCs, Us) -> - {more,{Cs,Col,Toks,Line,{B,NCs,BCs,Us},fun scan_based_int/6}}; -scan_based_int(Cs, St, Line, Col, Toks, B, Ncs0, Bcs, Us) -> +scan_based_int([$_]=Cs, St, Line, Col, Toks, B, NCs, BCs, Us) -> + {more,{Cs,St,Col,Toks,Line,{B,NCs,BCs,Us},fun scan_based_int/6}}; +scan_based_int([]=Cs, St, Line, Col, Toks, B, NCs, BCs, Us) -> + {more,{Cs,St,Col,Toks,Line,{B,NCs,BCs,Us},fun scan_based_int/6}}; +scan_based_int(Cs, _St, Line, Col, _Toks, _B, [], Bcs, _Us) -> + %% No actual digits following the base. + Len = length(Bcs), + Ncol = incr_column(Col, Len), + scan_error({illegal,integer}, Line, Col, Line, Ncol, Cs); +scan_based_int(Cs, St, Line, Col, Toks, B, Ncs0, [_|_]=Bcs, Us) -> Ncs = lists:reverse(Ncs0), - case catch erlang:list_to_integer(remove_digit_separators(Ncs, Us), B) of - N when is_integer(N) -> - tok3(Cs, St, Line, Col, Toks, integer, Bcs++Ncs, N); - _ -> + try list_to_integer(remove_digit_separators(Ncs, Us), B) of + N -> + tok3(Cs, St, Line, Col, Toks, integer, Bcs++Ncs, N) + catch + error:system_limit -> + %% Extremely unlikely to occur in practice. Len = length(Bcs)+length(Ncs), Ncol = incr_column(Col, Len), scan_error({illegal,integer}, Line, Col, Line, Ncol, Cs) end. -scan_fraction(Cs, St, Line, Col, Toks, {Ncs,Us}) -> +scan_fraction(Cs, #erl_scan{}=St, Line, Col, Toks, {Ncs,Us}) -> scan_fraction(Cs, St, Line, Col, Toks, Ncs, Us). scan_fraction([C|Cs], St, Line, Col, Toks, Ncs, Us) when ?DIGIT(C) -> @@ -1051,27 +1130,27 @@ scan_fraction([C|Cs], St, Line, Col, Toks, Ncs, Us) when ?DIGIT(C) -> scan_fraction([$_,Next|Cs], St, Line, Col, Toks, [Prev|_]=Ncs, _Us) when ?DIGIT(Next) andalso ?DIGIT(Prev) -> scan_fraction(Cs, St, Line, Col, Toks, [Next,$_|Ncs], with_underscore); -scan_fraction([$_]=Cs, _St, Line, Col, Toks, Ncs, Us) -> - {more,{Cs,Col,Toks,Line,{Ncs,Us},fun scan_fraction/6}}; +scan_fraction([$_]=Cs, St, Line, Col, Toks, Ncs, Us) -> + {more,{Cs,St,Col,Toks,Line,{Ncs,Us},fun scan_fraction/6}}; scan_fraction([E|Cs], St, Line, Col, Toks, Ncs, Us) when E =:= $e; E =:= $E -> scan_exponent_sign(Cs, St, Line, Col, Toks, [E|Ncs], Us); -scan_fraction([]=Cs, _St, Line, Col, Toks, Ncs, Us) -> - {more,{Cs,Col,Toks,Line,{Ncs,Us},fun scan_fraction/6}}; +scan_fraction([]=Cs, St, Line, Col, Toks, Ncs, Us) -> + {more,{Cs,St,Col,Toks,Line,{Ncs,Us},fun scan_fraction/6}}; scan_fraction(Cs, St, Line, Col, Toks, Ncs, Us) -> float_end(Cs, St, Line, Col, Toks, Ncs, Us). -scan_exponent_sign(Cs, St, Line, Col, Toks, {Ncs, Us}) -> +scan_exponent_sign(Cs, #erl_scan{}=St, Line, Col, Toks, {Ncs, Us}) -> scan_exponent_sign(Cs, St, Line, Col, Toks, Ncs, Us). scan_exponent_sign([C|Cs], St, Line, Col, Toks, Ncs, Us) when C =:= $+; C =:= $- -> scan_exponent(Cs, St, Line, Col, Toks, [C|Ncs], Us); -scan_exponent_sign([]=Cs, _St, Line, Col, Toks, Ncs, Us) -> - {more,{Cs,Col,Toks,Line,{Ncs,Us},fun scan_exponent_sign/6}}; +scan_exponent_sign([]=Cs, St, Line, Col, Toks, Ncs, Us) -> + {more,{Cs,St,Col,Toks,Line,{Ncs,Us},fun scan_exponent_sign/6}}; scan_exponent_sign(Cs, St, Line, Col, Toks, Ncs, Us) -> scan_exponent(Cs, St, Line, Col, Toks, Ncs, Us). -scan_exponent(Cs, St, Line, Col, Toks, {Ncs, Us}) -> +scan_exponent(Cs, #erl_scan{}=St, Line, Col, Toks, {Ncs, Us}) -> scan_exponent(Cs, St, Line, Col, Toks, Ncs, Us). scan_exponent([C|Cs], St, Line, Col, Toks, Ncs, Us) when ?DIGIT(C) -> @@ -1079,23 +1158,27 @@ scan_exponent([C|Cs], St, Line, Col, Toks, Ncs, Us) when ?DIGIT(C) -> scan_exponent([$_,Next|Cs], St, Line, Col, Toks, [Prev|_]=Ncs, _) when ?DIGIT(Next) andalso ?DIGIT(Prev) -> scan_exponent(Cs, St, Line, Col, Toks, [Next,$_|Ncs], with_underscore); -scan_exponent([$_]=Cs, _St, Line, Col, Toks, Ncs, Us) -> - {more,{Cs,Col,Toks,Line,{Ncs,Us},fun scan_exponent/6}}; -scan_exponent([]=Cs, _St, Line, Col, Toks, Ncs, Us) -> - {more,{Cs,Col,Toks,Line,{Ncs,Us},fun scan_exponent/6}}; +scan_exponent([$_]=Cs, St, Line, Col, Toks, Ncs, Us) -> + {more,{Cs,St,Col,Toks,Line,{Ncs,Us},fun scan_exponent/6}}; +scan_exponent([]=Cs, St, Line, Col, Toks, Ncs, Us) -> + {more,{Cs,St,Col,Toks,Line,{Ncs,Us},fun scan_exponent/6}}; scan_exponent(Cs, St, Line, Col, Toks, Ncs, Us) -> float_end(Cs, St, Line, Col, Toks, Ncs, Us). float_end(Cs, St, Line, Col, Toks, Ncs0, Us) -> Ncs = lists:reverse(Ncs0), - case catch list_to_float(remove_digit_separators(Ncs, Us)) of - F when is_float(F) -> - tok3(Cs, St, Line, Col, Toks, float, Ncs, F); - _ -> + try list_to_float(remove_digit_separators(Ncs, Us)) of + F -> + tok3(Cs, St, Line, Col, Toks, float, Ncs, F) + catch + _:_ -> Ncol = incr_column(Col, length(Ncs)), scan_error({illegal,float}, Line, Col, Line, Ncol, Cs) end. +skip_comment_fun(Cs, #erl_scan{}=St, Line, Col, Toks, N) -> + skip_comment(Cs, St, Line, Col, Toks, N). + skip_comment([C|Cs], St, Line, Col, Toks, N) when C =/= $\n, ?CHAR(C) -> case ?UNICODE(C) of true -> @@ -1104,12 +1187,16 @@ skip_comment([C|Cs], St, Line, Col, Toks, N) when C =/= $\n, ?CHAR(C) -> Ncol = incr_column(Col, N+1), scan_error({illegal,character}, Line, Col, Line, Ncol, Cs) end; -skip_comment([]=Cs, _St, Line, Col, Toks, N) -> - {more,{Cs,Col,Toks,Line,N,fun skip_comment/6}}; +skip_comment([]=Cs, St, Line, Col, Toks, N) -> + {more,{Cs,St,Col,Toks,Line,N,fun skip_comment_fun/6}}; skip_comment(Cs, St, Line, Col, Toks, N) -> scan1(Cs, St, Line, incr_column(Col, N), Toks). -scan_comment([C|Cs], St, Line, Col, Toks, Ncs) when C =/= $\n, ?CHAR(C) -> +scan_comment_fun(Cs, #erl_scan{}=St, Line, Col, Toks, Ncs) -> + scan_comment(Cs, St, Line, Col, Toks, Ncs). + +scan_comment([C|Cs], St, Line, Col, Toks, Ncs) + when C =/= $\n, ?CHAR(C) -> case ?UNICODE(C) of true -> scan_comment(Cs, St, Line, Col, Toks, [C|Ncs]); @@ -1117,34 +1204,76 @@ scan_comment([C|Cs], St, Line, Col, Toks, Ncs) when C =/= $\n, ?CHAR(C) -> Ncol = incr_column(Col, length(Ncs)+1), scan_error({illegal,character}, Line, Col, Line, Ncol, Cs) end; -scan_comment([]=Cs, _St, Line, Col, Toks, Ncs) -> - {more,{Cs,Col,Toks,Line,Ncs,fun scan_comment/6}}; +scan_comment([]=Cs, St, Line, Col, Toks, Ncs) -> + {more,{Cs,St,Col,Toks,Line,Ncs,fun scan_comment_fun/6}}; scan_comment(Cs, St, Line, Col, Toks, Ncs0) -> Ncs = lists:reverse(Ncs0), tok3(Cs, St, Line, Col, Toks, comment, Ncs, Ncs). +scan_check("%%ssa%" ++ Cs, St, Line, Col, Toks, _Ncs) -> + scan_check1(Cs, St, Line, Toks, Col, 7); +scan_check("%%ssa"=Cs, St, Line, Col, Toks, Ncs) -> + {more,{Cs,St,Col,Toks,Line,Ncs,fun scan_check/6}}; +scan_check("%%ss"=Cs, St, Line, Col, Toks, Ncs) -> + {more,{Cs,St,Col,Toks,Line,Ncs,fun scan_check/6}}; +scan_check("%%s"=Cs, St, Line, Col, Toks, Ncs) -> + {more,{Cs,St,Col,Toks,Line,Ncs,fun scan_check/6}}; +scan_check("%%"=Cs, St, Line, Col, Toks, Ncs) -> + {more,{Cs,St,Col,Toks,Line,Ncs,fun scan_check/6}}; +scan_check("%"=Cs, St, Line, Col, Toks, Ncs) -> + {more,{Cs,St,Col,Toks,Line,Ncs,fun scan_check/6}}; +scan_check("%ssa%" ++ Cs, St, Line, Col, Toks, _Ncs) -> + scan_check1(Cs, St, Line, Toks, Col, 6); +scan_check("%ssa"=Cs, St, Line, Col, Toks, Ncs) -> + {more,{Cs,St,Col,Toks,Line,Ncs,fun scan_check/6}}; +scan_check("%ss"=Cs, St, Line, Col, Toks, Ncs) -> + {more,{Cs,St,Col,Toks,Line,Ncs,fun scan_check/6}}; +scan_check("%s"=Cs, St, Line, Col, Toks, Ncs) -> + {more,{Cs,St,Col,Toks,Line,Ncs,fun scan_check/6}}; +scan_check("ssa%" ++ Cs, St, Line, Col, Toks, _Ncs) -> + scan_check1(Cs, St, Line, Toks, Col, 5); +scan_check("ssa"=Cs, St, Line, Col, Toks, Ncs) -> + {more,{Cs,St,Col,Toks,Line,Ncs,fun scan_check/6}}; +scan_check("ss"=Cs, St, Line, Col, Toks, Ncs) -> + {more,{Cs,St,Col,Toks,Line,Ncs,fun scan_check/6}}; +scan_check("s"=Cs, St, Line, Col, Toks, Ncs) -> + {more,{Cs,St,Col,Toks,Line,Ncs,fun scan_check/6}}; +scan_check([]=Cs, St, Line, Col, Toks, Ncs) -> + {more,{Cs,St,Col,Toks,Line,Ncs,fun scan_check/6}}; +scan_check(Cs, St=#erl_scan{comment=true}, Line, Col, Toks, Ncs) -> + scan_comment(Cs, St, Line, Col, Toks, Ncs); +scan_check(Cs, St, Line, Col, Toks, _Ncs) -> + skip_comment(Cs, St, Line, Col, Toks, 1). + +scan_check1(Cs, St=#erl_scan{in_check=true}, Line, Toks, Col, NoofCols) -> + %% Skip as we are already in the check mode + scan1(Cs, St, Line, incr_column(Col, NoofCols), Toks); +scan_check1(Cs, St, Line, Toks, Col, NoofCols) -> + tok2(Cs, St#erl_scan{in_check=true}, Line, + Col, Toks, "%ssa%", '%ssa%', NoofCols). + tok2(Cs, #erl_scan{has_fun = false}=St, Line, no_col=Col, Toks, _Wcs, P) -> scan1(Cs, St, Line, Col, [{P,anno(Line)}|Toks]); -tok2(Cs, St, Line, Col, Toks, Wcs, P) -> +tok2(Cs, #erl_scan{}=St, Line, Col, Toks, Wcs, P) -> Anno = anno(Line, Col, St, ?STR(P, St, Wcs)), scan1(Cs, St, Line, incr_column(Col, length(Wcs)), [{P,Anno}|Toks]). tok2(Cs, #erl_scan{has_fun = false}=St, Line, no_col=Col, Toks, _Wcs, P, _N) -> scan1(Cs, St, Line, Col, [{P,anno(Line)}|Toks]); -tok2(Cs, St, Line, Col, Toks, Wcs, P, N) -> +tok2(Cs, #erl_scan{}=St, Line, Col, Toks, Wcs, P, N) -> Anno = anno(Line, Col, St, ?STR(P,St,Wcs)), scan1(Cs, St, Line, incr_column(Col, N), [{P,Anno}|Toks]). tok3(Cs, #erl_scan{has_fun = false}=St, Line, no_col=Col, Toks, Item, _S, Sym) -> scan1(Cs, St, Line, Col, [{Item,anno(Line),Sym}|Toks]); -tok3(Cs, St, Line, Col, Toks, Item, String, Sym) -> +tok3(Cs, #erl_scan{}=St, Line, Col, Toks, Item, String, Sym) -> Token = {Item,anno(Line, Col, St, ?STR(Item, St, String)),Sym}, scan1(Cs, St, Line, incr_column(Col, length(String)), [Token|Toks]). tok3(Cs, #erl_scan{has_fun = false}=St, Line, no_col=Col, Toks, Item, _String, Sym, _Length) -> scan1(Cs, St, Line, Col, [{Item,anno(Line),Sym}|Toks]); -tok3(Cs, St, Line, Col, Toks, Item, String, Sym, Length) -> +tok3(Cs, #erl_scan{}=St, Line, Col, Toks, Item, String, Sym, Length) -> Token = {Item,anno(Line, Col, St, ?STR(Item, St, String)),Sym}, scan1(Cs, St, Line, incr_column(Col, Length), [Token|Toks]). diff --git a/lib/stdlib/src/erl_stdlib_errors.erl b/lib/stdlib/src/erl_stdlib_errors.erl index eeccb77db1..a90d6477a7 100644 --- a/lib/stdlib/src/erl_stdlib_errors.erl +++ b/lib/stdlib/src/erl_stdlib_errors.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2020-2022. All Rights Reserved. +%% Copyright Ericsson AB 2020-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -73,10 +73,17 @@ format_binary_error(encode_unsigned, [Subject, Endianness], _) -> [must_be_non_neg_integer(Subject), must_be_endianness(Endianness)]; format_binary_error(encode_hex, [Subject], _) -> [must_be_binary(Subject)]; +format_binary_error(encode_hex, [Subject, Case], _) -> + [must_be_binary(Subject), must_be_hex_case(Case)]; format_binary_error(decode_hex, [Subject], _) -> if - is_binary(Subject), byte_size(Subject) rem 2 == 1 -> - ["must contain an even number of bytes"]; + is_binary(Subject) -> + if + byte_size(Subject) rem 2 =:= 1 -> + [<<"must contain an even number of bytes">>]; + true -> + [<<"must only contain hex digits 0-9, A-F, and a-f">>] + end; true -> [must_be_binary(Subject)] end; @@ -237,8 +244,10 @@ format_maps_error(intersect_with, [Combiner, Map1, Map2]) -> [must_be_fun(Combiner, 3), must_be_map(Map1), must_be_map(Map2)]; format_maps_error(is_key, _Args) -> [[], not_map]; -format_maps_error(iterator, _Args) -> - [not_map]; +format_maps_error(iterator, [Map]) -> + [must_be_map(Map)]; +format_maps_error(iterator, [Map, Order]) -> + [must_be_map(Map), must_be_map_iterator_order(Order)]; format_maps_error(keys, _Args) -> [not_map]; format_maps_error(map, [Pred, Map]) -> @@ -247,10 +256,10 @@ format_maps_error(merge, [Map1, Map2]) -> [must_be_map(Map1), must_be_map(Map2)]; format_maps_error(merge_with, [Combiner, Map1, Map2]) -> [must_be_fun(Combiner, 3), must_be_map(Map1), must_be_map(Map2)]; -format_maps_error(put, _Args) -> - [[], [], not_map]; format_maps_error(next, _Args) -> [bad_iterator]; +format_maps_error(put, _Args) -> + [[], [], not_map]; format_maps_error(remove, _Args) -> [[], not_map]; format_maps_error(size, _Args) -> @@ -258,7 +267,7 @@ format_maps_error(size, _Args) -> format_maps_error(take, _Args) -> [[], not_map]; format_maps_error(to_list, _Args) -> - [not_map]; + [not_map_or_iterator]; format_maps_error(update, _Args) -> [[], [], not_map]; format_maps_error(update_with, [_Key, Fun, Map]) -> @@ -506,7 +515,7 @@ format_io_error_cause(_, _, _, _HasDevice) -> maybe_posix_message(Cause, HasDevice) -> case erl_posix_msg:message(Cause) of - "unknown POSIX error" -> + "unknown POSIX error" ++ _ -> unknown; PosixStr when HasDevice -> [io_lib:format("~ts (~tp)",[PosixStr, Cause])]; @@ -641,6 +650,9 @@ format_ets_error(lookup_element, [_,_,Pos]=Args, Cause) -> [TabCause, "", PosCause] end end; +format_ets_error(lookup_element, [Tab, Key, Pos, _Default], Cause) -> + % The default argument cannot cause an error. + format_ets_error(lookup_element, [Tab, Key, Pos], Cause); format_ets_error(match, [_], _Cause) -> [bad_continuation]; format_ets_error(match, [_,_,_]=Args, Cause) -> @@ -913,6 +925,10 @@ must_be_binary(Bin, Error) when is_binary(Bin) -> Error; must_be_binary(Bin, _Error) when is_bitstring(Bin) -> bitstring; must_be_binary(_, _) -> not_binary. +must_be_hex_case(uppercase) -> []; +must_be_hex_case(lowercase) -> []; +must_be_hex_case(_) -> bad_hex_case. + must_be_endianness(little) -> []; must_be_endianness(big) -> []; must_be_endianness(_) -> bad_endianness. @@ -959,14 +975,21 @@ must_be_list(_) -> must_be_map(#{}) -> []; must_be_map(_) -> not_map. +must_be_map_iterator_order(undefined) -> + []; +must_be_map_iterator_order(ordered) -> + []; +must_be_map_iterator_order(CmpFun) when is_function(CmpFun, 2) -> + []; +must_be_map_iterator_order(_) -> + not_map_iterator_order. + must_be_map_or_iter(Map) when is_map(Map) -> []; must_be_map_or_iter(Iter) -> - try maps:next(Iter) of - _ -> [] - catch - error:_ -> - not_map_or_iterator + case maps:is_iterator_valid(Iter) of + true -> []; + false -> not_map_or_iterator end. must_be_number(N) -> @@ -1069,6 +1092,8 @@ expand_error(not_atom) -> <<"not an atom">>; expand_error(not_binary) -> <<"not a binary">>; +expand_error(bad_hex_case) -> + <<"not 'uppercase' or 'lowercase'">>; expand_error(not_compiled_regexp) -> <<"not a compiled regular expression">>; expand_error(not_iodata) -> @@ -1083,6 +1108,8 @@ expand_error(not_integer) -> <<"not an integer">>; expand_error(not_list) -> <<"not a list">>; +expand_error(not_map_iterator_order) -> + <<"not 'undefined', 'ordered', or a fun that takes two arguments">>; expand_error(not_map_or_iterator) -> <<"not a map or an iterator">>; expand_error(not_number) -> diff --git a/lib/stdlib/src/ets.erl b/lib/stdlib/src/ets.erl index 979a75b231..3545c8a186 100644 --- a/lib/stdlib/src/ets.erl +++ b/lib/stdlib/src/ets.erl @@ -70,7 +70,7 @@ -export([all/0, delete/1, delete/2, delete_all_objects/1, delete_object/2, first/1, give_away/3, info/1, info/2, insert/2, insert_new/2, is_compiled_ms/1, last/1, lookup/2, - lookup_element/3, match/1, match/2, match/3, match_object/1, + lookup_element/3, lookup_element/4, match/1, match/2, match/3, match_object/1, match_object/2, match_object/3, match_spec_compile/1, match_spec_run_r/3, member/2, new/2, next/2, prev/2, rename/2, safe_fixtable/2, select/1, select/2, select/3, @@ -232,6 +232,16 @@ lookup(_, _) -> lookup_element(_, _, _) -> erlang:nif_error(undef). +-spec lookup_element(Table, Key, Pos, Default) -> Elem when + Table :: table(), + Key :: term(), + Pos :: pos_integer(), + Default :: term(), + Elem :: term() | [term()]. + +lookup_element(_, _, _, _) -> + erlang:nif_error(undef). + -spec match(Table, Pattern) -> [Match] when Table :: table(), Pattern :: match_pattern(), diff --git a/lib/stdlib/src/filename.erl b/lib/stdlib/src/filename.erl index 8bf4e97b9f..fc8e8110a0 100644 --- a/lib/stdlib/src/filename.erl +++ b/lib/stdlib/src/filename.erl @@ -78,8 +78,10 @@ -include_lib("kernel/include/file.hrl"). --define(IS_DRIVELETTER(Letter),(((Letter >= $A) andalso (Letter =< $Z)) orelse - ((Letter >= $a) andalso (Letter =< $z)))). +-define(IS_DRIVELETTER(Letter), + (is_integer(Letter) + andalso (($A =< Letter andalso Letter =< $Z) + orelse ($a =< Letter andalso Letter =< $z)))). %% Converts a relative filename to an absolute filename %% or the filename itself if it already is an absolute filename @@ -333,8 +335,7 @@ dirname([$/|Rest], Dir, File, Seps) -> dirname([DirSep|Rest], Dir, File, {DirSep,_}=Seps) when is_integer(DirSep) -> dirname(Rest, File++Dir, [$/], Seps); dirname([Dl,DrvSep|Rest], [], [], {_,DrvSep}=Seps) - when is_integer(DrvSep), ((($a =< Dl) and (Dl =< $z)) or - (($A =< Dl) and (Dl =< $Z))) -> + when is_integer(DrvSep), ?IS_DRIVELETTER(Dl) -> dirname(Rest, [DrvSep,Dl], [], Seps); dirname([Char|Rest], Dir, File, Seps) when is_integer(Char) -> dirname(Rest, Dir, [Char|File], Seps); @@ -757,7 +758,8 @@ win32_split([X, $\\|Rest]) when is_integer(X) -> win32_split([X, $/|Rest]); win32_split([X, Y, $\\|Rest]) when is_integer(X), is_integer(Y) -> win32_split([X, Y, $/|Rest]); -win32_split([UcLetter, $:|Rest]) when UcLetter >= $A, UcLetter =< $Z -> +win32_split([UcLetter, $:|Rest]) + when is_integer(UcLetter), $A =< UcLetter, UcLetter =< $Z -> win32_split([UcLetter+$a-$A, $:|Rest]); win32_split([Letter, $:, $/|Rest]) -> split(Rest, [], [[Letter, $:, $/]], win32); diff --git a/lib/stdlib/src/gb_sets.erl b/lib/stdlib/src/gb_sets.erl index 8dda0d4ee0..aba1bed156 100644 --- a/lib/stdlib/src/gb_sets.erl +++ b/lib/stdlib/src/gb_sets.erl @@ -205,13 +205,13 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -spec empty() -> Set when - Set :: set(). + Set :: set(none()). empty() -> {0, nil}. -spec new() -> Set when - Set :: set(). + Set :: set(none()). new() -> empty(). @@ -259,13 +259,13 @@ is_member_1(_, nil) -> Set1 :: set(Element), Set2 :: set(Element). -insert(Key, {S, T}) -> +insert(Key, {S, T}) when is_integer(S), S >= 0 -> S1 = S + 1, {S1, insert_1(Key, T, ?pow(S1, ?p))}. insert_1(Key, {Key1, Smaller, Bigger}, S) when Key < Key1 -> case insert_1(Key, Smaller, ?div2(S)) of - {T1, H1, S1} when is_integer(H1) -> + {T1, H1, S1} when is_integer(H1), is_integer(S1) -> T = {Key1, T1, Bigger}, {H2, S2} = count(Bigger), H = ?mul2(erlang:max(H1, H2)), @@ -282,7 +282,7 @@ insert_1(Key, {Key1, Smaller, Bigger}, S) when Key < Key1 -> end; insert_1(Key, {Key1, Smaller, Bigger}, S) when Key > Key1 -> case insert_1(Key, Bigger, ?div2(S)) of - {T1, H1, S1} when is_integer(H1) -> + {T1, H1, S1} when is_integer(H1), is_integer(S1) -> T = {Key1, Smaller, T1}, {H2, S2} = count(Smaller), H = ?mul2(erlang:max(H1, H2)), @@ -317,7 +317,7 @@ count(nil) -> Set1 :: set(Element), Set2 :: set(Element). -balance({S, T}) -> +balance({S, T}) when is_integer(S), S >= 0 -> {S, balance(T, S)}. balance(T, S) -> @@ -550,9 +550,9 @@ next([]) -> Set2 :: set(Element), Set3 :: set(Element). -union({N1, T1}, {N2, T2}) when N2 < N1 -> +union({N1, T1}, {N2, T2}) when is_integer(N1), is_integer(N2), N2 < N1 -> union(to_list_1(T2), N2, T1, N1); -union({N1, T1}, {N2, T2}) -> +union({N1, T1}, {N2, T2}) when is_integer(N1), is_integer(N2) -> union(to_list_1(T1), N1, T2, N2). %% We avoid the expensive mathematical computations if there is little @@ -633,7 +633,7 @@ push([X | Xs], As) -> push([], As) -> As. -balance_revlist(L, S) -> +balance_revlist(L, S) when is_integer(S) -> {T, _} = balance_revlist_1(L, S), T. @@ -670,9 +670,9 @@ union_list(S, []) -> S. Set2 :: set(Element), Set3 :: set(Element). -intersection({N1, T1}, {N2, T2}) when N2 < N1 -> +intersection({N1, T1}, {N2, T2}) when is_integer(N1), is_integer(N2), N2 < N1 -> intersection(to_list_1(T2), N2, T1, N1); -intersection({N1, T1}, {N2, T2}) -> +intersection({N1, T1}, {N2, T2}) when is_integer(N1), is_integer(N2) -> intersection(to_list_1(T1), N1, T2, N2). intersection(L, _N1, T2, N2) when N2 < 10 -> @@ -770,7 +770,8 @@ subtract(S1, S2) -> Set2 :: set(Element), Set3 :: set(Element). -difference({N1, T1}, {N2, T2}) -> +difference({N1, T1}, {N2, T2}) when is_integer(N1), N1 >= 0, + is_integer(N2), N2 >= 0 -> difference(to_list_1(T1), N1, T2, N2). difference(L, N1, T2, N2) when N2 < 10 -> @@ -820,7 +821,8 @@ difference_2(Xs, [], As, S) -> Set1 :: set(Element), Set2 :: set(Element). -is_subset({N1, T1}, {N2, T2}) -> +is_subset({N1, T1}, {N2, T2}) when is_integer(N1), N1 >= 0, + is_integer(N2), N2 >= 0 -> is_subset(to_list_1(T1), N1, T2, N2). is_subset(L, _N1, T2, N2) when N2 < 10 -> diff --git a/lib/stdlib/src/gb_trees.erl b/lib/stdlib/src/gb_trees.erl index c0cdde012e..54a5ab6690 100644 --- a/lib/stdlib/src/gb_trees.erl +++ b/lib/stdlib/src/gb_trees.erl @@ -169,7 +169,7 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% --spec empty() -> tree(). +-spec empty() -> tree(none(), none()). empty() -> {0, nil}. @@ -279,7 +279,7 @@ insert(Key, Val, {S, T}) when is_integer(S) -> insert_1(Key, Value, {Key1, V, Smaller, Bigger}, S) when Key < Key1 -> case insert_1(Key, Value, Smaller, ?div2(S)) of - {T1, H1, S1} -> + {T1, H1, S1} when is_integer(H1), is_integer(S1) -> T = {Key1, V, T1, Bigger}, {H2, S2} = count(Bigger), H = ?mul2(erlang:max(H1, H2)), @@ -296,7 +296,7 @@ insert_1(Key, Value, {Key1, V, Smaller, Bigger}, S) when Key < Key1 -> end; insert_1(Key, Value, {Key1, V, Smaller, Bigger}, S) when Key > Key1 -> case insert_1(Key, Value, Bigger, ?div2(S)) of - {T1, H1, S1} -> + {T1, H1, S1} when is_integer(H1), is_integer(S1) -> T = {Key1, V, Smaller, T1}, {H2, S2} = count(Smaller), H = ?mul2(erlang:max(H1, H2)), @@ -349,7 +349,7 @@ count(nil) -> Tree1 :: tree(Key, Value), Tree2 :: tree(Key, Value). -balance({S, T}) -> +balance({S, T}) when is_integer(S), S >= 0 -> {S, balance(T, S)}. balance(T, S) -> diff --git a/lib/stdlib/src/gen.erl b/lib/stdlib/src/gen.erl index 363094fb15..d03ca78e7c 100644 --- a/lib/stdlib/src/gen.erl +++ b/lib/stdlib/src/gen.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2022. All Rights Reserved. +%% Copyright Ericsson AB 1996-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -40,6 +40,8 @@ -export([format_status_header/2, format_status/4]). +-export(['@wait_response_recv_opt'/3]). + -define(MAX_INT_TIMEOUT, 4294967295). -define(default_timeout, 5000). @@ -193,7 +195,8 @@ init_it(GenMod, Starter, Parent, Name, Mod, Args, Options) -> true -> init_it2(GenMod, Starter, Parent, Name, Mod, Args, Options); {false, Pid} -> - proc_lib:init_ack(Starter, {error, {already_started, Pid}}) + proc_lib:init_fail( + Starter, {error, {already_started, Pid}}, {exit, normal}) end. init_it2(GenMod, Starter, Parent, Name, Mod, Args, Options) -> @@ -313,6 +316,17 @@ do_send_request(Process, Tag, Request) -> _ = erlang:send(Process, {Tag, {self(), [alias|ReqId]}, Request}, [noconnect]), ReqId. +-spec '@wait_response_recv_opt'(term(), term(), term()) -> ok. +'@wait_response_recv_opt'(Process, Tag, Request) -> + %% Enables reference optimization in wait_response/2 and + %% receive_response/2 + %% + %% This never actually runs and is only used to trigger the optimization, + %% see the module comment in beam_ssa_recv for details. + _ = wait_response(send_request(Process, Tag, Request), infinity), + _ = receive_response(send_request(Process, Tag, Request), infinity), + ok. + %% %% Wait for a reply to the client. %% Note: if timeout is returned monitors are kept. diff --git a/lib/stdlib/src/gen_fsm.erl b/lib/stdlib/src/gen_fsm.erl index 68ff5e2d4d..6dba530b85 100644 --- a/lib/stdlib/src/gen_fsm.erl +++ b/lib/stdlib/src/gen_fsm.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2021. All Rights Reserved. +%% Copyright Ericsson AB 1996-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -329,30 +329,26 @@ init_it(Starter, self, Name, Mod, Args, Options) -> init_it(Starter, Parent, Name0, Mod, Args, Options) -> Name = gen:name(Name0), Debug = gen:debug_options(Name, Options), - HibernateAfterTimeout = gen:hibernate_after(Options), - case catch Mod:init(Args) of + HibernateAfterTimeout = gen:hibernate_after(Options), + case catch Mod:init(Args) of {ok, StateName, StateData} -> - proc_lib:init_ack(Starter, {ok, self()}), + proc_lib:init_ack(Starter, {ok, self()}), loop(Parent, Name, StateName, StateData, Mod, infinity, HibernateAfterTimeout, Debug); {ok, StateName, StateData, Timeout} -> - proc_lib:init_ack(Starter, {ok, self()}), + proc_lib:init_ack(Starter, {ok, self()}), loop(Parent, Name, StateName, StateData, Mod, Timeout, HibernateAfterTimeout, Debug); {stop, Reason} -> - gen:unregister_name(Name0), - proc_lib:init_ack(Starter, {error, Reason}), - exit(Reason); + gen:unregister_name(Name0), + exit(Reason); ignore -> gen:unregister_name(Name0), - proc_lib:init_ack(Starter, ignore), - exit(normal); + proc_lib:init_fail(Starter, ignore, {exit, normal}); {'EXIT', Reason} -> gen:unregister_name(Name0), - proc_lib:init_ack(Starter, {error, Reason}), - exit(Reason); + exit(Reason); Else -> - Error = {bad_return_value, Else}, - proc_lib:init_ack(Starter, {error, Error}), - exit(Error) + Reason = {bad_return_value, Else}, + exit(Reason) end. %%----------------------------------------------------------------- diff --git a/lib/stdlib/src/gen_server.erl b/lib/stdlib/src/gen_server.erl index af5e04f78a..c574f2aeb2 100644 --- a/lib/stdlib/src/gen_server.erl +++ b/lib/stdlib/src/gen_server.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2022. All Rights Reserved. +%% Copyright Ericsson AB 1996-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -147,13 +147,29 @@ ( (X) =:= infinity orelse ( is_integer(X) andalso (X) >= 0 ) ) ). +-record(callback_cache,{module :: module(), + handle_call :: fun((Request :: term(), From :: from(), State :: term()) -> + {reply, Reply :: term(), NewState :: term()} | + {reply, Reply :: term(), NewState :: term(), timeout() | hibernate | {continue, term()}} | + {noreply, NewState :: term()} | + {noreply, NewState :: term(), timeout() | hibernate | {continue, term()}} | + {stop, Reason :: term(), Reply :: term(), NewState :: term()} | + {stop, Reason :: term(), NewState :: term()}), + handle_cast :: fun((Request :: term(), State :: term()) -> + {noreply, NewState :: term()} | + {noreply, NewState :: term(), timeout() | hibernate | {continue, term()}} | + {stop, Reason :: term(), NewState :: term()}), + handle_info :: fun((Info :: timeout | term(), State :: term()) -> + {noreply, NewState :: term()} | + {noreply, NewState :: term(), timeout() | hibernate | {continue, term()}} | + {stop, Reason :: term(), NewState :: term()})}). %%%========================================================================= %%% API %%%========================================================================= -callback init(Args :: term()) -> {ok, State :: term()} | {ok, State :: term(), timeout() | hibernate | {continue, term()}} | - {stop, Reason :: term()} | ignore. + {stop, Reason :: term()} | ignore | {error, Reason :: term()}. -callback handle_call(Request :: term(), From :: from(), State :: term()) -> {reply, Reply :: term(), NewState :: term()} | @@ -649,7 +665,7 @@ do_abcast([], _,_) -> abcast. %% multi_call(Name, Request) when is_atom(Name) -> - do_multi_call([node() | nodes()], Name, Request, infinity). + multi_call([node() | nodes()], Name, Request, infinity). -spec multi_call( Nodes :: [node()], @@ -663,7 +679,7 @@ multi_call(Name, Request) %% multi_call(Nodes, Name, Request) when is_list(Nodes), is_atom(Name) -> - do_multi_call(Nodes, Name, Request, infinity). + multi_call(Nodes, Name, Request, infinity). -spec multi_call( Nodes :: [node()], @@ -678,8 +694,93 @@ multi_call(Nodes, Name, Request) %% multi_call(Nodes, Name, Request, Timeout) when is_list(Nodes), is_atom(Name), ?is_timeout(Timeout) -> - do_multi_call(Nodes, Name, Request, Timeout). + Alias = alias(), + try + Timer = if Timeout == infinity -> undefined; + true -> erlang:start_timer(Timeout, self(), Alias) + end, + Reqs = mc_send(Nodes, Name, Alias, Request, Timer, []), + mc_recv(Reqs, Alias, Timer, [], []) + after + _ = unalias(Alias) + end. +-dialyzer({no_improper_lists, mc_send/6}). + +mc_send([], _Name, _Alias, _Request, _Timer, Reqs) -> + Reqs; +mc_send([Node|Nodes], Name, Alias, Request, Timer, Reqs) when is_atom(Node) -> + NN = {Name, Node}, + Mon = try + erlang:monitor(process, NN, [{tag, Alias}]) + catch + error:badarg -> + %% Node not alive... + M = make_ref(), + Alias ! {Alias, M, process, NN, noconnection}, + M + end, + try + %% We use 'noconnect' since it is no point in bringing up a new + %% connection if it was not brought up by the monitor signal... + _ = erlang:send(NN, + {'$gen_call', {self(), [[alias|Alias]|Mon]}, Request}, + [noconnect]), + ok + catch + _:_ -> + ok + end, + mc_send(Nodes, Name, Alias, Request, Timer, [[Node|Mon]|Reqs]); +mc_send(_BadNodes, _Name, Alias, _Request, Timer, Reqs) -> + %% Cleanup then fail... + unalias(Alias), + mc_cancel_timer(Timer, Alias), + _ = mc_recv_tmo(Reqs, Alias, [], []), + error(badarg). + +mc_recv([], Alias, Timer, Replies, BadNodes) -> + mc_cancel_timer(Timer, Alias), + unalias(Alias), + {Replies, BadNodes}; +mc_recv([[Node|Mon] | RestReqs] = Reqs, Alias, Timer, Replies, BadNodes) -> + receive + {[[alias|Alias]|Mon], Reply} -> + erlang:demonitor(Mon, [flush]), + mc_recv(RestReqs, Alias, Timer, [{Node,Reply}|Replies], BadNodes); + {Alias, Mon, process, _, _} -> + mc_recv(RestReqs, Alias, Timer, Replies, [Node|BadNodes]); + {timeout, Timer, Alias} -> + unalias(Alias), + mc_recv_tmo(Reqs, Alias, Replies, BadNodes) + end. + +mc_recv_tmo([], _Alias, Replies, BadNodes) -> + {Replies, BadNodes}; +mc_recv_tmo([[Node|Mon] | RestReqs], Alias, Replies, BadNodes) -> + erlang:demonitor(Mon), + receive + {[[alias|Alias]|Mon], Reply} -> + mc_recv_tmo(RestReqs, Alias, [{Node,Reply}|Replies], BadNodes); + {Alias, Mon, process, _, _} -> + mc_recv_tmo(RestReqs, Alias, Replies, [Node|BadNodes]) + after + 0 -> + mc_recv_tmo(RestReqs, Alias, Replies, [Node|BadNodes]) + end. + +mc_cancel_timer(undefined, _Alias) -> + ok; +mc_cancel_timer(Timer, Alias) -> + case erlang:cancel_timer(Timer) of + false -> + receive + {timeout, Timer, Alias} -> + ok + end; + _ -> + ok + end. %%----------------------------------------------------------------- %% enter_loop(Mod, Options, State, <ServerName>, <TimeOut>) ->_ @@ -783,7 +884,8 @@ enter_loop(Mod, Options, State, ServerName, TimeoutOrHibernate) Parent = gen:get_parent(), Debug = gen:debug_options(Name, Options), HibernateAfterTimeout = gen:hibernate_after(Options), - loop(Parent, Name, State, Mod, TimeoutOrHibernate, HibernateAfterTimeout, Debug); + CbCache = create_callback_cache(Mod), + loop(Parent, Name, State, CbCache, TimeoutOrHibernate, HibernateAfterTimeout, Debug); %% enter_loop(Mod, Options, State, ServerName, {continue, _}=Continue) when is_atom(Mod), is_list(Options) -> @@ -791,7 +893,8 @@ enter_loop(Mod, Options, State, ServerName, {continue, _}=Continue) Parent = gen:get_parent(), Debug = gen:debug_options(Name, Options), HibernateAfterTimeout = gen:hibernate_after(Options), - loop(Parent, Name, State, Mod, Continue, HibernateAfterTimeout, Debug). + CbCache = create_callback_cache(Mod), + loop(Parent, Name, State, CbCache, Continue, HibernateAfterTimeout, Debug). %%%======================================================================== %%% Gen-callback functions @@ -810,19 +913,25 @@ init_it(Starter, Parent, Name0, Mod, Args, Options) -> Name = gen:name(Name0), Debug = gen:debug_options(Name, Options), HibernateAfterTimeout = gen:hibernate_after(Options), - + CbCache = create_callback_cache(Mod), case init_it(Mod, Args) of {ok, {ok, State}} -> - proc_lib:init_ack(Starter, {ok, self()}), - loop(Parent, Name, State, Mod, infinity, HibernateAfterTimeout, Debug); - {ok, {ok, State, TimeoutOrHibernate}} + proc_lib:init_ack(Starter, {ok, self()}), + loop( + Parent, Name, State, CbCache, infinity, + HibernateAfterTimeout, Debug); + {ok, {ok, State, TimeoutOrHibernate}} when ?is_timeout(TimeoutOrHibernate); TimeoutOrHibernate =:= hibernate -> - proc_lib:init_ack(Starter, {ok, self()}), - loop(Parent, Name, State, Mod, TimeoutOrHibernate, HibernateAfterTimeout, Debug); + proc_lib:init_ack(Starter, {ok, self()}), + loop( + Parent, Name, State, CbCache, TimeoutOrHibernate, + HibernateAfterTimeout, Debug); {ok, {ok, State, {continue, _}=Continue}} -> - proc_lib:init_ack(Starter, {ok, self()}), - loop(Parent, Name, State, Mod, Continue, HibernateAfterTimeout, Debug); + proc_lib:init_ack(Starter, {ok, self()}), + loop( + Parent, Name, State, CbCache, Continue, + HibernateAfterTimeout, Debug); {ok, {stop, Reason}} -> %% For consistency, we must make sure that the %% registered name (if any) is unregistered before @@ -831,29 +940,32 @@ init_it(Starter, Parent, Name0, Mod, Args, Options) -> %% an 'already_started' error if it immediately %% tried starting the process again.) gen:unregister_name(Name0), - proc_lib:init_ack(Starter, {error, Reason}), - exit(Reason); + exit(Reason); + {ok, {error, _Reason} = ERROR} -> + %% The point of this clause is that we shall have a silent/graceful + %% termination. The error reason will be returned to the + %% 'Starter' ({error, Reason}), but *no* crash report. + gen:unregister_name(Name0), + proc_lib:init_fail(Starter, ERROR, {exit, normal}); {ok, ignore} -> gen:unregister_name(Name0), - proc_lib:init_ack(Starter, ignore), - exit(normal); + proc_lib:init_fail(Starter, ignore, {exit, normal}); {ok, Else} -> - Error = {bad_return_value, Else}, - proc_lib:init_ack(Starter, {error, Error}), - exit(Error); + gen:unregister_name(Name0), + exit({bad_return_value, Else}); {'EXIT', Class, Reason, Stacktrace} -> gen:unregister_name(Name0), - proc_lib:init_ack(Starter, {error, terminate_reason(Class, Reason, Stacktrace)}), - erlang:raise(Class, Reason, Stacktrace) + erlang:raise(Class, Reason, Stacktrace) end. init_it(Mod, Args) -> try - {ok, Mod:init(Args)} + {ok, Mod:init(Args)} catch - throw:R -> {ok, R}; - Class:R:S -> {'EXIT', Class, R, S} + throw:R -> {ok, R}; + Class:R:S -> {'EXIT', Class, R, S} end. + %%%======================================================================== %%% Internal functions %%%======================================================================== @@ -861,58 +973,68 @@ init_it(Mod, Args) -> %%% The MAIN loop. %%% --------------------------------------------------- -loop(Parent, Name, State, Mod, {continue, Continue} = Msg, HibernateAfterTimeout, Debug) -> - Reply = try_dispatch(Mod, handle_continue, Continue, State), +loop(Parent, Name, State, CbCache, {continue, Continue} = Msg, HibernateAfterTimeout, Debug) -> + Reply = try_handle_continue(CbCache, Continue, State), case Debug of - [] -> - handle_common_reply(Reply, Parent, Name, undefined, Msg, Mod, - HibernateAfterTimeout, State); - _ -> - Debug1 = sys:handle_debug(Debug, fun print_event/3, Name, Msg), - handle_common_reply(Reply, Parent, Name, undefined, Msg, Mod, - HibernateAfterTimeout, State, Debug1) + [] -> + handle_common_reply(Reply, Parent, Name, undefined, Msg, CbCache, + HibernateAfterTimeout, State); + _ -> + Debug1 = sys:handle_debug(Debug, fun print_event/3, Name, Msg), + handle_common_reply(Reply, Parent, Name, undefined, Msg, CbCache, + HibernateAfterTimeout, State, Debug1) end; -loop(Parent, Name, State, Mod, hibernate, HibernateAfterTimeout, Debug) -> +loop(Parent, Name, State, CbCache, hibernate, HibernateAfterTimeout, Debug) -> + Mod = CbCache#callback_cache.module, proc_lib:hibernate(?MODULE,wake_hib,[Parent, Name, State, Mod, HibernateAfterTimeout, Debug]); -loop(Parent, Name, State, Mod, infinity, HibernateAfterTimeout, Debug) -> - receive - Msg -> - decode_msg(Msg, Parent, Name, State, Mod, infinity, HibernateAfterTimeout, Debug, false) - after HibernateAfterTimeout -> - loop(Parent, Name, State, Mod, hibernate, HibernateAfterTimeout, Debug) - end; +loop(Parent, Name, State, CbCache, infinity, HibernateAfterTimeout, Debug) -> + receive + Msg -> + decode_msg(Msg, Parent, Name, State, CbCache, infinity, HibernateAfterTimeout, Debug, false) + after HibernateAfterTimeout -> + loop(Parent, Name, State, CbCache, hibernate, HibernateAfterTimeout, Debug) + end; -loop(Parent, Name, State, Mod, Time, HibernateAfterTimeout, Debug) -> +loop(Parent, Name, State, CbCache, Time, HibernateAfterTimeout, Debug) -> Msg = receive - Input -> - Input - after Time -> - timeout - end, - decode_msg(Msg, Parent, Name, State, Mod, Time, HibernateAfterTimeout, Debug, false). + Input -> + Input + after Time -> + timeout + end, + decode_msg(Msg, Parent, Name, State, CbCache, Time, HibernateAfterTimeout, Debug, false). + +-spec create_callback_cache(module()) -> #callback_cache{}. +create_callback_cache(Mod) -> + #callback_cache{module = Mod, + handle_call = fun Mod:handle_call/3, + handle_cast = fun Mod:handle_cast/2, + handle_info = fun Mod:handle_info/2}. wake_hib(Parent, Name, State, Mod, HibernateAfterTimeout, Debug) -> Msg = receive - Input -> - Input - end, - decode_msg(Msg, Parent, Name, State, Mod, hibernate, HibernateAfterTimeout, Debug, true). + Input -> + Input + end, + CbCache = create_callback_cache(Mod), + decode_msg(Msg, Parent, Name, State, CbCache, hibernate, HibernateAfterTimeout, Debug, true). -decode_msg(Msg, Parent, Name, State, Mod, Time, HibernateAfterTimeout, Debug, Hib) -> +decode_msg(Msg, Parent, Name, State, CbCache, Time, HibernateAfterTimeout, Debug, Hib) -> case Msg of - {system, From, Req} -> - sys:handle_system_msg(Req, From, Parent, ?MODULE, Debug, - [Name, State, Mod, Time, HibernateAfterTimeout], Hib); - {'EXIT', Parent, Reason} -> - terminate(Reason, ?STACKTRACE(), Name, undefined, Msg, Mod, State, Debug); - _Msg when Debug =:= [] -> - handle_msg(Msg, Parent, Name, State, Mod, HibernateAfterTimeout); - _Msg -> - Debug1 = sys:handle_debug(Debug, fun print_event/3, - Name, {in, Msg}), - handle_msg(Msg, Parent, Name, State, Mod, HibernateAfterTimeout, Debug1) + {system, From, Req} -> + sys:handle_system_msg(Req, From, Parent, ?MODULE, Debug, + [Name, State, CbCache, Time, HibernateAfterTimeout], Hib); + {'EXIT', Parent, Reason} -> + #callback_cache{module = Mod} = CbCache, + terminate(Reason, ?STACKTRACE(), Name, undefined, Msg, Mod, State, Debug); + _Msg when Debug =:= [] -> + handle_msg(Msg, Parent, Name, State, CbCache, HibernateAfterTimeout); + _Msg -> + Debug1 = sys:handle_debug(Debug, fun print_event/3, + Name, {in, Msg}), + handle_msg(Msg, Parent, Name, State, CbCache, HibernateAfterTimeout, Debug1) end. %%% --------------------------------------------------- @@ -925,184 +1047,6 @@ do_send(Dest, Msg) -> end, ok. -do_multi_call([Node], Name, Req, infinity) when Node =:= node() -> - % Special case when multi_call is used with local node only. - % In that case we can leverage the benefit of recv_mark optimisation - % existing in simple gen:call. - try gen:call(Name, '$gen_call', Req, infinity) of - {ok, Res} -> {[{Node, Res}],[]} - catch exit:_ -> - {[], [Node]} - end; -do_multi_call(Nodes, Name, Req, infinity) -> - Tag = make_ref(), - Monitors = send_nodes(Nodes, Name, Tag, Req), - rec_nodes(Tag, Monitors, Name, undefined); -do_multi_call(Nodes, Name, Req, Timeout) -> - Tag = make_ref(), - Caller = self(), - Receiver = - spawn( - fun() -> - %% Middleman process. Should be unsensitive to regular - %% exit signals. The sychronization is needed in case - %% the receiver would exit before the caller started - %% the monitor. - process_flag(trap_exit, true), - Mref = erlang:monitor(process, Caller), - receive - {Caller,Tag} -> - Monitors = send_nodes(Nodes, Name, Tag, Req), - TimerId = erlang:start_timer(Timeout, self(), ok), - Result = rec_nodes(Tag, Monitors, Name, TimerId), - exit({self(),Tag,Result}); - {'DOWN',Mref,_,_,_} -> - %% Caller died before sending us the go-ahead. - %% Give up silently. - exit(normal) - end - end), - Mref = erlang:monitor(process, Receiver), - Receiver ! {self(),Tag}, - receive - {'DOWN',Mref,_,_,{Receiver,Tag,Result}} -> - Result; - {'DOWN',Mref,_,_,Reason} -> - %% The middleman code failed. Or someone did - %% exit(_, kill) on the middleman process => Reason==killed - exit(Reason) - end. - -send_nodes(Nodes, Name, Tag, Req) -> - send_nodes(Nodes, Name, Tag, Req, []). - -send_nodes([Node|Tail], Name, Tag, Req, Monitors) - when is_atom(Node) -> - Monitor = start_monitor(Node, Name), - %% Handle non-existing names in rec_nodes. - catch {Name, Node} ! {'$gen_call', {self(), {Tag, Node}}, Req}, - send_nodes(Tail, Name, Tag, Req, [Monitor | Monitors]); -send_nodes([_Node|Tail], Name, Tag, Req, Monitors) -> - %% Skip non-atom Node - send_nodes(Tail, Name, Tag, Req, Monitors); -send_nodes([], _Name, _Tag, _Req, Monitors) -> - Monitors. - -%% Against old nodes: -%% If no reply has been delivered within 2 secs. (per node) check that -%% the server really exists and wait for ever for the answer. -%% -%% Against contemporary nodes: -%% Wait for reply, server 'DOWN', or timeout from TimerId. - -rec_nodes(Tag, Nodes, Name, TimerId) -> - rec_nodes(Tag, Nodes, Name, [], [], 2000, TimerId). - -rec_nodes(Tag, [{N,R}|Tail], Name, Badnodes, Replies, Time, TimerId ) -> - receive - {'DOWN', R, _, _, _} -> - rec_nodes(Tag, Tail, Name, [N|Badnodes], Replies, Time, TimerId); - {{Tag, N}, Reply} -> %% Tag is bound !!! - erlang:demonitor(R, [flush]), - rec_nodes(Tag, Tail, Name, Badnodes, - [{N,Reply}|Replies], Time, TimerId); - {timeout, TimerId, _} -> - erlang:demonitor(R, [flush]), - %% Collect all replies that already have arrived - rec_nodes_rest(Tag, Tail, Name, [N|Badnodes], Replies) - end; -rec_nodes(Tag, [N|Tail], Name, Badnodes, Replies, Time, TimerId) -> - %% R6 node - receive - {nodedown, N} -> - monitor_node(N, false), - rec_nodes(Tag, Tail, Name, [N|Badnodes], Replies, 2000, TimerId); - {{Tag, N}, Reply} -> %% Tag is bound !!! - receive {nodedown, N} -> ok after 0 -> ok end, - monitor_node(N, false), - rec_nodes(Tag, Tail, Name, Badnodes, - [{N,Reply}|Replies], 2000, TimerId); - {timeout, TimerId, _} -> - receive {nodedown, N} -> ok after 0 -> ok end, - monitor_node(N, false), - %% Collect all replies that already have arrived - rec_nodes_rest(Tag, Tail, Name, [N | Badnodes], Replies) - after Time -> - case rpc:call(N, erlang, whereis, [Name]) of - Pid when is_pid(Pid) -> % It exists try again. - rec_nodes(Tag, [N|Tail], Name, Badnodes, - Replies, infinity, TimerId); - _ -> % badnode - receive {nodedown, N} -> ok after 0 -> ok end, - monitor_node(N, false), - rec_nodes(Tag, Tail, Name, [N|Badnodes], - Replies, 2000, TimerId) - end - end; -rec_nodes(_, [], _, Badnodes, Replies, _, TimerId) -> - case catch erlang:cancel_timer(TimerId) of - false -> % It has already sent it's message - receive - {timeout, TimerId, _} -> ok - after 0 -> - ok - end; - _ -> % Timer was cancelled, or TimerId was 'undefined' - ok - end, - {Replies, Badnodes}. - -%% Collect all replies that already have arrived -rec_nodes_rest(Tag, [{N,R}|Tail], Name, Badnodes, Replies) -> - receive - {'DOWN', R, _, _, _} -> - rec_nodes_rest(Tag, Tail, Name, [N|Badnodes], Replies); - {{Tag, N}, Reply} -> %% Tag is bound !!! - erlang:demonitor(R, [flush]), - rec_nodes_rest(Tag, Tail, Name, Badnodes, [{N,Reply}|Replies]) - after 0 -> - erlang:demonitor(R, [flush]), - rec_nodes_rest(Tag, Tail, Name, [N|Badnodes], Replies) - end; -rec_nodes_rest(Tag, [N|Tail], Name, Badnodes, Replies) -> - %% R6 node - receive - {nodedown, N} -> - monitor_node(N, false), - rec_nodes_rest(Tag, Tail, Name, [N|Badnodes], Replies); - {{Tag, N}, Reply} -> %% Tag is bound !!! - receive {nodedown, N} -> ok after 0 -> ok end, - monitor_node(N, false), - rec_nodes_rest(Tag, Tail, Name, Badnodes, [{N,Reply}|Replies]) - after 0 -> - receive {nodedown, N} -> ok after 0 -> ok end, - monitor_node(N, false), - rec_nodes_rest(Tag, Tail, Name, [N|Badnodes], Replies) - end; -rec_nodes_rest(_Tag, [], _Name, Badnodes, Replies) -> - {Replies, Badnodes}. - - -%%% --------------------------------------------------- -%%% Monitor functions -%%% --------------------------------------------------- - -start_monitor(Node, Name) when is_atom(Node), is_atom(Name) -> - if node() =:= nonode@nohost, Node =/= nonode@nohost -> - Ref = make_ref(), - self() ! {'DOWN', Ref, process, {Name, Node}, noconnection}, - {Node, Ref}; - true -> - case catch erlang:monitor(process, {Name, Node}) of - {'EXIT', _} -> - %% Remote node is R6 - monitor_node(Node, true), - Node; - Ref when is_reference(Ref) -> - {Node, Ref} - end - end. - %% --------------------------------------------------- %% Helper functions for try-catch of callbacks. %% Returns the return value of the callback, or @@ -1113,60 +1057,80 @@ start_monitor(Node, Name) when is_atom(Node), is_atom(Name) -> %% stacktraces. %% --------------------------------------------------- -try_dispatch({'$gen_cast', Msg}, Mod, State) -> - try_dispatch(Mod, handle_cast, Msg, State); -try_dispatch(Info, Mod, State) -> - try_dispatch(Mod, handle_info, Info, State). +try_dispatch({'$gen_cast', Msg}, CbCache, State) -> + try_handle_cast(CbCache, Msg, State); +try_dispatch(Info, CbCache, State) -> + try_handle_info(CbCache, Info, State). + +try_handle_continue(#callback_cache{module = Mod}, Msg, State) -> + try + {ok, Mod:handle_continue(Msg, State)} + catch + throw:R -> + {ok, R}; + Class:R:Stacktrace -> + {'EXIT', Class, R, Stacktrace} + end. -try_dispatch(Mod, Func, Msg, State) -> +try_handle_info(#callback_cache{module = Mod, handle_info = HandleInfo}, Msg, State) -> try - {ok, Mod:Func(Msg, State)} + {ok, HandleInfo(Msg, State)} catch - throw:R -> - {ok, R}; - error:undef = R:Stacktrace when Func == handle_info -> + throw:R -> + {ok, R}; + error:undef = R:Stacktrace -> case erlang:function_exported(Mod, handle_info, 2) of false -> ?LOG_WARNING( - #{label=>{gen_server,no_handle_info}, - module=>Mod, - message=>Msg}, - #{domain=>[otp], - report_cb=>fun gen_server:format_log/2, - error_logger=> - #{tag=>warning_msg, - report_cb=>fun gen_server:format_log/1}}), + #{label=>{gen_server,no_handle_info}, + module=>Mod, + message=>Msg}, + #{domain=>[otp], + report_cb=>fun gen_server:format_log/2, + error_logger=> + #{tag=>warning_msg, + report_cb=>fun gen_server:format_log/1}}), {ok, {noreply, State}}; true -> {'EXIT', error, R, Stacktrace} end; - Class:R:Stacktrace -> - {'EXIT', Class, R, Stacktrace} + Class:R:Stacktrace -> + {'EXIT', Class, R, Stacktrace} end. -try_handle_call(Mod, Msg, From, State) -> +try_handle_cast(#callback_cache{handle_cast = HandleCast}, Msg, State) -> try - {ok, Mod:handle_call(Msg, From, State)} + {ok, HandleCast(Msg, State)} catch - throw:R -> - {ok, R}; - Class:R:Stacktrace -> - {'EXIT', Class, R, Stacktrace} + throw:R -> + {ok, R}; + Class:R:Stacktrace -> + {'EXIT', Class, R, Stacktrace} + end. + +try_handle_call(#callback_cache{handle_call = HandleCall}, Msg, From, State) -> + try + {ok, HandleCall(Msg, From, State)} + catch + throw:R -> + {ok, R}; + Class:R:Stacktrace -> + {'EXIT', Class, R, Stacktrace} end. try_terminate(Mod, Reason, State) -> case erlang:function_exported(Mod, terminate, 2) of - true -> - try - {ok, Mod:terminate(Reason, State)} - catch - throw:R -> - {ok, R}; - Class:R:Stacktrace -> - {'EXIT', Class, R, Stacktrace} - end; - false -> - {ok, ok} + true -> + try + {ok, Mod:terminate(Reason, State)} + catch + throw:R -> + {ok, R}; + Class:R:Stacktrace -> + {'EXIT', Class, R, Stacktrace} + end; + false -> + {ok, ok} end. @@ -1174,69 +1138,72 @@ try_terminate(Mod, Reason, State) -> %%% Message handling functions %%% --------------------------------------------------- -handle_msg({'$gen_call', From, Msg}, Parent, Name, State, Mod, HibernateAfterTimeout) -> - Result = try_handle_call(Mod, Msg, From, State), +handle_msg({'$gen_call', From, Msg}, Parent, Name, State, CbCache, HibernateAfterTimeout) -> + Result = try_handle_call(CbCache, Msg, From, State), case Result of {ok, {reply, Reply, NState}} -> reply(From, Reply), - loop(Parent, Name, NState, Mod, infinity, HibernateAfterTimeout, []); + loop(Parent, Name, NState, CbCache, infinity, HibernateAfterTimeout, []); {ok, {reply, Reply, NState, TimeoutOrHibernate}} when ?is_timeout(TimeoutOrHibernate); TimeoutOrHibernate =:= hibernate -> reply(From, Reply), - loop(Parent, Name, NState, Mod, TimeoutOrHibernate, HibernateAfterTimeout, []); + loop(Parent, Name, NState, CbCache, TimeoutOrHibernate, HibernateAfterTimeout, []); {ok, {reply, Reply, NState, {continue, _}=Continue}} -> reply(From, Reply), - loop(Parent, Name, NState, Mod, Continue, HibernateAfterTimeout, []); + loop(Parent, Name, NState, CbCache, Continue, HibernateAfterTimeout, []); {ok, {stop, Reason, Reply, NState}} -> try + Mod = CbCache#callback_cache.module, terminate(Reason, ?STACKTRACE(), Name, From, Msg, Mod, NState, []) after reply(From, Reply) end; - Other -> handle_common_reply(Other, Parent, Name, From, Msg, Mod, HibernateAfterTimeout, State) + Other -> handle_common_reply(Other, Parent, Name, From, Msg, CbCache, HibernateAfterTimeout, State) end; -handle_msg(Msg, Parent, Name, State, Mod, HibernateAfterTimeout) -> - Reply = try_dispatch(Msg, Mod, State), - handle_common_reply(Reply, Parent, Name, undefined, Msg, Mod, HibernateAfterTimeout, State). +handle_msg(Msg, Parent, Name, State, CbCache, HibernateAfterTimeout) -> + Reply = try_dispatch(Msg, CbCache, State), + handle_common_reply(Reply, Parent, Name, undefined, Msg, CbCache, HibernateAfterTimeout, State). -handle_msg({'$gen_call', From, Msg}, Parent, Name, State, Mod, HibernateAfterTimeout, Debug) -> - Result = try_handle_call(Mod, Msg, From, State), +handle_msg({'$gen_call', From, Msg}, Parent, Name, State, CbCache, HibernateAfterTimeout, Debug) -> + Result = try_handle_call(CbCache, Msg, From, State), case Result of {ok, {reply, Reply, NState}} -> Debug1 = reply(Name, From, Reply, NState, Debug), - loop(Parent, Name, NState, Mod, infinity, HibernateAfterTimeout, Debug1); + loop(Parent, Name, NState, CbCache, infinity, HibernateAfterTimeout, Debug1); {ok, {reply, Reply, NState, TimeoutOrHibernate}} when ?is_timeout(TimeoutOrHibernate); TimeoutOrHibernate =:= hibernate -> Debug1 = reply(Name, From, Reply, NState, Debug), - loop(Parent, Name, NState, Mod, TimeoutOrHibernate, HibernateAfterTimeout, Debug1); + loop(Parent, Name, NState, CbCache, TimeoutOrHibernate, HibernateAfterTimeout, Debug1); {ok, {reply, Reply, NState, {continue, _}=Continue}} -> Debug1 = reply(Name, From, Reply, NState, Debug), - loop(Parent, Name, NState, Mod, Continue, HibernateAfterTimeout, Debug1); + loop(Parent, Name, NState, CbCache, Continue, HibernateAfterTimeout, Debug1); {ok, {stop, Reason, Reply, NState}} -> try + Mod = CbCache#callback_cache.module, terminate(Reason, ?STACKTRACE(), Name, From, Msg, Mod, NState, Debug) after _ = reply(Name, From, Reply, NState, Debug) end; Other -> - handle_common_reply(Other, Parent, Name, From, Msg, Mod, HibernateAfterTimeout, State, Debug) + handle_common_reply(Other, Parent, Name, From, Msg, CbCache, HibernateAfterTimeout, State, Debug) end; -handle_msg(Msg, Parent, Name, State, Mod, HibernateAfterTimeout, Debug) -> - Reply = try_dispatch(Msg, Mod, State), - handle_common_reply(Reply, Parent, Name, undefined, Msg, Mod, HibernateAfterTimeout, State, Debug). +handle_msg(Msg, Parent, Name, State, CbCache, HibernateAfterTimeout, Debug) -> + Reply = try_dispatch(Msg, CbCache, State), + handle_common_reply(Reply, Parent, Name, undefined, Msg, CbCache, HibernateAfterTimeout, State, Debug). -handle_common_reply(Reply, Parent, Name, From, Msg, Mod, HibernateAfterTimeout, State) -> +handle_common_reply(Reply, Parent, Name, From, Msg, CbCache, HibernateAfterTimeout, State) -> + Mod = CbCache#callback_cache.module, case Reply of {ok, {noreply, NState}} -> - loop(Parent, Name, NState, Mod, infinity, HibernateAfterTimeout, []); + loop(Parent, Name, NState, CbCache, infinity, HibernateAfterTimeout, []); {ok, {noreply, NState, TimeoutOrHibernate}} when ?is_timeout(TimeoutOrHibernate); TimeoutOrHibernate =:= hibernate -> - loop(Parent, Name, NState, Mod, TimeoutOrHibernate, HibernateAfterTimeout, []); + loop(Parent, Name, NState, CbCache, TimeoutOrHibernate, HibernateAfterTimeout, []); {ok, {noreply, NState, {continue, _}=Continue}} -> - loop(Parent, Name, NState, Mod, Continue, HibernateAfterTimeout, []); + loop(Parent, Name, NState, CbCache, Continue, HibernateAfterTimeout, []); {ok, {stop, Reason, NState}} -> terminate(Reason, ?STACKTRACE(), Name, From, Msg, Mod, NState, []); {'EXIT', Class, Reason, Stacktrace} -> @@ -1245,20 +1212,21 @@ handle_common_reply(Reply, Parent, Name, From, Msg, Mod, HibernateAfterTimeout, terminate({bad_return_value, BadReply}, ?STACKTRACE(), Name, From, Msg, Mod, State, []) end. -handle_common_reply(Reply, Parent, Name, From, Msg, Mod, HibernateAfterTimeout, State, Debug) -> +handle_common_reply(Reply, Parent, Name, From, Msg, CbCache, HibernateAfterTimeout, State, Debug) -> + Mod = CbCache#callback_cache.module, case Reply of {ok, {noreply, NState}} -> Debug1 = sys:handle_debug(Debug, fun print_event/3, Name, {noreply, NState}), - loop(Parent, Name, NState, Mod, infinity, HibernateAfterTimeout, Debug1); + loop(Parent, Name, NState, CbCache, infinity, HibernateAfterTimeout, Debug1); {ok, {noreply, NState, TimeoutOrHibernate}} when ?is_timeout(TimeoutOrHibernate); TimeoutOrHibernate =:= hibernate -> Debug1 = sys:handle_debug(Debug, fun print_event/3, Name, {noreply, NState}), - loop(Parent, Name, NState, Mod, TimeoutOrHibernate, HibernateAfterTimeout, Debug1); + loop(Parent, Name, NState, CbCache, TimeoutOrHibernate, HibernateAfterTimeout, Debug1); {ok, {noreply, NState, {continue, _}=Continue}} -> Debug1 = sys:handle_debug(Debug, fun print_event/3, Name, {noreply, NState}), - loop(Parent, Name, NState, Mod, Continue, HibernateAfterTimeout, Debug1); + loop(Parent, Name, NState, CbCache, Continue, HibernateAfterTimeout, Debug1); {ok, {stop, Reason, NState}} -> terminate(Reason, ?STACKTRACE(), Name, From, Msg, Mod, NState, Debug); {'EXIT', Class, Reason, Stacktrace} -> @@ -1270,32 +1238,34 @@ handle_common_reply(Reply, Parent, Name, From, Msg, Mod, HibernateAfterTimeout, reply(Name, From, Reply, State, Debug) -> reply(From, Reply), sys:handle_debug(Debug, fun print_event/3, Name, - {out, Reply, From, State} ). + {out, Reply, From, State} ). %%----------------------------------------------------------------- %% Callback functions for system messages handling. %%----------------------------------------------------------------- -system_continue(Parent, Debug, [Name, State, Mod, Time, HibernateAfterTimeout]) -> - loop(Parent, Name, State, Mod, Time, HibernateAfterTimeout, Debug). +system_continue(Parent, Debug, [Name, State, CbCache, Time, HibernateAfterTimeout]) -> + loop(Parent, Name, State, CbCache, Time, HibernateAfterTimeout, Debug). -spec system_terminate(_, _, _, [_]) -> no_return(). -system_terminate(Reason, _Parent, Debug, [Name, State, Mod, _Time, _HibernateAfterTimeout]) -> +system_terminate(Reason, _Parent, Debug, [Name, State, CbCache, _Time, _HibernateAfterTimeout]) -> + Mod = CbCache#callback_cache.module, terminate(Reason, ?STACKTRACE(), Name, undefined, [], Mod, State, Debug). -system_code_change([Name, State, Mod, Time, HibernateAfterTimeout], _Module, OldVsn, Extra) -> +system_code_change([Name, State, CbCache, Time, HibernateAfterTimeout], _Module, OldVsn, Extra) -> + Mod = CbCache#callback_cache.module, case catch Mod:code_change(OldVsn, State, Extra) of - {ok, NewState} -> {ok, [Name, NewState, Mod, Time, HibernateAfterTimeout]}; - Else -> Else + {ok, NewState} -> {ok, [Name, NewState, CbCache, Time, HibernateAfterTimeout]}; + Else -> Else end. system_get_state([_Name, State, _Mod, _Time, _HibernateAfterTimeout]) -> {ok, State}. -system_replace_state(StateFun, [Name, State, Mod, Time, HibernateAfterTimeout]) -> +system_replace_state(StateFun, [Name, State, CbCache, Time, HibernateAfterTimeout]) -> NState = StateFun(State), - {ok, NState, [Name, NState, Mod, Time, HibernateAfterTimeout]}. + {ok, NState, [Name, NState, CbCache, Time, HibernateAfterTimeout]}. %%----------------------------------------------------------------- %% Format debug messages. Print them as the call-back module sees @@ -1348,7 +1318,7 @@ terminate(Class, Reason, Stacktrace, Name, From, Msg, Mod, State, Debug) -> -spec terminate(_, _, _, _, _, _, _, _, _, _) -> no_return(). terminate(Class, Reason, Stacktrace, ReportStacktrace, Name, From, Msg, Mod, State, Debug) -> - Reply = try_terminate(Mod, terminate_reason(Class, Reason, Stacktrace), State), + Reply = try_terminate(Mod, catch_result(Class, Reason, Stacktrace), State), case Reply of {'EXIT', C, R, S} -> error_info(R, S, Name, From, Msg, Mod, State, Debug), @@ -1371,8 +1341,9 @@ terminate(Class, Reason, Stacktrace, ReportStacktrace, Name, From, Msg, Mod, Sta erlang:raise(Class, Reason, Stacktrace) end. -terminate_reason(error, Reason, Stacktrace) -> {Reason, Stacktrace}; -terminate_reason(exit, Reason, _Stacktrace) -> Reason. +%% What an old style `catch` would return +catch_result(error, Reason, Stacktrace) -> {Reason, Stacktrace}; +catch_result(exit, Reason, _Stacktrace) -> Reason. error_info(_Reason, _ST, application_controller, _From, _Msg, _Mod, _State, _Debug) -> %% OTP-5811 Don't send an error report if it's the system process @@ -1659,7 +1630,8 @@ mod(_) -> "t". %% Status information %%----------------------------------------------------------------- format_status(Opt, StatusData) -> - [PDict, SysState, Parent, Debug, [Name, State, Mod, _Time, _HibernateAfterTimeout]] = StatusData, + [PDict, SysState, Parent, Debug, [Name, State, CbCache, _Time, _HibernateAfterTimeout]] = StatusData, + Mod = CbCache#callback_cache.module, Header = gen:format_status_header("Status for generic server", Name), Status = case gen:format_status(Mod, Opt, #{ state => State, log => sys:get_log(Debug) }, diff --git a/lib/stdlib/src/gen_statem.erl b/lib/stdlib/src/gen_statem.erl index e3fa14a8a8..849bf45561 100644 --- a/lib/stdlib/src/gen_statem.erl +++ b/lib/stdlib/src/gen_statem.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2016-2022. All Rights Reserved. +%% Copyright Ericsson AB 2016-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -97,6 +97,9 @@ start_ret/0, start_mon_ret/0]). +%% -define(DBG(T), erlang:display({{self(), ?MODULE, ?LINE, ?FUNCTION_NAME}, T})). + + %%%========================================================================== %%% Interface functions. %%%========================================================================== @@ -225,8 +228,9 @@ {ok, State :: StateType, Data :: DataType} | {ok, State :: StateType, Data :: DataType, Actions :: [action()] | action()} | - 'ignore' | - {'stop', Reason :: term()}. + 'ignore' | + {'stop', Reason :: term()} | + {'error', Reason :: term()}. %% Old, not advertised -type state_function_result() :: @@ -641,15 +645,15 @@ call(ServerRef, Request) -> {'dirty_timeout',T :: timeout()}) -> Reply :: term(). call(ServerRef, Request, infinity = T = Timeout) -> - call_dirty(ServerRef, Request, Timeout, T); + call(ServerRef, Request, Timeout, T); call(ServerRef, Request, {dirty_timeout, T} = Timeout) -> - call_dirty(ServerRef, Request, Timeout, T); + call(ServerRef, Request, Timeout, T); call(ServerRef, Request, {clean_timeout, T} = Timeout) -> - call_clean(ServerRef, Request, Timeout, T); + call(ServerRef, Request, Timeout, T); call(ServerRef, Request, {_, _} = Timeout) -> erlang:error(badarg, [ServerRef,Request,Timeout]); call(ServerRef, Request, Timeout) -> - call_clean(ServerRef, Request, Timeout, Timeout). + call(ServerRef, Request, Timeout, Timeout). -spec send_request(ServerRef::server_ref(), Request::term()) -> ReqId::request_id(). @@ -896,7 +900,8 @@ enter_loop(Module, Opts, State, Data, Server, Actions) -> wrap_cast(Event) -> {'$gen_cast',Event}. -call_dirty(ServerRef, Request, Timeout, T) -> +-compile({inline, [call/4]}). +call(ServerRef, Request, Timeout, T) -> try gen:call(ServerRef, '$gen_call', Request, T) of {ok,Reply} -> Reply @@ -910,63 +915,6 @@ call_dirty(ServerRef, Request, Timeout, T) -> Stacktrace) end. -call_clean(ServerRef, Request, Timeout, T) - when (is_pid(ServerRef) - andalso (node(ServerRef) == node())) - orelse (element(2, ServerRef) == node() - andalso is_atom(element(1, ServerRef)) - andalso (tuple_size(ServerRef) =:= 2)) -> - %% No need to use a proxy locally since we know alias will be - %% used as of OTP 24 which will prevent garbage responses... - call_dirty(ServerRef, Request, Timeout, T); -call_clean(ServerRef, Request, Timeout, T) -> - %% Call server through proxy process to dodge any late reply - %% - %% We still need a proxy in the distributed case since we may - %% communicate with a node that does not understand aliases. - %% This can be removed when alias support is mandatory. - %% Probably in OTP 26. - Ref = make_ref(), - Self = self(), - Pid = spawn( - fun () -> - Self ! - try gen:call( - ServerRef, '$gen_call', Request, T) of - Result -> - {Ref,Result} - catch Class:Reason:Stacktrace -> - {Ref,Class,Reason,Stacktrace} - end - end), - Mref = monitor(process, Pid), - receive - {Ref,Result} -> - demonitor(Mref, [flush]), - case Result of - {ok,Reply} -> - Reply - end; - {Ref,Class,Reason,Stacktrace} when Class =:= exit -> - %% 'gen' raises 'exit' for problems - demonitor(Mref, [flush]), - %% Pretend it happened in this process - erlang:raise( - Class, - %% Wrap the reason according to tradition - {Reason,{?MODULE,call,[ServerRef,Request,Timeout]}}, - Stacktrace); - {Ref,Class,Reason,Stacktrace} -> - demonitor(Mref, [flush]), - %% Pretend it happened in this process - erlang:raise(Class, Reason, Stacktrace); - {'DOWN',Mref,_,_,Reason} -> - %% There is a theoretical possibility that the - %% proxy process gets killed between try--of and ! - %% so this clause is in case of that - exit(Reason) - end. - replies([{reply,From,Reply}|Replies]) -> reply(From, Reply), replies(Replies); @@ -1027,12 +975,12 @@ init_it(Starter, Parent, ServerRef, Module, Args, Opts) -> Name, Debug, HibernateAfterTimeout); Class:Reason:Stacktrace -> gen:unregister_name(ServerRef), - proc_lib:init_ack(Starter, {error,Reason}), error_info( Class, Reason, Stacktrace, Debug, #params{parent = Parent, name = Name, modules = [Module]}, #state{}, []), - erlang:raise(Class, Reason, Stacktrace) + proc_lib:init_fail( + Starter, {error,Reason}, {Class,Reason,Stacktrace}) end. %%--------------------------------------------------------------------------- @@ -1054,21 +1002,24 @@ init_result( State, Data, Actions); {stop,Reason} -> gen:unregister_name(ServerRef), - proc_lib:init_ack(Starter, {error,Reason}), - exit(Reason); + exit(Reason); + {error, _Reason} = ERROR -> + %% The point of this clause is that we shall have a *silent* + %% termination. The error reason will be returned to the + %% 'Starter' ({error, Reason}), but *no* crash report. + gen:unregister_name(ServerRef), + proc_lib:init_fail(Starter, ERROR, {exit,normal}); ignore -> gen:unregister_name(ServerRef), - proc_lib:init_ack(Starter, ignore), - exit(normal); + proc_lib:init_fail(Starter, ignore, {exit,normal}); _ -> gen:unregister_name(ServerRef), - Error = {bad_return_from_init,Result}, - proc_lib:init_ack(Starter, {error,Error}), + Reason = {bad_return_from_init,Result}, error_info( - error, Error, ?STACKTRACE(), Debug, + error, Reason, ?STACKTRACE(), Debug, #params{parent = Parent, name = Name, modules = [Module]}, #state{}, []), - exit(Error) + exit(Reason) end. %%%========================================================================== @@ -3074,9 +3025,6 @@ cancel_timer(TimeoutType, Timers) -> %% Return a list of all pending timeouts list_timeouts(Timers) -> {maps:size(Timers) - 1, % Subtract fixed key 't0q' - maps:fold( - fun (t0q, _, Acc) -> - Acc; - (TimeoutType, {_TimerRef,TimeoutMsg}, Acc) -> - [{TimeoutType,TimeoutMsg}|Acc] - end, [], Timers)}. + [{TimeoutType, TimeoutMsg} + || TimeoutType := {_TimerRef, TimeoutMsg} <- Timers, + TimeoutType =/= t0q]}. diff --git a/lib/stdlib/src/io.erl b/lib/stdlib/src/io.erl index 18f6ef3dde..067177155e 100644 --- a/lib/stdlib/src/io.erl +++ b/lib/stdlib/src/io.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2021. All Rights Reserved. +%% Copyright Ericsson AB 1996-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -209,19 +209,22 @@ get_password(Io) -> -type encoding() :: 'latin1' | 'unicode' | 'utf8' | 'utf16' | 'utf32' | {'utf16', 'big' | 'little'} | {'utf32','big' | 'little'}. --type expand_fun() :: fun((term()) -> {'yes'|'no', string(), [string(), ...]}). +-type expand_fun() :: fun((string()) -> {'yes'|'no', string(), list()}). -type opt_pair() :: {'binary', boolean()} | {'echo', boolean()} | {'expand_fun', expand_fun()} - | {'encoding', encoding()}. + | {'encoding', encoding()} + | {atom(), term()}. +-type get_opt_pair() :: opt_pair() + | {'terminal', boolean()}. --spec getopts() -> [opt_pair()] | {'error', Reason} when +-spec getopts() -> [get_opt_pair()] | {'error', Reason} when Reason :: term(). getopts() -> getopts(default_input()). --spec getopts(IoDevice) -> [opt_pair()] | {'error', Reason} when +-spec getopts(IoDevice) -> [get_opt_pair()] | {'error', Reason} when IoDevice :: device(), Reason :: term(). diff --git a/lib/stdlib/src/io_lib.erl b/lib/stdlib/src/io_lib.erl index e2823b70f2..5f45165968 100644 --- a/lib/stdlib/src/io_lib.erl +++ b/lib/stdlib/src/io_lib.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2019. All Rights Reserved. +%% Copyright Ericsson AB 1996-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -124,7 +124,9 @@ precision := 'none' | integer(), pad_char := char(), encoding := 'unicode' | 'latin1', - strings := boolean() + strings := boolean(), + % `maps_order` has been added since OTP26 and is optional + maps_order => maps:iterator_order() }. %%---------------------------------------------------------------------- @@ -322,7 +324,7 @@ add_modifier(_, C) -> Term :: term(). write(Term) -> - write1(Term, -1, latin1). + write1(Term, -1, latin1, undefined). -spec write(term(), depth(), boolean()) -> chars(). @@ -347,64 +349,65 @@ write(Term, Options) when is_list(Options) -> Depth = get_option(depth, Options, -1), Encoding = get_option(encoding, Options, epp:default_encoding()), CharsLimit = get_option(chars_limit, Options, -1), + MapsOrder = get_option(maps_order, Options, undefined), if Depth =:= 0; CharsLimit =:= 0 -> "..."; - CharsLimit < 0 -> - write1(Term, Depth, Encoding); - CharsLimit > 0 -> + is_integer(CharsLimit), CharsLimit < 0, is_integer(Depth) -> + write1(Term, Depth, Encoding, MapsOrder); + is_integer(CharsLimit), CharsLimit > 0 -> RecDefFun = fun(_, _) -> no end, If = io_lib_pretty:intermediate - (Term, Depth, CharsLimit, RecDefFun, Encoding, _Str=false), + (Term, Depth, CharsLimit, RecDefFun, Encoding, _Str=false, MapsOrder), io_lib_pretty:write(If) end; write(Term, Depth) -> write(Term, [{depth, Depth}, {encoding, latin1}]). -write1(_Term, 0, _E) -> "..."; -write1(Term, _D, _E) when is_integer(Term) -> integer_to_list(Term); -write1(Term, _D, _E) when is_float(Term) -> io_lib_format:fwrite_g(Term); -write1(Atom, _D, latin1) when is_atom(Atom) -> write_atom_as_latin1(Atom); -write1(Atom, _D, _E) when is_atom(Atom) -> write_atom(Atom); -write1(Term, _D, _E) when is_port(Term) -> write_port(Term); -write1(Term, _D, _E) when is_pid(Term) -> pid_to_list(Term); -write1(Term, _D, _E) when is_reference(Term) -> write_ref(Term); -write1(<<_/bitstring>>=Term, D, _E) -> write_binary(Term, D); -write1([], _D, _E) -> "[]"; -write1({}, _D, _E) -> "{}"; -write1([H|T], D, E) -> +write1(_Term, 0, _E, _O) -> "..."; +write1(Term, _D, _E, _O) when is_integer(Term) -> integer_to_list(Term); +write1(Term, _D, _E, _O) when is_float(Term) -> io_lib_format:fwrite_g(Term); +write1(Atom, _D, latin1, _O) when is_atom(Atom) -> write_atom_as_latin1(Atom); +write1(Atom, _D, _E, _O) when is_atom(Atom) -> write_atom(Atom); +write1(Term, _D, _E, _O) when is_port(Term) -> write_port(Term); +write1(Term, _D, _E, _O) when is_pid(Term) -> pid_to_list(Term); +write1(Term, _D, _E, _O) when is_reference(Term) -> write_ref(Term); +write1(<<_/bitstring>>=Term, D, _E, _O) -> write_binary(Term, D); +write1([], _D, _E, _O) -> "[]"; +write1({}, _D, _E, _O) -> "{}"; +write1([H|T], D, E, O) -> if D =:= 1 -> "[...]"; true -> - [$[,[write1(H, D-1, E)|write_tail(T, D-1, E)],$]] + [$[,[write1(H, D-1, E, O)|write_tail(T, D-1, E, O)],$]] end; -write1(F, _D, _E) when is_function(F) -> +write1(F, _D, _E, _O) when is_function(F) -> erlang:fun_to_list(F); -write1(Term, D, E) when is_map(Term) -> - write_map(Term, D, E); -write1(T, D, E) when is_tuple(T) -> +write1(Term, D, E, O) when is_map(Term) -> + write_map(Term, D, E, O); +write1(T, D, E, O) when is_tuple(T) -> if D =:= 1 -> "{...}"; true -> [${, - [write1(element(1, T), D-1, E)|write_tuple(T, 2, D-1, E)], + [write1(element(1, T), D-1, E, O)|write_tuple(T, 2, D-1, E, O)], $}] end. %% write_tail(List, Depth, Encoding) %% Test the terminating case first as this looks better with depth. -write_tail([], _D, _E) -> ""; -write_tail(_, 1, _E) -> [$| | "..."]; -write_tail([H|T], D, E) -> - [$,,write1(H, D-1, E)|write_tail(T, D-1, E)]; -write_tail(Other, D, E) -> - [$|,write1(Other, D-1, E)]. +write_tail([], _D, _E, _O) -> ""; +write_tail(_, 1, _E, _O) -> [$| | "..."]; +write_tail([H|T], D, E, O) -> + [$,,write1(H, D-1, E, O)|write_tail(T, D-1, E, O)]; +write_tail(Other, D, E, O) -> + [$|,write1(Other, D-1, E, O)]. -write_tuple(T, I, _D, _E) when I > tuple_size(T) -> ""; -write_tuple(_, _I, 1, _E) -> [$, | "..."]; -write_tuple(T, I, D, E) -> - [$,,write1(element(I, T), D-1, E)|write_tuple(T, I+1, D-1, E)]. +write_tuple(T, I, _D, _E, _O) when I > tuple_size(T) -> ""; +write_tuple(_, _I, 1, _E, _O) -> [$, | "..."]; +write_tuple(T, I, D, E, O) -> + [$,,write1(element(I, T), D-1, E, O)|write_tuple(T, I+1, D-1, E, O)]. write_port(Port) -> erlang:port_to_list(Port). @@ -412,34 +415,34 @@ write_port(Port) -> write_ref(Ref) -> erlang:ref_to_list(Ref). -write_map(_, 1, _E) -> "#{}"; -write_map(Map, D, E) when is_integer(D) -> - I = maps:iterator(Map), +write_map(_, 1, _E, _O) -> "#{}"; +write_map(Map, D, E, O) when is_integer(D) -> + I = maps:iterator(Map, O), case maps:next(I) of {K, V, NextI} -> D0 = D - 1, - W = write_map_assoc(K, V, D0, E), - [$#,${,[W | write_map_body(NextI, D0, D0, E)],$}]; + W = write_map_assoc(K, V, D0, E, O), + [$#,${,[W | write_map_body(NextI, D0, D0, E, O)],$}]; none -> "#{}" end. -write_map_body(_, 1, _D0, _E) -> ",..."; -write_map_body(I, D, D0, E) -> +write_map_body(_, 1, _D0, _E, _O) -> ",..."; +write_map_body(I, D, D0, E, O) -> case maps:next(I) of {K, V, NextI} -> - W = write_map_assoc(K, V, D0, E), - [$,,W|write_map_body(NextI, D - 1, D0, E)]; + W = write_map_assoc(K, V, D0, E, O), + [$,,W|write_map_body(NextI, D - 1, D0, E, O)]; none -> "" end. -write_map_assoc(K, V, D, E) -> - [write1(K, D, E)," => ",write1(V, D, E)]. +write_map_assoc(K, V, D, E, O) -> + [write1(K, D, E, O)," => ",write1(V, D, E, O)]. write_binary(B, D) when is_integer(D) -> {S, _} = write_binary(B, D, -1), S. -write_binary(B, D, T) -> +write_binary(B, D, T) when is_integer(T) -> {S, Rest} = write_binary_body(B, D, tsub(T, 4), []), {[$<,$<,lists:reverse(S),$>,$>], Rest}. @@ -509,15 +512,16 @@ quote_atom(Atom, Cs0) -> true -> true; false -> case Cs0 of - [C|Cs] when C >= $a, C =< $z -> + [C|Cs] when is_integer(C), C >= $a, C =< $z -> not name_chars(Cs); - [C|Cs] when C >= $ร, C =< $รฟ, C =/= $รท -> + [C|Cs] when is_integer(C), C >= $ร, C =< $รฟ, C =/= $รท -> not name_chars(Cs); - _ -> true + [C|_] when is_integer(C) -> true; + [] -> true end end. -name_chars([C|Cs]) -> +name_chars([C|Cs]) when is_integer(C) -> case name_char(C) of true -> name_chars(Cs); false -> false @@ -580,7 +584,7 @@ write_string_as_latin1(S, Q) -> write_string1(_,[], Q) -> [Q]; -write_string1(Enc,[C|Cs], Q) -> +write_string1(Enc,[C|Cs], Q) when is_integer(C) -> string_char(Enc,C, Q, write_string1(Enc,Cs, Q)). string_char(_,Q, Q, Tail) -> [$\\,Q|Tail]; %Must check these first! @@ -803,53 +807,45 @@ collect_chars(Tag, Data, N) -> collect_chars(Tag, Data, latin1, N). %% Now we are aware of encoding... -collect_chars(start, Data, unicode, N) when is_binary(Data) -> +collect_chars(start, Data, unicode, N) when is_binary(Data), is_integer(N) -> {Size,Npos} = count_and_find_utf8(Data,N), - if Size > N -> + if Size >= N -> {B1,B2} = split_binary(Data, Npos), {stop,B1,B2}; Size < N -> - {binary,[Data],N-Size}; - true -> - {stop,Data,eof} + {binary,[Data],N-Size} end; -collect_chars(start, Data, latin1, N) when is_binary(Data) -> +collect_chars(start, Data, latin1, N) when is_binary(Data), is_integer(N) -> Size = byte_size(Data), - if Size > N -> + if Size >= N -> {B1,B2} = split_binary(Data, N), {stop,B1,B2}; Size < N -> - {binary,[Data],N-Size}; - true -> - {stop,Data,eof} + {binary,[Data],N-Size} end; -collect_chars(start,Data,_,N) when is_list(Data) -> +collect_chars(start,Data,_,N) when is_list(Data), is_integer(N) -> collect_chars_list([], N, Data); collect_chars(start, eof, _,_) -> {stop,eof,eof}; collect_chars({binary,Stack,_N}, eof, _,_) -> {stop,binrev(Stack),eof}; -collect_chars({binary,Stack,N}, Data,unicode, _) -> +collect_chars({binary,Stack,N}, Data,unicode, _) when is_integer(N) -> {Size,Npos} = count_and_find_utf8(Data,N), - if Size > N -> + if Size >= N -> {B1,B2} = split_binary(Data, Npos), {stop,binrev(Stack, [B1]),B2}; Size < N -> - {binary,[Data|Stack],N-Size}; - true -> - {stop,binrev(Stack, [Data]),eof} + {binary,[Data|Stack],N-Size} end; -collect_chars({binary,Stack,N}, Data,latin1, _) -> +collect_chars({binary,Stack,N}, Data,latin1, _) when is_integer(N) -> Size = byte_size(Data), - if Size > N -> + if Size >= N -> {B1,B2} = split_binary(Data, N), {stop,binrev(Stack, [B1]),B2}; Size < N -> - {binary,[Data|Stack],N-Size}; - true -> - {stop,binrev(Stack, [Data]),eof} + {binary,[Data|Stack],N-Size} end; -collect_chars({list,Stack,N}, Data, _,_) -> +collect_chars({list,Stack,N}, Data, _,_) when is_integer(N) -> collect_chars_list(Stack, N, Data); %% collect_chars(Continuation, MoreChars, Count) @@ -857,9 +853,9 @@ collect_chars({list,Stack,N}, Data, _,_) -> %% {done,Result,RestChars} %% {more,Continuation} -collect_chars([], Chars, _, N) -> +collect_chars([], Chars, _, N) when is_integer(N) -> collect_chars1(N, Chars, []); -collect_chars({Left,Sofar}, Chars, _, _N) -> +collect_chars({Left,Sofar}, Chars, _, _N) when is_integer(Left) -> collect_chars1(Left, Chars, Sofar). collect_chars1(N, Chars, Stack) when N =< 0 -> @@ -991,13 +987,13 @@ binrev(L) -> binrev(L, T) -> list_to_binary(lists:reverse(L, T)). --spec limit_term(term(), non_neg_integer()) -> term(). +-spec limit_term(term(), depth()) -> term(). %% The intention is to mimic the depth limitation of io_lib:write() %% and io_lib_pretty:print(). The leaves ('...') should never be %% seen when printed with the same depth. Bitstrings are never %% truncated, which is OK as long as they are not sent to other nodes. -limit_term(Term, Depth) -> +limit_term(Term, Depth) when is_integer(Depth), Depth >= -1 -> try test_limit(Term, Depth) of ok -> Term catch diff --git a/lib/stdlib/src/io_lib_format.erl b/lib/stdlib/src/io_lib_format.erl index fb77957036..813125abd9 100644 --- a/lib/stdlib/src/io_lib_format.erl +++ b/lib/stdlib/src/io_lib_format.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2021. All Rights Reserved. +%% Copyright Ericsson AB 1996-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -106,6 +106,8 @@ scan(Format, Args) -> unscan(Cs) -> {print(Cs), args(Cs)}. +args([#{args := As, maps_order := O} | Cs]) when is_function(O, 2); O =:= reversed -> + [O | As] ++ args(Cs); args([#{args := As} | Cs]) -> As ++ args(Cs); args([_C | Cs]) -> @@ -114,17 +116,20 @@ args([]) -> []. print([#{control_char := C, width := F, adjust := Ad, precision := P, - pad_char := Pad, encoding := Encoding, strings := Strings} | Cs]) -> - print(C, F, Ad, P, Pad, Encoding, Strings) ++ print(Cs); -print([C | Cs]) -> + pad_char := Pad, encoding := Encoding, strings := Strings + } = Map | Cs]) -> + MapsOrder = maps:get(maps_order, Map, undefined), + print(C, F, Ad, P, Pad, Encoding, Strings, MapsOrder) ++ print(Cs); +print([C | Cs]) when is_integer(C) -> [C | print(Cs)]; print([]) -> []. -print(C, F, Ad, P, Pad, Encoding, Strings) -> +print(C, F, Ad, P, Pad, Encoding, Strings, MapsOrder) -> [$~] ++ print_field_width(F, Ad) ++ print_precision(P, Pad) ++ print_pad_char(Pad) ++ print_encoding(Encoding) ++ - print_strings(Strings) ++ [C]. + print_strings(Strings) ++ print_maps_order(MapsOrder) ++ + [C]. print_field_width(none, _Ad) -> ""; print_field_width(F, left) -> integer_to_list(-F); @@ -143,6 +148,11 @@ print_encoding(latin1) -> "". print_strings(false) -> "l"; print_strings(true) -> "". +print_maps_order(undefined) -> ""; +print_maps_order(ordered) -> "k"; +print_maps_order(reversed) -> "K"; +print_maps_order(CmpFun) when is_function(CmpFun, 2) -> "K". + collect([$~|Fmt0], Args0) -> {C,Fmt1,Args1} = collect_cseq(Fmt0, Args0), [C|collect(Fmt1, Args1)]; @@ -159,18 +169,23 @@ collect_cseq(Fmt0, Args0) -> precision => P, pad_char => Pad, encoding => latin1, - strings => true}, - {Spec1,Fmt4} = modifiers(Fmt3, Spec0), - {C,As,Fmt5,Args4} = collect_cc(Fmt4, Args3), + strings => true, + maps_order => undefined}, + {Spec1,Fmt4,Args4} = modifiers(Fmt3, Args3, Spec0), + {C,As,Fmt5,Args5} = collect_cc(Fmt4, Args4), Spec2 = Spec1#{control_char => C, args => As}, - {Spec2,Fmt5,Args4}. - -modifiers([$t|Fmt], Spec) -> - modifiers(Fmt, Spec#{encoding => unicode}); -modifiers([$l|Fmt], Spec) -> - modifiers(Fmt, Spec#{strings => false}); -modifiers(Fmt, Spec) -> - {Spec, Fmt}. + {Spec2,Fmt5,Args5}. + +modifiers([$t|Fmt], Args, Spec) -> + modifiers(Fmt, Args, Spec#{encoding => unicode}); +modifiers([$l|Fmt], Args, Spec) -> + modifiers(Fmt, Args, Spec#{strings => false}); +modifiers([$k|Fmt], Args, Spec) -> + modifiers(Fmt, Args, Spec#{maps_order => ordered}); +modifiers([$K|Fmt], [MapsOrder | Args], Spec) -> + modifiers(Fmt, Args, Spec#{maps_order => MapsOrder}); +modifiers(Fmt, Args, Spec) -> + {Spec, Fmt, Args}. field_width([$-|Fmt0], Args0) -> {F,Fmt,Args} = field_value(Fmt0, Args0), @@ -274,12 +289,14 @@ build_small([]) -> []. build_limited([#{control_char := C, args := As, width := F, adjust := Ad, precision := P, pad_char := Pad, encoding := Enc, - strings := Str} | Cs], NumOfPs0, Count0, MaxLen0, I) -> + strings := Str} = Map | Cs], + NumOfPs0, Count0, MaxLen0, I) -> + Ord = maps:get(maps_order, Map, undefined), MaxChars = if MaxLen0 < 0 -> MaxLen0; true -> MaxLen0 div Count0 end, - S = control_limited(C, As, F, Ad, P, Pad, Enc, Str, MaxChars, I), + S = control_limited(C, As, F, Ad, P, Pad, Enc, Str, Ord, MaxChars, I), NumOfPs = decr_pc(C, NumOfPs0), Count = Count0 - 1, MaxLen = if @@ -371,24 +388,34 @@ control_small($n, [], F, Adj, P, Pad, _Enc) -> newline(F, Adj, P, Pad); control_small($i, [_A], _F, _Adj, _P, _Pad, _Enc) -> []; control_small(_C, _As, _F, _Adj, _P, _Pad, _Enc) -> not_small. -control_limited($s, [L0], F, Adj, P, Pad, latin1=Enc, _Str, CL, _I) -> +control_limited($s, [L0], F, Adj, P, Pad, latin1=Enc, _Str, _Ord, CL, _I) -> L = iolist_to_chars(L0, F, CL), string(L, limit_field(F, CL), Adj, P, Pad, Enc); -control_limited($s, [L0], F, Adj, P, Pad, unicode=Enc, _Str, CL, _I) -> +control_limited($s, [L0], F, Adj, P, Pad, unicode=Enc, _Str, _Ord, CL, _I) -> L = cdata_to_chars(L0, F, CL), uniconv(string(L, limit_field(F, CL), Adj, P, Pad, Enc)); -control_limited($w, [A], F, Adj, P, Pad, Enc, _Str, CL, _I) -> - Chars = io_lib:write(A, [{depth, -1}, {encoding, Enc}, {chars_limit, CL}]), +control_limited($w, [A], F, Adj, P, Pad, Enc, _Str, Ord, CL, _I) -> + Chars = io_lib:write(A, [ + {depth, -1}, + {encoding, Enc}, + {chars_limit, CL}, + {maps_order, Ord} + ]), term(Chars, F, Adj, P, Pad); -control_limited($p, [A], F, Adj, P, Pad, Enc, Str, CL, I) -> - print(A, -1, F, Adj, P, Pad, Enc, Str, CL, I); -control_limited($W, [A,Depth], F, Adj, P, Pad, Enc, _Str, CL, _I) +control_limited($p, [A], F, Adj, P, Pad, Enc, Str, Ord, CL, I) -> + print(A, -1, F, Adj, P, Pad, Enc, Str, Ord, CL, I); +control_limited($W, [A,Depth], F, Adj, P, Pad, Enc, _Str, Ord, CL, _I) when is_integer(Depth) -> - Chars = io_lib:write(A, [{depth, Depth}, {encoding, Enc}, {chars_limit, CL}]), + Chars = io_lib:write(A, [ + {depth, Depth}, + {encoding, Enc}, + {chars_limit, CL}, + {maps_order, Ord} + ]), term(Chars, F, Adj, P, Pad); -control_limited($P, [A,Depth], F, Adj, P, Pad, Enc, Str, CL, I) +control_limited($P, [A,Depth], F, Adj, P, Pad, Enc, Str, Ord, CL, I) when is_integer(Depth) -> - print(A, Depth, F, Adj, P, Pad, Enc, Str, CL, I). + print(A, Depth, F, Adj, P, Pad, Enc, Str, Ord, CL, I). -ifdef(UNICODE_AS_BINARIES). uniconv(C) -> @@ -425,17 +452,18 @@ term(T, F, Adj, P0, Pad) -> %% Print a term. Field width sets maximum line length, Precision sets %% initial indentation. -print(T, D, none, Adj, P, Pad, E, Str, ChLim, I) -> - print(T, D, 80, Adj, P, Pad, E, Str, ChLim, I); -print(T, D, F, Adj, none, Pad, E, Str, ChLim, I) -> - print(T, D, F, Adj, I+1, Pad, E, Str, ChLim, I); -print(T, D, F, right, P, _Pad, Enc, Str, ChLim, _I) -> +print(T, D, none, Adj, P, Pad, E, Str, Ord, ChLim, I) -> + print(T, D, 80, Adj, P, Pad, E, Str, Ord, ChLim, I); +print(T, D, F, Adj, none, Pad, E, Str, Ord, ChLim, I) -> + print(T, D, F, Adj, I+1, Pad, E, Str, Ord, ChLim, I); +print(T, D, F, right, P, _Pad, Enc, Str, Ord, ChLim, _I) -> Options = [{chars_limit, ChLim}, {column, P}, {line_length, F}, {depth, D}, {encoding, Enc}, - {strings, Str}], + {strings, Str}, + {maps_order, Ord}], io_lib_pretty:print(T, Options). %% fwrite_e(Float, Field, Adjust, Precision, PadChar) diff --git a/lib/stdlib/src/io_lib_pretty.erl b/lib/stdlib/src/io_lib_pretty.erl index 98eea64b0e..4ed42d6c9f 100644 --- a/lib/stdlib/src/io_lib_pretty.erl +++ b/lib/stdlib/src/io_lib_pretty.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2021. All Rights Reserved. +%% Copyright Ericsson AB 1996-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -27,7 +27,7 @@ -export([print/1,print/2,print/3,print/4,print/5,print/6]). %% To be used by io_lib only. --export([intermediate/6, write/1]). +-export([intermediate/7, write/1]). %%% %%% Exported functions @@ -64,7 +64,8 @@ print(Term) -> | {'line_length', line_length()} | {'line_max_chars', line_max_chars()} | {'record_print_fun', rec_print_fun()} - | {'strings', boolean()}. + | {'strings', boolean()} + | {'maps_order', maps:iterator_order()}. -type options() :: [option()]. -spec print(term(), rec_print_fun()) -> chars(); @@ -79,7 +80,8 @@ print(Term, Options) when is_list(Options) -> RecDefFun = get_option(record_print_fun, Options, no_fun), Encoding = get_option(encoding, Options, epp:default_encoding()), Strings = get_option(strings, Options, true), - print(Term, Col, Ll, D, M, T, RecDefFun, Encoding, Strings); + MapsOrder = get_option(maps_order, Options, undefined), + print(Term, Col, Ll, D, M, T, RecDefFun, Encoding, Strings, MapsOrder); print(Term, RecDefFun) -> print(Term, -1, RecDefFun). @@ -91,7 +93,7 @@ print(Term, Depth, RecDefFun) -> -spec print(term(), column(), line_length(), depth()) -> chars(). print(Term, Col, Ll, D) -> - print(Term, Col, Ll, D, _M=-1, _T=-1, no_fun, latin1, true). + print(Term, Col, Ll, D, _M=-1, _T=-1, no_fun, latin1, true, undefined). -spec print(term(), column(), line_length(), depth(), rec_print_fun()) -> chars(). @@ -102,7 +104,7 @@ print(Term, Col, Ll, D, RecDefFun) -> rec_print_fun()) -> chars(). print(Term, Col, Ll, D, M, RecDefFun) -> - print(Term, Col, Ll, D, M, _T=-1, RecDefFun, latin1, true). + print(Term, Col, Ll, D, M, _T=-1, RecDefFun, latin1, true, undefined). %% D = Depth, default -1 (infinite), or LINEMAX=30 when printing from shell %% T = chars_limit, that is, maximal number of characters, default -1 @@ -111,22 +113,22 @@ print(Term, Col, Ll, D, M, RecDefFun) -> %% Col = current column, default 1 %% Ll = line length/~p field width, default 80 %% M = CHAR_MAX (-1 if no max, 60 when printing from shell) -print(_, _, _, 0, _M, _T, _RF, _Enc, _Str) -> "..."; -print(_, _, _, _D, _M, 0, _RF, _Enc, _Str) -> "..."; -print(Term, Col, Ll, D, M, T, RecDefFun, Enc, Str) when Col =< 0 -> +print(_, _, _, 0, _M, _T, _RF, _Enc, _Str, _Ord) -> "..."; +print(_, _, _, _D, _M, 0, _RF, _Enc, _Str, _Ord) -> "..."; +print(Term, Col, Ll, D, M, T, RecDefFun, Enc, Str, Ord) when Col =< 0 -> %% ensure Col is at least 1 - print(Term, 1, Ll, D, M, T, RecDefFun, Enc, Str); -print(Atom, _Col, _Ll, _D, _M, _T, _RF, Enc, _Str) when is_atom(Atom) -> + print(Term, 1, Ll, D, M, T, RecDefFun, Enc, Str, Ord); +print(Atom, _Col, _Ll, _D, _M, _T, _RF, Enc, _Str, _Ord) when is_atom(Atom) -> write_atom(Atom, Enc); -print(Term, Col, Ll, D, M0, T, RecDefFun, Enc, Str) when is_tuple(Term); +print(Term, Col, Ll, D, M0, T, RecDefFun, Enc, Str, Ord) when is_tuple(Term); is_list(Term); is_map(Term); is_bitstring(Term) -> %% preprocess and compute total number of chars {_, Len, _Dots, _} = If = case T < 0 of - true -> print_length(Term, D, T, RecDefFun, Enc, Str); - false -> intermediate(Term, D, T, RecDefFun, Enc, Str) + true -> print_length(Term, D, T, RecDefFun, Enc, Str, Ord); + false -> intermediate(Term, D, T, RecDefFun, Enc, Str, Ord) end, %% use Len as CHAR_MAX if M0 = -1 M = max_cs(M0, Len), @@ -143,7 +145,7 @@ print(Term, Col, Ll, D, M0, T, RecDefFun, Enc, Str) when is_tuple(Term); 1), pp(If, Col, Ll, M, TInd, indent(Col), 0, 0) end; -print(Term, _Col, _Ll, _D, _M, _T, _RF, _Enc, _Str) -> +print(Term, _Col, _Ll, _D, _M, _T, _RF, _Enc, _Str, _Ord) -> %% atomic data types (bignums, atoms, ...) are never truncated io_lib:write(Term). @@ -442,19 +444,19 @@ write_tail(E, S) -> }. -spec intermediate(term(), depth(), pos_integer(), rec_print_fun(), - encoding(), boolean()) -> intermediate_format(). + encoding(), boolean(), boolean()) -> intermediate_format(). -intermediate(Term, D, T, RF, Enc, Str) when T > 0 -> +intermediate(Term, D, T, RF, Enc, Str, Ord) when T > 0 -> D0 = 1, - If = print_length(Term, D0, T, RF, Enc, Str), + If = print_length(Term, D0, T, RF, Enc, Str, Ord), case If of {_, Len, Dots, _} when Dots =:= 0; Len > T; D =:= 1 -> If; {_, Len, _, _} -> - find_upper(If, Term, T, D0, 2, D, RF, Enc, Str, Len) + find_upper(If, Term, T, D0, 2, D, RF, Enc, Str, Ord, Len) end. -find_upper(Lower, Term, T, Dl, Dd, D, RF, Enc, Str, LastLen) -> +find_upper(Lower, Term, T, Dl, Dd, D, RF, Enc, Str, Ord, LastLen) -> Dd2 = Dd * 2, D1 = case D < 0 of true -> Dl + Dd2; @@ -468,14 +470,14 @@ find_upper(Lower, Term, T, Dl, Dd, D, RF, Enc, Str, LastLen) -> %% Cannot happen if print_length() is free of bugs. If; {_, Len, _, _} when Len =< T, D1 < D orelse D < 0 -> - find_upper(If, Term, T, D1, Dd2, D, RF, Enc, Str, Len); + find_upper(If, Term, T, D1, Dd2, D, RF, Enc, Str, Ord, Len); _ -> - search_depth(Lower, If, Term, T, Dl, D1, RF, Enc, Str) + search_depth(Lower, If, Term, T, Dl, D1, RF, Enc, Str, Ord) end. %% Lower has NumOfDots > 0 and Len =< T. %% Upper has NumOfDots > 0 and Len > T. -search_depth(Lower, Upper, _Term, T, Dl, Du, _RF, _Enc, _Str) +search_depth(Lower, Upper, _Term, T, Dl, Du, _RF, _Enc, _Str, _Ord) when Du - Dl =:= 1 -> %% The returned intermediate format has Len >= T. case Lower of @@ -484,7 +486,7 @@ search_depth(Lower, Upper, _Term, T, Dl, Du, _RF, _Enc, _Str) _ -> Upper end; -search_depth(Lower, Upper, Term, T, Dl, Du, RF, Enc, Str) -> +search_depth(Lower, Upper, Term, T, Dl, Du, RF, Enc, Str, Ord) -> D1 = (Dl + Du) div 2, If = expand(Lower, T, D1 - Dl), case If of @@ -493,9 +495,9 @@ search_depth(Lower, Upper, Term, T, Dl, Du, RF, Enc, Str) -> %% This is a bit expensive since the work to %% crate Upper is wasted. It is the price %% to pay to get a more balanced output. - search_depth(Lower, If, Term, T, Dl, D1, RF, Enc, Str); + search_depth(Lower, If, Term, T, Dl, D1, RF, Enc, Str, Ord); _ -> - search_depth(If, Upper, Term, T, D1, Du, RF, Enc, Str) + search_depth(If, Upper, Term, T, D1, Du, RF, Enc, Str, Ord) end. %% The depth (D) is used for extracting and counting the characters to @@ -504,16 +506,16 @@ search_depth(Lower, Upper, Term, T, Dl, Du, RF, Enc, Str) -> %% counted but need to be added later. %% D =/= 0 -print_length([], _D, _T, _RF, _Enc, _Str) -> +print_length([], _D, _T, _RF, _Enc, _Str, _Ord) -> {"[]", 2, 0, no_more}; -print_length({}, _D, _T, _RF, _Enc, _Str) -> +print_length({}, _D, _T, _RF, _Enc, _Str, _Ord) -> {"{}", 2, 0, no_more}; -print_length(#{}=M, _D, _T, _RF, _Enc, _Str) when map_size(M) =:= 0 -> +print_length(#{}=M, _D, _T, _RF, _Enc, _Str, _Ord) when map_size(M) =:= 0 -> {"#{}", 3, 0, no_more}; -print_length(Atom, _D, _T, _RF, Enc, _Str) when is_atom(Atom) -> +print_length(Atom, _D, _T, _RF, Enc, _Str, _Ord) when is_atom(Atom) -> S = write_atom(Atom, Enc), {S, io_lib:chars_length(S), 0, no_more}; -print_length(List, D, T, RF, Enc, Str) when is_list(List) -> +print_length(List, D, T, RF, Enc, Str, Ord) when is_list(List) -> %% only flat lists are "printable" case Str andalso printable_list(List, D, T, Enc) of true -> @@ -527,37 +529,37 @@ print_length(List, D, T, RF, Enc, Str) when is_list(List) -> %% does not make Prefix longer. {[S | "..."], 3 + io_lib:chars_length(S), 0, no_more}; false -> - case print_length_list(List, D, T, RF, Enc, Str) of + case print_length_list(List, D, T, RF, Enc, Str, Ord) of {What, Len, Dots, _More} when Dots > 0 -> More = fun(T1, Dd) -> - ?FUNCTION_NAME(List, D+Dd, T1, RF, Enc, Str) + ?FUNCTION_NAME(List, D+Dd, T1, RF, Enc, Str, Ord) end, {What, Len, Dots, More}; If -> If end end; -print_length(Fun, _D, _T, _RF, _Enc, _Str) when is_function(Fun) -> +print_length(Fun, _D, _T, _RF, _Enc, _Str, _Ord) when is_function(Fun) -> S = io_lib:write(Fun), {S, iolist_size(S), 0, no_more}; -print_length(R, D, T, RF, Enc, Str) when is_atom(element(1, R)), - is_function(RF) -> +print_length(R, D, T, RF, Enc, Str, Ord) when is_atom(element(1, R)), + is_function(RF) -> case RF(element(1, R), tuple_size(R) - 1) of no -> - print_length_tuple(R, D, T, RF, Enc, Str); + print_length_tuple(R, D, T, RF, Enc, Str, Ord); RDefs -> - print_length_record(R, D, T, RF, RDefs, Enc, Str) + print_length_record(R, D, T, RF, RDefs, Enc, Str, Ord) end; -print_length(Tuple, D, T, RF, Enc, Str) when is_tuple(Tuple) -> - print_length_tuple(Tuple, D, T, RF, Enc, Str); -print_length(Map, D, T, RF, Enc, Str) when is_map(Map) -> - print_length_map(Map, D, T, RF, Enc, Str); -print_length(<<>>, _D, _T, _RF, _Enc, _Str) -> +print_length(Tuple, D, T, RF, Enc, Str, Ord) when is_tuple(Tuple) -> + print_length_tuple(Tuple, D, T, RF, Enc, Str, Ord); +print_length(Map, D, T, RF, Enc, Str, Ord) when is_map(Map) -> + print_length_map(Map, D, T, RF, Enc, Str, Ord); +print_length(<<>>, _D, _T, _RF, _Enc, _Str, _Ord) -> {"<<>>", 4, 0, no_more}; -print_length(<<_/bitstring>> = Bin, 1, _T, RF, Enc, Str) -> - More = fun(T1, Dd) -> ?FUNCTION_NAME(Bin, 1+Dd, T1, RF, Enc, Str) end, +print_length(<<_/bitstring>> = Bin, 1, _T, RF, Enc, Str, Ord) -> + More = fun(T1, Dd) -> ?FUNCTION_NAME(Bin, 1+Dd, T1, RF, Enc, Str, Ord) end, {"<<...>>", 7, 3, More}; -print_length(<<_/bitstring>> = Bin, D, T, RF, Enc, Str) -> +print_length(<<_/bitstring>> = Bin, D, T, RF, Enc, Str, Ord) -> D1 = D - 1, case Str andalso @@ -573,13 +575,13 @@ print_length(<<_/bitstring>> = Bin, D, T, RF, Enc, Str) -> {true, true, Prefix} -> S = io_lib:write_string(Prefix, $"), %" More = fun(T1, Dd) -> - ?FUNCTION_NAME(Bin, D+Dd, T1, RF, Enc, Str) + ?FUNCTION_NAME(Bin, D+Dd, T1, RF, Enc, Str, Ord) end, {[$<,$<,S|"...>>"], 7 + length(S), 3, More}; {false, true, Prefix} -> S = io_lib:write_string(Prefix, $"), %" More = fun(T1, Dd) -> - ?FUNCTION_NAME(Bin, D+Dd, T1, RF, Enc, Str) + ?FUNCTION_NAME(Bin, D+Dd, T1, RF, Enc, Str, Ord) end, {[$<,$<,S|"/utf8...>>"], 12 + io_lib:chars_length(S), 3, More}; false -> @@ -588,135 +590,135 @@ print_length(<<_/bitstring>> = Bin, D, T, RF, Enc, Str) -> {{bin, S}, iolist_size(S), 0, no_more}; {S, _Rest} -> More = fun(T1, Dd) -> - ?FUNCTION_NAME(Bin, D+Dd, T1, RF, Enc, Str) + ?FUNCTION_NAME(Bin, D+Dd, T1, RF, Enc, Str, Ord) end, {{bin, S}, iolist_size(S), 3, More} end end; -print_length(Term, _D, _T, _RF, _Enc, _Str) -> +print_length(Term, _D, _T, _RF, _Enc, _Str, _Ord) -> S = io_lib:write(Term), %% S can contain unicode, so iolist_size(S) cannot be used here {S, io_lib:chars_length(S), 0, no_more}. -print_length_map(Map, 1, _T, RF, Enc, Str) -> - More = fun(T1, Dd) -> ?FUNCTION_NAME(Map, 1+Dd, T1, RF, Enc, Str) end, +print_length_map(Map, 1, _T, RF, Enc, Str, Ord) -> + More = fun(T1, Dd) -> ?FUNCTION_NAME(Map, 1+Dd, T1, RF, Enc, Str, Ord) end, {"#{...}", 6, 3, More}; -print_length_map(Map, D, T, RF, Enc, Str) when is_map(Map) -> - Next = maps:next(maps:iterator(Map)), - PairsS = print_length_map_pairs(Next, D, D - 1, tsub(T, 3), RF, Enc, Str), +print_length_map(Map, D, T, RF, Enc, Str, Ord) when is_map(Map) -> + Next = maps:next(maps:iterator(Map, Ord)), + PairsS = print_length_map_pairs(Next, D, D - 1, tsub(T, 3), RF, Enc, Str, Ord), {Len, Dots} = list_length(PairsS, 3, 0), {{map, PairsS}, Len, Dots, no_more}. -print_length_map_pairs(none, _D, _D0, _T, _RF, _Enc, _Str) -> +print_length_map_pairs(none, _D, _D0, _T, _RF, _Enc, _Str, _Ord) -> []; -print_length_map_pairs(Term, D, D0, T, RF, Enc, Str) when D =:= 1; T =:= 0-> +print_length_map_pairs(Term, D, D0, T, RF, Enc, Str, Ord) when D =:= 1; T =:= 0-> More = fun(T1, Dd) -> - ?FUNCTION_NAME(Term, D+Dd, D0, T1, RF, Enc, Str) + ?FUNCTION_NAME(Term, D+Dd, D0, T1, RF, Enc, Str, Ord) end, {dots, 3, 3, More}; -print_length_map_pairs({K, V, Iter}, D, D0, T, RF, Enc, Str) -> +print_length_map_pairs({K, V, Iter}, D, D0, T, RF, Enc, Str, Ord) -> Next = maps:next(Iter), T1 = case Next =:= none of false -> tsub(T, 1); true -> T end, - Pair1 = print_length_map_pair(K, V, D0, T1, RF, Enc, Str), + Pair1 = print_length_map_pair(K, V, D0, T1, RF, Enc, Str, Ord), {_, Len1, _, _} = Pair1, [Pair1 | - print_length_map_pairs(Next, D - 1, D0, tsub(T1, Len1), RF, Enc, Str)]. + print_length_map_pairs(Next, D - 1, D0, tsub(T1, Len1), RF, Enc, Str, Ord)]. -print_length_map_pair(K, V, D, T, RF, Enc, Str) -> - {_, KL, KD, _} = P1 = print_length(K, D, T, RF, Enc, Str), +print_length_map_pair(K, V, D, T, RF, Enc, Str, Ord) -> + {_, KL, KD, _} = P1 = print_length(K, D, T, RF, Enc, Str, Ord), KL1 = KL + 4, - {_, VL, VD, _} = P2 = print_length(V, D, tsub(T, KL1), RF, Enc, Str), + {_, VL, VD, _} = P2 = print_length(V, D, tsub(T, KL1), RF, Enc, Str, Ord), {{map_pair, P1, P2}, KL1 + VL, KD + VD, no_more}. -print_length_tuple(Tuple, 1, _T, RF, Enc, Str) -> - More = fun(T1, Dd) -> ?FUNCTION_NAME(Tuple, 1+Dd, T1, RF, Enc, Str) end, +print_length_tuple(Tuple, 1, _T, RF, Enc, Str, Ord) -> + More = fun(T1, Dd) -> ?FUNCTION_NAME(Tuple, 1+Dd, T1, RF, Enc, Str, Ord) end, {"{...}", 5, 3, More}; -print_length_tuple(Tuple, D, T, RF, Enc, Str) -> - L = print_length_tuple1(Tuple, 1, D, tsub(T, 2), RF, Enc, Str), +print_length_tuple(Tuple, D, T, RF, Enc, Str, Ord) -> + L = print_length_tuple1(Tuple, 1, D, tsub(T, 2), RF, Enc, Str, Ord), IsTagged = is_atom(element(1, Tuple)) and (tuple_size(Tuple) > 1), {Len, Dots} = list_length(L, 2, 0), {{tuple,IsTagged,L}, Len, Dots, no_more}. -print_length_tuple1(Tuple, I, _D, _T, _RF, _Enc, _Str) +print_length_tuple1(Tuple, I, _D, _T, _RF, _Enc, _Str, _Ord) when I > tuple_size(Tuple) -> []; -print_length_tuple1(Tuple, I, D, T, RF, Enc, Str) when D =:= 1; T =:= 0-> - More = fun(T1, Dd) -> ?FUNCTION_NAME(Tuple, I, D+Dd, T1, RF, Enc, Str) end, +print_length_tuple1(Tuple, I, D, T, RF, Enc, Str, Ord) when D =:= 1; T =:= 0-> + More = fun(T1, Dd) -> ?FUNCTION_NAME(Tuple, I, D+Dd, T1, RF, Enc, Str, Ord) end, {dots, 3, 3, More}; -print_length_tuple1(Tuple, I, D, T, RF, Enc, Str) -> +print_length_tuple1(Tuple, I, D, T, RF, Enc, Str, Ord) -> E = element(I, Tuple), T1 = case I =:= tuple_size(Tuple) of false -> tsub(T, 1); true -> T end, - {_, Len1, _, _} = Elem1 = print_length(E, D - 1, T1, RF, Enc, Str), + {_, Len1, _, _} = Elem1 = print_length(E, D - 1, T1, RF, Enc, Str, Ord), T2 = tsub(T1, Len1), - [Elem1 | print_length_tuple1(Tuple, I + 1, D - 1, T2, RF, Enc, Str)]. + [Elem1 | print_length_tuple1(Tuple, I + 1, D - 1, T2, RF, Enc, Str, Ord)]. -print_length_record(Tuple, 1, _T, RF, RDefs, Enc, Str) -> +print_length_record(Tuple, 1, _T, RF, RDefs, Enc, Str, Ord) -> More = fun(T1, Dd) -> - ?FUNCTION_NAME(Tuple, 1+Dd, T1, RF, RDefs, Enc, Str) + ?FUNCTION_NAME(Tuple, 1+Dd, T1, RF, RDefs, Enc, Str, Ord) end, {"{...}", 5, 3, More}; -print_length_record(Tuple, D, T, RF, RDefs, Enc, Str) -> +print_length_record(Tuple, D, T, RF, RDefs, Enc, Str, Ord) -> Name = [$# | write_atom(element(1, Tuple), Enc)], NameL = io_lib:chars_length(Name), T1 = tsub(T, NameL+2), - L = print_length_fields(RDefs, D - 1, T1, Tuple, 2, RF, Enc, Str), + L = print_length_fields(RDefs, D - 1, T1, Tuple, 2, RF, Enc, Str, Ord), {Len, Dots} = list_length(L, NameL + 2, 0), {{record, [{Name,NameL} | L]}, Len, Dots, no_more}. -print_length_fields([], _D, _T, Tuple, I, _RF, _Enc, _Str) +print_length_fields([], _D, _T, Tuple, I, _RF, _Enc, _Str, _Ord) when I > tuple_size(Tuple) -> []; -print_length_fields(Term, D, T, Tuple, I, RF, Enc, Str) +print_length_fields(Term, D, T, Tuple, I, RF, Enc, Str, Ord) when D =:= 1; T =:= 0 -> More = fun(T1, Dd) -> - ?FUNCTION_NAME(Term, D+Dd, T1, Tuple, I, RF, Enc, Str) + ?FUNCTION_NAME(Term, D+Dd, T1, Tuple, I, RF, Enc, Str, Ord) end, {dots, 3, 3, More}; -print_length_fields([Def | Defs], D, T, Tuple, I, RF, Enc, Str) -> +print_length_fields([Def | Defs], D, T, Tuple, I, RF, Enc, Str, Ord) -> E = element(I, Tuple), T1 = case I =:= tuple_size(Tuple) of false -> tsub(T, 1); true -> T end, - Field1 = print_length_field(Def, D - 1, T1, E, RF, Enc, Str), + Field1 = print_length_field(Def, D - 1, T1, E, RF, Enc, Str, Ord), {_, Len1, _, _} = Field1, T2 = tsub(T1, Len1), [Field1 | - print_length_fields(Defs, D - 1, T2, Tuple, I + 1, RF, Enc, Str)]. + print_length_fields(Defs, D - 1, T2, Tuple, I + 1, RF, Enc, Str, Ord)]. -print_length_field(Def, D, T, E, RF, Enc, Str) -> +print_length_field(Def, D, T, E, RF, Enc, Str, Ord) -> Name = write_atom(Def, Enc), NameL = io_lib:chars_length(Name) + 3, {_, Len, Dots, _} = - Field = print_length(E, D, tsub(T, NameL), RF, Enc, Str), + Field = print_length(E, D, tsub(T, NameL), RF, Enc, Str, Ord), {{field, Name, NameL, Field}, NameL + Len, Dots, no_more}. -print_length_list(List, D, T, RF, Enc, Str) -> - L = print_length_list1(List, D, tsub(T, 2), RF, Enc, Str), +print_length_list(List, D, T, RF, Enc, Str, Ord) -> + L = print_length_list1(List, D, tsub(T, 2), RF, Enc, Str, Ord), {Len, Dots} = list_length(L, 2, 0), {{list, L}, Len, Dots, no_more}. -print_length_list1([], _D, _T, _RF, _Enc, _Str) -> +print_length_list1([], _D, _T, _RF, _Enc, _Str, _Ord) -> []; -print_length_list1(Term, D, T, RF, Enc, Str) when D =:= 1; T =:= 0-> - More = fun(T1, Dd) -> ?FUNCTION_NAME(Term, D+Dd, T1, RF, Enc, Str) end, +print_length_list1(Term, D, T, RF, Enc, Str, Ord) when D =:= 1; T =:= 0-> + More = fun(T1, Dd) -> ?FUNCTION_NAME(Term, D+Dd, T1, RF, Enc, Str, Ord) end, {dots, 3, 3, More}; -print_length_list1([E | Es], D, T, RF, Enc, Str) -> +print_length_list1([E | Es], D, T, RF, Enc, Str, Ord) -> %% If E is the last element in list, don't account length for a comma. T1 = case Es =:= [] of false -> tsub(T, 1); true -> T end, - {_, Len1, _, _} = Elem1 = print_length(E, D - 1, T1, RF, Enc, Str), - [Elem1 | print_length_list1(Es, D - 1, tsub(T1, Len1), RF, Enc, Str)]; -print_length_list1(E, D, T, RF, Enc, Str) -> - print_length(E, D - 1, T, RF, Enc, Str). + {_, Len1, _, _} = Elem1 = print_length(E, D - 1, T1, RF, Enc, Str, Ord), + [Elem1 | print_length_list1(Es, D - 1, tsub(T1, Len1), RF, Enc, Str, Ord)]; +print_length_list1(E, D, T, RF, Enc, Str, Ord) -> + print_length(E, D - 1, T, RF, Enc, Str, Ord). list_length([], Acc, DotsAcc) -> {Acc, DotsAcc}; diff --git a/lib/stdlib/src/lists.erl b/lib/stdlib/src/lists.erl index d2cb5aab3c..cb1c008ed1 100644 --- a/lib/stdlib/src/lists.erl +++ b/lib/stdlib/src/lists.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2022. All Rights Reserved. +%% Copyright Ericsson AB 1996-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -29,7 +29,7 @@ %% arguments. Please keep in alphabetical order. -export([append/1, append/2, concat/1, delete/2, droplast/1, duplicate/2, - enumerate/1, enumerate/2, + enumerate/1, enumerate/2, enumerate/3, flatlength/1, flatten/1, flatten/2, join/2, last/1, min/1, max/1, nth/2, nthtail/2, @@ -37,7 +37,7 @@ split/2, sublist/2, sublist/3, subtract/2, suffix/2, sum/1, uniq/1, unzip/1, unzip3/1, - zip/2, zip3/3]). + zip/2, zip/3, zip3/3, zip3/4]). %% Functions taking a list of tuples and a position within the tuple. -export([keydelete/3, keyreplace/4, keymap/3, @@ -60,7 +60,7 @@ map/2, mapfoldl/3, mapfoldr/3, partition/2, search/2, splitwith/2, takewhile/2, uniq/2, - zipwith/3, zipwith3/4]). + zipwith/3, zipwith/4, zipwith3/4, zipwith3/5]). %% Undocumented, but used within Erlang/OTP. -export([zf/2]). @@ -196,8 +196,12 @@ reverse([A, B | L]) -> T :: term(). nth(1, [H|_]) -> H; -nth(N, [_|T]) when N > 1 -> - nth(N - 1, T). +nth(N, [_|_]=L) when is_integer(N), N > 1 -> + nth_1(N, L). + +nth_1(1, [H|_]) -> H; +nth_1(N, [_|T]) -> + nth_1(N - 1, T). -spec nthtail(N, List) -> Tail when N :: non_neg_integer(), @@ -205,10 +209,15 @@ nth(N, [_|T]) when N > 1 -> Tail :: [T], T :: term(). +nthtail(0, []) -> []; +nthtail(0, [_|_]=L) -> L; nthtail(1, [_|T]) -> T; -nthtail(N, [_|T]) when N > 1 -> - nthtail(N - 1, T); -nthtail(0, L) when is_list(L) -> L. +nthtail(N, [_|_]=L) when is_integer(N), N > 1 -> + nthtail_1(N, L). + +nthtail_1(1, [_|T]) -> T; +nthtail_1(N, [_|T]) -> + nthtail_1(N - 1, T). %% prefix(Prefix, List) -> (true | false) @@ -416,8 +425,35 @@ delete(_, []) -> []. A :: term(), B :: term(). -zip([X | Xs], [Y | Ys]) -> [{X, Y} | zip(Xs, Ys)]; -zip([], []) -> []. +zip(Xs, Ys) -> zip(Xs, Ys, fail). + +-spec zip(List1, List2, How) -> List3 when + List1 :: [A], + List2 :: [B], + List3 :: [{A | DefaultA, B | DefaultB}], + A :: term(), + B :: term(), + How :: 'fail' | 'trim' | {'pad', {DefaultA, DefaultB}}, + DefaultA :: term(), + DefaultB :: term(). + +zip([X | Xs], [Y | Ys], How) -> + [{X, Y} | zip(Xs, Ys, How)]; +zip([], [], fail) -> + []; +zip([], [], trim) -> + []; +zip([], [], {pad, {_, _}}) -> + []; +zip([_ | _], [], trim) -> + []; +zip([], [_ | _], trim) -> + []; +zip([], [_ | _]=Ys, {pad, {X, _}}) -> + [{X, Y} || Y <- Ys]; +zip([_ | _]=Xs, [], {pad, {_, Y}}) -> + [{X, Y} || X <- Xs]. + %% Return {[X0, X1, ..., Xn], [Y0, Y1, ..., Yn]}, for a list [{X0, Y0}, %% {X1, Y1}, ..., {Xn, Yn}]. @@ -446,8 +482,43 @@ unzip([], Xs, Ys) -> {reverse(Xs), reverse(Ys)}. B :: term(), C :: term(). -zip3([X | Xs], [Y | Ys], [Z | Zs]) -> [{X, Y, Z} | zip3(Xs, Ys, Zs)]; -zip3([], [], []) -> []. +zip3(Xs, Ys, Zs) -> zip3(Xs, Ys, Zs, fail). + +-spec zip3(List1, List2, List3, How) -> List4 when + List1 :: [A], + List2 :: [B], + List3 :: [C], + List4 :: [{A | DefaultA, B | DefaultB, C | DefaultC}], + A :: term(), + B :: term(), + C :: term(), + How :: 'fail' | 'trim' | {'pad', {DefaultA, DefaultB, DefaultC}}, + DefaultA :: term(), + DefaultB :: term(), + DefaultC :: term(). + +zip3([X | Xs], [Y | Ys], [Z | Zs], How) -> + [{X, Y, Z} | zip3(Xs, Ys, Zs, How)]; +zip3([], [], [], fail) -> + []; +zip3([], [], [], trim) -> + []; +zip3(Xs, Ys, Zs, trim) when is_list(Xs), is_list(Ys), is_list(Zs) -> + []; +zip3([], [], [], {pad, {_, _, _}}) -> + []; +zip3([], [], [_ |_]=Zs, {pad, {X, Y, _}}) -> + [{X, Y, Z} || Z <- Zs]; +zip3([], [_ | _]=Ys, [], {pad, {X, _, Z}}) -> + [{X, Y, Z} || Y <- Ys]; +zip3([_ | _]=Xs, [], [], {pad, {_, Y, Z}}) -> + [{X, Y, Z} || X <- Xs]; +zip3([], [Y | Ys], [Z | Zs], {pad, {X, _, _}} = How) -> + [{X, Y, Z} | zip3([], Ys, Zs, How)]; +zip3([X | Xs], [], [Z | Zs], {pad, {_, Y, _}} = How) -> + [{X, Y, Z} | zip3(Xs, [], Zs, How)]; +zip3([X | Xs], [Y | Ys], [], {pad, {_, _, Z}} = How) -> + [{X, Y, Z} | zip3(Xs, Ys, [], How)]. %% Return {[X0, X1, ..., Xn], [Y0, Y1, ..., Yn], [Z0, Z1, ..., Zn]}, for %% a list [{X0, Y0, Z0}, {X1, Y1, Z1}, ..., {Xn, Yn, Zn}]. @@ -480,8 +551,36 @@ unzip3([], Xs, Ys, Zs) -> Y :: term(), T :: term(). -zipwith(F, [X | Xs], [Y | Ys]) -> [F(X, Y) | zipwith(F, Xs, Ys)]; -zipwith(F, [], []) when is_function(F, 2) -> []. +zipwith(F, Xs, Ys) -> zipwith(F, Xs, Ys, fail). + +-spec zipwith(Combine, List1, List2, How) -> List3 when + Combine :: fun((X | DefaultX, Y | DefaultY) -> T), + List1 :: [X], + List2 :: [Y], + List3 :: [T], + X :: term(), + Y :: term(), + How :: 'fail' | 'trim' | {'pad', {DefaultX, DefaultY}}, + DefaultX :: term(), + DefaultY :: term(), + T :: term(). + +zipwith(F, [X | Xs], [Y | Ys], How) -> + [F(X, Y) | zipwith(F, Xs, Ys, How)]; +zipwith(F, [], [], fail) when is_function(F, 2) -> + []; +zipwith(F, [], [], trim) when is_function(F, 2) -> + []; +zipwith(F, [], [], {pad, {_, _}}) when is_function(F, 2) -> + []; +zipwith(F, [_ | _], [], trim) when is_function(F, 2) -> + []; +zipwith(F, [], [_ | _], trim) when is_function(F, 2) -> + []; +zipwith(F, [], [_ | _]=Ys, {pad, {X, _}}) -> + [F(X, Y) || Y <- Ys]; +zipwith(F, [_ | _]=Xs, [], {pad, {_, Y}}) -> + [F(X, Y) || X <- Xs]. %% Return [F(X0, Y0, Z0), F(X1, Y1, Z1), ..., F(Xn, Yn, Zn)] for lists %% [X0, X1, ..., Xn], [Y0, Y1, ..., Yn] and [Z0, Z1, ..., Zn]. @@ -497,9 +596,45 @@ zipwith(F, [], []) when is_function(F, 2) -> []. Z :: term(), T :: term(). -zipwith3(F, [X | Xs], [Y | Ys], [Z | Zs]) -> - [F(X, Y, Z) | zipwith3(F, Xs, Ys, Zs)]; -zipwith3(F, [], [], []) when is_function(F, 3) -> []. +zipwith3(F, Xs, Ys, Zs) -> zipwith3(F, Xs, Ys, Zs, fail). + +-spec zipwith3(Combine, List1, List2, List3, How) -> List4 when + Combine :: fun((X | DefaultX, Y | DefaultY, Z | DefaultZ) -> T), + List1 :: [X], + List2 :: [Y], + List3 :: [Z], + List4 :: [T], + X :: term(), + Y :: term(), + Z :: term(), + How :: 'fail' | 'trim' | {'pad', {DefaultX, DefaultY, DefaultZ}}, + DefaultX :: term(), + DefaultY :: term(), + DefaultZ :: term(), + T :: term(). + +zipwith3(F, [X | Xs], [Y | Ys], [Z | Zs], How) -> + [F(X, Y, Z) | zipwith3(F, Xs, Ys, Zs, How)]; +zipwith3(F, [], [], [], fail) when is_function(F, 3) -> + []; +zipwith3(F, [], [], [], trim) when is_function(F, 3) -> + []; +zipwith3(F, Xs, Ys, Zs, trim) when is_function(F, 3), is_list(Xs), is_list(Ys), is_list(Zs) -> + []; +zipwith3(F, [], [], [], {pad, {_, _, _}}) when is_function(F, 3) -> + []; +zipwith3(F, [], [], [_ | _]=Zs, {pad, {X, Y, _}}) -> + [F(X, Y, Z) || Z <- Zs]; +zipwith3(F, [], [_ | _]=Ys, [], {pad, {X, _, Z}}) -> + [F(X, Y, Z) || Y <- Ys]; +zipwith3(F, [_ | _]=Xs, [], [], {pad, {_, Y, Z}}) -> + [F(X, Y, Z) || X <- Xs]; +zipwith3(F, [], [Y | Ys], [Z | Zs], {pad, {X, _, _}} = How) -> + [F(X, Y, Z) | zipwith3(F, [], Ys, Zs, How)]; +zipwith3(F, [X | Xs], [], [Z | Zs], {pad, {_, Y, _}} = How) -> + [F(X, Y, Z) | zipwith3(F, Xs, [], Zs, How)]; +zipwith3(F, [X | Xs], [Y | Ys], [], {pad, {_, _, Z}} = How) -> + [F(X, Y, Z) | zipwith3(F, Xs, Ys, [], How)]. %% sort(List) -> L %% sorts the list L @@ -575,24 +710,44 @@ merge(L) -> Y :: term(), Z :: term(). -merge3(L1, [], L3) -> - merge(L1, L3); -merge3(L1, L2, []) -> - merge(L1, L2); -merge3(L1, [H2 | T2], [H3 | T3]) -> - lists:reverse(merge3_1(L1, [], H2, T2, H3, T3), []). +merge3([_|_]=L1, [H2 | T2], [H3 | T3]) -> + lists:reverse(merge3_1(L1, [], H2, T2, H3, T3), []); +merge3([_|_]=L1, [_|_]=L2, []) -> + merge(L1, L2); +merge3([_|_]=L1, [], [_|_]=L3) -> + merge(L1, L3); +merge3([_|_]=L1, [], []) -> + L1; +merge3([], [_|_]=L2, [_|_]=L3) -> + merge(L2, L3); +merge3([], [_|_]=L2, []) -> + L2; +merge3([], [], [_|_]=L3) -> + L3; +merge3([], [], []) -> + []. %% rmerge3(X, Y, Z) -> L %% merges three reversed sorted lists X, Y and Z -spec rmerge3([X], [Y], [Z]) -> [(X | Y | Z)]. -rmerge3(L1, [], L3) -> - rmerge(L1, L3); -rmerge3(L1, L2, []) -> - rmerge(L1, L2); -rmerge3(L1, [H2 | T2], [H3 | T3]) -> - lists:reverse(rmerge3_1(L1, [], H2, T2, H3, T3), []). +rmerge3([_|_]=L1, [H2 | T2], [H3 | T3]) -> + lists:reverse(rmerge3_1(L1, [], H2, T2, H3, T3), []); +rmerge3([_|_]=L1, [_|_]=L2, []) -> + rmerge(L1, L2); +rmerge3([_|_]=L1, [], [_|_]=L3) -> + rmerge(L1, L3); +rmerge3([_|_]=L1, [], []) -> + L1; +rmerge3([], [_|_]=L2, [_|_]=L3) -> + rmerge(L2, L3); +rmerge3([], [_|_]=L2, []) -> + L2; +rmerge3([], [], [_|_]=L3) -> + L3; +rmerge3([], [], []) -> + []. %% merge(X, Y) -> L %% merges two sorted lists X and Y @@ -604,10 +759,14 @@ rmerge3(L1, [H2 | T2], [H3 | T3]) -> X :: term(), Y :: term(). -merge(T1, []) -> - T1; -merge(T1, [H2 | T2]) -> - lists:reverse(merge2_1(T1, H2, T2, []), []). +merge([_|_]=T1, [H2 | T2]) -> + lists:reverse(merge2_1(T1, H2, T2, []), []); +merge([_|_]=L1, []) -> + L1; +merge([], [_|_]=L2) -> + L2; +merge([], []) -> + []. %% rmerge(X, Y) -> L %% merges two reversed sorted lists X and Y @@ -616,10 +775,14 @@ merge(T1, [H2 | T2]) -> -spec rmerge([X], [Y]) -> [(X | Y)]. -rmerge(T1, []) -> - T1; -rmerge(T1, [H2 | T2]) -> - lists:reverse(rmerge2_1(T1, H2, T2, []), []). +rmerge([_|_]=T1, [H2 | T2]) -> + lists:reverse(rmerge2_1(T1, H2, T2, []), []); +rmerge([_|_]=L1, []) -> + L1; +rmerge([], [_|_]=L2) -> + L2; +rmerge([], []) -> + []. %% concat(L) concatenate the list representation of the elements %% in L - the elements in L can be atoms, numbers of strings. @@ -845,30 +1008,38 @@ keysort_1(_I, X, _EX, [], R) -> T2 :: Tuple, Tuple :: tuple(). -keymerge(Index, T1, L2) when is_integer(Index), Index > 0 -> - case L2 of - [] -> - T1; - [H2 | T2] -> - E2 = element(Index, H2), - M = keymerge2_1(Index, T1, E2, H2, T2, []), - lists:reverse(M, []) - end. +keymerge(Index, L1, L2) when is_integer(Index), Index > 0 -> + keymerge_1(Index, L1, L2). + +keymerge_1(Index, [_|_]=T1, [H2 | T2]) -> + E2 = element(Index, H2), + M = keymerge2_1(Index, T1, E2, H2, T2, []), + lists:reverse(M, []); +keymerge_1(_Index, [_|_]=L1, []) -> + L1; +keymerge_1(_Index, [], [_|_]=L2) -> + L2; +keymerge_1(_Index, [], []) -> + []. %% reverse(rkeymerge(I,reverse(A),reverse(B))) is equal to keymerge(I,A,B). -spec rkeymerge(pos_integer(), [X], [Y]) -> [R] when X :: tuple(), Y :: tuple(), R :: tuple(). -rkeymerge(Index, T1, L2) when is_integer(Index), Index > 0 -> - case L2 of - [] -> - T1; - [H2 | T2] -> - E2 = element(Index, H2), - M = rkeymerge2_1(Index, T1, E2, H2, T2, []), - lists:reverse(M, []) - end. +rkeymerge(Index, L1, L2) when is_integer(Index), Index > 0 -> + rkeymerge_1(Index, L1, L2). + +rkeymerge_1(Index, [_|_]=T1, [H2 | T2]) -> + E2 = element(Index, H2), + M = rkeymerge2_1(Index, T1, E2, H2, T2, []), + lists:reverse(M, []); +rkeymerge_1(_Index, [_|_]=L1, []) -> + L1; +rkeymerge_1(_Index, [], [_|_]=L2) -> + L2; +rkeymerge_1(_Index, [], []) -> + []. -spec ukeysort(N, TupleList1) -> TupleList2 when N :: pos_integer(), @@ -948,30 +1119,38 @@ ukeysort_1(_I, X, _EX, []) -> T2 :: Tuple, Tuple :: tuple(). -ukeymerge(Index, L1, T2) when is_integer(Index), Index > 0 -> - case L1 of - [] -> - T2; - [H1 | T1] -> - E1 = element(Index, H1), - M = ukeymerge2_2(Index, T1, E1, H1, T2, []), - lists:reverse(M, []) - end. +ukeymerge(Index, L1, L2) when is_integer(Index), Index > 0 -> + ukeymerge_1(Index, L1, L2). + +ukeymerge_1(Index, [H1 | T1], [_|_]=T2) -> + E1 = element(Index, H1), + M = ukeymerge2_2(Index, T1, E1, H1, T2, []), + lists:reverse(M, []); +ukeymerge_1(_Index, [_|_]=L1, []) -> + L1; +ukeymerge_1(_Index, [], [_|_]=L2) -> + L2; +ukeymerge_1(_Index, [], []) -> + []. %% reverse(rukeymerge(I,reverse(A),reverse(B))) is equal to ukeymerge(I,A,B). -spec rukeymerge(pos_integer(), [X], [Y]) -> [(X | Y)] when X :: tuple(), Y :: tuple(). -rukeymerge(Index, T1, L2) when is_integer(Index), Index > 0 -> - case L2 of - [] -> - T1; - [H2 | T2] -> - E2 = element(Index, H2), - M = rukeymerge2_1(Index, T1, E2, T2, [], H2), - lists:reverse(M, []) - end. +rukeymerge(Index, L1, L2) when is_integer(Index), Index > 0 -> + rukeymerge_1(Index, L1, L2). + +rukeymerge_1(Index, [_|_]=T1, [H2 | T2]) -> + E2 = element(Index, H2), + M = rukeymerge2_1(Index, T1, E2, T2, [], H2), + lists:reverse(M, []); +rukeymerge_1(_Index, [_|_]=L1, []) -> + L1; +rukeymerge_1(_Index, [], [_|_]=L2) -> + L2; +rukeymerge_1(_Index, [], []) -> + []. -spec keymap(Fun, N, TupleList1) -> TupleList2 when Fun :: fun((Term1 :: term()) -> Term2 :: term()), @@ -990,20 +1169,29 @@ keymap(Fun, Index, []) when is_integer(Index), Index >= 1, List2 :: [{Index, T}], Index :: integer(), T :: term(). -enumerate(List1) when is_list(List1) -> - enumerate_1(1, List1). +enumerate(List1) -> + enumerate(1, 1, List1). -spec enumerate(Index, List1) -> List2 when List1 :: [T], List2 :: [{Index, T}], Index :: integer(), T :: term(). -enumerate(Index, List1) when is_integer(Index), is_list(List1) -> - enumerate_1(Index, List1). +enumerate(Index, List1) -> + enumerate(Index, 1, List1). -enumerate_1(Index, [H|T]) -> - [{Index, H}|enumerate_1(Index + 1, T)]; -enumerate_1(_Index, []) -> +-spec enumerate(Index, Step, List1) -> List2 when + List1 :: [T], + List2 :: [{Index, T}], + Index :: integer(), + Step :: integer(), + T :: term(). +enumerate(Index, Step, List1) when is_integer(Index), is_integer(Step) -> + enumerate_1(Index, Step, List1). + +enumerate_1(Index, Step, [H|T]) -> + [{Index, H}|enumerate_1(Index + Step, Step, T)]; +enumerate_1(_Index, _Step, []) -> []. %%% Suggestion from OTP-2948: sort and merge with Fun. @@ -1034,19 +1222,33 @@ sort(Fun, [X, Y | T]) -> A :: term(), B :: term(). -merge(Fun, T1, [H2 | T2]) when is_function(Fun, 2) -> +merge(Fun, L1, L2) when is_function(Fun, 2) -> + merge_1(Fun, L1, L2). + +merge_1(Fun, [_|_]=T1, [H2 | T2]) -> lists:reverse(fmerge2_1(T1, H2, Fun, T2, []), []); -merge(Fun, T1, []) when is_function(Fun, 2) -> - T1. +merge_1(_Fun, [_|_]=L1, []) -> + L1; +merge_1(_Fun, [], [_|_]=L2) -> + L2; +merge_1(_Fun, [], []) -> + []. %% reverse(rmerge(F,reverse(A),reverse(B))) is equal to merge(F,A,B). -spec rmerge(fun((X, Y) -> boolean()), [X], [Y]) -> [(X | Y)]. -rmerge(Fun, T1, [H2 | T2]) when is_function(Fun, 2) -> +rmerge(Fun, L1, L2) when is_function(Fun, 2) -> + rmerge_1(Fun, L1, L2). + +rmerge_1(Fun, [_|_]=T1, [H2 | T2]) -> lists:reverse(rfmerge2_1(T1, H2, Fun, T2, []), []); -rmerge(Fun, T1, []) when is_function(Fun, 2) -> - T1. +rmerge_1(_Fun, [_|_]=L1, []) -> + L1; +rmerge_1(_Fun, [], [_|_]=L2) -> + L2; +rmerge_1(_Fun, [], []) -> + []. -spec usort(Fun, List1) -> List2 when Fun :: fun((T, T) -> boolean()), @@ -1087,19 +1289,33 @@ usort_1(Fun, X, [Y | L]) -> A :: term(), B :: term(). -umerge(Fun, [], T2) when is_function(Fun, 2) -> - T2; -umerge(Fun, [H1 | T1], T2) when is_function(Fun, 2) -> - lists:reverse(ufmerge2_2(H1, T1, Fun, T2, []), []). +umerge(Fun, L1, L2) when is_function(Fun, 2) -> + umerge_1(Fun, L1, L2). + +umerge_1(Fun, [H1 | T1], [_|_]=T2) -> + lists:reverse(ufmerge2_2(H1, T1, Fun, T2, []), []); +umerge_1(_Fun, [_|_]=L1, []) -> + L1; +umerge_1(_Fun, [], [_|_]=L2) -> + L2; +umerge_1(_Fun, [], []) -> + []. %% reverse(rumerge(F,reverse(A),reverse(B))) is equal to umerge(F,A,B). -spec rumerge(fun((X, Y) -> boolean()), [X], [Y]) -> [(X | Y)]. -rumerge(Fun, T1, []) when is_function(Fun, 2) -> - T1; -rumerge(Fun, T1, [H2 | T2]) when is_function(Fun, 2) -> - lists:reverse(rufmerge2_1(T1, H2, Fun, T2, []), []). +rumerge(Fun, L1, L2) when is_function(Fun, 2) -> + rumerge_1(Fun, L1, L2). + +rumerge_1(Fun, [_|_]=T1, [H2 | T2]) -> + lists:reverse(rufmerge2_1(T1, H2, Fun, T2, []), []); +rumerge_1(_Fun, [_|_]=L1, []) -> + L1; +rumerge_1(_Fun, [], [_|_]=L2) -> + L2; +rumerge_1(_Fun, [], []) -> + []. %% usort(List) -> L %% sorts the list L, removes duplicates @@ -1184,12 +1400,22 @@ umerge(L) -> Y :: term(), Z :: term(). -umerge3(L1, [], L3) -> - umerge(L1, L3); -umerge3(L1, L2, []) -> - umerge(L1, L2); -umerge3(L1, [H2 | T2], [H3 | T3]) -> - lists:reverse(umerge3_1(L1, [H2 | H3], T2, H2, [], T3, H3), []). +umerge3([_|_]=L1, [H2 | T2], [H3 | T3]) -> + lists:reverse(umerge3_1(L1, [H2 | H3], T2, H2, [], T3, H3), []); +umerge3([_|_]=L1, [_|_]=L2, []) -> + umerge(L1, L2); +umerge3([_|_]=L1, [], [_|_]=L3) -> + umerge(L1, L3); +umerge3([_|_]=L1, [], []) -> + L1; +umerge3([], [_|_]=L2, [_|_]=L3) -> + umerge(L2, L3); +umerge3([], [_|_]=L2, []) -> + L2; +umerge3([], [], [_|_]=L3) -> + L3; +umerge3([], [], []) -> + []. %% rumerge3(X, Y, Z) -> L %% merges three reversed sorted lists X, Y and Z without duplicates, @@ -1197,12 +1423,22 @@ umerge3(L1, [H2 | T2], [H3 | T3]) -> -spec rumerge3([X], [Y], [Z]) -> [(X | Y | Z)]. -rumerge3(L1, [], L3) -> - rumerge(L1, L3); -rumerge3(L1, L2, []) -> - rumerge(L1, L2); -rumerge3(L1, [H2 | T2], [H3 | T3]) -> - lists:reverse(rumerge3_1(L1, T2, H2, [], T3, H3),[]). +rumerge3([_|_]=L1, [H2 | T2], [H3 | T3]) -> + lists:reverse(rumerge3_1(L1, T2, H2, [], T3, H3),[]); +rumerge3([_|_]=L1, [_|_]=L2, []) -> + rumerge(L1, L2); +rumerge3([_|_]=L1, [], [_|_]=L3) -> + rumerge(L1, L3); +rumerge3([_|_]=L1, [], []) -> + L1; +rumerge3([], [_|_]=L2, [_|_]=L3) -> + rumerge(L2, L3); +rumerge3([], [_|_]=L2, []) -> + L2; +rumerge3([], [], [_|_]=L3) -> + L3; +rumerge3([], [], []) -> + []. %% umerge(X, Y) -> L %% merges two sorted lists X and Y without duplicates, removes duplicates @@ -1214,10 +1450,14 @@ rumerge3(L1, [H2 | T2], [H3 | T3]) -> X :: term(), Y :: term(). -umerge([], T2) -> - T2; -umerge([H1 | T1], T2) -> - lists:reverse(umerge2_2(T1, T2, [], H1), []). +umerge([H1 | T1], [_|_]=T2) -> + lists:reverse(umerge2_2(T1, T2, [], H1), []); +umerge([_|_]=L1, []) -> + L1; +umerge([], [_|_]=L2) -> + L2; +umerge([], []) -> + []. %% rumerge(X, Y) -> L %% merges two reversed sorted lists X and Y without duplicates, @@ -1227,10 +1467,14 @@ umerge([H1 | T1], T2) -> -spec rumerge([X], [Y]) -> [(X | Y)]. -rumerge(T1, []) -> - T1; -rumerge(T1, [H2 | T2]) -> - lists:reverse(rumerge2_1(T1, T2, [], H2), []). +rumerge([_|_]=T1, [H2 | T2]) -> + lists:reverse(rumerge2_1(T1, T2, [], H2), []); +rumerge([_|_]=L1, []) -> + L1; +rumerge([], [_|_]=L2) -> + L2; +rumerge([], []) -> + []. %% all(Predicate, List) %% any(Predicate, List) @@ -1663,24 +1907,24 @@ split_2_1(X, Y, [], R, Rs, S) -> %% merge/1 -mergel([[] | L], Acc) -> - mergel(L, Acc); -mergel([T1, [H2 | T2], [H3 | T3] | L], Acc) -> - mergel(L, [merge3_1(T1, [], H2, T2, H3, T3) | Acc]); -mergel([T1, [H2 | T2]], Acc) -> - rmergel([merge2_1(T1, H2, T2, []) | Acc], []); -mergel([L], []) -> - L; -mergel([L], Acc) -> - rmergel([lists:reverse(L, []) | Acc], []); mergel([], []) -> []; +mergel([[_|_]=L], []) -> + L; mergel([], Acc) -> rmergel(Acc, []); -mergel([A, [] | L], Acc) -> +mergel([[] | L], Acc) -> + mergel(L, Acc); +mergel([[_|_]=L], Acc) -> + rmergel([lists:reverse(L, []) | Acc], []); +mergel([[_|_]=A, [] | L], Acc) -> mergel([A | L], Acc); -mergel([A, B, [] | L], Acc) -> - mergel([A, B | L], Acc). +mergel([[_|_]=A, [_|_]=B, [] | L], Acc) -> + mergel([A, B | L], Acc); +mergel([[_|_]=T1, [H2 | T2], [H3 | T3] | L], Acc) -> + mergel(L, [merge3_1(T1, [], H2, T2, H3, T3) | Acc]); +mergel([[_|_]=T1, [H2 | T2]], Acc) -> + rmergel([merge2_1(T1, H2, T2, []) | Acc], []). rmergel([[H3 | T3], [H2 | T2], T1 | L], Acc) -> rmergel(L, [rmerge3_1(T1, [], H2, T2, H3, T3) | Acc]); @@ -1896,28 +2140,28 @@ usplit_2_1(X, Y, [], R, Rs, S) -> umergel(L) -> umergel(L, [], asc). +umergel([], [], _O) -> + []; +umergel([[_|_]=L], [], _O) -> + L; +umergel([], Acc, O) -> + rumergel(Acc, [], O); +umergel([[_|_]=L], Acc, O) -> + rumergel([lists:reverse(L, []) | Acc], [], O); umergel([[] | L], Acc, O) -> umergel(L, Acc, O); -umergel([T1, [H2 | T2], [H3 | T3] | L], Acc, asc) -> - umergel(L, [umerge3_1(T1, [H2 | H3], T2, H2, [], T3, H3) | Acc], asc); -umergel([[H3 | T3], [H2 | T2], T1 | L], Acc, desc) -> - umergel(L, [umerge3_1(T1, [H2 | H3], T2, H2, [], T3, H3) | Acc], desc); -umergel([A, [] | L], Acc, O) -> +umergel([[_|_]=A, [] | L], Acc, O) -> umergel([A | L], Acc, O); -umergel([A, B, [] | L], Acc, O) -> +umergel([[_|_]=A, [_|_]=B, [] | L], Acc, O) -> umergel([A, B | L], Acc, O); -umergel([[H1 | T1], T2 | L], Acc, asc) -> +umergel([[_|_]=T1, [H2 | T2], [H3 | T3] | L], Acc, asc) -> + umergel(L, [umerge3_1(T1, [H2 | H3], T2, H2, [], T3, H3) | Acc], asc); +umergel([[H3 | T3], [H2 | T2], [_|_]=T1 | L], Acc, desc) -> + umergel(L, [umerge3_1(T1, [H2 | H3], T2, H2, [], T3, H3) | Acc], desc); +umergel([[H1 | T1], [_|_]=T2 | L], Acc, asc) -> umergel(L, [umerge2_2(T1, T2, [], H1) | Acc], asc); -umergel([T2, [H1 | T1] | L], Acc, desc) -> - umergel(L, [umerge2_2(T1, T2, [], H1) | Acc], desc); -umergel([L], [], _O) -> - L; -umergel([L], Acc, O) -> - rumergel([lists:reverse(L, []) | Acc], [], O); -umergel([], [], _O) -> - []; -umergel([], Acc, O) -> - rumergel(Acc, [], O). +umergel([[_|_]=T2, [H1 | T1] | L], Acc, desc) -> + umergel(L, [umerge2_2(T1, T2, [], H1) | Acc], desc). rumergel([[H3 | T3], [H2 | T2], T1 | L], Acc, asc) -> rumergel(L, [rumerge3_1(T1, T2, H2, [], T3, H3) | Acc], asc); diff --git a/lib/stdlib/src/maps.erl b/lib/stdlib/src/maps.erl index c7107031fb..a7689f10b3 100644 --- a/lib/stdlib/src/maps.erl +++ b/lib/stdlib/src/maps.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2013-2022. All Rights Reserved. +%% Copyright Ericsson AB 2013-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -24,11 +24,15 @@ map/2, size/1, new/0, update_with/3, update_with/4, without/2, with/2, - iterator/1, next/1, + iterator/1, iterator/2, + next/1, intersect/2, intersect_with/3, merge_with/3, groups_from_list/2, groups_from_list/3]). +%% Internal +-export([is_iterator_valid/1]). + %% BIFs -export([get/2, find/2, from_list/1, from_keys/2, is_key/2, keys/1, merge/2, @@ -36,13 +40,18 @@ to_list/1, update/3, values/1]). -opaque iterator(Key, Value) :: {Key, Value, iterator(Key, Value)} | none - | nonempty_improper_list(integer(), #{Key => Value}). + | nonempty_improper_list(integer(), #{Key => Value}) + | nonempty_improper_list(list(Key), #{Key => Value}). -type iterator() :: iterator(term(), term()). --export_type([iterator/2, iterator/0]). +-type iterator_order(Key) :: undefined | ordered | reversed + | fun((A :: Key, B :: Key) -> boolean()). +-type iterator_order() :: iterator_order(term()). + +-export_type([iterator/2, iterator/0, iterator_order/1, iterator_order/0]). --dialyzer({no_improper_lists, iterator/1}). +-dialyzer({no_improper_lists, [iterator/1, iterator/2]}). %% We must inline these functions so that the stacktrace points to %% the correct function. @@ -220,13 +229,22 @@ remove(_,_) -> erlang:nif_error(undef). take(_,_) -> erlang:nif_error(undef). --spec to_list(Map) -> [{Key,Value}] when - Map :: #{Key => Value}. +-spec to_list(MapOrIterator) -> [{Key, Value}] when + MapOrIterator :: #{Key => Value} | iterator(Key, Value). to_list(Map) when is_map(Map) -> to_list_internal(erts_internal:map_next(0, Map, [])); -to_list(Map) -> - error_with_info({badmap,Map}, [Map]). +to_list(Iter) -> + try to_list_from_iterator(next(Iter)) + catch + error:_ -> + error_with_info({badmap, Iter}, [Iter]) + end. + +to_list_from_iterator({Key, Value, NextIter}) -> + [{Key, Value} | to_list_from_iterator(next(NextIter))]; +to_list_from_iterator(none) -> + []. to_list_internal([Iter, Map | Acc]) when is_integer(Iter) -> to_list_internal(erts_internal:map_next(Iter, Map, Acc)); @@ -298,31 +316,28 @@ get(Key,Map,Default) -> MapOrIter :: #{Key => Value} | iterator(Key, Value), Map :: #{Key => Value}. -filter(Pred, MapOrIter) when is_function(Pred, 2) -> - Iter = if - is_map(MapOrIter) -> - iterator(MapOrIter); - true -> - MapOrIter - end, - try next(Iter) of - Next -> - maps:from_list(filter_1(Pred, Next)) +filter(Pred, Map) when is_map(Map), is_function(Pred, 2) -> + maps:from_list(filter_1(Pred, next(iterator(Map)), undefined)); +filter(Pred, Iter) when is_function(Pred, 2) -> + ErrorTag = make_ref(), + try filter_1(Pred, try_next(Iter, ErrorTag), ErrorTag) of + Result -> + maps:from_list(Result) catch - error:_ -> - error_with_info({badmap,MapOrIter}, [Pred, MapOrIter]) + error:ErrorTag -> + error_with_info({badmap, Iter}, [Pred, Iter]) end; filter(Pred, Map) -> badarg_with_info([Pred, Map]). -filter_1(Pred, {K, V, Iter}) -> +filter_1(Pred, {K, V, Iter}, ErrorTag) -> case Pred(K, V) of true -> - [{K,V} | filter_1(Pred, next(Iter))]; + [{K,V} | filter_1(Pred, try_next(Iter, ErrorTag), ErrorTag)]; false -> - filter_1(Pred, next(Iter)) + filter_1(Pred, try_next(Iter, ErrorTag), ErrorTag) end; -filter_1(_Pred, none) -> +filter_1(_Pred, none, _ErrorTag) -> []. -spec filtermap(Fun, MapOrIter) -> Map when @@ -330,57 +345,52 @@ filter_1(_Pred, none) -> MapOrIter :: #{Key => Value1} | iterator(Key, Value1), Map :: #{Key => Value1 | Value2}. -filtermap(Fun, MapOrIter) when is_function(Fun, 2) -> - Iter = if - is_map(MapOrIter) -> - iterator(MapOrIter); - true -> - MapOrIter - end, - try next(Iter) of - Next -> - maps:from_list(filtermap_1(Fun, Next)) +filtermap(Fun, Map) when is_map(Map), is_function(Fun, 2) -> + maps:from_list(filtermap_1(Fun, next(iterator(Map)), undefined)); +filtermap(Fun, Iter) when is_function(Fun, 2) -> + ErrorTag = make_ref(), + try filtermap_1(Fun, try_next(Iter, ErrorTag), ErrorTag) of + Result -> + maps:from_list(Result) catch - error:_ -> - error_with_info({badmap,MapOrIter}, [Fun, MapOrIter]) + error:ErrorTag -> + error_with_info({badmap, Iter}, [Fun, Iter]) end; filtermap(Fun, Map) -> badarg_with_info([Fun, Map]). -filtermap_1(Pred, {K, V, Iter}) -> - case Pred(K, V) of +filtermap_1(Fun, {K, V, Iter}, ErrorTag) -> + case Fun(K, V) of true -> - [{K, V} | filtermap_1(Pred, next(Iter))]; + [{K, V} | filtermap_1(Fun, try_next(Iter, ErrorTag), ErrorTag)]; {true, NewV} -> - [{K, NewV} | filtermap_1(Pred, next(Iter))]; + [{K, NewV} | filtermap_1(Fun, try_next(Iter, ErrorTag), ErrorTag)]; false -> - filtermap_1(Pred, next(Iter)) + filtermap_1(Fun, try_next(Iter, ErrorTag), ErrorTag) end; -filtermap_1(_Pred, none) -> +filtermap_1(_Fun, none, _ErrorTag) -> []. -spec foreach(Fun,MapOrIter) -> ok when Fun :: fun((Key, Value) -> term()), MapOrIter :: #{Key => Value} | iterator(Key, Value). -foreach(Fun, MapOrIter) when is_function(Fun, 2) -> - Iter = if is_map(MapOrIter) -> iterator(MapOrIter); - true -> MapOrIter - end, - try next(Iter) of - Next -> - foreach_1(Fun, Next) +foreach(Fun, Map) when is_map(Map), is_function(Fun, 2) -> + foreach_1(Fun, next(iterator(Map)), undefined); +foreach(Fun, Iter) when is_function(Fun, 2) -> + ErrorTag = make_ref(), + try foreach_1(Fun, try_next(Iter, ErrorTag), ErrorTag) catch - error:_ -> - error_with_info({badmap, MapOrIter}, [Fun, MapOrIter]) + error:ErrorTag -> + error_with_info({badmap, Iter}, [Fun, Iter]) end; -foreach(Pred, Map) -> - badarg_with_info([Pred, Map]). +foreach(Fun, Map) -> + badarg_with_info([Fun, Map]). -foreach_1(Fun, {K, V, Iter}) -> +foreach_1(Fun, {K, V, Iter}, ErrorTag) -> Fun(K,V), - foreach_1(Fun, next(Iter)); -foreach_1(_Fun, none) -> + foreach_1(Fun, try_next(Iter, ErrorTag), ErrorTag); +foreach_1(_Fun, none, _ErrorTag) -> ok. -spec fold(Fun,Init,MapOrIter) -> Acc when @@ -390,26 +400,21 @@ foreach_1(_Fun, none) -> AccIn :: Init | AccOut, MapOrIter :: #{Key => Value} | iterator(Key, Value). -fold(Fun, Init, MapOrIter) when is_function(Fun, 3) -> - Iter = if - is_map(MapOrIter) -> - iterator(MapOrIter); - true -> - MapOrIter - end, - try next(Iter) of - Next -> - fold_1(Fun, Init, Next) +fold(Fun, Init, Map) when is_map(Map), is_function(Fun, 3) -> + fold_1(Fun, Init, next(iterator(Map)), undefined); +fold(Fun, Init, Iter) when is_function(Fun, 3) -> + ErrorTag = make_ref(), + try fold_1(Fun, Init, try_next(Iter, ErrorTag), ErrorTag) catch - error:_ -> - error_with_info({badmap,MapOrIter}, [Fun, Init, MapOrIter]) + error:ErrorTag -> + error_with_info({badmap, Iter}, [Fun, Init, Iter]) end; fold(Fun, Init, Map) -> badarg_with_info([Fun, Init, Map]). -fold_1(Fun, Acc, {K, V, Iter}) -> - fold_1(Fun, Fun(K,V,Acc), next(Iter)); -fold_1(_Fun, Acc, none) -> +fold_1(Fun, Acc, {K, V, Iter}, ErrorTag) -> + fold_1(Fun, Fun(K,V,Acc), try_next(Iter, ErrorTag), ErrorTag); +fold_1(_Fun, Acc, none, _ErrorTag) -> Acc. -spec map(Fun,MapOrIter) -> Map when @@ -417,27 +422,24 @@ fold_1(_Fun, Acc, none) -> MapOrIter :: #{Key => Value1} | iterator(Key, Value1), Map :: #{Key => Value2}. -map(Fun, MapOrIter) when is_function(Fun, 2) -> - Iter = if - is_map(MapOrIter) -> - iterator(MapOrIter); - true -> - MapOrIter - end, - try next(Iter) of - Next -> - maps:from_list(map_1(Fun, Next)) +map(Fun, Map) when is_map(Map), is_function(Fun, 2) -> + maps:from_list(map_1(Fun, next(iterator(Map)), undefined)); +map(Fun, Iter) when is_function(Fun, 2) -> + ErrorTag = make_ref(), + try map_1(Fun, try_next(Iter, ErrorTag), ErrorTag) of + Result -> + maps:from_list(Result) catch - error:_ -> - error_with_info({badmap,MapOrIter}, [Fun, MapOrIter]) + error:ErrorTag -> + error_with_info({badmap, Iter}, [Fun, Iter]) end; map(Fun, Map) -> badarg_with_info([Fun, Map]). -map_1(Fun, {K, V, Iter}) -> - [{K, Fun(K, V)} | map_1(Fun, next(Iter))]; -map_1(_Fun, none) -> +map_1(Fun, {K, V, Iter}, ErrorTag) -> + [{K, Fun(K, V)} | map_1(Fun, try_next(Iter, ErrorTag), ErrorTag)]; +map_1(_Fun, none, _ErrorTag) -> []. -spec size(Map) -> non_neg_integer() when @@ -455,16 +457,43 @@ size(Map) -> Map :: #{Key => Value}, Iterator :: iterator(Key, Value). -iterator(M) when is_map(M) -> [0 | M]; +iterator(M) when is_map(M) -> iterator(M, undefined); iterator(M) -> error_with_info({badmap, M}, [M]). +-spec iterator(Map, Order) -> Iterator when + Map :: #{Key => Value}, + Order :: iterator_order(Key), + Iterator :: iterator(Key, Value). + +iterator(M, undefined) when is_map(M) -> + [0 | M]; +iterator(M, ordered) when is_map(M) -> + CmpFun = fun(A, B) -> erts_internal:cmp_term(A, B) =< 0 end, + Keys = lists:sort(CmpFun, maps:keys(M)), + [Keys | M]; +iterator(M, reversed) when is_map(M) -> + CmpFun = fun(A, B) -> erts_internal:cmp_term(B, A) =< 0 end, + Keys = lists:sort(CmpFun, maps:keys(M)), + [Keys | M]; +iterator(M, CmpFun) when is_map(M), is_function(CmpFun, 2) -> + Keys = lists:sort(CmpFun, maps:keys(M)), + [Keys | M]; +iterator(M, Order) -> + badarg_with_info([M, Order]). + -spec next(Iterator) -> {Key, Value, NextIterator} | 'none' when Iterator :: iterator(Key, Value), NextIterator :: iterator(Key, Value). next({K, V, I}) -> {K, V, I}; -next([Path | Map]) when is_integer(Path), is_map(Map) -> - erts_internal:map_next(Path, Map, iterator); +next([Path | Map] = Iterator) + when (is_integer(Path) orelse is_list(Path)), is_map(Map) -> + try erts_internal:map_next(Path, Map, iterator) of + Result -> Result + catch + error:badarg -> + badarg_with_info([Iterator]) + end; next(none) -> none; next(Iter) -> @@ -500,12 +529,13 @@ with_1([], _Map) -> []. %% groups_from_list/2 & groups_from_list/3 --spec groups_from_list(Fun, List) -> MapOut when - Fun :: fun((Elem :: T) -> Selected), - MapOut :: #{Selected => List}, - Selected :: term(), - List :: [T], - T :: term(). +-spec groups_from_list(KeyFun, List) -> GroupsMap when + KeyFun :: fun((Elem) -> Key), + GroupsMap :: #{Key => Group}, + Key :: term(), + List :: [Elem], + Group :: [Elem], + Elem :: term(). groups_from_list(Fun, List0) when is_function(Fun, 1) -> try lists:reverse(List0) of @@ -528,15 +558,15 @@ groups_from_list_1(Fun, [H | Tail], Acc) -> groups_from_list_1(_Fun, [], Acc) -> Acc. --spec groups_from_list(Fun, ValueFun, List) -> MapOut when - Fun :: fun((Elem :: T) -> Key), - ValueFun :: fun((Elem :: T) -> ValOut), - MapOut :: #{Key := ListOut}, +-spec groups_from_list(KeyFun, ValueFun, List) -> GroupsMap when + KeyFun :: fun((Elem) -> Key), + ValueFun :: fun((Elem) -> Value), + GroupsMap :: #{Key := Group}, Key :: term(), - ValOut :: term(), - List :: [T], - ListOut :: [ValOut], - T :: term(). + Value :: term(), + List :: [Elem], + Group :: [Value], + Elem :: term(). groups_from_list(Fun, ValueFun, List0) when is_function(Fun, 1), is_function(ValueFun, 1) -> @@ -579,3 +609,34 @@ badarg_with_info(Args) -> error_with_info(Reason, Args) -> erlang:error(Reason, Args, [{error_info, #{module => erl_stdlib_errors}}]). + +-spec is_iterator_valid(MaybeIter) -> boolean() when + MaybeIter :: iterator() | term(). + +is_iterator_valid(Iter) -> + try is_iterator_valid_1(Iter) + catch + error:badarg -> + false + end. + +is_iterator_valid_1(none) -> + true; +is_iterator_valid_1({_, _, Next}) -> + is_iterator_valid_1(next(Next)); +is_iterator_valid_1(Iter) -> + _ = next(Iter), + true. + +try_next({_, _, _} = KVI, _ErrorTag) -> + KVI; +try_next(none, _ErrorTag) -> + none; +try_next(Iter, undefined) -> + next(Iter); +try_next(Iter, ErrorTag) -> + try next(Iter) + catch + error:badarg -> + error(ErrorTag) + end. diff --git a/lib/stdlib/src/math.erl b/lib/stdlib/src/math.erl index 3a3b384d8f..cf41cd2f1d 100644 --- a/lib/stdlib/src/math.erl +++ b/lib/stdlib/src/math.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2016. All Rights Reserved. +%% Copyright Ericsson AB 1996-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -19,7 +19,7 @@ %% -module(math). --export([pi/0]). +-export([pi/0,tau/0]). %%% BIFs @@ -154,5 +154,7 @@ tanh(_) -> %%% End of BIFs -spec pi() -> float(). - pi() -> 3.1415926535897932. + +-spec tau() -> float(). +tau() -> 6.2831853071795864. diff --git a/lib/stdlib/src/ms_transform.erl b/lib/stdlib/src/ms_transform.erl index dde8e572a3..01496adb55 100644 --- a/lib/stdlib/src/ms_transform.erl +++ b/lib/stdlib/src/ms_transform.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2002-2022. All Rights Reserved. +%% Copyright Ericsson AB 2002-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -977,6 +977,8 @@ real_guard_function(abs,1) -> true; real_guard_function(element,2) -> true; real_guard_function(hd,1) -> true; real_guard_function(length,1) -> true; +real_guard_function(max,2) -> true; +real_guard_function(min,2) -> true; real_guard_function(node,0) -> true; real_guard_function(node,1) -> true; real_guard_function(round,1) -> true; @@ -1015,6 +1017,9 @@ action_function(set_tcw,1) -> true; action_function(silent,1) -> true; action_function(trace,2) -> true; action_function(trace,3) -> true; +action_function(caller_line,0) -> true; +action_function(current_stacktrace,0) -> true; +action_function(current_stacktrace,1) -> true; action_function(_,_) -> false. bool_operator('and',2) -> diff --git a/lib/stdlib/src/orddict.erl b/lib/stdlib/src/orddict.erl index 9a2772949b..e78b0187f2 100644 --- a/lib/stdlib/src/orddict.erl +++ b/lib/stdlib/src/orddict.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2017. All Rights Reserved. +%% Copyright Ericsson AB 1996-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -36,7 +36,7 @@ %%--------------------------------------------------------------------------- --spec new() -> orddict(). +-spec new() -> orddict(none(), none()). new() -> []. diff --git a/lib/stdlib/src/otp_internal.erl b/lib/stdlib/src/otp_internal.erl index 3afbbfb881..5bc0704b68 100644 --- a/lib/stdlib/src/otp_internal.erl +++ b/lib/stdlib/src/otp_internal.erl @@ -31,50 +31,28 @@ obsolete(auth, is_auth, 1) -> {deprecated, "use net_adm:ping/1 instead"}; obsolete(calendar, local_time_to_universal_time, 1) -> {deprecated, "use calendar:local_time_to_universal_time_dst/1 instead"}; -obsolete(code, is_module_native, 1) -> - {deprecated, "HiPE has been removed", "OTP 26"}; -obsolete(code, rehash, 0) -> - {deprecated, "the code path cache feature has been removed", "OTP 26"}; obsolete(crypto, crypto_dyn_iv_init, 3) -> {deprecated, "see the documentation for details", "OTP 27"}; obsolete(crypto, crypto_dyn_iv_update, 3) -> {deprecated, "see the documentation for details", "OTP 27"}; obsolete(crypto, rand_uniform, 2) -> {deprecated, "use rand:uniform/1 instead"}; -obsolete(disk_log, accessible_logs, 0) -> - {deprecated, "use disk_log:all/0 instead", "OTP 26"}; -obsolete(disk_log, lclose, 1) -> - {deprecated, "use disk_log:close/1 instead", "OTP 26"}; -obsolete(disk_log, lclose, 2) -> - {deprecated, "use disk_log:close/1 instead", "OTP 26"}; +obsolete(dbg, stop_clear, 0) -> + {deprecated, "use dbg:stop/0 instead", "OTP 27"}; +obsolete(disk_log, inc_wrap_file, 1) -> + {deprecated, "use disk_log:next_file/1 instead", "OTP 28"}; obsolete(erlang, now, 0) -> {deprecated, "see the \"Time and Time Correction in Erlang\" chapter of the ERTS User's Guide for more information"}; obsolete(erlang, phash, 2) -> {deprecated, "use erlang:phash2/2 instead"}; -obsolete(ftp, start_service, 1) -> - {deprecated, "use ftp:open/2 instead", "OTP 26"}; -obsolete(ftp, stop_service, 1) -> - {deprecated, "use ftp:close/1 instead", "OTP 26"}; +obsolete(file, pid2name, 1) -> + {deprecated, "this functionality is no longer supported", "OTP 27"}; obsolete(http_uri, decode, 1) -> - {deprecated, "use uri_string:unquote function instead", "OTP 26"}; + {deprecated, "use uri_string:unquote function instead", "OTP 27"}; obsolete(http_uri, encode, 1) -> - {deprecated, "use uri_string:quote function instead", "OTP 26"}; + {deprecated, "use uri_string:quote function instead", "OTP 27"}; obsolete(httpd, parse_query, 1) -> {deprecated, "use uri_string:dissect_query/1 instead"}; -obsolete(httpd_util, decode_hex, 1) -> - {deprecated, "use uri_string:unquote function instead", "OTP 26"}; -obsolete(httpd_util, encode_hex, 1) -> - {deprecated, "use uri_string:quote function instead", "OTP 26"}; -obsolete(httpd_util, flatlength, 1) -> - {deprecated, "use erlang:iolist_size/1 instead", "OTP 26"}; -obsolete(httpd_util, hexlist_to_integer, 1) -> - {deprecated, "use erlang:list_to_integer/2 with base 16 instead", "OTP 26"}; -obsolete(httpd_util, integer_to_hexlist, 1) -> - {deprecated, "use erlang:integer_to_list/2 with base 16 instead", "OTP 26"}; -obsolete(httpd_util, strip, 1) -> - {deprecated, "use string:trim/1 instead", "OTP 26"}; -obsolete(httpd_util, suffix, 1) -> - {deprecated, "use filename:extension/1 and string:trim/2 instead", "OTP 26"}; obsolete(net, broadcast, 3) -> {deprecated, "use rpc:eval_everywhere/3 instead"}; obsolete(net, call, 4) -> @@ -115,6 +93,10 @@ obsolete(zlib, inflateChunk, 2) -> {deprecated, "use safeInflate/2 instead", "OTP 27"}; obsolete(zlib, setBufSize, 2) -> {deprecated, "this function will be removed in a future release", "OTP 27"}; +obsolete(code, is_module_native, 1) -> + {removed, "HiPE has been removed"}; +obsolete(code, rehash, 0) -> + {removed, "the code path cache feature has been removed"}; obsolete(core_lib, get_anno, 1) -> {removed, "use cerl:get_ann/1 instead"}; obsolete(core_lib, is_literal, 1) -> @@ -155,6 +137,12 @@ obsolete(crypto, stream_decrypt, 2) -> {removed, "use crypto:crypto_update/2 instead"}; obsolete(crypto, stream_encrypt, 2) -> {removed, "use crypto:crypto_update/2 instead"}; +obsolete(disk_log, accessible_logs, 0) -> + {removed, "use disk_log:all/0 instead"}; +obsolete(disk_log, lclose, 1) -> + {removed, "use disk_log:close/1 instead"}; +obsolete(disk_log, lclose, 2) -> + {removed, "use disk_log:close/1 instead"}; obsolete(erl_lint, modify_line, 2) -> {removed, "use erl_parse:map_anno/2 instead"}; obsolete(erl_parse, get_attribute, 2) -> @@ -171,6 +159,10 @@ obsolete(erlang, hash, 2) -> {removed, "use erlang:phash2/2 instead"}; obsolete(filename, safe_relative_path, 1) -> {removed, "use filelib:safe_relative_path/2 instead"}; +obsolete(ftp, start_service, 1) -> + {removed, "use ftp:open/2 instead"}; +obsolete(ftp, stop_service, 1) -> + {removed, "use ftp:close/1 instead"}; obsolete(http_uri, parse, 1) -> {removed, "use uri_string functions instead"}; obsolete(http_uri, parse, 2) -> @@ -189,6 +181,20 @@ obsolete(httpd_conf, is_file, 1) -> {removed, "use filelib:is_file/1 instead"}; obsolete(httpd_conf, make_integer, 1) -> {removed, "use erlang:list_to_integer/1 instead"}; +obsolete(httpd_util, decode_hex, 1) -> + {removed, "use uri_string:unquote function instead"}; +obsolete(httpd_util, encode_hex, 1) -> + {removed, "use uri_string:quote function instead"}; +obsolete(httpd_util, flatlength, 1) -> + {removed, "use erlang:iolist_size/1 instead"}; +obsolete(httpd_util, hexlist_to_integer, 1) -> + {removed, "use erlang:list_to_integer/2 with base 16 instead"}; +obsolete(httpd_util, integer_to_hexlist, 1) -> + {removed, "use erlang:integer_to_list/2 with base 16 instead"}; +obsolete(httpd_util, strip, 1) -> + {removed, "use string:trim/1 instead"}; +obsolete(httpd_util, suffix, 1) -> + {removed, "use filename:extension/1 and string:trim/2 instead"}; obsolete(net, relay, 1) -> {removed, "use fun Relay(Pid) -> receive X -> Pid ! X end, Relay(Pid) instead"}; obsolete(public_key, ssh_decode, 2) -> @@ -231,14 +237,14 @@ obsolete(ssl, ssl_accept, _) -> {removed, "use ssl_handshake/1,2,3 instead"}; obsolete(ct_slave, _, _) -> {deprecated, "use ?CT_PEER(), or the 'peer' module instead", "OTP 27"}; -obsolete(erts_alloc_config, _, _) -> - {deprecated, "this module will be removed in OTP 26.0. See the documentation for details", "OTP 26"}; obsolete(gen_fsm, _, _) -> {deprecated, "use the 'gen_statem' module instead"}; obsolete(random, _, _) -> {deprecated, "use the 'rand' module instead"}; obsolete(slave, _, _) -> {deprecated, "use the 'peer' module instead", "OTP 27"}; +obsolete(erts_alloc_config, _, _) -> + {removed, "this module has as of OTP 26.0 been removed"}; obsolete(os_mon_mib, _, _) -> {removed, "this module was removed in OTP 22.0"}; obsolete(pg2, _, _) -> @@ -276,6 +282,8 @@ obsolete_type(http_uri, query, 0) -> {removed, "use uri_string instead"}; obsolete_type(http_uri, scheme, 0) -> {removed, "use uri_string instead"}; +obsolete_type(http_uri, uri, 0) -> + {removed, "use uri_string instead"}; obsolete_type(http_uri, user_info, 0) -> {removed, "use uri_string instead"}; obsolete_type(_,_,_) -> no. diff --git a/lib/stdlib/src/peer.erl b/lib/stdlib/src/peer.erl index b4867c1bdf..0879b799c5 100644 --- a/lib/stdlib/src/peer.erl +++ b/lib/stdlib/src/peer.erl @@ -123,9 +123,12 @@ %% saving exit reason in the state %% crash: when peer terminates, origin process %% terminates with underlying reason - exec => exec(), %% path to executable, or SSH/Docker support connection => connection(), %% alternative connection specification + exec => exec(), %% path to executable, or SSH/Docker support + detached => boolean(), %% if the node should be start in detached mode args => [string()], %% additional command line parameters to append + post_process_args => + fun(([string()]) -> [string()]),%% fix the arguments env => [{string(), string()}], %% additional environment variables wait_boot => wait_boot(), %% default is synchronous start with 15 sec timeout shutdown => close | %% close supervision channel @@ -291,14 +294,20 @@ init([Notify, Options]) -> Env = maps:get(env, Options, []), + PostProcessArgs = maps:get(post_process_args, Options, fun(As) -> As end), + FinalArgs = PostProcessArgs(Args), + %% close port if running detached Conn = case maps:find(connection, Options) of {ok, standard_io} -> %% Cannot detach a peer that uses stdio. Request exit_status. - open_port({spawn_executable, Exec}, [{args, Args}, {env, Env}, hide, binary, exit_status]); + open_port({spawn_executable, Exec}, + [{args, FinalArgs}, {env, Env}, hide, + binary, exit_status, stderr_to_stdout]); _ -> - Port = open_port({spawn_executable, Exec}, [{args, Args}, {env, Env}, hide, binary]), + Port = open_port({spawn_executable, Exec}, + [{args, FinalArgs}, {env, Env}, hide, binary]), %% peer can close the port before we get here which will cause %% port_close to throw. Catch this and ignore. catch erlang:port_close(Port), @@ -339,6 +348,22 @@ handle_call({call, M, F, A}, From, origin_to_peer(tcp, Socket, {call, Seq, M, F, A}), {noreply, State#peer_state{outstanding = Out#{Seq => From}, seq = Seq + 1}}; +handle_call({starting, Node}, _From, #peer_state{ options = Options } = State) -> + case maps:find(shutdown, Options) of + {ok, {Timeout, MainCoverNode}} when is_integer(Timeout), + is_atom(MainCoverNode) -> + %% The node was started using test_server:start_peer/2 with cover enabled + %% so we should start cover on the starting node. + Modules = erpc:call(MainCoverNode,cover,modules,[]), + Sticky = [ begin erpc:call(Node, code, unstick_mod, [M]), M end + || M <- Modules, erpc:call(Node, code, is_sticky,[M])], + _ = erpc:call(MainCoverNode, cover, start, [Node]), + _ = [erpc:call(Node, code,stick_mod,[M]) || M <- Sticky], + ok; + _ -> + ok + end, + {reply, ok, State}; handle_call(get_node, _From, #peer_state{node = Node} = State) -> {reply, Node, State}; @@ -538,8 +563,9 @@ verify_args(Options) -> [error({invalid_arg, Arg}) || Arg <- Args, not io_lib:char_list(Arg)], %% alternative connection must be requested for non-distributed node, %% or a distributed node when origin is not alive - is_map_key(connection, Options) orelse - (is_map_key(name, Options) andalso erlang:is_alive()) orelse error(not_alive), + is_map_key(connection, Options) + orelse + (is_map_key(name, Options) andalso erlang:is_alive()) orelse error(not_alive), %% exec must be a string, or a tuple of string(), [string()] case maps:find(exec, Options) of {ok, {Exec, Strs}} -> @@ -579,8 +605,13 @@ verify_args(Options) -> ok; {ok, Err2} -> error({shutdown, Err2}) + end, + case maps:find(detached, Options) of + {ok, false} when map_get(connection, Options) =:= standard_io -> + error({detached, cannot_detach_with_standard_io}); + _ -> + ok end. - make_notify_ref(infinity) -> {self(), make_ref()}; @@ -796,6 +827,14 @@ command_line(Listen, Options) -> NameArg = name_arg(maps:find(name, Options), maps:find(host, Options), maps:find(longnames, Options)), %% additional command line args CmdOpts = maps:get(args, Options, []), + + %% If we should detach from the node. We use -detached to tell erl to detach + %% and -peer_detached to tell peer:start that we are detached. + DetachArgs = case maps:get(detached, Options, true) of + true -> ["-detached","-peer_detached"]; + false -> [] + end, + %% start command StartCmd = case Listen of @@ -803,14 +842,14 @@ command_line(Listen, Options) -> ["-user", atom_to_list(?MODULE)]; undefined -> Self = base64:encode_to_string(term_to_binary(self())), - ["-detached", "-noinput", "-user", atom_to_list(?MODULE), "-origin", Self]; + DetachArgs ++ ["-user", atom_to_list(?MODULE), "-origin", Self]; {Ips, Port} -> IpStr = lists:concat(lists:join(",", [inet:ntoa(Ip) || Ip <- Ips])), - ["-detached", "-noinput", "-user", atom_to_list(?MODULE), "-origin", IpStr, integer_to_list(Port)] + DetachArgs ++ ["-user", atom_to_list(?MODULE), "-origin", IpStr, integer_to_list(Port)] end, %% build command line {Exec, PreArgs} = exec(Options), - {Exec, PreArgs ++ NameArg ++ StartCmd ++ CmdOpts}. + {Exec, PreArgs ++ NameArg ++ CmdOpts ++ StartCmd}. exec(#{exec := Prog}) when is_list(Prog) -> {Prog, []}; @@ -1018,23 +1057,44 @@ start_peer_channel_handler() -> {ok, [[IpStr, PortString]]} -> %% enter this clause when -origin IpList Port is specified in the command line. Port = list_to_integer(PortString), - Ips = [begin {ok, Addr} = inet:parse_address(Ip), Addr end || Ip <- string:lexemes(IpStr, ",")], - spawn(fun () -> tcp_init(Ips, Port) end); + Ips = [begin {ok, Addr} = inet:parse_address(Ip), Addr end || + Ip <- string:lexemes(IpStr, ",")], + TCPConnection = spawn(fun () -> tcp_init(Ips, Port) end), + _ = case init:get_argument(peer_detached) of + {ok, _} -> + _ = register(user, TCPConnection); + error -> + _= user_sup:init( + [Flag || Flag <- init:get_arguments(), Flag =/= {user,["peer"]}]) + end, + TCPConnection; {ok, [[Base64EncProc]]} -> %% No alternative connection, but have "-origin Base64EncProc" OriginProcess = binary_to_term(base64:decode(Base64EncProc)), - %% setup 'user' process, I/O redirection: ask controlling process - %% who is the group leader. - GroupLeader = gen_server:call(OriginProcess, group_leader), - RelayPid = spawn(fun () -> relay(GroupLeader) end), - register(user, RelayPid), - spawn( - fun () -> - link(RelayPid), - MRef = monitor(process, OriginProcess), - notify_when_started(dist, OriginProcess), - origin_link(MRef, OriginProcess) - end); + OriginLink = spawn( + fun () -> + MRef = monitor(process, OriginProcess), + notify_when_started(dist, OriginProcess), + origin_link(MRef, OriginProcess) + end), + ok = gen_server:call(OriginProcess, {starting, node()}), + _ = case init:get_argument(peer_detached) of + {ok, _} -> + %% We are detached, so setup 'user' process, I/O redirection: + %% ask controlling process who is the group leader. + GroupLeader = gen_server:call(OriginProcess, group_leader), + RelayPid = spawn(fun () -> + link(OriginLink), + relay(GroupLeader) + end), + _ = register(user, RelayPid); + error -> + %% We are not detached, so after we spawn the link process we + %% start the terminal as normal but without the -user peer flag. + _ = user_sup:init( + [Flag || Flag <- init:get_arguments(), Flag =/= {user,["peer"]}]) + end, + OriginLink; error -> %% no -origin specified, meaning that standard I/O is used for alternative spawn(fun io_server/0) @@ -1074,7 +1134,6 @@ io_server() -> tcp_init(IpList, Port) -> try Sock = loop_connect(IpList, Port), - register(user, self()), erlang:group_leader(self(), self()), notify_when_started(tcp, Sock), io_server_loop(tcp, Sock, #{}, #{}, undefined) diff --git a/lib/stdlib/src/proc_lib.erl b/lib/stdlib/src/proc_lib.erl index a95d75b9f6..9c7235b954 100644 --- a/lib/stdlib/src/proc_lib.erl +++ b/lib/stdlib/src/proc_lib.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2022. All Rights Reserved. +%% Copyright Ericsson AB 1996-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -31,6 +31,7 @@ start_monitor/3, start_monitor/4, start_monitor/5, hibernate/3, init_ack/1, init_ack/2, + init_fail/2, init_fail/3, init_p/3,init_p/5,format/1,format/2,format/3,report_cb/2, initial_call/1, translate_initial_call/1, @@ -273,6 +274,7 @@ exit_reason(exit, Reason, _Stacktrace) -> exit_reason(throw, Reason, Stacktrace) -> {{nocatch, Reason}, Stacktrace}. + -spec start(Module, Function, Args) -> Ret when Module :: module(), Function :: atom(), @@ -309,14 +311,20 @@ sync_start({Pid, Ref}, Timeout) -> {ack, Pid, Return} -> erlang:demonitor(Ref, [flush]), Return; + {nack, Pid, Return} -> + flush_EXIT(Pid), + _ = await_DOWN(Pid, Ref), + Return; {'DOWN', Ref, process, Pid, Reason} -> + flush_EXIT(Pid), {error, Reason} after Timeout -> - erlang:demonitor(Ref, [flush]), - kill_flush(Pid), + kill_flush_EXIT(Pid), + _ = await_DOWN(Pid, Ref), {error, timeout} end. + -spec start_link(Module, Function, Args) -> Ret when Module :: module(), Function :: atom(), @@ -334,7 +342,7 @@ start_link(M, F, A) when is_atom(M), is_atom(F), is_list(A) -> Ret :: term() | {error, Reason :: term()}. start_link(M, F, A, Timeout) when is_atom(M), is_atom(F), is_list(A) -> - sync_start_link(?MODULE:spawn_link(M, F, A), Timeout). + sync_start(?MODULE:spawn_opt(M, F, A, [link,monitor]), Timeout). -spec start_link(Module, Function, Args, Time, SpawnOpts) -> Ret when Module :: module(), @@ -346,18 +354,9 @@ start_link(M, F, A, Timeout) when is_atom(M), is_atom(F), is_list(A) -> start_link(M,F,A,Timeout,SpawnOpts) when is_atom(M), is_atom(F), is_list(A) -> ?VERIFY_NO_MONITOR_OPT(M, F, A, Timeout, SpawnOpts), - sync_start_link(?MODULE:spawn_opt(M, F, A, [link|SpawnOpts]), Timeout). + sync_start( + ?MODULE:spawn_opt(M, F, A, [link,monitor|SpawnOpts]), Timeout). -sync_start_link(Pid, Timeout) -> - receive - {ack, Pid, Return} -> - Return; - {'EXIT', Pid, Reason} -> - {error, Reason} - after Timeout -> - kill_flush(Pid), - {error, timeout} - end. -spec start_monitor(Module, Function, Args) -> {Ret, Mon} when Module :: module(), @@ -393,29 +392,54 @@ start_monitor(M,F,A,Timeout,SpawnOpts) when is_atom(M), is_atom(F), is_list(A) -> ?VERIFY_NO_MONITOR_OPT(M, F, A, Timeout, SpawnOpts), - sync_start_monitor(?MODULE:spawn_opt(M, F, A, [monitor|SpawnOpts]), - Timeout). + sync_start_monitor( + ?MODULE:spawn_opt(M, F, A, [monitor|SpawnOpts]), Timeout). sync_start_monitor({Pid, Ref}, Timeout) -> receive {ack, Pid, Return} -> {Return, Ref}; + {nack, Pid, Return} -> + flush_EXIT(Pid), + self() ! await_DOWN(Pid, Ref), + {Return, Ref}; {'DOWN', Ref, process, Pid, Reason} = Down -> + flush_EXIT(Pid), self() ! Down, {{error, Reason}, Ref} after Timeout -> - kill_flush(Pid), + kill_flush_EXIT(Pid), + self() ! await_DOWN(Pid, Ref), {{error, timeout}, Ref} end. --spec kill_flush(Pid) -> 'ok' when - Pid :: pid(). -kill_flush(Pid) -> +%% We regard the existence of an {'EXIT', Pid, _} message +%% as proof enough that there was a link that fired and +%% we had process_flag(trap_exit, true), +%% so the message should be flushed. +%% It is the best we can do. +%% +%% After an unlink(Pid) an {'EXIT', Pid, _} link message +%% cannot arrive so receive after 0 will work, + +flush_EXIT(Pid) -> + unlink(Pid), + receive {'EXIT', Pid, _} -> ok after 0 -> ok end. + +kill_flush_EXIT(Pid) -> + %% unlink/1 has to be called before exit/2 + %% or we might be killed by the link unlink(Pid), exit(Pid, kill), - receive {'EXIT', Pid, _} -> ok after 0 -> ok end, - ok. + receive {'EXIT', Pid, _} -> ok after 0 -> ok end. + +await_DOWN(Pid, Ref) -> + receive + {'DOWN', Ref, process, Pid, _} = Down -> + Down + end. + -spec init_ack(Parent, Ret) -> 'ok' when Parent :: pid(), @@ -432,6 +456,27 @@ init_ack(Return) -> [Parent|_] = get('$ancestors'), init_ack(Parent, Return). +-spec init_fail(_, _, _) -> no_return(). +init_fail(Parent, Return, Exception) -> + _ = Parent ! {nack, self(), Return}, + case Exception of + {error, Reason} -> + erlang:error(Reason); + {exit, Reason} -> + erlang:exit(Reason); + {throw, Reason} -> + erlang:throw(Reason); + {Class, Reason, Stacktrace} -> + erlang:error( + erlang:raise(Class, Reason, Stacktrace), + [Parent, Return, Exception]) + end. + +-spec init_fail(_, _) -> no_return(). +init_fail(Return, Exception) -> + [Parent|_] = get('$ancestors'), + init_fail(Parent, Return, Exception). + %% ----------------------------------------------------- %% Fetch the initial call of a proc_lib spawned process. %% ----------------------------------------------------- diff --git a/lib/stdlib/src/proplists.erl b/lib/stdlib/src/proplists.erl index 8f25229cdd..a1198a972a 100644 --- a/lib/stdlib/src/proplists.erl +++ b/lib/stdlib/src/proplists.erl @@ -643,7 +643,7 @@ apply_stages(L, []) -> Rest :: [term()]. split(List, Keys) -> - {Store, Rest} = split(List, maps:from_list([{K, []} || K <- Keys]), []), + {Store, Rest} = split(List, #{K => [] || K <- Keys}, []), {[lists:reverse(map_get(K, Store)) || K <- Keys], lists:reverse(Rest)}. diff --git a/lib/stdlib/src/qlc.erl b/lib/stdlib/src/qlc.erl index cc4a988552..fab7207d13 100644 --- a/lib/stdlib/src/qlc.erl +++ b/lib/stdlib/src/qlc.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2021. All Rights Reserved. +%% Copyright Ericsson AB 2004-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -1326,7 +1326,7 @@ abstr_term(PPR, Anno) when is_pid(PPR); is_port(PPR); is_reference(PPR) -> abstr_term(Map, Anno) when is_map(Map) -> {map,Anno, [{map_field_assoc,Anno,abstr_term(K, Anno),abstr_term(V, Anno)} || - {K,V} <- maps:to_list(Map)]}; + K := V <- Map]}; abstr_term(Simple, Anno) -> erl_parse:abstract(Simple, erl_anno:line(Anno)). diff --git a/lib/stdlib/src/queue.erl b/lib/stdlib/src/queue.erl index a71121ad2d..82eb7b1df0 100644 --- a/lib/stdlib/src/queue.erl +++ b/lib/stdlib/src/queue.erl @@ -59,7 +59,7 @@ %% Creation, inspection and conversion %% O(1) --spec new() -> queue(). +-spec new() -> queue(none()). new() -> {[],[]}. %{RearList,FrontList} %% O(1) diff --git a/lib/stdlib/src/rand.erl b/lib/stdlib/src/rand.erl index 5f8d3a4c59..9c0f3a5c43 100644 --- a/lib/stdlib/src/rand.erl +++ b/lib/stdlib/src/rand.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2015-2022. All Rights Reserved. +%% Copyright Ericsson AB 2015-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -1466,7 +1466,7 @@ dummy_seed({A1, A2, A3}) -> %% the same sequence in the reverse order. The generator %% CX1 = (A * CX0) rem P %% that uses the multiplicative inverse mod P is, indeed, -%% an exact equevalent to the corresponding MWC generator. +%% an exact equivalent to the corresponding MWC generator. %% %% An MWC generator has, due to the power of two multiplier %% in the corresponding MCG, got known statistical weaknesses @@ -1502,8 +1502,7 @@ dummy_seed({A1, A2, A3}) -> -type mwc59_state() :: 1..?MWC59_P-1. -spec mwc59(CX0 :: mwc59_state()) -> CX1 :: mwc59_state(). -mwc59(CX0) -> % when is_integer(CX0), 1 =< CX0, CX0 < ?MWC59_P -> - CX = ?MASK(59, CX0), +mwc59(CX) when is_integer(CX), 1 =< CX, CX < ?MWC59_P -> C = CX bsr ?MWC59_B, X = ?MASK(?MWC59_B, CX), ?MWC59_A * X + C. @@ -1521,18 +1520,17 @@ mwc59(CX0) -> % when is_integer(CX0), 1 =< CX0, CX0 < ?MWC59_P -> %%% mwc59(CX1, N - 1). -spec mwc59_value32(CX :: mwc59_state()) -> V :: 0..?MASK(32). -mwc59_value32(CX1) -> % when is_integer(CX1), 1 =< CX1, CX1 < ?MWC59_P -> +mwc59_value32(CX1) when is_integer(CX1), 1 =< CX1, CX1 < ?MWC59_P -> CX = ?MASK(32, CX1), CX bxor ?BSL(32, CX, ?MWC59_XS). -spec mwc59_value(CX :: mwc59_state()) -> V :: 0..?MASK(59). -mwc59_value(CX1) -> % when is_integer(CX1), 1 =< CX1, CX1 < ?MWC59_P -> - CX = ?MASK(59, CX1), +mwc59_value(CX) when is_integer(CX), 1 =< CX, CX < ?MWC59_P -> CX2 = CX bxor ?BSL(59, CX, ?MWC59_XS1), CX2 bxor ?BSL(59, CX2, ?MWC59_XS2). -spec mwc59_float(CX :: mwc59_state()) -> V :: float(). -mwc59_float(CX1) -> +mwc59_float(CX1) when is_integer(CX1), 1 =< CX1, CX1 < ?MWC59_P -> CX = ?MASK(53, CX1), CX2 = CX bxor ?BSL(53, CX, ?MWC59_XS1), CX3 = CX2 bxor ?BSL(53, CX2, ?MWC59_XS2), diff --git a/lib/stdlib/src/re.erl b/lib/stdlib/src/re.erl index 863bbeb652..0f1f9a137a 100644 --- a/lib/stdlib/src/re.erl +++ b/lib/stdlib/src/re.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2020. All Rights Reserved. +%% Copyright Ericsson AB 2008-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -31,6 +31,8 @@ | bsr_anycrlf | bsr_unicode | no_start_optimize | ucp | never_utf. +-type replace_fun() :: fun((binary(), [binary()]) -> iodata() | unicode:charlist()). + %%% BIFs -export([internal_run/4]). @@ -353,7 +355,7 @@ compile_split(_,_) -> -spec replace(Subject, RE, Replacement) -> iodata() | unicode:charlist() when Subject :: iodata() | unicode:charlist(), RE :: mp() | iodata(), - Replacement :: iodata() | unicode:charlist(). + Replacement :: iodata() | unicode:charlist() | replace_fun(). replace(Subject,RE,Replacement) -> try @@ -366,7 +368,7 @@ replace(Subject,RE,Replacement) -> -spec replace(Subject, RE, Replacement, Options) -> iodata() | unicode:charlist() when Subject :: iodata() | unicode:charlist(), RE :: mp() | iodata() | unicode:charlist(), - Replacement :: iodata() | unicode:charlist(), + Replacement :: iodata() | unicode:charlist() | replace_fun(), Options :: [Option], Option :: anchored | global | notbol | noteol | notempty | notempty_atstart @@ -380,11 +382,11 @@ replace(Subject,RE,Replacement) -> replace(Subject,RE,Replacement,Options) -> try - {NewOpt,Convert} = process_repl_params(Options,iodata), - Unicode = check_for_unicode(RE, Options), - FlatSubject = to_binary(Subject, Unicode), - FlatReplacement = to_binary(Replacement, Unicode), - IoList = do_replace(FlatSubject,Subject,RE,FlatReplacement,NewOpt), + {NewOpt,Convert} = process_repl_params(Options,iodata), + Unicode = check_for_unicode(RE, Options), + FlatSubject = to_binary(Subject, Unicode), + Replacement1 = normalize_replacement(Replacement, Unicode), + IoList = do_replace(FlatSubject,Subject,RE,Replacement1,NewOpt), case Convert of iodata -> IoList; @@ -412,6 +414,10 @@ replace(Subject,RE,Replacement,Options) -> badarg_with_info([Subject,RE,Replacement,Options]) end. +normalize_replacement(Replacement, _Unicode) when is_function(Replacement, 2) -> + Replacement; +normalize_replacement(Replacement, Unicode) -> + to_binary(Replacement, Unicode). do_replace(FlatSubject,Subject,RE,Replacement,Options) -> case re:run(FlatSubject,RE,Options) of @@ -512,7 +518,9 @@ precomp_repl(<<X,Rest/binary>>) -> [<<X,BHead/binary>> | T0]; Other -> [<<X>> | Other] - end. + end; +precomp_repl(Repl) when is_function(Repl) -> + Repl. @@ -540,6 +548,16 @@ do_mlist(Whole,Subject,Pos,Repl,[[{MPos,Count} | Sub] | Tail]) [NewData | do_mlist(Whole,Rest,Pos+EatLength,Repl,Tail)]. +do_replace(Subject, Repl, SubExprs0) when is_function(Repl) -> + All = binary:part(Subject, hd(SubExprs0)), + SubExprs1 = + [ if + Pos >= 0, Len > 0 -> + binary:part(Subject, Pos, Len); + true -> + <<>> + end || {Pos, Len} <- tl(SubExprs0) ], + Repl(All, SubExprs1); do_replace(_,[Bin],_) when is_binary(Bin) -> Bin; do_replace(Subject,Repl,SubExprs0) -> diff --git a/lib/stdlib/src/sets.erl b/lib/stdlib/src/sets.erl index bdc0ed40f3..dccc6dcf3a 100644 --- a/lib/stdlib/src/sets.erl +++ b/lib/stdlib/src/sets.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2000-2021. All Rights Reserved. +%% Copyright Ericsson AB 2000-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -89,12 +89,12 @@ %%------------------------------------------------------------------------------ %% new() -> Set --spec new() -> set(). +-spec new() -> set(none()). new() -> Empty = mk_seg(?seg_size), #set{empty = Empty, segs = {Empty}}. --spec new([{version, 1..2}]) -> set(). +-spec new([{version, 1..2}]) -> set(none()). new([{version, 2}]) -> #{}; new(Opts) -> @@ -358,31 +358,51 @@ is_disjoint_1(Set, Iter) -> Set2 :: set(Element), Set3 :: set(Element). -subtract(#{}=S1, #{}=S2) -> - Next = maps:next(maps:iterator(S1)), - subtract_heuristic(Next, [], [], floor(map_size(S1) * 0.75), S1, S2); -subtract(S1, S2) -> - filter(fun (E) -> not is_element(E, S2) end, S1). +subtract(#{}=LHS, #{}=RHS) -> + LSize = map_size(LHS), + RSize = map_size(RHS), + + case RSize =< (LSize div 4) of + true -> + %% If we're guaranteed to keep more than 75% of the keys, it's + %% always cheaper to delete them one-by-one from the start. + Next = maps:next(maps:iterator(RHS)), + subtract_decided(Next, LHS, RHS); + false -> + %% We might delete more than 25% of the keys. Dynamically + %% transition to deleting elements one-by-one if we can determine + %% that we'll keep more than 75%. + KeepThreshold = (LSize * 3) div 4, + Next = maps:next(maps:iterator(LHS)), + subtract_heuristic(Next, [], [], KeepThreshold, LHS, RHS) + end; +subtract(LHS, RHS) -> + filter(fun (E) -> not is_element(E, RHS) end, LHS). -%% If we are keeping more than 75% of the keys, then it is -%% cheaper to delete them. Stop accumulating and start deleting. subtract_heuristic(Next, _Keep, Delete, 0, Acc, Reference) -> - subtract_decided(Next, remove_keys(Delete, Acc), Reference); -subtract_heuristic({Key, _Value, Iterator}, Keep, Delete, KeepCount, Acc, Reference) -> + %% We've kept more than 75% of the keys, transition to removing them + %% one-by-one. + subtract_decided(Next, remove_keys(Delete, Acc), Reference); +subtract_heuristic({Key, _Value, Iterator}, Keep, Delete, + KeepCount, Acc, Reference) -> Next = maps:next(Iterator), case Reference of - #{Key := _} -> - subtract_heuristic(Next, Keep, [Key | Delete], KeepCount, Acc, Reference); + #{ Key := _ } -> + subtract_heuristic(Next, Keep, [Key | Delete], + KeepCount, Acc, Reference); _ -> - subtract_heuristic(Next, [Key | Keep], Delete, KeepCount - 1, Acc, Reference) + subtract_heuristic(Next, [Key | Keep], Delete, + KeepCount - 1, Acc, Reference) end; subtract_heuristic(none, Keep, _Delete, _Count, _Acc, _Reference) -> maps:from_keys(Keep, ?VALUE). subtract_decided({Key, _Value, Iterator}, Acc, Reference) -> case Reference of - #{Key := _} -> - subtract_decided(maps:next(Iterator), maps:remove(Key, Acc), Reference); + #{ Key := _ } -> + subtract_decided(maps:next(Iterator), + maps:remove(Key, Acc), + Reference); _ -> subtract_decided(maps:next(Iterator), Acc, Reference) end; @@ -446,23 +466,12 @@ fold_1(Fun, Acc, Iter) -> Set1 :: set(Element), Set2 :: set(Element). filter(F, #{}=D) when is_function(F, 1)-> - maps:from_keys(filter_1(F, maps:iterator(D)), ?VALUE); + %% For this purpose, it is more efficient to use + %% maps:from_keys than a map comprehension. + maps:from_keys([K || K := _ <- D, F(K)], ?VALUE); filter(F, #set{}=D) when is_function(F, 1)-> filter_set(F, D). -filter_1(Fun, Iter) -> - case maps:next(Iter) of - {K, _, NextIter} -> - case Fun(K) of - true -> - [K | filter_1(Fun, NextIter)]; - false -> - filter_1(Fun, NextIter) - end; - none -> - [] - end. - %% get_slot(Hashdb, Key) -> Slot. %% Get the slot. First hash on the new range, if we hit a bucket %% which has not been split use the unsplit buddy bucket. diff --git a/lib/stdlib/src/shell.erl b/lib/stdlib/src/shell.erl index 7de78758b0..9f0bc0c811 100644 --- a/lib/stdlib/src/shell.erl +++ b/lib/stdlib/src/shell.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2022. All Rights Reserved. +%% Copyright Ericsson AB 1996-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -20,10 +20,13 @@ -module(shell). -export([start/0, start/1, start/2, server/1, server/2, history/1, results/1]). --export([whereis_evaluator/0, whereis_evaluator/1]). +-export([get_state/0, get_function/2]). -export([start_restricted/1, stop_restricted/0]). --export([local_allowed/3, non_local_allowed/3]). +-export([local_func/0, local_func/1, local_allowed/3, non_local_allowed/3]). -export([catch_exception/1, prompt_func/1, strings/1]). +-export([start_interactive/0, start_interactive/1]). +-export([read_and_add_records/5]). +-export([whereis/0]). -define(LINEMAX, 30). -define(CHAR_MAX, 60). @@ -36,7 +39,11 @@ -define(RECORDS, shell_records). -define(MAXSIZE_HEAPBINARY, 64). - +-record(shell_state,{ + bindings = [], + records = [], + functions = [] + }). %% When used as the fallback restricted shell callback module... local_allowed(q,[],State) -> {true,State}; @@ -48,6 +55,24 @@ non_local_allowed({init,stop},[],State) -> non_local_allowed(_,_,State) -> {false,State}. +-spec start_interactive() -> ok | {error, already_started}. +start_interactive() -> + user_drv:start_shell(). +-spec start_interactive(noshell | mfa()) -> + ok | {error, already_started}; + ({remote, string()}) -> + ok | {error, already_started | noconnection}; + ({node(), mfa()} | {remote, string(), mfa()}) -> + ok | {error, already_started | noconnection | badfile | nofile | on_load_failure}. +start_interactive({Node, {M, F, A}}) -> + user_drv:start_shell(#{ initial_shell => {Node, M, F ,A} }); +start_interactive(InitialShell) -> + user_drv:start_shell(#{ initial_shell => InitialShell }). + +-spec whereis() -> pid() | undefined. +whereis() -> + group:whereis_shell(). + -spec start() -> pid(). start() -> @@ -60,62 +85,16 @@ start(NoCtrlG) -> start(NoCtrlG, StartSync) -> _ = code:ensure_loaded(user_default), - spawn(fun() -> server(NoCtrlG, StartSync) end). - -%% Find the pid of the current evaluator process. --spec whereis_evaluator() -> 'undefined' | pid(). - -whereis_evaluator() -> - %% locate top group leader, always registered as user - %% can be implemented by group (normally) or user - %% (if oldshell or noshell) - case whereis(user) of - undefined -> - undefined; - User -> - %% get user_drv pid from group, or shell pid from user - case group:interfaces(User) of - [] -> % old- or noshell - case user:interfaces(User) of - [] -> - undefined; - [{shell,Shell}] -> - whereis_evaluator(Shell) - end; - [{user_drv,UserDrv}] -> - %% get current group pid from user_drv - case user_drv:interfaces(UserDrv) of - [] -> - undefined; - [{current_group,Group}] -> - %% get shell pid from group - GrIfs = group:interfaces(Group), - case lists:keyfind(shell, 1, GrIfs) of - {shell, Shell} -> - whereis_evaluator(Shell); - false -> - undefined - end - end - end - end. - --spec whereis_evaluator(pid()) -> 'undefined' | pid(). - -whereis_evaluator(Shell) -> - case process_info(Shell, dictionary) of - {dictionary,Dict} -> - case lists:keyfind(evaluator, 1, Dict) of - {_, Eval} when is_pid(Eval) -> - Eval; - _ -> - undefined - end; - _ -> - undefined - end. - -%% Call this function to start a user restricted shell + Ancestors = [self() | case get('$ancestors') of + undefined -> []; + Anc -> Anc + end], + spawn(fun() -> + put('$ancestors', Ancestors), + server(NoCtrlG, StartSync) + end). + +%% Call this function to start a user restricted shell %% from a normal shell session. -spec start_restricted(Module) -> {'error', Reason} when Module :: module(), @@ -123,16 +102,16 @@ whereis_evaluator(Shell) -> start_restricted(RShMod) when is_atom(RShMod) -> case code:ensure_loaded(RShMod) of - {module,RShMod} -> - application:set_env(stdlib, restricted_shell, RShMod), + {module,RShMod} -> + application:set_env(stdlib, restricted_shell, RShMod), exit(restricted_shell_started); - {error,What} = Error -> - error_logger:error_report( - lists:flatten( - io_lib:fwrite( - "Restricted shell module ~w not found: ~tp\n", - [RShMod,What]))), - Error + {error,What} = Error -> + error_logger:error_report( + lists:flatten( + io_lib:fwrite( + "Restricted shell module ~w not found: ~tp\n", + [RShMod,What]))), + Error end. -spec stop_restricted() -> no_return(). @@ -152,27 +131,27 @@ server(NoCtrlG, StartSync) -> %%% We subscribe with init to get a notification of when. %%% In older releases we didn't syncronize the shell with init, but let it -%%% start in parallell with other system processes. This was bad since +%%% start in parallell with other system processes. This was bad since %%% accessing the shell too early could interfere with the boot procedure. %%% Still, by means of a flag, we make it possible to start the shell the -%%% old way (for backwards compatibility reasons). This should however not +%%% old way (for backwards compatibility reasons). This should however not %%% be used unless for very special reasons necessary. -spec server(boolean()) -> 'terminated'. server(StartSync) -> case init:get_argument(async_shell_start) of - {ok,_} -> - ok; % no sync with init - _ when not StartSync -> - ok; - _ -> - case init:notify_when_started(self()) of - started -> - ok; - _ -> - init:wait_until_started() - end + {ok,_} -> + ok; % no sync with init + _ when not StartSync -> + ok; + _ -> + case init:notify_when_started(self()) of + started -> + ok; + _ -> + init:wait_until_started() + end end, %% Our spawner has fixed the process groups. Bs = erl_eval:new_bindings(), @@ -183,57 +162,71 @@ server(StartSync) -> RT = ets:new(?RECORDS, [public,ordered_set]), _ = initiate_records(Bs, RT), process_flag(trap_exit, true), + %% Store function definitions and types in an ets table. + FT = ets:new(user_functions, [public,ordered_set]), %% Check if we're in user restricted mode. - RShErr = - case application:get_env(stdlib, restricted_shell) of - {ok,RShMod} when is_atom(RShMod) -> - io:fwrite(<<"Restricted ">>, []), - case code:ensure_loaded(RShMod) of - {module,RShMod} -> - undefined; - {error,What} -> - {RShMod,What} - end; + RShErr = + case application:get_env(stdlib, restricted_shell) of + {ok,RShMod} when is_atom(RShMod) -> + io:fwrite(<<"Restricted ">>, []), + case code:ensure_loaded(RShMod) of + {module,RShMod} -> + undefined; + {error,What} -> + {RShMod,What} + end; {ok, Term} -> {Term,not_an_atom}; - undefined -> - undefined - end, - - case get(no_control_g) of - true -> - io:fwrite(<<"Eshell V~s\n">>, [erlang:system_info(version)]); - _undefined_or_false -> - io:fwrite(<<"Eshell V~s (abort with ^G)\n">>, - [erlang:system_info(version)]) + undefined -> + undefined + end, + + JCL = + case get(no_control_g) of + true -> " (type help(). for help)"; + _ -> " (press Ctrl+G to abort, type help(). for help)" + end, + DefaultSessionSlogan = + io_lib:format(<<"Eshell V~s">>, [erlang:system_info(version)]), + SessionSlogan = + case application:get_env(stdlib, shell_session_slogan, DefaultSessionSlogan) of + SloganFun when is_function(SloganFun, 0) -> + SloganFun(); + Slogan -> + Slogan + end, + try + io:fwrite("~ts~ts\n",[unicode:characters_to_list(SessionSlogan),JCL]) + catch _:_ -> + io:fwrite("Warning! The slogan \"~p\" could not be printed.\n",[SessionSlogan]) end, erase(no_control_g), case RShErr of - undefined -> + undefined -> ok; - {RShMod2,What2} -> + {RShMod2,What2} -> io:fwrite( - ("Warning! Restricted shell module ~w not found: ~tp.\n" - "Only the commands q() and init:stop() will be allowed!\n"), + ("Warning! Restricted shell module ~w not found: ~tp.\n" + "Only the commands q() and init:stop() will be allowed!\n"), [RShMod2,What2]), application:set_env(stdlib, restricted_shell, ?MODULE) end, {History,Results} = check_and_get_history_and_results(), - server_loop(0, start_eval(Bs, RT, []), Bs, RT, [], History, Results). + server_loop(0, start_eval(Bs, RT, FT, []), Bs, RT, FT, [], History, Results). -server_loop(N0, Eval_0, Bs00, RT, Ds00, History0, Results0) -> +server_loop(N0, Eval_0, Bs00, RT, FT, Ds00, History0, Results0) -> N = N0 + 1, - {Eval_1,Bs0,Ds0,Prompt} = prompt(N, Eval_0, Bs00, RT, Ds00), - {Res,Eval0} = get_command(Prompt, Eval_1, Bs0, RT, Ds0), + {Eval_1,Bs0,Ds0,Prompt} = prompt(N, Eval_0, Bs00, RT, FT, Ds00), + {Res,Eval0} = get_command(Prompt, Eval_1, Bs0, RT, FT, Ds0), - case Res of - {ok,Es0} -> + case Res of + {ok,Es0} -> case expand_hist(Es0, N) of {ok,Es} -> - {V,Eval,Bs,Ds} = shell_cmd(Es, Eval0, Bs0, RT, Ds0, cmd), + {V,Eval,Bs,Ds} = shell_cmd(Es, Eval0, Bs0, RT, FT, Ds0, cmd), {History,Results} = check_and_get_history_and_results(), add_cmd(N, Es, V), HB1 = del_cmd(command, N - History, N - History0, false), @@ -247,42 +240,83 @@ server_loop(N0, Eval_0, Bs00, RT, Ds00, History0, Results0) -> true -> ok end, - server_loop(N, Eval, Bs, RT, Ds, History, Results); + server_loop(N, Eval, Bs, RT, FT, Ds, History, Results); {error,E} -> fwrite_severity(benign, <<"~ts">>, [E]), - server_loop(N0, Eval0, Bs0, RT, Ds0, History0, Results0) + server_loop(N0, Eval0, Bs0, RT, FT, Ds0, History0, Results0) end; - {error,{Location,Mod,What}} -> + {error,{Location,Mod,What}} -> fwrite_severity(benign, <<"~s: ~ts">>, [pos(Location), Mod:format_error(What)]), - server_loop(N0, Eval0, Bs0, RT, Ds0, History0, Results0); - {error,terminated} -> %Io process terminated - exit(Eval0, kill), - terminated; - {error,interrupted} -> %Io process interrupted us - exit(Eval0, kill), - {_,Eval,_,_} = shell_rep(Eval0, Bs0, RT, Ds0), - server_loop(N0, Eval, Bs0, RT, Ds0, History0, Results0); - {error,tokens} -> %Most probably character > 255 - fwrite_severity(benign, <<"~w: Invalid tokens.">>, + server_loop(N0, Eval0, Bs0, RT, FT, Ds0, History0, Results0); + {error,terminated} -> %Io process terminated + exit(Eval0, kill), + terminated; + {error,interrupted} -> %Io process interrupted us + exit(Eval0, kill), + {_,Eval,_,_} = shell_rep(Eval0, Bs0, RT, FT, Ds0), + server_loop(N0, Eval, Bs0, RT, FT, Ds0, History0, Results0); + {error,tokens} -> %Most probably character > 255 + fwrite_severity(benign, <<"~w: Invalid tokens.">>, [N]), - server_loop(N0, Eval0, Bs0, RT, Ds0, History0, Results0); - eof -> + server_loop(N0, Eval0, Bs0, RT, FT, Ds0, History0, Results0); + eof -> fwrite_severity(fatal, <<"Terminating erlang (~w)">>, [node()]), - halt() + halt() end. -get_command(Prompt, Eval, Bs, RT, Ds) -> +get_command(Prompt, Eval, Bs, RT, FT, Ds) -> + Ancestors = [self() | get('$ancestors')], ResWordFun = fun erl_scan:reserved_word/1, Parse = fun() -> + put('$ancestors', Ancestors), exit( case io:scan_erl_exprs(group_leader(), Prompt, {1,1}, [text,{reserved_word_fun,ResWordFun}]) of {ok,Toks,_EndPos} -> - erl_eval:extended_parse_exprs(Toks); + %% NOTE: we can handle function definitions, records and soon type declarations + %% but this cannot be handled by the function which only expects erl_parse:abstract_expressions() + %% for now just pattern match against those types and pass the string to shell local func. + case Toks of + [{'-', _}, {atom, _, Atom}|_] -> + SpecialCase = fun(LocalFunc) -> + FakeLine = begin + case erl_parse:parse_form(Toks) of + {ok, Def} -> lists:flatten(erl_pp:form(Def)); + E -> + exit(E) + end + end, + {done, {ok, FakeResult, _}, _} = erl_scan:tokens( + [], atom_to_list(LocalFunc) ++ "(\""++FakeLine++"\").\n", + {1,1}, [text,{reserved_word_fun,fun erl_scan:reserved_word/1}]), + erl_eval:extended_parse_exprs(FakeResult) + end, + case Atom of + record -> SpecialCase(rd); + spec -> SpecialCase(ft); + type -> SpecialCase(td) + end; + [{atom, _, FunName}, {'(', _}|_] -> + case erl_parse:parse_form(Toks) of + {ok, FunDef} -> + case {edlin_expand:shell_default_or_bif(atom_to_list(FunName)), shell:local_func(FunName)} of + {"user_defined", false} -> + FakeLine =reconstruct(FunDef, FunName), + {done, {ok, FakeResult, _}, _} = erl_scan:tokens( + [], "fd("++ atom_to_list(FunName) ++ ", " ++ FakeLine ++ ").\n", + {1,1}, [text,{reserved_word_fun,fun erl_scan:reserved_word/1}]), + erl_eval:extended_parse_exprs(FakeResult); + _ -> erl_eval:extended_parse_exprs(Toks) + end; + _ -> erl_eval:extended_parse_exprs(Toks) + end; + _ -> + erl_eval:extended_parse_exprs(Toks) + end; {eof,_EndPos} -> eof; {error,ErrorInfo,_EndPos} -> @@ -297,30 +331,62 @@ get_command(Prompt, Eval, Bs, RT, Ds) -> Else -> Else end - ) + ) end, Pid = spawn_link(Parse), - get_command1(Pid, Eval, Bs, RT, Ds). - -get_command1(Pid, Eval, Bs, RT, Ds) -> + get_command1(Pid, Eval, Bs, RT, FT, Ds). + +reconstruct(Fun, Name) -> + lists:flatten(erl_pp:expr(reconstruct1(Fun, Name))). +reconstruct1({function, Anno, Name, Arity, Clauses}, Name) -> + {named_fun, Anno, 'RecursiveFuncVar', reconstruct1(Clauses, Name, Arity)}. +reconstruct1([{call, Anno, {atom, Anno1, Name}, Args}|Body], Name, Arity) when length(Args) =:= Arity -> + [{call, Anno, {var, Anno1, 'RecursiveFuncVar'}, reconstruct1(Args, Name, Arity)}| reconstruct1(Body, Name, Arity)]; +reconstruct1([{call, Anno, {atom, Anno1, Name}, Args}|Body], Name, Arity) -> % arity not the same + [{call, Anno, {remote, Anno1, {atom, Anno1, shell_default}, {atom, Anno1, Name}}, reconstruct1(Args, Name, Arity)}| + reconstruct1(Body, Name, Arity)]; +reconstruct1([{call, Anno, {atom, Anno1, Fun}, Args}|Body], Name, Arity) -> % Name not the same + case {edlin_expand:shell_default_or_bif(atom_to_list(Fun)), shell:local_func(Fun)} of + {"user_defined", false} -> + [{call, Anno, {remote, Anno1, {atom, Anno1, shell_default}, {atom, Anno1, Fun}}, reconstruct1(Args, Name, Arity)}| + reconstruct1(Body, Name, Arity)]; + {"shell_default", false} -> + [{call, Anno, {remote, Anno1, {atom, Anno1, shell_default}, {atom, Anno1, Fun}}, reconstruct1(Args, Name, Arity)}| reconstruct1(Body, Name, Arity)]; + {"erlang", false} -> + [{call, Anno, {remote, Anno1, {atom, Anno1, erlang}, {atom, Anno1, Fun}}, reconstruct1(Args, Name, Arity)}| reconstruct1(Body, Name, Arity)]; + {_, true} -> + [{call, Anno, {atom, Anno1, Fun}, reconstruct1(Args, Name, Arity)}| reconstruct1(Body, Name, Arity)] + end; +reconstruct1([E|Body], Name, Arity) when is_tuple(E) -> + [list_to_tuple(reconstruct1(tuple_to_list(E), Name, Arity))|reconstruct1(Body, Name, Arity)]; +reconstruct1([E|Body], Name, Arity) when is_list(E) -> + [reconstruct1(E, Name, Arity)|reconstruct1(Body, Name, Arity)]; +reconstruct1([E|Body], Name, Arity) -> + [E|reconstruct1(Body, Name, Arity)]; +reconstruct1([], _, _) -> []. + +get_command1(Pid, Eval, Bs, RT, FT, Ds) -> receive - {'EXIT', Pid, Res} -> - {Res, Eval}; - {'EXIT', Eval, {Reason,Stacktrace}} -> + {shell_state, From} -> + From ! {shell_state, Bs, RT, FT}, + get_command1(Pid, Eval, Bs, RT, FT, Ds); + {'EXIT', Pid, Res} -> + {Res, Eval}; + {'EXIT', Eval, {Reason,Stacktrace}} -> report_exception(error, {Reason,Stacktrace}, RT), - get_command1(Pid, start_eval(Bs, RT, Ds), Bs, RT, Ds); - {'EXIT', Eval, Reason} -> + get_command1(Pid, start_eval(Bs, RT, FT, Ds), Bs, RT, FT, Ds); + {'EXIT', Eval, Reason} -> report_exception(error, {Reason,[]}, RT), - get_command1(Pid, start_eval(Bs, RT, Ds), Bs, RT, Ds) + get_command1(Pid, start_eval(Bs, RT, FT, Ds), Bs, RT, FT, Ds) end. -prompt(N, Eval0, Bs0, RT, Ds0) -> +prompt(N, Eval0, Bs0, RT, FT, Ds0) -> case get_prompt_func() of {M,F} -> A = erl_anno:new(1), L = {cons,A,{tuple,A,[{atom,A,history},{integer,A,N}]},{nil,A}}, C = {call,A,{remote,A,{atom,A,M},{atom,A,F}},[L]}, - {V,Eval,Bs,Ds} = shell_cmd([C], Eval0, Bs0, RT, Ds0, pmt), + {V,Eval,Bs,Ds} = shell_cmd([C], Eval0, Bs0, RT, FT, Ds0, pmt), {Eval,Bs,Ds,case V of {pmt,Val} -> Val; @@ -333,15 +399,13 @@ prompt(N, Eval0, Bs0, RT, Ds0) -> end. get_prompt_func() -> - case application:get_env(stdlib, shell_prompt_func) of - {ok,{M,F}=PromptFunc} when is_atom(M), is_atom(F) -> + case application:get_env(stdlib, shell_prompt_func, default) of + {M,F}=PromptFunc when is_atom(M), is_atom(F) -> PromptFunc; - {ok,default=Default} -> + default=Default -> Default; - {ok,Term} -> + Term -> bad_prompt_func(Term), - default; - undefined -> default end. @@ -352,8 +416,8 @@ default_prompt(N) -> %% Don't bother flattening the list irrespective of what the %% I/O-protocol states. case is_alive() of - true -> io_lib:format(<<"(~s)~w> ">>, [node(), N]); - false -> io_lib:format(<<"~w> ">>, [N]) + true -> io_lib:format(<<"(~s)~w> ">>, [node(), N]); + false -> io_lib:format(<<"~w> ">>, [N]) end. %% expand_hist(Expressions, CommandNumber) @@ -392,7 +456,7 @@ expand_expr({record_field,A,R,Name,F}, C) -> {record_field,A,expand_expr(R, C),Name,expand_expr(F, C)}; expand_expr({record,A,R,Name,Ups}, C) -> {record,A,expand_expr(R, C),Name,expand_fields(Ups, C)}; -expand_expr({record_field,A,R,F}, C) -> %This is really illegal! +expand_expr({record_field,A,R,F}, C) -> %This is really illegal! {record_field,A,expand_expr(R, C),expand_expr(F, C)}; expand_expr({block,A,Es}, C) -> {block,A,expand_exprs(Es, C)}; @@ -410,17 +474,17 @@ expand_expr({'receive',A,Cs,To,ToEs}, C) -> expand_expr({call,A,{atom,_,e},[N]}, C) -> case get_cmd(N, C) of {undefined,_,_} -> - no_command(N); - {[Ce],_V,_CommandN} -> - Ce; - {Ces,_V,_CommandN} when is_list(Ces) -> - {block,A,Ces} + no_command(N); + {[Ce],_V,_CommandN} -> + Ce; + {Ces,_V,_CommandN} when is_list(Ces) -> + {block,A,Ces} end; expand_expr({call,CA,{atom,VA,v},[N]}, C) -> case get_cmd(N, C) of - {_,undefined,_} -> - no_command(N); - {Ces,_V,CommandN} when is_list(Ces) -> + {undefined,_,_} -> + no_command(N); + {Ces,_V,CommandN} when is_list(Ces) -> {call,CA,{atom,VA,v},[{integer,VA,CommandN}]} end; expand_expr({call,A,F,Args}, C) -> @@ -444,7 +508,8 @@ expand_expr({clause,A,H,G,B}, C) -> {clause,A,H, G, expand_exprs(B, C)}; expand_expr({bin,A,Fs}, C) -> {bin,A,expand_bin_elements(Fs, C)}; -expand_expr(E, _C) -> % Constants. + %expand_expr({'-'}) +expand_expr(E, _C) -> % Constants. E. expand_cs([{clause,A,P,G,B}|Cs], C) -> @@ -488,9 +553,9 @@ getc(N) -> get_cmd(Num, C) -> case catch erl_eval:expr(Num, erl_eval:new_bindings()) of - {value,N,_} when N < 0 -> getc(C+N); - {value,N,_} -> getc(N); - _Other -> {undefined,undefined,undefined} + {value,N,_} when N < 0 -> getc(C+N); + {value,N,_} -> getc(N); + _Other -> {undefined,undefined,undefined} end. del_cmd(_Type, N, N0, HasBin) when N < N0 -> @@ -521,61 +586,81 @@ has_bin(T, I) -> has_bin(element(I, T)), has_bin(T, I - 1). +get_state() -> + whereis() ! {shell_state, self()}, + receive + {shell_state, Bs, RT, FT} -> + #shell_state{bindings = Bs, records = ets:tab2list(RT), functions = ets:tab2list(FT)} + end. + +get_function(Func, Arity) -> + {shell_state, _Bs, _RT, FT} = get_state(), + try + {value, {_, Fun}} = lists:keysearch({function, {shell_default,Func,Arity}}, 1, FT), + Fun + catch _:_ -> + undefined + end. + %% shell_cmd(Sequence, Evaluator, Bindings, RecordTable, Dictionary, What) %% shell_rep(Evaluator, Bindings, RecordTable, Dictionary) -> -%% {Value,Evaluator,Bindings,Dictionary} +%% {Value,Evaluator,Bindings,Dictionary} %% Send a command to the evaluator and wait for the reply. Start a new %% evaluator if necessary. %% What = pmt | cmd. When evaluating a prompt ('pmt') the evaluated value %% must not be displayed, and it has to be returned. -shell_cmd(Es, Eval, Bs, RT, Ds, W) -> +shell_cmd(Es, Eval, Bs, RT, FT, Ds, W) -> Eval ! {shell_cmd,self(),{eval,Es}, W}, - shell_rep(Eval, Bs, RT, Ds). + shell_rep(Eval, Bs, RT, FT, Ds). -shell_rep(Ev, Bs0, RT, Ds0) -> +shell_rep(Ev, Bs0, RT, FT, Ds0) -> receive - {shell_rep,Ev,{value,V,Bs,Ds}} -> - {V,Ev,Bs,Ds}; + {shell_rep,Ev,{value,V,Bs,Ds}} -> + {V,Ev,Bs,Ds}; {shell_rep,Ev,{command_error,{Location,M,Error}}} -> fwrite_severity(benign, <<"~s: ~ts">>, [pos(Location), M:format_error(Error)]), {{'EXIT',Error},Ev,Bs0,Ds0}; {shell_req,Ev,{get_cmd,N}} -> - Ev ! {shell_rep,self(),getc(N)}, - shell_rep(Ev, Bs0, RT, Ds0); - {shell_req,Ev,get_cmd} -> - Ev ! {shell_rep,self(),get()}, - shell_rep(Ev, Bs0, RT, Ds0); - {shell_req,Ev,exit} -> - Ev ! {shell_rep,self(),exit}, - exit(normal); - {shell_req,Ev,{update_dict,Ds}} -> % Update dictionary - Ev ! {shell_rep,self(),ok}, - shell_rep(Ev, Bs0, RT, Ds); + Ev ! {shell_rep,self(),getc(N)}, + shell_rep(Ev, Bs0, RT, FT, Ds0); + {shell_req,Ev,get_cmd} -> + Ev ! {shell_rep,self(),get()}, + shell_rep(Ev, Bs0, RT, FT, Ds0); + {shell_req,Ev,exit} -> + Ev ! {shell_rep,self(),exit}, + exit(normal); + {shell_req,Ev,{update_dict,Ds}} -> % Update dictionary + Ev ! {shell_rep,self(),ok}, + shell_rep(Ev, Bs0, RT, FT, Ds); + {shell_state, From} -> + From ! {shell_state, Bs0, RT, FT}, + shell_rep(Ev, Bs0, RT, FT, Ds0); {ev_exit,{Ev,Class,Reason0}} -> % It has exited unnaturally receive {'EXIT',Ev,normal} -> ok end, - report_exception(Class, Reason0, RT), + report_exception(Class, Reason0, RT), Reason = nocatch(Class, Reason0), - {{'EXIT',Reason},start_eval(Bs0, RT, Ds0), Bs0, Ds0}; + {{'EXIT',Reason},start_eval(Bs0, RT, FT, Ds0), Bs0, Ds0}; {ev_caught,{Ev,Class,Reason0}} -> % catch_exception is in effect - report_exception(Class, benign, Reason0, RT), + report_exception(Class, benign, Reason0, RT), Reason = nocatch(Class, Reason0), {{'EXIT',Reason},Ev,Bs0,Ds0}; - {'EXIT',_Id,interrupt} -> % Someone interrupted us - exit(Ev, kill), - shell_rep(Ev, Bs0, RT, Ds0); + {'EXIT',_Id,interrupt} -> % Someone interrupted us + exit(Ev, kill), + shell_rep(Ev, Bs0, RT, FT, Ds0); {'EXIT',Ev,{Reason,Stacktrace}} -> report_exception(exit, {Reason,Stacktrace}, RT), - {{'EXIT',Reason},start_eval(Bs0, RT, Ds0), Bs0, Ds0}; + {{'EXIT',Reason},start_eval(Bs0, RT, FT, Ds0), Bs0, Ds0}; {'EXIT',Ev,Reason} -> report_exception(exit, {Reason,[]}, RT), - {{'EXIT',Reason},start_eval(Bs0, RT, Ds0), Bs0, Ds0}; - {'EXIT',_Id,R} -> - exit(Ev, R), - exit(R); - _Other -> % Ignore everything else - shell_rep(Ev, Bs0, RT, Ds0) + {{'EXIT',Reason},start_eval(Bs0, RT, FT, Ds0), Bs0, Ds0}; + {'EXIT',_Id,R} -> + exit(Ev, R), + exit(R); + _Other -> % Ignore everything else + io:format("Throwing ~p~n", [_Other]), + shell_rep(Ev, Bs0, RT, FT, Ds0) end. nocatch(throw, {Term,Stack}) -> @@ -599,9 +684,13 @@ report_exception(Class, Severity, {Reason,Stacktrace}, RT) -> {put_chars, unicode, Str}, nl]). -start_eval(Bs, RT, Ds) -> +start_eval(Bs, RT, FT, Ds) -> Self = self(), - Eval = spawn_link(fun() -> evaluator(Self, Bs, RT, Ds) end), + Ancestors = [self() | get('$ancestors')], + Eval = spawn_link(fun() -> + put('$ancestors', Ancestors), + evaluator(Self, Bs, RT, FT, Ds) + end), put(evaluator, Eval), Eval. @@ -609,45 +698,45 @@ start_eval(Bs, RT, Ds) -> %% Evaluate expressions from the shell. Use the "old" variable bindings %% and dictionary. -evaluator(Shell, Bs, RT, Ds) -> +evaluator(Shell, Bs, RT, FT, Ds) -> init_dict(Ds), case application:get_env(stdlib, restricted_shell) of - undefined -> - eval_loop(Shell, Bs, RT); - {ok,RShMod} -> - case get(restricted_shell_state) of - undefined -> put(restricted_shell_state, []); - _ -> ok - end, - put(restricted_expr_state, []), - restricted_eval_loop(Shell, Bs, RT, RShMod) + undefined -> + eval_loop(Shell, Bs, RT, FT); + {ok,RShMod} -> + case get(restricted_shell_state) of + undefined -> put(restricted_shell_state, []); + _ -> ok + end, + put(restricted_expr_state, []), + restricted_eval_loop(Shell, Bs, RT, FT, RShMod) end. -eval_loop(Shell, Bs0, RT) -> +eval_loop(Shell, Bs0, RT, FT) -> receive - {shell_cmd,Shell,{eval,Es},W} -> - Ef = {value, + {shell_cmd,Shell,{eval,Es},W} -> + Ef = {value, fun(MForFun, As) -> apply_fun(MForFun, As, Shell) end}, - Lf = local_func_handler(Shell, RT, Ef), + Lf = local_func_handler(Shell, RT, FT, Ef), Bs = eval_exprs(Es, Shell, Bs0, RT, Lf, Ef, W), - eval_loop(Shell, Bs, RT) + eval_loop(Shell, Bs, RT, FT) end. -restricted_eval_loop(Shell, Bs0, RT, RShMod) -> +restricted_eval_loop(Shell, Bs0, RT, FT, RShMod) -> receive - {shell_cmd,Shell,{eval,Es}, W} -> - {LFH,NLFH} = restrict_handlers(RShMod, Shell, RT), + {shell_cmd,Shell,{eval,Es}, W} -> + {LFH,NLFH} = restrict_handlers(RShMod, Shell, RT, FT), put(restricted_expr_state, []), Bs = eval_exprs(Es, Shell, Bs0, RT, {eval,LFH}, {value,NLFH}, W), - restricted_eval_loop(Shell, Bs, RT, RShMod) + restricted_eval_loop(Shell, Bs, RT, FT, RShMod) end. eval_exprs(Es, Shell, Bs0, RT, Lf, Ef, W) -> - try + try {R,Bs2} = exprs(Es, Bs0, RT, Lf, Ef, W), Shell ! {shell_rep,self(),R}, Bs2 - catch + catch exit:normal -> exit(normal); Class:Reason:Stacktrace -> @@ -672,8 +761,8 @@ do_catch(exit, restricted_shell_stopped) -> do_catch(exit, restricted_shell_started) -> false; do_catch(_Class, _Reason) -> - case application:get_env(stdlib, shell_catch_exception) of - {ok, true} -> + case application:get_env(stdlib, shell_catch_exception, false) of + true -> true; _ -> false @@ -704,14 +793,14 @@ exprs([E0|Es], Bs1, RT, Lf, Ef, Bs0, W) -> W =:= pmt -> {W,V0}; true -> case result_will_be_saved() of - true -> V0; - false -> - erlang:garbage_collect(), - ignored - end + true -> V0; + false -> + erlang:garbage_collect(), + ignored + end end, {{value,V,Bs,get()},Bs}; - true -> + true -> exprs(Es, Bs, RT, Lf, Ef, Bs0, W) end; {error,Error} -> @@ -734,7 +823,7 @@ used_record_defs(E, RT) -> %% Be careful to return a list where used records come before %% records that use them. The linter wants them ordered that way. UR = case used_records(E, [], RT, []) of - [] -> + [] -> []; L0 -> L1 = lists:zip(L0, lists:seq(1, length(L0))), @@ -782,7 +871,7 @@ used_records({call,_,{atom,_,record_info},[A,{atom,_,Name}]}) -> used_records({call,A,{tuple,_,[M,F]},As}) -> used_records({call,A,{remote,A,M,F},As}); used_records({type,_,record,[{atom,_,Name}|Fs]}) -> - {name, Name, Fs}; + {name, Name, Fs}; used_records(T) when is_tuple(T) -> {expr, tuple_to_list(T)}; used_records(E) -> @@ -801,63 +890,63 @@ severity_tag(fatal) -> <<"*** ">>; severity_tag(serious) -> <<"** ">>; severity_tag(benign) -> <<"* ">>. -restrict_handlers(RShMod, Shell, RT) -> - { fun(F,As,Binds) -> - local_allowed(F, As, RShMod, Binds, Shell, RT) +restrict_handlers(RShMod, Shell, RT, FT) -> + { fun(F,As,Binds) -> + local_allowed(F, As, RShMod, Binds, Shell, RT, FT) end, - fun(MF,As) -> - non_local_allowed(MF, As, RShMod, Shell) + fun(MF,As) -> + non_local_allowed(MF, As, RShMod, Shell) end }. -define(BAD_RETURN(M, F, V), try erlang:error(reason) - catch _:_:S -> erlang:raise(exit, {restricted_shell_bad_return,V}, + catch _:_:S -> erlang:raise(exit, {restricted_shell_bad_return,V}, [{M,F,3} | S]) end). -local_allowed(F, As, RShMod, Bs, Shell, RT) when is_atom(F) -> - {LFH,NLFH} = restrict_handlers(RShMod, Shell, RT), - case not_restricted(F, As) of % Not restricted is the same as builtin. - % variable and record manipulations local - % to the shell process. Those are never - % restricted. - true -> - local_func(F, As, Bs, Shell, RT, {eval,LFH}, {value,NLFH}); - false -> +local_allowed(F, As, RShMod, Bs, Shell, RT, FT) when is_atom(F) -> + {LFH,NLFH} = restrict_handlers(RShMod, Shell, RT, FT), + case not_restricted(F, As) of % Not restricted is the same as builtin. + % variable and record manipulations local + % to the shell process. Those are never + % restricted. + true -> + local_func(F, As, Bs, Shell, RT, FT, {eval,LFH}, {value,NLFH}); + false -> {AsEv,Bs1} = expr_list(As, Bs, {eval,LFH}, {value,NLFH}), - case RShMod:local_allowed(F, AsEv, {get(restricted_shell_state), - get(restricted_expr_state)}) of - {Result,{RShShSt,RShExprSt}} -> - put(restricted_shell_state, RShShSt), - put(restricted_expr_state, RShExprSt), - if not Result -> - shell_req(Shell, {update_dict,get()}), - exit({restricted_shell_disallowed,{F,AsEv}}); - true -> % This is never a builtin, - % those are handled above. - non_builtin_local_func(F,AsEv,Bs1) - end; - Unexpected -> % The user supplied non conforming module + case RShMod:local_allowed(F, AsEv, {get(restricted_shell_state), + get(restricted_expr_state)}) of + {Result,{RShShSt,RShExprSt}} -> + put(restricted_shell_state, RShShSt), + put(restricted_expr_state, RShExprSt), + if not Result -> + shell_req(Shell, {update_dict,get()}), + exit({restricted_shell_disallowed,{F,AsEv}}); + true -> % This is never a builtin, + % those are handled above. + non_builtin_local_func(F,AsEv,Bs1, FT) + end; + Unexpected -> % The user supplied non conforming module ?BAD_RETURN(RShMod, local_allowed, Unexpected) - end + end end. non_local_allowed(MForFun, As, RShMod, Shell) -> case RShMod:non_local_allowed(MForFun, As, {get(restricted_shell_state), - get(restricted_expr_state)}) of - {Result,{RShShSt,RShExprSt}} -> - put(restricted_shell_state, RShShSt), - put(restricted_expr_state, RShExprSt), - case Result of - false -> - shell_req(Shell, {update_dict,get()}), - exit({restricted_shell_disallowed,{MForFun,As}}); + get(restricted_expr_state)}) of + {Result,{RShShSt,RShExprSt}} -> + put(restricted_shell_state, RShShSt), + put(restricted_expr_state, RShExprSt), + case Result of + false -> + shell_req(Shell, {update_dict,get()}), + exit({restricted_shell_disallowed,{MForFun,As}}); {redirect, NewMForFun, NewAs} -> apply_fun(NewMForFun, NewAs, Shell); - _ -> - apply_fun(MForFun, As, Shell) - end; - Unexpected -> % The user supplied non conforming module + _ -> + apply_fun(MForFun, As, Shell) + end; + Unexpected -> % The user supplied non conforming module ?BAD_RETURN(RShMod, non_local_allowed, Unexpected) end. @@ -880,6 +969,16 @@ not_restricted(catch_exception, [_]) -> true; not_restricted(exit, []) -> true; +not_restricted(fl, []) -> + true; +not_restricted(fd, [_]) -> + true; +not_restricted(ft, [_]) -> + true; +not_restricted(td, [_]) -> + true; +not_restricted(rd, [_]) -> + true; not_restricted(rd, [_,_]) -> true; not_restricted(rf, []) -> @@ -902,7 +1001,7 @@ not_restricted(_, _) -> false. %% When erlang:garbage_collect() is called from the shell, -%% the shell process process that spawned the evaluating +%% the shell process process that spawned the evaluating %% process is garbage collected as well. %% To garbage collect the evaluating process only the command %% garbage_collect(self()). can be used. @@ -940,7 +1039,7 @@ expand_records(UsedRecords, E0) -> prep_rec({value,_CommandN,_V}=Value) -> %% erl_expand_records cannot handle the history expansion {value,_,_}. {atom,Value,ok}; -prep_rec({atom,{value,_CommandN,_V}=Value,ok}) -> +prep_rec({atom,{value,_CommandN,_V}=Value,ok}) -> %% Undo the effect of the previous clause... Value; prep_rec(T) when is_tuple(T) -> list_to_tuple(prep_rec(tuple_to_list(T))); @@ -952,39 +1051,101 @@ init_dict([{K,V}|Ds]) -> init_dict(Ds); init_dict([]) -> true. -%% local_func(Function, Args, Bindings, Shell, RecordTable, +%% local_func(Function, Args, Bindings, Shell, RecordTable, %% LocalFuncHandler, ExternalFuncHandler) -> {value,Val,Bs} %% Evaluate local functions, including shell commands. %% -%% Note that the predicate not_restricted/2 has to correspond to what's -%% handled internally - it should return 'true' for all local functions -%% handled in this module (i.e. those that are not eventually handled by +%% Note that the predicate not_restricted/2 has to correspond to what's +%% handled internally - it should return 'true' for all local functions +%% handled in this module (i.e. those that are not eventually handled by %% non_builtin_local_func/3 (user_default/shell_default). - -local_func(v, [{integer,_,V}], Bs, Shell, _RT, _Lf, _Ef) -> +local_func() -> [v,h,b,f,fl,rd,rf,rl,rp,rr,history,results,catch_exception]. +local_func(Func) -> + lists:member(Func, local_func()). +local_func(v, [{integer,_,V}], Bs, Shell, _RT, _FT, _Lf, _Ef) -> %% This command is validated and expanded prior. {_Ces,Value,_N} = shell_req(Shell, {get_cmd, V}), {value,Value,Bs}; -local_func(h, [], Bs, Shell, RT, _Lf, _Ef) -> +local_func(h, [], Bs, Shell, RT, _FT, _Lf, _Ef) -> Cs = shell_req(Shell, get_cmd), Cs1 = lists:filter(fun({{command, _},_}) -> true; - ({{result, _},_}) -> true; - (_) -> false - end, - Cs), + ({{result, _},_}) -> true; + (_) -> false + end, + Cs), Cs2 = lists:map(fun({{T, N}, V}) -> {{N, T}, V} end, - Cs1), + Cs1), Cs3 = lists:keysort(1, Cs2), {value,list_commands(Cs3, RT),Bs}; -local_func(b, [], Bs, _Shell, RT, _Lf, _Ef) -> +local_func(b, [], Bs, _Shell, RT, _FT, _Lf, _Ef) -> {value,list_bindings(erl_eval:bindings(Bs), RT),Bs}; -local_func(f, [], _Bs, _Shell, _RT, _Lf, _Ef) -> +local_func(f, [], _Bs, _Shell, _RT, _FT, _Lf, _Ef) -> {value,ok,erl_eval:new_bindings()}; -local_func(f, [{var,_,Name}], Bs, _Shell, _RT, _Lf, _Ef) -> +local_func(f, [{var,_,Name}], Bs, _Shell, _RT, _FT, _Lf, _Ef) -> {value,ok,erl_eval:del_binding(Name, Bs)}; -local_func(f, [_Other], _Bs, _Shell, _RT, _Lf, _Ef) -> +local_func(f, [_Other], _Bs, _Shell, _RT, _FT, _Lf, _Ef) -> erlang:raise(error, function_clause, [{shell,f,1}]); -local_func(rd, [{atom,_,RecName0},RecDef0], Bs, _Shell, RT, _Lf, _Ef) -> +local_func(fl, [], Bs, _Shell, _RT, FT, _Lf, _Ef) -> + {value, ets:tab2list(FT), Bs}; +local_func(fd, [{atom,_,FunName}, FunExpr], Bs, _Shell, _RT, FT, _Lf, _Ef) -> + {value, Fun, []} = erl_eval:expr(FunExpr, []), + {arity, Arity} = erlang:fun_info(Fun, arity), + ets:insert(FT, [{{function, {shell_default, FunName, Arity}}, Fun}]), + {value, ok, Bs}; +local_func(fd, [_], _Bs, _Shell, _RT, _FT, _Lf, _Ef) -> + erlang:raise(error, function_clause, [{shell, fd, 1}]); +local_func(ft, [{string, _, TypeDef}], Bs, _Shell, _RT, FT, _Lf, _Ef) -> + case erl_scan:tokens([], TypeDef, {1,1}, [text,{reserved_word_fun,fun erl_scan:reserved_word/1}]) of + {done, {ok, Toks, _}, _} -> + case erl_parse:parse_form(Toks) of + {ok, {attribute,_,spec,{{FunName, Arity},_}}=AttrForm} -> + ets:insert(FT, [{{function_type, {shell_default, FunName, Arity}}, AttrForm}]), + {value, ok, Bs}; + {error,{_Location,M,ErrDesc}} -> + ErrStr = io_lib:fwrite(<<"~ts">>, [M:format_error(ErrDesc)]), + exit(lists:flatten(ErrStr)) + end; + {done, {error,{_Location, M, ErrDesc}, _}, _} -> + ErrStr = io_lib:fwrite(<<"~ts">>, [M:format_error(ErrDesc)]), + exit(lists:flatten(ErrStr)) + end; +local_func(ft, [_], _Bs, _Shell, _RT, _FT, _Lf, _Ef) -> + erlang:raise(error, function_clause, [{shell, ft, 1}]); +local_func(td, [{string, _, TypeDef}], Bs, _Shell, _RT, FT, _Lf, _Ef) -> + case erl_scan:tokens([], TypeDef, {1,1}, [text,{reserved_word_fun,fun erl_scan:reserved_word/1}]) of + {done, {ok, Toks, _}, _} -> + case erl_parse:parse_form(Toks) of + {ok, {attribute,_,type,{TypeName, _, _}}=AttrForm} -> + ets:insert(FT, [{{type, TypeName}, AttrForm}]), + {value, ok, Bs}; + {error,{_Location,M,ErrDesc}} -> + ErrStr = io_lib:fwrite(<<"~ts">>, [M:format_error(ErrDesc)]), + exit(lists:flatten(ErrStr)) + end; + {done, {error,{_Location, M, ErrDesc}, _}, _} -> + ErrStr = io_lib:fwrite(<<"~ts">>, [M:format_error(ErrDesc)]), + exit(lists:flatten(ErrStr)) + end; +local_func(td, [_], _Bs, _Shell, _RT, _FT, _Lf, _Ef) -> + erlang:raise(error, function_clause, [{shell, td, 1}]); +local_func(rd, [{string, _, TypeDef}], Bs, _Shell, RT, _FT, _Lf, _Ef) -> + case erl_scan:tokens([], TypeDef, {1,1}, [text,{reserved_word_fun,fun erl_scan:reserved_word/1}]) of + {done, {ok, Toks, _}, _} -> + case erl_parse:parse_form(Toks) of + {ok,{attribute,_,_,_}=AttrForm} -> + [_] = add_records([AttrForm], Bs, RT), + {value,ok,Bs}; + {error,{_Location,M,ErrDesc}} -> + ErrStr = io_lib:fwrite(<<"~ts">>, [M:format_error(ErrDesc)]), + exit(lists:flatten(ErrStr)) + end; + {done, {error,{_Location, M, ErrDesc}, _}, _} -> + ErrStr = io_lib:fwrite(<<"~ts">>, [M:format_error(ErrDesc)]), + exit(lists:flatten(ErrStr)) + end; +local_func(rd, [_], _Bs, _Shell, _RT, _FT, _Lf, _Ef) -> + erlang:raise(error, function_clause, [{shell, rd, 1}]); +local_func(rd, [{atom,_,RecName0},RecDef0], Bs, _Shell, RT, _FT, _Lf, _Ef) -> RecDef = expand_value(RecDef0), RDs = lists:flatten(erl_pp:expr(RecDef)), RecName = io_lib:write_atom_as_latin1(RecName0), @@ -998,99 +1159,105 @@ local_func(rd, [{atom,_,RecName0},RecDef0], Bs, _Shell, RT, _Lf, _Ef) -> ErrStr = io_lib:fwrite(<<"~ts">>, [M:format_error(ErrDesc)]), exit(lists:flatten(ErrStr)) end; -local_func(rd, [_,_], _Bs, _Shell, _RT, _Lf, _Ef) -> +local_func(rd, [_,_], _Bs, _Shell, _RT, _FT, _Lf, _Ef) -> erlang:raise(error, function_clause, [{shell,rd,2}]); -local_func(rf, [], Bs, _Shell, RT, _Lf, _Ef) -> +local_func(rf, [], Bs, _Shell, RT, _FT, _Lf, _Ef) -> true = ets:delete_all_objects(RT), {value,initiate_records(Bs, RT),Bs}; -local_func(rf, [A], Bs0, _Shell, RT, Lf, Ef) -> +local_func(rf, [A], Bs0, _Shell, RT, _FT, Lf, Ef) -> {[Recs],Bs} = expr_list([A], Bs0, Lf, Ef), if '_' =:= Recs -> true = ets:delete_all_objects(RT); - true -> + true -> lists:foreach(fun(Name) -> true = ets:delete(RT, Name) end, listify(Recs)) end, {value,ok,Bs}; -local_func(rl, [], Bs, _Shell, RT, _Lf, _Ef) -> +local_func(rl, [], Bs, _Shell, RT, _FT, _Lf, _Ef) -> {value,list_records(ets:tab2list(RT)),Bs}; -local_func(rl, [A], Bs0, _Shell, RT, Lf, Ef) -> +local_func(rl, [A], Bs0, _Shell, RT, _FT, Lf, Ef) -> {[Recs],Bs} = expr_list([A], Bs0, Lf, Ef), {value,list_records(record_defs(RT, listify(Recs))),Bs}; -local_func(rp, [A], Bs0, _Shell, RT, Lf, Ef) -> +local_func(rp, [A], Bs0, _Shell, RT, _FT, Lf, Ef) -> {[V],Bs} = expr_list([A], Bs0, Lf, Ef), Cs = pp(V, _Column=1, _Depth=-1, RT), io:requests([{put_chars, unicode, Cs}, nl]), {value,ok,Bs}; -local_func(rr, [A], Bs0, _Shell, RT, Lf, Ef) -> +local_func(rr, [A], Bs0, _Shell, RT, _FT, Lf, Ef) -> {[File],Bs} = expr_list([A], Bs0, Lf, Ef), {value,read_and_add_records(File, '_', [], Bs, RT),Bs}; -local_func(rr, [_,_]=As0, Bs0, _Shell, RT, Lf, Ef) -> +local_func(rr, [_,_]=As0, Bs0, _Shell, RT, _FT, Lf, Ef) -> {[File,Sel],Bs} = expr_list(As0, Bs0, Lf, Ef), {value,read_and_add_records(File, Sel, [], Bs, RT),Bs}; -local_func(rr, [_,_,_]=As0, Bs0, _Shell, RT, Lf, Ef) -> +local_func(rr, [_,_,_]=As0, Bs0, _Shell, RT, _FT, Lf, Ef) -> {[File,Sel,Options],Bs} = expr_list(As0, Bs0, Lf, Ef), {value,read_and_add_records(File, Sel, Options, Bs, RT),Bs}; -local_func(history, [{integer,_,N}], Bs, _Shell, _RT, _Lf, _Ef) -> +local_func(history, [{integer,_,N}], Bs, _Shell, _RT, _FT, _Lf, _Ef) -> {value,history(N),Bs}; -local_func(history, [_Other], _Bs, _Shell, _RT, _Lf, _Ef) -> +local_func(history, [_Other], _Bs, _Shell, _RT, _FT, _Lf, _Ef) -> erlang:raise(error, function_clause, [{shell,history,1}]); -local_func(results, [{integer,_,N}], Bs, _Shell, _RT, _Lf, _Ef) -> +local_func(results, [{integer,_,N}], Bs, _Shell, _RT, _FT, _Lf, _Ef) -> {value,results(N),Bs}; -local_func(results, [_Other], _Bs, _Shell, _RT, _Lf, _Ef) -> +local_func(results, [_Other], _Bs, _Shell, _RT, _FT, _Lf, _Ef) -> erlang:raise(error, function_clause, [{shell,results,1}]); -local_func(catch_exception, [{atom,_,Bool}], Bs, _Shell, _RT, _Lf, _Ef) - when Bool; not Bool -> +local_func(catch_exception, [{atom,_,Bool}], Bs, _Shell, _RT, _FT, _Lf, _Ef) + when Bool; not Bool -> {value,catch_exception(Bool),Bs}; -local_func(catch_exception, [_Other], _Bs, _Shell, _RT, _Lf, _Ef) -> +local_func(catch_exception, [_Other], _Bs, _Shell, _RT, _FT, _Lf, _Ef) -> erlang:raise(error, function_clause, [{shell,catch_exception,1}]); -local_func(exit, [], _Bs, Shell, _RT, _Lf, _Ef) -> - shell_req(Shell, exit), %This terminates us +local_func(exit, [], _Bs, Shell, _RT, _FT, _Lf, _Ef) -> + shell_req(Shell, exit), %This terminates us exit(normal); -local_func(F, As0, Bs0, _Shell, _RT, Lf, Ef) when is_atom(F) -> +local_func(F, As0, Bs0, _Shell, _RT, FT, Lf, Ef) when is_atom(F) -> {As,Bs} = expr_list(As0, Bs0, Lf, Ef), - non_builtin_local_func(F,As,Bs). + non_builtin_local_func(F,As,Bs, FT). -non_builtin_local_func(F,As,Bs) -> +non_builtin_local_func(F,As,Bs, FT) -> Arity = length(As), case erlang:function_exported(user_default, F, Arity) of - true -> + true -> {eval,erlang:make_fun(user_default, F, Arity),As,Bs}; - false -> - shell_default(F,As,Bs) + false -> + shell_default(F,As,Bs, FT) end. -shell_default(F,As,Bs) -> +shell_default(F,As,Bs, FT) -> M = shell_default, A = length(As), case code:ensure_loaded(M) of - {module, _} -> - case erlang:function_exported(M,F,A) of - true -> - {eval,erlang:make_fun(M, F, A),As,Bs}; - false -> - shell_undef(F,A) - end; - {error, _} -> - shell_undef(F,A) + {module, _} -> + case erlang:function_exported(M,F,A) of + true -> + {eval,erlang:make_fun(M, F, A),As,Bs}; + false -> + shell_default_local_func(F,As, Bs, FT) + end; + {error, _} -> + shell_default_local_func(F,As, Bs, FT) + end. + +shell_default_local_func(F, As, Bs, FT) -> + case ets:lookup(FT, {function, {shell_default, F, length(As)}}) of + [] -> shell_undef(F, length(As)); + [{_, Fun}] -> {eval, Fun, As, Bs} end. shell_undef(F,A) -> erlang:error({shell_undef,F,A,[]}). -local_func_handler(Shell, RT, Ef) -> - H = fun(Lf) -> - fun(F, As, Bs) -> - local_func(F, As, Bs, Shell, RT, {eval,Lf(Lf)}, Ef) +local_func_handler(Shell, RT, FT, Ef) -> + H = fun(Lf) -> + fun(F, As, Bs) -> + local_func(F, As, Bs, Shell, RT, FT, {eval,Lf(Lf)}, Ef) end - end, + end, {eval,H(H)}. record_print_fun(RT) -> fun(Tag, NoFields) -> case ets:lookup(RT, Tag) of - [{_,{attribute,_,record,{Tag,Fields}}}] - when length(Fields) =:= NoFields -> + [{_,{attribute,_,record,{Tag,Fields}}}] + when length(Fields) =:= NoFields -> record_fields(Fields); _ -> no @@ -1109,7 +1276,7 @@ record_fields([]) -> initiate_records(Bs, RT) -> RNs1 = init_rec(shell_default, Bs, RT), RNs2 = case code:is_loaded(user_default) of - {file,_File} -> + {file,_File} -> init_rec(user_default, Bs, RT); false -> [] @@ -1145,11 +1312,12 @@ read_records(File, Selected, Options) -> RAs; RAs -> Sel = listify(Selected), - [RA || {attribute,_,_,{Name,_}}=RA <- RAs, + [RA || {attribute,_,_,{Name,_}}=RA <- RAs, lists:member(Name, Sel)] end. add_records(RAs, Bs0, RT) -> + %% TODO store File name to support type completion Recs = [{Name,D} || {attribute,_,_,{Name,_}}=D <- RAs], Bs1 = record_bindings(Recs, Bs0), case check_command([], Bs1) of @@ -1183,12 +1351,12 @@ expr_list(Es, Bs, Lf, Ef) -> record_bindings([], Bs) -> Bs; record_bindings(Recs0, Bs0) -> - {Recs1, _} = lists:mapfoldl(fun ({Name,Def}, I) -> {{Name,I,Def},I+1} + {Recs1, _} = lists:mapfoldl(fun ({Name,Def}, I) -> {{Name,I,Def},I+1} end, 0, Recs0), Recs2 = lists:keysort(2, lists:ukeysort(1, Recs1)), lists:foldl(fun ({Name,I,Def}, Bs) -> - erl_eval:add_binding({record,I,Name}, Def, Bs) - end, Bs0, Recs2). + erl_eval:add_binding({record,I,Name}, Def, Bs) + end, Bs0, Recs2). %%% Read record information from file(s) @@ -1214,7 +1382,7 @@ read_records(FileOrModule, Opts0) -> find_file(Mod) when is_atom(Mod) -> case code:which(Mod) of - File when is_list(File) -> + File when is_list(File) -> %% Special cases: %% - Modules not in the code path (loaded with code:load_abs/1): %% code:get_object_code/1 only searches in the code path @@ -1228,8 +1396,8 @@ find_file(Mod) when is_atom(Mod) -> error -> {error, nofile} end; - preloaded -> - {_M, Beam, File} = code:get_object_code(Mod), + preloaded -> + {_M, Beam, File} = code:get_object_code(Mod), {beam, Beam, File}; _Else -> % non_existing, interpreted, cover_compiled {error,nofile} @@ -1291,10 +1459,10 @@ try_sources([Src|Rest], Os) -> is_file(Name) -> case filelib:is_file(Name) of - true -> - not filelib:is_dir(Name); - false -> - false + true -> + not filelib:is_dir(Name); + false -> + false end. parse_file(File, Opts) -> @@ -1327,7 +1495,7 @@ record_attrs(Forms) -> shell_req(Shell, Req) -> Shell ! {shell_req,self(),Req}, receive - {shell_rep,Shell,Rep} -> Rep + {shell_rep,Shell,Rep} -> Rep end. list_commands([{{N,command},Es0}, {{N,result}, V} |Ds], RT) -> @@ -1337,7 +1505,7 @@ list_commands([{{N,command},Es0}, {{N,result}, V} |Ds], RT) -> I = iolist_size(Ns), io:requests([{put_chars, latin1, Ns}, {format,<<"~ts\n">>,[erl_pp:exprs(Es, I, enc())]}, - {format,<<"-> ">>,[]}, + {format,<<"-> ">>,[]}, {put_chars, unicode, VS}, nl]), list_commands(Ds, RT); @@ -1379,7 +1547,7 @@ list_bindings([], _RT) -> ok. list_records(Records) -> - lists:foreach(fun({_Name,Attr}) -> + lists:foreach(fun({_Name,Attr}) -> io:fwrite(<<"~ts">>, [erl_pp:attribute(Attr, enc())]) end, Records). @@ -1410,11 +1578,11 @@ prep_list_commands(E) -> substitute_v1(F, {value,_,_}=Value) -> F(Value); -substitute_v1(F, T) when is_tuple(T) -> +substitute_v1(F, T) when is_tuple(T) -> list_to_tuple(substitute_v1(F, tuple_to_list(T))); -substitute_v1(F, [E | Es]) -> +substitute_v1(F, [E | Es]) -> [substitute_v1(F, E) | substitute_v1(F, Es)]; -substitute_v1(_F, E) -> +substitute_v1(_F, E) -> E. a0() -> @@ -1439,13 +1607,7 @@ pp(V, I, RT) -> pp(V, I, _Depth=?LINEMAX, RT). pp(V, I, D, RT) -> - Strings = - case application:get_env(stdlib, shell_strings) of - {ok, false} -> - false; - _ -> - true - end, + Strings = application:get_env(stdlib, shell_strings, true) =/= false, io_lib_pretty:print(V, ([{column, I}, {line_length, columns()}, {depth, D}, {line_max_chars, ?CHAR_MAX}, {strings, Strings}, @@ -1464,8 +1626,8 @@ encoding() -> enc() -> case lists:keyfind(encoding, 1, io:getopts()) of - false -> [{encoding,latin1}]; % should never happen - Enc -> [Enc] + false -> [{encoding,latin1}]; % should never happen + Enc -> [Enc] end. garb(Shell) -> @@ -1475,33 +1637,31 @@ garb(Shell) -> erlang:garbage_collect(). get_env(V, Def) -> - case application:get_env(stdlib, V) of - {ok, Val} when is_integer(Val), Val >= 0 -> - Val; - _ -> - Def + case application:get_env(stdlib, V, Def) of + Val when is_integer(Val), Val >= 0 -> + Val; + _ -> + Def end. - + check_env(V) -> - case application:get_env(stdlib, V) of - undefined -> - ok; - {ok, Val} when is_integer(Val), Val >= 0 -> - ok; - {ok, Val} -> + case application:get_env(stdlib, V, 0) of + Val when is_integer(Val), Val >= 0 -> + ok; + Val -> Txt = io_lib:fwrite ("Invalid value of STDLIB configuration parameter" "~tw: ~tp\n", [V, Val]), - error_logger:info_report(lists:flatten(Txt)) + error_logger:info_report(lists:flatten(Txt)) end. - + set_env(App, Name, Val, Default) -> Prev = case application:get_env(App, Name) of - undefined -> - Default; - {ok, Old} -> - Old - end, + undefined -> + Default; + {ok, Old} -> + Old + end, application_controller:set_env(App, Name, Val), Prev. @@ -1521,7 +1681,7 @@ results(L) when is_integer(L), L >= 0 -> Bool :: boolean(). catch_exception(Bool) -> - set_env(stdlib, shell_catch_exception, Bool, ?DEF_CATCH_EXCEPTION). + set_env(stdlib, shell_catch_exception, Bool, ?DEF_CATCH_EXCEPTION). -spec prompt_func(PromptFunc) -> PromptFunc2 when PromptFunc :: 'default' | {module(),atom()}, diff --git a/lib/stdlib/src/shell_default.erl b/lib/stdlib/src/shell_default.erl index 3d820c9131..669266d255 100644 --- a/lib/stdlib/src/shell_default.erl +++ b/lib/stdlib/src/shell_default.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2021. All Rights Reserved. +%% Copyright Ericsson AB 1996-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -25,17 +25,17 @@ -export([help/0,lc/1,c/1,c/2,c/3,nc/1,nl/1,l/1,i/0,pid/3,i/3,m/0,m/1,lm/0,mm/0, memory/0,memory/1,uptime/0, - erlangrc/1,bi/1, regs/0, flush/0,pwd/0,ls/0,ls/1,cd/1, + erlangrc/1,bi/1, regs/0, flush/0,pwd/0,ls/0,ls/1,cd/1, y/1, y/2, - xm/1, bt/1, q/0, + xm/1, bt/1, q/0, h/1, h/2, h/3, ht/1, ht/2, ht/3, hcb/1, hcb/2, hcb/3, - ni/0, nregs/0]). + ni/0, nregs/0]). -export([ih/0,iv/0,im/0,ii/1,ii/2,iq/1,ini/1,ini/2,inq/1,ib/2,ib/3, - ir/2,ir/3,ibd/2,ibe/2,iba/3,ibc/3, - ic/0,ir/1,ir/0,il/0,ipb/0,ipb/1,iaa/1,iaa/2,ist/1,ia/1,ia/2,ia/3, - ia/4,ip/0]). - + ir/2,ir/3,ibd/2,ibe/2,iba/3,ibc/3, + ic/0,ir/1,ir/0,il/0,ipb/0,ipb/1,iaa/1,iaa/2,ist/1,ia/1,ia/2,ia/3, + ia/4,ip/0]). +-export(['$handle_undefined_function'/2]). -import(io, [format/1]). help() -> @@ -78,13 +78,13 @@ help() -> %% these are in alphabetic order it would be nice if they %% were to *stay* so! -bi(I) -> c:bi(I). -bt(Pid) -> c:bt(Pid). -c(File) -> c:c(File). +bi(I) -> c:bi(I). +bt(Pid) -> c:bt(Pid). +c(File) -> c:c(File). c(File, Opt) -> c:c(File, Opt). c(File, Opt, Filter) -> c:c(File, Opt, Filter). cd(D) -> c:cd(D). -erlangrc(X) -> c:erlangrc(X). +erlangrc(X) -> c:erlangrc(X). flush() -> c:flush(). h(M) -> c:h(M). h(M,F) -> c:h(M,F). @@ -95,25 +95,25 @@ ht(M,F,A) -> c:ht(M,F,A). hcb(M) -> c:hcb(M). hcb(M,F) -> c:hcb(M,F). hcb(M,F,A) -> c:hcb(M,F,A). -i() -> c:i(). -i(X,Y,Z) -> c:i(X,Y,Z). -l(Mod) -> c:l(Mod). -lc(X) -> c:lc(X). +i() -> c:i(). +i(X,Y,Z) -> c:i(X,Y,Z). +l(Mod) -> c:l(Mod). +lc(X) -> c:lc(X). ls() -> c:ls(). ls(S) -> c:ls(S). -m() -> c:m(). -m(Mod) -> c:m(Mod). +m() -> c:m(). +m(Mod) -> c:m(Mod). lm() -> c:lm(). mm() -> c:mm(). memory() -> c:memory(). memory(Type) -> c:memory(Type). -nc(X) -> c:nc(X). +nc(X) -> c:nc(X). ni() -> c:ni(). -nl(Mod) -> c:nl(Mod). +nl(Mod) -> c:nl(Mod). nregs() -> c:nregs(). -pid(X,Y,Z) -> c:pid(X,Y,Z). +pid(X,Y,Z) -> c:pid(X,Y,Z). pwd() -> c:pwd(). -q() -> c:q(). +q() -> c:q(). regs() -> c:regs(). uptime() -> c:uptime(). xm(Mod) -> c:xm(Mod). @@ -154,3 +154,11 @@ iv() -> calli(iv, []). calli(F, Args) -> c:appcall(debugger, i, F, Args). + +'$handle_undefined_function'(Func, Args) -> + case shell:get_function(Func, length(Args)) of + undefined -> + error_handler:raise_undef_exception(?MODULE, Func, Args); + Fun when is_function(Fun, length(Args)) -> + apply(Fun, Args) + end.
\ No newline at end of file diff --git a/lib/stdlib/src/shell_docs.erl b/lib/stdlib/src/shell_docs.erl index e42b5bb5b8..5cf1e0aed2 100644 --- a/lib/stdlib/src/shell_docs.erl +++ b/lib/stdlib/src/shell_docs.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2022. All Rights Reserved. +%% Copyright Ericsson AB 1996-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -872,10 +872,16 @@ render_element({li,[],Content},[l | _] = State, Pos, Ind,D) -> render_element({dl,_,Content},State,Pos,Ind,D) -> render_docs(Content, [dl|State], Pos, Ind,D); -render_element({dt,_,Content},[dl | _] = State,Pos,Ind,D) -> +render_element({dt,Attr,Content},[dl | _] = State,Pos,Ind,D) -> + Since = case Attr of + [{since, Vsn}] -> + [" (since ",unicode:characters_to_list(Vsn),$)]; + [] -> + [] + end, Underline = sansi(underline), {Docs, _NewPos} = render_docs(Content, [li | State], Pos, Ind, D), - {[Underline,Docs,ransi(underline),":","\n"], 0}; + {[Underline,Docs,ransi(underline),$:,Since,$\n], 0}; render_element({dd,_,Content},[dl | _] = State,Pos,Ind,D) -> trimnlnl(render_docs(Content, [li | State], Pos, Ind + 2, D)); @@ -1005,18 +1011,18 @@ nl(Chars) -> init_ansi(#config{ ansi = undefined, io_opts = Opts }) -> %% We use this as our heuristic to see if we should print ansi or not case {application:get_env(kernel, shell_docs_ansi), + proplists:get_value(terminal, Opts, false), proplists:is_defined(echo, Opts) andalso - proplists:is_defined(expand_fun, Opts), - os:type()} of + proplists:is_defined(expand_fun, Opts)} of {{ok,false}, _, _} -> put(ansi, noansi); {{ok,true}, _, _} -> put(ansi, []); - {_, _, {win32,_}} -> - put(ansi, noansi); - {_, true,_} -> + {_, true, _} -> + put(ansi, []); + {_, _, true} -> put(ansi, []); - {_, false,_} -> + {_, _, false} -> put(ansi, noansi) end; init_ansi(#config{ ansi = true }) -> diff --git a/lib/stdlib/src/stdlib.app.src b/lib/stdlib/src/stdlib.app.src index 358ebf471d..a71ad0a954 100644 --- a/lib/stdlib/src/stdlib.app.src +++ b/lib/stdlib/src/stdlib.app.src @@ -2,7 +2,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2022. All Rights Reserved. +%% Copyright Ericsson AB 1996-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -21,7 +21,8 @@ {application, stdlib, [{description, "ERTS CXC 138 10"}, {vsn, "%VSN%"}, - {modules, [array, + {modules, [argparse, + array, base64, beam_lib, binary, @@ -36,7 +37,9 @@ digraph, digraph_utils, edlin, + edlin_context, edlin_expand, + edlin_type_suggestion, epp, eval_bits, erl_abstract_code, @@ -112,6 +115,6 @@ dets]}, {applications, [kernel]}, {env, []}, - {runtime_dependencies, ["sasl-3.0","kernel-8.5.1","erts-13.1","crypto-4.5", + {runtime_dependencies, ["sasl-3.0","kernel-@OTP-17932@","erts-13.1","crypto-4.5", "compiler-5.0"]} ]}. diff --git a/lib/stdlib/src/string.erl b/lib/stdlib/src/string.erl index a418754caf..e0b765948c 100644 --- a/lib/stdlib/src/string.erl +++ b/lib/stdlib/src/string.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2019. All Rights Reserved. +%% Copyright Ericsson AB 1996-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -76,7 +76,9 @@ -import(lists,[member/2]). -compile({no_auto_import,[length/1]}). -compile({inline, [btoken/2, rev/1, append/2, stack/2, search_compile/1]}). --define(ASCII_LIST(CP1,CP2), CP1 < 256, CP2 < 256, CP1 =/= $\r). +-define(ASCII_LIST(CP1,CP2), + is_integer(CP1), 0 =< CP1, CP1 < 256, + is_integer(CP2), 0 =< CP2, CP2 < 256, CP1 =/= $\r). -export_type([grapheme_cluster/0]). @@ -198,7 +200,7 @@ slice(CD, N, Length) [] when is_binary(CD) -> <<>>; L -> slice_trail(L, Length) end; -slice(CD, N, infinity) -> +slice(CD, N, infinity) when is_integer(N), N >= 0 -> case slice_l0(CD, N) of [] when is_binary(CD) -> <<>>; Res -> Res @@ -261,11 +263,13 @@ trim(Str, Dir) -> Dir :: direction() | 'both', Characters :: [grapheme_cluster()]. trim(Str, _, []) -> Str; -trim(Str, leading, [Sep]) when is_list(Str), Sep < 256 -> +trim(Str, leading, [Sep]) + when is_list(Str), is_integer(Sep), 0 =< Sep, Sep < 256 -> trim_ls(Str, Sep); trim(Str, leading, Sep) when is_list(Sep) -> trim_l(Str, Sep); -trim(Str, trailing, [Sep]) when is_list(Str), Sep < 256 -> +trim(Str, trailing, [Sep]) + when is_list(Str), is_integer(Sep), 0 =< Sep, Sep < 256 -> trim_ts(Str, Sep); trim(Str, trailing, Seps0) when is_list(Seps0) -> Seps = search_pattern(Seps0), @@ -630,9 +634,10 @@ slice_l0(<<CP1/utf8, Bin/binary>>, N) when N > 0 -> slice_l0(L, N) -> slice_l(L, N). -slice_l([CP1|[CP2|_]=Cont], N) when ?ASCII_LIST(CP1,CP2),N > 0 -> +slice_l([CP1|[CP2|_]=Cont], N) + when ?ASCII_LIST(CP1,CP2), is_integer(N), N > 0 -> slice_l(Cont, N-1); -slice_l(CD, N) when N > 0 -> +slice_l(CD, N) when is_integer(N), N > 0 -> case unicode_util:gc(CD) of [_|Cont] -> slice_l(Cont, N-1); [] -> []; @@ -641,7 +646,8 @@ slice_l(CD, N) when N > 0 -> slice_l(Cont, 0) -> Cont. -slice_lb(<<CP2/utf8, Bin/binary>>, CP1, N) when ?ASCII_LIST(CP1,CP2), N > 1 -> +slice_lb(<<CP2/utf8, Bin/binary>>, CP1, N) + when ?ASCII_LIST(CP1,CP2), is_integer(N), N > 1 -> slice_lb(Bin, CP2, N-1); slice_lb(Bin, CP1, N) -> [_|Rest] = unicode_util:gc([CP1|Bin]), @@ -693,9 +699,13 @@ slice_bin(CD, CP1, N) when N > 0 -> slice_bin(CD, CP1, 0) -> byte_size(CD)+byte_size(<<CP1/utf8>>). -uppercase_list([CP1|[CP2|_]=Cont], _Changed) when $a =< CP1, CP1 =< $z, CP2 < 256 -> +uppercase_list([CP1|[CP2|_]=Cont], _Changed) + when is_integer(CP1), $a =< CP1, CP1 =< $z, + is_integer(CP2), 0 =< CP2, CP2 < 256 -> [CP1-32|uppercase_list(Cont, true)]; -uppercase_list([CP1|[CP2|_]=Cont], Changed) when CP1 < 128, CP2 < 256 -> +uppercase_list([CP1|[CP2|_]=Cont], Changed) + when is_integer(CP1), 0 =< CP1, CP1 < 128, + is_integer(CP2), 0 =< CP2, CP2 < 256 -> [CP1|uppercase_list(Cont, Changed)]; uppercase_list([], true) -> []; @@ -709,16 +719,16 @@ uppercase_list(CPs0, Changed) -> end. uppercase_bin(CP1, <<CP2/utf8, Bin/binary>>, _Changed) - when $a =< CP1, CP1 =< $z, CP2 < 256 -> + when is_integer(CP1), $a =< CP1, CP1 =< $z, CP2 < 256 -> [CP1-32|uppercase_bin(CP2, Bin, true)]; uppercase_bin(CP1, <<CP2/utf8, Bin/binary>>, Changed) - when CP1 < 128, CP2 < 256 -> + when is_integer(CP1), 0 =< CP1, CP1 < 128, CP2 < 256 -> [CP1|uppercase_bin(CP2, Bin, Changed)]; uppercase_bin(CP1, Bin, Changed) -> case unicode_util:uppercase([CP1|Bin]) of [CP1|CPs] -> case unicode_util:cp(CPs) of - [Next|Rest] -> + [Next|Rest] when is_integer(Next), Next >= 0 -> [CP1|uppercase_bin(Next, Rest, Changed)]; [] when Changed -> [CP1]; @@ -729,7 +739,7 @@ uppercase_bin(CP1, Bin, Changed) -> end; [Char|CPs] -> case unicode_util:cp(CPs) of - [Next|Rest] -> + [Next|Rest] when is_integer(Next), Next >= 0 -> [Char|uppercase_bin(Next, Rest, true)]; [] -> [Char]; @@ -738,9 +748,13 @@ uppercase_bin(CP1, Bin, Changed) -> end end. -lowercase_list([CP1|[CP2|_]=Cont], _Changed) when $A =< CP1, CP1 =< $Z, CP2 < 256 -> +lowercase_list([CP1|[CP2|_]=Cont], _Changed) + when is_integer(CP1), $A =< CP1, CP1 =< $Z, + is_integer(CP2), 0 =< CP2, CP2 < 256 -> [CP1+32|lowercase_list(Cont, true)]; -lowercase_list([CP1|[CP2|_]=Cont], Changed) when CP1 < 128, CP2 < 256 -> +lowercase_list([CP1|[CP2|_]=Cont], Changed) + when is_integer(CP1), 0 =< CP1, CP1 < 128, + is_integer(CP2), 0 =< CP2, CP2 < 256 -> [CP1|lowercase_list(Cont, Changed)]; lowercase_list([], true) -> []; @@ -754,16 +768,16 @@ lowercase_list(CPs0, Changed) -> end. lowercase_bin(CP1, <<CP2/utf8, Bin/binary>>, _Changed) - when $A =< CP1, CP1 =< $Z, CP2 < 256 -> + when is_integer(CP1), $A =< CP1, CP1 =< $Z, CP2 < 256 -> [CP1+32|lowercase_bin(CP2, Bin, true)]; lowercase_bin(CP1, <<CP2/utf8, Bin/binary>>, Changed) - when CP1 < 128, CP2 < 256 -> + when is_integer(CP1), 0 =< CP1, CP1 < 128, CP2 < 256 -> [CP1|lowercase_bin(CP2, Bin, Changed)]; lowercase_bin(CP1, Bin, Changed) -> case unicode_util:lowercase([CP1|Bin]) of [CP1|CPs] -> case unicode_util:cp(CPs) of - [Next|Rest] -> + [Next|Rest] when is_integer(Next), Next >= 0 -> [CP1|lowercase_bin(Next, Rest, Changed)]; [] when Changed -> [CP1]; @@ -774,7 +788,7 @@ lowercase_bin(CP1, Bin, Changed) -> end; [Char|CPs] -> case unicode_util:cp(CPs) of - [Next|Rest] -> + [Next|Rest] when is_integer(Next), Next >= 0 -> [Char|lowercase_bin(Next, Rest, true)]; [] -> [Char]; @@ -783,9 +797,13 @@ lowercase_bin(CP1, Bin, Changed) -> end end. -casefold_list([CP1|[CP2|_]=Cont], _Changed) when $A =< CP1, CP1 =< $Z, CP2 < 256 -> +casefold_list([CP1|[CP2|_]=Cont], _Changed) + when is_integer(CP1), $A =< CP1, CP1 =< $Z, + is_integer(CP2), 0 =< CP2, CP2 < 256 -> [CP1+32|casefold_list(Cont, true)]; -casefold_list([CP1|[CP2|_]=Cont], Changed) when CP1 < 128, CP2 < 256 -> +casefold_list([CP1|[CP2|_]=Cont], Changed) + when is_integer(CP1), 0 =< CP1, CP1 < 128, + is_integer(CP2), 0 =< CP2, CP2 < 256 -> [CP1|casefold_list(Cont, Changed)]; casefold_list([], true) -> []; @@ -799,16 +817,16 @@ casefold_list(CPs0, Changed) -> end. casefold_bin(CP1, <<CP2/utf8, Bin/binary>>, _Changed) - when $A =< CP1, CP1 =< $Z, CP2 < 256 -> + when is_integer(CP1), $A =< CP1, CP1 =< $Z, CP2 < 256 -> [CP1+32|casefold_bin(CP2, Bin, true)]; casefold_bin(CP1, <<CP2/utf8, Bin/binary>>, Changed) - when CP1 < 128, CP2 < 256 -> + when is_integer(CP1), 0 =< CP1, CP1 < 128, CP2 < 256 -> [CP1|casefold_bin(CP2, Bin, Changed)]; casefold_bin(CP1, Bin, Changed) -> case unicode_util:casefold([CP1|Bin]) of [CP1|CPs] -> case unicode_util:cp(CPs) of - [Next|Rest] -> + [Next|Rest] when is_integer(Next), Next >= 0 -> [CP1|casefold_bin(Next, Rest, Changed)]; [] when Changed -> [CP1]; @@ -819,7 +837,7 @@ casefold_bin(CP1, Bin, Changed) -> end; [Char|CPs] -> case unicode_util:cp(CPs) of - [Next|Rest] -> + [Next|Rest] when is_integer(Next), Next >= 0 -> [Char|casefold_bin(Next, Rest, true)]; [] -> [Char]; @@ -1734,7 +1752,7 @@ bin_search_str_2(Bin0, Start, Cont, First, SearchCPs) -> <<_:Start/binary, Bin/binary>> = Bin0, case binary:match(Bin, First) of nomatch -> {nomatch, byte_size(Bin0), Cont}; - {Where0, _} -> + {Where0, _} when is_integer(Where0) -> Where = Start+Where0, <<Keep:Where/binary, Cs0/binary>> = Bin0, [GC|Cs]=unicode_util:gc(Cs0), @@ -1979,7 +1997,7 @@ chars(C, N) -> chars(C, N, []). Tail :: string(), String :: string(). -chars(C, N, Tail) when N > 0 -> +chars(C, N, Tail) when is_integer(N), N > 0 -> chars(C, N-1, [C|Tail]); chars(C, 0, Tail) when is_integer(C) -> Tail. @@ -2109,7 +2127,7 @@ left(String, Len) when is_integer(Len) -> left(String, Len, $\s). Number :: non_neg_integer(), Character :: char(). -left(String, Len, Char) when is_integer(Char) -> +left(String, Len, Char) when is_integer(Len), is_integer(Char) -> Slen = erlang:length(String), if Slen > Len -> substr(String, 1, Len); @@ -2134,7 +2152,7 @@ right(String, Len) when is_integer(Len) -> right(String, Len, $\s). Number :: non_neg_integer(), Character :: char(). -right(String, Len, Char) when is_integer(Char) -> +right(String, Len, Char) when is_integer(Len), is_integer(Char) -> Slen = erlang:length(String), if Slen > Len -> substr(String, Slen-Len+1); @@ -2161,7 +2179,7 @@ centre(String, Len) when is_integer(Len) -> centre(String, Len, $\s). centre(String, 0, Char) when is_list(String), is_integer(Char) -> []; % Strange cases to centre string -centre(String, Len, Char) when is_integer(Char) -> +centre(String, Len, Char) when is_integer(Len), is_integer(Char) -> Slen = erlang:length(String), if Slen > Len -> substr(String, (Slen-Len) div 2 + 1, Len); @@ -2186,7 +2204,8 @@ sub_string(String, Start) -> substr(String, Start). Start :: pos_integer(), Stop :: pos_integer(). -sub_string(String, Start, Stop) -> substr(String, Start, Stop - Start + 1). +sub_string(String, Start, Stop) when is_integer(Start), is_integer(Stop) -> + substr(String, Start, Stop - Start + 1). %% ISO/IEC 8859-1 (latin1) letters are converted, others are ignored %% diff --git a/lib/stdlib/src/supervisor.erl b/lib/stdlib/src/supervisor.erl index 58b943d874..de44ce55ee 100644 --- a/lib/stdlib/src/supervisor.erl +++ b/lib/stdlib/src/supervisor.erl @@ -349,7 +349,10 @@ init_children(State, StartSpec) -> {ok, Children} -> case start_children(Children, SupName) of {ok, NChildren} -> - {ok, State#state{children = NChildren}}; + %% Static supervisor are not expected to + %% have much work to do so hibernate them + %% to improve memory handling. + {ok, State#state{children = NChildren}, hibernate}; {error, NChildren, Reason} -> _ = terminate_children(NChildren, SupName), {stop, {shutdown, Reason}} @@ -361,6 +364,9 @@ init_children(State, StartSpec) -> init_dynamic(State, [StartSpec]) -> case check_startspec([StartSpec], State#state.auto_shutdown) of {ok, Children} -> + %% Simple one for one supervisors are expected to + %% have many children coming and going so do not + %% hibernate. {ok, dyn_init(State#state{children = Children})}; Error -> {stop, {start_spec, Error}} diff --git a/lib/stdlib/src/timer.erl b/lib/stdlib/src/timer.erl index 182f5cb4f2..17c12f2931 100644 --- a/lib/stdlib/src/timer.erl +++ b/lib/stdlib/src/timer.erl @@ -22,8 +22,9 @@ -export([apply_after/4, send_after/3, send_after/2, exit_after/3, exit_after/2, kill_after/2, kill_after/1, - apply_interval/4, send_interval/3, send_interval/2, - cancel/1, sleep/1, tc/1, tc/2, tc/3, now_diff/2, + apply_interval/4, apply_repeatedly/4, + send_interval/3, send_interval/2, + cancel/1, sleep/1, tc/1, tc/2, tc/3, tc/4, now_diff/2, seconds/1, minutes/1, hours/1, hms/3]). -export([start_link/0, start/0, @@ -61,7 +62,7 @@ Reason :: term(). apply_after(0, M, F, A) when ?valid_mfa(M, F, A) -> - do_apply({M, F, A}), + _ = do_apply({M, F, A}, false), {ok, {instant, make_ref()}}; apply_after(Time, M, F, A) when ?valid_time(Time), @@ -160,6 +161,21 @@ apply_interval(Time, M, F, A) apply_interval(_Time, _M, _F, _A) -> {error, badarg}. +-spec apply_repeatedly(Time, Module, Function, Arguments) -> + {'ok', TRef} | {'error', Reason} + when Time :: time(), + Module :: module(), + Function :: atom(), + Arguments :: [term()], + TRef :: tref(), + Reason :: term(). +apply_repeatedly(Time, M, F, A) + when ?valid_time(Time), + ?valid_mfa(M, F, A) -> + req(apply_repeatedly, {system_time(), Time, self(), {M, F, A}}); +apply_repeatedly(_Time, _M, _F, _A) -> + {error, badarg}. + -spec send_interval(Time, Destination, Message) -> {'ok', TRef} | {'error', Reason} when Time :: time(), Destination :: pid() | (RegName :: atom()) | {RegName :: atom(), Node :: node()}, @@ -231,41 +247,71 @@ sleep(T) -> Time :: integer(), Value :: term(). tc(F) -> + tc(F, microsecond). + +%% +%% Measure the execution time (in microseconds) for Fun(Args) +%% or the execution time (in TimeUnit) for Fun(). +%% +-spec tc(Fun, Arguments) -> {Time, Value} + when Fun :: function(), + Arguments :: [term()], + Time :: integer(), + Value :: term(); + (Fun, TimeUnit) -> {Time, Value} + when Fun :: function(), + TimeUnit :: erlang:time_unit(), + Time :: integer(), + Value :: term(). +tc(F, A) when is_list(A) -> + tc(F, A, microsecond); +tc(F, TimeUnit) -> T1 = erlang:monotonic_time(), Val = F(), T2 = erlang:monotonic_time(), - Time = erlang:convert_time_unit(T2 - T1, native, microsecond), + Time = erlang:convert_time_unit(T2 - T1, native, TimeUnit), {Time, Val}. %% -%% Measure the execution time (in microseconds) for Fun(Args). +%% Measure the execution time (in microseconds) for an MFA +%% or the execution time (in TimeUnit) for Fun(Args). %% --spec tc(Fun, Arguments) -> {Time, Value} +-spec tc(Module, Function, Arguments) -> {Time, Value} + when Module :: module(), + Function :: atom(), + Arguments :: [term()], + Time :: integer(), + Value :: term(); + (Fun, Arguments, TimeUnit) -> {Time, Value} when Fun :: function(), Arguments :: [term()], + TimeUnit :: erlang:time_unit(), Time :: integer(), Value :: term(). -tc(F, A) -> +tc(M, F, A) when is_list(A) -> + tc(M, F, A, microsecond); +tc(F, A, TimeUnit) -> T1 = erlang:monotonic_time(), Val = apply(F, A), T2 = erlang:monotonic_time(), - Time = erlang:convert_time_unit(T2 - T1, native, microsecond), + Time = erlang:convert_time_unit(T2 - T1, native, TimeUnit), {Time, Val}. %% -%% Measure the execution time (in microseconds) for an MFA. +%% Measure the execution time (in TimeUnit) for an MFA. %% --spec tc(Module, Function, Arguments) -> {Time, Value} +-spec tc(Module, Function, Arguments, TimeUnit) -> {Time, Value} when Module :: module(), Function :: atom(), Arguments :: [term()], + TimeUnit :: erlang:time_unit(), Time :: integer(), Value :: term(). -tc(M, F, A) -> +tc(M, F, A, TimeUnit) -> T1 = erlang:monotonic_time(), Val = apply(M, F, A), T2 = erlang:monotonic_time(), - Time = erlang:convert_time_unit(T2 - T1, native, microsecond), + Time = erlang:convert_time_unit(T2 - T1, native, TimeUnit), {Time, Val}. %% @@ -382,15 +428,11 @@ maybe_req(Req, Arg) -> handle_call({apply_once, {Started, Time, MFA}}, _From, Tab) -> Timeout = Started + Time, Reply = try - erlang:start_timer( - Timeout, - self(), - {apply_once, MFA}, - [{abs, true}] - ) + erlang:start_timer(Timeout, self(), {apply_once, MFA}, + [{abs, true}]) of SRef -> - ets:insert(Tab, {SRef, SRef}), + ets:insert(Tab, {SRef}), {ok, {once, SRef}} catch error:badarg -> @@ -399,25 +441,13 @@ handle_call({apply_once, {Started, Time, MFA}}, _From, Tab) -> {reply, Reply, Tab}; %% Start an interval timer. handle_call({apply_interval, {Started, Time, Pid, MFA}}, _From, Tab) -> - NextTimeout = Started + Time, - TRef = monitor(process, Pid), - Reply = try - erlang:start_timer( - NextTimeout, - self(), - {apply_interval, NextTimeout, Time, TRef, MFA}, - [{abs, true}] - ) - of - SRef -> - ets:insert(Tab, {TRef, SRef}), - {ok, {interval, TRef}} - catch - error:badarg -> - demonitor(TRef, [flush]), - {error, badarg} - end, - {reply, Reply, Tab}; + {TRef, TPid, Tag} = start_interval_loop(Started, Time, Pid, MFA, false), + ets:insert(Tab, {TRef, TPid, Tag}), + {reply, {ok, {interval, TRef}}, Tab}; +handle_call({apply_repeatedly, {Started, Time, Pid, MFA}}, _From, Tab) -> + {TRef, TPid, Tag} = start_interval_loop(Started, Time, Pid, MFA, true), + ets:insert(Tab, {TRef, TPid, Tag}), + {reply, {ok, {interval, TRef}}, Tab}; %% Cancel a one-shot timer. handle_call({cancel, {once, TRef}}, _From, Tab) -> _ = remove_timer(TRef, Tab), @@ -440,31 +470,14 @@ handle_call(_Req, _From, Tab) -> when Tab :: ets:tid(). %% One-shot timer timeout. handle_info({timeout, TRef, {apply_once, MFA}}, Tab) -> - case ets:take(Tab, TRef) of - [{TRef, _SRef}] -> - do_apply(MFA); - [] -> - ok - end, - {noreply, Tab}; -%% Interval timer timeout. -handle_info({timeout, _, {apply_interval, CurTimeout, Time, TRef, MFA}}, Tab) -> - case ets:member(Tab, TRef) of - true -> - NextTimeout = CurTimeout + Time, - SRef = erlang:start_timer( - NextTimeout, - self(), - {apply_interval, NextTimeout, Time, TRef, MFA}, - [{abs, true}] - ), - ets:update_element(Tab, TRef, {2, SRef}), - do_apply(MFA); - false -> - ok - end, + _ = case ets:take(Tab, TRef) of + [{TRef}] -> + do_apply(MFA, false); + [] -> + ok + end, {noreply, Tab}; -%% A process related to an interval timer died. +%% An interval timer loop process died. handle_info({'DOWN', TRef, process, _Pid, _Reason}, Tab) -> _ = remove_timer(TRef, Tab), {noreply, Tab}; @@ -480,34 +493,121 @@ handle_cast(_Req, Tab) -> {noreply, Tab}. -spec terminate(term(), _Tab) -> 'ok'. -terminate(_Reason, _Tab) -> - ok. +terminate(_Reason, undefined) -> + ok; +terminate(Reason, Tab) -> + _ = ets:foldl(fun + ({TRef}, Acc) -> + _ = cancel_timer(TRef), + Acc; + ({_TRef, TPid, Tag}, Acc) -> + TPid ! {cancel, Tag}, + Acc + end, + undefined, + Tab), + true = ets:delete(Tab), + terminate(Reason, undefined). -spec code_change(term(), State, term()) -> {'ok', State}. code_change(_OldVsn, Tab, _Extra) -> %% According to the man for gen server no timer can be set here. {ok, Tab}. +start_interval_loop(Started, Time, TargetPid, MFA, WaitComplete) -> + Tag = make_ref(), + TimeServerPid = self(), + {TPid, TRef} = spawn_monitor(fun() -> + TimeServerRef = monitor(process, TimeServerPid), + TargetRef = monitor(process, TargetPid), + TimerRef = schedule_interval_timer(Started, Time, + MFA), + _ = interval_loop(TimeServerRef, TargetRef, Tag, + WaitComplete, TimerRef) + end), + {TRef, TPid, Tag}. + +%% Interval timer loop. +interval_loop(TimerServerMon, TargetMon, Tag, WaitComplete, TimerRef0) -> + receive + {cancel, Tag} -> + ok = cancel_timer(TimerRef0); + {'DOWN', TimerServerMon, process, _, _} -> + ok = cancel_timer(TimerRef0); + {'DOWN', TargetMon, process, _, _} -> + ok = cancel_timer(TimerRef0); + {timeout, TimerRef0, {apply_interval, CurTimeout, Time, MFA}} -> + case do_apply(MFA, WaitComplete) of + {ok, {spawn, ActionMon}} -> + receive + {cancel, Tag} -> + ok; + {'DOWN', TimerServerMon, process, _, _} -> + ok; + {'DOWN', TargetMon, process, _, _} -> + ok; + {'DOWN', ActionMon, process, _, _} -> + TimerRef1 = schedule_interval_timer(CurTimeout, Time, MFA), + interval_loop(TimerServerMon, TargetMon, Tag, WaitComplete, TimerRef1) + end; + _ -> + TimerRef1 = schedule_interval_timer(CurTimeout, Time, MFA), + interval_loop(TimerServerMon, TargetMon, Tag, WaitComplete, TimerRef1) + end + end. + +schedule_interval_timer(CurTimeout, Time, MFA) -> + NextTimeout = CurTimeout + Time, + case NextTimeout =< system_time() of + true -> + TimerRef = make_ref(), + self() ! {timeout, TimerRef, {apply_interval, NextTimeout, Time, MFA}}, + TimerRef; + false -> + erlang:start_timer(NextTimeout, self(), {apply_interval, NextTimeout, Time, MFA}, [{abs, true}]) + end. + %% Remove a timer. remove_timer(TRef, Tab) -> case ets:take(Tab, TRef) of - [{TRef, SRef}] -> - ok = erlang:cancel_timer(SRef, [{async, true}, {info, false}]), + [{TRef}] -> % One-shot timer. + ok = cancel_timer(TRef), + true; + [{TRef, TPid, Tag}] -> % Interval timer. + TPid ! {cancel, Tag}, true; [] -> % TimerReference does not exist, do nothing false end. +%% Cancel a timer. +cancel_timer(TRef) -> + erlang:cancel_timer(TRef, [{async, true}, {info, false}]). + %% Help functions %% If send op. send directly (faster than spawn) -do_apply({?MODULE, send, A}) -> - catch send(A); +do_apply({?MODULE, send, A}, _) -> + try send(A) + of _ -> {ok, send} + catch _:_ -> error + end; %% If exit op. resolve registered name -do_apply({erlang, exit, [Name, Reason]}) -> - catch exit(get_pid(Name), Reason); -do_apply({M,F,A}) -> - catch spawn(M, F, A). +do_apply({erlang, exit, [Name, Reason]}, _) -> + try exit(get_pid(Name), Reason) + of _ -> {ok, exit} + catch _:_ -> error + end; +do_apply({M,F,A}, false) -> + try spawn(M, F, A) + of _ -> {ok, spawn} + catch _:_ -> error + end; +do_apply({M, F, A}, true) -> + try spawn_monitor(M, F, A) + of {_, Ref} -> {ok, {spawn, Ref}} + catch _:_ -> error + end. %% Get current time in milliseconds, %% ceil'ed to the next millisecond. diff --git a/lib/stdlib/src/unicode.erl b/lib/stdlib/src/unicode.erl index f9d52d30d3..d2a1edce3d 100644 --- a/lib/stdlib/src/unicode.erl +++ b/lib/stdlib/src/unicode.erl @@ -91,9 +91,9 @@ characters_to_binary(_, _) -> -spec characters_to_list(Data, InEncoding) -> Result when Data :: latin1_chardata() | chardata() | external_chardata(), InEncoding :: encoding(), - Result :: list() - | {error, list(), RestData} - | {incomplete, list(), binary()}, + Result ::string() + | {error, string(), RestData} + | {incomplete, string(), binary()}, RestData :: latin1_chardata() | chardata() | external_chardata(). characters_to_list(_, _) -> @@ -103,9 +103,9 @@ characters_to_list(_, _) -> -spec characters_to_list(Data) -> Result when Data :: latin1_chardata() | chardata() | external_chardata(), - Result :: list() - | {error, list(), RestData} - | {incomplete, list(), binary()}, + Result :: string() + | {error, string(), RestData} + | {incomplete, string(), binary()}, RestData :: latin1_chardata() | chardata() | external_chardata(). characters_to_list(ML) -> diff --git a/lib/stdlib/src/zip.erl b/lib/stdlib/src/zip.erl index c59398a84e..0809dbb492 100644 --- a/lib/stdlib/src/zip.erl +++ b/lib/stdlib/src/zip.erl @@ -35,7 +35,7 @@ %% zip server -export([zip_open/1, zip_open/2, - zip_get/1, zip_get/2, + zip_get/1, zip_get/2, zip_get_crc32/2, zip_t/1, zip_tt/1, zip_list_dir/1, zip_list_dir/2, zip_close/1]). @@ -267,6 +267,13 @@ do_openzip_get(#openzip{files = Files, in = In0, input = Input, do_openzip_get(_) -> throw(einval). +%% retrieve the crc32 checksum from an open archive +openzip_get_crc32(FileName, #openzip{files = Files}) -> + case file_name_search(FileName, Files) of + {_,#zip_file_extra{crc32=CRC}} -> {ok, CRC}; + _ -> throw(file_not_found) + end. + %% retrieve a file from an open archive openzip_get(FileName, OpenZip) -> case ?CATCH(do_openzip_get(FileName, OpenZip)) of @@ -1165,6 +1172,9 @@ server_loop(Parent, OpenZip) -> {From, {get, FileName}} -> From ! {self(), openzip_get(FileName, OpenZip)}, server_loop(Parent, OpenZip); + {From, {get_crc32, FileName}} -> + From ! {self(), openzip_get_crc32(FileName, OpenZip)}, + server_loop(Parent, OpenZip); {From, list_dir} -> From ! {self(), openzip_list_dir(OpenZip)}, server_loop(Parent, OpenZip); @@ -1223,6 +1233,15 @@ zip_close(Pid) when is_pid(Pid) -> zip_get(FileName, Pid) when is_pid(Pid) -> request(self(), Pid, {get, FileName}). +-spec(zip_get_crc32(FileName, ZipHandle) -> {ok, CRC} | {error, Reason} when + FileName :: file:name(), + ZipHandle :: handle(), + CRC :: non_neg_integer(), + Reason :: term()). + +zip_get_crc32(FileName, Pid) when is_pid(Pid) -> + request(self(), Pid, {get_crc32, FileName}). + -spec(zip_list_dir(ZipHandle) -> {ok, Result} | {error, Reason} when Result :: [zip_comment() | zip_file()], ZipHandle :: handle(), diff --git a/lib/stdlib/test/Makefile b/lib/stdlib/test/Makefile index b8e4d89996..2597157004 100644 --- a/lib/stdlib/test/Makefile +++ b/lib/stdlib/test/Makefile @@ -7,10 +7,12 @@ include $(ERL_TOP)/make/$(TARGET)/otp.mk MODULES= \ array_SUITE \ + argparse_SUITE \ base64_SUITE \ base64_property_test_SUITE \ beam_lib_SUITE \ binary_module_SUITE \ + binary_property_test_SUITE \ binref \ c_SUITE \ calendar_SUITE \ @@ -23,6 +25,7 @@ MODULES= \ dummy_h \ dummy_via \ edlin_expand_SUITE \ + edlin_context_SUITE \ epp_SUITE \ erl_anno_SUITE \ erl_eval_SUITE \ @@ -49,6 +52,7 @@ MODULES= \ io_SUITE \ io_proto_SUITE \ lists_SUITE \ + lists_property_test_SUITE \ log_mf_h_SUITE \ math_SUITE \ ms_transform_SUITE \ @@ -104,10 +108,12 @@ MODULES= \ ERTS_MODULES= erts_test_utils SASL_MODULES= otp_vsns +KERNEL_MODULES= rtnode ERL_FILES= $(MODULES:%=%.erl) \ $(ERTS_MODULES:%=$(ERL_TOP)/erts/emulator/test/%.erl) \ - $(SASL_MODULES:%=$(ERL_TOP)/lib/sasl/test/%.erl) + $(SASL_MODULES:%=$(ERL_TOP)/lib/sasl/test/%.erl) \ + $(KERNEL_MODULES:%=$(ERL_TOP)/lib/kernel/test/%.erl) EXTRA_FILES= $(ERL_TOP)/otp_versions.table @@ -135,7 +141,7 @@ COVERFILE=stdlib.cover make_emakefile: $(ERL_TOP)/make/make_emakefile $(ERL_COMPILE_FLAGS) -o$(EBIN) \ - $(MODULES) $(ERTS_MODULES) $(SASL_MODULES) \ + $(MODULES) $(ERTS_MODULES) $(SASL_MODULES) $(KERNEL_MODULES) \ > $(EMAKEFILE) tests $(TYPES): make_emakefile @@ -161,6 +167,7 @@ release_tests_spec: make_emakefile $(ERL_FILES) $(COVERFILE) $(EXTRA_FILES) "$(RELSYSDIR)" chmod -R u+w "$(RELSYSDIR)" @tar cf - *_SUITE_data property_test | (cd "$(RELSYSDIR)"; tar xf -) + $(INSTALL_DIR) "$(RELSYSDIR)/stdlib_SUITE_data" $(INSTALL_DATA) $(ERL_TOP)/make/otp_version_tickets "$(RELSYSDIR)/stdlib_SUITE_data" release_docs_spec: diff --git a/lib/stdlib/test/argparse_SUITE.erl b/lib/stdlib/test/argparse_SUITE.erl new file mode 100644 index 0000000000..fb7eaecda1 --- /dev/null +++ b/lib/stdlib/test/argparse_SUITE.erl @@ -0,0 +1,1063 @@ +%% +%% +%% Copyright Maxim Fedorov +%% +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(argparse_SUITE). +-author("maximfca@gmail.com"). + +-export([suite/0, all/0, groups/0]). + +-export([ + readme/0, readme/1, + basic/0, basic/1, + long_form_eq/0, long_form_eq/1, + built_in_types/0, built_in_types/1, + type_validators/0, type_validators/1, + invalid_arguments/0, invalid_arguments/1, + complex_command/0, complex_command/1, + unicode/0, unicode/1, + parser_error/0, parser_error/1, + nargs/0, nargs/1, + argparse/0, argparse/1, + negative/0, negative/1, + nodigits/0, nodigits/1, + pos_mixed_with_opt/0, pos_mixed_with_opt/1, + default_for_not_required/0, default_for_not_required/1, + global_default/0, global_default/1, + subcommand/0, subcommand/1, + very_short/0, very_short/1, + multi_short/0, multi_short/1, + proxy_arguments/0, proxy_arguments/1, + + usage/0, usage/1, + usage_required_args/0, usage_required_args/1, + usage_template/0, usage_template/1, + parser_error_usage/0, parser_error_usage/1, + command_usage/0, command_usage/1, + usage_width/0, usage_width/1, + + validator_exception/0, validator_exception/1, + validator_exception_format/0, validator_exception_format/1, + + run_handle/0, run_handle/1 +]). + +-include_lib("stdlib/include/assert.hrl"). + +suite() -> + [{timetrap, {seconds, 30}}]. + +groups() -> + [ + {parser, [parallel], [ + readme, basic, long_form_eq, built_in_types, type_validators, + invalid_arguments, complex_command, unicode, parser_error, + nargs, argparse, negative, nodigits, pos_mixed_with_opt, + default_for_not_required, global_default, subcommand, + very_short, multi_short, proxy_arguments + ]}, + {usage, [parallel], [ + usage, usage_required_args, usage_template, + parser_error_usage, command_usage, usage_width + ]}, + {validator, [parallel], [ + validator_exception, validator_exception_format + ]}, + {run, [parallel], [ + run_handle + ]} + ]. + +all() -> + [{group, parser}, {group, validator}, {group, usage}]. + +%%-------------------------------------------------------------------- +%% Helpers + +prog() -> + {ok, [[ProgStr]]} = init:get_argument(progname), ProgStr. + +parser_error(CmdLine, CmdMap) -> + {error, Reason} = parse(CmdLine, CmdMap), + unicode:characters_to_list(argparse:format_error(Reason)). + +parse_opts(Args, Opts) -> + argparse:parse(string:lexemes(Args, " "), #{arguments => Opts}). + +parse(Args, Command) -> + argparse:parse(string:lexemes(Args, " "), Command). + +parse_cmd(Args, Command) -> + argparse:parse(string:lexemes(Args, " "), #{commands => Command}). + +%% ubiquitous command, containing sub-commands, and all possible option types +%% with all nargs. Not all combinations though. +ubiq_cmd() -> + #{ + arguments => [ + #{name => r, short => $r, type => boolean, help => "recursive"}, + #{name => f, short => $f, type => boolean, long => "-force", help => "force"}, + #{name => v, short => $v, type => boolean, action => count, help => "verbosity level"}, + #{name => interval, short => $i, type => {integer, [{min, 1}]}, help => "interval set"}, + #{name => weird, long => "-req", help => "required optional, right?"}, + #{name => float, long => "-float", type => float, default => 3.14, help => "floating-point long form argument"} + ], + commands => #{ + "start" => #{help => "verifies configuration and starts server", + arguments => [ + #{name => server, help => "server to start"}, + #{name => shard, short => $s, type => integer, nargs => nonempty_list, help => "initial shards"}, + #{name => part, short => $p, type => integer, nargs => list, help => hidden}, + #{name => z, short => $z, type => {integer, [{min, 1}, {max, 10}]}, help => "between"}, + #{name => l, short => $l, type => {integer, [{max, 10}]}, nargs => 'maybe', help => "maybe lower"}, + #{name => more, short => $m, type => {integer, [{max, 10}]}, help => "less than 10"}, + #{name => optpos, required => false, type => {integer, []}, help => "optional positional"}, + #{name => bin, short => $b, type => {binary, <<"m">>}, help => "binary with re"}, + #{name => g, short => $g, type => {binary, <<"m">>, []}, help => "binary with re"}, + #{name => t, short => $t, type => {string, "m"}, help => "string with re"}, + #{name => e, long => "--maybe-req", required => true, type => integer, nargs => 'maybe', help => "maybe required int"}, + #{name => y, required => true, long => "-yyy", short => $y, type => {string, "m", []}, help => "string with re"}, + #{name => u, short => $u, type => {string, ["1", "2"]}, help => "string choices"}, + #{name => choice, short => $c, type => {integer, [1,2,3]}, help => "tough choice"}, + #{name => fc, short => $q, type => {float, [2.1,1.2]}, help => "floating choice"}, + #{name => ac, short => $w, type => {atom, [one, two]}, help => "atom choice"}, + #{name => au, long => "-unsafe", type => {atom, unsafe}, help => "unsafe atom"}, + #{name => as, long => "-safe", type => atom, help => <<"safe atom">>}, + #{name => name, required => false, nargs => list, help => hidden}, + #{name => long, long => "foobar", required => false, help => [<<"foobaring option">>]} + ], commands => #{ + "crawler" => #{arguments => [ + #{name => extra, long => "--extra", help => "extra option very deep"} + ], + help => "controls crawler behaviour"}, + "doze" => #{help => "dozes a bit"}} + }, + "stop" => #{help => <<"stops running server">>, arguments => [] + }, + "status" => #{help => "prints server status", arguments => [], + commands => #{ + "crawler" => #{ + arguments => [#{name => extra, long => "--extra", help => "extra option very deep"}], + help => "crawler status"}} + }, + "restart" => #{help => hidden, arguments => [ + #{name => server, help => "server to restart"}, + #{name => duo, short => $d, long => "-duo", help => "dual option"} + ]} + } + }. + +%%-------------------------------------------------------------------- +%% Parser Test Cases + +readme() -> + [{doc, "Test cases covered in the README"}]. + +readme(Config) when is_list(Config) -> + Prog = prog(), + Rm = #{ + arguments => [ + #{name => dir}, + #{name => force, short => $f, type => boolean, default => false}, + #{name => recursive, short => $r, type => boolean} + ] + }, + ?assertEqual({ok, #{dir => "dir", force => true, recursive => true}, [Prog], Rm}, + argparse:parse(["-rf", "dir"], Rm)), + %% override progname + ?assertEqual("Usage:\n readme\n", + unicode:characters_to_list(argparse:help(#{}, #{progname => "readme"}))), + ?assertEqual("Usage:\n readme\n", + unicode:characters_to_list(argparse:help(#{}, #{progname => readme}))), + ?assertEqual("Usage:\n readme\n", + unicode:characters_to_list(argparse:help(#{}, #{progname => <<"readme">>}))), + %% test that command has priority over just a positional argument: + %% - parsing "opt sub" means "find positional argument "pos", then enter subcommand + %% - parsing "sub opt" means "enter sub-command, and then find positional argument" + Cmd = #{ + commands => #{"sub" => #{}}, + arguments => [#{name => pos}] + }, + ?assertEqual(parse("opt sub", Cmd), parse("sub opt", Cmd)). + +basic() -> + [{doc, "Basic cases"}]. + +basic(Config) when is_list(Config) -> + Prog = prog(), + %% empty command, with full options path + ?assertMatch({ok, #{}, [Prog, "cmd"], #{}}, + argparse:parse(["cmd"], #{commands => #{"cmd" => #{}}})), + %% sub-command, with no path, but user-supplied argument + ?assertEqual({ok, #{}, [Prog, "cmd", "sub"], #{attr => pos}}, + argparse:parse(["cmd", "sub"], #{commands => #{"cmd" => #{commands => #{"sub" => #{attr => pos}}}}})), + %% command with positional argument + PosCmd = #{arguments => [#{name => pos}]}, + ?assertEqual({ok, #{pos => "arg"}, [Prog, "cmd"], PosCmd}, + argparse:parse(["cmd", "arg"], #{commands => #{"cmd" => PosCmd}})), + %% command with optional argument + OptCmd = #{arguments => [#{name => force, short => $f, type => boolean}]}, + ?assertEqual({ok, #{force => true}, [Prog, "rm"], OptCmd}, + parse(["rm -f"], #{commands => #{"rm" => OptCmd}}), "rm -f"), + %% command with optional and positional argument + PosOptCmd = #{arguments => [#{name => force, short => $f, type => boolean}, #{name => dir}]}, + ?assertEqual({ok, #{force => true, dir => "dir"}, [Prog, "rm"], PosOptCmd}, + parse(["rm -f dir"], #{commands => #{"rm" => PosOptCmd}}), "rm -f dir"), + %% no command, just argument list + KernelCmd = #{arguments => [#{name => kernel, long => "kernel", type => atom, nargs => 2}]}, + ?assertEqual({ok, #{kernel => [port, dist]}, [Prog], KernelCmd}, + parse(["-kernel port dist"], KernelCmd)), + %% same but positional + ArgListCmd = #{arguments => [#{name => arg, nargs => 2, type => boolean}]}, + ?assertEqual({ok, #{arg => [true, false]}, [Prog], ArgListCmd}, + parse(["true false"], ArgListCmd)). + +long_form_eq() -> + [{doc, "Tests that long form supports --arg=value"}]. + +long_form_eq(Config) when is_list(Config) -> + Prog = prog(), + %% cmd --arg=value + PosOptCmd = #{arguments => [#{name => arg, long => "-arg"}]}, + ?assertEqual({ok, #{arg => "value"}, [Prog, "cmd"], PosOptCmd}, + parse(["cmd --arg=value"], #{commands => #{"cmd" => PosOptCmd}})), + %% --integer=10 + ?assertMatch({ok, #{int := 10}, _, _}, + parse(["--int=10"], #{arguments => [#{name => int, type => integer, long => "-int"}]})). + +built_in_types() -> + [{doc, "Tests all built-in types supplied as a single argument"}]. + +% built-in types testing +built_in_types(Config) when is_list(Config) -> + Prog = [prog()], + Bool = #{arguments => [#{name => meta, type => boolean, short => $b, long => "-boolean"}]}, + ?assertEqual({ok, #{}, Prog, Bool}, parse([""], Bool)), + ?assertEqual({ok, #{meta => true}, Prog, Bool}, parse(["-b"], Bool)), + ?assertEqual({ok, #{meta => true}, Prog, Bool}, parse(["--boolean"], Bool)), + ?assertEqual({ok, #{meta => false}, Prog, Bool}, parse(["--boolean false"], Bool)), + %% integer tests + Int = #{arguments => [#{name => int, type => integer, short => $i, long => "-int"}]}, + ?assertEqual({ok, #{int => 1}, Prog, Int}, parse([" -i 1"], Int)), + ?assertEqual({ok, #{int => 1}, Prog, Int}, parse(["--int 1"], Int)), + ?assertEqual({ok, #{int => -1}, Prog, Int}, parse(["-i -1"], Int)), + %% floating point + Float = #{arguments => [#{name => f, type => float, short => $f}]}, + ?assertEqual({ok, #{f => 44.44}, Prog, Float}, parse(["-f 44.44"], Float)), + %% atoms, existing + Atom = #{arguments => [#{name => atom, type => atom, short => $a, long => "-atom"}]}, + ?assertEqual({ok, #{atom => atom}, Prog, Atom}, parse(["-a atom"], Atom)), + ?assertEqual({ok, #{atom => atom}, Prog, Atom}, parse(["--atom atom"], Atom)). + +type_validators() -> + [{doc, "Test that parser return expected conversions for valid arguments"}]. + +type_validators(Config) when is_list(Config) -> + %% successful string regexes + ?assertMatch({ok, #{str := "me"}, _, _}, + parse_opts("me", [#{name => str, type => {string, "m."}}])), + ?assertMatch({ok, #{str := "me"}, _, _}, + parse_opts("me", [#{name => str, type => {string, "m.", []}}])), + ?assertMatch({ok, #{"str" := "me"}, _, _}, + parse_opts("me", [#{name => "str", type => {string, "m.", [{capture, none}]}}])), + %% and binary too... + ?assertMatch({ok, #{bin := <<"me">>}, _, _}, + parse_opts("me", [#{name => bin, type => {binary, <<"m.">>}}])), + ?assertMatch({ok, #{<<"bin">> := <<"me">>}, _, _}, + parse_opts("me", [#{name => <<"bin">>, type => {binary, <<"m.">>, []}}])), + ?assertMatch({ok, #{bin := <<"me">>}, _, _}, + parse_opts("me", [#{name => bin, type => {binary, <<"m.">>, [{capture, none}]}}])), + %% successful integer with range validators + ?assertMatch({ok, #{int := 5}, _, _}, + parse_opts("5", [#{name => int, type => {integer, [{min, 0}, {max, 10}]}}])), + ?assertMatch({ok, #{bin := <<"5">>}, _, _}, + parse_opts("5", [#{name => bin, type => binary}])), + ?assertMatch({ok, #{str := "011"}, _, _}, + parse_opts("11", [#{name => str, type => {custom, fun(S) -> [$0|S] end}}])), + %% choices: valid + ?assertMatch({ok, #{bin := <<"K">>}, _, _}, + parse_opts("K", [#{name => bin, type => {binary, [<<"M">>, <<"K">>]}}])), + ?assertMatch({ok, #{str := "K"}, _, _}, + parse_opts("K", [#{name => str, type => {string, ["K", "N"]}}])), + ?assertMatch({ok, #{atom := one}, _, _}, + parse_opts("one", [#{name => atom, type => {atom, [one, two]}}])), + ?assertMatch({ok, #{int := 12}, _, _}, + parse_opts("12", [#{name => int, type => {integer, [10, 12]}}])), + ?assertMatch({ok, #{float := 1.3}, _, _}, + parse_opts("1.3", [#{name => float, type => {float, [1.3, 1.4]}}])), + %% test for unsafe atom + %% ensure the atom does not exist + ?assertException(error, badarg, list_to_existing_atom("$can_never_be")), + {ok, ArgMap, _, _} = parse_opts("$can_never_be", [#{name => atom, type => {atom, unsafe}}]), + argparse:validate(#{arguments => [#{name => atom, type => {atom, unsafe}}]}), + %% now that atom exists, because argparse created it (in an unsafe way!) + ?assertEqual(list_to_existing_atom("$can_never_be"), maps:get(atom, ArgMap)), + %% test successful user-defined conversion + ?assertMatch({ok, #{user := "VER"}, _, _}, + parse_opts("REV", [#{name => user, type => {custom, fun (Str) -> lists:reverse(Str) end}}])). + +invalid_arguments() -> + [{doc, "Test that parser return errors for invalid arguments"}]. + +invalid_arguments(Config) when is_list(Config) -> + %% {float, [{min, float()} | {max, float()}]} | + Prog = [prog()], + MinFloat = #{name => float, type => {float, [{min, 1.0}]}}, + ?assertEqual({error, {Prog, MinFloat, "0.0", <<"is less than accepted minimum">>}}, + parse_opts("0.0", [MinFloat])), + MaxFloat = #{name => float, type => {float, [{max, 1.0}]}}, + ?assertEqual({error, {Prog, MaxFloat, "2.0", <<"is greater than accepted maximum">>}}, + parse_opts("2.0", [MaxFloat])), + %% {int, [{min, integer()} | {max, integer()}]} | + MinInt = #{name => int, type => {integer, [{min, 20}]}}, + ?assertEqual({error, {Prog, MinInt, "10", <<"is less than accepted minimum">>}}, + parse_opts("10", [MinInt])), + MaxInt = #{name => int, type => {integer, [{max, -10}]}}, + ?assertEqual({error, {Prog, MaxInt, "-5", <<"is greater than accepted maximum">>}}, + parse_opts("-5", [MaxInt])), + %% string: regex & regex with options + %% {string, string()} | {string, string(), []} + StrRegex = #{name => str, type => {string, "me.me"}}, + ?assertEqual({error, {Prog, StrRegex, "me", <<"does not match">>}}, + parse_opts("me", [StrRegex])), + StrRegexOpt = #{name => str, type => {string, "me.me", []}}, + ?assertEqual({error, {Prog, StrRegexOpt, "me", <<"does not match">>}}, + parse_opts("me", [StrRegexOpt])), + %% {binary, {re, binary()} | {re, binary(), []} + BinRegex = #{name => bin, type => {binary, <<"me.me">>}}, + ?assertEqual({error, {Prog, BinRegex, "me", <<"does not match">>}}, + parse_opts("me", [BinRegex])), + BinRegexOpt = #{name => bin, type => {binary, <<"me.me">>, []}}, + ?assertEqual({error, {Prog, BinRegexOpt, "me", <<"does not match">>}}, + parse_opts("me", [BinRegexOpt])), + %% invalid integer (comma , is not parsed) + ?assertEqual({error, {Prog, MinInt, "1,", <<"is not an integer">>}}, + parse_opts(["1,"], [MinInt])), + %% test invalid choices + BinChoices = #{name => bin, type => {binary, [<<"M">>, <<"N">>]}}, + ?assertEqual({error, {Prog, BinChoices, "K", <<"is not one of the choices">>}}, + parse_opts("K", [BinChoices])), + StrChoices = #{name => str, type => {string, ["M", "N"]}}, + ?assertEqual({error, {Prog, StrChoices, "K", <<"is not one of the choices">>}}, + parse_opts("K", [StrChoices])), + AtomChoices = #{name => atom, type => {atom, [one, two]}}, + ?assertEqual({error, {Prog, AtomChoices, "K", <<"is not one of the choices">>}}, + parse_opts("K", [AtomChoices])), + IntChoices = #{name => int, type => {integer, [10, 11]}}, + ?assertEqual({error, {Prog, IntChoices, "12", <<"is not one of the choices">>}}, + parse_opts("12", [IntChoices])), + FloatChoices = #{name => float, type => {float, [1.2, 1.4]}}, + ?assertEqual({error, {Prog, FloatChoices, "1.3", <<"is not one of the choices">>}}, + parse_opts("1.3", [FloatChoices])), + %% unsuccessful user-defined conversion + ?assertMatch({error, {Prog, _, "REV", <<"failed faildation">>}}, + parse_opts("REV", [#{name => user, type => {custom, fun (Str) -> integer_to_binary(Str) end}}])). + +complex_command() -> + [{doc, "Parses a complex command that has a mix of optional and positional arguments"}]. + +complex_command(Config) when is_list(Config) -> + Command = #{arguments => [ + %% options + #{name => string, short => $s, long => "-string", action => append, help => "String list option"}, + #{name => boolean, type => boolean, short => $b, action => append, help => "Boolean list option"}, + #{name => float, type => float, short => $f, long => "-float", action => append, help => "Float option"}, + %% positional args + #{name => integer, type => integer, help => "Integer variable"}, + #{name => string, help => "alias for string option", action => extend, nargs => list} + ]}, + CmdMap = #{commands => #{"start" => Command}}, + Parsed = argparse:parse(string:lexemes("start --float 1.04 -f 112 -b -b -s s1 42 --string s2 s3 s4", " "), CmdMap), + Expected = #{float => [1.04, 112], boolean => [true, true], integer => 42, string => ["s1", "s2", "s3", "s4"]}, + ?assertEqual({ok, Expected, [prog(), "start"], Command}, Parsed). + +unicode() -> + [{doc, "Tests basic unicode support"}]. + +unicode(Config) when is_list(Config) -> + %% test unicode short & long + ?assertMatch({ok, #{one := true}, _, _}, + parse(["-ะค"], #{arguments => [#{name => one, short => $ะค, type => boolean}]})), + ?assertMatch({ok, #{long := true}, _, _}, + parse(["--รฅรครถ"], #{arguments => [#{name => long, long => "-รฅรครถ", type => boolean}]})), + %% test default, help and value in unicode + Cmd = #{arguments => [#{name => text, type => binary, help => "รฅรครถ", default => <<"โ
"/utf8>>}]}, + Expected = #{text => <<"โ
"/utf8>>}, + Prog = [prog()], + ?assertEqual({ok, Expected, Prog, Cmd}, argparse:parse([], Cmd)), %% default + ?assertEqual({ok, Expected, Prog, Cmd}, argparse:parse(["โ
"], Cmd)), %% specified in the command line + ?assertEqual("Usage:\n " ++ prog() ++ " <text>\n\nArguments:\n text รฅรครถ (binary, โ
)\n", + unicode:characters_to_list(argparse:help(Cmd))), + %% test command name and argument name in unicode + Uni = #{commands => #{"รฅรครถ" => #{help => "รถะค"}}, handler => optional, + arguments => [#{name => "ะค", short => $รค, long => "รฅรครถ"}]}, + UniExpected = "Usage:\n " ++ prog() ++ + " {รฅรครถ} [-รค <ะค>] [-รฅรครถ <ะค>]\n\nSubcommands:\n รฅรครถ รถะค\n\nOptional arguments:\n -รค, -รฅรครถ ะค\n", + ?assertEqual(UniExpected, unicode:characters_to_list(argparse:help(Uni))), + ParsedExpected = #{"ะค" => "รถะค"}, + ?assertEqual({ok, ParsedExpected, Prog, Uni}, argparse:parse(["-รค", "รถะค"], Uni)). + +parser_error() -> + [{doc, "Tests error tuples that the parser returns"}]. + +parser_error(Config) when is_list(Config) -> + Prog = prog(), + %% unknown option at the top of the path + ?assertEqual({error, {[Prog], undefined, "arg", <<>>}}, + parse_cmd(["arg"], #{})), + %% positional argument missing in a sub-command + Opt = #{name => mode, required => true}, + ?assertMatch({error, {[Prog, "start"], _, undefined, <<>>}}, + parse_cmd(["start"], #{"start" => #{arguments => [Opt]}})), + %% optional argument missing in a sub-command + Opt1 = #{name => mode, short => $o, required => true}, + ?assertMatch({error, {[Prog, "start"], _, undefined, <<>>}}, + parse_cmd(["start"], #{"start" => #{arguments => [Opt1]}})), + %% positional argument: an atom that does not exist + Opt2 = #{name => atom, type => atom}, + ?assertEqual({error, {[Prog], Opt2, "boo-foo", <<"is not an existing atom">>}}, + parse_opts(["boo-foo"], [Opt2])), + %% optional argument missing some items + Opt3 = #{name => kernel, long => "kernel", type => atom, nargs => 2}, + ?assertEqual({error, {[Prog], Opt3, ["port"], "expected 2, found 1 argument(s)"}}, + parse_opts(["-kernel port"], [Opt3])), + %% positional argument missing some items + Opt4 = #{name => arg, type => atom, nargs => 3}, + ?assertEqual({error, {[Prog], Opt4, ["p1"], "expected 3, found 1 argument(s)"}}, + parse_opts(["p1"], [Opt4])), + %% short option with no argument, when it's needed + ?assertMatch({error, {_, _, undefined, <<"expected argument">>}}, + parse("-1", #{arguments => [#{name => short49, short => 49}]})). + +nargs() -> + [{doc, "Tests argument consumption option, with nargs"}]. + +nargs(Config) when is_list(Config) -> + Prog = [prog()], + %% consume optional list arguments + Opts = [ + #{name => arg, short => $s, nargs => list, type => integer}, + #{name => bool, short => $b, type => boolean} + ], + ?assertMatch({ok, #{arg := [1, 2, 3], bool := true}, _, _}, + parse_opts(["-s 1 2 3 -b"], Opts)), + %% consume one_or_more arguments in an optional list + Opts2 = [ + #{name => arg, short => $s, nargs => nonempty_list}, + #{name => extra, short => $x} + ], + ?assertMatch({ok, #{extra := "X", arg := ["a","b","c"]}, _, _}, + parse_opts(["-s port -s a b c -x X"], Opts2)), + %% error if there is no argument to consume + ?assertMatch({error, {_, _, ["-x"], <<"expected argument">>}}, + parse_opts(["-s -x"], Opts2)), + %% error when positional has nargs = nonempty_list or pos_integer + ?assertMatch({error, {_, _, undefined, <<>>}}, + parse_opts([""], [#{name => req, nargs => nonempty_list}])), + %% positional arguments consumption: one or more positional argument + OptsPos1 = #{arguments => [ + #{name => arg, nargs => nonempty_list}, + #{name => extra, short => $x} + ]}, + ?assertEqual({ok, #{extra => "X", arg => ["b","c"]}, Prog, OptsPos1}, + parse(["-x port -x a b c -x X"], OptsPos1)), + %% positional arguments consumption, any number (maybe zero) + OptsPos2 = #{arguments => [ + #{name => arg, nargs => list}, + #{name => extra, short => $x} + ]}, + ?assertEqual({ok, #{extra => "X", arg => ["a","b","c"]}, Prog, OptsPos2}, + parse(["-x port a b c -x X"], OptsPos2)), + %% positional: consume ALL arguments! + OptsAll = #{arguments => [ + #{name => arg, nargs => all}, + #{name => extra, short => $x} + ]}, + ?assertEqual({ok, #{extra => "port", arg => ["a","b","c", "-x", "X"]}, Prog, OptsAll}, + parse(["-x port a b c -x X"], OptsAll)), + %% maybe with a specified default + OptMaybe = [ + #{name => foo, long => "-foo", nargs => {'maybe', c}, default => d}, + #{name => bar, nargs => 'maybe', default => d} + ], + ?assertMatch({ok, #{foo := "YY", bar := "XX"}, Prog, _}, + parse_opts(["XX --foo YY"], OptMaybe)), + ?assertMatch({ok, #{foo := c, bar := "XX"}, Prog, _}, + parse_opts(["XX --foo"], OptMaybe)), + ?assertMatch({ok, #{foo := d, bar := d}, Prog, _}, + parse_opts([""], OptMaybe)), + %% maybe with default provided by argparse + ?assertMatch({ok, #{foo := d, bar := "XX", baz := ok}, _, _}, + parse_opts(["XX -b"], [#{name => baz, nargs => 'maybe', short => $b, default => ok} | OptMaybe])), + %% maybe arg - with no default given + ?assertMatch({ok, #{foo := d, bar := "XX", baz := 0}, _, _}, + parse_opts(["XX -b"], [#{name => baz, nargs => 'maybe', short => $b, type => integer} | OptMaybe])), + ?assertMatch({ok, #{foo := d, bar := "XX", baz := ""}, _, _}, + parse_opts(["XX -b"], [#{name => baz, nargs => 'maybe', short => $b, type => string} | OptMaybe])), + ?assertMatch({ok, #{foo := d, bar := "XX", baz := undefined}, _, _}, + parse_opts(["XX -b"], [#{name => baz, nargs => 'maybe', short => $b, type => atom} | OptMaybe])), + ?assertMatch({ok, #{foo := d, bar := "XX", baz := <<"">>}, _, _}, + parse_opts(["XX -b"], [#{name => baz, nargs => 'maybe', short => $b, type => binary} | OptMaybe])), + %% nargs: optional list, yet it still needs to be 'not required'! + OptList = [#{name => arg, nargs => list, required => false, type => integer}], + ?assertEqual({ok, #{}, Prog, #{arguments => OptList}}, parse_opts("", OptList)), + %% tests that action "count" with nargs "maybe" counts two times, first time + %% consuming an argument (for "maybe"), second time just counting + Cmd = #{arguments => [ + #{name => short49, short => $1, long => "-force", action => count, nargs => 'maybe'}]}, + ?assertEqual({ok, #{short49 => 2}, Prog, Cmd}, + parse("-1 arg1 --force", Cmd)). + +argparse() -> + [{doc, "Tests success cases, inspired by argparse in Python"}]. + +argparse(Config) when is_list(Config) -> + Prog = [prog()], + Parser = #{arguments => [ + #{name => sum, long => "-sum", action => {store, sum}, default => max}, + #{name => integers, type => integer, nargs => nonempty_list} + ]}, + ?assertEqual({ok, #{integers => [1, 2, 3, 4], sum => max}, Prog, Parser}, + parse("1 2 3 4", Parser)), + ?assertEqual({ok, #{integers => [1, 2, 3, 4], sum => sum}, Prog, Parser}, + parse("1 2 3 4 --sum", Parser)), + ?assertEqual({ok, #{integers => [7, -1, 42], sum => sum}, Prog, Parser}, + parse("--sum 7 -1 42", Parser)), + %% name or flags + Parser2 = #{arguments => [ + #{name => bar, required => true}, + #{name => foo, short => $f, long => "-foo"} + ]}, + ?assertEqual({ok, #{bar => "BAR"}, Prog, Parser2}, parse("BAR", Parser2)), + ?assertEqual({ok, #{bar => "BAR", foo => "FOO"}, Prog, Parser2}, parse("BAR --foo FOO", Parser2)), + %PROG: error: the following arguments are required: bar + ?assertMatch({error, {Prog, _, undefined, <<>>}}, parse("--foo FOO", Parser2)), + %% action tests: default + ?assertMatch({ok, #{foo := "1"}, Prog, _}, + parse("--foo 1", #{arguments => [#{name => foo, long => "-foo"}]})), + %% action test: store + ?assertMatch({ok, #{foo := 42}, Prog, _}, + parse("--foo", #{arguments => [#{name => foo, long => "-foo", action => {store, 42}}]})), + %% action tests: boolean (variants) + ?assertMatch({ok, #{foo := true}, Prog, _}, + parse("--foo", #{arguments => [#{name => foo, long => "-foo", action => {store, true}}]})), + ?assertMatch({ok, #{foo := 42}, Prog, _}, + parse("--foo", #{arguments => [#{name => foo, long => "-foo", type => boolean, action => {store, 42}}]})), + ?assertMatch({ok, #{foo := true}, Prog, _}, + parse("--foo", #{arguments => [#{name => foo, long => "-foo", type => boolean}]})), + ?assertMatch({ok, #{foo := true}, Prog, _}, + parse("--foo true", #{arguments => [#{name => foo, long => "-foo", type => boolean}]})), + ?assertMatch({ok, #{foo := false}, Prog, _}, + parse("--foo false", #{arguments => [#{name => foo, long => "-foo", type => boolean}]})), + %% action tests: append & append_const + ?assertMatch({ok, #{all := [1, "1"]}, Prog, _}, + parse("--x 1 -x 1", #{arguments => [ + #{name => all, long => "-x", type => integer, action => append}, + #{name => all, short => $x, action => append}]})), + ?assertMatch({ok, #{all := ["Z", 2]}, Prog, _}, + parse("--x -x", #{arguments => [ + #{name => all, long => "-x", action => {append, "Z"}}, + #{name => all, short => $x, action => {append, 2}}]})), + %% count: + ?assertMatch({ok, #{v := 3}, Prog, _}, + parse("-v -v -v", #{arguments => [#{name => v, short => $v, action => count}]})). + +negative() -> + [{doc, "Test negative number parser"}]. + +negative(Config) when is_list(Config) -> + Parser = #{arguments => [ + #{name => x, short => $x, type => integer, action => store}, + #{name => foo, nargs => 'maybe', required => false} + ]}, + ?assertMatch({ok, #{x := -1}, _, _}, parse("-x -1", Parser)), + ?assertMatch({ok, #{x := -1, foo := "-5"}, _, _}, parse("-x -1 -5", Parser)), + %% + Parser2 = #{arguments => [ + #{name => one, short => $1}, + #{name => foo, nargs => 'maybe', required => false} + ]}, + + %% negative number options present, so -1 is an option + ?assertMatch({ok, #{one := "X"}, _, _}, parse("-1 X", Parser2)), + %% negative number options present, so -2 is an option + ?assertMatch({error, {_, undefined, "-2", _}}, parse("-2", Parser2)), + + %% negative number options present, so both -1s are options + ?assertMatch({error, {_, _, undefined, _}}, parse("-1 -1", Parser2)), + %% no "-" prefix, can only be an integer + ?assertMatch({ok, #{foo := "-1"}, _, _}, argparse:parse(["-1"], Parser2, #{prefixes => "+"})), + %% no "-" prefix, can only be an integer, but just one integer! + ?assertMatch({error, {_, undefined, "-1", _}}, + argparse:parse(["-2", "-1"], Parser2, #{prefixes => "+"})), + %% just in case, floats work that way too... + ?assertMatch({error, {_, undefined, "-2", _}}, + parse("-2", #{arguments => [#{name => one, long => "1.2"}]})). + +nodigits() -> + [{doc, "Test prefixes and negative numbers together"}]. + +nodigits(Config) when is_list(Config) -> + %% verify nodigits working as expected + Parser3 = #{arguments => [ + #{name => extra, short => $3}, + #{name => arg, nargs => list} + ]}, + %% ensure not to consume optional prefix + ?assertEqual({ok, #{extra => "X", arg => ["a","b","3"]}, [prog()], Parser3}, + argparse:parse(string:lexemes("-3 port a b 3 +3 X", " "), Parser3, #{prefixes => "-+"})). + +pos_mixed_with_opt() -> + [{doc, "Tests that optional argument correctly consumes expected argument" + "inspired by https://github.com/python/cpython/issues/59317"}]. + +pos_mixed_with_opt(Config) when is_list(Config) -> + Parser = #{arguments => [ + #{name => pos}, + #{name => opt, default => 24, type => integer, long => "-opt"}, + #{name => vars, nargs => list} + ]}, + ?assertEqual({ok, #{pos => "1", opt => 8, vars => ["8", "9"]}, [prog()], Parser}, + parse("1 2 --opt 8 8 9", Parser)). + +default_for_not_required() -> + [{doc, "Tests that default value is used for non-required positional argument"}]. + +default_for_not_required(Config) when is_list(Config) -> + ?assertMatch({ok, #{def := 1}, _, _}, + parse("", #{arguments => [#{name => def, short => $d, required => false, default => 1}]})), + ?assertMatch({ok, #{def := 1}, _, _}, + parse("", #{arguments => [#{name => def, required => false, default => 1}]})). + +global_default() -> + [{doc, "Tests that a global default can be enabled for all non-required arguments"}]. + +global_default(Config) when is_list(Config) -> + ?assertMatch({ok, #{def := "global"}, _, _}, + argparse:parse("", #{arguments => [#{name => def, type => integer, required => false}]}, + #{default => "global"})). + +subcommand() -> + [{doc, "Tests subcommands parser"}]. + +subcommand(Config) when is_list(Config) -> + TwoCmd = #{arguments => [#{name => bar}]}, + Cmd = #{ + arguments => [#{name => force, type => boolean, short => $f}], + commands => #{"one" => #{ + arguments => [#{name => foo, type => boolean, long => "-foo"}, #{name => baz}], + commands => #{ + "two" => TwoCmd}}}}, + ?assertEqual({ok, #{force => true, baz => "N1O1O", foo => true, bar => "bar"}, [prog(), "one", "two"], TwoCmd}, + parse("one N1O1O -f two --foo bar", Cmd)), + %% it is an error not to choose subcommand + ?assertEqual({error, {[prog(), "one"], undefined, undefined, <<"subcommand expected">>}}, + parse("one N1O1O -f", Cmd)). + +very_short() -> + [{doc, "Tests short option appended to the optional itself"}]. + +very_short(Config) when is_list(Config) -> + ?assertMatch({ok, #{x := "V"}, _, _}, + parse("-xV", #{arguments => [#{name => x, short => $x}]})). + +multi_short() -> + [{doc, "Tests multiple short arguments blend into one"}]. + +multi_short(Config) when is_list(Config) -> + %% ensure non-flammable argument does not explode, even when it's possible + ?assertMatch({ok, #{v := "xv"}, _, _}, + parse("-vxv", #{arguments => [#{name => v, short => $v}, #{name => x, short => $x}]})), + %% ensure 'verbosity' use-case works + ?assertMatch({ok, #{v := 3}, _, _}, + parse("-vvv", #{arguments => [#{name => v, short => $v, action => count}]})), + %% + ?assertMatch({ok, #{recursive := true, force := true, path := "dir"}, _, _}, + parse("-rf dir", #{arguments => [ + #{name => recursive, short => $r, type => boolean}, + #{name => force, short => $f, type => boolean}, + #{name => path} + ]})). + +proxy_arguments() -> + [{doc, "Tests nargs => all used to proxy arguments to another script"}]. + +proxy_arguments(Config) when is_list(Config) -> + Cmd = #{ + commands => #{ + "start" => #{ + arguments => [ + #{name => shell, short => $s, long => "-shell", type => boolean}, + #{name => skip, short => $x, long => "-skip", type => boolean}, + #{name => args, required => false, nargs => all} + ] + }, + "stop" => #{}, + "status" => #{ + arguments => [ + #{name => skip, required => false, default => "ok"}, + #{name => args, required => false, nargs => all} + ]}, + "state" => #{ + arguments => [ + #{name => skip, required => false}, + #{name => args, required => false, nargs => all} + ]} + }, + arguments => [ + #{name => node} + ], + handler => fun (#{}) -> ok end + }, + Prog = prog(), + ?assertMatch({ok, #{node := "node1"}, _, _}, parse("node1", Cmd)), + ?assertMatch({ok, #{node := "node1"}, [Prog, "stop"], #{}}, parse("node1 stop", Cmd)), + ?assertMatch({ok, #{node := "node2.org", shell := true, skip := true}, _, _}, parse("node2.org start -x -s", Cmd)), + ?assertMatch({ok, #{args := ["-app","key","value"],node := "node1.org"}, [Prog, "start"], _}, + parse("node1.org start -app key value", Cmd)), + ?assertMatch({ok, #{args := ["-app","key","value", "-app2", "key2", "value2"], + node := "node3.org", shell := true}, [Prog, "start"], _}, + parse("node3.org start -s -app key value -app2 key2 value2", Cmd)), + %% test that any non-required positionals are skipped + ?assertMatch({ok, #{args := ["-a","bcd"], node := "node2.org", skip := "ok"}, _, _}, parse("node2.org status -a bcd", Cmd)), + ?assertMatch({ok, #{args := ["-app", "key"], node := "node2.org"}, _, _}, parse("node2.org state -app key", Cmd)). + +%%-------------------------------------------------------------------- +%% Usage Test Cases + +usage() -> + [{doc, "Basic tests for help formatter, including 'hidden' help"}]. + +usage(Config) when is_list(Config) -> + Cmd = ubiq_cmd(), + Usage = "Usage:\n erl start {crawler|doze} [-lrfv] [-s <shard>...] [-z <z>] [-m <more>] [-b <bin>]\n" + " [-g <g>] [-t <t>] ---maybe-req -y <y> --yyy <y> [-u <u>] [-c <choice>]\n" + " [-q <fc>] [-w <ac>] [--unsafe <au>] [--safe <as>] [-foobar <long>] [--force]\n" + " [-i <interval>] [--req <weird>] [--float <float>] <server> [<optpos>]\n\n" + "Subcommands:\n" + " crawler controls crawler behaviour\n" + " doze dozes a bit\n\n" + "Arguments:\n" + " server server to start\n" + " optpos optional positional (int)\n\n" + "Optional arguments:\n" + " -s initial shards (int)\n" + " -z between (1 <= int <= 10)\n" + " -l maybe lower (int <= 10)\n" + " -m less than 10 (int <= 10)\n" + " -b binary with re (binary re: m)\n" + " -g binary with re (binary re: m)\n" + " -t string with re (string re: m)\n" + " ---maybe-req maybe required int (int)\n" + " -y, --yyy string with re (string re: m)\n" + " -u string choices (choice: 1, 2)\n" + " -c tough choice (choice: 1, 2, 3)\n" + " -q floating choice (choice: 2.10000, 1.20000)\n" + " -w atom choice (choice: one, two)\n" + " --unsafe unsafe atom (atom)\n" + " --safe safe atom (existing atom)\n" + " -foobar foobaring option\n" + " -r recursive\n" + " -f, --force force\n" + " -v verbosity level\n" + " -i interval set (int >= 1)\n" + " --req required optional, right?\n" + " --float floating-point long form argument (float, 3.14)\n", + ?assertEqual(Usage, unicode:characters_to_list(argparse:help(Cmd, + #{progname => "erl", command => ["start"]}))), + FullCmd = "Usage:\n erl" + " <command> [-rfv] [--force] [-i <interval>] [--req <weird>] [--float <float>]\n\n" + "Subcommands:\n" + " start verifies configuration and starts server\n" + " status prints server status\n" + " stop stops running server\n\n" + "Optional arguments:\n" + " -r recursive\n" + " -f, --force force\n" + " -v verbosity level\n" + " -i interval set (int >= 1)\n" + " --req required optional, right?\n" + " --float floating-point long form argument (float, 3.14)\n", + ?assertEqual(FullCmd, unicode:characters_to_list(argparse:help(Cmd, + #{progname => erl}))), + CrawlerStatus = "Usage:\n erl status crawler [-rfv] [---extra <extra>] [--force] [-i <interval>]\n" + " [--req <weird>] [--float <float>]\n\nOptional arguments:\n" + " ---extra extra option very deep\n -r recursive\n" + " -f, --force force\n -v verbosity level\n" + " -i interval set (int >= 1)\n" + " --req required optional, right?\n" + " --float floating-point long form argument (float, 3.14)\n", + ?assertEqual(CrawlerStatus, unicode:characters_to_list(argparse:help(Cmd, + #{progname => "erl", command => ["status", "crawler"]}))), + ok. + +usage_required_args() -> + [{doc, "Verify that required args are printed as required in usage"}]. + +usage_required_args(Config) when is_list(Config) -> + Cmd = #{commands => #{"test" => #{arguments => [#{name => required, required => true, long => "-req"}]}}}, + Expected = "Usage:\n " ++ prog() ++ " test --req <required>\n\nOptional arguments:\n --req required\n", + ?assertEqual(Expected, unicode:characters_to_list(argparse:help(Cmd, #{command => ["test"]}))). + +usage_template() -> + [{doc, "Tests templates in help/usage"}]. + +usage_template(Config) when is_list(Config) -> + %% Argument (positional) + Cmd = #{arguments => [#{ + name => shard, + type => integer, + default => 0, + help => {"[-s SHARD]", ["initial number, ", type, <<" with a default value of ">>, default]}} + ]}, + ?assertEqual("Usage:\n " ++ prog() ++ " [-s SHARD]\n\nArguments:\n shard initial number, int with a default value of 0\n", + unicode:characters_to_list(argparse:help(Cmd, #{}))), + %% Optional + Cmd1 = #{arguments => [#{ + name => shard, + short => $s, + type => integer, + default => 0, + help => {<<"[-s SHARD]">>, ["initial number"]}} + ]}, + ?assertEqual("Usage:\n " ++ prog() ++ " [-s SHARD]\n\nOptional arguments:\n -s initial number\n", + unicode:characters_to_list(argparse:help(Cmd1, #{}))), + %% ISO Date example + DefaultRange = {{2020, 1, 1}, {2020, 6, 22}}, + CmdISO = #{ + arguments => [ + #{ + name => range, + long => "-range", + short => $r, + help => {"[--range RNG]", fun() -> + {{FY, FM, FD}, {TY, TM, TD}} = DefaultRange, + lists:flatten(io_lib:format("date range, ~b-~b-~b..~b-~b-~b", [FY, FM, FD, TY, TM, TD])) + end}, + type => {custom, fun(S) -> [S, DefaultRange] end}, + default => DefaultRange + } + ] + }, + ?assertEqual("Usage:\n " ++ prog() ++ " [--range RNG]\n\nOptional arguments:\n -r, --range date range, 2020-1-1..2020-6-22\n", + unicode:characters_to_list(argparse:help(CmdISO, #{}))), + ok. + +parser_error_usage() -> + [{doc, "Tests that parser errors have corresponding usage text"}]. + +parser_error_usage(Config) when is_list(Config) -> + %% unknown arguments + Prog = prog(), + ?assertEqual(Prog ++ ": unknown argument: arg", parser_error(["arg"], #{})), + ?assertEqual(Prog ++ ": unknown argument: -a", parser_error(["-a"], #{})), + %% missing argument + ?assertEqual(Prog ++ ": required argument missing: need", parser_error([""], + #{arguments => [#{name => need}]})), + ?assertEqual(Prog ++ ": required argument missing: need", parser_error([""], + #{arguments => [#{name => need, short => $n, required => true}]})), + %% invalid value + ?assertEqual(Prog ++ ": invalid argument for need: foo is not an integer", parser_error(["foo"], + #{arguments => [#{name => need, type => integer}]})), + ?assertEqual(Prog ++ ": invalid argument for need: cAnNotExIsT is not an existing atom", parser_error(["cAnNotExIsT"], + #{arguments => [#{name => need, type => atom}]})). + +command_usage() -> + [{doc, "Test command help template"}]. + +command_usage(Config) when is_list(Config) -> + Cmd = #{arguments => [ + #{name => arg, help => "argument help"}, #{name => opt, short => $o, help => "option help"}], + help => ["Options:\n", options, arguments, <<"NOTAUSAGE">>, usage, "\n"] + }, + ?assertEqual("Options:\n -o option help\n arg argument help\nNOTAUSAGE " ++ prog() ++ " [-o <opt>] <arg>\n", + unicode:characters_to_list(argparse:help(Cmd, #{}))). + +usage_width() -> + [{doc, "Test usage fitting in the viewport"}]. + +usage_width(Config) when is_list(Config) -> + Cmd = #{arguments => [ + #{name => arg, help => "argument help that spans way over allowed viewport width, wrapping words"}, + #{name => opt, short => $o, long => "-option_long_name", + help => "another quite long word wrapped thing spanning over several lines"}, + #{name => v, short => $v, type => boolean}, + #{name => q, short => $q, type => boolean}], + commands => #{ + "cmd1" => #{help => "Help for command number 1, not fitting at all"}, + "cmd2" => #{help => <<"Short help">>}, + "cmd3" => #{help => "Yet another instance of a very long help message"} + }, + help => " Very long help line taking much more than 40 characters allowed by the test case. +Also containing a few newlines. + + Indented new lines must be honoured!" + }, + + Expected = "Usage:\n erl {cmd1|cmd2|cmd3} [-vq] [-o <opt>]\n" + " [--option_long_name <opt>] <arg>\n\n" + " Very long help line taking much more\n" + "than 40 characters allowed by the test\n" + "case.\n" + "Also containing a few newlines.\n\n" + " Indented new lines must be honoured!\n\n" + "Subcommands:\n" + " cmd1 Help for\n" + " command number\n" + " 1, not fitting\n" + " at all\n" + " cmd2 Short help\n" + " cmd3 Yet another\n" + " instance of a\n" + " very long help\n" + " message\n\n" + "Arguments:\n" + " arg argument help\n" + " that spans way\n" + " over allowed\n" + " viewport width,\n" + " wrapping words\n\n" + "Optional arguments:\n" + " -o, --option_long_name another quite\n" + " long word\n" + " wrapped thing\n" + " spanning over\n" + " several lines\n" + " -v v\n" + " -q q\n", + + ?assertEqual(Expected, unicode:characters_to_list(argparse:help(Cmd, #{columns => 40, progname => "erl"}))). + +%%-------------------------------------------------------------------- +%% Validator Test Cases + +validator_exception() -> + [{doc, "Tests that the validator throws expected exceptions"}]. + +validator_exception(Config) when is_list(Config) -> + Prg = [prog()], + %% conflicting option names + ?assertException(error, {argparse, argument, Prg, short, "short conflicting with previously defined short for one"}, + argparse:validate(#{arguments => [#{name => one, short => $$}, #{name => two, short => $$}]})), + ?assertException(error, {argparse, argument, Prg, long, "long conflicting with previously defined long for one"}, + argparse:validate(#{arguments => [#{name => one, long => "a"}, #{name => two, long => "a"}]})), + %% broken options + %% long must be a string + ?assertException(error, {argparse, argument, Prg, long, _}, + argparse:validate(#{arguments => [#{name => one, long => ok}]})), + %% short must be a printable character + ?assertException(error, {argparse, argument, Prg, short, _}, + argparse:validate(#{arguments => [#{name => one, short => ok}]})), + ?assertException(error, {argparse, argument, Prg, short, _}, + argparse:validate(#{arguments => [#{name => one, short => 7}]})), + %% required is a boolean + ?assertException(error, {argparse, argument, Prg, required, _}, + argparse:validate(#{arguments => [#{name => one, required => ok}]})), + ?assertException(error, {argparse, argument, Prg, help, _}, + argparse:validate(#{arguments => [#{name => one, help => ok}]})), + %% broken commands + try argparse:help(#{}, #{progname => 123}), ?assert(false) + catch error:badarg:Stack -> + [{_, _, _, Ext} | _] = Stack, + #{cause := #{2 := Detail}} = proplists:get_value(error_info, Ext), + ?assertEqual(<<"progname is not valid">>, Detail) + end, + %% not-a-list of arguments provided to a subcommand + Prog = prog(), + ?assertException(error, {argparse, command, [Prog, "start"], arguments, <<"expected a list, [argument()]">>}, + argparse:validate(#{commands => #{"start" => #{arguments => atom}}})), + %% command is not a map + ?assertException(error, {argparse, command, Prg, commands, <<"expected map of #{string() => command()}">>}, + argparse:validate(#{commands => []})), + %% invalid commands field + ?assertException(error, {argparse, command, Prg, commands, _}, + argparse:validate(#{commands => ok})), + ?assertException(error, {argparse, command, _, commands, _}, + argparse:validate(#{commands => #{ok => #{}}})), + ?assertException(error, {argparse, command, _, help, + <<"must be a printable unicode list, or a command help template">>}, + argparse:validate(#{commands => #{"ok" => #{help => ok}}})), + ?assertException(error, {argparse, command, _, handler, _}, + argparse:validate(#{commands => #{"ok" => #{handler => fun validator_exception/0}}})), + %% extend + maybe: validator exception + ?assertException(error, {argparse, argument, _, action, <<"extend action works only with lists">>}, + parse("-1 -1", #{arguments => + [#{action => extend, name => short49, nargs => 'maybe', short => 49}]})). + +validator_exception_format() -> + [{doc, "Tests human-readable (EEP-54) format for exceptions thrown by the validator"}]. + +validator_exception_format(Config) when is_list(Config) -> + %% set up as a contract: test that EEP-54 transformation is done (but don't check strings) + try + argparse:validate(#{commands => #{"one" => #{commands => #{"two" => atom}}}}), + ?assert(false) + catch + error:R1:S1 -> + #{1 := Cmd, reason := RR1, general := G} = argparse:format_error(R1, S1), + ?assertEqual("command specification is invalid", unicode:characters_to_list(G)), + ?assertEqual("command \"" ++ prog() ++ " one two\": invalid field 'commands', reason: expected command()", + unicode:characters_to_list(RR1)), + ?assertEqual(["atom"], Cmd) + end, + %% check argument + try + argparse:validate(#{arguments => [#{}]}), + ?assert(false) + catch + error:R2:S2 -> + #{1 := Cmd2, reason := RR2, general := G2} = argparse:format_error(R2, S2), + ?assertEqual("argument specification is invalid", unicode:characters_to_list(G2)), + ?assertEqual("command \"" ++ prog() ++ + "\", argument '', invalid field 'name': argument must be a map containing 'name' field", + unicode:characters_to_list(RR2)), + ?assertEqual(["#{}"], Cmd2) + end. + +%%-------------------------------------------------------------------- +%% Validator Test Cases + +run_handle() -> + [{doc, "Very basic tests for argparse:run/3, choice of handlers formats"}]. + +%% fun((arg_map()) -> term()) | %% handler accepting arg_map +%% {module(), Fn :: atom()} | %% handler, accepting arg_map, Fn exported from module() +%% {fun(() -> term()), term()} | %% handler, positional form (term() is supplied for omitted args) +%% {module(), atom(), term()} + +run_handle(Config) when is_list(Config) -> + %% no subcommand, basic fun handler with argmap + ?assertEqual(6, + argparse:run(["-i", "3"], #{handler => fun (#{in := Val}) -> Val * 2 end, + arguments => [#{name => in, short => $i, type => integer}]}, #{})), + %% subcommand, positional fun() handler + ?assertEqual(6, + argparse:run(["mul", "2", "3"], #{commands => #{"mul" => #{ + handler => {fun (match, L, R) -> L * R end, match}, + arguments => [#{name => opt, short => $o}, + #{name => l, type => integer}, #{name => r, type => integer}]}}}, + #{})), + %% no subcommand, positional module-based function + ?assertEqual(6, + argparse:run(["2", "3"], #{handler => {erlang, '*', undefined}, + arguments => [#{name => l, type => integer}, #{name => r, type => integer}]}, + #{})), + %% subcommand, module-based function accepting argmap + ?assertEqual([{arg, "arg"}], + argparse:run(["map", "arg"], #{commands => #{"map" => #{ + handler => {maps, to_list}, + arguments => [#{name => arg}]}}}, + #{})).
\ No newline at end of file diff --git a/lib/stdlib/test/base64_SUITE.erl b/lib/stdlib/test/base64_SUITE.erl index 1fc4c3fc0e..0eae2d9f72 100644 --- a/lib/stdlib/test/base64_SUITE.erl +++ b/lib/stdlib/test/base64_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2017. All Rights Reserved. +%% Copyright Ericsson AB 2007-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -26,9 +26,11 @@ -export([all/0, suite/0, groups/0, group/1]). %% Test cases must be exported. --export([base64_encode/1, base64_decode/1, base64_otp_5635/1, - base64_otp_6279/1, big/1, illegal/1, mime_decode/1, - mime_decode_to_string/1, +-export([base64_encode/1, base64_encode_to_string/1, base64_encode_modes/1, + base64_decode/1, base64_decode_to_string/1, base64_decode_modes/1, + base64_otp_5635/1, base64_otp_6279/1, big/1, illegal/1, + mime_decode/1, mime_decode_modes/1, + mime_decode_to_string/1, mime_decode_to_string_modes/1, roundtrip_1/1, roundtrip_2/1, roundtrip_3/1, roundtrip_4/1]). %%------------------------------------------------------------------------- @@ -40,8 +42,11 @@ suite() -> {timetrap,{minutes,4}}]. all() -> - [base64_encode, base64_decode, base64_otp_5635, - base64_otp_6279, big, illegal, mime_decode, mime_decode_to_string, + [base64_encode, base64_encode_to_string, base64_encode_modes, + base64_decode, base64_decode_to_string, base64_decode_modes, + base64_otp_5635, base64_otp_6279, big, illegal, + mime_decode, mime_decode_modes, + mime_decode_to_string, mime_decode_to_string_modes, {group, roundtrip}]. groups() -> @@ -53,44 +58,176 @@ group(roundtrip) -> [{timetrap,{minutes,10}}]. %%------------------------------------------------------------------------- -%% Test base64:encode/1. +%% Test base64:encode/1,2. base64_encode(Config) when is_list(Config) -> %% Two pads <<"QWxhZGRpbjpvcGVuIHNlc2FtZQ==">> = base64:encode("Aladdin:open sesame"), + <<"QWxhZGRpbjpvcGVuIHNlc2FtZQ">> = + base64:encode("Aladdin:open sesame", #{padding => false}), %% One pad <<"SGVsbG8gV29ybGQ=">> = base64:encode(<<"Hello World">>), + <<"SGVsbG8gV29ybGQ">> = base64:encode(<<"Hello World">>, #{padding => false}), + %% No pad + <<"QWxhZGRpbjpvcGVuIHNlc2Ft">> = base64:encode("Aladdin:open sesam"), + <<"QWxhZGRpbjpvcGVuIHNlc2Ft">> = + base64:encode("Aladdin:open sesam", #{padding => false}), + + <<"MDEyMzQ1Njc4OSFAIzBeJiooKTs6PD4sLiBbXXt9">> = + base64:encode(<<"0123456789!@#0^&*();:<>,. []{}">>), + ok. + +%%------------------------------------------------------------------------- +%% Test base64:encode_to_string/1,2. +base64_encode_to_string(Config) when is_list(Config) -> + %% Two pads + "QWxhZGRpbjpvcGVuIHNlc2FtZQ==" = + base64:encode_to_string("Aladdin:open sesame"), + "QWxhZGRpbjpvcGVuIHNlc2FtZQ" = + base64:encode_to_string("Aladdin:open sesame", #{padding => false}), + %% One pad + "SGVsbG8gV29ybGQ=" = base64:encode_to_string(<<"Hello World">>), + "SGVsbG8gV29ybGQ" = + base64:encode_to_string(<<"Hello World">>, #{padding => false}), %% No pad "QWxhZGRpbjpvcGVuIHNlc2Ft" = base64:encode_to_string("Aladdin:open sesam"), + "QWxhZGRpbjpvcGVuIHNlc2Ft" = + base64:encode_to_string("Aladdin:open sesam", #{padding => false}), "MDEyMzQ1Njc4OSFAIzBeJiooKTs6PD4sLiBbXXt9" = base64:encode_to_string(<<"0123456789!@#0^&*();:<>,. []{}">>), ok. + +%%------------------------------------------------------------------------- +%% Test base64:encode/2 and base64:encode_to_string/2. +base64_encode_modes(Config) when is_list(Config) -> + Data = <<23, 234, 63, 163, 239, 129, 253, 175, 171>>, + + <<"F+o/o++B/a+r">> = base64:encode(Data, #{mode => standard}), + <<"F-o_o--B_a-r">> = base64:encode(Data, #{mode => urlsafe}), + + "F+o/o++B/a+r" = base64:encode_to_string(Data, #{mode => standard}), + "F-o_o--B_a-r" = base64:encode_to_string(Data, #{mode => urlsafe}), + + ok. + %%------------------------------------------------------------------------- -%% Test base64:decode/1. +%% Test base64:decode/1,2. base64_decode(Config) when is_list(Config) -> %% Two pads <<"Aladdin:open sesame">> = - base64:decode("QWxhZGRpbjpvcGVuIHNlc2FtZQ=="), + base64:decode(<<"QWxhZGRpbjpvcGVuIHNlc2FtZQ==">>), + <<"Aladdin:open sesame">> = + base64:decode("QWxhZGRpbjpvcGVuIHNlc2FtZQ=="), + <<"Aladdin:open sesame">> = + base64:decode(<<"QWxhZGRpbjpvcGVuIHNlc2FtZQ">>, #{padding => false}), + <<"Aladdin:open sesame">> = + base64:decode("QWxhZGRpbjpvcGVuIHNlc2FtZQ", #{padding => false}), + {'EXIT', {'missing_padding', _}} = + catch base64:decode(<<"QWxhZGRpbjpvcGVuIHNlc2FtZQ">>), + {'EXIT', {'missing_padding', _}} = + catch base64:decode("QWxhZGRpbjpvcGVuIHNlc2FtZQ"), %% One pad <<"Hello World">> = base64:decode(<<"SGVsbG8gV29ybGQ=">>), + <<"Hello World">> = base64:decode("SGVsbG8gV29ybGQ="), + <<"Hello World">> = base64:decode(<<"SGVsbG8gV29ybGQ">>, #{padding => false}), + <<"Hello World">> = base64:decode("SGVsbG8gV29ybGQ", #{padding => false}), + {'EXIT', {'missing_padding', _}} = + catch base64:decode_to_string(<<"SGVsbG8gV29ybGQ">>), + {'EXIT', {'missing_padding', _}} = + catch base64:decode_to_string("SGVsbG8gV29ybGQ"), %% No pad <<"Aladdin:open sesam">> = - base64:decode("QWxhZGRpbjpvcGVuIHNlc2Ft"), + base64:decode(<<"QWxhZGRpbjpvcGVuIHNlc2Ft">>), + <<"Aladdin:open sesam">> = + base64:decode("QWxhZGRpbjpvcGVuIHNlc2Ft"), + <<"Aladdin:open sesam">> = + base64:decode(<<"QWxhZGRpbjpvcGVuIHNlc2Ft">>, #{padding => false}), + <<"Aladdin:open sesam">> = + base64:decode("QWxhZGRpbjpvcGVuIHNlc2Ft", #{padding => false}), Alphabet = list_to_binary(lists:seq(0, 255)), Alphabet = base64:decode(base64:encode(Alphabet)), + Alphabet = base64:decode(base64:encode_to_string(Alphabet)), + + %% Encoded base 64 strings may be divided by non base 64 chars. + %% In this cases whitespaces. + <<"0123456789!@#0^&*();:<>,. []{}">> = + base64:decode( + "MDEy MzQ1Njc4 \tOSFAIzBeJ \niooKTs6 PD4sLi \r\nBbXXt9"), + <<"0123456789!@#0^&*();:<>,. []{}">> = + base64:decode( + <<"MDEy MzQ1Njc4 \tOSFAIzBeJ \niooKTs6 PD4sLi \r\nBbXXt9">>), + ok. + +%%------------------------------------------------------------------------- +%% Test base64:decode_to_string/1,2. +base64_decode_to_string(Config) when is_list(Config) -> + %% Two pads + "Aladdin:open sesame" = + base64:decode_to_string(<<"QWxhZGRpbjpvcGVuIHNlc2FtZQ==">>), + "Aladdin:open sesame" = + base64:decode_to_string("QWxhZGRpbjpvcGVuIHNlc2FtZQ=="), + "Aladdin:open sesame" = + base64:decode_to_string(<<"QWxhZGRpbjpvcGVuIHNlc2FtZQ">>, #{padding => false}), + "Aladdin:open sesame" = + base64:decode_to_string("QWxhZGRpbjpvcGVuIHNlc2FtZQ", #{padding => false}), + {'EXIT', {'missing_padding', _}} = + catch base64:decode_to_string(<<"QWxhZGRpbjpvcGVuIHNlc2FtZQ">>), + {'EXIT', {'missing_padding', _}} = + catch base64:decode_to_string("QWxhZGRpbjpvcGVuIHNlc2FtZQ"), + %% One pad + "Hello World" = base64:decode_to_string(<<"SGVsbG8gV29ybGQ=">>), + "Hello World" = base64:decode_to_string("SGVsbG8gV29ybGQ="), + "Hello World" = base64:decode_to_string(<<"SGVsbG8gV29ybGQ">>, #{padding => false}), + "Hello World" = base64:decode_to_string("SGVsbG8gV29ybGQ", #{padding => false}), + {'EXIT', {'missing_padding', _}} = + catch base64:decode_to_string(<<"SGVsbG8gV29ybGQ">>), + {'EXIT', {'missing_padding', _}} = + catch base64:decode_to_string("SGVsbG8gV29ybGQ"), + %% No pad + "Aladdin:open sesam" = + base64:decode_to_string(<<"QWxhZGRpbjpvcGVuIHNlc2Ft">>), + "Aladdin:open sesam" = + base64:decode_to_string("QWxhZGRpbjpvcGVuIHNlc2Ft"), + "Aladdin:open sesam" = + base64:decode_to_string(<<"QWxhZGRpbjpvcGVuIHNlc2Ft">>, #{padding => false}), + "Aladdin:open sesam" = + base64:decode_to_string("QWxhZGRpbjpvcGVuIHNlc2Ft", #{padding => false}), + + Alphabet = lists:seq(0, 255), + Alphabet = base64:decode_to_string(base64:encode(Alphabet)), + Alphabet = base64:decode_to_string(base64:encode_to_string(Alphabet)), %% Encoded base 64 strings may be divided by non base 64 chars. %% In this cases whitespaces. "0123456789!@#0^&*();:<>,. []{}" = - base64:decode_to_string( - "MDEy MzQ1Njc4 \tOSFAIzBeJ \niooKTs6 PD4sLi \r\nBbXXt9"), + base64:decode_to_string( + "MDEy MzQ1Njc4 \tOSFAIzBeJ \niooKTs6 PD4sLi \r\nBbXXt9"), "0123456789!@#0^&*();:<>,. []{}" = - base64:decode_to_string( - <<"MDEy MzQ1Njc4 \tOSFAIzBeJ \niooKTs6 PD4sLi \r\nBbXXt9">>), + base64:decode_to_string( + <<"MDEy MzQ1Njc4 \tOSFAIzBeJ \niooKTs6 PD4sLi \r\nBbXXt9">>), ok. + +%%------------------------------------------------------------------------- +%% Test base64:decode/2 and base64:decode_to_string/2 +base64_decode_modes(Config) when is_list(Config) -> + DataBin = <<23, 234, 63, 163, 239, 129, 253, 175, 171>>, + DataStr = [23, 234, 63, 163, 239, 129, 253, 175, 171], + + DataBin = base64:decode("F+o/o++B/a+r", #{mode => standard}), + DataBin = base64:decode("F-o_o--B_a-r", #{mode => urlsafe}), + {'EXIT', _} = catch base64:decode("F-o_o--B_a-r", #{mode => standard}), + {'EXIT', _} = catch base64:decode("F+o/o++B/a+r", #{mode => urlsafe}), + + DataStr = base64:decode_to_string("F+o/o++B/a+r", #{mode => standard}), + DataStr = base64:decode_to_string("F-o_o--B_a-r", #{mode => urlsafe}), + {'EXIT', _} = catch base64:decode_to_string("F-o_o--B_a-r", #{mode => standard}), + {'EXIT', _} = catch base64:decode_to_string("F+o/o++B/a+r", #{mode => urlsafe}), + + ok. + %%------------------------------------------------------------------------- %% OTP-5635: Some data doesn't pass through base64:decode/1 correctly. base64_otp_5635(Config) when is_list(Config) -> @@ -171,6 +308,31 @@ mime_decode(Config) when is_list(Config) -> <<"o">> = MimeDecode(<<"bw=\000=">>), ok. +%% Test base64:mime_decode/2. +mime_decode_modes(Config) when is_list(Config) -> + MimeDecode = fun (In, Options) -> + Out = base64:mime_decode(In, Options), + Out = base64:mime_decode(binary_to_list(In), Options) + end, + + %% The following all decode to the same data. + Data = <<23, 234, 63, 163, 239, 129, 253, 175, 171>>, + Data = MimeDecode(<<"F+o/o++B/a+r">>, #{mode => standard}), + Data = MimeDecode(<<"F-o_o--B_a-r">>, #{mode => urlsafe}), + + %% The following decodes to different data depending on mode. + Base64 = <<"AB+C+D/E/FG-H-I_J_KL">>, + %% In standard mode, "-" and "_" are invalid and thus ignored. + %% The base64 string to be decoded is equivalent to "AB+C+D/E/FGHIJKL". + <<0, 31, 130, 248, 63, 196, 252, 81, 135, 32, 146, 139>> = + MimeDecode(Base64, #{mode => standard}), + %% In urlsafe mode, "+" and "/" are invalid and thus ignored. + %% The base64 string to be decoded is equivalent to "ABCDEFG-H-I_J_KL". + <<0, 16, 131, 16, 81, 190, 31, 226, 63, 39, 242, 139>> = + MimeDecode(Base64, #{mode => urlsafe}), + + ok. + %%------------------------------------------------------------------------- %% Repeat of mime_decode() tests @@ -221,6 +383,32 @@ mime_decode_to_string(Config) when is_list(Config) -> "o" = MimeDecodeToString(<<"bw=\000=">>), ok. + +%% Test base64:mime_decode_to_string/2. +mime_decode_to_string_modes(Config) when is_list(Config) -> + MimeDecode = fun (In, Options) -> + Out = base64:mime_decode_to_string(In, Options), + Out = base64:mime_decode_to_string(binary_to_list(In), Options) + end, + + %% The following all decode to the same data. + Data = [23, 234, 63, 163, 239, 129, 253, 175, 171], + Data = MimeDecode(<<"F+o/o++B/a+r">>, #{mode => standard}), + Data = MimeDecode(<<"F-o_o--B_a-r">>, #{mode => urlsafe}), + + %% The following decodes to different data depending on mode. + Base64 = <<"AB+C+D/E/FG-H-I_J_KL">>, + %% In standard mode, "-" and "_" are invalid and thus ignored. + %% The base64 string to be decoded is equivalent to "AB+C+D/E/FGHIJKL". + [0, 31, 130, 248, 63, 196, 252, 81, 135, 32, 146, 139] = + MimeDecode(Base64, #{mode => standard}), + %% In urlsafe mode, "+" and "/" are invalid and thus ignored. + %% The base64 string to be decoded is equivalent to "ABCDEFG-H-I_J_KL". + [0, 16, 131, 16, 81, 190, 31, 226, 63, 39, 242, 139] = + MimeDecode(Base64, #{mode => urlsafe}), + + ok. + %%------------------------------------------------------------------------- roundtrip_1(Config) when is_list(Config) -> diff --git a/lib/stdlib/test/base64_property_test_SUITE.erl b/lib/stdlib/test/base64_property_test_SUITE.erl index 7717b62767..68ac5e9ee7 100644 --- a/lib/stdlib/test/base64_property_test_SUITE.erl +++ b/lib/stdlib/test/base64_property_test_SUITE.erl @@ -24,18 +24,18 @@ all() -> [ - encode_case, - encode_to_string_case, - decode_case, - decode_malformed_case, - decode_noisy_case, - decode_to_string_case, - decode_to_string_malformed_case, - decode_to_string_noisy_case, - mime_decode_case, - mime_decode_malformed_case, - mime_decode_to_string_case, - mime_decode_to_string_malformed_case + encode_1_case, encode_2_case, + encode_to_string_1_case, encode_to_string_2_case, + decode_1_case, decode_2_case, + decode_1_malformed_case, decode_2_malformed_case, + decode_1_noisy_case, decode_2_noisy_case, + decode_to_string_1_case, decode_to_string_2_case, + decode_to_string_1_malformed_case, decode_to_string_2_malformed_case, + decode_to_string_1_noisy_case, decode_to_string_2_noisy_case, + mime_decode_1_case, mime_decode_2_case, + mime_decode_1_malformed_case, mime_decode_2_malformed_case, + mime_decode_to_string_1_case, mime_decode_to_string_2_case, + mime_decode_to_string_1_malformed_case, mime_decode_to_string_2_malformed_case ]. init_per_suite(Config) -> @@ -44,41 +44,77 @@ init_per_suite(Config) -> end_per_suite(Config) -> Config. -encode_case(Config) -> - do_proptest(prop_encode, Config). +encode_1_case(Config) -> + do_proptest(prop_encode_1, Config). -encode_to_string_case(Config) -> - do_proptest(prop_encode_to_string, Config). +encode_2_case(Config) -> + do_proptest(prop_encode_2, Config). -decode_case(Config) -> - do_proptest(prop_decode, Config). +encode_to_string_1_case(Config) -> + do_proptest(prop_encode_to_string_1, Config). -decode_malformed_case(Config) -> - do_proptest(prop_decode_malformed, Config). +encode_to_string_2_case(Config) -> + do_proptest(prop_encode_to_string_2, Config). -decode_noisy_case(Config) -> - do_proptest(prop_decode_noisy, Config). +decode_1_case(Config) -> + do_proptest(prop_decode_1, Config). -decode_to_string_case(Config) -> - do_proptest(prop_decode_to_string, Config). +decode_2_case(Config) -> + do_proptest(prop_decode_2, Config). -decode_to_string_malformed_case(Config) -> - do_proptest(prop_decode_to_string_malformed, Config). +decode_1_malformed_case(Config) -> + do_proptest(prop_decode_1_malformed, Config). -decode_to_string_noisy_case(Config) -> - do_proptest(prop_decode_to_string_noisy, Config). +decode_2_malformed_case(Config) -> + do_proptest(prop_decode_2_malformed, Config). -mime_decode_case(Config) -> - do_proptest(prop_mime_decode, Config). +decode_1_noisy_case(Config) -> + do_proptest(prop_decode_1_noisy, Config). -mime_decode_malformed_case(Config) -> - do_proptest(prop_mime_decode_malformed, Config). +decode_2_noisy_case(Config) -> + do_proptest(prop_decode_2_noisy, Config). -mime_decode_to_string_case(Config) -> - do_proptest(prop_mime_decode_to_string, Config). +decode_to_string_1_case(Config) -> + do_proptest(prop_decode_to_string_1, Config). -mime_decode_to_string_malformed_case(Config) -> - do_proptest(prop_mime_decode_to_string_malformed, Config). +decode_to_string_2_case(Config) -> + do_proptest(prop_decode_to_string_2, Config). + +decode_to_string_1_malformed_case(Config) -> + do_proptest(prop_decode_to_string_1_malformed, Config). + +decode_to_string_2_malformed_case(Config) -> + do_proptest(prop_decode_to_string_2_malformed, Config). + +decode_to_string_1_noisy_case(Config) -> + do_proptest(prop_decode_to_string_1_noisy, Config). + +decode_to_string_2_noisy_case(Config) -> + do_proptest(prop_decode_to_string_2_noisy, Config). + +mime_decode_1_case(Config) -> + do_proptest(prop_mime_decode_1, Config). + +mime_decode_2_case(Config) -> + do_proptest(prop_mime_decode_2, Config). + +mime_decode_1_malformed_case(Config) -> + do_proptest(prop_mime_decode_1_malformed, Config). + +mime_decode_2_malformed_case(Config) -> + do_proptest(prop_mime_decode_2_malformed, Config). + +mime_decode_to_string_1_case(Config) -> + do_proptest(prop_mime_decode_to_string_1, Config). + +mime_decode_to_string_2_case(Config) -> + do_proptest(prop_mime_decode_to_string_2, Config). + +mime_decode_to_string_1_malformed_case(Config) -> + do_proptest(prop_mime_decode_to_string_1_malformed, Config). + +mime_decode_to_string_2_malformed_case(Config) -> + do_proptest(prop_mime_decode_to_string_2_malformed, Config). do_proptest(Prop, Config) -> ct_property_test:quickcheck( diff --git a/lib/stdlib/test/binary_module_SUITE.erl b/lib/stdlib/test/binary_module_SUITE.erl index 99f3566eab..954efae9b7 100644 --- a/lib/stdlib/test/binary_module_SUITE.erl +++ b/lib/stdlib/test/binary_module_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2021. All Rights Reserved. +%% Copyright Ericsson AB 1997-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -1462,10 +1462,15 @@ error_info(_Config) -> {split, [<<1,2,3>>, {bm,make_ref()}, []]}, {split, [<<1,2,3>>, <<"2">>, [bad_option]]}, - {encode_hex, [{no,a,binary}]}, - {decode_hex, [{no,a,binary}]}, - {decode_hex, [<<"000">>],[allow_rename]}, - {decode_hex, [<<"GG">>],[allow_rename]} + {encode_hex, [{no,binary}]}, + {encode_hex, [{no,binary}, lowercase]}, + {encode_hex, [<<"foobar">>, othercase]}, + {encode_hex, [no_binary, othercase], [{1,".*"}, {2,".*"}]}, + + {decode_hex, [{no,binary}]}, + {decode_hex, [<<"000">>]}, + {decode_hex, [<<"GG">>]}, + {decode_hex, [<<255>>]} ], error_info_lib:test_error_info(binary, L). @@ -1479,6 +1484,23 @@ hex_encoding(Config) when is_list(Config) -> <<"666F6F6261">> = binary:encode_hex(<<"fooba">>), <<"666F6F626172">> = binary:encode_hex(<<"foobar">>), + + <<>> = binary:encode_hex(<<>>, uppercase), + <<"66">> = binary:encode_hex(<<"f">>, uppercase), + <<"666F">> = binary:encode_hex(<<"fo">>, uppercase), + <<"666F6F">> = binary:encode_hex(<<"foo">>, uppercase), + <<"666F6F62">> = binary:encode_hex(<<"foob">>, uppercase), + <<"666F6F6261">> = binary:encode_hex(<<"fooba">>, uppercase), + <<"666F6F626172">> = binary:encode_hex(<<"foobar">>, uppercase), + + <<>> = binary:encode_hex(<<>>, lowercase), + <<"66">> = binary:encode_hex(<<"f">>, lowercase), + <<"666f">> = binary:encode_hex(<<"fo">>, lowercase), + <<"666f6f">> = binary:encode_hex(<<"foo">>, lowercase), + <<"666f6f62">> = binary:encode_hex(<<"foob">>, lowercase), + <<"666f6f6261">> = binary:encode_hex(<<"fooba">>, lowercase), + <<"666f6f626172">> = binary:encode_hex(<<"foobar">>, lowercase), + <<>> = binary:decode_hex(<<>>), <<"f">> = binary:decode_hex(<<"66">>), <<"fo">> = binary:decode_hex(<<"666F">>), @@ -1494,8 +1516,29 @@ hex_encoding(Config) when is_list(Config) -> <<"foobar">> = binary:decode_hex(<<"666f6f626172">>), <<"foobar">> = binary:decode_hex(<<"666f6F626172">>), + + rand:seed(default), + io:format("*** SEED: ~p ***\n", [rand:export_seed()]), + Bytes = iolist_to_binary([rand:bytes(256), lists:seq(0, 255)]), + do_hex_roundtrip(Bytes), + ok. +do_hex_roundtrip(Bytes) -> + UpperHex = binary:encode_hex(Bytes), + UpperHex = binary:encode_hex(Bytes, uppercase), + LowerHex = binary:encode_hex(Bytes, lowercase), + + Bytes = binary:decode_hex(UpperHex), + Bytes = binary:decode_hex(LowerHex), + + case Bytes of + <<_, Rest/binary>> -> + do_hex_roundtrip(Rest); + <<>> -> + ok + end. + %%% %%% Utilities. %%% diff --git a/lib/stdlib/test/binary_property_test_SUITE.erl b/lib/stdlib/test/binary_property_test_SUITE.erl new file mode 100644 index 0000000000..4ec1ba67ab --- /dev/null +++ b/lib/stdlib/test/binary_property_test_SUITE.erl @@ -0,0 +1,35 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2022. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(binary_property_test_SUITE). + +-compile([export_all, nowarn_export_all]). + +all() -> + [encode_hex_case]. + +init_per_suite(Config) -> + ct_property_test:init_per_suite(Config). + +end_per_suite(Config) -> + Config. + +encode_hex_case(Config) -> + ct_property_test:quickcheck(binary_prop:prop_hex_encode_2(), Config). + diff --git a/lib/stdlib/test/dets_SUITE.erl b/lib/stdlib/test/dets_SUITE.erl index c50a2d5baf..5a19717c28 100644 --- a/lib/stdlib/test/dets_SUITE.erl +++ b/lib/stdlib/test/dets_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2021. All Rights Reserved. +%% Copyright Ericsson AB 1996-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -46,7 +46,7 @@ otp_5487/1, otp_6206/1, otp_6359/1, otp_4738/1, otp_7146/1, otp_8070/1, otp_8856/1, otp_8898/1, otp_8899/1, otp_8903/1, otp_8923/1, otp_9282/1, otp_11245/1, otp_11709/1, otp_13229/1, - otp_13260/1, otp_13830/1]). + otp_13260/1, otp_13830/1, receive_optimisation/1]). -export([dets_dirty_loop/0]). @@ -94,7 +94,7 @@ all() -> insert_new, repair_continuation, otp_5487, otp_6206, otp_6359, otp_4738, otp_7146, otp_8070, otp_8856, otp_8898, otp_8899, otp_8903, otp_8923, otp_9282, otp_11245, otp_11709, - otp_13229, otp_13260, otp_13830 + otp_13229, otp_13260, otp_13830, receive_optimisation ]. groups() -> @@ -3492,6 +3492,34 @@ otp_13830(Config) -> {ok, Tab} = dets:open_file(Tab, [{file, File}, {version, default}]), ok = dets:close(Tab). +receive_optimisation(Config) -> + Tab = dets_receive_optimisation_test, + FName = filename(Tab, Config), + + % Spam message box + lists:foreach(fun(_) -> self() ! {spam, it} end, lists:seq(1, 1_000_000)), + + {ok, _} = dets:open_file(Tab,[{file, FName}]), + ok = dets:insert(Tab,{one, record}), + + StartTime = os:system_time(millisecond), + + % We expect one thousand of simple lookups to finish in one second + Lookups = 1000, + Timeout = 1000, + Loop = fun Loop(N) when N =< 0 -> ok; + Loop(N) -> + Now = os:system_time(millisecond), + (Now - StartTime > Timeout) andalso throw({timeout_after, Lookups - N}), + [{one, record}] = dets:lookup(Tab, one), + Loop(N-1) + end, + + ok = Loop(Lookups), + + ok = dets:close(Tab), + ok = file:delete(FName). + %% %% Parts common to several test cases %% diff --git a/lib/stdlib/test/edlin_context_SUITE.erl b/lib/stdlib/test/edlin_context_SUITE.erl new file mode 100644 index 0000000000..5e84a90199 --- /dev/null +++ b/lib/stdlib/test/edlin_context_SUITE.erl @@ -0,0 +1,189 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2023. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(edlin_context_SUITE). +-include_lib("common_test/include/ct.hrl"). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1]). + +-export([get_context/1]). + +suite() -> + [{timetrap,{minutes,1}}]. +all() -> + [get_context]. +groups() -> + []. +init_per_suite(Config) -> + Config. +end_per_suite(_Config) -> + ok. + +get_context(_Config) -> + {term, [], {atom, "h"}} = edlin_context:get_context(lists:reverse("h")), + {term} = edlin_context:get_context(lists:reverse("h(file")), + {term} = edlin_context:get_context(lists:reverse("h(file,open")), + {term, [{call, "h(file,open)"}], {atom, "h"}} = edlin_context:get_context(lists:reverse("h(file,open), h")), + {term} = edlin_context:get_context(lists:reverse("h(file,open), h(file")), + {term} = edlin_context:get_context(lists:reverse("h(file,open), h(file,open")), + {term, [{call, "h(file,open)"}], {call, "h(file,open)"}} = edlin_context:get_context(lists:reverse("h(file,open), h(file,open)")), + {function} = edlin_context:get_context(lists:reverse("file:")), + {function} = edlin_context:get_context(lists:reverse("file:open")), + {function, "file", "open", [], [], []} = edlin_context:get_context(lists:reverse("file:open(")), + {string} = edlin_context:get_context(lists:reverse("file:open(\"")), + {string} = edlin_context:get_context(lists:reverse("file:open(\"/")), + {string} = edlin_context:get_context(lists:reverse("file:open(\"Word")), + {function, "file", "open", [], {string, "\"\""}, []} = edlin_context:get_context(lists:reverse("file:open(\"\"")), + {function, "file", "open", [{string, "\"\""}], [], []} = edlin_context:get_context(lists:reverse("file:open(\"\",")), + {function, "file", "open", [{string, "\"\""}], [], [{list, [], []}]} = edlin_context:get_context(lists:reverse("file:open(\"\",[")), + {function, "file", "open", [{string, "\"\""}], [], [{tuple, [], []}]} = edlin_context:get_context(lists:reverse("file:open(\"\",{")), + {function, "file", "open", [{string, "\"\""}], [], [{list, [], []},{tuple, [], []}]} = edlin_context:get_context(lists:reverse("file:open(\"\",[{")), + {function, "file", "open", [{string, "\"\""}], [], [{list, [], {atom, "atom"}}]} = edlin_context:get_context(lists:reverse("file:open(\"\",[atom")), + {function, "file", "open", [{string, "\"\""}], [], [{tuple, [], {atom, "atom"}}]} = edlin_context:get_context(lists:reverse("file:open(\"\",{atom")), + {function, "file", "open", [{string, "\"\""}], [], [{list, [], []},{tuple, [], {atom, "atom"}}]} = edlin_context:get_context(lists:reverse("file:open(\"\",[{atom")), + {function, "file", "open", [{string, "\"\""}], [], [{list, [], []},{tuple, [{atom, "atom"}], []}]} = edlin_context:get_context(lists:reverse("file:open(\"\",[{atom,")), + {function, "file", "open", [{string, "\"\""}], [], [{map, ["atom"], "atom", [], []}]} = edlin_context:get_context(lists:reverse("file:open(\"\",#{ atom =>")), + {term, [], {atom, "list"}} = edlin_context:get_context(lists:reverse("#{list")), + {term, [], {atom, "list"}} = edlin_context:get_context(lists:reverse("{list")), + {term, [], {atom, "list"}} = edlin_context:get_context(lists:reverse("[list")), + {map, "M", []} = edlin_context:get_context(lists:reverse("M#{")), + {map, "M", []} = edlin_context:get_context(lists:reverse("M#{key")), + {map, "M", ["key"]} = edlin_context:get_context(lists:reverse("M#{key=>")), + {map, "M", ["key"]} = edlin_context:get_context(lists:reverse("M#{key:=")), %% map value + {map, "M", ["key"]} = edlin_context:get_context(lists:reverse("M#{key=>0")), + {map, "M", ["key"]} = edlin_context:get_context(lists:reverse("M#{key=>0,")), + {map, "M", ["key", "key2"]} = edlin_context:get_context(lists:reverse("M#{key=>0,key2=>")), + {map_or_record} = edlin_context:get_context(lists:reverse("#")), + {record, "record", [], [], [], [], []} = edlin_context:get_context(lists:reverse("#record{")), + {record, "record", [], [], [], [], []} = edlin_context:get_context(lists:reverse("#record.")), + {record, "record", [], [], [], {atom, "field"}, []} = edlin_context:get_context(lists:reverse("#record{field")), + {record, "record", ["field"], "field", [], [], []} = edlin_context:get_context(lists:reverse("#record{field=>")), + {record, "record", ["field"], "field", [], [], []} = edlin_context:get_context(lists:reverse("#record{field:=")), %% record_field value + {record, "record", ["field"], [], [{integer, "0"}], [], []} = edlin_context:get_context(lists:reverse("R#record{field=>0,")), + {record, "record", ["field", "field2"], "field2", [{integer,"0"}], [], [{list, [], []},{tuple, [{atom, "atom"}], []}]} = edlin_context:get_context(lists:reverse("R#record{field=>0,field2=>[{atom,")), + {term,[],{atom,"fun"}} = edlin_context:get_context(lists:reverse("fun")), + {term,[],{atom,"fun"}} = edlin_context:get_context(lists:reverse("fun ")), + {fun_} = edlin_context:get_context(lists:reverse("fun m")), + {fun_, "m"} = edlin_context:get_context(lists:reverse("fun m:")), + {fun_, "m"} = edlin_context:get_context(lists:reverse("fun m:f")), + {fun_, "m", "f"} = edlin_context:get_context(lists:reverse("fun m:f/")), + {fun_, "m", "f"} = edlin_context:get_context(lists:reverse("fun m:f/1")), + {fun_, "m", "f"} = edlin_context:get_context(lists:reverse("fun m:f/1 ")), + {term,[{fun_,"fun m:f/1"}],[]} = edlin_context:get_context(lists:reverse("fun m:f/1 ,")), + {function,"user_defined","my_fun", + [{keyword,"receive X -> X end"}], + [],[]} = edlin_context:get_context(lists:reverse("my_fun(receive X -> X end, ")), + {function,"user_defined","my_fun", + [{keyword,"maybe X -> X end"}], + [],[]} = edlin_context:get_context(lists:reverse("my_fun(maybe X -> X end, ")), + {function,"user_defined","my_fun", + [{keyword,"try a end"}], + [],[]} = edlin_context:get_context(lists:reverse("my_fun(try a end, ")), + {function,"user_defined","my_fun", + [{keyword,"catch X -> X end"}], + [],[]}= edlin_context:get_context(lists:reverse("my_fun(catch X -> X end, ")), + {function,"user_defined","my_fun", + [{keyword,"try a catch _:_ -> b end"}], + [],[]} = edlin_context:get_context(lists:reverse("my_fun(try a catch _:_ -> b end, ")), + {function,"user_defined","my_fun", + [{keyword,"begin X end"}], + [],[]}= edlin_context:get_context(lists:reverse("my_fun(begin X end, ")), + {function,"user_defined","my_fun", + [{keyword,"if X -> X end"}], + [],[]} = edlin_context:get_context(lists:reverse("my_fun(if X -> X end, ")), + {function,"user_defined","my_fun", + [{keyword,"case X of _ -> X end"}], + [],[]} = edlin_context:get_context(lists:reverse("my_fun(case X of _ -> X end, ")), + {binding} = edlin_context:get_context(lists:reverse("fun() -> X")), + {term,[],{atom,"x"}} = edlin_context:get_context(lists:reverse("fun() -> x")), + {term} = edlin_context:get_context(lists:reverse("fun() ->")), + {macro} = edlin_context:get_context(lists:reverse("?")), %% not supported in edlin_expand + % unknown map + {term,[{operation,"one = a"},{operation,"two = b"}],[]} = edlin_context:get_context(lists:reverse("#{ one = a, two = b, ")), + {term,[{atom,"a"},{atom,"b"}],[]} = edlin_context:get_context(lists:reverse("#{ one := a, two := b, ")), + {term,[{atom,"a"},{atom,"b"}],[]} = edlin_context:get_context(lists:reverse("#{ one => a, two => b, ")), + {term,[{operation,"A = a"},{operation,"B = b"}],[]} = edlin_context:get_context(lists:reverse("A = a, B = b, ")), + {term,[{operation,"one = a"}],[]} = edlin_context:get_context(lists:reverse("#{ one = a, two = ")), + {term,[{atom,"a"}],[]} = edlin_context:get_context(lists:reverse("#{ one := a, two := ")), + {term,[{atom,"a"}],[]} = edlin_context:get_context(lists:reverse("#{ one => a, two => ")), + {term,[{operation,"A = a"}],[]} = edlin_context:get_context(lists:reverse("A = a, B = ")), + {term,[],{operation,"A = a"}} = edlin_context:get_context(lists:reverse("A = a")), + {'end'} = edlin_context:get_context(lists:reverse("a.")), + {record,"record",[],[],[],[],[]} = edlin_context:get_context(lists:reverse("#record.")), + {record,"record",[],[],[],[],[]} = edlin_context:get_context(lists:reverse("{#record.")), + {record,"record",[],[],[],{atom,"a"},[]} = edlin_context:get_context(lists:reverse("#record.a")), + {record,"record",[],[],[],{atom,"a"},[]} = edlin_context:get_context(lists:reverse("{#record.a")), + {term,[],{record,"#record{}"}} = edlin_context:get_context(lists:reverse("#record{}")), + {term,[],{map,"#{ a => b}"}} = edlin_context:get_context(lists:reverse("#{ a => b}")), + {term,[{atom,"a"}],{atom,"tuple"}} = edlin_context:get_context(lists:reverse("{a, tuple")), + {term,[],{tuple,"{a, tuple}"}} = edlin_context:get_context(lists:reverse("{a, tuple}")), + {term,[],{call,"lists:my_fun()"}} = edlin_context:get_context(lists:reverse("lists:my_fun()")), + {term} = edlin_context:get_context(lists:reverse("(")), + {term,[],{parenthesis,"()"}} = edlin_context:get_context(lists:reverse("()")), + {new_fun,"()"} = edlin_context:get_context(lists:reverse("fun()")), + %% 256 $] + {term,[],{list,"[]"}} = edlin_context:get_context(lists:reverse("[]")), + {term,[{atom,"a"}],{atom,"b"}} = edlin_context:get_context(lists:reverse("fun() when a, b")), + {term,[{atom,"a"}],{atom,"b"}} = edlin_context:get_context(lists:reverse("fun() -> a, b")), + {term,[{atom,"a"},{atom,"b"}],[]} = edlin_context:get_context(lists:reverse("fun() -> a, b, ")), + {term, [], {pid, "<1.0.1>"}} = edlin_context:get_context(lists:reverse("<1.0.1>")), + {term, [], {funref, "#Fun<erl_eval.0.1>"}} = edlin_context:get_context(lists:reverse("#Fun<erl_eval.0.1>")), + {term, [], {ref, "#Ref<1.0.1>"}} = edlin_context:get_context(lists:reverse("#Ref<1.0.1>")), + %{term, [], {port, "#Port<1.0>"}} = edlin_context:get_context(lists:reverse("#Port<1.0>")), + {term, [], {binary, "<<0>>"}} = edlin_context:get_context(lists:reverse("<<0>>")), + {term,[],{keyword,"fun (X) -> X end"}} = edlin_context:get_context(lists:reverse("fun (X) -> X end")), + {term,[],{keyword,"fun(X) -> X end"}} = edlin_context:get_context(lists:reverse("fun(X) -> X end")), %% should be fun_ too + {term,[],{keyword,"receive X -> X end"}} = edlin_context:get_context(lists:reverse("receive X -> X end")), + {error, _} = edlin_context:get_context(lists:reverse("no_keyword -> X end")), + {term} = edlin_context:get_context(lists:reverse("@")), %% No valid argument 305 + {term,[],{char,"$@"}} = edlin_context:get_context(lists:reverse("$@")), + {term,[],{char,"$ "}} = edlin_context:get_context(lists:reverse("$ ")), + {term,[],{float,"1.0"}} = edlin_context:get_context(lists:reverse("1.0")), + {term,[],{integer,"10#10"}} = edlin_context:get_context(lists:reverse("10#10")), + {term,[],{integer,"1"}} = edlin_context:get_context(lists:reverse("1")), + {binding} = edlin_context:get_context(lists:reverse("{X")), + {term,[{var, "X"}], []} = edlin_context:get_context(lists:reverse("{X, ")), + {error,_} = edlin_context:get_context(lists:reverse("<abc)")), + {error,_} = edlin_context:get_context(lists:reverse("<abc]")), + {error,_} = edlin_context:get_context(lists:reverse("<abc}")), + {term} = edlin_context:get_context(lists:reverse("(abc>")), + {term} = edlin_context:get_context(lists:reverse("\"\\\"\"")), %% odd quotes "\"" + {error, _} = edlin_context:get_context(lists:reverse("{\"\", $\"}")), %% odd quotes + {term} = edlin_context:get_context(lists:reverse("receive X -> ")), + %% read operator and argument order validity + {error,_} = edlin_context:get_context(lists:reverse("foo bar")), %% TODO what should be returned here, illegal + {term,[],{operation,"\" \" \" \""}} = edlin_context:get_context(lists:reverse("\" \" \" \"")), %% " " " " + {term,[],{operation,"1 + 2"}} = edlin_context:get_context(lists:reverse("1+2")), + {term,[],{operation,"1 andalso 2"}} = edlin_context:get_context(lists:reverse("1 andalso 2")), + {term,[],{operation,"1 and 2"}} = edlin_context:get_context(lists:reverse("1 and 2")), + {term,[],{operation,"1 orelse 2"}} = edlin_context:get_context(lists:reverse("1 orelse 2")), + {term,[],{operation,"1 or 2"}} = edlin_context:get_context(lists:reverse("1 or 2")), + {term,[],{operation,"1 or 2"}} = edlin_context:get_context(lists:reverse("1 or 2")), + {term,[],{operation,"1 =/= 2"}} = edlin_context:get_context(lists:reverse("1 =/= 2")), + {term,[],{operation,"1 =:= 2"}} = edlin_context:get_context(lists:reverse("1 =:= 2")), + {term,[],{operation,"1 <=> 2"}} = edlin_context:get_context(lists:reverse("1 <=> 2")), + {term,[],{operation,"<<1>> > <<2>>"}} = edlin_context:get_context(lists:reverse("<<1>>><<2>>")), + %{term,[],{operation,"<<1>> > <<2>>"}} = edlin_context:get_context(lists:reverse("<<1>> > <<2>>")), + {error,_} = edlin_context:get_context(lists:reverse("1 + + 2")), + {term,[],{integer,"2"}} = edlin_context:get_context(lists:reverse("1 -> 2")), + {term,[],{integer,"2"}} = edlin_context:get_context(lists:reverse("receive X -> 2")), + {term} = edlin_context:get_context(lists:reverse("receive X ->")), + {term,[{integer,"2"}],{operation,"1 + 3"}} = edlin_context:get_context(lists:reverse("receive X -> 2, 1+3")), + {term,[],{integer,"-1"}} = edlin_context:get_context(lists:reverse("-1")), + {term,[],{float,"-1.2"}} = edlin_context:get_context(lists:reverse("-1.2")), + ok.
\ No newline at end of file diff --git a/lib/stdlib/test/edlin_expand_SUITE.erl b/lib/stdlib/test/edlin_expand_SUITE.erl index 22b6ba03af..206a8b7246 100644 --- a/lib/stdlib/test/edlin_expand_SUITE.erl +++ b/lib/stdlib/test/edlin_expand_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2021. All Rights Reserved. +%% Copyright Ericsson AB 2010-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -18,12 +18,22 @@ %% %CopyrightEnd% %% -module(edlin_expand_SUITE). --export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, - init_per_testcase/2, end_per_testcase/2, - init_per_group/2,end_per_group/2]). --export([normal/1, quoted_fun/1, quoted_module/1, quoted_both/1, erl_1152/1, - erl_352/1, unicode/1]). - +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_testcase/2, end_per_testcase/2, + init_per_group/2,end_per_group/2]). +-export([normal/1, type_completion/1, quoted_fun/1, quoted_module/1, quoted_both/1, erl_1152/1, get_coverage/1, + check_trailing/1, unicode/1, filename_completion/1, binding_completion/1, record_completion/1, + map_completion/1, function_parameter_completion/1, fun_completion/1]). +-record(a_record, + {a_field :: atom1 | atom2 | btom | 'my atom' | {atom3, {atom4, non_neg_integer()}} | 'undefined', + b_field :: boolean() | 'undefined', + c_field :: list(term()) | 'undefined', + d_field :: non_neg_integer() | 'undefined'}). +-record('Quoted_record', + {'A_field' :: atom1 | atom2 | btom | 'my atom' | {atom3, {atom4, non_neg_integer()}} | 'undefined', + b_field :: boolean() | 'undefined', + c_field :: list(term()) | 'undefined', + d_field :: non_neg_integer() | 'undefined'}). -include_lib("common_test/include/ct.hrl"). init_per_testcase(_Case, Config) -> @@ -37,17 +47,20 @@ suite() -> [{ct_hooks,[ts_install_cth]}, {timetrap,{minutes,1}}]. -all() -> - [normal, quoted_fun, quoted_module, quoted_both, erl_1152, erl_352, +all() -> + [normal, filename_completion, binding_completion, get_coverage, type_completion, + record_completion, fun_completion, map_completion, function_parameter_completion, + quoted_fun, quoted_module, quoted_both, erl_1152, check_trailing, unicode]. -groups() -> +groups() -> []. init_per_suite(Config) -> Config. end_per_suite(_Config) -> + cleanup(), ok. init_per_group(_GroupName, Config) -> @@ -60,53 +73,470 @@ cleanup() -> [try code:purge(M), code:delete(M) - catch _:_ -> ok end || M <- [expand_test, expand_test1, - 'ExpandTestCaps', 'ExpandTestCaps2']]. + catch _:_ -> ok end || M <- [expand_test, expand_test1, expand_function_parameter, + 'ExpandTestCaps', 'ExpandTestCaps1', + complete_function_parameter]]. normal(Config) when is_list(Config) -> {module,expand_test} = compile_and_load(Config,expand_test), %% These tests might fail if another module with the prefix %% "expand_" happens to also be loaded. - {yes, "test:", []} = do_expand("expand_"), + {yes,"test:",[]} = do_expand("expand_"), {no, [], []} = do_expand("expandXX_"), - {no,[], - [{"a_fun_name",1}, - {"a_less_fun_name",1}, - {"b_comes_after_a",1}, - {"expand0arity_entirely",0}, - {"module_info",0}, - {"module_info",1}]} = do_expand("expand_test:"), - {yes,[],[{"a_fun_name",1}, - {"a_less_fun_name",1}]} = do_expand("expand_test:a_"), + {no,[],[#{ + title:="functions", + elems:=[{"a_fun_name",[{ending,"("}]}, + {"a_less_fun_name",_}, + {"b_comes_after_a",_}, + {"expand0arity_entirely",_}, + {"module_info",_}] + }]} = do_expand("expand_test:"), + {yes,[],[#{title:="functions", + elems:=[{"a_fun_name",_},{"a_less_fun_name",_}]}]} = do_expand("expand_test:a_"), {yes,"arity_entirely()",[]} = do_expand("expand_test:expand0"), ok. +to_atom(Str) -> + case erl_scan:string(Str) of + {ok, [{atom,_,A}], _} -> + {ok, A}; + _ -> + error + end. + +type_completion(_Config) -> + ct:timetrap({minutes, 20}), + {Time,_} = timer:tc(fun() -> do_expand("erl_pp:expr(") end), + case Time of + Time when Time > 2600000 -> {skip, "Expansion too slow on this machine"}; + _ -> + parallelforeach( + fun(Mod) -> + Exports = edlin_expand:get_exports(Mod), + [try + Str = io_lib:write_atom(Mod) ++ ":" ++ atom_to_list(Func) ++ "(", + do_expand(Str) + catch E:R:ST -> + erlang:raise(E, {R, Mod, Func}, ST) + end || {Func, _}<- Exports] + end, [list_to_atom(M) || {M,_,_} <- code:all_available()]), + ok + end. + +parallelforeach(Fun, List) -> + case parallelforeach(Fun, List, #{}, erlang:system_info(schedulers_online)) of + [] -> ok; + Else -> ct:fail(Else) + end. +parallelforeach(_Fun, [], Workers, _MaxWorkers) when map_size(Workers) =:= 0 -> + []; +parallelforeach(Fun, List, Workers, MaxWorkers) when MaxWorkers =:= map_size(Workers); + List =:= [] -> + receive + {'DOWN', Ref, _, _, normal} -> + parallelforeach(Fun, List, maps:remove(Ref, Workers), MaxWorkers); + + {'DOWN', Ref, _, _, Reason} -> + {Arg, NewWorkers} = maps:take(Ref, Workers), + [{Arg, Reason} | parallelforeach(Fun, List, NewWorkers, MaxWorkers)] + end; +parallelforeach(Fun, [H|T], Workers, MaxWorkers) -> + {_Pid, Ref} = spawn_monitor(fun() -> Fun(H) end), + parallelforeach(Fun, T, Workers#{ Ref => H }, MaxWorkers). + +filename_completion(Config) -> + DataDir = proplists:get_value(data_dir, Config), + {ok, CWD} = file:get_cwd(), + file:set_cwd(DataDir), + {yes,"e\"",[]} = do_expand("\"visible\\ fil"), + {no,[],[]} = do_expand("\"visible fil"), + {no,[],[]} = do_expand("\" "), + {no,[],[]} = do_expand("\"\""), + {yes, "e\"", []} = do_expand("\".hidden\\ fil"), + {yes,"/", []} = do_expand("\".."), + {yes,"ta/", _} = do_expand("\"../edlin_expand_SUITE_da"), + {yes,"erl\"",[]} = do_expand("\"complete_function_parameter."), + R = case {os:type(), file:native_name_encoding()} of + {{win32,_}, _} -> {skip, "Unicode on filenames in windows are tricky"}; + {_,latin1} -> {skip, "Cannot interpret unicode filenames when native_name_encoding is latin1"}; + _ -> + {yes,"isible", + [{"visible file",_},{"visible_file",_},{"visible๐_file",_}]} = do_expand("\"v"), + {yes,"e\"",[]} = do_expand("\"visible๐_fil"), + {yes,[], + [{"../",[]}, + {".hidden file",_}, + {".hidden_file",_}, + {".hidden๐_file",_}]} = do_expand("\"."), + ok + end, + file:set_cwd(CWD), + R. + +record_completion(_Config) -> + %% test record completion for loaded records + %% test record field name completion + %% test record field completion + {yes,"ord{", []} = do_expand("#a_rec"), + {yes,"uoted_record'{", []} = do_expand("#'Q"), + {no, [], [#{title:="fields", elems:=[{"a_field",_}, {"b_field",_}, {"c_field",_}, {"d_field",_}]}]} = do_expand("#a_record{"), + {no, [], [#{title:="fields", elems:=[{"a_field",_}, {"b_field",_}, {"c_field",_}, {"d_field",_}]}]} = do_expand("#a_record."), + {yes,"eld=", []} = do_expand("#a_record{a_fi"), + {no,[],[#{title:="types",elems:= + [{"atom1",[]}, + {"atom2",[]}, + {"btom",[]}, + {"'my atom'",[]}, + {"{atom3, ...}",[]}], + options:=[{hide,title}]}]} = do_expand("#a_record{a_field="), + %% test that an already specified field does not get suggested again + {no,[], + [#{title:="fields", elems:= + [{"a_field",[{ending,"="}]}, + {"b_field",[{ending,"="}]}, + {"c_field",[{ending,"="}]}, + {"d_field",[{ending,"="}]}], + options:=[highlight_all]}]} = do_expand("#a_record{a_field={atom3,b},"), + %% test match argument and closing parenthesis completion + {yes,", ",[]} = do_expand("#a_record{a_field={atom3"), + {no,[],[#{title:="types",elems:=[{"{atom4, ...}",[]}],options:=[{hide,title}]}]} = do_expand("#a_record{a_field={atom3,"), + {no,[],[#{title:="types",elems:=[{"integer() >= 0",[]}],options:=[{hide,title}]}]} = do_expand("#a_record{a_field={atom3,{atom4, "), + {yes,"}",_} = do_expand("#a_record{a_field={atom3,{atom4, 1"), + {yes,"}",_} = do_expand("#a_record{a_field={atom3,{atom4, 1}"), + ok. + +fun_completion(_Config) -> + {yes, "/1", []} = do_expand("fun lists:unzip3"), + {no, [], []} = do_expand("fun lists:unzip3/1,"), + {no, [], []} = do_expand("lists:unzip3/1"), + {no, [], [{"2",_},{"3",_}]} = do_expand("lists:seq/"), + %{yes, ", ", _} = do_expand("lists:all(fun erlang:is_atom/1"), + {no, [], [#{}]} = do_expand("lists:all(fun erlang:is_atom/1, "), + ok. + +binding_completion(_Config) -> + %% test that bindings in the shell can be completed + {yes,"ding",[]} = do_expand("Bin"), + {yes,"ding",[]} = do_expand("file:open(Bin"), + {yes,"ding",[]} = do_expand("fun (X, Y) -> Bin"), + %% test unicode + {yes,"รถndag", []} = do_expand("S"), + {yes,"", []} = do_expand("ร"), + ok. + +map_completion(_Config) -> + %% test that key suggestion works for a known map in bindings + {no,[],[{"a_key",[{ending, "=>"}]},{"b_key",_},{"c_key",_}]} = do_expand("MapBinding#{"), + {yes, "_key=>", []} = do_expand("MapBinding#{b"), + {yes, "_key=>", []} = do_expand("MapBinding # { b"), + %% test that an already specified key does not get suggested again + {no, [], [{"a_key",_},{"c_key", _}]} = do_expand("MapBinding#{b_key=>1,"), + %% test that unicode works + ok. + +function_parameter_completion(Config) -> + %% test first and second parameter + %% test multiple arities with same type on first parameter + %% test multiple arities with different type on first parameter + %% test that recursive types does not trigger endless loop + %% test that getting type of out of bound parameter does not trigger crash + compile_and_load2(Config,complete_function_parameter), + {no, [], [#{elems:=[#{title:="complete_function_parameter:an_untyped_fun/2", elems:=[]}]}]} = do_expand("complete_function_parameter:an_untyped_fun("), + {yes,":",[]} = do_expand("complete_function_parameter:an_untyped_fun(complete_function_parameter"), + {no, [], [#{elems:=[#{elems:=[#{title:="types",elems:=[{"integer()",[]}]}]}]}]} = do_expand("complete_function_parameter:a_fun_name("), + {no, [], [#{elems:=[#{elems:=[#{elems:=[{"integer()",[]}]}]}]}]} = do_expand("complete_function_parameter:a_fun_name(1,"), + {no, [], [#{elems:=[#{elems:=[#{elems:=[{"integer()",[]}]}]}]}]} = do_expand("complete_function_parameter : a_fun_name ( 1 , "), + {yes, ")", []} = do_expand("complete_function_parameter:a_fun_name(1,2"), + {no, [], []} = do_expand("complete_function_parameter:a_fun_name(1,2,"), + {no, [], [#{elems:=[#{elems:=[#{elems:=[{"any()",[]},{"[any() | [Deeplist]]",[]}]}]}]}]} = do_expand("complete_function_parameter:a_deeplist_fun("), + {no,[],[#{title:="typespecs", + elems:=[#{title:= + "complete_function_parameter:multi_arity_fun(T1)", + elems:=[#{title:="types", + elems:=[{"integer()",[]}], + options:=[{hide,title}]}], + options:=[{highlight_param,1}]}, + #{title:= + "complete_function_parameter:multi_arity_fun(T1, T2)", + elems:=[#{title:="types", + elems:=[{"integer()",[]}], + options:=[{hide,title}]}], + options:=[{highlight_param,1}]}, + #{title:= + "complete_function_parameter:multi_arity_fun()", + options:=[], + elems:=[")"]}], + options:=[highlight_all]}]} = do_expand("complete_function_parameter:multi_arity_fun("), + {no, [], [#{elems:=[#{elems:=[#{elems:=[{"true",[]},{"false",[]}]}]}]}]} = do_expand("complete_function_parameter:multi_arity_fun(1,"), + {no,[], + [#{elems := + [#{elems := + [#{elems := [{"integer()",[]}], + options := [{hide,title}], + title := "types"}], + options := [{highlight_param,1}], + title := + "complete_function_parameter:different_multi_arity_fun(T1)"}, + #{elems := + [#{elems := [{"true",[]},{"false",[]}], + options := [{hide,title}], + title := "types"}], + options := [{highlight_param,1}], + title := + "complete_function_parameter:different_multi_arity_fun(B1, T1)"}], + options := [highlight_all], + title := "typespecs"}]} = do_expand("complete_function_parameter:different_multi_arity_fun("), + {no, [], [#{elems:=[#{elems:=[#{elems:=[{"integer()",[]}]}]}]}]} = do_expand("complete_function_parameter:different_multi_arity_fun(false,"), + {no, [], [#{elems:=[#{elems:=[#{elems:=[{"{atom1, ...}",[]}, + {"atom1",[]}, + {"atom2",[]}, + {"[atom4 | atom5]",[]}]}]}]}]} = do_expand("complete_function_parameter:advanced_nested_parameter("), + {no, [], [#{elems:=[#{elems:=[#{elems:=[{"atom1",[]}]}]}]}]} = do_expand("complete_function_parameter:advanced_nested_parameter({"), + {no, [], [#{elems:=[#{elems:=[#{elems:=[{"{integer() >= 0, ...}",[]}]}]}]}]} = do_expand("complete_function_parameter:advanced_nested_parameter({atom1,"), + {no, [], [#{elems:=[#{elems:=[#{elems:=[{"atom4",[]},{"atom5",[]}]}]}]}]} = do_expand("complete_function_parameter:advanced_nested_parameter(["), + {no, [], [#{elems:=[#{elems:=[#{elems:=[{"atom4",[]},{"atom5",[]}]}]}]}]} = do_expand("complete_function_parameter : advanced_nested_parameter ( [ , "), + {no,[], + [#{elems := + [#{elems := + [#{elems := [{"integer()",[]}], + options := [{hide,title}], + title := "types"}], + options := [{highlight_param,1}], + title := + "complete_function_parameter:different_multi_arity_fun(T1)"}, + #{elems := + [#{elems := [{"true",[]},{"false",[]}], + options := [{hide,title}], + title := "types"}], + options := [{highlight_param,1}], + title := + "complete_function_parameter:different_multi_arity_fun(B1, T1)"}], + options := [highlight_all], + title := "typespecs"}]} = do_expand("complete_function_parameter:different_multi_arity_fun("), + %% Hide results where the type of the first parameters does not match the prototype header + {no,[], + [#{title:="typespecs", + elems:=[#{title:="complete_function_parameter:different_multi_arity_fun(B1, T1)", + elems:=[#{title:="types", + elems:=[{"integer()",[]}], + options:=[{hide,title}]}], + options:=[{highlight_param,2}]}], + options:=[highlight_all]}]} = do_expand("complete_function_parameter:different_multi_arity_fun(false,"), + {no, _, []} = do_expand("complete_function_parameter:different_multi_arity_fun(atom,"), + {yes, _, _} = do_expand("complete_function_parameter:'emoji"), + + ok. + +get_coverage(Config) -> + compile_and_load2(Config,complete_function_parameter), + do_expand("\""), + do_expand("\"."), + do_expand("\"../"), + do_expand("\"/"), + do_expand("fun m:f"), + do_expand("fun m:f/"), + do_expand("#"), + do_expand("MapBinding#{"), + do_expand("B"), + do_expand("#a_record{"), + do_expand("#a_record{a_field=>"), + + %% match_arguments and is_type tests + do_expand("complete_function_parameter:map_parameter_function(#{"), + do_expand("complete_function_parameter:map_parameter_function(#{a"), + do_expand("complete_function_parameter:map_parameter_function(#{a=>"), + do_expand("complete_function_parameter:map_parameter_function(#{a=>1"), + do_expand("complete_function_parameter:map_parameter_function(#{a=>1,"), + do_expand("complete_function_parameter:map_parameter_function(#{a=>1,d=>err"), + do_expand("complete_function_parameter:map_parameter_function(#{a=>1,d=>error"), + do_expand("complete_function_parameter:map_parameter_function(#{a=>1,d=>error}"), + do_expand("complete_function_parameter:map_parameter_function(#{a=>1,b=>2,c=>3,d=>error"), + do_expand("complete_function_parameter:map_parameter_function(#{}, "), + do_expand("complete_function_parameter:map_parameter_function(#{V=>1}, "), + do_expand("complete_function_parameter:map_parameter_function(#{a=>V}, "), + do_expand("complete_function_parameter:tuple_parameter_function({a,b}, "), + do_expand("complete_function_parameter:tuple_parameter_function({a,V}, "), + do_expand("complete_function_parameter:list_parameter_function([], "), + do_expand("complete_function_parameter:list_parameter_function([atom], "), + true = {no, [], []} =/= do_expand("complete_function_parameter:list_parameter_function([V], "), + do_expand("complete_function_parameter:non_empty_list_parameter_function([atom], "), + do_expand("complete_function_parameter:non_empty_list_parameter_function([], "), + do_expand("complete_function_parameter:binary_parameter_function(<<0>>, "), + do_expand("complete_function_parameter:binary_parameter_function(<<V>>, "), + do_expand("complete_function_parameter:integer_parameter_function(0, "), + do_expand("complete_function_parameter:non_neg_integer_parameter_function(1, "), + do_expand("complete_function_parameter:neg_integer_parameter_function(-1, "), + do_expand("complete_function_parameter:float_parameter_function(0.1, "), + do_expand("complete_function_parameter:pid_parameter_function(<0.1.0>, "), + do_expand("complete_function_parameter:port_parameter_function(#Port<0.1>, "), + do_expand("complete_function_parameter:record_parameter_function(#a_record{a = 1}, "), + do_expand("complete_function_parameter:function_parameter_function(#Fun<erl_eval.1.0>, "), + do_expand("complete_function_parameter:function_parameter_function(fun(X) -> X end, "), %% Todo verify fun arity + do_expand("complete_function_parameter:function_parameter_function(receive X -> X end, "), + do_expand("complete_function_parameter:function_parameter_function(V, "), + do_expand("complete_function_parameter:function_parameter_function((1+2), "), + do_expand("complete_function_parameter:function_parameter_function(some_call(), "), + do_expand("complete_function_parameter:function_parameter_function(#Nope<1.0>, "), + do_expand("complete_function_parameter:reference_parameter_function(#Ref<1.0.1.0>, "), + do_expand("complete_function_parameter:any_parameter_function(#Ref<1.0.1.0>, "), + do_expand("complete_function_parameter:ann_type_parameter_function(atom, "), + true = {no, [], []} =/= do_expand("complete_function_parameter:ann_type_parameter_function2(1, "), + do_expand("complete_function_parameter:atom_parameter_function(atom, "), + do_expand("complete_function_parameter:ann_type_parameter_function(atom"), + do_expand("complete_function_parameter:atom_parameter_function(atom"), + %% user_defined function + {yes, "func(", _} = + do_expand("my_"), + {no,[],[#{elems := + [#{elems := + [#{elems := [{"#my_record",[]}], + options := [{hide,title}], + title := "types"}], + options := [{highlight_param,1}], + title := "shell_default:my_func(A)"}], + options := [highlight_all], + title := "typespecs"}]} = + do_expand("my_func("), + {yes,"=",[]} = + do_expand("my_func(#my_record{ field"), + {no,[], + [#{elems := + [#{elems := [{"a_value",[]},{"b_value",[]}], + options := [{separator," :: "},{highlight_all}], + title := "erlang:my_type()"}], + options := [{hide,title}], + title := "types"}]} = + do_expand("my_func(#my_record{ field=>"), + {yes,"ue, ",[]} = + do_expand("my_func(#my_record{field=>a_val"), + %% bifs() + {yes, "st(", _} = + do_expand("integer_to_li"), + %% commands() + {yes, "(", _} = + do_expand("bt"), + {yes, ":", _}=edlin_expand:expand(lists:reverse("complete_function_parameter")), + {no, [], _}=edlin_expand:expand(lists:reverse("#")), + {no, [], _}=edlin_expand:expand(lists:reverse("UnbindedMap#")), + {no, [], _}=edlin_expand:expand(lists:reverse("UnbindedMap#{")), + {no, [], _}=edlin_expand:expand(lists:reverse("#{")), + {yes, "(", _}=edlin_expand:expand(lists:reverse("complete_function_parameter:a_fun_name")), + + do_expand("fun l"), + %{yes, ">", _} = + do_expand("fun () -"), + %{yes, "n ", _} = + do_expand("fun () whe "), + %{no, [], []} = + do_expand("fun () when "), + %{no, [], []} = + do_expand("fun () -> "), + %% {keyword, ...} + do_expand("M#"), + do_expand("#non_existant_record"), + do_expand("#a_record{ non_existand_field"), + + + %% match_arguments coverage + do_expand("complete_function_parameter:integer_parameter_function(atom,"), %% match_argument -> false + do_expand("complete_function_parameter:a_zero_arity_fun()"), %% match_argument, parameters empty + do_expand("erlang:system_info(thread"), + do_expand("erlang:system_info(thread_nope"), + do_expand("erlang:system_info(threads"), + do_expand("erlang:process_flag(priori"), + do_expand("erlang:process_flag(priority_nope"), + do_expand("erlang:process_flag(priority, "), + do_expand("erlang:process_flag(priority, atom"), + do_expand("erlang:process_flag(priority, 1"), + do_expand("erlang:system_info({allocator,"), + do_expand("lists:seq(1"), + do_expand("lists:seq(1, 10"), + do_expand("ssh:connect({"), + do_expand("ssh:connect({255,"), + do_expand("ssh:connect({'$i"), + do_expand("ssh:connect({'$x"), + do_expand("ssh:connect({1000,"), + do_expand("ssh:connect({255,255,255,255,255,255,255,255"), + do_expand("ssh:connect({255,255,255,255}, ["), + do_expand("ssh:connect(receive V -> V end, "), + do_expand("ssh:connect((1+2), "), + do_expand("ssh:connect(hej(), "), + do_expand("ssh:connect(fun() -> ok end, "), + do_expand("ssh:connect(fun a:b/1, "), + do_expand("ssh:connect(V, [in"), + do_expand("ssh:connect(V, [inet"), + do_expand("ssh:connect(V, [inet, "), + do_expand("atom_to_list("), + do_expand("help("), + do_expand("h(lists"), + do_expand("ht(lists"), + do_expand("h(file,"), + do_expand("ht(file,"), + do_expand("ht(file,mode"), + do_expand("file:get_cwd()"), + do_expand("fl"), + do_expand("fl("), + do_expand("fl()"), + do_expand("MapBinding#"), + do_expand("RecordBinding#"), + do_expand("TupleBinding#"), + do_expand("Binding#"), + do_expand("MyVar"), + {_, _, M0} = do_expand("ssh:connect("), + do_format(M0), + {_, _, M1} = do_expand("ssh:connect({"), + do_format(M1), + {_, _, M6} = do_expand("ssh:connect({},["), + do_format(M6), + lists:flatten(edlin_expand:format_matches(M6, 20)), + {_, _, M2} = do_expand("e"), + do_format(M2), + {_, _, M3} = do_expand("erlang:i"), + do_format(M3), + {_, _, M4} = do_expand("complete_function_parameter:an_untyped_fun("), + do_format(M4), + lists:flatten(edlin_expand:format_matches(M4, 20)), + {_,_,M5}=edlin_expand:expand("e"), + do_format(M5), + {_,_,M7}=edlin_expand:expand("erlang:"), + do_format(M7), + {_,_,M8}=edlin_expand:expand("e"), + do_format(M8), + lists:flatten(edlin_expand:format_matches(M8, 20)), + {_,_,M9}=edlin_expand:expand("complete_function_parameter:an_untyped_fun("), + lists:flatten(edlin_expand:format_matches(M9, 20)), + do_format(M5), + {_, _, M10} = edlin_expand:expand("ssh:connect({},["), + do_format(M10), + lists:flatten(edlin_expand:format_matches(M10, 20)), + ok. + %% Normal module name, some function names using quoted atoms. quoted_fun(Config) when is_list(Config) -> {module,expand_test} = compile_and_load(Config,expand_test), {module,expand_test1} = compile_and_load(Config,expand_test1), %% should be no colon after test this time - {yes, "test", []} = do_expand("expand_"), + {yes, "test", [#{title:="modules", elems:=[{"expand_test",[{ending, ":"}]},{"expand_test1",_}]}]} = do_expand("expand_"), {no, [], []} = do_expand("expandXX_"), - {no,[],[{"'#weird-fun-name'",1}, - {"'Quoted_fun_name'",0}, - {"'Quoted_fun_too'",0}, - {"a_fun_name",1}, - {"a_less_fun_name",1}, - {"b_comes_after_a",1}, - {"module_info",0}, - {"module_info",1}]} = do_expand("expand_test1:"), - {yes,"_",[]} = do_expand("expand_test1:a"), - {yes,[],[{"a_fun_name",1}, - {"a_less_fun_name",1}]} = do_expand("expand_test1:a_"), - {yes,[], - [{"'#weird-fun-name'",1}, - {"'Quoted_fun_name'",0}, - {"'Quoted_fun_too'",0}]} = do_expand("expand_test1:'"), - {yes,"uoted_fun_",[]} = do_expand("expand_test1:'Q"), - {yes,[], - [{"'Quoted_fun_name'",0}, - {"'Quoted_fun_too'",0}]} = do_expand("expand_test1:'Quoted_fun_"), + {no,[],[#{title:="functions", + elems:=[{"'#weird-fun-name'",_}, + {"'Quoted_fun_name'",_}, + {"'Quoted_fun_too'",_}, + {"a_fun_name",_}, + {"a_less_fun_name",_}, + {"b_comes_after_a",_}, + {"module_info",_}]}]} = do_expand("expand_test1:"), + {yes,"_",[#{elems:=[{"a_fun_name",_}, + {"a_less_fun_name",_}]}]} = do_expand("expand_test1:a"), + {yes,[],[#{elems:=[{"a_fun_name",_}, + {"a_less_fun_name",_}]}]} = do_expand("expand_test1:a_"), + {yes,[],[#{elems:=[{"'#weird-fun-name'",_}, + {"'Quoted_fun_name'",_}, + {"'Quoted_fun_too'",_}]}]} = do_expand("expand_test1:'"), + {yes,"uoted_fun_",[#{elems:=[{"'Quoted_fun_name'",_}, + {"'Quoted_fun_too'",_}]}]} = do_expand("expand_test1:'Q"), + {yes,[],[#{elems:=[{"'Quoted_fun_name'",_}, + {"'Quoted_fun_too'",_}]}]} = do_expand("expand_test1:'Quoted_fun_"), {yes,"weird-fun-name'(",[]} = do_expand("expand_test1:'#"), %% Since there is a module_info/1 as well as a module_info/0 @@ -116,146 +546,127 @@ quoted_fun(Config) when is_list(Config) -> quoted_module(Config) when is_list(Config) -> {module,'ExpandTestCaps'} = compile_and_load(Config,'ExpandTestCaps'), - {yes, "Caps':", []} = do_expand("'ExpandTest"), - {no,[], - [{"a_fun_name",1}, - {"a_less_fun_name",1}, - {"b_comes_after_a",1}, - {"module_info",0}, - {"module_info",1}]} = do_expand("'ExpandTestCaps':"), - {yes,[],[{"a_fun_name",1}, - {"a_less_fun_name",1}]} = do_expand("'ExpandTestCaps':a_"), + {yes, "Caps':",[]} = do_expand("'ExpandTest"), + {no,[],[#{elems:=[{"a_fun_name",_}, + {"a_less_fun_name",_}, + {"b_comes_after_a",_}, + {"module_info",_}]}]} = do_expand("'ExpandTestCaps':"), + {yes,[],[#{title:="functions", elems:=[{"a_fun_name",_}, + {"a_less_fun_name",_}]}]} = do_expand("'ExpandTestCaps':a_"), ok. quoted_both(Config) when is_list(Config) -> {module,'ExpandTestCaps'} = compile_and_load(Config,'ExpandTestCaps'), {module,'ExpandTestCaps1'} = compile_and_load(Config,'ExpandTestCaps1'), %% should be no colon (or quote) after test this time - {yes, "Caps", []} = do_expand("'ExpandTest"), - {no,[],[{"'#weird-fun-name'",0}, - {"'Quoted_fun_name'",0}, - {"'Quoted_fun_too'",0}, - {"a_fun_name",1}, - {"a_less_fun_name",1}, - {"b_comes_after_a",1}, - {"module_info",0}, - {"module_info",1}]} = do_expand("'ExpandTestCaps1':"), - {yes,"_",[]} = do_expand("'ExpandTestCaps1':a"), - {yes,[],[{"a_fun_name",1}, - {"a_less_fun_name",1}]} = do_expand("'ExpandTestCaps1':a_"), - {yes,[], - [{"'#weird-fun-name'",0}, - {"'Quoted_fun_name'",0}, - {"'Quoted_fun_too'",0}]} = do_expand("'ExpandTestCaps1':'"), - {yes,"uoted_fun_",[]} = do_expand("'ExpandTestCaps1':'Q"), - {yes,[], - [{"'Quoted_fun_name'",0}, - {"'Quoted_fun_too'",0}]} = do_expand("'ExpandTestCaps1':'Quoted_fun_"), + {yes, "Caps", [#{elems:=[{"'ExpandTestCaps'",[{ending, ":"}]},{"'ExpandTestCaps1'",_}]}]} = do_expand("'ExpandTest"), + {no,[],[#{elems:=[{"'#weird-fun-name'",_}, + {"'Quoted_fun_name'",_}, + {"'Quoted_fun_too'",_}, + {"a_fun_name",_}, + {"a_less_fun_name",_}, + {"b_comes_after_a",_}, + {"module_info",_}]}]} = do_expand("'ExpandTestCaps1':"), + {yes,"_",[#{elems:=[{"a_fun_name",_},{"a_less_fun_name",_}]}]} = do_expand("'ExpandTestCaps1':a"), + {yes,[],[#{elems:=[{"a_fun_name",_}, + {"a_less_fun_name",_}]}]} = do_expand("'ExpandTestCaps1':a_"), + {yes,[],[#{elems:=[{"'#weird-fun-name'",_}, + {"'Quoted_fun_name'",_}, + {"'Quoted_fun_too'",_}]}]} = do_expand("'ExpandTestCaps1':'"), + {yes,"uoted_fun_",[#{elems:=[{"'Quoted_fun_name'",_},{"'Quoted_fun_too'",_}]}]} = do_expand("'ExpandTestCaps1':'Q"), + {yes,[],[#{elems:=[{"'Quoted_fun_name'",_}, + {"'Quoted_fun_too'",_}]}]} = do_expand("'ExpandTestCaps1':'Quoted_fun_"), {yes,"weird-fun-name'()",[]} = do_expand("'ExpandTestCaps1':'#"), ok. %% Note: pull request #1152. erl_1152(Config) when is_list(Config) -> - "\n"++"foo"++" "++[1089]++_ = do_format(["foo",[1089]]), + "foo"++" "++[1089]++_ = do_format(["foo",[1089]]), ok. -erl_352(Config) when is_list(Config) -> - erl_352_test(3, 3), - - erl_352_test(3, 75), - erl_352_test(3, 76, [trailing]), - erl_352_test(4, 74), - erl_352_test(4, 75, [leading]), - erl_352_test(4, 76, [leading, trailing]), - - erl_352_test(75, 3), - erl_352_test(76, 3, [leading]), - erl_352_test(74, 4), - erl_352_test(75, 4, [leading]), - erl_352_test(76, 4, [leading]), - - erl_352_test(74, 74, [leading]), - erl_352_test(74, 75, [leading]), - erl_352_test(74, 76, [leading, trailing]). - -erl_352_test(PrefixLen, SuffixLen) -> - erl_352_test(PrefixLen, SuffixLen, []). - -erl_352_test(PrefixLen, SuffixLen, Dots) -> - io:format("\nPrefixLen = ~w, SuffixLen = ~w\n", [PrefixLen, SuffixLen]), - - PrefixM = lists:duplicate(PrefixLen, $p), - SuffixM = lists:duplicate(SuffixLen, $s), - LM = [PrefixM ++ S ++ SuffixM || S <- ["1", "2"]], - StrM = do_format(LM), - check_leading(StrM, "", PrefixM, SuffixM, Dots), - - PrefixF = lists:duplicate(PrefixLen, $p), - SuffixF = lists:duplicate(SuffixLen-2, $s), - LF = [{PrefixF ++ S ++ SuffixF, 1} || S <- ["1", "2"]], - StrF = do_format(LF), - true = check_leading(StrF, "/1", PrefixF, SuffixF, Dots), - +check_trailing(Config) when is_list(Config) -> + Str = lists:duplicate(80, $1), + StrF = do_format([Str]), + {_, "...\n"} = lists:split(76, StrF), ok. -check_leading(FormStr, ArityStr, Prefix, Suffix, Dots) -> - List = string:tokens(FormStr, "\n "), - io:format("~p\n", [List]), - true = lists:all(fun(L) -> length(L) < 80 end, List), - case lists:member(leading, Dots) of - true -> - true = lists:all(fun(L) -> - {"...", Rest} = lists:split(3, L), - check_trailing(Rest, ArityStr, - Suffix, Dots) - end, List); - false -> - true = lists:all(fun(L) -> - {Prefix, Rest} = - lists:split(length(Prefix), L), - check_trailing(Rest, ArityStr, - Suffix, Dots) - end, List) - end. - -check_trailing([I|Str], ArityStr, Suffix, Dots) -> - true = lists:member(I, [$1, $2]), - case lists:member(trailing, Dots) of - true -> - {Rest, "..." ++ ArityStr} = - lists:split(length(Str) - (3 + length(ArityStr)), Str), - true = lists:prefix(Rest, Suffix); - false -> - {Rest, ArityStr} = - lists:split(length(Str) - length(ArityStr), Str), - Rest =:= Suffix - end. - unicode(Config) when is_list(Config) -> {module,unicode_expand} = compile_and_load(Config,'unicode_expand'), - {no,[],[{"'ะบlะธัะธะปะปะธฬัะตัะบะธะน ะฐัะพะผ'",0}, - {"'ะบlะธัะธะปะปะธฬัะตัะบะธะน ะฐัะพะผ'",1}, - {"'ะบlะธัะธะปะปะธฬัะตัะบะธะน ะฐัะพะผB'",1}, - {"module_info",0}, - {"module_info",1}]} = do_expand("unicode_expand:"), - {yes,"ัะธะปะปะธฬัะตัะบะธะน ะฐัะพะผ", []} = do_expand("unicode_expand:'ะบlะธ"), - {yes,"ะตัะบะธะน ะฐัะพะผ", []} = do_expand("unicode_expand:'ะบlะธัะธะปะปะธฬั"), - {yes,"(",[]} = do_expand("unicode_expand:'ะบlะธัะธะปะปะธฬัะตัะบะธะน ะฐัะพะผB'"), - "\n'ะบlะธัะธะปะปะธฬัะตัะบะธะน ะฐัะพะผ'/0 'ะบlะธัะธะปะปะธฬัะตัะบะธะน ะฐัะพะผ'/1 " - "'ะบlะธัะธะปะปะธฬัะตัะบะธะน ะฐัะพะผB'/1 \nmodule_info/0 " - "module_info/1 \n" = - do_format([{"'ะบlะธัะธะปะปะธฬัะตัะบะธะน ะฐัะพะผ'",0}, - {"'ะบlะธัะธะปะปะธฬัะตัะบะธะน ะฐัะพะผ'",1}, - {"'ะบlะธัะธะปะปะธฬัะตัะบะธะน ะฐัะพะผB'",1}, - {"module_info",0}, - {"module_info",1}]), + {no,[], [#{elems:=[{"'ะบlะธัะธะปะปะธฬัะตัะบะธะน ะฐัะพะผ'",_}, + {"'ะบlะธัะธะปะปะธฬัะตัะบะธะน ะฐัะพะผB'",_}, + {"module_info",_}]}]} = do_expand("unicode_expand:"), + {yes,"ัะธะปะปะธฬัะตัะบะธะน ะฐัะพะผ", [#{elems:=[{"'ะบlะธัะธะปะปะธฬัะตัะบะธะน ะฐัะพะผ'",_}, + {"'ะบlะธัะธะปะปะธฬัะตัะบะธะน ะฐัะพะผB'",_}]}]} = do_expand("unicode_expand:'ะบlะธ"), + {yes,"ะตัะบะธะน ะฐัะพะผ", [#{elems:=[{"'ะบlะธัะธะปะปะธฬัะตัะบะธะน ะฐัะพะผ'",_}, + {"'ะบlะธัะธะปะปะธฬัะตัะบะธะน ะฐัะพะผB'",_}]}]} = do_expand("unicode_expand:'ะบlะธัะธะปะปะธฬั"), + {yes,"(", []} = do_expand("unicode_expand:'ะบlะธัะธะปะปะธฬัะตัะบะธะน ะฐัะพะผB'"), + "'ะบlะธัะธะปะปะธฬัะตัะบะธะน ะฐัะพะผ' 'ะบlะธัะธะปะปะธฬัะตัะบะธะน ะฐัะพะผB' module_info\n" = + do_format([{"'ะบlะธัะธะปะปะธฬัะตัะบะธะน ะฐัะพะผ'",[]}, + {"'ะบlะธัะธะปะปะธฬัะตัะบะธะน ะฐัะพะผB'",[]}, + {"module_info",[]}]), ok. do_expand(String) -> - edlin_expand:expand(lists:reverse(String)). + erlang:display(String), + Bs = [ + {'Binding', 0}, + {'MapBinding', #{a_key=>0, b_key=>1, c_key=>2}}, + {'RecordBinding', {some_record, 1, 2}}, + {'TupleBinding', {0, 1, 2}}, + {'Sรถndag', 0}, + {'ร', 0}], + Rt = ets:new(records, []), + + Rt2 = [{my_record, {attribute,[{text,"record"}, + {location,{1,2}}], + record, + {my_record, [{typed_record_field,{record_field,[{text,"field"}, + {location,{1,20}}], + {atom,[{text,"field"},{location,{1,20}}],field}, + {atom,[{text,"a_value"},{location,{1,28}}],a_value}}, + {user_type,[{text,"my_type"},{location,{1,33}}], + my_type,[]}}]}}}], + Ft = [{{function,{shell_default,my_func,1}},fun(_A)->0 end}, + {{function_type,{shell_default,my_func,1}}, + {attribute,[{text,"spec"},{location,{1,2}}], + spec, + {{my_func,1}, + [{type,[{text,"("},{location,{1,14}}], + bounded_fun, + [{type,[{text,"("},{location,{1,14}}], + 'fun', + [{type,[{text,"("},{location,{1,14}}], + product, + [{var,[{text,"A"},{location,{1,15}}],'A'}]}, + {type,[{text,"integer"},{location,{1,21}}],integer,[]}]}, + [{type,[{text,"A"},{location,{1,36}}], + constraint, + [{atom,[{text,"A"},{location,{1,36}}],is_subtype}, + [{var,[{text,"A"},{location,{1,36}}],'A'}, + {type,[{text,"#"},{location,{1,41}}], + record, + [{atom,[{text,"my_record"},{location,{1,42}}], + my_record}]}]]}]]}]}}}, + {{type,my_type}, + {attribute,[{text,"type"},{location,{1,2}}], + type, + {my_type,{type,[{text,"a"},{location,{1,20}}], + union, + [{atom,[{text,"a_value"},{location,{1,20}}],a_value}, + {atom,[{text,"b_value"},{location,{1,24}}],b_value}]}, + []}}}], + shell:read_and_add_records(edlin_expand_SUITE, '_', [], Bs, Rt), + edlin_expand:expand(lists:reverse(String), [], {shell_state, Bs, ets:tab2list(Rt)++Rt2, Ft}). do_format(StringList) -> - lists:flatten(edlin_expand:format_matches(StringList)). + lists:flatten(edlin_expand:format_matches(StringList, 79)). + +compile_and_load2(Config, Module) -> + Filename = filename:join( + proplists:get_value(data_dir,Config), + atom_to_list(Module)), + PrivDir = proplists:get_value(priv_dir,Config), + c:c(Filename, [debug_info, {outdir, PrivDir}]). compile_and_load(Config,Module) -> Filename = filename:join( diff --git a/lib/stdlib/test/edlin_expand_SUITE_data/.hidden file b/lib/stdlib/test/edlin_expand_SUITE_data/.hidden file new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/stdlib/test/edlin_expand_SUITE_data/.hidden file diff --git a/lib/stdlib/test/edlin_expand_SUITE_data/.hidden_file b/lib/stdlib/test/edlin_expand_SUITE_data/.hidden_file new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/stdlib/test/edlin_expand_SUITE_data/.hidden_file diff --git a/lib/stdlib/test/edlin_expand_SUITE_data/.hidden๐_file b/lib/stdlib/test/edlin_expand_SUITE_data/.hidden๐_file new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/stdlib/test/edlin_expand_SUITE_data/.hidden๐_file diff --git a/lib/stdlib/test/edlin_expand_SUITE_data/complete_function_parameter.erl b/lib/stdlib/test/edlin_expand_SUITE_data/complete_function_parameter.erl new file mode 100644 index 0000000000..c4ad9e13f8 --- /dev/null +++ b/lib/stdlib/test/edlin_expand_SUITE_data/complete_function_parameter.erl @@ -0,0 +1,164 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2023. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(complete_function_parameter). + +-export( + [a_fun_name/2, + an_untyped_fun/2, + a_deeplist_fun/1, + a_zero_arity_fun/0, + multi_arity_fun/0, + multi_arity_fun/1, + multi_arity_fun/2, + different_multi_arity_fun/1, + different_multi_arity_fun/2, + advanced_nested_parameter/1, + test_year/1, + 'emoji_function๐คฏ'/1, + map_parameter_function/1, + map_parameter_function/2, + tuple_parameter_function/2, + list_parameter_function/2, + non_empty_list_parameter_function/2, + binary_parameter_function/2, + neg_integer_parameter_function/2, + non_neg_integer_parameter_function/2, + integer_parameter_function/2, + float_parameter_function/2, + port_parameter_function/2, + pid_parameter_function/2, + record_parameter_function/2, + function_parameter_function/2, + reference_parameter_function/2, + any_parameter_function/2, + ann_type_parameter_function/2, + ann_type_parameter_function2/2, + atom_parameter_function/2 + ]). +-record(a_record, {}). +%% test first and second parameter + %% test multiple arities with same type on first parameter + %% test multiple arities with different type on first parameter + %% test that recursive types does not trigger endless loop + %% test that getting type of out of bound parameter does not trigger crash +-spec a_fun_name(Start, End) -> Return when + Start :: integer(), + End :: integer(), + Return:: integer(). +a_fun_name(_Start, _End) -> 0. + +an_untyped_fun(_Start, _End) -> 1. + +-spec a_deeplist_fun(Deeplist) -> integer() when + Deeplist :: T | [Deeplist], + T :: term(). +a_deeplist_fun(Deeplist) -> lists:flatten(Deeplist). + +a_zero_arity_fun() -> 0. + +-spec multi_arity_fun() -> integer(). +multi_arity_fun() -> 0. + +-spec multi_arity_fun(T1) -> integer() when + T1 :: integer(). +multi_arity_fun(_T1) -> 1. + +-spec multi_arity_fun(T1,T2) -> integer() when + T1 :: integer(), + T2 :: boolean(). +multi_arity_fun(_T1, _T2) -> 2. + +-spec different_multi_arity_fun(T1) -> integer() when + T1 :: integer(). +different_multi_arity_fun(_T1) -> 1. +-spec different_multi_arity_fun(B1, T1) -> integer() when + B1 :: boolean(), + T1 :: integer(). +different_multi_arity_fun(_T1, _T2) -> 2. + +-spec advanced_nested_parameter(T1) -> integer() when + T1 :: {atom1, {non_neg_integer(), non_neg_integer()}} | 'atom1' | 'atom2' | ['atom4' | 'atom5']. +advanced_nested_parameter(_T1) -> 0. + +-spec test_year(Y) -> integer() when + Y :: calendar:year(). +test_year(_Y) -> 0. + +-spec 'emoji_function๐คฏ'(integer()) -> integer(). +'emoji_function๐คฏ'(_) -> 0. + +-spec map_parameter_function(Map) -> boolean() when + Map :: #{a => 1, b => 2, c => 3, d => error}. +map_parameter_function(_) -> false. +-spec map_parameter_function(Map, any()) -> boolean() when + Map :: #{a => 1, b => 2, c => 3, d => error}. +map_parameter_function(_,_) -> false. + +-spec binary_parameter_function(binary(), any()) -> boolean(). +binary_parameter_function(_,_) -> false. + +-spec tuple_parameter_function(tuple(), any()) -> boolean(). +tuple_parameter_function(_,_) -> false. + +-spec list_parameter_function(list(), any()) -> boolean(). +list_parameter_function(_,_) -> false. + +-spec non_empty_list_parameter_function(nonempty_list(), any()) -> boolean(). +non_empty_list_parameter_function(_,_) -> false. + +-spec integer_parameter_function(integer(), any()) -> boolean(). +integer_parameter_function(_,_) -> false. + +-spec non_neg_integer_parameter_function(non_neg_integer(), any()) -> boolean(). +non_neg_integer_parameter_function(_,_) -> false. + +-spec neg_integer_parameter_function(neg_integer(), any()) -> boolean(). +neg_integer_parameter_function(_,_) -> false. + +-spec float_parameter_function(float(), any()) -> boolean(). +float_parameter_function(_,_) -> false. + +-spec pid_parameter_function(pid(), any()) -> boolean(). +pid_parameter_function(_,_) -> false. + +-spec port_parameter_function(port(), any()) -> boolean(). +port_parameter_function(_,_) -> false. + +-spec record_parameter_function(A, any()) -> boolean() when + A :: #a_record{}. +record_parameter_function(_,_) -> false. + +-spec function_parameter_function(fun((any()) -> any()), any()) -> boolean(). +function_parameter_function(_,_) -> false. + +-spec reference_parameter_function(reference(), any()) -> boolean(). +reference_parameter_function(_,_) -> false. + +-spec any_parameter_function(any(), any()) -> boolean(). +any_parameter_function(_,_) -> false. + +-spec atom_parameter_function(atom, any()) -> boolean(). +atom_parameter_function(_,_) -> false. + +-spec ann_type_parameter_function(V::atom(), W::any()) -> boolean(). +ann_type_parameter_function(_,_) -> false. + +-spec ann_type_parameter_function2(W::any(), V::atom()) -> boolean(). +ann_type_parameter_function2(_,_) -> false. diff --git a/lib/stdlib/test/edlin_expand_SUITE_data/visible file b/lib/stdlib/test/edlin_expand_SUITE_data/visible file new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/stdlib/test/edlin_expand_SUITE_data/visible file diff --git a/lib/stdlib/test/edlin_expand_SUITE_data/visible_file b/lib/stdlib/test/edlin_expand_SUITE_data/visible_file new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/stdlib/test/edlin_expand_SUITE_data/visible_file diff --git a/lib/stdlib/test/edlin_expand_SUITE_data/visible๐_file b/lib/stdlib/test/edlin_expand_SUITE_data/visible๐_file new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/stdlib/test/edlin_expand_SUITE_data/visible๐_file diff --git a/lib/stdlib/test/erl_eval_SUITE.erl b/lib/stdlib/test/erl_eval_SUITE.erl index faaa9f727f..af6802254c 100644 --- a/lib/stdlib/test/erl_eval_SUITE.erl +++ b/lib/stdlib/test/erl_eval_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2022. All Rights Reserved. +%% Copyright Ericsson AB 1998-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -55,7 +55,9 @@ otp_14708/1, otp_16545/1, otp_16865/1, - eep49/1]). + eep49/1, + binary_and_map_aliases/1, + eep58/1]). %% %% Define to run outside of test server @@ -96,7 +98,7 @@ all() -> otp_8133, otp_10622, otp_13228, otp_14826, funs, custom_stacktrace, try_catch, eval_expr_5, zero_width, eep37, eep43, otp_15035, otp_16439, otp_14708, otp_16545, otp_16865, - eep49]. + eep49, binary_and_map_aliases, eep58]. groups() -> []. @@ -1971,6 +1973,64 @@ eep49(Config) when is_list(Config) -> {else_clause,simply_wrong}), ok. +%% GH-6348/OTP-18297: Lift restrictions for matching of binaries and maps. +binary_and_map_aliases(Config) when is_list(Config) -> + check(fun() -> + <<A:16>> = <<B:8,C:8>> = <<16#cafe:16>>, + {A,B,C} + end, + "begin <<A:16>> = <<B:8,C:8>> = <<16#cafe:16>>, {A,B,C} end.", + {16#cafe,16#ca,16#fe}), + check(fun() -> + <<A:8/bits,B:24/bits>> = + <<C:16,D:16>> = + <<E:8,F:8,G:8,H:8>> = + <<16#abcdef57:32>>, + {A,B,C,D,E,F,G,H} + end, + "begin <<A:8/bits,B:24/bits>> = + <<C:16,D:16>> = + <<E:8,F:8,G:8,H:8>> = + <<16#abcdef57:32>>, + {A,B,C,D,E,F,G,H} + end.", + {<<16#ab>>,<<16#cdef57:24>>, 16#abcd,16#ef57, 16#ab,16#cd,16#ef,16#57}), + check(fun() -> + #{K := V} = #{k := K} = #{k => my_key, my_key => 42}, + V + end, + "begin #{K := V} = #{k := K} = #{k => my_key, my_key => 42}, V end.", + 42), + ok. + +%% EEP 58: Map comprehensions. +eep58(Config) when is_list(Config) -> + check(fun() -> X = 32, #{X => X*X || X <- [1,2,3]} end, + "begin X = 32, #{X => X*X || X <- [1,2,3]} end.", + #{1 => 1, 2 => 4, 3 => 9}), + check(fun() -> + K = V = none, + #{K => V*V || K := V <- #{1 => 1, 2 => 2, 3 => 3}} + end, + "begin K = V = none, #{K => V*V || K := V <- #{1 => 1, 2 => 2, 3 => 3}} end.", + #{1 => 1, 2 => 4, 3 => 9}), + check(fun() -> + #{K => V*V || K := V <- maps:iterator(#{1 => 1, 2 => 2, 3 => 3})} + end, + "#{K => V*V || K := V <- maps:iterator(#{1 => 1, 2 => 2, 3 => 3})}.", + #{1 => 1, 2 => 4, 3 => 9}), + check(fun() -> << <<K:8,V:24>> || K := V <- #{42 => 7777} >> end, + "<< <<K:8,V:24>> || K := V <- #{42 => 7777} >>.", + <<42:8,7777:24>>), + check(fun() -> [X || X := X <- #{a => 1, b => b}] end, + "[X || X := X <- #{a => 1, b => b}].", + [b]), + + error_check("[K+V || K := V <- a].", {bad_generator,a}), + error_check("[K+V || K := V <- [-1|#{}]].", {bad_generator,[-1|#{}]}), + + ok. + %% Check the string in different contexts: as is; in fun; from compiled code. check(F, String, Result) -> check1(F, String, Result), diff --git a/lib/stdlib/test/erl_expand_records_SUITE.erl b/lib/stdlib/test/erl_expand_records_SUITE.erl index ea5cc4a354..13aaf0abdb 100644 --- a/lib/stdlib/test/erl_expand_records_SUITE.erl +++ b/lib/stdlib/test/erl_expand_records_SUITE.erl @@ -39,7 +39,7 @@ -export([attributes/1, expr/1, guard/1, init/1, pattern/1, strict/1, update/1, otp_5915/1, otp_7931/1, otp_5990/1, - otp_7078/1, otp_7101/1, maps/1, + otp_7078/1, maps/1, side_effects/1]). init_per_testcase(_Case, Config) -> @@ -59,7 +59,7 @@ all() -> groups() -> [{tickets, [], - [otp_5915, otp_7931, otp_5990, otp_7078, otp_7101]}]. + [otp_5915, otp_7931, otp_5990, otp_7078]}]. init_per_suite(Config) -> Config. @@ -719,67 +719,8 @@ otp_7078(Config) when is_list(Config) -> run(Config, Ts, [strict_record_tests]), ok. --record(otp_7101, {a,b,c=[],d=[],e=[]}). - id(I) -> I. -%% OTP-7101. Record update: more than one call to setelement/3. -otp_7101(Config) when is_list(Config) -> - %% Ensure the compiler won't do any funny constant propagation tricks. - id(#otp_7101{a=a,b=b,c=c,d=d,e=e}), - Rec = id(#otp_7101{}), - - %% Spawn a tracer process to count the number of setelement/3 calls. - %% The tracer will forward all trace messages to us. - Self = self(), - Tracer = spawn_link(fun() -> otp_7101_tracer(Self, 0) end), - 1 = erlang:trace_pattern({erlang,setelement,3}, true), - erlang:trace(self(), true, [{tracer,Tracer},call]), - - %% Update the record. - #otp_7101{a=2,b=1,c=[],d=[],e=[]} = otp_7101_update1(Rec), - #otp_7101{a=1,b=2,c=[],d=[],e=[]} = otp_7101_update2(Rec), - #otp_7101{a=2,b=1,c=[],d=[],e=[]} = otp_7101_update3(Rec), - #otp_7101{a=1,b=2,c=[],d=[],e=[]} = otp_7101_update4(Rec), - - %% Verify that setelement/3 was called the same number of times as - %% the number of record updates. - Ref = erlang:trace_delivered(Self), - receive - {trace_delivered, Self, Ref} -> - Tracer ! done - end, - 1 = erlang:trace_pattern({erlang,setelement,3}, false), - receive - 4 -> - ok; - Other -> - ct:fail({unexpected,Other}) - end. - -otp_7101_tracer(Parent, N) -> - receive - {trace,Parent,call,{erlang,setelement,[_,_,_]}} -> - otp_7101_tracer(Parent, N+1); - done -> - Parent ! N - end. - -otp_7101_update1(R) -> - R#otp_7101{b=1, - a=2}. - -otp_7101_update2(R) -> - R#otp_7101{a=1, - b=2}. - -otp_7101_update3(R) -> - R#otp_7101{b=1,a=2}. - -otp_7101_update4(R) -> - R#otp_7101{a=1,b=2}. - - -record(side_effects, {a,b,c}). %% Make sure that the record expression is only evaluated once. diff --git a/lib/stdlib/test/erl_lint_SUITE.erl b/lib/stdlib/test/erl_lint_SUITE.erl index 770e12e3f0..81bc3e9a0d 100644 --- a/lib/stdlib/test/erl_lint_SUITE.erl +++ b/lib/stdlib/test/erl_lint_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2022. All Rights Reserved. +%% Copyright Ericsson AB 1999-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -35,7 +35,8 @@ -export([all/0, suite/0, groups/0]). --export([unused_vars_warn_basic/1, +-export([singleton_type_var_errors/1, + unused_vars_warn_basic/1, unused_vars_warn_lc/1, unused_vars_warn_rec/1, unused_vars_warn_fun/1, @@ -49,7 +50,8 @@ unsafe_vars_try/1, unsized_binary_in_bin_gen_pattern/1, guard/1, otp_4886/1, otp_4988/1, otp_5091/1, otp_5276/1, otp_5338/1, - otp_5362/1, otp_5371/1, otp_7227/1, otp_5494/1, otp_5644/1, otp_5878/1, + otp_5362/1, otp_5371/1, otp_7227/1, binary_aliases/1, + otp_5494/1, otp_5644/1, otp_5878/1, otp_5917/1, otp_6585/1, otp_6885/1, otp_10436/1, otp_11254/1, otp_11772/1, otp_11771/1, otp_11872/1, export_all/1, @@ -78,7 +80,9 @@ underscore_match/1, unused_record/1, unused_type2/1, - eep49/1]). + eep49/1, + redefined_builtin_type/1, + tilde_k/1]). suite() -> [{ct_hooks,[ts_install_cth]}, @@ -90,7 +94,7 @@ all() -> unsafe_vars, unsafe_vars2, unsafe_vars_try, guard, unsized_binary_in_bin_gen_pattern, otp_4886, otp_4988, otp_5091, otp_5276, otp_5338, - otp_5362, otp_5371, otp_7227, otp_5494, otp_5644, + otp_5362, otp_5371, otp_7227, binary_aliases, otp_5494, otp_5644, otp_5878, otp_5917, otp_6585, otp_6885, otp_10436, otp_11254, otp_11772, otp_11771, otp_11872, export_all, bif_clash, behaviour_basic, behaviour_multiple, otp_11861, @@ -106,7 +110,10 @@ all() -> no_load_nif, inline_nifs, warn_missing_spec, otp_16824, underscore_match, unused_record, unused_type2, - eep49]. + eep49, + redefined_builtin_type, + tilde_k, + singleton_type_var_errors]. groups() -> [{unused_vars_warn, [], @@ -166,7 +173,13 @@ c(A) -> g({M, F}) -> (Z=M):(Z=F)(); g({M, F, Arg}) -> (Z=M):F(Z=Arg). h(X, Y) -> (Z=X) + (Z=Y).">>, - [warn_unused_vars], []}], + [warn_unused_vars], []}, + {basic3, + <<"f(E) -> + X = Y = E.">>, + [warn_unused_vars], + {warnings,[{{2,19},erl_lint,{unused_var,'X'}}, + {{2,23},erl_lint,{unused_var,'Y'}}]}}], [] = run(Config, Ts), ok. @@ -297,7 +310,7 @@ unused_vars_warn_lc(Config) when is_list(Config) -> j(X) -> [foo || X <- X, % X shadowed. X <- % X shadowed. X unused. - X = + X = Y = [[1,2,3]], % Y unused. X <- [], % X shadowed. X <- X]. % X shadowed. X unused. @@ -892,6 +905,126 @@ unused_import(Config) when is_list(Config) -> [] = run(Config, Ts), ok. +%% Test singleton type variables +singleton_type_var_errors(Config) when is_list(Config) -> + Ts = [{singleton_error1, + <<"-spec test_singleton_typevars_in_union(Opts) -> term() when + Opts :: {ok, Unknown} | {error, Unknown}. + test_singleton_typevars_in_union(_) -> + error. + ">>, + [], + {warnings,[{{2,36},erl_lint,{singleton_typevar,'Unknown'}}]}}, + + {singleton_error2, + <<"-spec test_singleton_list_typevars_in_union([Opts]) -> term() when + Opts :: {ok, Unknown} | {error, Unknown}. + test_singleton_list_typevars_in_union(_) -> + error.">>, + [], + {warnings,[{{2,36},erl_lint,{singleton_typevar,'Unknown'}}]}}, + + {singleton_error3, + <<"-spec test_singleton_list_typevars_in_list([Opts]) -> term() when + Opts :: {ok, Unknown}. + test_singleton_list_typevars_in_list(_) -> + error.">>, + [], + {errors, + [{{2,36},erl_lint,{singleton_typevar,'Unknown'}}],[]}}, + + {singleton_error4, + <<"-spec test_singleton_list_typevars_in_list_with_type_subst([{ok, Unknown}]) -> term(). + test_singleton_list_typevars_in_list_with_type_subst(_) -> + error.">>, + [], + {errors,[{{1,86},erl_lint,{singleton_typevar,'Unknown'}}],[]}}, + + {singleton_error5, + <<"-spec test_singleton_buried_typevars_in_union(Opts) -> term() when + Opts :: {ok, Foo} | {error, Foo}, + Foo :: {true, X} | {false, X}. + test_singleton_buried_typevars_in_union(_) -> + error.">>, + [], + {warnings,[{{3,38},erl_lint,{singleton_typevar,'X'}}]}}, + + {singleton_error6, + <<"-spec test_multiple_subtypes_to_same_typevar(Opts) -> term() when + Opts :: {Foo, Bar} | Y, + Foo :: X, + Bar :: X, + Y :: Z. + test_multiple_subtypes_to_same_typevar(_) -> + error.">>, + [], + {errors,[{{5,31},erl_lint,{singleton_typevar,'Z'}}],[]}}, + + {singleton_error7, + <<"-spec test_duplicate_non_terminal_var_in_union(Opts) -> term() when + Opts :: {ok, U, U} | {error, U, U}, + U :: Foo. + test_duplicate_non_terminal_var_in_union(_) -> + error.">>, + [], + {errors,[{{3,31},erl_lint,{singleton_typevar,'Foo'}}],[]}}, + + {singleton_error8, + <<"-spec test_unused_outside_union(Opts) -> term() when + Unused :: Unknown, + A :: Unknown, + Opts :: {Unknown | A}. + test_unused_outside_union(_) -> + error.">>, + [], + {errors,[{{2,21},erl_lint,{singleton_typevar,'Unused'}}],[]}}, + + {singleton_disabled_warning, + <<"-spec test_singleton_typevars_in_union(Opts) -> term() when + Opts :: {ok, Unknown} | {error, Unknown}. + test_singleton_typevars_in_union(_) -> + error. + ">>, + [nowarn_singleton_typevar], + []}, + + {singleton_ok1, + <<"-spec test_multiple_occurrences_singleton(Opts) -> term() when + Opts :: {Foo, Foo}. + test_multiple_occurrences_singleton(_) -> + ok.">>, + [], + []}, + + {singleton_ok2, + <<"-spec id(X) -> X. + id(X) -> + X.">>, + [], + []}, + + {singleton_ok3, + <<"-spec ok(Opts) -> term() when + Opts :: {Unknown, {ok, Unknown} | {error, Unknown}}. + ok(_) -> + error.">>, + [], + []}, + + {singleton_ok4, + <<"-spec ok(Opts) -> term() when + Union :: {ok, Unknown} | {error, Unknown}, + Opts :: {{tag, Unknown} | Union}. + ok(_) -> + error.">>, + [], + []} + + ], + + [] = run(Config, Ts), + ok. + %% Test warnings for unused functions. unused_function(Config) when is_list(Config) -> Ts = [{func1, @@ -966,13 +1099,13 @@ binary_types(Config) when is_list(Config) -> Ts = [{binary1, <<"-type nonempty_binary() :: term().">>, [nowarn_unused_type], - {errors,[{{1,22},erl_lint, - {builtin_type,{nonempty_binary,0}}}],[]}}, + {warnings,[{{1,22},erl_lint, + {redefine_builtin_type,{nonempty_binary,0}}}]}}, {binary2, <<"-type nonempty_bitstring() :: term().">>, [nowarn_unused_type], - {errors,[{{1,22},erl_lint, - {builtin_type,{nonempty_bitstring,0}}}],[]}}], + {warnings,[{{1,22},erl_lint, + {redefine_builtin_type,{nonempty_bitstring,0}}}]}}], [] = run(Config, Ts), ok. @@ -2296,22 +2429,22 @@ otp_15456(Config) when is_list(Config) -> ok. %% OTP-5371. Aliases for bit syntax expressions are no longer allowed. +%% GH-6348/OTP-18297: Updated for OTP 26 to allow aliases. otp_5371(Config) when is_list(Config) -> Ts = [{otp_5371_1, <<"t(<<A:8>> = <<B:8>>) -> {A,B}. ">>, [], - {errors,[{{1,23},erl_lint,illegal_bin_pattern}],[]}}, + []}, {otp_5371_2, <<"x([<<A:8>>] = [<<B:8>>]) -> {A,B}. y({a,<<A:8>>} = {b,<<B:8>>}) -> {A,B}. ">>, - [], - {errors,[{{1,24},erl_lint,illegal_bin_pattern}, - {{3,20},erl_lint,illegal_bin_pattern}],[]}}, + [], + {warnings,[{{3,15},v3_core,{nomatch,pattern}}]}}, {otp_5371_3, <<"-record(foo, {a,b,c}). -record(bar, {x,y,z}). @@ -2328,11 +2461,10 @@ otp_5371(Config) when is_list(Config) -> {X,Y}. ">>, [], - {errors,[{{4,26},erl_lint,illegal_bin_pattern}, - {{6,26},erl_lint,illegal_bin_pattern}, - {{8,26},erl_lint,illegal_bin_pattern}, - {{10,30},erl_lint,illegal_bin_pattern}, - {{12,30},erl_lint,illegal_bin_pattern}],[]}}, + {warnings,[{{4,15},v3_core,{nomatch,pattern}}, + {{8,15},v3_core,{nomatch,pattern}}, + {{10,15},v3_core,{nomatch,pattern}}, + {{12,15},v3_core,{nomatch,pattern}}]}}, {otp_5371_4, <<"-record(foo, {a,b,c}). -record(bar, {x,y,z}). @@ -2352,42 +2484,41 @@ otp_5371(Config) when is_list(Config) -> [] = run(Config, Ts), ok. -%% OTP_7227. Some aliases for bit syntax expressions were still allowed. +%% OTP-7227. Some aliases for bit syntax expressions were still allowed. +%% GH-6348/OTP-18297: Updated for OTP 26 to allow aliases. otp_7227(Config) when is_list(Config) -> Ts = [{otp_7227_1, <<"t([<<A:8>> = {C,D} = <<B:8>>]) -> {A,B,C,D}. ">>, [], - {errors,[{{1,42},erl_lint,illegal_bin_pattern}],[]}}, + {warnings,[{{1,21},v3_core,{nomatch,pattern}}]}}, {otp_7227_2, <<"t([(<<A:8>> = {C,D}) = <<B:8>>]) -> {A,B,C,D}. ">>, [], - {errors,[{{1,25},erl_lint,illegal_bin_pattern}],[]}}, + {warnings,[{{1,21},v3_core,{nomatch,pattern}}]}}, {otp_7227_3, <<"t([(<<A:8>> = {C,D}) = (<<B:8>> = <<C:8>>)]) -> {A,B,C,D}. ">>, [], - {errors,[{{1,45},erl_lint,illegal_bin_pattern}, - {{1,45},erl_lint,illegal_bin_pattern}, - {{1,55},erl_lint,illegal_bin_pattern}],[]}}, + {warnings,[{{1,21},v3_core,{nomatch,pattern}}]}}, {otp_7227_4, <<"t(Val) -> <<A:8>> = <<B:8>> = Val, {A,B}. ">>, [], - {errors,[{{2,19},erl_lint,illegal_bin_pattern}],[]}}, + []}, {otp_7227_5, <<"t(Val) -> <<A:8>> = X = <<B:8>> = Val, {A,B,X}. ">>, [], - {errors,[{{2,19},erl_lint,illegal_bin_pattern}],[]}}, + []}, {otp_7227_6, <<"t(X, Y) -> <<A:8>> = <<X:4,Y:4>>, @@ -2401,27 +2532,70 @@ otp_7227(Config) when is_list(Config) -> {A,B,X}. ">>, [], - {errors,[{{2,36},erl_lint,illegal_bin_pattern}, - {{2,36},erl_lint,illegal_bin_pattern}, - {{2,46},erl_lint,illegal_bin_pattern}],[]}}, - {otp_7227_8, + []}, + {otp_7227_8, <<"t(Val) -> (<<A:8>> = X) = (Y = <<B:8>>) = Val, {A,B,X,Y}. ">>, [], - {errors,[{{2,40},erl_lint,illegal_bin_pattern}],[]}}, + []}, {otp_7227_9, <<"t(Val) -> (Z = <<A:8>> = X) = (Y = <<B:8>> = W) = Val, {A,B,X,Y,Z,W}. ">>, [], - {errors,[{{2,44},erl_lint,illegal_bin_pattern}],[]}} + []} ], [] = run(Config, Ts), ok. +%% GH-6348/OTP-18297: Allow aliases of binary patterns. +binary_aliases(Config) when is_list(Config) -> + Ts = [{binary_aliases_1, + <<"t([<<Size:8,_/bits>> = <<_:8,Data:Size/bits>>]) -> + Data. + ">>, + [], + {errors,[{{1,55},erl_lint,{unbound_var,'Size'}}],[]}}, + {binary_aliases_2, + <<"t(#{key := <<Size:8,_/bits>>} = #{key := <<_:8,Data:Size/bits>>}) -> + Data. + ">>, + [], + {errors,[{{1,73},erl_lint,{unbound_var,'Size'}}],[]}}, + {binary_aliases_3, + <<"t(<<_:8,Data:Size/bits>> = <<Size:8,_/bits>>) -> + Data. + ">>, + [], + {errors,[{{1,34},erl_lint,{unbound_var,'Size'}}],[]}}, + {binary_aliases_4, + <<"t([<<_:8,Data:Size/bits>> = <<Size:8,_/bits>>]) -> + Data. + ">>, + [], + {errors,[{{1,35},erl_lint,{unbound_var,'Size'}}],[]}}, + {binary_aliases_5, + <<"t(Bin) -> + <<_:8,A:Size>> = <<_:8,B:Size/bits>> = <<Size:8,_/bits>> = Bin, + {A,B,Size}. + ">>, + [], + []}, + {binary_aliases_6, + <<"t(<<_:8,A:Size>> = <<_:8,B:Size/bits>> = <<Size:8,_/bits>>) -> + {A,B,Size}. + ">>, + [], + {errors,[{{1,31},erl_lint,{unbound_var,'Size'}}, + {{1,48},erl_lint,{unbound_var,'Size'}}], + []}} + ], + [] = run(Config, Ts), + ok. + %% OTP-5494. Warnings for functions exported more than once. otp_5494(Config) when is_list(Config) -> Ts = [{otp_5494_1, @@ -2780,14 +2954,6 @@ otp_10436(Config) when is_list(Config) -> {warnings,[{{4,14},erl_lint,{not_exported_opaque,{t2,0}}}, {{4,14},erl_lint,{unused_type,{t2,0}}}]} = run_test2(Config, Ts, []), - Ts2 = <<"-module(otp_10436_2). - -export_type([t1/0, t2/0]). - -opaque t1() :: term(). - -opaque t2() :: any(). - ">>, - {warnings,[{{3,15},erl_lint,{underspecified_opaque,{t1,0}}}, - {{4,15},erl_lint,{underspecified_opaque,{t2,0}}}]} = - run_test2(Config, Ts2, []), ok. %% OTP-11254. M:F/A could crash the linter. @@ -2822,9 +2988,9 @@ otp_11772(Config) when is_list(Config) -> t() -> 1. ">>, - {errors,[{{7,14},erl_lint,{builtin_type,{node,0}}}, - {{8,14},erl_lint,{builtin_type,{mfa,0}}}], - []} = run_test2(Config, Ts, []), + {warnings,[{{7,14},erl_lint,{redefine_builtin_type,{node,0}}}, + {{8,14},erl_lint,{redefine_builtin_type,{mfa,0}}}]} = + run_test2(Config, Ts, []), ok. %% OTP-11771. Do not allow redefinition of the types arity(_) &c.. @@ -2847,11 +3013,11 @@ otp_11771(Config) when is_list(Config) -> t() -> 1. ">>, - {errors,[{{7,14},erl_lint,{builtin_type,{arity,0}}}, - {{8,14},erl_lint,{builtin_type,{bitstring,0}}}, - {{9,14},erl_lint,{builtin_type,{iodata,0}}}, - {{10,14},erl_lint,{builtin_type,{boolean,0}}}], - []} = run_test2(Config, Ts, []), + {warnings,[{{7,14},erl_lint,{redefine_builtin_type,{arity,0}}}, + {{8,14},erl_lint,{redefine_builtin_type,{bitstring,0}}}, + {{9,14},erl_lint,{redefine_builtin_type,{iodata,0}}}, + {{10,14},erl_lint,{redefine_builtin_type,{boolean,0}}}]} = + run_test2(Config, Ts, []), ok. %% OTP-11872. The type map() undefined when exported. @@ -2863,15 +3029,16 @@ otp_11872(Config) when is_list(Config) -> -export_type([map/0, product/0]). - -opaque map() :: dict(). + -opaque map() :: unknown_type(). -spec t() -> map(). t() -> 1. ">>, - {errors,[{{6,14},erl_lint,{undefined_type,{product,0}}}, - {{8,14},erl_lint,{builtin_type,{map,0}}}], []} = + {error,[{{6,14},erl_lint,{undefined_type,{product,0}}}, + {{8,30},erl_lint,{undefined_type,{unknown_type,0}}}], + [{{8,14},erl_lint,{redefine_builtin_type,{map,0}}}]} = run_test2(Config, Ts, []), ok. @@ -3881,35 +4048,61 @@ maps_type(Config) when is_list(Config) -> t(M) -> M. ">>, [], - {errors,[{{3,7},erl_lint,{builtin_type,{map,0}}}],[]}}], + {warnings,[{{3,7},erl_lint,{redefine_builtin_type,{map,0}}}]}}], [] = run(Config, Ts), ok. +%% GH-6348/OTP-18297: In OTP 26 parallel matching of maps +%% has been extended. maps_parallel_match(Config) when is_list(Config) -> - Ts = [{parallel_map_patterns_unbound1, + Ts = [{parallel_map_patterns_unbound, <<" t(#{} = M) -> - #{K := V} = #{k := K} = M, + #{k := K} = #{K := V} = M, V. ">>, [], - {errors,[{{3,18},erl_lint,{unbound_var,'K'}}],[]}}, - {parallel_map_patterns_unbound2, + {errors,[{{3,30},erl_lint,{unbound_var,'K'}}],[]}}, + {parallel_map_patterns_not_toplevel1, <<" t(#{} = M) -> + [#{K1 := V1} = + #{K2 := V2} = + #{k1 := K1,k2 := K2}] = [M], + [V1,V2]. + ">>, + [], + {errors,[{{3,19},erl_lint,{unbound_var,'K1'}}, + {{4,19},erl_lint,{unbound_var,'K2'}}],[]}}, + {parallel_map_patterns_unbound_not_toplevel2, + <<" + t(#{} = M) -> + [#{k := K} = #{K := V}] = [M], + V. + ">>, + [], + {errors,[{{3,31},erl_lint,{unbound_var,'K'}}],[]}}, + {parallel_map_patterns_bound1, + <<" + t(#{} = M,K1,K2) -> #{K1 := V1} = #{K2 := V2} = #{k1 := K1,k2 := K2} = M, [V1,V2]. ">>, [], - {errors,[{{3,18},erl_lint,{unbound_var,'K1'}}, - {{3,18},erl_lint,{unbound_var,'K1'}}, - {{4,18},erl_lint,{unbound_var,'K2'}}, - {{4,18},erl_lint,{unbound_var,'K2'}}],[]}}, - {parallel_map_patterns_bound, + []}, + {parallel_map_patterns_bound2, <<" - t(#{} = M,K1,K2) -> + t(#{} = M) -> + #{K := V} = #{k := K} = M, + V. + ">>, + [], + []}, + {parallel_map_patterns_bound3, + <<" + t(#{} = M) -> #{K1 := V1} = #{K2 := V2} = #{k1 := K1,k2 := K2} = M, @@ -4248,8 +4441,8 @@ otp_14323(Config) -> -dialyzer(nowarn_function). % unknown option -dialyzer(1). % badly formed - -dialyzer(malformed). % unkonwn option - -dialyzer({malformed,f/0}). % unkonwn option + -dialyzer(malformed). % unknown option + -dialyzer({malformed,f/0}). % unknown option -dialyzer({nowarn_function,a/1}). % undefined function -dialyzer({nowarn_function,{a,-1}}). % badly formed @@ -4849,6 +5042,151 @@ eep49(Config) when is_list(Config) -> [] = run(Config, Ts), ok. +%% GH-6132: Allow local redefinition of types. +redefined_builtin_type(Config) -> + Ts = [{redef1, + <<"-type nonempty_binary() :: term(). + -type map() :: {_,_}.">>, + [nowarn_unused_type, + nowarn_redefined_builtin_type], + []}, + {redef2, + <<"-type nonempty_bitstring() :: term(). + -type map() :: {_,_}.">>, + [nowarn_unused_type, + {nowarn_redefined_builtin_type,{map,0}}], + {warnings,[{{1,22},erl_lint, + {redefine_builtin_type,{nonempty_bitstring,0}}}]}}, + {redef3, + <<"-compile({nowarn_redefined_builtin_type,{map,0}}). + -compile({nowarn_redefined_builtin_type,[{nonempty_bitstring,0}]}). + -type nonempty_bitstring() :: term(). + -type map() :: {_,_}. + -type list() :: erlang:map().">>, + [nowarn_unused_type, + {nowarn_redefined_builtin_type,{map,0}}], + {warnings,[{{5,16},erl_lint, + {redefine_builtin_type,{list,0}}}]}}, + {redef4, + <<"-type tuple() :: 'tuple'. + -type map() :: 'map'. + -type list() :: 'list'. + -spec t(tuple() | map()) -> list(). + t(_) -> ok. + ">>, + [], + {warnings,[{{1,22},erl_lint,{redefine_builtin_type,{tuple,0}}}, + {{2,16},erl_lint,{redefine_builtin_type,{map,0}}}, + {{3,16},erl_lint,{redefine_builtin_type,{list,0}}} + ]}}, + {redef5, + <<"-type atom() :: 'atom'. + -type integer() :: 'integer'. + -type reference() :: 'ref'. + -type pid() :: 'pid'. + -type port() :: 'port'. + -type float() :: 'float'. + -type iodata() :: 'iodata'. + -type ref_set() :: gb_sets:set(reference()). + -type pid_map() :: #{pid() => port()}. + -type atom_int_fun() :: fun((atom()) -> integer()). + -type collection(Type) :: {'collection', Type}. + -callback b1(I :: iodata()) -> atom(). + -spec t(collection(float())) -> {pid_map(), ref_set(), atom_int_fun()}. + t(_) -> ok. + ">>, + [], + {warnings,[{{1,22},erl_lint,{redefine_builtin_type,{atom,0}}}, + {{2,16},erl_lint,{redefine_builtin_type,{integer,0}}}, + {{3,16},erl_lint,{redefine_builtin_type,{reference,0}}}, + {{4,16},erl_lint,{redefine_builtin_type,{pid,0}}}, + {{5,16},erl_lint,{redefine_builtin_type,{port,0}}}, + {{6,16},erl_lint,{redefine_builtin_type,{float,0}}}, + {{7,16},erl_lint,{redefine_builtin_type,{iodata,0}}} + ]}}, + {redef6, + <<"-spec bar(function()) -> bar(). + bar({function, F}) -> F(). + -type function() :: {function, fun(() -> bar())}. + -type bar() :: {bar, binary()}. + ">>, + [], + {warnings,[{{3,16},erl_lint, + {redefine_builtin_type,{function,0}}}]}}, + {redef7, + <<"-type function() :: {function, fun(() -> bar())}. + -type bar() :: {bar, binary()}. + -spec bar(function()) -> bar(). + bar({function, F}) -> F(). + ">>, + [], + {warnings,[{{1,22},erl_lint, + {redefine_builtin_type,{function,0}}}]}}, + {redef8, + <<"-type function() :: {function, fun(() -> atom())}. + ">>, + [], + {warnings,[{{1,22},erl_lint, + {redefine_builtin_type,{function,0}}}, + {{1,22},erl_lint, + {unused_type,{function,0}}}]}}, + {redef9, + <<"-spec foo() -> fun(). + foo() -> fun() -> ok end. + ">>, + [], + []} + ], + [] = run(Config, Ts), + ok. + +tilde_k(Config) -> + Ts = [{tilde_k_1, + <<"t(Map) -> + io:format(\"~kp\n\", [Map]), + io:format(\"~kP\n\", [Map,10]), + io:format(\"~kw\n\", [Map]), + io:format(\"~kW\n\", [Map,5]), + io:format(\"~tkp\n\", [Map]), + io:format(\"~klp\n\", [Map]), + RevCmpFun = fun erlang:'>='/2, + io:format(\"~Kp\n\", [RevCmpFun,Map]), + io:format(\"~KP\n\", [RevCmpFun,Map,10]), + io:format(\"~Kw\n\", [RevCmpFun,Map]), + io:format(\"~KW\n\", [RevCmpFun,Map,5]), + ok.">>, + [], + []}, + {tilde_k_2, + <<"t(Map) -> + io:format(\"~kkp\n\", [Map]), + io:format(\"~kKp\n\", [Map]), + io:format(\"~ks\n\", [Map]), + ok.">>, + [], + {warnings, + [{{2,29}, + erl_lint, + {format_error,{"format string invalid (~ts)", + ["repeated modifier k"]}}}, + {{4,29}, + erl_lint, + {format_error,{"format string invalid (~ts)", + ["conflicting modifiers ~Kkp"]}}}, + {{6,29}, + erl_lint, + {format_error,{"format string invalid (~ts)", + ["invalid modifier/control combination ~ks"]}}}]} + } + ], + [] = run(Config, Ts), + + ok. + +%%% +%%% Common utilities. +%%% + format_error(E) -> lists:flatten(erl_lint:format_error(E)). diff --git a/lib/stdlib/test/erl_pp_SUITE.erl b/lib/stdlib/test/erl_pp_SUITE.erl index c8c1a206ca..ef021aa691 100644 --- a/lib/stdlib/test/erl_pp_SUITE.erl +++ b/lib/stdlib/test/erl_pp_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2006-2021. All Rights Reserved. +%% Copyright Ericsson AB 2006-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -56,7 +56,7 @@ otp_10302/1, otp_10820/1, otp_11100/1, otp_11861/1, pr_1014/1, otp_13662/1, otp_14285/1, otp_15592/1, otp_15751/1, otp_15755/1, otp_16435/1, gh_5093/1, - eep49/1]). + eep49/1, eep58/1]). %% Internal export. -export([ehook/6]). @@ -88,7 +88,7 @@ groups() -> otp_8473, otp_8522, otp_8567, otp_8664, otp_9147, otp_10302, otp_10820, otp_11100, otp_11861, pr_1014, otp_13662, otp_14285, otp_15592, otp_15751, otp_15755, otp_16435, - gh_5093, eep49]}]. + gh_5093, eep49, eep58]}]. init_per_suite(Config) -> Config. @@ -1393,6 +1393,27 @@ eep49(_Config) -> " end.\n"), ok. +eep58(_Config) -> + assert_same("lc_map(Map) ->\n" + " [ \n" + " {K, V} ||\n" + " K := V <- Map\n" + " ].\n"), + + assert_same("bc_map(Map) ->\n" + " << \n" + " <<K:32,V:32>> ||\n" + " K := V <- Map\n" + " >>.\n"), + + assert_same("mc(Map) ->\n" + " #{ \n" + " K => V + 1 ||\n" + " K := V <- Map\n" + " }.\n"), + + ok. + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% compile(Config, Tests) -> diff --git a/lib/stdlib/test/erl_scan_SUITE.erl b/lib/stdlib/test/erl_scan_SUITE.erl index ee8bc8420f..96c68039ae 100644 --- a/lib/stdlib/test/erl_scan_SUITE.erl +++ b/lib/stdlib/test/erl_scan_SUITE.erl @@ -516,12 +516,17 @@ chars() -> test_string(L, Ts) end || C <- lists:seq(0, 255)], - %% $\^\n now increments the line... + %% GH-6477. Test legal use of caret notation. [begin L = "$\\^" ++ [C], - Ts = [{char,{1,1},C band 2#11111}], + Ts = case C of + $? -> + [{char,{1,1},127}]; + _ -> + [{char,{1,1},C band 2#11111}] + end, test_string(L, Ts) - end || C <- lists:seq(0, 255)], + end || C <- lists:seq($?, $Z) ++ lists:seq($a, $z)], [begin L = "$\\" ++ [C], @@ -672,10 +677,18 @@ illegal() -> erl_scan:string(String, {1,1}), {done,{error,{{1,4},erl_scan,{illegal,character}},{1,14}},"34\". "} = erl_scan:tokens([], String++". ", {1,1}), + + %% GH-6477. Test for illegal characters in caret notation. + _ = [begin + S = [$$,$\\,$^,C], + {error,{1,erl_scan,{illegal,character}},1} = erl_scan:string(S) + end || C <- lists:seq(0, 16#3e) ++ [16#60] ++ lists:seq($z+1, 16#10ffff)], ok. crashes() -> {'EXIT',_} = (catch {foo, erl_scan:string([-1])}), % type error + {'EXIT',_} = (catch erl_scan:string("'a" ++ [999999999] ++ "c'")), + {'EXIT',_} = (catch {foo, erl_scan:string("$"++[-1])}), {'EXIT',_} = (catch {foo, erl_scan:string("$\\"++[-1])}), {'EXIT',_} = (catch {foo, erl_scan:string("$\\^"++[-1])}), @@ -698,6 +711,7 @@ crashes() -> (catch {foo, erl_scan:string("% foo"++[a],{1,1})}), {'EXIT',_} = (catch {foo, erl_scan:string([3.0])}), % type error + {'EXIT',_} = (catch {foo, erl_scan:string("A" ++ [999999999])}), ok. @@ -867,11 +881,11 @@ unicode() -> erl_scan:string([1089]), {error,{{1,1},erl_scan,{illegal,character}},{1,2}} = erl_scan:string([1089], {1,1}), - {error,{{1,3},erl_scan,{illegal,character}},{1,4}} = - erl_scan:string("'a" ++ [999999999] ++ "c'", {1,1}), + {error,{{1,1},erl_scan,{illegal,character}},{1,2}} = + erl_scan:string([16#D800], {1,1}), test("\"a"++[1089]++"b\""), - {ok,[{char,1,1}],1} = + {error,{1,erl_scan,{illegal,character}},1} = erl_scan_string([$$,$\\,$^,1089], 1), {error,{1,erl_scan,Error},1} = @@ -908,7 +922,7 @@ unicode() -> U3 = "\"a\n\\x{fff}\n\"", {ok,[{string,1,[$a,$\n,$\x{fff},$\n]}],3} = erl_scan_string(U3, 1), - U4 = "\"\\^\n\\x{aaa}\\^\n\"", + U4 = "\"\n\\x{aaa}\n\"", {ok,[{string,1,[$\n,$\x{aaa},$\n]}],3} = erl_scan_string(U4, 1), %% Keep these tests: @@ -1023,7 +1037,7 @@ otp_10302(Config) when is_list(Config) -> U3 = "\"a\n\\x{fff}\n\"", {ok,[{string,1,[97,10,4095,10]}],3} = erl_scan_string(U3, 1), - U4 = "\"\\^\n\\x{aaa}\\^\n\"", + U4 = "\"\n\\x{aaa}\n\"", {ok,[{string,1,[10,2730,10]}],3} = erl_scan_string(U4, 1,[]), Str1 = "\"ab" ++ [1089] ++ "cd\"", diff --git a/lib/stdlib/test/error_logger_h_SUITE.erl b/lib/stdlib/test/error_logger_h_SUITE.erl index d1f375459b..3dcbc2664a 100644 --- a/lib/stdlib/test/error_logger_h_SUITE.erl +++ b/lib/stdlib/test/error_logger_h_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2015-2021. All Rights Reserved. +%% Copyright Ericsson AB 2015-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -66,17 +66,17 @@ logfile(Config) -> analyse_events(Log, Ev, [AtNode], unlimited), %% Make sure that the file_io_server process has been stopped - [] = lists:filtermap( - fun(X) -> - case {process_info(X, [current_function]), - file:pid2name(X)} of - {[{current_function, {file_io_server, _, _}}], - {ok,P2N = Log}} -> - {true, {X, P2N}}; - _ -> - false - end - end, processes()), + [] = lists:filtermap(fun(X) -> + case process_info(X, [current_function]) of + [{current_function, {file_io_server, _, _}}] -> + case file:pid2name(X) of + {ok, Log} -> {true, {X, Log}}; + _ -> false + end; + _ -> + false + end + end, processes()), peer:stop(Peer), diff --git a/lib/stdlib/test/escript_SUITE_data/arg_overflow b/lib/stdlib/test/escript_SUITE_data/arg_overflow index dd5accc051..e3138cabbd 100755 --- a/lib/stdlib/test/escript_SUITE_data/arg_overflow +++ b/lib/stdlib/test/escript_SUITE_data/arg_overflow @@ -1,5 +1,5 @@ #! /usr/bin/env escript %% -*- erlang -*- -%%!x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x x +%%!-x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x -x main(_) -> halt(0). diff --git a/lib/stdlib/test/escript_SUITE_data/linebuf_overflow b/lib/stdlib/test/escript_SUITE_data/linebuf_overflow index 33133c1ce9..018be1f26d 100755 --- a/lib/stdlib/test/escript_SUITE_data/linebuf_overflow +++ b/lib/stdlib/test/escript_SUITE_data/linebuf_overflow @@ -1,5 +1,5 @@ #! /usr/bin/env escript %% -*- erlang -*- -%%!xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +%%!-v xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx main(_) -> halt(0). diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl index ffa088543d..52cf5b9e69 100644 --- a/lib/stdlib/test/ets_SUITE.erl +++ b/lib/stdlib/test/ets_SUITE.erl @@ -40,7 +40,7 @@ -export([tab2file/1, tab2file2/1, tabfile_ext1/1, tabfile_ext2/1, tabfile_ext3/1, tabfile_ext4/1, badfile/1]). -export([heavy_lookup/1, heavy_lookup_element/1, heavy_concurrent/1]). --export([lookup_element_mult/1]). +-export([lookup_element_mult/1, lookup_element_default/1]). -export([foldl_ordered/1, foldr_ordered/1, foldl/1, foldr/1, fold_empty/1]). -export([t_delete_object/1, t_init_table/1, t_whitebox/1, select_bound_chunk/1, t_delete_all_objects/1, t_test_ms/1, @@ -195,7 +195,7 @@ groups() -> privacy]}, {insert, [], [empty, badinsert]}, {lookup, [], [badlookup, lookup_order]}, - {lookup_element, [], [lookup_element_mult]}, + {lookup_element, [], [lookup_element_mult, lookup_element_default]}, {delete, [], [delete_elem, delete_tab, delete_large_tab, delete_large_named_table, evil_delete, table_leak, @@ -2407,13 +2407,19 @@ update_element_do(Tab,Tuple,Key,UpdPos) -> Big32 = 16#12345678, Big64 = 16#123456789abcdef0, - Values = { 623, -27, 0, Big32, -Big32, Big64, -Big64, Big32*Big32, + RefcBin = list_to_binary(lists:seq(1,100)), + BigMap1 = maps:from_list([{N,N} || N <- lists:seq(1,33)]), + BigMap2 = BigMap1#{key => RefcBin, RefcBin => value}, + Values = { 623, -27, Big32, -Big32, Big64, -Big64, Big32*Big32, -Big32*Big32, Big32*Big64, -Big32*Big64, Big64*Big64, -Big64*Big64, "A", "Sverker", [], {12,-132}, {}, - <<45,232,0,12,133>>, <<234,12,23>>, list_to_binary(lists:seq(1,100)), + <<45,232,0,12,133>>, <<234,12,23>>, RefcBin, (fun(X) -> X*Big32 end), - make_ref(), make_ref(), self(), ok, update_element, 28, 29 }, - Length = size(Values), + make_ref(), make_ref(), self(), ok, update_element, + #{a => value, "hello" => "world", 1.0 => RefcBin }, + BigMap1, BigMap2}, + Length = tuple_size(Values), + 29 = Length, PosValArgF = fun(ToIx, ResList, [Pos | PosTail], Rand, MeF) -> NextIx = (ToIx+Rand) rem Length, @@ -2433,7 +2439,12 @@ update_element_do(Tab,Tuple,Key,UpdPos) -> true = ets:update_element(Tab, Key, PosValArg), ArgHash = erlang:phash2({Tab,Key,PosValArg}), NewTuple = update_tuple(PosValArg,Tuple), - [NewTuple] = ets:lookup(Tab,Key) + [NewTuple] = ets:lookup(Tab,Key), + [begin + Elem = element(I, NewTuple), + Elem = ets:lookup_element(Tab, Key, I) + end + || I <- lists:seq(1, tuple_size(NewTuple))] end, LoopF = fun(_FromIx, Incr, _Times, Checksum, _MeF) when Incr >= Length -> @@ -4142,6 +4153,41 @@ fill_tab(Tab,Val) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +lookup_element_default(Config) when is_list(Config) -> + EtsMem = etsmem(), + + TabSet = ets_new(foo, [set]), + ets:insert(TabSet, {key, 42}), + 42 = ets:lookup_element(TabSet, key, 2, 13), + 13 = ets:lookup_element(TabSet, not_key, 2, 13), + {'EXIT',{badarg,_}} = catch ets:lookup_element(TabSet, key, 3, 13), + true = ets:delete(TabSet), + + TabOrderedSet = ets_new(foo, [ordered_set]), + ets:insert(TabOrderedSet, {key, 42}), + 42 = ets:lookup_element(TabOrderedSet, key, 2, 13), + 13 = ets:lookup_element(TabOrderedSet, not_key, 2, 13), + {'EXIT',{badarg,_}} = catch ets:lookup_element(TabOrderedSet, key, 3, 13), + true = ets:delete(TabOrderedSet), + + TabBag = ets_new(foo, [bag]), + ets:insert(TabBag, {key, 42}), + ets:insert(TabBag, {key, 43, 44}), + [42, 43] = ets:lookup_element(TabBag, key, 2, 13), + 13 = ets:lookup_element(TabBag, not_key, 2, 13), + {'EXIT',{badarg,_}} = catch ets:lookup_element(TabBag, key, 3, 13), + true = ets:delete(TabBag), + + TabDuplicateBag = ets_new(foo, [duplicate_bag]), + ets:insert(TabDuplicateBag, {key, 42}), + ets:insert(TabDuplicateBag, {key, 42}), + ets:insert(TabDuplicateBag, {key, 43, 44}), + [42, 42, 43] = ets:lookup_element(TabDuplicateBag, key, 2, 13), + 13 = ets:lookup_element(TabDuplicateBag, not_key, 2, 13), + {'EXIT',{badarg,_}} = catch ets:lookup_element(TabDuplicateBag, key, 3, 13), + true = ets:delete(TabDuplicateBag), + + verify_etsmem(EtsMem). %% OTP-2386. Multiple return elements. lookup_element_mult(Config) when is_list(Config) -> @@ -9187,6 +9233,9 @@ error_info(_Config) -> {lookup_element, [OneKeyTab, one, 4]}, + {lookup_element, ['$Tab', no_key, 1, default_value], [no_fail]}, + {lookup_element, [OneKeyTab, one, 4, default_value]}, + {match, [bad_continuation], [no_table]}, {match, ['$Tab', <<1,2,3>>], [no_fail]}, diff --git a/lib/stdlib/test/gen_server_SUITE.erl b/lib/stdlib/test/gen_server_SUITE.erl index 5fa604d4fd..52f6a2ceb9 100644 --- a/lib/stdlib/test/gen_server_SUITE.erl +++ b/lib/stdlib/test/gen_server_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2021. All Rights Reserved. +%% Copyright Ericsson AB 1996-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -26,12 +26,16 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2]). --export([start/1, crash/1, call/1, send_request/1, +-export([start/1, crash/1, loop_start_fail/1, call/1, send_request/1, send_request_receive_reqid_collection/1, send_request_wait_reqid_collection/1, send_request_check_reqid_collection/1, - cast/1, cast_fast/1, - continue/1, info/1, abcast/1, multicall/1, multicall_down/1, + cast/1, cast_fast/1, continue/1, info/1, abcast/1, + multicall/1, multicall_down/1, multicall_remote/1, + multicall_remote_old1/1, multicall_remote_old2/1, + multicall_recv_opt_success/1, + multicall_recv_opt_timeout/1, + multicall_recv_opt_noconnection/1, call_remote1/1, call_remote2/1, call_remote3/1, calling_self/1, call_remote_n1/1, call_remote_n2/1, call_remote_n3/1, spec_init/1, spec_init_local_registered_parent/1, @@ -59,20 +63,33 @@ spec_init_anonymous_default_timeout/1, spec_init_not_proc_lib/1, cast_fast_messup/0]). +%% Internal test specific exports +-export([multicall_srv_ctrlr/2, multicall_suspender/2]). %% The gen_server behaviour -export([init/1, handle_call/3, handle_cast/2, handle_continue/2, handle_info/2, code_change/3, terminate/2, format_status/2]). +%% This module needs to compile on old nodes... +-ifndef(CT_PEER). +-define(CT_PEER(), {ok, undefined, undefined}). +-define(CT_PEER(Opts), {ok, undefined, undefined}). +-endif. +-ifndef(CT_PEER_REL). +-define(CT_PEER_REL(Opts, Release, PrivDir), {ok, undefined, undefined}). +-endif. + suite() -> [{ct_hooks,[ts_install_cth]}, {timetrap,{minutes,1}}]. all() -> - [start, {group,stop}, crash, call, send_request, + [start, {group,stop}, crash, loop_start_fail, call, send_request, send_request_receive_reqid_collection, send_request_wait_reqid_collection, send_request_check_reqid_collection, cast, cast_fast, info, abcast, - continue, multicall, multicall_down, call_remote1, call_remote2, calling_self, + continue, + {group, multi_call}, + call_remote1, call_remote2, calling_self, call_remote3, call_remote_n1, call_remote_n2, call_remote_n3, spec_init, spec_init_local_registered_parent, @@ -87,6 +104,13 @@ all() -> groups() -> [{stop, [], [stop1, stop2, stop3, stop4, stop5, stop6, stop7, stop8, stop9, stop10]}, + {multi_call, [], [{group, multi_call_parallel}, {group, multi_call_sequence}]}, + {multi_call_parallel, [parallel], + [multicall, multicall_down, multicall_remote, multicall_remote_old1, + multicall_remote_old2]}, + {multi_call_sequence, [], + [multicall_recv_opt_success, multicall_recv_opt_timeout, + multicall_recv_opt_noconnection]}, {format_status, [], [call_format_status, error_format_status, terminate_crash_format, crash_in_format_status, throw_in_format_status, format_all_status]}, @@ -135,7 +159,10 @@ end_per_testcase(_Case, Config) -> undefined -> ok; Peer -> - peer:stop(Peer) + try peer:stop(Peer) + catch exit : noproc -> + ok + end end, ok. @@ -148,6 +175,7 @@ start(Config) when is_list(Config) -> OldFl = process_flag(trap_exit, true), %% anonymous + io:format("anonymous~n", []), {ok, Pid0} = gen_server:start(gen_server_SUITE, [], []), ok = gen_server:call(Pid0, started_p), ok = gen_server:call(Pid0, stop), @@ -155,6 +183,7 @@ start(Config) when is_list(Config) -> {'EXIT', {noproc,_}} = (catch gen_server:call(Pid0, started_p, 1)), %% anonymous with timeout + io:format("try init timeout~n", []), {ok, Pid00} = gen_server:start(gen_server_SUITE, [], [{timeout,1000}]), ok = gen_server:call(Pid00, started_p), @@ -163,9 +192,16 @@ start(Config) when is_list(Config) -> [{timeout,100}]), %% anonymous with ignore + io:format("try init ignore~n", []), ignore = gen_server:start(gen_server_SUITE, ignore, []), + %% anonymous with shutdown + io:format("try init shutdown~n", []), + {error, foobar} = + gen_server:start(gen_server_SUITE, {error, foobar}, []), + %% anonymous with stop + io:format("try init stop~n", []), {error, stopped} = gen_server:start(gen_server_SUITE, stop, []), %% anonymous linked @@ -485,6 +521,55 @@ crash(Config) when is_list(Config) -> ok. + +loop_start_fail(Config) -> + _ = process_flag(trap_exit, true), + loop_start_fail( + Config, + [{start, []}, {start, [link]}, + {start_link, []}, + {start_monitor, [link]}, {start_monitor, []}]). + +loop_start_fail(_Config, []) -> + ok; +loop_start_fail(Config, [{Start, Opts} | Start_Opts]) -> + loop_start_fail( + fun gen_server:Start/3, + {ets, {return, {stop, failed_to_start}}}, Opts, + fun ({error, failed_to_start}) -> ok end), + loop_start_fail( + fun gen_server:Start/3, + {ets, {return, ignore}}, Opts, + fun (ignore) -> ok end), + loop_start_fail( + fun gen_server:Start/3, + {ets, {return, 4711}}, Opts, + fun ({error, {bad_return_value, 4711}}) -> ok end), + loop_start_fail( + fun gen_server:Start/3, + {ets, {crash, error, bailout}}, Opts, + fun ({error, {bailout, ST}}) when is_list(ST) -> ok end), + loop_start_fail( + fun gen_server:Start/3, + {ets, {crash, exit, bailout}}, Opts, + fun ({error, bailout}) -> ok end), + loop_start_fail( + fun gen_server:Start/3, + {ets, {wait, 1000, void}}, [{timeout, 200} | Opts], + fun ({error, timeout}) -> ok end), + loop_start_fail(Config, Start_Opts). + +loop_start_fail(GenStartFun, Arg, Opts, ValidateFun) -> + loop_start_fail(GenStartFun, Arg, Opts, ValidateFun, 5). +%% +loop_start_fail(_GenStartFun, _Arg, _Opts, _ValidateFun, 0) -> + ok; +loop_start_fail(GenStartFun, Arg, Opts, ValidateFun, N) -> + ok = ValidateFun(GenStartFun(?MODULE, Arg, Opts)), + loop_start_fail(GenStartFun, Arg, Opts, ValidateFun, N - 1). + + + %% -------------------------------------- %% Test gen_server:call and handle_call. %% Test all different return values from @@ -1434,6 +1519,251 @@ busy_wait_for_process(Pid,N) -> _ -> ok end. + +multicall_remote(Config) when is_list(Config) -> + PNs = lists:map(fun (_) -> + {ok, P, N} = ?CT_PEER(), + {P, N} + end, lists:seq(1, 4)), + multicall_remote_test(PNs, ?FUNCTION_NAME), + ok. + +multicall_remote_old1(Config) when is_list(Config) -> + multicall_remote_old_test(Config, 1, ?FUNCTION_NAME). + +multicall_remote_old2(Config) when is_list(Config) -> + multicall_remote_old_test(Config, 2, ?FUNCTION_NAME). + + +multicall_remote_old_test(Config, OldN, Name) -> + try + {OldRelName, OldRel} = old_release(OldN), + PD = proplists:get_value(priv_dir, Config), + PNs = lists:map(fun (I) -> + Dir = atom_to_list(Name)++"-"++integer_to_list(I), + AbsDir = filename:join([PD, Dir]), + ok = file:make_dir(AbsDir), + case ?CT_PEER_REL(#{connection => 0}, OldRelName, AbsDir) of + not_available -> + throw({skipped, "No OTP "++OldRel++" available"}); + {ok, P, N} -> + {P, N} + end + end, lists:seq(1, 4)), + OldNodes = lists:map(fun ({_, N}) -> N end, PNs), + %% Recompile on one old node and load this on all old nodes... + SrcFile = filename:rootname(code:which(?MODULE)) ++ ".erl", + {ok, ?MODULE, BeamCode} = erpc:call(hd(OldNodes), compile, file, [SrcFile, [binary]]), + LoadResult = lists:duplicate(length(OldNodes), {ok, {module, ?MODULE}}), + LoadResult = erpc:multicall(OldNodes, code, load_binary, [?MODULE, SrcFile, BeamCode]), + multicall_remote_test(PNs, Name) + catch + throw:Res -> + Res + end. + +multicall_remote_test([{Peer1, Node1}, + {Peer2, Node2}, + {Peer3, Node3}, + {Peer4, Node4}], + Name) -> + Tester = self(), + ThisNode = node(), + + Nodes = [Node1, Node2, Node3, Node4, ThisNode], + + SrvList = + lists:map(fun (Node) -> + Ctrl = spawn_link(Node, ?MODULE, + multicall_srv_ctrlr, + [Tester, Name]), + receive + {Ctrl, _Srv} = Procs -> + {Node, Procs} + end + end, Nodes), + SrvMap = maps:from_list(SrvList), + + Res0 = {lists:map(fun (Node) -> + {Node,ok} + end, Nodes), []}, + + Res0 = gen_server:multi_call(Nodes, Name, started_p), + + true = try + _ = gen_server:multi_call([Node1, Node2, Node3, node(), {Node4}], + Name, {delayed_answer,1}), + false + catch + _:_ -> + true + end, + + Res1 = {lists:map(fun (Node) -> + {Node,delayed} + end, Nodes), []}, + + Res1 = gen_server:multi_call(Nodes, Name, {delayed_answer,1}), + + Res2 = {[], Nodes}, + + Start = erlang:monotonic_time(millisecond), + Res2 = gen_server:multi_call(Nodes, Name, {delayed_answer,1000}, 100), + End = erlang:monotonic_time(millisecond), + Time = End-Start, + ct:log("Time: ~p ms~n", [Time]), + true = 200 >= Time, + + {Ctrl2, Srv2} = maps:get(Node2, SrvMap), + unlink(Ctrl2), + exit(Ctrl2, kill), + wait_until(fun () -> + false == erpc:call(Node2, erlang, + is_process_alive, [Srv2]) + end), + + {Ctrl3, _Srv3} = maps:get(Node3, SrvMap), + unlink(Ctrl3), + peer:stop(Peer3), + + {Ctrl4, Srv4} = maps:get(Node4, SrvMap), + Spndr = spawn_link(Node4, ?MODULE, multicall_suspender, [Tester, Srv4]), + + Res3 = {[{Node1, delayed}, {ThisNode, delayed}], + [Node2, Node3, Node4]}, + + Res3 = gen_server:multi_call(Nodes, Name, {delayed_answer,1}, 1000), + + Spndr ! {Tester, resume_it}, + + receive Msg -> ct:fail({unexpected_msg, Msg}) + after 1000 -> ok + end, + + unlink(Ctrl4), + + {Ctrl1, _Srv1} = maps:get(Node1, SrvMap), + + unlink(Ctrl1), + + peer:stop(Peer1), + peer:stop(Peer2), + peer:stop(Peer4), + + ok. + +multicall_srv_ctrlr(Tester, Name) -> + {ok, Srv} = gen_server:start_link({local, Name}, + gen_server_SUITE, [], []), + Tester ! {self(), Srv}, + receive after infinity -> ok end. + +multicall_suspender(Tester, Suspendee) -> + true = erlang:suspend_process(Suspendee), + receive + {Tester, resume_it} -> + erlang:resume_process(Suspendee) + end. + +multicall_recv_opt_success(Config) when is_list(Config) -> + multicall_recv_opt_test(success). + +multicall_recv_opt_timeout(Config) when is_list(Config) -> + multicall_recv_opt_test(timeout). + +multicall_recv_opt_noconnection(Config) when is_list(Config) -> + multicall_recv_opt_test(noconnection). + +multicall_recv_opt_test(Type) -> + Tester = self(), + Name = ?FUNCTION_NAME, + Loops = 1000, + HugeMsgQ = 500000, + process_flag(message_queue_data, off_heap), + + {ok, Peer1, Node1} = ?CT_PEER(), + {ok, Peer2, Node2} = ?CT_PEER(), + + if Type == noconnection -> peer:stop(Peer2); + true -> ok + end, + + Nodes = [Node1, Node2], + + SrvList = + lists:map(fun (Node) -> + Ctrl = spawn_link(Node, ?MODULE, + multicall_srv_ctrlr, + [Tester, Name]), + receive + {Ctrl, _Srv} = Procs -> + {Node, Procs} + end + end, + if Type == noconnection -> [Node1]; + true -> Nodes + end), + + {Req, ExpRes, Tmo} = case Type of + success -> + {ping, + {[{Node1, pong}, {Node2, pong}], []}, + infinity}; + timeout -> + {{delayed_answer,100}, + {[], Nodes}, + 1}; + noconnection -> + {ping, + {[{Node1, pong}], [Node2]}, + infinity} + end, + + _Warmup = time_multicall(ExpRes, Nodes, Name, Req, Tmo, Loops div 10), + + Empty = time_multicall(ExpRes, Nodes, Name, Req, Tmo, Loops), + ct:pal("Time with empty message queue: ~p microsecond~n", + [erlang:convert_time_unit(Empty, native, microsecond)]), + + make_msgq(HugeMsgQ), + + Huge = time_multicall(ExpRes, Nodes, Name, Req, Tmo, Loops), + ct:pal("Time with huge message queue: ~p microsecond~n", + [erlang:convert_time_unit(Huge, native, microsecond)]), + + lists:foreach(fun ({_Node, {Ctrl, _Srv}}) -> unlink(Ctrl) end, SrvList), + + peer:stop(Peer1), + if Type == noconnection -> ok; + true -> peer:stop(Peer2) + end, + + Q = Huge / Empty, + HugeMsgQ = flush_msgq(), + case Q > 10 of + true -> + ct:fail({ratio, Q}); + false -> + {comment, "Ratio: "++erlang:float_to_list(Q)} + end. + +time_multicall(Expect, Nodes, Name, Req, Tmo, Times) -> + Start = erlang:monotonic_time(), + ok = do_time_multicall(Expect, Nodes, Name, Req, Tmo, Times), + erlang:monotonic_time() - Start. + +do_time_multicall(_Expect, _Nodes, _Name, _Req, _Tmo, 0) -> + ok; +do_time_multicall(Expect, Nodes, Name, Req, Tmo, N) -> + Expect = gen_server:multi_call(Nodes, Name, Req, Tmo), + do_time_multicall(Expect, Nodes, Name, Req, Tmo, N-1). + +make_msgq(0) -> + ok; +make_msgq(N) -> + self() ! {a, msg}, + make_msgq(N-1). + %%-------------------------------------------------------------- %% Test gen_server:enter_loop/[3,4,5]. Used when you want to write %% your own special init-phase. @@ -2049,11 +2379,10 @@ undef_init(_Config) -> {error, {undef, [{oc_init_server, init, [_], _}|_]}} = (catch gen_server:start_link(oc_init_server, [], [])), receive - {'EXIT', Server, - {undef, [{oc_init_server, init, [_], _}|_]}} when is_pid(Server) -> + Msg -> + ct:fail({unexpected_msg, Msg}) + after 500 -> ok - after 1000 -> - ct:fail(expected_exit_msg) end. %% The upgrade should fail if code_change is expected in the callback module @@ -2483,23 +2812,44 @@ spec_init_not_proc_lib(Options) -> init([]) -> {ok, []}; init(ignore) -> + io:format("init(ignore)~n"), ignore; +init({error, Reason}) -> + io:format("init(error) -> ~w~n", [Reason]), + {error, Reason}; init(stop) -> + io:format("init(stop)~n"), {stop, stopped}; init(hibernate) -> + io:format("init(hibernate)~n"), {ok,[],hibernate}; init(sleep) -> + io:format("init(sleep)~n"), ct:sleep(1000), {ok, []}; init({continue, Pid}) -> + io:format("init(continue) -> ~p~n", [Pid]), self() ! {after_continue, Pid}, {ok, [], {continue, {message, Pid}}}; init({state,State}) -> - {ok,State}. + io:format("init(state) -> ~p~n", [State]), + {ok,State}; +init({ets,InitResult}) -> + ?MODULE = ets:new(?MODULE, [named_table]), + case InitResult of + {return, Value} -> + Value; + {crash, Class, Reason} -> + erlang:Class(Reason); + {wait, Time, Value} -> + receive after Time -> Value end + end. handle_call(started_p, _From, State) -> io:format("FROZ"), {reply,ok,State}; +handle_call(ping, _From, State) -> + {reply,pong,State}; handle_call({delayed_answer, T}, From, State) -> {noreply,{reply_to,From,State},T}; handle_call({call_within, T}, _From, _) -> @@ -2507,6 +2857,7 @@ handle_call({call_within, T}, _From, _) -> handle_call(next_call, _From, call_within) -> {reply,ok,[]}; handle_call(next_call, _From, State) -> + io:format("handle_call(next_call) -> State: ~p~n", [State]), {reply,false,State}; handle_call(badreturn, _From, _State) -> badreturn; @@ -2645,3 +2996,28 @@ format_status(terminate, [_PDict, State]) -> {formatted, State}; format_status(normal, [_PDict, _State]) -> format_status_called. + +%% Utils... + +wait_until(Fun) -> + case catch Fun() of + true -> + ok; + _ -> + receive after 100 -> ok end, + wait_until(Fun) + end. + +old_release(N) -> + OldRel = integer_to_list(list_to_integer(erlang:system_info(otp_release))-N), + {OldRel++"_latest", OldRel}. + +flush_msgq() -> + flush_msgq(0). +flush_msgq(N) -> + receive + _ -> + flush_msgq(N+1) + after 0 -> + N + end. diff --git a/lib/stdlib/test/gen_statem_SUITE.erl b/lib/stdlib/test/gen_statem_SUITE.erl index 9b4ee9413f..d99c7e9786 100644 --- a/lib/stdlib/test/gen_statem_SUITE.erl +++ b/lib/stdlib/test/gen_statem_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2016-2022. All Rights Reserved. +%% Copyright Ericsson AB 2016-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -37,7 +37,7 @@ all() -> {group, stop_handle_event}, {group, abnormal}, {group, abnormal_handle_event}, - shutdown, stop_and_reply, state_enter, event_order, + shutdown, loop_start_fail, stop_and_reply, state_enter, event_order, state_timeout, timeout_cancel_and_update, event_types, generic_timers, code_change, {group, sys}, @@ -61,7 +61,7 @@ groups() -> {format_log, [], tcs(format_log)}]. tcs(start) -> - [start1, start2, start3, start4, start5, start6, start7, + [start1, start2, start3, start4, start5a, start5b, start6, start7, start8, start9, start10, start11, start12, next_events]; tcs(stop) -> [stop1, stop2, stop3, stop4, stop5, stop6, stop7, stop8, stop9, stop10]; @@ -210,7 +210,7 @@ start4(Config) -> ok = verify_empty_msgq(). %% anonymous with stop -start5(Config) -> +start5a(Config) -> OldFl = process_flag(trap_exit, true), {error,stopped} = gen_statem:start(?MODULE, start_arg(Config, stop), []), @@ -218,6 +218,16 @@ start5(Config) -> process_flag(trap_exit, OldFl), ok = verify_empty_msgq(). +%% anonymous with shutdown +start5b(Config) -> + OldFl = process_flag(trap_exit, true), + + {error, foobar} = + gen_statem:start(?MODULE, start_arg(Config, {error, foobar}), []), + + process_flag(trap_exit, OldFl), + ok = verify_empty_msgq(). + %% anonymous linked start6(Config) -> {ok,Pid} = gen_statem:start_link(?MODULE, start_arg(Config, []), []), @@ -676,6 +686,53 @@ shutdown(Config) -> end. +loop_start_fail(Config) -> + _ = process_flag(trap_exit, true), + loop_start_fail( + Config, + [{start, []}, {start, [link]}, + {start_link, []}, + {start_monitor, [link]}, {start_monitor, []}]). + +loop_start_fail(_Config, []) -> + ok; +loop_start_fail(Config, [{Start, Opts} | Start_Opts]) -> + loop_start_fail( + fun gen_statem:Start/3, + {ets, {return, {stop, failed_to_start}}}, Opts, + fun ({error, failed_to_start}) -> ok end), + loop_start_fail( + fun gen_statem:Start/3, + {ets, {return, ignore}}, Opts, + fun (ignore) -> ok end), + loop_start_fail( + fun gen_statem:Start/3, + {ets, {return, 4711}}, Opts, + fun ({error, {bad_return_from_init, 4711}}) -> ok end), + loop_start_fail( + fun gen_statem:Start/3, + {ets, {crash, error, bailout}}, Opts, + fun ({error, bailout}) -> ok end), + loop_start_fail( + fun gen_statem:Start/3, + {ets, {crash, exit, bailout}}, Opts, + fun ({error, bailout}) -> ok end), + loop_start_fail( + fun gen_statem:Start/3, + {ets, {wait, 1000, void}}, [{timeout, 200} | Opts], + fun ({error, timeout}) -> ok end), + loop_start_fail(Config, Start_Opts). + +loop_start_fail(GenStartFun, Arg, Opts, ValidateFun) -> + loop_start_fail(GenStartFun, Arg, Opts, ValidateFun, 5). +%% +loop_start_fail(_GenStartFun, _Arg, _Opts, _ValidateFun, 0) -> + ok; +loop_start_fail(GenStartFun, Arg, Opts, ValidateFun, N) -> + ok = ValidateFun(GenStartFun(?MODULE, Arg, Opts)), + loop_start_fail(GenStartFun, Arg, Opts, ValidateFun, N - 1). + + stop_and_reply(_Config) -> process_flag(trap_exit, true), @@ -1352,7 +1409,7 @@ terminate_crash_format(Config) -> terminate_crash_format(Config,format_status_statem, {{formatted,idle},{formatted,crash_terminate}}) after - dbg:stop_clear(), + dbg:stop(), process_flag(trap_exit, OldFl), error_logger_forwarder:unregister() end. @@ -1512,11 +1569,12 @@ replace_state(Config) -> %% Hibernation hibernate(Config) -> OldFl = process_flag(trap_exit, true), + WaitHibernate = 500, {ok,Pid0} = gen_statem:start_link( ?MODULE, start_arg(Config, hiber_now), []), - wait_erlang_hibernate(Pid0), + wait_erlang_hibernate(Pid0, WaitHibernate), stop_it(Pid0), receive {'EXIT',Pid0,normal} -> ok @@ -1529,38 +1587,38 @@ hibernate(Config) -> true = ({current_function,{erlang,hibernate,3}} =/= erlang:process_info(Pid,current_function)), hibernating = gen_statem:call(Pid, hibernate_sync), - wait_erlang_hibernate(Pid), + wait_erlang_hibernate(Pid, WaitHibernate), good_morning = gen_statem:call(Pid, wakeup_sync), - is_not_in_erlang_hibernate(Pid), + is_not_in_erlang_hibernate(Pid, WaitHibernate), hibernating = gen_statem:call(Pid, hibernate_sync), - wait_erlang_hibernate(Pid), + wait_erlang_hibernate(Pid, WaitHibernate), please_just_five_more = gen_statem:call(Pid, snooze_sync), - wait_erlang_hibernate(Pid), + wait_erlang_hibernate(Pid, WaitHibernate), good_morning = gen_statem:call(Pid, wakeup_sync), - is_not_in_erlang_hibernate(Pid), + is_not_in_erlang_hibernate(Pid, WaitHibernate), ok = gen_statem:cast(Pid, hibernate_async), - wait_erlang_hibernate(Pid), + wait_erlang_hibernate(Pid, WaitHibernate), ok = gen_statem:cast(Pid, wakeup_async), - is_not_in_erlang_hibernate(Pid), + is_not_in_erlang_hibernate(Pid, WaitHibernate), ok = gen_statem:cast(Pid, hibernate_async), - wait_erlang_hibernate(Pid), + wait_erlang_hibernate(Pid, WaitHibernate), ok = gen_statem:cast(Pid, snooze_async), - wait_erlang_hibernate(Pid), + wait_erlang_hibernate(Pid, WaitHibernate), ok = gen_statem:cast(Pid, wakeup_async), - is_not_in_erlang_hibernate(Pid), + is_not_in_erlang_hibernate(Pid, WaitHibernate), - Pid ! hibernate_later, + Pid ! {hibernate_later, WaitHibernate div 2}, true = ({current_function,{erlang,hibernate,3}} =/= erlang:process_info(Pid, current_function)), - wait_erlang_hibernate(Pid), + wait_erlang_hibernate(Pid, WaitHibernate), 'alive!' = gen_statem:call(Pid, 'alive?'), true = ({current_function,{erlang,hibernate,3}} =/= erlang:process_info(Pid, current_function)), Pid ! hibernate_now, - wait_erlang_hibernate(Pid), + wait_erlang_hibernate(Pid, WaitHibernate), 'alive!' = gen_statem:call(Pid, 'alive?'), true = @@ -1568,37 +1626,37 @@ hibernate(Config) -> erlang:process_info(Pid, current_function)), hibernating = gen_statem:call(Pid, hibernate_sync), - wait_erlang_hibernate(Pid), + wait_erlang_hibernate(Pid, WaitHibernate), good_morning = gen_statem:call(Pid, wakeup_sync), - is_not_in_erlang_hibernate(Pid), + is_not_in_erlang_hibernate(Pid, WaitHibernate), hibernating = gen_statem:call(Pid, hibernate_sync), - wait_erlang_hibernate(Pid), + wait_erlang_hibernate(Pid, WaitHibernate), please_just_five_more = gen_statem:call(Pid, snooze_sync), - wait_erlang_hibernate(Pid), + wait_erlang_hibernate(Pid, WaitHibernate), good_morning = gen_statem:call(Pid, wakeup_sync), - is_not_in_erlang_hibernate(Pid), + is_not_in_erlang_hibernate(Pid, WaitHibernate), ok = gen_statem:cast(Pid, hibernate_async), - wait_erlang_hibernate(Pid), + wait_erlang_hibernate(Pid, WaitHibernate), ok = gen_statem:cast(Pid, wakeup_async), - is_not_in_erlang_hibernate(Pid), + is_not_in_erlang_hibernate(Pid, WaitHibernate), ok = gen_statem:cast(Pid, hibernate_async), - wait_erlang_hibernate(Pid), + wait_erlang_hibernate(Pid, WaitHibernate), ok = gen_statem:cast(Pid, snooze_async), - wait_erlang_hibernate(Pid), + wait_erlang_hibernate(Pid, WaitHibernate), ok = gen_statem:cast(Pid, wakeup_async), - is_not_in_erlang_hibernate(Pid), + is_not_in_erlang_hibernate(Pid, WaitHibernate), hibernating = gen_statem:call(Pid, hibernate_sync), - wait_erlang_hibernate(Pid), + wait_erlang_hibernate(Pid, WaitHibernate), sys:suspend(Pid), - wait_erlang_hibernate(Pid), + wait_erlang_hibernate(Pid, WaitHibernate), sys:resume(Pid), - wait_erlang_hibernate(Pid), - receive after 1000 -> ok end, - wait_erlang_hibernate(Pid), + wait_erlang_hibernate(Pid, WaitHibernate), + receive after WaitHibernate -> ok end, + wait_erlang_hibernate(Pid, WaitHibernate), good_morning = gen_statem:call(Pid, wakeup_sync), - is_not_in_erlang_hibernate(Pid), + is_not_in_erlang_hibernate(Pid, WaitHibernate), stop_it(Pid), process_flag(trap_exit, OldFl), receive @@ -1611,73 +1669,74 @@ hibernate(Config) -> %% Auto-hibernation timeout auto_hibernate(Config) -> OldFl = process_flag(trap_exit, true), - HibernateAfterTimeout = 1000, + HibernateAfterTimeout = 500, + WaitTime = 1000, {ok,Pid} = gen_statem:start_link( ?MODULE, start_arg(Config, []), [{hibernate_after, HibernateAfterTimeout}]), %% After init test - is_not_in_erlang_hibernate(Pid), + is_not_in_erlang_hibernate(Pid, 2 * HibernateAfterTimeout), timer:sleep(HibernateAfterTimeout), - wait_erlang_hibernate(Pid), + wait_erlang_hibernate(Pid, 2 * HibernateAfterTimeout), %% After info test Pid ! {hping, self()}, receive {Pid, hpong} -> ok - after 1000 -> + after WaitTime -> ct:fail(info) end, - is_not_in_erlang_hibernate(Pid), + is_not_in_erlang_hibernate(Pid, 2 * HibernateAfterTimeout), timer:sleep(HibernateAfterTimeout), - wait_erlang_hibernate(Pid), + wait_erlang_hibernate(Pid, 2 * HibernateAfterTimeout), %% After cast test ok = gen_statem:cast(Pid, {hping, self()}), receive {Pid, hpong} -> ok - after 1000 -> + after WaitTime -> ct:fail(cast) end, - is_not_in_erlang_hibernate(Pid), + is_not_in_erlang_hibernate(Pid, 2 * HibernateAfterTimeout), timer:sleep(HibernateAfterTimeout), - wait_erlang_hibernate(Pid), + wait_erlang_hibernate(Pid, 2 * HibernateAfterTimeout), %% After call test hpong = gen_statem:call(Pid, hping), - is_not_in_erlang_hibernate(Pid), + is_not_in_erlang_hibernate(Pid, 2 * HibernateAfterTimeout), timer:sleep(HibernateAfterTimeout), - wait_erlang_hibernate(Pid), + wait_erlang_hibernate(Pid, 2 * HibernateAfterTimeout), %% Timer test 1 TimerTimeout1 = HibernateAfterTimeout div 2, ok = gen_statem:call(Pid, {start_htimer, self(), TimerTimeout1}), - is_not_in_erlang_hibernate(Pid), + is_not_in_erlang_hibernate(Pid, 2 * HibernateAfterTimeout), timer:sleep(TimerTimeout1), - is_not_in_erlang_hibernate(Pid), + is_not_in_erlang_hibernate(Pid, 2 * HibernateAfterTimeout), receive {Pid, htimer_timeout} -> ok - after 1000 -> + after WaitTime -> ct:fail(timer1) end, - is_not_in_erlang_hibernate(Pid), + is_not_in_erlang_hibernate(Pid, 2 * HibernateAfterTimeout), timer:sleep(HibernateAfterTimeout), - wait_erlang_hibernate(Pid), + wait_erlang_hibernate(Pid, 2 * HibernateAfterTimeout), %% Timer test 2 TimerTimeout2 = HibernateAfterTimeout * 2, ok = gen_statem:call(Pid, {start_htimer, self(), TimerTimeout2}), - is_not_in_erlang_hibernate(Pid), + is_not_in_erlang_hibernate(Pid, 2 * HibernateAfterTimeout), timer:sleep(HibernateAfterTimeout), - wait_erlang_hibernate(Pid), + wait_erlang_hibernate(Pid, 2 * HibernateAfterTimeout), receive {Pid, htimer_timeout} -> ok after TimerTimeout2 -> ct:fail(timer2) end, - is_not_in_erlang_hibernate(Pid), + is_not_in_erlang_hibernate(Pid, 2 * HibernateAfterTimeout), timer:sleep(HibernateAfterTimeout), - wait_erlang_hibernate(Pid), + wait_erlang_hibernate(Pid, 2 * HibernateAfterTimeout), stop_it(Pid), process_flag(trap_exit, OldFl), receive @@ -1688,35 +1747,35 @@ auto_hibernate(Config) -> ok = verify_empty_msgq(). -wait_erlang_hibernate(Pid) -> +wait_erlang_hibernate(Pid, Time) -> receive after 1 -> ok end, - wait_erlang_hibernate_1(200, Pid). + wait_erlang_hibernate_1(Pid, Time, Time div 100). -wait_erlang_hibernate_1(0, Pid) -> +wait_erlang_hibernate_1(Pid, Time, _T) when Time =< 0 -> ct:log("~p\n", [erlang:process_info(Pid, current_function)]), ct:fail(should_be_in_erlang_hibernate_3); -wait_erlang_hibernate_1(N, Pid) -> +wait_erlang_hibernate_1(Pid, Time, T) -> {current_function,MFA} = erlang:process_info(Pid, current_function), case MFA of {erlang,hibernate,3} -> ok; _ -> - receive after 10 -> ok end, - wait_erlang_hibernate_1(N-1, Pid) + receive after T -> ok end, + wait_erlang_hibernate_1(Pid, Time - T, T) end. -is_not_in_erlang_hibernate(Pid) -> +is_not_in_erlang_hibernate(Pid, Time) -> receive after 1 -> ok end, - is_not_in_erlang_hibernate_1(200, Pid). + is_not_in_erlang_hibernate_1(Pid, Time, Time div 100). -is_not_in_erlang_hibernate_1(0, _Pid) -> +is_not_in_erlang_hibernate_1(_Pid, Time, _T) when Time =< 0 -> ct:fail(should_not_be_in_erlang_hibernate_3); -is_not_in_erlang_hibernate_1(N, Pid) -> +is_not_in_erlang_hibernate_1(Pid, Time, T) -> {current_function,MFA} = erlang:process_info(Pid, current_function), case MFA of {erlang,hibernate,3} -> - receive after 10 -> ok end, - is_not_in_erlang_hibernate_1(N-1, Pid); + receive after T -> ok end, + is_not_in_erlang_hibernate_1(Pid, Time - T, T); _ -> ok end. @@ -2751,25 +2810,37 @@ start_arg(Config, Arg) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% init(ignore) -> + io:format("init(ignore)~n", []), ignore; init(stop) -> + io:format("init(stop)~n", []), {stop,stopped}; +init({error, Reason}) -> + io:format("init(error) -> Reason: ~p~n", [Reason]), + {error, Reason}; init(stop_shutdown) -> + io:format("init(stop_shutdown)~n", []), {stop,shutdown}; init(sleep) -> + io:format("init(sleep)~n", []), ct:sleep(1000), init_sup({ok,idle,data}); init(hiber) -> + io:format("init(hiber)~n", []), init_sup({ok,hiber_idle,[]}); init(hiber_now) -> + io:format("init(hiber_now)~n", []), init_sup({ok,hiber_idle,[],[hibernate]}); init({data, Data}) -> + io:format("init(data)~n", []), init_sup({ok,idle,Data}); init({callback_mode,CallbackMode,Arg}) -> + io:format("init(callback_mode)~n", []), ets:new(?MODULE, [named_table,private]), ets:insert(?MODULE, {callback_mode,CallbackMode}), init(Arg); init({map_statem,#{init := Init}=Machine,Modes}) -> + io:format("init(map_statem)~n", []), ets:new(?MODULE, [named_table,private]), ets:insert(?MODULE, {callback_mode,[handle_event_function|Modes]}), case Init() of @@ -2780,7 +2851,19 @@ init({map_statem,#{init := Init}=Machine,Modes}) -> Other -> init_sup(Other) end; +init({ets, InitResult}) -> + ?MODULE = ets:new(?MODULE, [named_table]), + init_sup( + case InitResult of + {return, Value} -> + Value; + {crash, Class, Reason} -> + erlang:Class(Reason); + {wait, Time, Value} -> + receive after Time -> Value end + end); init([]) -> + io:format("init~n", []), init_sup({ok,idle,data}). %% Supervise state machine parent i.e the test case, and if it dies @@ -2972,8 +3055,8 @@ hiber_idle({call,From}, hibernate_sync, Data) -> {next_state,hiber_wakeup,Data, [{reply,From,hibernating}, hibernate]}; -hiber_idle(info, hibernate_later, _) -> - Tref = erlang:start_timer(1000, self(), hibernate), +hiber_idle(info, {hibernate_later, Time}, _) -> + Tref = erlang:start_timer(Time, self(), hibernate), {keep_state,Tref}; hiber_idle(info, hibernate_now, Data) -> {keep_state,Data, diff --git a/lib/stdlib/test/io_SUITE.erl b/lib/stdlib/test/io_SUITE.erl index 17fd6d41fd..f26d98cc18 100644 --- a/lib/stdlib/test/io_SUITE.erl +++ b/lib/stdlib/test/io_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2022. All Rights Reserved. +%% Copyright Ericsson AB 1999-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -33,7 +33,8 @@ maps/1, coverage/1, otp_14178_unicode_atoms/1, otp_14175/1, otp_14285/1, limit_term/1, otp_14983/1, otp_15103/1, otp_15076/1, otp_15159/1, otp_15639/1, otp_15705/1, otp_15847/1, otp_15875/1, - github_4801/1, chars_limit/1, error_info/1, otp_17525/1]). + github_4801/1, chars_limit/1, error_info/1, otp_17525/1, + unscan_format_without_maps_order/1, build_text_without_maps_order/1]). -export([pretty/2, trf/3]). @@ -67,7 +68,8 @@ all() -> format_string, maps, coverage, otp_14178_unicode_atoms, otp_14175, otp_14285, limit_term, otp_14983, otp_15103, otp_15076, otp_15159, otp_15639, otp_15705, otp_15847, otp_15875, github_4801, chars_limit, - error_info, otp_17525]. + error_info, otp_17525, unscan_format_without_maps_order, + build_text_without_maps_order]. %% Error cases for output. error_1(Config) when is_list(Config) -> @@ -2184,14 +2186,16 @@ otp_10755(Suite) when is_list(Suite) -> " io:format(\"~ltw\", [S]),\n" " io:format(\"~tlw\", [S]),\n" " io:format(\"~ltW\", [S, 1]),\n" - " io:format(\"~tlW\", [S, 1]).\n", + " io:format(\"~tlW\", [S, 1]),\n" + " io:format(\"~ltp\", [S, 1]).\n", {ok,l_mod,[{_File,Ws}]} = compile_file("l_mod.erl", Text, Suite), - ["format string invalid (invalid control ~lw)", - "format string invalid (invalid control ~lW)", - "format string invalid (invalid control ~ltw)", - "format string invalid (invalid control ~ltw)", - "format string invalid (invalid control ~ltW)", - "format string invalid (invalid control ~ltW)"] = + ["format string invalid (invalid modifier/control combination ~lw)", + "format string invalid (invalid modifier/control combination ~lW)", + "format string invalid (invalid modifier/control combination ~lw)", + "format string invalid (invalid modifier/control combination ~lw)", + "format string invalid (invalid modifier/control combination ~lW)", + "format string invalid (invalid modifier/control combination ~lW)", + "format string invalid (conflicting modifiers ~ltp)"] = [lists:flatten(M:format_error(E)) || {_L,M,E} <- Ws], ok. @@ -2271,34 +2275,64 @@ format_string(_Config) -> ok. maps(_Config) -> - %% Note that order in which a map is printed is arbitrary. In - %% practice, small maps (non-HAMT) are printed in key order, but - %% the breakpoint for creating big maps (HAMT) is lower in the - %% debug-compiled run-time system than in the optimized run-time - %% system. - %% + %% Note that order in which a map is printed is arbitrary. %% Therefore, play it completely safe by not assuming any order %% in a map with more than one element. + AOrdCmpFun = fun(A, B) -> A =< B end, + ARevCmpFun = fun(A, B) -> B < A end, + + AtomMap1 = #{a => b}, + AtomMap2 = #{a => b, c => d}, + AtomMap3 = #{a => b, c => d, e => f}, + "#{}" = fmt("~w", [#{}]), - "#{a => b}" = fmt("~w", [#{a=>b}]), - re_fmt(<<"#\\{(a => b),[.][.][.]\\}">>, - "~W", [#{a => b,c => d},2]), - re_fmt(<<"#\\{(a => b),[.][.][.]\\}">>, - "~W", [#{a => b,c => d,e => f},2]), + "#{a => b}" = fmt("~w", [AtomMap1]), + re_fmt(<<"#\\{(a => b|c => d),[.][.][.]\\}">>, + "~W", [AtomMap2, 2]), + re_fmt(<<"#\\{(a => b|c => d|e => f),[.][.][.]\\}">>, + "~W", [AtomMap3, 2]), + "#{a => b,c => d,e => f}" = fmt("~kw", [AtomMap3]), + re_fmt(<<"#\\{(a => b|c => d|e => f),[.][.][.]\\}">>, + "~KW", [undefined, AtomMap3, 2]), + "#{a => b,c => d,e => f}" = fmt("~Kw", [ordered, AtomMap3]), + "#{e => f,c => d,a => b}" = fmt("~Kw", [reversed, AtomMap3]), + "#{a => b,c => d,e => f}" = fmt("~Kw", [AOrdCmpFun, AtomMap3]), + "#{e => f,c => d,a => b}" = fmt("~Kw", [ARevCmpFun, AtomMap3]), "#{}" = fmt("~p", [#{}]), - "#{a => b}" = fmt("~p", [#{a => b}]), - "#{...}" = fmt("~P", [#{a => b},1]), + "#{a => b}" = fmt("~p", [AtomMap1]), + "#{...}" = fmt("~P", [AtomMap1, 1]), re_fmt(<<"#\\{(a => b|c => d),[.][.][.]\\}">>, - "~P", [#{a => b,c => d},2]), + "~P", [AtomMap2, 2]), re_fmt(<<"#\\{(a => b|c => d|e => f),[.][.][.]\\}">>, - "~P", [#{a => b,c => d,e => f},2]), + "~P", [AtomMap3, 2]), + "#{a => b,c => d,e => f}" = fmt("~kp", [AtomMap3]), + re_fmt(<<"#\\{(a => b|c => d|e => f),[.][.][.]\\}">>, + "~KP", [undefined, AtomMap3, 2]), + "#{a => b,c => d,e => f}" = fmt("~Kp", [ordered, AtomMap3]), + "#{e => f,c => d,a => b}" = fmt("~Kp", [reversed, AtomMap3]), + "#{a => b,c => d,e => f}" = fmt("~Kp", [AOrdCmpFun, AtomMap3]), + "#{e => f,c => d,a => b}" = fmt("~Kp", [ARevCmpFun, AtomMap3]), - List = [{I,I*I} || I <- lists:seq(1, 20)], + List = [{I, I * I} || I <- lists:seq(1, 64)], Map = maps:from_list(List), - "#{...}" = fmt("~P", [Map,1]), + "#{...}" = fmt("~P", [Map, 1]), + "#{1 => 1,...}" = fmt("~kP", [Map, 2]), + "#{1 => 1,...}" = fmt("~KP", [ordered, Map, 2]), + "#{64 => 4096,...}" = fmt("~KP", [reversed, Map, 2]), + "#{1 => 1,...}" = fmt("~KP", [AOrdCmpFun, Map, 2]), + "#{64 => 4096,...}" = fmt("~KP", [ARevCmpFun, Map, 2]), + + FloatIntegerMap = #{-1.0 => a, 0.0 => b, -1 => c, 0 => d}, + re_fmt(<<"#\\{(-1.0 => a|0.0 => b|-1 => c|0 => d),[.][.][.]\\}">>, + "~P", [FloatIntegerMap, 2]), + "#{-1 => c,0 => d,-1.0 => a,0.0 => b}" = fmt("~kp", [FloatIntegerMap]), + re_fmt(<<"#\\{(-1.0 => a|0.0 => b|-1 => c|0 => d),[.][.][.]\\}">>, + "~KP", [undefined, FloatIntegerMap, 2]), + "#{-1 => c,0 => d,-1.0 => a,0.0 => b}" = fmt("~Kp", [ordered, FloatIntegerMap]), + "#{0.0 => b,-1.0 => a,0 => d,-1 => c}" = fmt("~Kp", [reversed, FloatIntegerMap]), %% Print a map and parse it back to a map. S = fmt("~p\n", [Map]), @@ -3150,3 +3184,29 @@ otp_17525(_Config) -> " {...}|...]" = lists:flatten(S), ok. + +unscan_format_without_maps_order(_Config) -> + FormatSpec = #{ + adjust => right, + args => [[<<"1">>]], + control_char => 115, + encoding => unicode, + pad_char => 32, + precision => none, + strings => true, + width => none + }, + {"~ts",[[<<"1">>]]} = io_lib:unscan_format([FormatSpec]). + +build_text_without_maps_order(_Config) -> + FormatSpec = #{ + adjust => right, + args => [[<<"1">>]], + control_char => 115, + encoding => unicode, + pad_char => 32, + precision => none, + strings => true, + width => none + }, + [["1"]] = io_lib:build_text([FormatSpec]). diff --git a/lib/stdlib/test/io_proto_SUITE.erl b/lib/stdlib/test/io_proto_SUITE.erl index 525b479fef..bc96992ce2 100644 --- a/lib/stdlib/test/io_proto_SUITE.erl +++ b/lib/stdlib/test/io_proto_SUITE.erl @@ -22,55 +22,27 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2]). --export([init_per_testcase/2, end_per_testcase/2]). - -export([setopts_getopts/1,unicode_options/1,unicode_options_gen/1, binary_options/1, read_modes_gl/1, - read_modes_ogl/1, broken_unicode/1,eof_on_pipe/1,unicode_prompt/1]). + read_modes_ogl/1, broken_unicode/1,eof_on_pipe/1, + unicode_prompt/1, shell_slogan/1]). -export([io_server_proxy/1,start_io_server_proxy/0, proxy_getall/1, proxy_setnext/2, proxy_quit/1]). %% For spawn --export([toerl_server/3,answering_machine1/3, - answering_machine2/3]). - --export([uprompt/1]). +-export([answering_machine1/3, answering_machine2/3]). -%%-define(without_test_server, true). - --ifdef(without_test_server). --define(line, put(line, ?LINE), ). --define(config(X,Y), foo). --define(t, test_server). --define(privdir(_), "./io_SUITE_priv"). --else. --include_lib("common_test/include/ct.hrl"). --define(privdir(Conf), proplists:get_value(priv_dir, Conf)). --endif. +-export([uprompt/1, slogan/0, session_slogan/0]). %%-define(debug, true). -ifdef(debug). --define(format(S, A), io:format(S, A)). -define(dbg(Data),io:format(standard_error, "DBG: ~p\r\n",[Data])). --define(RM_RF(Dir),begin io:format(standard_error, "Not Removed: ~p\r\n",[Dir]), - ok end). -else. --define(format(S, A), ok). -define(dbg(Data),noop). --define(RM_RF(Dir),rm_rf(Dir)). -endif. -init_per_testcase(_Case, Config) -> - Term = os:getenv("TERM", "dumb"), - os:putenv("TERM","vt100"), - [{term, Term} | Config]. -end_per_testcase(_Case, Config) -> - Term = proplists:get_value(term,Config), - os:putenv("TERM",Term), - ok. - suite() -> [{ct_hooks,[ts_install_cth]}, {timetrap,{minutes,5}}]. @@ -78,16 +50,21 @@ suite() -> all() -> [setopts_getopts, unicode_options, unicode_options_gen, binary_options, read_modes_gl, read_modes_ogl, - broken_unicode, eof_on_pipe, unicode_prompt]. + broken_unicode, eof_on_pipe, unicode_prompt, + shell_slogan]. groups() -> []. init_per_suite(Config) -> - DefShell = get_default_shell(), - [{default_shell,DefShell}|Config]. + Term = os:getenv("TERM", "dumb"), + os:putenv("TERM","vt100"), + DefShell = rtnode:get_default_shell(), + [{default_shell,DefShell},{term, Term}|Config]. -end_per_suite(_Config) -> +end_per_suite(Config) -> + Term = proplists:get_value(term,Config), + os:putenv("TERM",Term), ok. init_per_group(_GroupName, Config) -> @@ -96,13 +73,11 @@ init_per_group(_GroupName, Config) -> end_per_group(_GroupName, Config) -> Config. - - -record(state, { - q = [], - nxt = eof, - mode = list - }). + q = [], + nxt = eof, + mode = list + }). uprompt(_L) -> [1050,1072,1082,1074,1086,32,1077,32,85,110,105,99,111,100,101,32,63]. @@ -111,41 +86,73 @@ uprompt(_L) -> unicode_prompt(Config) when is_list(Config) -> PA = filename:dirname(code:which(?MODULE)), case proplists:get_value(default_shell,Config) of - old -> - ok; new -> - rtnode([{putline,""}, - {putline, "2."}, - {getline, "2"}, - {putline, "shell:prompt_func({io_proto_SUITE,uprompt})."}, - {getline, "default"}, - {putline, "io:get_line('')."}, - {putline, "hej"}, - {getline, "\"hej\\n\""}, - {putline, "io:setopts([{binary,true}])."}, - {getline, "ok"}, - {putline, "io:get_line('')."}, - {putline, "hej"}, - {getline, "<<\"hej\\n\">>"} - ],[],[],"-pa \""++ PA++"\"") + rtnode:run( + [{putline,""}, + {putline, "2."}, + {expect, "[\n ]2"}, + {putline, "shell:prompt_func({io_proto_SUITE,uprompt})."}, + {expect, "[\n ]default"}, + {putline, "io:get_line('')."}, + {putline, "hej"}, + {expect, "\\Q\"hej\\n\"\\E"}, + {putline, "io:setopts([{binary,true}])."}, + {expect, "[\n ]ok"}, + {putline, "io:get_line('')."}, + {putline, "hej"}, + {expect,"[\n ]hej"}, + {expect, "\\Q<<\"hej\\n\">>\\E"} + ],[],"",["-pa",PA]); + _ -> + ok end, %% And one with oldshell - rtnode([{putline,""}, - {putline, "2."}, - {getline_re, ".*2$"}, - {putline, "shell:prompt_func({io_proto_SUITE,uprompt})."}, - {getline_re, ".*default"}, - {putline, "io:get_line('')."}, - {putline, "hej"}, - {getline_re, ".*\"hej\\\\n\""}, - {putline, "io:setopts([{binary,true}])."}, - {getline_re, ".*ok"}, - {putline, "io:get_line('')."}, - {putline, "hej"}, - {getline_re, ".*<<\"hej\\\\n\">>"} - ],[],[],"-oldshell -pa \""++PA++"\""), + rtnode:run( + [{putline,""}, + {putline, "2."}, + {expect, "[\n ]2"}, + {putline, "shell:prompt_func({io_proto_SUITE,uprompt})."}, + {expect, "default"}, + {putline, "io:get_line('')."}, + {putline, "hej"}, + {expect, "\\Q\"hej\\n\"\\E"}, + {putline, "io:setopts([{binary,true}])."}, + {expect, "[\n ]\\?*ok"}, + {putline, "io:get_line('')."}, + {putline, "hej"}, + {expect,"[\n ]\\?*hej"}, + {expect, "\\Q<<\"hej\\n\">>\\E"} + ],[],"",["-oldshell","-pa",PA]), ok. +%% Test that an Unicode prompt does not crash the shell. +shell_slogan(Config) when is_list(Config) -> + PA = filename:dirname(code:which(?MODULE)), + case proplists:get_value(default_shell,Config) of + new -> + rtnode:run( + [{expect, "\\Q"++string:trim(erlang:system_info(system_version))++"\\E"}, + {expect, "\\Q"++io_lib:format("Eshell V~s (press Ctrl+G to abort, type help(). for help)",[erlang:system_info(version)])++"\\E"} + ],[],"",[]), + rtnode:run( + [{expect, "\nTest slogan"}, + {expect, "\nTest session slogan \\("} + ],[],"",["-stdlib","shell_slogan","\"Test slogan\"", + "-stdlib","shell_session_slogan","\"Test session slogan\""]), + rtnode:run( + [{expect, "\nTest slogan"}, + {expect, "\\Q\nTest session slogan (\\E"} + ],[],"",["-stdlib","shell_slogan","fun io_proto_SUITE:slogan/0", + "-stdlib","shell_session_slogan","fun io_proto_SUITE:session_slogan/0", + "-pa",PA]); + _ -> + ok + end. + +slogan() -> + "Test slogan". +session_slogan() -> + "Test session slogan". %% Check io:setopts and io:getopts functions. setopts_getopts(Config) when is_list(Config) -> @@ -222,40 +229,42 @@ setopts_getopts(Config) when is_list(Config) -> eof = io:get_line(RFile,''), file:close(RFile), case proplists:get_value(default_shell,Config) of - old -> - ok; new -> %% So, lets test another node with new interactive shell - rtnode([{putline,""}, - {putline, "2."}, - {getline, "2"}, - {putline, "lists:keyfind(binary,1,io:getopts())."}, - {getline, "{binary,false}"}, - {putline, "io:get_line('')."}, - {putline, "hej"}, - {getline, "\"hej\\n\""}, - {putline, "io:setopts([{binary,true}])."}, - {getline, "ok"}, - {putline, "io:get_line('')."}, - {putline, "hej"}, - {getline, "<<\"hej\\n\">>"} - ],[]) + rtnode:run( + [{putline,""}, + {putline, "2."}, + {expect, "[\n ]2[^.]"}, + {putline, "lists:keyfind(binary,1,io:getopts())."}, + {expect, "{binary,false}"}, + {putline, "io:get_line('')."}, + {putline, "hej"}, + {expect, "\\Q\"hej\\n\"\\E"}, + {putline, "io:setopts([{binary,true}])."}, + {expect, "[\n ]ok"}, + {putline, "io:get_line('')."}, + {putline, "hej"}, + {expect, "\\Q<<\"hej\\n\">>\\E"} + ],[]); + _ -> + ok end, %% And one with oldshell - rtnode([{putline,""}, - {putline, "2."}, - {getline_re, ".*2$"}, - {putline, "lists:keyfind(binary,1,io:getopts())."}, - {getline_re, ".*{binary,false}"}, - {putline, "io:get_line('')."}, - {putline, "hej"}, - {getline_re, ".*\"hej\\\\n\""}, - {putline, "io:setopts([{binary,true}])."}, - {getline_re, ".*ok"}, - {putline, "io:get_line('')."}, - {putline, "hej"}, - {getline_re, ".*<<\"hej\\\\n\">>"} - ],[],[],"-oldshell"), + rtnode:run( + [{putline,""}, + {putline, "2."}, + {expect, "[\n ]2[^.]"}, + {putline, "lists:keyfind(binary,1,io:getopts())."}, + {expect, "[\n ]{binary,false}"}, + {putline, "io:get_line('')."}, + {putline, "hej"}, + {expect, "\\Q\"hej\\n\"\\E"}, + {putline, "io:setopts([{binary,true}])."}, + {expect, "[\n ]ok"}, + {putline, "io:get_line('')."}, + {putline, "hej"}, + {expect, "\\Q<<\"hej\\n\">>\\E"} + ],[],"",["-oldshell"]), ok. @@ -419,42 +428,38 @@ unicode_options(Config) when is_list(Config) -> [ ok = CannotWriteFile(F,FailDir) || F <- AllNoBom ], case proplists:get_value(default_shell,Config) of - old -> - ok; new -> %% OK, time for the group_leaders... - rtnode([{putline,""}, - {putline, "2."}, - {getline, "2"}, - {putline, "lists:keyfind(encoding,1,io:getopts())."}, - {getline, "{encoding,latin1}"}, - {putline, "io:format(\"~ts~n\",[[1024]])."}, - {getline, "\\x{400}"}, - {putline, "io:setopts([unicode])."}, - {getline, "ok"}, - {putline, "io:format(\"~ts~n\",[[1024]])."}, - {getline, - binary_to_list(unicode:characters_to_binary( - [1024],unicode,utf8))} - ],[],"LC_CTYPE=\""++get_lc_ctype()++"\"; " - "export LC_CTYPE; ") + rtnode:run( + [{putline,""}, + {putline, "2."}, + {expect, "[\n ]2[^.]"}, + {putline, "lists:keyfind(encoding,1,io:getopts())."}, + {expect, "{encoding,latin1}"}, + {putline, "io:format(\"~ts~n\",[[1024]])."}, + {expect, "\\Q\\x{400}\\E"}, + {putline, "io:setopts([unicode])."}, + {expect, "[\n ]ok"}, + {putline, "io:format(\"~ts~n\",[[1024]])."}, + {expect, "[\n ]"++[1024]} + ],[],"",["-env","LC_ALL",get_lc_ctype()]); + _ -> + ok end, - rtnode([{putline,""}, - {putline, "2."}, - {getline_re, ".*2$"}, - {putline, "lists:keyfind(encoding,1,io:getopts())."}, - {getline_re, ".*{encoding,latin1}"}, - {putline, "io:format(\"~ts~n\",[[1024]])."}, - {getline_re, ".*\\\\x{400\\}"}, - {putline, "io:setopts([{encoding,unicode}])."}, - {getline_re, ".*ok"}, - {putline, "io:format(\"~ts~n\",[[1024]])."}, - {getline_re, - ".*"++binary_to_list(unicode:characters_to_binary( - [1024],unicode,utf8))} - ],[],"LC_CTYPE=\""++get_lc_ctype()++"\"; export LC_CTYPE; ", - " -oldshell "), - + rtnode:run( + [{putline,""}, + {putline, "2."}, + {expect, "[\n ]2[^.]"}, + {putline, "lists:keyfind(encoding,1,io:getopts())."}, + {expect, "[\n ]{encoding,latin1}"}, + {putline, "io:format(\"~ts~n\",[[1024]])."}, + {expect, "\\Q\\x{400}\\E"}, + {putline, "io:setopts([{encoding,unicode}])."}, + {expect, "[\n ]ok"}, + {putline, "io:format(\"~ts~n\",[[1024]])."}, + {expect, "[\n ]"++[1024]} + ],[],"", + ["-oldshell","-env","LC_ALL",get_lc_ctype()]), ok. %% Tests various unicode options on random generated files. @@ -709,115 +714,124 @@ binary_options(Config) when is_list(Config) -> %% OK, time for the group_leaders... case proplists:get_value(default_shell,Config) of - old -> - ok; new -> - rtnode([{putline, "2."}, - {getline, "2"}, - {putline, "lists:keyfind(binary,1,io:getopts())."}, - {getline, "{binary,false}"}, - {putline, "io:get_line('')."}, - {putline, "hej"}, - {getline, "\"hej\\n\""}, - {putline, "io:setopts([{binary,true},unicode])."}, - {getline, "ok"}, - {putline, "io:get_line('')."}, - {putline, "hej"}, - {getline, "<<\"hej\\n\">>"}, - {putline, "io:get_line('')."}, - {putline, binary_to_list(<<"\345\344\366"/utf8>>)}, - {getline, "<<\""++binary_to_list(<<"\345\344\366"/utf8>>)++"\\n\"/utf8>>"} - ],[]) + rtnode:run( + [{putline, "2."}, + {expect, "[\n ]2[^.]"}, + {putline, "lists:keyfind(binary,1,io:getopts())."}, + {expect, "[\n ]{binary,false}"}, + {putline, "io:get_line('')."}, + {putline, "hej"}, + {expect, "\\Q\"hej\\n\"\\E"}, + {putline, "io:setopts([{binary,true},unicode])."}, + {expect, "[\n ]ok"}, + {putline, "io:get_line('')."}, + {putline, "hej"}, + {expect, "\\Q<<\"hej\\n\">>\\E"}, + {putline, "io:get_line('')."}, + {putline, binary_to_list(<<"\345\344\366"/utf8>>)}, + {expect, latin1, "[\n ]\\Q<<\""++binary_to_list(<<"\345\344\366"/utf8>>)++"\\n\"/utf8>>\\E"} + ],[]); + _ -> + ok end, %% And one with oldshell - rtnode([{putline, "2."}, - {getline_re, ".*2$"}, - {putline, "lists:keyfind(binary,1,io:getopts())."}, - {getline_re, ".*{binary,false}"}, - {putline, "io:get_line('')."}, - {putline, "hej"}, - {getline_re, ".*\"hej\\\\n\""}, - {putline, "io:setopts([{binary,true},unicode])."}, - {getline_re, ".*ok"}, - {putline, "io:get_line('')."}, - {putline, "hej"}, - {getline_re, ".*<<\"hej\\\\n\">>"}, - {putline, "io:get_line('')."}, - {putline, binary_to_list(<<"\345\344\366"/utf8>>)}, - {getline_re, ".*<<\""++binary_to_list(<<"\345\344\366"/utf8>>)++"\\\\n\"/utf8>>"} - ],[],[],"-oldshell"), + rtnode:run( + [{putline, "2."}, + {expect, "[\n ]2[^.]"}, + {putline, "lists:keyfind(binary,1,io:getopts())."}, + {expect, "[\n ]{binary,false}"}, + {putline, "io:get_line('')."}, + {putline, "hej"}, + {expect, "[\n ]\\Q\"hej\\n\"\\E"}, + {putline, "io:setopts([{binary,true},unicode])."}, + {expect, "[\n ]ok"}, + {putline, "io:get_line('')."}, + {putline, "hej"}, + {expect, "\\Q<<\"hej\\n\">>\\E"}, + {putline, "io:get_line('')."}, + {putline, binary_to_list(<<"\345\344\366"/utf8>>)}, + {expect, latin1, "[\n ]\\Q<<\""++binary_to_list(<<"\345\344\366"/utf8>>)++"\\n\"/utf8>>\\E"} + ],[],"",["-oldshell"]), ok. - - - answering_machine1(OthNode,OthReg,Me) -> TestDataLine1 = [229,228,246], TestDataUtf = binary_to_list(unicode:characters_to_binary(TestDataLine1)), - rtnode([{putline,""}, - {putline, "2."}, - {getline, "2"}, - {putline, "{"++OthReg++","++OthNode++"} ! group_leader()."}, - {getline, "<"}, - %% get_line - {getline_re, ".*Prompt"}, - {putline, "Hej"}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, "Hej"}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataLine1}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataLine1}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataUtf}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataUtf}, - {getline_re, ".*Okej"}, - %% get_chars - {getline_re, ".*Prompt"}, - {putline, "Hej"}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, "Hej"}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataLine1}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataLine1}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataUtf}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataUtf}, - {getline_re, ".*Okej"}, - %% fread - {getline_re, ".*Prompt"}, - {putline, "Hej"}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, "Hej"}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataLine1}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataLine1}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataUtf}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataUtf}, - {getline_re, ".*Okej"} - - ],Me,"LC_CTYPE=\""++get_lc_ctype()++"\"; export LC_CTYPE; "), + TestDataLine1Oct = "\\\\345( \b)*\\\\344( \b)*\\\\366", + rtnode:run( + [{putline,""}, + {putline, "2."}, + {expect, "2"}, + {putline, "io:getopts()."}, + {expect, ">"}, + {putline, "{"++OthReg++","++OthNode++"} ! group_leader()."}, + {expect, "<"}, + %% get_line + {expect, "Prompt"}, + {putline, "Hej"}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, "Hej"}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataLine1}, + {expect, latin1, "\n" ++ TestDataLine1Oct}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataLine1}, + {expect, latin1, "\n" ++ TestDataLine1Oct}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataUtf}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataUtf}, + {expect, "Okej"}, + %% get_chars + {expect, "Prompt"}, + {putline, "Hej"}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, "Hej"}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataLine1}, + {expect, latin1, "\n" ++ TestDataLine1Oct}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataLine1}, + {expect, latin1, "\n" ++ TestDataLine1Oct}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataUtf}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataUtf}, + {expect, "Okej"}, + %% fread + {expect, "Prompt"}, + {putline, "Hej"}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, "Hej"}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataLine1}, + {expect, latin1, "\n" ++ TestDataLine1Oct}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataLine1}, + {expect, latin1, "\n" ++ TestDataLine1Oct}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataUtf}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataUtf}, + {expect, "Okej"} + + ],Me,"",["-env","LC_ALL",get_lc_ctype()]), O = list_to_atom(OthReg), O ! {self(),done}, ok. @@ -825,70 +839,77 @@ answering_machine1(OthNode,OthReg,Me) -> answering_machine2(OthNode,OthReg,Me) -> TestDataLine1 = [229,228,246], TestDataUtf = binary_to_list(unicode:characters_to_binary(TestDataLine1)), - rtnode([{putline,""}, - {putline, "2."}, - {getline, "2"}, - {putline, "{"++OthReg++","++OthNode++"} ! group_leader()."}, - {getline_re, ".*<[0-9].*"}, - %% get_line - {getline_re, ".*Prompt"}, - {putline, "Hej"}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, "Hej"}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataLine1}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataLine1}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataUtf}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataUtf}, - {getline_re, ".*Okej"}, - %% get_chars - {getline_re, ".*Prompt"}, - {putline, "Hej"}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, "Hej"}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataLine1}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataLine1}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataUtf}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataUtf}, - {getline_re, ".*Okej"}, - %% fread - {getline_re, ".*Prompt"}, - {putline, "Hej"}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, "Hej"}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataLine1}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataLine1}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataUtf}, - {getline_re, ".*Okej"}, - {getline_re, ".*Prompt"}, - {putline, TestDataUtf}, - {getline_re, ".*Okej"} - - ],Me,"LC_CTYPE=\""++get_lc_ctype()++"\"; export LC_CTYPE; "," -oldshell "), + rtnode:run( + [{putline,""}, + {putline, "2."}, + {expect, "2"}, + {putline, "{"++OthReg++","++OthNode++"} ! group_leader()."}, + {expect, "<[0-9].*"}, + %% get_line + {expect, "Prompt"}, + {putline, "Hej"}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, "Hej"}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataLine1}, + {expect, latin1, "\n" ++ TestDataLine1}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataLine1}, + {expect, latin1, "\n" ++ TestDataLine1}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataUtf}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataUtf}, + {expect, "Okej"}, + %% get_chars + {expect, "Prompt"}, + {putline, "Hej"}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, "Hej"}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataLine1}, + {expect, latin1, "\n" ++ TestDataLine1}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataLine1}, + {expect, latin1, "\n" ++ TestDataLine1}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataUtf}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataUtf}, + {expect, "Okej"}, + %% fread + {expect, "Prompt"}, + {putline, "Hej"}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, "Hej"}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataLine1}, + {expect, latin1, "\n" ++ TestDataLine1}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataLine1}, + {expect, latin1, "\n" ++ TestDataLine1}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataUtf}, + {expect, "Okej"}, + {expect, "Prompt"}, + {putline, TestDataUtf}, + {expect, "Okej"} + + ],Me,"",["-oldshell","-env","LC_ALL",get_lc_ctype()]), O = list_to_atom(OthReg), O ! {self(),done}, ok. @@ -896,20 +917,20 @@ answering_machine2(OthNode,OthReg,Me) -> %% Test various modes when reading from the group leade from another machine. read_modes_ogl(Config) when is_list(Config) -> - case get_progs() of - {error,Reason} -> - {skipped,Reason}; + case proplists:get_value(default_shell,Config) of + noshell -> + {skipped,"No run_erl"}; _ -> read_modes_gl_1(Config,answering_machine2) end. %% Test various modes when reading from the group leade from another machine. read_modes_gl(Config) when is_list(Config) -> - case {get_progs(),proplists:get_value(default_shell,Config)} of - {{error,Reason},_} -> - {skipped,Reason}; - {_,old} -> - {skipper,"No new shell"}; + case proplists:get_value(default_shell,Config) of + noshell -> + {skipped,"No run_erl"}; + old -> + {skipped,"No new shell"}; _ -> read_modes_gl_1(Config,answering_machine1) end. @@ -919,7 +940,7 @@ read_modes_gl_1(_Config,Machine) -> TestDataLine1BinUtf = unicode:characters_to_binary(TestDataLine1), TestDataLine1BinLatin = list_to_binary(TestDataLine1), - {ok,N2List} = create_nodename(), + N2List = peer:random_name(?FUNCTION_NAME), MyNodeList = atom2list(node()), register(io_proto_suite,self()), AM1 = spawn(?MODULE,Machine, @@ -1023,14 +1044,10 @@ loop_through_file2(_,{error,_Err},_,_) -> loop_through_file2(F,Bin,Chunk,Enc) when is_binary(Bin) -> loop_through_file2(F,io:get_chars(F,'',Chunk),Chunk,Enc). - - %% Test eof before newline on stdin when erlang is in pipe. eof_on_pipe(Config) when is_list(Config) -> - case {get_progs(),os:type()} of - {{error,Reason},_} -> - {skipped,Reason}; - {{_,_,Erl},{unix,linux}} -> + case {ct:get_progname(),os:type()} of + {Erl,{unix,linux}} -> %% Not even Linux is reliable - echo can be both styles try EchoLine = case os:cmd("echo -ne \"test\\ntest\"") of @@ -1074,453 +1091,6 @@ eof_on_pipe(Config) when is_list(Config) -> {skipped,"Only on linux"} end. - -%% -%% Tool for running interactive shell (stolen from the kernel -%% test suite interactive_shell_SUITE) -%% --undef(line). --define(line,). -rtnode(C,N) -> - rtnode(C,N,[]). -rtnode(Commands,Nodename,ErlPrefix) -> - rtnode(Commands,Nodename,ErlPrefix,[]). -rtnode(Commands,Nodename,ErlPrefix,Extra) -> - case get_progs() of - {error,_Reason} -> - {skip,"No runerl present"}; - {RunErl,ToErl,Erl} -> - case create_tempdir() of - {error, Reason2} -> - {skip, Reason2}; - Tempdir -> - SPid = start_runerl_node(RunErl, ErlPrefix++ - "\\\""++Erl++"\\\"", - Tempdir, Nodename, Extra), - CPid = start_toerl_server(ToErl, Tempdir), - put(getline_skipped, []), - Res = (catch get_and_put(CPid, Commands, 1)), - case stop_runerl_node(CPid) of - {error,_} -> - CPid2 = start_toerl_server(ToErl, Tempdir), - put(getline_skipped, []), - ok = get_and_put - (CPid2, - [{putline,[7]}, - {sleep, - timeout(short)}, - {putline,""}, - {getline," -->"}, - {putline,"s"}, - {putline,"c"}, - {putline,""}], 1), - stop_runerl_node(CPid2); - _ -> - ok - end, - wait_for_runerl_server(SPid), - ok = ?RM_RF(Tempdir), - ok = Res - end - end. - -timeout(long) -> - 2 * timeout(normal); -timeout(short) -> - timeout(normal) div 10; -timeout(normal) -> - 10000 * test_server:timetrap_scale_factor(). - - -%% start_noshell_node(Name) -> -%% PADir = filename:dirname(code:which(?MODULE)), -%% {ok, Node} = test_server:start_node(Name,slave,[{args," -noshell -pa "++ -%% PADir++" "}]), -%% Node. -%% stop_noshell_node(Node) -> -%% test_server:stop_node(Node). - --ifndef(debug). -rm_rf(Dir) -> - try - {ok,List} = file:list_dir(Dir), - Files = [filename:join([Dir,X]) || X <- List], - [case file:list_dir(Y) of - {error, enotdir} -> - ok = file:delete(Y); - _ -> - ok = rm_rf(Y) - end || Y <- Files], - ok = file:del_dir(Dir), - ok - catch - _:Exception -> {error, {Exception,Dir}} - end. --endif. - -get_and_put(_CPid,[],_) -> - ok; -get_and_put(CPid, [{sleep, X}|T],N) -> - ?dbg({sleep, X}), - receive - after X -> - get_and_put(CPid,T,N+1) - end; -get_and_put(CPid, [{getline_pred,Pred,Msg}|T]=T0, N) - when is_function(Pred) -> - ?dbg({getline, Match}), - CPid ! {self(), {get_line, timeout(normal)}}, - receive - {get_line, timeout} -> - error_logger:error_msg("~p: getline timeout waiting for \"~s\" " - "(command number ~p, skipped: ~p)~n", - [?MODULE,Msg,N,get(getline_skipped)]), - {error, timeout}; - {get_line, Data} -> - ?dbg({data,Data}), - case Pred(Data) of - yes -> - put(getline_skipped, []), - get_and_put(CPid, T,N+1); - no -> - error_logger:error_msg("~p: getline match failure " - "\"~s\" " - "(command number ~p)\n", - [?MODULE,Msg,N]), - {error, no_match}; - 'maybe' -> - List = get(getline_skipped), - put(getline_skipped, List ++ [Data]), - get_and_put(CPid, T0, N) - end - end; -get_and_put(CPid, [{getline, Match}|T],N) -> - ?dbg({getline, Match}), - F = fun(Data) -> - case lists:prefix(Match, Data) of - true -> yes; - false -> 'maybe' - end - end, - get_and_put(CPid, [{getline_pred,F,Match}|T], N); -get_and_put(CPid, [{getline_re, Match}|T],N) -> - F = fun(Data) -> - case re:run(Data, Match, [{capture,none}]) of - match -> yes; - _ -> 'maybe' - end - end, - get_and_put(CPid, [{getline_pred,F,Match}|T], N); -get_and_put(CPid, [{putline_raw, Line}|T],N) -> - ?dbg({putline_raw, Line}), - CPid ! {self(), {send_line, Line}}, - Timeout = timeout(normal), - receive - {send_line, ok} -> - get_and_put(CPid, T,N+1) - after Timeout -> - error_logger:error_msg("~p: putline_raw timeout (~p) sending " - "\"~s\" (command number ~p)~n", - [?MODULE, Timeout, Line, N]), - {error, timeout} - end; - -get_and_put(CPid, [{putline, Line}|T],N) -> - ?dbg({putline, Line}), - CPid ! {self(), {send_line, Line}}, - Timeout = timeout(normal), - receive - {send_line, ok} -> - get_and_put(CPid, [{getline, []}|T],N) - after Timeout -> - error_logger:error_msg("~p: putline timeout (~p) sending " - "\"~s\" (command number ~p)~n[~p]~n", - [?MODULE, Timeout, Line, N,get()]), - {error, timeout} - end. - -wait_for_runerl_server(SPid) -> - Ref = erlang:monitor(process, SPid), - Timeout = timeout(long), - receive - {'DOWN', Ref, process, SPid, _} -> - ok - after Timeout -> - {error, timeout} - end. - - - -stop_runerl_node(CPid) -> - Ref = erlang:monitor(process, CPid), - CPid ! {self(), kill_emulator}, - Timeout = timeout(long), - receive - {'DOWN', Ref, process, CPid, noproc} -> - ok; - {'DOWN', Ref, process, CPid, normal} -> - ok; - {'DOWN', Ref, process, CPid, {error, Reason}} -> - {error, Reason} - after Timeout -> - {error, timeout} - end. - -get_progs() -> - case os:type() of - {unix,freebsd} -> - {error,"cant use run_erl on freebsd"}; - {unix,openbsd} -> - {error,"cant use run_erl on openbsd"}; - {unix,_} -> - case os:find_executable("run_erl") of - RE when is_list(RE) -> - case os:find_executable("to_erl") of - TE when is_list(TE) -> - case os:find_executable("erl") of - E when is_list(E) -> - {RE,TE,E}; - _ -> - {error, "Could not find erl command"} - end; - _ -> - {error, "Could not find to_erl command"} - end; - _ -> - {error, "Could not find run_erl command"} - end; - _ -> - {error, "Not a unix OS"} - end. - -create_tempdir() -> - create_tempdir(filename:join(["/tmp","rtnode"++os:getpid()]),$A). - -create_tempdir(Dir,X) when X > $Z, X < $a -> - create_tempdir(Dir,$a); -create_tempdir(Dir,X) when X > $z -> - Estr = lists:flatten( - io_lib:format("Unable to create ~s, reason eexist", - [Dir++[$z]])), - {error, Estr}; -create_tempdir(Dir0, Ch) -> - %% Expect fairly standard unix. - Dir = Dir0++[Ch], - case file:make_dir(Dir) of - {error, eexist} -> - create_tempdir(Dir0, Ch+1); - {error, Reason} -> - Estr = lists:flatten( - io_lib:format("Unable to create ~s, reason ~p", - [Dir,Reason])), - {error,Estr}; - ok -> - Dir - end. - -create_nodename() -> - create_nodename($A). - -create_nodename(X) when X > $Z, X < $a -> - create_nodename($a); -create_nodename(X) when X > $z -> - {error,out_of_nodenames}; -create_nodename(X) -> - NN = "rtnode"++os:getpid()++[X], - case file:read_file_info(filename:join(["/tmp",NN])) of - {error,enoent} -> - Host = lists:nth(2,string:tokens(atom_to_list(node()),"@")), - {ok,NN++"@"++Host}; - _ -> - create_nodename(X+1) - end. - - -start_runerl_node(RunErl,Erl,Tempdir,Nodename,Extra) -> - XArg = case Nodename of - [] -> - []; - _ -> - " -sname "++(if is_atom(Nodename) -> atom_to_list(Nodename); - true -> Nodename - end)++ - " -setcookie "++atom_to_list(erlang:get_cookie()) - end, - XXArg = case Extra of - [] -> - []; - _ -> - " "++Extra - end, - spawn(fun() -> - ?dbg("\""++RunErl++"\" "++Tempdir++"/ "++Tempdir++ - " \""++Erl++XArg++XXArg++"\""), - os:cmd("\""++RunErl++"\" "++Tempdir++"/ "++Tempdir++ - " \""++Erl++XArg++XXArg++"\"") - end). - -start_toerl_server(ToErl,Tempdir) -> - Pid = spawn(?MODULE,toerl_server,[self(),ToErl,Tempdir]), - receive - {Pid,started} -> - Pid; - {Pid,error,Reason} -> - {error,Reason} - end. - -try_to_erl(_Command, 0) -> - {error, cannot_to_erl}; -try_to_erl(Command, N) -> - ?dbg({?LINE,N}), - Port = open_port({spawn, Command},[eof,{line,1000}]), - Timeout = timeout(normal) div 2, - receive - {Port, eof} -> - receive after Timeout -> - ok - end, - try_to_erl(Command, N-1) - after Timeout -> - ?dbg(Port), - Port - end. - -toerl_server(Parent,ToErl,Tempdir) -> - Port = try_to_erl("\""++ToErl++"\" "++Tempdir++"/ 2>/dev/null",8), - case Port of - P when is_port(P) -> - Parent ! {self(),started}; - {error,Other} -> - Parent ! {self(),error,Other}, - exit(Other) - end, - case toerl_loop(Port,[]) of - normal -> - ok; - {error, Reason} -> - error_logger:error_msg("toerl_server exit with reason ~p~n", - [Reason]), - exit(Reason) - end. - -toerl_loop(Port,Acc) -> - ?dbg({toerl_loop, Port, Acc}), - receive - {Port,{data,{Tag0,Data}}} when is_port(Port) -> - ?dbg({?LINE,Port,{data,{Tag0,Data}}}), - case Acc of - [{noeol,Data0}|T0] -> - toerl_loop(Port,[{Tag0, Data0++Data}|T0]); - _ -> - toerl_loop(Port,[{Tag0,Data}|Acc]) - end; - {Pid,{get_line,Timeout}} -> - case Acc of - [] -> - case get_data_within(Port,Timeout,[]) of - timeout -> - Pid ! {get_line, timeout}, - toerl_loop(Port,[]); - {noeol,Data1} -> - Pid ! {get_line, timeout}, - toerl_loop(Port,[{noeol,Data1}]); - {eol,Data2} -> - Pid ! {get_line, Data2}, - toerl_loop(Port,[]) - end; - [{noeol,Data3}] -> - case get_data_within(Port,Timeout,Data3) of - timeout -> - Pid ! {get_line, timeout}, - toerl_loop(Port,Acc); - {noeol,Data4} -> - Pid ! {get_line, timeout}, - toerl_loop(Port,[{noeol,Data4}]); - {eol,Data5} -> - Pid ! {get_line, Data5}, - toerl_loop(Port,[]) - end; - List -> - {NewAcc,[{eol,Data6}]} = lists:split(length(List)-1,List), - Pid ! {get_line,Data6}, - toerl_loop(Port,NewAcc) - end; - {Pid, {send_line, Data7}} -> - Port ! {self(),{command, Data7++"\n"}}, - Pid ! {send_line, ok}, - toerl_loop(Port,Acc); - {_Pid, kill_emulator} -> - Port ! {self(),{command, "init:stop().\n"}}, - Timeout1 = timeout(long), - receive - {Port,eof} -> - normal - after Timeout1 -> - {error, kill_timeout} - end; - {Port, eof} -> - {error, unexpected_eof}; - Other -> - {error, {unexpected, Other}} - end. - -millistamp() -> - erlang:monotonic_time(millisecond). - -get_data_within(Port, X, Acc) when X =< 0 -> - ?dbg({get_data_within, X, Acc, ?LINE}), - receive - {Port,{data,{Tag0,Data}}} -> - ?dbg({?LINE,Port,{data,{Tag0,Data}}}), - {Tag0, Acc++Data} - after 0 -> - case Acc of - [] -> - timeout; - Noeol -> - {noeol,Noeol} - end - end; - - -get_data_within(Port, Timeout, Acc) -> - ?dbg({get_data_within, Timeout, Acc, ?LINE}), - T1 = millistamp(), - receive - {Port,{data,{noeol,Data}}} -> - ?dbg({?LINE,Port,{data,{noeol,Data}}}), - Elapsed = millistamp() - T1 + 1, - get_data_within(Port, Timeout - Elapsed, Acc ++ Data); - {Port,{data,{eol,Data1}}} -> - ?dbg({?LINE,Port,{data,{eol,Data1}}}), - {eol, Acc ++ Data1} - after Timeout -> - timeout - end. - -get_default_shell() -> - Match = fun(Data) -> - case lists:prefix("undefined", Data) of - true -> - yes; - false -> - case re:run(Data, "<\\d+[.]\\d+[.]\\d+>", - [{capture,none}]) of - match -> no; - _ -> 'maybe' - end - end - end, - try - rtnode([{putline,""}, - {putline, "whereis(user_drv)."}, - {getline_pred, Match, "matching of user_drv pid"}], []), - old - catch _E:_R -> - ?dbg({_E,_R}), - new - end. - %% %% Test I/O-server %% diff --git a/lib/stdlib/test/lists_SUITE.erl b/lib/stdlib/test/lists_SUITE.erl index b369b6918e..59f2e8bd03 100644 --- a/lib/stdlib/test/lists_SUITE.erl +++ b/lib/stdlib/test/lists_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2022. All Rights Reserved. +%% Copyright Ericsson AB 1997-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -55,6 +55,10 @@ ufunsort_error/1, uniq_1/1, uniq_2/1, zip_unzip/1, zip_unzip3/1, zipwith/1, zipwith3/1, + zip_fail/1, zip_trim/1, zip_pad/1, + zip3_fail/1, zip3_trim/1, zip3_pad/1, + zipwith_fail/1, zipwith_trim/1, zipwith_pad/1, + zipwith3_fail/1, zipwith3_trim/1, zipwith3_pad/1, filter_partition/1, join/1, otp_5939/1, otp_6023/1, otp_6606/1, otp_7230/1, @@ -121,7 +125,11 @@ groups() -> {flatten, [parallel], [flatten_1, flatten_2, flatten_1_e, flatten_2_e]}, {tickets, [parallel], [otp_5939, otp_6023, otp_6606, otp_7230]}, - {zip, [parallel], [zip_unzip, zip_unzip3, zipwith, zipwith3]}, + {zip, [parallel], [zip_unzip, zip_unzip3, zipwith, zipwith3, + zip_fail, zip_trim, zip_pad, + zip3_fail, zip3_trim, zip3_pad, + zipwith_fail, zipwith_trim, zipwith_pad, + zipwith3_fail, zipwith3_trim, zipwith3_pad]}, {uniq, [parallel], [uniq_1, uniq_2]}, {misc, [parallel], [reverse, member, dropwhile, takewhile, filter_partition, suffix, subtract, join, @@ -433,6 +441,8 @@ keyreplace(Config) when is_list(Config) -> merge(Config) when is_list(Config) -> + Singleton = id([a, b, c]), + %% merge list of lists [] = lists:merge([]), [] = lists:merge([[]]), @@ -455,6 +465,33 @@ merge(Config) when is_list(Config) -> Seq = lists:seq(1,100), true = Seq == lists:merge(lists:map(fun(E) -> [E] end, Seq)), + true = erts_debug:same(Singleton, lists:merge([Singleton])), + true = erts_debug:same(Singleton, lists:merge([Singleton, []])), + true = erts_debug:same(Singleton, lists:merge([[], Singleton])), + true = erts_debug:same(Singleton, lists:merge([Singleton, [], []])), + true = erts_debug:same(Singleton, lists:merge([[], Singleton, []])), + true = erts_debug:same(Singleton, lists:merge([[], [], Singleton])), + + {'EXIT', _} = (catch lists:merge([a])), + {'EXIT', _} = (catch lists:merge([a, b])), + {'EXIT', _} = (catch lists:merge([a, []])), + {'EXIT', _} = (catch lists:merge([[], b])), + {'EXIT', _} = (catch lists:merge([a, [1, 2, 3]])), + {'EXIT', _} = (catch lists:merge([[1, 2, 3], b])), + {'EXIT', _} = (catch lists:merge([a, b, c])), + {'EXIT', _} = (catch lists:merge([a, b, []])), + {'EXIT', _} = (catch lists:merge([a, [], c])), + {'EXIT', _} = (catch lists:merge([a, [], []])), + {'EXIT', _} = (catch lists:merge([[], b, c])), + {'EXIT', _} = (catch lists:merge([[], b, []])), + {'EXIT', _} = (catch lists:merge([[], [], c])), + {'EXIT', _} = (catch lists:merge([a, b, [1, 2, 3]])), + {'EXIT', _} = (catch lists:merge([a, [1, 2, 3], c])), + {'EXIT', _} = (catch lists:merge([a, [1, 2, 3], [4, 5, 6]])), + {'EXIT', _} = (catch lists:merge([[1, 2, 3], b, c])), + {'EXIT', _} = (catch lists:merge([[1, 2, 3], b, [4, 5, 6]])), + {'EXIT', _} = (catch lists:merge([[1, 2, 3], [4, 5, 6], c])), + Two = [1,2], Six = [1,2,3,4,5,6], @@ -474,6 +511,15 @@ merge(Config) when is_list(Config) -> [1,2,3,4,5,7] = lists:merge([2,4], [1,3,5,7]), [1,2,3,4,5,6,7] = lists:merge([2,4,6], [1,3,5,7]), + true = erts_debug:same(Singleton, lists:merge([], Singleton)), + true = erts_debug:same(Singleton, lists:merge(Singleton, [])), + + {'EXIT', _} = (catch lists:merge(a, b)), + {'EXIT', _} = (catch lists:merge(a, [])), + {'EXIT', _} = (catch lists:merge([], b)), + {'EXIT', _} = (catch lists:merge(a, [1, 2, 3])), + {'EXIT', _} = (catch lists:merge([1, 2, 3], b)), + %% 3-way merge [] = lists:merge3([], [], []), Two = lists:merge3([], [], Two), @@ -490,11 +536,28 @@ merge(Config) when is_list(Config) -> Nine = lists:merge3([7,8,9],[4,5,6],[1,2,3]), Nine = lists:merge3([4,5,6],[7,8,9],[1,2,3]), + true = erts_debug:same(Singleton, lists:merge3([], [], Singleton)), + true = erts_debug:same(Singleton, lists:merge3([], Singleton, [])), + true = erts_debug:same(Singleton, lists:merge3(Singleton, [], [])), + + {'EXIT', _} = (catch lists:merge3(a, b, c)), + {'EXIT', _} = (catch lists:merge3(a, b, [])), + {'EXIT', _} = (catch lists:merge3(a, [], c)), + {'EXIT', _} = (catch lists:merge3(a, [], [])), + {'EXIT', _} = (catch lists:merge3([], b, [])), + {'EXIT', _} = (catch lists:merge3([], [], c)), + {'EXIT', _} = (catch lists:merge3(a, b, [1, 2, 3])), + {'EXIT', _} = (catch lists:merge3(a, [1, 2, 3], c)), + {'EXIT', _} = (catch lists:merge3(a, [1, 2, 3], [4, 5, 6])), + {'EXIT', _} = (catch lists:merge3([1, 2, 3], b, [4, 5, 6])), + {'EXIT', _} = (catch lists:merge3([1, 2, 3], [4, 5, 6], c)), + ok. %% reverse merge functions rmerge(Config) when is_list(Config) -> + Singleton = id([a, b, c]), Two = [2,1], Six = [6,5,4,3,2,1], @@ -514,6 +577,15 @@ rmerge(Config) when is_list(Config) -> [7,5,4,3,2,1] = lists:rmerge([4,2], [7,5,3,1]), [7,6,5,4,3,2,1] = lists:rmerge([6,4,2], [7,5,3,1]), + true = erts_debug:same(Singleton, lists:rmerge([], Singleton)), + true = erts_debug:same(Singleton, lists:rmerge(Singleton, [])), + + {'EXIT', _} = (catch lists:rmerge(a, b)), + {'EXIT', _} = (catch lists:rmerge(a, [])), + {'EXIT', _} = (catch lists:rmerge([], b)), + {'EXIT', _} = (catch lists:rmerge(a, [1, 2, 3])), + {'EXIT', _} = (catch lists:rmerge([1, 2, 3], b)), + Nine = [9,8,7,6,5,4,3,2,1], %% 3-way reversed merge @@ -532,6 +604,22 @@ rmerge(Config) when is_list(Config) -> Nine = lists:rmerge3([9,8,7],[6,5,4],[3,2,1]), Nine = lists:rmerge3([6,5,4],[9,8,7],[3,2,1]), + true = erts_debug:same(Singleton, lists:rmerge3([], [], Singleton)), + true = erts_debug:same(Singleton, lists:rmerge3([], Singleton, [])), + true = erts_debug:same(Singleton, lists:rmerge3(Singleton, [], [])), + + {'EXIT', _} = (catch lists:rmerge3(a, b, c)), + {'EXIT', _} = (catch lists:rmerge3(a, b, [])), + {'EXIT', _} = (catch lists:rmerge3(a, [], c)), + {'EXIT', _} = (catch lists:rmerge3(a, [], [])), + {'EXIT', _} = (catch lists:rmerge3([], b, [])), + {'EXIT', _} = (catch lists:rmerge3([], [], c)), + {'EXIT', _} = (catch lists:rmerge3(a, b, [1, 2, 3])), + {'EXIT', _} = (catch lists:rmerge3(a, [1, 2, 3], c)), + {'EXIT', _} = (catch lists:rmerge3(a, [1, 2, 3], [4, 5, 6])), + {'EXIT', _} = (catch lists:rmerge3([1, 2, 3], b, [4, 5, 6])), + {'EXIT', _} = (catch lists:rmerge3([1, 2, 3], [4, 5, 6], c)), + ok. sort_1(Config) when is_list(Config) -> @@ -632,6 +720,8 @@ usort_1(Conf) when is_list(Conf) -> ok. umerge(Conf) when is_list(Conf) -> + Singleton = id([a, b, c]), + %% merge list of lists [] = lists:umerge([]), [] = lists:umerge([[]]), @@ -655,6 +745,33 @@ umerge(Conf) when is_list(Conf) -> Seq = lists:seq(1,100), true = Seq == lists:umerge(lists:map(fun(E) -> [E] end, Seq)), + true = erts_debug:same(Singleton, lists:umerge([Singleton])), + true = erts_debug:same(Singleton, lists:umerge([Singleton, []])), + true = erts_debug:same(Singleton, lists:umerge([[], Singleton])), + true = erts_debug:same(Singleton, lists:umerge([Singleton, [], []])), + true = erts_debug:same(Singleton, lists:umerge([[], Singleton, []])), + true = erts_debug:same(Singleton, lists:umerge([[], [], Singleton])), + + {'EXIT', _} = (catch lists:umerge([a])), + {'EXIT', _} = (catch lists:umerge([a, b])), + {'EXIT', _} = (catch lists:umerge([a, []])), + {'EXIT', _} = (catch lists:umerge([[], b])), + {'EXIT', _} = (catch lists:umerge([a, [1, 2, 3]])), + {'EXIT', _} = (catch lists:umerge([[1, 2, 3], b])), + {'EXIT', _} = (catch lists:umerge([a, b, c])), + {'EXIT', _} = (catch lists:umerge([a, b, []])), + {'EXIT', _} = (catch lists:umerge([a, [], c])), + {'EXIT', _} = (catch lists:umerge([a, [], []])), + {'EXIT', _} = (catch lists:umerge([[], b, c])), + {'EXIT', _} = (catch lists:umerge([[], b, []])), + {'EXIT', _} = (catch lists:umerge([[], [], c])), + {'EXIT', _} = (catch lists:umerge([a, b, [1, 2, 3]])), + {'EXIT', _} = (catch lists:umerge([a, [1, 2, 3], c])), + {'EXIT', _} = (catch lists:umerge([a, [1, 2, 3], [4, 5, 6]])), + {'EXIT', _} = (catch lists:umerge([[1, 2, 3], b, c])), + {'EXIT', _} = (catch lists:umerge([[1, 2, 3], b, [4, 5, 6]])), + {'EXIT', _} = (catch lists:umerge([[1, 2, 3], [4, 5, 6], c])), + Two = [1,2], Six = [1,2,3,4,5,6], @@ -681,6 +798,15 @@ umerge(Conf) when is_list(Conf) -> [1,2,3,4,5,7] = lists:umerge([2,4], [1,2,3,4,5,7]), [1,2,3,4,5,6,7] = lists:umerge([2,4,6], [1,2,3,4,5,6,7]), + true = erts_debug:same(Singleton, lists:umerge([], Singleton)), + true = erts_debug:same(Singleton, lists:umerge(Singleton, [])), + + {'EXIT', _} = (catch lists:umerge(a, b)), + {'EXIT', _} = (catch lists:umerge(a, [])), + {'EXIT', _} = (catch lists:umerge([], b)), + {'EXIT', _} = (catch lists:umerge(a, [1, 2, 3])), + {'EXIT', _} = (catch lists:umerge([1, 2, 3], b)), + %% 3-way unique merge [] = lists:umerge3([], [], []), Two = lists:umerge3([], [], Two), @@ -702,9 +828,27 @@ umerge(Conf) when is_list(Conf) -> [1,2,3] = lists:umerge3([1,2,3],[2,3],[1,2,3]), [1,2,3,4] = lists:umerge3([2,3,4],[3,4],[1,2,3]), + true = erts_debug:same(Singleton, lists:umerge3([], [], Singleton)), + true = erts_debug:same(Singleton, lists:umerge3([], Singleton, [])), + true = erts_debug:same(Singleton, lists:umerge3(Singleton, [], [])), + + {'EXIT', _} = (catch lists:umerge3(a, b, c)), + {'EXIT', _} = (catch lists:umerge3(a, b, [])), + {'EXIT', _} = (catch lists:umerge3(a, [], c)), + {'EXIT', _} = (catch lists:umerge3(a, [], [])), + {'EXIT', _} = (catch lists:umerge3([], b, [])), + {'EXIT', _} = (catch lists:umerge3([], [], c)), + {'EXIT', _} = (catch lists:umerge3(a, b, [1, 2, 3])), + {'EXIT', _} = (catch lists:umerge3(a, [1, 2, 3], c)), + {'EXIT', _} = (catch lists:umerge3(a, [1, 2, 3], [4, 5, 6])), + {'EXIT', _} = (catch lists:umerge3([1, 2, 3], b, [4, 5, 6])), + {'EXIT', _} = (catch lists:umerge3([1, 2, 3], [4, 5, 6], c)), + ok. rumerge(Conf) when is_list(Conf) -> + Singleton = id([a, b, c]), + Two = [2,1], Six = [6,5,4,3,2,1], @@ -731,6 +875,15 @@ rumerge(Conf) when is_list(Conf) -> [7,5,4,3,2,1] = lists:rumerge([4,2], [7,5,4,3,2,1]), [7,6,5,4,3,2,1] = lists:rumerge([6,4,2], [7,6,5,4,3,2,1]), + true = erts_debug:same(Singleton, lists:rumerge([], Singleton)), + true = erts_debug:same(Singleton, lists:rumerge(Singleton, [])), + + {'EXIT', _} = (catch lists:rumerge(a, b)), + {'EXIT', _} = (catch lists:rumerge(a, [])), + {'EXIT', _} = (catch lists:rumerge([], b)), + {'EXIT', _} = (catch lists:rumerge(a, [1, 2, 3])), + {'EXIT', _} = (catch lists:rumerge([1, 2, 3], b)), + Nine = [9,8,7,6,5,4,3,2,1], %% 3-way reversed unique merge @@ -759,6 +912,23 @@ rumerge(Conf) when is_list(Conf) -> true = lists:umerge(L1, L2) == lists:reverse(lists:rumerge(lists:reverse(L1), lists:reverse(L2))), + + true = erts_debug:same(Singleton, lists:rumerge3([], [], Singleton)), + true = erts_debug:same(Singleton, lists:rumerge3([], Singleton, [])), + true = erts_debug:same(Singleton, lists:rumerge3(Singleton, [], [])), + + {'EXIT', _} = (catch lists:rumerge3(a, b, c)), + {'EXIT', _} = (catch lists:rumerge3(a, b, [])), + {'EXIT', _} = (catch lists:rumerge3(a, [], c)), + {'EXIT', _} = (catch lists:rumerge3(a, [], [])), + {'EXIT', _} = (catch lists:rumerge3([], b, [])), + {'EXIT', _} = (catch lists:rumerge3([], [], c)), + {'EXIT', _} = (catch lists:rumerge3(a, b, [1, 2, 3])), + {'EXIT', _} = (catch lists:rumerge3(a, [1, 2, 3], c)), + {'EXIT', _} = (catch lists:rumerge3(a, [1, 2, 3], [4, 5, 6])), + {'EXIT', _} = (catch lists:rumerge3([1, 2, 3], b, [4, 5, 6])), + {'EXIT', _} = (catch lists:rumerge3([1, 2, 3], [4, 5, 6], c)), + ok. %% usort/1 on big randomized lists. @@ -815,6 +985,7 @@ ucheck_stability(L) -> %% Key merge two lists. keymerge(Config) when is_list(Config) -> + Singleton = id([{1, a}, {2, b}, {3, c}]), Two = [{1,a},{2,b}], Six = [{1,a},{2,b},{3,c},{4,d},{5,e},{6,f}], @@ -843,11 +1014,21 @@ keymerge(Config) when is_list(Config) -> [{b,2},{c,11},{c,12},{c,21},{c,22},{e,5}] = lists:keymerge(1,[{c,11},{c,12},{e,5}], [{b,2},{c,21},{c,22}]), + true = erts_debug:same(Singleton, lists:keymerge(1, Singleton, [])), + true = erts_debug:same(Singleton, lists:keymerge(1, [], Singleton)), + + {'EXIT', _} = (catch lists:keymerge(1, a, b)), + {'EXIT', _} = (catch lists:keymerge(1, a, [])), + {'EXIT', _} = (catch lists:keymerge(1, [], b)), + {'EXIT', _} = (catch lists:keymerge(1, a, [{1, a}, {2, b}, {3, c}])), + {'EXIT', _} = (catch lists:keymerge(1, [{1, a}, {2, b}, {3, c}], b)), + ok. %% Reverse key merge two lists. rkeymerge(Config) when is_list(Config) -> + Singleton = id([{1, a}, {2, b}, {3, c}]), Two = [{2,b},{1,a}], Six = [{6,f},{5,e},{4,d},{3,c},{2,b},{1,a}], @@ -880,6 +1061,15 @@ rkeymerge(Config) when is_list(Config) -> lists:reverse(lists:rkeymerge(1,lists:reverse(L1), lists:reverse(L2))), + true = erts_debug:same(Singleton, lists:rkeymerge(1, Singleton, [])), + true = erts_debug:same(Singleton, lists:rkeymerge(1, [], Singleton)), + + {'EXIT', _} = (catch lists:rkeymerge(1, a, b)), + {'EXIT', _} = (catch lists:rkeymerge(1, a, [])), + {'EXIT', _} = (catch lists:rkeymerge(1, [], b)), + {'EXIT', _} = (catch lists:rkeymerge(1, a, [{1, a}, {2, b}, {3, c}])), + {'EXIT', _} = (catch lists:rkeymerge(1, [{1, a}, {2, b}, {3, c}], b)), + ok. keysort_1(Config) when is_list(Config) -> @@ -998,6 +1188,7 @@ keycompare(I, J, A, B) when element(I, A) == element(I, B), %% Merge two lists while removing duplicates. ukeymerge(Conf) when is_list(Conf) -> + Singleton = id([{1, a}, {2, b}, {3, c}]), Two = [{1,a},{2,b}], Six = [{1,a},{2,b},{3,c},{4,d},{5,e},{6,f}], @@ -1047,11 +1238,21 @@ ukeymerge(Conf) when is_list(Conf) -> L2 = [{b,1},{b,3},{b,5},{b,7}], L1 = lists:ukeymerge(2, L1, L2), + true = erts_debug:same(Singleton, lists:ukeymerge(1, Singleton, [])), + true = erts_debug:same(Singleton, lists:ukeymerge(1, [], Singleton)), + + {'EXIT', _} = (catch lists:ukeymerge(1, a, b)), + {'EXIT', _} = (catch lists:ukeymerge(1, a, [])), + {'EXIT', _} = (catch lists:ukeymerge(1, [], b)), + {'EXIT', _} = (catch lists:ukeymerge(1, a, [{1, a}, {2, b}, {3, c}])), + {'EXIT', _} = (catch lists:ukeymerge(1, [{1, a}, {2, b}, {3, c}], b)), + ok. %% Reverse merge two lists while removing duplicates. rukeymerge(Conf) when is_list(Conf) -> + Singleton = id([{1, a}, {2, b}, {3, c}]), Two = [{2,b},{1,a}], Six = [{6,f},{5,e},{4,d},{3,c},{2,b},{1,a}], @@ -1101,6 +1302,15 @@ rukeymerge(Conf) when is_list(Conf) -> lists:reverse(lists:rukeymerge(2, lists:reverse(L1), lists:reverse(L2))), + true = erts_debug:same(Singleton, lists:rukeymerge(1, Singleton, [])), + true = erts_debug:same(Singleton, lists:rukeymerge(1, [], Singleton)), + + {'EXIT', _} = (catch lists:rukeymerge(1, a, b)), + {'EXIT', _} = (catch lists:rukeymerge(1, a, [])), + {'EXIT', _} = (catch lists:rukeymerge(1, [], b)), + {'EXIT', _} = (catch lists:rukeymerge(1, a, [{1, a}, {2, b}, {3, c}])), + {'EXIT', _} = (catch lists:rukeymerge(1, [{1, a}, {2, b}, {3, c}], b)), + ok. ukeysort_1(Config) when is_list(Config) -> @@ -1278,6 +1488,7 @@ ukeycompare(I, J, A, B) when A =/= B, %% Merge two lists using a fun. funmerge(Config) when is_list(Config) -> + Singleton = id([a, b, c]), Two = [1,2], Six = [1,2,3,4,5,6], F = fun(X, Y) -> X =< Y end, @@ -1302,11 +1513,21 @@ funmerge(Config) when is_list(Config) -> [{b,2},{c,11},{c,12},{c,21},{c,22},{e,5}] = lists:merge(F2,[{c,11},{c,12},{e,5}], [{b,2},{c,21},{c,22}]), + true = erts_debug:same(Singleton, lists:merge(F, Singleton, [])), + true = erts_debug:same(Singleton, lists:merge(F, [], Singleton)), + + {'EXIT', _} = (catch lists:merge(F, a, b)), + {'EXIT', _} = (catch lists:merge(F, a, [])), + {'EXIT', _} = (catch lists:merge(F, [], b)), + {'EXIT', _} = (catch lists:merge(F, a, [1, 2, 3])), + {'EXIT', _} = (catch lists:merge(F, [1, 2, 3], b)), + ok. %% Reverse merge two lists using a fun. rfunmerge(Config) when is_list(Config) -> + Singleton = id([a, b, c]), Two = [2,1], Six = [6,5,4,3,2,1], F = fun(X, Y) -> X =< Y end, @@ -1334,6 +1555,15 @@ rfunmerge(Config) when is_list(Config) -> lists:merge(F2, L1, L2) == lists:reverse(lists:rmerge(F2,lists:reverse(L1), lists:reverse(L2))), + true = erts_debug:same(Singleton, lists:rmerge(F, Singleton, [])), + true = erts_debug:same(Singleton, lists:rmerge(F, [], Singleton)), + + {'EXIT', _} = (catch lists:rmerge(F, a, b)), + {'EXIT', _} = (catch lists:rmerge(F, a, [])), + {'EXIT', _} = (catch lists:rmerge(F, [], b)), + {'EXIT', _} = (catch lists:rmerge(F, a, [1, 2, 3])), + {'EXIT', _} = (catch lists:rmerge(F, [1, 2, 3], b)), + ok. @@ -1403,6 +1633,7 @@ funsort_check(I, Input, Expected) -> %% Merge two lists while removing duplicates using a fun. ufunmerge(Conf) when is_list(Conf) -> + Singleton = id([a, b, c]), Two = [1,2], Six = [1,2,3,4,5,6], F = fun(X, Y) -> X =< Y end, @@ -1437,10 +1668,20 @@ ufunmerge(Conf) when is_list(Conf) -> [{b,2},{e,5},{c,11},{c,12},{c,21},{c,22}] = lists:umerge(F2, [{e,5},{c,11},{c,12}], [{b,2},{c,21},{c,22}]), + true = erts_debug:same(Singleton, lists:umerge(F, Singleton, [])), + true = erts_debug:same(Singleton, lists:umerge(F, [], Singleton)), + + {'EXIT', _} = (catch lists:umerge(F, a, b)), + {'EXIT', _} = (catch lists:umerge(F, a, [])), + {'EXIT', _} = (catch lists:umerge(F, [], b)), + {'EXIT', _} = (catch lists:umerge(F, a, [1, 2, 3])), + {'EXIT', _} = (catch lists:umerge(F, [1, 2, 3], b)), + ok. %% Reverse merge two lists while removing duplicates using a fun. rufunmerge(Conf) when is_list(Conf) -> + Singleton = id([a, b, c]), Two = [2,1], Six = [6,5,4,3,2,1], F = fun(X, Y) -> X =< Y end, @@ -1480,6 +1721,15 @@ rufunmerge(Conf) when is_list(Conf) -> lists:umerge(F2, L3, L4) == lists:reverse(lists:rumerge(F2,lists:reverse(L3), lists:reverse(L4))), + true = erts_debug:same(Singleton, lists:rumerge(F, Singleton, [])), + true = erts_debug:same(Singleton, lists:rumerge(F, [], Singleton)), + + {'EXIT', _} = (catch lists:rumerge(F, a, b)), + {'EXIT', _} = (catch lists:rumerge(F, a, [])), + {'EXIT', _} = (catch lists:rumerge(F, [], b)), + {'EXIT', _} = (catch lists:rumerge(F, a, [1, 2, 3])), + {'EXIT', _} = (catch lists:rumerge(F, [1, 2, 3], b)), + ok. ufunsort_1(Config) when is_list(Config) -> @@ -2362,6 +2612,41 @@ zip_unzip(Config) when is_list(Config) -> {'EXIT',{function_clause,_}} = (catch lists:zip([a], [b,c])), ok. +zip_fail(Config) when is_list(Config) -> + [] = lists:zip([], [], fail), + {'EXIT', {function_clause, _}} = (catch lists:zip([a], [], fail)), + {'EXIT', {function_clause, _}} = (catch lists:zip([], [c], fail)), + + [{a, c}] = lists:zip([a], [c], fail), + {'EXIT', {function_clause, _}} = (catch lists:zip([a, b], [c], fail)), + {'EXIT', {function_clause, _}} = (catch lists:zip([a], [c, d], fail)), + + ok. + +zip_trim(Config) when is_list(Config) -> + [] = lists:zip([], [], trim), + [] = lists:zip([a], [], trim), + [] = lists:zip([], [c], trim), + + [{a, c}] = lists:zip([a], [c], trim), + [{a, c}] = lists:zip([a, b], [c], trim), + [{a, c}] = lists:zip([a], [c, d], trim), + + ok. + +zip_pad(Config) when is_list(Config) -> + How = {pad, {x, y}}, + + [] = lists:zip([], [], How), + [{a, y}] = lists:zip([a], [], How), + [{x, c}] = lists:zip([], [c], How), + + [{a, c}] = lists:zip([a], [c], How), + [{a, c}, {b, y}] = lists:zip([a, b], [c], How), + [{a, c}, {x, d}] = lists:zip([a], [c, d], How), + + ok. + %% Test lists:zip3/3, lists:unzip3/1. zip_unzip3(Config) when is_list(Config) -> [] = lists:zip3([], [], []), @@ -2388,6 +2673,65 @@ zip_unzip3(Config) when is_list(Config) -> ok. +zip3_fail(Config) when is_list(Config) -> + [] = lists:zip3([], [], [], fail), + {'EXIT', {function_clause, _}} = (catch lists:zip3([a], [], [], fail)), + {'EXIT', {function_clause, _}} = (catch lists:zip3([], [c], [], fail)), + {'EXIT', {function_clause, _}} = (catch lists:zip3([a], [c], [], fail)), + {'EXIT', {function_clause, _}} = (catch lists:zip3([], [], [e], fail)), + {'EXIT', {function_clause, _}} = (catch lists:zip3([a], [], [e], fail)), + {'EXIT', {function_clause, _}} = (catch lists:zip3([], [c], [e], fail)), + + [{a, c, e}] = lists:zip3([a], [c], [e], fail), + {'EXIT', {function_clause, _}} = (catch lists:zip3([a, b], [c], [e], fail)), + {'EXIT', {function_clause, _}} = (catch lists:zip3([a], [c, d], [e], fail)), + {'EXIT', {function_clause, _}} = (catch lists:zip3([a, b], [c, d], [e], fail)), + {'EXIT', {function_clause, _}} = (catch lists:zip3([a], [c], [e, f], fail)), + {'EXIT', {function_clause, _}} = (catch lists:zip3([a, b], [c], [e, f], fail)), + {'EXIT', {function_clause, _}} = (catch lists:zip3([a], [c, d], [e, f], fail)), + + ok. + +zip3_trim(Config) when is_list(Config) -> + [] = lists:zip3([], [], [], trim), + [] = lists:zip3([a], [], [], trim), + [] = lists:zip3([], [c], [], trim), + [] = lists:zip3([a], [c], [], trim), + [] = lists:zip3([], [], [e], trim), + [] = lists:zip3([a], [], [e], trim), + [] = lists:zip3([], [c], [e], trim), + + [{a, c, e}] = lists:zip3([a], [c], [e], trim), + [{a, c, e}] = lists:zip3([a, b], [c], [e], trim), + [{a, c, e}] = lists:zip3([a], [c, d], [e], trim), + [{a, c, e}] = lists:zip3([a, b], [c, d], [e], trim), + [{a, c, e}] = lists:zip3([a], [c], [e, f], trim), + [{a, c, e}] = lists:zip3([a, b], [c], [e, f], trim), + [{a, c, e}] = lists:zip3([a], [c, d], [e, f], trim), + + ok. + +zip3_pad(Config) when is_list(Config) -> + How = {pad, {x, y, z}}, + + [] = lists:zip3([], [], [], How), + [{a, y, z}] = lists:zip3([a], [], [], How), + [{x, c, z}] = lists:zip3([], [c], [], How), + [{a, c, z}] = lists:zip3([a], [c], [], How), + [{x, y, e}] = lists:zip3([], [], [e], How), + [{a, y, e}] = lists:zip3([a], [], [e], How), + [{x, c, e}] = lists:zip3([], [c], [e], How), + + [{a, c, e}] = lists:zip3([a], [c], [e], How), + [{a, c, e}, {b, y, z}] = lists:zip3([a, b], [c], [e], How), + [{a, c, e}, {x, d, z}] = lists:zip3([a], [c, d], [e], How), + [{a, c, e}, {b, d, z}] = lists:zip3([a, b], [c, d], [e], How), + [{a, c, e}, {x, y, f}] = lists:zip3([a], [c], [e, f], How), + [{a, c, e}, {b, y, f}] = lists:zip3([a, b], [c], [e, f], How), + [{a, c, e}, {x, d, f}] = lists:zip3([a], [c, d], [e, f], How), + + ok. + %% Test lists:zipwith/3. zipwith(Config) when is_list(Config) -> Zip = fun(A, B) -> [A|B] end, @@ -2410,6 +2754,47 @@ zipwith(Config) when is_list(Config) -> {'EXIT',{function_clause,_}} = (catch lists:zipwith(Zip, [a], [b,c])), ok. +zipwith_fail(Config) when is_list(Config) -> + Zip = fun(A, B) -> A * B end, + + [] = lists:zipwith(Zip, [], [], fail), + {'EXIT', {function_clause, _}} = (catch lists:zipwith(Zip, [2], [], fail)), + {'EXIT', {function_clause, _}} = (catch lists:zipwith(Zip, [], [5], fail)), + + [2 * 5] = lists:zipwith(Zip, [2], [5], fail), + {'EXIT', {function_clause, _}} = (catch lists:zipwith(Zip, [2, 3], [5], fail)), + {'EXIT', {function_clause, _}} = (catch lists:zipwith(Zip, [2], [5, 7], fail)), + + ok. + +zipwith_trim(Config) when is_list(Config) -> + Zip = fun(A, B) -> A * B end, + + [] = lists:zipwith(Zip, [], [], trim), + [] = lists:zipwith(Zip, [2], [], trim), + [] = lists:zipwith(Zip, [], [5], trim), + + [2 * 5] = lists:zipwith(Zip, [2], [5], trim), + [2 * 5] = lists:zipwith(Zip, [2, 3], [5], trim), + [2 * 5] = lists:zipwith(Zip, [2], [5, 7], trim), + + ok. + +zipwith_pad(Config) when is_list(Config) -> + How = {pad, {17, 19}}, + + Zip = fun(A, B) -> A * B end, + + [] = lists:zipwith(Zip, [], [], How), + [ 2 * 19] = lists:zipwith(Zip, [2], [], How), + [17 * 5] = lists:zipwith(Zip, [], [5], How), + + [2 * 5] = lists:zipwith(Zip, [2], [5], How), + [2 * 5, 3 * 19] = lists:zipwith(Zip, [2, 3], [5], How), + [2 * 5, 17 * 7] = lists:zipwith(Zip, [2], [5, 7], How), + + ok. + %% Test lists:zipwith3/4. zipwith3(Config) when is_list(Config) -> Zip = fun(A, B, C) -> [A,B,C] end, @@ -2434,6 +2819,69 @@ zipwith3(Config) when is_list(Config) -> ok. +zipwith3_fail(Config) when is_list(Config) -> + Zip = fun(A, B, C) -> A * B * C end, + + [] = lists:zipwith3(Zip, [], [], [], fail), + {'EXIT', {function_clause, _}} = (catch lists:zipwith3(Zip, [2], [], [], fail)), + {'EXIT', {function_clause, _}} = (catch lists:zipwith3(Zip, [], [5], [], fail)), + {'EXIT', {function_clause, _}} = (catch lists:zipwith3(Zip, [2], [5], [], fail)), + {'EXIT', {function_clause, _}} = (catch lists:zipwith3(Zip, [], [], [11], fail)), + {'EXIT', {function_clause, _}} = (catch lists:zipwith3(Zip, [2], [], [11], fail)), + {'EXIT', {function_clause, _}} = (catch lists:zipwith3(Zip, [], [5], [11], fail)), + + [2 * 5 * 11] = lists:zipwith3(Zip, [2], [5], [11], fail), + {'EXIT', {function_clause, _}} = (catch lists:zipwith3(Zip, [2, 3], [5], [11], fail)), + {'EXIT', {function_clause, _}} = (catch lists:zipwith3(Zip, [2], [5, 7], [11], fail)), + {'EXIT', {function_clause, _}} = (catch lists:zipwith3(Zip, [2, 3], [5, 7], [11], fail)), + {'EXIT', {function_clause, _}} = (catch lists:zipwith3(Zip, [2], [5], [11, 13], fail)), + {'EXIT', {function_clause, _}} = (catch lists:zipwith3(Zip, [2, 3], [5], [11, 13], fail)), + {'EXIT', {function_clause, _}} = (catch lists:zipwith3(Zip, [2], [5, 7], [11, 13], fail)), + + ok. + +zipwith3_trim(Config) when is_list(Config) -> + Zip = fun(A, B, C) -> A * B * C end, + + [] = lists:zipwith3(Zip, [], [], [], trim), + [] = lists:zipwith3(Zip, [2], [], [], trim), + [] = lists:zipwith3(Zip, [], [5], [], trim), + [] = lists:zipwith3(Zip, [], [], [11], trim), + [] = lists:zipwith3(Zip, [2], [], [11], trim), + [] = lists:zipwith3(Zip, [], [5], [11], trim), + + [2 * 5 * 11] = lists:zipwith3(Zip, [2], [5], [11], trim), + [2 * 5 * 11] = lists:zipwith3(Zip, [2, 3], [5], [11], trim), + [2 * 5 * 11] = lists:zipwith3(Zip, [2], [5, 7], [11], trim), + [2 * 5 * 11] = lists:zipwith3(Zip, [2], [5], [11, 13], trim), + [2 * 5 * 11] = lists:zipwith3(Zip, [2, 3], [5], [11, 13], trim), + [2 * 5 * 11] = lists:zipwith3(Zip, [2], [5, 7], [11, 13], trim), + + ok. + +zipwith3_pad(Config) when is_list(Config) -> + How = {pad, {17, 19, 23}}, + + Zip = fun(A, B, C) -> A * B * C end, + + [] = lists:zipwith3(Zip, [], [], [], How), + [ 2 * 19 * 23] = lists:zipwith3(Zip, [2], [], [], How), + [17 * 5 * 23] = lists:zipwith3(Zip, [], [5], [], How), + [ 2 * 5 * 23] = lists:zipwith3(Zip, [2], [5], [], How), + [17 * 19 * 11] = lists:zipwith3(Zip, [], [], [11], How), + [ 2 * 19 * 11] = lists:zipwith3(Zip, [2], [], [11], How), + [17 * 5 * 11] = lists:zipwith3(Zip, [], [5], [11], How), + + [2 * 5 * 11] = lists:zipwith3(Zip, [2], [5], [11], How), + [2 * 5 * 11, 3 * 19 * 23] = lists:zipwith3(Zip, [2, 3], [5], [11], How), + [2 * 5 * 11, 17 * 7 * 23] = lists:zipwith3(Zip, [2], [5, 7], [11], How), + [2 * 5 * 11, 3 * 7 * 23] = lists:zipwith3(Zip, [2, 3], [5, 7], [11], How), + [2 * 5 * 11, 17 * 19 * 13] = lists:zipwith3(Zip, [2], [5], [11, 13], How), + [2 * 5 * 11, 3 * 19 * 13] = lists:zipwith3(Zip, [2, 3], [5], [11, 13], How), + [2 * 5 * 11, 17 * 7 * 13] = lists:zipwith3(Zip, [2], [5, 7], [11, 13], How), + + ok. + %% Test lists:join/2 join(Config) when is_list(Config) -> A = [a,b,c], @@ -2750,15 +3198,39 @@ hof(Config) when is_list(Config) -> enumerate(Config) when is_list(Config) -> [] = lists:enumerate([]), [] = lists:enumerate(10, []), + [] = lists:enumerate(-10, []), + [] = lists:enumerate(10, 2, []), + [] = lists:enumerate(10, -2, []), + [] = lists:enumerate(-10, 2, []), + [] = lists:enumerate(-10, -2, []), [{1,a},{2,b},{3,c}] = lists:enumerate([a,b,c]), [{10,a},{11,b},{12,c}] = lists:enumerate(10, [a,b,c]), + [{-10,a},{-9,b},{-8,c}] = lists:enumerate(-10, [a,b,c]), + [{10,a},{12,b},{14,c}] = lists:enumerate(10, 2, [a,b,c]), + [{10,a},{8,b},{6,c}] = lists:enumerate(10, -2, [a,b,c]), + [{-10,a},{-12,b},{-14,c}] = lists:enumerate(-10, -2, [a,b,c]), {'EXIT', {function_clause, _}} = catch lists:enumerate(0), {'EXIT', {function_clause, _}} = catch lists:enumerate(0, 10), + {'EXIT', {function_clause, _}} = catch lists:enumerate(0, 10, 20), {'EXIT', {function_clause, _}} = catch lists:enumerate(1.0, []), {'EXIT', {function_clause, _}} = catch lists:enumerate(1.0, [a,b,c]), + {'EXIT', {function_clause, _}} = catch lists:enumerate(1.0, 2, []), + {'EXIT', {function_clause, _}} = catch lists:enumerate(1.0, 2, [a,b,c]), + {'EXIT', {function_clause, _}} = catch lists:enumerate(1, 2.0, []), + {'EXIT', {function_clause, _}} = catch lists:enumerate(1, 2.0, [a,b,c]), + {'EXIT', {function_clause, _}} = catch lists:enumerate(1.0, 2.0, []), + {'EXIT', {function_clause, _}} = catch lists:enumerate(1.0, 2.0, [a,b,c]), {'EXIT', {function_clause, _}} = catch lists:enumerate(<<1>>, []), {'EXIT', {function_clause, _}} = catch lists:enumerate(<<1>>, [a,b,c]), + {'EXIT', {function_clause, _}} = catch lists:enumerate(<<1>>, 2, []), + {'EXIT', {function_clause, _}} = catch lists:enumerate(<<1>>, 2, [a,b,c]), + {'EXIT', {function_clause, _}} = catch lists:enumerate(1, <<2>>, []), + {'EXIT', {function_clause, _}} = catch lists:enumerate(1, <<2>>, [a,b,c]), + {'EXIT', {function_clause, _}} = catch lists:enumerate(<<1>>, <<2>>, []), + {'EXIT', {function_clause, _}} = catch lists:enumerate(<<1>>, <<2>>, [a,b,c]), + {'EXIT', {function_clause, _}} = catch lists:enumerate(<<1,2,3>>), {'EXIT', {function_clause, _}} = catch lists:enumerate(1, <<1,2,3>>), + {'EXIT', {function_clause, _}} = catch lists:enumerate(1, 2, <<1,2,3>>), ok. diff --git a/lib/stdlib/test/lists_property_test_SUITE.erl b/lib/stdlib/test/lists_property_test_SUITE.erl new file mode 100644 index 0000000000..ecbf14309e --- /dev/null +++ b/lib/stdlib/test/lists_property_test_SUITE.erl @@ -0,0 +1,441 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2021-2023. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(lists_property_test_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-compile(export_all). + +all() -> + [ + all_true_case, all_false_case, + any_true_case, any_false_case, + append_1_case, + append_2_case, + concat_case, + delete_case, delete_absent_case, + droplast_case, + dropwhile_case, + duplicate_case, + enumerate_1_case, + enumerate_2_case, + enumerate_3_case, + filter_case, + filtermap_case, + flatlength_case, + flatmap_case, + flatten_1_case, + flatten_2_case, + foldl_case, + foldr_case, + foreach_case, + join_case, + keydelete_case, keydelete_absent_case, + keyfind_case, keyfind_absent_case, + keymap_case, + keymember_case, keymember_absent_case, + keymerge_case, keymerge_invalid_case, + keyreplace_case, keyreplace_absent_case, + keysearch_case, keysearch_absent_case, + keysort_case, + keystore_case, keystore_absent_case, + keytake_case, keytake_absent_case, + last_case, + map_case, + mapfoldl_case, + mapfoldr_case, + max_case, + member_case, member_absent_case, + merge_1_case, merge_1_invalid_case, + merge_2_case, merge_2_invalid_case, + merge_3_case, merge_3_invalid_case, + merge3_case, merge3_invalid_case, + min_case, + nth_case, nth_outofrange_case, + nthtail_case, nthtail_outofrange_case, + partition_case, + prefix_case, + reverse_1_case, + reverse_2_case, + search_case, search_absent_case, + seq2_case, + seq3_case, + sort_1_case, + sort_2_case, + split_case, split_outofrange_case, + splitwith_case, + sublist_2_case, + sublist_3_case, + subtract_case, + suffix_case, + sum_case, + takewhile_case, + ukeymerge_case, ukeymerge_invalid_case, + ukeysort_case, + umerge_1_case, umerge_1_invalid_case, + umerge_2_case, umerge_2_invalid_case, + umerge_3_case, umerge_3_invalid_case, + umerge3_case, umerge3_invalid_case, + uniq_1_case, + uniq_2_case, + unzip_case, + unzip3_case, + usort_1_case, + usort_2_case, + zip_2_case, + zip_3_case, + zip3_3_case, + zip3_4_case, + zipwith_3_case, + zipwith_4_case, + zipwith3_4_case, + zipwith3_5_case + ]. + +init_per_suite(Config) -> + ct_property_test:init_per_suite(Config). + +end_per_suite(Config) -> + persistent_term:erase({lists_prop, random_atoms}), + Config. + +do_proptest(Prop, Config) -> + ct_property_test:quickcheck(lists_prop:Prop(), Config). + +all_true_case(Config) -> + do_proptest(prop_all_true, Config). + +all_false_case(Config) -> + do_proptest(prop_all_false, Config). + +any_true_case(Config) -> + do_proptest(prop_any_true, Config). + +any_false_case(Config) -> + do_proptest(prop_any_false, Config). + +append_1_case(Config) -> + do_proptest(prop_append_1, Config). + +append_2_case(Config) -> + do_proptest(prop_append_2, Config). + +concat_case(Config) -> + do_proptest(prop_concat, Config). + +delete_case(Config) -> + do_proptest(prop_delete, Config). + +delete_absent_case(Config) -> + do_proptest(prop_delete_absent, Config). + +droplast_case(Config) -> + do_proptest(prop_droplast, Config). + +dropwhile_case(Config) -> + do_proptest(prop_dropwhile, Config). + +duplicate_case(Config) -> + do_proptest(prop_duplicate, Config). + +enumerate_1_case(Config) -> + do_proptest(prop_enumerate_1, Config). + +enumerate_2_case(Config) -> + do_proptest(prop_enumerate_2, Config). + +enumerate_3_case(Config) -> + do_proptest(prop_enumerate_3, Config). + +filter_case(Config) -> + do_proptest(prop_filter, Config). + +filtermap_case(Config) -> + do_proptest(prop_filtermap, Config). + +flatlength_case(Config) -> + do_proptest(prop_flatlength, Config). + +flatmap_case(Config) -> + do_proptest(prop_flatmap, Config). + +flatten_1_case(Config) -> + do_proptest(prop_flatten_1, Config). + +flatten_2_case(Config) -> + do_proptest(prop_flatten_2, Config). + +foldl_case(Config) -> + do_proptest(prop_foldl, Config). + +foldr_case(Config) -> + do_proptest(prop_foldr, Config). + +foreach_case(Config) -> + do_proptest(prop_foreach, Config). + +join_case(Config) -> + do_proptest(prop_join, Config). + +keydelete_case(Config) -> + do_proptest(prop_keydelete, Config). + +keydelete_absent_case(Config) -> + do_proptest(prop_keydelete_absent, Config). + +keyfind_case(Config) -> + do_proptest(prop_keyfind, Config). + +keyfind_absent_case(Config) -> + do_proptest(prop_keyfind_absent, Config). + +keymap_case(Config) -> + do_proptest(prop_keymap, Config). + +keymember_case(Config) -> + do_proptest(prop_keymember, Config). + +keymember_absent_case(Config) -> + do_proptest(prop_keymember_absent, Config). + +keymerge_case(Config) -> + do_proptest(prop_keymerge, Config). + +keymerge_invalid_case(Config) -> + do_proptest(prop_keymerge_invalid, Config). + +keyreplace_case(Config) -> + do_proptest(prop_keyreplace, Config). + +keyreplace_absent_case(Config) -> + do_proptest(prop_keyreplace_absent, Config). + +keysearch_case(Config) -> + do_proptest(prop_keysearch, Config). + +keysearch_absent_case(Config) -> + do_proptest(prop_keysearch_absent, Config). + +keysort_case(Config) -> + do_proptest(prop_keysort, Config). + +keystore_case(Config) -> + do_proptest(prop_keystore, Config). + +keystore_absent_case(Config) -> + do_proptest(prop_keystore_absent, Config). + +keytake_case(Config) -> + do_proptest(prop_keytake, Config). + +keytake_absent_case(Config) -> + do_proptest(prop_keytake_absent, Config). + +last_case(Config) -> + do_proptest(prop_last, Config). + +map_case(Config) -> + do_proptest(prop_map, Config). + +mapfoldl_case(Config) -> + do_proptest(prop_mapfoldl, Config). + +mapfoldr_case(Config) -> + do_proptest(prop_mapfoldr, Config). + +max_case(Config) -> + do_proptest(prop_max, Config). + +member_case(Config) -> + do_proptest(prop_member, Config). + +member_absent_case(Config) -> + do_proptest(prop_member_absent, Config). + +merge_1_case(Config) -> + do_proptest(prop_merge_1, Config). + +merge_1_invalid_case(Config) -> + do_proptest(prop_merge_1_invalid, Config). + +merge_2_case(Config) -> + do_proptest(prop_merge_2, Config). + +merge_2_invalid_case(Config) -> + do_proptest(prop_merge_2_invalid, Config). + +merge_3_case(Config) -> + do_proptest(prop_merge_3, Config). + +merge_3_invalid_case(Config) -> + do_proptest(prop_merge_3_invalid, Config). + +merge3_case(Config) -> + do_proptest(prop_merge3, Config). + +merge3_invalid_case(Config) -> + do_proptest(prop_merge3_invalid, Config). + +min_case(Config) -> + do_proptest(prop_min, Config). + +nth_case(Config) -> + do_proptest(prop_nth, Config). + +nth_outofrange_case(Config) -> + do_proptest(prop_nth_outofrange, Config). + +nthtail_case(Config) -> + do_proptest(prop_nthtail, Config). + +nthtail_outofrange_case(Config) -> + do_proptest(prop_nthtail_outofrange, Config). + +partition_case(Config) -> + do_proptest(prop_partition, Config). + +prefix_case(Config) -> + do_proptest(prop_prefix, Config). + +reverse_1_case(Config) -> + do_proptest(prop_reverse_1, Config). + +reverse_2_case(Config) -> + do_proptest(prop_reverse_2, Config). + +search_case(Config) -> + do_proptest(prop_search, Config). + +search_absent_case(Config) -> + do_proptest(prop_search_absent, Config). + +seq2_case(Config) -> + do_proptest(prop_seq2, Config). + +seq3_case(Config) -> + do_proptest(prop_seq3, Config). + +sort_1_case(Config) -> + do_proptest(prop_sort_1, Config). + +sort_2_case(Config) -> + do_proptest(prop_sort_2, Config). + +split_case(Config) -> + do_proptest(prop_split, Config). + +split_outofrange_case(Config) -> + do_proptest(prop_split_outofrange, Config). + +splitwith_case(Config) -> + do_proptest(prop_splitwith, Config). + +sublist_2_case(Config) -> + do_proptest(prop_sublist_2, Config). + +sublist_3_case(Config) -> + do_proptest(prop_sublist_3, Config). + +subtract_case(Config) -> + do_proptest(prop_subtract, Config). + +suffix_case(Config) -> + do_proptest(prop_suffix, Config). + +sum_case(Config) -> + do_proptest(prop_sum, Config). + +takewhile_case(Config) -> + do_proptest(prop_takewhile, Config). + +ukeymerge_case(Config) -> + do_proptest(prop_ukeymerge, Config). + +ukeymerge_invalid_case(Config) -> + do_proptest(prop_ukeymerge_invalid, Config). + +ukeysort_case(Config) -> + do_proptest(prop_ukeysort, Config). + +umerge_1_case(Config) -> + do_proptest(prop_umerge_1, Config). + +umerge_1_invalid_case(Config) -> + do_proptest(prop_umerge_1_invalid, Config). + +umerge_2_case(Config) -> + do_proptest(prop_umerge_2, Config). + +umerge_2_invalid_case(Config) -> + do_proptest(prop_umerge_2_invalid, Config). + +umerge_3_case(Config) -> + do_proptest(prop_umerge_3, Config). + +umerge_3_invalid_case(Config) -> + do_proptest(prop_umerge_3_invalid, Config). + +umerge3_case(Config) -> + do_proptest(prop_umerge3, Config). + +umerge3_invalid_case(Config) -> + do_proptest(prop_umerge3_invalid, Config). + +uniq_1_case(Config) -> + do_proptest(prop_uniq_1, Config). + +uniq_2_case(Config) -> + do_proptest(prop_uniq_2, Config). + +unzip_case(Config) -> + do_proptest(prop_unzip, Config). + +unzip3_case(Config) -> + do_proptest(prop_unzip3, Config). + +usort_1_case(Config) -> + do_proptest(prop_usort_1, Config). + +usort_2_case(Config) -> + do_proptest(prop_usort_2, Config). + +zip_2_case(Config) -> + do_proptest(prop_zip_2, Config). + +zip_3_case(Config) -> + do_proptest(prop_zip_3, Config). + +zip3_3_case(Config) -> + do_proptest(prop_zip3_3, Config). + +zip3_4_case(Config) -> + do_proptest(prop_zip3_4, Config). + +zipwith_3_case(Config) -> + do_proptest(prop_zipwith_3, Config). + +zipwith_4_case(Config) -> + do_proptest(prop_zipwith_4, Config). + +zipwith3_4_case(Config) -> + do_proptest(prop_zipwith3_4, Config). + +zipwith3_5_case(Config) -> + do_proptest(prop_zipwith3_5, Config). + diff --git a/lib/stdlib/test/maps_SUITE.erl b/lib/stdlib/test/maps_SUITE.erl index 8d479f8211..0f20398803 100644 --- a/lib/stdlib/test/maps_SUITE.erl +++ b/lib/stdlib/test/maps_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2022. All Rights Reserved. +%% Copyright Ericsson AB 1997-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -30,7 +30,9 @@ -export([t_update_with_3/1, t_update_with_4/1, t_get_3/1, t_filter_2/1, t_filtermap_2/1, t_fold_3/1,t_map_2/1,t_size_1/1, t_foreach_2/1, - t_iterator_1/1, t_put_opt/1, t_merge_opt/1, + t_iterator_1/1, t_iterator_2/1, + t_iterator_valid/1, + t_put_opt/1, t_merge_opt/1, t_with_2/1,t_without_2/1, t_intersect/1, t_intersect_with/1, t_merge_with/1, t_from_keys/1, @@ -57,7 +59,9 @@ all() -> [t_update_with_3,t_update_with_4, t_get_3,t_filter_2,t_filtermap_2, t_fold_3,t_map_2,t_size_1,t_foreach_2, - t_iterator_1,t_put_opt,t_merge_opt, + t_iterator_1,t_iterator_2, + t_iterator_valid, + t_put_opt,t_merge_opt, t_with_2,t_without_2, t_intersect, t_intersect_with, t_merge_with, t_from_keys, @@ -354,7 +358,8 @@ t_get_3(Config) when is_list(Config) -> DefaultValue = maps:get(key3, Map, DefaultValue), %% error case - ?badmap(a,get,[[a,b],a,def]) = (catch maps:get([a,b],id(a),def)), + {'EXIT', {{badmap,a}, _}} = (catch maps:get([a,b],id(a),def)), + ok. t_without_2(_Config) -> @@ -394,7 +399,8 @@ t_filter_2(Config) when is_list(Config) -> #{a := 2,c := 4} = maps:filter(Pred1,maps:iterator(M)), #{"b" := 2,"c" := 4} = maps:filter(Pred2,maps:iterator(M)), %% error case - ?badmap(a,filter,[_,a]) = (catch maps:filter(fun(_,_) -> ok end,id(a))), + ?badmap(a,filter,[_,a]) = (catch maps:filter(fun(_,_) -> true end,id(a))), + ?badmap({a,b,c},filter,[_,{a,b,c}]) = (catch maps:filter(fun(_,_) -> true end,id({a,b,c}))), ?badarg(filter,[<<>>,#{}]) = (catch maps:filter(id(<<>>),#{})), ok. @@ -408,7 +414,8 @@ t_filtermap_2(Config) when is_list(Config) -> false = maps:is_key(20, M1), true = M1 =:= M2, %% error case - ?badmap(a,filtermap,[_,a]) = (catch maps:filtermap(fun(_,_) -> ok end,id(a))), + ?badmap(a,filtermap,[_,a]) = (catch maps:filtermap(fun(_,_) -> true end,id(a))), + ?badmap({a,b,c},filtermap,[_,{a,b,c}]) = (catch maps:filtermap(fun(_,_) -> true end,id({a,b,c}))), ?badarg(filtermap,[<<>>,#{}]) = (catch maps:filtermap(id(<<>>),#{})), ok. @@ -424,6 +431,7 @@ t_fold_3(Config) when is_list(Config) -> %% error case ?badmap(a,fold,[_,0,a]) = (catch maps:fold(fun(_,_,_) -> ok end,0,id(a))), + ?badmap({a,b,c},fold,[_,0,{a,b,c}]) = (catch maps:fold(fun(_,_,_) -> ok end,0,id({a,b,c}))), ?badarg(fold,[<<>>,0,#{}]) = (catch maps:fold(id(<<>>),0,#{})), ok. @@ -438,6 +446,7 @@ t_map_2(Config) when is_list(Config) -> %% error case ?badmap(a,map,[_,a]) = (catch maps:map(fun(_,_) -> ok end, id(a))), + ?badmap({a,b,c},map,[_,{a,b,c}]) = (catch maps:map(fun(_,_) -> ok end, id({a,b,c}))), ?badarg(map,[<<>>,#{}]) = (catch maps:map(id(<<>>),#{})), ok. @@ -448,6 +457,7 @@ t_foreach_2(Config) when is_list(Config) -> ?badmap({},foreach,[_,{}]) = (catch maps:foreach(fun(_,_) -> ok end, id({}))), ?badmap(42,foreach,[_,42]) = (catch maps:foreach(fun(_,_) -> ok end, id(42))), ?badmap(<<>>,foreach,[_,<<>>]) = (catch maps:foreach(fun(_,_) -> ok end, id(<<>>))), + ?badmap({a,b,c},foreach,[_,{a,b,c}]) = (catch maps:foreach(fun(_,_) -> ok end, id({a,b,c}))), ?badarg(foreach,[<<>>,#{}]) = (catch maps:foreach(id(<<>>),#{})), F0 = fun() -> ok end, @@ -471,12 +481,15 @@ t_iterator_1(Config) when is_list(Config) -> KVList = lists:sort([{K1,V1},{K2,V2}]), KVList = lists:sort(maps:to_list(M0)), + KList = lists:sort([K1,K2]), + KList = lists:sort(maps:keys(M0)), %% Large map test Vs2 = lists:seq(1,200), M2 = maps:from_list([{{k,I},I}||I<-Vs2]), KVList2 = lists:sort(iter_kv(maps:iterator(M2))), + KVList2 = lists:sort(maps:to_list(maps:iterator(M2))), KVList2 = lists:sort(maps:to_list(M2)), %% Larger map test @@ -484,9 +497,101 @@ t_iterator_1(Config) when is_list(Config) -> Vs3 = lists:seq(1,10000), M3 = maps:from_list([{{k,I},I}||I<-Vs3]), KVList3 = lists:sort(iter_kv(maps:iterator(M3))), + KVList3 = lists:sort(maps:to_list(maps:iterator(M3))), KVList3 = lists:sort(maps:to_list(M3)), ok. +t_iterator_2(Config) when is_list(Config) -> + + AOrdCmpFun = fun(A, B) -> A =< B end, + ARevCmpFun = fun(A, B) -> B =< A end, + + %% Small map test + M0 = #{ a => 1, b => 2 }, + TOrdI0 = maps:iterator(M0, ordered), + {K1 = a, V1 = 1, TOrdI1} = maps:next(TOrdI0), + {K2 = b, V2 = 2, TOrdI2} = maps:next(TOrdI1), + none = maps:next(TOrdI2), + + TRevI0 = maps:iterator(M0, reversed), + {K2 = b, V2 = 2, TRevI1} = maps:next(TRevI0), + {K1 = a, V1 = 1, TRevI2} = maps:next(TRevI1), + none = maps:next(TRevI2), + + AOrdI0 = maps:iterator(M0, AOrdCmpFun), + {K1 = a, V1 = 1, AOrdI1} = maps:next(AOrdI0), + {K2 = b, V2 = 2, AOrdI2} = maps:next(AOrdI1), + none = maps:next(AOrdI2), + + ARevI0 = maps:iterator(M0, ARevCmpFun), + {K2 = b, V2 = 2, ARevI1} = maps:next(ARevI0), + {K1 = a, V1 = 1, ARevI2} = maps:next(ARevI1), + none = maps:next(ARevI2), + + OrdKVList = [{K1, V1}, {K2, V2}], + OrdKVList = maps:to_list(TOrdI0), + OrdKVList = maps:to_list(AOrdI0), + + RevKVList = [{K2, V2}, {K1, V1}], + RevKVList = maps:to_list(TRevI0), + RevKVList = maps:to_list(ARevI0), + + %% Large map test + + Vs2 = lists:seq(1, 200), + OrdKVList2 = [{{k, I}, I} || I <- Vs2], + M2 = maps:from_list(OrdKVList2), + ok = iterator_2_check_order(M2, ordered, reversed), + ok = iterator_2_check_order(M2, AOrdCmpFun, ARevCmpFun), + + %% Larger map test + + Vs3 = lists:seq(1, 10000), + OrdKVList3 = [{{k, I}, I} || I <- Vs3], + M3 = maps:from_list(OrdKVList3), + ok = iterator_2_check_order(M3, ordered, reversed), + ok = iterator_2_check_order(M3, AOrdCmpFun, ARevCmpFun), + + %% Float and integer keys + + M4 = #{-1.0 => a, 0.0 => b, -1 => c, 0 => d}, + OrdIter4 = maps:iterator(M4, ordered), + [{-1, c}, {0, d}, {-1.0, a}, {0.0, b}] = maps:to_list(OrdIter4), + ok = iterator_2_check_order(M4, ordered, reversed), + + ok. + +iterator_2_option_to_fun(ordered) -> + fun(A, B) -> erts_internal:cmp_term(A, B) =< 0 end; +iterator_2_option_to_fun(reversed) -> + fun(A, B) -> erts_internal:cmp_term(B, A) =< 0 end; +iterator_2_option_to_fun(F) when is_function(F, 2) -> + F. + +iterator_2_check_order(M, OrdOption, RevOption) -> + OrdCmpFun = iterator_2_option_to_fun(OrdOption), + RevCmpFun = iterator_2_option_to_fun(RevOption), + OrdKVCmpFun = fun({A, _}, {B, _}) -> OrdCmpFun(A, B) end, + RevKVCmpFun = fun({A, _}, {B, _}) -> RevCmpFun(A, B) end, + + OrdKVList = lists:sort(OrdKVCmpFun, maps:to_list(M)), + RevKVList = lists:sort(RevKVCmpFun, maps:to_list(M)), + RevKVList = lists:reverse(OrdKVList), + + Iter = maps:iterator(M, undefined), + OrdIter = maps:iterator(M, OrdOption), + RevIter = maps:iterator(M, RevOption), + + OrdKVList = lists:sort(OrdKVCmpFun, iter_kv(Iter)), + OrdKVList = lists:sort(OrdKVCmpFun, maps:to_list(Iter)), + OrdKVList = iter_kv(OrdIter), + OrdKVList = maps:to_list(OrdIter), + + RevKVList = iter_kv(RevIter), + RevKVList = maps:to_list(RevIter), + + ok. + iter_kv(I) -> case maps:next(I) of none -> @@ -495,6 +600,59 @@ iter_kv(I) -> [{K,V} | iter_kv(NI)] end. +t_iterator_valid(Config) when is_list(Config) -> + %% good iterators created via maps:iterator + true = maps:is_iterator_valid(maps:iterator(#{})), + true = maps:is_iterator_valid(maps:iterator(#{a => b})), + true = maps:is_iterator_valid(maps:iterator(#{a => b, c => d})), + true = maps:is_iterator_valid(maps:iterator(#{}, undefined)), + true = maps:is_iterator_valid(maps:iterator(#{a => b}, undefined)), + true = maps:is_iterator_valid(maps:iterator(#{a => b, c => d}, undefined)), + true = maps:is_iterator_valid(maps:iterator(#{}, ordered)), + true = maps:is_iterator_valid(maps:iterator(#{a => b}, ordered)), + true = maps:is_iterator_valid(maps:iterator(#{a => b, c => d}, ordered)), + true = maps:is_iterator_valid(maps:iterator(#{}, reversed)), + true = maps:is_iterator_valid(maps:iterator(#{a => b}, reversed)), + true = maps:is_iterator_valid(maps:iterator(#{a => b, c => d}, reversed)), + true = maps:is_iterator_valid(maps:iterator(#{}, fun erlang:'=<'/2)), + true = maps:is_iterator_valid(maps:iterator(#{a => b}, fun erlang:'=<'/2)), + true = maps:is_iterator_valid(maps:iterator(#{a => b, c => d}, fun erlang:'=<'/2)), + + %% good makeshift iterators + true = maps:is_iterator_valid(none), + true = maps:is_iterator_valid({a, b, none}), + true = maps:is_iterator_valid({a, b, {c, d, none}}), + true = maps:is_iterator_valid({a, b, {c, d, {e, f, none}}}), + true = maps:is_iterator_valid({a, b, maps:iterator(#{})}), + true = maps:is_iterator_valid({a, b, maps:iterator(#{c => d})}), + + %% not iterators + false = maps:is_iterator_valid(no_iter), + false = maps:is_iterator_valid(1), + false = maps:is_iterator_valid(1.0), + false = maps:is_iterator_valid([]), + false = maps:is_iterator_valid("foo"), + false = maps:is_iterator_valid(<<"foo">>), + false = maps:is_iterator_valid(fun() -> ok end), + false = maps:is_iterator_valid(self()), + false = maps:is_iterator_valid(make_ref()), + false = maps:is_iterator_valid(#{}), + false = maps:is_iterator_valid({}), + false = maps:is_iterator_valid({a}), + false = maps:is_iterator_valid({a, b}), + false = maps:is_iterator_valid({a, b, c, d}), + + %% bad makeshift iterators that only fail on later (ie, subsequent) calls to maps:next/1 + %% maps:next({a, b, c}) -> {a, b, c} + %% maps:next(c) -> badarg + false = maps:is_iterator_valid({a, b, c}), + %% maps:next({a, b, {c, d, e}}) -> {a, b, {c, d, e}} + %% maps:next({c, d, e}) -> {c, d, e} + %% maps:next(e) -> badarg + false = maps:is_iterator_valid({a, b, {c, d, e}}), + + ok. + t_put_opt(Config) when is_list(Config) -> Value = id(#{complex => map}), Small = id(#{a => Value}), @@ -784,15 +942,22 @@ t_groups_from_list(_Config) -> error_info(_Config) -> BadIterator = [-1|#{}], + BadIterator2 = {x, y, z}, GoodIterator = maps:iterator(#{}), + BadOrder = fun(_) -> true end, + GoodOrder = fun(A, B) -> A =< B end, - L = [{filter, [fun(_, _) -> true end, abc]}, + L = [ + {filter, [fun(_, _) -> true end, abc]}, {filter, [fun(_, _) -> true end, BadIterator]}, + {filter, [fun(_, _) -> true end, BadIterator2]}, {filter, [bad_fun, BadIterator],[{1,".*"},{2,".*"}]}, + {filter, [bad_fun, BadIterator2],[{1,".*"},{2,".*"}]}, {filter, [bad_fun, GoodIterator]}, {filtermap, [fun(_, _) -> true end, abc]}, {filtermap, [fun(_, _) -> true end, BadIterator]}, + {filtermap, [fun(_, _) -> true end, BadIterator2]}, {filtermap, [fun(_) -> true end, GoodIterator]}, {filtermap, [fun(_) -> ok end, #{}]}, @@ -800,11 +965,13 @@ error_info(_Config) -> {fold, [fun(_, _, _) -> true end, init, abc]}, {fold, [fun(_, _, _) -> true end, init, BadIterator]}, + {fold, [fun(_, _, _) -> true end, init, BadIterator2]}, {fold, [fun(_) -> true end, init, GoodIterator]}, {fold, [fun(_) -> ok end, init, #{}]}, {foreach, [fun(_, _) -> ok end, no_map]}, {foreach, [fun(_, _) -> ok end, BadIterator]}, + {foreach, [fun(_, _) -> ok end, BadIterator2]}, {foreach, [fun(_) -> ok end, GoodIterator]}, {foreach, [fun(_) -> ok end, #{}]}, @@ -834,14 +1001,26 @@ error_info(_Config) -> {intersect_with, [fun(_, _, _) -> ok end, x, y],[{2,".*"},{3,".*"}]}, {intersect_with, [fun(_, _) -> ok end, #{}, #{}]}, + {is_iterator_valid, [GoodIterator], [no_fail]}, + {is_iterator_valid, [BadIterator], [no_fail]}, + {is_iterator_valid, [BadIterator2], [no_fail]}, + {is_key,[key, no_map]}, {iterator,[{no,map}]}, + {iterator, [{no,map}, undefined], [{1, ".*"}]}, + {iterator, [{no,map}, ordered], [{1, ".*"}]}, + {iterator, [{no,map}, reversed], [{1, ".*"}]}, + {iterator, [{no,map}, GoodOrder], [{1, ".*"}]}, + {iterator, [#{a => b}, BadOrder], [{2, ".*"}]}, + {iterator, [{no,map}, BadOrder], [{1, ".*"}, {2, ".*"}]}, + {keys, [{no,map}]}, {map, [fun(_, _) -> true end, abc]}, {map, [fun(_, _) -> true end, BadIterator]}, + {map, [fun(_, _) -> true end, BadIterator2]}, {map, [fun(_) -> true end, GoodIterator]}, {map, [fun(_) -> ok end, #{}]}, @@ -853,6 +1032,7 @@ error_info(_Config) -> {merge_with, [a, b, c],[{1,".*"},{2,".*"},{3,".*"}]}, {next,[no_iterator]}, + {next,[BadIterator]}, {put, [key, value, {no,map}]}, @@ -863,6 +1043,8 @@ error_info(_Config) -> {take, [key, no_map]}, {to_list,[xyz]}, + {to_list,[BadIterator]}, + {to_list,[BadIterator2]}, {update,[key, value, no_map]}, diff --git a/lib/stdlib/test/math_SUITE.erl b/lib/stdlib/test/math_SUITE.erl index 5804c5bb46..5e92debd69 100644 --- a/lib/stdlib/test/math_SUITE.erl +++ b/lib/stdlib/test/math_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2021. All Rights Reserved. +%% Copyright Ericsson AB 2007-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -28,7 +28,7 @@ init_per_testcase/2, end_per_testcase/2]). %% Test cases --export([floor_ceil/1, error_info/1]). +-export([floor_ceil/1, error_info/1, constants/1]). suite() -> @@ -36,7 +36,7 @@ suite() -> {timetrap,{minutes,1}}]. all() -> - [floor_ceil, error_info]. + [floor_ceil, error_info, constants]. groups() -> []. @@ -60,6 +60,11 @@ init_per_testcase(_Case, Config) -> end_per_testcase(_Case, _Config) -> ok. +constants(_Config) -> + 3.1415926535897932 = math:pi(), + 6.2831853071795864 = math:tau(), + ok. + floor_ceil(_Config) -> MinusZero = 0.0/(-1.0), -43.0 = do_floor_ceil(-42.1), diff --git a/lib/stdlib/test/ms_transform_SUITE.erl b/lib/stdlib/test/ms_transform_SUITE.erl index c34c7e9e69..5e8e6076ae 100644 --- a/lib/stdlib/test/ms_transform_SUITE.erl +++ b/lib/stdlib/test/ms_transform_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2003-2021. All Rights Reserved. +%% Copyright Ericsson AB 2003-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -582,6 +582,8 @@ autoimported(Config) when is_list(Config) -> {element,2}, {hd,1}, {length,1}, + {max,2}, + {min,2}, {node,0}, {node,1}, {round,1}, @@ -857,6 +859,16 @@ action_function(Config) when is_list(Config) -> "silent(true), " "trace([send], [procs]), " "trace(Y, [procs], [send]) end)">>), + [{['$1','$2'], + [], + [{caller_line}, + {current_stacktrace}, + {current_stacktrace,3}]}] = + compile_and_run + (<<"dbg:fun2ms(fun([X,Y]) -> " + "caller_line()," + "current_stacktrace()," + "current_stacktrace(3) end)">>), ok. diff --git a/lib/stdlib/test/peer_SUITE.erl b/lib/stdlib/test/peer_SUITE.erl index 02bd16bd76..9b48f4f28e 100644 --- a/lib/stdlib/test/peer_SUITE.erl +++ b/lib/stdlib/test/peer_SUITE.erl @@ -13,6 +13,8 @@ suite/0, all/0, groups/0, + init_per_suite/1, + end_per_suite/1, init_per_group/2, end_per_group/2 ]). @@ -46,6 +48,9 @@ old_release/0, old_release/1, ssh/0, ssh/1, docker/0, docker/1, + post_process_args/0, post_process_args/1, + attached/0, attached/1, + attached_cntrl_channel_handler_crash/0, attached_cntrl_channel_handler_crash/1, cntrl_channel_handler_crash/0, cntrl_channel_handler_crash/1, cntrl_channel_handler_crash_old_release/0, cntrl_channel_handler_crash_old_release/1 ]). @@ -53,18 +58,29 @@ suite() -> [{timetrap, {minutes, 1}}]. +init_per_suite(Config) -> + %% Restrict number of schedulers so that we do not run out of + %% file descriptors during test + os:putenv("ERL_FLAGS","+S1 +SDio 1"), + Config. + +end_per_suite(_Config) -> + os:unsetenv("ERL_FLAGS"), + ok. + shutdown_alternatives() -> [shutdown_halt, shutdown_halt_timeout, shutdown_stop, shutdown_stop_timeout, shutdown_close]. alternative() -> [basic, peer_states, cast, detached, dyn_peer, stop_peer, - io_redirect, multi_node, duplicate_name, cntrl_channel_handler_crash, - cntrl_channel_handler_crash_old_release | shutdown_alternatives()]. + io_redirect, multi_node, duplicate_name, attached, attached_cntrl_channel_handler_crash, + cntrl_channel_handler_crash, cntrl_channel_handler_crash_old_release | shutdown_alternatives()]. groups() -> [ {dist, [parallel], [errors, dist, peer_down_crash, peer_down_continue, peer_down_boot, - dist_up_down, dist_localhost, cntrl_channel_handler_crash, + dist_up_down, dist_localhost, post_process_args, attached, + attached_cntrl_channel_handler_crash, cntrl_channel_handler_crash, cntrl_channel_handler_crash_old_release | shutdown_alternatives()]}, {dist_seq, [], [dist_io_redirect, %% Cannot be run in parallel in dist group peer_down_crash_tcp]}, @@ -80,7 +96,7 @@ all() -> init_per_group(remote, Config) -> %% check that SSH can connect to localhost, skip the test if not - try os:cmd("ssh localhost echo ok") of + try os:cmd("timeout 10s ssh localhost echo ok") of "ok\n" -> Config; _ -> {skip, "'ssh localhost echo ok' did not return ok"} catch @@ -487,6 +503,29 @@ duplicate_name(Config) when is_list(Config) -> ?assertException(exit, _, peer:start_link(#{connection => standard_io, name => ?FUNCTION_NAME})), peer:stop(Peer). +post_process_args() -> + [{doc, "Test that the post_process_args option works"}]. + +post_process_args(Config) when is_list(Config) -> + case {os:type(),os:find_executable("bash")} of + {{win32,_}, _Bash} -> + {skip,"Test does not work on windows"}; + {_, false} -> + {skip,"Test needs bash to run"}; + {_, Bash} -> + Erl = string:split(ct:get_progname()," ",all), + [throw({skip, "Needs bash to run"}) || Bash =:= false], + {ok, Peer, _Node} = + peer:start_link( + #{ name => ?CT_PEER_NAME(), + exec => {Bash,["-c"|Erl]}, + post_process_args => + fun(["-c"|Args]) -> + ["-c", lists:flatten(lists:join($\s, Args))] + end }), + peer:stop(Peer) + end. + %% ------------------------------------------------------------------- %% Compatibility: old releases old_release() -> @@ -580,6 +619,54 @@ docker(Config) when is_list(Config) -> peer:stop(Peer) end. +attached() -> + [{doc, "Test that it is possible to start a peer node using run_erl aka attached"}]. + +attached(Config) -> + attached_test(false, Config). + +attached_cntrl_channel_handler_crash() -> + [{doc, "Test that peer node is halted if peer control channel handler process crashes and peer node is attached"}]. + +attached_cntrl_channel_handler_crash(Config) -> + attached_test(true, Config). + +attached_test(CrashConnectionHandler, Config) -> + RunErl = os:find_executable("run_erl"), + [throw({skip, "Could not find run_erl"}) || RunErl =:= false], + Erl = string:split(ct:get_progname()," ",all), + RunErlDir = filename:join(proplists:get_value(priv_dir, Config),?FUNCTION_NAME), + filelib:ensure_path(RunErlDir), + Connection = proplists:get_value(connection, Config), + Conn = if Connection =:= undefined -> #{ name => ?CT_PEER_NAME() }; + true -> + case CrashConnectionHandler of + false -> + #{ connection => Connection }; + true -> + #{name => ?CT_PEER_NAME(), + connection => Connection} + end + end, + try peer:start( + Conn#{ + exec => {RunErl, Erl}, + detached => false, + post_process_args => + fun(Args) -> + [RunErlDir ++ "/", RunErlDir, + lists:flatten(lists:join(" ",[[$',A,$'] || A <- Args]))] + end + }) of + {ok, Peer, Node} when Connection =:= undefined; Connection =:= 0 -> + case CrashConnectionHandler of + false -> peer:stop(Peer); + true -> cntrl_channel_handler_crash_test(Node) + end + catch error:{detached,_} when Connection =:= standard_io -> + ok + end. + cntrl_channel_handler_crash() -> [{doc, "Test that peer node is halted if peer control channel handler process crashes"}]. diff --git a/lib/stdlib/test/property_test/base64_prop.erl b/lib/stdlib/test/property_test/base64_prop.erl index a2e3026dcc..f4d7f264f1 100644 --- a/lib/stdlib/test/property_test/base64_prop.erl +++ b/lib/stdlib/test/property_test/base64_prop.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2021-2022. All Rights Reserved. +%% Copyright Ericsson AB 2021-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -54,96 +54,200 @@ %%% Properties %%% %%%%%%%%%%%%%%%%%% -prop_encode() -> +prop_encode_1() -> ?FORALL( Str, oneof([list(byte()), binary()]), begin Enc = base64:encode(Str), Dec = base64:decode(Enc), - is_b64_binary(Enc) andalso str_equals(Str, Dec) + is_b64_binary(standard, Enc) andalso str_equals(Str, Dec) end ). -prop_encode_to_string() -> +prop_encode_2() -> + ?FORALL( + {Str, Mode}, + {oneof([list(byte()), binary()]), mode()}, + begin + Enc = base64:encode(Str, #{mode => Mode}), + Dec = base64:decode(Enc, #{mode => Mode}), + is_b64_binary(Mode, Enc) andalso str_equals(Str, Dec) + end + ). + +prop_encode_to_string_1() -> ?FORALL( Str, oneof([list(byte()), binary()]), begin Enc = base64:encode_to_string(Str), Dec = base64:decode_to_string(Enc), - is_b64_string(Enc) andalso str_equals(Str, Dec) + is_b64_string(standard, Enc) andalso str_equals(Str, Dec) + end + ). + +prop_encode_to_string_2() -> + ?FORALL( + {Str, Mode}, + {oneof([list(byte()), binary()]), mode()}, + begin + Enc = base64:encode_to_string(Str, #{mode => Mode}), + Dec = base64:decode_to_string(Enc, #{mode => Mode}), + is_b64_string(Mode, Enc) andalso str_equals(Str, Dec) end ). -prop_decode() -> +prop_decode_1() -> ?FORALL( {NormalizedB64, WspedB64}, - wsped_b64(), + wsped_b64(standard), begin Dec = base64:decode(WspedB64), Enc = base64:encode(Dec), - is_binary(Dec) andalso b64_equals(NormalizedB64, Enc) + is_binary(Dec) andalso b64_equals(standard, NormalizedB64, Enc) + end + ). + +prop_decode_2() -> + ?FORALL( + {{NormalizedB64, WspedB64}, Mode}, + ?LET( + Mode, + mode(), + {wsped_b64(Mode), Mode} + ), + begin + Dec = base64:decode(WspedB64, #{mode => Mode}), + Enc = base64:encode(Dec, #{mode => Mode}), + is_binary(Dec) andalso b64_equals(Mode, NormalizedB64, Enc) end ). -prop_decode_malformed() -> - common_decode_malformed(wsped_b64(), fun base64:decode/1). +prop_decode_1_malformed() -> + common_decode_malformed(fun wsped_b64/1, standard, fun(Data, _) -> base64:decode(Data) end). + +prop_decode_2_malformed() -> + common_decode_malformed(fun wsped_b64/1, mode(), fun base64:decode/2). -prop_decode_noisy() -> - common_decode_noisy(fun base64:decode/1). +prop_decode_1_noisy() -> + common_decode_noisy(standard, fun(Data, _) -> base64:decode(Data) end). -prop_decode_to_string() -> +prop_decode_2_noisy() -> + common_decode_noisy(mode(), fun base64:decode/2). + +prop_decode_to_string_1() -> ?FORALL( {NormalizedB64, WspedB64}, - wsped_b64(), + wsped_b64(standard), begin Dec = base64:decode_to_string(WspedB64), Enc = base64:encode(Dec), - is_bytelist(Dec) andalso b64_equals(NormalizedB64, Enc) + is_bytelist(Dec) andalso b64_equals(standard, NormalizedB64, Enc) + end + ). + +prop_decode_to_string_2() -> + ?FORALL( + {{NormalizedB64, WspedB64}, Mode}, + ?LET( + Mode, + mode(), + {wsped_b64(Mode), Mode} + ), + begin + Dec = base64:decode_to_string(WspedB64, #{mode => Mode}), + Enc = base64:encode(Dec, #{mode => Mode}), + is_bytelist(Dec) andalso b64_equals(Mode, NormalizedB64, Enc) end ). -prop_decode_to_string_malformed() -> - common_decode_malformed(wsped_b64(), fun base64:decode_to_string/1). +prop_decode_to_string_1_malformed() -> + common_decode_malformed(fun wsped_b64/1, standard, fun(Data, _) -> base64:decode_to_string(Data) end). + +prop_decode_to_string_2_malformed() -> + common_decode_malformed(fun wsped_b64/1, mode(), fun base64:decode_to_string/2). + +prop_decode_to_string_1_noisy() -> + common_decode_noisy(standard, fun(Data, _) -> base64:decode_to_string(Data) end). -prop_decode_to_string_noisy() -> - common_decode_noisy(fun base64:decode_to_string/1). +prop_decode_to_string_2_noisy() -> + common_decode_noisy(mode(), fun base64:decode_to_string/2). -prop_mime_decode() -> +prop_mime_decode_1() -> ?FORALL( {NormalizedB64, NoisyB64}, - noisy_b64(), + noisy_b64(standard), begin Dec = base64:mime_decode(NoisyB64), Enc = base64:encode(Dec), - is_binary(Dec) andalso b64_equals(NormalizedB64, Enc) + is_binary(Dec) andalso b64_equals(standard, NormalizedB64, Enc) end ). -prop_mime_decode_malformed() -> - common_decode_malformed(noisy_b64(), fun base64:mime_decode/1). +prop_mime_decode_2() -> + ?FORALL( + {{NormalizedB64, NoisyB64}, Mode}, + ?LET( + Mode, + mode(), + {wsped_b64(Mode), Mode} + ), + begin + Dec = base64:mime_decode(NoisyB64, #{mode => Mode}), + Enc = base64:encode(Dec, #{mode => Mode}), + is_binary(Dec) andalso b64_equals(Mode, NormalizedB64, Enc) + end + ). + +prop_mime_decode_1_malformed() -> + common_decode_malformed(fun noisy_b64/1, standard, fun(Data, _) -> base64:mime_decode(Data) end). -prop_mime_decode_to_string() -> +prop_mime_decode_2_malformed() -> + common_decode_malformed(fun noisy_b64/1, mode(), fun base64:mime_decode/2). + +prop_mime_decode_to_string_1() -> ?FORALL( {NormalizedB64, NoisyB64}, - noisy_b64(), + noisy_b64(standard), begin Dec = base64:mime_decode_to_string(NoisyB64), Enc = base64:encode(Dec), - is_bytelist(Dec) andalso b64_equals(NormalizedB64, Enc) + is_bytelist(Dec) andalso b64_equals(standard, NormalizedB64, Enc) end ). -prop_mime_decode_to_string_malformed() -> - common_decode_malformed(noisy_b64(), fun base64:mime_decode_to_string/1). +prop_mime_decode_to_string_2() -> + ?FORALL( + {{NormalizedB64, NoisyB64}, Mode}, + ?LET( + Mode, + mode(), + {wsped_b64(Mode), Mode} + ), + begin + Dec = base64:mime_decode_to_string(NoisyB64, #{mode => Mode}), + Enc = base64:encode(Dec, #{mode => Mode}), + is_bytelist(Dec) andalso b64_equals(Mode, NormalizedB64, Enc) + end + ). -common_decode_noisy(Fn) -> +prop_mime_decode_to_string_1_malformed() -> + common_decode_malformed(fun noisy_b64/1, standard, fun(Data, _) -> base64:mime_decode_to_string(Data) end). + +prop_mime_decode_to_string_2_malformed() -> + common_decode_malformed(fun noisy_b64/1, mode(), fun base64:mime_decode_to_string/2). + +common_decode_noisy(ModeGen, Fn) -> ?FORALL( - {_, NoisyB64}, - ?SUCHTHAT({NormalizedB64, NoisyB64}, noisy_b64(), NormalizedB64 =/= NoisyB64), + {{_, NoisyB64}, Mode}, + ?LET( + Mode, + ModeGen, + {?SUCHTHAT({NormalizedB64, NoisyB64}, noisy_b64(Mode), NormalizedB64 =/= NoisyB64), Mode} + ), try - Fn(NoisyB64) + Fn(NoisyB64, #{mode => Mode}) of _ -> false @@ -153,25 +257,30 @@ common_decode_noisy(Fn) -> end ). -common_decode_malformed(Gen, Fn) -> +common_decode_malformed(DataGen, ModeGen, Fn) -> ?FORALL( - MalformedB64, + {MalformedB64, Mode}, ?LET( - {{NormalizedB64, NoisyB64}, Malformings}, - { - Gen, - oneof( - [ - [b64_char()], - [b64_char(), b64_char()], - [b64_char(), b64_char(), b64_char()] - ] - ) - }, - {NormalizedB64, insert_noise(NoisyB64, Malformings)} + Mode, + ModeGen, + ?LET( + {{NormalizedB64, NoisyB64}, Malformings, InsertFn}, + { + DataGen(Mode), + oneof( + [ + [b64_char(Mode)], + [b64_char(Mode), b64_char(Mode)], + [b64_char(Mode), b64_char(Mode), b64_char(Mode)] + ] + ), + function1(boolean()) + }, + {{NormalizedB64, insert_noise(NoisyB64, Malformings, InsertFn)}, Mode} + ) ), try - Fn(MalformedB64) + Fn(MalformedB64, #{mode => Mode}) of _ -> false @@ -185,16 +294,20 @@ common_decode_malformed(Gen, Fn) -> %%% Generators %%% %%%%%%%%%%%%%%%%%% +%% Generate base64 encoding mode. +mode() -> + oneof([standard, urlsafe]). + %% Generate a single character from the base64 alphabet. -b64_char() -> - oneof(b64_chars()). +b64_char(Mode) -> + oneof(b64_chars(Mode)). %% Generate a string of characters from the base64 alphabet, %% including padding if needed. -b64_string() -> +b64_string(Mode) -> ?LET( {L, Filler}, - {list(b64_char()), b64_char()}, + {list(b64_char(Mode)), b64_char(Mode)}, case length(L) rem 4 of 0 -> L; 1 -> L ++ [Filler, $=, $=]; @@ -205,43 +318,43 @@ b64_string() -> %% Generate a binary of characters from the base64 alphabet, %% including padding if needed. -b64_binary() -> +b64_binary(Mode) -> ?LET( L, - b64_string(), + b64_string(Mode), list_to_binary(L) ). %% Generate a string or binary of characters from the %% base64 alphabet, including padding if needed. -b64() -> - oneof([b64_string(), b64_binary()]). +b64(Mode) -> + oneof([b64_string(Mode), b64_binary(Mode)]). %% Generate a string or binary of characters from the %% base64 alphabet, including padding if needed, with %% whitespaces inserted at random indexes. -wsped_b64() -> +wsped_b64(Mode) -> ?LET( - {B64, Wsps}, - {b64(), list(oneof([$\t, $\r, $\n, $\s]))}, - {B64, insert_noise(B64, Wsps)} + {B64, Wsps, InsertFn}, + {b64(Mode), list(oneof([$\t, $\r, $\n, $\s])), function1(boolean())}, + {B64, insert_noise(B64, Wsps, InsertFn)} ). %% Generate a single character outside of the base64 alphabet. %% As whitespaces are allowed but ignored in base64, this generator %% will produce no whitespaces, either. -non_b64_char() -> - oneof(lists:seq(16#00, 16#FF) -- b64_allowed_chars()). +non_b64_char(Mode) -> + oneof(lists:seq(16#00, 16#FF) -- b64_allowed_chars(Mode)). %% Generate a string or binary of characters from the %% base64 alphabet, including padding if needed, with %% whitespaces and non-base64 ("invalid") characters %% inserted at random indexes. -noisy_b64() -> +noisy_b64(Mode) -> ?LET( - {{B64, WspedB64}, Noise}, - {wsped_b64(), non_empty(list(non_b64_char()))}, - {B64, insert_noise(WspedB64, Noise)} + {{B64, WspedB64}, Noise, InsertFn}, + {wsped_b64(Mode), non_empty(list(non_b64_char(Mode))), function1(boolean())}, + {B64, insert_noise(WspedB64, Noise, InsertFn)} ). %%%%%%%%%%%%%%% @@ -252,81 +365,92 @@ noisy_b64() -> %% "=" is not included, as it is special in that it %% may only appear at the end of a base64 encoded string %% for padding. -b64_chars() -> +b64_chars_common() -> lists:seq($0, $9) ++ lists:seq($a, $z) ++ - lists:seq($A, $Z) ++ - [$+, $/]. + lists:seq($A, $Z). + +b64_chars(standard) -> + b64_chars_common() ++ [$+, $/]; +b64_chars(urlsafe) -> + b64_chars_common() ++ [$-, $_]. %% In addition to the above, the whitespace characters %% HTAB, CR, LF and SP are allowed to appear in a base64 %% encoded string and should be ignored. -b64_allowed_chars() -> - [$\t, $\r, $\n, $\s | b64_chars()]. +b64_allowed_chars(Mode) -> + [$\t, $\r, $\n, $\s | b64_chars(Mode)]. %% Insert the given list of noise characters at random %% places into the given base64 string. -insert_noise(B64, []) -> +insert_noise(B64, Noise, InsertFn) -> + insert_noise(B64, Noise, InsertFn, 0). + +insert_noise(B64, [], _, _) -> B64; -insert_noise([], Noise) -> +insert_noise([], Noise, _, _) -> Noise; -insert_noise(<<>>, Noise) -> +insert_noise(<<>>, Noise, _, _) -> list_to_binary(Noise); -insert_noise([B|Bs] = B64, [N|Ns] = Noise) -> - case rand:uniform(2) of - 1 -> - [B|insert_noise(Bs, Noise)]; - 2 -> - [N|insert_noise(B64, Ns)] +insert_noise([B|Bs] = B64, [N|Ns] = Noise, InsertFn, Idx) -> + case InsertFn(Idx) of + true -> + [B|insert_noise(Bs, Noise, InsertFn, Idx + 1)]; + false -> + [N|insert_noise(B64, Ns, InsertFn, Idx + 1)] end; -insert_noise(<<B, Bs/binary>> = B64, [N|Ns] = Noise) -> - case rand:uniform(2) of - 1 -> - <<B, (insert_noise(Bs, Noise))/binary>>; - 2 -> - <<N, (insert_noise(B64, Ns))/binary>> +insert_noise(<<B, Bs/binary>> = B64, [N|Ns] = Noise, InsertFn, Idx) -> + case InsertFn(Idx) of + true -> + <<B, (insert_noise(Bs, Noise, InsertFn, Idx + 1))/binary>>; + false -> + <<N, (insert_noise(B64, Ns, InsertFn, Idx + 1))/binary>> end. %% Check if the given character is in the base64 alphabet. %% This does not include the padding character "=". -is_b64_char($+) -> +is_b64_char(standard, $+) -> + true; +is_b64_char(standard, $/) -> + true; +is_b64_char(urlsafe, $-) -> true; -is_b64_char($/) -> +is_b64_char(urlsafe, $_) -> true; -is_b64_char(C) when C >= $0, C =< $9 -> +is_b64_char(_, C) when C >= $0, C =< $9 -> true; -is_b64_char(C) when C >= $A, C =< $Z -> +is_b64_char(_, C) when C >= $A, C =< $Z -> true; -is_b64_char(C) when C >= $a, C =< $z -> +is_b64_char(_, C) when C >= $a, C =< $z -> true; -is_b64_char(_) -> +is_b64_char(_, _) -> false. %% Check if the given argument is a base64 binary, %% ie that it consists of quadruplets of characters %% from the base64 alphabet, whereas the last quadruplet %% may be padded with one or two "="s -is_b64_binary(B) -> - is_b64_binary(B, 0). +is_b64_binary(Mode, B) -> + is_b64_binary(Mode, B, 0). -is_b64_binary(<<>>, N) -> +is_b64_binary(_, <<>>, N) -> N rem 4 =:= 0; -is_b64_binary(<<$=>>, N) -> +is_b64_binary(_, <<$=>>, N) -> N rem 4 =:= 3; -is_b64_binary(<<$=, $=>>, N) -> +is_b64_binary(_, <<$=, $=>>, N) -> N rem 4 =:= 2; -is_b64_binary(<<C, More/binary>>, N) -> - case is_b64_char(C) of +is_b64_binary(Mode, <<C, More/binary>>, N) -> + case is_b64_char(Mode, C) of true -> - is_b64_binary(More, N + 1); + is_b64_binary(Mode, More, N + 1); false -> false end. %% Check if the given argument is a base64 string %% (see is_b64_binary/1) -is_b64_string(S) -> - is_b64_binary(list_to_binary(S)). +is_b64_string(Mode, S) -> + is_b64_binary(Mode, list_to_binary(S)). %% Check if the argument is a list of bytes. is_bytelist(L) -> @@ -349,23 +473,23 @@ str_equals(Str1, Str2) when is_binary(Str1), is_binary(Str2) -> %% Assumes that the given arguments are in a normalized form, %% ie that they consist only of characters from the base64 %% alphabet and possible padding ("="). -b64_equals(L, B) when is_list(L) -> - b64_equals(list_to_binary(L), B); -b64_equals(B, L) when is_list(L) -> - b64_equals(B, list_to_binary(L)); -b64_equals(B1, B2) when is_binary(B1), is_binary(B2) -> - b64_equals1(B1, B2). - -b64_equals1(<<Eq:4/bytes>>, <<Eq:4/bytes>>) -> - is_b64_binary(Eq); -b64_equals1(<<Eq:4/bytes, More1/binary>>, <<Eq:4/bytes, More2/binary>>) -> - case lists:all(fun is_b64_char/1, binary_to_list(Eq)) of +b64_equals(Mode, L, B) when is_list(L) -> + b64_equals(Mode, list_to_binary(L), B); +b64_equals(Mode, B, L) when is_list(L) -> + b64_equals(Mode, B, list_to_binary(L)); +b64_equals(Mode, B1, B2) when is_binary(B1), is_binary(B2) -> + b64_equals1(Mode, B1, B2). + +b64_equals1(Mode, <<Eq:4/bytes>>, <<Eq:4/bytes>>) -> + is_b64_binary(Mode, Eq); +b64_equals1(Mode, <<Eq:4/bytes, More1/binary>>, <<Eq:4/bytes, More2/binary>>) -> + case lists:all(fun(C) -> is_b64_char(Mode, C) end, binary_to_list(Eq)) of true -> - b64_equals1(More1, More2); + b64_equals1(Mode, More1, More2); false -> false end; -b64_equals1(<<Eq, B1, $=, $=>>, <<Eq, B2, $=, $=>>) -> +b64_equals1(Mode, <<Eq, B1, $=, $=>>, <<Eq, B2, $=, $=>>) -> %% If the encoded string ends with "==", there exist multiple %% possibilities for the character preceding the "==" as only the %% 3rd and 4th bits of the encoded byte represented by that @@ -374,7 +498,7 @@ b64_equals1(<<Eq, B1, $=, $=>>, <<Eq, B2, $=, $=>>) -> %% For example, all of the encoded strings "QQ==", "QR==", ..., "QZ==" %% decode to the string "A", since all the bytes represented by Q to Z %% are the same in the significant 3rd and 4th bit. - case is_b64_char(Eq) of + case is_b64_char(Mode, Eq) of true -> Normalize = fun (C) when C >= $A, C =< $P -> $A; @@ -383,20 +507,21 @@ b64_equals1(<<Eq, B1, $=, $=>>, <<Eq, B2, $=, $=>>) -> (C) when C >= $g, C =< $v -> $g; (C) when C >= $w, C =< $z -> $w; (C) when C >= $0, C =< $9 -> $w; - ($+) -> $w; - ($/) -> $w + ($+) when Mode =:= standard -> $w; + ($-) when Mode =:= urlsafe -> $w; + ($/) when Mode =:= standard -> $w; + ($_) when Mode =:= urlsafe -> $w end, Normalize(B1) =:= Normalize(B2); false -> false end; -b64_equals1(<<Eq:2/bytes, B1, $=>>, <<Eq:2/bytes, B2, $=>>) -> +b64_equals1(Mode, <<Eq1, Eq2, B1, $=>>, <<Eq1, Eq2, B2, $=>>) -> %% Similar to the above, but with the encoded string ending with a %% single "=" the 3rd to 6th bits of the encoded byte are significant, %% such that, for example, all the encoded strings "QUE=" to "QUH=" %% decode to the same string "AA". - <<Eq1, Eq2>> = Eq, - case is_b64_char(Eq1) andalso is_b64_char(Eq2) of + case is_b64_char(Mode, Eq1) andalso is_b64_char(Mode, Eq2) of true -> Normalize = fun (C) when C >= $A, C =< $D -> $A; @@ -416,14 +541,16 @@ b64_equals1(<<Eq:2/bytes, B1, $=>>, <<Eq:2/bytes, B2, $=>>) -> (C) when C >= $0, C =< $3 -> $0; (C) when C >= $4, C =< $7 -> $4; (C) when C >= $8, C =< $9 -> $8; - ($+) -> $8; - ($/) -> $8 + ($+) when Mode =:= standard -> $8; + ($-) when Mode =:= urlsafe -> $8; + ($/) when Mode =:= standard -> $8; + ($_) when Mode =:= urlsafe -> $8 end, Normalize(B1) =:= Normalize(B2); false -> false end; -b64_equals1(<<>>, <<>>) -> +b64_equals1(_, <<>>, <<>>) -> true; -b64_equals1(_, _) -> +b64_equals1(_, _, _) -> false. diff --git a/lib/stdlib/test/property_test/binary_prop.erl b/lib/stdlib/test/property_test/binary_prop.erl new file mode 100644 index 0000000000..00efa77b8c --- /dev/null +++ b/lib/stdlib/test/property_test/binary_prop.erl @@ -0,0 +1,37 @@ +-module(binary_prop). + +-export([prop_hex_encode_2/0]). + +-compile([export_all, nowarn_export_all]). + +-include_lib("common_test/include/ct_property_test.hrl"). + +prop_hex_encode_2() -> + ?FORALL(Data, data(), + begin + UpperHex = binary:encode_hex(Data, uppercase), + LowerHex = binary:encode_hex(Data, lowercase), + binary:decode_hex(LowerHex) =:= Data andalso + binary:decode_hex(UpperHex) =:= Data andalso + check_hex_encoded(Data, UpperHex, LowerHex) + end). + +data() -> + ?SIZED(Size, resize(Size * 2, binary())). + + +%% @doc Credit to the <a href="https://github.com/erlang/otp/pull/6297#discussion_r1034771450">comment</a> of @Maria-12648430 +-spec check_hex_encoded(Data :: binary(), UpperHex :: binary(), LoweHex :: binary()) -> boolean(). +check_hex_encoded(<<I1:4, I2:4, Ins/binary>>, <<U1:8, U2:8, UCs/binary>>, <<L1:8, L2:8, LCs/binary>>) -> + check_hex_chars_match(I1, U1, L1) andalso + check_hex_chars_match(I2, U2, L2) andalso + check_hex_encoded(Ins, UCs, LCs); +check_hex_encoded(<<>>, <<>>, <<>>) -> + true; +check_hex_encoded(_, _, _) -> + false. + +check_hex_chars_match(X, U, L) when X < 10 -> + (U =:= $0 + X) andalso (L =:= $0 + X); +check_hex_chars_match(X, U, L) -> + (U =:= $A + X -10) andalso (L =:= $a + X -10). diff --git a/lib/stdlib/test/property_test/lists_prop.erl b/lib/stdlib/test/property_test/lists_prop.erl new file mode 100644 index 0000000000..68c087b76d --- /dev/null +++ b/lib/stdlib/test/property_test/lists_prop.erl @@ -0,0 +1,2146 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2021-2023. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(lists_prop). + +-compile([export_all, nowarn_export_all]). + +-proptest(eqc). +-proptest([triq, proper]). + +-ifndef(EQC). +-ifndef(PROPER). +-ifndef(TRIQ). +-define(EQC, true). +-endif. +-endif. +-endif. + +-ifdef(EQC). +-include_lib("eqc/include/eqc.hrl"). +-define(MOD_eqc,eqc). + +-else. +-ifdef(PROPER). +-include_lib("proper/include/proper.hrl"). +-define(MOD_eqc,proper). + +-else. +-ifdef(TRIQ). +-define(MOD_eqc,triq). +-include_lib("triq/include/triq.hrl"). + +-endif. +-endif. +-endif. + +-define(RANDOM_ATOMS, 1000). + +%%%%%%%%%%%%%%%%%% +%%% Properties %%% +%%%%%%%%%%%%%%%%%% + +%% all/2 +prop_all_true() -> + ?FORALL( + InList, + gen_list(), + lists:all(fun(_) -> true end, InList) + ). + +prop_all_false() -> + ?FORALL( + {InList, Elem}, + ?LET( + {F, R, E}, + {gen_list(), gen_list(), make_ref()}, + {F ++ [E|R], E} + ), + not lists:all(fun(T) -> T =/= Elem end, InList) + ). + +%% any/2 +prop_any_true() -> + ?FORALL( + {InList, Elem}, + ?LET( + {F, R, E}, + {gen_list(), gen_list(), make_ref()}, + {F ++ [E|R], E} + ), + lists:any(fun(T) -> T =:= Elem end, InList) + ). + +prop_any_false() -> + ?FORALL( + InList, + gen_list(), + not lists:any(fun(_) -> false end, InList) + ). + +%% append/1 +prop_append_1() -> + ?FORALL( + InLists, + list(gen_list()), + check_appended(InLists, lists:append(InLists)) + ). + +%% append/2 +prop_append_2() -> + ?FORALL( + {InList1, InList2}, + {gen_list(), gen_list()}, + lists:append(InList1, InList2) =:= InList1 ++ InList2 + ). + +%% concat/1 +prop_concat() -> + ?FORALL( + {InList, ExpString}, + gen_list_fold( + oneof([gen_atom(), number(), string()]), + fun + (A, Acc) when is_atom(A) -> Acc ++ atom_to_list(A); + (I, Acc) when is_integer(I) -> Acc ++ integer_to_list(I); + (F, Acc) when is_float(F) -> Acc ++ float_to_list(F); + (L, Acc) when is_list(L) -> Acc ++ L + end, + [] + ), + lists:concat(InList) =:= ExpString + ). + +%% delete/2 +prop_delete() -> + ?FORALL( + {InList, DelElem}, + ?LET( + {F, R, E}, + {gen_list(), gen_list(), gen_any()}, + {F ++ [E|R], E} + ), + begin + DeletedList = lists:delete(DelElem, InList), + length(DeletedList) =:= length(InList) - 1 andalso + check_deleted(DelElem, InList, DeletedList) + end + ). + +prop_delete_absent() -> + ?FORALL( + InList, + gen_list(), + lists:delete(make_ref(), InList) =:= InList + ). + +%% droplast/1 +prop_droplast() -> + ?FORALL( + InList, + gen_list(), + try + lists:droplast(InList) =:= lists:reverse(tl(lists:reverse(InList))) + catch + error:_ -> + InList =:= [] + end + ). + +%% dropwhile/2 +prop_dropwhile() -> + ?FORALL( + {Pred, InList, ExpList}, + ?LET( + Fn, + function1(boolean()), + ?LET( + {L, {_, DL}}, + gen_list_fold( + gen_any(), + fun(E, {Drop, Acc}) -> + case Drop andalso Fn(E) of + true -> {true, Acc}; + false -> {false, Acc ++ [E]} + end + end, + {true, []} + ), + {Fn, L, DL} + ) + ), + lists:dropwhile(Pred, InList) =:= ExpList + ). + +%% duplicate/2 +prop_duplicate() -> + ?FORALL( + {N, Term, ExpList}, + ?LET( + T, + gen_any(), + ?LET(L, list(T), {length(L), T, L}) + ), + lists:duplicate(N, Term) =:= ExpList + ). + +%% enumerate/1 +prop_enumerate_1() -> + ?FORALL( + {InList, ExpList}, + ?LET( + {L, {_, EL}}, + gen_list_fold( + gen_any(), + fun(T, {I, Acc}) -> + {I + 1, Acc ++ [{I, T}]} + end, + {1, []} + ), + {L, EL} + ), + lists:enumerate(InList) =:= ExpList + ). + +%% enumerate/2 +prop_enumerate_2() -> + ?FORALL( + {StartIndex, InList, ExpList}, + ?LET( + N, + integer(), + ?LET( + {L, {_, EL}}, + gen_list_fold( + gen_any(), + fun(T, {I, Acc}) -> + {I + 1, Acc ++ [{I, T}]} + end, + {N, []} + ), + {N, L, EL} + ) + ), + lists:enumerate(StartIndex, InList) =:= ExpList + ). + +%% enumerate/3 +prop_enumerate_3() -> + ?FORALL( + {StartIndex, Step, InList, ExpList}, + ?LET( + {N, S}, + {integer(), integer()}, + ?LET( + {L, {_, EL}}, + gen_list_fold( + gen_any(), + fun(T, {I, Acc}) -> + {I + S, Acc ++ [{I, T}]} + end, + {N, []} + ), + {N, S, L, EL} + ) + ), + lists:enumerate(StartIndex, Step, InList) =:= ExpList + ). + +%% filter/2 +prop_filter() -> + ?FORALL( + {Pred, InList, ExpList}, + ?LET( + P, + function1(boolean()), + ?LET( + {L, F}, + gen_list_fold( + gen_any(), + fun(T, Acc) -> + case P(T) of + true -> Acc ++ [T]; + false -> Acc + end + end, + [] + ), + {P, L, F} + ) + ), + lists:filter(Pred, InList) =:= ExpList + ). + +%% filtermap/2 +prop_filtermap() -> + ?FORALL( + {FilterMapFn, InList, ExpList}, + ?LET( + Fn, + function1(oneof([true, false, {true, gen_any()}])), + ?LET( + {L, FM}, + gen_list_fold( + gen_any(), + fun(T, Acc) -> + case Fn(T) of + false -> Acc; + true -> Acc ++ [T]; + {true, T1} -> Acc ++ [T1] + end + end, + [] + ), + {Fn, L, FM} + ) + ), + lists:filtermap(FilterMapFn, InList) =:= ExpList + ). + +%% flatlength/1 +prop_flatlength() -> + ?FORALL( + {DeepList, Len}, + gen_list_deepfold(fun(_, _, Cnt) -> Cnt + 1 end, 0), + lists:flatlength(DeepList) =:= Len + ). + +%% flatmap/2 +prop_flatmap() -> + ?FORALL( + {MapFn, InList, ExpList}, + ?LET( + Fn, + function1(gen_list()), + ?LET( + {L, FlatMapped}, + gen_list_fold( + gen_any(), + fun(T, Acc) -> + Acc ++ Fn(T) + end, + [] + ), + {Fn, L, FlatMapped} + ) + ), + lists:flatmap(MapFn, InList) =:= ExpList + ). + +%% flatten/1 +prop_flatten_1() -> + ?FORALL( + {DeepList, FlatList}, + gen_list_deepfold(fun(_, E, Acc) -> Acc ++ [E] end, []), + lists:flatten(DeepList) =:= FlatList + ). + +%% flatten/2 +prop_flatten_2() -> + ?FORALL( + {{DeepList, FlatList}, Tail}, + {gen_list_deepfold(fun(_, E, Acc) -> Acc ++ [E] end, []), gen_list()}, + lists:flatten(DeepList, Tail) =:= FlatList ++ Tail + ). + +%% foldl/3 +prop_foldl() -> + ?FORALL( + {FoldFn, InList, Acc0, Exp}, + ?LET( + {Fn, Acc0}, + {function2(gen_any()), gen_any()}, + ?LET( + {L, V}, + gen_list_fold(gen_any(), Fn, Acc0), + {Fn, L, Acc0, V} + ) + ), + lists:foldl(FoldFn, Acc0, InList) =:= Exp + ). + +%% foldr/3 +prop_foldr() -> + ?FORALL( + {FoldFn, InList, Acc0, Exp}, + ?LET( + {Fn, Acc0}, + {function2(gen_any()), gen_any()}, + ?LET( + {L, V}, + gen_list_fold(gen_any(), Fn, Acc0), + {Fn, lists:reverse(L), Acc0, V} + ) + ), + lists:foldr(FoldFn, Acc0, InList) =:= Exp + ). + +%% foreach/2 +prop_foreach() -> + ?FORALL( + InList, + gen_list(), + begin + Tag = make_ref(), + lists:foreach(fun(E) -> self() ! {Tag, E} end, InList), + [receive {Tag, T} -> T after 100 -> error(timeout) end || _ <- InList] =:= InList + end + ). + +%% join/2 +prop_join() -> + ?FORALL( + {Sep, InList}, + {gen_any(), gen_list()}, + check_joined(Sep, InList, lists:join(Sep, InList)) + ). + +%% keydelete/3 +prop_keydelete() -> + ?FORALL( + {Key, N, InList}, + ?LET( + {K, N}, + {gen_any(), range(1, 5)}, + ?LET( + {F, R, E}, + {gen_list(), gen_list(), gen_keytuple(K, N, N + 3)}, + {K, N, F ++ [E|R]} + ) + ), + begin + DeletedL = lists:keydelete(Key, N, InList), + length(DeletedL) =:= length(InList) - 1 andalso + check_keydeleted(Key, N, InList, DeletedL) + end + ). + +prop_keydelete_absent() -> + ?FORALL( + {N, InList}, + {pos_integer(), gen_list()}, + lists:keydelete(make_ref(), N, InList) =:= InList + ). + +%% keyfind/3 +prop_keyfind() -> + ?FORALL( + {Key, N, InList}, + ?LET( + {K, N}, + {gen_any(), range(1, 5)}, + ?LET( + {F, R, E}, + {gen_list(), gen_list(), gen_keytuple(K, N, N + 3)}, + {K, N, F ++ [E|R]} + ) + ), + begin + Found = lists:keyfind(Key, N, InList), + is_tuple(Found) andalso + tuple_size(Found) >= N andalso + element(N, Found) == Key + end + ). + +prop_keyfind_absent() -> + ?FORALL( + {N, InList}, + {pos_integer(), gen_list()}, + not lists:keyfind(make_ref(), N, InList) + ). + +%% keymap/3 +prop_keymap() -> + ?FORALL( + {MapFn, N, InList, ExpList}, + ?LET( + Fn, + function([gen_any()], gen_any()), + ?LET( + N, + range(1, 5), + ?LET( + {L, M}, + gen_list_fold( + gen_tuple(N, N + 3), + fun(T, Acc) -> + Acc ++ [setelement(N, T, Fn(element(N, T)))] + end, + [] + ), + {Fn, N, L, M} + ) + ) + ), + lists:keymap(MapFn, N, InList) =:= ExpList + ). + +%% keymember/3 +prop_keymember() -> + ?FORALL( + {Key, N, InList}, + ?LET( + {K, N}, + {gen_any(), range(1, 5)}, + ?LET( + {F, R, E}, + {gen_list(), gen_list(), gen_keytuple(K, N, N + 3)}, + {K, N, F ++ [E|R]} + ) + ), + lists:keymember(Key, N, InList) + ). + +prop_keymember_absent() -> + ?FORALL( + {N, InList}, + {pos_integer(), gen_list()}, + not lists:keymember(make_ref(), N, InList) + ). + +%% keymerge/3 +prop_keymerge() -> + ?FORALL( + {N, InList1, InList2}, + ?LET( + N, + range(1, 5), + ?LET( + {L1, L2}, + {list(gen_tuple(N, N+3)), list(gen_tuple(N, N+3))}, + {N, lists:sort(L1), lists:sort(L2)} + ) + ), + check_merged( + fun (E1, E2) -> element(N, E1) =< element(N, E2) end, + [InList1, InList2], + lists:keymerge(N, InList1, InList2) + ) + ). + +prop_keymerge_invalid() -> + ?FORALL( + {N, InList, X, Y}, + ?LET( + N, + range(1, 5), + ?LET( + {L, X, Y}, + {list(gen_tuple(N, N+3)), non_list(), non_list()}, + {N, L, X, Y} + ) + ), + expect_error(fun lists:keymerge/3, [N, InList, Y]) andalso + expect_error(fun lists:keymerge/3, [N, X, InList]) andalso + expect_error(fun lists:keymerge/3, [N, X, Y]) + ). + +%% keyreplace/4 +prop_keyreplace() -> + ?FORALL( + {Key, N, InList, Replacement}, + ?LET( + {K, N}, + {gen_any(), range(1, 5)}, + ?LET( + {F, R, E0, E1}, + {gen_list(), gen_list(), gen_keytuple(K, N, N + 3), gen_tuple()}, + {K, N, F ++ [E0|R], E1} + ) + ), + check_keyreplaced(Key, N, Replacement, InList, lists:keyreplace(Key, N, InList, Replacement)) + ). + +prop_keyreplace_absent() -> + ?FORALL( + {N, InList, Replacement}, + {pos_integer(), gen_list(), gen_tuple()}, + lists:keyreplace(make_ref(), N, InList, Replacement) =:= InList + ). + +%% keysearch/3 +prop_keysearch() -> + ?FORALL( + {Key, N, InList}, + ?LET( + {K, N}, + {gen_any(), range(1, 5)}, + ?LET( + {F, R, E}, + {gen_list(), gen_list(), gen_keytuple(K, N, N + 3)}, + {K, N, F ++ [E|R]} + ) + ), + begin + {value, Found} = lists:keysearch(Key, N, InList), + is_tuple(Found) andalso + tuple_size(Found) >= N andalso + element(N, Found) == Key + end + ). + +prop_keysearch_absent() -> + ?FORALL( + {N, InList}, + {pos_integer(), gen_list()}, + not lists:keysearch(make_ref(), N, InList) + ). + +%% keysort/2 +prop_keysort() -> + ?FORALL( + {N, InList}, + ?LET( + N, + range(1, 5), + {N, list(gen_tuple(N, N + 3))} + ), + begin + Sorted = lists:keysort(N, InList), + length(Sorted) =:= length(InList) andalso + check_sorted(fun(E1, E2) -> element(N, E1) =< element(N, E2) end, InList, Sorted) + end + ). + +%% keystore/4 +prop_keystore() -> + ?FORALL( + {Key, N, InList, ToStore}, + ?LET( + {K, N}, + {gen_any(), range(1, 5)}, + ?LET( + {F, R, E0, E1}, + {gen_list(), gen_list(), gen_keytuple(K, N, N + 3), gen_tuple()}, + {K, N, F ++ [E0|R], E1} + ) + ), + check_keyreplaced(Key, N, ToStore, InList, lists:keystore(Key, N, InList, ToStore)) + ). + +prop_keystore_absent() -> + ?FORALL( + {N, InList, ToStore}, + {pos_integer(), gen_list(), gen_tuple()}, + lists:keystore(make_ref(), N, InList, ToStore) =:= InList ++ [ToStore] + ). + +%% keytake/3 +prop_keytake() -> + ?FORALL( + {Key, N, InList, ExpList, ExpElem}, + ?LET( + {K, N}, + {make_ref(), range(1, 5)}, + ?LET( + {F, R, E}, + {gen_list(), gen_list(), gen_keytuple(K, N, N + 3)}, + {K, N, F ++ [E|R], F ++ R, E} + ) + ), + lists:keytake(Key, N, InList) =:= {value, ExpElem, ExpList} + ). + +prop_keytake_absent() -> + ?FORALL( + {N, InList}, + {pos_integer(), gen_list()}, + lists:keytake(make_ref(), N, InList) =:= false + ). + +%% last/1 +prop_last() -> + ?FORALL( + InList, + gen_list(), + try + lists:last(InList) =:= hd(lists:reverse(InList)) + catch + error:_ -> + InList =:= [] + end + ). + +%% map/2 +prop_map() -> + ?FORALL( + {MapFn, InList, ExpList}, + ?LET( + Fn, + function1(gen_any()), + ?LET( + {L, M}, + gen_list_fold( + gen_any(), + fun(T, Acc) -> + Acc ++ [Fn(T)] + end, + [] + ), + {Fn, L, M} + ) + ), + lists:map(MapFn, InList) =:= ExpList + ). + +%% mapfoldl/3 +prop_mapfoldl() -> + ?FORALL( + {MapFoldFn, InList, Acc0, Exp}, + ?LET( + {MapFn, FoldFn, Acc0}, + {function1(gen_any()), function2(gen_any()), gen_any()}, + ?LET( + {L, MV}, + gen_list_fold( + gen_any(), + fun(T, {AccM, AccF}) -> + {AccM ++ [MapFn(T)], FoldFn(T, AccF)} + end, + {[], Acc0} + ), + {fun(T, Acc) -> {MapFn(T), FoldFn(T, Acc)} end, L, Acc0, MV} + ) + ), + lists:mapfoldl(MapFoldFn, Acc0, InList) =:= Exp + ). + +%% mapfoldr/3 +prop_mapfoldr() -> + ?FORALL( + {MapFoldFn, InList, Acc0, Exp}, + ?LET( + {MapFn, FoldFn, Acc0}, + {function1(gen_any()), function2(gen_any()), gen_any()}, + ?LET( + {L, MV}, + gen_list_fold( + gen_any(), + fun(T, {AccM, AccF}) -> + {[MapFn(T)|AccM], FoldFn(T, AccF)} + end, + {[], Acc0} + ), + {fun(T, Acc) -> {MapFn(T), FoldFn(T, Acc)} end, lists:reverse(L), Acc0, MV} + ) + ), + lists:mapfoldr(MapFoldFn, Acc0, InList) =:= Exp + ). + +%% max/1 +prop_max() -> + ?FORALL( + {InList, ExpMax}, + gen_list_fold(gen_any(), fun erlang:max/2), + try + lists:max(InList) == ExpMax + catch + error:_ -> + InList =:= [] + end + ). + +%% member/2 +prop_member() -> + ?FORALL( + {InList, Member}, + ?LET( + {F, R, E}, + {gen_list(), gen_list(), gen_any()}, + {F ++ [E|R], E} + ), + lists:member(Member, InList) + ). + +prop_member_absent() -> + ?FORALL( + InList, + gen_list(), + not lists:member(make_ref(), InList) + ). + +%% merge/1 +prop_merge_1() -> + ?FORALL( + InLists, + list(?LET(L, gen_list(), lists:sort(L))), + check_merged(fun erlang:'=<'/2, InLists, lists:merge(InLists)) + ). + +prop_merge_1_invalid() -> + ?FORALL( + InLists, + ?LET( + {L1, X, L2}, + {list(oneof([non_list(), gen_list()])), non_list(), list(oneof([non_list(), gen_list()]))}, + L1 ++ [X|L2] + ), + expect_error(fun lists:merge/1, [InLists]) + ). + +%% merge/2 +prop_merge_2() -> + ?FORALL( + {InList1, InList2}, + ?LET( + {L1, L2}, + {gen_list(), gen_list()}, + {lists:sort(L1), lists:sort(L2)} + ), + check_merged(fun erlang:'=<'/2, [InList1, InList2], lists:merge(InList1, InList2)) + ). + +prop_merge_2_invalid() -> + ?FORALL( + {InList, X, Y}, + {gen_list(), non_list(), non_list()}, + expect_error(fun lists:merge/2, [InList, X]) andalso + expect_error(fun lists:merge/2, [X, InList]) andalso + expect_error(fun lists:merge/2, [X, Y]) + ). + +%% merge/3 +prop_merge_3() -> + ?FORALL( + {SortFn, InList1, InList2}, + ?LET( + {Fn, L1, L2}, + {gen_ordering_fun(), gen_list(), gen_list()}, + {Fn, lists:sort(Fn, L1), lists:sort(Fn, L2)} + ), + check_merged(SortFn, [InList1, InList2], lists:merge(SortFn, InList1, InList2)) + ). + +prop_merge_3_invalid() -> + ?FORALL( + {SortFn, InList, X, Y}, + {gen_ordering_fun(), gen_list(), non_list(), non_list()}, + expect_error(fun lists:merge/3, [SortFn, InList, Y]) andalso + expect_error(fun lists:merge/3, [SortFn, X, InList]) andalso + expect_error(fun lists:merge/3, [SortFn, X, Y]) + ). + +%% merge3/3 +prop_merge3() -> + ?FORALL( + {InList1, InList2, InList3}, + ?LET( + {L1, L2, L3}, + {gen_list(), gen_list(), gen_list()}, + {lists:sort(L1), lists:sort(L2), lists:sort(L3)} + ), + check_merged(fun erlang:'=<'/2, [InList1, InList2, InList3], lists:merge3(InList1, InList2, InList3)) + ). + +prop_merge3_invalid() -> + ?FORALL( + {InList, X, Y, Z}, + {gen_list(), non_list(), non_list(), non_list()}, + expect_error(fun lists:merge/3, [InList, InList, Z]) andalso + expect_error(fun lists:merge/3, [InList, Y, InList]) andalso + expect_error(fun lists:merge/3, [InList, Y, Z]) andalso + expect_error(fun lists:merge/3, [X, InList, Z]) andalso + expect_error(fun lists:merge/3, [X, Y, InList]) andalso + expect_error(fun lists:merge/3, [X, Y, Z]) + ). + +%% min/1 +prop_min() -> + ?FORALL( + {InList, ExpMin}, + gen_list_fold(gen_any(), fun erlang:min/2), + try + lists:min(InList) == ExpMin + catch + error:_ -> + InList =:= [] + end + ). + +%% nth/2 +prop_nth() -> + ?FORALL( + {InList, N, ExpElem}, + ?LET( + {F, R, E}, + {gen_list(), gen_list(), gen_any()}, + {F ++ [E|R], length(F)+1, E} + ), + lists:nth(N, InList) =:= ExpElem + ). + +prop_nth_outofrange() -> + ?FORALL( + {N, InList}, + ?LET( + {L, Offset}, + {gen_list(), pos_integer()}, + {length(L) + Offset, L} + ), + try + lists:nth(N, InList) + of + _ -> + false + catch + error:_ -> + true + end + ). + +%% nthtail/2 +prop_nthtail() -> + ?FORALL( + {InList, N, ExpTail}, + ?LET( + {F, R}, + {gen_list(), gen_list()}, + {F ++ R, length(F), R} + ), + lists:nthtail(N, InList) =:= ExpTail + ). + +prop_nthtail_outofrange() -> + ?FORALL( + {N, InList}, + ?LET( + {L, Offset}, + {gen_list(), pos_integer()}, + {length(L) + Offset, L} + ), + try + lists:nthtail(N, InList) + of + _ -> + false + catch + error:_ -> + true + end + ). + +%% partition/2 +prop_partition() -> + ?FORALL( + {Pred, InList}, + {function1(boolean()), gen_list()}, + begin + {Group1, Group2} = lists:partition(Pred, InList), + check_partitioned(Pred, InList, Group1, Group2) + end + ). + +%% prefix/2 +prop_prefix() -> + ?FORALL( + {InList, Prefix}, + ?LET( + {F, R}, + {gen_list(), gen_list()}, + {F ++ R, F} + ), + lists:prefix(Prefix, InList) andalso + not lists:prefix([make_ref()|Prefix], InList) andalso + not lists:prefix(Prefix ++ [make_ref()], InList) andalso + (not lists:prefix(Prefix, [make_ref()|InList]) orelse Prefix =:= []) + ). + +%% reverse/1 +prop_reverse_1() -> + ?FORALL( + InList, + gen_list(), + check_reversed(InList, lists:reverse(InList)) andalso + lists:reverse(lists:reverse(InList)) =:= InList + ). + +%% reverse/2 +prop_reverse_2() -> + ?FORALL( + {InList, InTail}, + {gen_list(), gen_list()}, + check_reversed(InList, lists:reverse(InList, InTail), InTail) + ). + +%% search/2 +prop_search() -> + ?FORALL( + {Pred, InList, ExpElem}, + ?LET( + {F, R, E}, + {gen_list(), gen_list(), make_ref()}, + {fun(T) -> T =:= E end, F ++ [E|R], E} + ), + lists:search(Pred, InList) =:= {value, ExpElem} + ). + +prop_search_absent() -> + ?FORALL( + InList, + gen_list(), + lists:search(fun(_) -> false end, InList) =:= false + ). + +%% seq/2 +prop_seq2() -> + ?FORALL( + {From, To}, + {integer(), integer()}, + try + lists:seq(From, To) + of + Seq -> + To >= From - 1 andalso + check_seq(Seq, From, To, 1) + catch + error:_ -> + To < From - 1 + end + ). + +%% seq/3 +prop_seq3() -> + ?FORALL( + {From, To, Step}, + {integer(), integer(), integer()}, + try + lists:seq(From, To, Step) + of + Seq when Step > 0 -> + To >= From - Step andalso + check_seq(Seq, From, To, Step); + Seq when Step < 0 -> + To =< From - Step andalso + check_seq(Seq, From, To, Step); + Seq when Step =:= 0 -> + From =:= To andalso + check_seq(Seq, From, To, Step) + catch + error:_ when Step > 0 -> + To < From - Step; + error:_ when Step < 0 -> + To > From - Step; + error:_ when Step =:= 0 -> + From =/= To + end + ). + +%% sort/1 +prop_sort_1() -> + ?FORALL( + InList, + gen_list(), + begin + Sorted = lists:sort(InList), + length(Sorted) =:= length(InList) andalso + check_sorted(InList, Sorted) + end + ). + +%% sort/2 +prop_sort_2() -> + ?FORALL( + {SortFn, InList}, + {gen_ordering_fun(), gen_list()}, + begin + Sorted = lists:sort(SortFn, InList), + length(Sorted) =:= length(InList) andalso + check_sorted(SortFn, InList, Sorted) + end + ). + +%% split/2 +prop_split() -> + ?FORALL( + {N, InList, ExpList1, ExpList2}, + ?LET( + {F, R}, + {gen_list(), gen_list()}, + {length(F), F ++ R, F, R} + ), + lists:split(N, InList) =:= {ExpList1, ExpList2} + ). + +prop_split_outofrange() -> + ?FORALL( + {N, InList}, + ?LET( + {L, Offset}, + {gen_list(), pos_integer()}, + {length(L) + Offset, L} + ), + try + lists:split(N, InList) + of + _ -> + false + catch + error:_ -> + true + end + ). + +%% splitwith/2 +prop_splitwith() -> + ?FORALL( + {Pred, InList}, + {function1(boolean()), gen_list()}, + begin + {Part1, Part2} = lists:splitwith(Pred, InList), + check_splitwithed(Pred, InList, Part1, Part2) + end + ). + +%% sublist/2 +prop_sublist_2() -> + ?FORALL( + {Len, InList, ExpList}, + ?LET( + {F, R}, + {gen_list(), gen_list()}, + {length(F), F ++ R, F} + ), + lists:sublist(InList, Len) =:= ExpList + ). + +%% sublist/3 +prop_sublist_3() -> + ?FORALL( + {Start, Len, InList, ExpList}, + ?LET( + {F, M, R}, + {gen_list(), gen_list(), gen_list()}, + {length(F)+1, length(M), F ++ M ++ R, M} + ), + lists:sublist(InList, Start, Len) =:= ExpList + ). + +%% subtract/2 +prop_subtract() -> + ?FORALL( + {InList, SubtractList}, + ?LET( + {L, B, S}, + {gen_list(), gen_list(), gen_list()}, + {L ++ B, S ++ B} + ), + lists:subtract(InList, SubtractList) =:= InList -- SubtractList + ). + +%% suffix/2 +prop_suffix() -> + ?FORALL( + {InList, Suffix}, + ?LET( + {F, R}, + {gen_list(), gen_list()}, + {F ++ R, R} + ), + lists:suffix(Suffix, InList) andalso + not lists:suffix([make_ref()|Suffix], InList) andalso + not lists:suffix(Suffix ++ [make_ref()], InList) andalso + (not lists:suffix(Suffix, InList ++ [make_ref()]) orelse Suffix =:= []) + ). + +%% sum/1 +prop_sum() -> + ?FORALL( + {InList, ExpSum}, + gen_list_fold(number(), fun erlang:'+'/2, 0), + lists:sum(InList) =:= ExpSum + ). + +%% takewhile/2 +prop_takewhile() -> + ?FORALL( + {Pred, InList, ExpList}, + ?LET( + Fn, + function1(boolean()), + ?LET( + {L, {_, TL}}, + gen_list_fold( + gen_any(), + fun(E, {Take, Acc}) -> + case Take andalso Fn(E) of + true -> {true, Acc ++ [E]}; + false -> {false, Acc} + end + end, + {true, []} + ), + {Fn, L, TL} + ) + ), + lists:takewhile(Pred, InList) =:= ExpList + ). + +%% ukeymerge/3 +prop_ukeymerge() -> + ?FORALL( + {N, InList1, InList2}, + ?LET( + N, + range(1, 5), + ?LET( + {L1, L2}, + {list(gen_tuple(N, N+3)), list(gen_tuple(N, N+3))}, + {N, lists:ukeysort(N, L1), lists:ukeysort(N, L2)} + ) + ), + check_umerged( + fun(E1, E2) -> element(N, E1) =< element(N, E2) end, + [InList1, InList2], + lists:ukeymerge(N, InList1, InList2) + ) + ). + +prop_ukeymerge_invalid() -> + ?FORALL( + {N, InList, X, Y}, + ?LET( + N, + range(1, 5), + ?LET( + {L, X, Y}, + {list(gen_tuple(N, N+3)), non_list(), non_list()}, + {N, L, X, Y} + ) + ), + expect_error(fun lists:ukeymerge/3, [N, InList, Y]) andalso + expect_error(fun lists:ukeymerge/3, [N, X, InList]) andalso + expect_error(fun lists:ukeymerge/3, [N, X, Y]) + ). + +%% ukeysort/2 +prop_ukeysort() -> + ?FORALL( + {N, InList}, + ?LET( + N, + range(1, 5), + {N, list(gen_tuple(N, N + 3))} + ), + begin + Sorted = lists:ukeysort(N, InList), + length(Sorted) =< length(InList) andalso + check_usorted(fun(E1, E2) -> element(N, E1) =< element(N, E2) end, InList, Sorted) + end + ). + +%% umerge/1 +prop_umerge_1() -> + ?FORALL( + InLists, + list(?LET(L, gen_list(), lists:usort(L))), + check_umerged(InLists, lists:umerge(InLists)) + ). + +prop_umerge_1_invalid() -> + ?FORALL( + InList, + ?LET( + {L1, X, L2}, + {list(oneof([non_list(), gen_list()])), non_list(), list(oneof([non_list(), gen_list()]))}, + L1 ++ [X|L2] + ), + expect_error(fun lists:umerge/1, [InList]) + ). + +%% umerge/2 +prop_umerge_2() -> + ?FORALL( + {InList1, InList2}, + ?LET( + {L1, L2}, + {gen_list(), gen_list()}, + {lists:usort(L1), lists:usort(L2)} + ), + check_umerged([InList1, InList2], lists:umerge(InList1, InList2)) + ). + +prop_umerge_2_invalid() -> + ?FORALL( + {InList, X, Y}, + {gen_list(), non_list(), non_list()}, + expect_error(fun lists:umerge/2, [InList, Y]) andalso + expect_error(fun lists:umerge/2, [X, InList]) andalso + expect_error(fun lists:umerge/2, [X, Y]) + ). + +%% umerge/3 +prop_umerge_3() -> + ?FORALL( + {SortFn, InList1, InList2}, + ?LET( + {Fn, L1, L2}, + {gen_ordering_fun(), gen_list(), gen_list()}, + {Fn, lists:usort(Fn, L1), lists:usort(Fn, L2)} + ), + check_umerged(SortFn, [InList1, InList2], lists:umerge(SortFn, InList1, InList2)) + ). + +prop_umerge_3_invalid() -> + ?FORALL( + {SortFn, InList, X, Y}, + {gen_ordering_fun(), gen_list(), non_list(), non_list()}, + expect_error(fun lists:umerge/3, [SortFn, InList, Y]) andalso + expect_error(fun lists:umerge/3, [SortFn, X, InList]) andalso + expect_error(fun lists:umerge/3, [SortFn, X, Y]) + ). + +%% umerge3/3 +prop_umerge3() -> + ?FORALL( + {InList1, InList2, InList3}, + ?LET( + {L1, L2, L3}, + {gen_list(), gen_list(), gen_list()}, + {lists:usort(L1), lists:usort(L2), lists:usort(L3)} + ), + check_umerged([InList1, InList2, InList3], lists:umerge3(InList1, InList2, InList3)) + ). + +prop_umerge3_invalid() -> + ?FORALL( + {InList, X, Y, Z}, + {gen_list(), non_list(), non_list(), non_list()}, + expect_error(fun lists:umerge3/3, [InList, InList, Z]) andalso + expect_error(fun lists:umerge3/3, [InList, Y, InList]) andalso + expect_error(fun lists:umerge3/3, [InList, Y, Z]) andalso + expect_error(fun lists:umerge3/3, [X, InList, InList]) andalso + expect_error(fun lists:umerge3/3, [X, InList, Z]) andalso + expect_error(fun lists:umerge3/3, [X, Y, InList]) andalso + expect_error(fun lists:umerge3/3, [X, Y, Z]) + ). + +%% uniq/1 +prop_uniq_1() -> + ?FORALL( + InList, + ?LET( + {L, M}, + {gen_list(), gen_list()}, + ?LET( + S, + vector(length(L) + 2 * length(M), integer()), + [E || {_, E} <- lists:sort(lists:zip(S, L ++ M ++ M))] + ) + ), + check_uniqed(InList, lists:uniq(InList)) + ). + +%% uniq/2 +prop_uniq_2() -> + ?FORALL( + {UniqFn, InList}, + {function1(oneof([a, b, c])), gen_list()}, + check_uniqed(UniqFn, InList, lists:uniq(UniqFn, InList)) + ). + +%% unzip/1 +prop_unzip() -> + ?FORALL( + {InList, {ExpList1, ExpList2}}, + gen_list_fold( + {gen_any(), gen_any()}, + fun({T1, T2}, {L1, L2}) -> + {L1 ++ [T1], L2 ++ [T2]} + end, + {[], []} + ), + lists:unzip(InList) =:= {ExpList1, ExpList2} + ). + +%% unzip3/1 +prop_unzip3() -> + ?FORALL( + {InList, {ExpList1, ExpList2, ExpList3}}, + gen_list_fold( + {gen_any(), gen_any(), gen_any()}, + fun({T1, T2, T3}, {L1, L2, L3}) -> + {L1 ++ [T1], L2 ++ [T2], L3 ++ [T3]} + end, + {[], [], []} + ), + lists:unzip3(InList) =:= {ExpList1, ExpList2, ExpList3} + ). + +%% usort/1 +prop_usort_1() -> + ?FORALL( + InList, + gen_list(), + begin + Sorted = lists:usort(InList), + length(Sorted) =< length(InList) andalso + check_usorted(InList, Sorted) + end + ). + +%% usort/2 +prop_usort_2() -> + ?FORALL( + {SortFn, InList}, + {gen_ordering_fun(), gen_list()}, + begin + Sorted = lists:usort(SortFn, InList), + length(Sorted) =< length(InList) andalso + check_usorted(SortFn, InList, Sorted) + end + ). + +%% zip/2 +prop_zip_2() -> + ?FORALL( + {ExpList, {InList1, InList2}}, + gen_list_fold( + {gen_any(), gen_any()}, + fun({T1, T2}, {L1, L2}) -> + {L1 ++ [T1], L2 ++ [T2]} + end, + {[], []} + ), + lists:zip(InList1, InList2) =:= ExpList + ). + +%% zip/3 +prop_zip_3() -> + ?FORALL( + {{ExpList, {InList1, InList2}}, ExtraList}, + { + gen_list_fold( + {gen_any(), gen_any()}, + fun({T1, T2}, {L1, L2}) -> + {L1 ++ [T1], L2 ++ [T2]} + end, + {[], []} + ), + non_empty(gen_list()) + }, + begin + Tag = make_ref(), + + Res1 = ExpList =:= lists:zip(InList1, InList2, fail) andalso + ExpList =:= lists:zip(InList1, InList2, trim) andalso + ExpList =:= lists:zip(InList1, InList2, {pad, {Tag, Tag}}), + + Res2 = try lists:zip(InList1, InList2 ++ ExtraList, fail) of _ -> false catch error:_ -> true end andalso + try lists:zip(InList1 ++ ExtraList, InList2, fail) of _ -> false catch error:_ -> true end, + + Res3 = ExpList =:= lists:zip(InList1, InList2 ++ ExtraList, trim) andalso + ExpList =:= lists:zip(InList1 ++ ExtraList, InList2, trim), + + Padded1 = lists:zip(InList1, InList2 ++ ExtraList, {pad, {Tag, Tag}}), + Padded2 = lists:zip(InList1 ++ ExtraList, InList2, {pad, {Tag, Tag}}), + Res4 = Padded1 =:= ExpList ++ [{Tag, X} || X <- ExtraList] andalso + Padded2 =:= ExpList ++ [{X, Tag} || X <- ExtraList], + + Res1 andalso Res2 andalso Res3 andalso Res4 + end + ). + +%% zip3/3 +prop_zip3_3() -> + ?FORALL( + {ExpList, {InList1, InList2, InList3}}, + gen_list_fold( + {gen_any(), gen_any(), gen_any()}, + fun({T1, T2, T3}, {L1, L2, L3}) -> + {L1 ++ [T1], L2 ++ [T2], L3 ++ [T3]} + end, + {[], [], []} + ), + lists:zip3(InList1, InList2, InList3) =:= ExpList + ). + +%% zip3/4 +prop_zip3_4() -> + ?FORALL( + {{ExpList, {InList1, InList2, InList3}}, ExtraList}, + { + gen_list_fold( + {gen_any(), gen_any(), gen_any()}, + fun({T1, T2, T3}, {L1, L2, L3}) -> + {L1 ++ [T1], L2 ++ [T2], L3 ++ [T3]} + end, + {[], [], []} + ), + non_empty(gen_list()) + }, + begin + Tag = make_ref(), + + Res1 = ExpList =:= lists:zip3(InList1, InList2, InList3, fail) andalso + ExpList =:= lists:zip3(InList1, InList2, InList3, trim) andalso + ExpList =:= lists:zip3(InList1, InList2, InList3, {pad, {Tag, Tag, Tag}}), + + Res2 = try lists:zip3(InList1, InList2, InList3 ++ ExtraList, fail) of _ -> false catch error:_ -> true end andalso + try lists:zip3(InList1, InList2 ++ ExtraList, InList3, fail) of _ -> false catch error:_ -> true end andalso + try lists:zip3(InList1, InList2 ++ ExtraList, InList3 ++ ExtraList, fail) of _ -> false catch error:_ -> true end andalso + try lists:zip3(InList1 ++ ExtraList, InList2, InList3, fail) of _ -> false catch error:_ -> true end andalso + try lists:zip3(InList1 ++ ExtraList, InList2, InList3 ++ ExtraList, fail) of _ -> false catch error:_ -> true end andalso + try lists:zip3(InList1 ++ ExtraList, InList2 ++ ExtraList, InList3, fail) of _ -> false catch error:_ -> true end, + + Res3 = ExpList =:= lists:zip3(InList1, InList2, InList3 ++ ExtraList, trim) andalso + ExpList =:= lists:zip3(InList1, InList2 ++ ExtraList, InList3, trim) andalso + ExpList =:= lists:zip3(InList1, InList2 ++ ExtraList, InList3 ++ ExtraList, trim) andalso + ExpList =:= lists:zip3(InList1 ++ ExtraList, InList2, InList3, trim) andalso + ExpList =:= lists:zip3(InList1 ++ ExtraList, InList2, InList3 ++ ExtraList, trim) andalso + ExpList =:= lists:zip3(InList1 ++ ExtraList, InList2 ++ ExtraList, InList3, trim), + + Padded1 = lists:zip3(InList1, InList2, InList3 ++ ExtraList, {pad, {Tag, Tag, Tag}}), + Padded2 = lists:zip3(InList1, InList2 ++ ExtraList, InList3, {pad, {Tag, Tag, Tag}}), + Padded3 = lists:zip3(InList1, InList2 ++ ExtraList, InList3 ++ ExtraList, {pad, {Tag, Tag, Tag}}), + Padded4 = lists:zip3(InList1 ++ ExtraList, InList2, InList3, {pad, {Tag, Tag, Tag}}), + Padded5 = lists:zip3(InList1 ++ ExtraList, InList2, InList3 ++ ExtraList, {pad, {Tag, Tag, Tag}}), + Padded6 = lists:zip3(InList1 ++ ExtraList, InList2 ++ ExtraList, InList3, {pad, {Tag, Tag, Tag}}), + Res4 = Padded1 =:= ExpList ++ [{Tag, Tag, X} || X <- ExtraList] andalso + Padded2 =:= ExpList ++ [{Tag, X, Tag} || X <- ExtraList] andalso + Padded3 =:= ExpList ++ [{Tag, X, X} || X <- ExtraList] andalso + Padded4 =:= ExpList ++ [{X, Tag, Tag} || X <- ExtraList] andalso + Padded5 =:= ExpList ++ [{X, Tag, X} || X <- ExtraList] andalso + Padded6 =:= ExpList ++ [{X, X, Tag} || X <- ExtraList], + + Res1 andalso Res2 andalso Res3 andalso Res4 + end + ). + +%% zipwith/3 +prop_zipwith_3() -> + ?FORALL( + {ZipFn, InList1, InList2, ExpList}, + ?LET( + Fn, + function2(gen_any()), + ?LET( + {_, {L1, L2, Z}}, + gen_list_fold( + {gen_any(), gen_any()}, + fun({T1, T2}, {L1, L2, Z}) -> + {L1 ++ [T1], L2 ++ [T2], Z ++ [Fn(T1, T2)]} + end, + {[], [], []} + ), + {Fn, L1, L2, Z} + ) + ), + lists:zipwith(ZipFn, InList1, InList2) =:= ExpList + ). + +%% zipwith/4 +prop_zipwith_4() -> + ?FORALL( + {ZipFn, InList1, InList2, ExpList, ExtraList}, + ?LET( + {Extra, Fn}, + {non_empty(gen_list()), function2(gen_any())}, + ?LET( + {_, {L1, L2, Z}}, + gen_list_fold( + {gen_any(), gen_any()}, + fun({T1, T2}, {L1, L2, Z}) -> + {L1 ++ [T1], L2 ++ [T2], Z ++ [Fn(T1, T2)]} + end, + {[], [], []} + ), + {Fn, L1, L2, Z, Extra} + ) + ), + begin + Tag = make_ref(), + + Res1 = ExpList =:= lists:zipwith(ZipFn, InList1, InList2, fail) andalso + ExpList =:= lists:zipwith(ZipFn, InList1, InList2, trim) andalso + ExpList =:= lists:zipwith(ZipFn, InList1, InList2, {pad, {Tag, Tag}}), + + Res2 = try lists:zipwith(ZipFn, InList1, InList2 ++ ExtraList, fail) of _ -> false catch error:_ -> true end andalso + try lists:zipwith(ZipFn, InList1 ++ ExtraList, InList2, fail) of _ -> false catch error:_ -> true end, + + Res3 = ExpList =:= lists:zipwith(ZipFn, InList1, InList2 ++ ExtraList, trim) andalso + ExpList =:= lists:zipwith(ZipFn, InList1 ++ ExtraList, InList2, trim), + + Padded1 = lists:zipwith(ZipFn, InList1, InList2 ++ ExtraList, {pad, {Tag, Tag}}), + Padded2 = lists:zipwith(ZipFn, InList1 ++ ExtraList, InList2, {pad, {Tag, Tag}}), + Res4 = Padded1 =:= ExpList ++ [ZipFn(Tag, X) || X <- ExtraList] andalso + Padded2 =:= ExpList ++ [ZipFn(X, Tag) || X <- ExtraList], + + Res1 andalso Res2 andalso Res3 andalso Res4 + end + ). + +%% zipwith3/4 +prop_zipwith3_4() -> + ?FORALL( + {ZipFn, InList1, InList2, InList3, ExpList}, + ?LET( + Fn, + function3(gen_any()), + ?LET( + {_, {L1, L2, L3, Z}}, + gen_list_fold( + {gen_any(), gen_any(), gen_any()}, + fun({T1, T2, T3}, {L1, L2, L3, Z}) -> + {L1 ++ [T1], L2 ++ [T2], L3 ++ [T3], Z ++ [Fn(T1, T2, T3)]} + end, + {[], [], [], []} + ), + {Fn, L1, L2, L3, Z} + ) + ), + lists:zipwith3(ZipFn, InList1, InList2, InList3) =:= ExpList + ). + +%% zipwith3/5 +prop_zipwith3_5() -> + ?FORALL( + {ZipFn, InList1, InList2, InList3, ExpList, ExtraList}, + ?LET( + {Extra, Fn}, + {non_empty(gen_list()), function3(gen_any())}, + ?LET( + {_, {L1, L2, L3, Z}}, + gen_list_fold( + {gen_any(), gen_any(), gen_any()}, + fun({T1, T2, T3}, {L1, L2, L3, Z}) -> + {L1 ++ [T1], L2 ++ [T2], L3 ++ [T3], Z ++ [Fn(T1, T2, T3)]} + end, + {[], [], [], []} + ), + {Fn, L1, L2, L3, Z, Extra} + ) + ), + begin + Tag = make_ref(), + + Res1 = ExpList =:= lists:zipwith3(ZipFn, InList1, InList2, InList3, fail) andalso + ExpList =:= lists:zipwith3(ZipFn, InList1, InList2, InList3, trim) andalso + ExpList =:= lists:zipwith3(ZipFn, InList1, InList2, InList3, {pad, {Tag, Tag, Tag}}), + + Res2 = try lists:zipwith3(ZipFn, InList1, InList2, InList3 ++ ExtraList, fail) of _ -> false catch error:_ -> true end andalso + try lists:zipwith3(ZipFn, InList1, InList2 ++ ExtraList, InList3, fail) of _ -> false catch error:_ -> true end andalso + try lists:zipwith3(ZipFn, InList1, InList2 ++ ExtraList, InList3 ++ ExtraList, fail) of _ -> false catch error:_ -> true end andalso + try lists:zipwith3(ZipFn, InList1 ++ ExtraList, InList2, InList3, fail) of _ -> false catch error:_ -> true end andalso + try lists:zipwith3(ZipFn, InList1 ++ ExtraList, InList2, InList3 ++ ExtraList, fail) of _ -> false catch error:_ -> true end andalso + try lists:zipwith3(ZipFn, InList1 ++ ExtraList, InList2 ++ ExtraList, InList3, fail) of _ -> false catch error:_ -> true end, + + Res3 = ExpList =:= lists:zipwith3(ZipFn, InList1, InList2, InList3 ++ ExtraList, trim) andalso + ExpList =:= lists:zipwith3(ZipFn, InList1, InList2 ++ ExtraList, InList3, trim) andalso + ExpList =:= lists:zipwith3(ZipFn, InList1, InList2 ++ ExtraList, InList3 ++ ExtraList, trim) andalso + ExpList =:= lists:zipwith3(ZipFn, InList1 ++ ExtraList, InList2, InList3, trim) andalso + ExpList =:= lists:zipwith3(ZipFn, InList1 ++ ExtraList, InList2, InList3 ++ ExtraList, trim) andalso + ExpList =:= lists:zipwith3(ZipFn, InList1 ++ ExtraList, InList2 ++ ExtraList, InList3, trim), + + Padded1 = lists:zipwith3(ZipFn, InList1, InList2, InList3 ++ ExtraList, {pad, {Tag, Tag, Tag}}), + Padded2 = lists:zipwith3(ZipFn, InList1, InList2 ++ ExtraList, InList3, {pad, {Tag, Tag, Tag}}), + Padded3 = lists:zipwith3(ZipFn, InList1, InList2 ++ ExtraList, InList3 ++ ExtraList, {pad, {Tag, Tag, Tag}}), + Padded4 = lists:zipwith3(ZipFn, InList1 ++ ExtraList, InList2, InList3, {pad, {Tag, Tag, Tag}}), + Padded5 = lists:zipwith3(ZipFn, InList1 ++ ExtraList, InList2, InList3 ++ ExtraList, {pad, {Tag, Tag, Tag}}), + Padded6 = lists:zipwith3(ZipFn, InList1 ++ ExtraList, InList2 ++ ExtraList, InList3, {pad, {Tag, Tag, Tag}}), + Res4 = Padded1 =:= ExpList ++ [ZipFn(Tag, Tag, X) || X <- ExtraList] andalso + Padded2 =:= ExpList ++ [ZipFn(Tag, X, Tag) || X <- ExtraList] andalso + Padded3 =:= ExpList ++ [ZipFn(Tag, X, X) || X <- ExtraList] andalso + Padded4 =:= ExpList ++ [ZipFn(X, Tag, Tag) || X <- ExtraList] andalso + Padded5 =:= ExpList ++ [ZipFn(X, Tag, X) || X <- ExtraList] andalso + Padded6 =:= ExpList ++ [ZipFn(X, X, Tag) || X <- ExtraList], + + Res1 andalso Res2 andalso Res3 andalso Res4 + end + ). + +%%%%%%%%%%%%%%%%%% +%%% Generators %%% +%%%%%%%%%%%%%%%%%% + +non_list() -> + ?SUCHTHAT(NonList, gen_any(), not is_list(NonList)). + +%% Generator for lists of the given type, folding the given function +%% over values on the top level as they are generated. The first generated +%% value serves as the initial accumulator. +gen_list_fold(Gen, FoldFn) -> + ?SIZED( + Size, + ?LET( + T, + Gen, + if + Size =< 1 -> + {[], T}; + true -> + gen_list_fold(max(0, Size - 1), Gen, [T], FoldFn, T) + end + ) + ). + +%% Generator for lists of the given type, folding the given function +%% over values on the top level as they are generated. +gen_list_fold(Gen, FoldFn, Acc0) -> + ?SIZED( + Size, + gen_list_fold(max(0, Size - 1), Gen, [], FoldFn, Acc0) + ). + +gen_list_fold(0, _Gen, L, _FoldFn, Acc) -> + {L, Acc}; +gen_list_fold(N, Gen, L, FoldFn, Acc) -> + ?LET( + E, + Gen, + gen_list_fold(N - 1, Gen, L ++ [E], FoldFn, FoldFn(E, Acc)) + ). + +%% Generator for key tuples of the given size, +%% with the given key in the given (ie, last) position. +gen_keytuple(Key, Size) -> + gen_keytuple(Key, Size, Size). + +%% Generator for key tuples of the given minimum and maximum +%% sizes, with the given key in the given minimum position. +gen_keytuple(Key, MinSize, MaxSize) -> + ?LET( + Tuple, + gen_tuple(MinSize, MaxSize), + setelement(MinSize, Tuple, Key) + ). + +%% Generator for tuples of random size. +gen_tuple() -> + ?LET( + N, + non_neg_integer(), + gen_tuple(N) + ). + +%% Generator for tuples of the given size. +gen_tuple(Size) -> + ?LET( + V, + vector(Size, gen_any()), + list_to_tuple(V) + ). + +%% Generator for tuples of the given minimum and +%% maximum sizes. +gen_tuple(MinSize, MaxSize) -> + ?LET( + N, + range(MinSize, MaxSize), + ?LET( + V, + vector(N, gen_any()), + list_to_tuple(V) + ) + ). + +%% Generator for lists of anything. +gen_list() -> + list(gen_any()). + +%% Generator for lists of anything, folding the given function +%% over values on all levels of list-nesting as they are generated. +gen_list_deepfold(FoldFn, Acc0) -> + ?SIZED( + Size, + ?LET( + {_, L, Acc}, + gen_list_deepfold(max(0, Size - 1), 0, [], FoldFn, Acc0), + {L, Acc} + ) + ). + +gen_list_deepfold(N, _Level, L, _FoldFn, Acc) when N =< 0 -> + {N, lists:reverse(L), Acc}; +gen_list_deepfold(N, Level, L, FoldFn, Acc) -> + ?LET( + X, + frequency([ + {4, {term, gen_any_simple()}}, + {1, deeplist}, + {1, tuple}, + {2, stop} + ]), + case X of + deeplist -> + ?LET( + {N1, L1, Acc1}, + gen_list_deepfold(N, Level + 1, [], FoldFn, Acc), + gen_list_deepfold(N1, Level, [L1|L], FoldFn, Acc1) + ); + tuple -> + ?LET( + {N1, L1, _}, + gen_list_deepfold(N, Level + 1, [], fun(_, _, _) -> undefined end, undefined), + begin + E = list_to_tuple(L1), + gen_list_deepfold(N1, Level, [E|L], FoldFn, FoldFn(Level, E, Acc)) + end + ); + stop -> + {N, lists:reverse(L), Acc}; + {term, E} -> + gen_list_deepfold(N - 1, Level, [E|L], FoldFn, FoldFn(Level, E, Acc)) + end + ). + +%% Generator for simple and composite (lists and tuples) types. +gen_any() -> + frequency( + [ + {4, gen_any_simple()}, + {1, ?LET({L, _}, gen_list_deepfold(fun(_, _, Acc) -> Acc end, undefined), L)}, + {1, ?LET({L, _}, gen_list_deepfold(fun(_, _, Acc) -> Acc end, undefined), list_to_tuple(L))} + ] + ). + +%% Generator for simple types: +%% - atoms +%% - integers +%% - floats +%% - bitstrings +gen_any_simple() -> + oneof([gen_atom(), integer(), float(), bitstring()]). + +%% Generator for interesting atoms: +%% - well-known atoms like `ok', `undefined', `infinity'... +%% - randomly generated "weird" atoms +gen_atom() -> + oneof( + [ + oneof([ok, error, true, false, undefined, infinity]), + oneof(['', '"', '\'', '(', ')', '()', '[', '[', '[]', '{', '}', '{}']), + gen_random_atom() + ] + ). + +%% Generator for a limited set of random atoms. The number of +%% atoms that will be generated is set in `?RANDOM_ATOMS'. +gen_random_atom() -> + ?LAZY( + ?LET( + N, + range(1, ?RANDOM_ATOMS), + try + persistent_term:get({?MODULE, random_atoms}) + of + Atoms -> + maps:get(N, Atoms) + catch + error:badarg -> + ?LET( + AtomsList, + vector(?RANDOM_ATOMS, ?SIZED(Size, resize(Size * 100, atom()))), + begin + Fn = fun + F(_, [], Acc) -> + Acc; + F(Index, [A|As], Acc) -> + F(Index + 1, As, Acc#{Index => A}) + end, + Atoms = Fn(1, AtomsList, #{}), + persistent_term:put({?MODULE, random_atoms}, Atoms), + maps:get(N, Atoms) + end + ) + end + ) + ). + +%% Generator for ordering functions, to be used for sorting and merging. +%% The generated ordering functions are designed to fulfill the requirements given +%% at the top of the `lists' documentation, namely to be antisymmetric, transitive, +%% and total. Further, the chances that two terms compare equal, less or greater +%% are equal. +gen_ordering_fun() -> + ?LET( + F, + function1(range(1, 3)), + fun(T1, T2) -> + F(T1) =< F(T2) + end + ). + +%%%%%%%%%%%%%%% +%%% Helpers %%% +%%%%%%%%%%%%%%% + +%% -------------------------------------------------------------------- +expect_error(Fn, Args) when is_function(Fn, length(Args))-> + try + erlang:apply(Fn, Args) + of + _ -> false + catch + error:_ -> true; + _:_ -> false + end. + +%% -------------------------------------------------------------------- +check_appended([], []) -> + true; +check_appended([[]|Ls], AL) -> + check_appended(Ls, AL); +check_appended([L], AL) -> + L =:= AL; +check_appended([[E1|L]|Ls], [E2|AL]) -> + E1 =:= E2 andalso + check_appended([L|Ls], AL); +check_appended(_Ls, _AL) -> + false. + +%% -------------------------------------------------------------------- +check_deleted(E, [E|L], DL) -> + L =:= DL; +check_deleted(E, [_|L], [_|DL]) -> + check_deleted(E, L, DL); +check_deleted(_E, [], []) -> + true; +check_deleted(_E, _L, _DL) -> + false. + +%% -------------------------------------------------------------------- +check_joined(Sep, [E|L], [E, Sep|JL]) -> + check_joined(Sep, L, JL); +check_joined(_Sep, [E], [E]) -> + true; +check_joined(_Sep, [], []) -> + true; +check_joined(_Sep, _L, _JL) -> + false. + +%% -------------------------------------------------------------------- +check_keydeleted(K, N, [E|L], KDL) when element(N, E) == K -> + L =:= KDL; +check_keydeleted(K, N, [_|L], [_|KDL]) -> + check_keydeleted(K, N, L, KDL); +check_keydeleted(_K, _N, _L, _KDL) -> + false. + +%% -------------------------------------------------------------------- +check_keyreplaced(K, N, R, [E1|L], [E2|KRL]) when element(N, E1) == K -> + E2 =:= R andalso L =:= KRL; +check_keyreplaced(K, N, R, [_|L], [_|KRL]) -> + check_keyreplaced(K, N, R, L, KRL); +check_keyreplaced(_K, _N, _R, _L, _KRL) -> + false. + +%% -------------------------------------------------------------------- +check_merged(Ls, ML) -> + check_merged(fun erlang:'=<'/2, Ls, ML). + +check_merged(Fn, [[]|Ls], ML) -> + check_merged(Fn, Ls, ML); +check_merged(_Fn, [], ML) -> + ML =:= []; +check_merged(_Fn, [L], ML) -> + ML =:= L; +check_merged(Fn, Ls, [E|ML]) -> + case find_in_heads(Fn, E, Ls) of + {true, Ls1} -> + check_merged(Fn, Ls1, ML); + false -> + false + end; +check_merged(_Fn, _Ls, _ML) -> + false. + +find_in_heads(Fn, E, Ls) -> + find_in_heads(Fn, E, Ls, []). + +find_in_heads(Fn, E, [[]|Ls], Seen) -> + find_in_heads(Fn, E, Ls, Seen); +find_in_heads(Fn, E, [[E1|LRest]=L|Ls], Seen) -> + case Fn(E, E1) andalso Fn(E1, E) of + true -> + {true, lists:reverse(Seen, [LRest|Ls])}; + false -> + find_in_heads(Fn, E, Ls, [L|Seen]) + end; +find_in_heads(_Fn, _E, _Ls, _Seen) -> + false. + +%% -------------------------------------------------------------------- +check_partitioned(Pred, [E|L], P1, P2) -> + case {Pred(E), P1, P2} of + {true, [E|Rest], _} -> + check_partitioned(Pred, L, Rest, P2); + {false, _, [E|Rest]} -> + check_partitioned(Pred, L, P1, Rest); + _ -> + false + end; +check_partitioned(_Pred, [], [], []) -> + true; +check_partitioned(_Pred, _L, _P1, _P2) -> + false. + +%% -------------------------------------------------------------------- +check_reversed(L1, L2) -> + check_reversed(L1, L2, []). + +check_reversed(L1, L2, Tail) -> + check_reversed1(L1, L2) =:= Tail. + +check_reversed1([], L2) -> + L2; +check_reversed1([E|L1], L2) -> + case check_reversed1(L1, L2) of + [E|L2Rest] -> L2Rest; + _ -> false + end. + +%% -------------------------------------------------------------------- +check_seq([F|Seq], F, T, S) -> + check_seq(Seq, F + S, T, S); +check_seq([], F, T, S) when S >= 0 -> + F >= T; +check_seq([], F, T, S) when S < 0 -> + F =< T; +check_seq(_Seq, _F, _T, _S) -> + false. + +%% -------------------------------------------------------------------- +check_sorted(L, Sorted) -> + check_sorted(fun erlang:'=<'/2, L, Sorted). + +check_sorted(SortFun, L, Sorted) -> + ExpElems = count_elems(L), + check_sorted(SortFun, Sorted, ExpElems, #{}). + +check_sorted(_SortFun, [], ExpElems, FoundElems) -> + ExpElems =:= FoundElems; +check_sorted(SortFun, [E], ExpElems, FoundElems) -> + maps:is_key(E, ExpElems) andalso + check_sorted(SortFun, [], ExpElems, maps:update_with(E, fun(Cnt) -> Cnt + 1 end, 1, FoundElems)); +check_sorted(SortFun, [E1|[E2|_]=L], ExpElems, FoundElems) -> + SortFun(E1, E2) andalso + maps:is_key(E1, ExpElems) andalso + check_sorted(SortFun, L, ExpElems, maps:update_with(E1, fun(Cnt) -> Cnt + 1 end, 1, FoundElems)); +check_sorted(_SortFun, _L, _ExpElems, _FoundElems) -> + false. + +count_elems(L) -> + count_elems(L, #{}). + +count_elems([E|Es], Acc) -> + count_elems(Es, maps:update_with(E, fun(Cnt) -> Cnt + 1 end, 1, Acc)); +count_elems([], Acc) -> + Acc. + +%% -------------------------------------------------------------------- +check_splitwithed(Pred, [E|L], [E|P1], P2) -> + Pred(E) andalso + check_splitwithed(Pred, L, P1, P2); +check_splitwithed(Pred, [E|_]=L, [], P2) -> + not Pred(E) andalso L =:= P2; +check_splitwithed(_Pred, [], [], []) -> + true; +check_splitwithed(_Pred, _L, _P1, _P2) -> + false. + +%% -------------------------------------------------------------------- +check_umerged(Ls, ML) -> + check_umerged(fun erlang:'=<'/2, Ls, ML). + +check_umerged(Fn, [[]|Ls], ML) -> + check_umerged(Fn, Ls, ML); +check_umerged(_Fn, [L], ML) -> + ML =:= L; +check_umerged(_Fn, [], ML) -> + ML =:= []; +check_umerged(Fn, Ls, [E|ML]) -> + case find_and_remove_from_heads(Fn, E, Ls) of + {true, Ls1} -> + check_umerged(Fn, Ls1, ML); + false -> + false + end; +check_umerged(_Fn, _Ls, _ML) -> + false. + +find_and_remove_from_heads(Fn, E, Ls) -> + find_and_remove_from_heads(false, Fn, E, Ls, []). + +find_and_remove_from_heads(Found, Fn, E, [[]|Ls], Seen) -> + find_and_remove_from_heads(Found, Fn, E, Ls, Seen); +find_and_remove_from_heads(false, _Fn, _E, [], _Seen) -> + false; +find_and_remove_from_heads(true, _Fn, _E, [], Seen) -> + {true, lists:reverse(Seen)}; +find_and_remove_from_heads(Found, Fn, E, [[E1|LRest]=L|Ls], Seen) -> + case Fn(E, E1) andalso Fn(E1, E) of + true -> + find_and_remove_from_heads(true, Fn, E, Ls, [LRest|Seen]); + false -> + find_and_remove_from_heads(Found, Fn, E, Ls, [L|Seen]) + end. + +%% -------------------------------------------------------------------- +check_uniqed(L, UL) -> + check_uniqed(fun(X) -> X end, L, UL). + +check_uniqed(Fn, L, UL) -> + check_uniqed1(Fn, L, UL, sets:new([{version, 2}])). + +check_uniqed1(Fn, [E|L], [], Seen) -> + sets:is_element(Fn(E), Seen) andalso + check_uniqed1(Fn, L, [], Seen); +check_uniqed1(Fn, [E1|L], [E2|URest]=U, Seen) -> + X1 = Fn(E1), + X2 = Fn(E2), + case sets:is_element(X1, Seen) of + true -> + X1 =/= X2 andalso + check_uniqed1(Fn, L, U, Seen); + false -> + X1 =:= X2 andalso + check_uniqed1(Fn, L, URest, sets:add_element(X1, Seen)) + end; +check_uniqed1(_Fn, [], [], _Seen) -> + true; +check_uniqed1(_Fn, _L, _UL, _Seen) -> + false. + +%% -------------------------------------------------------------------- +check_usorted(L, Sorted) -> + check_usorted(fun erlang:'=<'/2, L, Sorted). + +check_usorted(SortFun, L, Sorted) -> + ExpElems = ucount_elems(SortFun, L), + check_sorted(SortFun, Sorted, ExpElems, #{}). + +ucount_elems(SortFun, L) -> + ucount_elems(SortFun, L, #{}). + +ucount_elems(SortFun, [E|Es], Acc) -> + K = ufind_key(SortFun, E, maps:keys(Acc)), + ucount_elems(SortFun, Es, maps:put(K, 1, Acc)); +ucount_elems(_SortFun, [], Acc) -> + Acc. + +ufind_key(SortFun, E, [K|Keys]) -> + case SortFun(E, K) andalso SortFun(K, E) of + true -> + K; + false -> + ufind_key(SortFun, E, Keys) + end; +ufind_key(_SortFun, E, []) -> + E. diff --git a/lib/stdlib/test/property_test/uri_string_recompose.erl b/lib/stdlib/test/property_test/uri_string_recompose.erl index 1720ea78d3..4f37e4aa9c 100644 --- a/lib/stdlib/test/property_test/uri_string_recompose.erl +++ b/lib/stdlib/test/property_test/uri_string_recompose.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2021. All Rights Reserved. +%% Copyright Ericsson AB 2008-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -84,7 +84,7 @@ prop_recompose() -> Map =:= uri_string:parse(uri_string:recompose(Map))). prop_normalize() -> - ?FORALL(Map, map(), + ?FORALL(Map, property_map(), uri_string:percent_decode( uri_string:normalize(Map, [return_map])) =:= uri_string:percent_decode( @@ -94,11 +94,11 @@ prop_normalize() -> %% Stats prop_map_key_length_collect() -> - ?FORALL(List, map(), + ?FORALL(List, property_map(), collect(length(maps:keys(List)), true)). prop_map_collect() -> - ?FORALL(List, map(), + ?FORALL(List, property_map(), collect(lists:sort(maps:keys(List)), true)). prop_scheme_collect() -> @@ -110,7 +110,7 @@ prop_scheme_collect() -> %%% Generators %%%======================================================================== -map() -> +property_map() -> ?LET(Gen, comp_proplist(), proplist_to_map(Gen)). map_no_unicode() -> diff --git a/lib/stdlib/test/qlc_SUITE.erl b/lib/stdlib/test/qlc_SUITE.erl index 74c8aabf8e..2e1b722b8e 100644 --- a/lib/stdlib/test/qlc_SUITE.erl +++ b/lib/stdlib/test/qlc_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2021. All Rights Reserved. +%% Copyright Ericsson AB 2004-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -401,6 +401,7 @@ nomatch(Config) when is_list(Config) -> %% {warnings,[{{3,52},qlc,nomatch_pattern}]}}, {warnings,[{{3,37},v3_core,{nomatch,pattern}}]}}, + %% No longer illegal in OTP 26. {nomatch4, <<"nomatch() -> etsc(fun(E) -> @@ -411,7 +412,7 @@ nomatch(Config) when is_list(Config) -> end, [{<<34>>},{<<40>>}]). ">>, [], - {errors,[{{3,48},erl_lint,illegal_bin_pattern}],[]}}, + []}, {nomatch5, <<"nomatch() -> @@ -6556,15 +6557,15 @@ otp_7114(Config) when is_list(Config) -> %% OTP-7232. qlc:info() bug (pids, ports, refs, funs). otp_7232(Config) when is_list(Config) -> - Ts = [<<"L = [fun math:sqrt/1, list_to_pid(\"<0.4.1>\"), + Ts = [<<"L = [fun math:sqrt/1, list_to_pid(\"<0.4.0>\"), erlang:make_ref()], - \"[fun math:sqrt/1, <0.4.1>, #Ref<\" ++ _ = qlc:info(L), + \"[fun math:sqrt/1, <0.4.0>, #Ref<\" ++ _ = qlc:info(L), {call,_, {remote,_,{atom,_,qlc},{atom,_,sort}}, [{cons,_, {'fun',_,{function,{atom,_,math},{atom,_,sqrt},_}}, {cons,_, - {string,_,\"<0.4.1>\"}, % could use list_to_pid.. + {string,_,\"<0.4.0>\"}, % could use list_to_pid.. {cons,_,{string,_,\"#Ref<\"++_},{nil,_}}}}, {nil,_}]} = qlc:info(qlc:sort(L),{format,abstract_code})">>, diff --git a/lib/stdlib/test/re_SUITE.erl b/lib/stdlib/test/re_SUITE.erl index 09a65d8fdd..fc6e977942 100644 --- a/lib/stdlib/test/re_SUITE.erl +++ b/lib/stdlib/test/re_SUITE.erl @@ -22,7 +22,7 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, pcre/1,compile_options/1, run_options/1,combined_options/1,replace_autogen/1, - global_capture/1,replace_input_types/1,replace_return/1, + global_capture/1,replace_input_types/1,replace_with_fun/1,replace_return/1, split_autogen/1,split_options/1,split_specials/1, error_handling/1,pcre_cve_2008_2371/1,re_version/1, pcre_compile_workspace_overflow/1,re_infinite_loop/1, @@ -42,7 +42,7 @@ suite() -> all() -> [pcre, compile_options, run_options, combined_options, replace_autogen, global_capture, replace_input_types, - replace_return, split_autogen, split_options, + replace_with_fun, replace_return, split_autogen, split_options, split_specials, error_handling, pcre_cve_2008_2371, pcre_compile_workspace_overflow, re_infinite_loop, re_backwards_accented, opt_dupnames, opt_all_names, @@ -365,6 +365,16 @@ replace_input_types(Config) when is_list(Config) -> <<"a",208,128,"cd">> = re:replace(<<"abcd">>,"b","\x{400}",[{return,binary},unicode]), ok. +%% Test replace with a replacement function. +replace_with_fun(Config) when is_list(Config) -> + <<"ABCD">> = re:replace("abcd", ".", fun(<<C>>, []) -> <<(C - $a + $A)>> end, [global, {return, binary}]), + <<"AbCd">> = re:replace("abcd", ".", fun(<<C>>, []) when (C - $a) rem 2 =:= 0 -> <<(C - $a + $A)>>; (C, []) -> C end, [global, {return, binary}]), + <<"b-ad-c">> = re:replace("abcd", "(.)(.)", fun(_, [A, B]) -> <<B/binary, $-, A/binary>> end, [global, {return, binary}]), + <<"#ab-B#cd">> = re:replace("abcd", ".(.)", fun(Whole, [<<C>>]) -> <<$#, Whole/binary, $-, (C - $a + $A), $#>> end, [{return, binary}]), + <<"#ab#cd">> = re:replace("abcd", ".(x)?(.)", fun(Whole, [<<>>, _]) -> <<$#, Whole/binary, $#>> end, [{return, binary}]), + <<"#ab#cd">> = re:replace("abcd", ".(.)(x)?", fun(Whole, [_]) -> <<$#, Whole/binary, $#>> end, [{return, binary}]), + ok. + %% Test return options of replace together with global searching. replace_return(Config) when is_list(Config) -> {'EXIT',{badarg,_}} = (catch re:replace("na","(a","")), diff --git a/lib/stdlib/test/shell_SUITE.erl b/lib/stdlib/test/shell_SUITE.erl index b38dee47e7..142f8ad445 100644 --- a/lib/stdlib/test/shell_SUITE.erl +++ b/lib/stdlib/test/shell_SUITE.erl @@ -18,24 +18,25 @@ %% %CopyrightEnd% %% -module(shell_SUITE). --export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2]). - -export([forget/1, records/1, known_bugs/1, otp_5226/1, otp_5327/1, otp_5435/1, otp_5195/1, otp_5915/1, otp_5916/1, bs_match_misc_SUITE/1, bs_match_int_SUITE/1, bs_match_tail_SUITE/1, bs_match_bin_SUITE/1, bs_construct_SUITE/1, - refman_bit_syntax/1, - progex_bit_syntax/1, progex_records/1, + refman_bit_syntax/1, + progex_bit_syntax/1, progex_records/1, progex_lc/1, progex_funs/1, otp_5990/1, otp_6166/1, otp_6554/1, otp_7184/1, otp_7232/1, otp_8393/1, otp_10302/1, otp_13719/1, - otp_14285/1, otp_14296/1, typed_records/1]). + otp_14285/1, otp_14296/1, typed_records/1, types/1]). --export([ start_restricted_from_shell/1, +-export([ start_restricted_from_shell/1, start_restricted_on_command_line/1,restricted_local/1]). +-export([ start_interactive/1, whereis/1 ]). + %% Internal export. -export([otp_5435_2/0, prompt1/1, prompt2/1, prompt3/1, prompt4/1, prompt5/1]). @@ -73,19 +74,21 @@ suite() -> [{ct_hooks,[ts_install_cth]}, {timetrap,{minutes,10}}]. -all() -> +all() -> [forget, known_bugs, otp_5226, otp_5327, - otp_5435, otp_5195, otp_5915, otp_5916, {group, bits}, + otp_5435, otp_5195, otp_5915, otp_5916, + start_interactive, whereis, {group, bits}, {group, refman}, {group, progex}, {group, tickets}, - {group, restricted}, {group, records}]. + {group, restricted}, {group, records}, {group, definitions}]. -groups() -> +groups() -> [{restricted, [], [start_restricted_from_shell, start_restricted_on_command_line, restricted_local]}, {bits, [], [bs_match_misc_SUITE, bs_match_tail_SUITE, bs_match_bin_SUITE, bs_construct_SUITE]}, + {definitions, [], [types]}, {records, [], [records, typed_records]}, {refman, [], [refman_bit_syntax]}, @@ -300,7 +303,7 @@ restricted_local(Config) when is_list(Config) -> application:get_env(stdlib, restricted_shell), true = purge_and_delete(user_default), ok. - + %% f/0 and f/1. forget(Config) when is_list(Config) -> @@ -319,15 +322,73 @@ forget(Config) when is_list(Config) -> comm_err(<<"f(a).">>), ok. +%% type definition support +types(Config) when is_list(Config) -> + %% type + [ok] = scan(<<"-type baz() :: integer().">>), + %% record + [ok] = scan(<<"-record(foo, {bar :: baz()}).">>), + %% spec + [ok] = scan(<<"-spec foo(Bar) -> Baz when + Bar :: string(), + Baz :: integer().">>), + shell_attribute_test(Config), + ok. +shell_attribute_test(Config) -> + Path = filename:join([proplists:get_value(priv_dir, Config), + "shell_history", "function_def"]), + rtnode:run( + [{putline, "foo(Bar) -> Bar."}, + {expect, "ok"}, + {putline, "fl()."}, + {expect, "\\Q{function,{shell_default,foo,1}}\\E"}, + {putline, "foo(1)."}, + {expect, "1"}, + {putline, "shell_default:foo(2)."}, + {expect, "2"} + ],[],"", ["-kernel","shell_history","enabled", + "-kernel","shell_history_path","\"" ++ Path ++ "\"", + "-kernel","shell_history_drop","[\"init:stop().\"]"]), + receive after 1000 -> ok end, + rtnode:run( + [{putline, "-record(hej, {a = 0 :: integer()})."}, + {expect, "ok"}, + {putline, "rl()."}, + {expect, "\\Q-record(hej,{a = 0 :: integer()}).\\E"}, + {putline, "#hej{a=1}."}, + {expect, "\\Q#hej{a = 1}\\E"} + ],[],"", ["-kernel","shell_history","enabled", + "-kernel","shell_history_path","\"" ++ Path ++ "\"", + "-kernel","shell_history_drop","[\"init:stop().\"]"]), + receive after 1000 -> ok end, + rtnode:run( + [{putline, "-spec foo(Bar) -> Bar when Bar :: integer()."}, + {expect, "ok"}, + {putline, "fl()."}, + {expect, "\\Q{function_type,{shell_default,foo,1}}\\E"} + ],[],"", ["-kernel","shell_history","enabled", + "-kernel","shell_history_path","\"" ++ Path ++ "\"", + "-kernel","shell_history_drop","[\"init:stop().\"]"]), + receive after 1000 -> ok end, + rtnode:run( + [{putline, "-type my_type() :: boolean() | integer()."}, + {expect, "ok"}, + {putline, "fl()."}, + {expect, "\\Q{type,my_type}\\E"} + ],[],"", ["-kernel","shell_history","enabled", + "-kernel","shell_history_path","\"" ++ Path ++ "\"", + "-kernel","shell_history_drop","[\"init:stop().\"]"]), + ok. + %% Test of the record support. OTP-5063. records(Config) when is_list(Config) -> %% rd/2 [{attribute,_,record,{bar,_}},ok] = - scan(<<"rd(foo,{bar}), + scan(<<"rd(foo,{bar}), rd(bar,{foo = (#foo{})#foo.bar}), rl(bar).">>), "variable 'R' is unbound" = % used to work (before OTP-5878, R11B) - exit_string(<<"rd(foo,{bar}), + exit_string(<<"rd(foo,{bar}), R = #foo{}, rd(bar,{foo = R#foo.bar}).">>), "exception error: no function clause matching call to rd/2" = @@ -401,7 +462,7 @@ records(Config) when is_list(Config) -> [{attribute,A1,record,{test1,_}},ok] = scan(RR5), RR6 = "rr(\"" ++ Test ++ "\", '_', {d,test2}), rl([test1,test2]).", [{attribute,A1,record,{test2,_}},ok] = scan(RR6), - RR7 = "rr(\"" ++ Test ++ + RR7 = "rr(\"" ++ Test ++ "\", '_', [{d,test1},{d,test2,17}]), rl([test1,test2]).", [{attribute,A1,record,{test1,_}},{attribute,A1,record,{test2,_}},ok] = scan(RR7), @@ -503,7 +564,7 @@ records(Config) when is_list(Config) -> [ok] = scan(<<"rd(a,{}), is_record({a},a) andalso true, b().">>), - + %% nested record defs "#b{a = #a{}}.\n" = t(<<"rd(a,{}), rd(b, {a = #a{}}), #b{}.">>), @@ -672,7 +733,7 @@ otp_5435_2() -> %% application being in the path. %% OTP-5876. [{attribute,_,record,{bar,_}},ok] = - scan(<<"rd(foo,{bar}), + scan(<<"rd(foo,{bar}), rd(bar,{foo = (#foo{})#foo.bar}), rl(bar).">>), ok. @@ -685,7 +746,7 @@ otp_5195(Config) when is_list(Config) -> %% An experimental shell used to translate error tuples: %% "(qlc) \"1: generated variable 'X' must not be used in " - %% "list expression\".\n" = + %% "list expression\".\n" = %% t(<<"qlc:q([X || X <- [{a}], Y <- [X]]).">>), %% Same as last one (if the shell does not translate error tuples): [{error,qlc,{{1,31},qlc,{used_generator_variable,'X'}}}] = @@ -1204,7 +1265,7 @@ bs_match_int_SUITE(Config) when is_list(Config) -> Int -> ok; Other -> io:format(\"Bin = ~p,\", [Bin]), - io:format(\"SkipBef = ~p, N = ~p\", + io:format(\"SkipBef = ~p, N = ~p\", [SkipBef,N]), io:format(\"Expected ~p, got ~p\", [Int,Other]) @@ -1311,8 +1372,8 @@ ok = evaluate(C, []). %% OTP-5327. Adopted from emulator/test/bs_match_bin_SUITE.erl. bs_match_bin_SUITE(Config) when is_list(Config) -> - ByteSplitBinary = - <<"ByteSplit = + ByteSplitBinary = + <<"ByteSplit = fun(L, B, Pos, Fun) when Pos >= 0 -> Sz1 = Pos, Sz2 = size(B) - Pos, @@ -1343,7 +1404,7 @@ ok = evaluate(ByteSplitBinary, []), BitSplitBinary = <<"Mkbin = fun(L) when list(L) -> list_to_binary(L) end, - MakeInt = + MakeInt = fun(List, 0, Acc, _F) -> Acc; ([H|T], N, Acc, F) -> F(T, N-1, Acc bsl 1 bor H, F) end, @@ -1452,7 +1513,7 @@ bs_construct_SUITE(Config) when is_list(Config) -> ?FAIL(<<<<23,56,0,2>>:(-16)/binary>>) "," ?FAIL(<<<<23,56,0,2>>:(2.5)/binary>>) "," ?FAIL(<<<<23,56,0,2>>:(anka)>>) " - end, + end, TestF(), NotUsed1 = fun(I, BinString) -> <<I:32,BinString/binary>>, ok end, @@ -1512,7 +1573,7 @@ ok = evaluate(C1, []), C2 = <<" I = fun(X) -> X end, - Fail = fun() -> + Fail = fun() -> I_minus_777 = I(-777), I_minus_2047 = I(-2047), @@ -1634,8 +1695,8 @@ progex_bit_syntax(Config) when is_list(Config) -> Fun4 = fun(Dgram) -> DgramSize = byte_size(Dgram), - case Dgram of - <<?IP_VERSION:4, HLen:4, SrvcType:8, TotLen:16, + case Dgram of + <<?IP_VERSION:4, HLen:4, SrvcType:8, TotLen:16, ID:16, Flgs:3, FragOff:13, TTL:8, Proto:8, HdrChkSum:16, SrcIP:32, DestIP:32, @@ -1736,7 +1797,7 @@ triples_to_bin1(T) -> triples_to_bin1([{X,Y,Z} | T], Acc) -> triples_to_bin1(T, <<Acc/binary, X:32, Y:32, Z:32>>); % inefficient -triples_to_bin1([], Acc) -> +triples_to_bin1([], Acc) -> Acc. triples_to_bin2(T) -> @@ -1744,12 +1805,12 @@ triples_to_bin2(T) -> triples_to_bin2([{X,Y,Z} | T], Acc) -> triples_to_bin2(T, [<<X:32, Y:32, Z:32>> | Acc]); -triples_to_bin2([], Acc) -> +triples_to_bin2([], Acc) -> list_to_binary(lists:reverse(Acc)). %% Record examples from Programming Examples. OTP-5237. progex_records(Config) when is_list(Config) -> - Test1 = + Test1 = <<"-module(recs). -record(person, {name = \"\", phone = [], address}). -record(name, {first = \"Robert\", last = \"Ericsson\"}). @@ -1786,7 +1847,7 @@ t() -> c), P3 = #person{name=\"Joe\", phone=[0,0,7], address=\"A street\"}, - #person{name = Name} = P3, + #person{name = Name} = P3, \"Joe\" = Name, \"Robert\" = demo(), @@ -1854,7 +1915,7 @@ Test1_shell = Find), P3 = #person{name=\"Joe\", phone=[0,0,7], address=\"A street\"}, - #person{name = Name} = P3, + #person{name = Name} = P3, \"Joe\" = Name, Demo = fun() -> @@ -1884,7 +1945,7 @@ print(#person{name = Name, age = Age, io:format(\"Name: ~s, Age: ~w, Phone: ~w ~n\" \"Dictionary: ~w.~n\", [Name, Age, Phone, Dict]). - birthday(P) when record(P, person) -> + birthday(P) when record(P, person) -> P#person{age = P#person.age + 1}. register_two_hackers() -> @@ -1902,7 +1963,7 @@ ok. %% List comprehension examples from Programming Examples. OTP-5237. progex_lc(Config) when is_list(Config) -> - Test1 = + Test1 = <<"-module(lc). -export([t/0]). @@ -2036,7 +2097,7 @@ ok. %% Funs examples from Programming Examples. OTP-5237. progex_funs(Config) when is_list(Config) -> - Test1 = + Test1 = <<"-module(funs). -export([t/0]). @@ -2273,7 +2334,7 @@ Test2_shell = \"ERLANG\" = Upcase_word(\"Erlang\"), [\"I\",\"LIKE\",\"ERLANG\"] = lists:map(Upcase_word, L), {[\"I\",\"LIKE\",\"ERLANG\"],11} = - lists:mapfoldl(fun(Word, Sum) -> + lists:mapfoldl(fun(Word, Sum) -> {Upcase_word(Word), Sum + length(Word)} end, 0, L), [500,12,45] = lists:filter(Big, [500,12,2,45,6,7]), @@ -2318,10 +2379,10 @@ otp_6166(Config) when is_list(Config) -> -record(r6, {f = #r5{}}). % r6 > r0 -record(r0, {f = #r5{}, g = #r5{}}). % r0 < r5">>, ok = file:write_file(Test2, Contents2), - - RR12 = "[r1,r2,r3,r4,r5] = rr(\"" ++ Test1 ++ "\"), - [r0,r1,r2,r3,r4,r5,r6] = rr(\"" ++ Test2 ++ "\"), - R0 = #r0{}, R6 = #r6{}, + + RR12 = "[r1,r2,r3,r4,r5] = rr(\"" ++ Test1 ++ "\"), + [r0,r1,r2,r3,r4,r5,r6] = rr(\"" ++ Test2 ++ "\"), + R0 = #r0{}, R6 = #r6{}, true = is_record(R0, r0), true = is_record(R6, r6), ok. ", @@ -2438,7 +2499,7 @@ otp_6554(Config) when is_list(Config) -> "exception error: undefined function math:sqrt/2" = comm_err(<<"math:sqrt(2, 2).">>), "exception error: limit of number of arguments to interpreted function " - "exceeded" = + "exceeded" = comm_err(<<"fun(A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T,U) ->" " a end().">>), "exception error: bad filter a" = @@ -2536,7 +2597,7 @@ otp_6554(Config) when is_list(Config) -> t(<<"results(2). 1. v(2). h().">>), application:unset_env(stdlib, shell_saved_results), "1\nfoo\n17\nB = foo\nC = 17\nF = fun() ->\n foo" - "\n end.\nok.\n" = + "\n end.\nok.\n" = t(<<"begin F = fun() -> foo end, 1 end. B = F(). C = 17. b().">>), "3: command not found" = comm_err(<<"#{v(3) => v}.">>), @@ -2562,9 +2623,9 @@ otp_7184(Config) when is_list(Config) -> t(<<"P = self(), spawn_link(fun() -> process_flag(trap_exit,true), P ! up, - receive X -> + receive X -> otp_7184 ! {otp_7184, X} - end + end end), receive up -> ok end, erlang:raise(throw, thrown, []).">>), @@ -2574,9 +2635,9 @@ otp_7184(Config) when is_list(Config) -> t(<<"P = self(), spawn_link(fun() -> process_flag(trap_exit,true), P ! up, - receive X -> + receive X -> otp_7184 ! {otp_7184, X} - end + end end), receive up -> ok end, erlang:raise(exit, fini, []).">>), @@ -2586,9 +2647,9 @@ otp_7184(Config) when is_list(Config) -> t(<<"P = self(), spawn_link(fun() -> process_flag(trap_exit,true), P ! up, - receive X -> + receive X -> otp_7184 ! {otp_7184,X} - end + end end), receive up -> ok end, erlang:raise(error, bad, []).">>), @@ -2713,7 +2774,7 @@ exit_term(B) -> -endif. error_string(B) -> - "** exception error:" ++ Reply = t(B), + "** exception error:" ++ Reply = t(B), caught_string(Reply). exit_string(B) -> @@ -2820,7 +2881,7 @@ otp_10302(Config) when is_list(Config) -> {ok, Es} = erl_parse:parse_exprs(Ts), B = erl_eval:new_bindings(), erl_eval:exprs(Es, B).">>, - + "ok.\n** exception error: an error occurred when evaluating" " an arithmetic expression\n in operator '/'/2\n" " called as <<\"ยช\">> / <<\"ยช\">>.\n" = t({Node,Test7}), @@ -3014,11 +3075,198 @@ otp_14296(Config) when is_list(Config) -> {error, {_,_,"bad term"}} = TF("1, 2"), ok. +start_interactive(_Config) -> + start_interactive_shell([]), + start_interactive_shell(["-env","TERM","dumb"]). + +start_interactive_shell(ExtraArgs) -> + + %% Basic test case + rtnode:run( + [{expect, "eval_test"}, + {putline, "test."}, + {eval, fun() -> shell:start_interactive() end}, + {expect, "1>"}, + {expect, "2>"}, + {eval, fun() -> {error,already_started} = shell:start_interactive(), ok end} + ],[],"",["-noinput","-eval","io:format(\"eval_test~n\")"] ++ ExtraArgs), + + %% Test that custom MFA works + rtnode:run( + [{expect, "eval_test"}, + {putline, "test."}, + {eval, fun() -> shell:start_interactive({shell,start,[]}) end}, + {expect, "1>"}, + {expect, "2>"} + ],[],"",["-noinput","-eval","io:format(\"eval_test~n\")"] ++ ExtraArgs), + + %% Test that we can start noshell and then a shell + rtnode:run( + [{expect, "eval_test"}, + {putline, "test."}, + {eval, fun() -> shell:start_interactive(noshell) end}, + {eval, fun() -> io:format(user,"~ts",[io:get_line(user, "")]) end}, + {expect, "test\\."}, + {eval, fun() -> shell:start_interactive() end}, + {expect, "1>"}, + {putline, "test."}, + {expect, "2>"} + ],[],"",["-noinput","-eval","io:format(\"eval_test~n\")"] ++ ExtraArgs), + + %% Test that we can start various remote shell combos + [ begin + {ok, Peer, Node} = ?CT_PEER(), + SNode = atom_to_list(Node), + rtnode:run( + [{expect, "eval_test"}, + {putline, "test."}, + {eval, fun() -> shell:start_interactive(Arg(Node)) end}, + {expect, "\\Q("++SNode++")\\E2>"} + ] ++ quit_hosting_node(), + [],"",["-noinput","-eval","io:format(\"eval_test~n\")"] ++ ExtraArgs), + peer:stop(Peer) + end || Arg <- [fun(Node) -> {Node, {shell,start,[]}} end, + fun(Node) -> {remote, atom_to_list(Node)} end, + fun(Node) -> {remote, hd(string:split(atom_to_list(Node),"@"))} end, + fun(Node) -> {remote, atom_to_list(Node), {shell,start,[]}} end + ]], + + %% Test that errors work as they should + {ok, Peer, Node} = ?CT_PEER(), + rtnode:run( + [{expect, "eval_test"}, + {eval, fun() -> + {error,noconnection} = shell:start_interactive( + {remote,"invalid_node"}), + {error,noconnection} = shell:start_interactive( + {remote,"invalid_node", + {invalid_module, start, []}}), + {error,nofile} = shell:start_interactive( + {remote,atom_to_list(Node), + {invalid_module, start, []}}), + shell:start_interactive({remote, atom_to_list(Node)}) + end}, + {expect, "1> $"} + ] ++ quit_hosting_node(), + [],"",["-noinput","-eval","io:format(\"eval_test~n\")"] ++ ExtraArgs), + peer:stop(Peer), + + ok. + +whereis(_Config) -> + Proxy = spawn_link( + fun() -> + (fun F(P) -> + receive + {set,NewPid} -> + F(NewPid); + {get,From} -> + From ! P, + F(P) + end + end)(undefined) + end), + + %% Test that shell:whereis() works with JCL in newshell + rtnode:run( + [{expect,"1> $"}, + {putline,"shell:whereis()."}, + {expect,"2> $"}, + {eval,fun() -> + group_leader(erlang:whereis(user),self()), + Proxy ! {set,shell:whereis()}, + ok + end}, + {putline,"\^g"}, + {expect, "--> $"}, + {putline, "s"}, + {expect, "--> $"}, + {putline, "c"}, + {expect, "\r\nEshell"}, + {putline,"shell:whereis()."}, + {expect,"2> $"}, + {eval,fun() -> + group_leader(erlang:whereis(user),self()), + Proxy ! {get, self()}, + receive PrevPid -> PrevPid end, + io:format("~p =:= ~p~n",[PrevPid, shell:whereis()]), + false = PrevPid =:= shell:whereis(), + ok + end}, + {putline,"\^g"}, + {expect, "--> $"}, + {putline, "c 1"}, + {expect, "\r\n"}, + {putline, ""}, + {expect, "2> $"}, + {eval,fun() -> + group_leader(erlang:whereis(user),self()), + Proxy ! {get, self()}, + receive PrevPid -> PrevPid end, + true = PrevPid =:= shell:whereis(), + ok + end}]), + + %% Test that shell:whereis() works in oldshell + rtnode:run( + [{expect,"1>"}, + {eval,fun() -> + group_leader(erlang:whereis(user),self()), + true = is_pid(shell:whereis()), + ok + end}], + [],"",["-env","TERM","dumb"]), + + %% Test that noinput and noshell gives undefined shell process + rtnode:run( + [{eval,fun() -> + group_leader(erlang:whereis(user),self()), + undefined = shell:whereis(), + ok + end}], + [],"",["-noinput"]), + rtnode:run( + [{eval,fun() -> + group_leader(erlang:whereis(user),self()), + undefined = shell:whereis(), + ok + end}], + [],"",["-noshell"]), + + %% Test that remsh gives the correct shell process + {ok, Peer, Node} = ?CT_PEER(), + NodeStr = lists:flatten(io_lib:format("~w",[Node])), + rtnode:run( + [{expect, "1>"}, + {putline,"shell:whereis()."}, + {expect,"\n<0[.]"}, + {expect, "2>"}, + {eval, fun() -> + group_leader(erlang:whereis(user),self()), + true = Node =:= node(shell:whereis()), + ok + end}] ++ quit_hosting_node(), + peer:random_name(?FUNCTION_NAME), " ", "-remsh " ++ NodeStr ++ + " -pa " ++ filename:dirname(code:which(?MODULE))), + + peer:stop(Peer), + + ok. + +quit_hosting_node() -> + [{putline, "\^g"}, + {expect, "--> $"}, + {putline, "s"}, + {expect, "--> $"}, + {putline, "c"}, + {expect, ["Eshell"]}, + {expect, ["1> $"]}]. + term_to_string(T) -> lists:flatten(io_lib:format("~w", [T])). scan(B) -> - F = fun(Ts) -> + F = fun(Ts) -> case erl_parse:parse_term(Ts) of {ok,Term} -> Term; @@ -3059,18 +3307,18 @@ t1(Parent, {Bin,Enc}, F) -> S = #state{bin = Bin, unic = Enc, reply = [], leader = group_leader()}, group_leader(self(), self()), _Shell = F(), - try + try server_loop(S) catch exit:R -> Parent ! {self(), R}; throw:{?MODULE,LoopReply,latin1} -> - L0 = binary_to_list(list_to_binary(LoopReply)), - [$\n | L1] = lists:dropwhile(fun(X) -> X =/= $\n end, L0), - Parent ! {self(), dotify(L1)}; + L0 = binary_to_list(list_to_binary(LoopReply)), + [$\n | L1] = lists:dropwhile(fun(X) -> X =/= $\n end, L0), + Parent ! {self(), dotify(L1)}; throw:{?MODULE,LoopReply,_Uni} -> - Tmp = unicode:characters_to_binary(LoopReply), - L0 = unicode:characters_to_list(Tmp), - [$\n | L1] = lists:dropwhile(fun(X) -> X =/= $\n end, L0), - Parent ! {self(), dotify(L1)} + Tmp = unicode:characters_to_binary(LoopReply), + L0 = unicode:characters_to_list(Tmp), + [$\n | L1] = lists:dropwhile(fun(X) -> X =/= $\n end, L0), + Parent ! {self(), dotify(L1)} after group_leader(S#state.leader, self()) end. @@ -3102,20 +3350,20 @@ start_new_shell(Node) -> %% This is a very minimal implementation of the IO protocol... server_loop(S) -> - receive + receive {io_request, From, ReplyAs, Request} when is_pid(From) -> - server_loop(do_io_request(Request, From, S, ReplyAs)); - NotExpected -> + server_loop(do_io_request(Request, From, S, ReplyAs)); + NotExpected -> exit(NotExpected) end. - + do_io_request(Req, From, S, ReplyAs) -> case io_requests([Req], [], S) of {_Status,{eof,_},S1} -> - io_reply(From, ReplyAs, {error,terminated}), - throw({?MODULE,S1#state.reply,S1#state.unic}); - {_Status,Reply,S1} -> - io_reply(From, ReplyAs, Reply), + io_reply(From, ReplyAs, {error,terminated}), + throw({?MODULE,S1#state.reply,S1#state.unic}); + {_Status,Reply,S1} -> + io_reply(From, ReplyAs, Reply), S1 end. @@ -3133,7 +3381,7 @@ io_requests([R | Rs], Cont, S) -> end; io_requests([], [Rs|Cont], S) -> io_requests(Rs, Cont, S); -io_requests([], [], S) -> +io_requests([], [], S) -> {ok,ok,S}. io_request({setopts, Opts}, S) -> @@ -3165,7 +3413,7 @@ io_request({put_chars,unicode,Chars0}, S) -> {ok,ok,S#state{reply = [S#state.reply | Chars]}}; io_request({put_chars,Enc,Mod,Func,Args}, S) -> case catch apply(Mod, Func, Args) of - Chars when is_list(Chars) -> + Chars when is_list(Chars) -> io_request({put_chars,Enc,Chars}, S) end; io_request({get_until,Enc,_Prompt,Mod,Func,ExtraArgs}, S) -> @@ -3178,7 +3426,7 @@ get_until_loop(M, F, As, S, {more,Cont}, Enc) -> Bin = S#state.bin, case byte_size(Bin) of 0 -> - get_until_loop(M, F, As, S, + get_until_loop(M, F, As, S, catch apply(M, F, [Cont,eof|As]), Enc); _ when S#state.unic =:= latin1 -> get_until_loop(M, F, As, S#state{bin = <<>>}, @@ -3208,7 +3456,7 @@ run_file(Config, Module, Test) -> ok = file:write_file(FileName, Test), ok = compile_file(Config, FileName, Test, []), code:purge(Module), - {module, Module} = code:load_abs(LoadBeamFile), + {module, Module} = code:load_abs(LoadBeamFile), ok = Module:t(), file:delete(FileName), file:delete(BeamFile), diff --git a/lib/stdlib/test/shell_docs_SUITE.erl b/lib/stdlib/test/shell_docs_SUITE.erl index 028e2c0aba..97a9a73d71 100644 --- a/lib/stdlib/test/shell_docs_SUITE.erl +++ b/lib/stdlib/test/shell_docs_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2020-2021. All Rights Reserved. +%% Copyright Ericsson AB 2020-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -255,14 +255,15 @@ render_non_native(_Config) -> beam_language = not_erlang, format = <<"text/asciidoc">>, module_doc = #{<<"en">> => <<"This is\n\npure text">>}, - docs= [] + docs = [] }, <<"\n\tnot_an_erlang_module\n\n" " This is\n" " \n" " pure text\n">> = - unicode:characters_to_binary(shell_docs:render(not_an_erlang_module, Docs, #{})), + unicode:characters_to_binary( + shell_docs:render(not_an_erlang_module, Docs, #{ ansi => false })), ok. diff --git a/lib/stdlib/test/supervisor_SUITE.erl b/lib/stdlib/test/supervisor_SUITE.erl index 78d7e7d7bc..a9cf48e997 100644 --- a/lib/stdlib/test/supervisor_SUITE.erl +++ b/lib/stdlib/test/supervisor_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2021. All Rights Reserved. +%% Copyright Ericsson AB 1996-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -200,7 +200,7 @@ start_link(InitResult) -> %% Simulate different supervisors callback. init(fail) -> - erlang:error({badmatch,2}); + erlang:error(fail); init(InitResult) -> InitResult. @@ -227,7 +227,7 @@ sup_start_normal(Config) when is_list(Config) -> sup_start_ignore_init(Config) when is_list(Config) -> process_flag(trap_exit, true), ignore = start_link(ignore), - check_exit_reason(normal). + check_no_exit(100). %%------------------------------------------------------------------------- %% Tests what happens if init-callback returns ignore. @@ -325,15 +325,20 @@ sup_start_ignore_permanent_child_start_child_simple(Config) %% Tests what happens if init-callback returns a invalid value. sup_start_error_return(Config) when is_list(Config) -> process_flag(trap_exit, true), - {error, Term} = start_link(invalid), - check_exit_reason(Term). + %% The bad return is processed in supervisor:init/1 + InitResult = invalid, + {error, {bad_return, {?MODULE, init, InitResult}}} = + start_link(InitResult), + check_no_exit(100). %%------------------------------------------------------------------------- %% Tests what happens if init-callback fails. sup_start_fail(Config) when is_list(Config) -> process_flag(trap_exit, true), - {error, Term} = start_link(fail), - check_exit_reason(Term). + %% The exception is processed in gen_server:init_it/2 + ErrorReason = fail, + {error, {ErrorReason, _Stacktrace}} = start_link(ErrorReason), + check_no_exit(100). %%------------------------------------------------------------------------- %% Test what happens when the start function for a child returns @@ -706,11 +711,15 @@ child_adm(Config) when is_list(Config) -> process_flag(trap_exit, true), Child = {child1, {supervisor_1, start_child, []}, permanent, 1000, worker, []}, - {ok, _Pid} = start_link({ok, {{one_for_one, 2, 3600}, [Child]}}), + {ok, Pid} = start_link({ok, {{one_for_one, 2, 3600}, [Child]}}), + + %% Test that supervisors of static nature are hibernated after start + {current_function, {erlang, hibernate, 3}} = + process_info(Pid, current_function), + [{child1, CPid, worker, []}] = supervisor:which_children(sup_test), [1,1,0,1] = get_child_counts(sup_test), link(CPid), - %% Start of an already runnig process {error,{already_started, CPid}} = supervisor:start_child(sup_test, Child), @@ -771,7 +780,13 @@ child_adm(Config) when is_list(Config) -> child_adm_simple(Config) when is_list(Config) -> Child = {child, {supervisor_1, start_child, []}, permanent, 1000, worker, []}, - {ok, _Pid} = start_link({ok, {{simple_one_for_one, 2, 3600}, [Child]}}), + {ok, Pid} = start_link({ok, {{simple_one_for_one, 2, 3600}, [Child]}}), + + %% Test that supervisors of dynamic nature are not hibernated after start + {current_function, {_, Function, _}} = + process_info(Pid, current_function), + true = Function =/= hibernate, + %% In simple_one_for_one all children are added dynamically [] = supervisor:which_children(sup_test), [1,0,0,0] = get_child_counts(sup_test), @@ -2618,7 +2633,7 @@ order_of_children(_Config) -> [{ok,[_]} = dbg:p(P,procs) || P <- Expected1], terminate(Pid3, abnormal), receive {exited,ExitedPids1} -> - dbg:stop_clear(), + dbg:stop(), case ExitedPids1 of Expected1 -> ok; _ -> ct:fail({faulty_termination_order, @@ -2626,7 +2641,7 @@ order_of_children(_Config) -> {got,ExitedPids1}}) end after 3000 -> - dbg:stop_clear(), + dbg:stop(), ct:fail({shutdown_fail,timeout}) end, @@ -2647,7 +2662,7 @@ order_of_children(_Config) -> [{ok,[_]} = dbg:p(P,procs) || P <- Expected2], exit(SupPid,shutdown), receive {exited,ExitedPids2} -> - dbg:stop_clear(), + dbg:stop(), case ExitedPids2 of Expected2 -> ok; _ -> ct:fail({faulty_termination_order, @@ -2655,7 +2670,7 @@ order_of_children(_Config) -> {got,ExitedPids2}}) end after 3000 -> - dbg:stop_clear(), + dbg:stop(), ct:fail({shutdown_fail,timeout}) end, ok. @@ -3739,18 +3754,18 @@ check_exit([Pid | Pids], Timeout) -> error end. -check_exit_reason(Reason) -> +check_exit_reason(Pid, Reason) when is_pid(Pid) -> receive - {'EXIT', _, Reason} -> + {'EXIT', Pid, Reason} -> ok; - {'EXIT', _, Else} -> + {'EXIT', Pid, Else} -> ct:fail({bad_exit_reason, Else}) end. -check_exit_reason(Pid, Reason) -> +check_no_exit(Timeout) -> receive - {'EXIT', Pid, Reason} -> - ok; - {'EXIT', Pid, Else} -> - ct:fail({bad_exit_reason, Else}) + {'EXIT', Pid, _} = Exit when is_pid(Pid) -> + ct:fail({unexpected_message, Exit}) + after Timeout -> + ok end. diff --git a/lib/stdlib/test/timer_simple_SUITE.erl b/lib/stdlib/test/timer_simple_SUITE.erl index 98a8dd408d..761689dc51 100644 --- a/lib/stdlib/test/timer_simple_SUITE.erl +++ b/lib/stdlib/test/timer_simple_SUITE.erl @@ -51,7 +51,11 @@ kill_after2/1, kill_after3/1, apply_interval1/1, + apply_interval2/1, apply_interval_invalid_args/1, + apply_repeatedly1/1, + apply_repeatedly2/1, + apply_repeatedly_invalid_args/1, send_interval1/1, send_interval2/1, send_interval3/1, @@ -95,6 +99,7 @@ all() -> {group, exit_after}, {group, kill_after}, {group, apply_interval}, + {group, apply_repeatedly}, {group, send_interval}, {group, cancel}, {group, sleep}, @@ -152,10 +157,20 @@ groups() -> [], [ apply_interval1, + apply_interval2, apply_interval_invalid_args ] }, { + apply_repeatedly, + [], + [ + apply_repeatedly1, + apply_repeatedly2, + apply_repeatedly_invalid_args + ] + }, + { send_interval, [], [ @@ -406,6 +421,23 @@ apply_interval1(Config) when is_list(Config) -> {ok, cancel} = timer:cancel(Ref), nor = get_mess(1000, Msg). +%% Test apply_interval with the execution time of the action +%% longer than the timer interval. The timer should not wait for +%% the action to complete, ie start another action while the +%% previously started action is still running. +apply_interval2(Config) when is_list(Config) -> + Msg = make_ref(), + Self = self(), + {ok, Ref} = timer:apply_interval(500, erlang, apply, + [fun() -> + Self ! Msg, + receive after 1000 -> ok end + end, []]), + receive after 1800 -> ok end, + {ok, cancel} = timer:cancel(Ref), + ok = get_mess(1000, Msg, 3), + nor = get_mess(1000, Msg). + %% Test that apply_interval rejects invalid arguments. apply_interval_invalid_args(Config) when is_list(Config) -> {error, badarg} = timer:apply_interval(-1, foo, bar, []), @@ -414,6 +446,44 @@ apply_interval_invalid_args(Config) when is_list(Config) -> {error, badarg} = timer:apply_interval(0, foo, bar, baz), ok. +%% Test of apply_repeatedly by sending messages. Receive +%% 3 messages, cancel the timer, and check that we do +%% not get any more messages. In a case like this, ie where +%% the execution time of the action is shorter than the timer +%% interval, this should behave the same as apply_interval. +apply_repeatedly1(Config) when is_list(Config) -> + Msg = make_ref(), + {ok, Ref} = timer:apply_repeatedly(1000, ?MODULE, send, + [self(), Msg]), + ok = get_mess(1500, Msg, 3), + {ok, cancel} = timer:cancel(Ref), + nor = get_mess(1000, Msg). + +%% Test apply_repeatedly with the execution time of the action +%% longer than the timer interval. The timer should wait for +%% the action to complete, ie not start another action until it +%% has completed. +apply_repeatedly2(Config) when is_list(Config) -> + Msg = make_ref(), + Self = self(), + {ok, Ref} = timer:apply_repeatedly(1, erlang, apply, + [fun() -> + Self ! Msg, + receive after 1000 -> ok end + end, []]), + receive after 2500 -> ok end, + {ok, cancel} = timer:cancel(Ref), + ok = get_mess(1000, Msg, 3), + nor = get_mess(1000, Msg). + +%% Test that apply_repeatedly rejects invalid arguments. +apply_repeatedly_invalid_args(Config) when is_list(Config) -> + {error, badarg} = timer:apply_repeatedly(-1, foo, bar, []), + {error, badarg} = timer:apply_repeatedly(0, "foo", bar, []), + {error, badarg} = timer:apply_repeatedly(0, foo, "bar", []), + {error, badarg} = timer:apply_repeatedly(0, foo, bar, baz), + ok. + %% Test of send_interval/2. Receive 5 messages, cancel the timer, and %% check that we do not get any more messages. send_interval1(Config) when is_list(Config) -> @@ -661,7 +731,19 @@ tc(Config) when is_list(Config) -> true -> ok end, + %% tc/4 + {Res4, ok} = timer:tc(timer, sleep, [500], millisecond), + ok = if + Res4 < 500 -> {too_early, Res4}; + Res4 > 800 -> {too_late, Res4}; + true -> ok + end, + %% Check that timer:tc don't catch errors + ok = try timer:tc(erlang, exit, [foo], second) + catch exit:foo -> ok + end, + ok = try timer:tc(erlang, exit, [foo]) catch exit:foo -> ok end, @@ -676,6 +758,7 @@ tc(Config) when is_list(Config) -> %% Check that return values are propageted Self = self(), + {_, Self} = timer:tc(erlang, self, [], second), {_, Self} = timer:tc(erlang, self, []), {_, Self} = timer:tc(fun(P) -> P end, [self()]), {_, Self} = timer:tc(fun() -> self() end), diff --git a/lib/stdlib/test/unicode_util_SUITE.erl b/lib/stdlib/test/unicode_util_SUITE.erl index e4f3e5f379..42a119fba9 100644 --- a/lib/stdlib/test/unicode_util_SUITE.erl +++ b/lib/stdlib/test/unicode_util_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2017-2020. All Rights Reserved. +%% Copyright Ericsson AB 2017-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -27,6 +27,7 @@ nfd/1, nfc/1, nfkd/1, nfkc/1, whitespace/1, get/1, + lookup/1, count/1]). -export([debug/0, id/1, bin_split/1, uc_loaded_size/0, @@ -45,6 +46,7 @@ all() -> nfd, nfc, nfkd, nfkc, whitespace, get, + lookup, count ]. @@ -90,7 +92,7 @@ casefold(_) -> whitespace(_) -> WS = unicode_util:whitespace(), WS = lists:filter(fun unicode_util:is_whitespace/1, WS), - %% TODO add more tests + false = unicode_util:is_whitespace($A), ok. cp(_) -> @@ -101,6 +103,15 @@ cp(_) -> "hejsan" = fetch(["hej"|<<"san">>], Get), {error, <<128>>} = Get(<<128>>), {error, [<<128>>, 0]} = Get([<<128>>, 0]), + + {'EXIT', _} = catch Get([-1]), + {'EXIT', _} = catch Get([-1, $a]), + {'EXIT', _} = catch Get([foo, $a]), + {'EXIT', _} = catch Get([-1, $a]), + {'EXIT', _} = catch Get([[], -1]), + {'EXIT', _} = catch Get([[-1], $a]), + {'EXIT', _} = catch Get([[-1, $a], $a]), + ok. gc(Config) -> @@ -113,6 +124,15 @@ gc(Config) -> {error, <<128>>} = Get(<<128>>), {error, [<<128>>, 0]} = Get([<<128>>, 0]), + {'EXIT', _} = catch Get([-1]), + {'EXIT', _} = catch Get([-1, $a]), + {'EXIT', _} = catch Get([foo, $a]), + {'EXIT', _} = catch Get([-1, $a]), + {'EXIT', _} = catch Get([[], -1]), + {'EXIT', _} = catch Get([[-1], $a]), + {'EXIT', _} = catch Get([[-1, $a], $a]), + {'EXIT', _} = catch Get([<<$a>>, [-1, $a], $a]), %% Current impl + 0 = fold(fun verify_gc/3, 0, DataDir ++ "/GraphemeBreakTest.txt"), ok. @@ -324,6 +344,29 @@ verify_nfkc(Data0, LineNo, _Acc) -> get(_) -> add_get_tests. +lookup(Config) -> + DataDir = proplists:get_value(data_dir, Config), + {ok, Bin} = file:read_file(filename:join(DataDir, "unicode_table.bin")), + 0 = check_category(0, binary_to_term(Bin), 0), + ok. + +check_category(Id, [{Id, {_, _, _, What}}|Rest], Es) -> + case maps:get(category, unicode_util:lookup(Id)) of + What -> check_category(Id+1, Rest, Es); + _Err -> + io:format("~w Exp: ~w Got ~w~n",[Id, What, _Err]), exit(_Err), + check_category(Id+1, Rest, Es+1) + end; +check_category(Id, [{Next,_}|_] = Rest, Es) -> + case maps:get(category, unicode_util:lookup(Id)) of + {other, not_assigned} -> check_category(max(Id+1,Next-1), Rest, Es); + Err -> io:format("~w Exp: {other, not_assigned} Got ~w~n",[Id,Err]), + check_category(max(Id+1,Next-1), Rest, Es+1) + end; +check_category(_Id, [], Es) -> + Es. + + count(Config) -> Parent = self(), Exec = fun() -> diff --git a/lib/stdlib/test/unicode_util_SUITE_data/GraphemeBreakTest.txt b/lib/stdlib/test/unicode_util_SUITE_data/GraphemeBreakTest.txt index eff2fd33b0..3c73f97b7b 100644 --- a/lib/stdlib/test/unicode_util_SUITE_data/GraphemeBreakTest.txt +++ b/lib/stdlib/test/unicode_util_SUITE_data/GraphemeBreakTest.txt @@ -1,11 +1,11 @@ -# GraphemeBreakTest-14.0.0.txt -# Date: 2021-03-08, 06:22:32 GMT -# ยฉ 2021 Unicodeยฎ, Inc. +# GraphemeBreakTest-15.0.0.txt +# Date: 2022-02-26, 00:38:37 GMT +# ยฉ 2022 Unicodeยฎ, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. -# For terms of use, see http://www.unicode.org/terms_of_use.html +# For terms of use, see https://www.unicode.org/terms_of_use.html # # Unicode Character Database -# For documentation, see http://www.unicode.org/reports/tr44/ +# For documentation, see https://www.unicode.org/reports/tr44/ # # Default Grapheme_Cluster_Break Test # diff --git a/lib/stdlib/test/unicode_util_SUITE_data/LineBreakTest.txt b/lib/stdlib/test/unicode_util_SUITE_data/LineBreakTest.txt index 8d1cef0f78..3122a2e21e 100644 --- a/lib/stdlib/test/unicode_util_SUITE_data/LineBreakTest.txt +++ b/lib/stdlib/test/unicode_util_SUITE_data/LineBreakTest.txt @@ -1,11 +1,11 @@ -# LineBreakTest-14.0.0.txt -# Date: 2021-08-20, 21:08:45 GMT -# ยฉ 2021 Unicodeยฎ, Inc. +# LineBreakTest-15.0.0.txt +# Date: 2022-02-26, 00:38:39 GMT +# ยฉ 2022 Unicodeยฎ, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. -# For terms of use, see http://www.unicode.org/terms_of_use.html +# For terms of use, see https://www.unicode.org/terms_of_use.html # # Unicode Character Database -# For documentation, see http://www.unicode.org/reports/tr44/ +# For documentation, see https://www.unicode.org/reports/tr44/ # # Default Line_Break Test # diff --git a/lib/stdlib/test/unicode_util_SUITE_data/NormalizationTest.txt b/lib/stdlib/test/unicode_util_SUITE_data/NormalizationTest.txt index 302c35f37c..e75b4801c9 100644 --- a/lib/stdlib/test/unicode_util_SUITE_data/NormalizationTest.txt +++ b/lib/stdlib/test/unicode_util_SUITE_data/NormalizationTest.txt @@ -1,11 +1,11 @@ -# NormalizationTest-14.0.0.txt -# Date: 2021-05-28, 21:49:12 GMT -# ยฉ 2021 Unicodeยฎ, Inc. +# NormalizationTest-15.0.0.txt +# Date: 2022-04-02, 01:29:09 GMT +# ยฉ 2022 Unicodeยฎ, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. -# For terms of use, see http://www.unicode.org/terms_of_use.html +# For terms of use, see https://www.unicode.org/terms_of_use.html # # Unicode Character Database -# For documentation, see http://www.unicode.org/reports/tr44/ +# For documentation, see https://www.unicode.org/reports/tr44/ # # Normalization Test Suite # Format: @@ -16208,6 +16208,68 @@ FFEE;FFEE;FFEE;25CB;25CB; # (๏ฟฎ; ๏ฟฎ; ๏ฟฎ; โ; โ; ) HALFWIDTH WHITE CIRCLE 1D7FD;1D7FD;1D7FD;0037;0037; # (๐ฝ; ๐ฝ; ๐ฝ; 7; 7; ) MATHEMATICAL MONOSPACE DIGIT SEVEN 1D7FE;1D7FE;1D7FE;0038;0038; # (๐พ; ๐พ; ๐พ; 8; 8; ) MATHEMATICAL MONOSPACE DIGIT EIGHT 1D7FF;1D7FF;1D7FF;0039;0039; # (๐ฟ; ๐ฟ; ๐ฟ; 9; 9; ) MATHEMATICAL MONOSPACE DIGIT NINE +1E030;1E030;1E030;0430;0430; # (๐ฐ; ๐ฐ; ๐ฐ; ะฐ; ะฐ; ) MODIFIER LETTER CYRILLIC SMALL A +1E031;1E031;1E031;0431;0431; # (๐ฑ; ๐ฑ; ๐ฑ; ะฑ; ะฑ; ) MODIFIER LETTER CYRILLIC SMALL BE +1E032;1E032;1E032;0432;0432; # (๐ฒ; ๐ฒ; ๐ฒ; ะฒ; ะฒ; ) MODIFIER LETTER CYRILLIC SMALL VE +1E033;1E033;1E033;0433;0433; # (๐ณ; ๐ณ; ๐ณ; ะณ; ะณ; ) MODIFIER LETTER CYRILLIC SMALL GHE +1E034;1E034;1E034;0434;0434; # (๐ด; ๐ด; ๐ด; ะด; ะด; ) MODIFIER LETTER CYRILLIC SMALL DE +1E035;1E035;1E035;0435;0435; # (๐ต; ๐ต; ๐ต; ะต; ะต; ) MODIFIER LETTER CYRILLIC SMALL IE +1E036;1E036;1E036;0436;0436; # (๐ถ; ๐ถ; ๐ถ; ะถ; ะถ; ) MODIFIER LETTER CYRILLIC SMALL ZHE +1E037;1E037;1E037;0437;0437; # (๐ท; ๐ท; ๐ท; ะท; ะท; ) MODIFIER LETTER CYRILLIC SMALL ZE +1E038;1E038;1E038;0438;0438; # (๐ธ; ๐ธ; ๐ธ; ะธ; ะธ; ) MODIFIER LETTER CYRILLIC SMALL I +1E039;1E039;1E039;043A;043A; # (๐น; ๐น; ๐น; ะบ; ะบ; ) MODIFIER LETTER CYRILLIC SMALL KA +1E03A;1E03A;1E03A;043B;043B; # (๐บ; ๐บ; ๐บ; ะป; ะป; ) MODIFIER LETTER CYRILLIC SMALL EL +1E03B;1E03B;1E03B;043C;043C; # (๐ป; ๐ป; ๐ป; ะผ; ะผ; ) MODIFIER LETTER CYRILLIC SMALL EM +1E03C;1E03C;1E03C;043E;043E; # (๐ผ; ๐ผ; ๐ผ; ะพ; ะพ; ) MODIFIER LETTER CYRILLIC SMALL O +1E03D;1E03D;1E03D;043F;043F; # (๐ฝ; ๐ฝ; ๐ฝ; ะฟ; ะฟ; ) MODIFIER LETTER CYRILLIC SMALL PE +1E03E;1E03E;1E03E;0440;0440; # (๐พ; ๐พ; ๐พ; ั; ั; ) MODIFIER LETTER CYRILLIC SMALL ER +1E03F;1E03F;1E03F;0441;0441; # (๐ฟ; ๐ฟ; ๐ฟ; ั; ั; ) MODIFIER LETTER CYRILLIC SMALL ES +1E040;1E040;1E040;0442;0442; # (๐; ๐; ๐; ั; ั; ) MODIFIER LETTER CYRILLIC SMALL TE +1E041;1E041;1E041;0443;0443; # (๐; ๐; ๐; ั; ั; ) MODIFIER LETTER CYRILLIC SMALL U +1E042;1E042;1E042;0444;0444; # (๐; ๐; ๐; ั; ั; ) MODIFIER LETTER CYRILLIC SMALL EF +1E043;1E043;1E043;0445;0445; # (๐; ๐; ๐; ั
; ั
; ) MODIFIER LETTER CYRILLIC SMALL HA +1E044;1E044;1E044;0446;0446; # (๐; ๐; ๐; ั; ั; ) MODIFIER LETTER CYRILLIC SMALL TSE +1E045;1E045;1E045;0447;0447; # (๐
; ๐
; ๐
; ั; ั; ) MODIFIER LETTER CYRILLIC SMALL CHE +1E046;1E046;1E046;0448;0448; # (๐; ๐; ๐; ั; ั; ) MODIFIER LETTER CYRILLIC SMALL SHA +1E047;1E047;1E047;044B;044B; # (๐; ๐; ๐; ั; ั; ) MODIFIER LETTER CYRILLIC SMALL YERU +1E048;1E048;1E048;044D;044D; # (๐; ๐; ๐; ั; ั; ) MODIFIER LETTER CYRILLIC SMALL E +1E049;1E049;1E049;044E;044E; # (๐; ๐; ๐; ั; ั; ) MODIFIER LETTER CYRILLIC SMALL YU +1E04A;1E04A;1E04A;A689;A689; # (๐; ๐; ๐; ๊; ๊; ) MODIFIER LETTER CYRILLIC SMALL DZZE +1E04B;1E04B;1E04B;04D9;04D9; # (๐; ๐; ๐; ำ; ำ; ) MODIFIER LETTER CYRILLIC SMALL SCHWA +1E04C;1E04C;1E04C;0456;0456; # (๐; ๐; ๐; ั; ั; ) MODIFIER LETTER CYRILLIC SMALL BYELORUSSIAN-UKRAINIAN I +1E04D;1E04D;1E04D;0458;0458; # (๐; ๐; ๐; ั; ั; ) MODIFIER LETTER CYRILLIC SMALL JE +1E04E;1E04E;1E04E;04E9;04E9; # (๐; ๐; ๐; ำฉ; ำฉ; ) MODIFIER LETTER CYRILLIC SMALL BARRED O +1E04F;1E04F;1E04F;04AF;04AF; # (๐; ๐; ๐; าฏ; าฏ; ) MODIFIER LETTER CYRILLIC SMALL STRAIGHT U +1E050;1E050;1E050;04CF;04CF; # (๐; ๐; ๐; ำ; ำ; ) MODIFIER LETTER CYRILLIC SMALL PALOCHKA +1E051;1E051;1E051;0430;0430; # (๐; ๐; ๐; ะฐ; ะฐ; ) CYRILLIC SUBSCRIPT SMALL LETTER A +1E052;1E052;1E052;0431;0431; # (๐; ๐; ๐; ะฑ; ะฑ; ) CYRILLIC SUBSCRIPT SMALL LETTER BE +1E053;1E053;1E053;0432;0432; # (๐; ๐; ๐; ะฒ; ะฒ; ) CYRILLIC SUBSCRIPT SMALL LETTER VE +1E054;1E054;1E054;0433;0433; # (๐; ๐; ๐; ะณ; ะณ; ) CYRILLIC SUBSCRIPT SMALL LETTER GHE +1E055;1E055;1E055;0434;0434; # (๐; ๐; ๐; ะด; ะด; ) CYRILLIC SUBSCRIPT SMALL LETTER DE +1E056;1E056;1E056;0435;0435; # (๐; ๐; ๐; ะต; ะต; ) CYRILLIC SUBSCRIPT SMALL LETTER IE +1E057;1E057;1E057;0436;0436; # (๐; ๐; ๐; ะถ; ะถ; ) CYRILLIC SUBSCRIPT SMALL LETTER ZHE +1E058;1E058;1E058;0437;0437; # (๐; ๐; ๐; ะท; ะท; ) CYRILLIC SUBSCRIPT SMALL LETTER ZE +1E059;1E059;1E059;0438;0438; # (๐; ๐; ๐; ะธ; ะธ; ) CYRILLIC SUBSCRIPT SMALL LETTER I +1E05A;1E05A;1E05A;043A;043A; # (๐; ๐; ๐; ะบ; ะบ; ) CYRILLIC SUBSCRIPT SMALL LETTER KA +1E05B;1E05B;1E05B;043B;043B; # (๐; ๐; ๐; ะป; ะป; ) CYRILLIC SUBSCRIPT SMALL LETTER EL +1E05C;1E05C;1E05C;043E;043E; # (๐; ๐; ๐; ะพ; ะพ; ) CYRILLIC SUBSCRIPT SMALL LETTER O +1E05D;1E05D;1E05D;043F;043F; # (๐; ๐; ๐; ะฟ; ะฟ; ) CYRILLIC SUBSCRIPT SMALL LETTER PE +1E05E;1E05E;1E05E;0441;0441; # (๐; ๐; ๐; ั; ั; ) CYRILLIC SUBSCRIPT SMALL LETTER ES +1E05F;1E05F;1E05F;0443;0443; # (๐; ๐; ๐; ั; ั; ) CYRILLIC SUBSCRIPT SMALL LETTER U +1E060;1E060;1E060;0444;0444; # (๐ ; ๐ ; ๐ ; ั; ั; ) CYRILLIC SUBSCRIPT SMALL LETTER EF +1E061;1E061;1E061;0445;0445; # (๐ก; ๐ก; ๐ก; ั
; ั
; ) CYRILLIC SUBSCRIPT SMALL LETTER HA +1E062;1E062;1E062;0446;0446; # (๐ข; ๐ข; ๐ข; ั; ั; ) CYRILLIC SUBSCRIPT SMALL LETTER TSE +1E063;1E063;1E063;0447;0447; # (๐ฃ; ๐ฃ; ๐ฃ; ั; ั; ) CYRILLIC SUBSCRIPT SMALL LETTER CHE +1E064;1E064;1E064;0448;0448; # (๐ค; ๐ค; ๐ค; ั; ั; ) CYRILLIC SUBSCRIPT SMALL LETTER SHA +1E065;1E065;1E065;044A;044A; # (๐ฅ; ๐ฅ; ๐ฅ; ั; ั; ) CYRILLIC SUBSCRIPT SMALL LETTER HARD SIGN +1E066;1E066;1E066;044B;044B; # (๐ฆ; ๐ฆ; ๐ฆ; ั; ั; ) CYRILLIC SUBSCRIPT SMALL LETTER YERU +1E067;1E067;1E067;0491;0491; # (๐ง; ๐ง; ๐ง; า; า; ) CYRILLIC SUBSCRIPT SMALL LETTER GHE WITH UPTURN +1E068;1E068;1E068;0456;0456; # (๐จ; ๐จ; ๐จ; ั; ั; ) CYRILLIC SUBSCRIPT SMALL LETTER BYELORUSSIAN-UKRAINIAN I +1E069;1E069;1E069;0455;0455; # (๐ฉ; ๐ฉ; ๐ฉ; ั; ั; ) CYRILLIC SUBSCRIPT SMALL LETTER DZE +1E06A;1E06A;1E06A;045F;045F; # (๐ช; ๐ช; ๐ช; ั; ั; ) CYRILLIC SUBSCRIPT SMALL LETTER DZHE +1E06B;1E06B;1E06B;04AB;04AB; # (๐ซ; ๐ซ; ๐ซ; าซ; าซ; ) MODIFIER LETTER CYRILLIC SMALL ES WITH DESCENDER +1E06C;1E06C;1E06C;A651;A651; # (๐ฌ; ๐ฌ; ๐ฌ; ๊; ๊; ) MODIFIER LETTER CYRILLIC SMALL YERU WITH BACK YER +1E06D;1E06D;1E06D;04B1;04B1; # (๐ญ; ๐ญ; ๐ญ; าฑ; าฑ; ) MODIFIER LETTER CYRILLIC SMALL STRAIGHT U WITH STROKE 1EE00;1EE00;1EE00;0627;0627; # (๐ธ; ๐ธ; ๐ธ; ุง; ุง; ) ARABIC MATHEMATICAL ALEF 1EE01;1EE01;1EE01;0628;0628; # (๐ธ; ๐ธ; ๐ธ; ุจ; ุจ; ) ARABIC MATHEMATICAL BEH 1EE02;1EE02;1EE02;062C;062C; # (๐ธ; ๐ธ; ๐ธ; ุฌ; ุฌ; ) ARABIC MATHEMATICAL JEEM @@ -18496,6 +18558,12 @@ FFEE;FFEE;FFEE;25CB;25CB; # (๏ฟฎ; ๏ฟฎ; ๏ฟฎ; โ; โ; ) HALFWIDTH WHITE CIRCLE 0061 10EAB 0315 0300 05AE 0062;0061 05AE 10EAB 0300 0315 0062;0061 05AE 10EAB 0300 0315 0062;0061 05AE 10EAB 0300 0315 0062;0061 05AE 10EAB 0300 0315 0062; # (aโ๐บซโฬโฬโึฎb; aโึฎโ๐บซโฬโฬb; aโึฎโ๐บซโฬโฬb; aโึฎโ๐บซโฬโฬb; aโึฎโ๐บซโฬโฬb; ) LATIN SMALL LETTER A, YEZIDI COMBINING HAMZA MARK, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B 0061 0315 0300 05AE 10EAC 0062;00E0 05AE 10EAC 0315 0062;0061 05AE 0300 10EAC 0315 0062;00E0 05AE 10EAC 0315 0062;0061 05AE 0300 10EAC 0315 0062; # (aโฬโฬโึฎโ๐บฌb; ร โึฎโ๐บฌโฬb; aโึฎโฬโ๐บฌโฬb; ร โึฎโ๐บฌโฬb; aโึฎโฬโ๐บฌโฬb; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, YEZIDI COMBINING MADDA MARK, LATIN SMALL LETTER B 0061 10EAC 0315 0300 05AE 0062;0061 05AE 10EAC 0300 0315 0062;0061 05AE 10EAC 0300 0315 0062;0061 05AE 10EAC 0300 0315 0062;0061 05AE 10EAC 0300 0315 0062; # (aโ๐บฌโฬโฬโึฎb; aโึฎโ๐บฌโฬโฬb; aโึฎโ๐บฌโฬโฬb; aโึฎโ๐บฌโฬโฬb; aโึฎโ๐บฌโฬโฬb; ) LATIN SMALL LETTER A, YEZIDI COMBINING MADDA MARK, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 059A 0316 1DFA 10EFD 0062;0061 1DFA 0316 10EFD 059A 0062;0061 1DFA 0316 10EFD 059A 0062;0061 1DFA 0316 10EFD 059A 0062;0061 1DFA 0316 10EFD 059A 0062; # (aโึโฬโแทบโ๐ปฝb; aโแทบโฬโ๐ปฝโึb; aโแทบโฬโ๐ปฝโึb; aโแทบโฬโ๐ปฝโึb; aโแทบโฬโ๐ปฝโึb; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, COMBINING DOT BELOW LEFT, ARABIC SMALL LOW WORD SAKTA, LATIN SMALL LETTER B +0061 10EFD 059A 0316 1DFA 0062;0061 1DFA 10EFD 0316 059A 0062;0061 1DFA 10EFD 0316 059A 0062;0061 1DFA 10EFD 0316 059A 0062;0061 1DFA 10EFD 0316 059A 0062; # (aโ๐ปฝโึโฬโแทบb; aโแทบโ๐ปฝโฬโึb; aโแทบโ๐ปฝโฬโึb; aโแทบโ๐ปฝโฬโึb; aโแทบโ๐ปฝโฬโึb; ) LATIN SMALL LETTER A, ARABIC SMALL LOW WORD SAKTA, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, COMBINING DOT BELOW LEFT, LATIN SMALL LETTER B +0061 059A 0316 1DFA 10EFE 0062;0061 1DFA 0316 10EFE 059A 0062;0061 1DFA 0316 10EFE 059A 0062;0061 1DFA 0316 10EFE 059A 0062;0061 1DFA 0316 10EFE 059A 0062; # (aโึโฬโแทบโ๐ปพb; aโแทบโฬโ๐ปพโึb; aโแทบโฬโ๐ปพโึb; aโแทบโฬโ๐ปพโึb; aโแทบโฬโ๐ปพโึb; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, COMBINING DOT BELOW LEFT, ARABIC SMALL LOW WORD QASR, LATIN SMALL LETTER B +0061 10EFE 059A 0316 1DFA 0062;0061 1DFA 10EFE 0316 059A 0062;0061 1DFA 10EFE 0316 059A 0062;0061 1DFA 10EFE 0316 059A 0062;0061 1DFA 10EFE 0316 059A 0062; # (aโ๐ปพโึโฬโแทบb; aโแทบโ๐ปพโฬโึb; aโแทบโ๐ปพโฬโึb; aโแทบโ๐ปพโฬโึb; aโแทบโ๐ปพโฬโึb; ) LATIN SMALL LETTER A, ARABIC SMALL LOW WORD QASR, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, COMBINING DOT BELOW LEFT, LATIN SMALL LETTER B +0061 059A 0316 1DFA 10EFF 0062;0061 1DFA 0316 10EFF 059A 0062;0061 1DFA 0316 10EFF 059A 0062;0061 1DFA 0316 10EFF 059A 0062;0061 1DFA 0316 10EFF 059A 0062; # (aโึโฬโแทบโ๐ปฟb; aโแทบโฬโ๐ปฟโึb; aโแทบโฬโ๐ปฟโึb; aโแทบโฬโ๐ปฟโึb; aโแทบโฬโ๐ปฟโึb; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, COMBINING DOT BELOW LEFT, ARABIC SMALL LOW WORD MADDA, LATIN SMALL LETTER B +0061 10EFF 059A 0316 1DFA 0062;0061 1DFA 10EFF 0316 059A 0062;0061 1DFA 10EFF 0316 059A 0062;0061 1DFA 10EFF 0316 059A 0062;0061 1DFA 10EFF 0316 059A 0062; # (aโ๐ปฟโึโฬโแทบb; aโแทบโ๐ปฟโฬโึb; aโแทบโ๐ปฟโฬโึb; aโแทบโ๐ปฟโฬโึb; aโแทบโ๐ปฟโฬโึb; ) LATIN SMALL LETTER A, ARABIC SMALL LOW WORD MADDA, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, COMBINING DOT BELOW LEFT, LATIN SMALL LETTER B 0061 059A 0316 1DFA 10F46 0062;0061 1DFA 0316 10F46 059A 0062;0061 1DFA 0316 10F46 059A 0062;0061 1DFA 0316 10F46 059A 0062;0061 1DFA 0316 10F46 059A 0062; # (aโึโฬโแทบโ๐ฝb; aโแทบโฬโ๐ฝโึb; aโแทบโฬโ๐ฝโึb; aโแทบโฬโ๐ฝโึb; aโแทบโฬโ๐ฝโึb; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, COMBINING DOT BELOW LEFT, SOGDIAN COMBINING DOT BELOW, LATIN SMALL LETTER B 0061 10F46 059A 0316 1DFA 0062;0061 1DFA 10F46 0316 059A 0062;0061 1DFA 10F46 0316 059A 0062;0061 1DFA 10F46 0316 059A 0062;0061 1DFA 10F46 0316 059A 0062; # (aโ๐ฝโึโฬโแทบb; aโแทบโ๐ฝโฬโึb; aโแทบโ๐ฝโฬโึb; aโแทบโ๐ฝโฬโึb; aโแทบโ๐ฝโฬโึb; ) LATIN SMALL LETTER A, SOGDIAN COMBINING DOT BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, COMBINING DOT BELOW LEFT, LATIN SMALL LETTER B 0061 059A 0316 1DFA 10F47 0062;0061 1DFA 0316 10F47 059A 0062;0061 1DFA 0316 10F47 059A 0062;0061 1DFA 0316 10F47 059A 0062;0061 1DFA 0316 10F47 059A 0062; # (aโึโฬโแทบโ๐ฝb; aโแทบโฬโ๐ฝโึb; aโแทบโฬโ๐ฝโึb; aโแทบโฬโ๐ฝโึb; aโแทบโฬโ๐ฝโึb; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, COMBINING DOT BELOW LEFT, SOGDIAN COMBINING TWO DOTS BELOW, LATIN SMALL LETTER B @@ -18640,6 +18708,10 @@ FFEE;FFEE;FFEE;25CB;25CB; # (๏ฟฎ; ๏ฟฎ; ๏ฟฎ; โ; โ; ) HALFWIDTH WHITE CIRCLE 0061 11D45 05B0 094D 3099 0062;0061 3099 11D45 094D 05B0 0062;0061 3099 11D45 094D 05B0 0062;0061 3099 11D45 094D 05B0 0062;0061 3099 11D45 094D 05B0 0062; # (aโ๐ต
โึฐโเฅโใb; aโใโ๐ต
โเฅโึฐb; aโใโ๐ต
โเฅโึฐb; aโใโ๐ต
โเฅโึฐb; aโใโ๐ต
โเฅโึฐb; ) LATIN SMALL LETTER A, MASARAM GONDI VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B 0061 05B0 094D 3099 11D97 0062;0061 3099 094D 11D97 05B0 0062;0061 3099 094D 11D97 05B0 0062;0061 3099 094D 11D97 05B0 0062;0061 3099 094D 11D97 05B0 0062; # (aโึฐโเฅโใโ๐ถb; aโใโเฅโ๐ถโึฐb; aโใโเฅโ๐ถโึฐb; aโใโเฅโ๐ถโึฐb; aโใโเฅโ๐ถโึฐb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, GUNJALA GONDI VIRAMA, LATIN SMALL LETTER B 0061 11D97 05B0 094D 3099 0062;0061 3099 11D97 094D 05B0 0062;0061 3099 11D97 094D 05B0 0062;0061 3099 11D97 094D 05B0 0062;0061 3099 11D97 094D 05B0 0062; # (aโ๐ถโึฐโเฅโใb; aโใโ๐ถโเฅโึฐb; aโใโ๐ถโเฅโึฐb; aโใโ๐ถโเฅโึฐb; aโใโ๐ถโเฅโึฐb; ) LATIN SMALL LETTER A, GUNJALA GONDI VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B +0061 05B0 094D 3099 11F41 0062;0061 3099 094D 11F41 05B0 0062;0061 3099 094D 11F41 05B0 0062;0061 3099 094D 11F41 05B0 0062;0061 3099 094D 11F41 05B0 0062; # (aโึฐโเฅโใ๐ฝb; aโใโเฅ๐ฝโึฐb; aโใโเฅ๐ฝโึฐb; aโใโเฅ๐ฝโึฐb; aโใโเฅ๐ฝโึฐb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, KAWI SIGN KILLER, LATIN SMALL LETTER B +0061 11F41 05B0 094D 3099 0062;0061 3099 11F41 094D 05B0 0062;0061 3099 11F41 094D 05B0 0062;0061 3099 11F41 094D 05B0 0062;0061 3099 11F41 094D 05B0 0062; # (a๐ฝโึฐโเฅโใb; aโใ๐ฝโเฅโึฐb; aโใ๐ฝโเฅโึฐb; aโใ๐ฝโเฅโึฐb; aโใ๐ฝโเฅโึฐb; ) LATIN SMALL LETTER A, KAWI SIGN KILLER, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B +0061 05B0 094D 3099 11F42 0062;0061 3099 094D 11F42 05B0 0062;0061 3099 094D 11F42 05B0 0062;0061 3099 094D 11F42 05B0 0062;0061 3099 094D 11F42 05B0 0062; # (aโึฐโเฅโใโ๐ฝb; aโใโเฅโ๐ฝโึฐb; aโใโเฅโ๐ฝโึฐb; aโใโเฅโ๐ฝโึฐb; aโใโเฅโ๐ฝโึฐb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, KAWI CONJOINER, LATIN SMALL LETTER B +0061 11F42 05B0 094D 3099 0062;0061 3099 11F42 094D 05B0 0062;0061 3099 11F42 094D 05B0 0062;0061 3099 11F42 094D 05B0 0062;0061 3099 11F42 094D 05B0 0062; # (aโ๐ฝโึฐโเฅโใb; aโใโ๐ฝโเฅโึฐb; aโใโ๐ฝโเฅโึฐb; aโใโ๐ฝโเฅโึฐb; aโใโ๐ฝโเฅโึฐb; ) LATIN SMALL LETTER A, KAWI CONJOINER, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B 0061 16FF0 0334 16AF0 0062;0061 0334 16AF0 16FF0 0062;0061 0334 16AF0 16FF0 0062;0061 0334 16AF0 16FF0 0062;0061 0334 16AF0 16FF0 0062; # (a๐ฟฐโฬดโ๐ซฐb; aโฬดโ๐ซฐ๐ฟฐb; aโฬดโ๐ซฐ๐ฟฐb; aโฬดโ๐ซฐ๐ฟฐb; aโฬดโ๐ซฐ๐ฟฐb; ) LATIN SMALL LETTER A, VIETNAMESE ALTERNATE READING MARK CA, COMBINING TILDE OVERLAY, BASSA VAH COMBINING HIGH TONE, LATIN SMALL LETTER B 0061 16AF0 16FF0 0334 0062;0061 16AF0 0334 16FF0 0062;0061 16AF0 0334 16FF0 0062;0061 16AF0 0334 16FF0 0062;0061 16AF0 0334 16FF0 0062; # (aโ๐ซฐ๐ฟฐโฬดb; aโ๐ซฐโฬด๐ฟฐb; aโ๐ซฐโฬด๐ฟฐb; aโ๐ซฐโฬด๐ฟฐb; aโ๐ซฐโฬด๐ฟฐb; ) LATIN SMALL LETTER A, BASSA VAH COMBINING HIGH TONE, VIETNAMESE ALTERNATE READING MARK CA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B 0061 16FF0 0334 16AF1 0062;0061 0334 16AF1 16FF0 0062;0061 0334 16AF1 16FF0 0062;0061 0334 16AF1 16FF0 0062;0061 0334 16AF1 16FF0 0062; # (a๐ฟฐโฬดโ๐ซฑb; aโฬดโ๐ซฑ๐ฟฐb; aโฬดโ๐ซฑ๐ฟฐb; aโฬดโ๐ซฑ๐ฟฐb; aโฬดโ๐ซฑ๐ฟฐb; ) LATIN SMALL LETTER A, VIETNAMESE ALTERNATE READING MARK CA, COMBINING TILDE OVERLAY, BASSA VAH COMBINING LOW TONE, LATIN SMALL LETTER B @@ -18812,6 +18884,8 @@ FFEE;FFEE;FFEE;25CB;25CB; # (๏ฟฎ; ๏ฟฎ; ๏ฟฎ; โ; โ; ) HALFWIDTH WHITE CIRCLE 0061 1E029 0315 0300 05AE 0062;0061 05AE 1E029 0300 0315 0062;0061 05AE 1E029 0300 0315 0062;0061 05AE 1E029 0300 0315 0062;0061 05AE 1E029 0300 0315 0062; # (aโ๐ฉโฬโฬโึฎb; aโึฎโ๐ฉโฬโฬb; aโึฎโ๐ฉโฬโฬb; aโึฎโ๐ฉโฬโฬb; aโึฎโ๐ฉโฬโฬb; ) LATIN SMALL LETTER A, COMBINING GLAGOLITIC LETTER IOTATED BIG YUS, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B 0061 0315 0300 05AE 1E02A 0062;00E0 05AE 1E02A 0315 0062;0061 05AE 0300 1E02A 0315 0062;00E0 05AE 1E02A 0315 0062;0061 05AE 0300 1E02A 0315 0062; # (aโฬโฬโึฎโ๐ชb; ร โึฎโ๐ชโฬb; aโึฎโฬโ๐ชโฬb; ร โึฎโ๐ชโฬb; aโึฎโฬโ๐ชโฬb; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING GLAGOLITIC LETTER FITA, LATIN SMALL LETTER B 0061 1E02A 0315 0300 05AE 0062;0061 05AE 1E02A 0300 0315 0062;0061 05AE 1E02A 0300 0315 0062;0061 05AE 1E02A 0300 0315 0062;0061 05AE 1E02A 0300 0315 0062; # (aโ๐ชโฬโฬโึฎb; aโึฎโ๐ชโฬโฬb; aโึฎโ๐ชโฬโฬb; aโึฎโ๐ชโฬโฬb; aโึฎโ๐ชโฬโฬb; ) LATIN SMALL LETTER A, COMBINING GLAGOLITIC LETTER FITA, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 1E08F 0062;00E0 05AE 1E08F 0315 0062;0061 05AE 0300 1E08F 0315 0062;00E0 05AE 1E08F 0315 0062;0061 05AE 0300 1E08F 0315 0062; # (aโฬโฬโึฎโ๐b; ร โึฎโ๐โฬb; aโึฎโฬโ๐โฬb; ร โึฎโ๐โฬb; aโึฎโฬโ๐โฬb; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I, LATIN SMALL LETTER B +0061 1E08F 0315 0300 05AE 0062;0061 05AE 1E08F 0300 0315 0062;0061 05AE 1E08F 0300 0315 0062;0061 05AE 1E08F 0300 0315 0062;0061 05AE 1E08F 0300 0315 0062; # (aโ๐โฬโฬโึฎb; aโึฎโ๐โฬโฬb; aโึฎโ๐โฬโฬb; aโึฎโ๐โฬโฬb; aโึฎโ๐โฬโฬb; ) LATIN SMALL LETTER A, COMBINING CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B 0061 0315 0300 05AE 1E130 0062;00E0 05AE 1E130 0315 0062;0061 05AE 0300 1E130 0315 0062;00E0 05AE 1E130 0315 0062;0061 05AE 0300 1E130 0315 0062; # (aโฬโฬโึฎโ๐ฐb; ร โึฎโ๐ฐโฬb; aโึฎโฬโ๐ฐโฬb; ร โึฎโ๐ฐโฬb; aโึฎโฬโ๐ฐโฬb; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, NYIAKENG PUACHUE HMONG TONE-B, LATIN SMALL LETTER B 0061 1E130 0315 0300 05AE 0062;0061 05AE 1E130 0300 0315 0062;0061 05AE 1E130 0300 0315 0062;0061 05AE 1E130 0300 0315 0062;0061 05AE 1E130 0300 0315 0062; # (aโ๐ฐโฬโฬโึฎb; aโึฎโ๐ฐโฬโฬb; aโึฎโ๐ฐโฬโฬb; aโึฎโ๐ฐโฬโฬb; aโึฎโ๐ฐโฬโฬb; ) LATIN SMALL LETTER A, NYIAKENG PUACHUE HMONG TONE-B, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B 0061 0315 0300 05AE 1E131 0062;00E0 05AE 1E131 0315 0062;0061 05AE 0300 1E131 0315 0062;00E0 05AE 1E131 0315 0062;0061 05AE 0300 1E131 0315 0062; # (aโฬโฬโึฎโ๐ฑb; ร โึฎโ๐ฑโฬb; aโึฎโฬโ๐ฑโฬb; ร โึฎโ๐ฑโฬb; aโึฎโฬโ๐ฑโฬb; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, NYIAKENG PUACHUE HMONG TONE-M, LATIN SMALL LETTER B @@ -18836,6 +18910,14 @@ FFEE;FFEE;FFEE;25CB;25CB; # (๏ฟฎ; ๏ฟฎ; ๏ฟฎ; โ; โ; ) HALFWIDTH WHITE CIRCLE 0061 1E2EE 0315 0300 05AE 0062;0061 05AE 1E2EE 0300 0315 0062;0061 05AE 1E2EE 0300 0315 0062;0061 05AE 1E2EE 0300 0315 0062;0061 05AE 1E2EE 0300 0315 0062; # (aโ๐ฎโฬโฬโึฎb; aโึฎโ๐ฎโฬโฬb; aโึฎโ๐ฎโฬโฬb; aโึฎโ๐ฎโฬโฬb; aโึฎโ๐ฎโฬโฬb; ) LATIN SMALL LETTER A, WANCHO TONE KOI, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B 0061 0315 0300 05AE 1E2EF 0062;00E0 05AE 1E2EF 0315 0062;0061 05AE 0300 1E2EF 0315 0062;00E0 05AE 1E2EF 0315 0062;0061 05AE 0300 1E2EF 0315 0062; # (aโฬโฬโึฎโ๐ฏb; ร โึฎโ๐ฏโฬb; aโึฎโฬโ๐ฏโฬb; ร โึฎโ๐ฏโฬb; aโึฎโฬโ๐ฏโฬb; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, WANCHO TONE KOINI, LATIN SMALL LETTER B 0061 1E2EF 0315 0300 05AE 0062;0061 05AE 1E2EF 0300 0315 0062;0061 05AE 1E2EF 0300 0315 0062;0061 05AE 1E2EF 0300 0315 0062;0061 05AE 1E2EF 0300 0315 0062; # (aโ๐ฏโฬโฬโึฎb; aโึฎโ๐ฏโฬโฬb; aโึฎโ๐ฏโฬโฬb; aโึฎโ๐ฏโฬโฬb; aโึฎโ๐ฏโฬโฬb; ) LATIN SMALL LETTER A, WANCHO TONE KOINI, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 035C 0315 0300 1E4EC 0062;00E0 0315 1E4EC 035C 0062;0061 0300 0315 1E4EC 035C 0062;00E0 0315 1E4EC 035C 0062;0061 0300 0315 1E4EC 035C 0062; # (aโอโฬโฬโ๐ฌb; ร โฬโ๐ฌโอb; aโฬโฬโ๐ฌโอb; ร โฬโ๐ฌโอb; aโฬโฬโ๐ฌโอb; ) LATIN SMALL LETTER A, COMBINING DOUBLE BREVE BELOW, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, NAG MUNDARI SIGN MUHOR, LATIN SMALL LETTER B +0061 1E4EC 035C 0315 0300 0062;00E0 1E4EC 0315 035C 0062;0061 0300 1E4EC 0315 035C 0062;00E0 1E4EC 0315 035C 0062;0061 0300 1E4EC 0315 035C 0062; # (aโ๐ฌโอโฬโฬb; ร โ๐ฌโฬโอb; aโฬโ๐ฌโฬโอb; ร โ๐ฌโฬโอb; aโฬโ๐ฌโฬโอb; ) LATIN SMALL LETTER A, NAG MUNDARI SIGN MUHOR, COMBINING DOUBLE BREVE BELOW, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, LATIN SMALL LETTER B +0061 035C 0315 0300 1E4ED 0062;00E0 0315 1E4ED 035C 0062;0061 0300 0315 1E4ED 035C 0062;00E0 0315 1E4ED 035C 0062;0061 0300 0315 1E4ED 035C 0062; # (aโอโฬโฬโ๐ญb; ร โฬโ๐ญโอb; aโฬโฬโ๐ญโอb; ร โฬโ๐ญโอb; aโฬโฬโ๐ญโอb; ) LATIN SMALL LETTER A, COMBINING DOUBLE BREVE BELOW, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, NAG MUNDARI SIGN TOYOR, LATIN SMALL LETTER B +0061 1E4ED 035C 0315 0300 0062;00E0 1E4ED 0315 035C 0062;0061 0300 1E4ED 0315 035C 0062;00E0 1E4ED 0315 035C 0062;0061 0300 1E4ED 0315 035C 0062; # (aโ๐ญโอโฬโฬb; ร โ๐ญโฬโอb; aโฬโ๐ญโฬโอb; ร โ๐ญโฬโอb; aโฬโ๐ญโฬโอb; ) LATIN SMALL LETTER A, NAG MUNDARI SIGN TOYOR, COMBINING DOUBLE BREVE BELOW, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, LATIN SMALL LETTER B +0061 059A 0316 1DFA 1E4EE 0062;0061 1DFA 0316 1E4EE 059A 0062;0061 1DFA 0316 1E4EE 059A 0062;0061 1DFA 0316 1E4EE 059A 0062;0061 1DFA 0316 1E4EE 059A 0062; # (aโึโฬโแทบโ๐ฎb; aโแทบโฬโ๐ฎโึb; aโแทบโฬโ๐ฎโึb; aโแทบโฬโ๐ฎโึb; aโแทบโฬโ๐ฎโึb; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, COMBINING DOT BELOW LEFT, NAG MUNDARI SIGN IKIR, LATIN SMALL LETTER B +0061 1E4EE 059A 0316 1DFA 0062;0061 1DFA 1E4EE 0316 059A 0062;0061 1DFA 1E4EE 0316 059A 0062;0061 1DFA 1E4EE 0316 059A 0062;0061 1DFA 1E4EE 0316 059A 0062; # (aโ๐ฎโึโฬโแทบb; aโแทบโ๐ฎโฬโึb; aโแทบโ๐ฎโฬโึb; aโแทบโ๐ฎโฬโึb; aโแทบโ๐ฎโฬโึb; ) LATIN SMALL LETTER A, NAG MUNDARI SIGN IKIR, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, COMBINING DOT BELOW LEFT, LATIN SMALL LETTER B +0061 0315 0300 05AE 1E4EF 0062;00E0 05AE 1E4EF 0315 0062;0061 05AE 0300 1E4EF 0315 0062;00E0 05AE 1E4EF 0315 0062;0061 05AE 0300 1E4EF 0315 0062; # (aโฬโฬโึฎโ๐ฏb; ร โึฎโ๐ฏโฬb; aโึฎโฬโ๐ฏโฬb; ร โึฎโ๐ฏโฬb; aโึฎโฬโ๐ฏโฬb; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, NAG MUNDARI SIGN SUTUH, LATIN SMALL LETTER B +0061 1E4EF 0315 0300 05AE 0062;0061 05AE 1E4EF 0300 0315 0062;0061 05AE 1E4EF 0300 0315 0062;0061 05AE 1E4EF 0300 0315 0062;0061 05AE 1E4EF 0300 0315 0062; # (aโ๐ฏโฬโฬโึฎb; aโึฎโ๐ฏโฬโฬb; aโึฎโ๐ฏโฬโฬb; aโึฎโ๐ฏโฬโฬb; aโึฎโ๐ฏโฬโฬb; ) LATIN SMALL LETTER A, NAG MUNDARI SIGN SUTUH, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B 0061 059A 0316 1DFA 1E8D0 0062;0061 1DFA 0316 1E8D0 059A 0062;0061 1DFA 0316 1E8D0 059A 0062;0061 1DFA 0316 1E8D0 059A 0062;0061 1DFA 0316 1E8D0 059A 0062; # (aโึโฬโแทบโ๐ฃb; aโแทบโฬโ๐ฃโึb; aโแทบโฬโ๐ฃโึb; aโแทบโฬโ๐ฃโึb; aโแทบโฬโ๐ฃโึb; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, COMBINING DOT BELOW LEFT, MENDE KIKAKUI COMBINING NUMBER TEENS, LATIN SMALL LETTER B 0061 1E8D0 059A 0316 1DFA 0062;0061 1DFA 1E8D0 0316 059A 0062;0061 1DFA 1E8D0 0316 059A 0062;0061 1DFA 1E8D0 0316 059A 0062;0061 1DFA 1E8D0 0316 059A 0062; # (aโ๐ฃโึโฬโแทบb; aโแทบโ๐ฃโฬโึb; aโแทบโ๐ฃโฬโึb; aโแทบโ๐ฃโฬโึb; aโแทบโ๐ฃโฬโึb; ) LATIN SMALL LETTER A, MENDE KIKAKUI COMBINING NUMBER TEENS, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, COMBINING DOT BELOW LEFT, LATIN SMALL LETTER B 0061 059A 0316 1DFA 1E8D1 0062;0061 1DFA 0316 1E8D1 059A 0062;0061 1DFA 0316 1E8D1 059A 0062;0061 1DFA 0316 1E8D1 059A 0062;0061 1DFA 0316 1E8D1 059A 0062; # (aโึโฬโแทบโ๐ฃb; aโแทบโฬโ๐ฃโึb; aโแทบโฬโ๐ฃโึb; aโแทบโฬโ๐ฃโึb; aโแทบโฬโ๐ฃโึb; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, COMBINING DOT BELOW LEFT, MENDE KIKAKUI COMBINING NUMBER TENS, LATIN SMALL LETTER B diff --git a/lib/stdlib/test/unicode_util_SUITE_data/unicode_table.bin b/lib/stdlib/test/unicode_util_SUITE_data/unicode_table.bin Binary files differnew file mode 100644 index 0000000000..9ea04b334d --- /dev/null +++ b/lib/stdlib/test/unicode_util_SUITE_data/unicode_table.bin diff --git a/lib/stdlib/test/zip_SUITE.erl b/lib/stdlib/test/zip_SUITE.erl index 1709acc523..97e5c660dd 100644 --- a/lib/stdlib/test/zip_SUITE.erl +++ b/lib/stdlib/test/zip_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2006-2021. All Rights Reserved. +%% Copyright Ericsson AB 2006-2023. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -280,6 +280,8 @@ zip_api(Config) when is_list(Config) -> Name1 = hd(Names), {ok, Data1} = file:read_file(Name1), {ok, {Name1, Data1}} = zip:zip_get(Name1, ZipSrv), + Data1Crc = erlang:crc32(Data1), + {ok, Data1Crc} = zip:zip_get_crc32(Name1, ZipSrv), %% Get all files FilesDatas = lists:map(fun(Name) -> {ok, B} = file:read_file(Name), diff --git a/lib/stdlib/uc_spec/CaseFolding.txt b/lib/stdlib/uc_spec/CaseFolding.txt index 932ace29e6..65aa0fcd6b 100644 --- a/lib/stdlib/uc_spec/CaseFolding.txt +++ b/lib/stdlib/uc_spec/CaseFolding.txt @@ -1,11 +1,11 @@ -# CaseFolding-14.0.0.txt -# Date: 2021-03-08, 19:35:41 GMT -# ยฉ 2021 Unicodeยฎ, Inc. +# CaseFolding-15.0.0.txt +# Date: 2022-02-02, 23:35:35 GMT +# ยฉ 2022 Unicodeยฎ, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. -# For terms of use, see http://www.unicode.org/terms_of_use.html +# For terms of use, see https://www.unicode.org/terms_of_use.html # # Unicode Character Database -# For documentation, see http://www.unicode.org/reports/tr44/ +# For documentation, see https://www.unicode.org/reports/tr44/ # # Case Folding Properties # diff --git a/lib/stdlib/uc_spec/CompositionExclusions.txt b/lib/stdlib/uc_spec/CompositionExclusions.txt index 74e425e2a0..bbc8bd75d8 100644 --- a/lib/stdlib/uc_spec/CompositionExclusions.txt +++ b/lib/stdlib/uc_spec/CompositionExclusions.txt @@ -1,6 +1,6 @@ -# CompositionExclusions-14.0.0.txt -# Date: 2021-03-30, 23:59:00 GMT [KW, LI] -# ยฉ 2021 Unicodeยฎ, Inc. +# CompositionExclusions-15.0.0.txt +# Date: 2022-05-03, 18:50:00 GMT [KW, LI] +# ยฉ 2022 Unicodeยฎ, Inc. # For terms of use, see https://www.unicode.org/terms_of_use.html # # Unicode Character Database diff --git a/lib/stdlib/uc_spec/EastAsianWidth.txt b/lib/stdlib/uc_spec/EastAsianWidth.txt new file mode 100644 index 0000000000..38b7076c02 --- /dev/null +++ b/lib/stdlib/uc_spec/EastAsianWidth.txt @@ -0,0 +1,2619 @@ +# EastAsianWidth-15.0.0.txt +# Date: 2022-05-24, 17:40:20 GMT [KW, LI] +# ยฉ 2022 Unicodeยฎ, Inc. +# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. +# For terms of use, see https://www.unicode.org/terms_of_use.html +# +# Unicode Character Database +# For documentation, see https://www.unicode.org/reports/tr44/ +# +# East_Asian_Width Property +# +# This file is a normative contributory data file in the +# Unicode Character Database. +# +# The format is two fields separated by a semicolon. +# Field 0: Unicode code point value or range of code point values +# Field 1: East_Asian_Width property, consisting of one of the following values: +# "A", "F", "H", "N", "Na", "W" +# - All code points, assigned or unassigned, that are not listed +# explicitly are given the value "N". +# - The unassigned code points in the following blocks default to "W": +# CJK Unified Ideographs Extension A: U+3400..U+4DBF +# CJK Unified Ideographs: U+4E00..U+9FFF +# CJK Compatibility Ideographs: U+F900..U+FAFF +# - All undesignated code points in Planes 2 and 3, whether inside or +# outside of allocated blocks, default to "W": +# Plane 2: U+20000..U+2FFFD +# Plane 3: U+30000..U+3FFFD +# +# Character ranges are specified as for other property files in the +# Unicode Character Database. +# +# For legacy reasons, there are no spaces before or after the semicolon +# which separates the two fields. The comments following the number sign +# "#" list the General_Category property value or the L& alias of the +# derived value LC, the Unicode character name or names, and, in lines +# with ranges of code points, the code point count in square brackets. +# +# For more information, see UAX #11: East Asian Width, +# at https://www.unicode.org/reports/tr11/ +# +# @missing: 0000..10FFFF; N +0000..001F;N # Cc [32] <control-0000>..<control-001F> +0020;Na # Zs SPACE +0021..0023;Na # Po [3] EXCLAMATION MARK..NUMBER SIGN +0024;Na # Sc DOLLAR SIGN +0025..0027;Na # Po [3] PERCENT SIGN..APOSTROPHE +0028;Na # Ps LEFT PARENTHESIS +0029;Na # Pe RIGHT PARENTHESIS +002A;Na # Po ASTERISK +002B;Na # Sm PLUS SIGN +002C;Na # Po COMMA +002D;Na # Pd HYPHEN-MINUS +002E..002F;Na # Po [2] FULL STOP..SOLIDUS +0030..0039;Na # Nd [10] DIGIT ZERO..DIGIT NINE +003A..003B;Na # Po [2] COLON..SEMICOLON +003C..003E;Na # Sm [3] LESS-THAN SIGN..GREATER-THAN SIGN +003F..0040;Na # Po [2] QUESTION MARK..COMMERCIAL AT +0041..005A;Na # Lu [26] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER Z +005B;Na # Ps LEFT SQUARE BRACKET +005C;Na # Po REVERSE SOLIDUS +005D;Na # Pe RIGHT SQUARE BRACKET +005E;Na # Sk CIRCUMFLEX ACCENT +005F;Na # Pc LOW LINE +0060;Na # Sk GRAVE ACCENT +0061..007A;Na # Ll [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z +007B;Na # Ps LEFT CURLY BRACKET +007C;Na # Sm VERTICAL LINE +007D;Na # Pe RIGHT CURLY BRACKET +007E;Na # Sm TILDE +007F;N # Cc <control-007F> +0080..009F;N # Cc [32] <control-0080>..<control-009F> +00A0;N # Zs NO-BREAK SPACE +00A1;A # Po INVERTED EXCLAMATION MARK +00A2..00A3;Na # Sc [2] CENT SIGN..POUND SIGN +00A4;A # Sc CURRENCY SIGN +00A5;Na # Sc YEN SIGN +00A6;Na # So BROKEN BAR +00A7;A # Po SECTION SIGN +00A8;A # Sk DIAERESIS +00A9;N # So COPYRIGHT SIGN +00AA;A # Lo FEMININE ORDINAL INDICATOR +00AB;N # Pi LEFT-POINTING DOUBLE ANGLE QUOTATION MARK +00AC;Na # Sm NOT SIGN +00AD;A # Cf SOFT HYPHEN +00AE;A # So REGISTERED SIGN +00AF;Na # Sk MACRON +00B0;A # So DEGREE SIGN +00B1;A # Sm PLUS-MINUS SIGN +00B2..00B3;A # No [2] SUPERSCRIPT TWO..SUPERSCRIPT THREE +00B4;A # Sk ACUTE ACCENT +00B5;N # Ll MICRO SIGN +00B6..00B7;A # Po [2] PILCROW SIGN..MIDDLE DOT +00B8;A # Sk CEDILLA +00B9;A # No SUPERSCRIPT ONE +00BA;A # Lo MASCULINE ORDINAL INDICATOR +00BB;N # Pf RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK +00BC..00BE;A # No [3] VULGAR FRACTION ONE QUARTER..VULGAR FRACTION THREE QUARTERS +00BF;A # Po INVERTED QUESTION MARK +00C0..00C5;N # Lu [6] LATIN CAPITAL LETTER A WITH GRAVE..LATIN CAPITAL LETTER A WITH RING ABOVE +00C6;A # Lu LATIN CAPITAL LETTER AE +00C7..00CF;N # Lu [9] LATIN CAPITAL LETTER C WITH CEDILLA..LATIN CAPITAL LETTER I WITH DIAERESIS +00D0;A # Lu LATIN CAPITAL LETTER ETH +00D1..00D6;N # Lu [6] LATIN CAPITAL LETTER N WITH TILDE..LATIN CAPITAL LETTER O WITH DIAERESIS +00D7;A # Sm MULTIPLICATION SIGN +00D8;A # Lu LATIN CAPITAL LETTER O WITH STROKE +00D9..00DD;N # Lu [5] LATIN CAPITAL LETTER U WITH GRAVE..LATIN CAPITAL LETTER Y WITH ACUTE +00DE..00E1;A # L& [4] LATIN CAPITAL LETTER THORN..LATIN SMALL LETTER A WITH ACUTE +00E2..00E5;N # Ll [4] LATIN SMALL LETTER A WITH CIRCUMFLEX..LATIN SMALL LETTER A WITH RING ABOVE +00E6;A # Ll LATIN SMALL LETTER AE +00E7;N # Ll LATIN SMALL LETTER C WITH CEDILLA +00E8..00EA;A # Ll [3] LATIN SMALL LETTER E WITH GRAVE..LATIN SMALL LETTER E WITH CIRCUMFLEX +00EB;N # Ll LATIN SMALL LETTER E WITH DIAERESIS +00EC..00ED;A # Ll [2] LATIN SMALL LETTER I WITH GRAVE..LATIN SMALL LETTER I WITH ACUTE +00EE..00EF;N # Ll [2] LATIN SMALL LETTER I WITH CIRCUMFLEX..LATIN SMALL LETTER I WITH DIAERESIS +00F0;A # Ll LATIN SMALL LETTER ETH +00F1;N # Ll LATIN SMALL LETTER N WITH TILDE +00F2..00F3;A # Ll [2] LATIN SMALL LETTER O WITH GRAVE..LATIN SMALL LETTER O WITH ACUTE +00F4..00F6;N # Ll [3] LATIN SMALL LETTER O WITH CIRCUMFLEX..LATIN SMALL LETTER O WITH DIAERESIS +00F7;A # Sm DIVISION SIGN +00F8..00FA;A # Ll [3] LATIN SMALL LETTER O WITH STROKE..LATIN SMALL LETTER U WITH ACUTE +00FB;N # Ll LATIN SMALL LETTER U WITH CIRCUMFLEX +00FC;A # Ll LATIN SMALL LETTER U WITH DIAERESIS +00FD;N # Ll LATIN SMALL LETTER Y WITH ACUTE +00FE;A # Ll LATIN SMALL LETTER THORN +00FF;N # Ll LATIN SMALL LETTER Y WITH DIAERESIS +0100;N # Lu LATIN CAPITAL LETTER A WITH MACRON +0101;A # Ll LATIN SMALL LETTER A WITH MACRON +0102..0110;N # L& [15] LATIN CAPITAL LETTER A WITH BREVE..LATIN CAPITAL LETTER D WITH STROKE +0111;A # Ll LATIN SMALL LETTER D WITH STROKE +0112;N # Lu LATIN CAPITAL LETTER E WITH MACRON +0113;A # Ll LATIN SMALL LETTER E WITH MACRON +0114..011A;N # L& [7] LATIN CAPITAL LETTER E WITH BREVE..LATIN CAPITAL LETTER E WITH CARON +011B;A # Ll LATIN SMALL LETTER E WITH CARON +011C..0125;N # L& [10] LATIN CAPITAL LETTER G WITH CIRCUMFLEX..LATIN SMALL LETTER H WITH CIRCUMFLEX +0126..0127;A # L& [2] LATIN CAPITAL LETTER H WITH STROKE..LATIN SMALL LETTER H WITH STROKE +0128..012A;N # L& [3] LATIN CAPITAL LETTER I WITH TILDE..LATIN CAPITAL LETTER I WITH MACRON +012B;A # Ll LATIN SMALL LETTER I WITH MACRON +012C..0130;N # L& [5] LATIN CAPITAL LETTER I WITH BREVE..LATIN CAPITAL LETTER I WITH DOT ABOVE +0131..0133;A # L& [3] LATIN SMALL LETTER DOTLESS I..LATIN SMALL LIGATURE IJ +0134..0137;N # L& [4] LATIN CAPITAL LETTER J WITH CIRCUMFLEX..LATIN SMALL LETTER K WITH CEDILLA +0138;A # Ll LATIN SMALL LETTER KRA +0139..013E;N # L& [6] LATIN CAPITAL LETTER L WITH ACUTE..LATIN SMALL LETTER L WITH CARON +013F..0142;A # L& [4] LATIN CAPITAL LETTER L WITH MIDDLE DOT..LATIN SMALL LETTER L WITH STROKE +0143;N # Lu LATIN CAPITAL LETTER N WITH ACUTE +0144;A # Ll LATIN SMALL LETTER N WITH ACUTE +0145..0147;N # L& [3] LATIN CAPITAL LETTER N WITH CEDILLA..LATIN CAPITAL LETTER N WITH CARON +0148..014B;A # L& [4] LATIN SMALL LETTER N WITH CARON..LATIN SMALL LETTER ENG +014C;N # Lu LATIN CAPITAL LETTER O WITH MACRON +014D;A # Ll LATIN SMALL LETTER O WITH MACRON +014E..0151;N # L& [4] LATIN CAPITAL LETTER O WITH BREVE..LATIN SMALL LETTER O WITH DOUBLE ACUTE +0152..0153;A # L& [2] LATIN CAPITAL LIGATURE OE..LATIN SMALL LIGATURE OE +0154..0165;N # L& [18] LATIN CAPITAL LETTER R WITH ACUTE..LATIN SMALL LETTER T WITH CARON +0166..0167;A # L& [2] LATIN CAPITAL LETTER T WITH STROKE..LATIN SMALL LETTER T WITH STROKE +0168..016A;N # L& [3] LATIN CAPITAL LETTER U WITH TILDE..LATIN CAPITAL LETTER U WITH MACRON +016B;A # Ll LATIN SMALL LETTER U WITH MACRON +016C..017F;N # L& [20] LATIN CAPITAL LETTER U WITH BREVE..LATIN SMALL LETTER LONG S +0180..01BA;N # L& [59] LATIN SMALL LETTER B WITH STROKE..LATIN SMALL LETTER EZH WITH TAIL +01BB;N # Lo LATIN LETTER TWO WITH STROKE +01BC..01BF;N # L& [4] LATIN CAPITAL LETTER TONE FIVE..LATIN LETTER WYNN +01C0..01C3;N # Lo [4] LATIN LETTER DENTAL CLICK..LATIN LETTER RETROFLEX CLICK +01C4..01CD;N # L& [10] LATIN CAPITAL LETTER DZ WITH CARON..LATIN CAPITAL LETTER A WITH CARON +01CE;A # Ll LATIN SMALL LETTER A WITH CARON +01CF;N # Lu LATIN CAPITAL LETTER I WITH CARON +01D0;A # Ll LATIN SMALL LETTER I WITH CARON +01D1;N # Lu LATIN CAPITAL LETTER O WITH CARON +01D2;A # Ll LATIN SMALL LETTER O WITH CARON +01D3;N # Lu LATIN CAPITAL LETTER U WITH CARON +01D4;A # Ll LATIN SMALL LETTER U WITH CARON +01D5;N # Lu LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON +01D6;A # Ll LATIN SMALL LETTER U WITH DIAERESIS AND MACRON +01D7;N # Lu LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE +01D8;A # Ll LATIN SMALL LETTER U WITH DIAERESIS AND ACUTE +01D9;N # Lu LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON +01DA;A # Ll LATIN SMALL LETTER U WITH DIAERESIS AND CARON +01DB;N # Lu LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE +01DC;A # Ll LATIN SMALL LETTER U WITH DIAERESIS AND GRAVE +01DD..024F;N # L& [115] LATIN SMALL LETTER TURNED E..LATIN SMALL LETTER Y WITH STROKE +0250;N # Ll LATIN SMALL LETTER TURNED A +0251;A # Ll LATIN SMALL LETTER ALPHA +0252..0260;N # Ll [15] LATIN SMALL LETTER TURNED ALPHA..LATIN SMALL LETTER G WITH HOOK +0261;A # Ll LATIN SMALL LETTER SCRIPT G +0262..0293;N # Ll [50] LATIN LETTER SMALL CAPITAL G..LATIN SMALL LETTER EZH WITH CURL +0294;N # Lo LATIN LETTER GLOTTAL STOP +0295..02AF;N # Ll [27] LATIN LETTER PHARYNGEAL VOICED FRICATIVE..LATIN SMALL LETTER TURNED H WITH FISHHOOK AND TAIL +02B0..02C1;N # Lm [18] MODIFIER LETTER SMALL H..MODIFIER LETTER REVERSED GLOTTAL STOP +02C2..02C3;N # Sk [2] MODIFIER LETTER LEFT ARROWHEAD..MODIFIER LETTER RIGHT ARROWHEAD +02C4;A # Sk MODIFIER LETTER UP ARROWHEAD +02C5;N # Sk MODIFIER LETTER DOWN ARROWHEAD +02C6;N # Lm MODIFIER LETTER CIRCUMFLEX ACCENT +02C7;A # Lm CARON +02C8;N # Lm MODIFIER LETTER VERTICAL LINE +02C9..02CB;A # Lm [3] MODIFIER LETTER MACRON..MODIFIER LETTER GRAVE ACCENT +02CC;N # Lm MODIFIER LETTER LOW VERTICAL LINE +02CD;A # Lm MODIFIER LETTER LOW MACRON +02CE..02CF;N # Lm [2] MODIFIER LETTER LOW GRAVE ACCENT..MODIFIER LETTER LOW ACUTE ACCENT +02D0;A # Lm MODIFIER LETTER TRIANGULAR COLON +02D1;N # Lm MODIFIER LETTER HALF TRIANGULAR COLON +02D2..02D7;N # Sk [6] MODIFIER LETTER CENTRED RIGHT HALF RING..MODIFIER LETTER MINUS SIGN +02D8..02DB;A # Sk [4] BREVE..OGONEK +02DC;N # Sk SMALL TILDE +02DD;A # Sk DOUBLE ACUTE ACCENT +02DE;N # Sk MODIFIER LETTER RHOTIC HOOK +02DF;A # Sk MODIFIER LETTER CROSS ACCENT +02E0..02E4;N # Lm [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP +02E5..02EB;N # Sk [7] MODIFIER LETTER EXTRA-HIGH TONE BAR..MODIFIER LETTER YANG DEPARTING TONE MARK +02EC;N # Lm MODIFIER LETTER VOICING +02ED;N # Sk MODIFIER LETTER UNASPIRATED +02EE;N # Lm MODIFIER LETTER DOUBLE APOSTROPHE +02EF..02FF;N # Sk [17] MODIFIER LETTER LOW DOWN ARROWHEAD..MODIFIER LETTER LOW LEFT ARROW +0300..036F;A # Mn [112] COMBINING GRAVE ACCENT..COMBINING LATIN SMALL LETTER X +0370..0373;N # L& [4] GREEK CAPITAL LETTER HETA..GREEK SMALL LETTER ARCHAIC SAMPI +0374;N # Lm GREEK NUMERAL SIGN +0375;N # Sk GREEK LOWER NUMERAL SIGN +0376..0377;N # L& [2] GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA..GREEK SMALL LETTER PAMPHYLIAN DIGAMMA +037A;N # Lm GREEK YPOGEGRAMMENI +037B..037D;N # Ll [3] GREEK SMALL REVERSED LUNATE SIGMA SYMBOL..GREEK SMALL REVERSED DOTTED LUNATE SIGMA SYMBOL +037E;N # Po GREEK QUESTION MARK +037F;N # Lu GREEK CAPITAL LETTER YOT +0384..0385;N # Sk [2] GREEK TONOS..GREEK DIALYTIKA TONOS +0386;N # Lu GREEK CAPITAL LETTER ALPHA WITH TONOS +0387;N # Po GREEK ANO TELEIA +0388..038A;N # Lu [3] GREEK CAPITAL LETTER EPSILON WITH TONOS..GREEK CAPITAL LETTER IOTA WITH TONOS +038C;N # Lu GREEK CAPITAL LETTER OMICRON WITH TONOS +038E..0390;N # L& [3] GREEK CAPITAL LETTER UPSILON WITH TONOS..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS +0391..03A1;A # Lu [17] GREEK CAPITAL LETTER ALPHA..GREEK CAPITAL LETTER RHO +03A3..03A9;A # Lu [7] GREEK CAPITAL LETTER SIGMA..GREEK CAPITAL LETTER OMEGA +03AA..03B0;N # L& [7] GREEK CAPITAL LETTER IOTA WITH DIALYTIKA..GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS +03B1..03C1;A # Ll [17] GREEK SMALL LETTER ALPHA..GREEK SMALL LETTER RHO +03C2;N # Ll GREEK SMALL LETTER FINAL SIGMA +03C3..03C9;A # Ll [7] GREEK SMALL LETTER SIGMA..GREEK SMALL LETTER OMEGA +03CA..03F5;N # L& [44] GREEK SMALL LETTER IOTA WITH DIALYTIKA..GREEK LUNATE EPSILON SYMBOL +03F6;N # Sm GREEK REVERSED LUNATE EPSILON SYMBOL +03F7..03FF;N # L& [9] GREEK CAPITAL LETTER SHO..GREEK CAPITAL REVERSED DOTTED LUNATE SIGMA SYMBOL +0400;N # Lu CYRILLIC CAPITAL LETTER IE WITH GRAVE +0401;A # Lu CYRILLIC CAPITAL LETTER IO +0402..040F;N # Lu [14] CYRILLIC CAPITAL LETTER DJE..CYRILLIC CAPITAL LETTER DZHE +0410..044F;A # L& [64] CYRILLIC CAPITAL LETTER A..CYRILLIC SMALL LETTER YA +0450;N # Ll CYRILLIC SMALL LETTER IE WITH GRAVE +0451;A # Ll CYRILLIC SMALL LETTER IO +0452..0481;N # L& [48] CYRILLIC SMALL LETTER DJE..CYRILLIC SMALL LETTER KOPPA +0482;N # So CYRILLIC THOUSANDS SIGN +0483..0487;N # Mn [5] COMBINING CYRILLIC TITLO..COMBINING CYRILLIC POKRYTIE +0488..0489;N # Me [2] COMBINING CYRILLIC HUNDRED THOUSANDS SIGN..COMBINING CYRILLIC MILLIONS SIGN +048A..04FF;N # L& [118] CYRILLIC CAPITAL LETTER SHORT I WITH TAIL..CYRILLIC SMALL LETTER HA WITH STROKE +0500..052F;N # L& [48] CYRILLIC CAPITAL LETTER KOMI DE..CYRILLIC SMALL LETTER EL WITH DESCENDER +0531..0556;N # Lu [38] ARMENIAN CAPITAL LETTER AYB..ARMENIAN CAPITAL LETTER FEH +0559;N # Lm ARMENIAN MODIFIER LETTER LEFT HALF RING +055A..055F;N # Po [6] ARMENIAN APOSTROPHE..ARMENIAN ABBREVIATION MARK +0560..0588;N # Ll [41] ARMENIAN SMALL LETTER TURNED AYB..ARMENIAN SMALL LETTER YI WITH STROKE +0589;N # Po ARMENIAN FULL STOP +058A;N # Pd ARMENIAN HYPHEN +058D..058E;N # So [2] RIGHT-FACING ARMENIAN ETERNITY SIGN..LEFT-FACING ARMENIAN ETERNITY SIGN +058F;N # Sc ARMENIAN DRAM SIGN +0591..05BD;N # Mn [45] HEBREW ACCENT ETNAHTA..HEBREW POINT METEG +05BE;N # Pd HEBREW PUNCTUATION MAQAF +05BF;N # Mn HEBREW POINT RAFE +05C0;N # Po HEBREW PUNCTUATION PASEQ +05C1..05C2;N # Mn [2] HEBREW POINT SHIN DOT..HEBREW POINT SIN DOT +05C3;N # Po HEBREW PUNCTUATION SOF PASUQ +05C4..05C5;N # Mn [2] HEBREW MARK UPPER DOT..HEBREW MARK LOWER DOT +05C6;N # Po HEBREW PUNCTUATION NUN HAFUKHA +05C7;N # Mn HEBREW POINT QAMATS QATAN +05D0..05EA;N # Lo [27] HEBREW LETTER ALEF..HEBREW LETTER TAV +05EF..05F2;N # Lo [4] HEBREW YOD TRIANGLE..HEBREW LIGATURE YIDDISH DOUBLE YOD +05F3..05F4;N # Po [2] HEBREW PUNCTUATION GERESH..HEBREW PUNCTUATION GERSHAYIM +0600..0605;N # Cf [6] ARABIC NUMBER SIGN..ARABIC NUMBER MARK ABOVE +0606..0608;N # Sm [3] ARABIC-INDIC CUBE ROOT..ARABIC RAY +0609..060A;N # Po [2] ARABIC-INDIC PER MILLE SIGN..ARABIC-INDIC PER TEN THOUSAND SIGN +060B;N # Sc AFGHANI SIGN +060C..060D;N # Po [2] ARABIC COMMA..ARABIC DATE SEPARATOR +060E..060F;N # So [2] ARABIC POETIC VERSE SIGN..ARABIC SIGN MISRA +0610..061A;N # Mn [11] ARABIC SIGN SALLALLAHOU ALAYHE WASSALLAM..ARABIC SMALL KASRA +061B;N # Po ARABIC SEMICOLON +061C;N # Cf ARABIC LETTER MARK +061D..061F;N # Po [3] ARABIC END OF TEXT MARK..ARABIC QUESTION MARK +0620..063F;N # Lo [32] ARABIC LETTER KASHMIRI YEH..ARABIC LETTER FARSI YEH WITH THREE DOTS ABOVE +0640;N # Lm ARABIC TATWEEL +0641..064A;N # Lo [10] ARABIC LETTER FEH..ARABIC LETTER YEH +064B..065F;N # Mn [21] ARABIC FATHATAN..ARABIC WAVY HAMZA BELOW +0660..0669;N # Nd [10] ARABIC-INDIC DIGIT ZERO..ARABIC-INDIC DIGIT NINE +066A..066D;N # Po [4] ARABIC PERCENT SIGN..ARABIC FIVE POINTED STAR +066E..066F;N # Lo [2] ARABIC LETTER DOTLESS BEH..ARABIC LETTER DOTLESS QAF +0670;N # Mn ARABIC LETTER SUPERSCRIPT ALEF +0671..06D3;N # Lo [99] ARABIC LETTER ALEF WASLA..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE +06D4;N # Po ARABIC FULL STOP +06D5;N # Lo ARABIC LETTER AE +06D6..06DC;N # Mn [7] ARABIC SMALL HIGH LIGATURE SAD WITH LAM WITH ALEF MAKSURA..ARABIC SMALL HIGH SEEN +06DD;N # Cf ARABIC END OF AYAH +06DE;N # So ARABIC START OF RUB EL HIZB +06DF..06E4;N # Mn [6] ARABIC SMALL HIGH ROUNDED ZERO..ARABIC SMALL HIGH MADDA +06E5..06E6;N # Lm [2] ARABIC SMALL WAW..ARABIC SMALL YEH +06E7..06E8;N # Mn [2] ARABIC SMALL HIGH YEH..ARABIC SMALL HIGH NOON +06E9;N # So ARABIC PLACE OF SAJDAH +06EA..06ED;N # Mn [4] ARABIC EMPTY CENTRE LOW STOP..ARABIC SMALL LOW MEEM +06EE..06EF;N # Lo [2] ARABIC LETTER DAL WITH INVERTED V..ARABIC LETTER REH WITH INVERTED V +06F0..06F9;N # Nd [10] EXTENDED ARABIC-INDIC DIGIT ZERO..EXTENDED ARABIC-INDIC DIGIT NINE +06FA..06FC;N # Lo [3] ARABIC LETTER SHEEN WITH DOT BELOW..ARABIC LETTER GHAIN WITH DOT BELOW +06FD..06FE;N # So [2] ARABIC SIGN SINDHI AMPERSAND..ARABIC SIGN SINDHI POSTPOSITION MEN +06FF;N # Lo ARABIC LETTER HEH WITH INVERTED V +0700..070D;N # Po [14] SYRIAC END OF PARAGRAPH..SYRIAC HARKLEAN ASTERISCUS +070F;N # Cf SYRIAC ABBREVIATION MARK +0710;N # Lo SYRIAC LETTER ALAPH +0711;N # Mn SYRIAC LETTER SUPERSCRIPT ALAPH +0712..072F;N # Lo [30] SYRIAC LETTER BETH..SYRIAC LETTER PERSIAN DHALATH +0730..074A;N # Mn [27] SYRIAC PTHAHA ABOVE..SYRIAC BARREKH +074D..074F;N # Lo [3] SYRIAC LETTER SOGDIAN ZHAIN..SYRIAC LETTER SOGDIAN FE +0750..077F;N # Lo [48] ARABIC LETTER BEH WITH THREE DOTS HORIZONTALLY BELOW..ARABIC LETTER KAF WITH TWO DOTS ABOVE +0780..07A5;N # Lo [38] THAANA LETTER HAA..THAANA LETTER WAAVU +07A6..07B0;N # Mn [11] THAANA ABAFILI..THAANA SUKUN +07B1;N # Lo THAANA LETTER NAA +07C0..07C9;N # Nd [10] NKO DIGIT ZERO..NKO DIGIT NINE +07CA..07EA;N # Lo [33] NKO LETTER A..NKO LETTER JONA RA +07EB..07F3;N # Mn [9] NKO COMBINING SHORT HIGH TONE..NKO COMBINING DOUBLE DOT ABOVE +07F4..07F5;N # Lm [2] NKO HIGH TONE APOSTROPHE..NKO LOW TONE APOSTROPHE +07F6;N # So NKO SYMBOL OO DENNEN +07F7..07F9;N # Po [3] NKO SYMBOL GBAKURUNEN..NKO EXCLAMATION MARK +07FA;N # Lm NKO LAJANYALAN +07FD;N # Mn NKO DANTAYALAN +07FE..07FF;N # Sc [2] NKO DOROME SIGN..NKO TAMAN SIGN +0800..0815;N # Lo [22] SAMARITAN LETTER ALAF..SAMARITAN LETTER TAAF +0816..0819;N # Mn [4] SAMARITAN MARK IN..SAMARITAN MARK DAGESH +081A;N # Lm SAMARITAN MODIFIER LETTER EPENTHETIC YUT +081B..0823;N # Mn [9] SAMARITAN MARK EPENTHETIC YUT..SAMARITAN VOWEL SIGN A +0824;N # Lm SAMARITAN MODIFIER LETTER SHORT A +0825..0827;N # Mn [3] SAMARITAN VOWEL SIGN SHORT A..SAMARITAN VOWEL SIGN U +0828;N # Lm SAMARITAN MODIFIER LETTER I +0829..082D;N # Mn [5] SAMARITAN VOWEL SIGN LONG I..SAMARITAN MARK NEQUDAA +0830..083E;N # Po [15] SAMARITAN PUNCTUATION NEQUDAA..SAMARITAN PUNCTUATION ANNAAU +0840..0858;N # Lo [25] MANDAIC LETTER HALQA..MANDAIC LETTER AIN +0859..085B;N # Mn [3] MANDAIC AFFRICATION MARK..MANDAIC GEMINATION MARK +085E;N # Po MANDAIC PUNCTUATION +0860..086A;N # Lo [11] SYRIAC LETTER MALAYALAM NGA..SYRIAC LETTER MALAYALAM SSA +0870..0887;N # Lo [24] ARABIC LETTER ALEF WITH ATTACHED FATHA..ARABIC BASELINE ROUND DOT +0888;N # Sk ARABIC RAISED ROUND DOT +0889..088E;N # Lo [6] ARABIC LETTER NOON WITH INVERTED SMALL V..ARABIC VERTICAL TAIL +0890..0891;N # Cf [2] ARABIC POUND MARK ABOVE..ARABIC PIASTRE MARK ABOVE +0898..089F;N # Mn [8] ARABIC SMALL HIGH WORD AL-JUZ..ARABIC HALF MADDA OVER MADDA +08A0..08C8;N # Lo [41] ARABIC LETTER BEH WITH SMALL V BELOW..ARABIC LETTER GRAF +08C9;N # Lm ARABIC SMALL FARSI YEH +08CA..08E1;N # Mn [24] ARABIC SMALL HIGH FARSI YEH..ARABIC SMALL HIGH SIGN SAFHA +08E2;N # Cf ARABIC DISPUTED END OF AYAH +08E3..08FF;N # Mn [29] ARABIC TURNED DAMMA BELOW..ARABIC MARK SIDEWAYS NOON GHUNNA +0900..0902;N # Mn [3] DEVANAGARI SIGN INVERTED CANDRABINDU..DEVANAGARI SIGN ANUSVARA +0903;N # Mc DEVANAGARI SIGN VISARGA +0904..0939;N # Lo [54] DEVANAGARI LETTER SHORT A..DEVANAGARI LETTER HA +093A;N # Mn DEVANAGARI VOWEL SIGN OE +093B;N # Mc DEVANAGARI VOWEL SIGN OOE +093C;N # Mn DEVANAGARI SIGN NUKTA +093D;N # Lo DEVANAGARI SIGN AVAGRAHA +093E..0940;N # Mc [3] DEVANAGARI VOWEL SIGN AA..DEVANAGARI VOWEL SIGN II +0941..0948;N # Mn [8] DEVANAGARI VOWEL SIGN U..DEVANAGARI VOWEL SIGN AI +0949..094C;N # Mc [4] DEVANAGARI VOWEL SIGN CANDRA O..DEVANAGARI VOWEL SIGN AU +094D;N # Mn DEVANAGARI SIGN VIRAMA +094E..094F;N # Mc [2] DEVANAGARI VOWEL SIGN PRISHTHAMATRA E..DEVANAGARI VOWEL SIGN AW +0950;N # Lo DEVANAGARI OM +0951..0957;N # Mn [7] DEVANAGARI STRESS SIGN UDATTA..DEVANAGARI VOWEL SIGN UUE +0958..0961;N # Lo [10] DEVANAGARI LETTER QA..DEVANAGARI LETTER VOCALIC LL +0962..0963;N # Mn [2] DEVANAGARI VOWEL SIGN VOCALIC L..DEVANAGARI VOWEL SIGN VOCALIC LL +0964..0965;N # Po [2] DEVANAGARI DANDA..DEVANAGARI DOUBLE DANDA +0966..096F;N # Nd [10] DEVANAGARI DIGIT ZERO..DEVANAGARI DIGIT NINE +0970;N # Po DEVANAGARI ABBREVIATION SIGN +0971;N # Lm DEVANAGARI SIGN HIGH SPACING DOT +0972..097F;N # Lo [14] DEVANAGARI LETTER CANDRA A..DEVANAGARI LETTER BBA +0980;N # Lo BENGALI ANJI +0981;N # Mn BENGALI SIGN CANDRABINDU +0982..0983;N # Mc [2] BENGALI SIGN ANUSVARA..BENGALI SIGN VISARGA +0985..098C;N # Lo [8] BENGALI LETTER A..BENGALI LETTER VOCALIC L +098F..0990;N # Lo [2] BENGALI LETTER E..BENGALI LETTER AI +0993..09A8;N # Lo [22] BENGALI LETTER O..BENGALI LETTER NA +09AA..09B0;N # Lo [7] BENGALI LETTER PA..BENGALI LETTER RA +09B2;N # Lo BENGALI LETTER LA +09B6..09B9;N # Lo [4] BENGALI LETTER SHA..BENGALI LETTER HA +09BC;N # Mn BENGALI SIGN NUKTA +09BD;N # Lo BENGALI SIGN AVAGRAHA +09BE..09C0;N # Mc [3] BENGALI VOWEL SIGN AA..BENGALI VOWEL SIGN II +09C1..09C4;N # Mn [4] BENGALI VOWEL SIGN U..BENGALI VOWEL SIGN VOCALIC RR +09C7..09C8;N # Mc [2] BENGALI VOWEL SIGN E..BENGALI VOWEL SIGN AI +09CB..09CC;N # Mc [2] BENGALI VOWEL SIGN O..BENGALI VOWEL SIGN AU +09CD;N # Mn BENGALI SIGN VIRAMA +09CE;N # Lo BENGALI LETTER KHANDA TA +09D7;N # Mc BENGALI AU LENGTH MARK +09DC..09DD;N # Lo [2] BENGALI LETTER RRA..BENGALI LETTER RHA +09DF..09E1;N # Lo [3] BENGALI LETTER YYA..BENGALI LETTER VOCALIC LL +09E2..09E3;N # Mn [2] BENGALI VOWEL SIGN VOCALIC L..BENGALI VOWEL SIGN VOCALIC LL +09E6..09EF;N # Nd [10] BENGALI DIGIT ZERO..BENGALI DIGIT NINE +09F0..09F1;N # Lo [2] BENGALI LETTER RA WITH MIDDLE DIAGONAL..BENGALI LETTER RA WITH LOWER DIAGONAL +09F2..09F3;N # Sc [2] BENGALI RUPEE MARK..BENGALI RUPEE SIGN +09F4..09F9;N # No [6] BENGALI CURRENCY NUMERATOR ONE..BENGALI CURRENCY DENOMINATOR SIXTEEN +09FA;N # So BENGALI ISSHAR +09FB;N # Sc BENGALI GANDA MARK +09FC;N # Lo BENGALI LETTER VEDIC ANUSVARA +09FD;N # Po BENGALI ABBREVIATION SIGN +09FE;N # Mn BENGALI SANDHI MARK +0A01..0A02;N # Mn [2] GURMUKHI SIGN ADAK BINDI..GURMUKHI SIGN BINDI +0A03;N # Mc GURMUKHI SIGN VISARGA +0A05..0A0A;N # Lo [6] GURMUKHI LETTER A..GURMUKHI LETTER UU +0A0F..0A10;N # Lo [2] GURMUKHI LETTER EE..GURMUKHI LETTER AI +0A13..0A28;N # Lo [22] GURMUKHI LETTER OO..GURMUKHI LETTER NA +0A2A..0A30;N # Lo [7] GURMUKHI LETTER PA..GURMUKHI LETTER RA +0A32..0A33;N # Lo [2] GURMUKHI LETTER LA..GURMUKHI LETTER LLA +0A35..0A36;N # Lo [2] GURMUKHI LETTER VA..GURMUKHI LETTER SHA +0A38..0A39;N # Lo [2] GURMUKHI LETTER SA..GURMUKHI LETTER HA +0A3C;N # Mn GURMUKHI SIGN NUKTA +0A3E..0A40;N # Mc [3] GURMUKHI VOWEL SIGN AA..GURMUKHI VOWEL SIGN II +0A41..0A42;N # Mn [2] GURMUKHI VOWEL SIGN U..GURMUKHI VOWEL SIGN UU +0A47..0A48;N # Mn [2] GURMUKHI VOWEL SIGN EE..GURMUKHI VOWEL SIGN AI +0A4B..0A4D;N # Mn [3] GURMUKHI VOWEL SIGN OO..GURMUKHI SIGN VIRAMA +0A51;N # Mn GURMUKHI SIGN UDAAT +0A59..0A5C;N # Lo [4] GURMUKHI LETTER KHHA..GURMUKHI LETTER RRA +0A5E;N # Lo GURMUKHI LETTER FA +0A66..0A6F;N # Nd [10] GURMUKHI DIGIT ZERO..GURMUKHI DIGIT NINE +0A70..0A71;N # Mn [2] GURMUKHI TIPPI..GURMUKHI ADDAK +0A72..0A74;N # Lo [3] GURMUKHI IRI..GURMUKHI EK ONKAR +0A75;N # Mn GURMUKHI SIGN YAKASH +0A76;N # Po GURMUKHI ABBREVIATION SIGN +0A81..0A82;N # Mn [2] GUJARATI SIGN CANDRABINDU..GUJARATI SIGN ANUSVARA +0A83;N # Mc GUJARATI SIGN VISARGA +0A85..0A8D;N # Lo [9] GUJARATI LETTER A..GUJARATI VOWEL CANDRA E +0A8F..0A91;N # Lo [3] GUJARATI LETTER E..GUJARATI VOWEL CANDRA O +0A93..0AA8;N # Lo [22] GUJARATI LETTER O..GUJARATI LETTER NA +0AAA..0AB0;N # Lo [7] GUJARATI LETTER PA..GUJARATI LETTER RA +0AB2..0AB3;N # Lo [2] GUJARATI LETTER LA..GUJARATI LETTER LLA +0AB5..0AB9;N # Lo [5] GUJARATI LETTER VA..GUJARATI LETTER HA +0ABC;N # Mn GUJARATI SIGN NUKTA +0ABD;N # Lo GUJARATI SIGN AVAGRAHA +0ABE..0AC0;N # Mc [3] GUJARATI VOWEL SIGN AA..GUJARATI VOWEL SIGN II +0AC1..0AC5;N # Mn [5] GUJARATI VOWEL SIGN U..GUJARATI VOWEL SIGN CANDRA E +0AC7..0AC8;N # Mn [2] GUJARATI VOWEL SIGN E..GUJARATI VOWEL SIGN AI +0AC9;N # Mc GUJARATI VOWEL SIGN CANDRA O +0ACB..0ACC;N # Mc [2] GUJARATI VOWEL SIGN O..GUJARATI VOWEL SIGN AU +0ACD;N # Mn GUJARATI SIGN VIRAMA +0AD0;N # Lo GUJARATI OM +0AE0..0AE1;N # Lo [2] GUJARATI LETTER VOCALIC RR..GUJARATI LETTER VOCALIC LL +0AE2..0AE3;N # Mn [2] GUJARATI VOWEL SIGN VOCALIC L..GUJARATI VOWEL SIGN VOCALIC LL +0AE6..0AEF;N # Nd [10] GUJARATI DIGIT ZERO..GUJARATI DIGIT NINE +0AF0;N # Po GUJARATI ABBREVIATION SIGN +0AF1;N # Sc GUJARATI RUPEE SIGN +0AF9;N # Lo GUJARATI LETTER ZHA +0AFA..0AFF;N # Mn [6] GUJARATI SIGN SUKUN..GUJARATI SIGN TWO-CIRCLE NUKTA ABOVE +0B01;N # Mn ORIYA SIGN CANDRABINDU +0B02..0B03;N # Mc [2] ORIYA SIGN ANUSVARA..ORIYA SIGN VISARGA +0B05..0B0C;N # Lo [8] ORIYA LETTER A..ORIYA LETTER VOCALIC L +0B0F..0B10;N # Lo [2] ORIYA LETTER E..ORIYA LETTER AI +0B13..0B28;N # Lo [22] ORIYA LETTER O..ORIYA LETTER NA +0B2A..0B30;N # Lo [7] ORIYA LETTER PA..ORIYA LETTER RA +0B32..0B33;N # Lo [2] ORIYA LETTER LA..ORIYA LETTER LLA +0B35..0B39;N # Lo [5] ORIYA LETTER VA..ORIYA LETTER HA +0B3C;N # Mn ORIYA SIGN NUKTA +0B3D;N # Lo ORIYA SIGN AVAGRAHA +0B3E;N # Mc ORIYA VOWEL SIGN AA +0B3F;N # Mn ORIYA VOWEL SIGN I +0B40;N # Mc ORIYA VOWEL SIGN II +0B41..0B44;N # Mn [4] ORIYA VOWEL SIGN U..ORIYA VOWEL SIGN VOCALIC RR +0B47..0B48;N # Mc [2] ORIYA VOWEL SIGN E..ORIYA VOWEL SIGN AI +0B4B..0B4C;N # Mc [2] ORIYA VOWEL SIGN O..ORIYA VOWEL SIGN AU +0B4D;N # Mn ORIYA SIGN VIRAMA +0B55..0B56;N # Mn [2] ORIYA SIGN OVERLINE..ORIYA AI LENGTH MARK +0B57;N # Mc ORIYA AU LENGTH MARK +0B5C..0B5D;N # Lo [2] ORIYA LETTER RRA..ORIYA LETTER RHA +0B5F..0B61;N # Lo [3] ORIYA LETTER YYA..ORIYA LETTER VOCALIC LL +0B62..0B63;N # Mn [2] ORIYA VOWEL SIGN VOCALIC L..ORIYA VOWEL SIGN VOCALIC LL +0B66..0B6F;N # Nd [10] ORIYA DIGIT ZERO..ORIYA DIGIT NINE +0B70;N # So ORIYA ISSHAR +0B71;N # Lo ORIYA LETTER WA +0B72..0B77;N # No [6] ORIYA FRACTION ONE QUARTER..ORIYA FRACTION THREE SIXTEENTHS +0B82;N # Mn TAMIL SIGN ANUSVARA +0B83;N # Lo TAMIL SIGN VISARGA +0B85..0B8A;N # Lo [6] TAMIL LETTER A..TAMIL LETTER UU +0B8E..0B90;N # Lo [3] TAMIL LETTER E..TAMIL LETTER AI +0B92..0B95;N # Lo [4] TAMIL LETTER O..TAMIL LETTER KA +0B99..0B9A;N # Lo [2] TAMIL LETTER NGA..TAMIL LETTER CA +0B9C;N # Lo TAMIL LETTER JA +0B9E..0B9F;N # Lo [2] TAMIL LETTER NYA..TAMIL LETTER TTA +0BA3..0BA4;N # Lo [2] TAMIL LETTER NNA..TAMIL LETTER TA +0BA8..0BAA;N # Lo [3] TAMIL LETTER NA..TAMIL LETTER PA +0BAE..0BB9;N # Lo [12] TAMIL LETTER MA..TAMIL LETTER HA +0BBE..0BBF;N # Mc [2] TAMIL VOWEL SIGN AA..TAMIL VOWEL SIGN I +0BC0;N # Mn TAMIL VOWEL SIGN II +0BC1..0BC2;N # Mc [2] TAMIL VOWEL SIGN U..TAMIL VOWEL SIGN UU +0BC6..0BC8;N # Mc [3] TAMIL VOWEL SIGN E..TAMIL VOWEL SIGN AI +0BCA..0BCC;N # Mc [3] TAMIL VOWEL SIGN O..TAMIL VOWEL SIGN AU +0BCD;N # Mn TAMIL SIGN VIRAMA +0BD0;N # Lo TAMIL OM +0BD7;N # Mc TAMIL AU LENGTH MARK +0BE6..0BEF;N # Nd [10] TAMIL DIGIT ZERO..TAMIL DIGIT NINE +0BF0..0BF2;N # No [3] TAMIL NUMBER TEN..TAMIL NUMBER ONE THOUSAND +0BF3..0BF8;N # So [6] TAMIL DAY SIGN..TAMIL AS ABOVE SIGN +0BF9;N # Sc TAMIL RUPEE SIGN +0BFA;N # So TAMIL NUMBER SIGN +0C00;N # Mn TELUGU SIGN COMBINING CANDRABINDU ABOVE +0C01..0C03;N # Mc [3] TELUGU SIGN CANDRABINDU..TELUGU SIGN VISARGA +0C04;N # Mn TELUGU SIGN COMBINING ANUSVARA ABOVE +0C05..0C0C;N # Lo [8] TELUGU LETTER A..TELUGU LETTER VOCALIC L +0C0E..0C10;N # Lo [3] TELUGU LETTER E..TELUGU LETTER AI +0C12..0C28;N # Lo [23] TELUGU LETTER O..TELUGU LETTER NA +0C2A..0C39;N # Lo [16] TELUGU LETTER PA..TELUGU LETTER HA +0C3C;N # Mn TELUGU SIGN NUKTA +0C3D;N # Lo TELUGU SIGN AVAGRAHA +0C3E..0C40;N # Mn [3] TELUGU VOWEL SIGN AA..TELUGU VOWEL SIGN II +0C41..0C44;N # Mc [4] TELUGU VOWEL SIGN U..TELUGU VOWEL SIGN VOCALIC RR +0C46..0C48;N # Mn [3] TELUGU VOWEL SIGN E..TELUGU VOWEL SIGN AI +0C4A..0C4D;N # Mn [4] TELUGU VOWEL SIGN O..TELUGU SIGN VIRAMA +0C55..0C56;N # Mn [2] TELUGU LENGTH MARK..TELUGU AI LENGTH MARK +0C58..0C5A;N # Lo [3] TELUGU LETTER TSA..TELUGU LETTER RRRA +0C5D;N # Lo TELUGU LETTER NAKAARA POLLU +0C60..0C61;N # Lo [2] TELUGU LETTER VOCALIC RR..TELUGU LETTER VOCALIC LL +0C62..0C63;N # Mn [2] TELUGU VOWEL SIGN VOCALIC L..TELUGU VOWEL SIGN VOCALIC LL +0C66..0C6F;N # Nd [10] TELUGU DIGIT ZERO..TELUGU DIGIT NINE +0C77;N # Po TELUGU SIGN SIDDHAM +0C78..0C7E;N # No [7] TELUGU FRACTION DIGIT ZERO FOR ODD POWERS OF FOUR..TELUGU FRACTION DIGIT THREE FOR EVEN POWERS OF FOUR +0C7F;N # So TELUGU SIGN TUUMU +0C80;N # Lo KANNADA SIGN SPACING CANDRABINDU +0C81;N # Mn KANNADA SIGN CANDRABINDU +0C82..0C83;N # Mc [2] KANNADA SIGN ANUSVARA..KANNADA SIGN VISARGA +0C84;N # Po KANNADA SIGN SIDDHAM +0C85..0C8C;N # Lo [8] KANNADA LETTER A..KANNADA LETTER VOCALIC L +0C8E..0C90;N # Lo [3] KANNADA LETTER E..KANNADA LETTER AI +0C92..0CA8;N # Lo [23] KANNADA LETTER O..KANNADA LETTER NA +0CAA..0CB3;N # Lo [10] KANNADA LETTER PA..KANNADA LETTER LLA +0CB5..0CB9;N # Lo [5] KANNADA LETTER VA..KANNADA LETTER HA +0CBC;N # Mn KANNADA SIGN NUKTA +0CBD;N # Lo KANNADA SIGN AVAGRAHA +0CBE;N # Mc KANNADA VOWEL SIGN AA +0CBF;N # Mn KANNADA VOWEL SIGN I +0CC0..0CC4;N # Mc [5] KANNADA VOWEL SIGN II..KANNADA VOWEL SIGN VOCALIC RR +0CC6;N # Mn KANNADA VOWEL SIGN E +0CC7..0CC8;N # Mc [2] KANNADA VOWEL SIGN EE..KANNADA VOWEL SIGN AI +0CCA..0CCB;N # Mc [2] KANNADA VOWEL SIGN O..KANNADA VOWEL SIGN OO +0CCC..0CCD;N # Mn [2] KANNADA VOWEL SIGN AU..KANNADA SIGN VIRAMA +0CD5..0CD6;N # Mc [2] KANNADA LENGTH MARK..KANNADA AI LENGTH MARK +0CDD..0CDE;N # Lo [2] KANNADA LETTER NAKAARA POLLU..KANNADA LETTER FA +0CE0..0CE1;N # Lo [2] KANNADA LETTER VOCALIC RR..KANNADA LETTER VOCALIC LL +0CE2..0CE3;N # Mn [2] KANNADA VOWEL SIGN VOCALIC L..KANNADA VOWEL SIGN VOCALIC LL +0CE6..0CEF;N # Nd [10] KANNADA DIGIT ZERO..KANNADA DIGIT NINE +0CF1..0CF2;N # Lo [2] KANNADA SIGN JIHVAMULIYA..KANNADA SIGN UPADHMANIYA +0CF3;N # Mc KANNADA SIGN COMBINING ANUSVARA ABOVE RIGHT +0D00..0D01;N # Mn [2] MALAYALAM SIGN COMBINING ANUSVARA ABOVE..MALAYALAM SIGN CANDRABINDU +0D02..0D03;N # Mc [2] MALAYALAM SIGN ANUSVARA..MALAYALAM SIGN VISARGA +0D04..0D0C;N # Lo [9] MALAYALAM LETTER VEDIC ANUSVARA..MALAYALAM LETTER VOCALIC L +0D0E..0D10;N # Lo [3] MALAYALAM LETTER E..MALAYALAM LETTER AI +0D12..0D3A;N # Lo [41] MALAYALAM LETTER O..MALAYALAM LETTER TTTA +0D3B..0D3C;N # Mn [2] MALAYALAM SIGN VERTICAL BAR VIRAMA..MALAYALAM SIGN CIRCULAR VIRAMA +0D3D;N # Lo MALAYALAM SIGN AVAGRAHA +0D3E..0D40;N # Mc [3] MALAYALAM VOWEL SIGN AA..MALAYALAM VOWEL SIGN II +0D41..0D44;N # Mn [4] MALAYALAM VOWEL SIGN U..MALAYALAM VOWEL SIGN VOCALIC RR +0D46..0D48;N # Mc [3] MALAYALAM VOWEL SIGN E..MALAYALAM VOWEL SIGN AI +0D4A..0D4C;N # Mc [3] MALAYALAM VOWEL SIGN O..MALAYALAM VOWEL SIGN AU +0D4D;N # Mn MALAYALAM SIGN VIRAMA +0D4E;N # Lo MALAYALAM LETTER DOT REPH +0D4F;N # So MALAYALAM SIGN PARA +0D54..0D56;N # Lo [3] MALAYALAM LETTER CHILLU M..MALAYALAM LETTER CHILLU LLL +0D57;N # Mc MALAYALAM AU LENGTH MARK +0D58..0D5E;N # No [7] MALAYALAM FRACTION ONE ONE-HUNDRED-AND-SIXTIETH..MALAYALAM FRACTION ONE FIFTH +0D5F..0D61;N # Lo [3] MALAYALAM LETTER ARCHAIC II..MALAYALAM LETTER VOCALIC LL +0D62..0D63;N # Mn [2] MALAYALAM VOWEL SIGN VOCALIC L..MALAYALAM VOWEL SIGN VOCALIC LL +0D66..0D6F;N # Nd [10] MALAYALAM DIGIT ZERO..MALAYALAM DIGIT NINE +0D70..0D78;N # No [9] MALAYALAM NUMBER TEN..MALAYALAM FRACTION THREE SIXTEENTHS +0D79;N # So MALAYALAM DATE MARK +0D7A..0D7F;N # Lo [6] MALAYALAM LETTER CHILLU NN..MALAYALAM LETTER CHILLU K +0D81;N # Mn SINHALA SIGN CANDRABINDU +0D82..0D83;N # Mc [2] SINHALA SIGN ANUSVARAYA..SINHALA SIGN VISARGAYA +0D85..0D96;N # Lo [18] SINHALA LETTER AYANNA..SINHALA LETTER AUYANNA +0D9A..0DB1;N # Lo [24] SINHALA LETTER ALPAPRAANA KAYANNA..SINHALA LETTER DANTAJA NAYANNA +0DB3..0DBB;N # Lo [9] SINHALA LETTER SANYAKA DAYANNA..SINHALA LETTER RAYANNA +0DBD;N # Lo SINHALA LETTER DANTAJA LAYANNA +0DC0..0DC6;N # Lo [7] SINHALA LETTER VAYANNA..SINHALA LETTER FAYANNA +0DCA;N # Mn SINHALA SIGN AL-LAKUNA +0DCF..0DD1;N # Mc [3] SINHALA VOWEL SIGN AELA-PILLA..SINHALA VOWEL SIGN DIGA AEDA-PILLA +0DD2..0DD4;N # Mn [3] SINHALA VOWEL SIGN KETTI IS-PILLA..SINHALA VOWEL SIGN KETTI PAA-PILLA +0DD6;N # Mn SINHALA VOWEL SIGN DIGA PAA-PILLA +0DD8..0DDF;N # Mc [8] SINHALA VOWEL SIGN GAETTA-PILLA..SINHALA VOWEL SIGN GAYANUKITTA +0DE6..0DEF;N # Nd [10] SINHALA LITH DIGIT ZERO..SINHALA LITH DIGIT NINE +0DF2..0DF3;N # Mc [2] SINHALA VOWEL SIGN DIGA GAETTA-PILLA..SINHALA VOWEL SIGN DIGA GAYANUKITTA +0DF4;N # Po SINHALA PUNCTUATION KUNDDALIYA +0E01..0E30;N # Lo [48] THAI CHARACTER KO KAI..THAI CHARACTER SARA A +0E31;N # Mn THAI CHARACTER MAI HAN-AKAT +0E32..0E33;N # Lo [2] THAI CHARACTER SARA AA..THAI CHARACTER SARA AM +0E34..0E3A;N # Mn [7] THAI CHARACTER SARA I..THAI CHARACTER PHINTHU +0E3F;N # Sc THAI CURRENCY SYMBOL BAHT +0E40..0E45;N # Lo [6] THAI CHARACTER SARA E..THAI CHARACTER LAKKHANGYAO +0E46;N # Lm THAI CHARACTER MAIYAMOK +0E47..0E4E;N # Mn [8] THAI CHARACTER MAITAIKHU..THAI CHARACTER YAMAKKAN +0E4F;N # Po THAI CHARACTER FONGMAN +0E50..0E59;N # Nd [10] THAI DIGIT ZERO..THAI DIGIT NINE +0E5A..0E5B;N # Po [2] THAI CHARACTER ANGKHANKHU..THAI CHARACTER KHOMUT +0E81..0E82;N # Lo [2] LAO LETTER KO..LAO LETTER KHO SUNG +0E84;N # Lo LAO LETTER KHO TAM +0E86..0E8A;N # Lo [5] LAO LETTER PALI GHA..LAO LETTER SO TAM +0E8C..0EA3;N # Lo [24] LAO LETTER PALI JHA..LAO LETTER LO LING +0EA5;N # Lo LAO LETTER LO LOOT +0EA7..0EB0;N # Lo [10] LAO LETTER WO..LAO VOWEL SIGN A +0EB1;N # Mn LAO VOWEL SIGN MAI KAN +0EB2..0EB3;N # Lo [2] LAO VOWEL SIGN AA..LAO VOWEL SIGN AM +0EB4..0EBC;N # Mn [9] LAO VOWEL SIGN I..LAO SEMIVOWEL SIGN LO +0EBD;N # Lo LAO SEMIVOWEL SIGN NYO +0EC0..0EC4;N # Lo [5] LAO VOWEL SIGN E..LAO VOWEL SIGN AI +0EC6;N # Lm LAO KO LA +0EC8..0ECE;N # Mn [7] LAO TONE MAI EK..LAO YAMAKKAN +0ED0..0ED9;N # Nd [10] LAO DIGIT ZERO..LAO DIGIT NINE +0EDC..0EDF;N # Lo [4] LAO HO NO..LAO LETTER KHMU NYO +0F00;N # Lo TIBETAN SYLLABLE OM +0F01..0F03;N # So [3] TIBETAN MARK GTER YIG MGO TRUNCATED A..TIBETAN MARK GTER YIG MGO -UM GTER TSHEG MA +0F04..0F12;N # Po [15] TIBETAN MARK INITIAL YIG MGO MDUN MA..TIBETAN MARK RGYA GRAM SHAD +0F13;N # So TIBETAN MARK CARET -DZUD RTAGS ME LONG CAN +0F14;N # Po TIBETAN MARK GTER TSHEG +0F15..0F17;N # So [3] TIBETAN LOGOTYPE SIGN CHAD RTAGS..TIBETAN ASTROLOGICAL SIGN SGRA GCAN -CHAR RTAGS +0F18..0F19;N # Mn [2] TIBETAN ASTROLOGICAL SIGN -KHYUD PA..TIBETAN ASTROLOGICAL SIGN SDONG TSHUGS +0F1A..0F1F;N # So [6] TIBETAN SIGN RDEL DKAR GCIG..TIBETAN SIGN RDEL DKAR RDEL NAG +0F20..0F29;N # Nd [10] TIBETAN DIGIT ZERO..TIBETAN DIGIT NINE +0F2A..0F33;N # No [10] TIBETAN DIGIT HALF ONE..TIBETAN DIGIT HALF ZERO +0F34;N # So TIBETAN MARK BSDUS RTAGS +0F35;N # Mn TIBETAN MARK NGAS BZUNG NYI ZLA +0F36;N # So TIBETAN MARK CARET -DZUD RTAGS BZHI MIG CAN +0F37;N # Mn TIBETAN MARK NGAS BZUNG SGOR RTAGS +0F38;N # So TIBETAN MARK CHE MGO +0F39;N # Mn TIBETAN MARK TSA -PHRU +0F3A;N # Ps TIBETAN MARK GUG RTAGS GYON +0F3B;N # Pe TIBETAN MARK GUG RTAGS GYAS +0F3C;N # Ps TIBETAN MARK ANG KHANG GYON +0F3D;N # Pe TIBETAN MARK ANG KHANG GYAS +0F3E..0F3F;N # Mc [2] TIBETAN SIGN YAR TSHES..TIBETAN SIGN MAR TSHES +0F40..0F47;N # Lo [8] TIBETAN LETTER KA..TIBETAN LETTER JA +0F49..0F6C;N # Lo [36] TIBETAN LETTER NYA..TIBETAN LETTER RRA +0F71..0F7E;N # Mn [14] TIBETAN VOWEL SIGN AA..TIBETAN SIGN RJES SU NGA RO +0F7F;N # Mc TIBETAN SIGN RNAM BCAD +0F80..0F84;N # Mn [5] TIBETAN VOWEL SIGN REVERSED I..TIBETAN MARK HALANTA +0F85;N # Po TIBETAN MARK PALUTA +0F86..0F87;N # Mn [2] TIBETAN SIGN LCI RTAGS..TIBETAN SIGN YANG RTAGS +0F88..0F8C;N # Lo [5] TIBETAN SIGN LCE TSA CAN..TIBETAN SIGN INVERTED MCHU CAN +0F8D..0F97;N # Mn [11] TIBETAN SUBJOINED SIGN LCE TSA CAN..TIBETAN SUBJOINED LETTER JA +0F99..0FBC;N # Mn [36] TIBETAN SUBJOINED LETTER NYA..TIBETAN SUBJOINED LETTER FIXED-FORM RA +0FBE..0FC5;N # So [8] TIBETAN KU RU KHA..TIBETAN SYMBOL RDO RJE +0FC6;N # Mn TIBETAN SYMBOL PADMA GDAN +0FC7..0FCC;N # So [6] TIBETAN SYMBOL RDO RJE RGYA GRAM..TIBETAN SYMBOL NOR BU BZHI -KHYIL +0FCE..0FCF;N # So [2] TIBETAN SIGN RDEL NAG RDEL DKAR..TIBETAN SIGN RDEL NAG GSUM +0FD0..0FD4;N # Po [5] TIBETAN MARK BSKA- SHOG GI MGO RGYAN..TIBETAN MARK CLOSING BRDA RNYING YIG MGO SGAB MA +0FD5..0FD8;N # So [4] RIGHT-FACING SVASTI SIGN..LEFT-FACING SVASTI SIGN WITH DOTS +0FD9..0FDA;N # Po [2] TIBETAN MARK LEADING MCHAN RTAGS..TIBETAN MARK TRAILING MCHAN RTAGS +1000..102A;N # Lo [43] MYANMAR LETTER KA..MYANMAR LETTER AU +102B..102C;N # Mc [2] MYANMAR VOWEL SIGN TALL AA..MYANMAR VOWEL SIGN AA +102D..1030;N # Mn [4] MYANMAR VOWEL SIGN I..MYANMAR VOWEL SIGN UU +1031;N # Mc MYANMAR VOWEL SIGN E +1032..1037;N # Mn [6] MYANMAR VOWEL SIGN AI..MYANMAR SIGN DOT BELOW +1038;N # Mc MYANMAR SIGN VISARGA +1039..103A;N # Mn [2] MYANMAR SIGN VIRAMA..MYANMAR SIGN ASAT +103B..103C;N # Mc [2] MYANMAR CONSONANT SIGN MEDIAL YA..MYANMAR CONSONANT SIGN MEDIAL RA +103D..103E;N # Mn [2] MYANMAR CONSONANT SIGN MEDIAL WA..MYANMAR CONSONANT SIGN MEDIAL HA +103F;N # Lo MYANMAR LETTER GREAT SA +1040..1049;N # Nd [10] MYANMAR DIGIT ZERO..MYANMAR DIGIT NINE +104A..104F;N # Po [6] MYANMAR SIGN LITTLE SECTION..MYANMAR SYMBOL GENITIVE +1050..1055;N # Lo [6] MYANMAR LETTER SHA..MYANMAR LETTER VOCALIC LL +1056..1057;N # Mc [2] MYANMAR VOWEL SIGN VOCALIC R..MYANMAR VOWEL SIGN VOCALIC RR +1058..1059;N # Mn [2] MYANMAR VOWEL SIGN VOCALIC L..MYANMAR VOWEL SIGN VOCALIC LL +105A..105D;N # Lo [4] MYANMAR LETTER MON NGA..MYANMAR LETTER MON BBE +105E..1060;N # Mn [3] MYANMAR CONSONANT SIGN MON MEDIAL NA..MYANMAR CONSONANT SIGN MON MEDIAL LA +1061;N # Lo MYANMAR LETTER SGAW KAREN SHA +1062..1064;N # Mc [3] MYANMAR VOWEL SIGN SGAW KAREN EU..MYANMAR TONE MARK SGAW KAREN KE PHO +1065..1066;N # Lo [2] MYANMAR LETTER WESTERN PWO KAREN THA..MYANMAR LETTER WESTERN PWO KAREN PWA +1067..106D;N # Mc [7] MYANMAR VOWEL SIGN WESTERN PWO KAREN EU..MYANMAR SIGN WESTERN PWO KAREN TONE-5 +106E..1070;N # Lo [3] MYANMAR LETTER EASTERN PWO KAREN NNA..MYANMAR LETTER EASTERN PWO KAREN GHWA +1071..1074;N # Mn [4] MYANMAR VOWEL SIGN GEBA KAREN I..MYANMAR VOWEL SIGN KAYAH EE +1075..1081;N # Lo [13] MYANMAR LETTER SHAN KA..MYANMAR LETTER SHAN HA +1082;N # Mn MYANMAR CONSONANT SIGN SHAN MEDIAL WA +1083..1084;N # Mc [2] MYANMAR VOWEL SIGN SHAN AA..MYANMAR VOWEL SIGN SHAN E +1085..1086;N # Mn [2] MYANMAR VOWEL SIGN SHAN E ABOVE..MYANMAR VOWEL SIGN SHAN FINAL Y +1087..108C;N # Mc [6] MYANMAR SIGN SHAN TONE-2..MYANMAR SIGN SHAN COUNCIL TONE-3 +108D;N # Mn MYANMAR SIGN SHAN COUNCIL EMPHATIC TONE +108E;N # Lo MYANMAR LETTER RUMAI PALAUNG FA +108F;N # Mc MYANMAR SIGN RUMAI PALAUNG TONE-5 +1090..1099;N # Nd [10] MYANMAR SHAN DIGIT ZERO..MYANMAR SHAN DIGIT NINE +109A..109C;N # Mc [3] MYANMAR SIGN KHAMTI TONE-1..MYANMAR VOWEL SIGN AITON A +109D;N # Mn MYANMAR VOWEL SIGN AITON AI +109E..109F;N # So [2] MYANMAR SYMBOL SHAN ONE..MYANMAR SYMBOL SHAN EXCLAMATION +10A0..10C5;N # Lu [38] GEORGIAN CAPITAL LETTER AN..GEORGIAN CAPITAL LETTER HOE +10C7;N # Lu GEORGIAN CAPITAL LETTER YN +10CD;N # Lu GEORGIAN CAPITAL LETTER AEN +10D0..10FA;N # Ll [43] GEORGIAN LETTER AN..GEORGIAN LETTER AIN +10FB;N # Po GEORGIAN PARAGRAPH SEPARATOR +10FC;N # Lm MODIFIER LETTER GEORGIAN NAR +10FD..10FF;N # Ll [3] GEORGIAN LETTER AEN..GEORGIAN LETTER LABIAL SIGN +1100..115F;W # Lo [96] HANGUL CHOSEONG KIYEOK..HANGUL CHOSEONG FILLER +1160..11FF;N # Lo [160] HANGUL JUNGSEONG FILLER..HANGUL JONGSEONG SSANGNIEUN +1200..1248;N # Lo [73] ETHIOPIC SYLLABLE HA..ETHIOPIC SYLLABLE QWA +124A..124D;N # Lo [4] ETHIOPIC SYLLABLE QWI..ETHIOPIC SYLLABLE QWE +1250..1256;N # Lo [7] ETHIOPIC SYLLABLE QHA..ETHIOPIC SYLLABLE QHO +1258;N # Lo ETHIOPIC SYLLABLE QHWA +125A..125D;N # Lo [4] ETHIOPIC SYLLABLE QHWI..ETHIOPIC SYLLABLE QHWE +1260..1288;N # Lo [41] ETHIOPIC SYLLABLE BA..ETHIOPIC SYLLABLE XWA +128A..128D;N # Lo [4] ETHIOPIC SYLLABLE XWI..ETHIOPIC SYLLABLE XWE +1290..12B0;N # Lo [33] ETHIOPIC SYLLABLE NA..ETHIOPIC SYLLABLE KWA +12B2..12B5;N # Lo [4] ETHIOPIC SYLLABLE KWI..ETHIOPIC SYLLABLE KWE +12B8..12BE;N # Lo [7] ETHIOPIC SYLLABLE KXA..ETHIOPIC SYLLABLE KXO +12C0;N # Lo ETHIOPIC SYLLABLE KXWA +12C2..12C5;N # Lo [4] ETHIOPIC SYLLABLE KXWI..ETHIOPIC SYLLABLE KXWE +12C8..12D6;N # Lo [15] ETHIOPIC SYLLABLE WA..ETHIOPIC SYLLABLE PHARYNGEAL O +12D8..1310;N # Lo [57] ETHIOPIC SYLLABLE ZA..ETHIOPIC SYLLABLE GWA +1312..1315;N # Lo [4] ETHIOPIC SYLLABLE GWI..ETHIOPIC SYLLABLE GWE +1318..135A;N # Lo [67] ETHIOPIC SYLLABLE GGA..ETHIOPIC SYLLABLE FYA +135D..135F;N # Mn [3] ETHIOPIC COMBINING GEMINATION AND VOWEL LENGTH MARK..ETHIOPIC COMBINING GEMINATION MARK +1360..1368;N # Po [9] ETHIOPIC SECTION MARK..ETHIOPIC PARAGRAPH SEPARATOR +1369..137C;N # No [20] ETHIOPIC DIGIT ONE..ETHIOPIC NUMBER TEN THOUSAND +1380..138F;N # Lo [16] ETHIOPIC SYLLABLE SEBATBEIT MWA..ETHIOPIC SYLLABLE PWE +1390..1399;N # So [10] ETHIOPIC TONAL MARK YIZET..ETHIOPIC TONAL MARK KURT +13A0..13F5;N # Lu [86] CHEROKEE LETTER A..CHEROKEE LETTER MV +13F8..13FD;N # Ll [6] CHEROKEE SMALL LETTER YE..CHEROKEE SMALL LETTER MV +1400;N # Pd CANADIAN SYLLABICS HYPHEN +1401..166C;N # Lo [620] CANADIAN SYLLABICS E..CANADIAN SYLLABICS CARRIER TTSA +166D;N # So CANADIAN SYLLABICS CHI SIGN +166E;N # Po CANADIAN SYLLABICS FULL STOP +166F..167F;N # Lo [17] CANADIAN SYLLABICS QAI..CANADIAN SYLLABICS BLACKFOOT W +1680;N # Zs OGHAM SPACE MARK +1681..169A;N # Lo [26] OGHAM LETTER BEITH..OGHAM LETTER PEITH +169B;N # Ps OGHAM FEATHER MARK +169C;N # Pe OGHAM REVERSED FEATHER MARK +16A0..16EA;N # Lo [75] RUNIC LETTER FEHU FEOH FE F..RUNIC LETTER X +16EB..16ED;N # Po [3] RUNIC SINGLE PUNCTUATION..RUNIC CROSS PUNCTUATION +16EE..16F0;N # Nl [3] RUNIC ARLAUG SYMBOL..RUNIC BELGTHOR SYMBOL +16F1..16F8;N # Lo [8] RUNIC LETTER K..RUNIC LETTER FRANKS CASKET AESC +1700..1711;N # Lo [18] TAGALOG LETTER A..TAGALOG LETTER HA +1712..1714;N # Mn [3] TAGALOG VOWEL SIGN I..TAGALOG SIGN VIRAMA +1715;N # Mc TAGALOG SIGN PAMUDPOD +171F;N # Lo TAGALOG LETTER ARCHAIC RA +1720..1731;N # Lo [18] HANUNOO LETTER A..HANUNOO LETTER HA +1732..1733;N # Mn [2] HANUNOO VOWEL SIGN I..HANUNOO VOWEL SIGN U +1734;N # Mc HANUNOO SIGN PAMUDPOD +1735..1736;N # Po [2] PHILIPPINE SINGLE PUNCTUATION..PHILIPPINE DOUBLE PUNCTUATION +1740..1751;N # Lo [18] BUHID LETTER A..BUHID LETTER HA +1752..1753;N # Mn [2] BUHID VOWEL SIGN I..BUHID VOWEL SIGN U +1760..176C;N # Lo [13] TAGBANWA LETTER A..TAGBANWA LETTER YA +176E..1770;N # Lo [3] TAGBANWA LETTER LA..TAGBANWA LETTER SA +1772..1773;N # Mn [2] TAGBANWA VOWEL SIGN I..TAGBANWA VOWEL SIGN U +1780..17B3;N # Lo [52] KHMER LETTER KA..KHMER INDEPENDENT VOWEL QAU +17B4..17B5;N # Mn [2] KHMER VOWEL INHERENT AQ..KHMER VOWEL INHERENT AA +17B6;N # Mc KHMER VOWEL SIGN AA +17B7..17BD;N # Mn [7] KHMER VOWEL SIGN I..KHMER VOWEL SIGN UA +17BE..17C5;N # Mc [8] KHMER VOWEL SIGN OE..KHMER VOWEL SIGN AU +17C6;N # Mn KHMER SIGN NIKAHIT +17C7..17C8;N # Mc [2] KHMER SIGN REAHMUK..KHMER SIGN YUUKALEAPINTU +17C9..17D3;N # Mn [11] KHMER SIGN MUUSIKATOAN..KHMER SIGN BATHAMASAT +17D4..17D6;N # Po [3] KHMER SIGN KHAN..KHMER SIGN CAMNUC PII KUUH +17D7;N # Lm KHMER SIGN LEK TOO +17D8..17DA;N # Po [3] KHMER SIGN BEYYAL..KHMER SIGN KOOMUUT +17DB;N # Sc KHMER CURRENCY SYMBOL RIEL +17DC;N # Lo KHMER SIGN AVAKRAHASANYA +17DD;N # Mn KHMER SIGN ATTHACAN +17E0..17E9;N # Nd [10] KHMER DIGIT ZERO..KHMER DIGIT NINE +17F0..17F9;N # No [10] KHMER SYMBOL LEK ATTAK SON..KHMER SYMBOL LEK ATTAK PRAM-BUON +1800..1805;N # Po [6] MONGOLIAN BIRGA..MONGOLIAN FOUR DOTS +1806;N # Pd MONGOLIAN TODO SOFT HYPHEN +1807..180A;N # Po [4] MONGOLIAN SIBE SYLLABLE BOUNDARY MARKER..MONGOLIAN NIRUGU +180B..180D;N # Mn [3] MONGOLIAN FREE VARIATION SELECTOR ONE..MONGOLIAN FREE VARIATION SELECTOR THREE +180E;N # Cf MONGOLIAN VOWEL SEPARATOR +180F;N # Mn MONGOLIAN FREE VARIATION SELECTOR FOUR +1810..1819;N # Nd [10] MONGOLIAN DIGIT ZERO..MONGOLIAN DIGIT NINE +1820..1842;N # Lo [35] MONGOLIAN LETTER A..MONGOLIAN LETTER CHI +1843;N # Lm MONGOLIAN LETTER TODO LONG VOWEL SIGN +1844..1878;N # Lo [53] MONGOLIAN LETTER TODO E..MONGOLIAN LETTER CHA WITH TWO DOTS +1880..1884;N # Lo [5] MONGOLIAN LETTER ALI GALI ANUSVARA ONE..MONGOLIAN LETTER ALI GALI INVERTED UBADAMA +1885..1886;N # Mn [2] MONGOLIAN LETTER ALI GALI BALUDA..MONGOLIAN LETTER ALI GALI THREE BALUDA +1887..18A8;N # Lo [34] MONGOLIAN LETTER ALI GALI A..MONGOLIAN LETTER MANCHU ALI GALI BHA +18A9;N # Mn MONGOLIAN LETTER ALI GALI DAGALGA +18AA;N # Lo MONGOLIAN LETTER MANCHU ALI GALI LHA +18B0..18F5;N # Lo [70] CANADIAN SYLLABICS OY..CANADIAN SYLLABICS CARRIER DENTAL S +1900..191E;N # Lo [31] LIMBU VOWEL-CARRIER LETTER..LIMBU LETTER TRA +1920..1922;N # Mn [3] LIMBU VOWEL SIGN A..LIMBU VOWEL SIGN U +1923..1926;N # Mc [4] LIMBU VOWEL SIGN EE..LIMBU VOWEL SIGN AU +1927..1928;N # Mn [2] LIMBU VOWEL SIGN E..LIMBU VOWEL SIGN O +1929..192B;N # Mc [3] LIMBU SUBJOINED LETTER YA..LIMBU SUBJOINED LETTER WA +1930..1931;N # Mc [2] LIMBU SMALL LETTER KA..LIMBU SMALL LETTER NGA +1932;N # Mn LIMBU SMALL LETTER ANUSVARA +1933..1938;N # Mc [6] LIMBU SMALL LETTER TA..LIMBU SMALL LETTER LA +1939..193B;N # Mn [3] LIMBU SIGN MUKPHRENG..LIMBU SIGN SA-I +1940;N # So LIMBU SIGN LOO +1944..1945;N # Po [2] LIMBU EXCLAMATION MARK..LIMBU QUESTION MARK +1946..194F;N # Nd [10] LIMBU DIGIT ZERO..LIMBU DIGIT NINE +1950..196D;N # Lo [30] TAI LE LETTER KA..TAI LE LETTER AI +1970..1974;N # Lo [5] TAI LE LETTER TONE-2..TAI LE LETTER TONE-6 +1980..19AB;N # Lo [44] NEW TAI LUE LETTER HIGH QA..NEW TAI LUE LETTER LOW SUA +19B0..19C9;N # Lo [26] NEW TAI LUE VOWEL SIGN VOWEL SHORTENER..NEW TAI LUE TONE MARK-2 +19D0..19D9;N # Nd [10] NEW TAI LUE DIGIT ZERO..NEW TAI LUE DIGIT NINE +19DA;N # No NEW TAI LUE THAM DIGIT ONE +19DE..19DF;N # So [2] NEW TAI LUE SIGN LAE..NEW TAI LUE SIGN LAEV +19E0..19FF;N # So [32] KHMER SYMBOL PATHAMASAT..KHMER SYMBOL DAP-PRAM ROC +1A00..1A16;N # Lo [23] BUGINESE LETTER KA..BUGINESE LETTER HA +1A17..1A18;N # Mn [2] BUGINESE VOWEL SIGN I..BUGINESE VOWEL SIGN U +1A19..1A1A;N # Mc [2] BUGINESE VOWEL SIGN E..BUGINESE VOWEL SIGN O +1A1B;N # Mn BUGINESE VOWEL SIGN AE +1A1E..1A1F;N # Po [2] BUGINESE PALLAWA..BUGINESE END OF SECTION +1A20..1A54;N # Lo [53] TAI THAM LETTER HIGH KA..TAI THAM LETTER GREAT SA +1A55;N # Mc TAI THAM CONSONANT SIGN MEDIAL RA +1A56;N # Mn TAI THAM CONSONANT SIGN MEDIAL LA +1A57;N # Mc TAI THAM CONSONANT SIGN LA TANG LAI +1A58..1A5E;N # Mn [7] TAI THAM SIGN MAI KANG LAI..TAI THAM CONSONANT SIGN SA +1A60;N # Mn TAI THAM SIGN SAKOT +1A61;N # Mc TAI THAM VOWEL SIGN A +1A62;N # Mn TAI THAM VOWEL SIGN MAI SAT +1A63..1A64;N # Mc [2] TAI THAM VOWEL SIGN AA..TAI THAM VOWEL SIGN TALL AA +1A65..1A6C;N # Mn [8] TAI THAM VOWEL SIGN I..TAI THAM VOWEL SIGN OA BELOW +1A6D..1A72;N # Mc [6] TAI THAM VOWEL SIGN OY..TAI THAM VOWEL SIGN THAM AI +1A73..1A7C;N # Mn [10] TAI THAM VOWEL SIGN OA ABOVE..TAI THAM SIGN KHUEN-LUE KARAN +1A7F;N # Mn TAI THAM COMBINING CRYPTOGRAMMIC DOT +1A80..1A89;N # Nd [10] TAI THAM HORA DIGIT ZERO..TAI THAM HORA DIGIT NINE +1A90..1A99;N # Nd [10] TAI THAM THAM DIGIT ZERO..TAI THAM THAM DIGIT NINE +1AA0..1AA6;N # Po [7] TAI THAM SIGN WIANG..TAI THAM SIGN REVERSED ROTATED RANA +1AA7;N # Lm TAI THAM SIGN MAI YAMOK +1AA8..1AAD;N # Po [6] TAI THAM SIGN KAAN..TAI THAM SIGN CAANG +1AB0..1ABD;N # Mn [14] COMBINING DOUBLED CIRCUMFLEX ACCENT..COMBINING PARENTHESES BELOW +1ABE;N # Me COMBINING PARENTHESES OVERLAY +1ABF..1ACE;N # Mn [16] COMBINING LATIN SMALL LETTER W BELOW..COMBINING LATIN SMALL LETTER INSULAR T +1B00..1B03;N # Mn [4] BALINESE SIGN ULU RICEM..BALINESE SIGN SURANG +1B04;N # Mc BALINESE SIGN BISAH +1B05..1B33;N # Lo [47] BALINESE LETTER AKARA..BALINESE LETTER HA +1B34;N # Mn BALINESE SIGN REREKAN +1B35;N # Mc BALINESE VOWEL SIGN TEDUNG +1B36..1B3A;N # Mn [5] BALINESE VOWEL SIGN ULU..BALINESE VOWEL SIGN RA REPA +1B3B;N # Mc BALINESE VOWEL SIGN RA REPA TEDUNG +1B3C;N # Mn BALINESE VOWEL SIGN LA LENGA +1B3D..1B41;N # Mc [5] BALINESE VOWEL SIGN LA LENGA TEDUNG..BALINESE VOWEL SIGN TALING REPA TEDUNG +1B42;N # Mn BALINESE VOWEL SIGN PEPET +1B43..1B44;N # Mc [2] BALINESE VOWEL SIGN PEPET TEDUNG..BALINESE ADEG ADEG +1B45..1B4C;N # Lo [8] BALINESE LETTER KAF SASAK..BALINESE LETTER ARCHAIC JNYA +1B50..1B59;N # Nd [10] BALINESE DIGIT ZERO..BALINESE DIGIT NINE +1B5A..1B60;N # Po [7] BALINESE PANTI..BALINESE PAMENENG +1B61..1B6A;N # So [10] BALINESE MUSICAL SYMBOL DONG..BALINESE MUSICAL SYMBOL DANG GEDE +1B6B..1B73;N # Mn [9] BALINESE MUSICAL SYMBOL COMBINING TEGEH..BALINESE MUSICAL SYMBOL COMBINING GONG +1B74..1B7C;N # So [9] BALINESE MUSICAL SYMBOL RIGHT-HAND OPEN DUG..BALINESE MUSICAL SYMBOL LEFT-HAND OPEN PING +1B7D..1B7E;N # Po [2] BALINESE PANTI LANTANG..BALINESE PAMADA LANTANG +1B80..1B81;N # Mn [2] SUNDANESE SIGN PANYECEK..SUNDANESE SIGN PANGLAYAR +1B82;N # Mc SUNDANESE SIGN PANGWISAD +1B83..1BA0;N # Lo [30] SUNDANESE LETTER A..SUNDANESE LETTER HA +1BA1;N # Mc SUNDANESE CONSONANT SIGN PAMINGKAL +1BA2..1BA5;N # Mn [4] SUNDANESE CONSONANT SIGN PANYAKRA..SUNDANESE VOWEL SIGN PANYUKU +1BA6..1BA7;N # Mc [2] SUNDANESE VOWEL SIGN PANAELAENG..SUNDANESE VOWEL SIGN PANOLONG +1BA8..1BA9;N # Mn [2] SUNDANESE VOWEL SIGN PAMEPET..SUNDANESE VOWEL SIGN PANEULEUNG +1BAA;N # Mc SUNDANESE SIGN PAMAAEH +1BAB..1BAD;N # Mn [3] SUNDANESE SIGN VIRAMA..SUNDANESE CONSONANT SIGN PASANGAN WA +1BAE..1BAF;N # Lo [2] SUNDANESE LETTER KHA..SUNDANESE LETTER SYA +1BB0..1BB9;N # Nd [10] SUNDANESE DIGIT ZERO..SUNDANESE DIGIT NINE +1BBA..1BBF;N # Lo [6] SUNDANESE AVAGRAHA..SUNDANESE LETTER FINAL M +1BC0..1BE5;N # Lo [38] BATAK LETTER A..BATAK LETTER U +1BE6;N # Mn BATAK SIGN TOMPI +1BE7;N # Mc BATAK VOWEL SIGN E +1BE8..1BE9;N # Mn [2] BATAK VOWEL SIGN PAKPAK E..BATAK VOWEL SIGN EE +1BEA..1BEC;N # Mc [3] BATAK VOWEL SIGN I..BATAK VOWEL SIGN O +1BED;N # Mn BATAK VOWEL SIGN KARO O +1BEE;N # Mc BATAK VOWEL SIGN U +1BEF..1BF1;N # Mn [3] BATAK VOWEL SIGN U FOR SIMALUNGUN SA..BATAK CONSONANT SIGN H +1BF2..1BF3;N # Mc [2] BATAK PANGOLAT..BATAK PANONGONAN +1BFC..1BFF;N # Po [4] BATAK SYMBOL BINDU NA METEK..BATAK SYMBOL BINDU PANGOLAT +1C00..1C23;N # Lo [36] LEPCHA LETTER KA..LEPCHA LETTER A +1C24..1C2B;N # Mc [8] LEPCHA SUBJOINED LETTER YA..LEPCHA VOWEL SIGN UU +1C2C..1C33;N # Mn [8] LEPCHA VOWEL SIGN E..LEPCHA CONSONANT SIGN T +1C34..1C35;N # Mc [2] LEPCHA CONSONANT SIGN NYIN-DO..LEPCHA CONSONANT SIGN KANG +1C36..1C37;N # Mn [2] LEPCHA SIGN RAN..LEPCHA SIGN NUKTA +1C3B..1C3F;N # Po [5] LEPCHA PUNCTUATION TA-ROL..LEPCHA PUNCTUATION TSHOOK +1C40..1C49;N # Nd [10] LEPCHA DIGIT ZERO..LEPCHA DIGIT NINE +1C4D..1C4F;N # Lo [3] LEPCHA LETTER TTA..LEPCHA LETTER DDA +1C50..1C59;N # Nd [10] OL CHIKI DIGIT ZERO..OL CHIKI DIGIT NINE +1C5A..1C77;N # Lo [30] OL CHIKI LETTER LA..OL CHIKI LETTER OH +1C78..1C7D;N # Lm [6] OL CHIKI MU TTUDDAG..OL CHIKI AHAD +1C7E..1C7F;N # Po [2] OL CHIKI PUNCTUATION MUCAAD..OL CHIKI PUNCTUATION DOUBLE MUCAAD +1C80..1C88;N # Ll [9] CYRILLIC SMALL LETTER ROUNDED VE..CYRILLIC SMALL LETTER UNBLENDED UK +1C90..1CBA;N # Lu [43] GEORGIAN MTAVRULI CAPITAL LETTER AN..GEORGIAN MTAVRULI CAPITAL LETTER AIN +1CBD..1CBF;N # Lu [3] GEORGIAN MTAVRULI CAPITAL LETTER AEN..GEORGIAN MTAVRULI CAPITAL LETTER LABIAL SIGN +1CC0..1CC7;N # Po [8] SUNDANESE PUNCTUATION BINDU SURYA..SUNDANESE PUNCTUATION BINDU BA SATANGA +1CD0..1CD2;N # Mn [3] VEDIC TONE KARSHANA..VEDIC TONE PRENKHA +1CD3;N # Po VEDIC SIGN NIHSHVASA +1CD4..1CE0;N # Mn [13] VEDIC SIGN YAJURVEDIC MIDLINE SVARITA..VEDIC TONE RIGVEDIC KASHMIRI INDEPENDENT SVARITA +1CE1;N # Mc VEDIC TONE ATHARVAVEDIC INDEPENDENT SVARITA +1CE2..1CE8;N # Mn [7] VEDIC SIGN VISARGA SVARITA..VEDIC SIGN VISARGA ANUDATTA WITH TAIL +1CE9..1CEC;N # Lo [4] VEDIC SIGN ANUSVARA ANTARGOMUKHA..VEDIC SIGN ANUSVARA VAMAGOMUKHA WITH TAIL +1CED;N # Mn VEDIC SIGN TIRYAK +1CEE..1CF3;N # Lo [6] VEDIC SIGN HEXIFORM LONG ANUSVARA..VEDIC SIGN ROTATED ARDHAVISARGA +1CF4;N # Mn VEDIC TONE CANDRA ABOVE +1CF5..1CF6;N # Lo [2] VEDIC SIGN JIHVAMULIYA..VEDIC SIGN UPADHMANIYA +1CF7;N # Mc VEDIC SIGN ATIKRAMA +1CF8..1CF9;N # Mn [2] VEDIC TONE RING ABOVE..VEDIC TONE DOUBLE RING ABOVE +1CFA;N # Lo VEDIC SIGN DOUBLE ANUSVARA ANTARGOMUKHA +1D00..1D2B;N # Ll [44] LATIN LETTER SMALL CAPITAL A..CYRILLIC LETTER SMALL CAPITAL EL +1D2C..1D6A;N # Lm [63] MODIFIER LETTER CAPITAL A..GREEK SUBSCRIPT SMALL LETTER CHI +1D6B..1D77;N # Ll [13] LATIN SMALL LETTER UE..LATIN SMALL LETTER TURNED G +1D78;N # Lm MODIFIER LETTER CYRILLIC EN +1D79..1D7F;N # Ll [7] LATIN SMALL LETTER INSULAR G..LATIN SMALL LETTER UPSILON WITH STROKE +1D80..1D9A;N # Ll [27] LATIN SMALL LETTER B WITH PALATAL HOOK..LATIN SMALL LETTER EZH WITH RETROFLEX HOOK +1D9B..1DBF;N # Lm [37] MODIFIER LETTER SMALL TURNED ALPHA..MODIFIER LETTER SMALL THETA +1DC0..1DFF;N # Mn [64] COMBINING DOTTED GRAVE ACCENT..COMBINING RIGHT ARROWHEAD AND DOWN ARROWHEAD BELOW +1E00..1EFF;N # L& [256] LATIN CAPITAL LETTER A WITH RING BELOW..LATIN SMALL LETTER Y WITH LOOP +1F00..1F15;N # L& [22] GREEK SMALL LETTER ALPHA WITH PSILI..GREEK SMALL LETTER EPSILON WITH DASIA AND OXIA +1F18..1F1D;N # Lu [6] GREEK CAPITAL LETTER EPSILON WITH PSILI..GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA +1F20..1F45;N # L& [38] GREEK SMALL LETTER ETA WITH PSILI..GREEK SMALL LETTER OMICRON WITH DASIA AND OXIA +1F48..1F4D;N # Lu [6] GREEK CAPITAL LETTER OMICRON WITH PSILI..GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA +1F50..1F57;N # Ll [8] GREEK SMALL LETTER UPSILON WITH PSILI..GREEK SMALL LETTER UPSILON WITH DASIA AND PERISPOMENI +1F59;N # Lu GREEK CAPITAL LETTER UPSILON WITH DASIA +1F5B;N # Lu GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA +1F5D;N # Lu GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA +1F5F..1F7D;N # L& [31] GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI..GREEK SMALL LETTER OMEGA WITH OXIA +1F80..1FB4;N # L& [53] GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI +1FB6..1FBC;N # L& [7] GREEK SMALL LETTER ALPHA WITH PERISPOMENI..GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI +1FBD;N # Sk GREEK KORONIS +1FBE;N # Ll GREEK PROSGEGRAMMENI +1FBF..1FC1;N # Sk [3] GREEK PSILI..GREEK DIALYTIKA AND PERISPOMENI +1FC2..1FC4;N # Ll [3] GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI +1FC6..1FCC;N # L& [7] GREEK SMALL LETTER ETA WITH PERISPOMENI..GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI +1FCD..1FCF;N # Sk [3] GREEK PSILI AND VARIA..GREEK PSILI AND PERISPOMENI +1FD0..1FD3;N # Ll [4] GREEK SMALL LETTER IOTA WITH VRACHY..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA +1FD6..1FDB;N # L& [6] GREEK SMALL LETTER IOTA WITH PERISPOMENI..GREEK CAPITAL LETTER IOTA WITH OXIA +1FDD..1FDF;N # Sk [3] GREEK DASIA AND VARIA..GREEK DASIA AND PERISPOMENI +1FE0..1FEC;N # L& [13] GREEK SMALL LETTER UPSILON WITH VRACHY..GREEK CAPITAL LETTER RHO WITH DASIA +1FED..1FEF;N # Sk [3] GREEK DIALYTIKA AND VARIA..GREEK VARIA +1FF2..1FF4;N # Ll [3] GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI +1FF6..1FFC;N # L& [7] GREEK SMALL LETTER OMEGA WITH PERISPOMENI..GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI +1FFD..1FFE;N # Sk [2] GREEK OXIA..GREEK DASIA +2000..200A;N # Zs [11] EN QUAD..HAIR SPACE +200B..200F;N # Cf [5] ZERO WIDTH SPACE..RIGHT-TO-LEFT MARK +2010;A # Pd HYPHEN +2011..2012;N # Pd [2] NON-BREAKING HYPHEN..FIGURE DASH +2013..2015;A # Pd [3] EN DASH..HORIZONTAL BAR +2016;A # Po DOUBLE VERTICAL LINE +2017;N # Po DOUBLE LOW LINE +2018;A # Pi LEFT SINGLE QUOTATION MARK +2019;A # Pf RIGHT SINGLE QUOTATION MARK +201A;N # Ps SINGLE LOW-9 QUOTATION MARK +201B;N # Pi SINGLE HIGH-REVERSED-9 QUOTATION MARK +201C;A # Pi LEFT DOUBLE QUOTATION MARK +201D;A # Pf RIGHT DOUBLE QUOTATION MARK +201E;N # Ps DOUBLE LOW-9 QUOTATION MARK +201F;N # Pi DOUBLE HIGH-REVERSED-9 QUOTATION MARK +2020..2022;A # Po [3] DAGGER..BULLET +2023;N # Po TRIANGULAR BULLET +2024..2027;A # Po [4] ONE DOT LEADER..HYPHENATION POINT +2028;N # Zl LINE SEPARATOR +2029;N # Zp PARAGRAPH SEPARATOR +202A..202E;N # Cf [5] LEFT-TO-RIGHT EMBEDDING..RIGHT-TO-LEFT OVERRIDE +202F;N # Zs NARROW NO-BREAK SPACE +2030;A # Po PER MILLE SIGN +2031;N # Po PER TEN THOUSAND SIGN +2032..2033;A # Po [2] PRIME..DOUBLE PRIME +2034;N # Po TRIPLE PRIME +2035;A # Po REVERSED PRIME +2036..2038;N # Po [3] REVERSED DOUBLE PRIME..CARET +2039;N # Pi SINGLE LEFT-POINTING ANGLE QUOTATION MARK +203A;N # Pf SINGLE RIGHT-POINTING ANGLE QUOTATION MARK +203B;A # Po REFERENCE MARK +203C..203D;N # Po [2] DOUBLE EXCLAMATION MARK..INTERROBANG +203E;A # Po OVERLINE +203F..2040;N # Pc [2] UNDERTIE..CHARACTER TIE +2041..2043;N # Po [3] CARET INSERTION POINT..HYPHEN BULLET +2044;N # Sm FRACTION SLASH +2045;N # Ps LEFT SQUARE BRACKET WITH QUILL +2046;N # Pe RIGHT SQUARE BRACKET WITH QUILL +2047..2051;N # Po [11] DOUBLE QUESTION MARK..TWO ASTERISKS ALIGNED VERTICALLY +2052;N # Sm COMMERCIAL MINUS SIGN +2053;N # Po SWUNG DASH +2054;N # Pc INVERTED UNDERTIE +2055..205E;N # Po [10] FLOWER PUNCTUATION MARK..VERTICAL FOUR DOTS +205F;N # Zs MEDIUM MATHEMATICAL SPACE +2060..2064;N # Cf [5] WORD JOINER..INVISIBLE PLUS +2066..206F;N # Cf [10] LEFT-TO-RIGHT ISOLATE..NOMINAL DIGIT SHAPES +2070;N # No SUPERSCRIPT ZERO +2071;N # Lm SUPERSCRIPT LATIN SMALL LETTER I +2074;A # No SUPERSCRIPT FOUR +2075..2079;N # No [5] SUPERSCRIPT FIVE..SUPERSCRIPT NINE +207A..207C;N # Sm [3] SUPERSCRIPT PLUS SIGN..SUPERSCRIPT EQUALS SIGN +207D;N # Ps SUPERSCRIPT LEFT PARENTHESIS +207E;N # Pe SUPERSCRIPT RIGHT PARENTHESIS +207F;A # Lm SUPERSCRIPT LATIN SMALL LETTER N +2080;N # No SUBSCRIPT ZERO +2081..2084;A # No [4] SUBSCRIPT ONE..SUBSCRIPT FOUR +2085..2089;N # No [5] SUBSCRIPT FIVE..SUBSCRIPT NINE +208A..208C;N # Sm [3] SUBSCRIPT PLUS SIGN..SUBSCRIPT EQUALS SIGN +208D;N # Ps SUBSCRIPT LEFT PARENTHESIS +208E;N # Pe SUBSCRIPT RIGHT PARENTHESIS +2090..209C;N # Lm [13] LATIN SUBSCRIPT SMALL LETTER A..LATIN SUBSCRIPT SMALL LETTER T +20A0..20A8;N # Sc [9] EURO-CURRENCY SIGN..RUPEE SIGN +20A9;H # Sc WON SIGN +20AA..20AB;N # Sc [2] NEW SHEQEL SIGN..DONG SIGN +20AC;A # Sc EURO SIGN +20AD..20C0;N # Sc [20] KIP SIGN..SOM SIGN +20D0..20DC;N # Mn [13] COMBINING LEFT HARPOON ABOVE..COMBINING FOUR DOTS ABOVE +20DD..20E0;N # Me [4] COMBINING ENCLOSING CIRCLE..COMBINING ENCLOSING CIRCLE BACKSLASH +20E1;N # Mn COMBINING LEFT RIGHT ARROW ABOVE +20E2..20E4;N # Me [3] COMBINING ENCLOSING SCREEN..COMBINING ENCLOSING UPWARD POINTING TRIANGLE +20E5..20F0;N # Mn [12] COMBINING REVERSE SOLIDUS OVERLAY..COMBINING ASTERISK ABOVE +2100..2101;N # So [2] ACCOUNT OF..ADDRESSED TO THE SUBJECT +2102;N # Lu DOUBLE-STRUCK CAPITAL C +2103;A # So DEGREE CELSIUS +2104;N # So CENTRE LINE SYMBOL +2105;A # So CARE OF +2106;N # So CADA UNA +2107;N # Lu EULER CONSTANT +2108;N # So SCRUPLE +2109;A # So DEGREE FAHRENHEIT +210A..2112;N # L& [9] SCRIPT SMALL G..SCRIPT CAPITAL L +2113;A # Ll SCRIPT SMALL L +2114;N # So L B BAR SYMBOL +2115;N # Lu DOUBLE-STRUCK CAPITAL N +2116;A # So NUMERO SIGN +2117;N # So SOUND RECORDING COPYRIGHT +2118;N # Sm SCRIPT CAPITAL P +2119..211D;N # Lu [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-STRUCK CAPITAL R +211E..2120;N # So [3] PRESCRIPTION TAKE..SERVICE MARK +2121..2122;A # So [2] TELEPHONE SIGN..TRADE MARK SIGN +2123;N # So VERSICLE +2124;N # Lu DOUBLE-STRUCK CAPITAL Z +2125;N # So OUNCE SIGN +2126;A # Lu OHM SIGN +2127;N # So INVERTED OHM SIGN +2128;N # Lu BLACK-LETTER CAPITAL Z +2129;N # So TURNED GREEK SMALL LETTER IOTA +212A;N # Lu KELVIN SIGN +212B;A # Lu ANGSTROM SIGN +212C..212D;N # Lu [2] SCRIPT CAPITAL B..BLACK-LETTER CAPITAL C +212E;N # So ESTIMATED SYMBOL +212F..2134;N # L& [6] SCRIPT SMALL E..SCRIPT SMALL O +2135..2138;N # Lo [4] ALEF SYMBOL..DALET SYMBOL +2139;N # Ll INFORMATION SOURCE +213A..213B;N # So [2] ROTATED CAPITAL Q..FACSIMILE SIGN +213C..213F;N # L& [4] DOUBLE-STRUCK SMALL PI..DOUBLE-STRUCK CAPITAL PI +2140..2144;N # Sm [5] DOUBLE-STRUCK N-ARY SUMMATION..TURNED SANS-SERIF CAPITAL Y +2145..2149;N # L& [5] DOUBLE-STRUCK ITALIC CAPITAL D..DOUBLE-STRUCK ITALIC SMALL J +214A;N # So PROPERTY LINE +214B;N # Sm TURNED AMPERSAND +214C..214D;N # So [2] PER SIGN..AKTIESELSKAB +214E;N # Ll TURNED SMALL F +214F;N # So SYMBOL FOR SAMARITAN SOURCE +2150..2152;N # No [3] VULGAR FRACTION ONE SEVENTH..VULGAR FRACTION ONE TENTH +2153..2154;A # No [2] VULGAR FRACTION ONE THIRD..VULGAR FRACTION TWO THIRDS +2155..215A;N # No [6] VULGAR FRACTION ONE FIFTH..VULGAR FRACTION FIVE SIXTHS +215B..215E;A # No [4] VULGAR FRACTION ONE EIGHTH..VULGAR FRACTION SEVEN EIGHTHS +215F;N # No FRACTION NUMERATOR ONE +2160..216B;A # Nl [12] ROMAN NUMERAL ONE..ROMAN NUMERAL TWELVE +216C..216F;N # Nl [4] ROMAN NUMERAL FIFTY..ROMAN NUMERAL ONE THOUSAND +2170..2179;A # Nl [10] SMALL ROMAN NUMERAL ONE..SMALL ROMAN NUMERAL TEN +217A..2182;N # Nl [9] SMALL ROMAN NUMERAL ELEVEN..ROMAN NUMERAL TEN THOUSAND +2183..2184;N # L& [2] ROMAN NUMERAL REVERSED ONE HUNDRED..LATIN SMALL LETTER REVERSED C +2185..2188;N # Nl [4] ROMAN NUMERAL SIX LATE FORM..ROMAN NUMERAL ONE HUNDRED THOUSAND +2189;A # No VULGAR FRACTION ZERO THIRDS +218A..218B;N # So [2] TURNED DIGIT TWO..TURNED DIGIT THREE +2190..2194;A # Sm [5] LEFTWARDS ARROW..LEFT RIGHT ARROW +2195..2199;A # So [5] UP DOWN ARROW..SOUTH WEST ARROW +219A..219B;N # Sm [2] LEFTWARDS ARROW WITH STROKE..RIGHTWARDS ARROW WITH STROKE +219C..219F;N # So [4] LEFTWARDS WAVE ARROW..UPWARDS TWO HEADED ARROW +21A0;N # Sm RIGHTWARDS TWO HEADED ARROW +21A1..21A2;N # So [2] DOWNWARDS TWO HEADED ARROW..LEFTWARDS ARROW WITH TAIL +21A3;N # Sm RIGHTWARDS ARROW WITH TAIL +21A4..21A5;N # So [2] LEFTWARDS ARROW FROM BAR..UPWARDS ARROW FROM BAR +21A6;N # Sm RIGHTWARDS ARROW FROM BAR +21A7..21AD;N # So [7] DOWNWARDS ARROW FROM BAR..LEFT RIGHT WAVE ARROW +21AE;N # Sm LEFT RIGHT ARROW WITH STROKE +21AF..21B7;N # So [9] DOWNWARDS ZIGZAG ARROW..CLOCKWISE TOP SEMICIRCLE ARROW +21B8..21B9;A # So [2] NORTH WEST ARROW TO LONG BAR..LEFTWARDS ARROW TO BAR OVER RIGHTWARDS ARROW TO BAR +21BA..21CD;N # So [20] ANTICLOCKWISE OPEN CIRCLE ARROW..LEFTWARDS DOUBLE ARROW WITH STROKE +21CE..21CF;N # Sm [2] LEFT RIGHT DOUBLE ARROW WITH STROKE..RIGHTWARDS DOUBLE ARROW WITH STROKE +21D0..21D1;N # So [2] LEFTWARDS DOUBLE ARROW..UPWARDS DOUBLE ARROW +21D2;A # Sm RIGHTWARDS DOUBLE ARROW +21D3;N # So DOWNWARDS DOUBLE ARROW +21D4;A # Sm LEFT RIGHT DOUBLE ARROW +21D5..21E6;N # So [18] UP DOWN DOUBLE ARROW..LEFTWARDS WHITE ARROW +21E7;A # So UPWARDS WHITE ARROW +21E8..21F3;N # So [12] RIGHTWARDS WHITE ARROW..UP DOWN WHITE ARROW +21F4..21FF;N # Sm [12] RIGHT ARROW WITH SMALL CIRCLE..LEFT RIGHT OPEN-HEADED ARROW +2200;A # Sm FOR ALL +2201;N # Sm COMPLEMENT +2202..2203;A # Sm [2] PARTIAL DIFFERENTIAL..THERE EXISTS +2204..2206;N # Sm [3] THERE DOES NOT EXIST..INCREMENT +2207..2208;A # Sm [2] NABLA..ELEMENT OF +2209..220A;N # Sm [2] NOT AN ELEMENT OF..SMALL ELEMENT OF +220B;A # Sm CONTAINS AS MEMBER +220C..220E;N # Sm [3] DOES NOT CONTAIN AS MEMBER..END OF PROOF +220F;A # Sm N-ARY PRODUCT +2210;N # Sm N-ARY COPRODUCT +2211;A # Sm N-ARY SUMMATION +2212..2214;N # Sm [3] MINUS SIGN..DOT PLUS +2215;A # Sm DIVISION SLASH +2216..2219;N # Sm [4] SET MINUS..BULLET OPERATOR +221A;A # Sm SQUARE ROOT +221B..221C;N # Sm [2] CUBE ROOT..FOURTH ROOT +221D..2220;A # Sm [4] PROPORTIONAL TO..ANGLE +2221..2222;N # Sm [2] MEASURED ANGLE..SPHERICAL ANGLE +2223;A # Sm DIVIDES +2224;N # Sm DOES NOT DIVIDE +2225;A # Sm PARALLEL TO +2226;N # Sm NOT PARALLEL TO +2227..222C;A # Sm [6] LOGICAL AND..DOUBLE INTEGRAL +222D;N # Sm TRIPLE INTEGRAL +222E;A # Sm CONTOUR INTEGRAL +222F..2233;N # Sm [5] SURFACE INTEGRAL..ANTICLOCKWISE CONTOUR INTEGRAL +2234..2237;A # Sm [4] THEREFORE..PROPORTION +2238..223B;N # Sm [4] DOT MINUS..HOMOTHETIC +223C..223D;A # Sm [2] TILDE OPERATOR..REVERSED TILDE +223E..2247;N # Sm [10] INVERTED LAZY S..NEITHER APPROXIMATELY NOR ACTUALLY EQUAL TO +2248;A # Sm ALMOST EQUAL TO +2249..224B;N # Sm [3] NOT ALMOST EQUAL TO..TRIPLE TILDE +224C;A # Sm ALL EQUAL TO +224D..2251;N # Sm [5] EQUIVALENT TO..GEOMETRICALLY EQUAL TO +2252;A # Sm APPROXIMATELY EQUAL TO OR THE IMAGE OF +2253..225F;N # Sm [13] IMAGE OF OR APPROXIMATELY EQUAL TO..QUESTIONED EQUAL TO +2260..2261;A # Sm [2] NOT EQUAL TO..IDENTICAL TO +2262..2263;N # Sm [2] NOT IDENTICAL TO..STRICTLY EQUIVALENT TO +2264..2267;A # Sm [4] LESS-THAN OR EQUAL TO..GREATER-THAN OVER EQUAL TO +2268..2269;N # Sm [2] LESS-THAN BUT NOT EQUAL TO..GREATER-THAN BUT NOT EQUAL TO +226A..226B;A # Sm [2] MUCH LESS-THAN..MUCH GREATER-THAN +226C..226D;N # Sm [2] BETWEEN..NOT EQUIVALENT TO +226E..226F;A # Sm [2] NOT LESS-THAN..NOT GREATER-THAN +2270..2281;N # Sm [18] NEITHER LESS-THAN NOR EQUAL TO..DOES NOT SUCCEED +2282..2283;A # Sm [2] SUBSET OF..SUPERSET OF +2284..2285;N # Sm [2] NOT A SUBSET OF..NOT A SUPERSET OF +2286..2287;A # Sm [2] SUBSET OF OR EQUAL TO..SUPERSET OF OR EQUAL TO +2288..2294;N # Sm [13] NEITHER A SUBSET OF NOR EQUAL TO..SQUARE CUP +2295;A # Sm CIRCLED PLUS +2296..2298;N # Sm [3] CIRCLED MINUS..CIRCLED DIVISION SLASH +2299;A # Sm CIRCLED DOT OPERATOR +229A..22A4;N # Sm [11] CIRCLED RING OPERATOR..DOWN TACK +22A5;A # Sm UP TACK +22A6..22BE;N # Sm [25] ASSERTION..RIGHT ANGLE WITH ARC +22BF;A # Sm RIGHT TRIANGLE +22C0..22FF;N # Sm [64] N-ARY LOGICAL AND..Z NOTATION BAG MEMBERSHIP +2300..2307;N # So [8] DIAMETER SIGN..WAVY LINE +2308;N # Ps LEFT CEILING +2309;N # Pe RIGHT CEILING +230A;N # Ps LEFT FLOOR +230B;N # Pe RIGHT FLOOR +230C..2311;N # So [6] BOTTOM RIGHT CROP..SQUARE LOZENGE +2312;A # So ARC +2313..2319;N # So [7] SEGMENT..TURNED NOT SIGN +231A..231B;W # So [2] WATCH..HOURGLASS +231C..231F;N # So [4] TOP LEFT CORNER..BOTTOM RIGHT CORNER +2320..2321;N # Sm [2] TOP HALF INTEGRAL..BOTTOM HALF INTEGRAL +2322..2328;N # So [7] FROWN..KEYBOARD +2329;W # Ps LEFT-POINTING ANGLE BRACKET +232A;W # Pe RIGHT-POINTING ANGLE BRACKET +232B..237B;N # So [81] ERASE TO THE LEFT..NOT CHECK MARK +237C;N # Sm RIGHT ANGLE WITH DOWNWARDS ZIGZAG ARROW +237D..239A;N # So [30] SHOULDERED OPEN BOX..CLEAR SCREEN SYMBOL +239B..23B3;N # Sm [25] LEFT PARENTHESIS UPPER HOOK..SUMMATION BOTTOM +23B4..23DB;N # So [40] TOP SQUARE BRACKET..FUSE +23DC..23E1;N # Sm [6] TOP PARENTHESIS..BOTTOM TORTOISE SHELL BRACKET +23E2..23E8;N # So [7] WHITE TRAPEZIUM..DECIMAL EXPONENT SYMBOL +23E9..23EC;W # So [4] BLACK RIGHT-POINTING DOUBLE TRIANGLE..BLACK DOWN-POINTING DOUBLE TRIANGLE +23ED..23EF;N # So [3] BLACK RIGHT-POINTING DOUBLE TRIANGLE WITH VERTICAL BAR..BLACK RIGHT-POINTING TRIANGLE WITH DOUBLE VERTICAL BAR +23F0;W # So ALARM CLOCK +23F1..23F2;N # So [2] STOPWATCH..TIMER CLOCK +23F3;W # So HOURGLASS WITH FLOWING SAND +23F4..23FF;N # So [12] BLACK MEDIUM LEFT-POINTING TRIANGLE..OBSERVER EYE SYMBOL +2400..2426;N # So [39] SYMBOL FOR NULL..SYMBOL FOR SUBSTITUTE FORM TWO +2440..244A;N # So [11] OCR HOOK..OCR DOUBLE BACKSLASH +2460..249B;A # No [60] CIRCLED DIGIT ONE..NUMBER TWENTY FULL STOP +249C..24E9;A # So [78] PARENTHESIZED LATIN SMALL LETTER A..CIRCLED LATIN SMALL LETTER Z +24EA;N # No CIRCLED DIGIT ZERO +24EB..24FF;A # No [21] NEGATIVE CIRCLED NUMBER ELEVEN..NEGATIVE CIRCLED DIGIT ZERO +2500..254B;A # So [76] BOX DRAWINGS LIGHT HORIZONTAL..BOX DRAWINGS HEAVY VERTICAL AND HORIZONTAL +254C..254F;N # So [4] BOX DRAWINGS LIGHT DOUBLE DASH HORIZONTAL..BOX DRAWINGS HEAVY DOUBLE DASH VERTICAL +2550..2573;A # So [36] BOX DRAWINGS DOUBLE HORIZONTAL..BOX DRAWINGS LIGHT DIAGONAL CROSS +2574..257F;N # So [12] BOX DRAWINGS LIGHT LEFT..BOX DRAWINGS HEAVY UP AND LIGHT DOWN +2580..258F;A # So [16] UPPER HALF BLOCK..LEFT ONE EIGHTH BLOCK +2590..2591;N # So [2] RIGHT HALF BLOCK..LIGHT SHADE +2592..2595;A # So [4] MEDIUM SHADE..RIGHT ONE EIGHTH BLOCK +2596..259F;N # So [10] QUADRANT LOWER LEFT..QUADRANT UPPER RIGHT AND LOWER LEFT AND LOWER RIGHT +25A0..25A1;A # So [2] BLACK SQUARE..WHITE SQUARE +25A2;N # So WHITE SQUARE WITH ROUNDED CORNERS +25A3..25A9;A # So [7] WHITE SQUARE CONTAINING BLACK SMALL SQUARE..SQUARE WITH DIAGONAL CROSSHATCH FILL +25AA..25B1;N # So [8] BLACK SMALL SQUARE..WHITE PARALLELOGRAM +25B2..25B3;A # So [2] BLACK UP-POINTING TRIANGLE..WHITE UP-POINTING TRIANGLE +25B4..25B5;N # So [2] BLACK UP-POINTING SMALL TRIANGLE..WHITE UP-POINTING SMALL TRIANGLE +25B6;A # So BLACK RIGHT-POINTING TRIANGLE +25B7;A # Sm WHITE RIGHT-POINTING TRIANGLE +25B8..25BB;N # So [4] BLACK RIGHT-POINTING SMALL TRIANGLE..WHITE RIGHT-POINTING POINTER +25BC..25BD;A # So [2] BLACK DOWN-POINTING TRIANGLE..WHITE DOWN-POINTING TRIANGLE +25BE..25BF;N # So [2] BLACK DOWN-POINTING SMALL TRIANGLE..WHITE DOWN-POINTING SMALL TRIANGLE +25C0;A # So BLACK LEFT-POINTING TRIANGLE +25C1;A # Sm WHITE LEFT-POINTING TRIANGLE +25C2..25C5;N # So [4] BLACK LEFT-POINTING SMALL TRIANGLE..WHITE LEFT-POINTING POINTER +25C6..25C8;A # So [3] BLACK DIAMOND..WHITE DIAMOND CONTAINING BLACK SMALL DIAMOND +25C9..25CA;N # So [2] FISHEYE..LOZENGE +25CB;A # So WHITE CIRCLE +25CC..25CD;N # So [2] DOTTED CIRCLE..CIRCLE WITH VERTICAL FILL +25CE..25D1;A # So [4] BULLSEYE..CIRCLE WITH RIGHT HALF BLACK +25D2..25E1;N # So [16] CIRCLE WITH LOWER HALF BLACK..LOWER HALF CIRCLE +25E2..25E5;A # So [4] BLACK LOWER RIGHT TRIANGLE..BLACK UPPER RIGHT TRIANGLE +25E6..25EE;N # So [9] WHITE BULLET..UP-POINTING TRIANGLE WITH RIGHT HALF BLACK +25EF;A # So LARGE CIRCLE +25F0..25F7;N # So [8] WHITE SQUARE WITH UPPER LEFT QUADRANT..WHITE CIRCLE WITH UPPER RIGHT QUADRANT +25F8..25FC;N # Sm [5] UPPER LEFT TRIANGLE..BLACK MEDIUM SQUARE +25FD..25FE;W # Sm [2] WHITE MEDIUM SMALL SQUARE..BLACK MEDIUM SMALL SQUARE +25FF;N # Sm LOWER RIGHT TRIANGLE +2600..2604;N # So [5] BLACK SUN WITH RAYS..COMET +2605..2606;A # So [2] BLACK STAR..WHITE STAR +2607..2608;N # So [2] LIGHTNING..THUNDERSTORM +2609;A # So SUN +260A..260D;N # So [4] ASCENDING NODE..OPPOSITION +260E..260F;A # So [2] BLACK TELEPHONE..WHITE TELEPHONE +2610..2613;N # So [4] BALLOT BOX..SALTIRE +2614..2615;W # So [2] UMBRELLA WITH RAIN DROPS..HOT BEVERAGE +2616..261B;N # So [6] WHITE SHOGI PIECE..BLACK RIGHT POINTING INDEX +261C;A # So WHITE LEFT POINTING INDEX +261D;N # So WHITE UP POINTING INDEX +261E;A # So WHITE RIGHT POINTING INDEX +261F..263F;N # So [33] WHITE DOWN POINTING INDEX..MERCURY +2640;A # So FEMALE SIGN +2641;N # So EARTH +2642;A # So MALE SIGN +2643..2647;N # So [5] JUPITER..PLUTO +2648..2653;W # So [12] ARIES..PISCES +2654..265F;N # So [12] WHITE CHESS KING..BLACK CHESS PAWN +2660..2661;A # So [2] BLACK SPADE SUIT..WHITE HEART SUIT +2662;N # So WHITE DIAMOND SUIT +2663..2665;A # So [3] BLACK CLUB SUIT..BLACK HEART SUIT +2666;N # So BLACK DIAMOND SUIT +2667..266A;A # So [4] WHITE CLUB SUIT..EIGHTH NOTE +266B;N # So BEAMED EIGHTH NOTES +266C..266D;A # So [2] BEAMED SIXTEENTH NOTES..MUSIC FLAT SIGN +266E;N # So MUSIC NATURAL SIGN +266F;A # Sm MUSIC SHARP SIGN +2670..267E;N # So [15] WEST SYRIAC CROSS..PERMANENT PAPER SIGN +267F;W # So WHEELCHAIR SYMBOL +2680..2692;N # So [19] DIE FACE-1..HAMMER AND PICK +2693;W # So ANCHOR +2694..269D;N # So [10] CROSSED SWORDS..OUTLINED WHITE STAR +269E..269F;A # So [2] THREE LINES CONVERGING RIGHT..THREE LINES CONVERGING LEFT +26A0;N # So WARNING SIGN +26A1;W # So HIGH VOLTAGE SIGN +26A2..26A9;N # So [8] DOUBLED FEMALE SIGN..HORIZONTAL MALE WITH STROKE SIGN +26AA..26AB;W # So [2] MEDIUM WHITE CIRCLE..MEDIUM BLACK CIRCLE +26AC..26BC;N # So [17] MEDIUM SMALL WHITE CIRCLE..SESQUIQUADRATE +26BD..26BE;W # So [2] SOCCER BALL..BASEBALL +26BF;A # So SQUARED KEY +26C0..26C3;N # So [4] WHITE DRAUGHTS MAN..BLACK DRAUGHTS KING +26C4..26C5;W # So [2] SNOWMAN WITHOUT SNOW..SUN BEHIND CLOUD +26C6..26CD;A # So [8] RAIN..DISABLED CAR +26CE;W # So OPHIUCHUS +26CF..26D3;A # So [5] PICK..CHAINS +26D4;W # So NO ENTRY +26D5..26E1;A # So [13] ALTERNATE ONE-WAY LEFT WAY TRAFFIC..RESTRICTED LEFT ENTRY-2 +26E2;N # So ASTRONOMICAL SYMBOL FOR URANUS +26E3;A # So HEAVY CIRCLE WITH STROKE AND TWO DOTS ABOVE +26E4..26E7;N # So [4] PENTAGRAM..INVERTED PENTAGRAM +26E8..26E9;A # So [2] BLACK CROSS ON SHIELD..SHINTO SHRINE +26EA;W # So CHURCH +26EB..26F1;A # So [7] CASTLE..UMBRELLA ON GROUND +26F2..26F3;W # So [2] FOUNTAIN..FLAG IN HOLE +26F4;A # So FERRY +26F5;W # So SAILBOAT +26F6..26F9;A # So [4] SQUARE FOUR CORNERS..PERSON WITH BALL +26FA;W # So TENT +26FB..26FC;A # So [2] JAPANESE BANK SYMBOL..HEADSTONE GRAVEYARD SYMBOL +26FD;W # So FUEL PUMP +26FE..26FF;A # So [2] CUP ON BLACK SQUARE..WHITE FLAG WITH HORIZONTAL MIDDLE BLACK STRIPE +2700..2704;N # So [5] BLACK SAFETY SCISSORS..WHITE SCISSORS +2705;W # So WHITE HEAVY CHECK MARK +2706..2709;N # So [4] TELEPHONE LOCATION SIGN..ENVELOPE +270A..270B;W # So [2] RAISED FIST..RAISED HAND +270C..2727;N # So [28] VICTORY HAND..WHITE FOUR POINTED STAR +2728;W # So SPARKLES +2729..273C;N # So [20] STRESS OUTLINED WHITE STAR..OPEN CENTRE TEARDROP-SPOKED ASTERISK +273D;A # So HEAVY TEARDROP-SPOKED ASTERISK +273E..274B;N # So [14] SIX PETALLED BLACK AND WHITE FLORETTE..HEAVY EIGHT TEARDROP-SPOKED PROPELLER ASTERISK +274C;W # So CROSS MARK +274D;N # So SHADOWED WHITE CIRCLE +274E;W # So NEGATIVE SQUARED CROSS MARK +274F..2752;N # So [4] LOWER RIGHT DROP-SHADOWED WHITE SQUARE..UPPER RIGHT SHADOWED WHITE SQUARE +2753..2755;W # So [3] BLACK QUESTION MARK ORNAMENT..WHITE EXCLAMATION MARK ORNAMENT +2756;N # So BLACK DIAMOND MINUS WHITE X +2757;W # So HEAVY EXCLAMATION MARK SYMBOL +2758..2767;N # So [16] LIGHT VERTICAL BAR..ROTATED FLORAL HEART BULLET +2768;N # Ps MEDIUM LEFT PARENTHESIS ORNAMENT +2769;N # Pe MEDIUM RIGHT PARENTHESIS ORNAMENT +276A;N # Ps MEDIUM FLATTENED LEFT PARENTHESIS ORNAMENT +276B;N # Pe MEDIUM FLATTENED RIGHT PARENTHESIS ORNAMENT +276C;N # Ps MEDIUM LEFT-POINTING ANGLE BRACKET ORNAMENT +276D;N # Pe MEDIUM RIGHT-POINTING ANGLE BRACKET ORNAMENT +276E;N # Ps HEAVY LEFT-POINTING ANGLE QUOTATION MARK ORNAMENT +276F;N # Pe HEAVY RIGHT-POINTING ANGLE QUOTATION MARK ORNAMENT +2770;N # Ps HEAVY LEFT-POINTING ANGLE BRACKET ORNAMENT +2771;N # Pe HEAVY RIGHT-POINTING ANGLE BRACKET ORNAMENT +2772;N # Ps LIGHT LEFT TORTOISE SHELL BRACKET ORNAMENT +2773;N # Pe LIGHT RIGHT TORTOISE SHELL BRACKET ORNAMENT +2774;N # Ps MEDIUM LEFT CURLY BRACKET ORNAMENT +2775;N # Pe MEDIUM RIGHT CURLY BRACKET ORNAMENT +2776..277F;A # No [10] DINGBAT NEGATIVE CIRCLED DIGIT ONE..DINGBAT NEGATIVE CIRCLED NUMBER TEN +2780..2793;N # No [20] DINGBAT CIRCLED SANS-SERIF DIGIT ONE..DINGBAT NEGATIVE CIRCLED SANS-SERIF NUMBER TEN +2794;N # So HEAVY WIDE-HEADED RIGHTWARDS ARROW +2795..2797;W # So [3] HEAVY PLUS SIGN..HEAVY DIVISION SIGN +2798..27AF;N # So [24] HEAVY SOUTH EAST ARROW..NOTCHED LOWER RIGHT-SHADOWED WHITE RIGHTWARDS ARROW +27B0;W # So CURLY LOOP +27B1..27BE;N # So [14] NOTCHED UPPER RIGHT-SHADOWED WHITE RIGHTWARDS ARROW..OPEN-OUTLINED RIGHTWARDS ARROW +27BF;W # So DOUBLE CURLY LOOP +27C0..27C4;N # Sm [5] THREE DIMENSIONAL ANGLE..OPEN SUPERSET +27C5;N # Ps LEFT S-SHAPED BAG DELIMITER +27C6;N # Pe RIGHT S-SHAPED BAG DELIMITER +27C7..27E5;N # Sm [31] OR WITH DOT INSIDE..WHITE SQUARE WITH RIGHTWARDS TICK +27E6;Na # Ps MATHEMATICAL LEFT WHITE SQUARE BRACKET +27E7;Na # Pe MATHEMATICAL RIGHT WHITE SQUARE BRACKET +27E8;Na # Ps MATHEMATICAL LEFT ANGLE BRACKET +27E9;Na # Pe MATHEMATICAL RIGHT ANGLE BRACKET +27EA;Na # Ps MATHEMATICAL LEFT DOUBLE ANGLE BRACKET +27EB;Na # Pe MATHEMATICAL RIGHT DOUBLE ANGLE BRACKET +27EC;Na # Ps MATHEMATICAL LEFT WHITE TORTOISE SHELL BRACKET +27ED;Na # Pe MATHEMATICAL RIGHT WHITE TORTOISE SHELL BRACKET +27EE;N # Ps MATHEMATICAL LEFT FLATTENED PARENTHESIS +27EF;N # Pe MATHEMATICAL RIGHT FLATTENED PARENTHESIS +27F0..27FF;N # Sm [16] UPWARDS QUADRUPLE ARROW..LONG RIGHTWARDS SQUIGGLE ARROW +2800..28FF;N # So [256] BRAILLE PATTERN BLANK..BRAILLE PATTERN DOTS-12345678 +2900..297F;N # Sm [128] RIGHTWARDS TWO-HEADED ARROW WITH VERTICAL STROKE..DOWN FISH TAIL +2980..2982;N # Sm [3] TRIPLE VERTICAL BAR DELIMITER..Z NOTATION TYPE COLON +2983;N # Ps LEFT WHITE CURLY BRACKET +2984;N # Pe RIGHT WHITE CURLY BRACKET +2985;Na # Ps LEFT WHITE PARENTHESIS +2986;Na # Pe RIGHT WHITE PARENTHESIS +2987;N # Ps Z NOTATION LEFT IMAGE BRACKET +2988;N # Pe Z NOTATION RIGHT IMAGE BRACKET +2989;N # Ps Z NOTATION LEFT BINDING BRACKET +298A;N # Pe Z NOTATION RIGHT BINDING BRACKET +298B;N # Ps LEFT SQUARE BRACKET WITH UNDERBAR +298C;N # Pe RIGHT SQUARE BRACKET WITH UNDERBAR +298D;N # Ps LEFT SQUARE BRACKET WITH TICK IN TOP CORNER +298E;N # Pe RIGHT SQUARE BRACKET WITH TICK IN BOTTOM CORNER +298F;N # Ps LEFT SQUARE BRACKET WITH TICK IN BOTTOM CORNER +2990;N # Pe RIGHT SQUARE BRACKET WITH TICK IN TOP CORNER +2991;N # Ps LEFT ANGLE BRACKET WITH DOT +2992;N # Pe RIGHT ANGLE BRACKET WITH DOT +2993;N # Ps LEFT ARC LESS-THAN BRACKET +2994;N # Pe RIGHT ARC GREATER-THAN BRACKET +2995;N # Ps DOUBLE LEFT ARC GREATER-THAN BRACKET +2996;N # Pe DOUBLE RIGHT ARC LESS-THAN BRACKET +2997;N # Ps LEFT BLACK TORTOISE SHELL BRACKET +2998;N # Pe RIGHT BLACK TORTOISE SHELL BRACKET +2999..29D7;N # Sm [63] DOTTED FENCE..BLACK HOURGLASS +29D8;N # Ps LEFT WIGGLY FENCE +29D9;N # Pe RIGHT WIGGLY FENCE +29DA;N # Ps LEFT DOUBLE WIGGLY FENCE +29DB;N # Pe RIGHT DOUBLE WIGGLY FENCE +29DC..29FB;N # Sm [32] INCOMPLETE INFINITY..TRIPLE PLUS +29FC;N # Ps LEFT-POINTING CURVED ANGLE BRACKET +29FD;N # Pe RIGHT-POINTING CURVED ANGLE BRACKET +29FE..29FF;N # Sm [2] TINY..MINY +2A00..2AFF;N # Sm [256] N-ARY CIRCLED DOT OPERATOR..N-ARY WHITE VERTICAL BAR +2B00..2B1A;N # So [27] NORTH EAST WHITE ARROW..DOTTED SQUARE +2B1B..2B1C;W # So [2] BLACK LARGE SQUARE..WHITE LARGE SQUARE +2B1D..2B2F;N # So [19] BLACK VERY SMALL SQUARE..WHITE VERTICAL ELLIPSE +2B30..2B44;N # Sm [21] LEFT ARROW WITH SMALL CIRCLE..RIGHTWARDS ARROW THROUGH SUPERSET +2B45..2B46;N # So [2] LEFTWARDS QUADRUPLE ARROW..RIGHTWARDS QUADRUPLE ARROW +2B47..2B4C;N # Sm [6] REVERSE TILDE OPERATOR ABOVE RIGHTWARDS ARROW..RIGHTWARDS ARROW ABOVE REVERSE TILDE OPERATOR +2B4D..2B4F;N # So [3] DOWNWARDS TRIANGLE-HEADED ZIGZAG ARROW..SHORT BACKSLANTED SOUTH ARROW +2B50;W # So WHITE MEDIUM STAR +2B51..2B54;N # So [4] BLACK SMALL STAR..WHITE RIGHT-POINTING PENTAGON +2B55;W # So HEAVY LARGE CIRCLE +2B56..2B59;A # So [4] HEAVY OVAL WITH OVAL INSIDE..HEAVY CIRCLED SALTIRE +2B5A..2B73;N # So [26] SLANTED NORTH ARROW WITH HOOKED HEAD..DOWNWARDS TRIANGLE-HEADED ARROW TO BAR +2B76..2B95;N # So [32] NORTH WEST TRIANGLE-HEADED ARROW TO BAR..RIGHTWARDS BLACK ARROW +2B97..2BFF;N # So [105] SYMBOL FOR TYPE A ELECTRONICS..HELLSCHREIBER PAUSE SYMBOL +2C00..2C5F;N # L& [96] GLAGOLITIC CAPITAL LETTER AZU..GLAGOLITIC SMALL LETTER CAUDATE CHRIVI +2C60..2C7B;N # L& [28] LATIN CAPITAL LETTER L WITH DOUBLE BAR..LATIN LETTER SMALL CAPITAL TURNED E +2C7C..2C7D;N # Lm [2] LATIN SUBSCRIPT SMALL LETTER J..MODIFIER LETTER CAPITAL V +2C7E..2C7F;N # Lu [2] LATIN CAPITAL LETTER S WITH SWASH TAIL..LATIN CAPITAL LETTER Z WITH SWASH TAIL +2C80..2CE4;N # L& [101] COPTIC CAPITAL LETTER ALFA..COPTIC SYMBOL KAI +2CE5..2CEA;N # So [6] COPTIC SYMBOL MI RO..COPTIC SYMBOL SHIMA SIMA +2CEB..2CEE;N # L& [4] COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI..COPTIC SMALL LETTER CRYPTOGRAMMIC GANGIA +2CEF..2CF1;N # Mn [3] COPTIC COMBINING NI ABOVE..COPTIC COMBINING SPIRITUS LENIS +2CF2..2CF3;N # L& [2] COPTIC CAPITAL LETTER BOHAIRIC KHEI..COPTIC SMALL LETTER BOHAIRIC KHEI +2CF9..2CFC;N # Po [4] COPTIC OLD NUBIAN FULL STOP..COPTIC OLD NUBIAN VERSE DIVIDER +2CFD;N # No COPTIC FRACTION ONE HALF +2CFE..2CFF;N # Po [2] COPTIC FULL STOP..COPTIC MORPHOLOGICAL DIVIDER +2D00..2D25;N # Ll [38] GEORGIAN SMALL LETTER AN..GEORGIAN SMALL LETTER HOE +2D27;N # Ll GEORGIAN SMALL LETTER YN +2D2D;N # Ll GEORGIAN SMALL LETTER AEN +2D30..2D67;N # Lo [56] TIFINAGH LETTER YA..TIFINAGH LETTER YO +2D6F;N # Lm TIFINAGH MODIFIER LETTER LABIALIZATION MARK +2D70;N # Po TIFINAGH SEPARATOR MARK +2D7F;N # Mn TIFINAGH CONSONANT JOINER +2D80..2D96;N # Lo [23] ETHIOPIC SYLLABLE LOA..ETHIOPIC SYLLABLE GGWE +2DA0..2DA6;N # Lo [7] ETHIOPIC SYLLABLE SSA..ETHIOPIC SYLLABLE SSO +2DA8..2DAE;N # Lo [7] ETHIOPIC SYLLABLE CCA..ETHIOPIC SYLLABLE CCO +2DB0..2DB6;N # Lo [7] ETHIOPIC SYLLABLE ZZA..ETHIOPIC SYLLABLE ZZO +2DB8..2DBE;N # Lo [7] ETHIOPIC SYLLABLE CCHA..ETHIOPIC SYLLABLE CCHO +2DC0..2DC6;N # Lo [7] ETHIOPIC SYLLABLE QYA..ETHIOPIC SYLLABLE QYO +2DC8..2DCE;N # Lo [7] ETHIOPIC SYLLABLE KYA..ETHIOPIC SYLLABLE KYO +2DD0..2DD6;N # Lo [7] ETHIOPIC SYLLABLE XYA..ETHIOPIC SYLLABLE XYO +2DD8..2DDE;N # Lo [7] ETHIOPIC SYLLABLE GYA..ETHIOPIC SYLLABLE GYO +2DE0..2DFF;N # Mn [32] COMBINING CYRILLIC LETTER BE..COMBINING CYRILLIC LETTER IOTIFIED BIG YUS +2E00..2E01;N # Po [2] RIGHT ANGLE SUBSTITUTION MARKER..RIGHT ANGLE DOTTED SUBSTITUTION MARKER +2E02;N # Pi LEFT SUBSTITUTION BRACKET +2E03;N # Pf RIGHT SUBSTITUTION BRACKET +2E04;N # Pi LEFT DOTTED SUBSTITUTION BRACKET +2E05;N # Pf RIGHT DOTTED SUBSTITUTION BRACKET +2E06..2E08;N # Po [3] RAISED INTERPOLATION MARKER..DOTTED TRANSPOSITION MARKER +2E09;N # Pi LEFT TRANSPOSITION BRACKET +2E0A;N # Pf RIGHT TRANSPOSITION BRACKET +2E0B;N # Po RAISED SQUARE +2E0C;N # Pi LEFT RAISED OMISSION BRACKET +2E0D;N # Pf RIGHT RAISED OMISSION BRACKET +2E0E..2E16;N # Po [9] EDITORIAL CORONIS..DOTTED RIGHT-POINTING ANGLE +2E17;N # Pd DOUBLE OBLIQUE HYPHEN +2E18..2E19;N # Po [2] INVERTED INTERROBANG..PALM BRANCH +2E1A;N # Pd HYPHEN WITH DIAERESIS +2E1B;N # Po TILDE WITH RING ABOVE +2E1C;N # Pi LEFT LOW PARAPHRASE BRACKET +2E1D;N # Pf RIGHT LOW PARAPHRASE BRACKET +2E1E..2E1F;N # Po [2] TILDE WITH DOT ABOVE..TILDE WITH DOT BELOW +2E20;N # Pi LEFT VERTICAL BAR WITH QUILL +2E21;N # Pf RIGHT VERTICAL BAR WITH QUILL +2E22;N # Ps TOP LEFT HALF BRACKET +2E23;N # Pe TOP RIGHT HALF BRACKET +2E24;N # Ps BOTTOM LEFT HALF BRACKET +2E25;N # Pe BOTTOM RIGHT HALF BRACKET +2E26;N # Ps LEFT SIDEWAYS U BRACKET +2E27;N # Pe RIGHT SIDEWAYS U BRACKET +2E28;N # Ps LEFT DOUBLE PARENTHESIS +2E29;N # Pe RIGHT DOUBLE PARENTHESIS +2E2A..2E2E;N # Po [5] TWO DOTS OVER ONE DOT PUNCTUATION..REVERSED QUESTION MARK +2E2F;N # Lm VERTICAL TILDE +2E30..2E39;N # Po [10] RING POINT..TOP HALF SECTION SIGN +2E3A..2E3B;N # Pd [2] TWO-EM DASH..THREE-EM DASH +2E3C..2E3F;N # Po [4] STENOGRAPHIC FULL STOP..CAPITULUM +2E40;N # Pd DOUBLE HYPHEN +2E41;N # Po REVERSED COMMA +2E42;N # Ps DOUBLE LOW-REVERSED-9 QUOTATION MARK +2E43..2E4F;N # Po [13] DASH WITH LEFT UPTURN..CORNISH VERSE DIVIDER +2E50..2E51;N # So [2] CROSS PATTY WITH RIGHT CROSSBAR..CROSS PATTY WITH LEFT CROSSBAR +2E52..2E54;N # Po [3] TIRONIAN SIGN CAPITAL ET..MEDIEVAL QUESTION MARK +2E55;N # Ps LEFT SQUARE BRACKET WITH STROKE +2E56;N # Pe RIGHT SQUARE BRACKET WITH STROKE +2E57;N # Ps LEFT SQUARE BRACKET WITH DOUBLE STROKE +2E58;N # Pe RIGHT SQUARE BRACKET WITH DOUBLE STROKE +2E59;N # Ps TOP HALF LEFT PARENTHESIS +2E5A;N # Pe TOP HALF RIGHT PARENTHESIS +2E5B;N # Ps BOTTOM HALF LEFT PARENTHESIS +2E5C;N # Pe BOTTOM HALF RIGHT PARENTHESIS +2E5D;N # Pd OBLIQUE HYPHEN +2E80..2E99;W # So [26] CJK RADICAL REPEAT..CJK RADICAL RAP +2E9B..2EF3;W # So [89] CJK RADICAL CHOKE..CJK RADICAL C-SIMPLIFIED TURTLE +2F00..2FD5;W # So [214] KANGXI RADICAL ONE..KANGXI RADICAL FLUTE +2FF0..2FFB;W # So [12] IDEOGRAPHIC DESCRIPTION CHARACTER LEFT TO RIGHT..IDEOGRAPHIC DESCRIPTION CHARACTER OVERLAID +3000;F # Zs IDEOGRAPHIC SPACE +3001..3003;W # Po [3] IDEOGRAPHIC COMMA..DITTO MARK +3004;W # So JAPANESE INDUSTRIAL STANDARD SYMBOL +3005;W # Lm IDEOGRAPHIC ITERATION MARK +3006;W # Lo IDEOGRAPHIC CLOSING MARK +3007;W # Nl IDEOGRAPHIC NUMBER ZERO +3008;W # Ps LEFT ANGLE BRACKET +3009;W # Pe RIGHT ANGLE BRACKET +300A;W # Ps LEFT DOUBLE ANGLE BRACKET +300B;W # Pe RIGHT DOUBLE ANGLE BRACKET +300C;W # Ps LEFT CORNER BRACKET +300D;W # Pe RIGHT CORNER BRACKET +300E;W # Ps LEFT WHITE CORNER BRACKET +300F;W # Pe RIGHT WHITE CORNER BRACKET +3010;W # Ps LEFT BLACK LENTICULAR BRACKET +3011;W # Pe RIGHT BLACK LENTICULAR BRACKET +3012..3013;W # So [2] POSTAL MARK..GETA MARK +3014;W # Ps LEFT TORTOISE SHELL BRACKET +3015;W # Pe RIGHT TORTOISE SHELL BRACKET +3016;W # Ps LEFT WHITE LENTICULAR BRACKET +3017;W # Pe RIGHT WHITE LENTICULAR BRACKET +3018;W # Ps LEFT WHITE TORTOISE SHELL BRACKET +3019;W # Pe RIGHT WHITE TORTOISE SHELL BRACKET +301A;W # Ps LEFT WHITE SQUARE BRACKET +301B;W # Pe RIGHT WHITE SQUARE BRACKET +301C;W # Pd WAVE DASH +301D;W # Ps REVERSED DOUBLE PRIME QUOTATION MARK +301E..301F;W # Pe [2] DOUBLE PRIME QUOTATION MARK..LOW DOUBLE PRIME QUOTATION MARK +3020;W # So POSTAL MARK FACE +3021..3029;W # Nl [9] HANGZHOU NUMERAL ONE..HANGZHOU NUMERAL NINE +302A..302D;W # Mn [4] IDEOGRAPHIC LEVEL TONE MARK..IDEOGRAPHIC ENTERING TONE MARK +302E..302F;W # Mc [2] HANGUL SINGLE DOT TONE MARK..HANGUL DOUBLE DOT TONE MARK +3030;W # Pd WAVY DASH +3031..3035;W # Lm [5] VERTICAL KANA REPEAT MARK..VERTICAL KANA REPEAT MARK LOWER HALF +3036..3037;W # So [2] CIRCLED POSTAL MARK..IDEOGRAPHIC TELEGRAPH LINE FEED SEPARATOR SYMBOL +3038..303A;W # Nl [3] HANGZHOU NUMERAL TEN..HANGZHOU NUMERAL THIRTY +303B;W # Lm VERTICAL IDEOGRAPHIC ITERATION MARK +303C;W # Lo MASU MARK +303D;W # Po PART ALTERNATION MARK +303E;W # So IDEOGRAPHIC VARIATION INDICATOR +303F;N # So IDEOGRAPHIC HALF FILL SPACE +3041..3096;W # Lo [86] HIRAGANA LETTER SMALL A..HIRAGANA LETTER SMALL KE +3099..309A;W # Mn [2] COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK..COMBINING KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK +309B..309C;W # Sk [2] KATAKANA-HIRAGANA VOICED SOUND MARK..KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK +309D..309E;W # Lm [2] HIRAGANA ITERATION MARK..HIRAGANA VOICED ITERATION MARK +309F;W # Lo HIRAGANA DIGRAPH YORI +30A0;W # Pd KATAKANA-HIRAGANA DOUBLE HYPHEN +30A1..30FA;W # Lo [90] KATAKANA LETTER SMALL A..KATAKANA LETTER VO +30FB;W # Po KATAKANA MIDDLE DOT +30FC..30FE;W # Lm [3] KATAKANA-HIRAGANA PROLONGED SOUND MARK..KATAKANA VOICED ITERATION MARK +30FF;W # Lo KATAKANA DIGRAPH KOTO +3105..312F;W # Lo [43] BOPOMOFO LETTER B..BOPOMOFO LETTER NN +3131..318E;W # Lo [94] HANGUL LETTER KIYEOK..HANGUL LETTER ARAEAE +3190..3191;W # So [2] IDEOGRAPHIC ANNOTATION LINKING MARK..IDEOGRAPHIC ANNOTATION REVERSE MARK +3192..3195;W # No [4] IDEOGRAPHIC ANNOTATION ONE MARK..IDEOGRAPHIC ANNOTATION FOUR MARK +3196..319F;W # So [10] IDEOGRAPHIC ANNOTATION TOP MARK..IDEOGRAPHIC ANNOTATION MAN MARK +31A0..31BF;W # Lo [32] BOPOMOFO LETTER BU..BOPOMOFO LETTER AH +31C0..31E3;W # So [36] CJK STROKE T..CJK STROKE Q +31F0..31FF;W # Lo [16] KATAKANA LETTER SMALL KU..KATAKANA LETTER SMALL RO +3200..321E;W # So [31] PARENTHESIZED HANGUL KIYEOK..PARENTHESIZED KOREAN CHARACTER O HU +3220..3229;W # No [10] PARENTHESIZED IDEOGRAPH ONE..PARENTHESIZED IDEOGRAPH TEN +322A..3247;W # So [30] PARENTHESIZED IDEOGRAPH MOON..CIRCLED IDEOGRAPH KOTO +3248..324F;A # No [8] CIRCLED NUMBER TEN ON BLACK SQUARE..CIRCLED NUMBER EIGHTY ON BLACK SQUARE +3250;W # So PARTNERSHIP SIGN +3251..325F;W # No [15] CIRCLED NUMBER TWENTY ONE..CIRCLED NUMBER THIRTY FIVE +3260..327F;W # So [32] CIRCLED HANGUL KIYEOK..KOREAN STANDARD SYMBOL +3280..3289;W # No [10] CIRCLED IDEOGRAPH ONE..CIRCLED IDEOGRAPH TEN +328A..32B0;W # So [39] CIRCLED IDEOGRAPH MOON..CIRCLED IDEOGRAPH NIGHT +32B1..32BF;W # No [15] CIRCLED NUMBER THIRTY SIX..CIRCLED NUMBER FIFTY +32C0..32FF;W # So [64] IDEOGRAPHIC TELEGRAPH SYMBOL FOR JANUARY..SQUARE ERA NAME REIWA +3300..33FF;W # So [256] SQUARE APAATO..SQUARE GAL +3400..4DBF;W # Lo [6592] CJK UNIFIED IDEOGRAPH-3400..CJK UNIFIED IDEOGRAPH-4DBF +4DC0..4DFF;N # So [64] HEXAGRAM FOR THE CREATIVE HEAVEN..HEXAGRAM FOR BEFORE COMPLETION +4E00..9FFF;W # Lo [20992] CJK UNIFIED IDEOGRAPH-4E00..CJK UNIFIED IDEOGRAPH-9FFF +A000..A014;W # Lo [21] YI SYLLABLE IT..YI SYLLABLE E +A015;W # Lm YI SYLLABLE WU +A016..A48C;W # Lo [1143] YI SYLLABLE BIT..YI SYLLABLE YYR +A490..A4C6;W # So [55] YI RADICAL QOT..YI RADICAL KE +A4D0..A4F7;N # Lo [40] LISU LETTER BA..LISU LETTER OE +A4F8..A4FD;N # Lm [6] LISU LETTER TONE MYA TI..LISU LETTER TONE MYA JEU +A4FE..A4FF;N # Po [2] LISU PUNCTUATION COMMA..LISU PUNCTUATION FULL STOP +A500..A60B;N # Lo [268] VAI SYLLABLE EE..VAI SYLLABLE NG +A60C;N # Lm VAI SYLLABLE LENGTHENER +A60D..A60F;N # Po [3] VAI COMMA..VAI QUESTION MARK +A610..A61F;N # Lo [16] VAI SYLLABLE NDOLE FA..VAI SYMBOL JONG +A620..A629;N # Nd [10] VAI DIGIT ZERO..VAI DIGIT NINE +A62A..A62B;N # Lo [2] VAI SYLLABLE NDOLE MA..VAI SYLLABLE NDOLE DO +A640..A66D;N # L& [46] CYRILLIC CAPITAL LETTER ZEMLYA..CYRILLIC SMALL LETTER DOUBLE MONOCULAR O +A66E;N # Lo CYRILLIC LETTER MULTIOCULAR O +A66F;N # Mn COMBINING CYRILLIC VZMET +A670..A672;N # Me [3] COMBINING CYRILLIC TEN MILLIONS SIGN..COMBINING CYRILLIC THOUSAND MILLIONS SIGN +A673;N # Po SLAVONIC ASTERISK +A674..A67D;N # Mn [10] COMBINING CYRILLIC LETTER UKRAINIAN IE..COMBINING CYRILLIC PAYEROK +A67E;N # Po CYRILLIC KAVYKA +A67F;N # Lm CYRILLIC PAYEROK +A680..A69B;N # L& [28] CYRILLIC CAPITAL LETTER DWE..CYRILLIC SMALL LETTER CROSSED O +A69C..A69D;N # Lm [2] MODIFIER LETTER CYRILLIC HARD SIGN..MODIFIER LETTER CYRILLIC SOFT SIGN +A69E..A69F;N # Mn [2] COMBINING CYRILLIC LETTER EF..COMBINING CYRILLIC LETTER IOTIFIED E +A6A0..A6E5;N # Lo [70] BAMUM LETTER A..BAMUM LETTER KI +A6E6..A6EF;N # Nl [10] BAMUM LETTER MO..BAMUM LETTER KOGHOM +A6F0..A6F1;N # Mn [2] BAMUM COMBINING MARK KOQNDON..BAMUM COMBINING MARK TUKWENTIS +A6F2..A6F7;N # Po [6] BAMUM NJAEMLI..BAMUM QUESTION MARK +A700..A716;N # Sk [23] MODIFIER LETTER CHINESE TONE YIN PING..MODIFIER LETTER EXTRA-LOW LEFT-STEM TONE BAR +A717..A71F;N # Lm [9] MODIFIER LETTER DOT VERTICAL BAR..MODIFIER LETTER LOW INVERTED EXCLAMATION MARK +A720..A721;N # Sk [2] MODIFIER LETTER STRESS AND HIGH TONE..MODIFIER LETTER STRESS AND LOW TONE +A722..A76F;N # L& [78] LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF..LATIN SMALL LETTER CON +A770;N # Lm MODIFIER LETTER US +A771..A787;N # L& [23] LATIN SMALL LETTER DUM..LATIN SMALL LETTER INSULAR T +A788;N # Lm MODIFIER LETTER LOW CIRCUMFLEX ACCENT +A789..A78A;N # Sk [2] MODIFIER LETTER COLON..MODIFIER LETTER SHORT EQUALS SIGN +A78B..A78E;N # L& [4] LATIN CAPITAL LETTER SALTILLO..LATIN SMALL LETTER L WITH RETROFLEX HOOK AND BELT +A78F;N # Lo LATIN LETTER SINOLOGICAL DOT +A790..A7CA;N # L& [59] LATIN CAPITAL LETTER N WITH DESCENDER..LATIN SMALL LETTER S WITH SHORT STROKE OVERLAY +A7D0..A7D1;N # L& [2] LATIN CAPITAL LETTER CLOSED INSULAR G..LATIN SMALL LETTER CLOSED INSULAR G +A7D3;N # Ll LATIN SMALL LETTER DOUBLE THORN +A7D5..A7D9;N # L& [5] LATIN SMALL LETTER DOUBLE WYNN..LATIN SMALL LETTER SIGMOID S +A7F2..A7F4;N # Lm [3] MODIFIER LETTER CAPITAL C..MODIFIER LETTER CAPITAL Q +A7F5..A7F6;N # L& [2] LATIN CAPITAL LETTER REVERSED HALF H..LATIN SMALL LETTER REVERSED HALF H +A7F7;N # Lo LATIN EPIGRAPHIC LETTER SIDEWAYS I +A7F8..A7F9;N # Lm [2] MODIFIER LETTER CAPITAL H WITH STROKE..MODIFIER LETTER SMALL LIGATURE OE +A7FA;N # Ll LATIN LETTER SMALL CAPITAL TURNED M +A7FB..A7FF;N # Lo [5] LATIN EPIGRAPHIC LETTER REVERSED F..LATIN EPIGRAPHIC LETTER ARCHAIC M +A800..A801;N # Lo [2] SYLOTI NAGRI LETTER A..SYLOTI NAGRI LETTER I +A802;N # Mn SYLOTI NAGRI SIGN DVISVARA +A803..A805;N # Lo [3] SYLOTI NAGRI LETTER U..SYLOTI NAGRI LETTER O +A806;N # Mn SYLOTI NAGRI SIGN HASANTA +A807..A80A;N # Lo [4] SYLOTI NAGRI LETTER KO..SYLOTI NAGRI LETTER GHO +A80B;N # Mn SYLOTI NAGRI SIGN ANUSVARA +A80C..A822;N # Lo [23] SYLOTI NAGRI LETTER CO..SYLOTI NAGRI LETTER HO +A823..A824;N # Mc [2] SYLOTI NAGRI VOWEL SIGN A..SYLOTI NAGRI VOWEL SIGN I +A825..A826;N # Mn [2] SYLOTI NAGRI VOWEL SIGN U..SYLOTI NAGRI VOWEL SIGN E +A827;N # Mc SYLOTI NAGRI VOWEL SIGN OO +A828..A82B;N # So [4] SYLOTI NAGRI POETRY MARK-1..SYLOTI NAGRI POETRY MARK-4 +A82C;N # Mn SYLOTI NAGRI SIGN ALTERNATE HASANTA +A830..A835;N # No [6] NORTH INDIC FRACTION ONE QUARTER..NORTH INDIC FRACTION THREE SIXTEENTHS +A836..A837;N # So [2] NORTH INDIC QUARTER MARK..NORTH INDIC PLACEHOLDER MARK +A838;N # Sc NORTH INDIC RUPEE MARK +A839;N # So NORTH INDIC QUANTITY MARK +A840..A873;N # Lo [52] PHAGS-PA LETTER KA..PHAGS-PA LETTER CANDRABINDU +A874..A877;N # Po [4] PHAGS-PA SINGLE HEAD MARK..PHAGS-PA MARK DOUBLE SHAD +A880..A881;N # Mc [2] SAURASHTRA SIGN ANUSVARA..SAURASHTRA SIGN VISARGA +A882..A8B3;N # Lo [50] SAURASHTRA LETTER A..SAURASHTRA LETTER LLA +A8B4..A8C3;N # Mc [16] SAURASHTRA CONSONANT SIGN HAARU..SAURASHTRA VOWEL SIGN AU +A8C4..A8C5;N # Mn [2] SAURASHTRA SIGN VIRAMA..SAURASHTRA SIGN CANDRABINDU +A8CE..A8CF;N # Po [2] SAURASHTRA DANDA..SAURASHTRA DOUBLE DANDA +A8D0..A8D9;N # Nd [10] SAURASHTRA DIGIT ZERO..SAURASHTRA DIGIT NINE +A8E0..A8F1;N # Mn [18] COMBINING DEVANAGARI DIGIT ZERO..COMBINING DEVANAGARI SIGN AVAGRAHA +A8F2..A8F7;N # Lo [6] DEVANAGARI SIGN SPACING CANDRABINDU..DEVANAGARI SIGN CANDRABINDU AVAGRAHA +A8F8..A8FA;N # Po [3] DEVANAGARI SIGN PUSHPIKA..DEVANAGARI CARET +A8FB;N # Lo DEVANAGARI HEADSTROKE +A8FC;N # Po DEVANAGARI SIGN SIDDHAM +A8FD..A8FE;N # Lo [2] DEVANAGARI JAIN OM..DEVANAGARI LETTER AY +A8FF;N # Mn DEVANAGARI VOWEL SIGN AY +A900..A909;N # Nd [10] KAYAH LI DIGIT ZERO..KAYAH LI DIGIT NINE +A90A..A925;N # Lo [28] KAYAH LI LETTER KA..KAYAH LI LETTER OO +A926..A92D;N # Mn [8] KAYAH LI VOWEL UE..KAYAH LI TONE CALYA PLOPHU +A92E..A92F;N # Po [2] KAYAH LI SIGN CWI..KAYAH LI SIGN SHYA +A930..A946;N # Lo [23] REJANG LETTER KA..REJANG LETTER A +A947..A951;N # Mn [11] REJANG VOWEL SIGN I..REJANG CONSONANT SIGN R +A952..A953;N # Mc [2] REJANG CONSONANT SIGN H..REJANG VIRAMA +A95F;N # Po REJANG SECTION MARK +A960..A97C;W # Lo [29] HANGUL CHOSEONG TIKEUT-MIEUM..HANGUL CHOSEONG SSANGYEORINHIEUH +A980..A982;N # Mn [3] JAVANESE SIGN PANYANGGA..JAVANESE SIGN LAYAR +A983;N # Mc JAVANESE SIGN WIGNYAN +A984..A9B2;N # Lo [47] JAVANESE LETTER A..JAVANESE LETTER HA +A9B3;N # Mn JAVANESE SIGN CECAK TELU +A9B4..A9B5;N # Mc [2] JAVANESE VOWEL SIGN TARUNG..JAVANESE VOWEL SIGN TOLONG +A9B6..A9B9;N # Mn [4] JAVANESE VOWEL SIGN WULU..JAVANESE VOWEL SIGN SUKU MENDUT +A9BA..A9BB;N # Mc [2] JAVANESE VOWEL SIGN TALING..JAVANESE VOWEL SIGN DIRGA MURE +A9BC..A9BD;N # Mn [2] JAVANESE VOWEL SIGN PEPET..JAVANESE CONSONANT SIGN KERET +A9BE..A9C0;N # Mc [3] JAVANESE CONSONANT SIGN PENGKAL..JAVANESE PANGKON +A9C1..A9CD;N # Po [13] JAVANESE LEFT RERENGGAN..JAVANESE TURNED PADA PISELEH +A9CF;N # Lm JAVANESE PANGRANGKEP +A9D0..A9D9;N # Nd [10] JAVANESE DIGIT ZERO..JAVANESE DIGIT NINE +A9DE..A9DF;N # Po [2] JAVANESE PADA TIRTA TUMETES..JAVANESE PADA ISEN-ISEN +A9E0..A9E4;N # Lo [5] MYANMAR LETTER SHAN GHA..MYANMAR LETTER SHAN BHA +A9E5;N # Mn MYANMAR SIGN SHAN SAW +A9E6;N # Lm MYANMAR MODIFIER LETTER SHAN REDUPLICATION +A9E7..A9EF;N # Lo [9] MYANMAR LETTER TAI LAING NYA..MYANMAR LETTER TAI LAING NNA +A9F0..A9F9;N # Nd [10] MYANMAR TAI LAING DIGIT ZERO..MYANMAR TAI LAING DIGIT NINE +A9FA..A9FE;N # Lo [5] MYANMAR LETTER TAI LAING LLA..MYANMAR LETTER TAI LAING BHA +AA00..AA28;N # Lo [41] CHAM LETTER A..CHAM LETTER HA +AA29..AA2E;N # Mn [6] CHAM VOWEL SIGN AA..CHAM VOWEL SIGN OE +AA2F..AA30;N # Mc [2] CHAM VOWEL SIGN O..CHAM VOWEL SIGN AI +AA31..AA32;N # Mn [2] CHAM VOWEL SIGN AU..CHAM VOWEL SIGN UE +AA33..AA34;N # Mc [2] CHAM CONSONANT SIGN YA..CHAM CONSONANT SIGN RA +AA35..AA36;N # Mn [2] CHAM CONSONANT SIGN LA..CHAM CONSONANT SIGN WA +AA40..AA42;N # Lo [3] CHAM LETTER FINAL K..CHAM LETTER FINAL NG +AA43;N # Mn CHAM CONSONANT SIGN FINAL NG +AA44..AA4B;N # Lo [8] CHAM LETTER FINAL CH..CHAM LETTER FINAL SS +AA4C;N # Mn CHAM CONSONANT SIGN FINAL M +AA4D;N # Mc CHAM CONSONANT SIGN FINAL H +AA50..AA59;N # Nd [10] CHAM DIGIT ZERO..CHAM DIGIT NINE +AA5C..AA5F;N # Po [4] CHAM PUNCTUATION SPIRAL..CHAM PUNCTUATION TRIPLE DANDA +AA60..AA6F;N # Lo [16] MYANMAR LETTER KHAMTI GA..MYANMAR LETTER KHAMTI FA +AA70;N # Lm MYANMAR MODIFIER LETTER KHAMTI REDUPLICATION +AA71..AA76;N # Lo [6] MYANMAR LETTER KHAMTI XA..MYANMAR LOGOGRAM KHAMTI HM +AA77..AA79;N # So [3] MYANMAR SYMBOL AITON EXCLAMATION..MYANMAR SYMBOL AITON TWO +AA7A;N # Lo MYANMAR LETTER AITON RA +AA7B;N # Mc MYANMAR SIGN PAO KAREN TONE +AA7C;N # Mn MYANMAR SIGN TAI LAING TONE-2 +AA7D;N # Mc MYANMAR SIGN TAI LAING TONE-5 +AA7E..AA7F;N # Lo [2] MYANMAR LETTER SHWE PALAUNG CHA..MYANMAR LETTER SHWE PALAUNG SHA +AA80..AAAF;N # Lo [48] TAI VIET LETTER LOW KO..TAI VIET LETTER HIGH O +AAB0;N # Mn TAI VIET MAI KANG +AAB1;N # Lo TAI VIET VOWEL AA +AAB2..AAB4;N # Mn [3] TAI VIET VOWEL I..TAI VIET VOWEL U +AAB5..AAB6;N # Lo [2] TAI VIET VOWEL E..TAI VIET VOWEL O +AAB7..AAB8;N # Mn [2] TAI VIET MAI KHIT..TAI VIET VOWEL IA +AAB9..AABD;N # Lo [5] TAI VIET VOWEL UEA..TAI VIET VOWEL AN +AABE..AABF;N # Mn [2] TAI VIET VOWEL AM..TAI VIET TONE MAI EK +AAC0;N # Lo TAI VIET TONE MAI NUENG +AAC1;N # Mn TAI VIET TONE MAI THO +AAC2;N # Lo TAI VIET TONE MAI SONG +AADB..AADC;N # Lo [2] TAI VIET SYMBOL KON..TAI VIET SYMBOL NUENG +AADD;N # Lm TAI VIET SYMBOL SAM +AADE..AADF;N # Po [2] TAI VIET SYMBOL HO HOI..TAI VIET SYMBOL KOI KOI +AAE0..AAEA;N # Lo [11] MEETEI MAYEK LETTER E..MEETEI MAYEK LETTER SSA +AAEB;N # Mc MEETEI MAYEK VOWEL SIGN II +AAEC..AAED;N # Mn [2] MEETEI MAYEK VOWEL SIGN UU..MEETEI MAYEK VOWEL SIGN AAI +AAEE..AAEF;N # Mc [2] MEETEI MAYEK VOWEL SIGN AU..MEETEI MAYEK VOWEL SIGN AAU +AAF0..AAF1;N # Po [2] MEETEI MAYEK CHEIKHAN..MEETEI MAYEK AHANG KHUDAM +AAF2;N # Lo MEETEI MAYEK ANJI +AAF3..AAF4;N # Lm [2] MEETEI MAYEK SYLLABLE REPETITION MARK..MEETEI MAYEK WORD REPETITION MARK +AAF5;N # Mc MEETEI MAYEK VOWEL SIGN VISARGA +AAF6;N # Mn MEETEI MAYEK VIRAMA +AB01..AB06;N # Lo [6] ETHIOPIC SYLLABLE TTHU..ETHIOPIC SYLLABLE TTHO +AB09..AB0E;N # Lo [6] ETHIOPIC SYLLABLE DDHU..ETHIOPIC SYLLABLE DDHO +AB11..AB16;N # Lo [6] ETHIOPIC SYLLABLE DZU..ETHIOPIC SYLLABLE DZO +AB20..AB26;N # Lo [7] ETHIOPIC SYLLABLE CCHHA..ETHIOPIC SYLLABLE CCHHO +AB28..AB2E;N # Lo [7] ETHIOPIC SYLLABLE BBA..ETHIOPIC SYLLABLE BBO +AB30..AB5A;N # Ll [43] LATIN SMALL LETTER BARRED ALPHA..LATIN SMALL LETTER Y WITH SHORT RIGHT LEG +AB5B;N # Sk MODIFIER BREVE WITH INVERTED BREVE +AB5C..AB5F;N # Lm [4] MODIFIER LETTER SMALL HENG..MODIFIER LETTER SMALL U WITH LEFT HOOK +AB60..AB68;N # Ll [9] LATIN SMALL LETTER SAKHA YAT..LATIN SMALL LETTER TURNED R WITH MIDDLE TILDE +AB69;N # Lm MODIFIER LETTER SMALL TURNED W +AB6A..AB6B;N # Sk [2] MODIFIER LETTER LEFT TACK..MODIFIER LETTER RIGHT TACK +AB70..ABBF;N # Ll [80] CHEROKEE SMALL LETTER A..CHEROKEE SMALL LETTER YA +ABC0..ABE2;N # Lo [35] MEETEI MAYEK LETTER KOK..MEETEI MAYEK LETTER I LONSUM +ABE3..ABE4;N # Mc [2] MEETEI MAYEK VOWEL SIGN ONAP..MEETEI MAYEK VOWEL SIGN INAP +ABE5;N # Mn MEETEI MAYEK VOWEL SIGN ANAP +ABE6..ABE7;N # Mc [2] MEETEI MAYEK VOWEL SIGN YENAP..MEETEI MAYEK VOWEL SIGN SOUNAP +ABE8;N # Mn MEETEI MAYEK VOWEL SIGN UNAP +ABE9..ABEA;N # Mc [2] MEETEI MAYEK VOWEL SIGN CHEINAP..MEETEI MAYEK VOWEL SIGN NUNG +ABEB;N # Po MEETEI MAYEK CHEIKHEI +ABEC;N # Mc MEETEI MAYEK LUM IYEK +ABED;N # Mn MEETEI MAYEK APUN IYEK +ABF0..ABF9;N # Nd [10] MEETEI MAYEK DIGIT ZERO..MEETEI MAYEK DIGIT NINE +AC00..D7A3;W # Lo [11172] HANGUL SYLLABLE GA..HANGUL SYLLABLE HIH +D7B0..D7C6;N # Lo [23] HANGUL JUNGSEONG O-YEO..HANGUL JUNGSEONG ARAEA-E +D7CB..D7FB;N # Lo [49] HANGUL JONGSEONG NIEUN-RIEUL..HANGUL JONGSEONG PHIEUPH-THIEUTH +D800..DB7F;N # Cs [896] <surrogate-D800>..<surrogate-DB7F> +DB80..DBFF;N # Cs [128] <surrogate-DB80>..<surrogate-DBFF> +DC00..DFFF;N # Cs [1024] <surrogate-DC00>..<surrogate-DFFF> +E000..F8FF;A # Co [6400] <private-use-E000>..<private-use-F8FF> +F900..FA6D;W # Lo [366] CJK COMPATIBILITY IDEOGRAPH-F900..CJK COMPATIBILITY IDEOGRAPH-FA6D +FA6E..FA6F;W # Cn [2] <reserved-FA6E>..<reserved-FA6F> +FA70..FAD9;W # Lo [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COMPATIBILITY IDEOGRAPH-FAD9 +FADA..FAFF;W # Cn [38] <reserved-FADA>..<reserved-FAFF> +FB00..FB06;N # Ll [7] LATIN SMALL LIGATURE FF..LATIN SMALL LIGATURE ST +FB13..FB17;N # Ll [5] ARMENIAN SMALL LIGATURE MEN NOW..ARMENIAN SMALL LIGATURE MEN XEH +FB1D;N # Lo HEBREW LETTER YOD WITH HIRIQ +FB1E;N # Mn HEBREW POINT JUDEO-SPANISH VARIKA +FB1F..FB28;N # Lo [10] HEBREW LIGATURE YIDDISH YOD YOD PATAH..HEBREW LETTER WIDE TAV +FB29;N # Sm HEBREW LETTER ALTERNATIVE PLUS SIGN +FB2A..FB36;N # Lo [13] HEBREW LETTER SHIN WITH SHIN DOT..HEBREW LETTER ZAYIN WITH DAGESH +FB38..FB3C;N # Lo [5] HEBREW LETTER TET WITH DAGESH..HEBREW LETTER LAMED WITH DAGESH +FB3E;N # Lo HEBREW LETTER MEM WITH DAGESH +FB40..FB41;N # Lo [2] HEBREW LETTER NUN WITH DAGESH..HEBREW LETTER SAMEKH WITH DAGESH +FB43..FB44;N # Lo [2] HEBREW LETTER FINAL PE WITH DAGESH..HEBREW LETTER PE WITH DAGESH +FB46..FB4F;N # Lo [10] HEBREW LETTER TSADI WITH DAGESH..HEBREW LIGATURE ALEF LAMED +FB50..FBB1;N # Lo [98] ARABIC LETTER ALEF WASLA ISOLATED FORM..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE FINAL FORM +FBB2..FBC2;N # Sk [17] ARABIC SYMBOL DOT ABOVE..ARABIC SYMBOL WASLA ABOVE +FBD3..FD3D;N # Lo [363] ARABIC LETTER NG ISOLATED FORM..ARABIC LIGATURE ALEF WITH FATHATAN ISOLATED FORM +FD3E;N # Pe ORNATE LEFT PARENTHESIS +FD3F;N # Ps ORNATE RIGHT PARENTHESIS +FD40..FD4F;N # So [16] ARABIC LIGATURE RAHIMAHU ALLAAH..ARABIC LIGATURE RAHIMAHUM ALLAAH +FD50..FD8F;N # Lo [64] ARABIC LIGATURE TEH WITH JEEM WITH MEEM INITIAL FORM..ARABIC LIGATURE MEEM WITH KHAH WITH MEEM INITIAL FORM +FD92..FDC7;N # Lo [54] ARABIC LIGATURE MEEM WITH JEEM WITH KHAH INITIAL FORM..ARABIC LIGATURE NOON WITH JEEM WITH YEH FINAL FORM +FDCF;N # So ARABIC LIGATURE SALAAMUHU ALAYNAA +FDF0..FDFB;N # Lo [12] ARABIC LIGATURE SALLA USED AS KORANIC STOP SIGN ISOLATED FORM..ARABIC LIGATURE JALLAJALALOUHOU +FDFC;N # Sc RIAL SIGN +FDFD..FDFF;N # So [3] ARABIC LIGATURE BISMILLAH AR-RAHMAN AR-RAHEEM..ARABIC LIGATURE AZZA WA JALL +FE00..FE0F;A # Mn [16] VARIATION SELECTOR-1..VARIATION SELECTOR-16 +FE10..FE16;W # Po [7] PRESENTATION FORM FOR VERTICAL COMMA..PRESENTATION FORM FOR VERTICAL QUESTION MARK +FE17;W # Ps PRESENTATION FORM FOR VERTICAL LEFT WHITE LENTICULAR BRACKET +FE18;W # Pe PRESENTATION FORM FOR VERTICAL RIGHT WHITE LENTICULAR BRAKCET +FE19;W # Po PRESENTATION FORM FOR VERTICAL HORIZONTAL ELLIPSIS +FE20..FE2F;N # Mn [16] COMBINING LIGATURE LEFT HALF..COMBINING CYRILLIC TITLO RIGHT HALF +FE30;W # Po PRESENTATION FORM FOR VERTICAL TWO DOT LEADER +FE31..FE32;W # Pd [2] PRESENTATION FORM FOR VERTICAL EM DASH..PRESENTATION FORM FOR VERTICAL EN DASH +FE33..FE34;W # Pc [2] PRESENTATION FORM FOR VERTICAL LOW LINE..PRESENTATION FORM FOR VERTICAL WAVY LOW LINE +FE35;W # Ps PRESENTATION FORM FOR VERTICAL LEFT PARENTHESIS +FE36;W # Pe PRESENTATION FORM FOR VERTICAL RIGHT PARENTHESIS +FE37;W # Ps PRESENTATION FORM FOR VERTICAL LEFT CURLY BRACKET +FE38;W # Pe PRESENTATION FORM FOR VERTICAL RIGHT CURLY BRACKET +FE39;W # Ps PRESENTATION FORM FOR VERTICAL LEFT TORTOISE SHELL BRACKET +FE3A;W # Pe PRESENTATION FORM FOR VERTICAL RIGHT TORTOISE SHELL BRACKET +FE3B;W # Ps PRESENTATION FORM FOR VERTICAL LEFT BLACK LENTICULAR BRACKET +FE3C;W # Pe PRESENTATION FORM FOR VERTICAL RIGHT BLACK LENTICULAR BRACKET +FE3D;W # Ps PRESENTATION FORM FOR VERTICAL LEFT DOUBLE ANGLE BRACKET +FE3E;W # Pe PRESENTATION FORM FOR VERTICAL RIGHT DOUBLE ANGLE BRACKET +FE3F;W # Ps PRESENTATION FORM FOR VERTICAL LEFT ANGLE BRACKET +FE40;W # Pe PRESENTATION FORM FOR VERTICAL RIGHT ANGLE BRACKET +FE41;W # Ps PRESENTATION FORM FOR VERTICAL LEFT CORNER BRACKET +FE42;W # Pe PRESENTATION FORM FOR VERTICAL RIGHT CORNER BRACKET +FE43;W # Ps PRESENTATION FORM FOR VERTICAL LEFT WHITE CORNER BRACKET +FE44;W # Pe PRESENTATION FORM FOR VERTICAL RIGHT WHITE CORNER BRACKET +FE45..FE46;W # Po [2] SESAME DOT..WHITE SESAME DOT +FE47;W # Ps PRESENTATION FORM FOR VERTICAL LEFT SQUARE BRACKET +FE48;W # Pe PRESENTATION FORM FOR VERTICAL RIGHT SQUARE BRACKET +FE49..FE4C;W # Po [4] DASHED OVERLINE..DOUBLE WAVY OVERLINE +FE4D..FE4F;W # Pc [3] DASHED LOW LINE..WAVY LOW LINE +FE50..FE52;W # Po [3] SMALL COMMA..SMALL FULL STOP +FE54..FE57;W # Po [4] SMALL SEMICOLON..SMALL EXCLAMATION MARK +FE58;W # Pd SMALL EM DASH +FE59;W # Ps SMALL LEFT PARENTHESIS +FE5A;W # Pe SMALL RIGHT PARENTHESIS +FE5B;W # Ps SMALL LEFT CURLY BRACKET +FE5C;W # Pe SMALL RIGHT CURLY BRACKET +FE5D;W # Ps SMALL LEFT TORTOISE SHELL BRACKET +FE5E;W # Pe SMALL RIGHT TORTOISE SHELL BRACKET +FE5F..FE61;W # Po [3] SMALL NUMBER SIGN..SMALL ASTERISK +FE62;W # Sm SMALL PLUS SIGN +FE63;W # Pd SMALL HYPHEN-MINUS +FE64..FE66;W # Sm [3] SMALL LESS-THAN SIGN..SMALL EQUALS SIGN +FE68;W # Po SMALL REVERSE SOLIDUS +FE69;W # Sc SMALL DOLLAR SIGN +FE6A..FE6B;W # Po [2] SMALL PERCENT SIGN..SMALL COMMERCIAL AT +FE70..FE74;N # Lo [5] ARABIC FATHATAN ISOLATED FORM..ARABIC KASRATAN ISOLATED FORM +FE76..FEFC;N # Lo [135] ARABIC FATHA ISOLATED FORM..ARABIC LIGATURE LAM WITH ALEF FINAL FORM +FEFF;N # Cf ZERO WIDTH NO-BREAK SPACE +FF01..FF03;F # Po [3] FULLWIDTH EXCLAMATION MARK..FULLWIDTH NUMBER SIGN +FF04;F # Sc FULLWIDTH DOLLAR SIGN +FF05..FF07;F # Po [3] FULLWIDTH PERCENT SIGN..FULLWIDTH APOSTROPHE +FF08;F # Ps FULLWIDTH LEFT PARENTHESIS +FF09;F # Pe FULLWIDTH RIGHT PARENTHESIS +FF0A;F # Po FULLWIDTH ASTERISK +FF0B;F # Sm FULLWIDTH PLUS SIGN +FF0C;F # Po FULLWIDTH COMMA +FF0D;F # Pd FULLWIDTH HYPHEN-MINUS +FF0E..FF0F;F # Po [2] FULLWIDTH FULL STOP..FULLWIDTH SOLIDUS +FF10..FF19;F # Nd [10] FULLWIDTH DIGIT ZERO..FULLWIDTH DIGIT NINE +FF1A..FF1B;F # Po [2] FULLWIDTH COLON..FULLWIDTH SEMICOLON +FF1C..FF1E;F # Sm [3] FULLWIDTH LESS-THAN SIGN..FULLWIDTH GREATER-THAN SIGN +FF1F..FF20;F # Po [2] FULLWIDTH QUESTION MARK..FULLWIDTH COMMERCIAL AT +FF21..FF3A;F # Lu [26] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH LATIN CAPITAL LETTER Z +FF3B;F # Ps FULLWIDTH LEFT SQUARE BRACKET +FF3C;F # Po FULLWIDTH REVERSE SOLIDUS +FF3D;F # Pe FULLWIDTH RIGHT SQUARE BRACKET +FF3E;F # Sk FULLWIDTH CIRCUMFLEX ACCENT +FF3F;F # Pc FULLWIDTH LOW LINE +FF40;F # Sk FULLWIDTH GRAVE ACCENT +FF41..FF5A;F # Ll [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN SMALL LETTER Z +FF5B;F # Ps FULLWIDTH LEFT CURLY BRACKET +FF5C;F # Sm FULLWIDTH VERTICAL LINE +FF5D;F # Pe FULLWIDTH RIGHT CURLY BRACKET +FF5E;F # Sm FULLWIDTH TILDE +FF5F;F # Ps FULLWIDTH LEFT WHITE PARENTHESIS +FF60;F # Pe FULLWIDTH RIGHT WHITE PARENTHESIS +FF61;H # Po HALFWIDTH IDEOGRAPHIC FULL STOP +FF62;H # Ps HALFWIDTH LEFT CORNER BRACKET +FF63;H # Pe HALFWIDTH RIGHT CORNER BRACKET +FF64..FF65;H # Po [2] HALFWIDTH IDEOGRAPHIC COMMA..HALFWIDTH KATAKANA MIDDLE DOT +FF66..FF6F;H # Lo [10] HALFWIDTH KATAKANA LETTER WO..HALFWIDTH KATAKANA LETTER SMALL TU +FF70;H # Lm HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK +FF71..FF9D;H # Lo [45] HALFWIDTH KATAKANA LETTER A..HALFWIDTH KATAKANA LETTER N +FF9E..FF9F;H # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK +FFA0..FFBE;H # Lo [31] HALFWIDTH HANGUL FILLER..HALFWIDTH HANGUL LETTER HIEUH +FFC2..FFC7;H # Lo [6] HALFWIDTH HANGUL LETTER A..HALFWIDTH HANGUL LETTER E +FFCA..FFCF;H # Lo [6] HALFWIDTH HANGUL LETTER YEO..HALFWIDTH HANGUL LETTER OE +FFD2..FFD7;H # Lo [6] HALFWIDTH HANGUL LETTER YO..HALFWIDTH HANGUL LETTER YU +FFDA..FFDC;H # Lo [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL LETTER I +FFE0..FFE1;F # Sc [2] FULLWIDTH CENT SIGN..FULLWIDTH POUND SIGN +FFE2;F # Sm FULLWIDTH NOT SIGN +FFE3;F # Sk FULLWIDTH MACRON +FFE4;F # So FULLWIDTH BROKEN BAR +FFE5..FFE6;F # Sc [2] FULLWIDTH YEN SIGN..FULLWIDTH WON SIGN +FFE8;H # So HALFWIDTH FORMS LIGHT VERTICAL +FFE9..FFEC;H # Sm [4] HALFWIDTH LEFTWARDS ARROW..HALFWIDTH DOWNWARDS ARROW +FFED..FFEE;H # So [2] HALFWIDTH BLACK SQUARE..HALFWIDTH WHITE CIRCLE +FFF9..FFFB;N # Cf [3] INTERLINEAR ANNOTATION ANCHOR..INTERLINEAR ANNOTATION TERMINATOR +FFFC;N # So OBJECT REPLACEMENT CHARACTER +FFFD;A # So REPLACEMENT CHARACTER +10000..1000B;N # Lo [12] LINEAR B SYLLABLE B008 A..LINEAR B SYLLABLE B046 JE +1000D..10026;N # Lo [26] LINEAR B SYLLABLE B036 JO..LINEAR B SYLLABLE B032 QO +10028..1003A;N # Lo [19] LINEAR B SYLLABLE B060 RA..LINEAR B SYLLABLE B042 WO +1003C..1003D;N # Lo [2] LINEAR B SYLLABLE B017 ZA..LINEAR B SYLLABLE B074 ZE +1003F..1004D;N # Lo [15] LINEAR B SYLLABLE B020 ZO..LINEAR B SYLLABLE B091 TWO +10050..1005D;N # Lo [14] LINEAR B SYMBOL B018..LINEAR B SYMBOL B089 +10080..100FA;N # Lo [123] LINEAR B IDEOGRAM B100 MAN..LINEAR B IDEOGRAM VESSEL B305 +10100..10102;N # Po [3] AEGEAN WORD SEPARATOR LINE..AEGEAN CHECK MARK +10107..10133;N # No [45] AEGEAN NUMBER ONE..AEGEAN NUMBER NINETY THOUSAND +10137..1013F;N # So [9] AEGEAN WEIGHT BASE UNIT..AEGEAN MEASURE THIRD SUBUNIT +10140..10174;N # Nl [53] GREEK ACROPHONIC ATTIC ONE QUARTER..GREEK ACROPHONIC STRATIAN FIFTY MNAS +10175..10178;N # No [4] GREEK ONE HALF SIGN..GREEK THREE QUARTERS SIGN +10179..10189;N # So [17] GREEK YEAR SIGN..GREEK TRYBLION BASE SIGN +1018A..1018B;N # No [2] GREEK ZERO SIGN..GREEK ONE QUARTER SIGN +1018C..1018E;N # So [3] GREEK SINUSOID SIGN..NOMISMA SIGN +10190..1019C;N # So [13] ROMAN SEXTANS SIGN..ASCIA SYMBOL +101A0;N # So GREEK SYMBOL TAU RHO +101D0..101FC;N # So [45] PHAISTOS DISC SIGN PEDESTRIAN..PHAISTOS DISC SIGN WAVY BAND +101FD;N # Mn PHAISTOS DISC SIGN COMBINING OBLIQUE STROKE +10280..1029C;N # Lo [29] LYCIAN LETTER A..LYCIAN LETTER X +102A0..102D0;N # Lo [49] CARIAN LETTER A..CARIAN LETTER UUU3 +102E0;N # Mn COPTIC EPACT THOUSANDS MARK +102E1..102FB;N # No [27] COPTIC EPACT DIGIT ONE..COPTIC EPACT NUMBER NINE HUNDRED +10300..1031F;N # Lo [32] OLD ITALIC LETTER A..OLD ITALIC LETTER ESS +10320..10323;N # No [4] OLD ITALIC NUMERAL ONE..OLD ITALIC NUMERAL FIFTY +1032D..1032F;N # Lo [3] OLD ITALIC LETTER YE..OLD ITALIC LETTER SOUTHERN TSE +10330..10340;N # Lo [17] GOTHIC LETTER AHSA..GOTHIC LETTER PAIRTHRA +10341;N # Nl GOTHIC LETTER NINETY +10342..10349;N # Lo [8] GOTHIC LETTER RAIDA..GOTHIC LETTER OTHAL +1034A;N # Nl GOTHIC LETTER NINE HUNDRED +10350..10375;N # Lo [38] OLD PERMIC LETTER AN..OLD PERMIC LETTER IA +10376..1037A;N # Mn [5] COMBINING OLD PERMIC LETTER AN..COMBINING OLD PERMIC LETTER SII +10380..1039D;N # Lo [30] UGARITIC LETTER ALPA..UGARITIC LETTER SSU +1039F;N # Po UGARITIC WORD DIVIDER +103A0..103C3;N # Lo [36] OLD PERSIAN SIGN A..OLD PERSIAN SIGN HA +103C8..103CF;N # Lo [8] OLD PERSIAN SIGN AURAMAZDAA..OLD PERSIAN SIGN BUUMISH +103D0;N # Po OLD PERSIAN WORD DIVIDER +103D1..103D5;N # Nl [5] OLD PERSIAN NUMBER ONE..OLD PERSIAN NUMBER HUNDRED +10400..1044F;N # L& [80] DESERET CAPITAL LETTER LONG I..DESERET SMALL LETTER EW +10450..1047F;N # Lo [48] SHAVIAN LETTER PEEP..SHAVIAN LETTER YEW +10480..1049D;N # Lo [30] OSMANYA LETTER ALEF..OSMANYA LETTER OO +104A0..104A9;N # Nd [10] OSMANYA DIGIT ZERO..OSMANYA DIGIT NINE +104B0..104D3;N # Lu [36] OSAGE CAPITAL LETTER A..OSAGE CAPITAL LETTER ZHA +104D8..104FB;N # Ll [36] OSAGE SMALL LETTER A..OSAGE SMALL LETTER ZHA +10500..10527;N # Lo [40] ELBASAN LETTER A..ELBASAN LETTER KHE +10530..10563;N # Lo [52] CAUCASIAN ALBANIAN LETTER ALT..CAUCASIAN ALBANIAN LETTER KIW +1056F;N # Po CAUCASIAN ALBANIAN CITATION MARK +10570..1057A;N # Lu [11] VITHKUQI CAPITAL LETTER A..VITHKUQI CAPITAL LETTER GA +1057C..1058A;N # Lu [15] VITHKUQI CAPITAL LETTER HA..VITHKUQI CAPITAL LETTER RE +1058C..10592;N # Lu [7] VITHKUQI CAPITAL LETTER SE..VITHKUQI CAPITAL LETTER XE +10594..10595;N # Lu [2] VITHKUQI CAPITAL LETTER Y..VITHKUQI CAPITAL LETTER ZE +10597..105A1;N # Ll [11] VITHKUQI SMALL LETTER A..VITHKUQI SMALL LETTER GA +105A3..105B1;N # Ll [15] VITHKUQI SMALL LETTER HA..VITHKUQI SMALL LETTER RE +105B3..105B9;N # Ll [7] VITHKUQI SMALL LETTER SE..VITHKUQI SMALL LETTER XE +105BB..105BC;N # Ll [2] VITHKUQI SMALL LETTER Y..VITHKUQI SMALL LETTER ZE +10600..10736;N # Lo [311] LINEAR A SIGN AB001..LINEAR A SIGN A664 +10740..10755;N # Lo [22] LINEAR A SIGN A701 A..LINEAR A SIGN A732 JE +10760..10767;N # Lo [8] LINEAR A SIGN A800..LINEAR A SIGN A807 +10780..10785;N # Lm [6] MODIFIER LETTER SMALL CAPITAL AA..MODIFIER LETTER SMALL B WITH HOOK +10787..107B0;N # Lm [42] MODIFIER LETTER SMALL DZ DIGRAPH..MODIFIER LETTER SMALL V WITH RIGHT HOOK +107B2..107BA;N # Lm [9] MODIFIER LETTER SMALL CAPITAL Y..MODIFIER LETTER SMALL S WITH CURL +10800..10805;N # Lo [6] CYPRIOT SYLLABLE A..CYPRIOT SYLLABLE JA +10808;N # Lo CYPRIOT SYLLABLE JO +1080A..10835;N # Lo [44] CYPRIOT SYLLABLE KA..CYPRIOT SYLLABLE WO +10837..10838;N # Lo [2] CYPRIOT SYLLABLE XA..CYPRIOT SYLLABLE XE +1083C;N # Lo CYPRIOT SYLLABLE ZA +1083F;N # Lo CYPRIOT SYLLABLE ZO +10840..10855;N # Lo [22] IMPERIAL ARAMAIC LETTER ALEPH..IMPERIAL ARAMAIC LETTER TAW +10857;N # Po IMPERIAL ARAMAIC SECTION SIGN +10858..1085F;N # No [8] IMPERIAL ARAMAIC NUMBER ONE..IMPERIAL ARAMAIC NUMBER TEN THOUSAND +10860..10876;N # Lo [23] PALMYRENE LETTER ALEPH..PALMYRENE LETTER TAW +10877..10878;N # So [2] PALMYRENE LEFT-POINTING FLEURON..PALMYRENE RIGHT-POINTING FLEURON +10879..1087F;N # No [7] PALMYRENE NUMBER ONE..PALMYRENE NUMBER TWENTY +10880..1089E;N # Lo [31] NABATAEAN LETTER FINAL ALEPH..NABATAEAN LETTER TAW +108A7..108AF;N # No [9] NABATAEAN NUMBER ONE..NABATAEAN NUMBER ONE HUNDRED +108E0..108F2;N # Lo [19] HATRAN LETTER ALEPH..HATRAN LETTER QOPH +108F4..108F5;N # Lo [2] HATRAN LETTER SHIN..HATRAN LETTER TAW +108FB..108FF;N # No [5] HATRAN NUMBER ONE..HATRAN NUMBER ONE HUNDRED +10900..10915;N # Lo [22] PHOENICIAN LETTER ALF..PHOENICIAN LETTER TAU +10916..1091B;N # No [6] PHOENICIAN NUMBER ONE..PHOENICIAN NUMBER THREE +1091F;N # Po PHOENICIAN WORD SEPARATOR +10920..10939;N # Lo [26] LYDIAN LETTER A..LYDIAN LETTER C +1093F;N # Po LYDIAN TRIANGULAR MARK +10980..1099F;N # Lo [32] MEROITIC HIEROGLYPHIC LETTER A..MEROITIC HIEROGLYPHIC SYMBOL VIDJ-2 +109A0..109B7;N # Lo [24] MEROITIC CURSIVE LETTER A..MEROITIC CURSIVE LETTER DA +109BC..109BD;N # No [2] MEROITIC CURSIVE FRACTION ELEVEN TWELFTHS..MEROITIC CURSIVE FRACTION ONE HALF +109BE..109BF;N # Lo [2] MEROITIC CURSIVE LOGOGRAM RMT..MEROITIC CURSIVE LOGOGRAM IMN +109C0..109CF;N # No [16] MEROITIC CURSIVE NUMBER ONE..MEROITIC CURSIVE NUMBER SEVENTY +109D2..109FF;N # No [46] MEROITIC CURSIVE NUMBER ONE HUNDRED..MEROITIC CURSIVE FRACTION TEN TWELFTHS +10A00;N # Lo KHAROSHTHI LETTER A +10A01..10A03;N # Mn [3] KHAROSHTHI VOWEL SIGN I..KHAROSHTHI VOWEL SIGN VOCALIC R +10A05..10A06;N # Mn [2] KHAROSHTHI VOWEL SIGN E..KHAROSHTHI VOWEL SIGN O +10A0C..10A0F;N # Mn [4] KHAROSHTHI VOWEL LENGTH MARK..KHAROSHTHI SIGN VISARGA +10A10..10A13;N # Lo [4] KHAROSHTHI LETTER KA..KHAROSHTHI LETTER GHA +10A15..10A17;N # Lo [3] KHAROSHTHI LETTER CA..KHAROSHTHI LETTER JA +10A19..10A35;N # Lo [29] KHAROSHTHI LETTER NYA..KHAROSHTHI LETTER VHA +10A38..10A3A;N # Mn [3] KHAROSHTHI SIGN BAR ABOVE..KHAROSHTHI SIGN DOT BELOW +10A3F;N # Mn KHAROSHTHI VIRAMA +10A40..10A48;N # No [9] KHAROSHTHI DIGIT ONE..KHAROSHTHI FRACTION ONE HALF +10A50..10A58;N # Po [9] KHAROSHTHI PUNCTUATION DOT..KHAROSHTHI PUNCTUATION LINES +10A60..10A7C;N # Lo [29] OLD SOUTH ARABIAN LETTER HE..OLD SOUTH ARABIAN LETTER THETH +10A7D..10A7E;N # No [2] OLD SOUTH ARABIAN NUMBER ONE..OLD SOUTH ARABIAN NUMBER FIFTY +10A7F;N # Po OLD SOUTH ARABIAN NUMERIC INDICATOR +10A80..10A9C;N # Lo [29] OLD NORTH ARABIAN LETTER HEH..OLD NORTH ARABIAN LETTER ZAH +10A9D..10A9F;N # No [3] OLD NORTH ARABIAN NUMBER ONE..OLD NORTH ARABIAN NUMBER TWENTY +10AC0..10AC7;N # Lo [8] MANICHAEAN LETTER ALEPH..MANICHAEAN LETTER WAW +10AC8;N # So MANICHAEAN SIGN UD +10AC9..10AE4;N # Lo [28] MANICHAEAN LETTER ZAYIN..MANICHAEAN LETTER TAW +10AE5..10AE6;N # Mn [2] MANICHAEAN ABBREVIATION MARK ABOVE..MANICHAEAN ABBREVIATION MARK BELOW +10AEB..10AEF;N # No [5] MANICHAEAN NUMBER ONE..MANICHAEAN NUMBER ONE HUNDRED +10AF0..10AF6;N # Po [7] MANICHAEAN PUNCTUATION STAR..MANICHAEAN PUNCTUATION LINE FILLER +10B00..10B35;N # Lo [54] AVESTAN LETTER A..AVESTAN LETTER HE +10B39..10B3F;N # Po [7] AVESTAN ABBREVIATION MARK..LARGE ONE RING OVER TWO RINGS PUNCTUATION +10B40..10B55;N # Lo [22] INSCRIPTIONAL PARTHIAN LETTER ALEPH..INSCRIPTIONAL PARTHIAN LETTER TAW +10B58..10B5F;N # No [8] INSCRIPTIONAL PARTHIAN NUMBER ONE..INSCRIPTIONAL PARTHIAN NUMBER ONE THOUSAND +10B60..10B72;N # Lo [19] INSCRIPTIONAL PAHLAVI LETTER ALEPH..INSCRIPTIONAL PAHLAVI LETTER TAW +10B78..10B7F;N # No [8] INSCRIPTIONAL PAHLAVI NUMBER ONE..INSCRIPTIONAL PAHLAVI NUMBER ONE THOUSAND +10B80..10B91;N # Lo [18] PSALTER PAHLAVI LETTER ALEPH..PSALTER PAHLAVI LETTER TAW +10B99..10B9C;N # Po [4] PSALTER PAHLAVI SECTION MARK..PSALTER PAHLAVI FOUR DOTS WITH DOT +10BA9..10BAF;N # No [7] PSALTER PAHLAVI NUMBER ONE..PSALTER PAHLAVI NUMBER ONE HUNDRED +10C00..10C48;N # Lo [73] OLD TURKIC LETTER ORKHON A..OLD TURKIC LETTER ORKHON BASH +10C80..10CB2;N # Lu [51] OLD HUNGARIAN CAPITAL LETTER A..OLD HUNGARIAN CAPITAL LETTER US +10CC0..10CF2;N # Ll [51] OLD HUNGARIAN SMALL LETTER A..OLD HUNGARIAN SMALL LETTER US +10CFA..10CFF;N # No [6] OLD HUNGARIAN NUMBER ONE..OLD HUNGARIAN NUMBER ONE THOUSAND +10D00..10D23;N # Lo [36] HANIFI ROHINGYA LETTER A..HANIFI ROHINGYA MARK NA KHONNA +10D24..10D27;N # Mn [4] HANIFI ROHINGYA SIGN HARBAHAY..HANIFI ROHINGYA SIGN TASSI +10D30..10D39;N # Nd [10] HANIFI ROHINGYA DIGIT ZERO..HANIFI ROHINGYA DIGIT NINE +10E60..10E7E;N # No [31] RUMI DIGIT ONE..RUMI FRACTION TWO THIRDS +10E80..10EA9;N # Lo [42] YEZIDI LETTER ELIF..YEZIDI LETTER ET +10EAB..10EAC;N # Mn [2] YEZIDI COMBINING HAMZA MARK..YEZIDI COMBINING MADDA MARK +10EAD;N # Pd YEZIDI HYPHENATION MARK +10EB0..10EB1;N # Lo [2] YEZIDI LETTER LAM WITH DOT ABOVE..YEZIDI LETTER YOT WITH CIRCUMFLEX ABOVE +10EFD..10EFF;N # Mn [3] ARABIC SMALL LOW WORD SAKTA..ARABIC SMALL LOW WORD MADDA +10F00..10F1C;N # Lo [29] OLD SOGDIAN LETTER ALEPH..OLD SOGDIAN LETTER FINAL TAW WITH VERTICAL TAIL +10F1D..10F26;N # No [10] OLD SOGDIAN NUMBER ONE..OLD SOGDIAN FRACTION ONE HALF +10F27;N # Lo OLD SOGDIAN LIGATURE AYIN-DALETH +10F30..10F45;N # Lo [22] SOGDIAN LETTER ALEPH..SOGDIAN INDEPENDENT SHIN +10F46..10F50;N # Mn [11] SOGDIAN COMBINING DOT BELOW..SOGDIAN COMBINING STROKE BELOW +10F51..10F54;N # No [4] SOGDIAN NUMBER ONE..SOGDIAN NUMBER ONE HUNDRED +10F55..10F59;N # Po [5] SOGDIAN PUNCTUATION TWO VERTICAL BARS..SOGDIAN PUNCTUATION HALF CIRCLE WITH DOT +10F70..10F81;N # Lo [18] OLD UYGHUR LETTER ALEPH..OLD UYGHUR LETTER LESH +10F82..10F85;N # Mn [4] OLD UYGHUR COMBINING DOT ABOVE..OLD UYGHUR COMBINING TWO DOTS BELOW +10F86..10F89;N # Po [4] OLD UYGHUR PUNCTUATION BAR..OLD UYGHUR PUNCTUATION FOUR DOTS +10FB0..10FC4;N # Lo [21] CHORASMIAN LETTER ALEPH..CHORASMIAN LETTER TAW +10FC5..10FCB;N # No [7] CHORASMIAN NUMBER ONE..CHORASMIAN NUMBER ONE HUNDRED +10FE0..10FF6;N # Lo [23] ELYMAIC LETTER ALEPH..ELYMAIC LIGATURE ZAYIN-YODH +11000;N # Mc BRAHMI SIGN CANDRABINDU +11001;N # Mn BRAHMI SIGN ANUSVARA +11002;N # Mc BRAHMI SIGN VISARGA +11003..11037;N # Lo [53] BRAHMI SIGN JIHVAMULIYA..BRAHMI LETTER OLD TAMIL NNNA +11038..11046;N # Mn [15] BRAHMI VOWEL SIGN AA..BRAHMI VIRAMA +11047..1104D;N # Po [7] BRAHMI DANDA..BRAHMI PUNCTUATION LOTUS +11052..11065;N # No [20] BRAHMI NUMBER ONE..BRAHMI NUMBER ONE THOUSAND +11066..1106F;N # Nd [10] BRAHMI DIGIT ZERO..BRAHMI DIGIT NINE +11070;N # Mn BRAHMI SIGN OLD TAMIL VIRAMA +11071..11072;N # Lo [2] BRAHMI LETTER OLD TAMIL SHORT E..BRAHMI LETTER OLD TAMIL SHORT O +11073..11074;N # Mn [2] BRAHMI VOWEL SIGN OLD TAMIL SHORT E..BRAHMI VOWEL SIGN OLD TAMIL SHORT O +11075;N # Lo BRAHMI LETTER OLD TAMIL LLA +1107F;N # Mn BRAHMI NUMBER JOINER +11080..11081;N # Mn [2] KAITHI SIGN CANDRABINDU..KAITHI SIGN ANUSVARA +11082;N # Mc KAITHI SIGN VISARGA +11083..110AF;N # Lo [45] KAITHI LETTER A..KAITHI LETTER HA +110B0..110B2;N # Mc [3] KAITHI VOWEL SIGN AA..KAITHI VOWEL SIGN II +110B3..110B6;N # Mn [4] KAITHI VOWEL SIGN U..KAITHI VOWEL SIGN AI +110B7..110B8;N # Mc [2] KAITHI VOWEL SIGN O..KAITHI VOWEL SIGN AU +110B9..110BA;N # Mn [2] KAITHI SIGN VIRAMA..KAITHI SIGN NUKTA +110BB..110BC;N # Po [2] KAITHI ABBREVIATION SIGN..KAITHI ENUMERATION SIGN +110BD;N # Cf KAITHI NUMBER SIGN +110BE..110C1;N # Po [4] KAITHI SECTION MARK..KAITHI DOUBLE DANDA +110C2;N # Mn KAITHI VOWEL SIGN VOCALIC R +110CD;N # Cf KAITHI NUMBER SIGN ABOVE +110D0..110E8;N # Lo [25] SORA SOMPENG LETTER SAH..SORA SOMPENG LETTER MAE +110F0..110F9;N # Nd [10] SORA SOMPENG DIGIT ZERO..SORA SOMPENG DIGIT NINE +11100..11102;N # Mn [3] CHAKMA SIGN CANDRABINDU..CHAKMA SIGN VISARGA +11103..11126;N # Lo [36] CHAKMA LETTER AA..CHAKMA LETTER HAA +11127..1112B;N # Mn [5] CHAKMA VOWEL SIGN A..CHAKMA VOWEL SIGN UU +1112C;N # Mc CHAKMA VOWEL SIGN E +1112D..11134;N # Mn [8] CHAKMA VOWEL SIGN AI..CHAKMA MAAYYAA +11136..1113F;N # Nd [10] CHAKMA DIGIT ZERO..CHAKMA DIGIT NINE +11140..11143;N # Po [4] CHAKMA SECTION MARK..CHAKMA QUESTION MARK +11144;N # Lo CHAKMA LETTER LHAA +11145..11146;N # Mc [2] CHAKMA VOWEL SIGN AA..CHAKMA VOWEL SIGN EI +11147;N # Lo CHAKMA LETTER VAA +11150..11172;N # Lo [35] MAHAJANI LETTER A..MAHAJANI LETTER RRA +11173;N # Mn MAHAJANI SIGN NUKTA +11174..11175;N # Po [2] MAHAJANI ABBREVIATION SIGN..MAHAJANI SECTION MARK +11176;N # Lo MAHAJANI LIGATURE SHRI +11180..11181;N # Mn [2] SHARADA SIGN CANDRABINDU..SHARADA SIGN ANUSVARA +11182;N # Mc SHARADA SIGN VISARGA +11183..111B2;N # Lo [48] SHARADA LETTER A..SHARADA LETTER HA +111B3..111B5;N # Mc [3] SHARADA VOWEL SIGN AA..SHARADA VOWEL SIGN II +111B6..111BE;N # Mn [9] SHARADA VOWEL SIGN U..SHARADA VOWEL SIGN O +111BF..111C0;N # Mc [2] SHARADA VOWEL SIGN AU..SHARADA SIGN VIRAMA +111C1..111C4;N # Lo [4] SHARADA SIGN AVAGRAHA..SHARADA OM +111C5..111C8;N # Po [4] SHARADA DANDA..SHARADA SEPARATOR +111C9..111CC;N # Mn [4] SHARADA SANDHI MARK..SHARADA EXTRA SHORT VOWEL MARK +111CD;N # Po SHARADA SUTRA MARK +111CE;N # Mc SHARADA VOWEL SIGN PRISHTHAMATRA E +111CF;N # Mn SHARADA SIGN INVERTED CANDRABINDU +111D0..111D9;N # Nd [10] SHARADA DIGIT ZERO..SHARADA DIGIT NINE +111DA;N # Lo SHARADA EKAM +111DB;N # Po SHARADA SIGN SIDDHAM +111DC;N # Lo SHARADA HEADSTROKE +111DD..111DF;N # Po [3] SHARADA CONTINUATION SIGN..SHARADA SECTION MARK-2 +111E1..111F4;N # No [20] SINHALA ARCHAIC DIGIT ONE..SINHALA ARCHAIC NUMBER ONE THOUSAND +11200..11211;N # Lo [18] KHOJKI LETTER A..KHOJKI LETTER JJA +11213..1122B;N # Lo [25] KHOJKI LETTER NYA..KHOJKI LETTER LLA +1122C..1122E;N # Mc [3] KHOJKI VOWEL SIGN AA..KHOJKI VOWEL SIGN II +1122F..11231;N # Mn [3] KHOJKI VOWEL SIGN U..KHOJKI VOWEL SIGN AI +11232..11233;N # Mc [2] KHOJKI VOWEL SIGN O..KHOJKI VOWEL SIGN AU +11234;N # Mn KHOJKI SIGN ANUSVARA +11235;N # Mc KHOJKI SIGN VIRAMA +11236..11237;N # Mn [2] KHOJKI SIGN NUKTA..KHOJKI SIGN SHADDA +11238..1123D;N # Po [6] KHOJKI DANDA..KHOJKI ABBREVIATION SIGN +1123E;N # Mn KHOJKI SIGN SUKUN +1123F..11240;N # Lo [2] KHOJKI LETTER QA..KHOJKI LETTER SHORT I +11241;N # Mn KHOJKI VOWEL SIGN VOCALIC R +11280..11286;N # Lo [7] MULTANI LETTER A..MULTANI LETTER GA +11288;N # Lo MULTANI LETTER GHA +1128A..1128D;N # Lo [4] MULTANI LETTER CA..MULTANI LETTER JJA +1128F..1129D;N # Lo [15] MULTANI LETTER NYA..MULTANI LETTER BA +1129F..112A8;N # Lo [10] MULTANI LETTER BHA..MULTANI LETTER RHA +112A9;N # Po MULTANI SECTION MARK +112B0..112DE;N # Lo [47] KHUDAWADI LETTER A..KHUDAWADI LETTER HA +112DF;N # Mn KHUDAWADI SIGN ANUSVARA +112E0..112E2;N # Mc [3] KHUDAWADI VOWEL SIGN AA..KHUDAWADI VOWEL SIGN II +112E3..112EA;N # Mn [8] KHUDAWADI VOWEL SIGN U..KHUDAWADI SIGN VIRAMA +112F0..112F9;N # Nd [10] KHUDAWADI DIGIT ZERO..KHUDAWADI DIGIT NINE +11300..11301;N # Mn [2] GRANTHA SIGN COMBINING ANUSVARA ABOVE..GRANTHA SIGN CANDRABINDU +11302..11303;N # Mc [2] GRANTHA SIGN ANUSVARA..GRANTHA SIGN VISARGA +11305..1130C;N # Lo [8] GRANTHA LETTER A..GRANTHA LETTER VOCALIC L +1130F..11310;N # Lo [2] GRANTHA LETTER EE..GRANTHA LETTER AI +11313..11328;N # Lo [22] GRANTHA LETTER OO..GRANTHA LETTER NA +1132A..11330;N # Lo [7] GRANTHA LETTER PA..GRANTHA LETTER RA +11332..11333;N # Lo [2] GRANTHA LETTER LA..GRANTHA LETTER LLA +11335..11339;N # Lo [5] GRANTHA LETTER VA..GRANTHA LETTER HA +1133B..1133C;N # Mn [2] COMBINING BINDU BELOW..GRANTHA SIGN NUKTA +1133D;N # Lo GRANTHA SIGN AVAGRAHA +1133E..1133F;N # Mc [2] GRANTHA VOWEL SIGN AA..GRANTHA VOWEL SIGN I +11340;N # Mn GRANTHA VOWEL SIGN II +11341..11344;N # Mc [4] GRANTHA VOWEL SIGN U..GRANTHA VOWEL SIGN VOCALIC RR +11347..11348;N # Mc [2] GRANTHA VOWEL SIGN EE..GRANTHA VOWEL SIGN AI +1134B..1134D;N # Mc [3] GRANTHA VOWEL SIGN OO..GRANTHA SIGN VIRAMA +11350;N # Lo GRANTHA OM +11357;N # Mc GRANTHA AU LENGTH MARK +1135D..11361;N # Lo [5] GRANTHA SIGN PLUTA..GRANTHA LETTER VOCALIC LL +11362..11363;N # Mc [2] GRANTHA VOWEL SIGN VOCALIC L..GRANTHA VOWEL SIGN VOCALIC LL +11366..1136C;N # Mn [7] COMBINING GRANTHA DIGIT ZERO..COMBINING GRANTHA DIGIT SIX +11370..11374;N # Mn [5] COMBINING GRANTHA LETTER A..COMBINING GRANTHA LETTER PA +11400..11434;N # Lo [53] NEWA LETTER A..NEWA LETTER HA +11435..11437;N # Mc [3] NEWA VOWEL SIGN AA..NEWA VOWEL SIGN II +11438..1143F;N # Mn [8] NEWA VOWEL SIGN U..NEWA VOWEL SIGN AI +11440..11441;N # Mc [2] NEWA VOWEL SIGN O..NEWA VOWEL SIGN AU +11442..11444;N # Mn [3] NEWA SIGN VIRAMA..NEWA SIGN ANUSVARA +11445;N # Mc NEWA SIGN VISARGA +11446;N # Mn NEWA SIGN NUKTA +11447..1144A;N # Lo [4] NEWA SIGN AVAGRAHA..NEWA SIDDHI +1144B..1144F;N # Po [5] NEWA DANDA..NEWA ABBREVIATION SIGN +11450..11459;N # Nd [10] NEWA DIGIT ZERO..NEWA DIGIT NINE +1145A..1145B;N # Po [2] NEWA DOUBLE COMMA..NEWA PLACEHOLDER MARK +1145D;N # Po NEWA INSERTION SIGN +1145E;N # Mn NEWA SANDHI MARK +1145F..11461;N # Lo [3] NEWA LETTER VEDIC ANUSVARA..NEWA SIGN UPADHMANIYA +11480..114AF;N # Lo [48] TIRHUTA ANJI..TIRHUTA LETTER HA +114B0..114B2;N # Mc [3] TIRHUTA VOWEL SIGN AA..TIRHUTA VOWEL SIGN II +114B3..114B8;N # Mn [6] TIRHUTA VOWEL SIGN U..TIRHUTA VOWEL SIGN VOCALIC LL +114B9;N # Mc TIRHUTA VOWEL SIGN E +114BA;N # Mn TIRHUTA VOWEL SIGN SHORT E +114BB..114BE;N # Mc [4] TIRHUTA VOWEL SIGN AI..TIRHUTA VOWEL SIGN AU +114BF..114C0;N # Mn [2] TIRHUTA SIGN CANDRABINDU..TIRHUTA SIGN ANUSVARA +114C1;N # Mc TIRHUTA SIGN VISARGA +114C2..114C3;N # Mn [2] TIRHUTA SIGN VIRAMA..TIRHUTA SIGN NUKTA +114C4..114C5;N # Lo [2] TIRHUTA SIGN AVAGRAHA..TIRHUTA GVANG +114C6;N # Po TIRHUTA ABBREVIATION SIGN +114C7;N # Lo TIRHUTA OM +114D0..114D9;N # Nd [10] TIRHUTA DIGIT ZERO..TIRHUTA DIGIT NINE +11580..115AE;N # Lo [47] SIDDHAM LETTER A..SIDDHAM LETTER HA +115AF..115B1;N # Mc [3] SIDDHAM VOWEL SIGN AA..SIDDHAM VOWEL SIGN II +115B2..115B5;N # Mn [4] SIDDHAM VOWEL SIGN U..SIDDHAM VOWEL SIGN VOCALIC RR +115B8..115BB;N # Mc [4] SIDDHAM VOWEL SIGN E..SIDDHAM VOWEL SIGN AU +115BC..115BD;N # Mn [2] SIDDHAM SIGN CANDRABINDU..SIDDHAM SIGN ANUSVARA +115BE;N # Mc SIDDHAM SIGN VISARGA +115BF..115C0;N # Mn [2] SIDDHAM SIGN VIRAMA..SIDDHAM SIGN NUKTA +115C1..115D7;N # Po [23] SIDDHAM SIGN SIDDHAM..SIDDHAM SECTION MARK WITH CIRCLES AND FOUR ENCLOSURES +115D8..115DB;N # Lo [4] SIDDHAM LETTER THREE-CIRCLE ALTERNATE I..SIDDHAM LETTER ALTERNATE U +115DC..115DD;N # Mn [2] SIDDHAM VOWEL SIGN ALTERNATE U..SIDDHAM VOWEL SIGN ALTERNATE UU +11600..1162F;N # Lo [48] MODI LETTER A..MODI LETTER LLA +11630..11632;N # Mc [3] MODI VOWEL SIGN AA..MODI VOWEL SIGN II +11633..1163A;N # Mn [8] MODI VOWEL SIGN U..MODI VOWEL SIGN AI +1163B..1163C;N # Mc [2] MODI VOWEL SIGN O..MODI VOWEL SIGN AU +1163D;N # Mn MODI SIGN ANUSVARA +1163E;N # Mc MODI SIGN VISARGA +1163F..11640;N # Mn [2] MODI SIGN VIRAMA..MODI SIGN ARDHACANDRA +11641..11643;N # Po [3] MODI DANDA..MODI ABBREVIATION SIGN +11644;N # Lo MODI SIGN HUVA +11650..11659;N # Nd [10] MODI DIGIT ZERO..MODI DIGIT NINE +11660..1166C;N # Po [13] MONGOLIAN BIRGA WITH ORNAMENT..MONGOLIAN TURNED SWIRL BIRGA WITH DOUBLE ORNAMENT +11680..116AA;N # Lo [43] TAKRI LETTER A..TAKRI LETTER RRA +116AB;N # Mn TAKRI SIGN ANUSVARA +116AC;N # Mc TAKRI SIGN VISARGA +116AD;N # Mn TAKRI VOWEL SIGN AA +116AE..116AF;N # Mc [2] TAKRI VOWEL SIGN I..TAKRI VOWEL SIGN II +116B0..116B5;N # Mn [6] TAKRI VOWEL SIGN U..TAKRI VOWEL SIGN AU +116B6;N # Mc TAKRI SIGN VIRAMA +116B7;N # Mn TAKRI SIGN NUKTA +116B8;N # Lo TAKRI LETTER ARCHAIC KHA +116B9;N # Po TAKRI ABBREVIATION SIGN +116C0..116C9;N # Nd [10] TAKRI DIGIT ZERO..TAKRI DIGIT NINE +11700..1171A;N # Lo [27] AHOM LETTER KA..AHOM LETTER ALTERNATE BA +1171D..1171F;N # Mn [3] AHOM CONSONANT SIGN MEDIAL LA..AHOM CONSONANT SIGN MEDIAL LIGATING RA +11720..11721;N # Mc [2] AHOM VOWEL SIGN A..AHOM VOWEL SIGN AA +11722..11725;N # Mn [4] AHOM VOWEL SIGN I..AHOM VOWEL SIGN UU +11726;N # Mc AHOM VOWEL SIGN E +11727..1172B;N # Mn [5] AHOM VOWEL SIGN AW..AHOM SIGN KILLER +11730..11739;N # Nd [10] AHOM DIGIT ZERO..AHOM DIGIT NINE +1173A..1173B;N # No [2] AHOM NUMBER TEN..AHOM NUMBER TWENTY +1173C..1173E;N # Po [3] AHOM SIGN SMALL SECTION..AHOM SIGN RULAI +1173F;N # So AHOM SYMBOL VI +11740..11746;N # Lo [7] AHOM LETTER CA..AHOM LETTER LLA +11800..1182B;N # Lo [44] DOGRA LETTER A..DOGRA LETTER RRA +1182C..1182E;N # Mc [3] DOGRA VOWEL SIGN AA..DOGRA VOWEL SIGN II +1182F..11837;N # Mn [9] DOGRA VOWEL SIGN U..DOGRA SIGN ANUSVARA +11838;N # Mc DOGRA SIGN VISARGA +11839..1183A;N # Mn [2] DOGRA SIGN VIRAMA..DOGRA SIGN NUKTA +1183B;N # Po DOGRA ABBREVIATION SIGN +118A0..118DF;N # L& [64] WARANG CITI CAPITAL LETTER NGAA..WARANG CITI SMALL LETTER VIYO +118E0..118E9;N # Nd [10] WARANG CITI DIGIT ZERO..WARANG CITI DIGIT NINE +118EA..118F2;N # No [9] WARANG CITI NUMBER TEN..WARANG CITI NUMBER NINETY +118FF;N # Lo WARANG CITI OM +11900..11906;N # Lo [7] DIVES AKURU LETTER A..DIVES AKURU LETTER E +11909;N # Lo DIVES AKURU LETTER O +1190C..11913;N # Lo [8] DIVES AKURU LETTER KA..DIVES AKURU LETTER JA +11915..11916;N # Lo [2] DIVES AKURU LETTER NYA..DIVES AKURU LETTER TTA +11918..1192F;N # Lo [24] DIVES AKURU LETTER DDA..DIVES AKURU LETTER ZA +11930..11935;N # Mc [6] DIVES AKURU VOWEL SIGN AA..DIVES AKURU VOWEL SIGN E +11937..11938;N # Mc [2] DIVES AKURU VOWEL SIGN AI..DIVES AKURU VOWEL SIGN O +1193B..1193C;N # Mn [2] DIVES AKURU SIGN ANUSVARA..DIVES AKURU SIGN CANDRABINDU +1193D;N # Mc DIVES AKURU SIGN HALANTA +1193E;N # Mn DIVES AKURU VIRAMA +1193F;N # Lo DIVES AKURU PREFIXED NASAL SIGN +11940;N # Mc DIVES AKURU MEDIAL YA +11941;N # Lo DIVES AKURU INITIAL RA +11942;N # Mc DIVES AKURU MEDIAL RA +11943;N # Mn DIVES AKURU SIGN NUKTA +11944..11946;N # Po [3] DIVES AKURU DOUBLE DANDA..DIVES AKURU END OF TEXT MARK +11950..11959;N # Nd [10] DIVES AKURU DIGIT ZERO..DIVES AKURU DIGIT NINE +119A0..119A7;N # Lo [8] NANDINAGARI LETTER A..NANDINAGARI LETTER VOCALIC RR +119AA..119D0;N # Lo [39] NANDINAGARI LETTER E..NANDINAGARI LETTER RRA +119D1..119D3;N # Mc [3] NANDINAGARI VOWEL SIGN AA..NANDINAGARI VOWEL SIGN II +119D4..119D7;N # Mn [4] NANDINAGARI VOWEL SIGN U..NANDINAGARI VOWEL SIGN VOCALIC RR +119DA..119DB;N # Mn [2] NANDINAGARI VOWEL SIGN E..NANDINAGARI VOWEL SIGN AI +119DC..119DF;N # Mc [4] NANDINAGARI VOWEL SIGN O..NANDINAGARI SIGN VISARGA +119E0;N # Mn NANDINAGARI SIGN VIRAMA +119E1;N # Lo NANDINAGARI SIGN AVAGRAHA +119E2;N # Po NANDINAGARI SIGN SIDDHAM +119E3;N # Lo NANDINAGARI HEADSTROKE +119E4;N # Mc NANDINAGARI VOWEL SIGN PRISHTHAMATRA E +11A00;N # Lo ZANABAZAR SQUARE LETTER A +11A01..11A0A;N # Mn [10] ZANABAZAR SQUARE VOWEL SIGN I..ZANABAZAR SQUARE VOWEL LENGTH MARK +11A0B..11A32;N # Lo [40] ZANABAZAR SQUARE LETTER KA..ZANABAZAR SQUARE LETTER KSSA +11A33..11A38;N # Mn [6] ZANABAZAR SQUARE FINAL CONSONANT MARK..ZANABAZAR SQUARE SIGN ANUSVARA +11A39;N # Mc ZANABAZAR SQUARE SIGN VISARGA +11A3A;N # Lo ZANABAZAR SQUARE CLUSTER-INITIAL LETTER RA +11A3B..11A3E;N # Mn [4] ZANABAZAR SQUARE CLUSTER-FINAL LETTER YA..ZANABAZAR SQUARE CLUSTER-FINAL LETTER VA +11A3F..11A46;N # Po [8] ZANABAZAR SQUARE INITIAL HEAD MARK..ZANABAZAR SQUARE CLOSING DOUBLE-LINED HEAD MARK +11A47;N # Mn ZANABAZAR SQUARE SUBJOINER +11A50;N # Lo SOYOMBO LETTER A +11A51..11A56;N # Mn [6] SOYOMBO VOWEL SIGN I..SOYOMBO VOWEL SIGN OE +11A57..11A58;N # Mc [2] SOYOMBO VOWEL SIGN AI..SOYOMBO VOWEL SIGN AU +11A59..11A5B;N # Mn [3] SOYOMBO VOWEL SIGN VOCALIC R..SOYOMBO VOWEL LENGTH MARK +11A5C..11A89;N # Lo [46] SOYOMBO LETTER KA..SOYOMBO CLUSTER-INITIAL LETTER SA +11A8A..11A96;N # Mn [13] SOYOMBO FINAL CONSONANT SIGN G..SOYOMBO SIGN ANUSVARA +11A97;N # Mc SOYOMBO SIGN VISARGA +11A98..11A99;N # Mn [2] SOYOMBO GEMINATION MARK..SOYOMBO SUBJOINER +11A9A..11A9C;N # Po [3] SOYOMBO MARK TSHEG..SOYOMBO MARK DOUBLE SHAD +11A9D;N # Lo SOYOMBO MARK PLUTA +11A9E..11AA2;N # Po [5] SOYOMBO HEAD MARK WITH MOON AND SUN AND TRIPLE FLAME..SOYOMBO TERMINAL MARK-2 +11AB0..11ABF;N # Lo [16] CANADIAN SYLLABICS NATTILIK HI..CANADIAN SYLLABICS SPA +11AC0..11AF8;N # Lo [57] PAU CIN HAU LETTER PA..PAU CIN HAU GLOTTAL STOP FINAL +11B00..11B09;N # Po [10] DEVANAGARI HEAD MARK..DEVANAGARI SIGN MINDU +11C00..11C08;N # Lo [9] BHAIKSUKI LETTER A..BHAIKSUKI LETTER VOCALIC L +11C0A..11C2E;N # Lo [37] BHAIKSUKI LETTER E..BHAIKSUKI LETTER HA +11C2F;N # Mc BHAIKSUKI VOWEL SIGN AA +11C30..11C36;N # Mn [7] BHAIKSUKI VOWEL SIGN I..BHAIKSUKI VOWEL SIGN VOCALIC L +11C38..11C3D;N # Mn [6] BHAIKSUKI VOWEL SIGN E..BHAIKSUKI SIGN ANUSVARA +11C3E;N # Mc BHAIKSUKI SIGN VISARGA +11C3F;N # Mn BHAIKSUKI SIGN VIRAMA +11C40;N # Lo BHAIKSUKI SIGN AVAGRAHA +11C41..11C45;N # Po [5] BHAIKSUKI DANDA..BHAIKSUKI GAP FILLER-2 +11C50..11C59;N # Nd [10] BHAIKSUKI DIGIT ZERO..BHAIKSUKI DIGIT NINE +11C5A..11C6C;N # No [19] BHAIKSUKI NUMBER ONE..BHAIKSUKI HUNDREDS UNIT MARK +11C70..11C71;N # Po [2] MARCHEN HEAD MARK..MARCHEN MARK SHAD +11C72..11C8F;N # Lo [30] MARCHEN LETTER KA..MARCHEN LETTER A +11C92..11CA7;N # Mn [22] MARCHEN SUBJOINED LETTER KA..MARCHEN SUBJOINED LETTER ZA +11CA9;N # Mc MARCHEN SUBJOINED LETTER YA +11CAA..11CB0;N # Mn [7] MARCHEN SUBJOINED LETTER RA..MARCHEN VOWEL SIGN AA +11CB1;N # Mc MARCHEN VOWEL SIGN I +11CB2..11CB3;N # Mn [2] MARCHEN VOWEL SIGN U..MARCHEN VOWEL SIGN E +11CB4;N # Mc MARCHEN VOWEL SIGN O +11CB5..11CB6;N # Mn [2] MARCHEN SIGN ANUSVARA..MARCHEN SIGN CANDRABINDU +11D00..11D06;N # Lo [7] MASARAM GONDI LETTER A..MASARAM GONDI LETTER E +11D08..11D09;N # Lo [2] MASARAM GONDI LETTER AI..MASARAM GONDI LETTER O +11D0B..11D30;N # Lo [38] MASARAM GONDI LETTER AU..MASARAM GONDI LETTER TRA +11D31..11D36;N # Mn [6] MASARAM GONDI VOWEL SIGN AA..MASARAM GONDI VOWEL SIGN VOCALIC R +11D3A;N # Mn MASARAM GONDI VOWEL SIGN E +11D3C..11D3D;N # Mn [2] MASARAM GONDI VOWEL SIGN AI..MASARAM GONDI VOWEL SIGN O +11D3F..11D45;N # Mn [7] MASARAM GONDI VOWEL SIGN AU..MASARAM GONDI VIRAMA +11D46;N # Lo MASARAM GONDI REPHA +11D47;N # Mn MASARAM GONDI RA-KARA +11D50..11D59;N # Nd [10] MASARAM GONDI DIGIT ZERO..MASARAM GONDI DIGIT NINE +11D60..11D65;N # Lo [6] GUNJALA GONDI LETTER A..GUNJALA GONDI LETTER UU +11D67..11D68;N # Lo [2] GUNJALA GONDI LETTER EE..GUNJALA GONDI LETTER AI +11D6A..11D89;N # Lo [32] GUNJALA GONDI LETTER OO..GUNJALA GONDI LETTER SA +11D8A..11D8E;N # Mc [5] GUNJALA GONDI VOWEL SIGN AA..GUNJALA GONDI VOWEL SIGN UU +11D90..11D91;N # Mn [2] GUNJALA GONDI VOWEL SIGN EE..GUNJALA GONDI VOWEL SIGN AI +11D93..11D94;N # Mc [2] GUNJALA GONDI VOWEL SIGN OO..GUNJALA GONDI VOWEL SIGN AU +11D95;N # Mn GUNJALA GONDI SIGN ANUSVARA +11D96;N # Mc GUNJALA GONDI SIGN VISARGA +11D97;N # Mn GUNJALA GONDI VIRAMA +11D98;N # Lo GUNJALA GONDI OM +11DA0..11DA9;N # Nd [10] GUNJALA GONDI DIGIT ZERO..GUNJALA GONDI DIGIT NINE +11EE0..11EF2;N # Lo [19] MAKASAR LETTER KA..MAKASAR ANGKA +11EF3..11EF4;N # Mn [2] MAKASAR VOWEL SIGN I..MAKASAR VOWEL SIGN U +11EF5..11EF6;N # Mc [2] MAKASAR VOWEL SIGN E..MAKASAR VOWEL SIGN O +11EF7..11EF8;N # Po [2] MAKASAR PASSIMBANG..MAKASAR END OF SECTION +11F00..11F01;N # Mn [2] KAWI SIGN CANDRABINDU..KAWI SIGN ANUSVARA +11F02;N # Lo KAWI SIGN REPHA +11F03;N # Mc KAWI SIGN VISARGA +11F04..11F10;N # Lo [13] KAWI LETTER A..KAWI LETTER O +11F12..11F33;N # Lo [34] KAWI LETTER KA..KAWI LETTER JNYA +11F34..11F35;N # Mc [2] KAWI VOWEL SIGN AA..KAWI VOWEL SIGN ALTERNATE AA +11F36..11F3A;N # Mn [5] KAWI VOWEL SIGN I..KAWI VOWEL SIGN VOCALIC R +11F3E..11F3F;N # Mc [2] KAWI VOWEL SIGN E..KAWI VOWEL SIGN AI +11F40;N # Mn KAWI VOWEL SIGN EU +11F41;N # Mc KAWI SIGN KILLER +11F42;N # Mn KAWI CONJOINER +11F43..11F4F;N # Po [13] KAWI DANDA..KAWI PUNCTUATION CLOSING SPIRAL +11F50..11F59;N # Nd [10] KAWI DIGIT ZERO..KAWI DIGIT NINE +11FB0;N # Lo LISU LETTER YHA +11FC0..11FD4;N # No [21] TAMIL FRACTION ONE THREE-HUNDRED-AND-TWENTIETH..TAMIL FRACTION DOWNSCALING FACTOR KIIZH +11FD5..11FDC;N # So [8] TAMIL SIGN NEL..TAMIL SIGN MUKKURUNI +11FDD..11FE0;N # Sc [4] TAMIL SIGN KAACU..TAMIL SIGN VARAAKAN +11FE1..11FF1;N # So [17] TAMIL SIGN PAARAM..TAMIL SIGN VAKAIYARAA +11FFF;N # Po TAMIL PUNCTUATION END OF TEXT +12000..12399;N # Lo [922] CUNEIFORM SIGN A..CUNEIFORM SIGN U U +12400..1246E;N # Nl [111] CUNEIFORM NUMERIC SIGN TWO ASH..CUNEIFORM NUMERIC SIGN NINE U VARIANT FORM +12470..12474;N # Po [5] CUNEIFORM PUNCTUATION SIGN OLD ASSYRIAN WORD DIVIDER..CUNEIFORM PUNCTUATION SIGN DIAGONAL QUADCOLON +12480..12543;N # Lo [196] CUNEIFORM SIGN AB TIMES NUN TENU..CUNEIFORM SIGN ZU5 TIMES THREE DISH TENU +12F90..12FF0;N # Lo [97] CYPRO-MINOAN SIGN CM001..CYPRO-MINOAN SIGN CM114 +12FF1..12FF2;N # Po [2] CYPRO-MINOAN SIGN CM301..CYPRO-MINOAN SIGN CM302 +13000..1342F;N # Lo [1072] EGYPTIAN HIEROGLYPH A001..EGYPTIAN HIEROGLYPH V011D +13430..1343F;N # Cf [16] EGYPTIAN HIEROGLYPH VERTICAL JOINER..EGYPTIAN HIEROGLYPH END WALLED ENCLOSURE +13440;N # Mn EGYPTIAN HIEROGLYPH MIRROR HORIZONTALLY +13441..13446;N # Lo [6] EGYPTIAN HIEROGLYPH FULL BLANK..EGYPTIAN HIEROGLYPH WIDE LOST SIGN +13447..13455;N # Mn [15] EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP START..EGYPTIAN HIEROGLYPH MODIFIER DAMAGED +14400..14646;N # Lo [583] ANATOLIAN HIEROGLYPH A001..ANATOLIAN HIEROGLYPH A530 +16800..16A38;N # Lo [569] BAMUM LETTER PHASE-A NGKUE MFON..BAMUM LETTER PHASE-F VUEQ +16A40..16A5E;N # Lo [31] MRO LETTER TA..MRO LETTER TEK +16A60..16A69;N # Nd [10] MRO DIGIT ZERO..MRO DIGIT NINE +16A6E..16A6F;N # Po [2] MRO DANDA..MRO DOUBLE DANDA +16A70..16ABE;N # Lo [79] TANGSA LETTER OZ..TANGSA LETTER ZA +16AC0..16AC9;N # Nd [10] TANGSA DIGIT ZERO..TANGSA DIGIT NINE +16AD0..16AED;N # Lo [30] BASSA VAH LETTER ENNI..BASSA VAH LETTER I +16AF0..16AF4;N # Mn [5] BASSA VAH COMBINING HIGH TONE..BASSA VAH COMBINING HIGH-LOW TONE +16AF5;N # Po BASSA VAH FULL STOP +16B00..16B2F;N # Lo [48] PAHAWH HMONG VOWEL KEEB..PAHAWH HMONG CONSONANT CAU +16B30..16B36;N # Mn [7] PAHAWH HMONG MARK CIM TUB..PAHAWH HMONG MARK CIM TAUM +16B37..16B3B;N # Po [5] PAHAWH HMONG SIGN VOS THOM..PAHAWH HMONG SIGN VOS FEEM +16B3C..16B3F;N # So [4] PAHAWH HMONG SIGN XYEEM NTXIV..PAHAWH HMONG SIGN XYEEM FAIB +16B40..16B43;N # Lm [4] PAHAWH HMONG SIGN VOS SEEV..PAHAWH HMONG SIGN IB YAM +16B44;N # Po PAHAWH HMONG SIGN XAUS +16B45;N # So PAHAWH HMONG SIGN CIM TSOV ROG +16B50..16B59;N # Nd [10] PAHAWH HMONG DIGIT ZERO..PAHAWH HMONG DIGIT NINE +16B5B..16B61;N # No [7] PAHAWH HMONG NUMBER TENS..PAHAWH HMONG NUMBER TRILLIONS +16B63..16B77;N # Lo [21] PAHAWH HMONG SIGN VOS LUB..PAHAWH HMONG SIGN CIM NRES TOS +16B7D..16B8F;N # Lo [19] PAHAWH HMONG CLAN SIGN TSHEEJ..PAHAWH HMONG CLAN SIGN VWJ +16E40..16E7F;N # L& [64] MEDEFAIDRIN CAPITAL LETTER M..MEDEFAIDRIN SMALL LETTER Y +16E80..16E96;N # No [23] MEDEFAIDRIN DIGIT ZERO..MEDEFAIDRIN DIGIT THREE ALTERNATE FORM +16E97..16E9A;N # Po [4] MEDEFAIDRIN COMMA..MEDEFAIDRIN EXCLAMATION OH +16F00..16F4A;N # Lo [75] MIAO LETTER PA..MIAO LETTER RTE +16F4F;N # Mn MIAO SIGN CONSONANT MODIFIER BAR +16F50;N # Lo MIAO LETTER NASALIZATION +16F51..16F87;N # Mc [55] MIAO SIGN ASPIRATION..MIAO VOWEL SIGN UI +16F8F..16F92;N # Mn [4] MIAO TONE RIGHT..MIAO TONE BELOW +16F93..16F9F;N # Lm [13] MIAO LETTER TONE-2..MIAO LETTER REFORMED TONE-8 +16FE0..16FE1;W # Lm [2] TANGUT ITERATION MARK..NUSHU ITERATION MARK +16FE2;W # Po OLD CHINESE HOOK MARK +16FE3;W # Lm OLD CHINESE ITERATION MARK +16FE4;W # Mn KHITAN SMALL SCRIPT FILLER +16FF0..16FF1;W # Mc [2] VIETNAMESE ALTERNATE READING MARK CA..VIETNAMESE ALTERNATE READING MARK NHAY +17000..187F7;W # Lo [6136] TANGUT IDEOGRAPH-17000..TANGUT IDEOGRAPH-187F7 +18800..18AFF;W # Lo [768] TANGUT COMPONENT-001..TANGUT COMPONENT-768 +18B00..18CD5;W # Lo [470] KHITAN SMALL SCRIPT CHARACTER-18B00..KHITAN SMALL SCRIPT CHARACTER-18CD5 +18D00..18D08;W # Lo [9] TANGUT IDEOGRAPH-18D00..TANGUT IDEOGRAPH-18D08 +1AFF0..1AFF3;W # Lm [4] KATAKANA LETTER MINNAN TONE-2..KATAKANA LETTER MINNAN TONE-5 +1AFF5..1AFFB;W # Lm [7] KATAKANA LETTER MINNAN TONE-7..KATAKANA LETTER MINNAN NASALIZED TONE-5 +1AFFD..1AFFE;W # Lm [2] KATAKANA LETTER MINNAN NASALIZED TONE-7..KATAKANA LETTER MINNAN NASALIZED TONE-8 +1B000..1B0FF;W # Lo [256] KATAKANA LETTER ARCHAIC E..HENTAIGANA LETTER RE-2 +1B100..1B122;W # Lo [35] HENTAIGANA LETTER RE-3..KATAKANA LETTER ARCHAIC WU +1B132;W # Lo HIRAGANA LETTER SMALL KO +1B150..1B152;W # Lo [3] HIRAGANA LETTER SMALL WI..HIRAGANA LETTER SMALL WO +1B155;W # Lo KATAKANA LETTER SMALL KO +1B164..1B167;W # Lo [4] KATAKANA LETTER SMALL WI..KATAKANA LETTER SMALL N +1B170..1B2FB;W # Lo [396] NUSHU CHARACTER-1B170..NUSHU CHARACTER-1B2FB +1BC00..1BC6A;N # Lo [107] DUPLOYAN LETTER H..DUPLOYAN LETTER VOCALIC M +1BC70..1BC7C;N # Lo [13] DUPLOYAN AFFIX LEFT HORIZONTAL SECANT..DUPLOYAN AFFIX ATTACHED TANGENT HOOK +1BC80..1BC88;N # Lo [9] DUPLOYAN AFFIX HIGH ACUTE..DUPLOYAN AFFIX HIGH VERTICAL +1BC90..1BC99;N # Lo [10] DUPLOYAN AFFIX LOW ACUTE..DUPLOYAN AFFIX LOW ARROW +1BC9C;N # So DUPLOYAN SIGN O WITH CROSS +1BC9D..1BC9E;N # Mn [2] DUPLOYAN THICK LETTER SELECTOR..DUPLOYAN DOUBLE MARK +1BC9F;N # Po DUPLOYAN PUNCTUATION CHINOOK FULL STOP +1BCA0..1BCA3;N # Cf [4] SHORTHAND FORMAT LETTER OVERLAP..SHORTHAND FORMAT UP STEP +1CF00..1CF2D;N # Mn [46] ZNAMENNY COMBINING MARK GORAZDO NIZKO S KRYZHEM ON LEFT..ZNAMENNY COMBINING MARK KRYZH ON LEFT +1CF30..1CF46;N # Mn [23] ZNAMENNY COMBINING TONAL RANGE MARK MRACHNO..ZNAMENNY PRIZNAK MODIFIER ROG +1CF50..1CFC3;N # So [116] ZNAMENNY NEUME KRYUK..ZNAMENNY NEUME PAUK +1D000..1D0F5;N # So [246] BYZANTINE MUSICAL SYMBOL PSILI..BYZANTINE MUSICAL SYMBOL GORGON NEO KATO +1D100..1D126;N # So [39] MUSICAL SYMBOL SINGLE BARLINE..MUSICAL SYMBOL DRUM CLEF-2 +1D129..1D164;N # So [60] MUSICAL SYMBOL MULTIPLE MEASURE REST..MUSICAL SYMBOL ONE HUNDRED TWENTY-EIGHTH NOTE +1D165..1D166;N # Mc [2] MUSICAL SYMBOL COMBINING STEM..MUSICAL SYMBOL COMBINING SPRECHGESANG STEM +1D167..1D169;N # Mn [3] MUSICAL SYMBOL COMBINING TREMOLO-1..MUSICAL SYMBOL COMBINING TREMOLO-3 +1D16A..1D16C;N # So [3] MUSICAL SYMBOL FINGERED TREMOLO-1..MUSICAL SYMBOL FINGERED TREMOLO-3 +1D16D..1D172;N # Mc [6] MUSICAL SYMBOL COMBINING AUGMENTATION DOT..MUSICAL SYMBOL COMBINING FLAG-5 +1D173..1D17A;N # Cf [8] MUSICAL SYMBOL BEGIN BEAM..MUSICAL SYMBOL END PHRASE +1D17B..1D182;N # Mn [8] MUSICAL SYMBOL COMBINING ACCENT..MUSICAL SYMBOL COMBINING LOURE +1D183..1D184;N # So [2] MUSICAL SYMBOL ARPEGGIATO UP..MUSICAL SYMBOL ARPEGGIATO DOWN +1D185..1D18B;N # Mn [7] MUSICAL SYMBOL COMBINING DOIT..MUSICAL SYMBOL COMBINING TRIPLE TONGUE +1D18C..1D1A9;N # So [30] MUSICAL SYMBOL RINFORZANDO..MUSICAL SYMBOL DEGREE SLASH +1D1AA..1D1AD;N # Mn [4] MUSICAL SYMBOL COMBINING DOWN BOW..MUSICAL SYMBOL COMBINING SNAP PIZZICATO +1D1AE..1D1EA;N # So [61] MUSICAL SYMBOL PEDAL MARK..MUSICAL SYMBOL KORON +1D200..1D241;N # So [66] GREEK VOCAL NOTATION SYMBOL-1..GREEK INSTRUMENTAL NOTATION SYMBOL-54 +1D242..1D244;N # Mn [3] COMBINING GREEK MUSICAL TRISEME..COMBINING GREEK MUSICAL PENTASEME +1D245;N # So GREEK MUSICAL LEIMMA +1D2C0..1D2D3;N # No [20] KAKTOVIK NUMERAL ZERO..KAKTOVIK NUMERAL NINETEEN +1D2E0..1D2F3;N # No [20] MAYAN NUMERAL ZERO..MAYAN NUMERAL NINETEEN +1D300..1D356;N # So [87] MONOGRAM FOR EARTH..TETRAGRAM FOR FOSTERING +1D360..1D378;N # No [25] COUNTING ROD UNIT DIGIT ONE..TALLY MARK FIVE +1D400..1D454;N # L& [85] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL ITALIC SMALL G +1D456..1D49C;N # L& [71] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL SCRIPT CAPITAL A +1D49E..1D49F;N # Lu [2] MATHEMATICAL SCRIPT CAPITAL C..MATHEMATICAL SCRIPT CAPITAL D +1D4A2;N # Lu MATHEMATICAL SCRIPT CAPITAL G +1D4A5..1D4A6;N # Lu [2] MATHEMATICAL SCRIPT CAPITAL J..MATHEMATICAL SCRIPT CAPITAL K +1D4A9..1D4AC;N # Lu [4] MATHEMATICAL SCRIPT CAPITAL N..MATHEMATICAL SCRIPT CAPITAL Q +1D4AE..1D4B9;N # L& [12] MATHEMATICAL SCRIPT CAPITAL S..MATHEMATICAL SCRIPT SMALL D +1D4BB;N # Ll MATHEMATICAL SCRIPT SMALL F +1D4BD..1D4C3;N # Ll [7] MATHEMATICAL SCRIPT SMALL H..MATHEMATICAL SCRIPT SMALL N +1D4C5..1D505;N # L& [65] MATHEMATICAL SCRIPT SMALL P..MATHEMATICAL FRAKTUR CAPITAL B +1D507..1D50A;N # Lu [4] MATHEMATICAL FRAKTUR CAPITAL D..MATHEMATICAL FRAKTUR CAPITAL G +1D50D..1D514;N # Lu [8] MATHEMATICAL FRAKTUR CAPITAL J..MATHEMATICAL FRAKTUR CAPITAL Q +1D516..1D51C;N # Lu [7] MATHEMATICAL FRAKTUR CAPITAL S..MATHEMATICAL FRAKTUR CAPITAL Y +1D51E..1D539;N # L& [28] MATHEMATICAL FRAKTUR SMALL A..MATHEMATICAL DOUBLE-STRUCK CAPITAL B +1D53B..1D53E;N # Lu [4] MATHEMATICAL DOUBLE-STRUCK CAPITAL D..MATHEMATICAL DOUBLE-STRUCK CAPITAL G +1D540..1D544;N # Lu [5] MATHEMATICAL DOUBLE-STRUCK CAPITAL I..MATHEMATICAL DOUBLE-STRUCK CAPITAL M +1D546;N # Lu MATHEMATICAL DOUBLE-STRUCK CAPITAL O +1D54A..1D550;N # Lu [7] MATHEMATICAL DOUBLE-STRUCK CAPITAL S..MATHEMATICAL DOUBLE-STRUCK CAPITAL Y +1D552..1D6A5;N # L& [340] MATHEMATICAL DOUBLE-STRUCK SMALL A..MATHEMATICAL ITALIC SMALL DOTLESS J +1D6A8..1D6C0;N # Lu [25] MATHEMATICAL BOLD CAPITAL ALPHA..MATHEMATICAL BOLD CAPITAL OMEGA +1D6C1;N # Sm MATHEMATICAL BOLD NABLA +1D6C2..1D6DA;N # Ll [25] MATHEMATICAL BOLD SMALL ALPHA..MATHEMATICAL BOLD SMALL OMEGA +1D6DB;N # Sm MATHEMATICAL BOLD PARTIAL DIFFERENTIAL +1D6DC..1D6FA;N # L& [31] MATHEMATICAL BOLD EPSILON SYMBOL..MATHEMATICAL ITALIC CAPITAL OMEGA +1D6FB;N # Sm MATHEMATICAL ITALIC NABLA +1D6FC..1D714;N # Ll [25] MATHEMATICAL ITALIC SMALL ALPHA..MATHEMATICAL ITALIC SMALL OMEGA +1D715;N # Sm MATHEMATICAL ITALIC PARTIAL DIFFERENTIAL +1D716..1D734;N # L& [31] MATHEMATICAL ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD ITALIC CAPITAL OMEGA +1D735;N # Sm MATHEMATICAL BOLD ITALIC NABLA +1D736..1D74E;N # Ll [25] MATHEMATICAL BOLD ITALIC SMALL ALPHA..MATHEMATICAL BOLD ITALIC SMALL OMEGA +1D74F;N # Sm MATHEMATICAL BOLD ITALIC PARTIAL DIFFERENTIAL +1D750..1D76E;N # L& [31] MATHEMATICAL BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD CAPITAL OMEGA +1D76F;N # Sm MATHEMATICAL SANS-SERIF BOLD NABLA +1D770..1D788;N # Ll [25] MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD SMALL OMEGA +1D789;N # Sm MATHEMATICAL SANS-SERIF BOLD PARTIAL DIFFERENTIAL +1D78A..1D7A8;N # L& [31] MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMEGA +1D7A9;N # Sm MATHEMATICAL SANS-SERIF BOLD ITALIC NABLA +1D7AA..1D7C2;N # Ll [25] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMEGA +1D7C3;N # Sm MATHEMATICAL SANS-SERIF BOLD ITALIC PARTIAL DIFFERENTIAL +1D7C4..1D7CB;N # L& [8] MATHEMATICAL SANS-SERIF BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD SMALL DIGAMMA +1D7CE..1D7FF;N # Nd [50] MATHEMATICAL BOLD DIGIT ZERO..MATHEMATICAL MONOSPACE DIGIT NINE +1D800..1D9FF;N # So [512] SIGNWRITING HAND-FIST INDEX..SIGNWRITING HEAD +1DA00..1DA36;N # Mn [55] SIGNWRITING HEAD RIM..SIGNWRITING AIR SUCKING IN +1DA37..1DA3A;N # So [4] SIGNWRITING AIR BLOW SMALL ROTATIONS..SIGNWRITING BREATH EXHALE +1DA3B..1DA6C;N # Mn [50] SIGNWRITING MOUTH CLOSED NEUTRAL..SIGNWRITING EXCITEMENT +1DA6D..1DA74;N # So [8] SIGNWRITING SHOULDER HIP SPINE..SIGNWRITING TORSO-FLOORPLANE TWISTING +1DA75;N # Mn SIGNWRITING UPPER BODY TILTING FROM HIP JOINTS +1DA76..1DA83;N # So [14] SIGNWRITING LIMB COMBINATION..SIGNWRITING LOCATION DEPTH +1DA84;N # Mn SIGNWRITING LOCATION HEAD NECK +1DA85..1DA86;N # So [2] SIGNWRITING LOCATION TORSO..SIGNWRITING LOCATION LIMBS DIGITS +1DA87..1DA8B;N # Po [5] SIGNWRITING COMMA..SIGNWRITING PARENTHESIS +1DA9B..1DA9F;N # Mn [5] SIGNWRITING FILL MODIFIER-2..SIGNWRITING FILL MODIFIER-6 +1DAA1..1DAAF;N # Mn [15] SIGNWRITING ROTATION MODIFIER-2..SIGNWRITING ROTATION MODIFIER-16 +1DF00..1DF09;N # Ll [10] LATIN SMALL LETTER FENG DIGRAPH WITH TRILL..LATIN SMALL LETTER T WITH HOOK AND RETROFLEX HOOK +1DF0A;N # Lo LATIN LETTER RETROFLEX CLICK WITH RETROFLEX HOOK +1DF0B..1DF1E;N # Ll [20] LATIN SMALL LETTER ESH WITH DOUBLE BAR..LATIN SMALL LETTER S WITH CURL +1DF25..1DF2A;N # Ll [6] LATIN SMALL LETTER D WITH MID-HEIGHT LEFT HOOK..LATIN SMALL LETTER T WITH MID-HEIGHT LEFT HOOK +1E000..1E006;N # Mn [7] COMBINING GLAGOLITIC LETTER AZU..COMBINING GLAGOLITIC LETTER ZHIVETE +1E008..1E018;N # Mn [17] COMBINING GLAGOLITIC LETTER ZEMLJA..COMBINING GLAGOLITIC LETTER HERU +1E01B..1E021;N # Mn [7] COMBINING GLAGOLITIC LETTER SHTA..COMBINING GLAGOLITIC LETTER YATI +1E023..1E024;N # Mn [2] COMBINING GLAGOLITIC LETTER YU..COMBINING GLAGOLITIC LETTER SMALL YUS +1E026..1E02A;N # Mn [5] COMBINING GLAGOLITIC LETTER YO..COMBINING GLAGOLITIC LETTER FITA +1E030..1E06D;N # Lm [62] MODIFIER LETTER CYRILLIC SMALL A..MODIFIER LETTER CYRILLIC SMALL STRAIGHT U WITH STROKE +1E08F;N # Mn COMBINING CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I +1E100..1E12C;N # Lo [45] NYIAKENG PUACHUE HMONG LETTER MA..NYIAKENG PUACHUE HMONG LETTER W +1E130..1E136;N # Mn [7] NYIAKENG PUACHUE HMONG TONE-B..NYIAKENG PUACHUE HMONG TONE-D +1E137..1E13D;N # Lm [7] NYIAKENG PUACHUE HMONG SIGN FOR PERSON..NYIAKENG PUACHUE HMONG SYLLABLE LENGTHENER +1E140..1E149;N # Nd [10] NYIAKENG PUACHUE HMONG DIGIT ZERO..NYIAKENG PUACHUE HMONG DIGIT NINE +1E14E;N # Lo NYIAKENG PUACHUE HMONG LOGOGRAM NYAJ +1E14F;N # So NYIAKENG PUACHUE HMONG CIRCLED CA +1E290..1E2AD;N # Lo [30] TOTO LETTER PA..TOTO LETTER A +1E2AE;N # Mn TOTO SIGN RISING TONE +1E2C0..1E2EB;N # Lo [44] WANCHO LETTER AA..WANCHO LETTER YIH +1E2EC..1E2EF;N # Mn [4] WANCHO TONE TUP..WANCHO TONE KOINI +1E2F0..1E2F9;N # Nd [10] WANCHO DIGIT ZERO..WANCHO DIGIT NINE +1E2FF;N # Sc WANCHO NGUN SIGN +1E4D0..1E4EA;N # Lo [27] NAG MUNDARI LETTER O..NAG MUNDARI LETTER ELL +1E4EB;N # Lm NAG MUNDARI SIGN OJOD +1E4EC..1E4EF;N # Mn [4] NAG MUNDARI SIGN MUHOR..NAG MUNDARI SIGN SUTUH +1E4F0..1E4F9;N # Nd [10] NAG MUNDARI DIGIT ZERO..NAG MUNDARI DIGIT NINE +1E7E0..1E7E6;N # Lo [7] ETHIOPIC SYLLABLE HHYA..ETHIOPIC SYLLABLE HHYO +1E7E8..1E7EB;N # Lo [4] ETHIOPIC SYLLABLE GURAGE HHWA..ETHIOPIC SYLLABLE HHWE +1E7ED..1E7EE;N # Lo [2] ETHIOPIC SYLLABLE GURAGE MWI..ETHIOPIC SYLLABLE GURAGE MWEE +1E7F0..1E7FE;N # Lo [15] ETHIOPIC SYLLABLE GURAGE QWI..ETHIOPIC SYLLABLE GURAGE PWEE +1E800..1E8C4;N # Lo [197] MENDE KIKAKUI SYLLABLE M001 KI..MENDE KIKAKUI SYLLABLE M060 NYON +1E8C7..1E8CF;N # No [9] MENDE KIKAKUI DIGIT ONE..MENDE KIKAKUI DIGIT NINE +1E8D0..1E8D6;N # Mn [7] MENDE KIKAKUI COMBINING NUMBER TEENS..MENDE KIKAKUI COMBINING NUMBER MILLIONS +1E900..1E943;N # L& [68] ADLAM CAPITAL LETTER ALIF..ADLAM SMALL LETTER SHA +1E944..1E94A;N # Mn [7] ADLAM ALIF LENGTHENER..ADLAM NUKTA +1E94B;N # Lm ADLAM NASALIZATION MARK +1E950..1E959;N # Nd [10] ADLAM DIGIT ZERO..ADLAM DIGIT NINE +1E95E..1E95F;N # Po [2] ADLAM INITIAL EXCLAMATION MARK..ADLAM INITIAL QUESTION MARK +1EC71..1ECAB;N # No [59] INDIC SIYAQ NUMBER ONE..INDIC SIYAQ NUMBER PREFIXED NINE +1ECAC;N # So INDIC SIYAQ PLACEHOLDER +1ECAD..1ECAF;N # No [3] INDIC SIYAQ FRACTION ONE QUARTER..INDIC SIYAQ FRACTION THREE QUARTERS +1ECB0;N # Sc INDIC SIYAQ RUPEE MARK +1ECB1..1ECB4;N # No [4] INDIC SIYAQ NUMBER ALTERNATE ONE..INDIC SIYAQ ALTERNATE LAKH MARK +1ED01..1ED2D;N # No [45] OTTOMAN SIYAQ NUMBER ONE..OTTOMAN SIYAQ NUMBER NINETY THOUSAND +1ED2E;N # So OTTOMAN SIYAQ MARRATAN +1ED2F..1ED3D;N # No [15] OTTOMAN SIYAQ ALTERNATE NUMBER TWO..OTTOMAN SIYAQ FRACTION ONE SIXTH +1EE00..1EE03;N # Lo [4] ARABIC MATHEMATICAL ALEF..ARABIC MATHEMATICAL DAL +1EE05..1EE1F;N # Lo [27] ARABIC MATHEMATICAL WAW..ARABIC MATHEMATICAL DOTLESS QAF +1EE21..1EE22;N # Lo [2] ARABIC MATHEMATICAL INITIAL BEH..ARABIC MATHEMATICAL INITIAL JEEM +1EE24;N # Lo ARABIC MATHEMATICAL INITIAL HEH +1EE27;N # Lo ARABIC MATHEMATICAL INITIAL HAH +1EE29..1EE32;N # Lo [10] ARABIC MATHEMATICAL INITIAL YEH..ARABIC MATHEMATICAL INITIAL QAF +1EE34..1EE37;N # Lo [4] ARABIC MATHEMATICAL INITIAL SHEEN..ARABIC MATHEMATICAL INITIAL KHAH +1EE39;N # Lo ARABIC MATHEMATICAL INITIAL DAD +1EE3B;N # Lo ARABIC MATHEMATICAL INITIAL GHAIN +1EE42;N # Lo ARABIC MATHEMATICAL TAILED JEEM +1EE47;N # Lo ARABIC MATHEMATICAL TAILED HAH +1EE49;N # Lo ARABIC MATHEMATICAL TAILED YEH +1EE4B;N # Lo ARABIC MATHEMATICAL TAILED LAM +1EE4D..1EE4F;N # Lo [3] ARABIC MATHEMATICAL TAILED NOON..ARABIC MATHEMATICAL TAILED AIN +1EE51..1EE52;N # Lo [2] ARABIC MATHEMATICAL TAILED SAD..ARABIC MATHEMATICAL TAILED QAF +1EE54;N # Lo ARABIC MATHEMATICAL TAILED SHEEN +1EE57;N # Lo ARABIC MATHEMATICAL TAILED KHAH +1EE59;N # Lo ARABIC MATHEMATICAL TAILED DAD +1EE5B;N # Lo ARABIC MATHEMATICAL TAILED GHAIN +1EE5D;N # Lo ARABIC MATHEMATICAL TAILED DOTLESS NOON +1EE5F;N # Lo ARABIC MATHEMATICAL TAILED DOTLESS QAF +1EE61..1EE62;N # Lo [2] ARABIC MATHEMATICAL STRETCHED BEH..ARABIC MATHEMATICAL STRETCHED JEEM +1EE64;N # Lo ARABIC MATHEMATICAL STRETCHED HEH +1EE67..1EE6A;N # Lo [4] ARABIC MATHEMATICAL STRETCHED HAH..ARABIC MATHEMATICAL STRETCHED KAF +1EE6C..1EE72;N # Lo [7] ARABIC MATHEMATICAL STRETCHED MEEM..ARABIC MATHEMATICAL STRETCHED QAF +1EE74..1EE77;N # Lo [4] ARABIC MATHEMATICAL STRETCHED SHEEN..ARABIC MATHEMATICAL STRETCHED KHAH +1EE79..1EE7C;N # Lo [4] ARABIC MATHEMATICAL STRETCHED DAD..ARABIC MATHEMATICAL STRETCHED DOTLESS BEH +1EE7E;N # Lo ARABIC MATHEMATICAL STRETCHED DOTLESS FEH +1EE80..1EE89;N # Lo [10] ARABIC MATHEMATICAL LOOPED ALEF..ARABIC MATHEMATICAL LOOPED YEH +1EE8B..1EE9B;N # Lo [17] ARABIC MATHEMATICAL LOOPED LAM..ARABIC MATHEMATICAL LOOPED GHAIN +1EEA1..1EEA3;N # Lo [3] ARABIC MATHEMATICAL DOUBLE-STRUCK BEH..ARABIC MATHEMATICAL DOUBLE-STRUCK DAL +1EEA5..1EEA9;N # Lo [5] ARABIC MATHEMATICAL DOUBLE-STRUCK WAW..ARABIC MATHEMATICAL DOUBLE-STRUCK YEH +1EEAB..1EEBB;N # Lo [17] ARABIC MATHEMATICAL DOUBLE-STRUCK LAM..ARABIC MATHEMATICAL DOUBLE-STRUCK GHAIN +1EEF0..1EEF1;N # Sm [2] ARABIC MATHEMATICAL OPERATOR MEEM WITH HAH WITH TATWEEL..ARABIC MATHEMATICAL OPERATOR HAH WITH DAL +1F000..1F003;N # So [4] MAHJONG TILE EAST WIND..MAHJONG TILE NORTH WIND +1F004;W # So MAHJONG TILE RED DRAGON +1F005..1F02B;N # So [39] MAHJONG TILE GREEN DRAGON..MAHJONG TILE BACK +1F030..1F093;N # So [100] DOMINO TILE HORIZONTAL BACK..DOMINO TILE VERTICAL-06-06 +1F0A0..1F0AE;N # So [15] PLAYING CARD BACK..PLAYING CARD KING OF SPADES +1F0B1..1F0BF;N # So [15] PLAYING CARD ACE OF HEARTS..PLAYING CARD RED JOKER +1F0C1..1F0CE;N # So [14] PLAYING CARD ACE OF DIAMONDS..PLAYING CARD KING OF DIAMONDS +1F0CF;W # So PLAYING CARD BLACK JOKER +1F0D1..1F0F5;N # So [37] PLAYING CARD ACE OF CLUBS..PLAYING CARD TRUMP-21 +1F100..1F10A;A # No [11] DIGIT ZERO FULL STOP..DIGIT NINE COMMA +1F10B..1F10C;N # No [2] DINGBAT CIRCLED SANS-SERIF DIGIT ZERO..DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT ZERO +1F10D..1F10F;N # So [3] CIRCLED ZERO WITH SLASH..CIRCLED DOLLAR SIGN WITH OVERLAID BACKSLASH +1F110..1F12D;A # So [30] PARENTHESIZED LATIN CAPITAL LETTER A..CIRCLED CD +1F12E..1F12F;N # So [2] CIRCLED WZ..COPYLEFT SYMBOL +1F130..1F169;A # So [58] SQUARED LATIN CAPITAL LETTER A..NEGATIVE CIRCLED LATIN CAPITAL LETTER Z +1F16A..1F16F;N # So [6] RAISED MC SIGN..CIRCLED HUMAN FIGURE +1F170..1F18D;A # So [30] NEGATIVE SQUARED LATIN CAPITAL LETTER A..NEGATIVE SQUARED SA +1F18E;W # So NEGATIVE SQUARED AB +1F18F..1F190;A # So [2] NEGATIVE SQUARED WC..SQUARE DJ +1F191..1F19A;W # So [10] SQUARED CL..SQUARED VS +1F19B..1F1AC;A # So [18] SQUARED THREE D..SQUARED VOD +1F1AD;N # So MASK WORK SYMBOL +1F1E6..1F1FF;N # So [26] REGIONAL INDICATOR SYMBOL LETTER A..REGIONAL INDICATOR SYMBOL LETTER Z +1F200..1F202;W # So [3] SQUARE HIRAGANA HOKA..SQUARED KATAKANA SA +1F210..1F23B;W # So [44] SQUARED CJK UNIFIED IDEOGRAPH-624B..SQUARED CJK UNIFIED IDEOGRAPH-914D +1F240..1F248;W # So [9] TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-672C..TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-6557 +1F250..1F251;W # So [2] CIRCLED IDEOGRAPH ADVANTAGE..CIRCLED IDEOGRAPH ACCEPT +1F260..1F265;W # So [6] ROUNDED SYMBOL FOR FU..ROUNDED SYMBOL FOR CAI +1F300..1F320;W # So [33] CYCLONE..SHOOTING STAR +1F321..1F32C;N # So [12] THERMOMETER..WIND BLOWING FACE +1F32D..1F335;W # So [9] HOT DOG..CACTUS +1F336;N # So HOT PEPPER +1F337..1F37C;W # So [70] TULIP..BABY BOTTLE +1F37D;N # So FORK AND KNIFE WITH PLATE +1F37E..1F393;W # So [22] BOTTLE WITH POPPING CORK..GRADUATION CAP +1F394..1F39F;N # So [12] HEART WITH TIP ON THE LEFT..ADMISSION TICKETS +1F3A0..1F3CA;W # So [43] CAROUSEL HORSE..SWIMMER +1F3CB..1F3CE;N # So [4] WEIGHT LIFTER..RACING CAR +1F3CF..1F3D3;W # So [5] CRICKET BAT AND BALL..TABLE TENNIS PADDLE AND BALL +1F3D4..1F3DF;N # So [12] SNOW CAPPED MOUNTAIN..STADIUM +1F3E0..1F3F0;W # So [17] HOUSE BUILDING..EUROPEAN CASTLE +1F3F1..1F3F3;N # So [3] WHITE PENNANT..WAVING WHITE FLAG +1F3F4;W # So WAVING BLACK FLAG +1F3F5..1F3F7;N # So [3] ROSETTE..LABEL +1F3F8..1F3FA;W # So [3] BADMINTON RACQUET AND SHUTTLECOCK..AMPHORA +1F3FB..1F3FF;W # Sk [5] EMOJI MODIFIER FITZPATRICK TYPE-1-2..EMOJI MODIFIER FITZPATRICK TYPE-6 +1F400..1F43E;W # So [63] RAT..PAW PRINTS +1F43F;N # So CHIPMUNK +1F440;W # So EYES +1F441;N # So EYE +1F442..1F4FC;W # So [187] EAR..VIDEOCASSETTE +1F4FD..1F4FE;N # So [2] FILM PROJECTOR..PORTABLE STEREO +1F4FF..1F53D;W # So [63] PRAYER BEADS..DOWN-POINTING SMALL RED TRIANGLE +1F53E..1F54A;N # So [13] LOWER RIGHT SHADOWED WHITE CIRCLE..DOVE OF PEACE +1F54B..1F54E;W # So [4] KAABA..MENORAH WITH NINE BRANCHES +1F54F;N # So BOWL OF HYGIEIA +1F550..1F567;W # So [24] CLOCK FACE ONE OCLOCK..CLOCK FACE TWELVE-THIRTY +1F568..1F579;N # So [18] RIGHT SPEAKER..JOYSTICK +1F57A;W # So MAN DANCING +1F57B..1F594;N # So [26] LEFT HAND TELEPHONE RECEIVER..REVERSED VICTORY HAND +1F595..1F596;W # So [2] REVERSED HAND WITH MIDDLE FINGER EXTENDED..RAISED HAND WITH PART BETWEEN MIDDLE AND RING FINGERS +1F597..1F5A3;N # So [13] WHITE DOWN POINTING LEFT HAND INDEX..BLACK DOWN POINTING BACKHAND INDEX +1F5A4;W # So BLACK HEART +1F5A5..1F5FA;N # So [86] DESKTOP COMPUTER..WORLD MAP +1F5FB..1F5FF;W # So [5] MOUNT FUJI..MOYAI +1F600..1F64F;W # So [80] GRINNING FACE..PERSON WITH FOLDED HANDS +1F650..1F67F;N # So [48] NORTH WEST POINTING LEAF..REVERSE CHECKER BOARD +1F680..1F6C5;W # So [70] ROCKET..LEFT LUGGAGE +1F6C6..1F6CB;N # So [6] TRIANGLE WITH ROUNDED CORNERS..COUCH AND LAMP +1F6CC;W # So SLEEPING ACCOMMODATION +1F6CD..1F6CF;N # So [3] SHOPPING BAGS..BED +1F6D0..1F6D2;W # So [3] PLACE OF WORSHIP..SHOPPING TROLLEY +1F6D3..1F6D4;N # So [2] STUPA..PAGODA +1F6D5..1F6D7;W # So [3] HINDU TEMPLE..ELEVATOR +1F6DC..1F6DF;W # So [4] WIRELESS..RING BUOY +1F6E0..1F6EA;N # So [11] HAMMER AND WRENCH..NORTHEAST-POINTING AIRPLANE +1F6EB..1F6EC;W # So [2] AIRPLANE DEPARTURE..AIRPLANE ARRIVING +1F6F0..1F6F3;N # So [4] SATELLITE..PASSENGER SHIP +1F6F4..1F6FC;W # So [9] SCOOTER..ROLLER SKATE +1F700..1F776;N # So [119] ALCHEMICAL SYMBOL FOR QUINTESSENCE..LUNAR ECLIPSE +1F77B..1F77F;N # So [5] HAUMEA..ORCUS +1F780..1F7D9;N # So [90] BLACK LEFT-POINTING ISOSCELES RIGHT TRIANGLE..NINE POINTED WHITE STAR +1F7E0..1F7EB;W # So [12] LARGE ORANGE CIRCLE..LARGE BROWN SQUARE +1F7F0;W # So HEAVY EQUALS SIGN +1F800..1F80B;N # So [12] LEFTWARDS ARROW WITH SMALL TRIANGLE ARROWHEAD..DOWNWARDS ARROW WITH LARGE TRIANGLE ARROWHEAD +1F810..1F847;N # So [56] LEFTWARDS ARROW WITH SMALL EQUILATERAL ARROWHEAD..DOWNWARDS HEAVY ARROW +1F850..1F859;N # So [10] LEFTWARDS SANS-SERIF ARROW..UP DOWN SANS-SERIF ARROW +1F860..1F887;N # So [40] WIDE-HEADED LEFTWARDS LIGHT BARB ARROW..WIDE-HEADED SOUTH WEST VERY HEAVY BARB ARROW +1F890..1F8AD;N # So [30] LEFTWARDS TRIANGLE ARROWHEAD..WHITE ARROW SHAFT WIDTH TWO THIRDS +1F8B0..1F8B1;N # So [2] ARROW POINTING UPWARDS THEN NORTH WEST..ARROW POINTING RIGHTWARDS THEN CURVING SOUTH WEST +1F900..1F90B;N # So [12] CIRCLED CROSS FORMEE WITH FOUR DOTS..DOWNWARD FACING NOTCHED HOOK WITH DOT +1F90C..1F93A;W # So [47] PINCHED FINGERS..FENCER +1F93B;N # So MODERN PENTATHLON +1F93C..1F945;W # So [10] WRESTLERS..GOAL NET +1F946;N # So RIFLE +1F947..1F9FF;W # So [185] FIRST PLACE MEDAL..NAZAR AMULET +1FA00..1FA53;N # So [84] NEUTRAL CHESS KING..BLACK CHESS KNIGHT-BISHOP +1FA60..1FA6D;N # So [14] XIANGQI RED GENERAL..XIANGQI BLACK SOLDIER +1FA70..1FA7C;W # So [13] BALLET SHOES..CRUTCH +1FA80..1FA88;W # So [9] YO-YO..FLUTE +1FA90..1FABD;W # So [46] RINGED PLANET..WING +1FABF..1FAC5;W # So [7] GOOSE..PERSON WITH CROWN +1FACE..1FADB;W # So [14] MOOSE..PEA POD +1FAE0..1FAE8;W # So [9] MELTING FACE..SHAKING FACE +1FAF0..1FAF8;W # So [9] HAND WITH INDEX FINGER AND THUMB CROSSED..RIGHTWARDS PUSHING HAND +1FB00..1FB92;N # So [147] BLOCK SEXTANT-1..UPPER HALF INVERSE MEDIUM SHADE AND LOWER HALF BLOCK +1FB94..1FBCA;N # So [55] LEFT HALF INVERSE MEDIUM SHADE AND RIGHT HALF BLOCK..WHITE UP-POINTING CHEVRON +1FBF0..1FBF9;N # Nd [10] SEGMENTED DIGIT ZERO..SEGMENTED DIGIT NINE +20000..2A6DF;W # Lo [42720] CJK UNIFIED IDEOGRAPH-20000..CJK UNIFIED IDEOGRAPH-2A6DF +2A6E0..2A6FF;W # Cn [32] <reserved-2A6E0>..<reserved-2A6FF> +2A700..2B739;W # Lo [4154] CJK UNIFIED IDEOGRAPH-2A700..CJK UNIFIED IDEOGRAPH-2B739 +2B73A..2B73F;W # Cn [6] <reserved-2B73A>..<reserved-2B73F> +2B740..2B81D;W # Lo [222] CJK UNIFIED IDEOGRAPH-2B740..CJK UNIFIED IDEOGRAPH-2B81D +2B81E..2B81F;W # Cn [2] <reserved-2B81E>..<reserved-2B81F> +2B820..2CEA1;W # Lo [5762] CJK UNIFIED IDEOGRAPH-2B820..CJK UNIFIED IDEOGRAPH-2CEA1 +2CEA2..2CEAF;W # Cn [14] <reserved-2CEA2>..<reserved-2CEAF> +2CEB0..2EBE0;W # Lo [7473] CJK UNIFIED IDEOGRAPH-2CEB0..CJK UNIFIED IDEOGRAPH-2EBE0 +2EBE1..2F7FF;W # Cn [3103] <reserved-2EBE1>..<reserved-2F7FF> +2F800..2FA1D;W # Lo [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D +2FA1E..2FA1F;W # Cn [2] <reserved-2FA1E>..<reserved-2FA1F> +2FA20..2FFFD;W # Cn [1502] <reserved-2FA20>..<reserved-2FFFD> +30000..3134A;W # Lo [4939] CJK UNIFIED IDEOGRAPH-30000..CJK UNIFIED IDEOGRAPH-3134A +3134B..3134F;W # Cn [5] <reserved-3134B>..<reserved-3134F> +31350..323AF;W # Lo [4192] CJK UNIFIED IDEOGRAPH-31350..CJK UNIFIED IDEOGRAPH-323AF +323B0..3FFFD;W # Cn [56398] <reserved-323B0>..<reserved-3FFFD> +E0001;N # Cf LANGUAGE TAG +E0020..E007F;N # Cf [96] TAG SPACE..CANCEL TAG +E0100..E01EF;A # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256 +F0000..FFFFD;A # Co [65534] <private-use-F0000>..<private-use-FFFFD> +100000..10FFFD;A # Co [65534] <private-use-100000>..<private-use-10FFFD> + +# EOF diff --git a/lib/stdlib/uc_spec/GraphemeBreakProperty.txt b/lib/stdlib/uc_spec/GraphemeBreakProperty.txt index dd2569064a..a12b5eef1e 100644 --- a/lib/stdlib/uc_spec/GraphemeBreakProperty.txt +++ b/lib/stdlib/uc_spec/GraphemeBreakProperty.txt @@ -1,11 +1,11 @@ -# GraphemeBreakProperty-14.0.0.txt -# Date: 2021-08-12, 23:13:02 GMT -# ยฉ 2021 Unicodeยฎ, Inc. +# GraphemeBreakProperty-15.0.0.txt +# Date: 2022-04-27, 17:07:38 GMT +# ยฉ 2022 Unicodeยฎ, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. -# For terms of use, see http://www.unicode.org/terms_of_use.html +# For terms of use, see https://www.unicode.org/terms_of_use.html # # Unicode Character Database -# For documentation, see http://www.unicode.org/reports/tr44/ +# For documentation, see https://www.unicode.org/reports/tr44/ # ================================================ @@ -32,8 +32,9 @@ 11A3A ; Prepend # Lo ZANABAZAR SQUARE CLUSTER-INITIAL LETTER RA 11A84..11A89 ; Prepend # Lo [6] SOYOMBO SIGN JIHVAMULIYA..SOYOMBO CLUSTER-INITIAL LETTER SA 11D46 ; Prepend # Lo MASARAM GONDI REPHA +11F02 ; Prepend # Lo KAWI SIGN REPHA -# Total code points: 26 +# Total code points: 27 # ================================================ @@ -67,7 +68,7 @@ FEFF ; Control # Cf ZERO WIDTH NO-BREAK SPACE FFF0..FFF8 ; Control # Cn [9] <reserved-FFF0>..<reserved-FFF8> FFF9..FFFB ; Control # Cf [3] INTERLINEAR ANNOTATION ANCHOR..INTERLINEAR ANNOTATION TERMINATOR -13430..13438 ; Control # Cf [9] EGYPTIAN HIEROGLYPH VERTICAL JOINER..EGYPTIAN HIEROGLYPH END SEGMENT +13430..1343F ; Control # Cf [16] EGYPTIAN HIEROGLYPH VERTICAL JOINER..EGYPTIAN HIEROGLYPH END WALLED ENCLOSURE 1BCA0..1BCA3 ; Control # Cf [4] SHORTHAND FORMAT LETTER OVERLAP..SHORTHAND FORMAT UP STEP 1D173..1D17A ; Control # Cf [8] MUSICAL SYMBOL BEGIN BEAM..MUSICAL SYMBOL END PHRASE E0000 ; Control # Cn <reserved-E0000> @@ -76,7 +77,7 @@ E0002..E001F ; Control # Cn [30] <reserved-E0002>..<reserved-E001F> E0080..E00FF ; Control # Cn [128] <reserved-E0080>..<reserved-E00FF> E01F0..E0FFF ; Control # Cn [3600] <reserved-E01F0>..<reserved-E0FFF> -# Total code points: 3886 +# Total code points: 3893 # ================================================ @@ -185,7 +186,7 @@ E01F0..E0FFF ; Control # Cn [3600] <reserved-E01F0>..<reserved-E0FFF> 0E47..0E4E ; Extend # Mn [8] THAI CHARACTER MAITAIKHU..THAI CHARACTER YAMAKKAN 0EB1 ; Extend # Mn LAO VOWEL SIGN MAI KAN 0EB4..0EBC ; Extend # Mn [9] LAO VOWEL SIGN I..LAO SEMIVOWEL SIGN LO -0EC8..0ECD ; Extend # Mn [6] LAO TONE MAI EK..LAO NIGGAHITA +0EC8..0ECE ; Extend # Mn [7] LAO TONE MAI EK..LAO YAMAKKAN 0F18..0F19 ; Extend # Mn [2] TIBETAN ASTROLOGICAL SIGN -KHYUD PA..TIBETAN ASTROLOGICAL SIGN SDONG TSHUGS 0F35 ; Extend # Mn TIBETAN MARK NGAS BZUNG NYI ZLA 0F37 ; Extend # Mn TIBETAN MARK NGAS BZUNG SGOR RTAGS @@ -324,6 +325,7 @@ FF9E..FF9F ; Extend # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDT 10AE5..10AE6 ; Extend # Mn [2] MANICHAEAN ABBREVIATION MARK ABOVE..MANICHAEAN ABBREVIATION MARK BELOW 10D24..10D27 ; Extend # Mn [4] HANIFI ROHINGYA SIGN HARBAHAY..HANIFI ROHINGYA SIGN TASSI 10EAB..10EAC ; Extend # Mn [2] YEZIDI COMBINING HAMZA MARK..YEZIDI COMBINING MADDA MARK +10EFD..10EFF ; Extend # Mn [3] ARABIC SMALL LOW WORD SAKTA..ARABIC SMALL LOW WORD MADDA 10F46..10F50 ; Extend # Mn [11] SOGDIAN COMBINING DOT BELOW..SOGDIAN COMBINING STROKE BELOW 10F82..10F85 ; Extend # Mn [4] OLD UYGHUR COMBINING DOT ABOVE..OLD UYGHUR COMBINING TWO DOTS BELOW 11001 ; Extend # Mn BRAHMI SIGN ANUSVARA @@ -346,6 +348,7 @@ FF9E..FF9F ; Extend # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDT 11234 ; Extend # Mn KHOJKI SIGN ANUSVARA 11236..11237 ; Extend # Mn [2] KHOJKI SIGN NUKTA..KHOJKI SIGN SHADDA 1123E ; Extend # Mn KHOJKI SIGN SUKUN +11241 ; Extend # Mn KHOJKI VOWEL SIGN VOCALIC R 112DF ; Extend # Mn KHUDAWADI SIGN ANUSVARA 112E3..112EA ; Extend # Mn [8] KHUDAWADI VOWEL SIGN U..KHUDAWADI SIGN VIRAMA 11300..11301 ; Extend # Mn [2] GRANTHA SIGN COMBINING ANUSVARA ABOVE..GRANTHA SIGN CANDRABINDU @@ -413,6 +416,12 @@ FF9E..FF9F ; Extend # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDT 11D95 ; Extend # Mn GUNJALA GONDI SIGN ANUSVARA 11D97 ; Extend # Mn GUNJALA GONDI VIRAMA 11EF3..11EF4 ; Extend # Mn [2] MAKASAR VOWEL SIGN I..MAKASAR VOWEL SIGN U +11F00..11F01 ; Extend # Mn [2] KAWI SIGN CANDRABINDU..KAWI SIGN ANUSVARA +11F36..11F3A ; Extend # Mn [5] KAWI VOWEL SIGN I..KAWI VOWEL SIGN VOCALIC R +11F40 ; Extend # Mn KAWI VOWEL SIGN EU +11F42 ; Extend # Mn KAWI CONJOINER +13440 ; Extend # Mn EGYPTIAN HIEROGLYPH MIRROR HORIZONTALLY +13447..13455 ; Extend # Mn [15] EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP START..EGYPTIAN HIEROGLYPH MODIFIER DAMAGED 16AF0..16AF4 ; Extend # Mn [5] BASSA VAH COMBINING HIGH TONE..BASSA VAH COMBINING HIGH-LOW TONE 16B30..16B36 ; Extend # Mn [7] PAHAWH HMONG MARK CIM TUB..PAHAWH HMONG MARK CIM TAUM 16F4F ; Extend # Mn MIAO SIGN CONSONANT MODIFIER BAR @@ -439,16 +448,18 @@ FF9E..FF9F ; Extend # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDT 1E01B..1E021 ; Extend # Mn [7] COMBINING GLAGOLITIC LETTER SHTA..COMBINING GLAGOLITIC LETTER YATI 1E023..1E024 ; Extend # Mn [2] COMBINING GLAGOLITIC LETTER YU..COMBINING GLAGOLITIC LETTER SMALL YUS 1E026..1E02A ; Extend # Mn [5] COMBINING GLAGOLITIC LETTER YO..COMBINING GLAGOLITIC LETTER FITA +1E08F ; Extend # Mn COMBINING CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I 1E130..1E136 ; Extend # Mn [7] NYIAKENG PUACHUE HMONG TONE-B..NYIAKENG PUACHUE HMONG TONE-D 1E2AE ; Extend # Mn TOTO SIGN RISING TONE 1E2EC..1E2EF ; Extend # Mn [4] WANCHO TONE TUP..WANCHO TONE KOINI +1E4EC..1E4EF ; Extend # Mn [4] NAG MUNDARI SIGN MUHOR..NAG MUNDARI SIGN SUTUH 1E8D0..1E8D6 ; Extend # Mn [7] MENDE KIKAKUI COMBINING NUMBER TEENS..MENDE KIKAKUI COMBINING NUMBER MILLIONS 1E944..1E94A ; Extend # Mn [7] ADLAM ALIF LENGTHENER..ADLAM NUKTA 1F3FB..1F3FF ; Extend # Sk [5] EMOJI MODIFIER FITZPATRICK TYPE-1-2..EMOJI MODIFIER FITZPATRICK TYPE-6 E0020..E007F ; Extend # Cf [96] TAG SPACE..CANCEL TAG E0100..E01EF ; Extend # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256 -# Total code points: 2095 +# Total code points: 2130 # ================================================ @@ -489,6 +500,7 @@ E0100..E01EF ; Extend # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256 0CC3..0CC4 ; SpacingMark # Mc [2] KANNADA VOWEL SIGN VOCALIC R..KANNADA VOWEL SIGN VOCALIC RR 0CC7..0CC8 ; SpacingMark # Mc [2] KANNADA VOWEL SIGN EE..KANNADA VOWEL SIGN AI 0CCA..0CCB ; SpacingMark # Mc [2] KANNADA VOWEL SIGN O..KANNADA VOWEL SIGN OO +0CF3 ; SpacingMark # Mc KANNADA SIGN COMBINING ANUSVARA ABOVE RIGHT 0D02..0D03 ; SpacingMark # Mc [2] MALAYALAM SIGN ANUSVARA..MALAYALAM SIGN VISARGA 0D3F..0D40 ; SpacingMark # Mc [2] MALAYALAM VOWEL SIGN I..MALAYALAM VOWEL SIGN II 0D46..0D48 ; SpacingMark # Mc [3] MALAYALAM VOWEL SIGN E..MALAYALAM VOWEL SIGN AI @@ -614,12 +626,16 @@ ABEC ; SpacingMark # Mc MEETEI MAYEK LUM IYEK 11D93..11D94 ; SpacingMark # Mc [2] GUNJALA GONDI VOWEL SIGN OO..GUNJALA GONDI VOWEL SIGN AU 11D96 ; SpacingMark # Mc GUNJALA GONDI SIGN VISARGA 11EF5..11EF6 ; SpacingMark # Mc [2] MAKASAR VOWEL SIGN E..MAKASAR VOWEL SIGN O +11F03 ; SpacingMark # Mc KAWI SIGN VISARGA +11F34..11F35 ; SpacingMark # Mc [2] KAWI VOWEL SIGN AA..KAWI VOWEL SIGN ALTERNATE AA +11F3E..11F3F ; SpacingMark # Mc [2] KAWI VOWEL SIGN E..KAWI VOWEL SIGN AI +11F41 ; SpacingMark # Mc KAWI SIGN KILLER 16F51..16F87 ; SpacingMark # Mc [55] MIAO SIGN ASPIRATION..MIAO VOWEL SIGN UI 16FF0..16FF1 ; SpacingMark # Mc [2] VIETNAMESE ALTERNATE READING MARK CA..VIETNAMESE ALTERNATE READING MARK NHAY 1D166 ; SpacingMark # Mc MUSICAL SYMBOL COMBINING SPRECHGESANG STEM 1D16D ; SpacingMark # Mc MUSICAL SYMBOL COMBINING AUGMENTATION DOT -# Total code points: 388 +# Total code points: 395 # ================================================ diff --git a/lib/stdlib/uc_spec/PropList.txt b/lib/stdlib/uc_spec/PropList.txt index 0a5a934682..b49d6460c1 100644 --- a/lib/stdlib/uc_spec/PropList.txt +++ b/lib/stdlib/uc_spec/PropList.txt @@ -1,11 +1,11 @@ -# PropList-14.0.0.txt -# Date: 2021-08-12, 23:13:05 GMT -# ยฉ 2021 Unicodeยฎ, Inc. +# PropList-15.0.0.txt +# Date: 2022-08-05, 22:17:16 GMT +# ยฉ 2022 Unicodeยฎ, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. -# For terms of use, see http://www.unicode.org/terms_of_use.html +# For terms of use, see https://www.unicode.org/terms_of_use.html # # Unicode Character Database -# For documentation, see http://www.unicode.org/reports/tr44/ +# For documentation, see https://www.unicode.org/reports/tr44/ # ================================================ @@ -215,6 +215,7 @@ FF64 ; Terminal_Punctuation # Po HALFWIDTH IDEOGRAPHIC COMMA 11C41..11C43 ; Terminal_Punctuation # Po [3] BHAIKSUKI DANDA..BHAIKSUKI WORD SEPARATOR 11C71 ; Terminal_Punctuation # Po MARCHEN MARK SHAD 11EF7..11EF8 ; Terminal_Punctuation # Po [2] MAKASAR PASSIMBANG..MAKASAR END OF SECTION +11F43..11F44 ; Terminal_Punctuation # Po [2] KAWI DANDA..KAWI DOUBLE DANDA 12470..12474 ; Terminal_Punctuation # Po [5] CUNEIFORM PUNCTUATION SIGN OLD ASSYRIAN WORD DIVIDER..CUNEIFORM PUNCTUATION SIGN DIAGONAL QUADCOLON 16A6E..16A6F ; Terminal_Punctuation # Po [2] MRO DANDA..MRO DOUBLE DANDA 16AF5 ; Terminal_Punctuation # Po BASSA VAH FULL STOP @@ -224,7 +225,7 @@ FF64 ; Terminal_Punctuation # Po HALFWIDTH IDEOGRAPHIC COMMA 1BC9F ; Terminal_Punctuation # Po DUPLOYAN PUNCTUATION CHINOOK FULL STOP 1DA87..1DA8A ; Terminal_Punctuation # Po [4] SIGNWRITING COMMA..SIGNWRITING COLON -# Total code points: 276 +# Total code points: 278 # ================================================ @@ -507,6 +508,7 @@ FF41..FF46 ; Hex_Digit # L& [6] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH L 0BD7 ; Other_Alphabetic # Mc TAMIL AU LENGTH MARK 0C00 ; Other_Alphabetic # Mn TELUGU SIGN COMBINING CANDRABINDU ABOVE 0C01..0C03 ; Other_Alphabetic # Mc [3] TELUGU SIGN CANDRABINDU..TELUGU SIGN VISARGA +0C04 ; Other_Alphabetic # Mn TELUGU SIGN COMBINING ANUSVARA ABOVE 0C3E..0C40 ; Other_Alphabetic # Mn [3] TELUGU VOWEL SIGN AA..TELUGU VOWEL SIGN II 0C41..0C44 ; Other_Alphabetic # Mc [4] TELUGU VOWEL SIGN U..TELUGU VOWEL SIGN VOCALIC RR 0C46..0C48 ; Other_Alphabetic # Mn [3] TELUGU VOWEL SIGN E..TELUGU VOWEL SIGN AI @@ -524,6 +526,7 @@ FF41..FF46 ; Hex_Digit # L& [6] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH L 0CCC ; Other_Alphabetic # Mn KANNADA VOWEL SIGN AU 0CD5..0CD6 ; Other_Alphabetic # Mc [2] KANNADA LENGTH MARK..KANNADA AI LENGTH MARK 0CE2..0CE3 ; Other_Alphabetic # Mn [2] KANNADA VOWEL SIGN VOCALIC L..KANNADA VOWEL SIGN VOCALIC LL +0CF3 ; Other_Alphabetic # Mc KANNADA SIGN COMBINING ANUSVARA ABOVE RIGHT 0D00..0D01 ; Other_Alphabetic # Mn [2] MALAYALAM SIGN COMBINING ANUSVARA ABOVE..MALAYALAM SIGN CANDRABINDU 0D02..0D03 ; Other_Alphabetic # Mc [2] MALAYALAM SIGN ANUSVARA..MALAYALAM SIGN VISARGA 0D3E..0D40 ; Other_Alphabetic # Mc [3] MALAYALAM VOWEL SIGN AA..MALAYALAM VOWEL SIGN II @@ -548,7 +551,7 @@ FF41..FF46 ; Hex_Digit # L& [6] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH L 0ECD ; Other_Alphabetic # Mn LAO NIGGAHITA 0F71..0F7E ; Other_Alphabetic # Mn [14] TIBETAN VOWEL SIGN AA..TIBETAN SIGN RJES SU NGA RO 0F7F ; Other_Alphabetic # Mc TIBETAN SIGN RNAM BCAD -0F80..0F81 ; Other_Alphabetic # Mn [2] TIBETAN VOWEL SIGN REVERSED I..TIBETAN VOWEL SIGN REVERSED II +0F80..0F83 ; Other_Alphabetic # Mn [4] TIBETAN VOWEL SIGN REVERSED I..TIBETAN SIGN SNA LDAN 0F8D..0F97 ; Other_Alphabetic # Mn [11] TIBETAN SUBJOINED SIGN LCE TSA CAN..TIBETAN SUBJOINED LETTER JA 0F99..0FBC ; Other_Alphabetic # Mn [36] TIBETAN SUBJOINED LETTER NYA..TIBETAN SUBJOINED LETTER FIXED-FORM RA 102B..102C ; Other_Alphabetic # Mc [2] MYANMAR VOWEL SIGN TALL AA..MYANMAR VOWEL SIGN AA @@ -692,6 +695,7 @@ FB1E ; Other_Alphabetic # Mn HEBREW POINT JUDEO-SPANISH VARIKA 11002 ; Other_Alphabetic # Mc BRAHMI SIGN VISARGA 11038..11045 ; Other_Alphabetic # Mn [14] BRAHMI VOWEL SIGN AA..BRAHMI VOWEL SIGN AU 11073..11074 ; Other_Alphabetic # Mn [2] BRAHMI VOWEL SIGN OLD TAMIL SHORT E..BRAHMI VOWEL SIGN OLD TAMIL SHORT O +11080..11081 ; Other_Alphabetic # Mn [2] KAITHI SIGN CANDRABINDU..KAITHI SIGN ANUSVARA 11082 ; Other_Alphabetic # Mc KAITHI SIGN VISARGA 110B0..110B2 ; Other_Alphabetic # Mc [3] KAITHI VOWEL SIGN AA..KAITHI VOWEL SIGN II 110B3..110B6 ; Other_Alphabetic # Mn [4] KAITHI VOWEL SIGN U..KAITHI VOWEL SIGN AI @@ -715,6 +719,7 @@ FB1E ; Other_Alphabetic # Mn HEBREW POINT JUDEO-SPANISH VARIKA 11234 ; Other_Alphabetic # Mn KHOJKI SIGN ANUSVARA 11237 ; Other_Alphabetic # Mn KHOJKI SIGN SHADDA 1123E ; Other_Alphabetic # Mn KHOJKI SIGN SUKUN +11241 ; Other_Alphabetic # Mn KHOJKI VOWEL SIGN VOCALIC R 112DF ; Other_Alphabetic # Mn KHUDAWADI SIGN ANUSVARA 112E0..112E2 ; Other_Alphabetic # Mc [3] KHUDAWADI VOWEL SIGN AA..KHUDAWADI VOWEL SIGN II 112E3..112E8 ; Other_Alphabetic # Mn [6] KHUDAWADI VOWEL SIGN U..KHUDAWADI VOWEL SIGN AU @@ -807,6 +812,12 @@ FB1E ; Other_Alphabetic # Mn HEBREW POINT JUDEO-SPANISH VARIKA 11D96 ; Other_Alphabetic # Mc GUNJALA GONDI SIGN VISARGA 11EF3..11EF4 ; Other_Alphabetic # Mn [2] MAKASAR VOWEL SIGN I..MAKASAR VOWEL SIGN U 11EF5..11EF6 ; Other_Alphabetic # Mc [2] MAKASAR VOWEL SIGN E..MAKASAR VOWEL SIGN O +11F00..11F01 ; Other_Alphabetic # Mn [2] KAWI SIGN CANDRABINDU..KAWI SIGN ANUSVARA +11F03 ; Other_Alphabetic # Mc KAWI SIGN VISARGA +11F34..11F35 ; Other_Alphabetic # Mc [2] KAWI VOWEL SIGN AA..KAWI VOWEL SIGN ALTERNATE AA +11F36..11F3A ; Other_Alphabetic # Mn [5] KAWI VOWEL SIGN I..KAWI VOWEL SIGN VOCALIC R +11F3E..11F3F ; Other_Alphabetic # Mc [2] KAWI VOWEL SIGN E..KAWI VOWEL SIGN AI +11F40 ; Other_Alphabetic # Mn KAWI VOWEL SIGN EU 16F4F ; Other_Alphabetic # Mn MIAO SIGN CONSONANT MODIFIER BAR 16F51..16F87 ; Other_Alphabetic # Mc [55] MIAO SIGN ASPIRATION..MIAO VOWEL SIGN UI 16F8F..16F92 ; Other_Alphabetic # Mn [4] MIAO TONE RIGHT..MIAO TONE BELOW @@ -817,12 +828,13 @@ FB1E ; Other_Alphabetic # Mn HEBREW POINT JUDEO-SPANISH VARIKA 1E01B..1E021 ; Other_Alphabetic # Mn [7] COMBINING GLAGOLITIC LETTER SHTA..COMBINING GLAGOLITIC LETTER YATI 1E023..1E024 ; Other_Alphabetic # Mn [2] COMBINING GLAGOLITIC LETTER YU..COMBINING GLAGOLITIC LETTER SMALL YUS 1E026..1E02A ; Other_Alphabetic # Mn [5] COMBINING GLAGOLITIC LETTER YO..COMBINING GLAGOLITIC LETTER FITA +1E08F ; Other_Alphabetic # Mn COMBINING CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I 1E947 ; Other_Alphabetic # Mn ADLAM HAMZA 1F130..1F149 ; Other_Alphabetic # So [26] SQUARED LATIN CAPITAL LETTER A..SQUARED LATIN CAPITAL LETTER Z 1F150..1F169 ; Other_Alphabetic # So [26] NEGATIVE CIRCLED LATIN CAPITAL LETTER A..NEGATIVE CIRCLED LATIN CAPITAL LETTER Z 1F170..1F189 ; Other_Alphabetic # So [26] NEGATIVE SQUARED LATIN CAPITAL LETTER A..NEGATIVE SQUARED LATIN CAPITAL LETTER Z -# Total code points: 1404 +# Total code points: 1425 # ================================================ @@ -840,14 +852,15 @@ FA70..FAD9 ; Ideographic # Lo [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COM 18D00..18D08 ; Ideographic # Lo [9] TANGUT IDEOGRAPH-18D00..TANGUT IDEOGRAPH-18D08 1B170..1B2FB ; Ideographic # Lo [396] NUSHU CHARACTER-1B170..NUSHU CHARACTER-1B2FB 20000..2A6DF ; Ideographic # Lo [42720] CJK UNIFIED IDEOGRAPH-20000..CJK UNIFIED IDEOGRAPH-2A6DF -2A700..2B738 ; Ideographic # Lo [4153] CJK UNIFIED IDEOGRAPH-2A700..CJK UNIFIED IDEOGRAPH-2B738 +2A700..2B739 ; Ideographic # Lo [4154] CJK UNIFIED IDEOGRAPH-2A700..CJK UNIFIED IDEOGRAPH-2B739 2B740..2B81D ; Ideographic # Lo [222] CJK UNIFIED IDEOGRAPH-2B740..CJK UNIFIED IDEOGRAPH-2B81D 2B820..2CEA1 ; Ideographic # Lo [5762] CJK UNIFIED IDEOGRAPH-2B820..CJK UNIFIED IDEOGRAPH-2CEA1 2CEB0..2EBE0 ; Ideographic # Lo [7473] CJK UNIFIED IDEOGRAPH-2CEB0..CJK UNIFIED IDEOGRAPH-2EBE0 2F800..2FA1D ; Ideographic # Lo [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D 30000..3134A ; Ideographic # Lo [4939] CJK UNIFIED IDEOGRAPH-30000..CJK UNIFIED IDEOGRAPH-3134A +31350..323AF ; Ideographic # Lo [4192] CJK UNIFIED IDEOGRAPH-31350..CJK UNIFIED IDEOGRAPH-323AF -# Total code points: 101661 +# Total code points: 105854 # ================================================ @@ -1028,6 +1041,7 @@ FFE3 ; Diacritic # Sk FULLWIDTH MACRON 10AE5..10AE6 ; Diacritic # Mn [2] MANICHAEAN ABBREVIATION MARK ABOVE..MANICHAEAN ABBREVIATION MARK BELOW 10D22..10D23 ; Diacritic # Lo [2] HANIFI ROHINGYA MARK SAKIN..HANIFI ROHINGYA MARK NA KHONNA 10D24..10D27 ; Diacritic # Mn [4] HANIFI ROHINGYA SIGN HARBAHAY..HANIFI ROHINGYA SIGN TASSI +10EFD..10EFF ; Diacritic # Mn [3] ARABIC SMALL LOW WORD SAKTA..ARABIC SMALL LOW WORD MADDA 10F46..10F50 ; Diacritic # Mn [11] SOGDIAN COMBINING DOT BELOW..SOGDIAN COMBINING STROKE BELOW 10F82..10F85 ; Diacritic # Mn [4] OLD UYGHUR COMBINING DOT ABOVE..OLD UYGHUR COMBINING TWO DOTS BELOW 11046 ; Diacritic # Mn BRAHMI VIRAMA @@ -1064,6 +1078,7 @@ FFE3 ; Diacritic # Sk FULLWIDTH MACRON 11D42 ; Diacritic # Mn MASARAM GONDI SIGN NUKTA 11D44..11D45 ; Diacritic # Mn [2] MASARAM GONDI SIGN HALANTA..MASARAM GONDI VIRAMA 11D97 ; Diacritic # Mn GUNJALA GONDI VIRAMA +13447..13455 ; Diacritic # Mn [15] EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP START..EGYPTIAN HIEROGLYPH MODIFIER DAMAGED 16AF0..16AF4 ; Diacritic # Mn [5] BASSA VAH COMBINING HIGH TONE..BASSA VAH COMBINING HIGH-LOW TONE 16B30..16B36 ; Diacritic # Mn [7] PAHAWH HMONG MARK CIM TUB..PAHAWH HMONG MARK CIM TAUM 16F8F..16F92 ; Diacritic # Mn [4] MIAO TONE RIGHT..MIAO TONE BELOW @@ -1079,6 +1094,7 @@ FFE3 ; Diacritic # Sk FULLWIDTH MACRON 1D17B..1D182 ; Diacritic # Mn [8] MUSICAL SYMBOL COMBINING ACCENT..MUSICAL SYMBOL COMBINING LOURE 1D185..1D18B ; Diacritic # Mn [7] MUSICAL SYMBOL COMBINING DOIT..MUSICAL SYMBOL COMBINING TRIPLE TONGUE 1D1AA..1D1AD ; Diacritic # Mn [4] MUSICAL SYMBOL COMBINING DOWN BOW..MUSICAL SYMBOL COMBINING SNAP PIZZICATO +1E030..1E06D ; Diacritic # Lm [62] MODIFIER LETTER CYRILLIC SMALL A..MODIFIER LETTER CYRILLIC SMALL STRAIGHT U WITH STROKE 1E130..1E136 ; Diacritic # Mn [7] NYIAKENG PUACHUE HMONG TONE-B..NYIAKENG PUACHUE HMONG TONE-D 1E2AE ; Diacritic # Mn TOTO SIGN RISING TONE 1E2EC..1E2EF ; Diacritic # Mn [4] WANCHO TONE TUP..WANCHO TONE KOINI @@ -1086,7 +1102,7 @@ FFE3 ; Diacritic # Sk FULLWIDTH MACRON 1E944..1E946 ; Diacritic # Mn [3] ADLAM ALIF LENGTHENER..ADLAM GEMINATION MARK 1E948..1E94A ; Diacritic # Mn [3] ADLAM CONSONANT MODIFIER..ADLAM NUKTA -# Total code points: 1064 +# Total code points: 1144 # ================================================ @@ -1135,6 +1151,7 @@ FF70 ; Extender # Lm HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND 02E0..02E4 ; Other_Lowercase # Lm [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP 0345 ; Other_Lowercase # Mn COMBINING GREEK YPOGEGRAMMENI 037A ; Other_Lowercase # Lm GREEK YPOGEGRAMMENI +10FC ; Other_Lowercase # Lm MODIFIER LETTER GEORGIAN NAR 1D2C..1D6A ; Other_Lowercase # Lm [63] MODIFIER LETTER CAPITAL A..GREEK SUBSCRIPT SMALL LETTER CHI 1D78 ; Other_Lowercase # Lm MODIFIER LETTER CYRILLIC EN 1D9B..1DBF ; Other_Lowercase # Lm [37] MODIFIER LETTER SMALL TURNED ALPHA..MODIFIER LETTER SMALL THETA @@ -1146,14 +1163,17 @@ FF70 ; Extender # Lm HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND 2C7C..2C7D ; Other_Lowercase # Lm [2] LATIN SUBSCRIPT SMALL LETTER J..MODIFIER LETTER CAPITAL V A69C..A69D ; Other_Lowercase # Lm [2] MODIFIER LETTER CYRILLIC HARD SIGN..MODIFIER LETTER CYRILLIC SOFT SIGN A770 ; Other_Lowercase # Lm MODIFIER LETTER US +A7F2..A7F4 ; Other_Lowercase # Lm [3] MODIFIER LETTER CAPITAL C..MODIFIER LETTER CAPITAL Q A7F8..A7F9 ; Other_Lowercase # Lm [2] MODIFIER LETTER CAPITAL H WITH STROKE..MODIFIER LETTER SMALL LIGATURE OE AB5C..AB5F ; Other_Lowercase # Lm [4] MODIFIER LETTER SMALL HENG..MODIFIER LETTER SMALL U WITH LEFT HOOK +AB69 ; Other_Lowercase # Lm MODIFIER LETTER SMALL TURNED W 10780 ; Other_Lowercase # Lm MODIFIER LETTER SMALL CAPITAL AA 10783..10785 ; Other_Lowercase # Lm [3] MODIFIER LETTER SMALL AE..MODIFIER LETTER SMALL B WITH HOOK 10787..107B0 ; Other_Lowercase # Lm [42] MODIFIER LETTER SMALL DZ DIGRAPH..MODIFIER LETTER SMALL V WITH RIGHT HOOK 107B2..107BA ; Other_Lowercase # Lm [9] MODIFIER LETTER SMALL CAPITAL Y..MODIFIER LETTER SMALL S WITH CURL +1E030..1E06D ; Other_Lowercase # Lm [62] MODIFIER LETTER CYRILLIC SMALL A..MODIFIER LETTER CYRILLIC SMALL STRAIGHT U WITH STROKE -# Total code points: 244 +# Total code points: 311 # ================================================ @@ -1251,13 +1271,14 @@ FA21 ; Unified_Ideograph # Lo CJK COMPATIBILITY IDEOGRAPH-FA21 FA23..FA24 ; Unified_Ideograph # Lo [2] CJK COMPATIBILITY IDEOGRAPH-FA23..CJK COMPATIBILITY IDEOGRAPH-FA24 FA27..FA29 ; Unified_Ideograph # Lo [3] CJK COMPATIBILITY IDEOGRAPH-FA27..CJK COMPATIBILITY IDEOGRAPH-FA29 20000..2A6DF ; Unified_Ideograph # Lo [42720] CJK UNIFIED IDEOGRAPH-20000..CJK UNIFIED IDEOGRAPH-2A6DF -2A700..2B738 ; Unified_Ideograph # Lo [4153] CJK UNIFIED IDEOGRAPH-2A700..CJK UNIFIED IDEOGRAPH-2B738 +2A700..2B739 ; Unified_Ideograph # Lo [4154] CJK UNIFIED IDEOGRAPH-2A700..CJK UNIFIED IDEOGRAPH-2B739 2B740..2B81D ; Unified_Ideograph # Lo [222] CJK UNIFIED IDEOGRAPH-2B740..CJK UNIFIED IDEOGRAPH-2B81D 2B820..2CEA1 ; Unified_Ideograph # Lo [5762] CJK UNIFIED IDEOGRAPH-2B820..CJK UNIFIED IDEOGRAPH-2CEA1 2CEB0..2EBE0 ; Unified_Ideograph # Lo [7473] CJK UNIFIED IDEOGRAPH-2CEB0..CJK UNIFIED IDEOGRAPH-2EBE0 30000..3134A ; Unified_Ideograph # Lo [4939] CJK UNIFIED IDEOGRAPH-30000..CJK UNIFIED IDEOGRAPH-3134A +31350..323AF ; Unified_Ideograph # Lo [4192] CJK UNIFIED IDEOGRAPH-31350..CJK UNIFIED IDEOGRAPH-323AF -# Total code points: 92865 +# Total code points: 97058 # ================================================ @@ -1323,8 +1344,10 @@ E0001 ; Deprecated # Cf LANGUAGE TAG 1D65E..1D65F ; Soft_Dotted # L& [2] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL I..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL J 1D692..1D693 ; Soft_Dotted # L& [2] MATHEMATICAL MONOSPACE SMALL I..MATHEMATICAL MONOSPACE SMALL J 1DF1A ; Soft_Dotted # L& LATIN SMALL LETTER I WITH STROKE AND RETROFLEX HOOK +1E04C..1E04D ; Soft_Dotted # Lm [2] MODIFIER LETTER CYRILLIC SMALL BYELORUSSIAN-UKRAINIAN I..MODIFIER LETTER CYRILLIC SMALL JE +1E068 ; Soft_Dotted # Lm CYRILLIC SUBSCRIPT SMALL LETTER BYELORUSSIAN-UKRAINIAN I -# Total code points: 47 +# Total code points: 50 # ================================================ @@ -1430,6 +1453,7 @@ FF61 ; Sentence_Terminal # Po HALFWIDTH IDEOGRAPHIC FULL STOP 11A9B..11A9C ; Sentence_Terminal # Po [2] SOYOMBO MARK SHAD..SOYOMBO MARK DOUBLE SHAD 11C41..11C42 ; Sentence_Terminal # Po [2] BHAIKSUKI DANDA..BHAIKSUKI DOUBLE DANDA 11EF7..11EF8 ; Sentence_Terminal # Po [2] MAKASAR PASSIMBANG..MAKASAR END OF SECTION +11F43..11F44 ; Sentence_Terminal # Po [2] KAWI DANDA..KAWI DOUBLE DANDA 16A6E..16A6F ; Sentence_Terminal # Po [2] MRO DANDA..MRO DOUBLE DANDA 16AF5 ; Sentence_Terminal # Po BASSA VAH FULL STOP 16B37..16B38 ; Sentence_Terminal # Po [2] PAHAWH HMONG SIGN VOS THOM..PAHAWH HMONG SIGN VOS TSHAB CEEB @@ -1438,7 +1462,7 @@ FF61 ; Sentence_Terminal # Po HALFWIDTH IDEOGRAPHIC FULL STOP 1BC9F ; Sentence_Terminal # Po DUPLOYAN PUNCTUATION CHINOOK FULL STOP 1DA88 ; Sentence_Terminal # Po SIGNWRITING FULL STOP -# Total code points: 152 +# Total code points: 154 # ================================================ diff --git a/lib/stdlib/uc_spec/README-UPDATE.txt b/lib/stdlib/uc_spec/README-UPDATE.txt index 8af7b54a07..0bf6117611 100644 --- a/lib/stdlib/uc_spec/README-UPDATE.txt +++ b/lib/stdlib/uc_spec/README-UPDATE.txt @@ -1,7 +1,7 @@ -Unicode 14.0.0 was updated from: -- https://www.unicode.org/Public/14.0.0/ucd/ -- https://www.unicode.org/Public/14.0.0/ucd/auxiliary/ -- https://www.unicode.org/Public/14.0.0/ucd/emoji/ +Unicode 15.0.0 was updated from: +- https://www.unicode.org/Public/15.0.0/ucd/ +- https://www.unicode.org/Public/15.0.0/ucd/auxiliary/ +- https://www.unicode.org/Public/15.0.0/ucd/emoji/ When updating the Unicode version please follow these steps: @@ -17,6 +17,7 @@ No subfolder should be created. - UnicodeData.txt - auxiliary/GraphemeBreakProperty.txt - emoji/emoji-data.txt + - EastAsianWidth.txt 2. Copy the following test files to lib/stdlib/test/unicode_util_SUITE_data/ replacing existing ones. No subfolder should be created. @@ -35,7 +36,12 @@ this very same file (lib/stdlib/uc_spec/README-UPDATE.txt). Remember to update these instructions if a new file is added or any other change is required for future version updates. -6. Run the test for the Unicode suite from the OTP repository root dir. +6. Check if the test file needs to be updated: + (cd $ERL_TOP/lib/stdlib/uc_spec; escript gen_unicode_mod.escript update_tests) + If ../test/unicode_util_SUITE_data/unicode_table.bin is updated include it in + the commit. + +7. Run the test for the Unicode suite from the OTP repository root dir. $ export ERL_TOP=$PWD $ export PATH=$ERL_TOP/bin:$PATH $ ./otp_build all -a && ./otp_build tests diff --git a/lib/stdlib/uc_spec/SpecialCasing.txt b/lib/stdlib/uc_spec/SpecialCasing.txt index 1c2e968a8c..08d04fa942 100644 --- a/lib/stdlib/uc_spec/SpecialCasing.txt +++ b/lib/stdlib/uc_spec/SpecialCasing.txt @@ -1,11 +1,11 @@ -# SpecialCasing-14.0.0.txt -# Date: 2021-03-08, 19:35:55 GMT -# ยฉ 2021 Unicodeยฎ, Inc. +# SpecialCasing-15.0.0.txt +# Date: 2022-02-02, 23:35:52 GMT +# ยฉ 2022 Unicodeยฎ, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. -# For terms of use, see http://www.unicode.org/terms_of_use.html +# For terms of use, see https://www.unicode.org/terms_of_use.html # # Unicode Character Database -# For documentation, see http://www.unicode.org/reports/tr44/ +# For documentation, see https://www.unicode.org/reports/tr44/ # # Special Casing # diff --git a/lib/stdlib/uc_spec/UnicodeData.txt b/lib/stdlib/uc_spec/UnicodeData.txt index b5abef7ed4..ea963a7162 100644 --- a/lib/stdlib/uc_spec/UnicodeData.txt +++ b/lib/stdlib/uc_spec/UnicodeData.txt @@ -2975,6 +2975,7 @@ 0CEF;KANNADA DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; 0CF1;KANNADA SIGN JIHVAMULIYA;Lo;0;L;;;;;N;;;;; 0CF2;KANNADA SIGN UPADHMANIYA;Lo;0;L;;;;;N;;;;; +0CF3;KANNADA SIGN COMBINING ANUSVARA ABOVE RIGHT;Mc;0;L;;;;;N;;;;; 0D00;MALAYALAM SIGN COMBINING ANUSVARA ABOVE;Mn;0;NSM;;;;;N;;;;; 0D01;MALAYALAM SIGN CANDRABINDU;Mn;0;NSM;;;;;N;;;;; 0D02;MALAYALAM SIGN ANUSVARA;Mc;0;L;;;;;N;;;;; @@ -3339,6 +3340,7 @@ 0ECB;LAO TONE MAI CATAWA;Mn;122;NSM;;;;;N;;;;; 0ECC;LAO CANCELLATION MARK;Mn;0;NSM;;;;;N;;;;; 0ECD;LAO NIGGAHITA;Mn;0;NSM;;;;;N;;;;; +0ECE;LAO YAMAKKAN;Mn;0;NSM;;;;;N;;;;; 0ED0;LAO DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; 0ED1;LAO DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; 0ED2;LAO DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; @@ -19393,6 +19395,9 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 10EAD;YEZIDI HYPHENATION MARK;Pd;0;R;;;;;N;;;;; 10EB0;YEZIDI LETTER LAM WITH DOT ABOVE;Lo;0;R;;;;;N;;;;; 10EB1;YEZIDI LETTER YOT WITH CIRCUMFLEX ABOVE;Lo;0;R;;;;;N;;;;; +10EFD;ARABIC SMALL LOW WORD SAKTA;Mn;220;NSM;;;;;N;;;;; +10EFE;ARABIC SMALL LOW WORD QASR;Mn;220;NSM;;;;;N;;;;; +10EFF;ARABIC SMALL LOW WORD MADDA;Mn;220;NSM;;;;;N;;;;; 10F00;OLD SOGDIAN LETTER ALEPH;Lo;0;R;;;;;N;;;;; 10F01;OLD SOGDIAN LETTER FINAL ALEPH;Lo;0;R;;;;;N;;;;; 10F02;OLD SOGDIAN LETTER BETH;Lo;0;R;;;;;N;;;;; @@ -20058,6 +20063,9 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1123C;KHOJKI DOUBLE SECTION MARK;Po;0;L;;;;;N;;;;; 1123D;KHOJKI ABBREVIATION SIGN;Po;0;L;;;;;N;;;;; 1123E;KHOJKI SIGN SUKUN;Mn;0;NSM;;;;;N;;;;; +1123F;KHOJKI LETTER QA;Lo;0;L;;;;;N;;;;; +11240;KHOJKI LETTER SHORT I;Lo;0;L;;;;;N;;;;; +11241;KHOJKI VOWEL SIGN VOCALIC R;Mn;0;NSM;;;;;N;;;;; 11280;MULTANI LETTER A;Lo;0;L;;;;;N;;;;; 11281;MULTANI LETTER I;Lo;0;L;;;;;N;;;;; 11282;MULTANI LETTER U;Lo;0;L;;;;;N;;;;; @@ -21256,6 +21264,16 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 11AF6;PAU CIN HAU LOW-FALLING TONE LONG FINAL;Lo;0;L;;;;;N;;;;; 11AF7;PAU CIN HAU LOW-FALLING TONE FINAL;Lo;0;L;;;;;N;;;;; 11AF8;PAU CIN HAU GLOTTAL STOP FINAL;Lo;0;L;;;;;N;;;;; +11B00;DEVANAGARI HEAD MARK;Po;0;L;;;;;N;;;;; +11B01;DEVANAGARI HEAD MARK WITH HEADSTROKE;Po;0;L;;;;;N;;;;; +11B02;DEVANAGARI SIGN BHALE;Po;0;L;;;;;N;;;;; +11B03;DEVANAGARI SIGN BHALE WITH HOOK;Po;0;L;;;;;N;;;;; +11B04;DEVANAGARI SIGN EXTENDED BHALE;Po;0;L;;;;;N;;;;; +11B05;DEVANAGARI SIGN EXTENDED BHALE WITH HOOK;Po;0;L;;;;;N;;;;; +11B06;DEVANAGARI SIGN WESTERN FIVE-LIKE BHALE;Po;0;L;;;;;N;;;;; +11B07;DEVANAGARI SIGN WESTERN NINE-LIKE BHALE;Po;0;L;;;;;N;;;;; +11B08;DEVANAGARI SIGN REVERSED NINE-LIKE BHALE;Po;0;L;;;;;N;;;;; +11B09;DEVANAGARI SIGN MINDU;Po;0;L;;;;;N;;;;; 11C00;BHAIKSUKI LETTER A;Lo;0;L;;;;;N;;;;; 11C01;BHAIKSUKI LETTER AA;Lo;0;L;;;;;N;;;;; 11C02;BHAIKSUKI LETTER I;Lo;0;L;;;;;N;;;;; @@ -21584,6 +21602,92 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 11EF6;MAKASAR VOWEL SIGN O;Mc;0;L;;;;;N;;;;; 11EF7;MAKASAR PASSIMBANG;Po;0;L;;;;;N;;;;; 11EF8;MAKASAR END OF SECTION;Po;0;L;;;;;N;;;;; +11F00;KAWI SIGN CANDRABINDU;Mn;0;NSM;;;;;N;;;;; +11F01;KAWI SIGN ANUSVARA;Mn;0;NSM;;;;;N;;;;; +11F02;KAWI SIGN REPHA;Lo;0;L;;;;;N;;;;; +11F03;KAWI SIGN VISARGA;Mc;0;L;;;;;N;;;;; +11F04;KAWI LETTER A;Lo;0;L;;;;;N;;;;; +11F05;KAWI LETTER AA;Lo;0;L;;;;;N;;;;; +11F06;KAWI LETTER I;Lo;0;L;;;;;N;;;;; +11F07;KAWI LETTER II;Lo;0;L;;;;;N;;;;; +11F08;KAWI LETTER U;Lo;0;L;;;;;N;;;;; +11F09;KAWI LETTER UU;Lo;0;L;;;;;N;;;;; +11F0A;KAWI LETTER VOCALIC R;Lo;0;L;;;;;N;;;;; +11F0B;KAWI LETTER VOCALIC RR;Lo;0;L;;;;;N;;;;; +11F0C;KAWI LETTER VOCALIC L;Lo;0;L;;;;;N;;;;; +11F0D;KAWI LETTER VOCALIC LL;Lo;0;L;;;;;N;;;;; +11F0E;KAWI LETTER E;Lo;0;L;;;;;N;;;;; +11F0F;KAWI LETTER AI;Lo;0;L;;;;;N;;;;; +11F10;KAWI LETTER O;Lo;0;L;;;;;N;;;;; +11F12;KAWI LETTER KA;Lo;0;L;;;;;N;;;;; +11F13;KAWI LETTER KHA;Lo;0;L;;;;;N;;;;; +11F14;KAWI LETTER GA;Lo;0;L;;;;;N;;;;; +11F15;KAWI LETTER GHA;Lo;0;L;;;;;N;;;;; +11F16;KAWI LETTER NGA;Lo;0;L;;;;;N;;;;; +11F17;KAWI LETTER CA;Lo;0;L;;;;;N;;;;; +11F18;KAWI LETTER CHA;Lo;0;L;;;;;N;;;;; +11F19;KAWI LETTER JA;Lo;0;L;;;;;N;;;;; +11F1A;KAWI LETTER JHA;Lo;0;L;;;;;N;;;;; +11F1B;KAWI LETTER NYA;Lo;0;L;;;;;N;;;;; +11F1C;KAWI LETTER TTA;Lo;0;L;;;;;N;;;;; +11F1D;KAWI LETTER TTHA;Lo;0;L;;;;;N;;;;; +11F1E;KAWI LETTER DDA;Lo;0;L;;;;;N;;;;; +11F1F;KAWI LETTER DDHA;Lo;0;L;;;;;N;;;;; +11F20;KAWI LETTER NNA;Lo;0;L;;;;;N;;;;; +11F21;KAWI LETTER TA;Lo;0;L;;;;;N;;;;; +11F22;KAWI LETTER THA;Lo;0;L;;;;;N;;;;; +11F23;KAWI LETTER DA;Lo;0;L;;;;;N;;;;; +11F24;KAWI LETTER DHA;Lo;0;L;;;;;N;;;;; +11F25;KAWI LETTER NA;Lo;0;L;;;;;N;;;;; +11F26;KAWI LETTER PA;Lo;0;L;;;;;N;;;;; +11F27;KAWI LETTER PHA;Lo;0;L;;;;;N;;;;; +11F28;KAWI LETTER BA;Lo;0;L;;;;;N;;;;; +11F29;KAWI LETTER BHA;Lo;0;L;;;;;N;;;;; +11F2A;KAWI LETTER MA;Lo;0;L;;;;;N;;;;; +11F2B;KAWI LETTER YA;Lo;0;L;;;;;N;;;;; +11F2C;KAWI LETTER RA;Lo;0;L;;;;;N;;;;; +11F2D;KAWI LETTER LA;Lo;0;L;;;;;N;;;;; +11F2E;KAWI LETTER WA;Lo;0;L;;;;;N;;;;; +11F2F;KAWI LETTER SHA;Lo;0;L;;;;;N;;;;; +11F30;KAWI LETTER SSA;Lo;0;L;;;;;N;;;;; +11F31;KAWI LETTER SA;Lo;0;L;;;;;N;;;;; +11F32;KAWI LETTER HA;Lo;0;L;;;;;N;;;;; +11F33;KAWI LETTER JNYA;Lo;0;L;;;;;N;;;;; +11F34;KAWI VOWEL SIGN AA;Mc;0;L;;;;;N;;;;; +11F35;KAWI VOWEL SIGN ALTERNATE AA;Mc;0;L;;;;;N;;;;; +11F36;KAWI VOWEL SIGN I;Mn;0;NSM;;;;;N;;;;; +11F37;KAWI VOWEL SIGN II;Mn;0;NSM;;;;;N;;;;; +11F38;KAWI VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;; +11F39;KAWI VOWEL SIGN UU;Mn;0;NSM;;;;;N;;;;; +11F3A;KAWI VOWEL SIGN VOCALIC R;Mn;0;NSM;;;;;N;;;;; +11F3E;KAWI VOWEL SIGN E;Mc;0;L;;;;;N;;;;; +11F3F;KAWI VOWEL SIGN AI;Mc;0;L;;;;;N;;;;; +11F40;KAWI VOWEL SIGN EU;Mn;0;NSM;;;;;N;;;;; +11F41;KAWI SIGN KILLER;Mc;9;L;;;;;N;;;;; +11F42;KAWI CONJOINER;Mn;9;NSM;;;;;N;;;;; +11F43;KAWI DANDA;Po;0;L;;;;;N;;;;; +11F44;KAWI DOUBLE DANDA;Po;0;L;;;;;N;;;;; +11F45;KAWI PUNCTUATION SECTION MARKER;Po;0;L;;;;;N;;;;; +11F46;KAWI PUNCTUATION ALTERNATE SECTION MARKER;Po;0;L;;;;;N;;;;; +11F47;KAWI PUNCTUATION FLOWER;Po;0;L;;;;;N;;;;; +11F48;KAWI PUNCTUATION SPACE FILLER;Po;0;L;;;;;N;;;;; +11F49;KAWI PUNCTUATION DOT;Po;0;L;;;;;N;;;;; +11F4A;KAWI PUNCTUATION DOUBLE DOT;Po;0;L;;;;;N;;;;; +11F4B;KAWI PUNCTUATION TRIPLE DOT;Po;0;L;;;;;N;;;;; +11F4C;KAWI PUNCTUATION CIRCLE;Po;0;L;;;;;N;;;;; +11F4D;KAWI PUNCTUATION FILLED CIRCLE;Po;0;L;;;;;N;;;;; +11F4E;KAWI PUNCTUATION SPIRAL;Po;0;L;;;;;N;;;;; +11F4F;KAWI PUNCTUATION CLOSING SPIRAL;Po;0;L;;;;;N;;;;; +11F50;KAWI DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +11F51;KAWI DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +11F52;KAWI DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +11F53;KAWI DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +11F54;KAWI DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +11F55;KAWI DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +11F56;KAWI DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +11F57;KAWI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +11F58;KAWI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +11F59;KAWI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; 11FB0;LISU LETTER YHA;Lo;0;L;;;;;N;;;;; 11FC0;TAMIL FRACTION ONE THREE-HUNDRED-AND-TWENTIETH;No;0;L;;;;1/320;N;;;;; 11FC1;TAMIL FRACTION ONE ONE-HUNDRED-AND-SIXTIETH;No;0;L;;;;1/160;N;;;;; @@ -24040,6 +24144,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1342C;EGYPTIAN HIEROGLYPH AA030;Lo;0;L;;;;;N;;;;; 1342D;EGYPTIAN HIEROGLYPH AA031;Lo;0;L;;;;;N;;;;; 1342E;EGYPTIAN HIEROGLYPH AA032;Lo;0;L;;;;;N;;;;; +1342F;EGYPTIAN HIEROGLYPH V011D;Lo;0;L;;;;;N;;;;; 13430;EGYPTIAN HIEROGLYPH VERTICAL JOINER;Cf;0;L;;;;;N;;;;; 13431;EGYPTIAN HIEROGLYPH HORIZONTAL JOINER;Cf;0;L;;;;;N;;;;; 13432;EGYPTIAN HIEROGLYPH INSERT AT TOP START;Cf;0;L;;;;;N;;;;; @@ -24049,6 +24154,35 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 13436;EGYPTIAN HIEROGLYPH OVERLAY MIDDLE;Cf;0;L;;;;;N;;;;; 13437;EGYPTIAN HIEROGLYPH BEGIN SEGMENT;Cf;0;L;;;;;N;;;;; 13438;EGYPTIAN HIEROGLYPH END SEGMENT;Cf;0;L;;;;;N;;;;; +13439;EGYPTIAN HIEROGLYPH INSERT AT MIDDLE;Cf;0;L;;;;;N;;;;; +1343A;EGYPTIAN HIEROGLYPH INSERT AT TOP;Cf;0;L;;;;;N;;;;; +1343B;EGYPTIAN HIEROGLYPH INSERT AT BOTTOM;Cf;0;L;;;;;N;;;;; +1343C;EGYPTIAN HIEROGLYPH BEGIN ENCLOSURE;Cf;0;L;;;;;N;;;;; +1343D;EGYPTIAN HIEROGLYPH END ENCLOSURE;Cf;0;L;;;;;N;;;;; +1343E;EGYPTIAN HIEROGLYPH BEGIN WALLED ENCLOSURE;Cf;0;L;;;;;N;;;;; +1343F;EGYPTIAN HIEROGLYPH END WALLED ENCLOSURE;Cf;0;L;;;;;N;;;;; +13440;EGYPTIAN HIEROGLYPH MIRROR HORIZONTALLY;Mn;0;NSM;;;;;N;;;;; +13441;EGYPTIAN HIEROGLYPH FULL BLANK;Lo;0;L;;;;;N;;;;; +13442;EGYPTIAN HIEROGLYPH HALF BLANK;Lo;0;L;;;;;N;;;;; +13443;EGYPTIAN HIEROGLYPH LOST SIGN;Lo;0;L;;;;;N;;;;; +13444;EGYPTIAN HIEROGLYPH HALF LOST SIGN;Lo;0;L;;;;;N;;;;; +13445;EGYPTIAN HIEROGLYPH TALL LOST SIGN;Lo;0;L;;;;;N;;;;; +13446;EGYPTIAN HIEROGLYPH WIDE LOST SIGN;Lo;0;L;;;;;N;;;;; +13447;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP START;Mn;0;NSM;;;;;N;;;;; +13448;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT BOTTOM START;Mn;0;NSM;;;;;N;;;;; +13449;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT START;Mn;0;NSM;;;;;N;;;;; +1344A;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP END;Mn;0;NSM;;;;;N;;;;; +1344B;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP;Mn;0;NSM;;;;;N;;;;; +1344C;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT BOTTOM START AND TOP END;Mn;0;NSM;;;;;N;;;;; +1344D;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT START AND TOP;Mn;0;NSM;;;;;N;;;;; +1344E;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT BOTTOM END;Mn;0;NSM;;;;;N;;;;; +1344F;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP START AND BOTTOM END;Mn;0;NSM;;;;;N;;;;; +13450;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT BOTTOM;Mn;0;NSM;;;;;N;;;;; +13451;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT START AND BOTTOM;Mn;0;NSM;;;;;N;;;;; +13452;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT END;Mn;0;NSM;;;;;N;;;;; +13453;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP AND END;Mn;0;NSM;;;;;N;;;;; +13454;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT BOTTOM AND END;Mn;0;NSM;;;;;N;;;;; +13455;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED;Mn;0;NSM;;;;;N;;;;; 14400;ANATOLIAN HIEROGLYPH A001;Lo;0;L;;;;;N;;;;; 14401;ANATOLIAN HIEROGLYPH A002;Lo;0;L;;;;;N;;;;; 14402;ANATOLIAN HIEROGLYPH A003;Lo;0;L;;;;;N;;;;; @@ -27289,9 +27423,11 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1B120;KATAKANA LETTER ARCHAIC YI;Lo;0;L;;;;;N;;;;; 1B121;KATAKANA LETTER ARCHAIC YE;Lo;0;L;;;;;N;;;;; 1B122;KATAKANA LETTER ARCHAIC WU;Lo;0;L;;;;;N;;;;; +1B132;HIRAGANA LETTER SMALL KO;Lo;0;L;;;;;N;;;;; 1B150;HIRAGANA LETTER SMALL WI;Lo;0;L;;;;;N;;;;; 1B151;HIRAGANA LETTER SMALL WE;Lo;0;L;;;;;N;;;;; 1B152;HIRAGANA LETTER SMALL WO;Lo;0;L;;;;;N;;;;; +1B155;KATAKANA LETTER SMALL KO;Lo;0;L;;;;;N;;;;; 1B164;KATAKANA LETTER SMALL WI;Lo;0;L;;;;;N;;;;; 1B165;KATAKANA LETTER SMALL WE;Lo;0;L;;;;;N;;;;; 1B166;KATAKANA LETTER SMALL WO;Lo;0;L;;;;;N;;;;; @@ -28573,6 +28709,26 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1D243;COMBINING GREEK MUSICAL TETRASEME;Mn;230;NSM;;;;;N;;;;; 1D244;COMBINING GREEK MUSICAL PENTASEME;Mn;230;NSM;;;;;N;;;;; 1D245;GREEK MUSICAL LEIMMA;So;0;ON;;;;;N;;;;; +1D2C0;KAKTOVIK NUMERAL ZERO;No;0;L;;;;0;N;;;;; +1D2C1;KAKTOVIK NUMERAL ONE;No;0;L;;;;1;N;;;;; +1D2C2;KAKTOVIK NUMERAL TWO;No;0;L;;;;2;N;;;;; +1D2C3;KAKTOVIK NUMERAL THREE;No;0;L;;;;3;N;;;;; +1D2C4;KAKTOVIK NUMERAL FOUR;No;0;L;;;;4;N;;;;; +1D2C5;KAKTOVIK NUMERAL FIVE;No;0;L;;;;5;N;;;;; +1D2C6;KAKTOVIK NUMERAL SIX;No;0;L;;;;6;N;;;;; +1D2C7;KAKTOVIK NUMERAL SEVEN;No;0;L;;;;7;N;;;;; +1D2C8;KAKTOVIK NUMERAL EIGHT;No;0;L;;;;8;N;;;;; +1D2C9;KAKTOVIK NUMERAL NINE;No;0;L;;;;9;N;;;;; +1D2CA;KAKTOVIK NUMERAL TEN;No;0;L;;;;10;N;;;;; +1D2CB;KAKTOVIK NUMERAL ELEVEN;No;0;L;;;;11;N;;;;; +1D2CC;KAKTOVIK NUMERAL TWELVE;No;0;L;;;;12;N;;;;; +1D2CD;KAKTOVIK NUMERAL THIRTEEN;No;0;L;;;;13;N;;;;; +1D2CE;KAKTOVIK NUMERAL FOURTEEN;No;0;L;;;;14;N;;;;; +1D2CF;KAKTOVIK NUMERAL FIFTEEN;No;0;L;;;;15;N;;;;; +1D2D0;KAKTOVIK NUMERAL SIXTEEN;No;0;L;;;;16;N;;;;; +1D2D1;KAKTOVIK NUMERAL SEVENTEEN;No;0;L;;;;17;N;;;;; +1D2D2;KAKTOVIK NUMERAL EIGHTEEN;No;0;L;;;;18;N;;;;; +1D2D3;KAKTOVIK NUMERAL NINETEEN;No;0;L;;;;19;N;;;;; 1D2E0;MAYAN NUMERAL ZERO;No;0;L;;;;0;N;;;;; 1D2E1;MAYAN NUMERAL ONE;No;0;L;;;;1;N;;;;; 1D2E2;MAYAN NUMERAL TWO;No;0;L;;;;2;N;;;;; @@ -30404,6 +30560,12 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1DF1C;LATIN SMALL LETTER TESH DIGRAPH WITH RETROFLEX HOOK;Ll;0;L;;;;;N;;;;; 1DF1D;LATIN SMALL LETTER C WITH RETROFLEX HOOK;Ll;0;L;;;;;N;;;;; 1DF1E;LATIN SMALL LETTER S WITH CURL;Ll;0;L;;;;;N;;;;; +1DF25;LATIN SMALL LETTER D WITH MID-HEIGHT LEFT HOOK;Ll;0;L;;;;;N;;;;; +1DF26;LATIN SMALL LETTER L WITH MID-HEIGHT LEFT HOOK;Ll;0;L;;;;;N;;;;; +1DF27;LATIN SMALL LETTER N WITH MID-HEIGHT LEFT HOOK;Ll;0;L;;;;;N;;;;; +1DF28;LATIN SMALL LETTER R WITH MID-HEIGHT LEFT HOOK;Ll;0;L;;;;;N;;;;; +1DF29;LATIN SMALL LETTER S WITH MID-HEIGHT LEFT HOOK;Ll;0;L;;;;;N;;;;; +1DF2A;LATIN SMALL LETTER T WITH MID-HEIGHT LEFT HOOK;Ll;0;L;;;;;N;;;;; 1E000;COMBINING GLAGOLITIC LETTER AZU;Mn;230;NSM;;;;;N;;;;; 1E001;COMBINING GLAGOLITIC LETTER BUKY;Mn;230;NSM;;;;;N;;;;; 1E002;COMBINING GLAGOLITIC LETTER VEDE;Mn;230;NSM;;;;;N;;;;; @@ -30442,6 +30604,69 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1E028;COMBINING GLAGOLITIC LETTER BIG YUS;Mn;230;NSM;;;;;N;;;;; 1E029;COMBINING GLAGOLITIC LETTER IOTATED BIG YUS;Mn;230;NSM;;;;;N;;;;; 1E02A;COMBINING GLAGOLITIC LETTER FITA;Mn;230;NSM;;;;;N;;;;; +1E030;MODIFIER LETTER CYRILLIC SMALL A;Lm;0;L;<super> 0430;;;;N;;;;; +1E031;MODIFIER LETTER CYRILLIC SMALL BE;Lm;0;L;<super> 0431;;;;N;;;;; +1E032;MODIFIER LETTER CYRILLIC SMALL VE;Lm;0;L;<super> 0432;;;;N;;;;; +1E033;MODIFIER LETTER CYRILLIC SMALL GHE;Lm;0;L;<super> 0433;;;;N;;;;; +1E034;MODIFIER LETTER CYRILLIC SMALL DE;Lm;0;L;<super> 0434;;;;N;;;;; +1E035;MODIFIER LETTER CYRILLIC SMALL IE;Lm;0;L;<super> 0435;;;;N;;;;; +1E036;MODIFIER LETTER CYRILLIC SMALL ZHE;Lm;0;L;<super> 0436;;;;N;;;;; +1E037;MODIFIER LETTER CYRILLIC SMALL ZE;Lm;0;L;<super> 0437;;;;N;;;;; +1E038;MODIFIER LETTER CYRILLIC SMALL I;Lm;0;L;<super> 0438;;;;N;;;;; +1E039;MODIFIER LETTER CYRILLIC SMALL KA;Lm;0;L;<super> 043A;;;;N;;;;; +1E03A;MODIFIER LETTER CYRILLIC SMALL EL;Lm;0;L;<super> 043B;;;;N;;;;; +1E03B;MODIFIER LETTER CYRILLIC SMALL EM;Lm;0;L;<super> 043C;;;;N;;;;; +1E03C;MODIFIER LETTER CYRILLIC SMALL O;Lm;0;L;<super> 043E;;;;N;;;;; +1E03D;MODIFIER LETTER CYRILLIC SMALL PE;Lm;0;L;<super> 043F;;;;N;;;;; +1E03E;MODIFIER LETTER CYRILLIC SMALL ER;Lm;0;L;<super> 0440;;;;N;;;;; +1E03F;MODIFIER LETTER CYRILLIC SMALL ES;Lm;0;L;<super> 0441;;;;N;;;;; +1E040;MODIFIER LETTER CYRILLIC SMALL TE;Lm;0;L;<super> 0442;;;;N;;;;; +1E041;MODIFIER LETTER CYRILLIC SMALL U;Lm;0;L;<super> 0443;;;;N;;;;; +1E042;MODIFIER LETTER CYRILLIC SMALL EF;Lm;0;L;<super> 0444;;;;N;;;;; +1E043;MODIFIER LETTER CYRILLIC SMALL HA;Lm;0;L;<super> 0445;;;;N;;;;; +1E044;MODIFIER LETTER CYRILLIC SMALL TSE;Lm;0;L;<super> 0446;;;;N;;;;; +1E045;MODIFIER LETTER CYRILLIC SMALL CHE;Lm;0;L;<super> 0447;;;;N;;;;; +1E046;MODIFIER LETTER CYRILLIC SMALL SHA;Lm;0;L;<super> 0448;;;;N;;;;; +1E047;MODIFIER LETTER CYRILLIC SMALL YERU;Lm;0;L;<super> 044B;;;;N;;;;; +1E048;MODIFIER LETTER CYRILLIC SMALL E;Lm;0;L;<super> 044D;;;;N;;;;; +1E049;MODIFIER LETTER CYRILLIC SMALL YU;Lm;0;L;<super> 044E;;;;N;;;;; +1E04A;MODIFIER LETTER CYRILLIC SMALL DZZE;Lm;0;L;<super> A689;;;;N;;;;; +1E04B;MODIFIER LETTER CYRILLIC SMALL SCHWA;Lm;0;L;<super> 04D9;;;;N;;;;; +1E04C;MODIFIER LETTER CYRILLIC SMALL BYELORUSSIAN-UKRAINIAN I;Lm;0;L;<super> 0456;;;;N;;;;; +1E04D;MODIFIER LETTER CYRILLIC SMALL JE;Lm;0;L;<super> 0458;;;;N;;;;; +1E04E;MODIFIER LETTER CYRILLIC SMALL BARRED O;Lm;0;L;<super> 04E9;;;;N;;;;; +1E04F;MODIFIER LETTER CYRILLIC SMALL STRAIGHT U;Lm;0;L;<super> 04AF;;;;N;;;;; +1E050;MODIFIER LETTER CYRILLIC SMALL PALOCHKA;Lm;0;L;<super> 04CF;;;;N;;;;; +1E051;CYRILLIC SUBSCRIPT SMALL LETTER A;Lm;0;L;<sub> 0430;;;;N;;;;; +1E052;CYRILLIC SUBSCRIPT SMALL LETTER BE;Lm;0;L;<sub> 0431;;;;N;;;;; +1E053;CYRILLIC SUBSCRIPT SMALL LETTER VE;Lm;0;L;<sub> 0432;;;;N;;;;; +1E054;CYRILLIC SUBSCRIPT SMALL LETTER GHE;Lm;0;L;<sub> 0433;;;;N;;;;; +1E055;CYRILLIC SUBSCRIPT SMALL LETTER DE;Lm;0;L;<sub> 0434;;;;N;;;;; +1E056;CYRILLIC SUBSCRIPT SMALL LETTER IE;Lm;0;L;<sub> 0435;;;;N;;;;; +1E057;CYRILLIC SUBSCRIPT SMALL LETTER ZHE;Lm;0;L;<sub> 0436;;;;N;;;;; +1E058;CYRILLIC SUBSCRIPT SMALL LETTER ZE;Lm;0;L;<sub> 0437;;;;N;;;;; +1E059;CYRILLIC SUBSCRIPT SMALL LETTER I;Lm;0;L;<sub> 0438;;;;N;;;;; +1E05A;CYRILLIC SUBSCRIPT SMALL LETTER KA;Lm;0;L;<sub> 043A;;;;N;;;;; +1E05B;CYRILLIC SUBSCRIPT SMALL LETTER EL;Lm;0;L;<sub> 043B;;;;N;;;;; +1E05C;CYRILLIC SUBSCRIPT SMALL LETTER O;Lm;0;L;<sub> 043E;;;;N;;;;; +1E05D;CYRILLIC SUBSCRIPT SMALL LETTER PE;Lm;0;L;<sub> 043F;;;;N;;;;; +1E05E;CYRILLIC SUBSCRIPT SMALL LETTER ES;Lm;0;L;<sub> 0441;;;;N;;;;; +1E05F;CYRILLIC SUBSCRIPT SMALL LETTER U;Lm;0;L;<sub> 0443;;;;N;;;;; +1E060;CYRILLIC SUBSCRIPT SMALL LETTER EF;Lm;0;L;<sub> 0444;;;;N;;;;; +1E061;CYRILLIC SUBSCRIPT SMALL LETTER HA;Lm;0;L;<sub> 0445;;;;N;;;;; +1E062;CYRILLIC SUBSCRIPT SMALL LETTER TSE;Lm;0;L;<sub> 0446;;;;N;;;;; +1E063;CYRILLIC SUBSCRIPT SMALL LETTER CHE;Lm;0;L;<sub> 0447;;;;N;;;;; +1E064;CYRILLIC SUBSCRIPT SMALL LETTER SHA;Lm;0;L;<sub> 0448;;;;N;;;;; +1E065;CYRILLIC SUBSCRIPT SMALL LETTER HARD SIGN;Lm;0;L;<sub> 044A;;;;N;;;;; +1E066;CYRILLIC SUBSCRIPT SMALL LETTER YERU;Lm;0;L;<sub> 044B;;;;N;;;;; +1E067;CYRILLIC SUBSCRIPT SMALL LETTER GHE WITH UPTURN;Lm;0;L;<sub> 0491;;;;N;;;;; +1E068;CYRILLIC SUBSCRIPT SMALL LETTER BYELORUSSIAN-UKRAINIAN I;Lm;0;L;<sub> 0456;;;;N;;;;; +1E069;CYRILLIC SUBSCRIPT SMALL LETTER DZE;Lm;0;L;<sub> 0455;;;;N;;;;; +1E06A;CYRILLIC SUBSCRIPT SMALL LETTER DZHE;Lm;0;L;<sub> 045F;;;;N;;;;; +1E06B;MODIFIER LETTER CYRILLIC SMALL ES WITH DESCENDER;Lm;0;L;<super> 04AB;;;;N;;;;; +1E06C;MODIFIER LETTER CYRILLIC SMALL YERU WITH BACK YER;Lm;0;L;<super> A651;;;;N;;;;; +1E06D;MODIFIER LETTER CYRILLIC SMALL STRAIGHT U WITH STROKE;Lm;0;L;<super> 04B1;;;;N;;;;; +1E08F;COMBINING CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I;Mn;230;NSM;;;;;N;;;;; 1E100;NYIAKENG PUACHUE HMONG LETTER MA;Lo;0;L;;;;;N;;;;; 1E101;NYIAKENG PUACHUE HMONG LETTER TSA;Lo;0;L;;;;;N;;;;; 1E102;NYIAKENG PUACHUE HMONG LETTER NTA;Lo;0;L;;;;;N;;;;; @@ -30603,6 +30828,48 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1E2F8;WANCHO DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; 1E2F9;WANCHO DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; 1E2FF;WANCHO NGUN SIGN;Sc;0;ET;;;;;N;;;;; +1E4D0;NAG MUNDARI LETTER O;Lo;0;L;;;;;N;;;;; +1E4D1;NAG MUNDARI LETTER OP;Lo;0;L;;;;;N;;;;; +1E4D2;NAG MUNDARI LETTER OL;Lo;0;L;;;;;N;;;;; +1E4D3;NAG MUNDARI LETTER OY;Lo;0;L;;;;;N;;;;; +1E4D4;NAG MUNDARI LETTER ONG;Lo;0;L;;;;;N;;;;; +1E4D5;NAG MUNDARI LETTER A;Lo;0;L;;;;;N;;;;; +1E4D6;NAG MUNDARI LETTER AJ;Lo;0;L;;;;;N;;;;; +1E4D7;NAG MUNDARI LETTER AB;Lo;0;L;;;;;N;;;;; +1E4D8;NAG MUNDARI LETTER ANY;Lo;0;L;;;;;N;;;;; +1E4D9;NAG MUNDARI LETTER AH;Lo;0;L;;;;;N;;;;; +1E4DA;NAG MUNDARI LETTER I;Lo;0;L;;;;;N;;;;; +1E4DB;NAG MUNDARI LETTER IS;Lo;0;L;;;;;N;;;;; +1E4DC;NAG MUNDARI LETTER IDD;Lo;0;L;;;;;N;;;;; +1E4DD;NAG MUNDARI LETTER IT;Lo;0;L;;;;;N;;;;; +1E4DE;NAG MUNDARI LETTER IH;Lo;0;L;;;;;N;;;;; +1E4DF;NAG MUNDARI LETTER U;Lo;0;L;;;;;N;;;;; +1E4E0;NAG MUNDARI LETTER UC;Lo;0;L;;;;;N;;;;; +1E4E1;NAG MUNDARI LETTER UD;Lo;0;L;;;;;N;;;;; +1E4E2;NAG MUNDARI LETTER UK;Lo;0;L;;;;;N;;;;; +1E4E3;NAG MUNDARI LETTER UR;Lo;0;L;;;;;N;;;;; +1E4E4;NAG MUNDARI LETTER E;Lo;0;L;;;;;N;;;;; +1E4E5;NAG MUNDARI LETTER ENN;Lo;0;L;;;;;N;;;;; +1E4E6;NAG MUNDARI LETTER EG;Lo;0;L;;;;;N;;;;; +1E4E7;NAG MUNDARI LETTER EM;Lo;0;L;;;;;N;;;;; +1E4E8;NAG MUNDARI LETTER EN;Lo;0;L;;;;;N;;;;; +1E4E9;NAG MUNDARI LETTER ETT;Lo;0;L;;;;;N;;;;; +1E4EA;NAG MUNDARI LETTER ELL;Lo;0;L;;;;;N;;;;; +1E4EB;NAG MUNDARI SIGN OJOD;Lm;0;L;;;;;N;;;;; +1E4EC;NAG MUNDARI SIGN MUHOR;Mn;232;NSM;;;;;N;;;;; +1E4ED;NAG MUNDARI SIGN TOYOR;Mn;232;NSM;;;;;N;;;;; +1E4EE;NAG MUNDARI SIGN IKIR;Mn;220;NSM;;;;;N;;;;; +1E4EF;NAG MUNDARI SIGN SUTUH;Mn;230;NSM;;;;;N;;;;; +1E4F0;NAG MUNDARI DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +1E4F1;NAG MUNDARI DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +1E4F2;NAG MUNDARI DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +1E4F3;NAG MUNDARI DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +1E4F4;NAG MUNDARI DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +1E4F5;NAG MUNDARI DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +1E4F6;NAG MUNDARI DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +1E4F7;NAG MUNDARI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +1E4F8;NAG MUNDARI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +1E4F9;NAG MUNDARI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; 1E7E0;ETHIOPIC SYLLABLE HHYA;Lo;0;L;;;;;N;;;;; 1E7E1;ETHIOPIC SYLLABLE HHYU;Lo;0;L;;;;;N;;;;; 1E7E2;ETHIOPIC SYLLABLE HHYI;Lo;0;L;;;;;N;;;;; @@ -32678,6 +32945,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F6D5;HINDU TEMPLE;So;0;ON;;;;;N;;;;; 1F6D6;HUT;So;0;ON;;;;;N;;;;; 1F6D7;ELEVATOR;So;0;ON;;;;;N;;;;; +1F6DC;WIRELESS;So;0;ON;;;;;N;;;;; 1F6DD;PLAYGROUND SLIDE;So;0;ON;;;;;N;;;;; 1F6DE;WHEEL;So;0;ON;;;;;N;;;;; 1F6DF;RING BUOY;So;0;ON;;;;;N;;;;; @@ -32823,6 +33091,14 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F771;ALCHEMICAL SYMBOL FOR MONTH;So;0;ON;;;;;N;;;;; 1F772;ALCHEMICAL SYMBOL FOR HALF DRAM;So;0;ON;;;;;N;;;;; 1F773;ALCHEMICAL SYMBOL FOR HALF OUNCE;So;0;ON;;;;;N;;;;; +1F774;LOT OF FORTUNE;So;0;ON;;;;;N;;;;; +1F775;OCCULTATION;So;0;ON;;;;;N;;;;; +1F776;LUNAR ECLIPSE;So;0;ON;;;;;N;;;;; +1F77B;HAUMEA;So;0;ON;;;;;N;;;;; +1F77C;MAKEMAKE;So;0;ON;;;;;N;;;;; +1F77D;GONGGONG;So;0;ON;;;;;N;;;;; +1F77E;QUAOAR;So;0;ON;;;;;N;;;;; +1F77F;ORCUS;So;0;ON;;;;;N;;;;; 1F780;BLACK LEFT-POINTING ISOSCELES RIGHT TRIANGLE;So;0;ON;;;;;N;;;;; 1F781;BLACK UP-POINTING ISOSCELES RIGHT TRIANGLE;So;0;ON;;;;;N;;;;; 1F782;BLACK RIGHT-POINTING ISOSCELES RIGHT TRIANGLE;So;0;ON;;;;;N;;;;; @@ -32912,6 +33188,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F7D6;NEGATIVE CIRCLED TRIANGLE;So;0;ON;;;;;N;;;;; 1F7D7;CIRCLED SQUARE;So;0;ON;;;;;N;;;;; 1F7D8;NEGATIVE CIRCLED SQUARE;So;0;ON;;;;;N;;;;; +1F7D9;NINE POINTED WHITE STAR;So;0;ON;;;;;N;;;;; 1F7E0;LARGE ORANGE CIRCLE;So;0;ON;;;;;N;;;;; 1F7E1;LARGE YELLOW CIRCLE;So;0;ON;;;;;N;;;;; 1F7E2;LARGE GREEN CIRCLE;So;0;ON;;;;;N;;;;; @@ -33434,6 +33711,9 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FA72;BRIEFS;So;0;ON;;;;;N;;;;; 1FA73;SHORTS;So;0;ON;;;;;N;;;;; 1FA74;THONG SANDAL;So;0;ON;;;;;N;;;;; +1FA75;LIGHT BLUE HEART;So;0;ON;;;;;N;;;;; +1FA76;GREY HEART;So;0;ON;;;;;N;;;;; +1FA77;PINK HEART;So;0;ON;;;;;N;;;;; 1FA78;DROP OF BLOOD;So;0;ON;;;;;N;;;;; 1FA79;ADHESIVE BANDAGE;So;0;ON;;;;;N;;;;; 1FA7A;STETHOSCOPE;So;0;ON;;;;;N;;;;; @@ -33446,6 +33726,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FA84;MAGIC WAND;So;0;ON;;;;;N;;;;; 1FA85;PINATA;So;0;ON;;;;;N;;;;; 1FA86;NESTING DOLLS;So;0;ON;;;;;N;;;;; +1FA87;MARACAS;So;0;ON;;;;;N;;;;; +1FA88;FLUTE;So;0;ON;;;;;N;;;;; 1FA90;RINGED PLANET;So;0;ON;;;;;N;;;;; 1FA91;CHAIR;So;0;ON;;;;;N;;;;; 1FA92;RAZOR;So;0;ON;;;;;N;;;;; @@ -33475,6 +33757,9 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FAAA;IDENTIFICATION CARD;So;0;ON;;;;;N;;;;; 1FAAB;LOW BATTERY;So;0;ON;;;;;N;;;;; 1FAAC;HAMSA;So;0;ON;;;;;N;;;;; +1FAAD;FOLDING HAND FAN;So;0;ON;;;;;N;;;;; +1FAAE;HAIR PICK;So;0;ON;;;;;N;;;;; +1FAAF;KHANDA;So;0;ON;;;;;N;;;;; 1FAB0;FLY;So;0;ON;;;;;N;;;;; 1FAB1;WORM;So;0;ON;;;;;N;;;;; 1FAB2;BEETLE;So;0;ON;;;;;N;;;;; @@ -33486,12 +33771,18 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FAB8;CORAL;So;0;ON;;;;;N;;;;; 1FAB9;EMPTY NEST;So;0;ON;;;;;N;;;;; 1FABA;NEST WITH EGGS;So;0;ON;;;;;N;;;;; +1FABB;HYACINTH;So;0;ON;;;;;N;;;;; +1FABC;JELLYFISH;So;0;ON;;;;;N;;;;; +1FABD;WING;So;0;ON;;;;;N;;;;; +1FABF;GOOSE;So;0;ON;;;;;N;;;;; 1FAC0;ANATOMICAL HEART;So;0;ON;;;;;N;;;;; 1FAC1;LUNGS;So;0;ON;;;;;N;;;;; 1FAC2;PEOPLE HUGGING;So;0;ON;;;;;N;;;;; 1FAC3;PREGNANT MAN;So;0;ON;;;;;N;;;;; 1FAC4;PREGNANT PERSON;So;0;ON;;;;;N;;;;; 1FAC5;PERSON WITH CROWN;So;0;ON;;;;;N;;;;; +1FACE;MOOSE;So;0;ON;;;;;N;;;;; +1FACF;DONKEY;So;0;ON;;;;;N;;;;; 1FAD0;BLUEBERRIES;So;0;ON;;;;;N;;;;; 1FAD1;BELL PEPPER;So;0;ON;;;;;N;;;;; 1FAD2;OLIVE;So;0;ON;;;;;N;;;;; @@ -33502,6 +33793,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FAD7;POURING LIQUID;So;0;ON;;;;;N;;;;; 1FAD8;BEANS;So;0;ON;;;;;N;;;;; 1FAD9;JAR;So;0;ON;;;;;N;;;;; +1FADA;GINGER ROOT;So;0;ON;;;;;N;;;;; +1FADB;PEA POD;So;0;ON;;;;;N;;;;; 1FAE0;MELTING FACE;So;0;ON;;;;;N;;;;; 1FAE1;SALUTING FACE;So;0;ON;;;;;N;;;;; 1FAE2;FACE WITH OPEN EYES AND HAND OVER MOUTH;So;0;ON;;;;;N;;;;; @@ -33510,6 +33803,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FAE5;DOTTED LINE FACE;So;0;ON;;;;;N;;;;; 1FAE6;BITING LIP;So;0;ON;;;;;N;;;;; 1FAE7;BUBBLES;So;0;ON;;;;;N;;;;; +1FAE8;SHAKING FACE;So;0;ON;;;;;N;;;;; 1FAF0;HAND WITH INDEX FINGER AND THUMB CROSSED;So;0;ON;;;;;N;;;;; 1FAF1;RIGHTWARDS HAND;So;0;ON;;;;;N;;;;; 1FAF2;LEFTWARDS HAND;So;0;ON;;;;;N;;;;; @@ -33517,6 +33811,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FAF4;PALM UP HAND;So;0;ON;;;;;N;;;;; 1FAF5;INDEX POINTING AT THE VIEWER;So;0;ON;;;;;N;;;;; 1FAF6;HEART HANDS;So;0;ON;;;;;N;;;;; +1FAF7;LEFTWARDS PUSHING HAND;So;0;ON;;;;;N;;;;; +1FAF8;RIGHTWARDS PUSHING HAND;So;0;ON;;;;;N;;;;; 1FB00;BLOCK SEXTANT-1;So;0;ON;;;;;N;;;;; 1FB01;BLOCK SEXTANT-2;So;0;ON;;;;;N;;;;; 1FB02;BLOCK SEXTANT-12;So;0;ON;;;;;N;;;;; @@ -33732,7 +34028,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 20000;<CJK Ideograph Extension B, First>;Lo;0;L;;;;;N;;;;; 2A6DF;<CJK Ideograph Extension B, Last>;Lo;0;L;;;;;N;;;;; 2A700;<CJK Ideograph Extension C, First>;Lo;0;L;;;;;N;;;;; -2B738;<CJK Ideograph Extension C, Last>;Lo;0;L;;;;;N;;;;; +2B739;<CJK Ideograph Extension C, Last>;Lo;0;L;;;;;N;;;;; 2B740;<CJK Ideograph Extension D, First>;Lo;0;L;;;;;N;;;;; 2B81D;<CJK Ideograph Extension D, Last>;Lo;0;L;;;;;N;;;;; 2B820;<CJK Ideograph Extension E, First>;Lo;0;L;;;;;N;;;;; @@ -34283,6 +34579,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 2FA1D;CJK COMPATIBILITY IDEOGRAPH-2FA1D;Lo;0;L;2A600;;;;N;;;;; 30000;<CJK Ideograph Extension G, First>;Lo;0;L;;;;;N;;;;; 3134A;<CJK Ideograph Extension G, Last>;Lo;0;L;;;;;N;;;;; +31350;<CJK Ideograph Extension H, First>;Lo;0;L;;;;;N;;;;; +323AF;<CJK Ideograph Extension H, Last>;Lo;0;L;;;;;N;;;;; E0001;LANGUAGE TAG;Cf;0;BN;;;;;N;;;;; E0020;TAG SPACE;Cf;0;BN;;;;;N;;;;; E0021;TAG EXCLAMATION MARK;Cf;0;BN;;;;;N;;;;; diff --git a/lib/stdlib/uc_spec/emoji-data.txt b/lib/stdlib/uc_spec/emoji-data.txt index 7806c7ab53..999a436779 100644 --- a/lib/stdlib/uc_spec/emoji-data.txt +++ b/lib/stdlib/uc_spec/emoji-data.txt @@ -1,13 +1,13 @@ -# emoji-data-14.0.0.txt -# Date: 2021-08-26, 17:22:22 GMT -# ยฉ 2021 Unicodeยฎ, Inc. +# emoji-data.txt +# Date: 2022-08-02, 00:26:10 GMT +# ยฉ 2022 Unicodeยฎ, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. -# For terms of use, see http://www.unicode.org/terms_of_use.html +# For terms of use, see https://www.unicode.org/terms_of_use.html # # Emoji Data for UTS #51 -# Used with Emoji Version 14.0 and subsequent minor revisions (if any) +# Used with Emoji Version 15.0 and subsequent minor revisions (if any) # -# For documentation and usage, see http://www.unicode.org/reports/tr51 +# For documentation and usage, see https://www.unicode.org/reports/tr51 # # Format: # <codepoint(s)> ; <property> # <comments> @@ -19,8 +19,7 @@ # ================================================ -# All omitted code points have Emoji=No -# @missing: 0000..10FFFF ; Emoji ; No +# All omitted code points have Emoji=No 0023 ; Emoji # E0.0 [1] (#๏ธ) hash sign 002A ; Emoji # E0.0 [1] (*๏ธ) asterisk @@ -341,6 +340,7 @@ 1F6D1..1F6D2 ; Emoji # E3.0 [2] (๐..๐) stop sign..shopping cart 1F6D5 ; Emoji # E12.0 [1] (๐) hindu temple 1F6D6..1F6D7 ; Emoji # E13.0 [2] (๐..๐) hut..elevator +1F6DC ; Emoji # E15.0 [1] (๐) wireless 1F6DD..1F6DF ; Emoji # E14.0 [3] (๐..๐) playground slide..ring buoy 1F6E0..1F6E5 ; Emoji # E0.7 [6] (๐ ๏ธ..๐ฅ๏ธ) hammer and wrench..motor boat 1F6E9 ; Emoji # E0.7 [1] (๐ฉ๏ธ) small airplane @@ -401,28 +401,36 @@ 1F9E7..1F9FF ; Emoji # E11.0 [25] (๐งง..๐งฟ) red envelope..nazar amulet 1FA70..1FA73 ; Emoji # E12.0 [4] (๐ฉฐ..๐ฉณ) ballet shoes..shorts 1FA74 ; Emoji # E13.0 [1] (๐ฉด) thong sandal +1FA75..1FA77 ; Emoji # E15.0 [3] (๐ฉต..๐ฉท) light blue heart..pink heart 1FA78..1FA7A ; Emoji # E12.0 [3] (๐ฉธ..๐ฉบ) drop of blood..stethoscope 1FA7B..1FA7C ; Emoji # E14.0 [2] (๐ฉป..๐ฉผ) x-ray..crutch 1FA80..1FA82 ; Emoji # E12.0 [3] (๐ช..๐ช) yo-yo..parachute 1FA83..1FA86 ; Emoji # E13.0 [4] (๐ช..๐ช) boomerang..nesting dolls +1FA87..1FA88 ; Emoji # E15.0 [2] (๐ช..๐ช) maracas..flute 1FA90..1FA95 ; Emoji # E12.0 [6] (๐ช..๐ช) ringed planet..banjo 1FA96..1FAA8 ; Emoji # E13.0 [19] (๐ช..๐ชจ) military helmet..rock 1FAA9..1FAAC ; Emoji # E14.0 [4] (๐ชฉ..๐ชฌ) mirror ball..hamsa +1FAAD..1FAAF ; Emoji # E15.0 [3] (๐ชญ..๐ชฏ) folding hand fan..khanda 1FAB0..1FAB6 ; Emoji # E13.0 [7] (๐ชฐ..๐ชถ) fly..feather 1FAB7..1FABA ; Emoji # E14.0 [4] (๐ชท..๐ชบ) lotus..nest with eggs +1FABB..1FABD ; Emoji # E15.0 [3] (๐ชป..๐ชฝ) hyacinth..wing +1FABF ; Emoji # E15.0 [1] (๐ชฟ) goose 1FAC0..1FAC2 ; Emoji # E13.0 [3] (๐ซ..๐ซ) anatomical heart..people hugging 1FAC3..1FAC5 ; Emoji # E14.0 [3] (๐ซ..๐ซ
) pregnant man..person with crown +1FACE..1FACF ; Emoji # E15.0 [2] (๐ซ..๐ซ) moose..donkey 1FAD0..1FAD6 ; Emoji # E13.0 [7] (๐ซ..๐ซ) blueberries..teapot 1FAD7..1FAD9 ; Emoji # E14.0 [3] (๐ซ..๐ซ) pouring liquid..jar +1FADA..1FADB ; Emoji # E15.0 [2] (๐ซ..๐ซ) ginger root..pea pod 1FAE0..1FAE7 ; Emoji # E14.0 [8] (๐ซ ..๐ซง) melting face..bubbles +1FAE8 ; Emoji # E15.0 [1] (๐ซจ) shaking face 1FAF0..1FAF6 ; Emoji # E14.0 [7] (๐ซฐ..๐ซถ) hand with index finger and thumb crossed..heart hands +1FAF7..1FAF8 ; Emoji # E15.0 [2] (๐ซท..๐ซธ) leftwards pushing hand..rightwards pushing hand -# Total elements: 1404 +# Total elements: 1424 # ================================================ -# All omitted code points have Emoji_Presentation=No -# @missing: 0000..10FFFF ; Emoji_Presentation ; No +# All omitted code points have Emoji_Presentation=No 231A..231B ; Emoji_Presentation # E0.6 [2] (โ..โ) watch..hourglass done 23E9..23EC ; Emoji_Presentation # E0.6 [4] (โฉ..โฌ) fast-forward button..fast down button @@ -625,6 +633,7 @@ 1F6D1..1F6D2 ; Emoji_Presentation # E3.0 [2] (๐..๐) stop sign..shopping cart 1F6D5 ; Emoji_Presentation # E12.0 [1] (๐) hindu temple 1F6D6..1F6D7 ; Emoji_Presentation # E13.0 [2] (๐..๐) hut..elevator +1F6DC ; Emoji_Presentation # E15.0 [1] (๐) wireless 1F6DD..1F6DF ; Emoji_Presentation # E14.0 [3] (๐..๐) playground slide..ring buoy 1F6EB..1F6EC ; Emoji_Presentation # E1.0 [2] (๐ซ..๐ฌ) airplane departure..airplane arrival 1F6F4..1F6F6 ; Emoji_Presentation # E3.0 [3] (๐ด..๐ถ) kick scooter..canoe @@ -681,28 +690,36 @@ 1F9E7..1F9FF ; Emoji_Presentation # E11.0 [25] (๐งง..๐งฟ) red envelope..nazar amulet 1FA70..1FA73 ; Emoji_Presentation # E12.0 [4] (๐ฉฐ..๐ฉณ) ballet shoes..shorts 1FA74 ; Emoji_Presentation # E13.0 [1] (๐ฉด) thong sandal +1FA75..1FA77 ; Emoji_Presentation # E15.0 [3] (๐ฉต..๐ฉท) light blue heart..pink heart 1FA78..1FA7A ; Emoji_Presentation # E12.0 [3] (๐ฉธ..๐ฉบ) drop of blood..stethoscope 1FA7B..1FA7C ; Emoji_Presentation # E14.0 [2] (๐ฉป..๐ฉผ) x-ray..crutch 1FA80..1FA82 ; Emoji_Presentation # E12.0 [3] (๐ช..๐ช) yo-yo..parachute 1FA83..1FA86 ; Emoji_Presentation # E13.0 [4] (๐ช..๐ช) boomerang..nesting dolls +1FA87..1FA88 ; Emoji_Presentation # E15.0 [2] (๐ช..๐ช) maracas..flute 1FA90..1FA95 ; Emoji_Presentation # E12.0 [6] (๐ช..๐ช) ringed planet..banjo 1FA96..1FAA8 ; Emoji_Presentation # E13.0 [19] (๐ช..๐ชจ) military helmet..rock 1FAA9..1FAAC ; Emoji_Presentation # E14.0 [4] (๐ชฉ..๐ชฌ) mirror ball..hamsa +1FAAD..1FAAF ; Emoji_Presentation # E15.0 [3] (๐ชญ..๐ชฏ) folding hand fan..khanda 1FAB0..1FAB6 ; Emoji_Presentation # E13.0 [7] (๐ชฐ..๐ชถ) fly..feather 1FAB7..1FABA ; Emoji_Presentation # E14.0 [4] (๐ชท..๐ชบ) lotus..nest with eggs +1FABB..1FABD ; Emoji_Presentation # E15.0 [3] (๐ชป..๐ชฝ) hyacinth..wing +1FABF ; Emoji_Presentation # E15.0 [1] (๐ชฟ) goose 1FAC0..1FAC2 ; Emoji_Presentation # E13.0 [3] (๐ซ..๐ซ) anatomical heart..people hugging 1FAC3..1FAC5 ; Emoji_Presentation # E14.0 [3] (๐ซ..๐ซ
) pregnant man..person with crown +1FACE..1FACF ; Emoji_Presentation # E15.0 [2] (๐ซ..๐ซ) moose..donkey 1FAD0..1FAD6 ; Emoji_Presentation # E13.0 [7] (๐ซ..๐ซ) blueberries..teapot 1FAD7..1FAD9 ; Emoji_Presentation # E14.0 [3] (๐ซ..๐ซ) pouring liquid..jar +1FADA..1FADB ; Emoji_Presentation # E15.0 [2] (๐ซ..๐ซ) ginger root..pea pod 1FAE0..1FAE7 ; Emoji_Presentation # E14.0 [8] (๐ซ ..๐ซง) melting face..bubbles +1FAE8 ; Emoji_Presentation # E15.0 [1] (๐ซจ) shaking face 1FAF0..1FAF6 ; Emoji_Presentation # E14.0 [7] (๐ซฐ..๐ซถ) hand with index finger and thumb crossed..heart hands +1FAF7..1FAF8 ; Emoji_Presentation # E15.0 [2] (๐ซท..๐ซธ) leftwards pushing hand..rightwards pushing hand -# Total elements: 1185 +# Total elements: 1205 # ================================================ -# All omitted code points have Emoji_Modifier=No -# @missing: 0000..10FFFF ; Emoji_Modifier ; No +# All omitted code points have Emoji_Modifier=No 1F3FB..1F3FF ; Emoji_Modifier # E1.0 [5] (๐ป..๐ฟ) light skin tone..dark skin tone @@ -710,8 +727,7 @@ # ================================================ -# All omitted code points have Emoji_Modifier_Base=No -# @missing: 0000..10FFFF ; Emoji_Modifier_Base ; No +# All omitted code points have Emoji_Modifier_Base=No 261D ; Emoji_Modifier_Base # E0.6 [1] (โ๏ธ) index pointing up 26F9 ; Emoji_Modifier_Base # E0.7 [1] (โน๏ธ) person bouncing ball @@ -762,13 +778,13 @@ 1F9D1..1F9DD ; Emoji_Modifier_Base # E5.0 [13] (๐ง..๐ง) person..elf 1FAC3..1FAC5 ; Emoji_Modifier_Base # E14.0 [3] (๐ซ..๐ซ
) pregnant man..person with crown 1FAF0..1FAF6 ; Emoji_Modifier_Base # E14.0 [7] (๐ซฐ..๐ซถ) hand with index finger and thumb crossed..heart hands +1FAF7..1FAF8 ; Emoji_Modifier_Base # E15.0 [2] (๐ซท..๐ซธ) leftwards pushing hand..rightwards pushing hand -# Total elements: 132 +# Total elements: 134 # ================================================ -# All omitted code points have Emoji_Component=No -# @missing: 0000..10FFFF ; Emoji_Component ; No +# All omitted code points have Emoji_Component=No 0023 ; Emoji_Component # E0.0 [1] (#๏ธ) hash sign 002A ; Emoji_Component # E0.0 [1] (*๏ธ) asterisk @@ -785,8 +801,7 @@ E0020..E007F ; Emoji_Component # E0.0 [96] (๓ ..๓ ฟ) tag space..c # ================================================ -# All omitted code points have Extended_Pictographic=No -# @missing: 0000..10FFFF ; Extended_Pictographic ; No +# All omitted code points have Extended_Pictographic=No 00A9 ; Extended_Pictographic# E0.6 [1] (ยฉ๏ธ) copyright 00AE ; Extended_Pictographic# E0.6 [1] (ยฎ๏ธ) registered @@ -1190,7 +1205,8 @@ E0020..E007F ; Emoji_Component # E0.0 [96] (๓ ..๓ ฟ) tag space..c 1F6D3..1F6D4 ; Extended_Pictographic# E0.0 [2] (๐..๐) STUPA..PAGODA 1F6D5 ; Extended_Pictographic# E12.0 [1] (๐) hindu temple 1F6D6..1F6D7 ; Extended_Pictographic# E13.0 [2] (๐..๐) hut..elevator -1F6D8..1F6DC ; Extended_Pictographic# E0.0 [5] (๐..๐) <reserved-1F6D8>..<reserved-1F6DC> +1F6D8..1F6DB ; Extended_Pictographic# E0.0 [4] (๐..๐) <reserved-1F6D8>..<reserved-1F6DB> +1F6DC ; Extended_Pictographic# E15.0 [1] (๐) wireless 1F6DD..1F6DF ; Extended_Pictographic# E14.0 [3] (๐..๐) playground slide..ring buoy 1F6E0..1F6E5 ; Extended_Pictographic# E0.7 [6] (๐ ๏ธ..๐ฅ๏ธ) hammer and wrench..motor boat 1F6E6..1F6E8 ; Extended_Pictographic# E0.0 [3] (๐ฆ..๐จ) UP-POINTING MILITARY AIRPLANE..UP-POINTING SMALL AIRPLANE @@ -1207,7 +1223,7 @@ E0020..E007F ; Emoji_Component # E0.0 [96] (๓ ..๓ ฟ) tag space..c 1F6FA ; Extended_Pictographic# E12.0 [1] (๐บ) auto rickshaw 1F6FB..1F6FC ; Extended_Pictographic# E13.0 [2] (๐ป..๐ผ) pickup truck..roller skate 1F6FD..1F6FF ; Extended_Pictographic# E0.0 [3] (๐ฝ..๐ฟ) <reserved-1F6FD>..<reserved-1F6FF> -1F774..1F77F ; Extended_Pictographic# E0.0 [12] (๐ด..๐ฟ) <reserved-1F774>..<reserved-1F77F> +1F774..1F77F ; Extended_Pictographic# E0.0 [12] (๐ด..๐ฟ) LOT OF FORTUNE..ORCUS 1F7D5..1F7DF ; Extended_Pictographic# E0.0 [11] (๐..๐) CIRCLED TRIANGLE..<reserved-1F7DF> 1F7E0..1F7EB ; Extended_Pictographic# E12.0 [12] (๐ ..๐ซ) orange circle..brown square 1F7EC..1F7EF ; Extended_Pictographic# E0.0 [4] (๐ฌ..๐ฏ) <reserved-1F7EC>..<reserved-1F7EF> @@ -1266,30 +1282,37 @@ E0020..E007F ; Emoji_Component # E0.0 [96] (๓ ..๓ ฟ) tag space..c 1FA00..1FA6F ; Extended_Pictographic# E0.0 [112] (๐จ..๐ฉฏ) NEUTRAL CHESS KING..<reserved-1FA6F> 1FA70..1FA73 ; Extended_Pictographic# E12.0 [4] (๐ฉฐ..๐ฉณ) ballet shoes..shorts 1FA74 ; Extended_Pictographic# E13.0 [1] (๐ฉด) thong sandal -1FA75..1FA77 ; Extended_Pictographic# E0.0 [3] (๐ฉต..๐ฉท) <reserved-1FA75>..<reserved-1FA77> +1FA75..1FA77 ; Extended_Pictographic# E15.0 [3] (๐ฉต..๐ฉท) light blue heart..pink heart 1FA78..1FA7A ; Extended_Pictographic# E12.0 [3] (๐ฉธ..๐ฉบ) drop of blood..stethoscope 1FA7B..1FA7C ; Extended_Pictographic# E14.0 [2] (๐ฉป..๐ฉผ) x-ray..crutch 1FA7D..1FA7F ; Extended_Pictographic# E0.0 [3] (๐ฉฝ..๐ฉฟ) <reserved-1FA7D>..<reserved-1FA7F> 1FA80..1FA82 ; Extended_Pictographic# E12.0 [3] (๐ช..๐ช) yo-yo..parachute 1FA83..1FA86 ; Extended_Pictographic# E13.0 [4] (๐ช..๐ช) boomerang..nesting dolls -1FA87..1FA8F ; Extended_Pictographic# E0.0 [9] (๐ช..๐ช) <reserved-1FA87>..<reserved-1FA8F> +1FA87..1FA88 ; Extended_Pictographic# E15.0 [2] (๐ช..๐ช) maracas..flute +1FA89..1FA8F ; Extended_Pictographic# E0.0 [7] (๐ช..๐ช) <reserved-1FA89>..<reserved-1FA8F> 1FA90..1FA95 ; Extended_Pictographic# E12.0 [6] (๐ช..๐ช) ringed planet..banjo 1FA96..1FAA8 ; Extended_Pictographic# E13.0 [19] (๐ช..๐ชจ) military helmet..rock 1FAA9..1FAAC ; Extended_Pictographic# E14.0 [4] (๐ชฉ..๐ชฌ) mirror ball..hamsa -1FAAD..1FAAF ; Extended_Pictographic# E0.0 [3] (๐ชญ..๐ชฏ) <reserved-1FAAD>..<reserved-1FAAF> +1FAAD..1FAAF ; Extended_Pictographic# E15.0 [3] (๐ชญ..๐ชฏ) folding hand fan..khanda 1FAB0..1FAB6 ; Extended_Pictographic# E13.0 [7] (๐ชฐ..๐ชถ) fly..feather 1FAB7..1FABA ; Extended_Pictographic# E14.0 [4] (๐ชท..๐ชบ) lotus..nest with eggs -1FABB..1FABF ; Extended_Pictographic# E0.0 [5] (๐ชป..๐ชฟ) <reserved-1FABB>..<reserved-1FABF> +1FABB..1FABD ; Extended_Pictographic# E15.0 [3] (๐ชป..๐ชฝ) hyacinth..wing +1FABE ; Extended_Pictographic# E0.0 [1] (๐ชพ) <reserved-1FABE> +1FABF ; Extended_Pictographic# E15.0 [1] (๐ชฟ) goose 1FAC0..1FAC2 ; Extended_Pictographic# E13.0 [3] (๐ซ..๐ซ) anatomical heart..people hugging 1FAC3..1FAC5 ; Extended_Pictographic# E14.0 [3] (๐ซ..๐ซ
) pregnant man..person with crown -1FAC6..1FACF ; Extended_Pictographic# E0.0 [10] (๐ซ..๐ซ) <reserved-1FAC6>..<reserved-1FACF> +1FAC6..1FACD ; Extended_Pictographic# E0.0 [8] (๐ซ..๐ซ) <reserved-1FAC6>..<reserved-1FACD> +1FACE..1FACF ; Extended_Pictographic# E15.0 [2] (๐ซ..๐ซ) moose..donkey 1FAD0..1FAD6 ; Extended_Pictographic# E13.0 [7] (๐ซ..๐ซ) blueberries..teapot 1FAD7..1FAD9 ; Extended_Pictographic# E14.0 [3] (๐ซ..๐ซ) pouring liquid..jar -1FADA..1FADF ; Extended_Pictographic# E0.0 [6] (๐ซ..๐ซ) <reserved-1FADA>..<reserved-1FADF> +1FADA..1FADB ; Extended_Pictographic# E15.0 [2] (๐ซ..๐ซ) ginger root..pea pod +1FADC..1FADF ; Extended_Pictographic# E0.0 [4] (๐ซ..๐ซ) <reserved-1FADC>..<reserved-1FADF> 1FAE0..1FAE7 ; Extended_Pictographic# E14.0 [8] (๐ซ ..๐ซง) melting face..bubbles -1FAE8..1FAEF ; Extended_Pictographic# E0.0 [8] (๐ซจ..๐ซฏ) <reserved-1FAE8>..<reserved-1FAEF> +1FAE8 ; Extended_Pictographic# E15.0 [1] (๐ซจ) shaking face +1FAE9..1FAEF ; Extended_Pictographic# E0.0 [7] (๐ซฉ..๐ซฏ) <reserved-1FAE9>..<reserved-1FAEF> 1FAF0..1FAF6 ; Extended_Pictographic# E14.0 [7] (๐ซฐ..๐ซถ) hand with index finger and thumb crossed..heart hands -1FAF7..1FAFF ; Extended_Pictographic# E0.0 [9] (๐ซท..๐ซฟ) <reserved-1FAF7>..<reserved-1FAFF> +1FAF7..1FAF8 ; Extended_Pictographic# E15.0 [2] (๐ซท..๐ซธ) leftwards pushing hand..rightwards pushing hand +1FAF9..1FAFF ; Extended_Pictographic# E0.0 [7] (๐ซน..๐ซฟ) <reserved-1FAF9>..<reserved-1FAFF> 1FC00..1FFFD ; Extended_Pictographic# E0.0[1022] (๐ฐ..๐ฟฝ) <reserved-1FC00>..<reserved-1FFFD> # Total elements: 3537 diff --git a/lib/stdlib/uc_spec/gen_unicode_mod.escript b/lib/stdlib/uc_spec/gen_unicode_mod.escript index ecc30b40c6..a6ac3b0e60 100644 --- a/lib/stdlib/uc_spec/gen_unicode_mod.escript +++ b/lib/stdlib/uc_spec/gen_unicode_mod.escript @@ -23,55 +23,69 @@ -mode(compile). --record(cp, {name, class, dec, comp, cs}). +-record(cp, {name, class, dec, comp, cs, cat}). -define(MOD, "unicode_util"). -main(_) -> +main(Args) -> %% Parse main table - {ok, UD} = file:open("../uc_spec/UnicodeData.txt", [read, raw, {read_ahead, 1000000}]), + UD = file_open("../uc_spec/UnicodeData.txt"), Data0 = foldl(fun parse_unicode_data/2, [], UD), Data1 = array:from_orddict(lists:reverse(Data0)), ok = file:close(UD), %% Special Casing table - {ok, SC} = file:open("../uc_spec/SpecialCasing.txt", [read, raw, {read_ahead, 1000000}]), + SC = file_open("../uc_spec/SpecialCasing.txt"), Data2 = foldl(fun parse_special_casing/2, Data1, SC), ok = file:close(SC), %% Casing Folding table - {ok, CF} = file:open("../uc_spec/CaseFolding.txt", [read, raw, {read_ahead, 1000000}]), + CF = file_open("../uc_spec/CaseFolding.txt"), Data = foldl(fun parse_case_folding/2, Data2, CF), ok = file:close(CF), %% Normalization - {ok, ExclF} = file:open("../uc_spec/CompositionExclusions.txt", [read, raw, {read_ahead, 1000000}]), + ExclF = file_open("../uc_spec/CompositionExclusions.txt"), ExclData = foldl(fun parse_comp_excl/2, Data, ExclF), ok = file:close(ExclF), %% GraphemeBreakProperty table - {ok, Emoji} = file:open("../uc_spec/emoji-data.txt", [read, raw, {read_ahead, 1000000}]), + Emoji = file_open("../uc_spec/emoji-data.txt"), Props00 = foldl(fun parse_properties/2, [], Emoji), %% Filter Extended_Pictographic class which we are interested in. Props0 = [EP || {extended_pictographic, _} = EP <- Props00], ok = file:close(Emoji), - {ok, GBPF} = file:open("../uc_spec/GraphemeBreakProperty.txt", [read, raw, {read_ahead, 1000000}]), + GBPF = file_open("../uc_spec/GraphemeBreakProperty.txt"), Props1 = foldl(fun parse_properties/2, Props0, GBPF), ok = file:close(GBPF), - {ok, PropF} = file:open("../uc_spec/PropList.txt", [read, raw, {read_ahead, 1000000}]), + PropF = file_open("../uc_spec/PropList.txt"), Props2 = foldl(fun parse_properties/2, Props1, PropF), ok = file:close(PropF), Props = sofs:to_external(sofs:relation_to_family(sofs:relation(Props2))), + WidthF = file_open("../uc_spec/EastAsianWidth.txt"), + WideCs = foldl(fun parse_widths/2, [], WidthF), + ok = file:close(WidthF), + %% Make module + UpdateTests = case Args of + ["update_tests"] -> true; + _ -> false + end, + {ok, Out} = file:open(?MOD++".erl", [write]), - gen_file(Out, Data, ExclData, maps:from_list(Props)), + gen_file(Out, Data, ExclData, maps:from_list(Props), WideCs, UpdateTests), ok = file:close(Out), ok. +file_open(File) -> + {ok, Fd} = file:open(File, [read, raw, {read_ahead, 1000000}]), + Fd. + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% parse_unicode_data(Line0, Acc) -> Line = string:chomp(Line0), - [CodePoint,Name,_Cat,Class,_BiDi,Decomp, + [CodePoint,Name,Cat,Class,_BiDi,Decomp, _N1,_N2,_N3,_BDMirror,_Uni1,_Iso|Case] = tokens(Line, ";"), {Dec,Comp} = case to_decomp(Decomp) of {_, _} = Compabil -> {[], Compabil}; @@ -79,7 +93,7 @@ parse_unicode_data(Line0, Acc) -> end, [{hex_to_int(CodePoint), #cp{name=list_to_binary(Name),class=to_class(Class), - dec=Dec, comp=Comp, cs=to_case(Case)}} + dec=Dec, comp=Comp, cs=to_case(Case), cat=Cat}} |Acc]. to_class(String) -> @@ -152,7 +166,62 @@ parse_properties(Line0, Acc) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -gen_file(Fd, Data, ExclData, Props) -> +%% Pick ranges that are wide when seen from a non East Asian context, +%% That way we can minimize the data, every other code point is considered narrow. +%% We loose information but hopefully keep the important width for a standard +%% terminal. +parse_widths(Line0, Acc) -> + [{WidthClass, {From, _To}=Range}] = parse_properties(Line0, []), + case is_default_width(From, WidthClass) of + {true, narrow} -> + Acc; + {false, narrow} -> + [Range|Acc]; + {true, RuleRange} -> + [RuleRange|Acc] +%%% {false, rule_execption} -> i.e. narrow codepoint in wide range +%%% Should not happen in current specs + end. + +is_default_width(Index, WD) -> + if + 16#3400 =< Index, Index =< 16#4DBF -> + if WD =:= w orelse WD =:= f -> + {true, {16#3400, 16#4DBF}}; + true -> + {false, rule_execption} + end; + 16#4E00 =< Index, Index =< 16#9FFF -> + if WD =:= w orelse WD =:= f -> + {true, {16#4E00, 16#9FFF}}; + true -> + {false, rule_execption} + end; + 16#F900 =< Index, Index =< 16#FAFF -> + if WD =:= w orelse WD =:= f -> + {true, {16#F900, 16#FAFF}}; + true -> + {false, rule_execption} + end; + 16#20000 =< Index, Index =< 16#2FFFD -> + if WD =:= w orelse WD =:= f -> + {true, {16#20000, 16#2FFFD}}; + true -> + {false, rule_execption} + end; + 16#30000 =< Index, Index =< 16#3FFFD -> + if WD =:= w orelse WD =:= f -> + {true, {16#30000, 16#3FFFD}}; + true -> + {false, rule_execption} + end; + true -> + {WD =:= n orelse WD =:= na orelse WD == h orelse WD =:= a, narrow} + end. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +gen_file(Fd, Data, ExclData, Props, WideCs, UpdateTests) -> gen_header(Fd), gen_static(Fd), gen_norm(Fd), @@ -161,7 +230,8 @@ gen_file(Fd, Data, ExclData, Props) -> gen_gc(Fd, Props), gen_compose_pairs(Fd, ExclData, Data), gen_case_table(Fd, Data), - gen_unicode_table(Fd, Data), + gen_unicode_table(Fd, Data, UpdateTests), + gen_width_table(Fd, WideCs), ok. gen_header(Fd) -> @@ -173,26 +243,32 @@ gen_header(Fd) -> io:put_chars(Fd, "-export([whitespace/0, is_whitespace/1]).\n"), io:put_chars(Fd, "-export([uppercase/1, lowercase/1, titlecase/1, casefold/1]).\n\n"), io:put_chars(Fd, "-export([spec_version/0, lookup/1, get_case/1]).\n"), + io:put_chars(Fd, "-export([is_wide/1]).\n"), io:put_chars(Fd, "-compile({inline, [class/1]}).\n"), io:put_chars(Fd, "-compile(nowarn_unused_vars).\n"), io:put_chars(Fd, "-dialyzer({no_improper_lists, [cp/1, gc/1, gc_prepend/2]}).\n"), - io:put_chars(Fd, "-type gc() :: char()|[char()].\n\n\n"), + io:put_chars(Fd, "-type gc() :: char()|[char()].\n\n"), + io:put_chars(Fd, "-define(IS_CP(CP), (is_integer(CP) andalso 0 =< CP andalso CP < 16#110000)).\n\n\n"), ok. gen_static(Fd) -> io:put_chars(Fd, "-spec lookup(char()) -> #{'canon':=[{byte(),char()}], 'ccc':=byte(), " - "'compat':=[] | {atom(),[{byte(),char()}]}}.\n"), - io:put_chars(Fd, "lookup(Codepoint) ->\n" - " {CCC,Can,Comp} = unicode_table(Codepoint),\n" - " #{ccc=>CCC, canon=>Can, compat=>Comp}.\n\n"), + "'compat':=[] | {atom(),[{byte(),char()}]}, 'category':={atom(),atom()}}.\n"), + io:put_chars(Fd, "lookup(Codepoint) when ?IS_CP(Codepoint) ->\n" + " {CCC,Can,Comp,Cat} = unicode_table(Codepoint),\n" + " #{ccc=>CCC, canon=>Can, compat=>Comp, category=>category(Codepoint,Cat)}.\n\n"), + io:put_chars(Fd, "-spec get_case(char()) -> #{'fold':=gc(), 'lower':=gc(), 'title':=gc(), 'upper':=gc()}.\n"), - io:put_chars(Fd, "get_case(Codepoint) ->\n" + io:put_chars(Fd, "get_case(Codepoint) when ?IS_CP(Codepoint) ->\n" " case case_table(Codepoint) of\n" " {U,L} -> #{upper=>U,lower=>L,title=>U,fold=>L};\n" " {U,L,T,F} -> #{upper=>U,lower=>L,title=>T,fold=>F}\n" " end.\n\n"), - io:put_chars(Fd, "spec_version() -> {14,0}.\n\n\n"), - io:put_chars(Fd, "class(Codepoint) -> {CCC,_,_} = unicode_table(Codepoint),\n CCC.\n\n"), + + io:put_chars(Fd, "spec_version() -> {15,0}.\n\n\n"), + io:put_chars(Fd, "class(Codepoint) when ?IS_CP(Codepoint) -> \n" + " {CCC,_,_,_} = unicode_table(Codepoint),\n CCC.\n\n"), + io:put_chars(Fd, "-spec uppercase(unicode:chardata()) -> " "maybe_improper_list(gc(),unicode:chardata()).\n"), io:put_chars(Fd, "uppercase(Str0) ->\n"), @@ -217,6 +293,7 @@ gen_static(Fd) -> io:put_chars(Fd, " [] -> [];\n"), io:put_chars(Fd, " {error,Err} -> error({badarg, Err})\n"), io:put_chars(Fd, " end.\n\n"), + io:put_chars(Fd, "-spec titlecase(unicode:chardata()) -> " "maybe_improper_list(gc(),unicode:chardata()).\n"), io:put_chars(Fd, "titlecase(Str0) ->\n"), @@ -229,6 +306,7 @@ gen_static(Fd) -> io:put_chars(Fd, " [] -> [];\n"), io:put_chars(Fd, " {error,Err} -> error({badarg, Err})\n"), io:put_chars(Fd, " end.\n\n"), + io:put_chars(Fd, "-spec casefold(unicode:chardata()) -> " "maybe_improper_list(gc(),unicode:chardata()).\n"), io:put_chars(Fd, "casefold(Str0) ->\n"), @@ -242,6 +320,19 @@ gen_static(Fd) -> io:put_chars(Fd, " {error,Err} -> error({badarg, Err})\n"), io:put_chars(Fd, " end.\n\n"), + io:put_chars(Fd, "%% Returns true if the character is considered wide in non east asian context.\n"), + io:put_chars(Fd, "-spec is_wide(gc()) -> boolean().\n"), + io:put_chars(Fd, "is_wide(C) when ?IS_CP(C) ->\n"), + io:put_chars(Fd, " is_wide_cp(C);\n"), + io:put_chars(Fd, "is_wide([_, 16#FE0E|Cs]) -> true; %% Presentation sequence\n"), + io:put_chars(Fd, "is_wide([_, 16#FE0F|Cs]) -> true; %% Presentation sequence\n"), + io:put_chars(Fd, "is_wide([C|Cs]) when ?IS_CP(C) ->\n"), + io:put_chars(Fd, " is_wide_cp(C) orelse is_wide(Cs);\n"), + io:put_chars(Fd, "is_wide([]) ->\n false.\n\n"), + + io:put_chars(Fd, "category(CP, lookup_category) ->\n" + " cat_translate(lookup_category(CP));\n" + "category(_, Def) -> cat_translate(Def).\n\n"), ok. gen_norm(Fd) -> @@ -249,7 +340,7 @@ gen_norm(Fd) -> "-spec nfd(unicode:chardata()) -> maybe_improper_list(gc(),unicode:chardata()) | {error, unicode:chardata()}.\n" "nfd(Str0) ->\n" " case gc(Str0) of\n" - " [GC|R] when GC < 128 -> [GC|R];\n" + " [GC|R] when is_integer(GC), 0 =< GC, GC < 128 -> [GC|R];\n" " [GC|Str] -> [decompose(GC)|Str];\n" " [] -> [];\n" " {error,_}=Error -> Error\n end.\n\n" @@ -259,7 +350,7 @@ gen_norm(Fd) -> "-spec nfkd(unicode:chardata()) -> maybe_improper_list(gc(),unicode:chardata()) | {error, unicode:chardata()}.\n" "nfkd(Str0) ->\n" " case gc(Str0) of\n" - " [GC|R] when GC < 128 -> [GC|R];\n" + " [GC|R] when is_integer(GC), 0 =< GC, GC < 128 -> [GC|R];\n" " [GC|Str] -> [decompose_compat(GC)|Str];\n" " [] -> [];\n" " {error,_}=Error -> Error\n end.\n\n" @@ -269,7 +360,7 @@ gen_norm(Fd) -> "-spec nfc(unicode:chardata()) -> maybe_improper_list(gc(),unicode:chardata()) | {error, unicode:chardata()}.\n" "nfc(Str0) ->\n" " case gc(Str0) of\n" - " [GC|R] when GC < 256 -> [GC|R];\n" + " [GC|R] when is_integer(GC), 0 =< GC, GC < 256 -> [GC|R];\n" " [GC|Str] -> [compose(decompose(GC))|Str];\n" " [] -> [];\n" " {error,_}=Error -> Error\n end.\n\n" @@ -279,7 +370,7 @@ gen_norm(Fd) -> "-spec nfkc(unicode:chardata()) -> maybe_improper_list(gc(),unicode:chardata()) | {error, unicode:chardata()}.\n" "nfkc(Str0) ->\n" " case gc(Str0) of\n" - " [GC|R] when GC < 128 -> [GC|R];\n" + " [GC|R] when is_integer(GC), 0 =< GC, GC < 128 -> [GC|R];\n" " [GC|Str] -> [compose_compat_0(decompose_compat(GC))|Str];\n" " [] -> [];\n" " {error,_}=Error -> Error\n end.\n\n" @@ -288,13 +379,13 @@ gen_norm(Fd) -> io:put_chars(Fd, "decompose(CP) when is_integer(CP), CP < 16#AC00, 16#D7A3 > CP ->\n" " case unicode_table(CP) of\n" - " {_,[],_} -> CP;\n" - " {_,CPs,_} -> canonical_order(CPs)\n" + " {_,[],_,_} -> CP;\n" + " {_,CPs,_,_} -> canonical_order(CPs)\n" " end;\n" "decompose(CP) ->\n" " canonical_order(decompose_1(CP)).\n" "\n" - "decompose_1(CP) when 16#AC00 =< CP, CP =< 16#D7A3 ->\n" + "decompose_1(CP) when is_integer(CP), 16#AC00 =< CP, CP =< 16#D7A3 ->\n" " Syll = CP-16#AC00,\n" " T = 28,\n" " N = 588,\n" @@ -306,8 +397,8 @@ gen_norm(Fd) -> " end;\n" "decompose_1(CP) when is_integer(CP) ->\n" " case unicode_table(CP) of\n" - " {CCC, [],_} -> [{CCC,CP}];\n" - " {_, CPs, _} -> CPs\n" + " {CCC, [],_,_} -> [{CCC,CP}];\n" + " {_,CPs,_,_} -> CPs\n" " end;\n" "decompose_1([CP|CPs]) ->\n" " decompose_1(CP) ++ decompose_1(CPs);\n" @@ -331,14 +422,14 @@ gen_norm(Fd) -> io:put_chars(Fd, "decompose_compat(CP) when is_integer(CP), CP < 16#AC00, 16#D7A3 > CP ->\n" " case unicode_table(CP) of\n" - " {_, [], []} -> CP;\n" - " {_, _, {_,CPs}} -> canonical_order(CPs);\n" - " {_, CPs, _} -> canonical_order(CPs)\n" + " {_, [], [], _} -> CP;\n" + " {_, _, {_,CPs}, _} -> canonical_order(CPs);\n" + " {_, CPs, _, _} -> canonical_order(CPs)\n" " end;\n" "decompose_compat(CP) ->\n" " canonical_order(decompose_compat_1(CP)).\n" "\n" - "decompose_compat_1(CP) when 16#AC00 =< CP, CP =< 16#D7A3 ->\n" + "decompose_compat_1(CP) when is_integer(CP), 16#AC00 =< CP, CP =< 16#D7A3 ->\n" " Syll = CP-16#AC00,\n" " T = 28,\n" " N = 588,\n" @@ -350,23 +441,24 @@ gen_norm(Fd) -> " end;\n" "decompose_compat_1(CP) when is_integer(CP) ->\n" " case unicode_table(CP) of\n" - " {CCC, [], []} -> [{CCC,CP}];\n" - " {_, _, {_,CPs}} -> CPs;\n" - " {_, CPs, _} -> CPs\n" + " {CCC, [], [], _} -> [{CCC,CP}];\n" + " {_, _, {_,CPs}, _} -> CPs;\n" + " {_, CPs, _, _} -> CPs\n" " end;\n" "decompose_compat_1([CP|CPs]) ->\n" " decompose_compat_1(CP) ++ decompose_compat_1(CPs);\n" - "decompose_compat_1([]) -> [].\n"), + "decompose_compat_1([]) -> [].\n\n"), io:put_chars(Fd, "compose(CP) when is_integer(CP) -> CP;\n" "compose([Lead,Vowel|Trail]) %% Hangul\n" - " when 16#1100 =< Lead, Lead =< 16#1112 ->\n" + " when is_integer(Lead), 16#1100 =< Lead, Lead =< 16#1112, is_integer(Vowel) ->\n" " if 16#1161 =< Vowel, Vowel =< 16#1175 ->\n" " CP = 16#AC00 + ((Lead - 16#1100) * 588) + ((Vowel - 16#1161) * 28),\n" " case Trail of\n" - " [T|Acc] when 16#11A7 =< T, T =< 16#11C2 -> nolist(CP+T-16#11A7,Acc);\n" + " [T|Acc] when is_integer(T), 16#11A7 =< T, T =< 16#11C2 ->" + " nolist(CP+T-16#11A7,Acc);\n" " Acc -> nolist(CP,Acc)\n" " end;\n" " true ->\n" @@ -408,11 +500,12 @@ gen_norm(Fd) -> " end.\n\n" "compose_compat(CP) when is_integer(CP) -> CP;\n" "compose_compat([Lead,Vowel|Trail]) %% Hangul\n" - " when 16#1100 =< Lead, Lead =< 16#1112 ->\n" + " when is_integer(Lead), 16#1100 =< Lead, Lead =< 16#1112, is_integer(Vowel) ->\n" " if 16#1161 =< Vowel, Vowel =< 16#1175 ->\n" " CP = 16#AC00 + ((Lead - 16#1100) * 588) + ((Vowel - 16#1161) * 28),\n" " case Trail of\n" - " [T|Acc] when 16#11A7 =< T, T =< 16#11C2 -> nolist(CP+T-16#11A7,Acc);\n" + " [T|Acc] when is_integer(T), 16#11A7 =< T, T =< 16#11C2 ->" + " nolist(CP+T-16#11A7,Acc);\n" " Acc -> nolist(CP,Acc)\n" " end;\n" " true ->\n" @@ -462,7 +555,7 @@ gen_ws(Fd, Props) -> gen_cp(Fd) -> io:put_chars(Fd, "-spec cp(String::unicode:chardata()) ->" " maybe_improper_list() | {error, unicode:chardata()}.\n"), - io:put_chars(Fd, "cp([C|_]=L) when is_integer(C) -> L;\n"), + io:put_chars(Fd, "cp([C|_]=L) when ?IS_CP(C) -> L;\n"), io:put_chars(Fd, "cp([List]) -> cp(List);\n"), io:put_chars(Fd, "cp([List|R]) -> cpl(List, R);\n"), io:put_chars(Fd, "cp([]) -> [];\n"), @@ -470,8 +563,8 @@ gen_cp(Fd) -> io:put_chars(Fd, "cp(<<>>) -> [];\n"), io:put_chars(Fd, "cp(<<R/binary>>) -> {error,R}.\n"), io:put_chars(Fd, "\n"), - io:put_chars(Fd, "cpl([C], R) when is_integer(C) -> [C|cpl_1_cont(R)];\n"), - io:put_chars(Fd, "cpl([C|T], R) when is_integer(C) -> [C|cpl_cont(T, R)];\n"), + io:put_chars(Fd, "cpl([C], R) when ?IS_CP(C) -> [C|cpl_1_cont(R)];\n"), + io:put_chars(Fd, "cpl([C|T], R) when ?IS_CP(C) -> [C|cpl_cont(T, R)];\n"), io:put_chars(Fd, "cpl([List], R) -> cpl(List, R);\n"), io:put_chars(Fd, "cpl([List|T], R) -> cpl(List, [T|R]);\n"), io:put_chars(Fd, "cpl([], R) -> cp(R);\n"), @@ -542,18 +635,18 @@ gen_gc(Fd, GBP) -> " maybe_improper_list() | {error, unicode:chardata()}.\n"), io:put_chars(Fd, "gc([]=R) -> R;\n" - "gc([CP]=R) when is_integer(CP) -> R;\n" + "gc([CP]=R) when ?IS_CP(CP) -> R;\n" "gc([$\\r=CP|R0]) ->\n" " case cp(R0) of % Don't break CRLF\n" " [$\\n|R1] -> [[$\\r,$\\n]|R1];\n" " T -> [CP|T]\n" " end;\n" - "gc([CP1|T1]=T) when CP1 < 256 ->\n" + "gc([CP1|T1]=T) when ?IS_CP(CP1), CP1 < 256 ->\n" " case T1 of\n" - " [CP2|_] when CP2 < 256 -> T; %% Ascii Fast path\n" + " [CP2|_] when is_integer(CP2), 0 =< CP2, CP2 < 256 -> T; %% Ascii Fast path\n" " _ -> %% Keep the tail binary.\n" " case cp_no_bin(T1) of\n" - " [CP2|_]=T3 when CP2 < 256 -> [CP1|T3]; %% Asciii Fast path\n" + " [CP2|_]=T3 when is_integer(CP2), 0 =< CP2, CP2 < 256 -> [CP1|T3]; %% Asciii Fast path\n" " binary_found -> gc_1(T);\n" " T4 -> gc_1([CP1|T4])\n" " end\n" @@ -568,7 +661,7 @@ gen_gc(Fd, GBP) -> " end;\n" " true -> gc_1([CP1|Rest])\n" " end;\n" - "gc([CP|_]=T) when is_integer(CP) -> gc_1(T);\n" + "gc([CP|_]=T) when ?IS_CP(CP) -> gc_1(T);\n" "gc(Str) ->\n" " case cp(Str) of\n" " {error,_}=Error -> Error;\n" @@ -596,13 +689,14 @@ gen_gc(Fd, GBP) -> io:put_chars(Fd, "\n%% Optimize Latin-1\n"), [GenExtP(CP) || CP <- merge_ranges(ExtendedPictographicLow)], - io:format(Fd, - "gc_1([CP|R]=R0) when CP < 256 ->\n" - " case R of\n" - " [CP2|_] when CP2 < 256 -> R0;\n" - " _ -> gc_extend(cp(R), R, CP)\n" - " end;\n", - []), + io:put_chars(Fd, + "gc_1([CP|R]=R0) when is_integer(CP), 0 =< CP, CP < 256 ->\n" + " case R of\n" + " [CP2|_] when is_integer(CP2), 0 =< CP2, CP2 < 256 -> R0;\n" + " _ -> gc_extend(cp(R), R, CP)\n" + " end;\n" + "gc_1([CP|_]) when not ?IS_CP(CP) ->\n" + " error({badarg,CP});\n"), io:put_chars(Fd, "\n%% Continue control\n"), [GenControl(CP) || CP <- Crs], %% One clause per CP @@ -623,7 +717,7 @@ gen_gc(Fd, GBP) -> GenHangulT = fun(Range) -> io:format(Fd, "gc_1~s gc_h_T(R1,[CP]);\n", [gen_clause(Range)]) end, [GenHangulT(CP) || CP <- merge_ranges(maps:get(t,GBP))], io:put_chars(Fd, "%% Handle Hangul LV and LVT special, since they are large\n"), - io:put_chars(Fd, "gc_1([CP|_]=R0) when 44000 < CP, CP < 56000 -> gc_h_lv_lvt(R0, R0, []);\n"), + io:put_chars(Fd, "gc_1([CP|_]=R0) when is_integer(CP), 44000 < CP, CP < 56000 -> gc_h_lv_lvt(R0, R0, []);\n"), io:put_chars(Fd, "\n%% Handle Regional\n"), GenRegional = fun(Range) -> io:format(Fd, "gc_1~s gc_regional(R1,CP);\n", [gen_clause(Range)]) end, @@ -739,7 +833,9 @@ gen_gc(Fd, GBP) -> [{RLess,RLarge}] = merge_ranges(maps:get(regional_indicator,GBP)), io:put_chars(Fd,"gc_regional(R0, CP0) ->\n" " case cp(R0) of\n"), - io:format(Fd, " [CP|R1] when ~w =< CP,CP =< ~w-> gc_extend2(cp(R1),R1,[CP,CP0]);~n",[RLess, RLarge]), + io:format(Fd, " [CP|R1] when is_integer(CP), ~w =< CP, CP =< ~w ->\n" + " gc_extend2(cp(R1),R1,[CP,CP0]);~n", + [RLess, RLarge]), io:put_chars(Fd," R1 -> gc_extend(R1, R0, CP0)\n" " end.\n\n"), @@ -780,6 +876,7 @@ gen_gc(Fd, GBP) -> " _ -> gc_extend2(R1, R0, Acc)\n" " end\n end.\n\n"), io:put_chars(Fd, "%% Handle Hangul LV\n"), + io:put_chars(Fd, "gc_h_lv_lvt([CP|_], _R0, _Acc) when not ?IS_CP(CP) -> error(badarg);\n"), GenHangulLV = fun(Range) -> io:format(Fd, "gc_h_lv_lvt~s gc_h_V(R1,[CP|Acc]);\n", [gen_clause2(Range)]) end, [GenHangulLV(CP) || CP <- merge_ranges(maps:get(lv,GBP))], @@ -806,50 +903,241 @@ gen_compose_pairs(Fd, ExclData, Data) -> [io:format(Fd, "compose_pair(~w,~w) -> ~w;~n", [A,B,CP]) || {[A,B],CP} <- lists:sort(DeCmp2)], io:put_chars(Fd, "compose_pair(_,_) -> false.\n\n"), - io:put_chars(Fd, "nolist(CP, []) -> CP;\nnolist(CP,L) -> [CP|L].\n\n"), + io:put_chars(Fd, "nolist(CP, []) when ?IS_CP(CP) -> CP;\n" + "nolist(CP, L) when ?IS_CP(CP) -> [CP|L].\n\n"), ok. gen_case_table(Fd, Data) -> - Case = array:foldr(fun(CP, #cp{cs={U0,L0,T0,F0}}, Acc) -> - U = def_cp(U0,CP), - L = def_cp(L0,CP), - T = def_cp(T0,CP), - F = def_cp(F0,CP), - case T =:= U andalso F =:= L of - true -> - [{CP,{U,L}}|Acc]; - false -> - [{CP,{U,L,T,F}}|Acc] - end; - (_CP, _, Acc) -> Acc - end, [], Data), + HC = fun(CP, #cp{cs=Cs}, Acc) -> + case case_data(CP, Cs) of + default -> Acc; + CaseData -> [{CP,CaseData}|Acc] + end + end, + Case = array:sparse_foldr(HC, [], Data), [io:format(Fd, "case_table(~w) -> ~w;\n", [CP, Map])|| {CP,Map} <- Case], io:format(Fd, "case_table(CP) -> {CP, CP}.\n\n",[]), ok. +case_data(CP, {U0,L0,T0,F0}) -> + U = def_cp(U0,CP), + L = def_cp(L0,CP), + T = def_cp(T0,CP), + F = def_cp(F0,CP), + case T =:= U andalso F =:= L of + true -> {U,L}; + false -> {U,L,T,F} + end; +case_data(_, _) -> + default. + def_cp([], CP) -> CP; def_cp(CP, _) -> CP. -gen_unicode_table(Fd, Data) -> - FixCanon = fun(_, #cp{class=CCC, dec=Dec, comp=Comp}) -> +gen_unicode_table(Fd, Data, UpdateTests) -> + FixCanon = fun(_, #cp{class=CCC, dec=Dec, comp=Comp, cat=Cat}) -> Canon = decompose(Dec,Data), - #{ccc=>CCC, canonical=>Canon, compat=>Comp} + #{ccc=>CCC, canonical=>Canon, compat=>Comp, cat=>Cat} end, AofMaps0 = array:sparse_map(FixCanon, Data), - FixCompat = fun(_, #{ccc:=CCC, canonical:=Canon, compat:=Comp}) -> + FixCompat = fun(_, #{ccc:=CCC, canonical:=Canon, compat:=Comp, cat:=Cat}) -> Compat = decompose_compat(Canon, Comp, AofMaps0), - {CCC, Canon, Compat} + {CCC, Canon, Compat, category(Cat)} end, AofMaps1 = array:sparse_map(FixCompat, AofMaps0), Dict0 = array:sparse_to_orddict(AofMaps1), - Def = {0, [], []}, - Dict = lists:filter(fun({_, Map}) -> Map =/= Def end, Dict0), + Def = {0, [], [], lookup_category}, + {NonDef, CatTable} = lists:partition(fun({_, {0,[],[],_Cat}}) -> false; + (_) -> true + end, Dict0), + + %% Export testfile + case UpdateTests of + true -> + Dict1 = lists:map(fun({Id,{CCC, Canon, Compat, Cat}}) -> + {_, ECat} = lists:keyfind(Cat, 1, category_translate()), + {Id, {CCC, Canon, Compat, ECat}} + end, Dict0), + TestFile = "../test/unicode_util_SUITE_data/unicode_table.bin", + io:format("Updating: ~s~n", [TestFile]), + file:write_file(TestFile, term_to_binary(Dict1, [compressed])); + false -> + ignore + end, - [io:format(Fd, "unicode_table(~w) -> ~w;~n", [CP, Map]) || {CP,Map} <- Dict], + [io:format(Fd, "unicode_table(~w) -> ~w;~n", [CP, Map]) || {CP,Map} <- NonDef], io:format(Fd, "unicode_table(_) -> ~w.~n~n",[Def]), + + [io:format(Fd, "cat_translate(~w) -> ~w;~n", [Cat, EC]) || {Cat,EC} <- category_translate()], + io:format(Fd, "cat_translate(Cat) -> error({internal_error, Cat}).~n~n",[]), + gen_category(Fd, CatTable, Data), + ok. + +category([C,Sub]) -> + list_to_atom([C-$A+$a, Sub]). + +category_translate() -> + [{lu, {letter, uppercase}}, % Letter, Uppercase + {ll, {letter, lowercase}}, % Letter, Lowercase + {lt, {letter, titlecase}}, % Letter, Titlecase + {mn, {mark, non_spacing}}, % Mark, Non-Spacing + {mc, {mark, spacing_combining}}, % Mark, Spacing Combining + {me, {mark, enclosing}}, % Mark, Enclosing + {nd, {number, decimal}}, % Number, Decimal Digit + {nl, {number, letter}}, % Number, Letter + {no, {number, other}}, % Number, Other + {zs, {separator, space}}, % Separator, Space + {zl, {separator, line}}, % Separator, Line + {zp, {separator, paragraph}}, % Separator, Paragraph + {cc, {other, control}}, % Other, Control + {cf, {other, format}}, % Other, Format + {cs, {other, surrogate}}, % Other, Surrogate + {co, {other, private}}, % Other, Private Use + {cn, {other, not_assigned}}, % Other, Not Assigned (no characters in the file have this property) + {lm, {letter, modifier}}, % Letter, Modifier + {lo, {letter, other}}, % Letter, Other + {pc, {punctuation, connector}}, % Punctuation, Connector + {pd, {punctuation, dash}}, % Punctuation, Dash + {ps, {punctuation, open}}, % Punctuation, Open + {pe, {punctuation, close}}, % Punctuation, Close + {pi, {punctuation, initial}}, % Punctuation, Initial quote (may behave like Ps or Pe depending on usage) + {pf, {punctuation, final}}, % Punctuation, Final quote (may behave like Ps or Pe depending on usage) + {po, {punctuation, other}}, % Punctuation, Other + {sm, {symbol, math}}, % Symbol, Math + {sc, {symbol, currency}}, % Symbol, Currency + {sk, {symbol, modifier}}, % Symbol, Modifier + {so, {symbol, other}}]. % Symbol, Other + +gen_category(Fd, [{CP, {_, _, _, Cat}}|Rest], All) -> + gen_category(Fd, Rest, Cat, CP, CP, All, []). + +gen_category(Fd, [{CP, {_, _, _, NextCat}}|Rest], Cat, Start, End, All, Acc) + when End+1 =:= CP -> + IsLetterCat = letter_cat(NextCat, Cat), + if NextCat =:= Cat -> + gen_category(Fd, Rest, Cat, Start, CP, All, Acc); + IsLetterCat -> + gen_category(Fd, Rest, letter, Start, CP, All, Acc); + Start =:= End -> + io:format(Fd, "lookup_category(~w) -> ~w;~n", [Start, Cat]), + gen_category(Fd, Rest, NextCat, CP, CP, All, Acc); + true -> + case Cat of + letter -> + io:format(Fd, "lookup_category(CP) when ~w =< CP, CP =< ~w-> subcat_letter(CP);~n", + [Start, End]), + gen_category(Fd, Rest, NextCat, CP, CP, All, + lists:reverse(lists:seq(Start, End)) ++ Acc); + _ -> + io:format(Fd, "lookup_category(CP) when ~w =< CP, CP =< ~w-> ~w;~n", [Start, End, Cat]), + gen_category(Fd, Rest, NextCat, CP, CP, All, Acc) + end + end; +gen_category(Fd, [{CP, {_, _, _, NewCat}}|Rest]=Cont, Cat, Start, End, All, Acc) -> + case array:get(End+1, All) of + undefined -> + if Start =:= End -> + io:format(Fd, "lookup_category(~w) -> ~w;~n", [Start, Cat]), + gen_category(Fd, Rest, NewCat, CP, CP, All, Acc); + true -> + case Cat of + letter -> + io:format(Fd, "lookup_category(CP) when ~w =< CP, CP =< ~w-> subcat_letter(CP);~n", + [Start, End]), + gen_category(Fd, Rest, NewCat, CP, CP, All, + lists:reverse(lists:seq(Start, End)) ++ Acc); + _ -> + io:format(Fd, "lookup_category(CP) when ~w =< CP, CP =< ~w -> ~w;~n", + [Start, End, Cat]), + gen_category(Fd, Rest, NewCat, CP, CP, All, Acc) + end + end; + _ -> %% We can make ranges larger by setting already assigned category + gen_category(Fd, Cont, Cat, Start, End+1, All, Acc) + end; +gen_category(Fd, [], Cat, Start, End, All, Acc) -> + case Start =:= End of + true -> + io:format(Fd, "lookup_category(~w) -> ~w;~n", [Start, Cat]); + false -> + io:format(Fd, "lookup_category(CP) when ~w =< CP, CP =< ~w -> ~w;~n", [Start, End, Cat]) + end, + io:put_chars(Fd, "lookup_category(Cp) -> cn.\n\n"), + gen_letter(Fd, lists:reverse(Acc), All), ok. +letter_cat(lm, _) -> + false; +letter_cat(_, lm) -> + false; +letter_cat(L1, L2) -> + is_letter(L1) andalso (L2 =:= letter orelse is_letter(L2)). + +is_letter(LC) -> + lists:member(LC, [lu,ll,lt,lo,lm]). + +gen_letter(Fd, Letters, All) -> + gen_letter(Fd, Letters, All, []). +gen_letter(Fd, [CP|Rest], All, Acc) -> + case array:get(CP, All) of + undefined -> + gen_letter(Fd, Rest, All, Acc); + #cp{cat=Cat0, cs=Cs} -> + case {category(Cat0), case_table(CP,case_data(CP, Cs))} of + {Sub,Sub} -> + gen_letter(Fd, Rest, All, Acc); + {lm,_} -> + gen_letter(Fd, Rest, All, Acc); + {Cat, _Dbg} -> + case is_letter(Cat) of + true -> + gen_letter(Fd, Rest, All, [{CP, Cat}|Acc]); + false -> + gen_letter(Fd, Rest, All, Acc) + end + end + end; +gen_letter(Fd, [], _, Acc) -> + [{Start, Cat}|SCletters] = lists:reverse(Acc), + subcat_letter(Fd, SCletters, Start, Start, Cat), + io:put_chars(Fd, + "subcat_letter(CP) ->\n" + " case case_table(CP) of\n" + " {CP, CP} -> lo; %{letter,other};\n" + " {CP, _} -> lu; %{letter,uppercase};\n" + " {_, CP} -> ll; %{letter,lowercase};\n" + " {_, _, CP, _} -> lt; %{letter,titlecase};\n" + " {CP, _, _, _} -> lu; %{letter,uppercase};\n" + " {_,CP,_,_} -> ll %{letter,lowercase}\n" + " end.\n\n"). + +subcat_letter(Fd, [{CP, Cat}|R], Start, End, Cat) when End+1 =:= CP -> + subcat_letter(Fd, R, Start, CP, Cat); +subcat_letter(Fd, Rest, Start, Start, Cat) -> + io:format(Fd, "subcat_letter(~w) -> ~w;\n",[Start,Cat]), + case Rest of + [] -> ok; + [{CP, NewCat}|R] -> subcat_letter(Fd, R, CP, CP, NewCat) + end; +subcat_letter(Fd, Rest, Start, End, Cat) -> + io:format(Fd, "subcat_letter(CP) when ~w =< CP, CP =< ~w -> ~w;\n",[Start,End,Cat]), + case Rest of + [] -> ok; + [{CP, NewCat}|R] -> subcat_letter(Fd, R, CP, CP, NewCat) + end. + +case_table(CP, CaseData) -> + case CaseData of + {CP, CP} -> lo; + {CP, _} -> lu; + {_, CP} -> ll; + {_, _, CP, _} -> lt; + {CP, _, _, _} -> lu; + {_,CP,_,_} -> ll; + default -> lo + end. + decompose([], _Data) -> []; decompose([CP|CPs], Data) when is_integer(CP) -> case array:get(CP,Data) of @@ -883,35 +1171,41 @@ decompose_compat([{_,CP}|CPs], Data) -> decompose_compat([CP|CPs], Data). +gen_width_table(Fd, WideChars) -> + MergedWCs = merge_ranges(WideChars), + Write = fun(Range) -> io:format(Fd, "is_wide_cp~s true;~n", [gen_single_clause(Range)]) end, + [Write(Range) || Range <- MergedWCs], + io:format(Fd, "is_wide_cp(_) -> false.~n", []). + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% gen_clause({R0, undefined}) -> io_lib:format("([~w=CP|R1]=R0) ->", [R0]); gen_clause({R0, R1}) -> - io_lib:format("([CP|R1]=R0) when ~w =< CP, CP =< ~w ->", [R0,R1]). + io_lib:format("([CP|R1]=R0) when is_integer(CP), ~w =< CP, CP =< ~w ->", [R0,R1]). gen_clause2({R0, undefined}) -> io_lib:format("([~w=CP|R1], R0, Acc) ->", [R0]); gen_clause2({R0, R1}) -> - io_lib:format("([CP|R1], R0, Acc) when ~w =< CP, CP =< ~w ->", [R0,R1]). + io_lib:format("([CP|R1], R0, Acc) when is_integer(CP), ~w =< CP, CP =< ~w ->", [R0,R1]). gen_case_clause({R0, undefined}) -> io_lib:format("[~w=CP|R1] ->", [R0]); gen_case_clause({R0, R1}) -> - io_lib:format("[CP|R1] when ~w =< CP, CP =< ~w ->", [R0,R1]). + io_lib:format("[CP|R1] when is_integer(CP), ~w =< CP, CP =< ~w ->", [R0,R1]). gen_single_clause({R0, undefined}) -> io_lib:format("(~w) ->", [R0]); gen_single_clause({R0, R1}) -> - io_lib:format("(CP) when ~w =< CP, CP =< ~w ->", [R0,R1]). + io_lib:format("(CP) when is_integer(CP), ~w =< CP, CP =< ~w ->", [R0,R1]). merge_ranges(List) -> merge_ranges(List, true). merge_ranges(List, Opt) -> - Res0 = merge_ranges_1(lists:sort(List), []), + Res0 = merge_ranges_1(lists:usort(List), []), case Opt of split -> split_ranges(Res0,[]); % One clause per CP |