diff options
65 files changed, 2707 insertions, 1175 deletions
diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index d74b31f4eb..1c65e56799 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -1149,7 +1149,8 @@ RUN_OBJS += \ $(ESOCK_RUN_OBJS) $(OBJDIR)/erl_flxctr.o \ $(OBJDIR)/erl_nfunc_sched.o \ $(OBJDIR)/erl_global_literals.o \ - $(OBJDIR)/beam_file.o + $(OBJDIR)/beam_file.o \ + $(OBJDIR)/beam_types.o LTTNG_OBJS = $(OBJDIR)/erlang_lttng.o diff --git a/erts/emulator/beam/beam_file.c b/erts/emulator/beam/beam_file.c index 2f003ac0e7..39ceea7f56 100644 --- a/erts/emulator/beam/beam_file.c +++ b/erts/emulator/beam/beam_file.c @@ -580,6 +580,58 @@ static int parse_line_chunk(BeamFile *beam, IFF_Chunk *chunk) { return 1; } +/* We assume the presence of a type table to simplify loading, so we'll need to + * create a dummy table (with single entry for the "any type") when we don't + * have one. */ +static void init_fallback_type_table(BeamFile *beam) { + BeamFile_TypeTable *types; + + types = &beam->types; + ASSERT(types->entries == NULL); + + types->entries = erts_alloc(ERTS_ALC_T_PREPARED_CODE, + sizeof(types->entries[0])); + types->count = 1; + + types->entries[0].type_union = BEAM_TYPE_ANY; +} + +static int parse_type_chunk(BeamFile *beam, IFF_Chunk *chunk) { + BeamFile_TypeTable *types; + BeamReader reader; + + Sint32 version, count; + int i; + + beamreader_init(chunk->data, chunk->size, &reader); + + LoadAssert(beamreader_read_i32(&reader, &version)); + LoadAssert(version == BEAM_TYPES_VERSION); + + types = &beam->types; + ASSERT(types->entries == NULL); + + LoadAssert(beamreader_read_i32(&reader, &count)); + LoadAssert(CHECK_ITEM_COUNT(count, 0, sizeof(types->entries[0]))); + LoadAssert(count >= 1); + + types->entries = erts_alloc(ERTS_ALC_T_PREPARED_CODE, + count * sizeof(types->entries[0])); + types->count = count; + + for (i = 0; i < count; i++) { + const byte *type_data; + + LoadAssert(beamreader_read_bytes(&reader, 2, &type_data)); + LoadAssert(beam_types_decode(type_data, 2, &types->entries[i])); + } + + /* The first entry MUST be the "any type." */ + LoadAssert(types->entries[0].type_union == BEAM_TYPE_ANY); + + return 1; +} + static ErlHeapFragment *new_literal_fragment(Uint size) { ErlHeapFragment *bp; @@ -807,6 +859,7 @@ beamfile_read(const byte *data, size_t size, BeamFile *beam) { MakeIffId('L', 'i', 'n', 'e'), /* 9 */ MakeIffId('L', 'o', 'c', 'T'), /* 10 */ MakeIffId('A', 't', 'o', 'm'), /* 11 */ + MakeIffId('T', 'y', 'p', 'e'), /* 12 */ }; static const int UTF8_ATOM_CHUNK = 0; @@ -823,6 +876,7 @@ beamfile_read(const byte *data, size_t size, BeamFile *beam) { static const int LOC_CHUNK = 10; #endif static const int OBSOLETE_ATOM_CHUNK = 11; + static const int TYPE_CHUNK = 12; static const int NUM_CHUNKS = sizeof(chunk_iffs) / sizeof(chunk_iffs[0]); @@ -911,6 +965,15 @@ beamfile_read(const byte *data, size_t size, BeamFile *beam) { } } + if (chunks[TYPE_CHUNK].size > 0) { + if (!parse_type_chunk(beam, &chunks[TYPE_CHUNK])) { + error = BEAMFILE_READ_CORRUPT_TYPE_TABLE; + goto error; + } + } else { + init_fallback_type_table(beam); + } + beam->strings.data = chunks[STR_CHUNK].data; beam->strings.size = chunks[STR_CHUNK].size; @@ -1040,6 +1103,11 @@ void beamfile_free(BeamFile *beam) { beam->lines.names = NULL; } + if (beam->types.entries) { + erts_free(ERTS_ALC_T_PREPARED_CODE, beam->types.entries); + beam->types.entries = NULL; + } + if (beam->static_literals.entries) { beamfile_literal_dtor(&beam->static_literals); } @@ -1523,7 +1591,7 @@ static int beamcodereader_read_next(BeamCodeReader *code_reader, BeamOp **out) { case 4: /* Literal */ { - BeamFile_LiteralTable *literals; + const BeamFile_LiteralTable *literals; TaggedNumber index; LoadAssert(beamreader_read_tagged(reader, &index)); @@ -1539,6 +1607,25 @@ static int beamcodereader_read_next(BeamCodeReader *code_reader, BeamOp **out) { break; } + case 5: + /* Register with type hint */ + { + const BeamFile_TypeTable *types; + TaggedNumber index; + + LoadAssert(beamreader_read_tagged(reader, &raw_arg)); + LoadAssert(raw_arg.tag == TAG_x || raw_arg.tag == TAG_y); + + LoadAssert(beamreader_read_tagged(reader, &index)); + LoadAssert(index.tag == TAG_u); + + types = &(code_reader->file)->types; + LoadAssert(index.word_value < types->count); + + ERTS_CT_ASSERT(REG_MASK < (1 << 10)); + raw_arg.word_value |= index.word_value << 10; + break; + } default: LoadError("Unrecognized extended tag"); } diff --git a/erts/emulator/beam/beam_file.h b/erts/emulator/beam/beam_file.h index 51e3a48c89..278a116318 100644 --- a/erts/emulator/beam/beam_file.h +++ b/erts/emulator/beam/beam_file.h @@ -25,6 +25,7 @@ #include "sys.h" #include "atom.h" +#include "beam_types.h" #define CHECKSUM_SIZE 16 @@ -137,6 +138,13 @@ typedef struct { } BeamFile_LiteralTable; typedef struct { + /* To simplify code that queries types, the first entry (which must be + * present) is always the "any type." */ + Sint32 count; + BeamType *entries; +} BeamFile_TypeTable; + +typedef struct { IFF_File iff; Eterm module; @@ -151,6 +159,7 @@ typedef struct { #endif BeamFile_LambdaTable lambdas; BeamFile_LineTable lines; + BeamFile_TypeTable types; /* Static literals are those defined in the file, and dynamic literals are * those created when loading. The former is positively indexed starting @@ -190,13 +199,14 @@ enum beamfile_read_result { /* Optional chunks */ BEAMFILE_READ_CORRUPT_LAMBDA_TABLE, BEAMFILE_READ_CORRUPT_LINE_TABLE, - BEAMFILE_READ_CORRUPT_LITERAL_TABLE + BEAMFILE_READ_CORRUPT_LITERAL_TABLE, + BEAMFILE_READ_CORRUPT_TYPE_TABLE }; typedef struct { /* TAG_xyz */ - int type; - BeamInstr val; + UWord type; + UWord val; } BeamOpArg; typedef struct beamop { diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index d8a868aa37..efdeca3b6d 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -154,6 +154,8 @@ erts_prepare_loading(Binary* magic, Process *c_p, Eterm group_leader, BeamLoadError0(stp, "corrupt literal table"); case BEAMFILE_READ_CORRUPT_LOCALS_TABLE: BeamLoadError0(stp, "corrupt locals table"); + case BEAMFILE_READ_CORRUPT_TYPE_TABLE: + BeamLoadError0(stp, "corrupt type table"); case BEAMFILE_READ_SUCCESS: break; } diff --git a/erts/emulator/beam/beam_types.c b/erts/emulator/beam/beam_types.c new file mode 100644 index 0000000000..705cdec046 --- /dev/null +++ b/erts/emulator/beam/beam_types.c @@ -0,0 +1,42 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2021-2021. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "beam_types.h" + +#include "global.h" +#include "erl_message.h" +#include "external.h" + +int beam_types_decode(const byte *data, Uint size, BeamType *out) { + int types; + + if (size != 2) { + return 0; + } + + types = (Uint16)data[0] << 8 | (Uint16)data[1]; + out->type_union = types; + + return 1; +} diff --git a/erts/emulator/beam/beam_types.h b/erts/emulator/beam/beam_types.h new file mode 100644 index 0000000000..7480a5e83a --- /dev/null +++ b/erts/emulator/beam/beam_types.h @@ -0,0 +1,97 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2021-2021. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +/** + * @description Basic type representation for BEAM instruction operands. + * @file beam_types.h + * + * While the compiler is good eliminating redundant type tests and simplifying + * instructions, we're limited by the available instructions and it's not + * always worthwhile to add new variants. + * + * The idea behind this module is to allow minor optimizations _inside_ + * instructions based on what we know about their operand types. For example, + * when we know that the source passed to `is_tagged_tuple` is always boxed, we + * can skip the boxed check. + */ + +#ifndef _BEAM_TYPES_H +#define _BEAM_TYPES_H + +#include "sys.h" + +#define BEAM_TYPES_VERSION 0 + +#define BEAM_TYPE_NONE (0) + +#define BEAM_TYPE_ATOM (1 << 0) +#define BEAM_TYPE_BITSTRING (1 << 1) +#define BEAM_TYPE_BS_MATCHSTATE (1 << 2) +#define BEAM_TYPE_CONS (1 << 3) +#define BEAM_TYPE_FLOAT (1 << 4) +#define BEAM_TYPE_FUN (1 << 5) +#define BEAM_TYPE_INTEGER (1 << 6) +#define BEAM_TYPE_MAP (1 << 7) +#define BEAM_TYPE_NIL (1 << 8) +#define BEAM_TYPE_PID (1 << 9) +#define BEAM_TYPE_PORT (1 << 10) +#define BEAM_TYPE_REFERENCE (1 << 11) +#define BEAM_TYPE_TUPLE (1 << 12) + +#define BEAM_TYPE_ANY ((1 << 13) - 1) + +#define BEAM_TYPE_MASK_BOXED \ + (BEAM_TYPE_BITSTRING | \ + BEAM_TYPE_BS_MATCHSTATE | \ + BEAM_TYPE_FLOAT | \ + BEAM_TYPE_FUN | \ + BEAM_TYPE_INTEGER | \ + BEAM_TYPE_MAP | \ + BEAM_TYPE_PID | \ + BEAM_TYPE_PORT | \ + BEAM_TYPE_REFERENCE | \ + BEAM_TYPE_TUPLE) + +#define BEAM_TYPE_MASK_IMMEDIATE \ + (BEAM_TYPE_ATOM | \ + BEAM_TYPE_INTEGER | \ + BEAM_TYPE_NIL | \ + BEAM_TYPE_PID | \ + BEAM_TYPE_PORT) + +#define BEAM_TYPE_MASK_CELL \ + (BEAM_TYPE_CONS) + +#define BEAM_TYPE_MASK_ALWAYS_IMMEDIATE \ + (BEAM_TYPE_MASK_IMMEDIATE & ~(BEAM_TYPE_MASK_BOXED | BEAM_TYPE_MASK_CELL)) +#define BEAM_TYPE_MASK_ALWAYS_BOXED \ + (BEAM_TYPE_MASK_BOXED & ~(BEAM_TYPE_MASK_CELL | BEAM_TYPE_MASK_IMMEDIATE)) +#define BEAM_TYPE_MASK_ALWAYS_CELL \ + (BEAM_TYPE_MASK_CELL & ~(BEAM_TYPE_MASK_BOXED | BEAM_TYPE_MASK_IMMEDIATE)) + +typedef struct { + /** @brief A set of the possible types (atom, tuple, etc) this term may + * be. When a single bit is set, the term will always be of that type. */ + int type_union; +} BeamType; + +int beam_types_decode(const byte *data, Uint size, BeamType *out); + +#endif diff --git a/erts/emulator/beam/emu/emu_load.c b/erts/emulator/beam/emu/emu_load.c index 83183db896..5166a39e5f 100644 --- a/erts/emulator/beam/emu/emu_load.c +++ b/erts/emulator/beam/emu/emu_load.c @@ -831,11 +831,11 @@ int beam_load_emit_op(LoaderState *stp, BeamOp *tmp_op) { break; case 'x': /* x(N) */ BeamLoadVerifyTag(stp, tag_to_letter[tag], *sign); - code[ci++] = tmp_op->a[arg].val * sizeof(Eterm); + code[ci++] = (tmp_op->a[arg].val & REG_MASK) * sizeof(Eterm); break; case 'y': /* y(N) */ BeamLoadVerifyTag(stp, tag_to_letter[tag], *sign); - code[ci++] = (tmp_op->a[arg].val + CP_SIZE) * sizeof(Eterm); + code[ci++] = ((tmp_op->a[arg].val & REG_MASK) + CP_SIZE) * sizeof(Eterm); break; case 'a': /* Tagged atom */ BeamLoadVerifyTag(stp, tag_to_letter[tag], *sign); @@ -865,10 +865,10 @@ int beam_load_emit_op(LoaderState *stp, BeamOp *tmp_op) { case 's': /* Any source (tagged constant or register) */ switch (tag) { case TAG_x: - code[ci++] = make_loader_x_reg(tmp_op->a[arg].val); + code[ci++] = make_loader_x_reg(tmp_op->a[arg].val & REG_MASK); break; case TAG_y: - code[ci++] = make_loader_y_reg(tmp_op->a[arg].val + CP_SIZE); + code[ci++] = make_loader_y_reg((tmp_op->a[arg].val & REG_MASK) + CP_SIZE); break; case TAG_i: code[ci++] = (BeamInstr) make_small((Uint)tmp_op->a[arg].val); @@ -903,10 +903,10 @@ int beam_load_emit_op(LoaderState *stp, BeamOp *tmp_op) { case 'S': /* Source (x(N), y(N)) */ switch (tag) { case TAG_x: - code[ci++] = tmp_op->a[arg].val * sizeof(Eterm); + code[ci++] = (tmp_op->a[arg].val & REG_MASK) * sizeof(Eterm); break; case TAG_y: - code[ci++] = (tmp_op->a[arg].val + CP_SIZE) * sizeof(Eterm) + 1; + code[ci++] = ((tmp_op->a[arg].val & REG_MASK) + CP_SIZE) * sizeof(Eterm) + 1; break; default: BeamLoadError1(stp, "bad tag %d for destination", diff --git a/erts/emulator/beam/emu/generators.tab b/erts/emulator/beam/emu/generators.tab index 45a29a7bab..1c7780d354 100644 --- a/erts/emulator/beam/emu/generators.tab +++ b/erts/emulator/beam/emu/generators.tab @@ -19,6 +19,36 @@ // %CopyrightEnd% // +// Rewrites call_fun2 as call_fun: we're not yet helped by the Safe parameter, +// and can't do anything clever with Func either. +gen.call_fun2(Safe, Arity, Func) { + BeamOp *call; + + (void)Safe; + + $NewBeamOp(S, call); + $BeamOpNameArity(call, call_fun, 1); + call->a[0] = Arity; + call->next = NULL; + + /* Move the fun in place when needed. We don't generate this at the moment, + * but a future version of the compiler might keep Func in a Y register. */ + if (Func.type != TAG_x || Arity.val != Func.val) { + BeamOp *move; + + $NewBeamOp(S, move); + $BeamOpNameArity(move, move, 2); + move->a[0] = Func; + move->a[1].type = TAG_x; + move->a[1].val = Arity.val; + move->next = call; + + return move; + } + + return call; +} + // Generate the fastest instruction to fetch an integer from a binary. gen.get_integer2(Fail, Ms, Live, Size, Unit, Flags, Dst) { BeamOp* op; diff --git a/erts/emulator/beam/emu/ops.tab b/erts/emulator/beam/emu/ops.tab index 4821ef9836..1c1181e95e 100644 --- a/erts/emulator/beam/emu/ops.tab +++ b/erts/emulator/beam/emu/ops.tab @@ -1021,6 +1021,8 @@ call_fun Arity => i_call_fun Arity i_call_fun t i_call_fun_last t Q +call_fun2 Safe Arity Func => call_fun2(Safe, Arity, Func) + # # A fun with an empty environment can be converted to a literal. # As a further optimization, the we try to move the fun to its diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h index 7c2fe55796..da52f3b151 100644 --- a/erts/emulator/beam/erl_vm.h +++ b/erts/emulator/beam/erl_vm.h @@ -48,9 +48,10 @@ #define EMULATOR "BEAM" #define SEQ_TRACE 1 -#define CONTEXT_REDS 4000 /* Swap process out after this number */ -#define MAX_ARG 255 /* Max number of arguments allowed */ -#define MAX_REG 1024 /* Max number of x(N) registers used */ +#define CONTEXT_REDS 4000 /* Swap process out after this number */ +#define MAX_ARG 255 /* Max number of arguments allowed */ +#define MAX_REG 1024 /* Max number of x(N) registers used */ +#define REG_MASK (MAX_REG - 1) /* * Guard BIFs and the new trapping length/1 implementation need 3 extra diff --git a/erts/emulator/beam/jit/arm/beam_asm.hpp b/erts/emulator/beam/jit/arm/beam_asm.hpp index b750ca9246..7b64695867 100644 --- a/erts/emulator/beam/jit/arm/beam_asm.hpp +++ b/erts/emulator/beam/jit/arm/beam_asm.hpp @@ -434,8 +434,8 @@ protected: * registers when we don't know the exact number. * * Furthermore, we only save the callee-save registers when told to sync - * sync all registers with the `Update::eXRegs` flag, as this is very - * rarely needed. */ + * all registers with the `Update::eXRegs` flag, as this is very rarely + * needed. */ template<int Spec = 0> void emit_enter_runtime(int live = num_register_backed_xregs) { @@ -756,6 +756,12 @@ public: void setLogger(std::string log); void setLogger(FILE *log); + void comment(const char *format) { + if (logger.file()) { + a.commentf("# %s", format); + } + } + template<typename... Ts> void comment(const char *format, Ts... args) { if (logger.file()) { @@ -879,8 +885,8 @@ class BeamGlobalAssembler : public BeamAssembler { }; #undef DECL_ENUM + static const std::map<GlobalLabels, const std::string> labelNames; static const std::map<GlobalLabels, emitFptr> emitPtrs; - static const std::map<GlobalLabels, std::string> labelNames; std::unordered_map<GlobalLabels, Label> labels; std::unordered_map<GlobalLabels, fptr> ptrs; @@ -925,15 +931,9 @@ class BeamModuleAssembler : public BeamAssembler { /* Map of BEAM label number to asmjit Label. These should not be used * directly by most instructions because of displacement limits, use * `resolve_beam_label` instead. */ - typedef std::unordered_map<BeamLabel, Label> LabelMap; + typedef std::unordered_map<BeamLabel, const Label> LabelMap; LabelMap rawLabels; - /* Map of label number to function name. Only defined for the - * entry label of a function. This map will be populated and - * used only when assembly output has been requested. */ - typedef std::unordered_map<BeamLabel, std::string> LabelNames; - LabelNames labelNames; - /* Sequence number used to create unique named labels by * resolve_label(). Only used when assembly output has been * requested. */ @@ -979,6 +979,9 @@ class BeamModuleAssembler : public BeamAssembler { /* All functions that have been seen so far */ std::vector<BeamLabel> functions; + /* The BEAM file we've been loaded from, if any. */ + BeamFile *beam; + BeamGlobalAssembler *ga; /* Used by emit to populate the labelToMFA map */ @@ -1082,13 +1085,13 @@ class BeamModuleAssembler : public BeamAssembler { public: BeamModuleAssembler(BeamGlobalAssembler *ga, Eterm mod, - unsigned num_labels, - BeamFile_ExportTable *named_labels = NULL); + int num_labels, + BeamFile *file = NULL); BeamModuleAssembler(BeamGlobalAssembler *ga, Eterm mod, - unsigned num_labels, - unsigned num_functions, - BeamFile_ExportTable *named_labels = NULL); + int num_labels, + int num_functions, + BeamFile *file = NULL); bool emit(unsigned op, const Span<ArgVal> &args); @@ -1136,6 +1139,67 @@ public: void patchStrings(char *rw_base, const byte *string); protected: + bool always_immediate(const ArgVal &arg) const { + if (arg.isImmed()) { + return true; + } + + int type_union = beam->types.entries[arg.typeIndex()].type_union; + return (type_union & BEAM_TYPE_MASK_ALWAYS_IMMEDIATE) == type_union; + } + + bool always_same_types(const ArgVal &lhs, const ArgVal &rhs) const { + int lhs_types = beam->types.entries[lhs.typeIndex()].type_union; + int rhs_types = beam->types.entries[rhs.typeIndex()].type_union; + + /* We can only be certain that the types are the same when there's + * one possible type. For example, if one is a number and the other + * is an integer, they could differ if the former is a float. */ + if ((lhs_types & (lhs_types - 1)) == 0) { + return lhs_types == rhs_types; + } + + return false; + } + + bool always_one_of(const ArgVal &arg, int types) const { + if (arg.isImmed()) { + if (is_small(arg.getValue())) { + return !!(types & BEAM_TYPE_INTEGER); + } else if (is_atom(arg.getValue())) { + return !!(types & BEAM_TYPE_ATOM); + } else if (is_nil(arg.getValue())) { + return !!(types & BEAM_TYPE_NIL); + } + + return false; + } else { + int type_union = beam->types.entries[arg.typeIndex()].type_union; + return type_union == (type_union & types); + } + } + + int masked_types(const ArgVal &arg, int mask) const { + if (arg.isImmed()) { + if (is_small(arg.getValue())) { + return mask & BEAM_TYPE_INTEGER; + } else if (is_atom(arg.getValue())) { + return mask & BEAM_TYPE_ATOM; + } else if (is_nil(arg.getValue())) { + return mask & BEAM_TYPE_NIL; + } + + return BEAM_TYPE_NONE; + } else { + int type_union = beam->types.entries[arg.typeIndex()].type_union; + return type_union & mask; + } + } + + bool exact_type(const ArgVal &arg, int type_id) const { + return always_one_of(arg, type_id); + } + /* Helpers */ void emit_gc_test(const ArgVal &Stack, const ArgVal &Heap, @@ -1147,13 +1211,28 @@ protected: arm::Mem emit_variable_apply(bool includeI); arm::Mem emit_fixed_apply(const ArgVal &arity, bool includeI); - arm::Gp emit_call_fun(); + arm::Gp emit_call_fun(bool skip_box_test = false, + bool skip_fun_test = false, + bool skip_arity_test = false); arm::Gp emit_is_binary(const ArgVal &Fail, const ArgVal &Src, Label next, Label subbin); + void emit_is_boxed(Label Fail, arm::Gp Src) { + BeamAssembler::emit_is_boxed(Fail, Src); + } + + void emit_is_boxed(Label Fail, const ArgVal &Arg, arm::Gp Src) { + if (always_one_of(Arg, BEAM_TYPE_MASK_ALWAYS_BOXED)) { + comment("skipped box test since argument is always boxed"); + return; + } + + BeamAssembler::emit_is_boxed(Fail, Src); + } + void emit_get_list(const arm::Gp boxed_ptr, const ArgVal &Hd, const ArgVal &Tl); @@ -1233,17 +1312,18 @@ protected: * * When the branch type is not `dispUnknown`, this must be used * _IMMEDIATELY BEFORE_ the instruction that the label is used in. */ - Label resolve_beam_label(const ArgVal &Label, enum Displacement disp); - Label resolve_label(Label target, - enum Displacement disp, - std::string *name = nullptr); + const Label &resolve_beam_label(const ArgVal &Label, + enum Displacement disp); + const Label &resolve_label(const Label &target, + enum Displacement disp, + const char *name = nullptr); /* Resolves a shared fragment, creating a trampoline that loads the * appropriate address before jumping there. * * When the branch type is not `dispUnknown`, this must be used * _IMMEDIATELY BEFORE_ the instruction that the label is used in. */ - Label resolve_fragment(void (*fragment)(), enum Displacement disp); + const Label &resolve_fragment(void (*fragment)(), enum Displacement disp); /* Embeds a constant argument and returns its address. All kinds of * constants are accepted, including labels and export entries. diff --git a/erts/emulator/beam/jit/arm/beam_asm_global.cpp b/erts/emulator/beam/jit/arm/beam_asm_global.cpp index 20007763d7..bcecc584ff 100644 --- a/erts/emulator/beam/jit/arm/beam_asm_global.cpp +++ b/erts/emulator/beam/jit/arm/beam_asm_global.cpp @@ -38,7 +38,7 @@ const std::map<BeamGlobalAssembler::GlobalLabels, BeamGlobalAssembler::emitFptr> #define DECL_LABEL_NAME(NAME) {NAME, STRINGIFY(NAME)}, -const std::map<BeamGlobalAssembler::GlobalLabels, std::string> +const std::map<BeamGlobalAssembler::GlobalLabels, const std::string> BeamGlobalAssembler::labelNames = {BEAM_GLOBAL_FUNCS( DECL_LABEL_NAME) PROCESS_MAIN_LABELS(DECL_LABEL_NAME)}; #undef DECL_LABEL_NAME diff --git a/erts/emulator/beam/jit/arm/beam_asm_module.cpp b/erts/emulator/beam/jit/arm/beam_asm_module.cpp index 0b0937e4f9..791b89769e 100644 --- a/erts/emulator/beam/jit/arm/beam_asm_module.cpp +++ b/erts/emulator/beam/jit/arm/beam_asm_module.cpp @@ -25,11 +25,6 @@ #include "beam_asm.hpp" using namespace asmjit; -static std::string getAtom(Eterm atom) { - Atom *ap = atom_tab(atom_val(atom)); - return std::string((char *)ap->name, ap->len); -} - #ifdef BEAMASM_DUMP_SIZES # include <mutex> @@ -78,56 +73,14 @@ ErtsCodePtr BeamModuleAssembler::getLambda(unsigned index) { return (ErtsCodePtr)getCode(lambda.trampoline); } -BeamModuleAssembler::BeamModuleAssembler(BeamGlobalAssembler *_ga, - Eterm _mod, - unsigned num_labels, - BeamFile_ExportTable *named_labels) - : BeamAssembler(getAtom(_mod)), ga(_ga), mod(_mod) { - _veneers.reserve(num_labels + 1); - rawLabels.reserve(num_labels + 1); - - if (logger.file() && named_labels) { - BeamFile_ExportEntry *e = &named_labels->entries[0]; - for (unsigned int i = 1; i < num_labels; i++) { - /* Large enough to hold most realistic function names. We will - * truncate too long names, but as the label name is not important - * for the functioning of the JIT and this functionality is - * probably only used by developers, we don't bother with dynamic - * allocation. */ - char tmp[MAX_ATOM_SZ_LIMIT]; - - /* The named_labels are sorted, so no need for a search. */ - if ((unsigned int)e->label == i) { - erts_snprintf(tmp, sizeof(tmp), "%T/%d", e->function, e->arity); - rawLabels[i] = a.newNamedLabel(tmp); - e++; - } else { - std::string lblName = "label_" + std::to_string(i); - rawLabels[i] = a.newNamedLabel(lblName.data()); - } - } - } else if (logger.file()) { - /* There is no naming info, but dumping of the assembly code - * has been requested, so do the best we can and number the - * labels. */ - for (unsigned int i = 1; i < num_labels; i++) { - std::string lblName = "label_" + std::to_string(i); - rawLabels[i] = a.newNamedLabel(lblName.data()); - } - } else { - /* No output is requested, go with unnamed labels */ - for (unsigned int i = 1; i < num_labels; i++) { - rawLabels[i] = a.newLabel(); - } - } -} - BeamModuleAssembler::BeamModuleAssembler(BeamGlobalAssembler *ga, Eterm mod, - unsigned num_labels, - unsigned num_functions, - BeamFile_ExportTable *named_labels) - : BeamModuleAssembler(ga, mod, num_labels, named_labels) { + int num_labels, + int num_functions, + BeamFile *file) + : BeamModuleAssembler(ga, mod, num_labels, file) { + _veneers.reserve(num_labels + 1); + codeHeader = a.newLabel(); a.align(kAlignCode, 8); a.bind(codeHeader); @@ -398,6 +351,8 @@ void BeamModuleAssembler::emit_i_func_info(const ArgVal &Label, } void BeamModuleAssembler::emit_label(const ArgVal &Label) { + ASSERT(Label.isLabel()); + currLabel = rawLabels[Label.getValue()]; bind_veneer_target(currLabel); } @@ -566,20 +521,25 @@ void BeamModuleAssembler::patchStrings(char *rw_base, } } -Label BeamModuleAssembler::resolve_beam_label(const ArgVal &Lbl, - enum Displacement disp) { +const Label &BeamModuleAssembler::resolve_beam_label(const ArgVal &Lbl, + enum Displacement disp) { ASSERT(Lbl.isLabel()); - auto it = labelNames.find(Lbl.getValue()); - if (it != labelNames.end()) { - return resolve_label(rawLabels[Lbl.getValue()], disp, &it->second); + + const Label &beamLabel = rawLabels.at(Lbl.getValue()); + const auto &labelEntry = code.labelEntry(beamLabel); + + if (labelEntry->hasName()) { + return resolve_label(rawLabels.at(Lbl.getValue()), + disp, + labelEntry->name()); } else { - return resolve_label(rawLabels[Lbl.getValue()], disp); + return resolve_label(rawLabels.at(Lbl.getValue()), disp); } } -Label BeamModuleAssembler::resolve_label(Label target, - enum Displacement disp, - std::string *labelName) { +const Label &BeamModuleAssembler::resolve_label(const Label &target, + enum Displacement disp, + const char *labelName) { ssize_t currOffset = a.offset(); ssize_t minOffset = currOffset - disp; @@ -615,6 +575,7 @@ Label BeamModuleAssembler::resolve_label(Label target, } Label anchor; + if (!labelName) { anchor = a.newLabel(); } else { @@ -624,9 +585,10 @@ Label BeamModuleAssembler::resolve_label(Label target, * huge more than one veneer can be created for each entry * label. */ std::stringstream name; - name << '@' << *labelName << '-' << labelSeq++; + name << '@' << labelName << '-' << labelSeq++; anchor = a.newNamedLabel(name.str().c_str()); } + auto it = _veneers.emplace(target.id(), Veneer{.latestOffset = maxOffset, .anchor = anchor, @@ -638,8 +600,8 @@ Label BeamModuleAssembler::resolve_label(Label target, return veneer.anchor; } -Label BeamModuleAssembler::resolve_fragment(void (*fragment)(), - enum Displacement disp) { +const Label &BeamModuleAssembler::resolve_fragment(void (*fragment)(), + enum Displacement disp) { Label target; auto it = _dispatchTable.find(fragment); @@ -818,7 +780,7 @@ void BeamModuleAssembler::emit_constant(const Constant &constant) { if (value.isImmed() || value.isWord()) { a.embedUInt64(rawValue); } else if (value.isLabel()) { - a.embedLabel(rawLabels[rawValue]); + a.embedLabel(rawLabels.at(rawValue)); } else { a.embedUInt64(LLONG_MAX); diff --git a/erts/emulator/beam/jit/arm/generators.tab b/erts/emulator/beam/jit/arm/generators.tab index 677d741258..00df1d876b 100644 --- a/erts/emulator/beam/jit/arm/generators.tab +++ b/erts/emulator/beam/jit/arm/generators.tab @@ -580,3 +580,13 @@ gen.create_bin(Fail, Alloc, Live, Unit, Dst, N, Segments) { return op; } + +gen.remove_tuple_type(Tuple) { + BeamOp* op; + + $NewBeamOp(S, op); + $BeamOpNameArity(op, current_tuple, 1); + op->a[0] = Tuple; + op->a[0].val = Tuple.val & REG_MASK; + return op; +} diff --git a/erts/emulator/beam/jit/arm/instr_bs.cpp b/erts/emulator/beam/jit/arm/instr_bs.cpp index 96d9c31bfd..4b41277f0f 100644 --- a/erts/emulator/beam/jit/arm/instr_bs.cpp +++ b/erts/emulator/beam/jit/arm/instr_bs.cpp @@ -514,7 +514,7 @@ void BeamModuleAssembler::emit_i_bs_start_match3(const ArgVal &Src, mov_arg(ARG2, Src); if (Fail.getValue() != 0) { - emit_is_boxed(resolve_beam_label(Fail, dispUnknown), ARG2); + emit_is_boxed(resolve_beam_label(Fail, dispUnknown), Src, ARG2); } else { /* bs_start_match3 may not throw, and the compiler will only emit {f,0} * when it knows that the source is a match state or binary, so we're diff --git a/erts/emulator/beam/jit/arm/instr_common.cpp b/erts/emulator/beam/jit/arm/instr_common.cpp index 4ed112f0cd..5db7676086 100644 --- a/erts/emulator/beam/jit/arm/instr_common.cpp +++ b/erts/emulator/beam/jit/arm/instr_common.cpp @@ -675,7 +675,7 @@ arm::Gp BeamModuleAssembler::emit_is_binary(const ArgVal &Fail, Label subbin) { auto src = load_source(Src, ARG1); - emit_is_boxed(resolve_beam_label(Fail, dispUnknown), src.reg); + emit_is_boxed(resolve_beam_label(Fail, dispUnknown), Src, src.reg); arm::Gp boxed_ptr = emit_ptr_val(ARG1, src.reg); a.ldur(TMP1, emit_boxed_val(boxed_ptr)); @@ -683,11 +683,17 @@ arm::Gp BeamModuleAssembler::emit_is_binary(const ArgVal &Fail, a.cmp(TMP1, imm(_TAG_HEADER_SUB_BIN)); a.cond_eq().b(subbin); - ERTS_CT_ASSERT(_TAG_HEADER_REFC_BIN + 4 == _TAG_HEADER_HEAP_BIN); - a.and_(TMP1, TMP1, imm(~4)); - a.cmp(TMP1, imm(_TAG_HEADER_REFC_BIN)); - a.cond_eq().b(next); - a.b(resolve_beam_label(Fail, disp128MB)); + if (masked_types(Src, BEAM_TYPE_MASK_ALWAYS_BOXED) == BEAM_TYPE_BITSTRING) { + comment("simplified binary test since source is always a bitstring " + "when boxed"); + } else { + ERTS_CT_ASSERT(_TAG_HEADER_REFC_BIN + 4 == _TAG_HEADER_HEAP_BIN); + a.and_(TMP1, TMP1, imm(~4)); + a.cmp(TMP1, imm(_TAG_HEADER_REFC_BIN)); + a.cond_ne().b(resolve_beam_label(Fail, disp1MB)); + } + + a.b(next); return boxed_ptr; } @@ -695,9 +701,8 @@ arm::Gp BeamModuleAssembler::emit_is_binary(const ArgVal &Fail, void BeamModuleAssembler::emit_is_binary(const ArgVal &Fail, const ArgVal &Src) { Label next = a.newLabel(), subbin = a.newLabel(); - arm::Gp boxed_ptr; - boxed_ptr = emit_is_binary(Fail, Src, next, subbin); + arm::Gp boxed_ptr = emit_is_binary(Fail, Src, next, subbin); a.bind(subbin); { @@ -721,35 +726,35 @@ void BeamModuleAssembler::emit_is_bitstring(const ArgVal &Fail, } void BeamModuleAssembler::emit_is_float(const ArgVal &Fail, const ArgVal &Src) { - Label next = a.newLabel(); - auto src = load_source(Src, TMP1); - emit_is_boxed(resolve_beam_label(Fail, dispUnknown), src.reg); - - arm::Gp boxed_ptr = emit_ptr_val(TMP1, src.reg); - a.ldur(TMP1, emit_boxed_val(boxed_ptr)); + emit_is_boxed(resolve_beam_label(Fail, dispUnknown), Src, src.reg); - a.cmp(TMP1, imm(HEADER_FLONUM)); - a.cond_ne().b(resolve_beam_label(Fail, disp1MB)); + if (masked_types(Src, BEAM_TYPE_MASK_BOXED) == BEAM_TYPE_FLOAT) { + comment("skipped header test since we know it's a float when boxed"); + } else { + arm::Gp boxed_ptr = emit_ptr_val(TMP1, src.reg); + a.ldur(TMP1, emit_boxed_val(boxed_ptr)); - a.bind(next); + a.cmp(TMP1, imm(HEADER_FLONUM)); + a.cond_ne().b(resolve_beam_label(Fail, disp1MB)); + } } void BeamModuleAssembler::emit_is_function(const ArgVal &Fail, const ArgVal &Src) { - Label next = a.newLabel(); - auto src = load_source(Src, TMP1); - emit_is_boxed(resolve_beam_label(Fail, dispUnknown), src.reg); - - arm::Gp boxed_ptr = emit_ptr_val(TMP1, src.reg); - a.ldur(TMP1, emit_boxed_val(boxed_ptr)); - a.cmp(TMP1, imm(HEADER_FUN)); - a.cond_ne().b(resolve_beam_label(Fail, disp1MB)); + emit_is_boxed(resolve_beam_label(Fail, dispUnknown), Src, src.reg); - a.bind(next); + if (masked_types(Src, BEAM_TYPE_MASK_BOXED) == BEAM_TYPE_FUN) { + comment("skipped header test since we know it's a fun when boxed"); + } else { + arm::Gp boxed_ptr = emit_ptr_val(TMP1, src.reg); + a.ldur(TMP1, emit_boxed_val(boxed_ptr)); + a.cmp(TMP1, imm(HEADER_FUN)); + a.cond_ne().b(resolve_beam_label(Fail, disp1MB)); + } } void BeamModuleAssembler::emit_is_function2(const ArgVal &Fail, @@ -785,12 +790,17 @@ void BeamModuleAssembler::emit_is_function2(const ArgVal &Fail, auto src = load_source(Src, TMP1); - emit_is_boxed(resolve_beam_label(Fail, dispUnknown), src.reg); + emit_is_boxed(resolve_beam_label(Fail, dispUnknown), Src, src.reg); arm::Gp boxed_ptr = emit_ptr_val(TMP1, src.reg); - a.ldur(TMP2, emit_boxed_val(boxed_ptr)); - a.cmp(TMP2, imm(HEADER_FUN)); - a.cond_ne().b(resolve_beam_label(Fail, disp1MB)); + + if (masked_types(Src, BEAM_TYPE_MASK_BOXED) == BEAM_TYPE_FUN) { + comment("skipped header test since we know it's a fun when boxed"); + } else { + a.ldur(TMP2, emit_boxed_val(boxed_ptr)); + a.cmp(TMP2, imm(HEADER_FUN)); + a.cond_ne().b(resolve_beam_label(Fail, disp1MB)); + } a.ldur(TMP2, emit_boxed_val(boxed_ptr, offsetof(ErlFunThing, arity))); emit_branch_if_ne(TMP2, arity, resolve_beam_label(Fail, dispUnknown)); @@ -798,27 +808,36 @@ void BeamModuleAssembler::emit_is_function2(const ArgVal &Fail, void BeamModuleAssembler::emit_is_integer(const ArgVal &Fail, const ArgVal &Src) { - Label next = a.newLabel(); - auto src = load_source(Src, TMP1); + Label next = a.newLabel(); - a.and_(TMP2, src.reg, imm(_TAG_IMMED1_MASK)); - a.cmp(TMP2, imm(_TAG_IMMED1_SMALL)); - a.cond_eq().b(next); + if (always_one_of(Src, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_ALWAYS_BOXED)) { + comment("simplified small test since all other types are boxed"); + emit_is_boxed(next, Src, src.reg); + } else { + a.and_(TMP2, src.reg, imm(_TAG_IMMED1_MASK)); + a.cmp(TMP2, imm(_TAG_IMMED1_SMALL)); + a.cond_eq().b(next); - emit_is_boxed(resolve_beam_label(Fail, dispUnknown), TMP2); + emit_is_boxed(resolve_beam_label(Fail, dispUnknown), Src, TMP2); + } - arm::Gp boxed_ptr = emit_ptr_val(TMP1, src.reg); - a.ldur(TMP1, emit_boxed_val(boxed_ptr)); + if (masked_types(Src, BEAM_TYPE_MASK_BOXED) == BEAM_TYPE_INTEGER) { + comment("skipped header test since we know it's a bignum when " + "boxed"); + } else { + arm::Gp boxed_ptr = emit_ptr_val(TMP1, src.reg); + a.ldur(TMP1, emit_boxed_val(boxed_ptr)); - /* The following value (0b111011) is not possible to use as - * an immediate operand for 'and'. See the note at the beginning - * of the file. - */ - mov_imm(TMP2, _TAG_HEADER_MASK - _BIG_SIGN_BIT); - a.and_(TMP1, TMP1, TMP2); - a.cmp(TMP1, imm(_TAG_HEADER_POS_BIG)); - a.cond_ne().b(resolve_beam_label(Fail, disp1MB)); + /* The following value (0b111011) is not possible to use as + * an immediate operand for 'and'. See the note at the beginning + * of the file. + */ + mov_imm(TMP2, _TAG_HEADER_MASK - _BIG_SIGN_BIT); + a.and_(TMP1, TMP1, TMP2); + a.cmp(TMP1, imm(_TAG_HEADER_POS_BIG)); + a.cond_ne().b(resolve_beam_label(Fail, disp1MB)); + } a.bind(next); } @@ -835,12 +854,19 @@ void BeamModuleAssembler::emit_is_list(const ArgVal &Fail, const ArgVal &Src) { void BeamModuleAssembler::emit_is_map(const ArgVal &Fail, const ArgVal &Src) { auto src = load_source(Src, TMP1); - emit_is_boxed(resolve_beam_label(Fail, dispUnknown), src.reg); - arm::Gp boxed_ptr = emit_ptr_val(TMP1, src.reg); - a.ldur(TMP1, emit_boxed_val(boxed_ptr)); - a.and_(TMP1, TMP1, imm(_TAG_HEADER_MASK)); - a.cmp(TMP1, imm(_TAG_HEADER_MAP)); - a.cond_ne().b(resolve_beam_label(Fail, disp1MB)); + emit_is_boxed(resolve_beam_label(Fail, dispUnknown), Src, src.reg); + + /* As an optimization for the `error | #{}` case, skip checking the header + * word when we know that the only possible boxed type is a map. */ + if (masked_types(Src, BEAM_TYPE_MASK_BOXED) == BEAM_TYPE_MAP) { + comment("skipped header test since we know it's a map when boxed"); + } else { + arm::Gp boxed_ptr = emit_ptr_val(TMP1, src.reg); + a.ldur(TMP1, emit_boxed_val(boxed_ptr)); + a.and_(TMP1, TMP1, imm(_TAG_HEADER_MASK)); + a.cmp(TMP1, imm(_TAG_HEADER_MAP)); + a.cond_ne().b(resolve_beam_label(Fail, disp1MB)); + } } void BeamModuleAssembler::emit_is_nil(const ArgVal &Fail, const ArgVal &Src) { @@ -851,91 +877,117 @@ void BeamModuleAssembler::emit_is_nil(const ArgVal &Fail, const ArgVal &Src) { void BeamModuleAssembler::emit_is_number(const ArgVal &Fail, const ArgVal &Src) { - Label next = a.newLabel(); - auto src = load_source(Src, TMP1); + Label next = a.newLabel(); - a.and_(TMP2, src.reg, imm(_TAG_IMMED1_MASK)); - a.cmp(TMP2, imm(_TAG_IMMED1_SMALL)); - a.cond_eq().b(next); + if (always_one_of(Src, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_ALWAYS_BOXED)) { + comment("simplified small test since all other types are boxed"); + emit_is_boxed(next, Src, src.reg); + } else { + a.and_(TMP2, src.reg, imm(_TAG_IMMED1_MASK)); + a.cmp(TMP2, imm(_TAG_IMMED1_SMALL)); + a.cond_eq().b(next); - emit_is_boxed(resolve_beam_label(Fail, dispUnknown), TMP2); + emit_is_boxed(resolve_beam_label(Fail, dispUnknown), Src, src.reg); + } - arm::Gp boxed_ptr = emit_ptr_val(TMP1, src.reg); - a.ldur(TMP1, emit_boxed_val(boxed_ptr)); + if (masked_types(Src, BEAM_TYPE_MASK_BOXED) == + (BEAM_TYPE_FLOAT | BEAM_TYPE_INTEGER)) { + comment("skipped header test since we know it's a number when boxed"); + } else { + arm::Gp boxed_ptr = emit_ptr_val(TMP1, src.reg); + a.ldur(TMP1, emit_boxed_val(boxed_ptr)); - /* The following value (0b111011) is not possible to use as - * an immediate operand for 'and'. See the note at the beginning - * of the file. - */ - mov_imm(TMP2, _TAG_HEADER_MASK - _BIG_SIGN_BIT); - a.and_(TMP2, TMP1, TMP2); - a.cmp(TMP2, imm(_TAG_HEADER_POS_BIG)); + /* The following value (0b111011) is not possible to use as + * an immediate operand for 'and'. See the note at the beginning + * of the file. + */ + mov_imm(TMP2, _TAG_HEADER_MASK - _BIG_SIGN_BIT); + a.and_(TMP2, TMP1, TMP2); + a.cmp(TMP2, imm(_TAG_HEADER_POS_BIG)); - a.mov(TMP3, imm(HEADER_FLONUM)); - a.ccmp(TMP1, TMP3, 4, arm::Cond::kNE); - a.cond_ne().b(resolve_beam_label(Fail, disp1MB)); + a.mov(TMP3, imm(HEADER_FLONUM)); + a.ccmp(TMP1, TMP3, 4, arm::Cond::kNE); + a.cond_ne().b(resolve_beam_label(Fail, disp1MB)); + } a.bind(next); } void BeamModuleAssembler::emit_is_pid(const ArgVal &Fail, const ArgVal &Src) { - Label next = a.newLabel(); - auto src = load_source(Src, TMP1); + Label next = a.newLabel(); - a.and_(TMP2, src.reg, imm(_TAG_IMMED1_MASK)); - a.cmp(TMP2, imm(_TAG_IMMED1_PID)); - a.cond_eq().b(next); + if (always_one_of(Src, BEAM_TYPE_PID | BEAM_TYPE_MASK_ALWAYS_BOXED)) { + comment("simplified local pid test since all other types are boxed"); + emit_is_boxed(next, Src, src.reg); + } else { + a.and_(TMP2, src.reg, imm(_TAG_IMMED1_MASK)); + a.cmp(TMP2, imm(_TAG_IMMED1_PID)); + a.cond_eq().b(next); - /* Reuse TMP2 as the important bits are still available. */ - emit_is_boxed(resolve_beam_label(Fail, dispUnknown), TMP2); + /* Reuse TMP2 as the important bits are still available. */ + emit_is_boxed(resolve_beam_label(Fail, dispUnknown), Src, TMP2); + } - arm::Gp boxed_ptr = emit_ptr_val(TMP1, src.reg); - a.ldur(TMP2, emit_boxed_val(boxed_ptr)); - a.and_(TMP2, TMP2, imm(_TAG_HEADER_MASK)); - a.cmp(TMP2, imm(_TAG_HEADER_EXTERNAL_PID)); - a.cond_ne().b(resolve_beam_label(Fail, disp1MB)); + if (masked_types(Src, BEAM_TYPE_MASK_BOXED) == BEAM_TYPE_PID) { + comment("skipped header test since we know it's a pid when boxed"); + } else { + arm::Gp boxed_ptr = emit_ptr_val(TMP1, src.reg); + a.ldur(TMP2, emit_boxed_val(boxed_ptr)); + a.and_(TMP2, TMP2, imm(_TAG_HEADER_MASK)); + a.cmp(TMP2, imm(_TAG_HEADER_EXTERNAL_PID)); + a.cond_ne().b(resolve_beam_label(Fail, disp1MB)); + } a.bind(next); } void BeamModuleAssembler::emit_is_port(const ArgVal &Fail, const ArgVal &Src) { - Label next = a.newLabel(); - auto src = load_source(Src, TMP1); + Label next = a.newLabel(); - a.and_(TMP2, src.reg, imm(_TAG_IMMED1_MASK)); - a.cmp(TMP2, imm(_TAG_IMMED1_PORT)); - a.cond_eq().b(next); + if (always_one_of(Src, BEAM_TYPE_PORT | BEAM_TYPE_MASK_ALWAYS_BOXED)) { + comment("simplified local port test since all other types are boxed"); + emit_is_boxed(next, Src, src.reg); + } else { + a.and_(TMP2, src.reg, imm(_TAG_IMMED1_MASK)); + a.cmp(TMP2, imm(_TAG_IMMED1_PORT)); + a.cond_eq().b(next); - /* Reuse TMP2 as the important bits are still available. */ - emit_is_boxed(resolve_beam_label(Fail, dispUnknown), TMP2); + /* Reuse TMP2 as the important bits are still available. */ + emit_is_boxed(resolve_beam_label(Fail, dispUnknown), Src, TMP2); + } - arm::Gp boxed_ptr = emit_ptr_val(TMP1, src.reg); - a.ldur(TMP2, emit_boxed_val(boxed_ptr)); - a.and_(TMP2, TMP2, imm(_TAG_HEADER_MASK)); - a.cmp(TMP2, imm(_TAG_HEADER_EXTERNAL_PORT)); - a.cond_ne().b(resolve_beam_label(Fail, disp1MB)); + if (masked_types(Src, BEAM_TYPE_MASK_BOXED) == BEAM_TYPE_PORT) { + comment("skipped header test since we know it's a port when boxed"); + } else { + arm::Gp boxed_ptr = emit_ptr_val(TMP1, src.reg); + a.ldur(TMP2, emit_boxed_val(boxed_ptr)); + a.and_(TMP2, TMP2, imm(_TAG_HEADER_MASK)); + a.cmp(TMP2, imm(_TAG_HEADER_EXTERNAL_PORT)); + a.cond_ne().b(resolve_beam_label(Fail, disp1MB)); + } a.bind(next); } void BeamModuleAssembler::emit_is_reference(const ArgVal &Fail, const ArgVal &Src) { - Label next = a.newLabel(); - auto src = load_source(Src, TMP1); - emit_is_boxed(resolve_beam_label(Fail, dispUnknown), src.reg); - arm::Gp boxed_ptr = emit_ptr_val(TMP1, src.reg); - a.ldur(TMP1, emit_boxed_val(boxed_ptr)); - a.and_(TMP1, TMP1, imm(_TAG_HEADER_MASK)); - a.cmp(TMP1, imm(_TAG_HEADER_EXTERNAL_REF)); - a.ccmp(TMP1, imm(_TAG_HEADER_REF), 4, arm::Cond::kNE); - a.cond_ne().b(resolve_beam_label(Fail, disp1MB)); + emit_is_boxed(resolve_beam_label(Fail, dispUnknown), Src, src.reg); - a.bind(next); + if (masked_types(Src, BEAM_TYPE_MASK_BOXED) == BEAM_TYPE_REFERENCE) { + comment("skipped header test since we know it's a ref when boxed"); + } else { + arm::Gp boxed_ptr = emit_ptr_val(TMP1, src.reg); + a.ldur(TMP1, emit_boxed_val(boxed_ptr)); + a.and_(TMP1, TMP1, imm(_TAG_HEADER_MASK)); + a.cmp(TMP1, imm(_TAG_HEADER_EXTERNAL_REF)); + a.ccmp(TMP1, imm(_TAG_HEADER_REF), 4, arm::Cond::kNE); + a.cond_ne().b(resolve_beam_label(Fail, disp1MB)); + } } /* Note: This instruction leaves the untagged pointer to the tuple in @@ -945,8 +997,11 @@ void BeamModuleAssembler::emit_i_is_tagged_tuple(const ArgVal &Fail, const ArgVal &Arity, const ArgVal &Tag) { auto src = load_source(Src, ARG1); - emit_is_boxed(resolve_beam_label(Fail, dispUnknown), src.reg); + + emit_is_boxed(resolve_beam_label(Fail, dispUnknown), Src, src.reg); + emit_untag_ptr(ARG1, src.reg); + /* It is safe to fetch the both the header word and the first * element of the tuple with ldp because the empty tuple is always * a literal that is padded so that the word after arity is @@ -975,7 +1030,8 @@ void BeamModuleAssembler::emit_i_is_tagged_tuple_ff(const ArgVal &NotTuple, Label correct_arity = a.newLabel(); auto src = load_source(Src, ARG1); - emit_is_boxed(resolve_beam_label(NotTuple, dispUnknown), src.reg); + emit_is_boxed(resolve_beam_label(NotTuple, dispUnknown), Src, src.reg); + emit_untag_ptr(ARG1, src.reg); /* It is safe to fetch the both the header word and the first @@ -1006,13 +1062,27 @@ void BeamModuleAssembler::emit_i_is_tuple(const ArgVal &Fail, const ArgVal &Src) { auto src = load_source(Src, ARG1); - emit_is_boxed(resolve_beam_label(Fail, dispUnknown), src.reg); - emit_untag_ptr(ARG1, src.reg); + emit_is_boxed(resolve_beam_label(Fail, dispUnknown), Src, src.reg); - a.ldr(TMP1, arm::Mem(ARG1)); - ERTS_CT_ASSERT(_TAG_HEADER_ARITYVAL == 0); - a.tst(TMP1, imm(_TAG_HEADER_MASK)); - a.cond_ne().b(resolve_beam_label(Fail, disp1MB)); + /* As an optimization for the `error | {ok, Value}` case, skip checking the + * header word when we know that the only possible boxed type is a tuple. */ + if (masked_types(Src, BEAM_TYPE_MASK_BOXED) == BEAM_TYPE_TUPLE) { + comment("skipped header test since we know it's a tuple when boxed"); + + /* Note: ARG1 will NOT be set. This is OK since the operand + * for `current_tuple` has a type; that operand will not match + * the type-less operand for `get_tuple_element`. Thus, there + * will always be a `load_tuple_ptr` instruction emitted if + * this instruction is immediately followed by a + * `get_tuple_element` instruction. */ + } else { + emit_untag_ptr(ARG1, src.reg); + + a.ldr(TMP1, arm::Mem(ARG1)); + ERTS_CT_ASSERT(_TAG_HEADER_ARITYVAL == 0); + a.tst(TMP1, imm(_TAG_HEADER_MASK)); + a.cond_ne().b(resolve_beam_label(Fail, disp1MB)); + } } /* Note: This instruction leaves the untagged pointer to the tuple in @@ -1022,7 +1092,8 @@ void BeamModuleAssembler::emit_i_is_tuple_of_arity(const ArgVal &Fail, const ArgVal &Arity) { auto src = load_source(Src, ARG1); - emit_is_boxed(resolve_beam_label(Fail, dispUnknown), src.reg); + emit_is_boxed(resolve_beam_label(Fail, dispUnknown), Src, src.reg); + emit_untag_ptr(ARG1, src.reg); a.ldr(TMP1, arm::Mem(ARG1)); @@ -1048,9 +1119,13 @@ void BeamModuleAssembler::emit_is_eq_exact(const ArgVal &Fail, const ArgVal &Y) { auto x = load_source(X, ARG1); - if (Y.isImmed()) { - /* If the second operand is a known to be an immediate, we can - * fail immediately if the operands are not equal. */ + /* If either argument is known to be an immediate, we can fail immediately + * if they're not equal. */ + if (always_immediate(X) || always_immediate(Y)) { + if (!X.isImmed() && !Y.isImmed()) { + comment("simplified check since one argument is an immediate"); + } + cmp_arg(x.reg, Y); a.cond_ne().b(resolve_beam_label(Fail, disp1MB)); @@ -1064,10 +1139,14 @@ void BeamModuleAssembler::emit_is_eq_exact(const ArgVal &Fail, a.cmp(x.reg, y.reg); a.cond_eq().b(next); - /* The terms could still be equal if both operands are pointers - * having the same tag. */ - emit_is_unequal_based_on_tags(x.reg, y.reg); - a.cond_eq().b(resolve_beam_label(Fail, disp1MB)); + if (always_same_types(X, Y)) { + comment("skipped tag test since they are always equal"); + } else { + /* The terms could still be equal if both operands are pointers + * having the same tag. */ + emit_is_unequal_based_on_tags(x.reg, y.reg); + a.cond_eq().b(resolve_beam_label(Fail, disp1MB)); + } /* Both operands are pointers having the same tag. Must do a * deeper comparison. */ @@ -1090,9 +1169,13 @@ void BeamModuleAssembler::emit_is_ne_exact(const ArgVal &Fail, const ArgVal &Y) { auto x = load_source(X, ARG1); - if (Y.isImmed()) { - /* If the second operand is a known to be an immediate, we can - * fail immediately if the operands are equal. */ + /* If either argument is known to be an immediate, we can fail immediately + * if they're equal. */ + if (always_immediate(X) || always_immediate(Y)) { + if (!X.isImmed() && !Y.isImmed()) { + comment("simplified check since one argument is an immediate"); + } + cmp_arg(x.reg, Y); a.cond_eq().b(resolve_beam_label(Fail, disp1MB)); @@ -1106,10 +1189,14 @@ void BeamModuleAssembler::emit_is_ne_exact(const ArgVal &Fail, a.cmp(x.reg, y.reg); a.cond_eq().b(resolve_beam_label(Fail, disp1MB)); - /* Test whether the terms are definitely unequal based on the tags - * alone. */ - emit_is_unequal_based_on_tags(x.reg, y.reg); - a.cond_eq().b(next); + if (always_same_types(X, Y)) { + comment("skipped tag test since they are always equal"); + } else { + /* Test whether the terms are definitely unequal based on the tags + * alone. */ + emit_is_unequal_based_on_tags(x.reg, y.reg); + a.cond_eq().b(next); + } /* Both operands are pointers having the same tag. Must do a * deeper comparison. */ @@ -1248,85 +1335,114 @@ void BeamGlobalAssembler::emit_arith_compare_shared() { void BeamModuleAssembler::emit_is_lt(const ArgVal &Fail, const ArgVal &LHS, const ArgVal &RHS) { - Label generic = a.newLabel(), next = a.newLabel(); + mov_arg(ARG1, LHS); + mov_arg(ARG2, RHS); - auto lhs = load_source(LHS, ARG1); - auto rhs = load_source(RHS, ARG2); + if (always_one_of(LHS, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_BOXED) && + always_one_of(RHS, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_BOXED)) { + Label branch_compare = a.newLabel(); - a.cmp(lhs.reg, rhs.reg); - a.cond_eq().b(resolve_beam_label(Fail, disp1MB)); + a.cmp(ARG1, ARG2); - /* Relative comparisons are overwhelmingly likely to be used on smalls, so - * we'll specialize those and keep the rest in a shared fragment. */ + /* The only possible kind of immediate is a small and all other values + * are boxed, so we can test for smalls by testing boxed. */ + comment("simplified small test since all other types are boxed"); + ERTS_CT_ASSERT(_TAG_PRIMARY_MASK - TAG_PRIMARY_BOXED == (1 << 0)); + a.and_(TMP1, ARG1, ARG2); + a.tbnz(TMP1, imm(0), branch_compare); + fragment_call(ga->get_arith_compare_shared()); - if (RHS.isImmed() && is_small(RHS.getValue())) { - a.and_(TMP1, lhs.reg, imm(_TAG_IMMED1_MASK)); - } else if (LHS.isImmed() && is_small(LHS.getValue())) { - a.and_(TMP1, rhs.reg, imm(_TAG_IMMED1_MASK)); + /* The flags will either be from the initial comparison, or from the + * shared fragment. */ + a.bind(branch_compare); + a.cond_ge().b(resolve_beam_label(Fail, disp1MB)); } else { - ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK); - a.and_(TMP1, lhs.reg, rhs.reg); - a.and_(TMP1, TMP1, imm(_TAG_IMMED1_MASK)); - } + Label generic = a.newLabel(), next = a.newLabel(); + + /* Relative comparisons are overwhelmingly likely to be used on smalls, + * so we'll specialize those and keep the rest in a shared fragment. */ + if (RHS.isImmed() && is_small(RHS.getValue())) { + a.and_(TMP1, ARG1, imm(_TAG_IMMED1_MASK)); + } else if (LHS.isImmed() && is_small(LHS.getValue())) { + a.and_(TMP1, ARG2, imm(_TAG_IMMED1_MASK)); + } else { + ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK); + a.and_(TMP1, ARG1, ARG2); + a.and_(TMP1, TMP1, imm(_TAG_IMMED1_MASK)); + } - a.cmp(TMP1, imm(_TAG_IMMED1_SMALL)); - a.cond_ne().b(generic); + a.cmp(TMP1, imm(_TAG_IMMED1_SMALL)); + a.cond_ne().b(generic); - a.cmp(lhs.reg, rhs.reg); - a.cond_lt().b(next); - a.b(resolve_beam_label(Fail, disp128MB)); + a.cmp(ARG1, ARG2); + a.cond_lt().b(next); + a.b(resolve_beam_label(Fail, disp128MB)); - a.bind(generic); - { - mov_var(ARG1, lhs); - mov_var(ARG2, rhs); - fragment_call(ga->get_arith_compare_shared()); - a.cond_ge().b(resolve_beam_label(Fail, disp1MB)); - } + a.bind(generic); + { + fragment_call(ga->get_arith_compare_shared()); + a.cond_ge().b(resolve_beam_label(Fail, disp1MB)); + } - a.bind(next); + a.bind(next); + } } void BeamModuleAssembler::emit_is_ge(const ArgVal &Fail, const ArgVal &LHS, const ArgVal &RHS) { - Label generic = a.newLabel(), next = a.newLabel(); + mov_arg(ARG1, LHS); + mov_arg(ARG2, RHS); - auto lhs = load_source(LHS, ARG1); - auto rhs = load_source(RHS, ARG2); + if (always_one_of(LHS, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_BOXED) && + always_one_of(RHS, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_BOXED)) { + Label branch_compare = a.newLabel(); - a.cmp(lhs.reg, rhs.reg); - a.cond_eq().b(next); + a.cmp(ARG1, ARG2); + + /* The only possible kind of immediate is a small and all other values + * are boxed, so we can test for smalls by testing boxed. */ + comment("simplified small test since all other types are boxed"); + ERTS_CT_ASSERT(_TAG_PRIMARY_MASK - TAG_PRIMARY_BOXED == (1 << 0)); + a.and_(TMP1, ARG1, ARG2); + a.tbnz(TMP1, imm(0), branch_compare); - /* Relative comparisons are overwhelmingly likely to be used on smalls, so - * we'll specialize those and keep the rest in a shared fragment. */ + fragment_call(ga->get_arith_compare_shared()); - if (RHS.isImmed() && is_small(RHS.getValue())) { - a.and_(TMP1, lhs.reg, imm(_TAG_IMMED1_MASK)); - } else if (LHS.isImmed() && is_small(LHS.getValue())) { - a.and_(TMP1, rhs.reg, imm(_TAG_IMMED1_MASK)); + /* The flags will either be from the initial comparison, or from the + * shared fragment. */ + a.bind(branch_compare); + a.cond_lt().b(resolve_beam_label(Fail, disp1MB)); } else { - ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK); - a.and_(TMP1, lhs.reg, rhs.reg); - a.and_(TMP1, TMP1, imm(_TAG_IMMED1_MASK)); - } + Label generic = a.newLabel(), next = a.newLabel(); + + /* Relative comparisons are overwhelmingly likely to be used on smalls, + * so we'll specialize those and keep the rest in a shared fragment. */ + if (RHS.isImmed() && is_small(RHS.getValue())) { + a.and_(TMP1, ARG1, imm(_TAG_IMMED1_MASK)); + } else if (LHS.isImmed() && is_small(LHS.getValue())) { + a.and_(TMP1, ARG2, imm(_TAG_IMMED1_MASK)); + } else { + ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK); + a.and_(TMP1, ARG1, ARG2); + a.and_(TMP1, TMP1, imm(_TAG_IMMED1_MASK)); + } - a.cmp(TMP1, imm(_TAG_IMMED1_SMALL)); - a.cond_ne().b(generic); + a.cmp(TMP1, imm(_TAG_IMMED1_SMALL)); + a.cond_ne().b(generic); - a.cmp(lhs.reg, rhs.reg); - a.cond_ge().b(next); - a.b(resolve_beam_label(Fail, disp128MB)); + a.cmp(ARG1, ARG2); + a.cond_ge().b(next); + a.b(resolve_beam_label(Fail, disp128MB)); - a.bind(generic); - { - mov_var(ARG1, lhs); - mov_var(ARG2, rhs); - fragment_call(ga->get_arith_compare_shared()); - a.cond_lt().b(resolve_beam_label(Fail, disp1MB)); - } + a.bind(generic); + { + fragment_call(ga->get_arith_compare_shared()); + a.cond_lt().b(resolve_beam_label(Fail, disp1MB)); + } - a.bind(next); + a.bind(next); + } } void BeamModuleAssembler::emit_badmatch(const ArgVal &Src) { diff --git a/erts/emulator/beam/jit/arm/instr_fun.cpp b/erts/emulator/beam/jit/arm/instr_fun.cpp index 345a83cf3d..2d6bcf1b2a 100644 --- a/erts/emulator/beam/jit/arm/instr_fun.cpp +++ b/erts/emulator/beam/jit/arm/instr_fun.cpp @@ -333,64 +333,118 @@ void BeamModuleAssembler::emit_i_apply_fun_only() { /* Asssumes that: * ARG3 = arity * ARG4 = fun thing */ -arm::Gp BeamModuleAssembler::emit_call_fun() { +arm::Gp BeamModuleAssembler::emit_call_fun(bool skip_box_test, + bool skip_fun_test, + bool skip_arity_test) { + const bool never_fails = skip_box_test && skip_fun_test && skip_arity_test; Label next = a.newLabel(); /* Speculatively untag the ErlFunThing. */ emit_untag_ptr(TMP2, ARG4); - /* Load the error fragment into TMP3 so we can CSEL ourselves there on - * error. */ - a.adr(TMP3, resolve_fragment(ga->get_handle_call_fun_error(), disp1MB)); + if (!never_fails) { + /* Load the error fragment into TMP3 so we can CSEL ourselves there on + * error. */ + a.adr(TMP3, resolve_fragment(ga->get_handle_call_fun_error(), disp1MB)); + } - /* The `handle_call_fun_error` fragment expects current PC in ARG5. */ + /* The `handle_call_fun_error` and `unloaded_fun` fragments expect current + * PC in ARG5. */ a.adr(ARG5, next); - /* As emit_is_boxed(), but explicitly sets ZF so we can rely on that for - * error checking in `next`. */ - a.tst(ARG4, imm(_TAG_PRIMARY_MASK - TAG_PRIMARY_BOXED)); - a.cond_ne().b(next); - - /* Load header word and `ErlFunThing->entry`. We can safely do this before - * testing the header because boxed terms are guaranteed to be at least two - * words long. */ - a.ldp(TMP1, ARG1, arm::Mem(TMP2)); + if (!skip_box_test) { + /* As emit_is_boxed(), but explicitly sets ZF so we can rely on that + * for error checking in `next`. */ + a.tst(ARG4, imm(_TAG_PRIMARY_MASK - TAG_PRIMARY_BOXED)); + a.cond_ne().b(next); + } else { + comment("skipped box test since source is always boxed"); + } - a.cmp(TMP1, imm(HEADER_FUN)); - a.cond_ne().b(next); + if (!skip_fun_test) { + /* Load header word and `ErlFunThing->entry`. We can safely do this + * before testing the header because boxed terms are guaranteed to be + * at least two words long. */ + ERTS_CT_ASSERT_FIELD_PAIR(ErlFunThing, thing_word, entry); + a.ldp(TMP1, ARG1, arm::Mem(TMP2)); + + a.cmp(TMP1, imm(HEADER_FUN)); + a.cond_ne().b(next); + } else { + comment("skipped fun test since source is always a fun when boxed"); + a.ldr(ARG1, arm::Mem(TMP2, offsetof(ErlFunThing, entry))); + } - a.ldr(TMP2, arm::Mem(TMP2, offsetof(ErlFunThing, arity))); - a.cmp(TMP2, ARG3); + if (!skip_arity_test) { + a.ldr(TMP2, arm::Mem(TMP2, offsetof(ErlFunThing, arity))); + a.cmp(TMP2, ARG3); + } else { + comment("skipped arity test since source always has right arity"); + } a.ldr(TMP1, emit_setup_dispatchable_call(ARG1)); /* Assumes that ZF is set on success and clear on error, overwriting our * destination with the error fragment's address. */ a.bind(next); - a.csel(TMP1, TMP1, TMP3, imm(arm::Cond::kEQ)); + + if (!never_fails) { + a.csel(TMP1, TMP1, TMP3, imm(arm::Cond::kEQ)); + } return TMP1; } -void BeamModuleAssembler::emit_i_call_fun(const ArgVal &Arity) { - mov_arg(ARG4, ArgVal(ArgVal::XReg, Arity.getValue())); +void BeamModuleAssembler::emit_i_call_fun2(const ArgVal &Safe, + const ArgVal &Arity, + const ArgVal &Func) { + arm::Gp target; + mov_imm(ARG3, Arity.getValue()); + mov_arg(ARG4, Func); - erlang_call(emit_call_fun()); + target = emit_call_fun(always_one_of(Func, BEAM_TYPE_MASK_ALWAYS_BOXED), + masked_types(Func, BEAM_TYPE_MASK_BOXED) == + BEAM_TYPE_FUN, + Safe.getValue() == am_true); + + erlang_call(target); } -void BeamModuleAssembler::emit_i_call_fun_last(const ArgVal &Arity, - const ArgVal &Deallocate) { - emit_deallocate(Deallocate); +void BeamModuleAssembler::emit_i_call_fun2_last(const ArgVal &Safe, + const ArgVal &Arity, + const ArgVal &Func, + const ArgVal &Deallocate) { + arm::Gp target; - mov_arg(ARG4, ArgVal(ArgVal::XReg, Arity.getValue())); mov_imm(ARG3, Arity.getValue()); + mov_arg(ARG4, Func); - arm::Gp target = emit_call_fun(); + target = emit_call_fun(always_one_of(Func, BEAM_TYPE_MASK_ALWAYS_BOXED), + masked_types(Func, BEAM_TYPE_MASK_BOXED) == + BEAM_TYPE_FUN, + Safe.getValue() == am_true); + + emit_deallocate(Deallocate); emit_leave_erlang_frame(); a.br(target); } +void BeamModuleAssembler::emit_i_call_fun(const ArgVal &Arity) { + const ArgVal Func(ArgVal::XReg, Arity.getValue()); + const ArgVal Safe(ArgVal::Immediate, am_false); + + emit_i_call_fun2(Safe, Arity, Func); +} + +void BeamModuleAssembler::emit_i_call_fun_last(const ArgVal &Arity, + const ArgVal &Deallocate) { + const ArgVal Func(ArgVal::XReg, Arity.getValue()); + const ArgVal Safe(ArgVal::Immediate, am_false); + + emit_i_call_fun2_last(Safe, Arity, Func, Deallocate); +} + /* Psuedo-instruction for signalling lambda load errors. Never actually runs. */ void BeamModuleAssembler::emit_i_lambda_error(const ArgVal &Dummy) { emit_nyi("emit_i_lambda_error"); diff --git a/erts/emulator/beam/jit/arm/instr_guard_bifs.cpp b/erts/emulator/beam/jit/arm/instr_guard_bifs.cpp index ae403db726..8735a5c4c8 100644 --- a/erts/emulator/beam/jit/arm/instr_guard_bifs.cpp +++ b/erts/emulator/beam/jit/arm/instr_guard_bifs.cpp @@ -125,15 +125,21 @@ void BeamGlobalAssembler::emit_bif_is_ne_exact_shared() { } } -void BeamModuleAssembler::emit_bif_is_eq_ne_exact_immed(const ArgVal &Src, - const ArgVal &Immed, +void BeamModuleAssembler::emit_bif_is_eq_ne_exact_immed(const ArgVal &LHS, + const ArgVal &RHS, const ArgVal &Dst, Eterm fail_value, Eterm succ_value) { - auto src = load_source(Src, TMP1); auto dst = init_destination(Dst, TMP2); - cmp_arg(src.reg, Immed); + if (RHS.isImmed()) { + auto lhs = load_source(LHS, TMP1); + cmp_arg(lhs.reg, RHS); + } else { + auto [lhs, rhs] = load_sources(LHS, TMP1, RHS, TMP2); + a.cmp(lhs.reg, rhs.reg); + } + mov_imm(TMP3, succ_value); mov_imm(TMP4, fail_value); a.csel(dst.reg, TMP3, TMP4, arm::Cond::kEQ); @@ -143,7 +149,10 @@ void BeamModuleAssembler::emit_bif_is_eq_ne_exact_immed(const ArgVal &Src, void BeamModuleAssembler::emit_bif_is_eq_exact(const ArgVal &LHS, const ArgVal &RHS, const ArgVal &Dst) { - if (RHS.isImmed()) { + if (always_immediate(LHS) || always_immediate(RHS)) { + if (!LHS.isImmed() && !RHS.isImmed()) { + comment("simplified check since one argument is an immediate"); + } emit_bif_is_eq_ne_exact_immed(LHS, RHS, Dst, am_false, am_true); } else { auto [lhs, rhs] = load_sources(LHS, ARG1, RHS, ARG2); @@ -160,7 +169,10 @@ void BeamModuleAssembler::emit_bif_is_eq_exact(const ArgVal &LHS, void BeamModuleAssembler::emit_bif_is_ne_exact(const ArgVal &LHS, const ArgVal &RHS, const ArgVal &Dst) { - if (RHS.isImmed()) { + if (always_immediate(LHS) || always_immediate(RHS)) { + if (!LHS.isImmed() && !RHS.isImmed()) { + comment("simplified check since one argument is an immediate"); + } emit_bif_is_eq_ne_exact_immed(LHS, RHS, Dst, am_true, am_false); } else { auto [lhs, rhs] = load_sources(LHS, ARG1, RHS, ARG2); @@ -515,9 +527,9 @@ void BeamModuleAssembler::emit_bif_map_size(const ArgVal &Fail, auto dst = init_destination(Dst, TMP2); if (Fail.getValue() == 0) { - emit_is_boxed(error, src.reg); + emit_is_boxed(error, Src, src.reg); } else { - emit_is_boxed(resolve_beam_label(Fail, dispUnknown), src.reg); + emit_is_boxed(resolve_beam_label(Fail, dispUnknown), Src, src.reg); } arm::Gp boxed_ptr = emit_ptr_val(TMP3, src.reg); diff --git a/erts/emulator/beam/jit/arm/instr_select.cpp b/erts/emulator/beam/jit/arm/instr_select.cpp index bac2238be7..194a41a71b 100644 --- a/erts/emulator/beam/jit/arm/instr_select.cpp +++ b/erts/emulator/beam/jit/arm/instr_select.cpp @@ -212,14 +212,19 @@ void BeamModuleAssembler::emit_i_select_tuple_arity(const ArgVal &Src, const Span<ArgVal> &args) { auto src = load_source(Src, TMP1); - emit_is_boxed(resolve_beam_label(Fail, dispUnknown), src.reg); + emit_is_boxed(resolve_beam_label(Fail, dispUnknown), Src, src.reg); arm::Gp boxed_ptr = emit_ptr_val(TMP1, src.reg); a.ldur(TMP1, emit_boxed_val(boxed_ptr, 0)); - ERTS_CT_ASSERT(_TAG_HEADER_ARITYVAL == 0); - a.tst(TMP1, imm(_TAG_HEADER_MASK)); - a.cond_ne().b(resolve_beam_label(Fail, disp1MB)); + if (masked_types(Src, BEAM_TYPE_MASK_BOXED) == BEAM_TYPE_TUPLE) { + comment("simplified tuple test since the source is always a tuple " + "when boxed"); + } else { + ERTS_CT_ASSERT(_TAG_HEADER_ARITYVAL == 0); + a.tst(TMP1, imm(_TAG_HEADER_MASK)); + a.cond_ne().b(resolve_beam_label(Fail, disp1MB)); + } Label fail = rawLabels[Fail.getValue()]; emit_linear_search(TMP1, fail, args); diff --git a/erts/emulator/beam/jit/arm/ops.tab b/erts/emulator/beam/jit/arm/ops.tab index 28f04fee9f..73d2c5f3b9 100644 --- a/erts/emulator/beam/jit/arm/ops.tab +++ b/erts/emulator/beam/jit/arm/ops.tab @@ -212,6 +212,7 @@ set_tuple_element s S P current_tuple/1 current_tuple/2 +typed_current_tuple/1 is_tuple Fail=f Src | test_arity Fail Src Arity => \ i_is_tuple_of_arity Fail Src Arity | current_tuple Src @@ -222,10 +223,12 @@ is_tuple NotTupleFail Tuple | is_tagged_tuple WrongRecordFail Tuple Arity Atom = i_is_tagged_tuple_ff NotTupleFail WrongRecordFail Tuple Arity Atom | current_tuple Tuple is_tagged_tuple Fail Tuple Arity Atom => \ - i_is_tagged_tuple Fail Tuple Arity Atom | current_tuple Tuple + i_is_tagged_tuple Fail Tuple Arity Atom | typed_current_tuple Tuple is_tuple Fail=f Src => i_is_tuple Fail Src | current_tuple Src +typed_current_tuple Tuple => remove_tuple_type(Tuple) + i_is_tuple_of_arity f? s A i_test_arity f? s A @@ -704,6 +707,14 @@ call_fun Arity => i_call_fun Arity i_call_fun t i_call_fun_last t t +call_fun2 Safe Arity Func | deallocate D | return => \ + i_call_fun2_last Safe Arity Func D +call_fun2 Safe Arity Func => \ + i_call_fun2 Safe Arity Func + +i_call_fun2 a t S +i_call_fun2_last a t S t + # # A fun with an empty environment can be converted to a literal. # As a further optimization, the we try to move the fun to its diff --git a/erts/emulator/beam/jit/asm_load.c b/erts/emulator/beam/jit/asm_load.c index a819f3385c..ace86da539 100644 --- a/erts/emulator/beam/jit/asm_load.c +++ b/erts/emulator/beam/jit/asm_load.c @@ -40,50 +40,14 @@ static void init_label(Label *lp); -static int named_labels_compare(BeamFile_ExportEntry *a, - BeamFile_ExportEntry *b) { - if (a->label < b->label) - return -1; - else if (a->label == b->label) - return 0; - else - return 1; -} - int beam_load_prepare_emit(LoaderState *stp) { BeamCodeHeader *hdr; - BeamFile_ExportTable *named_labels_ptr = NULL, named_labels; int i; - if (erts_jit_asm_dump) { - /* Dig out all named labels from the BEAM-file and sort them on the - label id. */ - named_labels.count = stp->beam.exports.count + stp->beam.locals.count; - named_labels.entries = erts_alloc( - ERTS_ALC_T_PREPARED_CODE, - named_labels.count * sizeof(named_labels.entries[0])); - - for (unsigned i = 0; i < stp->beam.exports.count; i++) - memcpy(&named_labels.entries[i], - &stp->beam.exports.entries[i], - sizeof(stp->beam.exports.entries[i])); - for (unsigned i = 0; i < stp->beam.locals.count; i++) - memcpy(&named_labels.entries[i + stp->beam.exports.count], - &stp->beam.locals.entries[i], - sizeof(stp->beam.locals.entries[i])); - - qsort(named_labels.entries, - named_labels.count, - sizeof(named_labels.entries[0]), - (int (*)(const void *, const void *))named_labels_compare); - named_labels_ptr = &named_labels; - } stp->ba = beamasm_new_assembler(stp->module, stp->beam.code.label_count, stp->beam.code.function_count, - named_labels_ptr); - if (named_labels_ptr != NULL) - erts_free(ERTS_ALC_T_PREPARED_CODE, named_labels_ptr->entries); + &stp->beam); /* Initialize code header */ stp->codev_size = stp->beam.code.function_count + 1; @@ -451,6 +415,7 @@ int beam_load_emit_op(LoaderState *stp, BeamOp *tmp_op) { stp->last_label); } stp->labels[stp->last_label].value = 1; + curr->type = TAG_f; break; case 'e': /* Export entry */ BeamLoadVerifyTag(stp, tag, TAG_u); diff --git a/erts/emulator/beam/jit/beam_asm.h b/erts/emulator/beam/jit/beam_asm.h index d592decec0..8444dbe02a 100644 --- a/erts/emulator/beam/jit/beam_asm.h +++ b/erts/emulator/beam/jit/beam_asm.h @@ -42,7 +42,7 @@ void beamasm_init(void); void *beamasm_new_assembler(Eterm mod, int num_labels, int num_functions, - BeamFile_ExportTable *named_labels); + BeamFile *beam); void beamasm_codegen(void *ba, const void **native_module_exec, void **native_module_rw, diff --git a/erts/emulator/beam/jit/beam_jit_common.cpp b/erts/emulator/beam/jit/beam_jit_common.cpp index 14824e5a43..8e56560b25 100644 --- a/erts/emulator/beam/jit/beam_jit_common.cpp +++ b/erts/emulator/beam/jit/beam_jit_common.cpp @@ -253,6 +253,88 @@ void BeamModuleAssembler::codegen(char *buff, size_t len) { CodeHolder::kCopyPadSectionBuffer); } +BeamModuleAssembler::BeamModuleAssembler(BeamGlobalAssembler *_ga, + Eterm _mod, + int num_labels, + BeamFile *file) + : BeamAssembler(getAtom(_mod)), beam(file), ga(_ga), mod(_mod) { + rawLabels.reserve(num_labels + 1); + + if (logger.file() && beam) { + /* Dig out all named labels from the BEAM-file and sort them on the + * label id. */ + const int named_count = beam->exports.count + beam->locals.count; + BeamFile_ExportEntry *entries; + + entries = (BeamFile_ExportEntry *)erts_alloc( + ERTS_ALC_T_PREPARED_CODE, + (named_count + 1) * sizeof(entries[0])); + + for (int i = 0; i < beam->exports.count; i++) { + entries[i] = beam->exports.entries[i]; + } + + for (int i = 0; i < beam->locals.count; i++) { + entries[i + beam->exports.count] = beam->locals.entries[i]; + } + + /* Place a sentinel entry with an invalid label number. */ + entries[named_count].label = 0; + + std::qsort(entries, + named_count, + sizeof(entries[0]), + [](const void *lhs__, const void *rhs__) { + auto lhs = (const BeamFile_ExportEntry *)lhs__; + auto rhs = (const BeamFile_ExportEntry *)rhs__; + + if (lhs->label < rhs->label) { + return -1; + } else if (lhs->label == rhs->label) { + return 0; + } else { + return 1; + } + }); + + BeamFile_ExportEntry *e = &entries[0]; + + for (int i = 1; i < num_labels; i++) { + /* Large enough to hold most realistic function names. We will + * truncate too long names, but as the label name is not important + * for the functioning of the JIT and this functionality is + * probably only used by developers, we don't bother with dynamic + * allocation. */ + char tmp[MAX_ATOM_SZ_LIMIT]; + + /* The named_labels are sorted, so no need for a search. */ + if (e->label == i) { + erts_snprintf(tmp, sizeof(tmp), "%T/%d", e->function, e->arity); + rawLabels.emplace(i, a.newNamedLabel(tmp)); + e++; + } else { + std::string lblName = "label_" + std::to_string(i); + rawLabels.emplace(i, a.newNamedLabel(lblName.data())); + } + } + + erts_free(ERTS_ALC_T_PREPARED_CODE, entries); + } else if (logger.file()) { + /* There is no naming info, but dumping of the assembly code + * has been requested, so do the best we can and number the + * labels. */ + for (int i = 1; i < num_labels; i++) { + std::string lblName = "label_" + std::to_string(i); + rawLabels.emplace(i, a.newNamedLabel(lblName.data())); + } + } else { + /* No output is requested, go with unnamed labels */ + for (int i = 1; i < num_labels; i++) { + rawLabels.emplace(i, a.newLabel()); + } + } +} + void BeamModuleAssembler::register_metadata(const BeamCodeHeader *header) { #ifndef WIN32 const BeamCodeLineTab *line_table = header->line_table; diff --git a/erts/emulator/beam/jit/beam_jit_common.hpp b/erts/emulator/beam/jit/beam_jit_common.hpp index b43926e8e8..6521cd6f00 100644 --- a/erts/emulator/beam/jit/beam_jit_common.hpp +++ b/erts/emulator/beam/jit/beam_jit_common.hpp @@ -55,7 +55,7 @@ struct ArgVal : public BeamOpArg { Immediate = 'I' }; - ArgVal(int t, UWord value) { + ArgVal(UWord t, UWord value) { #ifdef DEBUG switch (t) { case TAG_f: @@ -88,12 +88,30 @@ struct ArgVal : public BeamOpArg { val = value; } + constexpr int typeIndex() const { + switch (getType()) { + case TYPE::XReg: + case TYPE::YReg: + return (int)(val >> 10); + default: + /* Type index 0 always points to the "any type," making it a safe + * fallback whenever we lack type information. */ + return 0; + } + } + constexpr enum TYPE getType() const { return (enum TYPE)type; } constexpr uint64_t getValue() const { - return val; + switch (getType()) { + case TYPE::XReg: + case TYPE::YReg: + return val & REG_MASK; + default: + return val; + } } constexpr bool isRegister() const { diff --git a/erts/emulator/beam/jit/beam_jit_main.cpp b/erts/emulator/beam/jit/beam_jit_main.cpp index bb2e292d7e..7db7554b37 100644 --- a/erts/emulator/beam/jit/beam_jit_main.cpp +++ b/erts/emulator/beam/jit/beam_jit_main.cpp @@ -268,17 +268,17 @@ void beamasm_init() { func_label = label++; entry_label = label++; - args = {ArgVal(ArgVal::Immediate, func_label), + args = {ArgVal(ArgVal::Label, func_label), ArgVal(ArgVal::Word, sizeof(UWord))}; bma->emit(op_aligned_label_Lt, args); - args = {ArgVal(ArgVal::Immediate, func_label), + args = {ArgVal(ArgVal::Label, func_label), ArgVal(ArgVal::Immediate, mod_name), ArgVal(ArgVal::Immediate, op.name), ArgVal(ArgVal::Immediate, op.arity)}; bma->emit(op_i_func_info_IaaI, args); - args = {ArgVal(ArgVal::Immediate, entry_label), + args = {ArgVal(ArgVal::Label, entry_label), ArgVal(ArgVal::Word, sizeof(UWord))}; bma->emit(op_aligned_label_Lt, args); @@ -386,12 +386,12 @@ extern "C" void *beamasm_new_assembler(Eterm mod, int num_labels, int num_functions, - BeamFile_ExportTable *named_labels) { + BeamFile *file) { return new BeamModuleAssembler(bga, mod, num_labels, num_functions, - named_labels); + file); } int beamasm_emit(void *instance, unsigned specific_op, BeamOp *op) { @@ -418,18 +418,16 @@ extern "C" BeamModuleAssembler ba(bga, info->mfa.module, 3); std::vector<ArgVal> args; - args = {ArgVal(ArgVal::Immediate, 1), - ArgVal(ArgVal::Word, sizeof(UWord))}; + args = {ArgVal(ArgVal::Label, 1), ArgVal(ArgVal::Word, sizeof(UWord))}; ba.emit(op_aligned_label_Lt, args); - args = {ArgVal(ArgVal::Immediate, 1), + args = {ArgVal(ArgVal::Label, 1), ArgVal(ArgVal::Immediate, info->mfa.module), ArgVal(ArgVal::Immediate, info->mfa.function), ArgVal(ArgVal::Immediate, info->mfa.arity)}; ba.emit(op_i_func_info_IaaI, args); - args = {ArgVal(ArgVal::Immediate, 2), - ArgVal(ArgVal::Word, sizeof(UWord))}; + args = {ArgVal(ArgVal::Label, 2), ArgVal(ArgVal::Word, sizeof(UWord))}; ba.emit(op_aligned_label_Lt, args); args = {}; diff --git a/erts/emulator/beam/jit/x86/beam_asm.hpp b/erts/emulator/beam/jit/x86/beam_asm.hpp index 4b6199dd5e..525b561fff 100644 --- a/erts/emulator/beam/jit/x86/beam_asm.hpp +++ b/erts/emulator/beam/jit/x86/beam_asm.hpp @@ -288,7 +288,7 @@ protected: a.short_().jle(ok); a.bind(crash); - a.comment("# Redzone touched"); + comment("Redzone touched"); a.ud2(); a.bind(ok); @@ -336,7 +336,7 @@ protected: Label next = a.newLabel(); a.cmp(x86::rsp, getInitialSPRef()); a.short_().je(next); - a.comment("# The stack has grown"); + comment("The stack has grown"); a.ud2(); a.bind(next); #endif @@ -547,7 +547,7 @@ protected: a.short_().je(next); a.bind(crash); - a.comment("# Runtime stack is corrupt"); + comment("Runtime stack is corrupt"); a.ud2(); a.bind(next); @@ -568,7 +568,7 @@ protected: a.short_().jle(next); a.bind(crash); - a.comment("Erlang stack is corrupt"); + comment("Erlang stack is corrupt"); a.ud2(); a.bind(next); #endif @@ -794,6 +794,30 @@ protected: } } + /* Set the Z flag if Reg1 and Reg2 are definitely not equal based on their + * tags alone. (They may still be equal if both are immediates and all other + * bits are equal too.) */ + void emit_is_unequal_based_on_tags(x86::Gp Reg1, x86::Gp Reg2) { + ASSERT(Reg1 != RET && Reg2 != RET); + emit_is_unequal_based_on_tags(Reg1, Reg2, RET); + } + + void emit_is_unequal_based_on_tags(x86::Gp Reg1, + x86::Gp Reg2, + const x86::Gp &spill) { + ERTS_CT_ASSERT(TAG_PRIMARY_IMMED1 == _TAG_PRIMARY_MASK); + ERTS_CT_ASSERT((TAG_PRIMARY_LIST | TAG_PRIMARY_BOXED) == + TAG_PRIMARY_IMMED1); + a.mov(RETd, Reg1.r32()); + a.or_(RETd, Reg2.r32()); + a.and_(RETb, imm(_TAG_PRIMARY_MASK)); + + /* RET will be now be TAG_PRIMARY_IMMED1 if either one or both + * registers are immediates, or if one register is a list and the other + * a boxed. */ + a.cmp(RETb, imm(TAG_PRIMARY_IMMED1)); + } + /* * Generate the shortest instruction for setting a register to an immediate * value. May clear flags. @@ -845,6 +869,12 @@ public: void setLogger(std::string log); void setLogger(FILE *log); + void comment(const char *format) { + if (logger.file()) { + a.commentf("# %s", format); + } + } + template<typename... Ts> void comment(const char *format, Ts... args) { if (logger.file()) { @@ -961,8 +991,8 @@ class BeamGlobalAssembler : public BeamAssembler { }; #undef DECL_ENUM + static const std::map<GlobalLabels, const std::string> labelNames; static const std::map<GlobalLabels, emitFptr> emitPtrs; - static const std::map<GlobalLabels, std::string> labelNames; std::unordered_map<GlobalLabels, Label> labels; std::unordered_map<GlobalLabels, fptr> ptrs; @@ -1001,8 +1031,8 @@ class BeamModuleAssembler : public BeamAssembler { typedef unsigned BeamLabel; /* Map of label number to asmjit Label */ - typedef std::unordered_map<BeamLabel, Label> LabelMap; - LabelMap labels; + typedef std::unordered_map<BeamLabel, const Label> LabelMap; + LabelMap rawLabels; struct patch { Label where; @@ -1045,6 +1075,9 @@ class BeamModuleAssembler : public BeamAssembler { /* All functions that have been seen so far */ std::vector<BeamLabel> functions; + /* The BEAM file we've been loaded from, if any. */ + BeamFile *beam; + BeamGlobalAssembler *ga; Label codeHeader; @@ -1072,13 +1105,13 @@ class BeamModuleAssembler : public BeamAssembler { public: BeamModuleAssembler(BeamGlobalAssembler *ga, Eterm mod, - unsigned num_labels, - BeamFile_ExportTable *named_labels = NULL); + int num_labels, + BeamFile *file = NULL); BeamModuleAssembler(BeamGlobalAssembler *ga, Eterm mod, - unsigned num_labels, - unsigned num_functions, - BeamFile_ExportTable *named_labels = NULL); + int num_labels, + int num_functions, + BeamFile *file = NULL); bool emit(unsigned op, const Span<ArgVal> &args); @@ -1125,6 +1158,67 @@ public: void patchStrings(char *rw_base, const byte *string); protected: + bool always_immediate(const ArgVal &arg) const { + if (arg.isImmed()) { + return true; + } + + int type_union = beam->types.entries[arg.typeIndex()].type_union; + return (type_union & BEAM_TYPE_MASK_ALWAYS_IMMEDIATE) == type_union; + } + + bool always_same_types(const ArgVal &lhs, const ArgVal &rhs) const { + int lhs_types = beam->types.entries[lhs.typeIndex()].type_union; + int rhs_types = beam->types.entries[rhs.typeIndex()].type_union; + + /* We can only be certain that the types are the same when there's + * one possible type. For example, if one is a number and the other + * is an integer, they could differ if the former is a float. */ + if ((lhs_types & (lhs_types - 1)) == 0) { + return lhs_types == rhs_types; + } + + return false; + } + + bool always_one_of(const ArgVal &arg, int types) const { + if (arg.isImmed()) { + if (is_small(arg.getValue())) { + return !!(types & BEAM_TYPE_INTEGER); + } else if (is_atom(arg.getValue())) { + return !!(types & BEAM_TYPE_ATOM); + } else if (is_nil(arg.getValue())) { + return !!(types & BEAM_TYPE_NIL); + } + + return false; + } else { + int type_union = beam->types.entries[arg.typeIndex()].type_union; + return type_union == (type_union & types); + } + } + + int masked_types(const ArgVal &arg, int mask) const { + if (arg.isImmed()) { + if (is_small(arg.getValue())) { + return mask & BEAM_TYPE_INTEGER; + } else if (is_atom(arg.getValue())) { + return mask & BEAM_TYPE_ATOM; + } else if (is_nil(arg.getValue())) { + return mask & BEAM_TYPE_NIL; + } + + return BEAM_TYPE_NONE; + } else { + int type_union = beam->types.entries[arg.typeIndex()].type_union; + return type_union & mask; + } + } + + bool exact_type(const ArgVal &arg, int type_id) const { + return always_one_of(arg, type_id); + } + /* Helpers */ void emit_gc_test(const ArgVal &Stack, const ArgVal &Heap, @@ -1136,9 +1230,30 @@ protected: x86::Mem emit_variable_apply(bool includeI); x86::Mem emit_fixed_apply(const ArgVal &arity, bool includeI); - x86::Gp emit_call_fun(void); + x86::Gp emit_call_fun(bool skip_box_test = false, + bool skip_fun_test = false, + bool skip_arity_test = false); + + x86::Gp emit_is_binary(const ArgVal &Fail, + const ArgVal &Src, + Label next, + Label subbin); + + void emit_is_boxed(Label Fail, x86::Gp Src, Distance dist = dLong) { + BeamAssembler::emit_is_boxed(Fail, Src, dist); + } + + void emit_is_boxed(Label Fail, + const ArgVal &Arg, + x86::Gp Src, + Distance dist = dLong) { + if (always_one_of(Arg, BEAM_TYPE_MASK_ALWAYS_BOXED)) { + comment("skipped box test since argument is always boxed"); + return; + } - void emit_is_binary(Label Fail, x86::Gp Src, Label next, Label subbin); + BeamAssembler::emit_is_boxed(Fail, Src, dist); + } void emit_get_list(const x86::Gp boxed_ptr, const ArgVal &Hd, @@ -1187,16 +1302,20 @@ protected: const ArgVal &RHS, const ArgVal &Dst); - void emit_is_small(Label fail, x86::Gp Reg); - void emit_is_both_small(Label fail, x86::Gp A, x86::Gp B); + void emit_is_small(Label fail, const ArgVal &Arg, x86::Gp Reg); + void emit_are_both_small(Label fail, + const ArgVal &LHS, + x86::Gp A, + const ArgVal &RHS, + x86::Gp B); void emit_validate_unicode(Label next, Label fail, x86::Gp value); - void emit_bif_is_eq_ne_exact_immed(const ArgVal &Src, - const ArgVal &Immed, - const ArgVal &Dst, - Eterm fail_value, - Eterm succ_value); + void emit_bif_is_eq_ne_exact(const ArgVal &LHS, + const ArgVal &RHS, + const ArgVal &Dst, + Eterm fail_value, + Eterm succ_value); void emit_proc_lc_unrequire(void); void emit_proc_lc_require(void); @@ -1218,6 +1337,11 @@ protected: #include "beamasm_protos.h" + const Label &resolve_beam_label(const ArgVal &Lbl) const { + ASSERT(Lbl.isLabel()); + return rawLabels.at(Lbl.getValue()); + } + void make_move_patch(x86::Gp to, std::vector<struct patch> &patches, int64_t offset = 0) { diff --git a/erts/emulator/beam/jit/x86/beam_asm_global.cpp b/erts/emulator/beam/jit/x86/beam_asm_global.cpp index 3fea108adb..00fffcc56c 100644 --- a/erts/emulator/beam/jit/x86/beam_asm_global.cpp +++ b/erts/emulator/beam/jit/x86/beam_asm_global.cpp @@ -38,7 +38,7 @@ const std::map<BeamGlobalAssembler::GlobalLabels, BeamGlobalAssembler::emitFptr> #define DECL_LABEL_NAME(NAME) {NAME, STRINGIFY(NAME)}, -const std::map<BeamGlobalAssembler::GlobalLabels, std::string> +const std::map<BeamGlobalAssembler::GlobalLabels, const std::string> BeamGlobalAssembler::labelNames = {BEAM_GLOBAL_FUNCS( DECL_LABEL_NAME) PROCESS_MAIN_LABELS(DECL_LABEL_NAME)}; #undef DECL_LABEL_NAME @@ -178,7 +178,7 @@ void BeamGlobalAssembler::emit_export_trampoline() { a.je(jump_trace); /* Must never happen. */ - a.comment("# Unexpected export trampoline op"); + comment("Unexpected export trampoline op"); a.ud2(); a.bind(call_bif); @@ -289,7 +289,7 @@ void BeamGlobalAssembler::emit_process_exit() { a.test(RET, RET); a.je(labels[do_schedule]); - a.comment("# End of process"); + comment("End of process"); a.ud2(); } @@ -341,7 +341,7 @@ void BeamGlobalAssembler::emit_raise_exception_shared() { a.jmp(RET); a.bind(crash); - a.comment("# Error address is not a CP or NULL or ARG2 and ARG4 are unset"); + comment("Error address is not a CP or NULL or ARG2 and ARG4 are unset"); a.ud2(); } diff --git a/erts/emulator/beam/jit/x86/beam_asm_module.cpp b/erts/emulator/beam/jit/x86/beam_asm_module.cpp index de7504c22e..a69961c24b 100644 --- a/erts/emulator/beam/jit/x86/beam_asm_module.cpp +++ b/erts/emulator/beam/jit/x86/beam_asm_module.cpp @@ -24,11 +24,6 @@ #include "beam_asm.hpp" using namespace asmjit; -static std::string getAtom(Eterm atom) { - Atom *ap = atom_tab(atom_val(atom)); - return std::string((char *)ap->name, ap->len); -} - #ifdef BEAMASM_DUMP_SIZES # include <mutex> @@ -68,8 +63,8 @@ extern "C" void beamasm_dump_sizes() { #endif ErtsCodePtr BeamModuleAssembler::getCode(BeamLabel label) { - ASSERT(label < labels.size() + 1); - return (ErtsCodePtr)getCode(labels[label]); + ASSERT(label < rawLabels.size() + 1); + return (ErtsCodePtr)getCode(rawLabels[label]); } ErtsCodePtr BeamModuleAssembler::getLambda(unsigned index) { @@ -77,55 +72,12 @@ ErtsCodePtr BeamModuleAssembler::getLambda(unsigned index) { return (ErtsCodePtr)getCode(lambda.trampoline); } -BeamModuleAssembler::BeamModuleAssembler(BeamGlobalAssembler *_ga, - Eterm _mod, - unsigned num_labels, - BeamFile_ExportTable *named_labels) - : BeamAssembler(getAtom(_mod)), ga(_ga), mod(_mod) { - labels.reserve(num_labels + 1); - - if (logger.file() && named_labels) { - BeamFile_ExportEntry *e = &named_labels->entries[0]; - for (unsigned int i = 1; i < num_labels; i++) { - /* Large enough to hold most realistic function names. We will - * truncate too long names, but as the label name is not important - * for the functioning of the JIT and this functionality is - * probably only used by developers, we don't bother with dynamic - * allocation. */ - char tmp[MAX_ATOM_SZ_LIMIT]; - - /* The named_labels are sorted, so no need for a search. */ - if ((unsigned int)e->label == i) { - erts_snprintf(tmp, sizeof(tmp), "%T/%d", e->function, e->arity); - labels[i] = a.newNamedLabel(tmp); - e++; - } else { - std::string lblName = "label_" + std::to_string(i); - labels[i] = a.newNamedLabel(lblName.data()); - } - } - } else if (logger.file()) { - /* There is no naming info, but dumping of the assembly code - * has been requested, so do the best we can and number the - * labels. */ - for (unsigned int i = 1; i < num_labels; i++) { - std::string lblName = "label_" + std::to_string(i); - labels[i] = a.newNamedLabel(lblName.data()); - } - } else { - /* No output is requested, go with unnamed labels */ - for (unsigned int i = 1; i < num_labels; i++) { - labels[i] = a.newLabel(); - } - } -} - BeamModuleAssembler::BeamModuleAssembler(BeamGlobalAssembler *ga, Eterm mod, - unsigned num_labels, - unsigned num_functions, - BeamFile_ExportTable *named_labels) - : BeamModuleAssembler(ga, mod, num_labels, named_labels) { + int num_labels, + int num_functions, + BeamFile *file) + : BeamModuleAssembler(ga, mod, num_labels, file) { codeHeader = a.newLabel(); a.align(kAlignCode, 8); a.bind(codeHeader); @@ -206,7 +158,7 @@ Label BeamModuleAssembler::embed_vararg_rodata(const Span<ArgVal> &args, make_word_patch(literals[arg.getValue()].patches); break; case ArgVal::Label: - a.embedLabel(labels[arg.getValue()]); + a.embedLabel(resolve_beam_label(arg)); break; case ArgVal::Immediate: case ArgVal::Word: @@ -228,7 +180,7 @@ void BeamModuleAssembler::emit_i_nif_padding() { const size_t minimum_size = sizeof(UWord[BEAM_NATIVE_MIN_FUNC_SZ]); size_t prev_func_start, diff; - prev_func_start = code.labelOffsetFromBase(labels[functions.back() + 1]); + prev_func_start = code.labelOffsetFromBase(rawLabels[functions.back() + 1]); diff = a.offset() - prev_func_start; if (diff < minimum_size) { @@ -373,7 +325,9 @@ void BeamModuleAssembler::emit_i_func_info(const ArgVal &Label, } void BeamModuleAssembler::emit_label(const ArgVal &Label) { - currLabel = labels[Label.getValue()]; + ASSERT(Label.isLabel()); + + currLabel = rawLabels[Label.getValue()]; a.bind(currLabel); } diff --git a/erts/emulator/beam/jit/x86/generators.tab b/erts/emulator/beam/jit/x86/generators.tab index 176192a960..148fd1131b 100644 --- a/erts/emulator/beam/jit/x86/generators.tab +++ b/erts/emulator/beam/jit/x86/generators.tab @@ -648,3 +648,13 @@ gen.create_bin(Fail, Alloc, Live, Unit, Dst, N, Segments) { return op; } + +gen.remove_tuple_type(Tuple) { + BeamOp* op; + + $NewBeamOp(S, op); + $BeamOpNameArity(op, current_tuple, 1); + op->a[0] = Tuple; + op->a[0].val = Tuple.val & REG_MASK; + return op; +} diff --git a/erts/emulator/beam/jit/x86/instr_arith.cpp b/erts/emulator/beam/jit/x86/instr_arith.cpp index fd9c38f9a2..32cf40af3d 100644 --- a/erts/emulator/beam/jit/x86/instr_arith.cpp +++ b/erts/emulator/beam/jit/x86/instr_arith.cpp @@ -30,32 +30,62 @@ extern "C" #include "erl_bif_table.h" } -void BeamModuleAssembler::emit_is_small(Label fail, x86::Gp Reg) { +void BeamModuleAssembler::emit_is_small(Label fail, + const ArgVal &Arg, + x86::Gp Reg) { ASSERT(ARG1 != Reg); - comment("is_small(X)"); - a.mov(ARG1d, Reg.r32()); - a.and_(ARG1d, imm(_TAG_IMMED1_MASK)); - a.cmp(ARG1d, imm(_TAG_IMMED1_SMALL)); - a.short_().jne(fail); + if (always_one_of(Arg, BEAM_TYPE_FLOAT | BEAM_TYPE_INTEGER)) { + comment("simplified test for small operand since it is a number"); + a.test(Reg.r32(), imm(TAG_PRIMARY_LIST)); + a.short_().je(fail); + } else { + comment("is the operand small?"); + a.mov(ARG1d, Reg.r32()); + a.and_(ARG1d, imm(_TAG_IMMED1_MASK)); + a.cmp(ARG1d, imm(_TAG_IMMED1_SMALL)); + a.short_().jne(fail); + } } -void BeamModuleAssembler::emit_is_both_small(Label fail, x86::Gp A, x86::Gp B) { +void BeamModuleAssembler::emit_are_both_small(Label fail, + const ArgVal &LHS, + x86::Gp A, + const ArgVal &RHS, + x86::Gp B) { ASSERT(ARG1 != A && ARG1 != B); - - comment("is_both_small(X, Y)"); - if (A != RET && B != RET) { - a.mov(RETd, A.r32()); - a.and_(RETd, B.r32()); - a.and_(RETb, imm(_TAG_IMMED1_MASK)); - a.cmp(RETb, imm(_TAG_IMMED1_SMALL)); + if (always_one_of(LHS, BEAM_TYPE_FLOAT | BEAM_TYPE_INTEGER) && + always_one_of(RHS, BEAM_TYPE_FLOAT | BEAM_TYPE_INTEGER)) { + comment("simplified test for small operands since both are numbers"); + if (RHS.isImmed() && is_small(RHS.getValue())) { + a.test(A.r32(), imm(TAG_PRIMARY_LIST)); + } else if (LHS.isImmed() && is_small(LHS.getValue())) { + a.test(B.r32(), imm(TAG_PRIMARY_LIST)); + } else if (A != RET && B != RET) { + a.mov(RETd, A.r32()); + a.and_(RETd, B.r32()); + a.test(RETb, imm(TAG_PRIMARY_LIST)); + } else { + a.mov(ARG1d, A.r32()); + a.and_(ARG1d, B.r32()); + a.test(ARG1d, imm(TAG_PRIMARY_LIST)); + } + a.short_().je(fail); } else { - a.mov(ARG1d, A.r32()); - a.and_(ARG1d, B.r32()); - a.and_(ARG1d, imm(_TAG_IMMED1_MASK)); - a.cmp(ARG1d, imm(_TAG_IMMED1_SMALL)); + comment("are both operands small?"); + if (A != RET && B != RET) { + a.mov(RETd, A.r32()); + a.and_(RETd, B.r32()); + a.and_(RETb, imm(_TAG_IMMED1_MASK)); + a.cmp(RETb, imm(_TAG_IMMED1_SMALL)); + } else { + a.mov(ARG1d, A.r32()); + a.and_(ARG1d, B.r32()); + a.and_(ARG1d, imm(_TAG_IMMED1_MASK)); + a.cmp(ARG1d, imm(_TAG_IMMED1_SMALL)); + } + a.short_().jne(fail); } - a.short_().jne(fail); } void BeamGlobalAssembler::emit_increment_body_shared() { @@ -91,14 +121,23 @@ void BeamModuleAssembler::emit_i_increment(const ArgVal &Src, * that ARG3 is untagged at this point */ mov_arg(ARG2, Src); mov_imm(ARG3, Val.getValue() << _TAG_IMMED1_SIZE); - a.mov(RETd, ARG2d); - a.and_(RETb, imm(_TAG_IMMED1_MASK)); - a.cmp(RETb, imm(_TAG_IMMED1_SMALL)); - a.short_().jne(mixed); - a.mov(RET, ARG2); - a.add(RET, ARG3); - a.short_().jno(next); + if (always_one_of(Src, BEAM_TYPE_FLOAT | BEAM_TYPE_INTEGER)) { + comment("simplified test for small operand since it is a number"); + a.mov(RET, ARG2); + a.test(RETb, imm(TAG_PRIMARY_LIST)); + a.short_().je(mixed); + a.add(RET, ARG3); + a.short_().jno(next); + } else { + a.mov(RETd, ARG2d); + a.and_(RETb, imm(_TAG_IMMED1_MASK)); + a.cmp(RETb, imm(_TAG_IMMED1_SMALL)); + a.short_().jne(mixed); + a.mov(RET, ARG2); + a.add(RET, ARG3); + a.short_().jno(next); + } a.bind(mixed); safe_fragment_call(ga->get_increment_body_shared()); @@ -167,7 +206,7 @@ void BeamModuleAssembler::emit_i_plus(const ArgVal &LHS, mov_arg(ARG2, LHS); /* Used by erts_mixed_plus in this slot */ mov_arg(ARG3, RHS); /* Used by erts_mixed_plus in this slot */ - emit_is_both_small(mixed, ARG2, ARG3); + emit_are_both_small(mixed, LHS, ARG2, RHS, ARG3); comment("add with overflow check"); a.mov(RET, ARG2); @@ -181,7 +220,7 @@ void BeamModuleAssembler::emit_i_plus(const ArgVal &LHS, { if (Fail.getValue() != 0) { safe_fragment_call(ga->get_plus_guard_shared()); - a.je(labels[Fail.getValue()]); + a.je(resolve_beam_label(Fail)); } else { safe_fragment_call(ga->get_plus_body_shared()); } @@ -251,17 +290,7 @@ void BeamModuleAssembler::emit_i_minus(const ArgVal &LHS, mov_arg(ARG2, LHS); /* Used by erts_mixed_plus in this slot */ mov_arg(ARG3, RHS); /* Used by erts_mixed_plus in this slot */ - if (RHS.isImmed() && is_small(RHS.getValue())) { - a.mov(RETd, ARG2d); - } else if (LHS.isImmed() && is_small(LHS.getValue())) { - a.mov(RETd, ARG3d); - } else { - a.mov(RETd, ARG2d); - a.and_(RETd, ARG3d); - } - a.and_(RETb, imm(_TAG_IMMED1_MASK)); - a.cmp(RETb, imm(_TAG_IMMED1_SMALL)); - a.short_().jne(mixed); + emit_are_both_small(mixed, LHS, ARG2, RHS, ARG3); comment("sub with overflow check"); a.mov(RET, ARG2); @@ -273,7 +302,7 @@ void BeamModuleAssembler::emit_i_minus(const ArgVal &LHS, a.bind(mixed); if (Fail.getValue() != 0) { safe_fragment_call(ga->get_minus_guard_shared()); - a.je(labels[Fail.getValue()]); + a.je(resolve_beam_label(Fail)); } else { safe_fragment_call(ga->get_minus_body_shared()); } @@ -352,7 +381,7 @@ void BeamModuleAssembler::emit_i_unary_minus(const ArgVal &Src, a.bind(mixed); if (Fail.getValue() != 0) { safe_fragment_call(ga->get_unary_minus_guard_shared()); - a.je(labels[Fail.getValue()]); + a.je(resolve_beam_label(Fail)); } else { safe_fragment_call(ga->get_unary_minus_body_shared()); } @@ -560,7 +589,7 @@ void BeamModuleAssembler::emit_div_rem(const ArgVal &Fail, * compiler. */ if (Fail.getValue() != 0) { safe_fragment_call(ga->get_int_div_rem_guard_shared()); - a.je(labels[Fail.getValue()]); + a.je(resolve_beam_label(Fail)); } else { a.mov(ARG5, imm(error_mfa)); safe_fragment_call(ga->get_int_div_rem_body_shared()); @@ -637,7 +666,7 @@ void BeamModuleAssembler::emit_i_m_div(const ArgVal &Fail, emit_test_the_non_value(RET); if (Fail.getValue() != 0) { - a.je(labels[Fail.getValue()]); + a.je(resolve_beam_label(Fail)); } else { a.short_().jne(next); @@ -720,18 +749,18 @@ void BeamModuleAssembler::emit_i_times(const ArgVal &Fail, if (RHS.isImmed() && is_small(RHS.getValue())) { Sint val = signed_val(RHS.getValue()); - emit_is_small(mixed, ARG2); + emit_is_small(mixed, LHS, ARG2); comment("mul with overflow check, imm RHS"); a.mov(RET, ARG2); a.mov(ARG4, imm(val)); } else if (LHS.isImmed() && is_small(LHS.getValue())) { Sint val = signed_val(LHS.getValue()); - emit_is_small(mixed, ARG3); + emit_is_small(mixed, RHS, ARG3); comment("mul with overflow check, imm LHS"); a.mov(RET, ARG3); a.mov(ARG4, imm(val)); } else { - emit_is_both_small(mixed, ARG2, ARG3); + emit_are_both_small(mixed, LHS, ARG2, RHS, ARG3); comment("mul with overflow check"); a.mov(RET, ARG2); a.mov(ARG4, ARG3); @@ -749,7 +778,7 @@ void BeamModuleAssembler::emit_i_times(const ArgVal &Fail, { if (Fail.getValue() != 0) { safe_fragment_call(ga->get_times_guard_shared()); - a.je(labels[Fail.getValue()]); + a.je(resolve_beam_label(Fail)); } else { safe_fragment_call(ga->get_times_body_shared()); } @@ -838,9 +867,9 @@ void BeamModuleAssembler::emit_i_band(const ArgVal &LHS, mov_arg(RET, RHS); if (RHS.isImmed() && is_small(RHS.getValue())) { - emit_is_small(generic, ARG2); + emit_is_small(generic, LHS, ARG2); } else { - emit_is_both_small(generic, RET, ARG2); + emit_are_both_small(generic, LHS, RET, RHS, ARG2); } /* TAG & TAG = TAG, so we don't need to tag it again. */ @@ -851,7 +880,7 @@ void BeamModuleAssembler::emit_i_band(const ArgVal &LHS, { if (Fail.getValue() != 0) { safe_fragment_call(ga->get_i_band_guard_shared()); - a.je(labels[Fail.getValue()]); + a.je(resolve_beam_label(Fail)); } else { safe_fragment_call(ga->get_i_band_body_shared()); } @@ -886,9 +915,9 @@ void BeamModuleAssembler::emit_i_bor(const ArgVal &Fail, mov_arg(RET, RHS); if (RHS.isImmed() && is_small(RHS.getValue())) { - emit_is_small(generic, ARG2); + emit_is_small(generic, LHS, ARG2); } else { - emit_is_both_small(generic, RET, ARG2); + emit_are_both_small(generic, LHS, RET, RHS, ARG2); } /* TAG | TAG = TAG, so we don't need to tag it again. */ @@ -899,7 +928,7 @@ void BeamModuleAssembler::emit_i_bor(const ArgVal &Fail, { if (Fail.getValue() != 0) { safe_fragment_call(ga->get_i_bor_guard_shared()); - a.je(labels[Fail.getValue()]); + a.je(resolve_beam_label(Fail)); } else { safe_fragment_call(ga->get_i_bor_body_shared()); } @@ -934,9 +963,9 @@ void BeamModuleAssembler::emit_i_bxor(const ArgVal &Fail, mov_arg(RET, RHS); if (RHS.isImmed() && is_small(RHS.getValue())) { - emit_is_small(generic, ARG2); + emit_is_small(generic, LHS, ARG2); } else { - emit_is_both_small(generic, RET, ARG2); + emit_are_both_small(generic, LHS, RET, RHS, ARG2); } /* TAG ^ TAG = 0, so we need to tag it again. */ @@ -948,7 +977,7 @@ void BeamModuleAssembler::emit_i_bxor(const ArgVal &Fail, { if (Fail.getValue() != 0) { safe_fragment_call(ga->get_i_bxor_guard_shared()); - a.je(labels[Fail.getValue()]); + a.je(resolve_beam_label(Fail)); } else { safe_fragment_call(ga->get_i_bxor_body_shared()); } @@ -1032,14 +1061,20 @@ void BeamModuleAssembler::emit_i_bnot(const ArgVal &Fail, /* Fall through to the generic path if the result is not a small, where the * above operation will be reverted. */ - a.mov(ARG1d, RETd); - a.and_(ARG1d, imm(_TAG_IMMED1_MASK)); - a.cmp(ARG1d, imm(_TAG_IMMED1_SMALL)); - a.short_().je(next); + if (always_one_of(Src, BEAM_TYPE_FLOAT | BEAM_TYPE_INTEGER)) { + comment("simplified test for small operand since it is a number"); + a.test(RETb, imm(TAG_PRIMARY_LIST)); + a.short_().jne(next); + } else { + a.mov(ARG1d, RETd); + a.and_(ARG1d, imm(_TAG_IMMED1_MASK)); + a.cmp(ARG1d, imm(_TAG_IMMED1_SMALL)); + a.short_().je(next); + } if (Fail.getValue() != 0) { safe_fragment_call(ga->get_i_bnot_guard_shared()); - a.je(labels[Fail.getValue()]); + a.je(resolve_beam_label(Fail)); } else { safe_fragment_call(ga->get_i_bnot_body_shared()); } @@ -1075,7 +1110,7 @@ void BeamModuleAssembler::emit_i_bsr(const ArgVal &LHS, Sint shift = signed_val(RHS.getValue()); if (shift >= 0 && shift < SMALL_BITS - 1) { - emit_is_small(generic, ARG2); + emit_is_small(generic, LHS, ARG2); a.mov(RET, ARG2); @@ -1098,7 +1133,7 @@ void BeamModuleAssembler::emit_i_bsr(const ArgVal &LHS, if (Fail.getValue() != 0) { safe_fragment_call(ga->get_i_bsr_guard_shared()); - a.je(labels[Fail.getValue()]); + a.je(resolve_beam_label(Fail)); } else { safe_fragment_call(ga->get_i_bsr_body_shared()); } @@ -1235,7 +1270,7 @@ void BeamModuleAssembler::emit_i_bsl(const ArgVal &LHS, { if (Fail.getValue() != 0) { safe_fragment_call(ga->get_i_bsl_guard_shared()); - a.je(labels[Fail.getValue()]); + a.je(resolve_beam_label(Fail)); } else { safe_fragment_call(ga->get_i_bsl_body_shared()); } diff --git a/erts/emulator/beam/jit/x86/instr_bif.cpp b/erts/emulator/beam/jit/x86/instr_bif.cpp index 6c8160a7de..4c8db7de3f 100644 --- a/erts/emulator/beam/jit/x86/instr_bif.cpp +++ b/erts/emulator/beam/jit/x86/instr_bif.cpp @@ -136,7 +136,7 @@ void BeamModuleAssembler::emit_i_bif1(const ArgVal &Src1, if (Fail.getValue() != 0) { safe_fragment_call(ga->get_i_bif_guard_shared()); - a.je(labels[Fail.getValue()]); + a.je(resolve_beam_label(Fail)); } else { safe_fragment_call(ga->get_i_bif_body_shared()); } @@ -153,7 +153,7 @@ void BeamModuleAssembler::emit_i_bif2(const ArgVal &Src1, if (Fail.getValue() != 0) { safe_fragment_call(ga->get_i_bif_guard_shared()); - a.je(labels[Fail.getValue()]); + a.je(resolve_beam_label(Fail)); } else { safe_fragment_call(ga->get_i_bif_body_shared()); } @@ -171,7 +171,7 @@ void BeamModuleAssembler::emit_i_bif3(const ArgVal &Src1, if (Fail.getValue() != 0) { safe_fragment_call(ga->get_i_bif_guard_shared()); - a.je(labels[Fail.getValue()]); + a.je(resolve_beam_label(Fail)); } else { safe_fragment_call(ga->get_i_bif_body_shared()); } @@ -340,7 +340,7 @@ void BeamModuleAssembler::emit_i_length(const ArgVal &Fail, /* The return address is discarded when yielding, so it doesn't need to * be aligned. */ safe_fragment_call(ga->get_i_length_guard_shared()); - a.je(labels[Fail.getValue()]); + a.je(resolve_beam_label(Fail)); } else { safe_fragment_call(ga->get_i_length_body_shared()); } @@ -883,7 +883,7 @@ void BeamGlobalAssembler::emit_call_nif_early() { a.test(ARG2, imm(sizeof(UWord) - 1)); a.short_().je(next); - a.comment("# Return address isn't word-aligned"); + comment("# Return address isn't word-aligned"); a.ud2(); a.bind(next); diff --git a/erts/emulator/beam/jit/x86/instr_bs.cpp b/erts/emulator/beam/jit/x86/instr_bs.cpp index d6e0ddd4dd..01e2ac593b 100644 --- a/erts/emulator/beam/jit/x86/instr_bs.cpp +++ b/erts/emulator/beam/jit/x86/instr_bs.cpp @@ -133,7 +133,7 @@ void BeamModuleAssembler::emit_i_bs_init_fail_heap(const ArgVal &Size, Label fail; if (Fail.getValue() != 0) { - fail = labels[Fail.getValue()]; + fail = resolve_beam_label(Fail); } else { fail = a.newLabel(); } @@ -234,7 +234,7 @@ void BeamModuleAssembler::emit_i_bs_init_bits_fail_heap(const ArgVal &NumBits, Label fail; if (Fail.getValue() != 0) { - fail = labels[Fail.getValue()]; + fail = resolve_beam_label(Fail); } else { fail = a.newLabel(); } @@ -311,7 +311,7 @@ void BeamModuleAssembler::emit_i_new_bs_put_integer_imm(const ArgVal &Src, a.test(RET, RET); if (Fail.getValue() != 0) { - a.je(labels[Fail.getValue()]); + a.je(resolve_beam_label(Fail)); } else { a.short_().jne(next); emit_error(BADARG); @@ -327,7 +327,7 @@ void BeamModuleAssembler::emit_i_new_bs_put_integer(const ArgVal &Fail, Label next, fail; if (Fail.getValue() != 0) { - fail = labels[Fail.getValue()]; + fail = resolve_beam_label(Fail); } else { fail = a.newLabel(); next = a.newLabel(); @@ -369,7 +369,7 @@ void BeamModuleAssembler::emit_i_new_bs_put_binary(const ArgVal &Fail, Label next, fail; if (Fail.getValue() != 0) { - fail = labels[Fail.getValue()]; + fail = resolve_beam_label(Fail); } else { fail = a.newLabel(); next = a.newLabel(); @@ -428,7 +428,7 @@ void BeamModuleAssembler::emit_i_new_bs_put_binary_all(const ArgVal &Src, emit_error(BADARG); a.bind(next); } else { - a.je(labels[Fail.getValue()]); + a.je(resolve_beam_label(Fail)); } } @@ -458,7 +458,7 @@ void BeamModuleAssembler::emit_i_new_bs_put_binary_imm(const ArgVal &Fail, emit_error(BADARG); a.bind(next); } else { - a.je(labels[Fail.getValue()]); + a.je(resolve_beam_label(Fail)); } } @@ -470,7 +470,7 @@ void BeamModuleAssembler::emit_i_new_bs_put_float(const ArgVal &Fail, Label next, fail; if (Fail.getValue() != 0) { - fail = labels[Fail.getValue()]; + fail = resolve_beam_label(Fail); } else { fail = a.newLabel(); next = a.newLabel(); @@ -528,7 +528,7 @@ void BeamModuleAssembler::emit_i_new_bs_put_float_imm(const ArgVal &Fail, emit_test_the_non_value(RET); if (Fail.getValue() != 0) { - a.jne(labels[Fail.getValue()]); + a.jne(resolve_beam_label(Fail)); } else { a.short_().je(next); emit_error(BADARG); @@ -545,7 +545,7 @@ void BeamModuleAssembler::emit_i_bs_start_match3(const ArgVal &Src, mov_arg(ARG2, Src); if (Fail.getValue() != 0) { - emit_is_boxed(labels[Fail.getValue()], ARG2); + emit_is_boxed(resolve_beam_label(Fail), Src, ARG2); } else { /* bs_start_match3 may not throw, and the compiler will only emit {f,0} * when it knows that the source is a match state or binary, so we're @@ -570,7 +570,7 @@ void BeamModuleAssembler::emit_i_bs_start_match3(const ArgVal &Src, ERTS_CT_ASSERT(_TAG_HEADER_REFC_BIN + 4 == _TAG_HEADER_HEAP_BIN); a.and_(RETb, imm(~4)); a.cmp(RETb, imm(_TAG_HEADER_REFC_BIN)); - a.jne(labels[Fail.getValue()]); + a.jne(resolve_beam_label(Fail)); } a.bind(is_binary); @@ -602,7 +602,7 @@ void BeamModuleAssembler::emit_i_bs_match_string(const ArgVal &Ctx, const ArgVal &Bits, const ArgVal &Ptr) { const UWord size = Bits.getValue(); - Label fail = labels[Fail.getValue()]; + Label fail = resolve_beam_label(Fail); mov_arg(ARG1, Ctx); @@ -723,7 +723,7 @@ void BeamModuleAssembler::emit_i_bs_get_integer_8(const ArgVal &Ctx, mov_arg(ARG4, Ctx); address = emit_bs_get_integer_prologue(next, - labels[Fail.getValue()], + resolve_beam_label(Fail), flags, 8); @@ -751,7 +751,7 @@ void BeamModuleAssembler::emit_i_bs_get_integer_16(const ArgVal &Ctx, mov_arg(ARG4, Ctx); address = emit_bs_get_integer_prologue(next, - labels[Fail.getValue()], + resolve_beam_label(Fail), flags, 16); @@ -794,7 +794,7 @@ void BeamModuleAssembler::emit_i_bs_get_integer_32(const ArgVal &Ctx, mov_arg(ARG4, Ctx); address = emit_bs_get_integer_prologue(next, - labels[Fail.getValue()], + resolve_beam_label(Fail), flags, 32); @@ -843,7 +843,7 @@ void BeamModuleAssembler::emit_i_bs_get_integer_64(const ArgVal &Ctx, ARG4); address = emit_bs_get_integer_prologue(next, - labels[Fail.getValue()], + resolve_beam_label(Fail), flags, 64); @@ -901,7 +901,7 @@ void BeamModuleAssembler::emit_i_bs_get_integer(const ArgVal &Ctx, Label fail; int unit; - fail = labels[Fail.getValue()]; + fail = resolve_beam_label(Fail); unit = FlagsAndUnit.getValue() >> 3; /* Clobbers RET + ARG3, returns a negative result if we always fail and @@ -942,7 +942,7 @@ void BeamModuleAssembler::emit_bs_test_tail2(const ArgVal &Fail, a.cmp(ARG2, imm(Offset.getValue())); } - a.jne(labels[Fail.getValue()]); + a.jne(resolve_beam_label(Fail)); } void BeamModuleAssembler::emit_bs_set_position(const ArgVal &Ctx, @@ -982,7 +982,7 @@ void BeamModuleAssembler::emit_i_bs_get_binary_all2(const ArgVal &Ctx, a.test(RETb, imm(unit - 1)); } - a.jne(labels[Fail.getValue()]); + a.jne(resolve_beam_label(Fail)); emit_enter_runtime<Update::eHeap>(); @@ -1037,7 +1037,7 @@ void BeamModuleAssembler::emit_bs_skip_bits(const ArgVal &Fail, a.add(RET, emit_boxed_val(ARG1, offsetof(ErlBinMatchState, mb.offset))); a.cmp(RET, emit_boxed_val(ARG1, offsetof(ErlBinMatchState, mb.size))); - a.ja(labels[Fail.getValue()]); + a.ja(resolve_beam_label(Fail)); a.mov(emit_boxed_val(ARG1, offsetof(ErlBinMatchState, mb.offset)), RET); } @@ -1048,7 +1048,7 @@ void BeamModuleAssembler::emit_i_bs_skip_bits2(const ArgVal &Ctx, const ArgVal &Unit) { Label fail; - fail = labels[Fail.getValue()]; + fail = resolve_beam_label(Fail); if (emit_bs_get_field_size(Bits, Unit.getValue(), fail, RET) >= 0) { emit_bs_skip_bits(Fail, Ctx); @@ -1072,7 +1072,7 @@ void BeamModuleAssembler::emit_i_bs_get_binary2(const ArgVal &Ctx, Label fail; int unit; - fail = labels[Fail.getValue()]; + fail = resolve_beam_label(Fail); unit = Flags.getValue() >> 3; /* Clobbers RET + ARG3 */ @@ -1114,7 +1114,7 @@ void BeamModuleAssembler::emit_i_bs_get_float2(const ArgVal &Ctx, Label fail; Sint unit; - fail = labels[Fail.getValue()]; + fail = resolve_beam_label(Fail); unit = Flags.getValue() >> 3; mov_arg(ARG4, Ctx); @@ -1183,7 +1183,7 @@ void BeamModuleAssembler::emit_i_bs_put_utf8(const ArgVal &Fail, a.test(RET, RET); if (Fail.getValue() != 0) { - a.je(labels[Fail.getValue()]); + a.je(resolve_beam_label(Fail)); } else { a.short_().jne(next); emit_error(BADARG); @@ -1203,7 +1203,7 @@ void BeamModuleAssembler::emit_bs_get_utf8(const ArgVal &Ctx, emit_leave_runtime(); emit_test_the_non_value(RET); - a.je(labels[Fail.getValue()]); + a.je(resolve_beam_label(Fail)); } void BeamModuleAssembler::emit_i_bs_get_utf8(const ArgVal &Ctx, @@ -1253,7 +1253,7 @@ void BeamModuleAssembler::emit_i_bs_put_utf16(const ArgVal &Fail, a.test(RET, RET); if (Fail.getValue() != 0) { - a.je(labels[Fail.getValue()]); + a.je(resolve_beam_label(Fail)); } else { a.short_().jne(next); emit_error(BADARG); @@ -1275,7 +1275,7 @@ void BeamModuleAssembler::emit_bs_get_utf16(const ArgVal &Ctx, emit_leave_runtime(); emit_test_the_non_value(RET); - a.je(labels[Fail.getValue()]); + a.je(resolve_beam_label(Fail)); } void BeamModuleAssembler::emit_i_bs_get_utf16(const ArgVal &Ctx, @@ -1315,7 +1315,7 @@ void BeamModuleAssembler::emit_i_bs_validate_unicode(const ArgVal &Fail, Label fail, next = a.newLabel(); if (Fail.getValue() != 0) { - fail = labels[Fail.getValue()]; + fail = resolve_beam_label(Fail); } else { fail = a.newLabel(); } @@ -1348,7 +1348,7 @@ void BeamModuleAssembler::emit_i_bs_validate_unicode_retract(const ArgVal &Fail, imm(32)); if (Fail.getValue() != 0) { - a.jmp(labels[Fail.getValue()]); + a.jmp(resolve_beam_label(Fail)); } else { emit_error(BADARG); } @@ -1377,7 +1377,7 @@ void BeamModuleAssembler::emit_bs_test_unit(const ArgVal &Fail, a.test(RETb, imm(unit - 1)); } - a.jnz(labels[Fail.getValue()]); + a.jnz(resolve_beam_label(Fail)); } /* Set the error reason when bs_add has failed. */ @@ -1397,7 +1397,7 @@ void BeamModuleAssembler::emit_bs_add(const ArgVal &Fail, Label fail; if (Fail.getValue() != 0) { - fail = labels[Fail.getValue()]; + fail = resolve_beam_label(Fail); } else { fail = a.newLabel(); } @@ -1502,7 +1502,7 @@ void BeamModuleAssembler::emit_i_bs_append(const ArgVal &Fail, emit_test_the_non_value(RET); if (Fail.getValue() != 0) { - a.je(labels[Fail.getValue()]); + a.je(resolve_beam_label(Fail)); } else { a.short_().jne(next); /* The error has been prepared in `erts_bs_append` */ @@ -1538,7 +1538,7 @@ void BeamModuleAssembler::emit_i_bs_private_append(const ArgVal &Fail, emit_test_the_non_value(RET); if (Fail.getValue() != 0) { - a.je(labels[Fail.getValue()]); + a.je(resolve_beam_label(Fail)); } else { a.short_().jne(next); /* The error has been prepared in `erts_bs_private_append` */ @@ -1726,7 +1726,7 @@ void BeamModuleAssembler::emit_i_bs_create_bin(const ArgVal &Fail, emit_leave_runtime<Update::eReductions | Update::eStack | Update::eHeap>(); if (Fail.getValue() != 0) { - a.jmp(labels[Fail.getValue()]); + a.jmp(resolve_beam_label(Fail)); } else { safe_fragment_call(ga->get_bs_create_bin_error_shared()); } diff --git a/erts/emulator/beam/jit/x86/instr_call.cpp b/erts/emulator/beam/jit/x86/instr_call.cpp index 33b9903a6b..5b191c73dd 100644 --- a/erts/emulator/beam/jit/x86/instr_call.cpp +++ b/erts/emulator/beam/jit/x86/instr_call.cpp @@ -75,24 +75,19 @@ void BeamModuleAssembler::emit_return() { } void BeamModuleAssembler::emit_i_call(const ArgVal &CallDest) { - Label dest = labels[CallDest.getValue()]; - - erlang_call(dest, RET); + erlang_call(resolve_beam_label(CallDest), RET); } void BeamModuleAssembler::emit_i_call_last(const ArgVal &CallDest, const ArgVal &Deallocate) { emit_deallocate(Deallocate); - - emit_leave_frame(); - - a.jmp(labels[CallDest.getValue()]); + emit_i_call_only(CallDest); } void BeamModuleAssembler::emit_i_call_only(const ArgVal &CallDest) { emit_leave_frame(); - a.jmp(labels[CallDest.getValue()]); + a.jmp(resolve_beam_label(CallDest)); } /* Handles save_calls. Export entry is in RET. diff --git a/erts/emulator/beam/jit/x86/instr_common.cpp b/erts/emulator/beam/jit/x86/instr_common.cpp index 2b6d3405bb..73621e5d9c 100644 --- a/erts/emulator/beam/jit/x86/instr_common.cpp +++ b/erts/emulator/beam/jit/x86/instr_common.cpp @@ -331,7 +331,7 @@ void BeamModuleAssembler::emit_is_nonempty_list_get_list(const ArgVal &Fail, const ArgVal &Tl) { mov_arg(RET, Src); a.test(RETb, imm(_TAG_PRIMARY_MASK - TAG_PRIMARY_LIST)); - a.jne(labels[Fail.getValue()]); + a.jne(resolve_beam_label(Fail)); emit_get_list(RET, Hd, Tl); } @@ -340,7 +340,7 @@ void BeamModuleAssembler::emit_is_nonempty_list_get_hd(const ArgVal &Fail, const ArgVal &Hd) { mov_arg(RET, Src); a.test(RETb, imm(_TAG_PRIMARY_MASK - TAG_PRIMARY_LIST)); - a.jne(labels[Fail.getValue()]); + a.jne(resolve_beam_label(Fail)); x86::Gp boxed_ptr = emit_ptr_val(RET, RET); @@ -354,7 +354,7 @@ void BeamModuleAssembler::emit_is_nonempty_list_get_tl(const ArgVal &Fail, const ArgVal &Tl) { mov_arg(RET, Src); a.test(RETb, imm(_TAG_PRIMARY_MASK - TAG_PRIMARY_LIST)); - a.jne(labels[Fail.getValue()]); + a.jne(resolve_beam_label(Fail)); x86::Gp boxed_ptr = emit_ptr_val(RET, RET); @@ -413,7 +413,7 @@ void BeamModuleAssembler::emit_tuple_assertion(const ArgVal &Src, a.bind(fatal); { - a.comment("# Tuple assertion failure"); + comment("tuple assertion failure"); a.ud2(); } a.bind(ok); @@ -763,19 +763,26 @@ void BeamModuleAssembler::emit_is_nonempty_list(const ArgVal &Fail, x86::Mem list_ptr = getArgRef(Src, 1); a.test(list_ptr, imm(_TAG_PRIMARY_MASK - TAG_PRIMARY_LIST)); - a.jne(labels[Fail.getValue()]); + a.jne(resolve_beam_label(Fail)); } void BeamModuleAssembler::emit_jump(const ArgVal &Fail) { - a.jmp(labels[Fail.getValue()]); + a.jmp(resolve_beam_label(Fail)); } void BeamModuleAssembler::emit_is_atom(const ArgVal &Fail, const ArgVal &Src) { mov_arg(RET, Src); - ERTS_CT_ASSERT(_TAG_IMMED2_MASK < 256); - a.and_(RETb, imm(_TAG_IMMED2_MASK)); - a.cmp(RETb, imm(_TAG_IMMED2_ATOM)); - a.jne(labels[Fail.getValue()]); + + if (always_one_of(Src, BEAM_TYPE_ATOM | BEAM_TYPE_MASK_ALWAYS_BOXED)) { + comment("simplified atom test since all other types are boxed"); + a.test(RETb, imm(_TAG_PRIMARY_MASK - TAG_PRIMARY_BOXED)); + a.je(resolve_beam_label(Fail)); + } else { + ERTS_CT_ASSERT(_TAG_IMMED2_MASK < 256); + a.and_(RETb, imm(_TAG_IMMED2_MASK)); + a.cmp(RETb, imm(_TAG_IMMED2_ATOM)); + a.jne(resolve_beam_label(Fail)); + } } void BeamModuleAssembler::emit_is_boolean(const ArgVal &Fail, @@ -789,45 +796,54 @@ void BeamModuleAssembler::emit_is_boolean(const ArgVal &Fail, a.and_(ARG1, imm(~(am_true & ~_TAG_IMMED1_MASK))); a.cmp(ARG1, imm(am_false)); - a.jne(labels[Fail.getValue()]); + a.jne(resolve_beam_label(Fail)); } -void BeamModuleAssembler::emit_is_binary(Label fail, - x86::Gp src, - Label next, - Label subbin) { - ASSERT(src != RET && src != ARG2); +x86::Gp BeamModuleAssembler::emit_is_binary(const ArgVal &Fail, + const ArgVal &Src, + Label next, + Label subbin) { + mov_arg(ARG1, Src); - emit_is_boxed(fail, src); + emit_is_boxed(resolve_beam_label(Fail), Src, ARG1); - x86::Gp boxed_ptr = emit_ptr_val(src, src); + x86::Gp boxed_ptr = emit_ptr_val(ARG1, ARG1); a.mov(RETd, emit_boxed_val(boxed_ptr, 0, sizeof(Uint32))); - a.and_(RETb, imm(_TAG_HEADER_MASK)); a.cmp(RETb, imm(_TAG_HEADER_SUB_BIN)); a.short_().je(subbin); - ERTS_CT_ASSERT(_TAG_HEADER_REFC_BIN + 4 == _TAG_HEADER_HEAP_BIN); - a.and_(RETb, imm(~4)); - a.cmp(RETb, imm(_TAG_HEADER_REFC_BIN)); - a.short_().je(next); - a.jmp(fail); + + if (masked_types(Src, BEAM_TYPE_MASK_ALWAYS_BOXED) == BEAM_TYPE_BITSTRING) { + comment("simplified binary test since source is always a bitstring " + "when boxed"); + } else { + ERTS_CT_ASSERT(_TAG_HEADER_REFC_BIN + 4 == _TAG_HEADER_HEAP_BIN); + a.and_(RETb, imm(~4)); + a.cmp(RETb, imm(_TAG_HEADER_REFC_BIN)); + a.jne(resolve_beam_label(Fail)); + } + + a.short_().jmp(next); + + return boxed_ptr; } void BeamModuleAssembler::emit_is_binary(const ArgVal &Fail, const ArgVal &Src) { Label next = a.newLabel(), subbin = a.newLabel(); + x86::Gp boxed_ptr; - mov_arg(ARG1, Src); - - emit_is_binary(labels[Fail.getValue()], ARG1, next, subbin); + boxed_ptr = emit_is_binary(Fail, Src, next, subbin); a.bind(subbin); { /* emit_is_binary has already removed the literal tag from Src, if * applicable. */ - a.cmp(emit_boxed_val(ARG1, offsetof(ErlSubBin, bitsize), sizeof(byte)), + a.cmp(emit_boxed_val(boxed_ptr, + offsetof(ErlSubBin, bitsize), + sizeof(byte)), imm(0)); - a.jne(labels[Fail.getValue()]); + a.jne(resolve_beam_label(Fail)); } a.bind(next); @@ -837,9 +853,7 @@ void BeamModuleAssembler::emit_is_bitstring(const ArgVal &Fail, const ArgVal &Src) { Label next = a.newLabel(); - mov_arg(ARG1, Src); - - emit_is_binary(labels[Fail.getValue()], ARG1, next, next); + emit_is_binary(Fail, Src, next, next); a.bind(next); } @@ -847,23 +861,31 @@ void BeamModuleAssembler::emit_is_bitstring(const ArgVal &Fail, void BeamModuleAssembler::emit_is_float(const ArgVal &Fail, const ArgVal &Src) { mov_arg(ARG1, Src); - emit_is_boxed(labels[Fail.getValue()], ARG1); + emit_is_boxed(resolve_beam_label(Fail), Src, ARG1); - x86::Gp boxed_ptr = emit_ptr_val(ARG1, ARG1); - a.cmp(emit_boxed_val(boxed_ptr), imm(HEADER_FLONUM)); - a.jne(labels[Fail.getValue()]); + if (masked_types(Src, BEAM_TYPE_MASK_BOXED) == BEAM_TYPE_FLOAT) { + comment("skipped header test since we know it's a float when boxed"); + } else { + x86::Gp boxed_ptr = emit_ptr_val(ARG1, ARG1); + a.cmp(emit_boxed_val(boxed_ptr), imm(HEADER_FLONUM)); + a.jne(resolve_beam_label(Fail)); + } } void BeamModuleAssembler::emit_is_function(const ArgVal &Fail, const ArgVal &Src) { mov_arg(RET, Src); - emit_is_boxed(labels[Fail.getValue()], RET); + emit_is_boxed(resolve_beam_label(Fail), Src, RET); - x86::Gp boxed_ptr = emit_ptr_val(RET, RET); - a.mov(RETd, emit_boxed_val(boxed_ptr, 0, sizeof(Uint32))); - a.cmp(RET, imm(HEADER_FUN)); - a.jne(labels[Fail.getValue()]); + if (masked_types(Src, BEAM_TYPE_MASK_BOXED) == BEAM_TYPE_FUN) { + comment("skipped header test since we know it's a fun when boxed"); + } else { + x86::Gp boxed_ptr = emit_ptr_val(RET, RET); + a.mov(RETd, emit_boxed_val(boxed_ptr, 0, sizeof(Uint32))); + a.cmp(RET, imm(HEADER_FUN)); + a.jne(resolve_beam_label(Fail)); + } } void BeamModuleAssembler::emit_is_function2(const ArgVal &Fail, @@ -884,50 +906,64 @@ void BeamModuleAssembler::emit_is_function2(const ArgVal &Fail, emit_leave_runtime(); a.cmp(RET, imm(am_true)); - a.jne(labels[Fail.getValue()]); + a.jne(resolve_beam_label(Fail)); return; } unsigned arity = unsigned_val(Arity.getValue()); if (arity > MAX_ARG) { /* Arity is negative or too large. */ - a.jmp(labels[Fail.getValue()]); + a.jmp(resolve_beam_label(Fail)); return; } mov_arg(ARG1, Src); - emit_is_boxed(labels[Fail.getValue()], ARG1); + emit_is_boxed(resolve_beam_label(Fail), Src, ARG1); x86::Gp boxed_ptr = emit_ptr_val(ARG1, ARG1); - a.mov(RETd, emit_boxed_val(boxed_ptr, 0, sizeof(Uint32))); - a.cmp(RETd, imm(HEADER_FUN)); - a.jne(labels[Fail.getValue()]); + + if (masked_types(Src, BEAM_TYPE_MASK_BOXED) == BEAM_TYPE_FUN) { + comment("skipped header test since we know it's a fun when boxed"); + } else { + a.mov(RETd, emit_boxed_val(boxed_ptr, 0, sizeof(Uint32))); + a.cmp(RETd, imm(HEADER_FUN)); + a.jne(resolve_beam_label(Fail)); + } a.cmp(emit_boxed_val(boxed_ptr, offsetof(ErlFunThing, arity)), imm(arity)); - a.jne(labels[Fail.getValue()]); + a.jne(resolve_beam_label(Fail)); } void BeamModuleAssembler::emit_is_integer(const ArgVal &Fail, const ArgVal &Src) { Label next = a.newLabel(); - Label fail = labels[Fail.getValue()]; mov_arg(ARG1, Src); - a.mov(RETd, ARG1d); - a.and_(RETb, imm(_TAG_IMMED1_MASK)); - a.cmp(RETb, imm(_TAG_IMMED1_SMALL)); - a.short_().je(next); + if (always_one_of(Src, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_ALWAYS_BOXED)) { + comment("simplified small test since all other types are boxed"); + emit_is_boxed(next, Src, ARG1); + } else { + a.mov(RETd, ARG1d); + a.and_(RETb, imm(_TAG_IMMED1_MASK)); + a.cmp(RETb, imm(_TAG_IMMED1_SMALL)); + a.short_().je(next); - emit_is_boxed(fail, RET); + emit_is_boxed(resolve_beam_label(Fail), Src, RET); + } - x86::Gp boxed_ptr = emit_ptr_val(ARG1, ARG1); - a.mov(RETd, emit_boxed_val(boxed_ptr, 0, sizeof(Uint32))); + if (masked_types(Src, BEAM_TYPE_MASK_BOXED) == BEAM_TYPE_INTEGER) { + comment("skipped header test since we know it's a bignum when " + "boxed"); + } else { + x86::Gp boxed_ptr = emit_ptr_val(ARG1, ARG1); + a.mov(RETd, emit_boxed_val(boxed_ptr, 0, sizeof(Uint32))); - a.and_(RETb, imm(_TAG_HEADER_MASK - _BIG_SIGN_BIT)); - a.cmp(RETb, imm(_TAG_HEADER_POS_BIG)); - a.jne(fail); + a.and_(RETb, imm(_TAG_HEADER_MASK - _BIG_SIGN_BIT)); + a.cmp(RETb, imm(_TAG_HEADER_POS_BIG)); + a.jne(resolve_beam_label(Fail)); + } a.bind(next); } @@ -940,51 +976,68 @@ void BeamModuleAssembler::emit_is_list(const ArgVal &Fail, const ArgVal &Src) { a.cmp(RET, imm(NIL)); a.short_().je(next); a.test(RETb, imm(_TAG_PRIMARY_MASK - TAG_PRIMARY_LIST)); - a.jne(labels[Fail.getValue()]); + a.jne(resolve_beam_label(Fail)); a.bind(next); } void BeamModuleAssembler::emit_is_map(const ArgVal &Fail, const ArgVal &Src) { mov_arg(RET, Src); - emit_is_boxed(labels[Fail.getValue()], RET); + emit_is_boxed(resolve_beam_label(Fail), Src, RET); - x86::Gp boxed_ptr = emit_ptr_val(RET, RET); - a.mov(RETd, emit_boxed_val(boxed_ptr, 0, sizeof(Uint32))); - a.and_(RETb, imm(_TAG_HEADER_MASK)); - a.cmp(RETb, imm(_TAG_HEADER_MAP)); - a.jne(labels[Fail.getValue()]); + /* As an optimization for the `error | #{}` case, skip checking the header + * word when we know that the only possible boxed type is a map. */ + if (masked_types(Src, BEAM_TYPE_MASK_BOXED) == BEAM_TYPE_MAP) { + comment("skipped header test since we know it's a map when boxed"); + } else { + x86::Gp boxed_ptr = emit_ptr_val(RET, RET); + a.mov(RETd, emit_boxed_val(boxed_ptr, 0, sizeof(Uint32))); + a.and_(RETb, imm(_TAG_HEADER_MASK)); + a.cmp(RETb, imm(_TAG_HEADER_MAP)); + a.jne(resolve_beam_label(Fail)); + } } void BeamModuleAssembler::emit_is_nil(const ArgVal &Fail, const ArgVal &Src) { a.cmp(getArgRef(Src), imm(NIL)); - a.jne(labels[Fail.getValue()]); + a.jne(resolve_beam_label(Fail)); } void BeamModuleAssembler::emit_is_number(const ArgVal &Fail, const ArgVal &Src) { + Label fail = resolve_beam_label(Fail); Label next = a.newLabel(); - Label fail = labels[Fail.getValue()]; mov_arg(ARG1, Src); - a.mov(RETd, ARG1d); - a.and_(RETb, imm(_TAG_IMMED1_MASK)); - a.cmp(RETb, imm(_TAG_IMMED1_SMALL)); - a.short_().je(next); + if (always_one_of(Src, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_ALWAYS_BOXED)) { + comment("simplified small test test since all other types are boxed"); + emit_is_boxed(next, Src, ARG1); + } else { + a.mov(RETd, ARG1d); + a.and_(RETb, imm(_TAG_IMMED1_MASK)); + a.cmp(RETb, imm(_TAG_IMMED1_SMALL)); + a.short_().je(next); - emit_is_boxed(fail, RET); + /* Reuse RET as the important bits are still available. */ + emit_is_boxed(fail, Src, RET); + } - x86::Gp boxed_ptr = emit_ptr_val(ARG1, ARG1); - a.mov(ARG1, emit_boxed_val(boxed_ptr)); + if (masked_types(Src, BEAM_TYPE_MASK_BOXED) == + (BEAM_TYPE_FLOAT | BEAM_TYPE_INTEGER)) { + comment("skipped header test since we know it's a number when boxed"); + } else { + x86::Gp boxed_ptr = emit_ptr_val(ARG1, ARG1); + a.mov(ARG1, emit_boxed_val(boxed_ptr)); - a.mov(RETd, ARG1d); - a.and_(RETb, imm(_TAG_HEADER_MASK - _BIG_SIGN_BIT)); - a.cmp(RETb, imm(_TAG_HEADER_POS_BIG)); - a.short_().je(next); + a.mov(RETd, ARG1d); + a.and_(RETb, imm(_TAG_HEADER_MASK - _BIG_SIGN_BIT)); + a.cmp(RETb, imm(_TAG_HEADER_POS_BIG)); + a.short_().je(next); - a.cmp(ARG1d, imm(HEADER_FLONUM)); - a.jne(fail); + a.cmp(ARG1d, imm(HEADER_FLONUM)); + a.jne(fail); + } a.bind(next); } @@ -994,59 +1047,84 @@ void BeamModuleAssembler::emit_is_pid(const ArgVal &Fail, const ArgVal &Src) { mov_arg(ARG1, Src); - a.mov(RETd, ARG1d); - a.and_(RETb, imm(_TAG_IMMED1_MASK)); - a.cmp(RETb, imm(_TAG_IMMED1_PID)); - a.short_().je(next); + if (always_one_of(Src, BEAM_TYPE_PID | BEAM_TYPE_MASK_ALWAYS_BOXED)) { + comment("simplified local pid test since all other types are boxed"); + emit_is_boxed(next, Src, ARG1); + } else { + a.mov(RETd, ARG1d); + a.and_(RETb, imm(_TAG_IMMED1_MASK)); + a.cmp(RETb, imm(_TAG_IMMED1_PID)); + a.short_().je(next); - /* Reuse RET as the important bits are still available. */ - emit_is_boxed(labels[Fail.getValue()], RET); + /* Reuse RET as the important bits are still available. */ + emit_is_boxed(resolve_beam_label(Fail), Src, RET); + } + + if (masked_types(Src, BEAM_TYPE_MASK_BOXED) == BEAM_TYPE_PID) { + comment("skipped header test since we know it's a pid when boxed"); + } else { + x86::Gp boxed_ptr = emit_ptr_val(ARG1, ARG1); + a.mov(RETd, emit_boxed_val(boxed_ptr, 0, sizeof(Uint32))); + a.and_(RETb, imm(_TAG_HEADER_MASK)); + a.cmp(RETb, imm(_TAG_HEADER_EXTERNAL_PID)); + a.jne(resolve_beam_label(Fail)); + } - x86::Gp boxed_ptr = emit_ptr_val(ARG1, ARG1); - a.mov(RETd, emit_boxed_val(boxed_ptr, 0, sizeof(Uint32))); - a.and_(RETb, _TAG_HEADER_MASK); - a.cmp(RETb, _TAG_HEADER_EXTERNAL_PID); - a.jne(labels[Fail.getValue()]); a.bind(next); } void BeamModuleAssembler::emit_is_port(const ArgVal &Fail, const ArgVal &Src) { Label next = a.newLabel(); + mov_arg(ARG1, Src); - a.mov(RETd, ARG1d); - a.and_(RETb, imm(_TAG_IMMED1_MASK)); - a.cmp(RETb, imm(_TAG_IMMED1_PORT)); - a.short_().je(next); + if (always_one_of(Src, BEAM_TYPE_PORT | BEAM_TYPE_MASK_ALWAYS_BOXED)) { + comment("simplified local port test since all other types are boxed"); + emit_is_boxed(next, Src, ARG1); + } else { + a.mov(RETd, ARG1d); + a.and_(RETb, imm(_TAG_IMMED1_MASK)); + a.cmp(RETb, imm(_TAG_IMMED1_PORT)); + a.short_().je(next); - /* Reuse RET as the important bits are still available. */ - emit_is_boxed(labels[Fail.getValue()], RET); + /* Reuse RET as the important bits are still available. */ + emit_is_boxed(resolve_beam_label(Fail), Src, RET); + } + + if (masked_types(Src, BEAM_TYPE_MASK_BOXED) == BEAM_TYPE_PORT) { + comment("skipped header test since we know it's a port when boxed"); + } else { + x86::Gp boxed_ptr = emit_ptr_val(ARG1, ARG1); + a.mov(RETd, emit_boxed_val(boxed_ptr, 0, sizeof(Uint32))); + a.and_(RETb, imm(_TAG_HEADER_MASK)); + a.cmp(RETb, imm(_TAG_HEADER_EXTERNAL_PORT)); + a.jne(resolve_beam_label(Fail)); + } - x86::Gp boxed_ptr = emit_ptr_val(ARG1, ARG1); - a.mov(RETd, emit_boxed_val(boxed_ptr, 0, sizeof(Uint32))); - a.and_(RETb, imm(_TAG_HEADER_MASK)); - a.cmp(RETb, imm(_TAG_HEADER_EXTERNAL_PORT)); - a.jne(labels[Fail.getValue()]); a.bind(next); } void BeamModuleAssembler::emit_is_reference(const ArgVal &Fail, const ArgVal &Src) { - Label next = a.newLabel(); - mov_arg(RET, Src); - emit_is_boxed(labels[Fail.getValue()], RET); + emit_is_boxed(resolve_beam_label(Fail), Src, RET); - x86::Gp boxed_ptr = emit_ptr_val(RET, RET); - a.mov(RETd, emit_boxed_val(boxed_ptr, 0, sizeof(Uint32))); - a.and_(RETb, imm(_TAG_HEADER_MASK)); - a.cmp(RETb, imm(_TAG_HEADER_REF)); - a.short_().je(next); - a.cmp(RETb, imm(_TAG_HEADER_EXTERNAL_REF)); - a.jne(labels[Fail.getValue()]); + if (masked_types(Src, BEAM_TYPE_MASK_BOXED) == BEAM_TYPE_REFERENCE) { + comment("skipped header test since we know it's a ref when boxed"); + } else { + Label next = a.newLabel(); - a.bind(next); + x86::Gp boxed_ptr = emit_ptr_val(RET, RET); + a.mov(RETd, emit_boxed_val(boxed_ptr, 0, sizeof(Uint32))); + a.and_(RETb, imm(_TAG_HEADER_MASK)); + a.cmp(RETb, imm(_TAG_HEADER_REF)); + a.short_().je(next); + a.cmp(RETb, imm(_TAG_HEADER_EXTERNAL_REF)); + a.jne(resolve_beam_label(Fail)); + + a.bind(next); + } } /* Note: This instruction leaves the pointer to the tuple in ARG2. */ @@ -1056,15 +1134,15 @@ void BeamModuleAssembler::emit_i_is_tagged_tuple(const ArgVal &Fail, const ArgVal &Tag) { mov_arg(ARG2, Src); - emit_is_boxed(labels[Fail.getValue()], ARG2); + emit_is_boxed(resolve_beam_label(Fail), Src, ARG2); x86::Gp boxed_ptr = emit_ptr_val(ARG2, ARG2); ERTS_CT_ASSERT(Support::isInt32(make_arityval(MAX_ARITYVAL))); a.cmp(emit_boxed_val(boxed_ptr, 0, sizeof(Uint32)), imm(Arity.getValue())); - a.jne(labels[Fail.getValue()]); + a.jne(resolve_beam_label(Fail)); a.cmp(emit_boxed_val(boxed_ptr, sizeof(Eterm)), imm(Tag.getValue())); - a.jne(labels[Fail.getValue()]); + a.jne(resolve_beam_label(Fail)); } /* Note: This instruction leaves the pointer to the tuple in ARG2. */ @@ -1074,34 +1152,51 @@ void BeamModuleAssembler::emit_i_is_tagged_tuple_ff(const ArgVal &NotTuple, const ArgVal &Arity, const ArgVal &Tag) { mov_arg(ARG2, Src); - emit_is_boxed(labels[NotTuple.getValue()], ARG2); + + emit_is_boxed(resolve_beam_label(NotTuple), Src, ARG2); + (void)emit_ptr_val(ARG2, ARG2); a.mov(ARG1, emit_boxed_val(ARG2)); ERTS_CT_ASSERT(_TAG_HEADER_ARITYVAL == 0); a.test(ARG1.r8(), imm(_TAG_HEADER_MASK)); - a.jne(labels[NotTuple.getValue()]); + a.jne(resolve_beam_label(NotTuple)); ERTS_CT_ASSERT(Support::isInt32(make_arityval(MAX_ARITYVAL))); a.cmp(ARG1d, imm(Arity.getValue())); - a.jne(labels[NotRecord.getValue()]); + a.jne(resolve_beam_label(NotRecord)); a.cmp(emit_boxed_val(ARG2, sizeof(Eterm)), imm(Tag.getValue())); - a.jne(labels[NotRecord.getValue()]); + a.jne(resolve_beam_label(NotRecord)); } /* Note: This instruction leaves the pointer to the tuple in ARG2. */ void BeamModuleAssembler::emit_i_is_tuple(const ArgVal &Fail, const ArgVal &Src) { - mov_arg(ARG2, Src); + if (masked_types(Src, BEAM_TYPE_MASK_BOXED) == BEAM_TYPE_TUPLE) { + /* Fast path for the `error | {ok, Value}` case. */ + comment("simplified tuple test since the source is always a tuple " + "when boxed"); + a.test(getArgRef(Src, sizeof(byte)), + imm(_TAG_PRIMARY_MASK - TAG_PRIMARY_BOXED)); + + /* Note: ARG2 will NOT be set. This is OK since the operand + * for `current_tuple` has a type; that operand will not match + * the type-less operand for `get_tuple_element`. Thus, there + * will always be a `load_tuple_ptr` instruction emitted if + * this instruction is immediately followed by a + * `get_tuple_element` instruction. */ + } else { + mov_arg(ARG2, Src); - emit_is_boxed(labels[Fail.getValue()], ARG2); + emit_is_boxed(resolve_beam_label(Fail), Src, ARG2); - (void)emit_ptr_val(ARG2, ARG2); - ERTS_CT_ASSERT(_TAG_HEADER_ARITYVAL == 0); - a.test(emit_boxed_val(ARG2, 0, sizeof(byte)), imm(_TAG_HEADER_MASK)); + (void)emit_ptr_val(ARG2, ARG2); + ERTS_CT_ASSERT(_TAG_HEADER_ARITYVAL == 0); + a.test(emit_boxed_val(ARG2, 0, sizeof(byte)), imm(_TAG_HEADER_MASK)); + } - a.jne(labels[Fail.getValue()]); + a.jne(resolve_beam_label(Fail)); } /* Note: This instruction leaves the pointer to the tuple in ARG2. */ @@ -1110,12 +1205,12 @@ void BeamModuleAssembler::emit_i_is_tuple_of_arity(const ArgVal &Fail, const ArgVal &Arity) { mov_arg(ARG2, Src); - emit_is_boxed(labels[Fail.getValue()], ARG2); + emit_is_boxed(resolve_beam_label(Fail), Src, ARG2); (void)emit_ptr_val(ARG2, ARG2); ERTS_CT_ASSERT(Support::isInt32(make_arityval(MAX_ARITYVAL))); a.cmp(emit_boxed_val(ARG2, 0, sizeof(Uint32)), imm(Arity.getValue())); - a.jne(labels[Fail.getValue()]); + a.jne(resolve_beam_label(Fail)); } /* Note: This instruction leaves the pointer to the tuple in ARG2. */ @@ -1127,26 +1222,25 @@ void BeamModuleAssembler::emit_i_test_arity(const ArgVal &Fail, (void)emit_ptr_val(ARG2, ARG2); ERTS_CT_ASSERT(Support::isInt32(make_arityval(MAX_ARITYVAL))); a.cmp(emit_boxed_val(ARG2, 0, sizeof(Uint32)), imm(Arity.getValue())); - a.jne(labels[Fail.getValue()]); -} - -void BeamModuleAssembler::emit_i_is_eq_exact_immed(const ArgVal &Fail, - const ArgVal &X, - const ArgVal &Y) { - cmp_arg(getArgRef(X), Y); - a.jne(labels[Fail.getValue()]); -} - -void BeamModuleAssembler::emit_i_is_ne_exact_immed(const ArgVal &Fail, - const ArgVal &X, - const ArgVal &Y) { - cmp_arg(getArgRef(X), Y); - a.je(labels[Fail.getValue()]); + a.jne(resolve_beam_label(Fail)); } void BeamModuleAssembler::emit_is_eq_exact(const ArgVal &Fail, const ArgVal &X, const ArgVal &Y) { + /* If either argument is known to be an immediate, we can fail immediately + * if they're not equal. */ + if (always_immediate(X) || always_immediate(Y)) { + if (!X.isImmed() && !Y.isImmed()) { + comment("simplified check since one argument is an immediate"); + } + + cmp_arg(getArgRef(X), Y); + a.jne(resolve_beam_label(Fail)); + + return; + } + Label next = a.newLabel(); mov_arg(ARG2, Y); /* May clobber ARG1 */ @@ -1159,12 +1253,14 @@ void BeamModuleAssembler::emit_is_eq_exact(const ArgVal &Fail, a.short_().je(next); #endif - /* Fancy way of checking if both are immediates. */ - a.mov(RETd, ARG1d); - a.and_(RETd, ARG2d); - a.and_(RETb, imm(_TAG_PRIMARY_MASK)); - a.cmp(RETb, imm(TAG_PRIMARY_IMMED1)); - a.je(labels[Fail.getValue()]); + if (always_same_types(X, Y)) { + comment("skipped test of tags since they are always equal"); + } else { + /* The terms could still be equal if both operands are pointers + * having the same tag. */ + emit_is_unequal_based_on_tags(ARG1, ARG2); + a.je(resolve_beam_label(Fail)); + } emit_enter_runtime(); @@ -1172,8 +1268,8 @@ void BeamModuleAssembler::emit_is_eq_exact(const ArgVal &Fail, emit_leave_runtime(); - a.test(RET, RET); - a.je(labels[Fail.getValue()]); + a.test(RETd, RETd); + a.je(resolve_beam_label(Fail)); a.bind(next); } @@ -1188,7 +1284,7 @@ void BeamModuleAssembler::emit_i_is_eq_exact_literal(const ArgVal &Fail, /* Fail immediately unless Src is the same type of pointer as the literal. */ a.test(ARG1.r8(), imm(tag_test.getValue())); - a.jne(labels[Fail.getValue()]); + a.jne(resolve_beam_label(Fail)); emit_enter_runtime(); @@ -1196,31 +1292,47 @@ void BeamModuleAssembler::emit_i_is_eq_exact_literal(const ArgVal &Fail, emit_leave_runtime(); - a.test(RET, RET); - a.jz(labels[Fail.getValue()]); + a.test(RETd, RETd); + a.jz(resolve_beam_label(Fail)); } void BeamModuleAssembler::emit_is_ne_exact(const ArgVal &Fail, const ArgVal &X, const ArgVal &Y) { + /* If either argument is known to be an immediate, we can fail immediately + * if they're equal. */ + if (always_immediate(X) || always_immediate(Y)) { + if (!X.isImmed() && !Y.isImmed()) { + comment("simplified check since one argument is an immediate"); + } + + cmp_arg(getArgRef(X), Y); + a.je(resolve_beam_label(Fail)); + + return; + } + Label next = a.newLabel(); mov_arg(ARG2, Y); /* May clobber ARG1 */ mov_arg(ARG1, X); a.cmp(ARG1, ARG2); - a.je(labels[Fail.getValue()]); + a.je(resolve_beam_label(Fail)); + + if (always_same_types(X, Y)) { + comment("skipped tag test since they are always equal"); + } else { + /* Test whether the terms are definitely unequal based on the tags + * alone. */ + emit_is_unequal_based_on_tags(ARG1, ARG2); - /* Fancy way of checking if both are immediates. */ - a.mov(RETd, ARG1d); - a.and_(RETd, ARG2d); - a.and_(RETb, imm(_TAG_PRIMARY_MASK)); - a.cmp(RETb, imm(TAG_PRIMARY_IMMED1)); #ifdef JIT_HARD_DEBUG - a.je(next); + a.je(next); #else - a.short_().je(next); + a.short_().je(next); #endif + } emit_enter_runtime(); @@ -1228,8 +1340,8 @@ void BeamModuleAssembler::emit_is_ne_exact(const ArgVal &Fail, emit_leave_runtime(); - a.test(RET, RET); - a.jnz(labels[Fail.getValue()]); + a.test(RETd, RETd); + a.jnz(resolve_beam_label(Fail)); a.bind(next); } @@ -1253,8 +1365,8 @@ void BeamModuleAssembler::emit_i_is_ne_exact_literal(const ArgVal &Fail, emit_leave_runtime(); - a.test(RET, RET); - a.jnz(labels[Fail.getValue()]); + a.test(RETd, RETd); + a.jnz(resolve_beam_label(Fail)); a.bind(next); } @@ -1311,7 +1423,7 @@ void BeamGlobalAssembler::emit_arith_eq_shared() { void BeamModuleAssembler::emit_is_eq(const ArgVal &Fail, const ArgVal &A, const ArgVal &B) { - Label fail = labels[Fail.getValue()], next = a.newLabel(); + Label fail = resolve_beam_label(Fail), next = a.newLabel(); mov_arg(ARG2, B); /* May clobber ARG1 */ mov_arg(ARG1, A); @@ -1334,7 +1446,7 @@ void BeamModuleAssembler::emit_is_eq(const ArgVal &Fail, void BeamModuleAssembler::emit_is_ne(const ArgVal &Fail, const ArgVal &A, const ArgVal &B) { - Label fail = labels[Fail.getValue()], next = a.newLabel(); + Label fail = resolve_beam_label(Fail), next = a.newLabel(); mov_arg(ARG2, B); /* May clobber ARG1 */ mov_arg(ARG1, A); @@ -1444,39 +1556,47 @@ void BeamGlobalAssembler::emit_arith_compare_shared() { void BeamModuleAssembler::emit_is_lt(const ArgVal &Fail, const ArgVal &LHS, const ArgVal &RHS) { - Label fail = labels[Fail.getValue()]; Label generic = a.newLabel(), next = a.newLabel(); mov_arg(ARG2, RHS); /* May clobber ARG1 */ mov_arg(ARG1, LHS); - a.cmp(ARG1, ARG2); - a.je(fail); - - /* Relative comparisons are overwhelmingly likely to be used on smalls, so - * we'll specialize those and keep the rest in a shared fragment. */ - - if (RHS.isImmed() && is_small(RHS.getValue())) { - a.mov(RETd, ARG1d); - } else if (LHS.isImmed() && is_small(LHS.getValue())) { - a.mov(RETd, ARG2d); - } else { + if (always_one_of(LHS, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_BOXED) && + always_one_of(RHS, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_BOXED)) { + /* The only possible kind of immediate is a small and all other + * values are boxed, so we can test for smalls by testing boxed. */ + comment("simplified small test since all other types are boxed"); a.mov(RETd, ARG1d); a.and_(RETd, ARG2d); - } + a.test(RETb, imm(_TAG_PRIMARY_MASK - TAG_PRIMARY_BOXED)); + a.short_().je(generic); + } else { + /* Relative comparisons are overwhelmingly likely to be used on + * smalls, so we'll specialize those and keep the rest in a shared + * fragment. */ + if (RHS.isImmed() && is_small(RHS.getValue())) { + a.mov(RETd, ARG1d); + } else if (LHS.isImmed() && is_small(LHS.getValue())) { + a.mov(RETd, ARG2d); + } else { + a.mov(RETd, ARG1d); + a.and_(RETd, ARG2d); + } - a.and_(RETb, imm(_TAG_IMMED1_MASK)); - a.cmp(RETb, imm(_TAG_IMMED1_SMALL)); - a.short_().jne(generic); + a.and_(RETb, imm(_TAG_IMMED1_MASK)); + a.cmp(RETb, imm(_TAG_IMMED1_SMALL)); + a.short_().jne(generic); + } + /* Both arguments are smalls. */ a.cmp(ARG1, ARG2); a.short_().jl(next); - a.jmp(fail); + a.jmp(resolve_beam_label(Fail)); a.bind(generic); { safe_fragment_call(ga->get_arith_compare_shared()); - a.jge(fail); + a.jge(resolve_beam_label(Fail)); } a.bind(next); @@ -1485,66 +1605,103 @@ void BeamModuleAssembler::emit_is_lt(const ArgVal &Fail, void BeamModuleAssembler::emit_is_ge(const ArgVal &Fail, const ArgVal &LHS, const ArgVal &RHS) { - Label fail = labels[Fail.getValue()]; Label generic = a.newLabel(), next = a.newLabel(); mov_arg(ARG2, RHS); /* May clobber ARG1 */ mov_arg(ARG1, LHS); - a.cmp(ARG1, ARG2); - a.short_().je(next); - - /* Relative comparisons are overwhelmingly likely to be used on smalls, so - * we'll specialize those and keep the rest in a shared fragment. */ - - if (RHS.isImmed() && is_small(RHS.getValue())) { - a.mov(RETd, ARG1d); - } else if (LHS.isImmed() && is_small(LHS.getValue())) { - a.mov(RETd, ARG2d); - } else { + if (always_one_of(LHS, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_BOXED) && + always_one_of(RHS, BEAM_TYPE_INTEGER | BEAM_TYPE_MASK_BOXED)) { + /* The only possible kind of immediate is a small and all other + * values are boxed, so we can test for smalls by testing boxed. */ + comment("simplified small test since all other types are boxed"); a.mov(RETd, ARG1d); a.and_(RETd, ARG2d); - } + a.test(RETb, imm(_TAG_PRIMARY_MASK - TAG_PRIMARY_BOXED)); + a.short_().je(generic); + } else { + /* Relative comparisons are overwhelmingly likely to be used on + * smalls, so we'll specialize those and keep the rest in a shared + * fragment. */ + if (RHS.isImmed() && is_small(RHS.getValue())) { + a.mov(RETd, ARG1d); + } else if (LHS.isImmed() && is_small(LHS.getValue())) { + a.mov(RETd, ARG2d); + } else { + a.mov(RETd, ARG1d); + a.and_(RETd, ARG2d); + } - a.and_(RETb, imm(_TAG_IMMED1_MASK)); - a.cmp(RETb, imm(_TAG_IMMED1_SMALL)); - a.short_().jne(generic); + a.and_(RETb, imm(_TAG_IMMED1_MASK)); + a.cmp(RETb, imm(_TAG_IMMED1_SMALL)); + a.short_().jne(generic); + } + /* Both arguments are smalls. */ a.cmp(ARG1, ARG2); a.short_().jge(next); - a.jmp(fail); + a.jmp(resolve_beam_label(Fail)); a.bind(generic); { safe_fragment_call(ga->get_arith_compare_shared()); - a.jl(fail); + a.jl(resolve_beam_label(Fail)); } a.bind(next); } -void BeamModuleAssembler::emit_bif_is_eq_ne_exact_immed(const ArgVal &Src, - const ArgVal &Immed, - const ArgVal &Dst, - Eterm fail_value, - Eterm succ_value) { - cmp_arg(getArgRef(Src), Immed); - mov_imm(RET, fail_value); - mov_imm(ARG1, succ_value); - a.cmove(RET, ARG1); +void BeamModuleAssembler::emit_bif_is_eq_ne_exact(const ArgVal &LHS, + const ArgVal &RHS, + const ArgVal &Dst, + Eterm fail_value, + Eterm succ_value) { + /* `mov_imm` may clobber the flags if either value is zero. */ + ASSERT(fail_value && succ_value); + + mov_imm(RET, succ_value); + cmp_arg(getArgRef(LHS), RHS); + + if (always_immediate(LHS) || always_immediate(RHS)) { + if (!LHS.isImmed() && !RHS.isImmed()) { + comment("simplified check since one argument is an immediate"); + } + mov_imm(ARG1, fail_value); + a.cmovne(RET, ARG1); + } else { + Label next = a.newLabel(); + + a.je(next); + + mov_arg(ARG1, LHS); + mov_arg(ARG2, RHS); + + emit_enter_runtime(); + runtime_call<2>(eq); + emit_leave_runtime(); + + a.test(RET, RET); + + mov_imm(RET, succ_value); + mov_imm(ARG1, fail_value); + a.cmove(RET, ARG1); + + a.bind(next); + } + mov_arg(Dst, RET); } -void BeamModuleAssembler::emit_bif_is_eq_exact_immed(const ArgVal &Src, - const ArgVal &Immed, - const ArgVal &Dst) { - emit_bif_is_eq_ne_exact_immed(Src, Immed, Dst, am_false, am_true); +void BeamModuleAssembler::emit_bif_is_eq_exact(const ArgVal &LHS, + const ArgVal &RHS, + const ArgVal &Dst) { + emit_bif_is_eq_ne_exact(LHS, RHS, Dst, am_false, am_true); } -void BeamModuleAssembler::emit_bif_is_ne_exact_immed(const ArgVal &Src, - const ArgVal &Immed, - const ArgVal &Dst) { - emit_bif_is_eq_ne_exact_immed(Src, Immed, Dst, am_true, am_false); +void BeamModuleAssembler::emit_bif_is_ne_exact(const ArgVal &LHS, + const ArgVal &RHS, + const ArgVal &Dst) { + emit_bif_is_eq_ne_exact(LHS, RHS, Dst, am_true, am_false); } void BeamModuleAssembler::emit_badmatch(const ArgVal &Src) { @@ -1586,7 +1743,7 @@ void BeamModuleAssembler::emit_catch(const ArgVal &Y, const ArgVal &Fail) { mov_arg(Y, RET); /* Offset = 1 for `mov` payload */ - catches.push_back({{patch_addr, 0x1, 0}, labels[Fail.getValue()]}); + catches.push_back({{patch_addr, 0x1, 0}, resolve_beam_label(Fail)}); } /* diff --git a/erts/emulator/beam/jit/x86/instr_fun.cpp b/erts/emulator/beam/jit/x86/instr_fun.cpp index 27600cf128..01790cf426 100644 --- a/erts/emulator/beam/jit/x86/instr_fun.cpp +++ b/erts/emulator/beam/jit/x86/instr_fun.cpp @@ -164,7 +164,7 @@ void BeamModuleAssembler::emit_i_lambda_trampoline(const ArgVal &Index, if (NumFree.getValue() == 0) { /* No free variables, let the lambda jump directly to our target. */ - lambda.trampoline = labels[Lbl.getValue()]; + lambda.trampoline = resolve_beam_label(Lbl); return; } @@ -187,7 +187,7 @@ void BeamModuleAssembler::emit_i_lambda_trampoline(const ArgVal &Index, a.mov(getXRef(i + effective_arity), RET); } - a.jmp(labels[Lbl.getValue()]); + a.jmp(resolve_beam_label(Lbl)); } void BeamModuleAssembler::emit_i_make_fun3(const ArgVal &Fun, @@ -297,9 +297,9 @@ void BeamGlobalAssembler::emit_apply_fun_shared() { void BeamModuleAssembler::emit_i_apply_fun() { safe_fragment_call(ga->get_apply_fun_shared()); - x86::Gp dest = emit_call_fun(); - ASSERT(dest != ARG6); - erlang_call(dest, ARG6); + x86::Gp target = emit_call_fun(); + ASSERT(target != ARG6); + erlang_call(target, ARG6); } void BeamModuleAssembler::emit_i_apply_fun_last(const ArgVal &Deallocate) { @@ -310,70 +310,121 @@ void BeamModuleAssembler::emit_i_apply_fun_last(const ArgVal &Deallocate) { void BeamModuleAssembler::emit_i_apply_fun_only() { safe_fragment_call(ga->get_apply_fun_shared()); - x86::Gp dest = emit_call_fun(); + x86::Gp target = emit_call_fun(); emit_leave_frame(); - a.jmp(dest); + a.jmp(target); } /* Asssumes that: * ARG3 = arity * ARG4 = fun thing */ -x86::Gp BeamModuleAssembler::emit_call_fun() { +x86::Gp BeamModuleAssembler::emit_call_fun(bool skip_box_test, + bool skip_fun_test, + bool skip_arity_test) { + const bool never_fails = skip_box_test && skip_fun_test && skip_arity_test; Label next = a.newLabel(); /* Speculatively strip the literal tag when needed. */ x86::Gp fun_thing = emit_ptr_val(RET, ARG4); - /* Load the error fragment into ARG2 so we can CMOV ourselves there on - * error. */ - a.mov(ARG2, ga->get_handle_call_fun_error()); + if (!never_fails) { + /* Load the error fragment into ARG2 so we can CMOV ourselves there on + * error. */ + a.mov(ARG2, ga->get_handle_call_fun_error()); + } - /* The `handle_call_fun_error` fragment expects current PC in ARG5. */ + /* The `handle_call_fun_error` and `unloaded_fun` fragments expect current + * PC in ARG5. */ a.lea(ARG5, x86::qword_ptr(next)); - /* As emit_is_boxed(), but explicitly sets ZF so we can rely on that for - * error checking in `next`. */ - a.test(ARG4d, imm(_TAG_PRIMARY_MASK - TAG_PRIMARY_BOXED)); - a.short_().jne(next); + if (!skip_box_test) { + /* As emit_is_boxed(), but explicitly sets ZF so we can rely on that + * for error checking in `next`. */ + a.test(ARG4d, imm(_TAG_PRIMARY_MASK - TAG_PRIMARY_BOXED)); + a.short_().jne(next); + } else { + comment("skipped box test since source is always boxed"); + } - a.cmp(emit_boxed_val(fun_thing), imm(HEADER_FUN)); - a.short_().jne(next); + if (skip_fun_test) { + comment("skipped fun test since source is always a fun when boxed"); + } else { + a.cmp(emit_boxed_val(fun_thing), imm(HEADER_FUN)); + a.short_().jne(next); + } - a.cmp(emit_boxed_val(fun_thing, offsetof(ErlFunThing, arity)), ARG3); + if (skip_arity_test) { + comment("skipped arity test since source always has right arity"); + } else { + a.cmp(emit_boxed_val(fun_thing, offsetof(ErlFunThing, arity)), ARG3); + } a.mov(RET, emit_boxed_val(fun_thing, offsetof(ErlFunThing, entry))); a.mov(ARG1, emit_setup_dispatchable_call(RET)); - /* Assumes that ZF is set on success and clear on error, overwriting our - * destination with the error fragment's address. */ a.bind(next); - a.cmovne(ARG1, ARG2); + + if (!never_fails) { + /* Assumes that ZF is set on success and clear on error, overwriting + * our destination with the error fragment's address. */ + a.cmovne(ARG1, ARG2); + } return ARG1; } -void BeamModuleAssembler::emit_i_call_fun(const ArgVal &Arity) { - mov_arg(ARG4, ArgVal(ArgVal::XReg, Arity.getValue())); +void BeamModuleAssembler::emit_i_call_fun2(const ArgVal &Safe, + const ArgVal &Arity, + const ArgVal &Func) { + x86::Gp target; + mov_imm(ARG3, Arity.getValue()); + mov_arg(ARG4, Func); - x86::Gp dest = emit_call_fun(); - erlang_call(dest, ARG6); + target = emit_call_fun(always_one_of(Func, BEAM_TYPE_MASK_ALWAYS_BOXED), + masked_types(Func, BEAM_TYPE_MASK_BOXED) == + BEAM_TYPE_FUN, + Safe.getValue() == am_true); + + erlang_call(target, ARG6); } -void BeamModuleAssembler::emit_i_call_fun_last(const ArgVal &Arity, - const ArgVal &Deallocate) { - emit_deallocate(Deallocate); +void BeamModuleAssembler::emit_i_call_fun2_last(const ArgVal &Safe, + const ArgVal &Arity, + const ArgVal &Func, + const ArgVal &Deallocate) { + x86::Gp target; - mov_arg(ARG4, ArgVal(ArgVal::XReg, Arity.getValue())); mov_imm(ARG3, Arity.getValue()); + mov_arg(ARG4, Func); + + target = emit_call_fun(always_one_of(Func, BEAM_TYPE_MASK_ALWAYS_BOXED), + masked_types(Func, BEAM_TYPE_MASK_BOXED) == + BEAM_TYPE_FUN, + Safe.getValue() == am_true); - x86::Gp dest = emit_call_fun(); + emit_deallocate(Deallocate); emit_leave_frame(); - a.jmp(dest); + a.jmp(target); +} + +void BeamModuleAssembler::emit_i_call_fun(const ArgVal &Arity) { + const ArgVal Func(ArgVal::XReg, Arity.getValue()); + const ArgVal Safe(ArgVal::Immediate, am_false); + + emit_i_call_fun2(Safe, Arity, Func); +} + +void BeamModuleAssembler::emit_i_call_fun_last(const ArgVal &Arity, + const ArgVal &Deallocate) { + const ArgVal Func(ArgVal::XReg, Arity.getValue()); + const ArgVal Safe(ArgVal::Immediate, am_false); + + emit_i_call_fun2_last(Safe, Arity, Func, Deallocate); } /* Psuedo-instruction for signalling lambda load errors. Never actually runs. */ void BeamModuleAssembler::emit_i_lambda_error(const ArgVal &Dummy) { - a.comment("# Lambda error"); + comment("lambda error"); a.ud2(); } diff --git a/erts/emulator/beam/jit/x86/instr_guard_bifs.cpp b/erts/emulator/beam/jit/x86/instr_guard_bifs.cpp index cae99afa08..e55d964674 100644 --- a/erts/emulator/beam/jit/x86/instr_guard_bifs.cpp +++ b/erts/emulator/beam/jit/x86/instr_guard_bifs.cpp @@ -66,9 +66,8 @@ void BeamModuleAssembler::emit_bif_hd(const ArgVal &Fail, mov_arg(RET, Src); a.test(RETb, imm(_TAG_PRIMARY_MASK - TAG_PRIMARY_LIST)); - Uint fail = Fail.getValue(); - if (fail) { - a.jne(labels[fail]); + if (Fail.getValue() != 0) { + a.jne(resolve_beam_label(Fail)); } else { Label next = a.newLabel(); a.short_().je(next); @@ -169,47 +168,70 @@ void BeamModuleAssembler::emit_bif_element(const ArgVal &Fail, * The size of the code is 40 bytes, while the size of the bif2 * instruction is 36 bytes. */ Uint position = signed_val(Pos.getValue()); - Label error; mov_arg(ARG2, Tuple); - if (Fail.getValue() == 0) { - error = a.newLabel(); + x86::Gp boxed_ptr = emit_ptr_val(ARG3, ARG2); - emit_is_boxed(error, ARG2, dShort); - } else { - emit_is_boxed(labels[Fail.getValue()], ARG2); - } + if (exact_type(Tuple, BEAM_TYPE_TUPLE)) { + comment("skipped tuple test since source is always a tuple"); + ERTS_CT_ASSERT(Support::isInt32(make_arityval(MAX_ARITYVAL))); + a.cmp(emit_boxed_val(boxed_ptr, 0, sizeof(Uint32)), + imm(make_arityval_unchecked(position))); - x86::Gp boxed_ptr = emit_ptr_val(ARG3, ARG2); - a.mov(RETd, emit_boxed_val(boxed_ptr, 0, sizeof(Uint32))); + if (Fail.getValue() == 0) { + Label next = a.newLabel(); - ERTS_CT_ASSERT(Support::isInt32(make_arityval(MAX_ARITYVAL))); - a.cmp(RETd, imm(make_arityval_unchecked(position))); + a.short_().jae(next); - if (Fail.getValue() == 0) { - a.short_().jb(error); - } else { - a.jb(labels[Fail.getValue()]); - } + mov_imm(ARG1, make_small(position)); + safe_fragment_call(ga->get_handle_element_error()); - ERTS_CT_ASSERT(make_arityval_zero() == 0); - a.and_(RETb, imm(_TAG_HEADER_MASK)); + a.bind(next); + } else { + a.jb(resolve_beam_label(Fail)); + } + } else { + Distance dist; + Label error; + + if (Fail.getValue() == 0) { + error = a.newLabel(); + dist = dShort; + } else { + error = resolve_beam_label(Fail); + dist = dLong; + } - if (Fail.getValue() == 0) { - Label next = a.newLabel(); + emit_is_boxed(error, Tuple, ARG2, dist); - a.short_().je(next); + a.mov(RETd, emit_boxed_val(boxed_ptr, 0, sizeof(Uint32))); + a.cmp(RETd, imm(make_arityval_unchecked(position))); - a.bind(error); - { - mov_imm(ARG1, make_small(position)); - safe_fragment_call(ga->get_handle_element_error()); + if (Fail.getValue() == 0) { + a.short_().jb(error); + } else { + a.jb(error); } - a.bind(next); - } else { - a.jne(labels[Fail.getValue()]); + ERTS_CT_ASSERT(make_arityval_zero() == 0); + a.and_(RETb, imm(_TAG_HEADER_MASK)); + + if (Fail.getValue() == 0) { + Label next = a.newLabel(); + + a.short_().je(next); + + a.bind(error); + { + mov_imm(ARG1, make_small(position)); + safe_fragment_call(ga->get_handle_element_error()); + } + + a.bind(next); + } else { + a.jne(error); + } } a.mov(RET, emit_boxed_val(boxed_ptr, position * sizeof(Eterm))); @@ -222,7 +244,7 @@ void BeamModuleAssembler::emit_bif_element(const ArgVal &Fail, mov_arg(ARG1, Pos); if (Fail.getValue() != 0) { - a.lea(ARG3, x86::qword_ptr(labels[Fail.getValue()])); + a.lea(ARG3, x86::qword_ptr(resolve_beam_label(Fail))); } else { mov_imm(ARG3, 0); } diff --git a/erts/emulator/beam/jit/x86/instr_map.cpp b/erts/emulator/beam/jit/x86/instr_map.cpp index e413fbd72b..d928f75488 100644 --- a/erts/emulator/beam/jit/x86/instr_map.cpp +++ b/erts/emulator/beam/jit/x86/instr_map.cpp @@ -105,7 +105,7 @@ void BeamModuleAssembler::emit_i_get_map_element(const ArgVal &Fail, emit_leave_runtime(); emit_test_the_non_value(RET); - a.je(labels[Fail.getValue()]); + a.je(resolve_beam_label(Fail)); /* * Don't store the result if the destination is the scratch X register. @@ -137,7 +137,7 @@ void BeamModuleAssembler::emit_i_get_map_elements(const ArgVal &Fail, emit_leave_runtime(); a.test(RET, RET); - a.je(labels[Fail.getValue()]); + a.je(resolve_beam_label(Fail)); } void BeamModuleAssembler::emit_i_get_map_element_hash(const ArgVal &Fail, @@ -156,7 +156,7 @@ void BeamModuleAssembler::emit_i_get_map_element_hash(const ArgVal &Fail, emit_leave_runtime(); emit_test_the_non_value(RET); - a.je(labels[Fail.getValue()]); + a.je(resolve_beam_label(Fail)); /* * Don't store the result if the destination is the scratch X register. @@ -266,7 +266,7 @@ void BeamModuleAssembler::emit_update_map_exact(const ArgVal &Src, if (Fail.getValue() != 0) { fragment_call(ga->get_update_map_exact_guard_shared()); - a.je(labels[Fail.getValue()]); + a.je(resolve_beam_label(Fail)); } else { fragment_call(ga->get_update_map_exact_body_shared()); } diff --git a/erts/emulator/beam/jit/x86/instr_msg.cpp b/erts/emulator/beam/jit/x86/instr_msg.cpp index 43097600ab..71bd881718 100644 --- a/erts/emulator/beam/jit/x86/instr_msg.cpp +++ b/erts/emulator/beam/jit/x86/instr_msg.cpp @@ -283,7 +283,7 @@ void BeamModuleAssembler::emit_i_loop_rec(const ArgVal &Wait) { a.bind(entry); a.lea(ARG1, x86::qword_ptr(entry)); - a.lea(ARG2, x86::qword_ptr(labels[Wait.getValue()])); + a.lea(ARG2, x86::qword_ptr(resolve_beam_label(Wait))); fragment_call(ga->get_i_loop_rec_shared()); } @@ -313,14 +313,14 @@ void BeamModuleAssembler::emit_loop_rec_end(const ArgVal &Dest) { emit_leave_runtime(); a.dec(FCALLS); - a.jmp(labels[Dest.getValue()]); + a.jmp(resolve_beam_label(Dest)); } void BeamModuleAssembler::emit_wait_unlocked(const ArgVal &Dest) { emit_enter_runtime(); a.mov(ARG1, c_p); - a.lea(ARG2, x86::qword_ptr(labels[Dest.getValue()])); + a.lea(ARG2, x86::qword_ptr(resolve_beam_label(Dest))); runtime_call<2>(beam_jit_wait_unlocked); emit_leave_runtime(); @@ -332,7 +332,7 @@ void BeamModuleAssembler::emit_wait_locked(const ArgVal &Dest) { emit_enter_runtime(); a.mov(ARG1, c_p); - a.lea(ARG2, x86::qword_ptr(labels[Dest.getValue()])); + a.lea(ARG2, x86::qword_ptr(resolve_beam_label(Dest))); runtime_call<2>(beam_jit_wait_locked); emit_leave_runtime(); diff --git a/erts/emulator/beam/jit/x86/instr_select.cpp b/erts/emulator/beam/jit/x86/instr_select.cpp index 4fef1ea6d7..93d30b9081 100644 --- a/erts/emulator/beam/jit/x86/instr_select.cpp +++ b/erts/emulator/beam/jit/x86/instr_select.cpp @@ -32,11 +32,11 @@ void BeamModuleAssembler::emit_linear_search(x86::Gp comparand, const ArgVal &label = args[i + count]; cmp_arg(comparand, value, ARG1); - a.je(labels[label.getValue()]); + a.je(resolve_beam_label(label)); } if (Fail.isLabel()) { - a.jmp(labels[Fail.getValue()]); + a.jmp(resolve_beam_label(Fail)); } else { /* NIL means fallthrough to the next instruction. */ ASSERT(Fail.getType() == ArgVal::Immediate && Fail.getValue() == NIL); @@ -48,13 +48,21 @@ void BeamModuleAssembler::emit_i_select_tuple_arity(const ArgVal &Src, const ArgVal &Size, const Span<ArgVal> &args) { mov_arg(ARG2, Src); - emit_is_boxed(labels[Fail.getValue()], ARG2); + + emit_is_boxed(resolve_beam_label(Fail), Src, ARG2); + x86::Gp boxed_ptr = emit_ptr_val(ARG2, ARG2); ERTS_CT_ASSERT(Support::isInt32(make_arityval(MAX_ARITYVAL))); a.mov(ARG2d, emit_boxed_val(boxed_ptr, 0, sizeof(Uint32))); - ERTS_CT_ASSERT(_TAG_HEADER_ARITYVAL == 0); - a.test(ARG2.r8(), imm(_TAG_HEADER_MASK)); - a.jne(labels[Fail.getValue()]); + + if (masked_types(Src, BEAM_TYPE_MASK_BOXED) == BEAM_TYPE_TUPLE) { + comment("simplified tuple test since the source is always a tuple " + "when boxed"); + } else { + ERTS_CT_ASSERT(_TAG_HEADER_ARITYVAL == 0); + a.test(ARG2.r8(), imm(_TAG_HEADER_MASK)); + a.jne(resolve_beam_label(Fail)); + } ERTS_CT_ASSERT(Support::isInt32(make_arityval(MAX_ARITYVAL))); @@ -64,10 +72,10 @@ void BeamModuleAssembler::emit_i_select_tuple_arity(const ArgVal &Src, const ArgVal &label = args[i + count]; a.cmp(ARG2d, imm(value.getValue())); - a.je(labels[label.getValue()]); + a.je(resolve_beam_label(label)); } - a.jne(labels[Fail.getValue()]); + a.jne(resolve_beam_label(Fail)); } void BeamModuleAssembler::emit_i_select_val_lins(const ArgVal &Src, @@ -90,7 +98,7 @@ void BeamModuleAssembler::emit_i_select_val_bins(const ArgVal &Src, Label fail; if (Fail.isLabel()) { - fail = labels[Fail.getValue()]; + fail = resolve_beam_label(Fail); } else { /* NIL means fallthrough to the next instruction. */ ASSERT(Fail.getType() == ArgVal::Immediate && Fail.getValue() == NIL); @@ -152,15 +160,15 @@ void BeamModuleAssembler::emit_binsearch_nodes(size_t Left, cmp_arg(ARG2, midval, ARG1); if (Left == Right) { - a.je(labels[args[mid + count].getValue()]); - a.jmp(labels[Fail.getValue()]); + a.je(resolve_beam_label(args[mid + count])); + a.jmp(resolve_beam_label(Fail)); return; } - a.je(labels[args[mid + count].getValue()]); + a.je(resolve_beam_label(args[mid + count])); if (Left == mid) { - a.jb(labels[Fail.getValue()]); + a.jb(resolve_beam_label(Fail)); } else { Label right_tree = a.newLabel(); a.ja(right_tree); @@ -188,7 +196,7 @@ void BeamModuleAssembler::emit_i_jump_on_val(const ArgVal &Src, a.cmp(RETb, imm(_TAG_IMMED1_SMALL)); if (Fail.isLabel()) { - a.jne(labels[Fail.getValue()]); + a.jne(resolve_beam_label(Fail)); } else { /* NIL means fallthrough to the next instruction. */ ASSERT(Fail.getType() == ArgVal::Immediate && Fail.getValue() == NIL); @@ -209,7 +217,7 @@ void BeamModuleAssembler::emit_i_jump_on_val(const ArgVal &Src, a.cmp(ARG1, imm(args.size())); if (Fail.isLabel()) { - a.jae(labels[Fail.getValue()]); + a.jae(resolve_beam_label(Fail)); } else { a.short_().jae(fail); } @@ -261,10 +269,12 @@ bool BeamModuleAssembler::emit_optimized_three_way_select( a.mov(ARG1, imm(diff)); a.or_(ARG2, ARG1); } + cmp_arg(ARG2, val, ARG1); - a.je(labels[args[2].getValue()]); + a.je(resolve_beam_label(args[2])); + if (Fail.isLabel()) { - a.jmp(labels[Fail.getValue()]); + a.jmp(resolve_beam_label(Fail)); } else { /* NIL means fallthrough to the next instruction. */ ASSERT(Fail.getType() == ArgVal::Immediate && Fail.getValue() == NIL); diff --git a/erts/emulator/beam/jit/x86/ops.tab b/erts/emulator/beam/jit/x86/ops.tab index c754fde6bc..24e1274e94 100644 --- a/erts/emulator/beam/jit/x86/ops.tab +++ b/erts/emulator/beam/jit/x86/ops.tab @@ -221,6 +221,7 @@ set_tuple_element s S P current_tuple/1 current_tuple/2 +typed_current_tuple/1 is_tuple Fail=f Src | test_arity Fail Src Arity => \ i_is_tuple_of_arity Fail Src Arity | current_tuple Src @@ -231,10 +232,12 @@ is_tuple NotTupleFail Tuple | is_tagged_tuple WrongRecordFail Tuple Arity Atom = i_is_tagged_tuple_ff NotTupleFail WrongRecordFail Tuple Arity Atom | current_tuple Tuple is_tagged_tuple Fail Tuple Arity Atom => \ - i_is_tagged_tuple Fail Tuple Arity Atom | current_tuple Tuple + i_is_tagged_tuple Fail Tuple Arity Atom | typed_current_tuple Tuple is_tuple Fail=f Src => i_is_tuple Fail Src | current_tuple Src +typed_current_tuple Tuple => remove_tuple_type(Tuple) + i_is_tuple_of_arity f? s A i_test_arity f? s A @@ -385,22 +388,16 @@ is_eq_exact Lbl S S => is_eq_exact Lbl C=c R=xy => is_eq_exact Lbl R C is_eq_exact Lbl R=xy n => is_nil Lbl R -is_eq_exact Lbl R=xy C=ia => i_is_eq_exact_immed Lbl R C is_eq_exact Lbl R=xy C=q => is_eq_exact_literal(Lbl, R, C) is_ne_exact Lbl S S => jump Lbl is_ne_exact Lbl C=c R=xy => is_ne_exact Lbl R C -is_ne_exact Lbl R=xy C=ian => i_is_ne_exact_immed Lbl R C is_ne_exact Lbl R=xy C=q => i_is_ne_exact_literal Lbl R C -i_is_eq_exact_immed f? s c - i_is_eq_exact_literal/4 i_is_eq_exact_literal f? s c I -i_is_ne_exact_immed f? s c - i_is_ne_exact_literal f? s c is_eq_exact f? s s @@ -644,8 +641,8 @@ bif2 Fail Bif S1 S2 Dst | never_fails(Bif) => nofail_bif2 S1 S2 Bif Dst bif1 Fail Bif S1 Dst => i_bif1 S1 Fail Bif Dst bif2 Fail Bif S1 S2 Dst => i_bif2 S1 S2 Fail Bif Dst -nofail_bif2 S1=d S2=ian Bif Dst | is_eq_exact_bif(Bif) => bif_is_eq_exact_immed S1 S2 Dst -nofail_bif2 S1=d S2=ian Bif Dst | is_ne_exact_bif(Bif) => bif_is_ne_exact_immed S1 S2 Dst +nofail_bif2 S1=d S2 Bif Dst | is_eq_exact_bif(Bif) => bif_is_eq_exact S1 S2 Dst +nofail_bif2 S1=d S2 Bif Dst | is_ne_exact_bif(Bif) => bif_is_ne_exact S1 S2 Dst i_get_hash c I d i_get s d @@ -661,8 +658,8 @@ i_bif1 s j? b d i_bif2 s s j? b d i_bif3 s s s j? b d -bif_is_eq_exact_immed S c d -bif_is_ne_exact_immed S c d +bif_is_eq_exact S s d +bif_is_ne_exact S s d # # Internal calls. @@ -688,6 +685,14 @@ call_fun Arity => i_call_fun Arity i_call_fun t i_call_fun_last t t +call_fun2 Safe Arity Func | deallocate D | return => \ + i_call_fun2_last Safe Arity Func D +call_fun2 Safe Arity Func => \ + i_call_fun2 Safe Arity Func + +i_call_fun2 a t S +i_call_fun2_last a t S t + # # A fun with an empty environment can be converted to a literal. # As a further optimization, the we try to move the fun to its diff --git a/erts/emulator/beam/jit/x86/process_main.cpp b/erts/emulator/beam/jit/x86/process_main.cpp index ebd7b356cf..cf4dd8bd4f 100644 --- a/erts/emulator/beam/jit/x86/process_main.cpp +++ b/erts/emulator/beam/jit/x86/process_main.cpp @@ -154,7 +154,7 @@ void BeamGlobalAssembler::emit_process_main() { /* Check that ARG3 is set to a valid CP. */ a.test(ARG3, imm(_CPMASK)); a.je(check_i); - a.comment("# ARG3 is not a valid CP"); + comment("# ARG3 is not a valid CP"); a.ud2(); a.bind(check_i); #endif diff --git a/lib/compiler/src/Makefile b/lib/compiler/src/Makefile index 24414fef77..4fb1ab7631 100644 --- a/lib/compiler/src/Makefile +++ b/lib/compiler/src/Makefile @@ -106,6 +106,7 @@ MODULES = \ BEAM_H = $(wildcard ../priv/beam_h/*.h) HRL_FILES= \ + beam_asm.hrl \ beam_disasm.hrl \ beam_ssa_opt.hrl \ beam_ssa.hrl \ @@ -192,25 +193,29 @@ release_docs_spec: # Dependencies -- alphabetically, please # ---------------------------------------------------- +$(EBIN)/beam_asm.beam: beam_asm.hrl $(EBIN)/beam_call_types.beam: beam_types.hrl -$(EBIN)/beam_disasm.beam: $(EGEN)/beam_opcodes.hrl beam_disasm.hrl -$(EBIN)/beam_listing.beam: core_parse.hrl v3_kernel.hrl beam_ssa.hrl +$(EBIN)/beam_block.beam: beam_asm.hrl +$(EBIN)/beam_disasm.beam: $(EGEN)/beam_opcodes.hrl beam_disasm.hrl beam_asm.hrl +$(EBIN)/beam_jump.beam: beam_asm.hrl $(EBIN)/beam_kernel_to_ssa.beam: v3_kernel.hrl beam_ssa.hrl +$(EBIN)/beam_listing.beam: core_parse.hrl v3_kernel.hrl beam_ssa.hrl beam_asm.hrl $(EBIN)/beam_ssa.beam: beam_ssa.hrl $(EBIN)/beam_ssa_bsm.beam: beam_ssa.hrl -$(EBIN)/beam_ssa_codegen.beam: beam_ssa.hrl +$(EBIN)/beam_ssa_codegen.beam: beam_ssa.hrl beam_asm.hrl $(EBIN)/beam_ssa_dead.beam: beam_ssa.hrl $(EBIN)/beam_ssa_funs.beam: beam_ssa.hrl $(EBIN)/beam_ssa_lint.beam: beam_ssa.hrl $(EBIN)/beam_ssa_opt.beam: beam_ssa.hrl $(EBIN)/beam_ssa_pp.beam: beam_ssa.hrl -$(EBIN)/beam_ssa_pre_codegen.beam: beam_ssa.hrl +$(EBIN)/beam_ssa_pre_codegen.beam: beam_ssa.hrl beam_asm.hrl $(EBIN)/beam_ssa_recv.beam: beam_ssa.hrl $(EBIN)/beam_ssa_share.beam: beam_ssa.hrl $(EBIN)/beam_ssa_throw.beam: beam_ssa.hrl $(EBIN)/beam_ssa_type.beam: beam_ssa.hrl beam_types.hrl +$(EBIN)/beam_trim.beam: beam_asm.hrl $(EBIN)/beam_types.beam: beam_types.hrl -$(EBIN)/beam_validator.beam: beam_types.hrl +$(EBIN)/beam_validator.beam: beam_types.hrl beam_asm.hrl $(EBIN)/cerl.beam: core_parse.hrl $(EBIN)/compile.beam: core_parse.hrl ../../stdlib/include/erl_compile.hrl $(EBIN)/core_lib.beam: core_parse.hrl diff --git a/lib/compiler/src/beam_asm.erl b/lib/compiler/src/beam_asm.erl index c5d26146e8..f4151b9693 100644 --- a/lib/compiler/src/beam_asm.erl +++ b/lib/compiler/src/beam_asm.erl @@ -24,15 +24,15 @@ -export([module/4]). -export([encode/2]). --export_type([fail/0,label/0,reg/0,reg_num/0,src/0,module_code/0,function_name/0]). +-export_type([fail/0,label/0,src/0,module_code/0,function_name/0]). -import(lists, [map/2,member/2,keymember/3,duplicate/2,splitwith/2]). + -include("beam_opcodes.hrl"). +-include("beam_asm.hrl"). %% Common types for describing operands for BEAM instructions. --type reg_num() :: 0..1023. --type reg() :: {'x',reg_num()} | {'y',reg_num()}. --type src() :: reg() | +-type src() :: beam_reg() | {'literal',term()} | {'atom',atom()} | {'integer',integer()} | @@ -64,11 +64,12 @@ module(Code, ExtraChunks, CompileInfo, CompilerOpts) -> assemble({Mod,Exp0,Attr0,Asm0,NumLabels}, ExtraChunks, CompileInfo, CompilerOpts) -> {1,Dict0} = beam_dict:atom(Mod, beam_dict:new()), {0,Dict1} = beam_dict:fname(atom_to_list(Mod) ++ ".erl", Dict0), - Dict2 = shared_fun_wrappers(CompilerOpts, Dict1), + {0,Dict2} = beam_dict:type(any, Dict1), + Dict3 = shared_fun_wrappers(CompilerOpts, Dict2), NumFuncs = length(Asm0), {Asm,Attr} = on_load(Asm0, Attr0), Exp = sets:from_list(Exp0, [{version, 2}]), - {Code,Dict} = assemble_1(Asm, Exp, Dict2, []), + {Code,Dict} = assemble_1(Asm, Exp, Dict3, []), build_file(Code, Attr, Dict, NumLabels, NumFuncs, ExtraChunks, CompileInfo, CompilerOpts). @@ -190,9 +191,14 @@ build_file(Code, Attr, Dict, NumLabels, NumFuncs, ExtraChunks, CompileInfo, Comp end, %% Create the line chunk. - LineChunk = chunk(<<"Line">>, build_line_table(Dict)), + %% Create the type table chunk. + {NumTypes, TypeTab} = beam_dict:type_table(Dict), + TypeChunk = chunk(<<"Type">>, + <<?BEAM_TYPES_VERSION:32, NumTypes:32>>, + TypeTab), + %% Create the attributes and compile info chunks. Essentials0 = [AtomChunk,CodeChunk,StringChunk,ImportChunk, @@ -211,12 +217,14 @@ build_file(Code, Attr, Dict, NumLabels, NumFuncs, ExtraChunks, CompileInfo, Comp %% Create IFF chunk. Chunks = case member(slim, CompilerOpts) of - true -> - [Essentials,AttrChunk]; - false -> - [Essentials,LocChunk,AttrChunk, - CompileChunk,CheckedChunks,LineChunk] - end, + true when NumTypes > 0 -> + [Essentials,AttrChunk,TypeChunk]; + true when NumTypes =:= 0 -> + [Essentials,AttrChunk]; + false -> + [Essentials,LocChunk,AttrChunk, + CompileChunk,CheckedChunks,LineChunk,TypeChunk] + end, build_form(<<"BEAM">>, Chunks). %% finalize_fun_table(Essentials, MD5) -> FinalizedEssentials @@ -407,6 +415,23 @@ encode_op_1([A0|As], Dict0, Acc) -> encode_op_1(As, Dict, [Acc,A]); encode_op_1([], Dict, Acc) -> {Acc,Dict}. +encode_arg(#tr{r={x, X},t=Type}, Dict0) when is_integer(X), X >= 0 -> + %% Gracefully prevent this module from being loaded in OTP 24 and below by + %% forcing an opcode it doesn't understand. It would of course fail to load + %% without this, but the error message wouldn't be very helpful. + Canary = beam_opcodes:opcode(call_fun2, 3), + {Index, Dict} = beam_dict:type(Type, beam_dict:opcode(Canary, Dict0)), + Data = [encode(?tag_z, 5), + encode(?tag_x, X), + encode(?tag_u, Index)], + {Data, Dict}; +encode_arg(#tr{r={y, Y},t=Type}, Dict0) when is_integer(Y), Y >= 0 -> + Canary = beam_opcodes:opcode(call_fun2, 3), + {Index, Dict} = beam_dict:type(Type, beam_dict:opcode(Canary, Dict0)), + Data = [encode(?tag_z, 5), + encode(?tag_y, Y), + encode(?tag_u, Index)], + {Data, Dict}; encode_arg({x, X}, Dict) when is_integer(X), X >= 0 -> {encode(?tag_x, X), Dict}; encode_arg({y, Y}, Dict) when is_integer(Y), Y >= 0 -> diff --git a/lib/compiler/src/beam_asm.hrl b/lib/compiler/src/beam_asm.hrl new file mode 100644 index 0000000000..5e557bbe74 --- /dev/null +++ b/lib/compiler/src/beam_asm.hrl @@ -0,0 +1,44 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2021. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-include("beam_types.hrl"). + +-type reg_num() :: 0 .. 1023. + +-type xreg() :: {'x', reg_num()}. +-type yreg() :: {'y', reg_num()}. +-type freg() :: {'fr', reg_num()}. +-type zreg() :: {'z', reg_num()}. + +-type beam_reg() :: xreg() | yreg() | freg(). + +-type beam_literal() :: {atom, [] | atom()} | + {float, [] | float()} | + {integer, [] | integer()} | + {literal, term()} | + nil. + +%% Type-tagged beam register. Assembly passes that care about registers (e.g. +%% beam_trim) must be prepared to handle these whenever they inspect a +%% register. +%% +%% To aid in the above, the validator will explode upon encountering them in an +%% unfamiliar context. +-record(tr, {r :: beam_reg(), t :: type()}). diff --git a/lib/compiler/src/beam_call_types.erl b/lib/compiler/src/beam_call_types.erl index c38bf806c3..b00fdd08c0 100644 --- a/lib/compiler/src/beam_call_types.erl +++ b/lib/compiler/src/beam_call_types.erl @@ -235,6 +235,35 @@ types(erlang, 'list_to_bitstring', [_]) -> %% As list_to_binary but with bitstrings rather than binaries. sub_unsafe(#t_bitstring{}, [proper_list()]); +%% Process operations +types(erlang, alias, [_]) -> + sub_unsafe(reference, [any]); +types(erlang, alias, [_, _]) -> + sub_unsafe(reference, [any, proper_list()]); +types(erlang, monitor, [_, _]) -> + sub_unsafe(reference, [any, any]); +types(erlang, monitor, [_, _, _]) -> + sub_unsafe(reference, [any, any, proper_list()]); +types(erlang, 'spawn', [_]) -> + sub_unsafe(pid, [#t_fun{arity=0}]); +types(erlang, 'spawn', [_, _]) -> + sub_unsafe(pid, [#t_atom{}, #t_fun{arity=0}]); +types(erlang, 'spawn', [_, _, _]) -> + sub_unsafe(pid, [#t_atom{}, #t_atom{}, proper_list()]); +types(erlang, 'spawn_link', Args) -> + types(erlang, 'spawn', Args); +types(erlang, 'spawn_monitor', [_]) -> + RetType = make_two_tuple(pid, reference), + sub_unsafe(RetType, [#t_fun{arity=0}]); +types(erlang, 'spawn_monitor', [_, _]) -> + RetType = make_two_tuple(pid, reference), + sub_unsafe(RetType, [#t_atom{}, #t_fun{arity=0}]); +types(erlang, 'spawn_monitor', [_, _, _]) -> + RetType = make_two_tuple(pid, reference), + sub_unsafe(RetType, [#t_atom{}, #t_atom{}, proper_list()]); +types(erlang, 'spawn_request', [_ | _]=Args) when length(Args) =< 5 -> + sub_unsafe(reference, [any || _ <- Args]); + %% Misc ops. types(erlang, 'binary_part', [_, _]) -> PosLen = make_two_tuple(#t_integer{}, #t_integer{}), @@ -249,6 +278,8 @@ types(erlang, 'is_map_key', [Key, Map]) -> _ -> beam_types:make_boolean() end, sub_unsafe(RetType, [any, #t_map{}]); +types(erlang, make_ref, []) -> + sub_unsafe(reference, []); types(erlang, 'map_get', [Key, Map]) -> RetType = erlang_map_get_type(Key, Map), sub_unsafe(RetType, [any, #t_map{}]); diff --git a/lib/compiler/src/beam_dict.erl b/lib/compiler/src/beam_dict.erl index 43a8ea4fb1..f9c973145c 100644 --- a/lib/compiler/src/beam_dict.erl +++ b/lib/compiler/src/beam_dict.erl @@ -23,10 +23,12 @@ -export([new/0,opcode/2,highest_opcode/1, atom/2,local/4,export/4,import/4, - string/2,lambda/3,literal/2,line/2,fname/2, + string/2,lambda/3,literal/2,line/2,fname/2,type/2, atom_table/1,local_table/1,export_table/1,import_table/1, string_table/1,lambda_table/1,literal_table/1, - line_table/1]). + line_table/1,type_table/1]). + +-include("beam_types.hrl"). -type label() :: beam_asm:label(). @@ -37,28 +39,31 @@ -type fname_tab() :: #{Name :: term() => index()}. -type line_tab() :: #{{Fname :: index(), Line :: term()} => index()}. -type literal_tab() :: #{Literal :: term() => index()}. +-type type_tab() :: #{ Type :: binary() => index()}. -type lambda_info() :: {label(),{index(),label(),non_neg_integer()}}. -type lambda_tab() :: {non_neg_integer(),[lambda_info()]}. -type wrapper() :: #{label() => index()}. -record(asm, - {atoms = #{} :: atom_tab(), - exports = [] :: [{label(), arity(), label()}], - locals = [] :: [{label(), arity(), label()}], - imports = gb_trees:empty() :: import_tab(), - strings = <<>> :: binary(), %String pool - lambdas = {0,[]} :: lambda_tab(), + {atoms = #{} :: atom_tab(), + exports = [] :: [{label(), arity(), label()}], + locals = [] :: [{label(), arity(), label()}], + imports = gb_trees:empty() :: import_tab(), + strings = <<>> :: binary(), %String pool + lambdas = {0,[]} :: lambda_tab(), + types = #{} :: type_tab(), wrappers = #{} :: wrapper(), - literals = #{} :: literal_tab(), - fnames = #{} :: fname_tab(), - lines = #{} :: line_tab(), - num_lines = 0 :: non_neg_integer(), %Number of line instructions - next_import = 0 :: non_neg_integer(), - string_offset = 0 :: non_neg_integer(), - next_literal = 0 :: non_neg_integer(), - highest_opcode = 0 :: non_neg_integer() - }). + literals = #{} :: literal_tab(), + fnames = #{} :: fname_tab(), + lines = #{} :: line_tab(), + num_lines = 0 :: non_neg_integer(), %Number of line instructions + next_import = 0 :: non_neg_integer(), + string_offset = 0 :: non_neg_integer(), + next_literal = 0 :: non_neg_integer(), + highest_opcode = 0 :: non_neg_integer() + }). + -type bdict() :: #asm{}. %%----------------------------------------------------------------------------- @@ -225,6 +230,19 @@ fname(Name, #asm{fnames=Fnames}=Dict) -> {Index,Dict#asm{fnames=Fnames#{Name=>Index}}} end. +-spec type(type(), bdict()) -> {non_neg_integer(), bdict()} | none. + +type(Type, #asm{types=Types0}=Dict) -> + ExtType = beam_types:encode_ext(Type), + case Types0 of + #{ ExtType := Index } -> + {Index, Dict}; + #{} -> + Index = map_size(Types0), + Types = Types0#{ ExtType => Index }, + {Index, Dict#asm{types=Types}} + end. + %% Returns the atom table. %% atom_table(Dict, Encoding) -> {LastIndex,[Length,AtomString...]} -spec atom_table(bdict()) -> {non_neg_integer(), [[non_neg_integer(),...]]}. @@ -301,6 +319,18 @@ my_term_to_binary(Term) -> term_to_binary(Term, [{minor_version,2},deterministic]). +%% Returns the type table. +-spec type_table(bdict()) -> {non_neg_integer(), binary()}. + +type_table(#asm{types=Types}) -> + Sorted = lists:keysort(2, maps:to_list(Types)), + {map_size(Types), build_type_table(Sorted, <<>>)}. + +build_type_table([{ExtType, _} | Sorted], Acc) -> + build_type_table(Sorted, <<Acc/binary, ExtType/binary>>); +build_type_table([], Acc) -> + Acc. + %% Return the line table. -spec line_table(bdict()) -> {non_neg_integer(), %Number of line instructions. diff --git a/lib/compiler/src/beam_disasm.erl b/lib/compiler/src/beam_disasm.erl index 0ab095e364..33fcfd0c03 100644 --- a/lib/compiler/src/beam_disasm.erl +++ b/lib/compiler/src/beam_disasm.erl @@ -35,11 +35,13 @@ -include("beam_opcodes.hrl"). -include("beam_disasm.hrl"). +-include("beam_asm.hrl"). %%----------------------------------------------------------------------- -type index() :: non_neg_integer(). -type literals() :: 'none' | gb_trees:tree(index(), term()). +-type types() :: 'none' | gb_trees:tree(index(), term()). -type symbolic_tag() :: 'a' | 'f' | 'h' | 'i' | 'u' | 'x' | 'y' | 'z'. -type disasm_tag() :: symbolic_tag() | 'fr' | 'atom' | 'float' | 'literal'. -type disasm_term() :: 'nil' | {disasm_tag(), _}. @@ -190,8 +192,10 @@ process_chunks(F) -> Lambdas = beam_disasm_lambdas(LambdaBin, Atoms), LiteralBin = optional_chunk(F, "LitT"), Literals = beam_disasm_literals(LiteralBin), + TypeBin = optional_chunk(F, "Type"), + Types = beam_disasm_types(TypeBin), Code = beam_disasm_code(CodeBin, Atoms, mk_imports(ImportsList), - StrBin, Lambdas, Literals, Module), + StrBin, Lambdas, Literals, Types, Module), Attributes = case optional_chunk(F, attributes) of none -> []; @@ -255,6 +259,24 @@ disasm_literals(<<Sz:32,Ext:Sz/binary,T/binary>>, Index) -> disasm_literals(<<>>, _) -> []. %%----------------------------------------------------------------------- +%% Disassembles the literal table (constant pool) of a BEAM file. +%%----------------------------------------------------------------------- + +-spec beam_disasm_types('none' | binary()) -> literals(). + +beam_disasm_types(none) -> + none; +beam_disasm_types(<<?BEAM_TYPES_VERSION:32,Count:32,Table/binary>>) -> + Res = gb_trees:from_orddict(disasm_types(Table, 0)), + Count = gb_trees:size(Res), %Assertion. + Res. + +disasm_types(<<Type:2/binary,Rest/binary>>, Index) -> + [{Index,beam_types:decode_ext(Type)}|disasm_types(Rest, Index+1)]; +disasm_types(<<>>, _) -> + []. + +%%----------------------------------------------------------------------- %% Disassembles the code chunk of a BEAM file: %% - The code is first disassembled into a long list of instructions. %% - This list is then split into functions and all names are resolved. @@ -265,9 +287,9 @@ beam_disasm_code(<<_SS:32, % Sub-Size (length of information before code) _OM:32, % Opcode Max _L:32,_F:32, CodeBin/binary>>, Atoms, Imports, - Str, Lambdas, Literals, M) -> + Str, Lambdas, Literals, Types, M) -> Code = binary_to_list(CodeBin), - try disasm_code(Code, Atoms, Literals) of + try disasm_code(Code, Atoms, Literals, Types) of DisasmCode -> Functions = get_function_chunks(DisasmCode), Labels = mk_labels(local_labels(Functions)), @@ -283,10 +305,11 @@ beam_disasm_code(<<_SS:32, % Sub-Size (length of information before code) %%----------------------------------------------------------------------- -disasm_code([B|Bs], Atoms, Literals) -> - {Instr,RestBs} = disasm_instr(B, Bs, Atoms, Literals), - [Instr|disasm_code(RestBs, Atoms, Literals)]; -disasm_code([], _, _) -> []. +disasm_code([B|Bs], Atoms, Literals, Types) -> + {Instr,RestBs} = disasm_instr(B, Bs, Atoms, Literals, Types), + [Instr|disasm_code(RestBs, Atoms, Literals, Types)]; +disasm_code([], _, _, _) -> + []. %%----------------------------------------------------------------------- %% Splits the code stream into chunks representing the code of functions. @@ -366,31 +389,31 @@ local_labels_2(_, R, _) -> R. %% in a generic way; indexing instructions are handled separately. %%----------------------------------------------------------------------- -disasm_instr(B, Bs, Atoms, Literals) -> +disasm_instr(B, Bs, Atoms, Literals, Types) -> {SymOp, Arity} = beam_opcodes:opname(B), case SymOp of select_val -> - disasm_select_inst(select_val, Bs, Atoms, Literals); + disasm_select_inst(select_val, Bs, Atoms, Literals, Types); select_tuple_arity -> - disasm_select_inst(select_tuple_arity, Bs, Atoms, Literals); + disasm_select_inst(select_tuple_arity, Bs, Atoms, Literals, Types); put_map_assoc -> - disasm_map_inst(put_map_assoc, Arity, Bs, Atoms, Literals); + disasm_map_inst(put_map_assoc, Arity, Bs, Atoms, Literals, Types); put_map_exact -> - disasm_map_inst(put_map_exact, Arity, Bs, Atoms, Literals); + disasm_map_inst(put_map_exact, Arity, Bs, Atoms, Literals, Types); get_map_elements -> - disasm_map_inst(get_map_elements, Arity, Bs, Atoms, Literals); + disasm_map_inst(get_map_elements, Arity, Bs, Atoms, Literals, Types); has_map_fields -> - disasm_map_inst(has_map_fields, Arity, Bs, Atoms, Literals); + disasm_map_inst(has_map_fields, Arity, Bs, Atoms, Literals, Types); put_tuple2 -> - disasm_put_tuple2(Bs, Atoms, Literals); + disasm_put_tuple2(Bs, Atoms, Literals, Types); make_fun3 -> - disasm_make_fun3(Bs, Atoms, Literals); + disasm_make_fun3(Bs, Atoms, Literals, Types); init_yregs -> - disasm_init_yregs(Bs, Atoms, Literals); + disasm_init_yregs(Bs, Atoms, Literals, Types); bs_create_bin -> - disasm_bs_create_bin(Bs, Atoms, Literals); + disasm_bs_create_bin(Bs, Atoms, Literals, Types); _ -> - try decode_n_args(Arity, Bs, Atoms, Literals) of + try decode_n_args(Arity, Bs, Atoms, Literals, Types) of {Args, RestBs} -> ?NO_DEBUG("instr ~p~n", [{SymOp, Args}]), {{SymOp, Args}, RestBs} @@ -410,59 +433,59 @@ disasm_instr(B, Bs, Atoms, Literals) -> %% where each case is of the form [symbol,{f,Label}]. %%----------------------------------------------------------------------- -disasm_select_inst(Inst, Bs, Atoms, Literals) -> - {X, Bs1} = decode_arg(Bs, Atoms, Literals), - {F, Bs2} = decode_arg(Bs1, Atoms, Literals), - {Z, Bs3} = decode_arg(Bs2, Atoms, Literals), - {U, Bs4} = decode_arg(Bs3, Atoms, Literals), +disasm_select_inst(Inst, Bs, Atoms, Literals, Types) -> + {X, Bs1} = decode_arg(Bs, Atoms, Literals, Types), + {F, Bs2} = decode_arg(Bs1, Atoms, Literals, Types), + {Z, Bs3} = decode_arg(Bs2, Atoms, Literals, Types), + {U, Bs4} = decode_arg(Bs3, Atoms, Literals, Types), {u, Len} = U, - {List, RestBs} = decode_n_args(Len, Bs4, Atoms, Literals), + {List, RestBs} = decode_n_args(Len, Bs4, Atoms, Literals, Types), {{Inst, [X,F,{Z,U,List}]}, RestBs}. -disasm_map_inst(Inst, Arity, Bs0, Atoms, Literals) -> - {Args0,Bs1} = decode_n_args(Arity, Bs0, Atoms, Literals), +disasm_map_inst(Inst, Arity, Bs0, Atoms, Literals, Types) -> + {Args0,Bs1} = decode_n_args(Arity, Bs0, Atoms, Literals, Types), %% no droplast .. [Z|Args1] = lists:reverse(Args0), Args = lists:reverse(Args1), - {U, Bs2} = decode_arg(Bs1, Atoms, Literals), + {U, Bs2} = decode_arg(Bs1, Atoms, Literals, Types), {u, Len} = U, - {List, RestBs} = decode_n_args(Len, Bs2, Atoms, Literals), + {List, RestBs} = decode_n_args(Len, Bs2, Atoms, Literals, Types), {{Inst, Args ++ [{Z,U,List}]}, RestBs}. -disasm_put_tuple2(Bs, Atoms, Literals) -> - {X, Bs1} = decode_arg(Bs, Atoms, Literals), - {Z, Bs2} = decode_arg(Bs1, Atoms, Literals), - {U, Bs3} = decode_arg(Bs2, Atoms, Literals), +disasm_put_tuple2(Bs, Atoms, Literals, Types) -> + {X, Bs1} = decode_arg(Bs, Atoms, Literals, Types), + {Z, Bs2} = decode_arg(Bs1, Atoms, Literals, Types), + {U, Bs3} = decode_arg(Bs2, Atoms, Literals, Types), {u, Len} = U, - {List, RestBs} = decode_n_args(Len, Bs3, Atoms, Literals), + {List, RestBs} = decode_n_args(Len, Bs3, Atoms, Literals, Types), {{put_tuple2, [X,{Z,U,List}]}, RestBs}. -disasm_make_fun3(Bs, Atoms, Literals) -> - {Fun, Bs1} = decode_arg(Bs, Atoms, Literals), - {Dst, Bs2} = decode_arg(Bs1, Atoms, Literals), - {Z, Bs3} = decode_arg(Bs2, Atoms, Literals), - {U, Bs4} = decode_arg(Bs3, Atoms, Literals), +disasm_make_fun3(Bs, Atoms, Literals, Types) -> + {Fun, Bs1} = decode_arg(Bs, Atoms, Literals, Types), + {Dst, Bs2} = decode_arg(Bs1, Atoms, Literals, Types), + {Z, Bs3} = decode_arg(Bs2, Atoms, Literals, Types), + {U, Bs4} = decode_arg(Bs3, Atoms, Literals, Types), {u, Len} = U, - {List, RestBs} = decode_n_args(Len, Bs4, Atoms, Literals), + {List, RestBs} = decode_n_args(Len, Bs4, Atoms, Literals, Types), {{make_fun3, [Fun,Dst,{Z,U,List}]}, RestBs}. -disasm_init_yregs(Bs1, Atoms, Literals) -> - {Z, Bs2} = decode_arg(Bs1, Atoms, Literals), - {U, Bs3} = decode_arg(Bs2, Atoms, Literals), +disasm_init_yregs(Bs1, Atoms, Literals, Types) -> + {Z, Bs2} = decode_arg(Bs1, Atoms, Literals, Types), + {U, Bs3} = decode_arg(Bs2, Atoms, Literals, Types), {u, Len} = U, - {List, RestBs} = decode_n_args(Len, Bs3, Atoms, Literals), + {List, RestBs} = decode_n_args(Len, Bs3, Atoms, Literals, Types), {{init_yregs, [{Z,U,List}]}, RestBs}. -disasm_bs_create_bin(Bs0, Atoms, Literals) -> - {A1, Bs1} = decode_arg(Bs0, Atoms, Literals), - {A2, Bs2} = decode_arg(Bs1, Atoms, Literals), - {A3, Bs3} = decode_arg(Bs2, Atoms, Literals), - {A4, Bs4} = decode_arg(Bs3, Atoms, Literals), - {A5, Bs5} = decode_arg(Bs4, Atoms, Literals), - {Z, Bs6} = decode_arg(Bs5, Atoms, Literals), - {U, Bs7} = decode_arg(Bs6, Atoms, Literals), +disasm_bs_create_bin(Bs0, Atoms, Literals, Types) -> + {A1, Bs1} = decode_arg(Bs0, Atoms, Literals, Types), + {A2, Bs2} = decode_arg(Bs1, Atoms, Literals, Types), + {A3, Bs3} = decode_arg(Bs2, Atoms, Literals, Types), + {A4, Bs4} = decode_arg(Bs3, Atoms, Literals, Types), + {A5, Bs5} = decode_arg(Bs4, Atoms, Literals, Types), + {Z, Bs6} = decode_arg(Bs5, Atoms, Literals, Types), + {U, Bs7} = decode_arg(Bs6, Atoms, Literals, Types), {u, Len} = U, - {List, RestBs} = decode_n_args(Len, Bs7, Atoms, Literals), + {List, RestBs} = decode_n_args(Len, Bs7, Atoms, Literals, Types), {{bs_create_bin, [{A1,A2,A3,A4,A5,Z,U,List}]}, RestBs}. %%----------------------------------------------------------------------- @@ -481,21 +504,22 @@ decode_arg([B|Bs]) -> ?NO_DEBUG('Tag = ~p, B = ~p, Bs = ~p~n', [Tag, B, Bs]), case Tag of z -> - decode_z_tagged(Tag, B, Bs, no_literals); + decode_z_tagged(Tag, B, Bs, no_literals, no_types); _ -> %% all other cases are handled as if they were integers decode_int(Tag, B, Bs) end. --spec decode_arg([byte(),...], gb_trees:tree(index(), _), literals()) -> +-spec decode_arg([byte(),...], + gb_trees:tree(index(), _), literals(), types()) -> {disasm_term(), [byte()]}. -decode_arg([B|Bs0], Atoms, Literals) -> +decode_arg([B|Bs0], Atoms, Literals, Types) -> Tag = decode_tag(B band 2#111), ?NO_DEBUG('Tag = ~p, B = ~p, Bs = ~p~n', [Tag, B, Bs0]), case Tag of z -> - decode_z_tagged(Tag, B, Bs0, Literals); + decode_z_tagged(Tag, B, Bs0, Literals, Types); a -> %% atom or nil case decode_int(Tag, B, Bs0) of @@ -570,7 +594,7 @@ decode_negative(N, Len) -> %% Decodes lists and floating point numbers. %%----------------------------------------------------------------------- -decode_z_tagged(Tag,B,Bs,Literals) when (B band 16#08) =:= 0 -> +decode_z_tagged(Tag,B,Bs,Literals,Types) when (B band 16#08) =:= 0 -> N = B bsr 4, case N of 0 -> % float @@ -589,10 +613,12 @@ decode_z_tagged(Tag,B,Bs,Literals) when (B band 16#08) =:= 0 -> Literal -> {{literal,Literal},RestBs} end; + 5 -> % type-tagged register + decode_tr(Bs, Types); _ -> ?exit({decode_z_tagged,{invalid_extended_tag,N}}) end; -decode_z_tagged(_,B,_,_) -> +decode_z_tagged(_,B,_,_,_) -> ?exit({decode_z_tagged,{weird_value,B}}). -spec decode_float([byte(),...]) -> {{'float', float()}, [byte()]}. @@ -602,6 +628,12 @@ decode_float(Bs) -> <<Float:64/float>> = list_to_binary(FL), {{float,Float},RestBs}. +-spec decode_tr([byte(),...], term()) -> {#tr{}, [byte()]}. +decode_tr(Bs, Types) -> + {Reg, RestBs0} = decode_arg(Bs), + {{u, TypeIdx}, RestBs} = decode_arg(RestBs0), + {#tr{r=Reg,t=gb_trees:get(TypeIdx, Types)}, RestBs}. + -spec decode_fr([byte(),...]) -> {{'fr', non_neg_integer()}, [byte()]}. decode_fr(Bs) -> @@ -655,13 +687,13 @@ build_arg([], N) -> %% Decodes a bunch of arguments and returns them in a list %%----------------------------------------------------------------------- -decode_n_args(N, Bs, Atoms, Literals) when N >= 0 -> - decode_n_args(N, [], Bs, Atoms, Literals). +decode_n_args(N, Bs, Atoms, Literals, Types) when N >= 0 -> + decode_n_args(N, [], Bs, Atoms, Literals, Types). -decode_n_args(N, Acc, Bs0, Atoms, Literals) when N > 0 -> - {A1,Bs} = decode_arg(Bs0, Atoms, Literals), - decode_n_args(N-1, [A1|Acc], Bs, Atoms, Literals); -decode_n_args(0, Acc, Bs, _, _) -> +decode_n_args(N, Acc, Bs0, Atoms, Literals, Types) when N > 0 -> + {A1,Bs} = decode_arg(Bs0, Atoms, Literals, Types), + decode_n_args(N-1, [A1|Acc], Bs, Atoms, Literals, Types); +decode_n_args(0, Acc, Bs, _, _, _) -> {lists:reverse(Acc),Bs}. %%----------------------------------------------------------------------- @@ -1200,8 +1232,9 @@ resolve_inst({recv_marker_use,[Reg]},_,_,_) -> %% resolve_inst({bs_create_bin,Args},_,_,_) -> - io:format("~p\n", [Args]), {bs_create_bin,Args}; +resolve_inst({call_fun2,[Safe,{u,Arity},Func]},_,_,_) -> + {call_fun2,Safe,Arity,Func}; %% %% Catches instructions that are not yet handled. @@ -1214,6 +1247,7 @@ resolve_inst(X,_,_,_) -> ?exit({resolve_inst,X}). resolve_args(Args) -> [resolve_arg(A) || A <- Args]. +resolve_arg(#tr{r=Reg} = Arg) -> _ = resolve_arg(Reg), Arg; resolve_arg({x,N} = Arg) when is_integer(N), N >= 0 -> Arg; resolve_arg({y,N} = Arg) when is_integer(N), N >= 0 -> Arg; resolve_arg({fr,N} = Arg) when is_integer(N), N >= 0 -> Arg; diff --git a/lib/compiler/src/beam_jump.erl b/lib/compiler/src/beam_jump.erl index 0c4ad8fae5..25f7a1fb88 100644 --- a/lib/compiler/src/beam_jump.erl +++ b/lib/compiler/src/beam_jump.erl @@ -135,7 +135,7 @@ -type instruction() :: beam_utils:instruction(). --include("beam_types.hrl"). +-include("beam_asm.hrl"). -spec module(beam_utils:module_code(), [compile:option()]) -> {'ok',beam_utils:module_code()}. @@ -184,9 +184,10 @@ eliminate_moves([{select,select_val,Reg,{f,Fail},List}=I|Is], D0, Acc) -> D1 = add_unsafe_label(Fail, D0), D = update_value_dict(List, Reg, D1), eliminate_moves(Is, D, [I|Acc]); -eliminate_moves([{test,is_eq_exact,_,[Reg,Val]}=I, +eliminate_moves([{test,is_eq_exact,_,[Reg0,Val]}=I, {block,BlkIs0}|Is], D0, Acc) -> D = update_unsafe_labels(I, D0), + Reg = unpack_typed_reg(Reg0), RegVal = {Reg,Val}, BlkIs = eliminate_moves_blk(BlkIs0, RegVal), eliminate_moves([{block,BlkIs}|Is], D, [I|Acc]); @@ -269,7 +270,8 @@ value_to_literal(F) when is_float(F) -> {float,F}; value_to_literal(I) when is_integer(I) -> {integer,I}; value_to_literal(Other) -> {literal,Other}. -update_value_dict([Lit,{f,Lbl}|T], Reg, D0) -> +update_value_dict([Lit,{f,Lbl}|T], Reg0, D0) -> + Reg = unpack_typed_reg(Reg0), D = case D0 of #{Lbl:=unsafe} -> D0; #{Lbl:={Reg,Lit}} -> D0; @@ -279,6 +281,9 @@ update_value_dict([Lit,{f,Lbl}|T], Reg, D0) -> update_value_dict(T, Reg, D); update_value_dict([], _, D) -> D. +unpack_typed_reg(#tr{r=Reg}) -> Reg; +unpack_typed_reg(Reg) -> Reg. + add_unsafe_label(L, D) -> D#{L=>unsafe}. diff --git a/lib/compiler/src/beam_ssa_codegen.erl b/lib/compiler/src/beam_ssa_codegen.erl index 8454d02944..3ed13064b8 100644 --- a/lib/compiler/src/beam_ssa_codegen.erl +++ b/lib/compiler/src/beam_ssa_codegen.erl @@ -27,6 +27,7 @@ -export_type([ssa_register/0]). -include("beam_ssa.hrl"). +-include("beam_asm.hrl"). -import(lists, [foldl/3,keymember/3,keysort/2,map/2,mapfoldl/3, member/2,reverse/1,reverse/2,sort/1, @@ -104,11 +105,7 @@ module(#b_module{name=Mod,exports=Es,attributes=Attrs,body=Fs}, _Opts) -> -type sw_list_item() :: {b_literal(),ssa_label()}. --type reg_num() :: beam_asm:reg_num(). --type xreg() :: {'x',reg_num()}. --type yreg() :: {'y',reg_num()}. - --type ssa_register() :: xreg() | yreg() | {'fr',reg_num()} | {'z',reg_num()}. +-type ssa_register() :: xreg() | yreg() | freg() | zreg(). functions(Forms, AtomMod) -> mapfoldl(fun (F, St) -> function(F, AtomMod, St) end, @@ -1041,7 +1038,8 @@ cg_block([#cg_set{op=new_try_tag,dst=Tag,args=Args}], {Tag,Fail0}, St) -> {[{Kind,Reg,Fail}],St}; cg_block([#cg_set{anno=Anno,op={bif,Name},dst=Dst0,args=Args0}=I, #cg_set{op=succeeded,dst=Bool}], {Bool,Fail0}, St) -> - [Dst|Args] = beam_args([Dst0|Args0], St), + Args = typed_args(Args0, Anno, St), + Dst = beam_arg(Dst0, St), Line0 = call_line(body, {extfunc,erlang,Name,length(Args)}, Anno), Fail = bif_fail(Fail0), Line = case Fail of @@ -1072,9 +1070,10 @@ cg_block([#cg_set{op={bif,tuple_size},dst=Arity0,args=[Tuple0]}, {Is,St} = cg_block([Eq], Context, St0), {[TupleSize|Is],St} end; -cg_block([#cg_set{op={bif,Name},dst=Dst0,args=Args0}]=Is0, {Dst0,Fail}, St0) -> - [Dst|Args] = beam_args([Dst0|Args0], St0), - case Dst of +cg_block([#cg_set{anno=Anno,op={bif,Name},dst=Dst0,args=Args0}]=Is0, + {Dst0,Fail}, St0) -> + Args = typed_args(Args0, Anno, St0), + case beam_arg(Dst0, St0) of {z,_} -> %% The result of the BIF call will only be used once. Convert to %% a test instruction. @@ -1089,7 +1088,8 @@ cg_block([#cg_set{op={bif,Name},dst=Dst0,args=Args0}]=Is0, {Dst0,Fail}, St0) -> end; cg_block([#cg_set{anno=Anno,op={bif,Name},dst=Dst0,args=Args0}=I|T], Context, St0) -> - [Dst|Args] = beam_args([Dst0|Args0], St0), + Args = typed_args(Args0, Anno, St0), + Dst = beam_arg(Dst0, St0), {Is0,St} = cg_block(T, Context, St0), case is_gc_bif(Name, Args) of true -> @@ -1187,7 +1187,7 @@ cg_block([#cg_set{op=is_tagged_tuple,anno=Anno,dst=Bool,args=Args0}], {Bool,Fail {[{test,is_tuple,ensure_label(Fail, St),[Src]}, {test,test_arity,ensure_label(Fail, St),[Src,Arity]}],St}; #{} -> - [Src,{integer,Arity},Tag] = beam_args(Args0, St), + [Src,{integer,Arity},Tag] = typed_args(Args0, Anno, St), {[{test,is_tagged_tuple,ensure_label(Fail, St),[Src,Arity,Tag]}],St} end; cg_block([#cg_set{op=is_nonempty_list,dst=Bool,args=Args0}], {Bool,Fail}, St) -> @@ -1568,13 +1568,20 @@ cg_call(#cg_set{anno=Anno0,op=call,dst=Dst0,args=[#b_remote{}=Func0|Args0]}, [line(Anno0)] ++ Apply, {Is,St} end; -cg_call(#cg_set{anno=Anno,op=call,dst=Dst0,args=Args0}, +cg_call(#cg_set{anno=Anno,op=call,dst=Dst0,args=[Func | Args0]}, Where, Context, St) -> - [Dst,Func|Args] = beam_args([Dst0|Args0], St), Line = call_line(Where, Func, Anno), - Arity = length(Args), - Call = build_call(call_fun, Arity, Func, Context, Dst), - Is = setup_args(Args++[Func], Anno, Context, St) ++ Line ++ Call, + Args = beam_args(Args0 ++ [Func], St), + + Arity = length(Args0), + Dst = beam_arg(Dst0, St), + + %% Note that we only inspect the (possible) type of the fun while building + %% the call, we don't want the arguments to be typed. + [TypedFunc] = typed_args([Func], Anno, St), + Call = build_call(call_fun, Arity, TypedFunc, Context, Dst), + + Is = setup_args(Args, Anno, Context, St) ++ Line ++ Call, case Anno of #{ result_type := Type } -> Info = {var_info, Dst, [{type,Type}]}, @@ -1619,6 +1626,24 @@ build_stk([V|Vs], TmpReg, Tail) -> build_stk([], _TmpReg, nil) -> [{move,nil,{x,1}}]. +build_call(call_fun, Arity, #tr{}=Func0, none, Dst) -> + %% Func0 was the source register prior to copying arguments, and has been + %% moved to {x, Arity}. Update it to match. + Func = Func0#tr{r={x,Arity}}, + Safe = is_fun_call_safe(Arity, Func), + [{call_fun2,{atom,Safe},Arity,Func}|copy({x,0}, Dst)]; +build_call(call_fun, Arity, #tr{}=Func0, {return,Dst,N}, Dst) + when is_integer(N) -> + Func = Func0#tr{r={x,Arity}}, + Safe = is_fun_call_safe(Arity, Func), + [{call_fun2,{atom,Safe},Arity,Func},{deallocate,N},return]; +build_call(call_fun, Arity, #tr{}=Func0, {return,Val,N}, _Dst) + when is_integer(N) -> + Func = Func0#tr{r={x,Arity}}, + Safe = is_fun_call_safe(Arity, Func), + [{call_fun2,{atom,Safe},Arity,Func}, + {move,Val,{x,0}}, + {deallocate,N},return]; build_call(call_fun, Arity, _Func, none, Dst) -> [{call_fun,Arity}|copy({x,0}, Dst)]; build_call(call_fun, Arity, _Func, {return,Dst,N}, Dst) when is_integer(N) -> @@ -1650,6 +1675,12 @@ build_call(I, Arity, Func, {return,Val,N}, _Dst) when is_integer(N) -> build_call(I, Arity, Func, none, Dst) -> [{I,Arity,Func}|copy({x,0}, Dst)]. +%% Returns whether a call of the given fun is guaranteed to succeed. +is_fun_call_safe(Arity, #tr{t=#t_fun{arity=Arity}}) -> + true; +is_fun_call_safe(_Arity, _Func) -> + false. + build_apply(Arity, {return,Dst,N}, Dst) when is_integer(N) -> [{apply_last,Arity,N}]; build_apply(Arity, {return,Val,N}, _Dst) when is_integer(N) -> @@ -1819,6 +1850,9 @@ cg_extract([#cg_set{op=extract,dst=Dst0,args=Args0}|Is0], Agg, St) -> cg_extract(Is, _, _) -> {[],Is}. +-spec copy(Src, Dst) -> [{move,Src,Dst}] when + Src :: beam_reg() | beam_literal(), + Dst :: beam_reg(). copy(Src, Src) -> []; copy(Src, Dst) -> [{move,Src,Dst}]. @@ -2058,6 +2092,20 @@ get_register(V, Regs) -> false -> maps:get(V, Regs) end. +typed_args(As, Anno, St) -> + typed_args_1(As, Anno, St, 0). + +typed_args_1([Arg | Args], Anno, St, Index) -> + case Anno of + #{ arg_types := #{ Index := Type } } -> + Typed = #tr{r=beam_arg(Arg, St),t=Type}, + [Typed | typed_args_1(Args, Anno, St, Index + 1)]; + #{} -> + [beam_arg(Arg, St) | typed_args_1(Args, Anno, St, Index + 1)] + end; +typed_args_1([], _Anno, _St, _Index) -> + []. + beam_args(As, St) -> [beam_arg(A, St) || A <- As]. diff --git a/lib/compiler/src/beam_ssa_pp.erl b/lib/compiler/src/beam_ssa_pp.erl index 601e2be5df..769afb17a9 100644 --- a/lib/compiler/src/beam_ssa_pp.erl +++ b/lib/compiler/src/beam_ssa_pp.erl @@ -116,7 +116,7 @@ format_param_info([], _Break) -> format_type(T, Break) -> %% Gross hack, but it's short and simple. - Indented = lists:flatten(format_type(T)), + Indented = unicode:characters_to_list(format_type(T)), string:replace(Indented, [$\n], Break, all). format_blocks(Ls, Blocks, Anno) -> @@ -143,7 +143,7 @@ format_instrs([], _FuncAnno, _First) -> format_instr(#b_set{anno=Anno,op=Op,dst=Dst,args=Args}, FuncAnno, First) -> - AnnoStr = format_anno(Anno), + AnnoStr = format_instr_anno(Anno, FuncAnno, Args), LiveIntervalStr = format_live_interval(Dst, FuncAnno), [if First -> @@ -242,20 +242,37 @@ format_switch_list(List, FuncAnno) -> format_label(L) -> io_lib:format("^~w", [L]). -format_anno(#{n:=_}=Anno) -> - format_anno(maps:remove(n, Anno)); -format_anno(#{location:={File,Line}}=Anno0) -> +format_instr_anno(#{n:=_}=Anno, FuncAnno, Args) -> + format_instr_anno(maps:remove(n, Anno), FuncAnno, Args); +format_instr_anno(#{location:={File,Line}}=Anno0, FuncAnno, Args) -> Anno = maps:remove(location, Anno0), - [io_lib:format(" %% ~ts:~p\n", [File,Line])|format_anno(Anno)]; -format_anno(#{result_type:=T}=Anno0) -> + [io_lib:format(" %% ~ts:~p\n", [File,Line]) | + format_instr_anno(Anno, FuncAnno, Args)]; +format_instr_anno(#{result_type:=T}=Anno0, FuncAnno, Args) -> Anno = maps:remove(result_type, Anno0), Break = "\n %% ", [io_lib:format(" %% Result type:~s~s\n", - [Break, format_type(T, Break)]) | format_anno(Anno)]; -format_anno(Anno) -> - format_anno_1(Anno). + [Break, format_type(T, Break)]) | + format_instr_anno(Anno, FuncAnno, Args)]; +format_instr_anno(#{arg_types:=Ts}=Anno0, FuncAnno, Args) -> + Anno = maps:remove(arg_types, Anno0), -format_anno_1(Anno) -> + Break = "\n %% ", + + Iota = lists:seq(0, length(Args) - 1), + Formatted0 = [[format_arg(Arg, FuncAnno), " => ", + format_type(map_get(Idx, Ts), + Break)] + || {Idx, Arg} <- lists:zip(Iota, Args), is_map_key(Idx, Ts)], + Formatted = lists:join(Break, Formatted0), + + [io_lib:format(" %% Argument types:~s~ts\n", + [Break, unicode:characters_to_list(Formatted)]) | + format_instr_anno(Anno, FuncAnno, Args)]; +format_instr_anno(Anno, _FuncAnno, _Args) -> + format_instr_anno_1(Anno). + +format_instr_anno_1(Anno) -> case map_size(Anno) of 0 -> []; @@ -330,6 +347,12 @@ format_type(#t_tuple{elements=Es,exact=Ex,size=S}) -> ["{", string:join(format_tuple_elems(S, Ex, Es, 1), ", "), "}"]; +format_type(pid) -> + "pid()"; +format_type(port) -> + "pid()"; +format_type(reference) -> + "reference()"; format_type(none) -> "none()"; format_type(#t_union{atom=A,list=L,number=N,tuple_set=Ts,other=O}) -> diff --git a/lib/compiler/src/beam_ssa_pre_codegen.erl b/lib/compiler/src/beam_ssa_pre_codegen.erl index 2e804462ea..ba11360974 100644 --- a/lib/compiler/src/beam_ssa_pre_codegen.erl +++ b/lib/compiler/src/beam_ssa_pre_codegen.erl @@ -69,6 +69,7 @@ -export([module/2]). -include("beam_ssa.hrl"). +-include("beam_asm.hrl"). -import(lists, [all/2,any/2,append/1,duplicate/2, foldl/3,last/1,member/2,partition/2, @@ -90,9 +91,6 @@ functions([], _Ps) -> []. -type var_name() :: beam_ssa:var_name(). -type instr_number() :: pos_integer(). -type range() :: {instr_number(),instr_number()}. --type reg_num() :: beam_asm:reg_num(). --type xreg() :: {'x',reg_num()}. --type yreg() :: {'y',reg_num()}. -type ypool() :: {'y',beam_ssa:label()}. -type reservation() :: 'fr' | {'prefer',xreg()} | 'x' | {'x',xreg()} | ypool() | {yreg(),ypool()} | 'z'. diff --git a/lib/compiler/src/beam_ssa_type.erl b/lib/compiler/src/beam_ssa_type.erl index b1ffd46255..65acb3f813 100644 --- a/lib/compiler/src/beam_ssa_type.erl +++ b/lib/compiler/src/beam_ssa_type.erl @@ -534,7 +534,8 @@ opt_is([#b_set{op=call, dst=Dst}=I0 | Is], Ts0, Ds0, Ls, Fdb, Sub, Meta, Acc) -> Args = simplify_args(Args0, Ts0, Sub), - I1 = I0#b_set{args=Args}, + + I1 = opt_anno_types(I0#b_set{args=Args}, Ts0), [Fun | _] = Args, I = case normalized_type(Fun, Ts0) of @@ -560,7 +561,8 @@ opt_is([#b_set{op=MakeFun,args=Args0,dst=Dst}=I0|Is], opt_is(Is, Ts, Ds, Ls, Fdb, Sub0, Meta, [I|Acc]); opt_is([I0 | Is], Ts0, Ds0, Ls, Fdb, Sub0, Meta, Acc) -> case simplify(I0, Ts0, Ds0, Ls, Sub0) of - {#b_set{}=I, Ts, Ds} -> + {#b_set{}=I1, Ts, Ds} -> + I = opt_anno_types(I1, Ts), opt_is(Is, Ts, Ds, Ls, Fdb, Sub0, Meta, [I | Acc]); Sub when is_map(Sub) -> opt_is(Is, Ts0, Ds0, Ls, Fdb, Sub, Meta, Acc) @@ -568,6 +570,49 @@ opt_is([I0 | Is], Ts0, Ds0, Ls, Fdb, Sub0, Meta, Acc) -> opt_is([], Ts, Ds, _Ls, Fdb, Sub, _Meta, Acc) -> {reverse(Acc), Ts, Ds, Fdb, Sub}. +opt_anno_types(#b_set{op=Op,args=Args}=I, Ts) -> + case benefits_from_type_anno(Op, Args) of + true -> opt_anno_types_1(I, Args, Ts, 0, #{}); + false -> I + end. + +opt_anno_types_1(I, [#b_var{}=Var | Args], Ts, Index, Acc0) -> + case Ts of + #{ Var := Type } when Type =/= any -> + %% Note that we annotate arguments by their index instead of their + %% variable name, as they may be renamed by `beam_ssa_pre_codegen`. + Acc = Acc0#{ Index => Type }, + opt_anno_types_1(I, Args, Ts, Index + 1, Acc); + #{} -> + opt_anno_types_1(I, Args, Ts, Index + 1, Acc0) + end; +opt_anno_types_1(I, [_Arg | Args], Ts, Index, Acc) -> + opt_anno_types_1(I, Args, Ts, Index + 1, Acc); +opt_anno_types_1(#b_set{}=I, [], _Ts, _Index, Acc) when Acc =:= #{} -> + I; +opt_anno_types_1(#b_set{anno=Anno0}=I, [], _Ts, _Index, Acc) -> + case Anno0 of + #{ arg_types := Acc } -> + I; + #{} -> + Anno = Anno0#{ arg_types => Acc }, + I#b_set{anno=Anno} + end. + +%% Only add type annotations when we know we'll make good use of them. +benefits_from_type_anno({bif,'=:='}, _Args) -> + true; +benefits_from_type_anno({bif,'=/='}, _Args) -> + true; +benefits_from_type_anno({bif,Op}, Args) -> + not erl_internal:bool_op(Op, length(Args)); +benefits_from_type_anno(is_tagged_tuple, _Args) -> + true; +benefits_from_type_anno(call, [#b_var{} | _]) -> + true; +benefits_from_type_anno(_Op, _Args) -> + false. + opt_local_call(I0, Callee, Args, Dst, Ts, Fdb, Meta) -> ArgTypes = argument_types(Args, Ts), I = opt_local_return(I0, Callee, ArgTypes, Fdb), @@ -1395,6 +1440,9 @@ eval_type_test_bif(I, is_boolean, [Type]) -> end; eval_type_test_bif(I, is_float, [Type]) -> eval_type_test_bif_1(I, Type, #t_float{}); +eval_type_test_bif(I, is_function, [Type, #t_integer{elements={Arity,Arity}}]) + when Arity >= 0, Arity =< 255 -> + eval_type_test_bif_1(I, Type, #t_fun{arity=Arity}); eval_type_test_bif(I, is_function, [Type]) -> eval_type_test_bif_1(I, Type, #t_fun{}); eval_type_test_bif(I, is_integer, [Type]) -> @@ -1405,6 +1453,12 @@ eval_type_test_bif(I, is_map, [Type]) -> eval_type_test_bif_1(I, Type, #t_map{}); eval_type_test_bif(I, is_number, [Type]) -> eval_type_test_bif_1(I, Type, number); +eval_type_test_bif(I, is_pid, [Type]) -> + eval_type_test_bif_1(I, Type, pid); +eval_type_test_bif(I, is_port, [Type]) -> + eval_type_test_bif_1(I, Type, port); +eval_type_test_bif(I, is_reference, [Type]) -> + eval_type_test_bif_1(I, Type, reference); eval_type_test_bif(I, is_tuple, [Type]) -> eval_type_test_bif_1(I, Type, #t_tuple{}); eval_type_test_bif(I, Op, Types) -> @@ -2025,34 +2079,53 @@ infer_type(is_tagged_tuple, [#b_var{}=Src,#b_literal{val=Size}, infer_type(is_nonempty_list, [#b_var{}=Src], _Ts, _Ds) -> T = {Src,#t_cons{}}, {[T], [T]}; -infer_type({bif,is_atom}, [Arg], _Ts, _Ds) -> +infer_type({bif,is_atom}, [#b_var{}=Arg], _Ts, _Ds) -> T = {Arg, #t_atom{}}, {[T], [T]}; -infer_type({bif,is_binary}, [Arg], _Ts, _Ds) -> +infer_type({bif,is_binary}, [#b_var{}=Arg], _Ts, _Ds) -> T = {Arg, #t_bitstring{size_unit=8}}, {[T], [T]}; -infer_type({bif,is_bitstring}, [Arg], _Ts, _Ds) -> +infer_type({bif,is_bitstring}, [#b_var{}=Arg], _Ts, _Ds) -> T = {Arg, #t_bitstring{}}, {[T], [T]}; -infer_type({bif,is_boolean}, [Arg], _Ts, _Ds) -> +infer_type({bif,is_boolean}, [#b_var{}=Arg], _Ts, _Ds) -> T = {Arg, beam_types:make_boolean()}, {[T], [T]}; -infer_type({bif,is_float}, [Arg], _Ts, _Ds) -> +infer_type({bif,is_float}, [#b_var{}=Arg], _Ts, _Ds) -> T = {Arg, #t_float{}}, {[T], [T]}; -infer_type({bif,is_integer}, [Arg], _Ts, _Ds) -> +infer_type({bif,is_function}, [#b_var{}=Arg], _Ts, _Ds) -> + T = {Arg, #t_fun{}}, + {[T], [T]}; +infer_type({bif,is_function}, [#b_var{}=Arg, Arity0], _Ts, _Ds) -> + Arity = case Arity0 of + #b_literal{val=V} when is_integer(V), V >= 0, V =< 255 -> V; + _ -> any + end, + T = {Arg, #t_fun{arity=Arity}}, + {[T], [T]}; +infer_type({bif,is_integer}, [#b_var{}=Arg], _Ts, _Ds) -> T = {Arg, #t_integer{}}, {[T], [T]}; -infer_type({bif,is_list}, [Arg], _Ts, _Ds) -> +infer_type({bif,is_list}, [#b_var{}=Arg], _Ts, _Ds) -> T = {Arg, #t_list{}}, {[T], [T]}; -infer_type({bif,is_map}, [Arg], _Ts, _Ds) -> +infer_type({bif,is_map}, [#b_var{}=Arg], _Ts, _Ds) -> T = {Arg, #t_map{}}, {[T], [T]}; -infer_type({bif,is_number}, [Arg], _Ts, _Ds) -> +infer_type({bif,is_number}, [#b_var{}=Arg], _Ts, _Ds) -> T = {Arg, number}, {[T], [T]}; -infer_type({bif,is_tuple}, [Arg], _Ts, _Ds) -> +infer_type({bif,is_pid}, [#b_var{}=Arg], _Ts, _Ds) -> + T = {Arg, pid}, + {[T], [T]}; +infer_type({bif,is_port}, [#b_var{}=Arg], _Ts, _Ds) -> + T = {Arg, port}, + {[T], [T]}; +infer_type({bif,is_reference}, [#b_var{}=Arg], _Ts, _Ds) -> + T = {Arg, reference}, + {[T], [T]}; +infer_type({bif,is_tuple}, [#b_var{}=Arg], _Ts, _Ds) -> T = {Arg, #t_tuple{}}, {[T], [T]}; infer_type({bif,'=:='}, [#b_var{}=LHS,#b_var{}=RHS], Ts, _Ds) -> diff --git a/lib/compiler/src/beam_trim.erl b/lib/compiler/src/beam_trim.erl index a22753ddc4..ada608cc27 100644 --- a/lib/compiler/src/beam_trim.erl +++ b/lib/compiler/src/beam_trim.erl @@ -21,7 +21,9 @@ -module(beam_trim). -export([module/2]). --import(lists, [any/2,member/2,reverse/1,reverse/2,sort/1]). +-import(lists, [any/2,reverse/1,reverse/2,sort/1]). + +-include("beam_asm.hrl"). -record(st, {safe :: sets:set(beam_asm:label()) %Safe labels. @@ -210,10 +212,18 @@ expand_recipe({Layout,Trim,Moves}, FrameSize) -> end. create_map(Trim, []) -> - fun({y,Y}) when Y < Trim -> throw(not_possible); - ({y,Y}) -> {y,Y-Trim}; - ({frame_size,N}) -> N - Trim; - (Any) -> Any + fun({y,Y}) when Y < Trim -> + throw(not_possible); + ({y,Y}) -> + {y,Y-Trim}; + (#tr{r={y,Y}}) when Y < Trim -> + throw(not_possible); + (#tr{r={y,Y},t=Type}) -> + #tr{r={y,Y-Trim},t=Type}; + ({frame_size,N}) -> + N - Trim; + (Any) -> + Any end; create_map(Trim, Moves) -> Map0 = [{Src,Dst-Trim} || {move,{y,Src},{y,Dst}} <- Moves], @@ -225,12 +235,24 @@ create_map(Trim, Moves) -> #{} -> throw(not_possible) end; ({y,Y}) -> - case sets:is_element(Y, IllegalTargets) of - true -> throw(not_possible); - false -> {y,Y-Trim} - end; - ({frame_size,N}) -> N - Trim; - (Any) -> Any + case sets:is_element(Y, IllegalTargets) of + true -> throw(not_possible); + false -> {y,Y-Trim} + end; + (#tr{r={y,Y0},t=Type}) when Y0 < Trim -> + case Map of + #{Y0:=Y} -> #tr{r={y,Y},t=Type}; + #{} -> throw(not_possible) + end; + (#tr{r={y,Y},t=Type}) -> + case sets:is_element(Y, IllegalTargets) of + true -> throw(not_possible); + false -> #tr{r={y,Y-Trim},t=Type} + end; + ({frame_size,N}) -> + N - Trim; + (Any) -> + Any end. remap([{'%',Comment}=I|Is], Map, Acc) -> @@ -349,7 +371,8 @@ is_safe_label([{call_ext,_,{extfunc,M,F,A}}|_]) -> is_safe_label(_) -> false. is_safe_label_block([{set,Ds,Ss,_}|Is]) -> - IsYreg = fun({y,_}) -> true; + IsYreg = fun(#tr{r={y,_}}) -> true; + ({y,_}) -> true; (_) -> false end, %% This instruction is safe if the instruction @@ -486,14 +509,14 @@ is_not_used(Y, [{gc_bif,_,{f,_},_Live,Ss,Dst}|Is]) -> is_not_used_ss_dst(Y, Ss, Dst, Is); is_not_used(Y, [{get_map_elements,{f,_},S,{list,List}}|Is]) -> {Ss,Ds} = beam_utils:split_even(List), - case member(Y, [S|Ss]) of + case is_y_member(Y, [S|Ss]) of true -> false; false -> - member(Y, Ds) orelse is_not_used(Y, Is) + is_y_member(Y, Ds) orelse is_not_used(Y, Is) end; is_not_used(Y, [{init_yregs,{list,Yregs}}|Is]) -> - member(Y, Yregs) orelse is_not_used(Y, Is); + is_y_member(Y, Yregs) orelse is_not_used(Y, Is); is_not_used(Y, [{line,_}|Is]) -> is_not_used(Y, Is); is_not_used(Y, [{make_fun2,_,_,_,_}|Is]) -> @@ -507,16 +530,16 @@ is_not_used(Y, [{recv_marker_reserve,Dst}|Is]) -> is_not_used(Y, [{swap,Reg1,Reg2}|Is]) -> Y =/= Reg1 andalso Y =/= Reg2 andalso is_not_used(Y, Is); is_not_used(Y, [{test,_,_,Ss}|Is]) -> - not member(Y, Ss) andalso is_not_used(Y, Is); + not is_y_member(Y, Ss) andalso is_not_used(Y, Is); is_not_used(Y, [{test,_Op,{f,_},_Live,Ss,Dst}|Is]) -> is_not_used_ss_dst(Y, Ss, Dst, Is). is_not_used_block(Y, [{set,Ds,Ss,_}|Is]) -> - case member(Y, Ss) of + case is_y_member(Y, Ss) of true -> used; false -> - case member(Y, Ds) of + case is_y_member(Y, Ds) of true -> killed; false -> @@ -526,4 +549,13 @@ is_not_used_block(Y, [{set,Ds,Ss,_}|Is]) -> is_not_used_block(_Y, []) -> transparent. is_not_used_ss_dst(Y, Ss, Dst, Is) -> - not member(Y, Ss) andalso (Y =:= Dst orelse is_not_used(Y, Is)). + not is_y_member(Y, Ss) andalso (Y =:= Dst orelse is_not_used(Y, Is)). + +is_y_member({y,N0}, Ss) -> + any(fun(#tr{r={y,N}}) -> + N =:= N0; + ({y,N}) -> + N =:= N0; + (_) -> + false + end, Ss). diff --git a/lib/compiler/src/beam_types.erl b/lib/compiler/src/beam_types.erl index a6ec9fdd77..e49c245956 100644 --- a/lib/compiler/src/beam_types.erl +++ b/lib/compiler/src/beam_types.erl @@ -49,6 +49,8 @@ -export([limit_depth/1]). +-export([decode_ext/1, encode_ext/1]). + %% This is exported to help catch errors in property test generators and is not %% meant to be used outside of test suites. -export([verified_type/1]). @@ -615,6 +617,8 @@ make_integer(Min, Max) when is_integer(Min), is_integer(Max), Min =< Max -> limit_depth(Type) -> limit_depth(Type, ?MAX_TYPE_DEPTH). +-spec limit_depth(type(), non_neg_integer()) -> type(). + limit_depth(#t_cons{}=T, Depth) -> limit_depth_list(T, Depth); limit_depth(#t_list{}=T, Depth) -> @@ -1111,6 +1115,9 @@ verified_normal_type(#t_list{type=Type,terminator=Term}=T) -> verified_normal_type(#t_map{}=T) -> T; verified_normal_type(nil=T) -> T; verified_normal_type(number=T) -> T; +verified_normal_type(pid=T) -> T; +verified_normal_type(port=T) -> T; +verified_normal_type(reference=T) -> T; verified_normal_type(#t_tuple{size=Size,elements=Es}=T) -> %% All known elements must have a valid index and type (which may be a %% union). 'any' is prohibited since it's implicit and should never be @@ -1123,3 +1130,68 @@ verified_normal_type(#t_tuple{size=Size,elements=Es}=T) -> verified_type(Element) end, [], Es), T. + +%%% +%%% External type format +%%% +%%% This is a stripped-down version of our internal format, focusing solely on +%%% primary types and unions thereof. The idea is to help the JIT skip minor +%%% details inside instructions when we know they're pointless, such as +%%% checking whether the source of `is_tagged_tuple` is boxed, or reducing an +%%% equality check to a single machine compare instruction when we know that +%%% both arguments are immediates. +%%% + +-define(BEAM_TYPE_ATOM, (1 bsl 0)). +-define(BEAM_TYPE_BITSTRING, (1 bsl 1)). +-define(BEAM_TYPE_BS_MATCHSTATE, (1 bsl 2)). +-define(BEAM_TYPE_CONS, (1 bsl 3)). +-define(BEAM_TYPE_FLOAT, (1 bsl 4)). +-define(BEAM_TYPE_FUN, (1 bsl 5)). +-define(BEAM_TYPE_INTEGER, (1 bsl 6)). +-define(BEAM_TYPE_MAP, (1 bsl 7)). +-define(BEAM_TYPE_NIL, (1 bsl 8)). +-define(BEAM_TYPE_PID, (1 bsl 9)). +-define(BEAM_TYPE_PORT, (1 bsl 10)). +-define(BEAM_TYPE_REFERENCE, (1 bsl 11)). +-define(BEAM_TYPE_TUPLE, (1 bsl 12)). + +ext_type_mapping() -> + [{?BEAM_TYPE_ATOM, #t_atom{}}, + {?BEAM_TYPE_BITSTRING, #t_bitstring{}}, + {?BEAM_TYPE_BS_MATCHSTATE, #t_bs_matchable{}}, + {?BEAM_TYPE_CONS, #t_cons{}}, + {?BEAM_TYPE_FLOAT, #t_float{}}, + {?BEAM_TYPE_FUN, #t_fun{}}, + {?BEAM_TYPE_INTEGER, #t_integer{}}, + {?BEAM_TYPE_MAP, #t_map{}}, + {?BEAM_TYPE_NIL, nil}, + {?BEAM_TYPE_PID, pid}, + {?BEAM_TYPE_PORT, port}, + {?BEAM_TYPE_REFERENCE, reference}, + {?BEAM_TYPE_TUPLE, #t_tuple{}}]. + +-spec decode_ext(binary()) -> type(). +decode_ext(<<TypeBits:16/signed-big>>) -> + foldl(fun({Id, Type}, Acc) -> + decode_ext_bits(TypeBits, Id, Type, Acc) + end, none, ext_type_mapping()). + +decode_ext_bits(Input, TypeBit, Type, Acc) -> + case Input band TypeBit of + 0 -> Acc; + _ -> join(Type, Acc) + end. + +-spec encode_ext(type()) -> binary(). +encode_ext(Input) -> + TypeBits = foldl(fun({Id, Type}, Acc) -> + encode_ext_bits(Input, Id, Type, Acc) + end, 0, ext_type_mapping()), + <<TypeBits:16/signed-big>>. + +encode_ext_bits(Input, TypeBit, Type, Acc) -> + case meet(Input, Type) of + none -> Acc; + _ -> Acc bor TypeBit + end. diff --git a/lib/compiler/src/beam_types.hrl b/lib/compiler/src/beam_types.hrl index fb13936c48..48f5f1a049 100644 --- a/lib/compiler/src/beam_types.hrl +++ b/lib/compiler/src/beam_types.hrl @@ -18,6 +18,9 @@ %% %CopyrightEnd% %% +%% Type version, must be bumped whenever the external type format changes. +-define(BEAM_TYPES_VERSION, 0). + %% Common term types for passes operating on beam SSA and assembly. Helper %% functions for wrangling these can be found in beam_types.erl %% @@ -37,6 +40,9 @@ %% - #t_list{} Any list. %% -- #t_cons{} Cons (nonempty list). %% -- nil The empty list. +%% - pid +%% - port +%% - reference %% - #t_tuple{} Tuple. %% %% none No type (bottom element). @@ -131,6 +137,9 @@ #t_fun{} | #t_list{} | #t_cons{} | nil | #t_map{} | + pid | + port | + reference | #t_tuple{}. -type record_key() :: {Arity :: integer(), Tag :: normal_type() }. diff --git a/lib/compiler/src/beam_validator.erl b/lib/compiler/src/beam_validator.erl index 17327b3d5d..fe6bb2e7c9 100644 --- a/lib/compiler/src/beam_validator.erl +++ b/lib/compiler/src/beam_validator.erl @@ -19,7 +19,7 @@ -module(beam_validator). --include("beam_types.hrl"). +-include("beam_asm.hrl"). -define(UNICODE_MAX, (16#10FFFF)). @@ -134,16 +134,10 @@ validate_0([{function, Name, Arity, Entry, Code} | Fs], Module, Level, Ft) -> -record(value_ref, {id :: index()}). -record(value, {op :: term(), args :: [argument()], type :: validator_type()}). --type argument() :: #value_ref{} | literal(). +-type argument() :: #value_ref{} | beam_literal(). -type index() :: non_neg_integer(). --type literal() :: {atom, [] | atom()} | - {float, [] | float()} | - {integer, [] | integer()} | - {literal, term()} | - nil. - %% Register tags describe the state of the register rather than the value they %% contain (if any). %% @@ -274,7 +268,7 @@ validate_1(Is, MFA0, Entry, Level, Ft) -> validate_branches(MFA, Vst). extract_header([{func_info, {atom,Mod}, {atom,Name}, Arity}=I | Is], - MFA0, Entry, Offset, Acc) -> + MFA0, Entry, Offset, Acc) -> {_, Name, Arity} = MFA0, %Assertion. MFA = {Mod, Name, Arity}, @@ -425,7 +419,9 @@ vi({select_val,Src,{f,Fail},{list,Choices}}, Vst) -> assert_term(Src, Vst), assert_choices(Choices), validate_select_val(Fail, Choices, Src, Vst); -vi({select_tuple_arity,Tuple,{f,Fail},{list,Choices}}, Vst) -> +vi({select_tuple_arity,Tuple0,{f,Fail},{list,Choices}}, Vst) -> + Tuple = unpack_typed_arg(Tuple0), + assert_type(#t_tuple{}, Tuple, Vst), assert_arities(Choices), validate_select_tuple_arity(Fail, Choices, Tuple, Vst); @@ -441,6 +437,11 @@ vi({test,is_boolean,{f,Lbl},[Src]}, Vst) -> type_test(Lbl, beam_types:make_boolean(), Src, Vst); vi({test,is_float,{f,Lbl},[Src]}, Vst) -> type_test(Lbl, #t_float{}, Src, Vst); +vi({test,is_function,{f,Lbl},[Src]}, Vst) -> + type_test(Lbl, #t_fun{}, Src, Vst); +vi({test,is_function2,{f,Lbl},[Src,{integer,Arity}]}, Vst) + when Arity >= 0, Arity =< 255 -> + type_test(Lbl, #t_fun{arity=Arity}, Src, Vst); vi({test,is_tuple,{f,Lbl},[Src]}, Vst) -> type_test(Lbl, #t_tuple{}, Src, Vst); vi({test,is_integer,{f,Lbl},[Src]}, Vst) -> @@ -453,9 +454,10 @@ vi({test,is_list,{f,Lbl},[Src]}, Vst) -> type_test(Lbl, #t_list{}, Src, Vst); vi({test,is_map,{f,Lbl},[Src]}, Vst) -> type_test(Lbl, #t_map{}, Src, Vst); -vi({test,is_nil,{f,Lbl},[Src]}, Vst) -> +vi({test,is_nil,{f,Lbl},[Src0]}, Vst) -> %% is_nil is an exact check against the 'nil' value, and should not be %% treated as a simple type test. + Src = unpack_typed_arg(Src0), assert_term(Src, Vst), branch(Lbl, Vst, fun(FailVst) -> @@ -464,18 +466,28 @@ vi({test,is_nil,{f,Lbl},[Src]}, Vst) -> fun(SuccVst) -> update_eq_types(Src, nil, SuccVst) end); +vi({test,is_pid,{f,Lbl},[Src]}, Vst) -> + type_test(Lbl, pid, Src, Vst); +vi({test,is_port,{f,Lbl},[Src]}, Vst) -> + type_test(Lbl, port, Src, Vst); +vi({test,is_reference,{f,Lbl},[Src]}, Vst) -> + type_test(Lbl, reference, Src, Vst); vi({test,test_arity,{f,Lbl},[Tuple,Sz]}, Vst) when is_integer(Sz) -> assert_type(#t_tuple{}, Tuple, Vst), Type = #t_tuple{exact=true,size=Sz}, type_test(Lbl, Type, Tuple, Vst); -vi({test,is_tagged_tuple,{f,Lbl},[Src,Sz,Atom]}, Vst) -> +vi({test,is_tagged_tuple,{f,Lbl},[Src0,Sz,Atom]}, Vst) -> + Src = unpack_typed_arg(Src0), assert_term(Src, Vst), Es = #{ 1 => get_literal_type(Atom) }, Type = #t_tuple{exact=true,size=Sz,elements=Es}, type_test(Lbl, Type, Src, Vst); -vi({test,is_eq_exact,{f,Lbl},[Src,Val]=Ss}, Vst) -> +vi({test,is_eq_exact,{f,Lbl},[Src0,Val0]}, Vst) -> assert_no_exception(Lbl), - validate_src(Ss, Vst), + Src = unpack_typed_arg(Src0), + assert_term(Src, Vst), + Val = unpack_typed_arg(Val0), + assert_term(Val, Vst), branch(Lbl, Vst, fun(FailVst) -> update_ne_types(Src, Val, FailVst) @@ -483,9 +495,12 @@ vi({test,is_eq_exact,{f,Lbl},[Src,Val]=Ss}, Vst) -> fun(SuccVst) -> update_eq_types(Src, Val, SuccVst) end); -vi({test,is_ne_exact,{f,Lbl},[Src,Val]=Ss}, Vst) -> +vi({test,is_ne_exact,{f,Lbl},[Src0,Val0]}, Vst) -> assert_no_exception(Lbl), - validate_src(Ss, Vst), + Src = unpack_typed_arg(Src0), + assert_term(Src, Vst), + Val = unpack_typed_arg(Val0), + assert_term(Val, Vst), branch(Lbl, Vst, fun(FailVst) -> update_eq_types(Src, Val, FailVst) @@ -644,6 +659,18 @@ vi({call_last,Live,Func,N}, Vst) -> validate_tail_call(N, Func, Live, Vst); vi({call_ext_last,Live,Func,N}, Vst) -> validate_tail_call(N, Func, Live, Vst); +vi({call_fun2,{atom,Safe},Live,Fun0}, Vst) -> + Fun = unpack_typed_arg(Fun0), + assert_term(Fun, Vst), + + true = is_boolean(Safe), %Assertion. + + branch(?EXCEPTION_LABEL, Vst, + fun(SuccVst0) -> + SuccVst = update_type(fun meet/2, #t_fun{arity=Live}, + Fun, SuccVst0), + validate_body_call('fun', Live, SuccVst) + end); vi({call_fun,Live}, Vst) -> Fun = {x,Live}, assert_term(Fun, Vst), @@ -684,7 +711,10 @@ vi(return, Vst) -> %% BIF calls %% -vi({bif,Op,{f,Fail},Ss,Dst}, Vst0) -> +vi({bif,Op,{f,Fail},Ss0,Dst0}, Vst0) -> + Ss = [unpack_typed_arg(Arg) || Arg <- Ss0], + Dst = unpack_typed_arg(Dst0), + case is_float_arith_bif(Op, Ss) of true -> ?EXCEPTION_LABEL = Fail, %Assertion. @@ -693,7 +723,10 @@ vi({bif,Op,{f,Fail},Ss,Dst}, Vst0) -> validate_src(Ss, Vst0), validate_bif(bif, Op, Fail, Ss, Dst, Vst0, Vst0) end; -vi({gc_bif,Op,{f,Fail},Live,Ss,Dst}, Vst0) -> +vi({gc_bif,Op,{f,Fail},Live,Ss0,Dst0}, Vst0) -> + Ss = [unpack_typed_arg(Arg) || Arg <- Ss0], + Dst = unpack_typed_arg(Dst0), + validate_src(Ss, Vst0), verify_live(Live, Vst0), verify_y_init(Vst0), @@ -918,9 +951,9 @@ vi({test,bs_get_utf16=Op,{f,Fail},Live,[Ctx,_],Dst}, Vst) -> vi({test,bs_get_utf32=Op,{f,Fail},Live,[Ctx,_],Dst}, Vst) -> Type = beam_types:make_integer(0, ?UNICODE_MAX), validate_bs_get(Op, Fail, Ctx, Live, 32, Type, Dst, Vst); -vi({test,_Op,{f,Lbl},Src}, Vst) -> - %% is_pid, is_reference, et cetera. - validate_src(Src, Vst), +vi({test,_Op,{f,Lbl},Ss}, Vst) -> + %% is_lt, is_gt, et cetera. + validate_src([unpack_typed_arg(Arg) || Arg <- Ss], Vst), branch(Lbl, Vst); %% @@ -1480,7 +1513,9 @@ validate_bs_start_match({f,Fail}, Live, Src, Dst, Vst) -> %% %% Common code for validating bs_get* instructions. %% -validate_bs_get(Op, Fail, Ctx, Live, Stride, Type, Dst, Vst) -> +validate_bs_get(Op, Fail, Ctx0, Live, Stride, Type, Dst, Vst) -> + Ctx = unpack_typed_arg(Ctx0), + assert_no_exception(Fail), assert_type(#t_bs_context{}, Ctx, Vst), @@ -1494,7 +1529,9 @@ validate_bs_get(Op, Fail, Ctx, Live, Stride, Type, Dst, Vst) -> extract_term(Type, Op, [Ctx], Dst, SuccVst, SuccVst0) end). -validate_bs_get_all(Op, Fail, Ctx, Live, Stride, Type, Dst, Vst) -> +validate_bs_get_all(Op, Fail, Ctx0, Live, Stride, Type, Dst, Vst) -> + Ctx = unpack_typed_arg(Ctx0), + assert_no_exception(Fail), assert_type(#t_bs_context{}, Ctx, Vst), @@ -1593,7 +1630,9 @@ invalidate_current_ms_position(Ctx, #vst{current=St0}=Vst) -> %% %% Common code for is_$type instructions. %% -type_test(Fail, Type, Reg, Vst) -> +type_test(Fail, Type, Reg0, Vst) -> + Reg = unpack_typed_arg(Reg0), + assert_term(Reg, Vst), assert_no_exception(Fail), branch(Fail, Vst, @@ -1801,10 +1840,10 @@ schedule_out(Live, Vst0) when is_integer(Live) -> prune_x_regs(Live, #vst{current=St0}=Vst) when is_integer(Live) -> #st{fragile=Fragile0,xs=Xs0} = St0, Fragile = sets:filter(fun({x,X}) -> - X < Live; - ({y,_}) -> - true - end, Fragile0), + X < Live; + ({y,_}) -> + true + end, Fragile0), Xs = maps:filter(fun({x,X}, _) -> X < Live end, Xs0), @@ -1997,6 +2036,12 @@ infer_types_1(#value{op={bif,is_bitstring},args=[Src]}, Val, Op, Vst) -> infer_type_test_bif(#t_bitstring{}, Src, Val, Op, Vst); infer_types_1(#value{op={bif,is_float},args=[Src]}, Val, Op, Vst) -> infer_type_test_bif(#t_float{}, Src, Val, Op, Vst); +infer_types_1(#value{op={bif,is_function},args=[Src,{integer,Arity}]}, + Val, Op, Vst) + when Arity >= 0, Arity =< 255 -> + infer_type_test_bif(#t_fun{arity=Arity}, Src, Val, Op, Vst); +infer_types_1(#value{op={bif,is_function},args=[Src]}, Val, Op, Vst) -> + infer_type_test_bif(#t_fun{}, Src, Val, Op, Vst); infer_types_1(#value{op={bif,is_integer},args=[Src]}, Val, Op, Vst) -> infer_type_test_bif(#t_integer{}, Src, Val, Op, Vst); infer_types_1(#value{op={bif,is_list},args=[Src]}, Val, Op, Vst) -> @@ -2005,6 +2050,12 @@ infer_types_1(#value{op={bif,is_map},args=[Src]}, Val, Op, Vst) -> infer_type_test_bif(#t_map{}, Src, Val, Op, Vst); infer_types_1(#value{op={bif,is_number},args=[Src]}, Val, Op, Vst) -> infer_type_test_bif(number, Src, Val, Op, Vst); +infer_types_1(#value{op={bif,is_pid},args=[Src]}, Val, Op, Vst) -> + infer_type_test_bif(pid, Src, Val, Op, Vst); +infer_types_1(#value{op={bif,is_port},args=[Src]}, Val, Op, Vst) -> + infer_type_test_bif(port, Src, Val, Op, Vst); +infer_types_1(#value{op={bif,is_reference},args=[Src]}, Val, Op, Vst) -> + infer_type_test_bif(reference, Src, Val, Op, Vst); infer_types_1(#value{op={bif,is_tuple},args=[Src]}, Val, Op, Vst) -> infer_type_test_bif(#t_tuple{}, Src, Val, Op, Vst); infer_types_1(#value{op={bif,tuple_size}, args=[Tuple]}, @@ -2407,6 +2458,11 @@ validate_src(Ss, Vst) when is_list(Ss) -> _ = [assert_term(S, Vst) || S <- Ss], ok. +unpack_typed_arg(#tr{r=Reg}) -> + Reg; +unpack_typed_arg(Arg) -> + Arg. + %% get_term_type(Src, ValidatorState) -> Type %% Get the type of the source Src. The returned type Type will be %% a standard Erlang type (no catch/try tags or match contexts). diff --git a/lib/compiler/src/compile.erl b/lib/compiler/src/compile.erl index f5f61cbd96..79086d522f 100644 --- a/lib/compiler/src/compile.erl +++ b/lib/compiler/src/compile.erl @@ -274,9 +274,10 @@ expand_opt(r22, Os) -> no_swap | expand_opt(no_bsm4, Os)]); expand_opt(r23, Os) -> expand_opt(no_make_fun3, [no_bs_create_bin, no_ssa_opt_float, - no_recv_opt, no_init_yregs | Os]); + no_recv_opt, no_init_yregs | + expand_opt(r24, Os)]); expand_opt(r24, Os) -> - [no_bs_create_bin | Os]; + expand_opt(no_type_opt, [no_bs_create_bin | Os]); expand_opt(no_make_fun3, Os) -> [no_make_fun3, no_fun_opt | Os]; expand_opt({debug_info_key,_}=O, Os) -> diff --git a/lib/compiler/src/genop.tab b/lib/compiler/src/genop.tab index a4b10acff2..68935b1980 100755 --- a/lib/compiler/src/genop.tab +++ b/lib/compiler/src/genop.tab @@ -653,3 +653,8 @@ BEAM_FORMAT_NUMBER=0 ## @spec bs_create_bin Fail Alloc Live Unit Dst OpList ## @doc Builda a new binary using the binary syntax. 177: bs_create_bin/6 + +## @spec call_fun2 Safe Arity Func +## @doc Calls the fun Func with arity Arity. Assume arguments in registers x(0) +## to x(Arity-1). The call will never fail when Safe is 'true'. +178: call_fun2/3 diff --git a/lib/compiler/test/compile_SUITE.erl b/lib/compiler/test/compile_SUITE.erl index efee6b9674..0ece9fe7b5 100644 --- a/lib/compiler/test/compile_SUITE.erl +++ b/lib/compiler/test/compile_SUITE.erl @@ -1575,42 +1575,45 @@ bc_options(Config) -> L = [{101, small_float, [no_shared_fun_wrappers,no_line_info]}, {125, small_float, [no_shared_fun_wrappers, no_line_info, - no_ssa_opt_float]}, + no_ssa_opt_float, + no_type_opt]}, {153, small_float, [no_shared_fun_wrappers]}, - {164, small_maps, [no_init_yregs,no_shared_fun_wrappers]}, + {164, small_maps, [no_init_yregs,no_shared_fun_wrappers,no_type_opt]}, {164, small_maps, [r22]}, {164, big, [r22]}, {164, funs, [r22]}, {164, funs, [no_init_yregs,no_shared_fun_wrappers, no_ssa_opt_record, no_line_info,no_stack_trimming, - no_make_fun3]}, + no_make_fun3,no_type_opt]}, {168, small, [r22]}, + {168, small, [no_init_yregs,no_shared_fun_wrappers, + no_ssa_opt_record, + no_ssa_opt_float,no_line_info,no_type_opt]}, + {169, big, [no_init_yregs,no_shared_fun_wrappers, no_ssa_opt_record, no_line_info,no_stack_trimming, - no_make_fun3]}, + no_make_fun3,no_type_opt]}, {169, big, [r23]}, - {169, small_maps, [no_init_yregs]}, - - {170, small, [no_init_yregs,no_shared_fun_wrappers, - no_ssa_opt_record, - no_ssa_opt_float,no_line_info]}, + {169, small_maps, [no_init_yregs,no_type_opt]}, {171, big, [no_init_yregs,no_shared_fun_wrappers, no_ssa_opt_record, - no_ssa_opt_float,no_line_info]}, + no_ssa_opt_float,no_line_info, + no_type_opt]}, {171, funs, [no_init_yregs,no_shared_fun_wrappers, no_ssa_opt_record, - no_ssa_opt_float,no_line_info]}, + no_ssa_opt_float,no_line_info, + no_type_opt]}, - {172, funs, []}, - {172, big, []} + {178, funs, []}, + {178, big, []} ], Test = fun({Expected,Mod,Options}) -> diff --git a/lib/debugger/test/fun_SUITE.erl b/lib/debugger/test/fun_SUITE.erl index 7eb53e4ce4..962c54f22a 100644 --- a/lib/debugger/test/fun_SUITE.erl +++ b/lib/debugger/test/fun_SUITE.erl @@ -289,7 +289,7 @@ eep37(Config) when is_list(Config) -> 10 = Add(9), 50 = UnusedName(8), [1,1,2,6,24,120] = lists:map(F, lists:seq(0, 5)), - {'EXIT',{{badarity,_},_}} = (catch lists:map(fun G() -> G() end, [1])), + {'EXIT',{function_clause,_}} = (catch lists:map(fun G() -> G() end, [1])), {'EXIT',{{badarity,_},_}} = (catch F()), ok. diff --git a/lib/kernel/test/code_SUITE.erl b/lib/kernel/test/code_SUITE.erl index 740f05ac94..5227346217 100644 --- a/lib/kernel/test/code_SUITE.erl +++ b/lib/kernel/test/code_SUITE.erl @@ -869,13 +869,15 @@ check_funs({'$M_EXPR','$F_EXPR',_}, [{code_server,do_mod_call,4}, {code_server,handle_call,3}|_]) -> 0; check_funs({'$M_EXPR','$F_EXPR',_}, - [{lists,flatmap,2}, + [{lists,flatmap_1,2}, + {lists,flatmap,2}, {lists,concat,1}, {code_server,load_abs,4}, {code_server,handle_call,3}, {code_server,loop,1}|_]) -> 0; check_funs({'$M_EXPR','$F_EXPR',_}, - [{lists,foreach,2}, + [{lists,foreach_1,2}, + {lists,foreach,2}, {code_server,stick_dir,3}, {code_server,handle_call,3}, {code_server,loop,1}|_]) -> 0; @@ -892,6 +894,19 @@ check_funs({'$M_EXPR','$F_EXPR',1}, {code_server,init,3}, {code_server,start_link,1}]) -> 0; check_funs({'$M_EXPR','$F_EXPR',1}, + [{lists,all_1,2}, + {lists,all,2}, + {code_server,is_numstr,1}, + {code_server,is_vsn,1}, + {code_server,vsn_to_num,1}, + {code_server,create_bundle,2}, + {code_server,choose_bundles,1}, + {code_server,make_path,2}, + {code_server,get_user_lib_dirs_1,1}, + {code_server,get_user_lib_dirs,0}, + {code_server,init,3}, + {code_server,start_link,1}]) -> 0; +check_funs({'$M_EXPR','$F_EXPR',1}, [{lists,filter,2}, {code_server,try_archive_subdirs,3}|_]) -> 0; check_funs({'$M_EXPR','$F_EXPR',_}, diff --git a/lib/stdlib/src/gb_sets.erl b/lib/stdlib/src/gb_sets.erl index 6d6f7d40ac..8dda0d4ee0 100644 --- a/lib/stdlib/src/gb_sets.erl +++ b/lib/stdlib/src/gb_sets.erl @@ -871,7 +871,7 @@ is_set(_) -> false. Set1 :: set(Element), Set2 :: set(Element). -filter(F, S) -> +filter(F, S) when is_function(F, 1) -> from_ordset([X || X <- to_list(S), F(X)]). -spec fold(Function, Acc0, Set) -> Acc1 when diff --git a/lib/stdlib/src/lists.erl b/lib/stdlib/src/lists.erl index b82732e0ca..63d55f5630 100644 --- a/lib/stdlib/src/lists.erl +++ b/lib/stdlib/src/lists.erl @@ -1231,24 +1231,46 @@ rumerge(T1, [H2 | T2]) -> List :: [T], T :: term(). -all(Pred, [Hd|Tail]) -> +all(Pred, List) when is_function(Pred, 1) -> + case List of + [Hd | Tail] -> + case Pred(Hd) of + true -> all_1(Pred, Tail); + false -> false + end; + [] -> true + end. + +all_1(Pred, [Hd | Tail]) -> case Pred(Hd) of - true -> all(Pred, Tail); - false -> false + true -> all_1(Pred, Tail); + false -> false end; -all(Pred, []) when is_function(Pred, 1) -> true. +all_1(_Pred, []) -> + true. -spec any(Pred, List) -> boolean() when Pred :: fun((Elem :: T) -> boolean()), List :: [T], T :: term(). -any(Pred, [Hd|Tail]) -> +any(Pred, List) when is_function(Pred, 1) -> + case List of + [Hd | Tail] -> + case Pred(Hd) of + true -> true; + false -> any_1(Pred, Tail) + end; + [] -> false + end. + +any_1(Pred, [Hd | Tail]) -> case Pred(Hd) of - true -> true; - false -> any(Pred, Tail) + true -> true; + false -> any_1(Pred, Tail) end; -any(Pred, []) when is_function(Pred, 1) -> false. +any_1(_Pred, []) -> + false. -spec map(Fun, List1) -> List2 when Fun :: fun((A) -> B), @@ -1257,9 +1279,16 @@ any(Pred, []) when is_function(Pred, 1) -> false. A :: term(), B :: term(). -map(F, [H|T]) -> - [F(H)|map(F, T)]; -map(F, []) when is_function(F, 1) -> []. +map(F, List) when is_function(F, 1) -> + case List of + [Hd | Tail] -> [F(Hd) | map_1(F, Tail)]; + [] -> [] + end. + +map_1(F, [Hd | Tail]) -> + [F(Hd) | map_1(F, Tail)]; +map_1(_F, []) -> + []. -spec flatmap(Fun, List1) -> List2 when Fun :: fun((A) -> [B]), @@ -1268,9 +1297,13 @@ map(F, []) when is_function(F, 1) -> []. A :: term(), B :: term(). -flatmap(F, [Hd|Tail]) -> - F(Hd) ++ flatmap(F, Tail); -flatmap(F, []) when is_function(F, 1) -> []. +flatmap(F, List) when is_function(F, 1) -> + flatmap_1(F, List). + +flatmap_1(F, [Hd | Tail]) -> + F(Hd) ++ flatmap_1(F, Tail); +flatmap_1(_F, []) -> + []. -spec foldl(Fun, Acc0, List) -> Acc1 when Fun :: fun((Elem :: T, AccIn) -> AccOut), @@ -1281,9 +1314,16 @@ flatmap(F, []) when is_function(F, 1) -> []. List :: [T], T :: term(). -foldl(F, Accu, [Hd|Tail]) -> - foldl(F, F(Hd, Accu), Tail); -foldl(F, Accu, []) when is_function(F, 2) -> Accu. +foldl(F, Accu, List) when is_function(F, 2) -> + case List of + [Hd | Tail] -> foldl_1(F, F(Hd, Accu), Tail); + [] -> Accu + end. + +foldl_1(F, Accu, [Hd | Tail]) -> + foldl_1(F, F(Hd, Accu), Tail); +foldl_1(_F, Accu, []) -> + Accu. -spec foldr(Fun, Acc0, List) -> Acc1 when Fun :: fun((Elem :: T, AccIn) -> AccOut), @@ -1294,9 +1334,13 @@ foldl(F, Accu, []) when is_function(F, 2) -> Accu. List :: [T], T :: term(). -foldr(F, Accu, [Hd|Tail]) -> - F(Hd, foldr(F, Accu, Tail)); -foldr(F, Accu, []) when is_function(F, 2) -> Accu. +foldr(F, Accu, List) when is_function(F, 2) -> + foldr_1(F, Accu, List). + +foldr_1(F, Accu, [Hd | Tail]) -> + F(Hd, foldr_1(F, Accu, Tail)); +foldr_1(_F, Accu, []) -> + Accu. -spec filter(Pred, List1) -> List2 when Pred :: fun((Elem :: T) -> boolean()), @@ -1317,15 +1361,15 @@ filter(Pred, List) when is_function(Pred, 1) -> NotSatisfying :: [T], T :: term(). -partition(Pred, L) -> - partition(Pred, L, [], []). +partition(Pred, L) when is_function(Pred, 1) -> + partition_1(Pred, L, [], []). -partition(Pred, [H | T], As, Bs) -> +partition_1(Pred, [H | T], As, Bs) -> case Pred(H) of - true -> partition(Pred, T, [H | As], Bs); - false -> partition(Pred, T, As, [H | Bs]) + true -> partition_1(Pred, T, [H | As], Bs); + false -> partition_1(Pred, T, As, [H | Bs]) end; -partition(Pred, [], As, Bs) when is_function(Pred, 1) -> +partition_1(_Pred, [], As, Bs) -> {reverse(As), reverse(Bs)}. -spec filtermap(Fun, List1) -> List2 when @@ -1335,16 +1379,20 @@ partition(Pred, [], As, Bs) when is_function(Pred, 1) -> Elem :: term(), Value :: term(). -filtermap(F, [Hd|Tail]) -> +filtermap(F, List) when is_function(F, 1) -> + filtermap_1(F, List). + +filtermap_1(F, [Hd|Tail]) -> case F(Hd) of - true -> - [Hd|filtermap(F, Tail)]; - {true,Val} -> - [Val|filtermap(F, Tail)]; - false -> - filtermap(F, Tail) + true -> + [Hd | filtermap_1(F, Tail)]; + {true,Val} -> + [Val | filtermap_1(F, Tail)]; + false -> + filtermap_1(F, Tail) end; -filtermap(F, []) when is_function(F, 1) -> []. +filtermap_1(_F, []) -> + []. -spec zf(fun((T) -> boolean() | {'true', X}), [T]) -> [(T | X)]. @@ -1356,10 +1404,14 @@ zf(F, L) -> List :: [T], T :: term(). -foreach(F, [Hd|Tail]) -> +foreach(F, List) when is_function(F, 1) -> + foreach_1(F, List). + +foreach_1(F, [Hd | Tail]) -> F(Hd), - foreach(F, Tail); -foreach(F, []) when is_function(F, 1) -> ok. + foreach_1(F, Tail); +foreach_1(_F, []) -> + ok. -spec mapfoldl(Fun, Acc0, List1) -> {List2, Acc1} when Fun :: fun((A, AccIn) -> {B, AccOut}), @@ -1372,11 +1424,15 @@ foreach(F, []) when is_function(F, 1) -> ok. A :: term(), B :: term(). -mapfoldl(F, Accu0, [Hd|Tail]) -> - {R,Accu1} = F(Hd, Accu0), - {Rs,Accu2} = mapfoldl(F, Accu1, Tail), - {[R|Rs],Accu2}; -mapfoldl(F, Accu, []) when is_function(F, 2) -> {[],Accu}. +mapfoldl(F, Accu, List) when is_function(F, 2) -> + mapfoldl_1(F, Accu, List). + +mapfoldl_1(F, Accu0, [Hd | Tail]) -> + {R, Accu1} = F(Hd, Accu0), + {Rs, Accu2} = mapfoldl_1(F, Accu1, Tail), + {[R | Rs], Accu2}; +mapfoldl_1(_F, Accu, []) -> + {[], Accu}. -spec mapfoldr(Fun, Acc0, List1) -> {List2, Acc1} when Fun :: fun((A, AccIn) -> {B, AccOut}), @@ -1389,11 +1445,15 @@ mapfoldl(F, Accu, []) when is_function(F, 2) -> {[],Accu}. A :: term(), B :: term(). -mapfoldr(F, Accu0, [Hd|Tail]) -> - {Rs,Accu1} = mapfoldr(F, Accu0, Tail), - {R,Accu2} = F(Hd, Accu1), - {[R|Rs],Accu2}; -mapfoldr(F, Accu, []) when is_function(F, 2) -> {[],Accu}. +mapfoldr(F, Accu, List) when is_function(F, 2) -> + mapfoldr_1(F, Accu, List). + +mapfoldr_1(F, Accu0, [Hd|Tail]) -> + {Rs, Accu1} = mapfoldr_1(F, Accu0, Tail), + {R, Accu2} = F(Hd, Accu1), + {[R | Rs], Accu2}; +mapfoldr_1(_F, Accu, []) -> + {[], Accu}. -spec takewhile(Pred, List1) -> List2 when Pred :: fun((Elem :: T) -> boolean()), @@ -1401,12 +1461,16 @@ mapfoldr(F, Accu, []) when is_function(F, 2) -> {[],Accu}. List2 :: [T], T :: term(). -takewhile(Pred, [Hd|Tail]) -> +takewhile(Pred, List) when is_function(Pred, 1) -> + takewhile_1(Pred, List). + +takewhile_1(Pred, [Hd | Tail]) -> case Pred(Hd) of - true -> [Hd|takewhile(Pred, Tail)]; - false -> [] + true -> [Hd | takewhile_1(Pred, Tail)]; + false -> [] end; -takewhile(Pred, []) when is_function(Pred, 1) -> []. +takewhile_1(_Pred, []) -> + []. -spec dropwhile(Pred, List1) -> List2 when Pred :: fun((Elem :: T) -> boolean()), @@ -1414,24 +1478,31 @@ takewhile(Pred, []) when is_function(Pred, 1) -> []. List2 :: [T], T :: term(). -dropwhile(Pred, [Hd|Tail]=Rest) -> +dropwhile(Pred, List) when is_function(Pred, 1) -> + dropwhile_1(Pred, List). + +dropwhile_1(Pred, [Hd | Tail]=Rest) -> case Pred(Hd) of - true -> dropwhile(Pred, Tail); - false -> Rest + true -> dropwhile_1(Pred, Tail); + false -> Rest end; -dropwhile(Pred, []) when is_function(Pred, 1) -> []. +dropwhile_1(_Pred, []) -> + []. -spec search(Pred, List) -> {value, Value} | false when Pred :: fun((T) -> boolean()), List :: [T], Value :: T. -search(Pred, [Hd|Tail]) -> +search(Pred, List) when is_function(Pred, 1) -> + search_1(Pred, List). + +search_1(Pred, [Hd | Tail]) -> case Pred(Hd) of true -> {value, Hd}; - false -> search(Pred, Tail) + false -> search_1(Pred, Tail) end; -search(Pred, []) when is_function(Pred, 1) -> +search_1(_Pred, []) -> false. -spec splitwith(Pred, List) -> {List1, List2} when @@ -1442,14 +1513,14 @@ search(Pred, []) when is_function(Pred, 1) -> T :: term(). splitwith(Pred, List) when is_function(Pred, 1) -> - splitwith(Pred, List, []). + splitwith_1(Pred, List, []). -splitwith(Pred, [Hd|Tail], Taken) -> +splitwith_1(Pred, [Hd|Tail], Taken) -> case Pred(Hd) of - true -> splitwith(Pred, Tail, [Hd|Taken]); + true -> splitwith_1(Pred, Tail, [Hd|Taken]); false -> {reverse(Taken), [Hd|Tail]} end; -splitwith(Pred, [], Taken) when is_function(Pred, 1) -> +splitwith_1(_Pred, [], Taken) -> {reverse(Taken),[]}. -spec split(N, List1) -> {List2, List3} when diff --git a/lib/stdlib/src/sets.erl b/lib/stdlib/src/sets.erl index 086de0f202..e272b33bf6 100644 --- a/lib/stdlib/src/sets.erl +++ b/lib/stdlib/src/sets.erl @@ -426,8 +426,10 @@ is_subset_1(Set, Iter) -> Acc1 :: Acc, AccIn :: Acc, AccOut :: Acc. -fold(F, Acc, #{}=D) -> fold_1(F, Acc, maps:iterator(D)); -fold(F, Acc, #set{}=D) -> fold_set(F, Acc, D). +fold(F, Acc, #{}=D) when is_function(F, 2)-> + fold_1(F, Acc, maps:iterator(D)); +fold(F, Acc, #set{}=D) when is_function(F, 2)-> + fold_set(F, Acc, D). fold_1(Fun, Acc, Iter) -> case maps:next(Iter) of @@ -443,8 +445,10 @@ fold_1(Fun, Acc, Iter) -> Pred :: fun((Element) -> boolean()), Set1 :: set(Element), Set2 :: set(Element). -filter(F, #{}=D) -> maps:from_keys(filter_1(F, maps:iterator(D)), ?VALUE); -filter(F, #set{}=D) -> filter_set(F, D). +filter(F, #{}=D) when is_function(F, 1)-> + maps:from_keys(filter_1(F, maps:iterator(D)), ?VALUE); +filter(F, #set{}=D) when is_function(F, 1)-> + filter_set(F, D). filter_1(Fun, Iter) -> case maps:next(Iter) of @@ -482,7 +486,7 @@ get_bucket(T, Slot) -> get_bucket_s(T#set.segs, Slot). %% implemented map and hash using fold but these should be faster. %% We hope! -fold_set(F, Acc, D) when is_function(F, 2) -> +fold_set(F, Acc, D) -> Segs = D#set.segs, fold_segs(F, Acc, Segs, tuple_size(Segs)). @@ -499,7 +503,7 @@ fold_bucket(F, Acc, [E|Bkt]) -> fold_bucket(F, F(E, Acc), Bkt); fold_bucket(_, Acc, []) -> Acc. -filter_set(F, D) when is_function(F, 1) -> +filter_set(F, D) -> Segs0 = tuple_to_list(D#set.segs), {Segs1,Fc} = filter_seg_list(F, Segs0, [], 0), maybe_contract(D#set{segs = list_to_tuple(Segs1)}, Fc). diff --git a/lib/stdlib/test/beam_lib_SUITE.erl b/lib/stdlib/test/beam_lib_SUITE.erl index 2b9226dbc2..cd15528302 100644 --- a/lib/stdlib/test/beam_lib_SUITE.erl +++ b/lib/stdlib/test/beam_lib_SUITE.erl @@ -430,7 +430,7 @@ strip_add_chunks(Conf) when is_list(Conf) -> compare_chunks(B1, NB1, NBId1), %% Keep all the extra chunks - ExtraChunks = ["Abst" , "Dbgi" , "Attr" , "CInf" , "LocT" , "Atom" ], + ExtraChunks = ["Abst", "Dbgi", "Attr", "CInf", "LocT", "Atom", "Type"], {ok, {simple, AB1}} = beam_lib:strip(B1, ExtraChunks), ABId1 = chunk_ids(AB1), true = length(BId1) == length(ABId1), |