diff options
Diffstat (limited to 'deps/v8/src/compiler/turboshaft/operations.h')
| -rw-r--r-- | deps/v8/src/compiler/turboshaft/operations.h | 1903 |
1 files changed, 1653 insertions, 250 deletions
diff --git a/deps/v8/src/compiler/turboshaft/operations.h b/deps/v8/src/compiler/turboshaft/operations.h index e240c41115..8ef0f79b8e 100644 --- a/deps/v8/src/compiler/turboshaft/operations.h +++ b/deps/v8/src/compiler/turboshaft/operations.h @@ -17,14 +17,18 @@ #include "src/base/macros.h" #include "src/base/optional.h" #include "src/base/platform/mutex.h" +#include "src/base/small-vector.h" #include "src/base/template-utils.h" #include "src/base/vector.h" #include "src/codegen/external-reference.h" #include "src/common/globals.h" #include "src/compiler/common-operator.h" #include "src/compiler/globals.h" +#include "src/compiler/turboshaft/deopt-data.h" #include "src/compiler/turboshaft/fast-hash.h" +#include "src/compiler/turboshaft/index.h" #include "src/compiler/turboshaft/representations.h" +#include "src/compiler/turboshaft/types.h" #include "src/compiler/turboshaft/utils.h" #include "src/compiler/write-barrier-kind.h" @@ -41,8 +45,8 @@ enum class TrapId : uint32_t; namespace v8::internal::compiler::turboshaft { class Block; struct FrameStateData; -class Variable; class Graph; +struct FrameStateOp; // DEFINING NEW OPERATIONS // ======================= @@ -65,7 +69,14 @@ class Graph; // non-static method `Properties()` if the properties depend on the particular // operation and not just the opcode. +#ifdef V8_INTL_SUPPORT +#define TURBOSHAFT_INTL_OPERATION_LIST(V) V(StringToCaseIntl) +#else +#define TURBOSHAFT_INTL_OPERATION_LIST(V) +#endif // V8_INTL_SUPPORT + #define TURBOSHAFT_OPERATION_LIST(V) \ + TURBOSHAFT_INTL_OPERATION_LIST(V) \ V(WordBinop) \ V(FloatBinop) \ V(OverflowCheckedBinop) \ @@ -75,6 +86,7 @@ class Graph; V(Equal) \ V(Comparison) \ V(Change) \ + V(ChangeOrDeopt) \ V(TryChange) \ V(Float64InsertWord32) \ V(TaggedBitcast) \ @@ -83,6 +95,8 @@ class Graph; V(Constant) \ V(Load) \ V(Store) \ + V(Allocate) \ + V(DecodeExternalPointer) \ V(Retain) \ V(Parameter) \ V(OsrValue) \ @@ -90,21 +104,49 @@ class Graph; V(StackPointerGreaterThan) \ V(StackSlot) \ V(FrameConstant) \ - V(CheckLazyDeopt) \ V(Deoptimize) \ V(DeoptimizeIf) \ V(TrapIf) \ V(Phi) \ V(FrameState) \ V(Call) \ + V(CallAndCatchException) \ + V(LoadException) \ V(TailCall) \ V(Unreachable) \ V(Return) \ V(Branch) \ - V(CatchException) \ V(Switch) \ V(Tuple) \ - V(Projection) + V(Projection) \ + V(StaticAssert) \ + V(CheckTurboshaftTypeOf) \ + V(ObjectIs) \ + V(FloatIs) \ + V(ConvertToObject) \ + V(ConvertToObjectOrDeopt) \ + V(ConvertObjectToPrimitive) \ + V(ConvertObjectToPrimitiveOrDeopt) \ + V(TruncateObjectToPrimitive) \ + V(Tag) \ + V(Untag) \ + V(NewConsString) \ + V(NewArray) \ + V(DoubleArrayMinMax) \ + V(LoadFieldByIndex) \ + V(DebugBreak) \ + V(BigIntBinop) \ + V(BigIntEqual) \ + V(BigIntComparison) \ + V(BigIntUnary) \ + V(LoadRootRegister) \ + V(StringAt) \ + V(StringLength) \ + V(StringIndexOf) \ + V(StringFromCodePointAt) \ + V(StringSubstring) \ + V(StringEqual) \ + V(StringComparison) enum class Opcode : uint8_t { #define ENUM_CONSTANT(Name) k##Name, @@ -122,110 +164,17 @@ constexpr uint16_t kNumberOfOpcodes = 0 TURBOSHAFT_OPERATION_LIST(COUNT_OPCODES); #undef COUNT_OPCODES -// Operations are stored in possibly muliple sequential storage slots. -using OperationStorageSlot = std::aligned_storage_t<8, 8>; -// Operations occupy at least 2 slots, therefore we assign one id per two slots. -constexpr size_t kSlotsPerId = 2; - -// `OpIndex` is an offset from the beginning of the operations buffer. -// Compared to `Operation*`, it is more memory efficient (32bit) and stable when -// the operations buffer is re-allocated. -class OpIndex { - public: - explicit constexpr OpIndex(uint32_t offset) : offset_(offset) { - DCHECK_EQ(offset % sizeof(OperationStorageSlot), 0); - } - constexpr OpIndex() : offset_(std::numeric_limits<uint32_t>::max()) {} - - uint32_t id() const { - // Operations are stored at an offset that's a multiple of - // `sizeof(OperationStorageSlot)`. In addition, an operation occupies at - // least `kSlotsPerId` many `OperationSlot`s. Therefore, we can assign id's - // by dividing by `kSlotsPerId`. A compact id space is important, because it - // makes side-tables smaller. - DCHECK_EQ(offset_ % sizeof(OperationStorageSlot), 0); - return offset_ / sizeof(OperationStorageSlot) / kSlotsPerId; - } - uint32_t offset() const { - DCHECK_EQ(offset_ % sizeof(OperationStorageSlot), 0); - return offset_; - } - - bool valid() const { return *this != Invalid(); } - - static constexpr OpIndex Invalid() { return OpIndex(); } - - // Encode a sea-of-nodes node id in the `OpIndex` type. - // Only used for node origins that actually point to sea-of-nodes graph nodes. - static OpIndex EncodeTurbofanNodeId(uint32_t id) { - OpIndex result = OpIndex(id * sizeof(OperationStorageSlot)); - result.offset_ += kTurbofanNodeIdFlag; - return result; - } - uint32_t DecodeTurbofanNodeId() const { - DCHECK(IsTurbofanNodeId()); - return offset_ / sizeof(OperationStorageSlot); - } - bool IsTurbofanNodeId() const { - return offset_ % sizeof(OperationStorageSlot) == kTurbofanNodeIdFlag; - } - - bool operator==(OpIndex other) const { return offset_ == other.offset_; } - bool operator!=(OpIndex other) const { return offset_ != other.offset_; } - bool operator<(OpIndex other) const { return offset_ < other.offset_; } - bool operator>(OpIndex other) const { return offset_ > other.offset_; } - bool operator<=(OpIndex other) const { return offset_ <= other.offset_; } - bool operator>=(OpIndex other) const { return offset_ >= other.offset_; } - - private: - uint32_t offset_; - - static constexpr uint32_t kTurbofanNodeIdFlag = 1; -}; - -template <> -struct fast_hash<OpIndex> { - V8_INLINE size_t operator()(OpIndex op) { return op.id(); } -}; - -// `BlockIndex` is the index of a bound block. -// A dominating block always has a smaller index. -// It corresponds to the ordering of basic blocks in the operations buffer. -class BlockIndex { - public: - explicit constexpr BlockIndex(uint32_t id) : id_(id) {} - constexpr BlockIndex() : id_(std::numeric_limits<uint32_t>::max()) {} - - uint32_t id() const { return id_; } - bool valid() const { return *this != Invalid(); } - - static constexpr BlockIndex Invalid() { return BlockIndex(); } - - bool operator==(BlockIndex other) const { return id_ == other.id_; } - bool operator!=(BlockIndex other) const { return id_ != other.id_; } - bool operator<(BlockIndex other) const { return id_ < other.id_; } - bool operator>(BlockIndex other) const { return id_ > other.id_; } - bool operator<=(BlockIndex other) const { return id_ <= other.id_; } - bool operator>=(BlockIndex other) const { return id_ >= other.id_; } - - private: - uint32_t id_; -}; - -template <> -struct fast_hash<BlockIndex> { - V8_INLINE size_t operator()(BlockIndex op) { return op.id(); } -}; - -std::ostream& operator<<(std::ostream& os, BlockIndex b); -std::ostream& operator<<(std::ostream& os, const Block* b); - struct OpProperties { // The operation may read memory or depend on other information beyond its - // inputs. + // inputs. Generating random numbers or nondeterministic behavior counts as + // reading. const bool can_read; // The operation may write memory or have other observable side-effects. + // Writing to memory allocated as part of the operation does not count, since + // it is not observable. const bool can_write; + // The operation can allocate memory on the heap, which might also trigger GC. + const bool can_allocate; // The operation can abort the current execution by throwing an exception or // deoptimizing. const bool can_abort; @@ -233,47 +182,50 @@ struct OpProperties { const bool is_block_terminator; // By being const and not being set in the constructor, these properties are // guaranteed to be derived. - const bool is_pure = - !(can_read || can_write || can_abort || is_block_terminator); + const bool is_pure_no_allocation = !(can_read || can_write || can_allocate || + can_abort || is_block_terminator); const bool is_required_when_unused = can_write || can_abort || is_block_terminator; - // Nodes that don't read, write and aren't block terminators can be eliminated - // via value numbering. + // Operations that don't read, write, allocate and aren't block terminators + // can be eliminated via value numbering, which means that if there are two + // identical operations where one dominates the other, then the second can be + // replaced with the first one. This is safe for deopting or throwing + // operations, because the first instance would have aborted the execution + // already as + // `!can_read` guarantees deterministic behavior. const bool can_be_eliminated = - !(can_read || can_write || is_block_terminator); + !(can_read || can_write || can_allocate || is_block_terminator); - constexpr OpProperties(bool can_read, bool can_write, bool can_abort, - bool is_block_terminator) + constexpr OpProperties(bool can_read, bool can_write, bool can_allocate, + bool can_abort, bool is_block_terminator) : can_read(can_read), can_write(can_write), + can_allocate(can_allocate), can_abort(can_abort), is_block_terminator(is_block_terminator) {} - static constexpr OpProperties Pure() { return {false, false, false, false}; } - static constexpr OpProperties Reading() { - return {true, false, false, false}; - } - static constexpr OpProperties Writing() { - return {false, true, false, false}; - } - static constexpr OpProperties CanAbort() { - return {false, false, true, false}; - } - static constexpr OpProperties AnySideEffects() { - return {true, true, true, false}; - } - static constexpr OpProperties BlockTerminator() { - return {false, false, false, true}; - } - static constexpr OpProperties BlockTerminatorWithAnySideEffect() { - return {true, true, true, true}; - } - static constexpr OpProperties ReadingAndCanAbort() { - return {true, false, true, false}; - } - static constexpr OpProperties WritingAndCanAbort() { - return {false, true, true, false}; +#define ALL_OP_PROPERTIES(V) \ + V(PureNoAllocation, false, false, false, false, false) \ + V(PureMayAllocate, false, false, true, false, false) \ + V(Reading, true, false, false, false, false) \ + V(Writing, false, true, false, false, false) \ + V(CanAbort, false, false, false, true, false) \ + V(AnySideEffects, true, true, true, true, false) \ + V(BlockTerminator, false, false, false, false, true) \ + V(BlockTerminatorWithAnySideEffect, true, true, true, true, true) \ + V(ReadingAndCanAbort, true, false, false, true, false) \ + V(WritingAndCanAbort, false, true, false, true, false) + +#define DEFINE_OP_PROPERTY(Name, can_read, can_write, can_allocate, can_abort, \ + is_block_terminator) \ + static constexpr OpProperties Name() { \ + return {can_read, can_write, can_allocate, can_abort, \ + is_block_terminator}; \ } + + ALL_OP_PROPERTIES(DEFINE_OP_PROPERTY) +#undef DEFINE_OP_PROPERTY + bool operator==(const OpProperties& other) const { return can_read == other.can_read && can_write == other.can_write && can_abort == other.can_abort && @@ -310,6 +262,8 @@ struct alignas(OpIndex) Operation { return StorageSlotCount(opcode, input_count); } + base::Vector<const RegisterRepresentation> outputs_rep() const; + template <class Op> bool Is() const { return opcode == Op::opcode; @@ -362,9 +316,10 @@ std::ostream& operator<<(std::ostream& os, OperationPrintStyle op); inline std::ostream& operator<<(std::ostream& os, const Operation& op) { return os << OperationPrintStyle{op}; } -inline void Print(const Operation& op) { std::cout << op << "\n"; } +void Print(const Operation& op); OperationStorageSlot* AllocateOpStorage(Graph* graph, size_t slot_count); +const Operation& Get(const Graph& graph, OpIndex index); // Determine if an operation declares `properties`, which means that its // properties are static and don't depend on inputs or options. @@ -435,6 +390,9 @@ struct OperationT : Operation { OperationStorageSlot* ptr = AllocateOpStorage(graph, StorageSlotCount(input_count)); Derived* result = new (ptr) Derived(std::move(args)...); +#ifdef DEBUG + result->Validate(*graph); +#endif // If this DCHECK fails, then the number of inputs specified in the // operation constructor and in the static New function disagree. DCHECK_EQ(input_count, result->Operation::input_count); @@ -489,6 +447,11 @@ struct OperationT : Operation { PrintOptionsHelper(os, options, std::make_index_sequence<options_count>()); } + // Check graph invariants for this operation. Will be invoked in debug mode + // immediately upon construction. + // Concrete Operator classes are expected to re-define it. + void Validate(const Graph& graph) const = delete; + private: template <class... T, size_t... I> static void PrintOptionsHelper(std::ostream& os, @@ -502,6 +465,15 @@ struct OperationT : Operation { ...); os << "]"; } + + // All Operations have to define the outputs_rep function, to which + // Operation::outputs_rep() will forward, based on their opcode. If you forget + // to define it, then Operation::outputs_rep() would forward to itself, + // resulting in an infinite loop. To avoid this, we define here in OperationT + // a private version outputs_rep (with no implementation): if an operation + // forgets to define outputs_rep, then Operation::outputs_rep() tries to call + // this private version, which fails at compile time. + base::Vector<const RegisterRepresentation> outputs_rep() const; }; template <size_t InputCount, class Derived> @@ -587,6 +559,20 @@ class SupportedOperations { #undef DECLARE_GETTER }; +template <RegisterRepresentation::Enum... reps> +base::Vector<const RegisterRepresentation> RepVector() { + static const std::array<RegisterRepresentation, sizeof...(reps)> rep_array{ + RegisterRepresentation{reps}...}; + return base::VectorOf(rep_array); +} + +bool ValidOpInputRep(const Graph& graph, OpIndex input, + std::initializer_list<RegisterRepresentation> expected_rep, + base::Optional<size_t> projection_index = {}); +bool ValidOpInputRep(const Graph& graph, OpIndex input, + RegisterRepresentation expected_rep, + base::Optional<size_t> projection_index = {}); + struct WordBinopOp : FixedArityOperationT<2, WordBinopOp> { enum class Kind : uint8_t { kAdd, @@ -605,7 +591,10 @@ struct WordBinopOp : FixedArityOperationT<2, WordBinopOp> { Kind kind; WordRepresentation rep; - static constexpr OpProperties properties = OpProperties::Pure(); + static constexpr OpProperties properties = OpProperties::PureNoAllocation(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return base::VectorOf(static_cast<const RegisterRepresentation*>(&rep), 1); + } OpIndex left() const { return input(0); } OpIndex right() const { return input(1); } @@ -670,6 +659,11 @@ struct WordBinopOp : FixedArityOperationT<2, WordBinopOp> { WordBinopOp(OpIndex left, OpIndex right, Kind kind, WordRepresentation rep) : Base(left, right), kind(kind), rep(rep) {} + + void Validate(const Graph& graph) const { + DCHECK(ValidOpInputRep(graph, left(), rep)); + DCHECK(ValidOpInputRep(graph, right(), rep)); + } auto options() const { return std::tuple{kind, rep}; } void PrintOptions(std::ostream& os) const; }; @@ -689,7 +683,10 @@ struct FloatBinopOp : FixedArityOperationT<2, FloatBinopOp> { Kind kind; FloatRepresentation rep; - static constexpr OpProperties properties = OpProperties::Pure(); + static constexpr OpProperties properties = OpProperties::PureNoAllocation(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return base::VectorOf(static_cast<const RegisterRepresentation*>(&rep), 1); + } OpIndex left() const { return input(0); } OpIndex right() const { return input(1); } @@ -711,9 +708,13 @@ struct FloatBinopOp : FixedArityOperationT<2, FloatBinopOp> { } FloatBinopOp(OpIndex left, OpIndex right, Kind kind, FloatRepresentation rep) - : Base(left, right), kind(kind), rep(rep) { + : Base(left, right), kind(kind), rep(rep) {} + + void Validate(const Graph& graph) const { DCHECK_IMPLIES(kind == any_of(Kind::kPower, Kind::kAtan2, Kind::kMod), rep == FloatRepresentation::Float64()); + DCHECK(ValidOpInputRep(graph, left(), rep)); + DCHECK(ValidOpInputRep(graph, right(), rep)); } auto options() const { return std::tuple{kind, rep}; } void PrintOptions(std::ostream& os) const; @@ -729,7 +730,17 @@ struct OverflowCheckedBinopOp Kind kind; WordRepresentation rep; - static constexpr OpProperties properties = OpProperties::Pure(); + static constexpr OpProperties properties = OpProperties::PureNoAllocation(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + switch (rep.value()) { + case WordRepresentation::Word32(): + return RepVector<RegisterRepresentation::Word32(), + RegisterRepresentation::Word32()>(); + case WordRepresentation::Word64(): + return RepVector<RegisterRepresentation::Word64(), + RegisterRepresentation::Word32()>(); + } + } OpIndex left() const { return input(0); } OpIndex right() const { return input(1); } @@ -747,6 +758,11 @@ struct OverflowCheckedBinopOp OverflowCheckedBinopOp(OpIndex left, OpIndex right, Kind kind, WordRepresentation rep) : Base(left, right), kind(kind), rep(rep) {} + + void Validate(const Graph& graph) const { + DCHECK(ValidOpInputRep(graph, left(), rep)); + DCHECK(ValidOpInputRep(graph, right(), rep)); + } auto options() const { return std::tuple{kind, rep}; } void PrintOptions(std::ostream& os) const; }; @@ -762,15 +778,21 @@ struct WordUnaryOp : FixedArityOperationT<1, WordUnaryOp> { }; Kind kind; WordRepresentation rep; - static constexpr OpProperties properties = OpProperties::Pure(); + static constexpr OpProperties properties = OpProperties::PureNoAllocation(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return base::VectorOf(static_cast<const RegisterRepresentation*>(&rep), 1); + } OpIndex input() const { return Base::input(0); } static bool IsSupported(Kind kind, WordRepresentation rep); explicit WordUnaryOp(OpIndex input, Kind kind, WordRepresentation rep) - : Base(input), kind(kind), rep(rep) { + : Base(input), kind(kind), rep(rep) {} + + void Validate(const Graph& graph) const { DCHECK(IsSupported(kind, rep)); + DCHECK(ValidOpInputRep(graph, input(), rep)); } auto options() const { return std::tuple{kind, rep}; } }; @@ -808,15 +830,21 @@ struct FloatUnaryOp : FixedArityOperationT<1, FloatUnaryOp> { }; Kind kind; FloatRepresentation rep; - static constexpr OpProperties properties = OpProperties::Pure(); + static constexpr OpProperties properties = OpProperties::PureNoAllocation(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return base::VectorOf(static_cast<const RegisterRepresentation*>(&rep), 1); + } OpIndex input() const { return Base::input(0); } static bool IsSupported(Kind kind, FloatRepresentation rep); explicit FloatUnaryOp(OpIndex input, Kind kind, FloatRepresentation rep) - : Base(input), kind(kind), rep(rep) { + : Base(input), kind(kind), rep(rep) {} + + void Validate(const Graph& graph) const { DCHECK(IsSupported(kind, rep)); + DCHECK(ValidOpInputRep(graph, input(), rep)); } auto options() const { return std::tuple{kind, rep}; } }; @@ -834,7 +862,10 @@ struct ShiftOp : FixedArityOperationT<2, ShiftOp> { Kind kind; WordRepresentation rep; - static constexpr OpProperties properties = OpProperties::Pure(); + static constexpr OpProperties properties = OpProperties::PureNoAllocation(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return base::VectorOf(static_cast<const RegisterRepresentation*>(&rep), 1); + } OpIndex left() const { return input(0); } OpIndex right() const { return input(1); } @@ -868,6 +899,11 @@ struct ShiftOp : FixedArityOperationT<2, ShiftOp> { ShiftOp(OpIndex left, OpIndex right, Kind kind, WordRepresentation rep) : Base(left, right), kind(kind), rep(rep) {} + + void Validate(const Graph& graph) const { + DCHECK(ValidOpInputRep(graph, left(), rep)); + DCHECK(ValidOpInputRep(graph, right(), WordRepresentation::Word32())); + } auto options() const { return std::tuple{kind, rep}; } }; std::ostream& operator<<(std::ostream& os, ShiftOp::Kind kind); @@ -875,17 +911,37 @@ std::ostream& operator<<(std::ostream& os, ShiftOp::Kind kind); struct EqualOp : FixedArityOperationT<2, EqualOp> { RegisterRepresentation rep; - static constexpr OpProperties properties = OpProperties::Pure(); + static constexpr OpProperties properties = OpProperties::PureNoAllocation(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return RepVector<RegisterRepresentation::Word32()>(); + } OpIndex left() const { return input(0); } OpIndex right() const { return input(1); } + bool ValidInputRep( + base::Vector<const RegisterRepresentation> input_reps) const; + EqualOp(OpIndex left, OpIndex right, RegisterRepresentation rep) - : Base(left, right), rep(rep) { + : Base(left, right), rep(rep) {} + + void Validate(const Graph& graph) const { +#ifdef DEBUG DCHECK(rep == any_of(RegisterRepresentation::Word32(), RegisterRepresentation::Word64(), RegisterRepresentation::Float32(), - RegisterRepresentation::Float64())); + RegisterRepresentation::Float64(), + RegisterRepresentation::Tagged())); + RegisterRepresentation input_rep = rep; +#ifdef V8_COMPRESS_POINTERS + // In the presence of pointer compression, we only compare the lower 32bit. + if (input_rep == RegisterRepresentation::Tagged()) { + input_rep = RegisterRepresentation::Compressed(); + } +#endif // V8_COMPRESS_POINTERS + DCHECK(ValidOpInputRep(graph, left(), input_rep)); + DCHECK(ValidOpInputRep(graph, right(), input_rep)); +#endif // DEBUG } auto options() const { return std::tuple{rep}; } }; @@ -900,14 +956,19 @@ struct ComparisonOp : FixedArityOperationT<2, ComparisonOp> { Kind kind; RegisterRepresentation rep; - static constexpr OpProperties properties = OpProperties::Pure(); + static constexpr OpProperties properties = OpProperties::PureNoAllocation(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return RepVector<RegisterRepresentation::Word32()>(); + } OpIndex left() const { return input(0); } OpIndex right() const { return input(1); } ComparisonOp(OpIndex left, OpIndex right, Kind kind, RegisterRepresentation rep) - : Base(left, right), kind(kind), rep(rep) { + : Base(left, right), kind(kind), rep(rep) {} + + void Validate(const Graph& graph) const { DCHECK_EQ(rep, any_of(RegisterRepresentation::Word32(), RegisterRepresentation::Word64(), RegisterRepresentation::Float32(), @@ -916,9 +977,21 @@ struct ComparisonOp : FixedArityOperationT<2, ComparisonOp> { rep == any_of(RegisterRepresentation::Float32(), RegisterRepresentation::Float64()), kind == any_of(Kind::kSignedLessThan, Kind::kSignedLessThanOrEqual)); + DCHECK(ValidOpInputRep(graph, left(), rep)); + DCHECK(ValidOpInputRep(graph, right(), rep)); } auto options() const { return std::tuple{kind, rep}; } + static bool IsLessThan(Kind kind) { + switch (kind) { + case Kind::kSignedLessThan: + case Kind::kUnsignedLessThan: + return true; + case Kind::kSignedLessThanOrEqual: + case Kind::kUnsignedLessThanOrEqual: + return false; + } + } static bool IsSigned(Kind kind) { switch (kind) { case Kind::kSignedLessThan: @@ -1036,21 +1109,92 @@ struct ChangeOp : FixedArityOperationT<1, ChangeOp> { signalling_nan_possible); } - static constexpr OpProperties properties = OpProperties::Pure(); + static constexpr OpProperties properties = OpProperties::PureNoAllocation(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return base::VectorOf(&to, 1); + } OpIndex input() const { return Base::input(0); } ChangeOp(OpIndex input, Kind kind, Assumption assumption, RegisterRepresentation from, RegisterRepresentation to) : Base(input), kind(kind), assumption(assumption), from(from), to(to) {} + + void Validate(const Graph& graph) const { + DCHECK(ValidOpInputRep(graph, input(), from)); + } auto options() const { return std::tuple{kind, assumption, from, to}; } }; std::ostream& operator<<(std::ostream& os, ChangeOp::Kind kind); std::ostream& operator<<(std::ostream& os, ChangeOp::Assumption assumption); +struct ChangeOrDeoptOp : FixedArityOperationT<2, ChangeOrDeoptOp> { + enum class Kind : uint8_t { + kUint32ToInt32, + kInt64ToInt32, + kUint64ToInt32, + kUint64ToInt64, + kFloat64ToInt32, + kFloat64ToInt64, + }; + Kind kind; + CheckForMinusZeroMode minus_zero_mode; + FeedbackSource feedback; + + static constexpr OpProperties properties = OpProperties::PureNoAllocation(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + switch (kind) { + case Kind::kUint32ToInt32: + case Kind::kInt64ToInt32: + case Kind::kUint64ToInt32: + case Kind::kFloat64ToInt32: + return RepVector<RegisterRepresentation::Word32()>(); + case Kind::kUint64ToInt64: + case Kind::kFloat64ToInt64: + return RepVector<RegisterRepresentation::Word64()>(); + } + } + + OpIndex input() const { return Base::input(0); } + OpIndex frame_state() const { return Base::input(1); } + + ChangeOrDeoptOp(OpIndex input, OpIndex frame_state, Kind kind, + CheckForMinusZeroMode minus_zero_mode, + const FeedbackSource& feedback) + : Base(input, frame_state), + kind(kind), + minus_zero_mode(minus_zero_mode), + feedback(feedback) {} + + void Validate(const Graph& graph) const { + switch (kind) { + case Kind::kUint32ToInt32: + DCHECK( + ValidOpInputRep(graph, input(), RegisterRepresentation::Word32())); + break; + case Kind::kInt64ToInt32: + case Kind::kUint64ToInt32: + case Kind::kUint64ToInt64: + DCHECK( + ValidOpInputRep(graph, input(), RegisterRepresentation::Word64())); + break; + case Kind::kFloat64ToInt32: + case Kind::kFloat64ToInt64: + DCHECK( + ValidOpInputRep(graph, input(), RegisterRepresentation::Float64())); + break; + } + DCHECK(Get(graph, frame_state()).Is<FrameStateOp>()); + } + auto options() const { return std::tuple{kind, minus_zero_mode, feedback}; } +}; +std::ostream& operator<<(std::ostream& os, ChangeOrDeoptOp::Kind kind); + // Perform a conversion and return a pair of the result and a bit if it was // successful. struct TryChangeOp : FixedArityOperationT<1, TryChangeOp> { + static constexpr uint32_t kSuccessValue = 1; + static constexpr uint32_t kFailureValue = 0; enum class Kind : uint8_t { // The result of the truncation is undefined if the result is out of range. kSignedFloatTruncateOverflowUndefined, @@ -1060,13 +1204,27 @@ struct TryChangeOp : FixedArityOperationT<1, TryChangeOp> { FloatRepresentation from; WordRepresentation to; - static constexpr OpProperties properties = OpProperties::Pure(); + static constexpr OpProperties properties = OpProperties::PureNoAllocation(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + switch (to.value()) { + case WordRepresentation::Word32(): + return RepVector<RegisterRepresentation::Word32(), + RegisterRepresentation::Word32()>(); + case WordRepresentation::Word64(): + return RepVector<RegisterRepresentation::Word64(), + RegisterRepresentation::Word32()>(); + } + } OpIndex input() const { return Base::input(0); } TryChangeOp(OpIndex input, Kind kind, FloatRepresentation from, WordRepresentation to) : Base(input), kind(kind), from(from), to(to) {} + + void Validate(const Graph& graph) const { + DCHECK(ValidOpInputRep(graph, input(), from)); + } auto options() const { return std::tuple{kind, from, to}; } }; std::ostream& operator<<(std::ostream& os, TryChangeOp::Kind kind); @@ -1076,13 +1234,22 @@ struct Float64InsertWord32Op : FixedArityOperationT<2, Float64InsertWord32Op> { enum class Kind { kLowHalf, kHighHalf }; Kind kind; - static constexpr OpProperties properties = OpProperties::Pure(); + static constexpr OpProperties properties = OpProperties::PureNoAllocation(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return RepVector<RegisterRepresentation::Float64()>(); + } OpIndex float64() const { return input(0); } OpIndex word32() const { return input(1); } Float64InsertWord32Op(OpIndex float64, OpIndex word32, Kind kind) : Base(float64, word32), kind(kind) {} + + void Validate(const Graph& graph) const { + DCHECK( + ValidOpInputRep(graph, float64(), RegisterRepresentation::Float64())); + DCHECK(ValidOpInputRep(graph, word32(), RegisterRepresentation::Word32())); + } auto options() const { return std::tuple{kind}; } }; std::ostream& operator<<(std::ostream& os, Float64InsertWord32Op::Kind kind); @@ -1093,16 +1260,24 @@ struct TaggedBitcastOp : FixedArityOperationT<1, TaggedBitcastOp> { // Due to moving GC, converting from or to pointers doesn't commute with GC. static constexpr OpProperties properties = OpProperties::Reading(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return base::VectorOf(&to, 1); + } OpIndex input() const { return Base::input(0); } TaggedBitcastOp(OpIndex input, RegisterRepresentation from, RegisterRepresentation to) - : Base(input), from(from), to(to) { + : Base(input), from(from), to(to) {} + + void Validate(const Graph& graph) const { DCHECK((from == RegisterRepresentation::PointerSized() && to == RegisterRepresentation::Tagged()) || (from == RegisterRepresentation::Tagged() && - to == RegisterRepresentation::PointerSized())); + to == RegisterRepresentation::PointerSized()) || + (from == RegisterRepresentation::Compressed() && + to == RegisterRepresentation::Word32())); + DCHECK(ValidOpInputRep(graph, input(), from)); } auto options() const { return std::tuple{from, to}; } }; @@ -1110,22 +1285,28 @@ struct TaggedBitcastOp : FixedArityOperationT<1, TaggedBitcastOp> { struct SelectOp : FixedArityOperationT<3, SelectOp> { enum class Implementation : uint8_t { kBranch, kCMove }; - static constexpr OpProperties properties = OpProperties::Pure(); RegisterRepresentation rep; BranchHint hint; Implementation implem; + static constexpr OpProperties properties = OpProperties::PureNoAllocation(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return base::VectorOf(&rep, 1); + } + SelectOp(OpIndex cond, OpIndex vtrue, OpIndex vfalse, RegisterRepresentation rep, BranchHint hint, Implementation implem) - : Base(cond, vtrue, vfalse), rep(rep), hint(hint), implem(implem) { -#ifdef DEBUG - if (implem == Implementation::kCMove) { - DCHECK((rep == RegisterRepresentation::Word32() && - SupportedOperations::word32_select()) || - (rep == RegisterRepresentation::Word64() && - SupportedOperations::word64_select())); - } -#endif + : Base(cond, vtrue, vfalse), rep(rep), hint(hint), implem(implem) {} + + void Validate(const Graph& graph) const { + DCHECK_IMPLIES(implem == Implementation::kCMove, + (rep == RegisterRepresentation::Word32() && + SupportedOperations::word32_select()) || + (rep == RegisterRepresentation::Word64() && + SupportedOperations::word64_select())); + DCHECK(ValidOpInputRep(graph, cond(), RegisterRepresentation::Word32())); + DCHECK(ValidOpInputRep(graph, vtrue(), rep)); + DCHECK(ValidOpInputRep(graph, vfalse(), rep)); } OpIndex cond() const { return input(0); } @@ -1139,39 +1320,64 @@ std::ostream& operator<<(std::ostream& os, SelectOp::Implementation kind); struct PhiOp : OperationT<PhiOp> { RegisterRepresentation rep; - static constexpr OpProperties properties = OpProperties::Pure(); + static constexpr OpProperties properties = OpProperties::PureNoAllocation(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return base::VectorOf(&rep, 1); + } static constexpr size_t kLoopPhiBackEdgeIndex = 1; explicit PhiOp(base::Vector<const OpIndex> inputs, RegisterRepresentation rep) : Base(inputs), rep(rep) {} + + void Validate(const Graph& graph) const { +#ifdef DEBUG + for (OpIndex input : inputs()) { + DCHECK(ValidOpInputRep(graph, input, rep)); + } +#endif + } auto options() const { return std::tuple{rep}; } }; // Only used when moving a loop phi to a new graph while the loop backedge has // not been emitted yet. struct PendingLoopPhiOp : FixedArityOperationT<1, PendingLoopPhiOp> { - RegisterRepresentation rep; - union { - // Used when transforming a Turboshaft graph. - // This is not an input because it refers to the old graph. - OpIndex old_backedge_index = OpIndex::Invalid(); - // Used when translating from sea-of-nodes. - Node* old_backedge_node; + struct PhiIndex { + int index; + }; + struct Data { + union { + // Used when transforming a Turboshaft graph. + // This is not an input because it refers to the old graph. + OpIndex old_backedge_index = OpIndex::Invalid(); + // Used when translating from sea-of-nodes. + Node* old_backedge_node; + // Used when building loops with the assembler macros. + PhiIndex phi_index; + }; + explicit Data(OpIndex old_backedge_index) + : old_backedge_index(old_backedge_index) {} + explicit Data(Node* old_backedge_node) + : old_backedge_node(old_backedge_node) {} + explicit Data(PhiIndex phi_index) : phi_index(phi_index) {} }; + RegisterRepresentation rep; + Data data; - static constexpr OpProperties properties = OpProperties::Pure(); + static constexpr OpProperties properties = OpProperties::PureNoAllocation(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return base::VectorOf(&rep, 1); + } OpIndex first() const { return input(0); } - PendingLoopPhiOp(OpIndex first, RegisterRepresentation rep, - OpIndex old_backedge_index) - : Base(first), rep(rep), old_backedge_index(old_backedge_index) { - DCHECK(old_backedge_index.valid()); + PendingLoopPhiOp(OpIndex first, RegisterRepresentation rep, Data data) + : Base(first), rep(rep), data(data) {} + + void Validate(const Graph& graph) const { + DCHECK(ValidOpInputRep(graph, first(), rep)); } - PendingLoopPhiOp(OpIndex first, RegisterRepresentation rep, - Node* old_backedge_node) - : Base(first), rep(rep), old_backedge_node(old_backedge_node) {} std::tuple<> options() const { UNREACHABLE(); } void PrintOptions(std::ostream& os) const; }; @@ -1192,6 +1398,7 @@ struct ConstantOp : FixedArityOperationT<0, ConstantOp> { }; Kind kind; + RegisterRepresentation rep = Representation(kind); union Storage { uint64_t integral; float float32; @@ -1206,9 +1413,12 @@ struct ConstantOp : FixedArityOperationT<0, ConstantOp> { Storage(Handle<HeapObject> constant) : handle(constant) {} } storage; - static constexpr OpProperties properties = OpProperties::Pure(); + static constexpr OpProperties properties = OpProperties::PureNoAllocation(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return base::VectorOf(&rep, 1); + } - RegisterRepresentation Representation() const { + static RegisterRepresentation Representation(Kind kind) { switch (kind) { case Kind::kWord32: return RegisterRepresentation::Word32(); @@ -1232,7 +1442,9 @@ struct ConstantOp : FixedArityOperationT<0, ConstantOp> { } ConstantOp(Kind kind, Storage storage) - : Base(), kind(kind), storage(storage) { + : Base(), kind(kind), storage(storage) {} + + void Validate(const Graph& graph) const { DCHECK_IMPLIES( kind == Kind::kWord32, storage.integral <= WordRepresentation::Word32().MaxUnsignedValue()); @@ -1418,6 +1630,14 @@ struct LoadOp : OperationT<LoadOp> { // There is a Wasm trap handler for out-of-bounds accesses. bool with_trap_handler : 1; + static constexpr Kind Aligned(BaseTaggedness base_is_tagged) { + switch (base_is_tagged) { + case BaseTaggedness::kTaggedBase: + return TaggedBase(); + case BaseTaggedness::kUntaggedBase: + return RawAligned(); + } + } static constexpr Kind TaggedBase() { return Kind{true, false, false}; } static constexpr Kind RawAligned() { return Kind{false, false, false}; } static constexpr Kind RawUnaligned() { return Kind{false, true, false}; } @@ -1439,6 +1659,9 @@ struct LoadOp : OperationT<LoadOp> { return kind.with_trap_handler ? OpProperties::ReadingAndCanAbort() : OpProperties::Reading(); } + base::Vector<const RegisterRepresentation> outputs_rep() const { + return base::VectorOf(&result_rep, 1); + } OpIndex base() const { return input(0); } OpIndex index() const { @@ -1454,12 +1677,27 @@ struct LoadOp : OperationT<LoadOp> { result_rep(result_rep), element_size_log2(element_size_log2), offset(offset) { + input(0) = base; + if (index.valid()) { + input(1) = index; + } + } + + void Validate(const Graph& graph) const { DCHECK(loaded_rep.ToRegisterRepresentation() == result_rep || (loaded_rep.IsTagged() && result_rep == RegisterRepresentation::Compressed())); - DCHECK_IMPLIES(element_size_log2 > 0, index.valid()); - input(0) = base; - if (index.valid()) input(1) = index; + DCHECK_IMPLIES(element_size_log2 > 0, index().valid()); + DCHECK( + kind.tagged_base + ? ValidOpInputRep(graph, base(), RegisterRepresentation::Tagged()) + : ValidOpInputRep(graph, base(), + {RegisterRepresentation::PointerSized(), + RegisterRepresentation::Tagged()})); + if (index().valid()) { + DCHECK(ValidOpInputRep(graph, index(), + RegisterRepresentation::PointerSized())); + } } static LoadOp& New(Graph* graph, OpIndex base, OpIndex index, Kind kind, MemoryRepresentation loaded_rep, @@ -1496,6 +1734,7 @@ struct StoreOp : OperationT<StoreOp> { return kind.with_trap_handler ? OpProperties::WritingAndCanAbort() : OpProperties::Writing(); } + base::Vector<const RegisterRepresentation> outputs_rep() const { return {}; } OpIndex base() const { return input(0); } OpIndex value() const { return input(1); } @@ -1512,10 +1751,27 @@ struct StoreOp : OperationT<StoreOp> { write_barrier(write_barrier), element_size_log2(element_size_log2), offset(offset) { - DCHECK_IMPLIES(element_size_log2 > 0, index.valid()); input(0) = base; input(1) = value; - if (index.valid()) input(2) = index; + if (index.valid()) { + input(2) = index; + } + } + + void Validate(const Graph& graph) const { + DCHECK_IMPLIES(element_size_log2 > 0, index().valid()); + DCHECK( + kind.tagged_base + ? ValidOpInputRep(graph, base(), RegisterRepresentation::Tagged()) + : ValidOpInputRep(graph, base(), + {RegisterRepresentation::PointerSized(), + RegisterRepresentation::Tagged()})); + DCHECK(ValidOpInputRep(graph, value(), + stored_rep.ToRegisterRepresentationForStore())); + if (index().valid()) { + DCHECK(ValidOpInputRep(graph, index(), + RegisterRepresentation::PointerSized())); + } } static StoreOp& New(Graph* graph, OpIndex base, OpIndex index, OpIndex value, Kind kind, MemoryRepresentation stored_rep, @@ -1533,6 +1789,51 @@ struct StoreOp : OperationT<StoreOp> { } }; +struct AllocateOp : FixedArityOperationT<1, AllocateOp> { + AllocationType type; + AllowLargeObjects allow_large_objects; + + static constexpr OpProperties properties = OpProperties::PureMayAllocate(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return RepVector<RegisterRepresentation::Tagged()>(); + } + + OpIndex size() const { return input(0); } + + AllocateOp(OpIndex size, AllocationType type, + AllowLargeObjects allow_large_objects) + : Base(size), type(type), allow_large_objects(allow_large_objects) {} + + void Validate(const Graph& graph) const { + DCHECK( + ValidOpInputRep(graph, size(), RegisterRepresentation::PointerSized())); + } + void PrintOptions(std::ostream& os) const; + auto options() const { return std::tuple{type, allow_large_objects}; } +}; + +struct DecodeExternalPointerOp + : FixedArityOperationT<1, DecodeExternalPointerOp> { + ExternalPointerTag tag; + + static constexpr OpProperties properties = OpProperties::PureNoAllocation(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return RepVector<RegisterRepresentation::PointerSized()>(); + } + + OpIndex handle() const { return input(0); } + + DecodeExternalPointerOp(OpIndex handle, ExternalPointerTag tag) + : Base(handle), tag(tag) {} + + void Validate(const Graph& graph) const { + DCHECK_NE(tag, kExternalPointerNullTag); + DCHECK(ValidOpInputRep(graph, handle(), RegisterRepresentation::Word32())); + } + void PrintOptions(std::ostream& os) const; + auto options() const { return std::tuple{tag}; } +}; + // Retain a HeapObject to prevent it from being garbage collected too early. struct RetainOp : FixedArityOperationT<1, RetainOp> { OpIndex retained() const { return input(0); } @@ -1541,8 +1842,14 @@ struct RetainOp : FixedArityOperationT<1, RetainOp> { // this must not be reordered with operations reading from the heap, we mark // it as writing to prevent such reorderings. static constexpr OpProperties properties = OpProperties::Writing(); + base::Vector<const RegisterRepresentation> outputs_rep() const { return {}; } explicit RetainOp(OpIndex retained) : Base(retained) {} + + void Validate(const Graph& graph) const { + DCHECK( + ValidOpInputRep(graph, retained(), RegisterRepresentation::Tagged())); + } auto options() const { return std::tuple{}; } }; @@ -1551,11 +1858,19 @@ struct StackPointerGreaterThanOp StackCheckKind kind; static constexpr OpProperties properties = OpProperties::Reading(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return RepVector<RegisterRepresentation::Word32()>(); + } OpIndex stack_limit() const { return input(0); } StackPointerGreaterThanOp(OpIndex stack_limit, StackCheckKind kind) : Base(stack_limit), kind(kind) {} + + void Validate(const Graph& graph) const { + DCHECK(ValidOpInputRep(graph, stack_limit(), + RegisterRepresentation::PointerSized())); + } auto options() const { return std::tuple{kind}; } }; @@ -1567,8 +1882,12 @@ struct StackSlotOp : FixedArityOperationT<0, StackSlotOp> { int alignment; static constexpr OpProperties properties = OpProperties::Writing(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return RepVector<RegisterRepresentation::PointerSized()>(); + } StackSlotOp(int size, int alignment) : size(size), alignment(alignment) {} + void Validate(const Graph& graph) const {} auto options() const { return std::tuple{size, alignment}; } }; @@ -1579,9 +1898,19 @@ struct FrameConstantOp : FixedArityOperationT<0, FrameConstantOp> { enum class Kind { kStackCheckOffset, kFramePointer, kParentFramePointer }; Kind kind; - static constexpr OpProperties properties = OpProperties::Pure(); + static constexpr OpProperties properties = OpProperties::PureNoAllocation(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + switch (kind) { + case Kind::kStackCheckOffset: + return RepVector<RegisterRepresentation::Tagged()>(); + case Kind::kFramePointer: + case Kind::kParentFramePointer: + return RepVector<RegisterRepresentation::PointerSized()>(); + } + } explicit FrameConstantOp(Kind kind) : Base(), kind(kind) {} + void Validate(const Graph& graph) const {} auto options() const { return std::tuple{kind}; } }; std::ostream& operator<<(std::ostream& os, FrameConstantOp::Kind kind); @@ -1590,7 +1919,8 @@ struct FrameStateOp : OperationT<FrameStateOp> { bool inlined; const FrameStateData* data; - static constexpr OpProperties properties = OpProperties::Pure(); + static constexpr OpProperties properties = OpProperties::PureNoAllocation(); + base::Vector<const RegisterRepresentation> outputs_rep() const { return {}; } OpIndex parent_frame_state() const { DCHECK(inlined); @@ -1601,37 +1931,44 @@ struct FrameStateOp : OperationT<FrameStateOp> { if (inlined) result += 1; return result; } + uint16_t state_values_count() const { + DCHECK_EQ(input_count - inlined, state_values().size()); + return input_count - inlined; + } + const OpIndex state_value(size_t idx) const { return state_values()[idx]; } + + RegisterRepresentation state_value_rep(size_t idx) const { + return RegisterRepresentation::FromMachineRepresentation( + data->machine_types[idx].representation()); + } FrameStateOp(base::Vector<const OpIndex> inputs, bool inlined, const FrameStateData* data) : Base(inputs), inlined(inlined), data(data) {} + + void Validate(const Graph& graph) const { + if (inlined) { + DCHECK(Get(graph, parent_frame_state()).Is<FrameStateOp>()); + } + // TODO(tebbi): Check frame state inputs using `FrameStateData`. + } void PrintOptions(std::ostream& os) const; auto options() const { return std::tuple{inlined, data}; } }; -// CheckLazyDeoptOp should always immediately follow a call. -// Semantically, it deopts if the current code object has been -// deoptimized. But this might also be implemented differently. -struct CheckLazyDeoptOp : FixedArityOperationT<2, CheckLazyDeoptOp> { - static constexpr OpProperties properties = OpProperties::CanAbort(); - - OpIndex call() const { return input(0); } - OpIndex frame_state() const { return input(1); } - - CheckLazyDeoptOp(OpIndex call, OpIndex frame_state) - : Base(call, frame_state) {} - auto options() const { return std::tuple{}; } -}; - struct DeoptimizeOp : FixedArityOperationT<1, DeoptimizeOp> { const DeoptimizeParameters* parameters; static constexpr OpProperties properties = OpProperties::BlockTerminator(); + base::Vector<const RegisterRepresentation> outputs_rep() const { return {}; } OpIndex frame_state() const { return input(0); } DeoptimizeOp(OpIndex frame_state, const DeoptimizeParameters* parameters) : Base(frame_state), parameters(parameters) {} + void Validate(const Graph& graph) const { + DCHECK(Get(graph, frame_state()).Is<FrameStateOp>()); + } auto options() const { return std::tuple{parameters}; } }; @@ -1640,6 +1977,7 @@ struct DeoptimizeIfOp : FixedArityOperationT<2, DeoptimizeIfOp> { const DeoptimizeParameters* parameters; static constexpr OpProperties properties = OpProperties::CanAbort(); + base::Vector<const RegisterRepresentation> outputs_rep() const { return {}; } OpIndex condition() const { return input(0); } OpIndex frame_state() const { return input(1); } @@ -1649,6 +1987,11 @@ struct DeoptimizeIfOp : FixedArityOperationT<2, DeoptimizeIfOp> { : Base(condition, frame_state), negated(negated), parameters(parameters) {} + + void Validate(const Graph& graph) const { + DCHECK( + ValidOpInputRep(graph, condition(), RegisterRepresentation::Word32())); + } auto options() const { return std::tuple{negated, parameters}; } }; @@ -1657,66 +2000,223 @@ struct TrapIfOp : FixedArityOperationT<1, TrapIfOp> { const TrapId trap_id; static constexpr OpProperties properties = OpProperties::CanAbort(); + base::Vector<const RegisterRepresentation> outputs_rep() const { return {}; } OpIndex condition() const { return input(0); } TrapIfOp(OpIndex condition, bool negated, const TrapId trap_id) : Base(condition), negated(negated), trap_id(trap_id) {} + + void Validate(const Graph& graph) const { + DCHECK( + ValidOpInputRep(graph, condition(), RegisterRepresentation::Word32())); + } auto options() const { return std::tuple{negated, trap_id}; } }; +struct StaticAssertOp : FixedArityOperationT<1, StaticAssertOp> { + static constexpr OpProperties properties = OpProperties::CanAbort(); + base::Vector<const RegisterRepresentation> outputs_rep() const { return {}; } + const char* source; + + OpIndex condition() const { return Base::input(0); } + + StaticAssertOp(OpIndex condition, const char* source) + : Base(condition), source(source) {} + + void Validate(const Graph& graph) const { + DCHECK( + ValidOpInputRep(graph, condition(), RegisterRepresentation::Word32())); + } + auto options() const { return std::tuple{source}; } +}; + struct ParameterOp : FixedArityOperationT<0, ParameterOp> { int32_t parameter_index; + RegisterRepresentation rep; const char* debug_name; - static constexpr OpProperties properties = OpProperties::Pure(); + static constexpr OpProperties properties = OpProperties::PureNoAllocation(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return {&rep, 1}; + } - explicit ParameterOp(int32_t parameter_index, const char* debug_name = "") - : Base(), parameter_index(parameter_index), debug_name(debug_name) {} - auto options() const { return std::tuple{parameter_index, debug_name}; } + explicit ParameterOp(int32_t parameter_index, RegisterRepresentation rep, + const char* debug_name = "") + : Base(), + parameter_index(parameter_index), + rep(rep), + debug_name(debug_name) {} + void Validate(const Graph& graph) const {} + auto options() const { return std::tuple{parameter_index, rep, debug_name}; } void PrintOptions(std::ostream& os) const; }; struct OsrValueOp : FixedArityOperationT<0, OsrValueOp> { int32_t index; - static constexpr OpProperties properties = OpProperties::Pure(); + static constexpr OpProperties properties = OpProperties::PureNoAllocation(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return RepVector<RegisterRepresentation::Tagged()>(); + } explicit OsrValueOp(int32_t index) : Base(), index(index) {} + void Validate(const Graph& graph) const {} auto options() const { return std::tuple{index}; } }; -struct CallOp : OperationT<CallOp> { +struct TSCallDescriptor : public NON_EXPORTED_BASE(ZoneObject) { const CallDescriptor* descriptor; + base::Vector<const RegisterRepresentation> out_reps; + + TSCallDescriptor(const CallDescriptor* descriptor, + base::Vector<const RegisterRepresentation> out_reps) + : descriptor(descriptor), out_reps(out_reps) {} + + static const TSCallDescriptor* Create(const CallDescriptor* descriptor, + Zone* graph_zone) { + base::Vector<RegisterRepresentation> out_reps = + graph_zone->NewVector<RegisterRepresentation>( + descriptor->ReturnCount()); + for (size_t i = 0; i < descriptor->ReturnCount(); ++i) { + out_reps[i] = RegisterRepresentation::FromMachineRepresentation( + descriptor->GetReturnType(i).representation()); + } + return graph_zone->New<TSCallDescriptor>(descriptor, out_reps); + } +}; + +struct CallOp : OperationT<CallOp> { + const TSCallDescriptor* descriptor; static constexpr OpProperties properties = OpProperties::AnySideEffects(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return descriptor->out_reps; + } + + bool HasFrameState() const { + return descriptor->descriptor->NeedsFrameState(); + } OpIndex callee() const { return input(0); } + OpIndex frame_state() const { + return HasFrameState() ? input(1) : OpIndex::Invalid(); + } base::Vector<const OpIndex> arguments() const { - return inputs().SubVector(1, input_count); + return inputs().SubVector(1 + HasFrameState(), input_count); } - CallOp(OpIndex callee, base::Vector<const OpIndex> arguments, - const CallDescriptor* descriptor) - : Base(1 + arguments.size()), descriptor(descriptor) { + CallOp(OpIndex callee, OpIndex frame_state, + base::Vector<const OpIndex> arguments, + const TSCallDescriptor* descriptor) + : Base(1 + frame_state.valid() + arguments.size()), + descriptor(descriptor) { base::Vector<OpIndex> inputs = this->inputs(); inputs[0] = callee; - inputs.SubVector(1, inputs.size()).OverwriteWith(arguments); + if (frame_state.valid()) { + inputs[1] = frame_state; + } + inputs.SubVector(1 + frame_state.valid(), inputs.size()) + .OverwriteWith(arguments); } - static CallOp& New(Graph* graph, OpIndex callee, + void Validate(const Graph& graph) const { + if (frame_state().valid()) { + DCHECK(Get(graph, frame_state()).Is<FrameStateOp>()); + } + // TODO(tebbi): Check call inputs based on `TSCallDescriptor`. + } + + static CallOp& New(Graph* graph, OpIndex callee, OpIndex frame_state, base::Vector<const OpIndex> arguments, - const CallDescriptor* descriptor) { - return Base::New(graph, 1 + arguments.size(), callee, arguments, - descriptor); + const TSCallDescriptor* descriptor) { + return Base::New(graph, 1 + frame_state.valid() + arguments.size(), callee, + frame_state, arguments, descriptor); } auto options() const { return std::tuple{descriptor}; } }; +struct CallAndCatchExceptionOp : OperationT<CallAndCatchExceptionOp> { + const TSCallDescriptor* descriptor; + Block* if_success; + Block* if_exception; + + static constexpr OpProperties properties = + OpProperties::BlockTerminatorWithAnySideEffect(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return descriptor->out_reps; + } + + bool HasFrameState() const { + return descriptor->descriptor->NeedsFrameState(); + } + + OpIndex callee() const { return input(0); } + OpIndex frame_state() const { + return HasFrameState() ? input(1) : OpIndex::Invalid(); + } + base::Vector<const OpIndex> arguments() const { + return inputs().SubVector(1 + HasFrameState(), input_count); + } + + CallAndCatchExceptionOp(OpIndex callee, OpIndex frame_state, + base::Vector<const OpIndex> arguments, + Block* if_success, Block* if_exception, + const TSCallDescriptor* descriptor) + : Base(1 + frame_state.valid() + arguments.size()), + descriptor(descriptor), + if_success(if_success), + if_exception(if_exception) { + base::Vector<OpIndex> inputs = this->inputs(); + inputs[0] = callee; + if (frame_state.valid()) { + inputs[1] = frame_state; + } + inputs.SubVector(1 + frame_state.valid(), inputs.size()) + .OverwriteWith(arguments); + } + + void Validate(const Graph& graph) const { + if (frame_state().valid()) { + DCHECK(Get(graph, frame_state()).Is<FrameStateOp>()); + } + } + + static CallAndCatchExceptionOp& New(Graph* graph, OpIndex callee, + OpIndex frame_state, + base::Vector<const OpIndex> arguments, + Block* if_success, Block* if_exception, + const TSCallDescriptor* descriptor) { + return Base::New(graph, 1 + frame_state.valid() + arguments.size(), callee, + frame_state, arguments, if_success, if_exception, + descriptor); + } + + auto options() const { + return std::tuple{descriptor, if_success, if_exception}; + } +}; + +struct LoadExceptionOp : FixedArityOperationT<0, LoadExceptionOp> { + static constexpr OpProperties properties = OpProperties::Reading(); + + base::Vector<const RegisterRepresentation> outputs_rep() const { + return RepVector<RegisterRepresentation::Tagged()>(); + } + + LoadExceptionOp() : Base() {} + void Validate(const Graph& graph) const {} + + auto options() const { return std::tuple{}; } +}; + struct TailCallOp : OperationT<TailCallOp> { - const CallDescriptor* descriptor; + const TSCallDescriptor* descriptor; static constexpr OpProperties properties = OpProperties::BlockTerminatorWithAnySideEffect(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return descriptor->out_reps; + } OpIndex callee() const { return input(0); } base::Vector<const OpIndex> arguments() const { @@ -1724,15 +2224,16 @@ struct TailCallOp : OperationT<TailCallOp> { } TailCallOp(OpIndex callee, base::Vector<const OpIndex> arguments, - const CallDescriptor* descriptor) + const TSCallDescriptor* descriptor) : Base(1 + arguments.size()), descriptor(descriptor) { base::Vector<OpIndex> inputs = this->inputs(); inputs[0] = callee; inputs.SubVector(1, inputs.size()).OverwriteWith(arguments); } + void Validate(const Graph& graph) const {} static TailCallOp& New(Graph* graph, OpIndex callee, base::Vector<const OpIndex> arguments, - const CallDescriptor* descriptor) { + const TSCallDescriptor* descriptor) { return Base::New(graph, 1 + arguments.size(), callee, arguments, descriptor); } @@ -1742,13 +2243,16 @@ struct TailCallOp : OperationT<TailCallOp> { // Control-flow should never reach here. struct UnreachableOp : FixedArityOperationT<0, UnreachableOp> { static constexpr OpProperties properties = OpProperties::BlockTerminator(); + base::Vector<const RegisterRepresentation> outputs_rep() const { return {}; } UnreachableOp() : Base() {} + void Validate(const Graph& graph) const {} auto options() const { return std::tuple{}; } }; struct ReturnOp : OperationT<ReturnOp> { static constexpr OpProperties properties = OpProperties::BlockTerminator(); + base::Vector<const RegisterRepresentation> outputs_rep() const { return {}; } // Number of additional stack slots to be removed. OpIndex pop_count() const { return input(0); } @@ -1763,6 +2267,11 @@ struct ReturnOp : OperationT<ReturnOp> { inputs[0] = pop_count; inputs.SubVector(1, inputs.size()).OverwriteWith(return_values); } + + void Validate(const Graph& graph) const { + DCHECK( + ValidOpInputRep(graph, pop_count(), RegisterRepresentation::Word32())); + } static ReturnOp& New(Graph* graph, OpIndex pop_count, base::Vector<const OpIndex> return_values) { return Base::New(graph, 1 + return_values.size(), pop_count, return_values); @@ -1774,64 +2283,68 @@ struct GotoOp : FixedArityOperationT<0, GotoOp> { Block* destination; static constexpr OpProperties properties = OpProperties::BlockTerminator(); + base::Vector<const RegisterRepresentation> outputs_rep() const { return {}; } explicit GotoOp(Block* destination) : Base(), destination(destination) {} + void Validate(const Graph& graph) const {} auto options() const { return std::tuple{destination}; } }; struct BranchOp : FixedArityOperationT<1, BranchOp> { Block* if_true; Block* if_false; + BranchHint hint; static constexpr OpProperties properties = OpProperties::BlockTerminator(); + base::Vector<const RegisterRepresentation> outputs_rep() const { return {}; } OpIndex condition() const { return input(0); } - BranchOp(OpIndex condition, Block* if_true, Block* if_false) - : Base(condition), if_true(if_true), if_false(if_false) {} - auto options() const { return std::tuple{if_true, if_false}; } -}; - -// `CatchExceptionOp` has to follow a `CallOp` with a subsequent -// `CheckLazyDeoptOp`. It provides the exception value, which might only be used -// from the `if_exception` successor. -struct CatchExceptionOp : FixedArityOperationT<1, CatchExceptionOp> { - Block* if_success; - Block* if_exception; - - static constexpr OpProperties properties = OpProperties::BlockTerminator(); + BranchOp(OpIndex condition, Block* if_true, Block* if_false, BranchHint hint) + : Base(condition), if_true(if_true), if_false(if_false), hint(hint) {} - OpIndex call() const { return input(0); } - - explicit CatchExceptionOp(OpIndex call, Block* if_success, - Block* if_exception) - : Base(call), if_success(if_success), if_exception(if_exception) {} - auto options() const { return std::tuple{if_success, if_exception}; } + void Validate(const Graph& graph) const { + DCHECK( + ValidOpInputRep(graph, condition(), RegisterRepresentation::Word32())); + } + auto options() const { return std::tuple{if_true, if_false, hint}; } }; struct SwitchOp : FixedArityOperationT<1, SwitchOp> { struct Case { int32_t value; Block* destination; + BranchHint hint; - Case(int32_t value, Block* destination) - : value(value), destination(destination) {} + Case(int32_t value, Block* destination, BranchHint hint) + : value(value), destination(destination), hint(hint) {} bool operator==(const Case& other) const { - return value == other.value && destination == other.destination; + return value == other.value && destination == other.destination && + hint == other.hint; } }; base::Vector<const Case> cases; Block* default_case; + BranchHint default_hint; static constexpr OpProperties properties = OpProperties::BlockTerminator(); + base::Vector<const RegisterRepresentation> outputs_rep() const { return {}; } OpIndex input() const { return Base::input(0); } - SwitchOp(OpIndex input, base::Vector<const Case> cases, Block* default_case) - : Base(input), cases(cases), default_case(default_case) {} + SwitchOp(OpIndex input, base::Vector<const Case> cases, Block* default_case, + BranchHint default_hint) + : Base(input), + cases(cases), + default_case(default_case), + default_hint(default_hint) {} + + void Validate(const Graph& graph) const { + DCHECK(ValidOpInputRep(graph, input(), RegisterRepresentation::Word32())); + } void PrintOptions(std::ostream& os) const; - auto options() const { return std::tuple{cases, default_case}; } + auto options() const { return std::tuple{cases, default_case, default_hint}; } }; template <> @@ -1841,12 +2354,47 @@ struct fast_hash<SwitchOp::Case> { } }; +inline base::SmallVector<Block*, 4> SuccessorBlocks(const Operation& op) { + DCHECK(op.Properties().is_block_terminator); + switch (op.opcode) { + case Opcode::kCallAndCatchException: { + auto& casted = op.Cast<CallAndCatchExceptionOp>(); + return {casted.if_success, casted.if_exception}; + } + case Opcode::kGoto: { + auto& casted = op.Cast<GotoOp>(); + return {casted.destination}; + } + case Opcode::kBranch: { + auto& casted = op.Cast<BranchOp>(); + return {casted.if_true, casted.if_false}; + } + case Opcode::kReturn: + case Opcode::kDeoptimize: + case Opcode::kUnreachable: + return base::SmallVector<Block*, 4>{}; + case Opcode::kSwitch: { + auto& casted = op.Cast<SwitchOp>(); + base::SmallVector<Block*, 4> result; + for (const SwitchOp::Case& c : casted.cases) { + result.push_back(c.destination); + } + result.push_back(casted.default_case); + return result; + } + default: + UNREACHABLE(); + } +} + // Tuples are only used to lower operations with multiple outputs. // `TupleOp` should be folded away by subsequent `ProjectionOp`s. struct TupleOp : OperationT<TupleOp> { - static constexpr OpProperties properties = OpProperties::Pure(); + static constexpr OpProperties properties = OpProperties::PureNoAllocation(); + base::Vector<const RegisterRepresentation> outputs_rep() const { return {}; } explicit TupleOp(base::Vector<const OpIndex> inputs) : Base(inputs) {} + void Validate(const Graph& graph) const {} auto options() const { return std::tuple{}; } }; @@ -1854,15 +2402,845 @@ struct TupleOp : OperationT<TupleOp> { // distinguish them. struct ProjectionOp : FixedArityOperationT<1, ProjectionOp> { uint16_t index; + RegisterRepresentation rep; - static constexpr OpProperties properties = OpProperties::Pure(); + static constexpr OpProperties properties = OpProperties::PureNoAllocation(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return base::VectorOf(&rep, 1); + } OpIndex input() const { return Base::input(0); } - ProjectionOp(OpIndex input, uint16_t index) : Base(input), index(index) {} + ProjectionOp(OpIndex input, uint16_t index, RegisterRepresentation rep) + : Base(input), index(index), rep(rep) {} + + void Validate(const Graph& graph) const { + DCHECK(ValidOpInputRep(graph, input(), rep, index)); + } auto options() const { return std::tuple{index}; } }; +struct CheckTurboshaftTypeOfOp + : FixedArityOperationT<1, CheckTurboshaftTypeOfOp> { + RegisterRepresentation rep; + Type type; + bool successful; + + static constexpr OpProperties properties = OpProperties::AnySideEffects(); + base::Vector<const RegisterRepresentation> outputs_rep() const { return {}; } + + OpIndex input() const { return Base::input(0); } + + CheckTurboshaftTypeOfOp(OpIndex input, RegisterRepresentation rep, Type type, + bool successful) + : Base(input), rep(rep), type(std::move(type)), successful(successful) {} + + void Validate(const Graph& graph) const { + DCHECK(ValidOpInputRep(graph, input(), rep)); + } + auto options() const { return std::tuple{rep, type, successful}; } +}; + +struct ObjectIsOp : FixedArityOperationT<1, ObjectIsOp> { + enum class Kind : uint8_t { + kArrayBufferView, + kBigInt, + kBigInt64, + kCallable, + kConstructor, + kDetectableCallable, + kInternalizedString, + kNonCallable, + kNumber, + kReceiver, + kReceiverOrNullOrUndefined, + kSmi, + kString, + kSymbol, + kUndetectable, + }; + enum class InputAssumptions : uint8_t { + kNone, + kHeapObject, + kBigInt, + }; + Kind kind; + InputAssumptions input_assumptions; + + static constexpr OpProperties properties = OpProperties::Reading(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return RepVector<RegisterRepresentation::Word32()>(); + } + + OpIndex input() const { return Base::input(0); } + + ObjectIsOp(OpIndex input, Kind kind, InputAssumptions input_assumptions) + : Base(input), kind(kind), input_assumptions(input_assumptions) {} + void Validate(const Graph& graph) const { + DCHECK(ValidOpInputRep(graph, input(), RegisterRepresentation::Tagged())); + } + auto options() const { return std::tuple{kind, input_assumptions}; } +}; +std::ostream& operator<<(std::ostream& os, ObjectIsOp::Kind kind); +std::ostream& operator<<(std::ostream& os, + ObjectIsOp::InputAssumptions input_assumptions); + +struct FloatIsOp : FixedArityOperationT<1, FloatIsOp> { + enum class Kind : uint8_t { + kNaN, + }; + Kind kind; + FloatRepresentation input_rep; + + static constexpr OpProperties properties = OpProperties::PureNoAllocation(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return RepVector<RegisterRepresentation::Word32()>(); + } + + OpIndex input() const { return Base::input(0); } + + FloatIsOp(OpIndex input, Kind kind, FloatRepresentation input_rep) + : Base(input), kind(kind), input_rep(input_rep) {} + void Validate(const Graph& graph) const { + DCHECK(ValidOpInputRep(graph, input(), input_rep)); + } + auto options() const { return std::tuple{kind, input_rep}; } +}; +std::ostream& operator<<(std::ostream& os, FloatIsOp::Kind kind); + +struct ConvertToObjectOp : FixedArityOperationT<1, ConvertToObjectOp> { + enum class Kind : uint8_t { + kBigInt, + kBoolean, + kHeapNumber, + kNumber, + kSmi, + kString, + }; + enum class InputInterpretation : uint8_t { + kSigned, + kUnsigned, + kCharCode, + kCodePoint, + }; + Kind kind; + RegisterRepresentation input_rep; + InputInterpretation input_interpretation; + CheckForMinusZeroMode minus_zero_mode; + + static constexpr OpProperties properties = OpProperties::PureMayAllocate(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return RepVector<RegisterRepresentation::Tagged()>(); + } + + OpIndex input() const { return Base::input(0); } + + ConvertToObjectOp(OpIndex input, Kind kind, RegisterRepresentation input_rep, + InputInterpretation input_interpretation, + CheckForMinusZeroMode minus_zero_mode) + : Base(input), + kind(kind), + input_rep(input_rep), + input_interpretation(input_interpretation), + minus_zero_mode(minus_zero_mode) {} + + void Validate(const Graph& graph) const { + switch (kind) { + case Kind::kBigInt: + DCHECK_EQ(input_rep, RegisterRepresentation::Word64()); + DCHECK(ValidOpInputRep(graph, input(), input_rep)); + DCHECK_EQ(minus_zero_mode, + CheckForMinusZeroMode::kDontCheckForMinusZero); + break; + case Kind::kBoolean: + DCHECK_EQ(input_rep, RegisterRepresentation::Word32()); + DCHECK(ValidOpInputRep(graph, input(), input_rep)); + DCHECK_EQ(minus_zero_mode, + CheckForMinusZeroMode::kDontCheckForMinusZero); + break; + case Kind::kNumber: + case Kind::kHeapNumber: + DCHECK(ValidOpInputRep(graph, input(), input_rep)); + DCHECK_IMPLIES( + minus_zero_mode == CheckForMinusZeroMode::kCheckForMinusZero, + input_rep == RegisterRepresentation::Float64()); + break; + case Kind::kSmi: + DCHECK_EQ(input_rep, WordRepresentation::Word32()); + DCHECK_EQ(minus_zero_mode, + CheckForMinusZeroMode::kDontCheckForMinusZero); + DCHECK(ValidOpInputRep(graph, input(), input_rep)); + break; + case Kind::kString: + DCHECK_EQ(input_rep, WordRepresentation::Word32()); + DCHECK_EQ(input_interpretation, + any_of(InputInterpretation::kCharCode, + InputInterpretation::kCodePoint)); + DCHECK(ValidOpInputRep(graph, input(), input_rep)); + break; + } + } + + auto options() const { + return std::tuple{kind, input_rep, input_interpretation, minus_zero_mode}; + } +}; +std::ostream& operator<<(std::ostream& os, ConvertToObjectOp::Kind kind); + +struct ConvertToObjectOrDeoptOp + : FixedArityOperationT<2, ConvertToObjectOrDeoptOp> { + enum class Kind : uint8_t { + kSmi, + }; + enum class InputInterpretation : uint8_t { + kSigned, + kUnsigned, + }; + Kind kind; + RegisterRepresentation input_rep; + InputInterpretation input_interpretation; + FeedbackSource feedback; + + static constexpr OpProperties properties = OpProperties::CanAbort(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return RepVector<RegisterRepresentation::Tagged()>(); + } + + OpIndex input() const { return Base::input(0); } + OpIndex frame_state() const { return Base::input(1); } + + ConvertToObjectOrDeoptOp(OpIndex input, OpIndex frame_state, Kind kind, + RegisterRepresentation input_rep, + InputInterpretation input_interpretation, + const FeedbackSource& feedback) + : Base(input, frame_state), + kind(kind), + input_rep(input_rep), + input_interpretation(input_interpretation), + feedback(feedback) {} + + void Validate(const Graph& graph) const { + DCHECK(ValidOpInputRep(graph, input(), input_rep)); + } + + auto options() const { + return std::tuple{kind, input_rep, input_interpretation, feedback}; + } +}; +std::ostream& operator<<(std::ostream& os, ConvertToObjectOrDeoptOp::Kind kind); +std::ostream& operator<<( + std::ostream& os, + ConvertToObjectOrDeoptOp::InputInterpretation input_interpretation); + +struct ConvertObjectToPrimitiveOp + : FixedArityOperationT<1, ConvertObjectToPrimitiveOp> { + enum class Kind : uint8_t { + kInt32, + kInt64, + kUint32, + kBit, + kFloat64, + }; + enum class InputAssumptions : uint8_t { + kObject, + kSmi, + kNumberOrOddball, + }; + Kind kind; + InputAssumptions input_assumptions; + + static constexpr OpProperties properties = OpProperties::PureNoAllocation(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + switch (kind) { + case Kind::kInt32: + case Kind::kUint32: + case Kind::kBit: + return RepVector<RegisterRepresentation::Word32()>(); + case Kind::kInt64: + return RepVector<RegisterRepresentation::Word64()>(); + case Kind::kFloat64: + return RepVector<RegisterRepresentation::Float64()>(); + } + } + + OpIndex input() const { return Base::input(0); } + + ConvertObjectToPrimitiveOp(OpIndex input, Kind kind, + InputAssumptions input_assumptions) + : Base(input), kind(kind), input_assumptions(input_assumptions) {} + void Validate(const Graph& graph) const { + DCHECK(ValidOpInputRep(graph, input(), RegisterRepresentation::Tagged())); + } + + auto options() const { return std::tuple{kind, input_assumptions}; } +}; +std::ostream& operator<<(std::ostream& os, + ConvertObjectToPrimitiveOp::Kind kind); +std::ostream& operator<<( + std::ostream& os, + ConvertObjectToPrimitiveOp::InputAssumptions input_assumptions); + +struct ConvertObjectToPrimitiveOrDeoptOp + : FixedArityOperationT<2, ConvertObjectToPrimitiveOrDeoptOp> { + enum class PrimitiveKind : uint8_t { + kInt32, + kInt64, + kFloat64, + kArrayIndex, + }; + enum class ObjectKind : uint8_t { + kNumber, + kNumberOrBoolean, + kNumberOrOddball, + kNumberOrString, + kSmi, + }; + ObjectKind from_kind; + PrimitiveKind to_kind; + CheckForMinusZeroMode minus_zero_mode; + FeedbackSource feedback; + + static constexpr OpProperties properties = OpProperties::ReadingAndCanAbort(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + switch (to_kind) { + case PrimitiveKind::kInt32: + return RepVector<RegisterRepresentation::Word32()>(); + case PrimitiveKind::kInt64: + return RepVector<RegisterRepresentation::Word64()>(); + case PrimitiveKind::kFloat64: + return RepVector<RegisterRepresentation::Float64()>(); + case PrimitiveKind::kArrayIndex: + return Is64() ? RepVector<RegisterRepresentation::Word64()>() + : RepVector<RegisterRepresentation::Word32()>(); + } + } + + OpIndex input() const { return Base::input(0); } + OpIndex frame_state() const { return Base::input(1); } + + ConvertObjectToPrimitiveOrDeoptOp(OpIndex input, OpIndex frame_state, + ObjectKind from_kind, PrimitiveKind to_kind, + CheckForMinusZeroMode minus_zero_mode, + const FeedbackSource& feedback) + : Base(input, frame_state), + from_kind(from_kind), + to_kind(to_kind), + minus_zero_mode(minus_zero_mode), + feedback(feedback) {} + void Validate(const Graph& graph) const { + DCHECK(ValidOpInputRep(graph, input(), RegisterRepresentation::Tagged())); + DCHECK(Get(graph, frame_state()).Is<FrameStateOp>()); + } + + auto options() const { + return std::tuple{from_kind, to_kind, minus_zero_mode, feedback}; + } +}; +std::ostream& operator<<(std::ostream& os, + ConvertObjectToPrimitiveOrDeoptOp::ObjectKind kind); +std::ostream& operator<<(std::ostream& os, + ConvertObjectToPrimitiveOrDeoptOp::PrimitiveKind kind); + +struct TruncateObjectToPrimitiveOp + : FixedArityOperationT<1, TruncateObjectToPrimitiveOp> { + enum class Kind : uint8_t { + kInt32, + kInt64, + kBit, + }; + enum class InputAssumptions : uint8_t { + kBigInt, + kNumberOrOddball, + kHeapObject, + kObject, + }; + Kind kind; + InputAssumptions input_assumptions; + + static constexpr OpProperties properties = OpProperties::PureNoAllocation(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + switch (kind) { + case Kind::kInt32: + case Kind::kBit: + return RepVector<RegisterRepresentation::Word32()>(); + case Kind::kInt64: + return RepVector<RegisterRepresentation::Word64()>(); + } + } + + OpIndex input() const { return Base::input(0); } + + TruncateObjectToPrimitiveOp(OpIndex input, Kind kind, + InputAssumptions input_assumptions) + : Base(input), kind(kind), input_assumptions(input_assumptions) {} + void Validate(const Graph& graph) const { + DCHECK(ValidOpInputRep(graph, input(), RegisterRepresentation::Tagged())); + } + + auto options() const { return std::tuple{kind, input_assumptions}; } +}; +std::ostream& operator<<(std::ostream& os, + TruncateObjectToPrimitiveOp::Kind kind); +std::ostream& operator<<( + std::ostream& os, + TruncateObjectToPrimitiveOp::InputAssumptions input_assumptions); + +enum class TagKind { + kSmiTag, +}; +std::ostream& operator<<(std::ostream& os, TagKind kind); + +struct TagOp : FixedArityOperationT<1, TagOp> { + TagKind kind; + + static constexpr OpProperties properties = OpProperties::PureNoAllocation(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return RepVector<RegisterRepresentation::Tagged()>(); + } + + OpIndex input() const { return Base::input(0); } + + TagOp(OpIndex input, TagKind kind) : Base(input), kind(kind) {} + void Validate(const Graph& graph) const { + DCHECK(ValidOpInputRep(graph, input(), RegisterRepresentation::Word32())); + } + + auto options() const { return std::tuple{kind}; } +}; + +struct UntagOp : FixedArityOperationT<1, UntagOp> { + TagKind kind; + RegisterRepresentation rep; + + static constexpr OpProperties properties = OpProperties::PureNoAllocation(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return base::VectorOf(&rep, 1); + } + + OpIndex input() const { return Base::input(0); } + + UntagOp(OpIndex input, TagKind kind, RegisterRepresentation rep) + : Base(input), kind(kind), rep(rep) {} + void Validate(const Graph& graph) const { + DCHECK(ValidOpInputRep(graph, input(), RegisterRepresentation::Tagged())); + } + + auto options() const { return std::tuple{kind, rep}; } +}; + +struct NewConsStringOp : FixedArityOperationT<3, NewConsStringOp> { + static constexpr OpProperties properties = OpProperties::PureMayAllocate(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return RepVector<RegisterRepresentation::Tagged()>(); + } + + OpIndex length() const { return Base::input(0); } + OpIndex first() const { return Base::input(1); } + OpIndex second() const { return Base::input(2); } + + NewConsStringOp(OpIndex length, OpIndex first, OpIndex second) + : Base(length, first, second) {} + void Validate(const Graph& graph) const { + DCHECK(ValidOpInputRep(graph, length(), RegisterRepresentation::Word32())); + DCHECK(ValidOpInputRep(graph, first(), RegisterRepresentation::Tagged())); + DCHECK(ValidOpInputRep(graph, second(), RegisterRepresentation::Tagged())); + } + + auto options() const { return std::tuple{}; } +}; + +struct NewArrayOp : FixedArityOperationT<1, NewArrayOp> { + enum class Kind : uint8_t { + kDouble, + kObject, + }; + Kind kind; + AllocationType allocation_type; + + static constexpr OpProperties properties = OpProperties::PureMayAllocate(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return RepVector<RegisterRepresentation::Tagged()>(); + } + + OpIndex length() const { return Base::input(0); } + + NewArrayOp(OpIndex length, Kind kind, AllocationType allocation_type) + : Base(length), kind(kind), allocation_type(allocation_type) {} + void Validate(const Graph& graph) const { + DCHECK(ValidOpInputRep(graph, length(), + RegisterRepresentation::PointerSized())); + } + + auto options() const { return std::tuple{kind, allocation_type}; } +}; +std::ostream& operator<<(std::ostream& os, NewArrayOp::Kind kind); + +struct DoubleArrayMinMaxOp : FixedArityOperationT<1, DoubleArrayMinMaxOp> { + enum class Kind : uint8_t { + kMin, + kMax, + }; + Kind kind; + + static constexpr OpProperties properties = OpProperties::PureMayAllocate(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return RepVector<RegisterRepresentation::Tagged()>(); + } + + OpIndex array() const { return Base::input(0); } + + DoubleArrayMinMaxOp(OpIndex array, Kind kind) : Base(array), kind(kind) {} + void Validate(const Graph& graph) const { + DCHECK(ValidOpInputRep(graph, array(), RegisterRepresentation::Tagged())); + } + + auto options() const { return std::tuple{kind}; } +}; +std::ostream& operator<<(std::ostream& os, DoubleArrayMinMaxOp::Kind kind); + +// TODO(nicohartmann@): We should consider getting rid of the LoadFieldByIndex +// operation. +struct LoadFieldByIndexOp : FixedArityOperationT<2, LoadFieldByIndexOp> { + static constexpr OpProperties properties = OpProperties::PureMayAllocate(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return RepVector<RegisterRepresentation::Tagged()>(); + } + + OpIndex object() const { return Base::input(0); } + // Index encoding (see `src/objects/field-index-inl.h`): + // For efficiency, the LoadByFieldIndex instruction takes an index that is + // optimized for quick access. If the property is inline, the index is + // positive. If it's out-of-line, the encoded index is -raw_index - 1 to + // disambiguate the zero out-of-line index from the zero inobject case. + // The index itself is shifted up by one bit, the lower-most bit + // signifying if the field is a mutable double box (1) or not (0). + OpIndex index() const { return Base::input(1); } + + LoadFieldByIndexOp(OpIndex object, OpIndex index) : Base(object, index) {} + void Validate(const Graph& graph) const { + DCHECK(ValidOpInputRep(graph, object(), RegisterRepresentation::Tagged())); + DCHECK(ValidOpInputRep(graph, index(), RegisterRepresentation::Word32())); + } + + auto options() const { return std::tuple{}; } +}; + +struct DebugBreakOp : FixedArityOperationT<0, DebugBreakOp> { + static constexpr OpProperties properties = OpProperties::AnySideEffects(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return RepVector<>(); + } + + DebugBreakOp() : Base() {} + void Validate(const Graph& graph) const {} + + auto options() const { return std::tuple{}; } +}; + +struct BigIntBinopOp : FixedArityOperationT<3, BigIntBinopOp> { + enum class Kind : uint8_t { + kAdd, + kSub, + kMul, + kDiv, + kMod, + kBitwiseAnd, + kBitwiseOr, + kBitwiseXor, + kShiftLeft, + kShiftRightArithmetic, + }; + Kind kind; + + // TODO(nicohartmann@): Maybe we can specify more precise properties here. + // These operations can deopt (abort), allocate and read immutable data. + static constexpr OpProperties properties = OpProperties::AnySideEffects(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return RepVector<RegisterRepresentation::Tagged()>(); + } + + OpIndex left() const { return Base::input(0); } + OpIndex right() const { return Base::input(1); } + OpIndex frame_state() const { return Base::input(2); } + + BigIntBinopOp(OpIndex left, OpIndex right, OpIndex frame_state, Kind kind) + : Base(left, right, frame_state), kind(kind) {} + void Validate(const Graph& graph) const { + DCHECK(ValidOpInputRep(graph, left(), RegisterRepresentation::Tagged())); + DCHECK(ValidOpInputRep(graph, right(), RegisterRepresentation::Tagged())); + DCHECK(Get(graph, frame_state()).Is<FrameStateOp>()); + } + + auto options() const { return std::tuple{kind}; } +}; +std::ostream& operator<<(std::ostream& os, BigIntBinopOp::Kind kind); + +struct BigIntEqualOp : FixedArityOperationT<2, BigIntEqualOp> { + static constexpr OpProperties properties = OpProperties::PureNoAllocation(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return RepVector<RegisterRepresentation::Tagged()>(); + } + + OpIndex left() const { return Base::input(0); } + OpIndex right() const { return Base::input(1); } + + BigIntEqualOp(OpIndex left, OpIndex right) : Base(left, right) {} + + void Validate(const Graph& graph) const { + DCHECK(ValidOpInputRep(graph, left(), RegisterRepresentation::Tagged())); + DCHECK(ValidOpInputRep(graph, right(), RegisterRepresentation::Tagged())); + } + + auto options() const { return std::tuple{}; } +}; + +struct BigIntComparisonOp : FixedArityOperationT<2, BigIntComparisonOp> { + enum class Kind : uint8_t { + kLessThan, + kLessThanOrEqual, + }; + Kind kind; + + static constexpr OpProperties properties = OpProperties::PureNoAllocation(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return RepVector<RegisterRepresentation::Tagged()>(); + } + + OpIndex left() const { return Base::input(0); } + OpIndex right() const { return Base::input(1); } + + BigIntComparisonOp(OpIndex left, OpIndex right, Kind kind) + : Base(left, right), kind(kind) {} + + void Validate(const Graph& graph) const { + DCHECK(ValidOpInputRep(graph, left(), RegisterRepresentation::Tagged())); + DCHECK(ValidOpInputRep(graph, right(), RegisterRepresentation::Tagged())); + } + + auto options() const { return std::tuple{kind}; } +}; +std::ostream& operator<<(std::ostream& os, BigIntComparisonOp::Kind kind); + +struct BigIntUnaryOp : FixedArityOperationT<1, BigIntUnaryOp> { + enum class Kind : uint8_t { + kNegate, + }; + Kind kind; + + static constexpr OpProperties properties = OpProperties::PureMayAllocate(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return RepVector<RegisterRepresentation::Tagged()>(); + } + + OpIndex input() const { return Base::input(0); } + + BigIntUnaryOp(OpIndex input, Kind kind) : Base(input), kind(kind) {} + + void Validate(const Graph& graph) const { + DCHECK(ValidOpInputRep(graph, input(), RegisterRepresentation::Tagged())); + } + + auto options() const { return std::tuple{kind}; } +}; +std::ostream& operator<<(std::ostream& os, BigIntUnaryOp::Kind kind); + +struct LoadRootRegisterOp : FixedArityOperationT<0, LoadRootRegisterOp> { + static constexpr OpProperties properties = OpProperties::PureNoAllocation(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return RepVector<RegisterRepresentation::PointerSized()>(); + } + + LoadRootRegisterOp() : Base() {} + void Validate(const Graph& graph) const {} + std::tuple<> options() const { return {}; } +}; + +struct StringAtOp : FixedArityOperationT<2, StringAtOp> { + enum class Kind : uint8_t { + kCharCode, + kCodePoint, + }; + Kind kind; + + static constexpr OpProperties properties = OpProperties::PureNoAllocation(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return RepVector<RegisterRepresentation::Word32()>(); + } + + OpIndex string() const { return Base::input(0); } + OpIndex position() const { return Base::input(1); } + + StringAtOp(OpIndex string, OpIndex position, Kind kind) + : Base(string, position), kind(kind) {} + + void Validate(const Graph& graph) const { + DCHECK(ValidOpInputRep(graph, string(), RegisterRepresentation::Tagged())); + DCHECK(ValidOpInputRep(graph, position(), + RegisterRepresentation::PointerSized())); + } + + auto options() const { return std::tuple{kind}; } +}; +std::ostream& operator<<(std::ostream& os, StringAtOp::Kind kind); + +#ifdef V8_INTL_SUPPORT +struct StringToCaseIntlOp : FixedArityOperationT<1, StringToCaseIntlOp> { + enum class Kind : uint8_t { + kLower, + kUpper, + }; + Kind kind; + + static constexpr OpProperties properties = OpProperties::PureMayAllocate(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return RepVector<RegisterRepresentation::Tagged()>(); + } + + OpIndex string() const { return Base::input(0); } + + StringToCaseIntlOp(OpIndex string, Kind kind) : Base(string), kind(kind) {} + + void Validate(const Graph& graph) const { + DCHECK(ValidOpInputRep(graph, string(), RegisterRepresentation::Tagged())); + } + + auto options() const { return std::tuple{kind}; } +}; +std::ostream& operator<<(std::ostream& os, StringToCaseIntlOp::Kind kind); +#endif // V8_INTL_SUPPORT + +struct StringLengthOp : FixedArityOperationT<1, StringLengthOp> { + static constexpr OpProperties properties = OpProperties::PureNoAllocation(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return RepVector<RegisterRepresentation::Word32()>(); + } + + OpIndex string() const { return Base::input(0); } + + explicit StringLengthOp(OpIndex string) : Base(string) {} + + void Validate(const Graph& graph) const { + DCHECK(ValidOpInputRep(graph, string(), RegisterRepresentation::Tagged())); + } + + auto options() const { return std::tuple{}; } +}; + +struct StringIndexOfOp : FixedArityOperationT<3, StringIndexOfOp> { + static constexpr OpProperties properties = OpProperties::PureMayAllocate(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return RepVector<RegisterRepresentation::Tagged()>(); + } + + // Search the string `search` within the string `string` starting at + // `position`. + OpIndex string() const { return Base::input(0); } + OpIndex search() const { return Base::input(1); } + OpIndex position() const { return Base::input(2); } + + StringIndexOfOp(OpIndex string, OpIndex search, OpIndex position) + : Base(string, search, position) {} + + void Validate(const Graph& graph) const { + DCHECK(ValidOpInputRep(graph, string(), RegisterRepresentation::Tagged())); + DCHECK(ValidOpInputRep(graph, search(), RegisterRepresentation::Tagged())); + DCHECK( + ValidOpInputRep(graph, position(), RegisterRepresentation::Tagged())); + } + + auto options() const { return std::tuple{}; } +}; + +struct StringFromCodePointAtOp + : FixedArityOperationT<2, StringFromCodePointAtOp> { + static constexpr OpProperties properties = OpProperties::PureMayAllocate(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return RepVector<RegisterRepresentation::Tagged()>(); + } + + OpIndex string() const { return Base::input(0); } + OpIndex index() const { return Base::input(1); } + + StringFromCodePointAtOp(OpIndex string, OpIndex index) + : Base(string, index) {} + + void Validate(const Graph& graph) const { + DCHECK(ValidOpInputRep(graph, string(), RegisterRepresentation::Tagged())); + DCHECK(ValidOpInputRep(graph, index(), + RegisterRepresentation::PointerSized())); + } + + auto options() const { return std::tuple{}; } +}; + +struct StringSubstringOp : FixedArityOperationT<3, StringSubstringOp> { + static constexpr OpProperties properties = OpProperties::PureMayAllocate(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return RepVector<RegisterRepresentation::Tagged()>(); + } + + OpIndex string() const { return Base::input(0); } + OpIndex start() const { return Base::input(1); } + OpIndex end() const { return Base::input(2); } + + StringSubstringOp(OpIndex string, OpIndex start, OpIndex end) + : Base(string, start, end) {} + + void Validate(const Graph& graph) const { + DCHECK(ValidOpInputRep(graph, string(), RegisterRepresentation::Tagged())); + DCHECK(ValidOpInputRep(graph, start(), RegisterRepresentation::Word32())); + DCHECK(ValidOpInputRep(graph, end(), RegisterRepresentation::Word32())); + } + + auto options() const { return std::tuple{}; } +}; + +struct StringEqualOp : FixedArityOperationT<2, StringEqualOp> { + static constexpr OpProperties properties = OpProperties::PureNoAllocation(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return RepVector<RegisterRepresentation::Tagged()>(); + } + + OpIndex left() const { return Base::input(0); } + OpIndex right() const { return Base::input(1); } + + StringEqualOp(OpIndex left, OpIndex right) : Base(left, right) {} + + void Validate(const Graph& graph) const { + DCHECK(ValidOpInputRep(graph, left(), RegisterRepresentation::Tagged())); + DCHECK(ValidOpInputRep(graph, right(), RegisterRepresentation::Tagged())); + } + + auto options() const { return std::tuple{}; } +}; + +struct StringComparisonOp : FixedArityOperationT<2, StringComparisonOp> { + enum class Kind : uint8_t { + kLessThan, + kLessThanOrEqual, + }; + Kind kind; + + static constexpr OpProperties properties = OpProperties::PureNoAllocation(); + base::Vector<const RegisterRepresentation> outputs_rep() const { + return RepVector<RegisterRepresentation::Tagged()>(); + } + + OpIndex left() const { return Base::input(0); } + OpIndex right() const { return Base::input(1); } + + StringComparisonOp(OpIndex left, OpIndex right, Kind kind) + : Base(left, right), kind(kind) {} + + void Validate(const Graph& graph) const { + DCHECK(ValidOpInputRep(graph, left(), RegisterRepresentation::Tagged())); + DCHECK(ValidOpInputRep(graph, right(), RegisterRepresentation::Tagged())); + } + + auto options() const { return std::tuple{kind}; } +}; +std::ostream& operator<<(std::ostream& os, StringComparisonOp::Kind kind); + #define OPERATION_PROPERTIES_CASE(Name) Name##Op::PropertiesIfStatic(), static constexpr base::Optional<OpProperties> kOperationPropertiesTable[kNumberOfOpcodes] = { @@ -1929,6 +3307,31 @@ inline size_t Operation::StorageSlotCount(Opcode opcode, size_t input_count) { return std::max<size_t>(2, (r - 1 + size + input_count) / r); } +template <class Op> +V8_INLINE bool CanBeUsedAsInput(const Op& op) { + if (std::is_same<Op, FrameStateOp>::value) { + // FrameStateOp is the only Operation that can be used as an input but has + // empty `outputs_rep`. + return true; + } + // For all other Operations, they can only be used as an input if they have at + // least one output. + return op.outputs_rep().size() > 0; +} + +inline base::Vector<const RegisterRepresentation> Operation::outputs_rep() + const { + switch (opcode) { +#define CASE(type) \ + case Opcode::k##type: { \ + const type##Op& op = Cast<type##Op>(); \ + return op.outputs_rep(); \ + } + TURBOSHAFT_OPERATION_LIST(CASE) +#undef CASE + } +} + } // namespace v8::internal::compiler::turboshaft #endif // V8_COMPILER_TURBOSHAFT_OPERATIONS_H_ |