/* * Copyright 2016 Mozilla Foundation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // This is an INTERNAL header for Wasm baseline compiler: the compiler object // and its supporting types. #ifndef wasm_wasm_baseline_object_h #define wasm_wasm_baseline_object_h #include "jit/PerfSpewer.h" #include "wasm/WasmBCDefs.h" #include "wasm/WasmBCFrame.h" #include "wasm/WasmBCRegDefs.h" #include "wasm/WasmBCStk.h" #include "wasm/WasmConstants.h" namespace js { namespace wasm { struct StackMap; // Container for a piece of out-of-line code, the slow path that supports an // operation. class OutOfLineCode; // Part of the inter-bytecode state for the boolean-evaluation-for-control // optimization. struct BranchState; // Representation of wasm local variables. using Local = BaseStackFrame::Local; // Bitset used for simple bounds check elimination. Capping this at 64 locals // makes sense; even 32 locals would probably be OK in practice. // // For more information about BCE, see the block comment in WasmBCMemory.cpp. using BCESet = uint64_t; // Information stored in the control node for generating exception handling // landing pads. struct CatchInfo { uint32_t tagIndex; // Index for the associated exception. NonAssertingLabel label; // The entry label for the handler. explicit CatchInfo(uint32_t tagIndex_) : tagIndex(tagIndex_) {} }; using CatchInfoVector = Vector; // Control node, representing labels and stack heights at join points. struct Control { NonAssertingLabel label; // The "exit" label NonAssertingLabel otherLabel; // Used for the "else" branch of if-then-else // and to allow delegate to jump to catches. StackHeight stackHeight; // From BaseStackFrame uint32_t stackSize; // Value stack height BCESet bceSafeOnEntry; // Bounds check info flowing into the item BCESet bceSafeOnExit; // Bounds check info flowing out of the item bool deadOnArrival; // deadCode_ was set on entry to the region bool deadThenBranch; // deadCode_ was set on exit from "then" size_t tryNoteIndex; // For tracking try branch code ranges. CatchInfoVector catchInfos; // Used for try-catch handlers. size_t loopBytecodeStart; // For LT: bytecode offset of start of a loop. CodeOffset offsetOfCtrDec; // For LT: masm offset of loop's counter decr. Control() : stackHeight(StackHeight::Invalid()), stackSize(UINT32_MAX), bceSafeOnEntry(0), bceSafeOnExit(~BCESet(0)), deadOnArrival(false), deadThenBranch(false), tryNoteIndex(0), loopBytecodeStart(UINTPTR_MAX), offsetOfCtrDec(CodeOffset()) {} Control(Control&&) = default; Control(const Control&) = delete; }; // A vector of Nothing values, used for reading opcodes. class BaseNothingVector { mozilla::Nothing unused_; public: bool reserve(size_t size) { return true; } bool resize(size_t length) { return true; } mozilla::Nothing& operator[](size_t) { return unused_; } mozilla::Nothing& back() { return unused_; } size_t length() const { return 0; } bool append(mozilla::Nothing& nothing) { return true; } void infallibleAppend(mozilla::Nothing& nothing) {} }; // The baseline compiler tracks values on a stack of its own -- it needs to scan // that stack for spilling -- and thus has no need for the values maintained by // the iterator. struct BaseCompilePolicy { using Value = mozilla::Nothing; using ValueVector = BaseNothingVector; // The baseline compiler uses the iterator's control stack, attaching // its own control information. using ControlItem = Control; }; using BaseOpIter = OpIter; // Latent operation for boolean-evaluation-for-control optimization. enum class LatentOp { None, Compare, Eqz }; // Encapsulate the checking needed for a memory access. struct AccessCheck { AccessCheck() : omitBoundsCheck(false), omitAlignmentCheck(false), onlyPointerAlignment(false) {} // If `omitAlignmentCheck` is true then we need check neither the // pointer nor the offset. Otherwise, if `onlyPointerAlignment` is true // then we need check only the pointer. Otherwise, check the sum of // pointer and offset. bool omitBoundsCheck; bool omitAlignmentCheck; bool onlyPointerAlignment; }; // Encapsulate all the information about a function call. struct FunctionCall { FunctionCall(ABIKind abiKind, RestoreState restoreState) : abi(abiKind), restoreState(restoreState), abiKind(abiKind), #ifdef JS_CODEGEN_ARM hardFP(true), #endif frameAlignAdjustment(0), stackArgAreaSize(0) { // The system ABI preserves the instance register (as it's in a // non-volatile register) and realm. We just need to reload the HeapReg in // case the memory has been moved. MOZ_ASSERT_IF(abiKind == ABIKind::System, restoreState == RestoreState::None || restoreState == RestoreState::PinnedRegs); if (abiKind == ABIKind::System) { // Builtin calls use the system hardFP setting on ARM32. #if defined(JS_CODEGEN_ARM) hardFP = ARMFlags::UseHardFpABI(); abi.setUseHardFp(hardFP); #endif } else { #if defined(JS_CODEGEN_ARM) MOZ_ASSERT(hardFP, "The WASM ABI passes FP arguments in registers"); #endif } } ABIArgGenerator abi; RestoreState restoreState; ABIKind abiKind; #ifdef JS_CODEGEN_ARM bool hardFP; #endif size_t frameAlignAdjustment; size_t stackArgAreaSize; }; enum class PreBarrierKind { // No pre-write barrier is required because the previous value is undefined. None, // Perform a pre-write barrier to mark the previous value if an incremental // GC is underway. Normal, }; enum class PostBarrierKind { // No post barrier. None, // Add a store buffer entry if the new value requires it, but do not attempt // to remove a pre-existing entry. Imprecise, // Remove an existing store buffer entry if the new value does not require // one. This is required to preserve invariants with HeapPtr when used for // movable storage. Precise, // Add a store buffer entry for the entire cell (e.g. the entire struct or // array who now has a field pointing into the nursery). WholeCell, }; struct BranchIfRefSubtypeRegisters { RegPtr superSTV; RegI32 scratch1; RegI32 scratch2; }; ////////////////////////////////////////////////////////////////////////////// // // Wasm baseline compiler proper. // // This is a struct and not a class because there is no real benefit to hiding // anything, and because many static functions that are wrappers for masm // methods need to reach into it and would otherwise have to be declared as // friends. // // (Members generally have a '_' suffix but some don't because they are // referenced everywhere and it would be tedious to spell that out.) struct BaseCompiler final { /////////////////////////////////////////////////////////////////////////// // // Private types using LabelVector = Vector; /////////////////////////////////////////////////////////////////////////// // // Read-only and write-once members. // Static compilation environment. const CodeMetadata& codeMeta_; const CompilerEnvironment& compilerEnv_; const FuncCompileInput& func_; const ValTypeVector& locals_; // Information about the locations of locals, this is set up during // initialization and read-only after that. BaseStackFrame::LocalVector localInfo_; // On specific platforms we sometimes need to use specific registers. const SpecificRegs specific_; // SigD and SigF are single-entry parameter lists for f64 and f32, these are // created during initialization. ValTypeVector SigD_; ValTypeVector SigF_; // Where to go to to return, bound as compilation ends. NonAssertingLabel returnLabel_; // Prologue and epilogue offsets, initialized during prologue and epilogue // generation and only used by the caller. FuncOffsets offsets_; // We call this address from the breakable point when the (per-module) debug // stub pointer in the Instance is not null. NonAssertingLabel perFunctionDebugStub_; uint32_t previousBreakablePoint_; // BaselineCompileFunctions() "lends" us the StkVector to use in this // BaseCompiler object, and that is installed in |stk_| in our constructor. // This is so as to avoid having to malloc/free the vector's contents at // each creation/destruction of a BaseCompiler object. It does however mean // that we need to hold on to a reference to BaselineCompileFunctions()'s // vector, so we can swap (give) its contents back when this BaseCompiler // object is destroyed. This significantly reduces the heap turnover of the // baseline compiler. See bug 1532592. StkVector& stkSource_; /////////////////////////////////////////////////////////////////////////// // // Output-only data structures. // Bump allocator for temporary memory, used for the value stack and // out-of-line code blobs. Bump-allocated memory is not freed until the end // of the compilation. TempAllocator::Fallible alloc_; // Machine code emitter. MacroAssembler& masm; // Perf spewer for annotated JIT code while profiling. WasmBaselinePerfSpewer perfSpewer_; /////////////////////////////////////////////////////////////////////////// // // Compilation state. // Decoder for this function, used for misc error reporting. Decoder& decoder_; // Opcode reader. BaseOpIter iter_; // Register allocator. BaseRegAlloc ra; // Stack frame abstraction. BaseStackFrame fr; // Latent out of line support code for some operations, code for these will be // emitted at the end of compilation. Vector outOfLine_; // The stack maps for this compilation. StackMaps* stackMaps_; // Stack map state. This keeps track of live pointer slots and allows precise // stack maps to be generated at safe points. StackMapGenerator stackMapGenerator_; // Wasm value stack. This maps values on the wasm stack to values in the // running code and their locations. // // The value stack facilitates on-the-fly register allocation and the use of // immediates in instructions. It tracks latent constants, latent references // to locals, register contents, and values that have been flushed to the CPU // stack. // // The stack can be flushed to the CPU stack using sync(). // // The stack is a StkVector rather than a StkVector& since constantly // dereferencing a StkVector& has been shown to add 0.5% or more to the // compiler's dynamic instruction count. StkVector stk_; // Flag indicating that the compiler is currently in a dead code region. bool deadCode_; // Store previously finished note to know if we need to insert a nop in // finishTryNote. size_t mostRecentFinishedTryNoteIndex_; /////////////////////////////////////////////////////////////////////////// // // State for bounds check elimination. // Locals that have been bounds checked and not updated since BCESet bceSafe_; /////////////////////////////////////////////////////////////////////////// // // State for boolean-evaluation-for-control. // Latent operation for branch (seen next) LatentOp latentOp_; // Operand type, if latentOp_ is true ValType latentType_; // Comparison operator, if latentOp_ == Compare, int types Assembler::Condition latentIntCmp_; // Comparison operator, if latentOp_ == Compare, float types Assembler::DoubleCondition latentDoubleCmp_; /////////////////////////////////////////////////////////////////////////// // // Main compilation API. // // A client will create a compiler object, and then call init(), // emitFunction(), and finish() in that order. BaseCompiler(const CodeMetadata& codeMetadata, const CompilerEnvironment& compilerEnv, const FuncCompileInput& func, const ValTypeVector& locals, const RegisterOffsets& trapExitLayout, size_t trapExitLayoutNumWords, Decoder& decoder, StkVector& stkSource, TempAllocator* alloc, MacroAssembler* masm, StackMaps* stackMaps); ~BaseCompiler(); [[nodiscard]] bool init(); [[nodiscard]] bool emitFunction(); [[nodiscard]] FuncOffsets finish(); ////////////////////////////////////////////////////////////////////////////// // // Sundry accessor abstractions and convenience predicates. // // WasmBaselineObject-inl.h. inline const FuncType& funcType() const; inline bool usesMemory() const; inline bool usesSharedMemory(uint32_t memoryIndex) const; inline bool isMem32(uint32_t memoryIndex) const; inline bool isMem64(uint32_t memoryIndex) const; inline bool hugeMemoryEnabled(uint32_t memoryIndex) const; inline uint32_t instanceOffsetOfMemoryBase(uint32_t memoryIndex) const; inline uint32_t instanceOffsetOfBoundsCheckLimit(uint32_t memoryIndex, unsigned byteSize) const; // The casts are used by some of the ScratchRegister implementations. operator MacroAssembler&() const { return masm; } operator BaseRegAlloc&() { return ra; } ////////////////////////////////////////////////////////////////////////////// // // Locals. // // WasmBaselineObject-inl.h. // Assert that the local at the given index has the given type, and return a // reference to the Local. inline const Local& localFromSlot(uint32_t slot, MIRType type); ////////////////////////////////////////////////////////////////////////////// // // Out of line code management. [[nodiscard]] OutOfLineCode* addOutOfLineCode(OutOfLineCode* ool); [[nodiscard]] bool generateOutOfLineCode(); ///////////////////////////////////////////////////////////////////////////// // // Layering in the compiler (briefly). // // At the lowest layers are abstractions for registers (managed by the // BaseRegAlloc and the wrappers below) and the stack frame (managed by the // BaseStackFrame). // // The registers and frame are in turn used by the value abstraction, which is // implemented by the Stk type and backed by the value stack. Values may be // stored in registers, in the frame, or may be latent constants, and the // value stack handles storage mostly transparently in its push and pop // routines. // // In turn, the pop routines bring values into registers so that we can // compute on them, and the push routines move values to the stack (where they // may still reside in registers until the registers are needed or the value // must be in memory). // // Routines for managing parameters and results (for blocks or calls) may also // manipulate the stack directly. // // At the top are the code generators: methods that use the poppers and // pushers and other utilities to move values into place, and that emit code // to compute on those values or change control flow. ///////////////////////////////////////////////////////////////////////////// // // Register management. These are simply strongly-typed wrappers that // delegate to the register allocator. inline bool isAvailableI32(RegI32 r); inline bool isAvailableI64(RegI64 r); inline bool isAvailableRef(RegRef r); inline bool isAvailablePtr(RegPtr r); inline bool isAvailableF32(RegF32 r); inline bool isAvailableF64(RegF64 r); #ifdef ENABLE_WASM_SIMD inline bool isAvailableV128(RegV128 r); #endif // Allocate any register [[nodiscard]] inline RegI32 needI32(); [[nodiscard]] inline RegI64 needI64(); [[nodiscard]] inline RegRef needRef(); [[nodiscard]] inline RegPtr needPtr(); [[nodiscard]] inline RegF32 needF32(); [[nodiscard]] inline RegF64 needF64(); #ifdef ENABLE_WASM_SIMD [[nodiscard]] inline RegV128 needV128(); #endif // Allocate a specific register inline void needI32(RegI32 specific); inline void needI64(RegI64 specific); inline void needRef(RegRef specific); inline void needPtr(RegPtr specific); inline void needF32(RegF32 specific); inline void needF64(RegF64 specific); #ifdef ENABLE_WASM_SIMD inline void needV128(RegV128 specific); #endif template inline RegType need(); // Just a shorthand. inline void need2xI32(RegI32 r0, RegI32 r1); inline void need2xI64(RegI64 r0, RegI64 r1); // Get a register but do not sync the stack to free one up. This will crash // if no register is available. inline void needI32NoSync(RegI32 r); #if defined(JS_CODEGEN_ARM) // Allocate a specific register pair (even-odd register numbers). [[nodiscard]] inline RegI64 needI64Pair(); #endif inline void freeAny(AnyReg r); inline void freeI32(RegI32 r); inline void freeI64(RegI64 r); inline void freeRef(RegRef r); inline void freePtr(RegPtr r); inline void freeF32(RegF32 r); inline void freeF64(RegF64 r); #ifdef ENABLE_WASM_SIMD inline void freeV128(RegV128 r); #endif template inline void free(RegType r); // Free r if it is not invalid. inline void maybeFree(RegI32 r); inline void maybeFree(RegI64 r); inline void maybeFree(RegF32 r); inline void maybeFree(RegF64 r); inline void maybeFree(RegRef r); inline void maybeFree(RegPtr r); #ifdef ENABLE_WASM_SIMD inline void maybeFree(RegV128 r); #endif // On 64-bit systems, `except` must equal r and this is a no-op. On 32-bit // systems, `except` must equal the high or low part of a pair and the other // part of the pair is freed. inline void freeI64Except(RegI64 r, RegI32 except); // Return the 32-bit low part of the 64-bit register, do not free anything. inline RegI32 fromI64(RegI64 r); // If r is valid, return fromI64(r), otherwise an invalid RegI32. inline RegI32 maybeFromI64(RegI64 r); #ifdef JS_PUNBOX64 // On 64-bit systems, reinterpret r as 64-bit. inline RegI64 fromI32(RegI32 r); #endif // Widen r to 64 bits; this may allocate another register to form a pair. // Note this does not generate code for sign/zero extension. inline RegI64 widenI32(RegI32 r); // Narrow r to 32 bits; this may free part of a pair. Note this does not // generate code to canonicalize the value on 64-bit systems. inline RegI32 narrowI64(RegI64 r); inline RegI32 narrowRef(RegRef r); // Return the 32-bit low part of r. inline RegI32 lowPart(RegI64 r); // On 64-bit systems, return an invalid register. On 32-bit systems, return // the low part of a pair. inline RegI32 maybeHighPart(RegI64 r); ////////////////////////////////////////////////////////////////////////////// // // Values and value stack: Low-level methods for moving Stk values of specific // kinds to registers. inline void loadConstI32(const Stk& src, RegI32 dest); inline void loadMemI32(const Stk& src, RegI32 dest); inline void loadLocalI32(const Stk& src, RegI32 dest); inline void loadRegisterI32(const Stk& src, RegI32 dest); inline void loadConstI64(const Stk& src, RegI64 dest); inline void loadMemI64(const Stk& src, RegI64 dest); inline void loadLocalI64(const Stk& src, RegI64 dest); inline void loadRegisterI64(const Stk& src, RegI64 dest); inline void loadConstRef(const Stk& src, RegRef dest); inline void loadMemRef(const Stk& src, RegRef dest); inline void loadLocalRef(const Stk& src, RegRef dest); inline void loadRegisterRef(const Stk& src, RegRef dest); inline void loadConstF64(const Stk& src, RegF64 dest); inline void loadMemF64(const Stk& src, RegF64 dest); inline void loadLocalF64(const Stk& src, RegF64 dest); inline void loadRegisterF64(const Stk& src, RegF64 dest); inline void loadConstF32(const Stk& src, RegF32 dest); inline void loadMemF32(const Stk& src, RegF32 dest); inline void loadLocalF32(const Stk& src, RegF32 dest); inline void loadRegisterF32(const Stk& src, RegF32 dest); #ifdef ENABLE_WASM_SIMD inline void loadConstV128(const Stk& src, RegV128 dest); inline void loadMemV128(const Stk& src, RegV128 dest); inline void loadLocalV128(const Stk& src, RegV128 dest); inline void loadRegisterV128(const Stk& src, RegV128 dest); #endif ////////////////////////////////////////////////////////////////////////// // // Values and value stack: Mid-level routines for moving Stk values of any // kind to registers. inline void loadI32(const Stk& src, RegI32 dest); inline void loadI64(const Stk& src, RegI64 dest); #if !defined(JS_PUNBOX64) inline void loadI64Low(const Stk& src, RegI32 dest); inline void loadI64High(const Stk& src, RegI32 dest); #endif inline void loadF64(const Stk& src, RegF64 dest); inline void loadF32(const Stk& src, RegF32 dest); #ifdef ENABLE_WASM_SIMD inline void loadV128(const Stk& src, RegV128 dest); #endif inline void loadRef(const Stk& src, RegRef dest); ////////////////////////////////////////////////////////////////////// // // Value stack: stack management. // Flush all local and register value stack elements to memory. inline void sync(); // Save a register on the value stack temporarily. void saveTempPtr(const RegPtr& r); // Restore a temporarily saved register from the value stack. void restoreTempPtr(const RegPtr& r); // This is an optimization used to avoid calling sync for setLocal: if the // local does not exist unresolved on the value stack then we can skip the // sync. inline bool hasLocal(uint32_t slot); // Sync the local if necessary. (This currently syncs everything if a sync is // needed at all.) inline void syncLocal(uint32_t slot); // Return the amount of execution stack consumed by the top numval // values on the value stack. inline size_t stackConsumed(size_t numval); // Drop one value off the stack, possibly also moving the physical stack // pointer. inline void dropValue(); #ifdef DEBUG // Check that we're not leaking registers by comparing the // state of the stack + available registers with the set of // all available registers. // Call this between opcodes. void performRegisterLeakCheck(); // This can be called at any point, really, but typically just after // performRegisterLeakCheck(). void assertStackInvariants() const; // Count the number of memory references on the value stack. inline size_t countMemRefsOnStk(); // Check if there are any live registers on the value stack. inline bool hasLiveRegsOnStk(); // Print the stack to stderr. void showStack(const char* who) const; #endif ////////////////////////////////////////////////////////////////////// // // Value stack: pushers of values. // Push a register onto the value stack. inline void pushAny(AnyReg r); inline void pushI32(RegI32 r); inline void pushI64(RegI64 r); inline void pushRef(RegRef r); inline void pushPtr(RegPtr r); inline void pushF64(RegF64 r); inline void pushF32(RegF32 r); #ifdef ENABLE_WASM_SIMD inline void pushV128(RegV128 r); #endif // Template variation of the foregoing, for use by templated emitters. template inline void push(RegType item); // Push a constant value onto the stack. pushI32 can also take uint32_t, and // pushI64 can take uint64_t; the semantics are the same. Appropriate sign // extension for a 32-bit value on a 64-bit architecture happens when the // value is popped, see the definition of moveImm32. inline void pushI32(int32_t v); inline void pushI64(int64_t v); inline void pushRef(intptr_t v); inline void pushPtr(intptr_t v); inline void pushF64(double v); inline void pushF32(float v); #ifdef ENABLE_WASM_SIMD inline void pushV128(V128 v); #endif inline void pushConstRef(intptr_t v); // Push the local slot onto the stack. The slot will not be read here; it // will be read when it is consumed, or when a side effect to the slot forces // its value to be saved. inline void pushLocalI32(uint32_t slot); inline void pushLocalI64(uint32_t slot); inline void pushLocalRef(uint32_t slot); inline void pushLocalF64(uint32_t slot); inline void pushLocalF32(uint32_t slot); #ifdef ENABLE_WASM_SIMD inline void pushLocalV128(uint32_t slot); #endif // Push an U32 as an I64, zero-extending it in the process inline void pushU32AsI64(RegI32 rs); ////////////////////////////////////////////////////////////////////// // // Value stack: poppers and peekers of values. // Pop some value off the stack. inline AnyReg popAny(); inline AnyReg popAny(AnyReg specific); // Call only from other popI32() variants. v must be the stack top. May pop // the CPU stack. inline void popI32(const Stk& v, RegI32 dest); [[nodiscard]] inline RegI32 popI32(); inline RegI32 popI32(RegI32 specific); #ifdef ENABLE_WASM_SIMD // Call only from other popV128() variants. v must be the stack top. May pop // the CPU stack. inline void popV128(const Stk& v, RegV128 dest); [[nodiscard]] inline RegV128 popV128(); inline RegV128 popV128(RegV128 specific); #endif // Call only from other popI64() variants. v must be the stack top. May pop // the CPU stack. inline void popI64(const Stk& v, RegI64 dest); [[nodiscard]] inline RegI64 popI64(); inline RegI64 popI64(RegI64 specific); // Call only from other popRef() variants. v must be the stack top. May pop // the CPU stack. inline void popRef(const Stk& v, RegRef dest); inline RegRef popRef(RegRef specific); [[nodiscard]] inline RegRef popRef(); // Call only from other popPtr() variants. v must be the stack top. May pop // the CPU stack. inline void popPtr(const Stk& v, RegPtr dest); inline RegPtr popPtr(RegPtr specific); [[nodiscard]] inline RegPtr popPtr(); // Call only from other popF64() variants. v must be the stack top. May pop // the CPU stack. inline void popF64(const Stk& v, RegF64 dest); [[nodiscard]] inline RegF64 popF64(); inline RegF64 popF64(RegF64 specific); // Call only from other popF32() variants. v must be the stack top. May pop // the CPU stack. inline void popF32(const Stk& v, RegF32 dest); [[nodiscard]] inline RegF32 popF32(); inline RegF32 popF32(RegF32 specific); // Templated variation of the foregoing, for use by templated emitters. template inline RegType pop(); // Constant poppers will return true and pop the value if the stack top is a // constant of the appropriate type; otherwise pop nothing and return false. [[nodiscard]] inline bool hasConst() const; [[nodiscard]] inline bool popConst(int32_t* c); [[nodiscard]] inline bool popConst(int64_t* c); [[nodiscard]] inline bool peekConst(int32_t* c); [[nodiscard]] inline bool peekConst(int64_t* c); [[nodiscard]] inline bool peek2xConst(int32_t* c0, int32_t* c1); [[nodiscard]] inline bool popConstPositivePowerOfTwo(int32_t* c, uint_fast8_t* power, int32_t cutoff); [[nodiscard]] inline bool popConstPositivePowerOfTwo(int64_t* c, uint_fast8_t* power, int64_t cutoff); // Shorthand: Pop r1, then r0. inline void pop2xI32(RegI32* r0, RegI32* r1); inline void pop2xI64(RegI64* r0, RegI64* r1); inline void pop2xF32(RegF32* r0, RegF32* r1); inline void pop2xF64(RegF64* r0, RegF64* r1); #ifdef ENABLE_WASM_SIMD inline void pop2xV128(RegV128* r0, RegV128* r1); #endif inline void pop2xRef(RegRef* r0, RegRef* r1); // Pop to a specific register inline RegI32 popI32ToSpecific(RegI32 specific); inline RegI64 popI64ToSpecific(RegI64 specific); #ifdef JS_CODEGEN_ARM // Pop an I64 as a valid register pair. inline RegI64 popI64Pair(); #endif // Pop an I64 but narrow it and return the narrowed part. inline RegI32 popI64ToI32(); inline RegI32 popI64ToSpecificI32(RegI32 specific); // Pop an I32 or I64 as an I64. The value is zero extended out to 64-bits. inline RegI64 popAddressToInt64(AddressType addressType); // Pop an I32 or I64 as an I32. The value is clamped to UINT32_MAX to ensure // that it trips bounds checks. inline RegI32 popTableAddressToClampedInt32(AddressType addressType); // A combined push/pop that replaces an I32 or I64 on the stack with a clamped // I32, which will trip bounds checks if out of I32 range. inline void replaceTableAddressWithClampedInt32(AddressType addressType); // Pop the stack until it has the desired size, but do not move the physical // stack pointer. inline void popValueStackTo(uint32_t stackSize); // Pop the given number of elements off the value stack, but do not move // the physical stack pointer. inline void popValueStackBy(uint32_t items); // Peek into the stack at relativeDepth from the top. inline Stk& peek(uint32_t relativeDepth); // Peek the reference value at the specified depth and load it into a // register. inline void peekRefAt(uint32_t depth, RegRef dest); // Peek at the value on the top of the stack and return true if it is a Local // of any type. [[nodiscard]] inline bool peekLocal(uint32_t* local); //////////////////////////////////////////////////////////////////////////// // // Block parameters and results. // // Blocks may have multiple parameters and multiple results. Blocks can also // be the target of branches: the entry for loops, and the exit for // non-loops. // // Passing multiple values to a non-branch target (i.e., the entry of a // "block") falls out naturally: any items on the value stack can flow // directly from one block to another. // // However, for branch targets, we need to allocate well-known locations for // the branch values. The approach taken in the baseline compiler is to // allocate registers to the top N values (currently N=1), and then stack // locations for the rest. // // Types of result registers that interest us for result-manipulating // functions. enum class ResultRegKind { // General and floating result registers. All, // General result registers only. OnlyGPRs }; // This is a flag ultimately intended for popBlockResults() that specifies how // the CPU stack should be handled after the result values have been // processed. enum class ContinuationKind { // Adjust the stack for a fallthrough: do nothing. Fallthrough, // Adjust the stack for a jump: make the stack conform to the // expected stack at the target Jump }; // TODO: It's definitely disputable whether the result register management is // hot enough to warrant inlining at the outermost level. inline void needResultRegisters(ResultType type, ResultRegKind which); #ifdef JS_64BIT inline void widenInt32ResultRegisters(ResultType type); #endif inline void freeResultRegisters(ResultType type, ResultRegKind which); inline void needIntegerResultRegisters(ResultType type); inline void freeIntegerResultRegisters(ResultType type); inline void needResultRegisters(ResultType type); inline void freeResultRegisters(ResultType type); void assertResultRegistersAvailable(ResultType type); inline void captureResultRegisters(ResultType type); inline void captureCallResultRegisters(ResultType type); void popRegisterResults(ABIResultIter& iter); void popStackResults(ABIResultIter& iter, StackHeight stackBase); void popBlockResults(ResultType type, StackHeight stackBase, ContinuationKind kind); // This function is similar to popBlockResults, but additionally handles the // implicit exception pointer that is pushed to the value stack on entry to // a catch handler by dropping it appropriately. void popCatchResults(ResultType type, StackHeight stackBase); Stk captureStackResult(const ABIResult& result, StackHeight resultsBase, uint32_t stackResultBytes); [[nodiscard]] bool pushResults(ResultType type, StackHeight resultsBase); [[nodiscard]] bool pushBlockResults(ResultType type); // A combination of popBlockResults + pushBlockResults, used when entering a // block with a control-flow join (loops) or split (if) to shuffle the // fallthrough block parameters into the locations expected by the // continuation. // // This function should only be called when entering a block with a // control-flow join at the entry, where there are no live temporaries in // the current block. [[nodiscard]] bool topBlockParams(ResultType type); // A combination of popBlockResults + pushBlockResults, used before branches // where we don't know the target (br_if / br_table). If and when the branch // is taken, the stack results will be shuffled down into place. For br_if // that has fallthrough, the parameters for the untaken branch flow through to // the continuation. [[nodiscard]] bool topBranchParams(ResultType type, StackHeight* height); // Conditional branches with fallthrough are preceded by a topBranchParams, so // we know that there are no stack results that need to be materialized. In // that case, we can just shuffle the whole block down before popping the // stack. void shuffleStackResultsBeforeBranch(StackHeight srcHeight, StackHeight destHeight, ResultType type); // If in debug mode, adds LeaveFrame breakpoint. bool insertDebugCollapseFrame(); ////////////////////////////////////////////////////////////////////// // // Stack maps // Various methods for creating a stackmap. Stackmaps are indexed by the // lowest address of the instruction immediately *after* the instruction of // interest. In practice that means either: the return point of a call, the // instruction immediately after a trap instruction (the "resume" // instruction), or the instruction immediately following a no-op (when // debugging is enabled). // Create a vanilla stackmap. [[nodiscard]] bool createStackMap(const char* who); // Create a stackmap as vanilla, but for a custom assembler offset. [[nodiscard]] bool createStackMap(const char* who, CodeOffset assemblerOffset); // Create a stack map as vanilla, and note the presence of a ref-typed // DebugFrame on the stack. [[nodiscard]] bool createStackMap( const char* who, HasDebugFrameWithLiveRefs debugFrameWithLiveRefs); // Creates a stack map for an aborting trap instruction that will be emitted // OOL. [[nodiscard]] bool createAbortingOutOfLineTrapStackMap(StackMap** result); //////////////////////////////////////////////////////////// // // Control stack inline void initControl(Control& item, ResultType params); inline Control& controlItem(); inline Control& controlItem(uint32_t relativeDepth); inline Control& controlOutermost(); inline LabelKind controlKind(uint32_t relativeDepth); //////////////////////////////////////////////////////////// // // Debugger API // Insert a breakpoint almost anywhere. This will create a call, with all the // overhead that entails. void insertBreakablePoint(CallSiteKind kind); // Insert code at the end of a function for breakpoint filtering. void insertPerFunctionDebugStub(); // Debugger API used at the return point: shuffle register return values off // to memory for the debugger to see; and get them back again. void saveRegisterReturnValues(const ResultType& resultType); void restoreRegisterReturnValues(const ResultType& resultType); ////////////////////////////////////////////////////////////////////// // // Function prologue and epilogue. // Set up and tear down frame, execute prologue and epilogue. [[nodiscard]] bool beginFunction(); [[nodiscard]] bool endFunction(); // Move return values to memory before returning, as appropriate void popStackReturnValues(const ResultType& resultType); ////////////////////////////////////////////////////////////////////// // // Calls. void beginCall(FunctionCall& call); void endCall(FunctionCall& call, size_t stackSpace); void startCallArgs(size_t stackArgAreaSizeUnaligned, FunctionCall* call); ABIArg reservePointerArgument(FunctionCall* call); void passArg(ValType type, const Stk& arg, FunctionCall* call); // A flag passed to emitCallArgs, describing how the value stack is laid out. enum class CalleeOnStack { // After the arguments to the call, there is a callee pushed onto value // stack. This is only the case for callIndirect. To get the arguments to // the call, emitCallArgs has to reach one element deeper into the value // stack, to skip the callee. True, // No callee on the stack. False }; // The typename T for emitCallArgs can be one of the following: // NormalCallResults, TailCallResults, or NoCallResults. template [[nodiscard]] bool emitCallArgs(const ValTypeVector& argTypes, T results, FunctionCall* baselineCall, CalleeOnStack calleeOnStack); [[nodiscard]] bool pushStackResultsForWasmCall(const ResultType& type, RegPtr temp, StackResultsLoc* loc); void popStackResultsAfterWasmCall(const StackResultsLoc& results, uint32_t stackArgBytes); void pushBuiltinCallResult(const FunctionCall& call, MIRType type); [[nodiscard]] bool pushWasmCallResults(const FunctionCall& call, ResultType type, const StackResultsLoc& loc); // Precondition for the call*() methods: sync() CodeOffset callDefinition(uint32_t funcIndex, const FunctionCall& call); CodeOffset callSymbolic(SymbolicAddress callee, const FunctionCall& call); bool callIndirect(uint32_t funcTypeIndex, uint32_t tableIndex, const Stk& indexVal, const FunctionCall& call, bool tailCall, CodeOffset* fastCallOffset, CodeOffset* slowCallOffset); CodeOffset callImport(unsigned instanceDataOffset, const FunctionCall& call); bool updateCallRefMetrics(size_t callRefIndex); bool callRef(const Stk& calleeRef, const FunctionCall& call, mozilla::Maybe callRefIndex, CodeOffset* fastCallOffset, CodeOffset* slowCallOffset); void returnCallRef(const Stk& calleeRef, const FunctionCall& call, const FuncType& funcType); CodeOffset builtinCall(SymbolicAddress builtin, const FunctionCall& call); void builtinInstanceMethodCall(const SymbolicAddressSignature& builtin, const ABIArg& instanceArg, const FunctionCall& call, CodeOffset* callStackMapKey, CodeOffset* trapStackMapKey); // Helpers to pick up the returned value from the return register. inline RegI32 captureReturnedI32(); inline RegI64 captureReturnedI64(); inline RegF32 captureReturnedF32(const FunctionCall& call); inline RegF64 captureReturnedF64(const FunctionCall& call); #ifdef ENABLE_WASM_SIMD inline RegV128 captureReturnedV128(const FunctionCall& call); #endif inline RegRef captureReturnedRef(); ////////////////////////////////////////////////////////////////////// // // Register-to-register moves. These emit nothing if src == dest. inline void moveI32(RegI32 src, RegI32 dest); inline void moveI64(RegI64 src, RegI64 dest); inline void moveRef(RegRef src, RegRef dest); inline void movePtr(RegPtr src, RegPtr dest); inline void moveF64(RegF64 src, RegF64 dest); inline void moveF32(RegF32 src, RegF32 dest); #ifdef ENABLE_WASM_SIMD inline void moveV128(RegV128 src, RegV128 dest); #endif template inline void move(RegType src, RegType dest); ////////////////////////////////////////////////////////////////////// // // Immediate-to-register moves. // // The compiler depends on moveImm32() clearing the high bits of a 64-bit // register on 64-bit systems except MIPS64 And LoongArch64 where high bits // are sign extended from lower bits, see doc block "64-bit GPRs carrying // 32-bit values" in MacroAssembler.h. inline void moveImm32(int32_t v, RegI32 dest); inline void moveImm64(int64_t v, RegI64 dest); inline void moveImmRef(intptr_t v, RegRef dest); ////////////////////////////////////////////////////////////////////// // // Sundry low-level code generators. // Decrement the per-instance function hotness counter by `step` and request // optimized compilation for this function if the updated counter is negative // when regarded as an int32_t. The amount to decrement is to be filled in // later by ::patchHotnessCheck. [[nodiscard]] Maybe addHotnessCheck(); // Patch in the counter decrement for a hotness check, using the offset // previously obtained from ::addHotnessCheck. We require 1 <= `step` <= 127. void patchHotnessCheck(CodeOffset offset, uint32_t step); // Check the interrupt flag, trap if it is set. [[nodiscard]] bool addInterruptCheck(); // Check that the value is not zero, trap if it is. void checkDivideByZero(RegI32 rhs); void checkDivideByZero(RegI64 r); // Check that a signed division will not overflow, trap or flush-to-zero if it // will according to `zeroOnOverflow`. void checkDivideSignedOverflow(RegI32 rhs, RegI32 srcDest, Label* done, bool zeroOnOverflow); void checkDivideSignedOverflow(RegI64 rhs, RegI64 srcDest, Label* done, bool zeroOnOverflow); // Emit a jump table to be used by tableSwitch() void jumpTable(const LabelVector& labels, Label* theTable); // Emit a table switch, `theTable` is the jump table. void tableSwitch(Label* theTable, RegI32 switchValue, Label* dispatchCode); // Compare i64 and set an i32 boolean result according to the condition. inline void cmp64Set(Assembler::Condition cond, RegI64 lhs, RegI64 rhs, RegI32 dest); // Round floating to integer. [[nodiscard]] inline bool supportsRoundInstruction(RoundingMode mode); inline void roundF32(RoundingMode roundingMode, RegF32 f0); inline void roundF64(RoundingMode roundingMode, RegF64 f0); // These are just wrappers around assembler functions, but without // type-specific names, and using our register abstractions for better type // discipline. inline void branchTo(Assembler::DoubleCondition c, RegF64 lhs, RegF64 rhs, Label* l); inline void branchTo(Assembler::DoubleCondition c, RegF32 lhs, RegF32 rhs, Label* l); inline void branchTo(Assembler::Condition c, RegI32 lhs, RegI32 rhs, Label* l); inline void branchTo(Assembler::Condition c, RegI32 lhs, Imm32 rhs, Label* l); inline void branchTo(Assembler::Condition c, RegI64 lhs, RegI64 rhs, Label* l); inline void branchTo(Assembler::Condition c, RegI64 lhs, Imm64 rhs, Label* l); inline void branchTo(Assembler::Condition c, RegRef lhs, ImmWord rhs, Label* l); // Helpers for accessing Instance::baselineScratchWords_. Note that Word // and I64 versions of these routines access the same area and it is up to // the caller to use it in some way which makes sense. // Store/load `r`, a machine word, to/from the `index`th scratch storage // slot in the current Instance. `instancePtr` must point at the current // Instance; it will not be modified. For ::stashWord, `r` must not be the // same as `instancePtr`. void stashWord(RegPtr instancePtr, size_t index, RegPtr r); void unstashWord(RegPtr instancePtr, size_t index, RegPtr r); #ifdef JS_CODEGEN_X86 // Store r in instance scratch storage after first loading the instance from // the frame into the regForInstance. regForInstance must be neither of the // registers in r. void stashI64(RegPtr regForInstance, RegI64 r); // Load r from the instance scratch storage after first loading the instance // from the frame into the regForInstance. regForInstance can be one of the // registers in r. void unstashI64(RegPtr regForInstance, RegI64 r); #endif ////////////////////////////////////////////////////////////////////// // // Code generators for actual operations. template void quotientOrRemainder(RegType rs, RegType rsd, RegType reserved, IsUnsigned isUnsigned, ZeroOnOverflow zeroOnOverflow, bool isConst, IntType c, void (*operate)(MacroAssembler&, RegType, RegType, RegType, IsUnsigned)); [[nodiscard]] bool truncateF32ToI32(RegF32 src, RegI32 dest, TruncFlags flags); [[nodiscard]] bool truncateF64ToI32(RegF64 src, RegI32 dest, TruncFlags flags); #ifndef RABALDR_FLOAT_TO_I64_CALLOUT [[nodiscard]] RegF64 needTempForFloatingToI64(TruncFlags flags); [[nodiscard]] bool truncateF32ToI64(RegF32 src, RegI64 dest, TruncFlags flags, RegF64 temp); [[nodiscard]] bool truncateF64ToI64(RegF64 src, RegI64 dest, TruncFlags flags, RegF64 temp); #endif // RABALDR_FLOAT_TO_I64_CALLOUT #ifndef RABALDR_I64_TO_FLOAT_CALLOUT [[nodiscard]] RegI32 needConvertI64ToFloatTemp(ValType to, bool isUnsigned); void convertI64ToF32(RegI64 src, bool isUnsigned, RegF32 dest, RegI32 temp); void convertI64ToF64(RegI64 src, bool isUnsigned, RegF64 dest, RegI32 temp); #endif // RABALDR_I64_TO_FLOAT_CALLOUT ////////////////////////////////////////////////////////////////////// // // Global variable access. Address addressOfGlobalVar(const GlobalDesc& global, RegPtr tmp); ////////////////////////////////////////////////////////////////////// // // Table access. Address addressOfTableField(uint32_t tableIndex, uint32_t fieldOffset, RegPtr instance); void loadTableLength(uint32_t tableIndex, RegPtr instance, RegI32 length); void loadTableElements(uint32_t tableIndex, RegPtr instance, RegPtr elements); ////////////////////////////////////////////////////////////////////// // // Heap access. void bceCheckLocal(MemoryAccessDesc* access, AccessCheck* check, uint32_t local); void bceLocalIsUpdated(uint32_t local); // Fold offsets into ptr and bounds check as necessary. The instance will be // valid in cases where it's needed. template void prepareMemoryAccess(MemoryAccessDesc* access, AccessCheck* check, RegPtr instance, RegAddressType ptr); void branchAddNoOverflow(uint64_t offset, RegI32 ptr, Label* ok); void branchTestLowZero(RegI32 ptr, Imm32 mask, Label* ok); void boundsCheck4GBOrLargerAccess(uint32_t memoryIndex, unsigned byteSize, RegPtr instance, RegI32 ptr, Label* ok); void boundsCheckBelow4GBAccess(uint32_t memoryIndex, unsigned byteSize, RegPtr instance, RegI32 ptr, Label* ok); void branchAddNoOverflow(uint64_t offset, RegI64 ptr, Label* ok); void branchTestLowZero(RegI64 ptr, Imm32 mask, Label* ok); void boundsCheck4GBOrLargerAccess(uint32_t memoryIndex, unsigned byteSize, RegPtr instance, RegI64 ptr, Label* ok); void boundsCheckBelow4GBAccess(uint32_t memoryIndex, unsigned byteSize, RegPtr instance, RegI64 ptr, Label* ok); // Some consumers depend on the returned Address not incorporating instance, // as instance may be the scratch register. template Address prepareAtomicMemoryAccess(MemoryAccessDesc* access, AccessCheck* check, RegPtr instance, RegAddressType ptr); template void computeEffectiveAddress(MemoryAccessDesc* access); [[nodiscard]] bool needInstanceForAccess(const MemoryAccessDesc* access, const AccessCheck& check); // ptr and dest may be the same iff dest is I32. // This may destroy ptr even if ptr and dest are not the same. void executeLoad(MemoryAccessDesc* access, AccessCheck* check, RegPtr instance, RegPtr memoryBase, RegI32 ptr, AnyReg dest, RegI32 temp); void load(MemoryAccessDesc* access, AccessCheck* check, RegPtr instance, RegPtr memoryBase, RegI32 ptr, AnyReg dest, RegI32 temp); void load(MemoryAccessDesc* access, AccessCheck* check, RegPtr instance, RegPtr memoryBase, RegI64 ptr, AnyReg dest, RegI64 temp); template void doLoadCommon(MemoryAccessDesc* access, AccessCheck check, ValType type); void loadCommon(MemoryAccessDesc* access, AccessCheck check, ValType type); // ptr and src must not be the same register. // This may destroy ptr and src. void executeStore(MemoryAccessDesc* access, AccessCheck* check, RegPtr instance, RegPtr memoryBase, RegI32 ptr, AnyReg src, RegI32 temp); void store(MemoryAccessDesc* access, AccessCheck* check, RegPtr instance, RegPtr memoryBase, RegI32 ptr, AnyReg src, RegI32 temp); void store(MemoryAccessDesc* access, AccessCheck* check, RegPtr instance, RegPtr memoryBase, RegI64 ptr, AnyReg src, RegI64 temp); template void doStoreCommon(MemoryAccessDesc* access, AccessCheck check, ValType resultType); void storeCommon(MemoryAccessDesc* access, AccessCheck check, ValType resultType); void atomicLoad(MemoryAccessDesc* access, ValType type); #if !defined(JS_64BIT) template void atomicLoad64(MemoryAccessDesc* desc); #endif void atomicStore(MemoryAccessDesc* access, ValType type); void atomicRMW(MemoryAccessDesc* access, ValType type, AtomicOp op); template void atomicRMW32(MemoryAccessDesc* access, ValType type, AtomicOp op); template void atomicRMW64(MemoryAccessDesc* access, ValType type, AtomicOp op); void atomicXchg(MemoryAccessDesc* access, ValType type); template void atomicXchg64(MemoryAccessDesc* access, WantResult wantResult); template void atomicXchg32(MemoryAccessDesc* access, ValType type); void atomicCmpXchg(MemoryAccessDesc* access, ValType type); template void atomicCmpXchg32(MemoryAccessDesc* access, ValType type); template void atomicCmpXchg64(MemoryAccessDesc* access, ValType type); template RegType popConstMemoryAccess(MemoryAccessDesc* access, AccessCheck* check); template RegType popMemoryAccess(MemoryAccessDesc* access, AccessCheck* check); void pushHeapBase(uint32_t memoryIndex); //////////////////////////////////////////////////////////////////////////// // // Platform-specific popping and register targeting. // The simple popping methods pop values into targeted registers; the caller // can free registers using standard functions. These are always called // popXForY where X says something about types and Y something about the // operation being targeted. RegI32 needRotate64Temp(); void popAndAllocateForDivAndRemI32(RegI32* r0, RegI32* r1, RegI32* reserved); void popAndAllocateForMulI64(RegI64* r0, RegI64* r1, RegI32* temp); #ifndef RABALDR_INT_DIV_I64_CALLOUT void popAndAllocateForDivAndRemI64(RegI64* r0, RegI64* r1, RegI64* reserved, IsRemainder isRemainder); #endif RegI32 popI32RhsForShift(); RegI32 popI32RhsForShiftI64(); RegI64 popI64RhsForShift(); RegI32 popI32RhsForRotate(); RegI64 popI64RhsForRotate(); void popI32ForSignExtendI64(RegI64* r0); void popI64ForSignExtendI64(RegI64* r0); //////////////////////////////////////////////////////////// // // Sundry helpers. // Retrieve the current bytecodeOffset. inline BytecodeOffset bytecodeOffset() const; // Get a trap site description for a trap that would occur in the current // opcode. inline TrapSiteDesc trapSiteDesc() const; // Generate a trap instruction for the current bytecodeOffset. inline void trap(Trap t); // Generate a trap instruction for given location and stack map. inline void trap(Trap t, const TrapSiteDesc& trapSite, StackMap* stackMap); // Abstracted helper for throwing, used for throw, rethrow, and rethrowing // at the end of a series of catch blocks (if none matched the exception). [[nodiscard]] bool throwFrom(RegRef exn); // Load the specified tag object from the Instance. void loadTag(RegPtr instance, uint32_t tagIndex, RegRef tagDst); // Load the pending exception state from the Instance and then reset it. void consumePendingException(RegPtr instance, RegRef* exnDst, RegRef* tagDst); [[nodiscard]] bool startTryNote(size_t* tryNoteIndex); void finishTryNote(size_t tryNoteIndex); //////////////////////////////////////////////////////////// // // Barriers support. // This emits a GC pre-write barrier. The pre-barrier is needed when we // replace a member field with a new value, and the previous field value // might have no other referents, and incremental GC is ongoing. The field // might belong to an object or be a stack slot or a register or a heap // allocated value. // // let obj = { field: previousValue }; // obj.field = newValue; // previousValue must be marked with a pre-barrier. // // The `valueAddr` is the address of the location that we are about to // update. This function preserves that register. void emitPreBarrier(RegPtr valueAddr); // These emit GC post-write barriers. The post-barrier is needed when we // replace a member field with a new value, the new value is in the nursery, // and the containing object is a tenured object. The field (or the entire // containing object) must then be added to the store buffer so that the // nursery can be correctly collected. The field might belong to an object or // be a stack slot or a register or a heap allocated value. // // For the difference between 'precise' and 'imprecise', look at the // documentation on PostBarrierKind. // Emits a post-write barrier that creates a whole-cell store buffer entry. // See above for details. // // - `object` is a pointer to the object that contains the field. This // register is preserved by this function. // - `value` is the value that was stored in the field. This register is // preserved by this function. // - `temp` is consumed by this function. [[nodiscard]] bool emitPostBarrierWholeCell(RegRef object, RegRef value, RegPtr temp); // Emits a post-write barrier of type WasmAnyRefEdge, imprecisely. See above // for details. // // - `object` is a pointer to the object that contains the field. It is used, // if present, to skip adding a store buffer entry when the containing // object is in the nursery. This register is preserved by this function. // - `valueAddr` is the address of the location that we are writing to. This // register is consumed by this function. // - `value` is the value that was stored in the field. This register is // preserved by this function. [[nodiscard]] bool emitPostBarrierEdgeImprecise( const mozilla::Maybe& object, RegPtr valueAddr, RegRef value); // Emits a post-write barrier of type WasmAnyRefEdge, precisely. See above for // details. // // - `object` is a pointer to the object that contains the field. It is used, // if present, to skip adding a store buffer entry when the containing // object is in the nursery. This register is preserved by this function. // - `valueAddr` is the address of the location that we are writing to. This // register is consumed by this function. // - `prevValue` is the value that existed in the field before `value` was // stored. This register is consumed by this function. // - `value` is the value that was stored in the field. This register is // preserved by this function. [[nodiscard]] bool emitPostBarrierEdgePrecise( const mozilla::Maybe& object, RegPtr valueAddr, RegRef prevValue, RegRef value); // Emits a store to a JS object pointer at the address `valueAddr`, which is // inside the GC cell `object`. // // Preserves `object` and `value`. Consumes `valueAddr`. [[nodiscard]] bool emitBarrieredStore(const mozilla::Maybe& object, RegPtr valueAddr, RegRef value, PreBarrierKind preBarrierKind, PostBarrierKind postBarrierKind); // Emits a store of nullptr to a JS object pointer at the address valueAddr. // Preserves `valueAddr`. void emitBarrieredClear(RegPtr valueAddr); //////////////////////////////////////////////////////////// // // Machinery for optimized conditional branches. See comments in the // implementation. void setLatentCompare(Assembler::Condition compareOp, ValType operandType); void setLatentCompare(Assembler::DoubleCondition compareOp, ValType operandType); void setLatentEqz(ValType operandType); bool hasLatentOp() const; void resetLatentOp(); // Jump to the given branch, passing results, if the condition, `cond` // matches between `lhs` and `rhs. template [[nodiscard]] bool jumpConditionalWithResults(BranchState* b, Cond cond, Lhs lhs, Rhs rhs); // Jump to the given branch, passing results, if the WasmGcObject, `object`, // is a subtype of `destType`. [[nodiscard]] bool jumpConditionalWithResults(BranchState* b, RegRef object, MaybeRefType sourceType, RefType destType, bool onSuccess); template [[nodiscard]] bool sniffConditionalControlCmp(Cond compareOp, ValType operandType); [[nodiscard]] bool sniffConditionalControlEqz(ValType operandType); void emitBranchSetup(BranchState* b); [[nodiscard]] bool emitBranchPerform(BranchState* b); ////////////////////////////////////////////////////////////////////// [[nodiscard]] bool emitBody(); [[nodiscard]] bool emitBlock(); [[nodiscard]] bool emitLoop(); [[nodiscard]] bool emitIf(); [[nodiscard]] bool emitElse(); // Used for common setup for catch and catch_all. void emitCatchSetup(LabelKind kind, Control& tryCatch, const ResultType& resultType); [[nodiscard]] bool emitTry(); [[nodiscard]] bool emitTryTable(); [[nodiscard]] bool emitCatch(); [[nodiscard]] bool emitCatchAll(); [[nodiscard]] bool emitDelegate(); [[nodiscard]] bool emitThrow(); [[nodiscard]] bool emitThrowRef(); [[nodiscard]] bool emitRethrow(); [[nodiscard]] bool emitEnd(); [[nodiscard]] bool emitBr(); [[nodiscard]] bool emitBrIf(); [[nodiscard]] bool emitBrTable(); [[nodiscard]] bool emitDrop(); [[nodiscard]] bool emitReturn(); [[nodiscard]] bool emitCall(); [[nodiscard]] bool emitReturnCall(); [[nodiscard]] bool emitCallIndirect(); [[nodiscard]] bool emitReturnCallIndirect(); [[nodiscard]] bool emitUnaryMathBuiltinCall(SymbolicAddress callee, ValType operandType); [[nodiscard]] bool emitGetLocal(); [[nodiscard]] bool emitSetLocal(); [[nodiscard]] bool emitTeeLocal(); [[nodiscard]] bool emitGetGlobal(); [[nodiscard]] bool emitSetGlobal(); [[nodiscard]] RegPtr maybeLoadMemoryBaseForAccess( RegPtr instance, const MemoryAccessDesc* access); [[nodiscard]] RegPtr maybeLoadInstanceForAccess( const MemoryAccessDesc* access, const AccessCheck& check); [[nodiscard]] RegPtr maybeLoadInstanceForAccess( const MemoryAccessDesc* access, const AccessCheck& check, RegPtr specific); [[nodiscard]] bool emitLoad(ValType type, Scalar::Type viewType); [[nodiscard]] bool emitStore(ValType resultType, Scalar::Type viewType); [[nodiscard]] bool emitSelect(bool typed); template [[nodiscard]] bool emitSetOrTeeLocal(uint32_t slot); [[nodiscard]] bool endBlock(ResultType type); [[nodiscard]] bool endIfThen(ResultType type); [[nodiscard]] bool endIfThenElse(ResultType type); [[nodiscard]] bool endTryCatch(ResultType type); [[nodiscard]] bool endTryTable(ResultType type); void doReturn(ContinuationKind kind); void emitCompareI32(Assembler::Condition compareOp, ValType compareType); void emitCompareI64(Assembler::Condition compareOp, ValType compareType); void emitCompareF32(Assembler::DoubleCondition compareOp, ValType compareType); void emitCompareF64(Assembler::DoubleCondition compareOp, ValType compareType); void emitCompareRef(Assembler::Condition compareOp, ValType compareType); template inline CompilerType& selectCompiler(); template inline void emitUnop(void (*op)(MacroAssembler& masm, SourceType rs, DestType rd)); template inline void emitUnop(void (*op)(MacroAssembler& masm, SourceType rs, DestType rd, TempType temp)); template inline void emitUnop(ImmType immediate, void (*op)(MacroAssembler&, ImmType, SourceType, DestType)); template inline void emitUnop(void (*op)(CompilerType& compiler, RegType rsd)); template inline void emitUnop(void (*op)(BaseCompiler& bc, RegType rsd, TempType rt), TempType (*getSpecializedTemp)(BaseCompiler& bc)); template inline void emitBinop(void (*op)(CompilerType& masm, RhsType src, LhsDestType srcDest)); template inline void emitBinop(void (*op)(MacroAssembler& masm, RhsDestType src, LhsType srcDest, RhsDestOp)); template inline void emitBinop(void (*)(MacroAssembler& masm, RhsType rs, LhsDestType rsd, TempType temp)); template inline void emitBinop(void (*)(MacroAssembler& masm, RhsType rs, LhsDestType rsd, TempType1 temp1, TempType2 temp2)); template inline void emitBinop(ImmType immediate, void (*op)(MacroAssembler&, ImmType, RhsType, LhsDestType)); template inline void emitBinop(ImmType immediate, void (*op)(MacroAssembler&, ImmType, RhsType, LhsDestType, TempType1 temp1, TempType2 temp2)); template inline void emitBinop(void (*op)(CompilerType1& compiler1, RegType rs, RegType rd), void (*opConst)(CompilerType2& compiler2, ImmType c, RegType rd), RegType (BaseCompiler::*rhsPopper)() = nullptr); template inline void emitTernary(void (*op)(CompilerType&, ValType src0, ValType src1, ValType srcDest)); template inline void emitTernary(void (*op)(CompilerType&, ValType src0, ValType src1, ValType srcDest, ValType temp)); template inline void emitTernaryResultLast(void (*op)(CompilerType&, ValType src0, ValType src1, ValType srcDest)); template [[nodiscard]] inline bool emitInstanceCallOp( const SymbolicAddressSignature& fn, R reader); template [[nodiscard]] inline bool emitInstanceCallOp( const SymbolicAddressSignature& fn, R reader); template [[nodiscard]] inline bool emitInstanceCallOp( const SymbolicAddressSignature& fn, R reader); void emitMultiplyI64(); void emitQuotientI32(); void emitQuotientU32(); void emitRemainderI32(); void emitRemainderU32(); #ifdef RABALDR_INT_DIV_I64_CALLOUT [[nodiscard]] bool emitDivOrModI64BuiltinCall(SymbolicAddress callee, ValType operandType); #else void emitQuotientI64(); void emitQuotientU64(); void emitRemainderI64(); void emitRemainderU64(); #endif void emitRotrI64(); void emitRotlI64(); void emitEqzI32(); void emitEqzI64(); template [[nodiscard]] bool emitTruncateF32ToI32(); template [[nodiscard]] bool emitTruncateF64ToI32(); #ifdef RABALDR_FLOAT_TO_I64_CALLOUT [[nodiscard]] bool emitConvertFloatingToInt64Callout(SymbolicAddress callee, ValType operandType, ValType resultType); #else template [[nodiscard]] bool emitTruncateF32ToI64(); template [[nodiscard]] bool emitTruncateF64ToI64(); #endif void emitExtendI64_8(); void emitExtendI64_16(); void emitExtendI64_32(); void emitExtendI32ToI64(); void emitExtendU32ToI64(); #ifdef RABALDR_I64_TO_FLOAT_CALLOUT [[nodiscard]] bool emitConvertInt64ToFloatingCallout(SymbolicAddress callee, ValType operandType, ValType resultType); #else void emitConvertU64ToF32(); void emitConvertU64ToF64(); #endif void emitRound(RoundingMode roundingMode, ValType operandType); // Generate a call to the instance function denoted by `builtin`, passing as // args the top elements of the compiler's value stack and optionally an // Instance* too. The relationship between the top of stack and arg // ordering is as follows. If the value stack looks like this: // // A <- least recently pushed // B // C <- most recently pushed // // then the called function is expected to have signature [if an Instance* // is also to be passed]: // // static Instance::foo(Instance*, A, B, C) // // and the SymbolicAddressSignature::argTypes array will be // // {_PTR, _A, _B, _C, _END} // _PTR is for the Instance* // // (see WasmBuiltins.cpp). In short, the most recently pushed value is the // rightmost argument to the function. [[nodiscard]] bool emitInstanceCall(const SymbolicAddressSignature& builtin); [[nodiscard]] bool emitMemoryGrow(); [[nodiscard]] bool emitMemorySize(); [[nodiscard]] bool emitRefFunc(); [[nodiscard]] bool emitRefNull(); [[nodiscard]] bool emitRefIsNull(); [[nodiscard]] bool emitRefAsNonNull(); [[nodiscard]] bool emitBrOnNull(); [[nodiscard]] bool emitBrOnNonNull(); [[nodiscard]] bool emitCallRef(); [[nodiscard]] bool emitReturnCallRef(); [[nodiscard]] bool emitAtomicCmpXchg(ValType type, Scalar::Type viewType); [[nodiscard]] bool emitAtomicLoad(ValType type, Scalar::Type viewType); [[nodiscard]] bool emitAtomicRMW(ValType type, Scalar::Type viewType, AtomicOp op); [[nodiscard]] bool emitAtomicStore(ValType type, Scalar::Type viewType); [[nodiscard]] bool emitWait(ValType type, uint32_t byteSize); [[nodiscard]] bool atomicWait(ValType type, MemoryAccessDesc* access); [[nodiscard]] bool emitNotify(); [[nodiscard]] bool atomicNotify(MemoryAccessDesc* access); [[nodiscard]] bool emitFence(); [[nodiscard]] bool emitAtomicXchg(ValType type, Scalar::Type viewType); [[nodiscard]] bool emitMemInit(); [[nodiscard]] bool emitMemCopy(); [[nodiscard]] bool memCopyCall(uint32_t dstMemIndex, uint32_t srcMemIndex); void memCopyInlineM32(); [[nodiscard]] bool emitTableCopy(); [[nodiscard]] bool emitDataOrElemDrop(bool isData); [[nodiscard]] bool emitMemFill(); [[nodiscard]] bool memFillCall(uint32_t memoryIndex); void memFillInlineM32(); [[nodiscard]] bool emitTableInit(); [[nodiscard]] bool emitTableFill(); [[nodiscard]] bool emitMemDiscard(); [[nodiscard]] bool emitTableGet(); [[nodiscard]] bool emitTableGrow(); [[nodiscard]] bool emitTableSet(); [[nodiscard]] bool emitTableSize(); [[nodiscard]] bool emitI64AddSub128(bool isAdd); [[nodiscard]] bool emitI64MulWide(bool isSigned); void emitTableBoundsCheck(uint32_t tableIndex, RegI32 address, RegPtr instance); [[nodiscard]] bool emitTableGetAnyRef(uint32_t tableIndex); [[nodiscard]] bool emitTableSetAnyRef(uint32_t tableIndex); [[nodiscard]] bool emitStructNew(); [[nodiscard]] bool emitStructNewDefault(); [[nodiscard]] bool emitStructGet(FieldWideningOp wideningOp); [[nodiscard]] bool emitStructSet(); [[nodiscard]] bool emitArrayNew(); [[nodiscard]] bool emitArrayNewFixed(); [[nodiscard]] bool emitArrayNewDefault(); [[nodiscard]] bool emitArrayNewData(); [[nodiscard]] bool emitArrayNewElem(); [[nodiscard]] bool emitArrayInitData(); [[nodiscard]] bool emitArrayInitElem(); [[nodiscard]] bool emitArrayGet(FieldWideningOp wideningOp); [[nodiscard]] bool emitArraySet(); [[nodiscard]] bool emitArrayLen(); [[nodiscard]] bool emitArrayCopy(); [[nodiscard]] bool emitArrayFill(); [[nodiscard]] bool emitRefI31(); [[nodiscard]] bool emitI31Get(FieldWideningOp wideningOp); [[nodiscard]] bool emitRefTest(bool nullable); [[nodiscard]] bool emitRefCast(bool nullable); [[nodiscard]] bool emitBrOnCastCommon(bool onSuccess, uint32_t labelRelativeDepth, const ResultType& labelType, MaybeRefType sourceType, RefType destType); [[nodiscard]] bool emitBrOnCast(bool onSuccess); [[nodiscard]] bool emitAnyConvertExtern(); [[nodiscard]] bool emitExternConvertAny(); // Utility classes/methods to add trap information related to // null pointer dereferences/accesses. struct NoNullCheck { static void emitNullCheck(BaseCompiler* bc, RegRef rp) {} static void emitTrapSite(BaseCompiler* bc, FaultingCodeOffset fco, TrapMachineInsn tmi) {} }; struct SignalNullCheck { static void emitNullCheck(BaseCompiler* bc, RegRef rp); static void emitTrapSite(BaseCompiler* bc, FaultingCodeOffset fco, TrapMachineInsn tmi); }; // Load a pointer to the AllocSite for current bytecode offset RegPtr loadAllocSiteInstanceData(uint32_t allocSiteIndex); // Gets alloc site allociated with current instruction [[nodiscard]] bool readAllocSiteIndex(uint32_t* index); // Load a pointer to the SuperTypeVector for a given type index RegPtr loadSuperTypeVector(uint32_t typeIndex); // Emits allocation code for a GC struct. The struct may have an out-of-line // data area; if so, `isOutlineStruct` will be true and `outlineBase` will be // allocated and must be freed. template bool emitStructAlloc(uint32_t typeIndex, RegRef* object, bool* isOutlineStruct, RegPtr* outlineBase, uint32_t allocSiteIndex); // Emits allocation code for a dynamically-sized GC array. template bool emitArrayAlloc(uint32_t typeIndex, RegRef object, RegI32 numElements, uint32_t elemSize, uint32_t allocSiteIndex); // Emits allocation code for a fixed-size GC array. template bool emitArrayAllocFixed(uint32_t typeIndex, RegRef object, uint32_t numElements, uint32_t elemSize, uint32_t allocSiteIndex); template RegPtr emitGcArrayGetData(RegRef rp); template RegI32 emitGcArrayGetNumElements(RegRef rp); void emitGcArrayBoundsCheck(RegI32 index, RegI32 numElements); template void emitGcGet(StorageType type, FieldWideningOp wideningOp, const T& src); template void emitGcSetScalar(const T& dst, StorageType type, AnyReg value); BranchIfRefSubtypeRegisters allocRegistersForBranchIfRefSubtype( RefType destType); void freeRegistersForBranchIfRefSubtype( const BranchIfRefSubtypeRegisters& regs); // Write `value` to wasm struct `object`, at `areaBase + areaOffset`. The // caller must decide on the in- vs out-of-lineness before the call and set // the latter two accordingly; this routine does not take that into account. // The value in `object` is unmodified, but `areaBase` and `value` may get // trashed. template [[nodiscard]] bool emitGcStructSet(RegRef object, RegPtr areaBase, uint32_t areaOffset, StorageType type, AnyReg value, PreBarrierKind preBarrierKind); [[nodiscard]] bool emitGcArraySet(RegRef object, RegPtr data, RegI32 index, const ArrayType& array, AnyReg value, PreBarrierKind preBarrierKind, PostBarrierKind postBarrierKind); #ifdef ENABLE_WASM_SIMD void emitVectorAndNot(); # ifdef ENABLE_WASM_RELAXED_SIMD void emitDotI8x16I7x16AddS(); # endif void loadSplat(MemoryAccessDesc* access); void loadZero(MemoryAccessDesc* access); void loadExtend(MemoryAccessDesc* access, Scalar::Type viewType); void loadLane(MemoryAccessDesc* access, uint32_t laneIndex); void storeLane(MemoryAccessDesc* access, uint32_t laneIndex); [[nodiscard]] bool emitLoadSplat(Scalar::Type viewType); [[nodiscard]] bool emitLoadZero(Scalar::Type viewType); [[nodiscard]] bool emitLoadExtend(Scalar::Type viewType); [[nodiscard]] bool emitLoadLane(uint32_t laneSize); [[nodiscard]] bool emitStoreLane(uint32_t laneSize); [[nodiscard]] bool emitVectorShuffle(); [[nodiscard]] bool emitVectorLaneSelect(); # if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) [[nodiscard]] bool emitVectorShiftRightI64x2(); # endif #endif [[nodiscard]] bool emitCallBuiltinModuleFunc(); }; } // namespace wasm } // namespace js #endif // wasm_wasm_baseline_object_h