/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#ifndef jit_riscv64_MacroAssembler_riscv64_inl_h
#define jit_riscv64_MacroAssembler_riscv64_inl_h

#include "jit/riscv64/MacroAssembler-riscv64.h"

namespace js {
namespace jit {

template <>
inline void MacroAssembler::cmpPtrSet(Assembler::Condition cond, Address lhs,
                                      ImmPtr rhs, Register dest) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  loadPtr(lhs, scratch2);
  cmpPtrSet(cond, Register(scratch2), rhs, dest);
}

template <>
inline void MacroAssembler::cmpPtrSet(Assembler::Condition cond, Register lhs,
                                      Address rhs, Register dest) {
  UseScratchRegisterScope temps(this);
  Register scratch = temps.Acquire();
  MOZ_ASSERT(lhs != scratch);
  loadPtr(rhs, scratch);
  cmpPtrSet(cond, lhs, Register(scratch), dest);
}

template <>
inline void MacroAssembler::cmpPtrSet(Assembler::Condition cond, Address lhs,
                                      Register rhs, Register dest) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  MOZ_ASSERT(rhs != scratch2);
  loadPtr(lhs, scratch2);
  cmpPtrSet(cond, Register(scratch2), rhs, dest);
}

template <>
inline void MacroAssembler::cmp32Set(Assembler::Condition cond, Register lhs,
                                     Address rhs, Register dest) {
  UseScratchRegisterScope temps(this);
  Register scratch = temps.Acquire();
  MOZ_ASSERT(lhs != scratch);
  load32(rhs, scratch);
  cmp32Set(cond, lhs, Register(scratch), dest);
}

template <>
inline void MacroAssembler::cmp32Set(Assembler::Condition cond, Address lhs,
                                     Register rhs, Register dest) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  MOZ_ASSERT(rhs != scratch2);
  load32(lhs, scratch2);
  cmp32Set(cond, Register(scratch2), rhs, dest);
}

//{{{ check_macroassembler_style
CodeOffset MacroAssembler::sub32FromStackPtrWithPatch(Register dest) {
  // 2 instruction to materialize the constant.
  // + 1 instruction for sub.
  AutoForbidPoolsAndNops afp(this, 3);

  BufferOffset offset = MacroAssemblerRiscv64::ma_liPatchable(dest, Imm32(0));
  sub(dest, StackPointer, dest);
  return CodeOffset(offset.getOffset());
}

void MacroAssembler::patchSub32FromStackPtr(CodeOffset offset, Imm32 imm) {
  patchLi32(offset, imm);
}

void MacroAssembler::branchTest32(Condition cond, Register lhs, Imm32 rhs,
                                  Label* label) {
  MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
             cond == NotSigned);
  UseScratchRegisterScope temps(this);
  Register scratch = temps.Acquire();
  ma_and(scratch, lhs, rhs);
  ma_b(scratch, scratch, label, cond, ShortJump);
}
void MacroAssembler::branchTest32(Condition cond, Register lhs, Register rhs,
                                  Label* label) {
  MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
             cond == NotSigned);
  if (lhs == rhs) {
    ma_b(lhs, rhs, label, cond, ShortJump);
  } else {
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();
    and_(scratch, lhs, rhs);
    ma_b(scratch, scratch, label, cond, ShortJump);
  }
}
void MacroAssembler::branchTest64(Condition cond, Register64 lhs,
                                  Register64 rhs, Register temp, Label* success,
                                  Label* fail) {
  branchTestPtr(cond, lhs.reg, rhs.reg, success);
  if (fail) {
    jump(fail);
  }
}
void MacroAssembler::branchTest64(Condition cond, Register64 lhs, Imm64 rhs,
                                  Label* success, Label* fail) {
  branchTestPtr(cond, lhs.reg, ImmWord(rhs.value), success);
  if (fail) {
    jump(fail);
  }
}

template <typename T>
void MacroAssembler::branchAdd32(Condition cond, T src, Register dest,
                                 Label* overflow) {
  switch (cond) {
    case Overflow:
      ma_add32TestOverflow(dest, dest, src, overflow);
      break;
    case CarryClear:
    case CarrySet:
      ma_add32TestCarry(cond, dest, dest, src, overflow);
      break;
    default:
      MOZ_CRASH("NYI");
  }
}
template <typename T>
void MacroAssembler::branchAddPtr(Condition cond, T src, Register dest,
                                  Label* label) {
  switch (cond) {
    case Overflow:
      ma_addPtrTestOverflow(dest, dest, src, label);
      break;
    case CarryClear:
    case CarrySet:
      ma_addPtrTestCarry(cond, dest, dest, src, label);
      break;
    case Signed:
    case NotSigned:
      ma_addPtrTestSigned(cond, dest, dest, src, label);
      break;
    default:
      MOZ_CRASH("NYI");
  }
}
template <typename T>
void MacroAssembler::branchMul32(Condition cond, T src, Register dest,
                                 Label* overflow) {
  MOZ_ASSERT(cond == Assembler::Overflow);
  ma_mul32TestOverflow(dest, dest, src, overflow);
}
template <typename T>
void MacroAssembler::branchRshift32(Condition cond, T src, Register dest,
                                    Label* label) {
  MOZ_ASSERT(cond == Zero || cond == NonZero);
  rshift32(src, dest);
  branch32(cond == Zero ? Equal : NotEqual, dest, Imm32(0), label);
}
// the type of 'T src' maybe a Register, maybe a Imm32,depends on who call it.
template <typename T>
void MacroAssembler::branchSub32(Condition cond, T src, Register dest,
                                 Label* label) {
  switch (cond) {
    case Overflow:
      ma_sub32TestOverflow(dest, dest, src, label);
      break;
    case NonZero:
    case Zero:
    case Signed:
    case NotSigned:
      sub32(src, dest);
      ma_b(dest, dest, label, cond, ShortJump);
      break;
    default:
      MOZ_CRASH("NYI");
  }
}
template <typename T>
void MacroAssembler::branchSubPtr(Condition cond, T src, Register dest,
                                  Label* label) {
  switch (cond) {
    case Overflow:
      ma_subPtrTestOverflow(dest, dest, src, label);
      break;
    case NonZero:
    case Zero:
    case Signed:
    case NotSigned:
      subPtr(src, dest);
      ma_b(dest, dest, label, cond, ShortJump);
      break;
    default:
      MOZ_CRASH("NYI");
  }
}
template <typename T>
void MacroAssembler::branchTestGCThingImpl(Condition cond, const T& address,
                                           Label* label) {
  MOZ_ASSERT(cond == Equal || cond == NotEqual);
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  Register tag = extractTag(address, scratch2);
  ma_b(tag, ImmTagSignExt(JS::detail::ValueLowerInclGCThingTag), label,
       (cond == Equal) ? AboveOrEqual : Below, ShortJump);
}
template <typename T>
void MacroAssembler::testBigIntSet(Condition cond, const T& src,
                                   Register dest) {
  MOZ_ASSERT(cond == Equal || cond == NotEqual);
  Register tag = extractTag(src, dest);
  ma_cmp_set(dest, tag, ImmTagSignExt(JSVAL_TAG_BIGINT), cond);
}

template <typename T>
void MacroAssembler::testBooleanSet(Condition cond, const T& src,
                                    Register dest) {
  MOZ_ASSERT(cond == Equal || cond == NotEqual);
  Register tag = extractTag(src, dest);
  ma_cmp_set(dest, tag, ImmTagSignExt(JSVAL_TAG_BOOLEAN), cond);
}

template <typename T>
void MacroAssembler::testNumberSet(Condition cond, const T& src,
                                   Register dest) {
  MOZ_ASSERT(cond == Equal || cond == NotEqual);
  Register tag = extractTag(src, dest);
  ma_cmp_set(dest, tag, ImmTagSignExt(JS::detail::ValueUpperInclNumberTag),
             cond == Equal ? BelowOrEqual : Above);
}

template <typename T>
void MacroAssembler::testStringSet(Condition cond, const T& src,
                                   Register dest) {
  MOZ_ASSERT(cond == Equal || cond == NotEqual);
  Register tag = extractTag(src, dest);
  ma_cmp_set(dest, tag, ImmTagSignExt(JSVAL_TAG_STRING), cond);
}

template <typename T>
void MacroAssembler::testSymbolSet(Condition cond, const T& src,
                                   Register dest) {
  MOZ_ASSERT(cond == Equal || cond == NotEqual);
  Register tag = extractTag(src, dest);
  ma_cmp_set(dest, tag, ImmTagSignExt(JSVAL_TAG_SYMBOL), cond);
}

// Also see below for specializations of cmpPtrSet.
template <typename T1, typename T2>
void MacroAssembler::cmp32Set(Condition cond, T1 lhs, T2 rhs, Register dest) {
  ma_cmp_set(dest, lhs, rhs, cond);
}
template <typename T1, typename T2>
void MacroAssembler::cmpPtrSet(Condition cond, T1 lhs, T2 rhs, Register dest) {
  ma_cmp_set(dest, lhs, rhs, cond);
}
void MacroAssembler::abs32(Register src, Register dest) {
  UseScratchRegisterScope temps(this);
  Register scratch = temps.Acquire();
  sraiw(scratch, src, 31);
  xor_(dest, src, scratch);
  subw(dest, dest, scratch);
}
void MacroAssembler::absFloat32(FloatRegister src, FloatRegister dest) {
  fabs_s(dest, src);
}

void MacroAssembler::absDouble(FloatRegister src, FloatRegister dest) {
  fabs_d(dest, src);
}
void MacroAssembler::add32(Register src, Register dest) {
  addw(dest, dest, src);
}

void MacroAssembler::add32(Imm32 imm, Register dest) {
  ma_add32(dest, dest, imm);
}

void MacroAssembler::add32(Imm32 imm, Register src, Register dest) {
  ma_add32(dest, src, imm);
}

void MacroAssembler::add32(Imm32 imm, const Address& dest) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  load32(dest, scratch2);
  ma_add32(scratch2, scratch2, imm);
  store32(scratch2, dest);
}

void MacroAssembler::add32(const Address& src, Register dest) {
  UseScratchRegisterScope temps(this);
  Register scratch = temps.Acquire();
  load32(src, scratch);
  addw(dest, dest, scratch);
}

void MacroAssembler::add64(Register64 src, Register64 dest) {
  addPtr(src.reg, dest.reg);
}

void MacroAssembler::add64(Imm32 imm, Register64 dest) {
  ma_add64(dest.reg, dest.reg, imm);
}

void MacroAssembler::add64(Imm64 imm, Register64 dest) {
  ma_add64(dest.reg, dest.reg, imm);
}

void MacroAssembler::addDouble(FloatRegister src, FloatRegister dest) {
  fadd_d(dest, dest, src);
}

void MacroAssembler::addFloat32(FloatRegister src, FloatRegister dest) {
  fadd_s(dest, dest, src);
}
void MacroAssembler::addPtr(Register src, Register dest) {
  add(dest, dest, src);
}

void MacroAssembler::addPtr(Imm32 imm, Register dest) {
  ma_add64(dest, dest, imm);
}

void MacroAssembler::addPtr(ImmWord imm, Register dest) {
  UseScratchRegisterScope temps(this);
  Register scratch = temps.Acquire();
  movePtr(imm, scratch);
  addPtr(scratch, dest);
}
void MacroAssembler::addPtr(Imm32 imm, const Address& dest) {
  UseScratchRegisterScope temps(this);
  Register scratch = temps.Acquire();
  loadPtr(dest, scratch);
  addPtr(imm, scratch);
  storePtr(scratch, dest);
}

void MacroAssembler::addPtr(const Address& src, Register dest) {
  UseScratchRegisterScope temps(this);
  Register scratch = temps.Acquire();
  loadPtr(src, scratch);
  addPtr(scratch, dest);
}
void MacroAssembler::and32(Register src, Register dest) {
  and_(dest, dest, src);
}

void MacroAssembler::and32(Imm32 imm, Register dest) {
  ma_and(dest, dest, imm);
}

void MacroAssembler::and32(Imm32 imm, Register src, Register dest) {
  ma_and(dest, src, imm);
}

void MacroAssembler::and32(Imm32 imm, const Address& dest) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  load32(dest, scratch2);
  ma_and(scratch2, scratch2, imm);
  store32(scratch2, dest);
}

void MacroAssembler::and32(const Address& src, Register dest) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  load32(src, scratch2);
  and_(dest, dest, scratch2);
}
void MacroAssembler::and64(Imm64 imm, Register64 dest) {
  ma_and(dest.reg, dest.reg, imm);
}

void MacroAssembler::and64(Register64 src, Register64 dest) {
  and_(dest.reg, dest.reg, src.reg);
}

void MacroAssembler::andPtr(Register src, Register dest) {
  and_(dest, dest, src);
}

void MacroAssembler::andPtr(Imm32 imm, Register dest) {
  ma_and(dest, dest, imm);
}

void MacroAssembler::andPtr(Imm32 imm, Register src, Register dest) {
  ma_and(dest, src, imm);
}

void MacroAssembler::branch8(Condition cond, const Address& lhs, Imm32 rhs,
                             Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  MOZ_ASSERT(scratch2 != lhs.base);

  switch (cond) {
    case Assembler::Equal:
    case Assembler::NotEqual:
    case Assembler::Above:
    case Assembler::AboveOrEqual:
    case Assembler::Below:
    case Assembler::BelowOrEqual:
      load8ZeroExtend(lhs, scratch2);
      branch32(cond, scratch2, Imm32(uint8_t(rhs.value)), label);
      break;

    case Assembler::GreaterThan:
    case Assembler::GreaterThanOrEqual:
    case Assembler::LessThan:
    case Assembler::LessThanOrEqual:
      load8SignExtend(lhs, scratch2);
      branch32(cond, scratch2, Imm32(int8_t(rhs.value)), label);
      break;

    default:
      MOZ_CRASH("unexpected condition");
  }
}

void MacroAssembler::branch8(Condition cond, const BaseIndex& lhs, Register rhs,
                             Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  MOZ_ASSERT(scratch2 != lhs.base);

  computeScaledAddress(lhs, scratch2);

  switch (cond) {
    case Assembler::Equal:
    case Assembler::NotEqual:
    case Assembler::Above:
    case Assembler::AboveOrEqual:
    case Assembler::Below:
    case Assembler::BelowOrEqual:
      load8ZeroExtend(Address(scratch2, lhs.offset), scratch2);
      branch32(cond, scratch2, rhs, label);
      break;

    case Assembler::GreaterThan:
    case Assembler::GreaterThanOrEqual:
    case Assembler::LessThan:
    case Assembler::LessThanOrEqual:
      load8SignExtend(Address(scratch2, lhs.offset), scratch2);
      branch32(cond, scratch2, rhs, label);
      break;

    default:
      MOZ_CRASH("unexpected condition");
  }
}

void MacroAssembler::branch16(Condition cond, const Address& lhs, Imm32 rhs,
                              Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  MOZ_ASSERT(scratch2 != lhs.base);

  switch (cond) {
    case Assembler::Equal:
    case Assembler::NotEqual:
    case Assembler::Above:
    case Assembler::AboveOrEqual:
    case Assembler::Below:
    case Assembler::BelowOrEqual:
      load16ZeroExtend(lhs, scratch2);
      branch32(cond, scratch2, Imm32(uint16_t(rhs.value)), label);
      break;

    case Assembler::GreaterThan:
    case Assembler::GreaterThanOrEqual:
    case Assembler::LessThan:
    case Assembler::LessThanOrEqual:
      load16SignExtend(lhs, scratch2);
      branch32(cond, scratch2, Imm32(int16_t(rhs.value)), label);
      break;

    default:
      MOZ_CRASH("unexpected condition");
  }
}
void MacroAssembler::branch32(Condition cond, Register lhs, Register rhs,
                              Label* label) {
  ma_b(lhs, rhs, label, cond, ShortJump);
}

void MacroAssembler::branch32(Condition cond, Register lhs, Imm32 imm,
                              Label* label) {
  ma_b(lhs, imm, label, cond, ShortJump);
}

void MacroAssembler::branch32(Condition cond, const Address& lhs, Register rhs,
                              Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  load32(lhs, scratch2);
  ma_b(scratch2, rhs, label, cond, ShortJump);
}

void MacroAssembler::branch32(Condition cond, const Address& lhs, Imm32 rhs,
                              Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  load32(lhs, scratch2);
  ma_b(scratch2, rhs, label, cond, ShortJump);
}

void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs,
                              Register rhs, Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  load32(lhs, scratch2);
  ma_b(scratch2, rhs, label, cond, ShortJump);
}

void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs,
                              Imm32 rhs, Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  load32(lhs, scratch2);
  ma_b(scratch2, rhs, label, cond, ShortJump);
}

void MacroAssembler::branch32(Condition cond, const BaseIndex& lhs, Imm32 rhs,
                              Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  load32(lhs, scratch2);
  ma_b(scratch2, rhs, label, cond, ShortJump);
}

void MacroAssembler::branch32(Condition cond, wasm::SymbolicAddress addr,
                              Imm32 imm, Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  load32(addr, scratch2);
  ma_b(scratch2, imm, label, cond, ShortJump);
}
void MacroAssembler::branch64(Condition cond, Register64 lhs, Imm64 val,
                              Label* success, Label* fail) {
  MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal ||
                 cond == Assembler::LessThan ||
                 cond == Assembler::LessThanOrEqual ||
                 cond == Assembler::GreaterThan ||
                 cond == Assembler::GreaterThanOrEqual ||
                 cond == Assembler::Below || cond == Assembler::BelowOrEqual ||
                 cond == Assembler::Above || cond == Assembler::AboveOrEqual,
             "other condition codes not supported");

  branchPtr(cond, lhs.reg, ImmWord(val.value), success);
  if (fail) {
    jump(fail);
  }
}

void MacroAssembler::branch64(Condition cond, Register64 lhs, Register64 rhs,
                              Label* success, Label* fail) {
  MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal ||
                 cond == Assembler::LessThan ||
                 cond == Assembler::LessThanOrEqual ||
                 cond == Assembler::GreaterThan ||
                 cond == Assembler::GreaterThanOrEqual ||
                 cond == Assembler::Below || cond == Assembler::BelowOrEqual ||
                 cond == Assembler::Above || cond == Assembler::AboveOrEqual,
             "other condition codes not supported");

  branchPtr(cond, lhs.reg, rhs.reg, success);
  if (fail) {
    jump(fail);
  }
}

void MacroAssembler::branch64(Condition cond, const Address& lhs, Imm64 val,
                              Label* success, Label* fail) {
  MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal ||
                 cond == Assembler::LessThan ||
                 cond == Assembler::LessThanOrEqual ||
                 cond == Assembler::GreaterThan ||
                 cond == Assembler::GreaterThanOrEqual ||
                 cond == Assembler::Below || cond == Assembler::BelowOrEqual ||
                 cond == Assembler::Above || cond == Assembler::AboveOrEqual,
             "other condition codes not supported");

  branchPtr(cond, lhs, ImmWord(val.value), success);
  if (fail) {
    jump(fail);
  }
}

void MacroAssembler::branch64(Condition cond, const Address& lhs,
                              Register64 rhs, Label* success, Label* fail) {
  MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal ||
                 cond == Assembler::LessThan ||
                 cond == Assembler::LessThanOrEqual ||
                 cond == Assembler::GreaterThan ||
                 cond == Assembler::GreaterThanOrEqual ||
                 cond == Assembler::Below || cond == Assembler::BelowOrEqual ||
                 cond == Assembler::Above || cond == Assembler::AboveOrEqual,
             "other condition codes not supported");

  branchPtr(cond, lhs, rhs.reg, success);
  if (fail) {
    jump(fail);
  }
}

void MacroAssembler::branch64(Condition cond, const Address& lhs,
                              const Address& rhs, Register scratch,
                              Label* label) {
  MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal,
             "other condition codes not supported");
  MOZ_ASSERT(lhs.base != scratch);
  MOZ_ASSERT(rhs.base != scratch);

  loadPtr(rhs, scratch);
  branchPtr(cond, lhs, scratch, label);
}

void MacroAssembler::branchDouble(DoubleCondition cc, FloatRegister frs1,
                                  FloatRegister frs2, Label* L) {
  BranchFloat64(cc, frs1, frs2, L, ShortJump);
}
void MacroAssembler::branchFloat(DoubleCondition cc, FloatRegister frs1,
                                 FloatRegister frs2, Label* L) {
  BranchFloat32(cc, frs1, frs2, L, ShortJump);
}

void MacroAssembler::branchMulPtr(Condition cond, Register src, Register dest,
                                  Label* label) {
  MOZ_ASSERT(cond == Assembler::Overflow);
  ma_mulPtrTestOverflow(dest, dest, src, label);
}

void MacroAssembler::branchNegPtr(Condition cond, Register reg, Label* label) {
  MOZ_ASSERT(cond == Overflow);
  negPtr(reg);
  branchPtr(Assembler::Equal, reg, ImmWord(INTPTR_MIN), label);
}
void MacroAssembler::branchNeg32(Condition cond, Register reg, Label* label) {
  MOZ_ASSERT(cond == Overflow);
  neg32(reg);
  branch32(Assembler::Equal, reg, Imm32(INT32_MIN), label);
}

void MacroAssembler::branchPrivatePtr(Condition cond, const Address& lhs,
                                      Register rhs, Label* label) {
  branchPtr(cond, lhs, rhs, label);
}

void MacroAssembler::branchPtr(Condition cond, Register lhs, Register rhs,
                               Label* label) {
  ma_b(lhs, rhs, label, cond, ShortJump);
}

void MacroAssembler::branchPtr(Condition cond, Register lhs, Imm32 rhs,
                               Label* label) {
  ma_b(lhs, rhs, label, cond, ShortJump);
}

void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmPtr rhs,
                               Label* label) {
  if (rhs.value == nullptr && (cond == Zero || cond == NonZero)) {
    ma_b(lhs, lhs, label, cond, ShortJump);
  } else {
    ma_b(lhs, rhs, label, cond, ShortJump);
  }
}

void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmGCPtr rhs,
                               Label* label) {
  ma_b(lhs, rhs, label, cond, ShortJump);
}

void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmWord rhs,
                               Label* label) {
  ma_b(lhs, rhs, label, cond, ShortJump);
}

void MacroAssembler::branchPtr(Condition cond, const Address& lhs, Register rhs,
                               Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  loadPtr(lhs, scratch2);
  branchPtr(cond, scratch2, rhs, label);
}

void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmPtr rhs,
                               Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  loadPtr(lhs, scratch2);
  branchPtr(cond, scratch2, rhs, label);
}

void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmGCPtr rhs,
                               Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  loadPtr(lhs, scratch2);
  branchPtr(cond, scratch2, rhs, label);
}

void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmWord rhs,
                               Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  loadPtr(lhs, scratch2);
  branchPtr(cond, scratch2, rhs, label);
}

void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs,
                               Register rhs, Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  loadPtr(lhs, scratch2);
  branchPtr(cond, scratch2, rhs, label);
}

void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs,
                               ImmWord rhs, Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  loadPtr(lhs, scratch2);
  branchPtr(cond, scratch2, rhs, label);
}

void MacroAssembler::branchPtr(Condition cond, wasm::SymbolicAddress lhs,
                               Register rhs, Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  loadPtr(lhs, scratch2);
  branchPtr(cond, scratch2, rhs, label);
}

void MacroAssembler::branchPtr(Condition cond, const BaseIndex& lhs,
                               Register rhs, Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  loadPtr(lhs, scratch2);
  branchPtr(cond, scratch2, rhs, label);
}

void MacroAssembler::branchPtr(Condition cond, const BaseIndex& lhs,
                               ImmWord rhs, Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  loadPtr(lhs, scratch2);
  branchPtr(cond, scratch2, rhs, label);
}

void MacroAssembler::branchTest32(Condition cond, const AbsoluteAddress& lhs,
                                  Imm32 rhs, Label* label) {
  MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
             cond == NotSigned);
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  load32(lhs, scratch2);
  and32(rhs, scratch2);
  ma_b(scratch2, scratch2, label, cond, ShortJump);
}

void MacroAssembler::branchTest32(Condition cond, const Address& lhs, Imm32 rhs,
                                  Label* label) {
  MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
             cond == NotSigned);
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  load32(lhs, scratch2);
  and32(rhs, scratch2);
  ma_b(scratch2, scratch2, label, cond, ShortJump);
}
void MacroAssembler::branchTestBigInt(Condition cond, Register tag,
                                      Label* label) {
  MOZ_ASSERT(cond == Equal || cond == NotEqual);
  ma_b(tag, ImmTagSignExt(JSVAL_TAG_BIGINT), label, cond, ShortJump);
}

void MacroAssembler::branchTestBigInt(Condition cond, const ValueOperand& value,
                                      Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  splitSignExtTag(value, scratch2);
  branchTestBigInt(cond, scratch2, label);
}

void MacroAssembler::branchTestBigInt(Condition cond, const Address& address,
                                      Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  Register tag = extractTag(address, scratch2);
  branchTestBigInt(cond, tag, label);
}

void MacroAssembler::branchTestBigInt(Condition cond, const BaseIndex& address,
                                      Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  Register tag = extractTag(address, scratch2);
  branchTestBigInt(cond, tag, label);
}
void MacroAssembler::branchTestBigIntTruthy(bool b, const ValueOperand& value,
                                            Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  unboxBigInt(value, scratch2);
  load32(Address(scratch2, BigInt::offsetOfDigitLength()), scratch2);
  ma_b(scratch2, Imm32(0), label, b ? NotEqual : Equal, ShortJump);
}
void MacroAssembler::branchTestBoolean(Condition cond, Register tag,
                                       Label* label) {
  MOZ_ASSERT(cond == Equal || cond == NotEqual);
  ma_b(tag, ImmTagSignExt(JSVAL_TAG_BOOLEAN), label, cond, ShortJump);
}

void MacroAssembler::branchTestBoolean(Condition cond,
                                       const ValueOperand& value,
                                       Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  splitSignExtTag(value, scratch2);
  branchTestBoolean(cond, scratch2, label);
}

void MacroAssembler::branchTestBoolean(Condition cond, const Address& address,
                                       Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  Register tag = extractTag(address, scratch2);
  branchTestBoolean(cond, tag, label);
}

void MacroAssembler::branchTestBoolean(Condition cond, const BaseIndex& address,
                                       Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  Register tag = extractTag(address, scratch2);
  branchTestBoolean(cond, tag, label);
}
void MacroAssembler::branchTestBooleanTruthy(bool b, const ValueOperand& value,
                                             Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  unboxBoolean(value, scratch2);
  ma_b(scratch2, scratch2, label, b ? NonZero : Zero, ShortJump);
}
void MacroAssembler::branchTestDouble(Condition cond, Register tag,
                                      Label* label) {
  MOZ_ASSERT(cond == Equal || cond == NotEqual);
  Condition actual = (cond == Equal) ? BelowOrEqual : Above;
  ma_b(tag, ImmTagSignExt(JSVAL_TAG_MAX_DOUBLE), label, actual, ShortJump);
}

void MacroAssembler::branchTestDouble(Condition cond, const ValueOperand& value,
                                      Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  splitSignExtTag(value, scratch2);
  branchTestDouble(cond, scratch2, label);
}

void MacroAssembler::branchTestDouble(Condition cond, const Address& address,
                                      Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  Register tag = extractTag(address, scratch2);
  branchTestDouble(cond, tag, label);
}

void MacroAssembler::branchTestDouble(Condition cond, const BaseIndex& address,
                                      Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  Register tag = extractTag(address, scratch2);
  branchTestDouble(cond, tag, label);
}

void MacroAssembler::branchTestDoubleTruthy(bool b, FloatRegister value,
                                            Label* label) {
  ScratchDoubleScope fpscratch(*this);
  loadConstantDouble(0.0, fpscratch);
  DoubleCondition cond = b ? DoubleNotEqual : DoubleEqualOrUnordered;
  branchDouble(cond, value, fpscratch, label);
}
void MacroAssembler::branchTestGCThing(Condition cond, const Address& address,
                                       Label* label) {
  branchTestGCThingImpl(cond, address, label);
}

void MacroAssembler::branchTestGCThing(Condition cond, const BaseIndex& address,
                                       Label* label) {
  branchTestGCThingImpl(cond, address, label);
}

void MacroAssembler::branchTestGCThing(Condition cond,
                                       const ValueOperand& address,
                                       Label* label) {
  branchTestGCThingImpl(cond, address, label);
}
void MacroAssembler::branchTestInt32(Condition cond, Register tag,
                                     Label* label) {
  MOZ_ASSERT(cond == Equal || cond == NotEqual);
  ma_b(tag, ImmTagSignExt(JSVAL_TAG_INT32), label, cond, ShortJump);
}

void MacroAssembler::branchTestInt32(Condition cond, const ValueOperand& value,
                                     Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  splitSignExtTag(value, scratch2);
  branchTestInt32(cond, scratch2, label);
}

void MacroAssembler::branchTestInt32(Condition cond, const Address& address,
                                     Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  Register tag = extractTag(address, scratch2);
  branchTestInt32(cond, tag, label);
}

void MacroAssembler::branchTestInt32(Condition cond, const BaseIndex& address,
                                     Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  Register tag = extractTag(address, scratch2);
  branchTestInt32(cond, tag, label);
}
void MacroAssembler::branchTestInt32Truthy(bool b, const ValueOperand& value,
                                           Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch = temps.Acquire();
  SignExtendWord(scratch, value.valueReg());
  ma_b(scratch, scratch, label, b ? NonZero : Zero, ShortJump);
}
void MacroAssembler::branchTestMagic(Condition cond, Register tag,
                                     Label* label) {
  MOZ_ASSERT(cond == Equal || cond == NotEqual);
  ma_b(tag, ImmTagSignExt(JSVAL_TAG_MAGIC), label, cond, ShortJump);
}

void MacroAssembler::branchTestMagic(Condition cond, const Address& address,
                                     Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  Register tag = extractTag(address, scratch2);
  branchTestMagic(cond, tag, label);
}

void MacroAssembler::branchTestMagic(Condition cond, const BaseIndex& address,
                                     Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  Register tag = extractTag(address, scratch2);
  branchTestMagic(cond, tag, label);
}

void MacroAssembler::branchTestMagic(Condition cond, const ValueOperand& value,
                                     Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  splitSignExtTag(value, scratch2);
  branchTestMagic(cond, scratch2, label);
}

void MacroAssembler::branchTestMagic(Condition cond, const Address& valaddr,
                                     JSWhyMagic why, Label* label) {
  uint64_t magic = MagicValue(why).asRawBits();
  UseScratchRegisterScope temps(this);
  Register scratch = temps.Acquire();
  loadPtr(valaddr, scratch);
  ma_b(scratch, ImmWord(magic), label, cond, ShortJump);
}

void MacroAssembler::branchTestMagic(Condition cond, const BaseIndex& valaddr,
                                     JSWhyMagic why, Label* label) {
  uint64_t magic = MagicValue(why).asRawBits();
  UseScratchRegisterScope temps(this);
  Register scratch = temps.Acquire();
  loadPtr(valaddr, scratch);
  ma_b(scratch, ImmWord(magic), label, cond, ShortJump);
}
void MacroAssembler::branchTestNull(Condition cond, Register tag,
                                    Label* label) {
  MOZ_ASSERT(cond == Equal || cond == NotEqual);
  ma_b(tag, ImmTagSignExt(JSVAL_TAG_NULL), label, cond, ShortJump);
}

void MacroAssembler::branchTestNull(Condition cond, const ValueOperand& value,
                                    Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  splitSignExtTag(value, scratch2);
  branchTestNull(cond, scratch2, label);
}

void MacroAssembler::branchTestNull(Condition cond, const Address& address,
                                    Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  Register tag = extractTag(address, scratch2);
  branchTestNull(cond, tag, label);
}

void MacroAssembler::branchTestNull(Condition cond, const BaseIndex& address,
                                    Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  Register tag = extractTag(address, scratch2);
  branchTestNull(cond, tag, label);
}
void MacroAssembler::branchTestNumber(Condition cond, Register tag,
                                      Label* label) {
  MOZ_ASSERT(cond == Equal || cond == NotEqual);
  Condition actual = cond == Equal ? BelowOrEqual : Above;
  ma_b(tag, ImmTagSignExt(JS::detail::ValueUpperInclNumberTag), label, actual,
       ShortJump);
}

void MacroAssembler::branchTestNumber(Condition cond, const ValueOperand& value,
                                      Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  splitSignExtTag(value, scratch2);
  branchTestNumber(cond, scratch2, label);
}
void MacroAssembler::branchTestObject(Condition cond, Register tag,
                                      Label* label) {
  MOZ_ASSERT(cond == Equal || cond == NotEqual);
  ma_b(tag, ImmTagSignExt(JSVAL_TAG_OBJECT), label, cond, ShortJump);
}

void MacroAssembler::branchTestObject(Condition cond, const ValueOperand& value,
                                      Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  splitSignExtTag(value, scratch2);
  branchTestObject(cond, scratch2, label);
}

void MacroAssembler::branchTestObject(Condition cond, const Address& address,
                                      Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  Register tag = extractTag(address, scratch2);
  branchTestObject(cond, tag, label);
}

void MacroAssembler::branchTestObject(Condition cond, const BaseIndex& address,
                                      Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  Register tag = extractTag(address, scratch2);
  branchTestObject(cond, tag, label);
}
void MacroAssembler::branchTestPrimitive(Condition cond,
                                         const ValueOperand& value,
                                         Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  splitSignExtTag(value, scratch2);
  branchTestPrimitive(cond, scratch2, label);
}
void MacroAssembler::branchTestPrimitive(Condition cond, Register tag,
                                         Label* label) {
  MOZ_ASSERT(cond == Equal || cond == NotEqual);
  ma_b(tag, ImmTagSignExt(JS::detail::ValueUpperExclPrimitiveTag), label,
       (cond == Equal) ? Below : AboveOrEqual, ShortJump);
}
void MacroAssembler::branchTestPtr(Condition cond, Register lhs, Register rhs,
                                   Label* label) {
  MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
             cond == NotSigned);
  if (lhs == rhs) {
    ma_b(lhs, rhs, label, cond, ShortJump);
  } else {
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();
    and_(scratch, lhs, rhs);
    ma_b(scratch, scratch, label, cond, ShortJump);
  }
}

void MacroAssembler::branchTestPtr(Condition cond, Register lhs, Imm32 rhs,
                                   Label* label) {
  MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
             cond == NotSigned);
  UseScratchRegisterScope temps(this);
  Register scratch = temps.Acquire();
  ma_and(scratch, lhs, rhs);
  ma_b(scratch, scratch, label, cond, ShortJump);
}

void MacroAssembler::branchTestPtr(Condition cond, Register lhs, ImmWord rhs,
                                   Label* label) {
  MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
             cond == NotSigned);
  UseScratchRegisterScope temps(this);
  Register scratch = temps.Acquire();
  ma_and(scratch, lhs, rhs);
  ma_b(scratch, scratch, label, cond, ShortJump);
}

void MacroAssembler::branchTestPtr(Condition cond, const Address& lhs,
                                   Imm32 rhs, Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  loadPtr(lhs, scratch2);
  branchTestPtr(cond, scratch2, rhs, label);
}
void MacroAssembler::branchTestString(Condition cond, Register tag,
                                      Label* label) {
  MOZ_ASSERT(cond == Equal || cond == NotEqual);
  ma_b(tag, ImmTagSignExt(JSVAL_TAG_STRING), label, cond, ShortJump);
}

void MacroAssembler::branchTestString(Condition cond, const ValueOperand& value,
                                      Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  splitSignExtTag(value, scratch2);
  branchTestString(cond, scratch2, label);
}

void MacroAssembler::branchTestString(Condition cond, const Address& address,
                                      Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  Register tag = extractTag(address, scratch2);
  branchTestString(cond, tag, label);
}

void MacroAssembler::branchTestString(Condition cond, const BaseIndex& address,
                                      Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  Register tag = extractTag(address, scratch2);
  branchTestString(cond, tag, label);
}
void MacroAssembler::branchTestStringTruthy(bool b, const ValueOperand& value,
                                            Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  unboxString(value, scratch2);
  load32(Address(scratch2, JSString::offsetOfLength()), scratch2);
  ma_b(scratch2, Imm32(0), label, b ? NotEqual : Equal, ShortJump);
}
void MacroAssembler::branchTestSymbol(Condition cond, Register tag,
                                      Label* label) {
  MOZ_ASSERT(cond == Equal || cond == NotEqual);
  ma_b(tag, ImmTagSignExt(JSVAL_TAG_SYMBOL), label, cond, ShortJump);
}

void MacroAssembler::branchTestSymbol(Condition cond, const ValueOperand& value,
                                      Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  splitSignExtTag(value, scratch2);
  branchTestSymbol(cond, scratch2, label);
}

void MacroAssembler::branchTestSymbol(Condition cond, const BaseIndex& address,
                                      Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  Register tag = extractTag(address, scratch2);
  branchTestSymbol(cond, tag, label);
}

void MacroAssembler::branchTestSymbol(Condition cond, const Address& address,
                                      Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  Register tag = extractTag(address, scratch2);
  branchTestSymbol(cond, tag, label);
}
void MacroAssembler::branchTestUndefined(Condition cond,
                                         const ValueOperand& value,
                                         Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  splitSignExtTag(value, scratch2);
  branchTestUndefined(cond, scratch2, label);
}

void MacroAssembler::branchTestUndefined(Condition cond, const Address& address,
                                         Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  Register tag = extractTag(address, scratch2);
  branchTestUndefined(cond, tag, label);
}

void MacroAssembler::branchTestUndefined(Condition cond,
                                         const BaseIndex& address,
                                         Label* label) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  Register tag = extractTag(address, scratch2);
  branchTestUndefined(cond, tag, label);
}
void MacroAssembler::branchTestUndefined(Condition cond, Register tag,
                                         Label* label) {
  MOZ_ASSERT(cond == Equal || cond == NotEqual);
  ma_b(tag, ImmTagSignExt(JSVAL_TAG_UNDEFINED), label, cond, ShortJump);
}
template <typename T>
void MacroAssembler::branchTestValue(Condition cond, const T& lhs,
                                     const ValueOperand& rhs, Label* label) {
  MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
  branchPtr(cond, lhs, rhs.valueReg(), label);
}
void MacroAssembler::branchToComputedAddress(const BaseIndex& addr) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  loadPtr(addr, scratch2);
  branch(scratch2);
}
void MacroAssembler::branchTruncateDoubleMaybeModUint32(FloatRegister src,
                                                        Register dest,
                                                        Label* fail) {
  // The Zfa fcvtmod.w.d instruction handles exactly this conversion.
  if (HasZfaExtension()) {
    fcvtmod_w_d(dest, src);
    return;
  }

  UseScratchRegisterScope temps(this);

  // Convert scalar to signed 64-bit fixed-point, rounding toward zero.
  // In the case of overflow or NaN, the output is saturated.
  // In the case of -0, the output is zero.
  Trunc_l_d(dest, src);

  // Zero if NaN, so we don't take the failure path below.
  Clear_if_nan_d(dest, src);

  // Unsigned subtraction of INT64_MAX returns 1 resp. 0 for INT64_{MIN,MAX}.
  Register scratch = temps.Acquire();
  ma_li(scratch, Imm64(0x7fff'ffff'ffff'ffff));
  sub(scratch, dest, scratch);

  // Fail if the result is saturated.
  branchPtr(Assembler::BelowOrEqual, scratch, ImmWord(1), fail);

  // Clear upper 32 bits.
  move32SignExtendToPtr(dest, dest);
}

void MacroAssembler::branchTruncateDoubleToInt32(FloatRegister src,
                                                 Register dest, Label* fail) {
  // Convert scalar to signed 64-bit fixed-point, rounding toward zero.
  // In the case of overflow or NaN, the output is saturated.
  // In the case of -0, the output is zero.
  Trunc_l_d(dest, src);

  // Zero if NaN, so we don't take the failure path below.
  Clear_if_nan_d(dest, src);

  // Sign extend lower 32 bits to test if the result isn't an Int32.
  {
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();

    move32SignExtendToPtr(dest, scratch);
    branchPtr(Assembler::NotEqual, dest, scratch, fail);
  }
}
void MacroAssembler::branchTruncateFloat32MaybeModUint32(FloatRegister src,
                                                         Register dest,
                                                         Label* fail) {
  // Infallible operation on riscv64.
  truncateFloat32ModUint32(src, dest);
}

void MacroAssembler::branchTruncateFloat32ToInt32(FloatRegister src,
                                                  Register dest, Label* fail) {
  // Convert scalar to signed 64-bit fixed-point, rounding toward zero.
  // In the case of overflow or NaN, the output is saturated.
  // In the case of -0, the output is zero.
  Trunc_l_s(dest, src);

  // Zero if NaN, so we don't take the failure path below.
  Clear_if_nan_s(dest, src);

  // Sign extend lower 32 bits to test if the result isn't an Int32.
  {
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();

    move32SignExtendToPtr(dest, scratch);
    branchPtr(Assembler::NotEqual, dest, scratch, fail);
  }
}

void MacroAssembler::branchInt64NotInPtrRange(Register64 src, Label* label) {
  // No-op on 64-bit platforms.
}

void MacroAssembler::branchUInt64NotInPtrRange(Register64 src, Label* label) {
  branchTest64(Assembler::Signed, src, src, label);
}

void MacroAssembler::byteSwap16SignExtend(Register src) {
  ByteSwap(src, src, 2);
}

void MacroAssembler::byteSwap16ZeroExtend(Register src) {
  ByteSwap(src, src, 2, /* zeroExtend= */ true);
}

void MacroAssembler::byteSwap32(Register src) { ByteSwap(src, src, 4); }

void MacroAssembler::byteSwap64(Register64 src) {
  ByteSwap(src.reg, src.reg, 8);
}

void MacroAssembler::clampIntToUint8(Register reg) {
  // If reg is < 0, then we want to clamp to 0.
  {
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();

    // reg = -(reg > 0 ? 1 : 0) & reg
    sgtz(scratch, reg);
    neg(scratch, scratch);
    and_(reg, reg, scratch);
  }

  // If reg is >= 255, then we want to clamp to 255.
  Label skip;
  ma_b(reg, Imm32(255), &skip, LessThanOrEqual, ShortJump);
  {
    ma_li(reg, Imm32(255));
  }
  bind(&skip);
}

void MacroAssembler::clz32(Register src, Register dest, bool knownNotZero) {
  Clz32(dest, src);
}
void MacroAssembler::clz64(Register64 src, Register64 dest) {
  Clz64(dest.reg, src.reg);
}

void MacroAssembler::ctz64(Register64 src, Register64 dest) {
  Ctz64(dest.reg, src.reg);
}

void MacroAssembler::cmp16Set(Condition cond, Address lhs, Imm32 rhs,
                              Register dest) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  MOZ_ASSERT(scratch2 != lhs.base);

  switch (cond) {
    case Assembler::Equal:
    case Assembler::NotEqual:
    case Assembler::Above:
    case Assembler::AboveOrEqual:
    case Assembler::Below:
    case Assembler::BelowOrEqual:
      load16ZeroExtend(lhs, scratch2);
      ma_cmp_set(dest, scratch2, Imm32(uint16_t(rhs.value)), cond);
      break;

    case Assembler::GreaterThan:
    case Assembler::GreaterThanOrEqual:
    case Assembler::LessThan:
    case Assembler::LessThanOrEqual:
      load16SignExtend(lhs, scratch2);
      ma_cmp_set(dest, scratch2, Imm32(int16_t(rhs.value)), cond);
      break;

    default:
      MOZ_CRASH("unexpected condition");
  }
}

void MacroAssembler::cmp32Load32(Condition cond, Register lhs,
                                 const Address& rhs, const Address& src,
                                 Register dest) {
  UseScratchRegisterScope temps(this);
  Register scratch = temps.Acquire();
  MOZ_ASSERT(lhs != scratch && dest != scratch);
  load32(rhs, scratch);
  cmp32Load32(cond, lhs, scratch, src, dest);
}

void MacroAssembler::cmp32Load32(Condition cond, Register lhs, Register rhs,
                                 const Address& src, Register dest) {
  Label skip;
  branch32(Assembler::InvertCondition(cond), lhs, rhs, &skip);
  load32(src, dest);
  bind(&skip);
}

void MacroAssembler::cmp32Load32(Condition cond, Register lhs, Imm32 rhs,
                                 const Address& src, Register dest) {
  Label skip;
  branch32(Assembler::InvertCondition(cond), lhs, rhs, &skip);
  load32(src, dest);
  bind(&skip);
}

void MacroAssembler::cmp32LoadPtr(Condition cond, const Address& lhs, Imm32 rhs,
                                  const Address& src, Register dest) {
  Label skip;
  branch32(Assembler::InvertCondition(cond), lhs, rhs, &skip);
  loadPtr(src, dest);
  bind(&skip);
}

void MacroAssembler::cmp32Move32(Condition cond, Register lhs, Imm32 rhs,
                                 Register src, Register dest) {
  ma_cmp_mv(dest, lhs, rhs, src, cond);
}

void MacroAssembler::cmp32Move32(Condition cond, Register lhs, Register rhs,
                                 Register src, Register dest) {
  ma_cmp_mv(dest, lhs, rhs, src, cond);
}

void MacroAssembler::cmp32Move32(Condition cond, Register lhs,
                                 const Address& rhs, Register src,
                                 Register dest) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  MOZ_ASSERT(lhs != scratch2 && src != scratch2 && dest != scratch2);
  load32(rhs, scratch2);
  cmp32Move32(cond, lhs, scratch2, src, dest);
}
void MacroAssembler::cmp32MovePtr(Condition cond, Register lhs, Imm32 rhs,
                                  Register src, Register dest) {
  ma_cmp_mv(dest, lhs, rhs, src, cond);
}
void MacroAssembler::cmp64Set(Condition cond, Register64 lhs, Register64 rhs,
                              Register dest) {
  ma_cmp_set(dest, lhs.reg, rhs.reg, cond);
}
void MacroAssembler::cmp64Set(Condition cond, Register64 lhs, Imm64 rhs,
                              Register dest) {
  ma_cmp_set(dest, lhs.reg, ImmWord(uint64_t(rhs.value)), cond);
}
void MacroAssembler::cmp64Set(Condition cond, Address lhs, Register64 rhs,
                              Register dest) {
  ma_cmp_set(dest, lhs, rhs.reg, cond);
}
void MacroAssembler::cmp64Set(Condition cond, Address lhs, Imm64 rhs,
                              Register dest) {
  ma_cmp_set(dest, lhs, ImmWord(uint64_t(rhs.value)), cond);
}
void MacroAssembler::cmp8Set(Condition cond, Address lhs, Imm32 rhs,
                             Register dest) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  MOZ_ASSERT(scratch2 != lhs.base);

  switch (cond) {
    case Assembler::Equal:
    case Assembler::NotEqual:
    case Assembler::Above:
    case Assembler::AboveOrEqual:
    case Assembler::Below:
    case Assembler::BelowOrEqual:
      load8ZeroExtend(lhs, scratch2);
      ma_cmp_set(dest, scratch2, Imm32(uint8_t(rhs.value)), cond);
      break;

    case Assembler::GreaterThan:
    case Assembler::GreaterThanOrEqual:
    case Assembler::LessThan:
    case Assembler::LessThanOrEqual:
      load8SignExtend(lhs, scratch2);
      ma_cmp_set(dest, scratch2, Imm32(int8_t(rhs.value)), cond);
      break;

    default:
      MOZ_CRASH("unexpected condition");
  }
}
void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs, Register rhs,
                                   Register src, Register dest) {
  ma_cmp_mv(dest, lhs, rhs, src, cond);
}

void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs, Imm32 rhs,
                                   Register src, Register dest) {
  ma_cmp_mv(dest, lhs, rhs, src, cond);
}

void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs,
                                   const Address& rhs, Register src,
                                   Register dest) {
  MOZ_CRASH("NYI");
}

void MacroAssembler::ctz32(Register src, Register dest, bool knownNotZero) {
  Ctz32(dest, src);
}

void MacroAssembler::decBranchPtr(Condition cond, Register lhs, Imm32 rhs,
                                  Label* label) {
  subPtr(rhs, lhs);
  branchPtr(cond, lhs, Imm32(0), label);
}
void MacroAssembler::divFloat32(FloatRegister src, FloatRegister dest) {
  fdiv_s(dest, dest, src);
}

void MacroAssembler::divDouble(FloatRegister src, FloatRegister dest) {
  fdiv_d(dest, dest, src);
}
void MacroAssembler::fallibleUnboxPtr(const ValueOperand& src, Register dest,
                                      JSValueType type, Label* fail) {
  MOZ_ASSERT(type == JSVAL_TYPE_OBJECT || type == JSVAL_TYPE_STRING ||
             type == JSVAL_TYPE_SYMBOL || type == JSVAL_TYPE_BIGINT);
  // dest := src XOR mask
  // scratch := dest >> JSVAL_TAG_SHIFT
  // fail if scratch != 0
  //
  // Note: src and dest can be the same register
  UseScratchRegisterScope temps(this);
  Register scratch = temps.Acquire();
  MOZ_ASSERT(src.valueReg() != scratch);
  mov(ImmShiftedTag(type), scratch);
  xor_(dest, src.valueReg(), scratch);
  srli(scratch, dest, JSVAL_TAG_SHIFT);
  ma_b(scratch, Imm32(0), fail, Assembler::NotEqual, ShortJump);
}

void MacroAssembler::fallibleUnboxPtr(const Address& src, Register dest,
                                      JSValueType type, Label* fail) {
  loadValue(src, ValueOperand(dest));
  fallibleUnboxPtr(ValueOperand(dest), dest, type, fail);
}

void MacroAssembler::fallibleUnboxPtr(const BaseIndex& src, Register dest,
                                      JSValueType type, Label* fail) {
  loadValue(src, ValueOperand(dest));
  fallibleUnboxPtr(ValueOperand(dest), dest, type, fail);
}
void MacroAssembler::flexibleLshift32(Register src, Register dest) {
  lshift32(src, dest);
}
void MacroAssembler::flexibleRshift32Arithmetic(Register src, Register dest) {
  rshift32Arithmetic(src, dest);
}
void MacroAssembler::flexibleRshift32(Register src, Register dest) {
  rshift32(src, dest);
}
void MacroAssembler::inc64(AbsoluteAddress dest) {
  UseScratchRegisterScope temps(this);
  Register scratch = temps.Acquire();
  Register scratch2 = temps.Acquire();
  ma_li(scratch, ImmWord(uintptr_t(dest.addr)));
  ld(scratch2, scratch, 0);
  addi(scratch2, scratch2, 1);
  sd(scratch2, scratch, 0);
}

void MacroAssembler::load32SignExtendToPtr(const Address& src, Register dest) {
  load32(src, dest);
}
void MacroAssembler::loadAbiReturnAddress(Register dest) { movePtr(ra, dest); }

void MacroAssembler::lshift32(Register src, Register dest) {
  sllw(dest, dest, src);
}

void MacroAssembler::lshift32(Imm32 imm, Register dest) {
  lshift32(imm, dest, dest);
}

void MacroAssembler::lshift32(Imm32 imm, Register src, Register dest) {
  slliw(dest, src, imm.value & 0x1f);
}

void MacroAssembler::lshift64(Register shift, Register64 dest) {
  sll(dest.reg, dest.reg, shift);
}

void MacroAssembler::lshift64(Imm32 imm, Register64 dest) {
  MOZ_ASSERT(0 <= imm.value && imm.value < 64);
  slli(dest.reg, dest.reg, imm.value);
}
void MacroAssembler::lshiftPtr(Register shift, Register dest) {
  sll(dest, dest, shift);
}

void MacroAssembler::lshiftPtr(Imm32 imm, Register dest) {
  lshiftPtr(imm, dest, dest);
}

void MacroAssembler::lshiftPtr(Imm32 imm, Register src, Register dest) {
  MOZ_ASSERT(0 <= imm.value && imm.value < 64);
  slli(dest, src, imm.value);
}

void MacroAssembler::flexibleLshiftPtr(Register shift, Register srcDest) {
  lshiftPtr(shift, srcDest);
}

void MacroAssembler::min32(Register lhs, Register rhs, Register dest) {
  minMax32(lhs, rhs, dest, /* isMax = */ false);
}

void MacroAssembler::min32(Register lhs, Imm32 rhs, Register dest) {
  minMax32(lhs, rhs, dest, /* isMax = */ false);
}

void MacroAssembler::max32(Register lhs, Register rhs, Register dest) {
  minMax32(lhs, rhs, dest, /* isMax = */ true);
}

void MacroAssembler::max32(Register lhs, Imm32 rhs, Register dest) {
  minMax32(lhs, rhs, dest, /* isMax = */ true);
}

void MacroAssembler::minPtr(Register lhs, Register rhs, Register dest) {
  minMaxPtr(lhs, rhs, dest, /* isMax = */ false);
}

void MacroAssembler::minPtr(Register lhs, ImmWord rhs, Register dest) {
  minMaxPtr(lhs, rhs, dest, /* isMax = */ false);
}

void MacroAssembler::maxPtr(Register lhs, Register rhs, Register dest) {
  minMaxPtr(lhs, rhs, dest, /* isMax = */ true);
}

void MacroAssembler::maxPtr(Register lhs, ImmWord rhs, Register dest) {
  minMaxPtr(lhs, rhs, dest, /* isMax = */ true);
}

void MacroAssembler::maxDouble(FloatRegister other, FloatRegister srcDest,
                               bool handleNaN) {
  Float64Max(srcDest, srcDest, other);
}
void MacroAssembler::maxFloat32(FloatRegister other, FloatRegister srcDest,
                                bool handleNaN) {
  Float32Max(srcDest, srcDest, other);
}
void MacroAssembler::memoryBarrier(MemoryBarrier barrier) {
  if (!barrier.isNone()) {
    sync();
  }
}
void MacroAssembler::minDouble(FloatRegister other, FloatRegister srcDest,
                               bool handleNaN) {
  Float64Min(srcDest, srcDest, other);
}
void MacroAssembler::minFloat32(FloatRegister other, FloatRegister srcDest,
                                bool handleNaN) {
  Float32Min(srcDest, srcDest, other);
}
void MacroAssembler::move16SignExtend(Register src, Register dest) {
  SignExtendShort(dest, src);
}
void MacroAssembler::move16To64SignExtend(Register src, Register64 dest) {
  SignExtendShort(dest.reg, src);
}
void MacroAssembler::move8SignExtendToPtr(Register src, Register dest) {
  move8To64SignExtend(src, Register64(dest));
}
void MacroAssembler::move16SignExtendToPtr(Register src, Register dest) {
  move16To64SignExtend(src, Register64(dest));
}
void MacroAssembler::move32SignExtendToPtr(Register src, Register dest) {
  SignExtendWord(dest, src);
}
void MacroAssembler::move32To64SignExtend(Register src, Register64 dest) {
  SignExtendWord(dest.reg, src);
}
void MacroAssembler::move32To64ZeroExtend(Register src, Register64 dest) {
  ZeroExtendWord(dest.reg, src);
}
void MacroAssembler::move32ZeroExtendToPtr(Register src, Register dest) {
  ZeroExtendWord(dest, src);
}
void MacroAssembler::move64(Register64 src, Register64 dest) {
  movePtr(src.reg, dest.reg);
}

void MacroAssembler::move64(Imm64 imm, Register64 dest) {
  movePtr(ImmWord(imm.value), dest.reg);
}

void MacroAssembler::move64To32(Register64 src, Register dest) {
  SignExtendWord(dest, src.reg);
}

void MacroAssembler::move8ZeroExtend(Register src, Register dest) {
  andi(dest, src, 0xFF);
}

void MacroAssembler::move8SignExtend(Register src, Register dest) {
  SignExtendByte(dest, src);
}
void MacroAssembler::move8To64SignExtend(Register src, Register64 dest) {
  SignExtendByte(dest.reg, src);
}
void MacroAssembler::moveDoubleToGPR64(FloatRegister src, Register64 dest) {
  fmv_x_d(dest.reg, src);
}

void MacroAssembler::moveGPR64ToDouble(Register64 src, FloatRegister dest) {
  fmv_d_x(dest, src.reg);
}
void MacroAssembler::moveLowDoubleToGPR(FloatRegister src, Register dest) {
  MOZ_CRASH("Not supported for this target");
}
void MacroAssembler::moveFloat32ToGPR(FloatRegister src, Register dest) {
  fmv_x_w(dest, src);
}
void MacroAssembler::moveGPRToFloat32(Register src, FloatRegister dest) {
  fmv_w_x(dest, src);
}
void MacroAssembler::moveFloat16ToGPR(FloatRegister src, Register dest) {
  MOZ_ASSERT(HasZfhminExtension());
  fmv_x_h(dest, src);
}
void MacroAssembler::moveGPRToFloat16(Register src, FloatRegister dest) {
  MOZ_ASSERT(HasZfhminExtension());
  fmv_h_x(dest, src);
}
void MacroAssembler::mul32(Register rhs, Register srcDest) {
  mulw(srcDest, srcDest, rhs);
}

void MacroAssembler::mul32(Imm32 imm, Register srcDest) {
  ma_mul32(srcDest, srcDest, imm);
}

void MacroAssembler::mulHighUnsigned32(Imm32 imm, Register src, Register dest) {
  ma_mulhu32(dest, src, imm);
}

void MacroAssembler::mul64(Imm64 imm, const Register64& dest) {
  ma_mul64(dest.reg, dest.reg, imm);
}

void MacroAssembler::mul64(Imm64 imm, const Register64& dest,
                           const Register temp) {
  MOZ_ASSERT(temp == Register::Invalid());
  mul64(imm, dest);
}

void MacroAssembler::mul64(const Register64& src, const Register64& dest,
                           const Register temp) {
  MOZ_ASSERT(temp == Register::Invalid());
  mul(dest.reg, dest.reg, src.reg);
}

void MacroAssembler::mulBy3(Register src, Register dest) {
  UseScratchRegisterScope temps(this);
  Register scratch = temps.Acquire();
  MOZ_ASSERT(src != scratch);
  add(scratch, src, src);
  add(dest, scratch, src);
}
void MacroAssembler::mulDouble(FloatRegister src, FloatRegister dest) {
  fmul_d(dest, dest, src);
}
void MacroAssembler::mulDoublePtr(ImmPtr imm, Register temp,
                                  FloatRegister dest) {
  UseScratchRegisterScope temps(this);
  Register scratch = temps.Acquire();
  ScratchDoubleScope fpscratch(asMasm());
  movePtr(imm, scratch);
  loadDouble(Address(scratch, 0), fpscratch);
  mulDouble(fpscratch, dest);
}
void MacroAssembler::mulFloat32(FloatRegister src, FloatRegister dest) {
  fmul_s(dest, dest, src);
}
void MacroAssembler::mul64(const Register64& rhs, const Register64& srcDest) {
  mul(srcDest.reg, srcDest.reg, rhs.reg);
}
void MacroAssembler::mulPtr(Register rhs, Register srcDest) {
  mul(srcDest, srcDest, rhs);
}
void MacroAssembler::mulPtr(ImmWord rhs, Register srcDest) {
  ma_mul64(srcDest, srcDest, rhs);
}

void MacroAssembler::negateDouble(FloatRegister reg) { fneg_d(reg, reg); }

void MacroAssembler::negateFloat(FloatRegister reg) { fneg_s(reg, reg); }

void MacroAssembler::neg64(Register64 reg) { neg(reg.reg, reg.reg); }

void MacroAssembler::negPtr(Register reg) { neg(reg, reg); }

void MacroAssembler::neg32(Register reg) { negw(reg, reg); }
void MacroAssembler::not32(Register reg) { not_(reg, reg); }

void MacroAssembler::notPtr(Register reg) { not_(reg, reg); }

void MacroAssembler::or32(Register src, Register dest) { or_(dest, dest, src); }

void MacroAssembler::or32(Imm32 imm, Register dest) { ma_or(dest, dest, imm); }

void MacroAssembler::or32(Imm32 imm, Register src, Register dest) {
  ma_or(dest, src, imm);
}

void MacroAssembler::or32(Imm32 imm, const Address& dest) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  load32(dest, scratch2);
  ma_or(scratch2, scratch2, imm);
  store32(scratch2, dest);
}

void MacroAssembler::or64(Register64 src, Register64 dest) {
  or_(dest.reg, dest.reg, src.reg);
}

void MacroAssembler::or64(Imm64 imm, Register64 dest) {
  ma_or(dest.reg, dest.reg, imm);
}

void MacroAssembler::orPtr(Register src, Register dest) {
  or_(dest, dest, src);
}

void MacroAssembler::orPtr(Imm32 imm, Register dest) { ma_or(dest, dest, imm); }

void MacroAssembler::orPtr(Imm32 imm, Register src, Register dest) {
  ma_or(dest, src, imm);
}

void MacroAssembler::popcnt32(Register input, Register output, Register tmp) {
  Popcnt32(output, input, tmp);
}
void MacroAssembler::popcnt64(Register64 input, Register64 output,
                              Register tmp) {
  Popcnt64(output.reg, input.reg, tmp);
}
void MacroAssembler::quotient32(Register lhs, Register rhs, Register dest,
                                bool isUnsigned) {
  if (isUnsigned) {
    divuw(dest, lhs, rhs);
  } else {
    divw(dest, lhs, rhs);
  }
}

void MacroAssembler::quotient64(Register lhs, Register rhs, Register dest,
                                bool isUnsigned) {
  if (isUnsigned) {
    divu(dest, lhs, rhs);
  } else {
    div(dest, lhs, rhs);
  }
}

void MacroAssembler::remainder32(Register lhs, Register rhs, Register dest,
                                 bool isUnsigned) {
  if (isUnsigned) {
    remuw(dest, lhs, rhs);
  } else {
    remw(dest, lhs, rhs);
  }
}

void MacroAssembler::remainder64(Register lhs, Register rhs, Register dest,
                                 bool isUnsigned) {
  if (isUnsigned) {
    remu(dest, lhs, rhs);
  } else {
    rem(dest, lhs, rhs);
  }
}

void MacroAssembler::rotateLeft64(Imm32 count, Register64 src, Register64 dest,
                                  Register temp) {
  Drol(dest.reg, src.reg, count);
}
void MacroAssembler::rotateLeft64(Register count, Register64 src,
                                  Register64 dest, Register temp) {
  Drol(dest.reg, src.reg, count);
}

void MacroAssembler::rotateLeft(Imm32 count, Register input, Register dest) {
  Rol(dest, input, count);
}
void MacroAssembler::rotateLeft(Register count, Register input, Register dest) {
  Rol(dest, input, count);
}

void MacroAssembler::rotateRight64(Imm32 count, Register64 src, Register64 dest,
                                   Register temp) {
  Dror(dest.reg, src.reg, count);
}
void MacroAssembler::rotateRight64(Register count, Register64 src,
                                   Register64 dest, Register temp) {
  Dror(dest.reg, src.reg, count);
}

void MacroAssembler::rotateRight(Imm32 count, Register input, Register dest) {
  Ror(dest, input, count);
}
void MacroAssembler::rotateRight(Register count, Register input,
                                 Register dest) {
  Ror(dest, input, count);
}

void MacroAssembler::rshift32Arithmetic(Register src, Register dest) {
  sraw(dest, dest, src);
}

void MacroAssembler::rshift32Arithmetic(Imm32 imm, Register dest) {
  rshift32Arithmetic(imm, dest, dest);
}

void MacroAssembler::rshift32Arithmetic(Imm32 imm, Register src,
                                        Register dest) {
  sraiw(dest, src, imm.value & 0x1f);
}

void MacroAssembler::rshift32(Register src, Register dest) {
  srlw(dest, dest, src);
}

void MacroAssembler::rshift32(Imm32 imm, Register dest) {
  rshift32(imm, dest, dest);
}

void MacroAssembler::rshift32(Imm32 imm, Register src, Register dest) {
  srliw(dest, src, imm.value & 0x1f);
}

void MacroAssembler::rshift64Arithmetic(Imm32 imm, Register64 dest) {
  MOZ_ASSERT(0 <= imm.value && imm.value < 64);
  srai(dest.reg, dest.reg, imm.value);
}

void MacroAssembler::rshift64Arithmetic(Register shift, Register64 dest) {
  sra(dest.reg, dest.reg, shift);
}

void MacroAssembler::rshift64(Register shift, Register64 dest) {
  srl(dest.reg, dest.reg, shift);
}

void MacroAssembler::rshift64(Imm32 imm, Register64 dest) {
  MOZ_ASSERT(0 <= imm.value && imm.value < 64);
  srli(dest.reg, dest.reg, imm.value);
}

void MacroAssembler::rshiftPtrArithmetic(Imm32 imm, Register dest) {
  rshiftPtrArithmetic(imm, dest, dest);
}

void MacroAssembler::rshiftPtrArithmetic(Imm32 imm, Register src,
                                         Register dest) {
  MOZ_ASSERT(0 <= imm.value && imm.value < 64);
  srai(dest, src, imm.value);
}

void MacroAssembler::rshiftPtrArithmetic(Register shift, Register dest) {
  sra(dest, dest, shift);
}

void MacroAssembler::flexibleRshiftPtrArithmetic(Register shift,
                                                 Register srcDest) {
  rshiftPtrArithmetic(shift, srcDest);
}

void MacroAssembler::rshiftPtr(Register shift, Register dest) {
  srl(dest, dest, shift);
}

void MacroAssembler::rshiftPtr(Imm32 imm, Register dest) {
  rshiftPtr(imm, dest, dest);
}

void MacroAssembler::rshiftPtr(Imm32 imm, Register src, Register dest) {
  MOZ_ASSERT(0 <= imm.value && imm.value < 64);
  srli(dest, src, imm.value);
}

void MacroAssembler::flexibleRshiftPtr(Register shift, Register srcDest) {
  rshiftPtr(shift, srcDest);
}

void MacroAssembler::spectreBoundsCheck32(Register index, Register length,
                                          Register maybeScratch,
                                          Label* failure) {
  MOZ_RELEASE_ASSERT(!JitOptions.spectreIndexMasking);
  branch32(Assembler::BelowOrEqual, length, index, failure);
}

void MacroAssembler::spectreBoundsCheck32(Register index, const Address& length,
                                          Register maybeScratch,
                                          Label* failure) {
  MOZ_RELEASE_ASSERT(!JitOptions.spectreIndexMasking);
  branch32(Assembler::BelowOrEqual, length, index, failure);
}
void MacroAssembler::spectreBoundsCheckPtr(Register index, Register length,
                                           Register maybeScratch,
                                           Label* failure) {
  MOZ_RELEASE_ASSERT(!JitOptions.spectreIndexMasking);
  branchPtr(Assembler::BelowOrEqual, length, index, failure);
}

void MacroAssembler::spectreBoundsCheckPtr(Register index,
                                           const Address& length,
                                           Register maybeScratch,
                                           Label* failure) {
  MOZ_RELEASE_ASSERT(!JitOptions.spectreIndexMasking);
  branchPtr(Assembler::BelowOrEqual, length, index, failure);
}
void MacroAssembler::spectreMovePtr(Condition, Register, Register) {
  MOZ_CRASH("spectreMovePtr");
}
void MacroAssembler::spectreZeroRegister(Condition cond, Register scratch,
                                         Register dest) {
  MOZ_CRASH("spectreZeroRegister");
}
void MacroAssembler::sqrtDouble(FloatRegister src, FloatRegister dest) {
  fsqrt_d(dest, src);
}
void MacroAssembler::sqrtFloat32(FloatRegister src, FloatRegister dest) {
  fsqrt_s(dest, src);
}

FaultingCodeOffset MacroAssembler::storeFloat16(FloatRegister src,
                                                const Address& dest, Register) {
  return ma_storeFloat16(src, dest);
}
FaultingCodeOffset MacroAssembler::storeFloat16(FloatRegister src,
                                                const BaseIndex& dest,
                                                Register) {
  return ma_storeFloat16(src, dest);
}

FaultingCodeOffset MacroAssembler::storeFloat32(FloatRegister src,
                                                const Address& addr) {
  return ma_storeFloat(src, addr);
}
FaultingCodeOffset MacroAssembler::storeFloat32(FloatRegister src,
                                                const BaseIndex& addr) {
  return ma_storeFloat(src, addr);
}

FaultingCodeOffset MacroAssembler::storeDouble(FloatRegister src,
                                               const Address& addr) {
  return ma_storeDouble(src, addr);
}
FaultingCodeOffset MacroAssembler::storeDouble(FloatRegister src,
                                               const BaseIndex& addr) {
  return ma_storeDouble(src, addr);
}

void MacroAssembler::sub32(Register src, Register dest) {
  subw(dest, dest, src);
}

void MacroAssembler::sub32(Imm32 imm, Register dest) {
  ma_sub32(dest, dest, imm);
}

void MacroAssembler::sub32(const Address& src, Register dest) {
  UseScratchRegisterScope temps(this);
  Register scratch = temps.Acquire();
  load32(src, scratch);
  subw(dest, dest, scratch);
}

void MacroAssembler::sub64(Register64 src, Register64 dest) {
  sub(dest.reg, dest.reg, src.reg);
}

void MacroAssembler::sub64(Imm64 imm, Register64 dest) {
  ma_sub64(dest.reg, dest.reg, imm);
}

void MacroAssembler::subDouble(FloatRegister src, FloatRegister dest) {
  fsub_d(dest, dest, src);
}

void MacroAssembler::subFloat32(FloatRegister src, FloatRegister dest) {
  fsub_s(dest, dest, src);
}

void MacroAssembler::subPtr(Register src, const Address& dest) {
  UseScratchRegisterScope temps(this);
  Register scratch = temps.Acquire();
  loadPtr(dest, scratch);
  subPtr(src, scratch);
  storePtr(scratch, dest);
}

void MacroAssembler::subPtr(const Address& addr, Register dest) {
  UseScratchRegisterScope temps(this);
  Register scratch = temps.Acquire();
  loadPtr(addr, scratch);
  subPtr(scratch, dest);
}
void MacroAssembler::subPtr(Imm32 imm, Register dest) {
  ma_sub64(dest, dest, imm);
}
void MacroAssembler::subPtr(Register src, Register dest) {
  sub(dest, dest, src);
}
void MacroAssembler::test32LoadPtr(Condition cond, const Address& addr,
                                   Imm32 mask, const Address& src,
                                   Register dest) {
  MOZ_RELEASE_ASSERT(!JitOptions.spectreStringMitigations);
  Label skip;
  branchTest32(Assembler::InvertCondition(cond), addr, mask, &skip);
  loadPtr(src, dest);
  bind(&skip);
}
void MacroAssembler::test32MovePtr(Condition cond, const Address& addr,
                                   Imm32 mask, Register src, Register dest) {
  MOZ_ASSERT(cond == Assembler::Zero || cond == Assembler::NonZero);
  Label skip;
  branchTest32(Assembler::InvertCondition(cond), addr, mask, &skip);
  movePtr(src, dest);
  bind(&skip);
}
void MacroAssembler::test32MovePtr(Condition cond, Register operand, Imm32 mask,
                                   Register src, Register dest) {
  MOZ_ASSERT(cond == Assembler::Zero || cond == Assembler::NonZero);
  Label skip;
  branchTest32(Assembler::InvertCondition(cond), operand, mask, &skip);
  movePtr(src, dest);
  bind(&skip);
}
void MacroAssembler::xor32(Register src, Register dest) {
  xor_(dest, dest, src);
}

void MacroAssembler::xor32(Imm32 imm, Register dest) {
  ma_xor(dest, dest, imm);
}

void MacroAssembler::xor32(Imm32 imm, Register src, Register dest) {
  ma_xor(dest, src, imm);
}

void MacroAssembler::xor32(Imm32 imm, const Address& dest) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  load32(dest, scratch2);
  xor32(imm, scratch2);
  store32(scratch2, dest);
}

void MacroAssembler::xor32(const Address& src, Register dest) {
  UseScratchRegisterScope temps(this);
  Register scratch2 = temps.Acquire();
  load32(src, scratch2);
  xor32(scratch2, dest);
}
void MacroAssembler::xor64(Register64 src, Register64 dest) {
  xor_(dest.reg, dest.reg, src.reg);
}

void MacroAssembler::xor64(Imm64 imm, Register64 dest) {
  ma_xor(dest.reg, dest.reg, imm);
}
void MacroAssembler::xorPtr(Register src, Register dest) {
  xor_(dest, dest, src);
}

void MacroAssembler::xorPtr(Imm32 imm, Register dest) {
  ma_xor(dest, dest, imm);
}

void MacroAssembler::xorPtr(Imm32 imm, Register src, Register dest) {
  ma_xor(dest, src, imm);
}

// ===============================================================
// 128-bit arithmetic

void MacroAssembler::wasmAddSubI128HI64(Register lhsLo, Register lhsHi,
                                        Register rhsLo, Register rhsHi,
                                        Register output, bool isAdd) {
  // Require: the output is not the same as any of the inputs.
  MOZ_RELEASE_ASSERT(output != lhsLo && output != lhsHi && output != rhsLo &&
                     output != rhsHi);
  // We use `output` as a temp to hold the carry or borrow.
  if (isAdd) {
    add(output, lhsLo, rhsLo);    // output = lhsLo + rhsLo
    sltu(output, output, lhsLo);  // output = carry from `lhsLo + rhsLo`
    add(output, output, lhsHi);   // output = carry + lhsHi
    add(output, output, rhsHi);   // output = carry + lhsHi + rhsHi
  } else {
    sltu(output, lhsLo, rhsLo);  // output = borrow from `lhsLo - rhsLo`
    sub(output, lhsHi, output);  // output = lhsHi - borrow
    sub(output, output, rhsHi);  // output = lhsHi - borrow - rhsHi
  }
}

void MacroAssembler::wasmMulI64WideHI64(Register lhs, Register rhs,
                                        Register output, bool isSigned) {
  if (isSigned) {
    mulh(output, lhs, rhs);
  } else {
    mulhu(output, lhs, rhs);
  }
}

//}}} check_macroassembler_style

void MacroAssemblerRiscv64Compat::incrementInt32Value(const Address& addr) {
  asMasm().add32(Imm32(1), addr);
}

void MacroAssemblerRiscv64Compat::retn(Imm32 n) {
  // pc <- [sp]; sp += n
  loadPtr(Address(StackPointer, 0), ra);
  asMasm().addPtr(n, StackPointer);
  jr(ra);
}
}  // namespace jit
}  // namespace js

#endif /* jit_riscv64_MacroAssembler_riscv64_inl_h */