Source/JavaScriptCore/jit/AssemblyHelpers.cpp

/*
 * Copyright (C) 2011-2022 Apple Inc. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
 */

#include "config.h"
#include "AssemblyHelpers.h"

#if ENABLE(JIT)

#include "AccessCase.h"
#include "AssemblyHelpersSpoolers.h"
#include "JITOperations.h"
#include "JSArrayBufferView.h"
#include "JSCJSValueInlines.h"
#include "LinkBuffer.h"
#include "MaxFrameExtentForSlowPathCall.h"
#include "SuperSampler.h"
#include "ThunkGenerators.h"

#if ENABLE(WEBASSEMBLY)
#include "WasmMemoryInformation.h"
#include "WasmContextInlines.h"
#endif

namespace JSC {

AssemblyHelpers::Jump AssemblyHelpers::branchIfFastTypedArray(GPRReg baseGPR)
{
    return branch32(
        Equal,
        Address(baseGPR, JSArrayBufferView::offsetOfMode()),
        TrustedImm32(FastTypedArray));
}

AssemblyHelpers::Jump AssemblyHelpers::branchIfNotFastTypedArray(GPRReg baseGPR)
{
    return branch32(
        NotEqual,
        Address(baseGPR, JSArrayBufferView::offsetOfMode()),
        TrustedImm32(FastTypedArray));
}

void AssemblyHelpers::incrementSuperSamplerCount()
{
    add32(TrustedImm32(1), AbsoluteAddress(bitwise_cast<const void*>(&g_superSamplerCount)));
}

void AssemblyHelpers::decrementSuperSamplerCount()
{
    sub32(TrustedImm32(1), AbsoluteAddress(bitwise_cast<const void*>(&g_superSamplerCount)));
}
    
void AssemblyHelpers::purifyNaN(FPRReg fpr)
{
    MacroAssembler::Jump notNaN = branchIfNotNaN(fpr);
    static const double NaN = PNaN;
    loadDouble(TrustedImmPtr(&NaN), fpr);
    notNaN.link(this);
}

#if ENABLE(SAMPLING_FLAGS)
void AssemblyHelpers::setSamplingFlag(int32_t flag)
{
    ASSERT(flag >= 1);
    ASSERT(flag <= 32);
    or32(TrustedImm32(1u << (flag - 1)), AbsoluteAddress(SamplingFlags::addressOfFlags()));
}

void AssemblyHelpers::clearSamplingFlag(int32_t flag)
{
    ASSERT(flag >= 1);
    ASSERT(flag <= 32);
    and32(TrustedImm32(~(1u << (flag - 1))), AbsoluteAddress(SamplingFlags::addressOfFlags()));
}
#endif

#if ASSERT_ENABLED
#if USE(JSVALUE64)
void AssemblyHelpers::jitAssertIsInt32(GPRReg gpr)
{
#if CPU(X86_64) || CPU(ARM64)
    Jump checkInt32 = branch64(BelowOrEqual, gpr, TrustedImm64(static_cast<uintptr_t>(0xFFFFFFFFu)));
    abortWithReason(AHIsNotInt32);
    checkInt32.link(this);
#else
    UNUSED_PARAM(gpr);
#endif
}

void AssemblyHelpers::jitAssertIsJSInt32(GPRReg gpr)
{
    Jump checkJSInt32 = branch64(AboveOrEqual, gpr, GPRInfo::numberTagRegister);
    abortWithReason(AHIsNotJSInt32);
    checkJSInt32.link(this);
}

void AssemblyHelpers::jitAssertIsJSNumber(GPRReg gpr)
{
    Jump checkJSNumber = branchTest64(MacroAssembler::NonZero, gpr, GPRInfo::numberTagRegister);
    abortWithReason(AHIsNotJSNumber);
    checkJSNumber.link(this);
}

void AssemblyHelpers::jitAssertIsJSDouble(GPRReg gpr)
{
    Jump checkJSInt32 = branch64(AboveOrEqual, gpr, GPRInfo::numberTagRegister);
    Jump checkJSNumber = branchTest64(MacroAssembler::NonZero, gpr, GPRInfo::numberTagRegister);
    checkJSInt32.link(this);
    abortWithReason(AHIsNotJSDouble);
    checkJSNumber.link(this);
}

void AssemblyHelpers::jitAssertIsCell(GPRReg gpr)
{
    Jump checkCell = branchTest64(MacroAssembler::Zero, gpr, GPRInfo::notCellMaskRegister);
    abortWithReason(AHIsNotCell);
    checkCell.link(this);
}

void AssemblyHelpers::jitAssertTagsInPlace()
{
    Jump ok = branch64(Equal, GPRInfo::numberTagRegister, TrustedImm64(JSValue::NumberTag));
    abortWithReason(AHNumberTagNotInPlace);
    breakpoint();
    ok.link(this);
    
    ok = branch64(Equal, GPRInfo::notCellMaskRegister, TrustedImm64(JSValue::NotCellMask));
    abortWithReason(AHNotCellMaskNotInPlace);
    ok.link(this);
}
#elif USE(JSVALUE32_64)
void AssemblyHelpers::jitAssertIsInt32(GPRReg gpr)
{
    UNUSED_PARAM(gpr);
}

void AssemblyHelpers::jitAssertIsJSInt32(GPRReg gpr)
{
    Jump checkJSInt32 = branch32(Equal, gpr, TrustedImm32(JSValue::Int32Tag));
    abortWithReason(AHIsNotJSInt32);
    checkJSInt32.link(this);
}

void AssemblyHelpers::jitAssertIsJSNumber(GPRReg gpr)
{
    Jump checkJSInt32 = branch32(Equal, gpr, TrustedImm32(JSValue::Int32Tag));
    Jump checkJSDouble = branch32(Below, gpr, TrustedImm32(JSValue::LowestTag));
    abortWithReason(AHIsNotJSNumber);
    checkJSInt32.link(this);
    checkJSDouble.link(this);
}

void AssemblyHelpers::jitAssertIsJSDouble(GPRReg gpr)
{
    Jump checkJSDouble = branch32(Below, gpr, TrustedImm32(JSValue::LowestTag));
    abortWithReason(AHIsNotJSDouble);
    checkJSDouble.link(this);
}

void AssemblyHelpers::jitAssertIsCell(GPRReg gpr)
{
    Jump checkCell = branchIfCell(gpr);
    abortWithReason(AHIsNotCell);
    checkCell.link(this);
}

void AssemblyHelpers::jitAssertTagsInPlace()
{
}
#endif // USE(JSVALUE32_64)

void AssemblyHelpers::jitAssertHasValidCallFrame()
{
    Jump checkCFR = branchTestPtr(Zero, GPRInfo::callFrameRegister, TrustedImm32(7));
    abortWithReason(AHCallFrameMisaligned);
    checkCFR.link(this);
}

void AssemblyHelpers::jitAssertIsNull(GPRReg gpr)
{
    Jump checkNull = branchTestPtr(Zero, gpr);
    abortWithReason(AHIsNotNull);
    checkNull.link(this);
}

void AssemblyHelpers::jitAssertArgumentCountSane()
{
    Jump ok = branch32(Below, payloadFor(CallFrameSlot::argumentCountIncludingThis), TrustedImm32(10000000));
    abortWithReason(AHInsaneArgumentCount);
    ok.link(this);
}

void AssemblyHelpers::jitAssertCodeBlockOnCallFrameWithType(GPRReg scratchGPR, JITType type)
{
    emitGetFromCallFrameHeaderPtr(CallFrameSlot::codeBlock, scratchGPR);
    loadPtr(Address(scratchGPR, CodeBlock::jitCodeOffset()), scratchGPR);
    load8(Address(scratchGPR, JITCode::offsetOfJITType()), scratchGPR);
    Jump ok = branch32(Equal, scratchGPR, TrustedImm32(static_cast<unsigned>(type)));
    abortWithReason(AHInvalidCodeBlock);
    ok.link(this);
}

void AssemblyHelpers::jitAssertCodeBlockOnCallFrameIsOptimizingJIT(GPRReg scratchGPR)
{
    emitGetFromCallFrameHeaderPtr(CallFrameSlot::codeBlock, scratchGPR);
    loadPtr(Address(scratchGPR, CodeBlock::jitCodeOffset()), scratchGPR);
    load8(Address(scratchGPR, JITCode::offsetOfJITType()), scratchGPR);
    JumpList ok;
    ok.append(branch32(Equal, scratchGPR, TrustedImm32(static_cast<unsigned>(JITType::DFGJIT))));
    ok.append(branch32(Equal, scratchGPR, TrustedImm32(static_cast<unsigned>(JITType::FTLJIT))));
    abortWithReason(AHInvalidCodeBlock);
    ok.link(this);
}

#endif // ASSERT_ENABLED

void AssemblyHelpers::jitReleaseAssertNoException(VM& vm)
{
    Jump noException;
#if USE(JSVALUE64)
    noException = branchTest64(Zero, AbsoluteAddress(vm.addressOfException()));
#elif USE(JSVALUE32_64)
    noException = branch32(Equal, AbsoluteAddress(vm.addressOfException()), TrustedImm32(0));
#endif
    abortWithReason(JITUncaughtExceptionAfterCall);
    noException.link(this);
}

void AssemblyHelpers::callExceptionFuzz(VM& vm)
{
    RELEASE_ASSERT(Options::useExceptionFuzz());

    EncodedJSValue* buffer = vm.exceptionFuzzingBuffer(sizeof(EncodedJSValue) * (GPRInfo::numberOfRegisters + FPRInfo::numberOfRegisters));

    for (unsigned i = 0; i < GPRInfo::numberOfRegisters; ++i) {
#if USE(JSVALUE64)
        store64(GPRInfo::toRegister(i), buffer + i);
#else
        store32(GPRInfo::toRegister(i), buffer + i);
#endif
    }
    for (unsigned i = 0; i < FPRInfo::numberOfRegisters; ++i) {
        move(TrustedImmPtr(buffer + GPRInfo::numberOfRegisters + i), GPRInfo::regT0);
        storeDouble(FPRInfo::toRegister(i), Address(GPRInfo::regT0));
    }

    // Set up one argument.
    move(TrustedImmPtr(&vm), GPRInfo::argumentGPR0);
    move(TrustedImmPtr(tagCFunction<OperationPtrTag>(operationExceptionFuzzWithCallFrame)), GPRInfo::nonPreservedNonReturnGPR);
    prepareCallOperation(vm);
    call(GPRInfo::nonPreservedNonReturnGPR, OperationPtrTag);

    for (unsigned i = 0; i < FPRInfo::numberOfRegisters; ++i) {
        move(TrustedImmPtr(buffer + GPRInfo::numberOfRegisters + i), GPRInfo::regT0);
        loadDouble(Address(GPRInfo::regT0), FPRInfo::toRegister(i));
    }
    for (unsigned i = 0; i < GPRInfo::numberOfRegisters; ++i) {
#if USE(JSVALUE64)
        load64(buffer + i, GPRInfo::toRegister(i));
#else
        load32(buffer + i, GPRInfo::toRegister(i));
#endif
    }
}

AssemblyHelpers::Jump AssemblyHelpers::emitJumpIfException(VM& vm)
{
    return emitExceptionCheck(vm, NormalExceptionCheck);
}

AssemblyHelpers::Jump AssemblyHelpers::emitExceptionCheck(VM& vm, ExceptionCheckKind kind, ExceptionJumpWidth width)
{
    if (UNLIKELY(Options::useExceptionFuzz()))
        callExceptionFuzz(vm);

    if (width == FarJumpWidth)
        kind = (kind == NormalExceptionCheck ? InvertedExceptionCheck : NormalExceptionCheck);
    
    Jump result;
#if USE(JSVALUE64)
    result = branchTest64(kind == NormalExceptionCheck ? NonZero : Zero, AbsoluteAddress(vm.addressOfException()));
#elif USE(JSVALUE32_64)
    result = branch32(kind == NormalExceptionCheck ? NotEqual : Equal, AbsoluteAddress(vm.addressOfException()), TrustedImm32(0));
#endif
    
    if (width == NormalJumpWidth)
        return result;

    PatchableJump realJump = patchableJump();
    result.link(this);
    
    return realJump.m_jump;
}

AssemblyHelpers::Jump AssemblyHelpers::emitNonPatchableExceptionCheck(VM& vm)
{
    if (UNLIKELY(Options::useExceptionFuzz()))
        callExceptionFuzz(vm);

    Jump result;
#if USE(JSVALUE64)
    result = branchTest64(NonZero, AbsoluteAddress(vm.addressOfException()));
#elif USE(JSVALUE32_64)
    result = branch32(NotEqual, AbsoluteAddress(vm.addressOfException()), TrustedImm32(0));
#endif
    
    return result;
}

void AssemblyHelpers::emitStoreStructureWithTypeInfo(AssemblyHelpers& jit, TrustedImmPtr structure, RegisterID dest)
{
    const Structure* structurePtr = reinterpret_cast<const Structure*>(structure.m_value);
#if USE(JSVALUE64)
    jit.store32(TrustedImm32(structurePtr->id().bits()), MacroAssembler::Address(dest, JSCell::structureIDOffset()));
    jit.store32(TrustedImm32(structurePtr->typeInfoBlob()), MacroAssembler::Address(dest, JSCell::indexingTypeAndMiscOffset()));
    if (ASSERT_ENABLED) {
        Jump correctStructure = jit.branch32(Equal, MacroAssembler::Address(dest, JSCell::structureIDOffset()), TrustedImm32(structurePtr->id().bits()));
        jit.abortWithReason(AHStructureIDIsValid);
        correctStructure.link(&jit);

        Jump correctIndexingType = jit.branch8(Equal, MacroAssembler::Address(dest, JSCell::indexingTypeAndMiscOffset()), TrustedImm32(structurePtr->indexingModeIncludingHistory()));
        jit.abortWithReason(AHIndexingTypeIsValid);
        correctIndexingType.link(&jit);

        Jump correctType = jit.branch8(Equal, MacroAssembler::Address(dest, JSCell::typeInfoTypeOffset()), TrustedImm32(structurePtr->typeInfo().type()));
        jit.abortWithReason(AHTypeInfoIsValid);
        correctType.link(&jit);

        Jump correctFlags = jit.branch8(Equal, MacroAssembler::Address(dest, JSCell::typeInfoFlagsOffset()), TrustedImm32(structurePtr->typeInfo().inlineTypeFlags()));
        jit.abortWithReason(AHTypeInfoInlineTypeFlagsAreValid);
        correctFlags.link(&jit);
    }
#else
    // Do a 32-bit wide store to initialize the cell's fields.
    jit.store32(TrustedImm32(structurePtr->typeInfoBlob()), MacroAssembler::Address(dest, JSCell::indexingTypeAndMiscOffset()));
    jit.storePtr(structure, MacroAssembler::Address(dest, JSCell::structureIDOffset()));
#endif
}

void AssemblyHelpers::loadProperty(GPRReg object, GPRReg offset, JSValueRegs result)
{
    Jump isInline = branch32(LessThan, offset, TrustedImm32(firstOutOfLineOffset));
    
    loadPtr(Address(object, JSObject::butterflyOffset()), result.payloadGPR());
    neg32(offset);
    signExtend32ToPtr(offset, offset);
    Jump ready = jump();
    
    isInline.link(this);
    addPtr(
        TrustedImm32(
            static_cast<int32_t>(sizeof(JSObject)) -
            (static_cast<int32_t>(firstOutOfLineOffset) - 2) * static_cast<int32_t>(sizeof(EncodedJSValue))),
        object, result.payloadGPR());
    
    ready.link(this);
    
    loadValue(
        BaseIndex(
            result.payloadGPR(), offset, TimesEight, (firstOutOfLineOffset - 2) * sizeof(EncodedJSValue)),
        result);
}

void AssemblyHelpers::storeProperty(JSValueRegs value, GPRReg object, GPRReg offset, GPRReg scratch)
{
    Jump isInline = branch32(LessThan, offset, TrustedImm32(firstOutOfLineOffset));
    
    loadPtr(Address(object, JSObject::butterflyOffset()), scratch);
    neg32(offset);
    signExtend32ToPtr(offset, offset);
    Jump ready = jump();
    
    isInline.link(this);
    addPtr(
        TrustedImm32(
            static_cast<int32_t>(sizeof(JSObject)) -
            (static_cast<int32_t>(firstOutOfLineOffset) - 2) * static_cast<int32_t>(sizeof(EncodedJSValue))),
        object, scratch);
    
    ready.link(this);
    
    storeValue(value,
        BaseIndex(scratch, offset, TimesEight, (firstOutOfLineOffset - 2) * sizeof(EncodedJSValue)));
}

void AssemblyHelpers::emitNonNullDecodeZeroExtendedStructureID(RegisterID source, RegisterID dest)
{
#if ENABLE(STRUCTURE_ID_WITH_SHIFT)
    lshift64(source, TrustedImm32(StructureID::encodeShiftAmount), dest);
#elif CPU(ADDRESS64)
    // This could use BFI on arm64 but that only helps if the start of structure heap is encodable as a mov and not as an immediate in the add so it's probably not super important.
    if constexpr (structureHeapAddressSize >= 4 * GB) {
        ASSERT(structureHeapAddressSize == 4 * GB);
        move(source, dest);
        add64(TrustedImm64(g_jscConfig.startOfStructureHeap), dest);
    } else {
        and32(TrustedImm32(StructureID::structureIDMask), source, dest);
        add64(TrustedImm64(g_jscConfig.startOfStructureHeap), dest);
    }
#else // not CPU(ADDRESS64)
    move(source, dest);
#endif
}

void AssemblyHelpers::emitLoadStructure(VM&, RegisterID source, RegisterID dest)
{
    load32(MacroAssembler::Address(source, JSCell::structureIDOffset()), dest);
    emitNonNullDecodeZeroExtendedStructureID(dest, dest);
}

void AssemblyHelpers::emitEncodeStructureID(RegisterID source, RegisterID dest)
{
#if ENABLE(STRUCTURE_ID_WITH_SHIFT)
    urshift64(source, TrustedImm32(StructureID::encodeShiftAmount), dest);
#elif CPU(ADDRESS64)
    move(source, dest);
    static_assert(StructureID::structureIDMask <= UINT32_MAX);
    and64(TrustedImm32(static_cast<uint32_t>(StructureID::structureIDMask)), dest);
#else
    move(source, dest);
#endif
}

void AssemblyHelpers::emitLoadPrototype(VM& vm, GPRReg objectGPR, JSValueRegs resultRegs, JumpList& slowPath)
{
    ASSERT(resultRegs.payloadGPR() != objectGPR);

    emitLoadStructure(vm, objectGPR, resultRegs.payloadGPR());

    auto overridesGetPrototype = branchTest32(MacroAssembler::NonZero, MacroAssembler::Address(resultRegs.payloadGPR(), Structure::outOfLineTypeFlagsOffset()), TrustedImm32(OverridesGetPrototypeOutOfLine));
    slowPath.append(overridesGetPrototype);

    loadValue(MacroAssembler::Address(resultRegs.payloadGPR(), Structure::prototypeOffset()), resultRegs);
    auto hasMonoProto = branchIfNotEmpty(resultRegs);
    loadValue(MacroAssembler::Address(objectGPR, offsetRelativeToBase(knownPolyProtoOffset)), resultRegs);
    hasMonoProto.link(this);
}

void AssemblyHelpers::makeSpaceOnStackForCCall()
{
    unsigned stackOffset = WTF::roundUpToMultipleOf(stackAlignmentBytes(), maxFrameExtentForSlowPathCall);
    if (stackOffset)
        subPtr(TrustedImm32(stackOffset), stackPointerRegister);
}

void AssemblyHelpers::reclaimSpaceOnStackForCCall()
{
    unsigned stackOffset = WTF::roundUpToMultipleOf(stackAlignmentBytes(), maxFrameExtentForSlowPathCall);
    if (stackOffset)
        addPtr(TrustedImm32(stackOffset), stackPointerRegister);
}

#if USE(JSVALUE64)
template<typename LoadFromHigh, typename StoreToHigh, typename LoadFromLow, typename StoreToLow>
void emitRandomThunkImpl(AssemblyHelpers& jit, GPRReg scratch0, GPRReg scratch1, GPRReg scratch2, FPRReg result, const LoadFromHigh& loadFromHigh, const StoreToHigh& storeToHigh, const LoadFromLow& loadFromLow, const StoreToLow& storeToLow)
{
    // Inlined WeakRandom::advance().
    // uint64_t x = m_low;
    loadFromLow(scratch0);
    // uint64_t y = m_high;
    loadFromHigh(scratch1);
    // m_low = y;
    storeToLow(scratch1);

    // x ^= x << 23;
    jit.move(scratch0, scratch2);
    jit.lshift64(AssemblyHelpers::TrustedImm32(23), scratch2);
    jit.xor64(scratch2, scratch0);

    // x ^= x >> 17;
    jit.move(scratch0, scratch2);
    jit.rshift64(AssemblyHelpers::TrustedImm32(17), scratch2);
    jit.xor64(scratch2, scratch0);

    // x ^= y ^ (y >> 26);
    jit.move(scratch1, scratch2);
    jit.rshift64(AssemblyHelpers::TrustedImm32(26), scratch2);
    jit.xor64(scratch1, scratch2);
    jit.xor64(scratch2, scratch0);

    // m_high = x;
    storeToHigh(scratch0);

    // return x + y;
    jit.add64(scratch1, scratch0);

    // Extract random 53bit. [0, 53] bit is safe integer number ranges in double representation.
    jit.move(AssemblyHelpers::TrustedImm64((1ULL << 53) - 1), scratch1);
    jit.and64(scratch1, scratch0);
    // Now, scratch0 is always in range of int64_t. Safe to convert it to double with cvtsi2sdq.
    jit.convertInt64ToDouble(scratch0, result);

    // Convert `(53bit double integer value) / (1 << 53)` to `(53bit double integer value) * (1.0 / (1 << 53))`.
    // In latter case, `1.0 / (1 << 53)` will become a double value represented as (mantissa = 0 & exp = 970, it means 1e-(2**54)).
    static constexpr double scale = 1.0 / (1ULL << 53);

    // Multiplying 1e-(2**54) with the double integer does not change anything of the mantissa part of the double integer.
    // It just reduces the exp part of the given 53bit double integer.
    // (Except for 0.0. This is specially handled and in this case, exp just becomes 0.)
    // Now we get 53bit precision random double value in [0, 1).
    jit.move(AssemblyHelpers::TrustedImmPtr(&scale), scratch1);
    jit.mulDouble(AssemblyHelpers::Address(scratch1), result);
}

void AssemblyHelpers::emitRandomThunk(JSGlobalObject* globalObject, GPRReg scratch0, GPRReg scratch1, GPRReg scratch2, FPRReg result)
{
    void* lowAddress = reinterpret_cast<uint8_t*>(globalObject) + JSGlobalObject::weakRandomOffset() + WeakRandom::lowOffset();
    void* highAddress = reinterpret_cast<uint8_t*>(globalObject) + JSGlobalObject::weakRandomOffset() + WeakRandom::highOffset();

    auto loadFromHigh = [&](GPRReg high) {
        load64(highAddress, high);
    };
    auto storeToHigh = [&](GPRReg high) {
        store64(high, highAddress);
    };
    auto loadFromLow = [&](GPRReg low) {
        load64(lowAddress, low);
    };
    auto storeToLow = [&](GPRReg low) {
        store64(low, lowAddress);
    };

    emitRandomThunkImpl(*this, scratch0, scratch1, scratch2, result, loadFromHigh, storeToHigh, loadFromLow, storeToLow);
}

void AssemblyHelpers::emitRandomThunk(VM& vm, GPRReg scratch0, GPRReg scratch1, GPRReg scratch2, GPRReg scratch3, FPRReg result)
{
    emitGetFromCallFrameHeaderPtr(CallFrameSlot::callee, scratch3);
    emitLoadStructure(vm, scratch3, scratch3);
    loadPtr(Address(scratch3, Structure::globalObjectOffset()), scratch3);
    // Now, scratch3 holds JSGlobalObject*.

    auto loadFromHigh = [&](GPRReg high) {
        load64(Address(scratch3, JSGlobalObject::weakRandomOffset() + WeakRandom::highOffset()), high);
    };
    auto storeToHigh = [&](GPRReg high) {
        store64(high, Address(scratch3, JSGlobalObject::weakRandomOffset() + WeakRandom::highOffset()));
    };
    auto loadFromLow = [&](GPRReg low) {
        load64(Address(scratch3, JSGlobalObject::weakRandomOffset() + WeakRandom::lowOffset()), low);
    };
    auto storeToLow = [&](GPRReg low) {
        store64(low, Address(scratch3, JSGlobalObject::weakRandomOffset() + WeakRandom::lowOffset()));
    };

    emitRandomThunkImpl(*this, scratch0, scratch1, scratch2, result, loadFromHigh, storeToHigh, loadFromLow, storeToLow);
}
#endif

void AssemblyHelpers::emitAllocateWithNonNullAllocator(GPRReg resultGPR, const JITAllocator& allocator, GPRReg allocatorGPR, GPRReg scratchGPR, JumpList& slowPath)
{
    if (Options::forceGCSlowPaths()) {
        slowPath.append(jump());
        return;
    }

    // NOTE, some invariants of this function:
    // - When going to the slow path, we must leave resultGPR with zero in it.
    // - We *can not* use RegisterSet::macroScratchRegisters on x86.
    // - We *can* use RegisterSet::macroScratchRegisters on ARM.

    Jump popPath;
    Jump done;
    
    if (allocator.isConstant())
        move(TrustedImmPtr(allocator.allocator().localAllocator()), allocatorGPR);

    load32(Address(allocatorGPR, LocalAllocator::offsetOfFreeList() + FreeList::offsetOfRemaining()), resultGPR);
    popPath = branchTest32(Zero, resultGPR);
    if (allocator.isConstant())
        add32(TrustedImm32(-allocator.allocator().cellSize()), resultGPR, scratchGPR);
    else {
        move(resultGPR, scratchGPR);
        sub32(Address(allocatorGPR, LocalAllocator::offsetOfCellSize()), scratchGPR);
    }
    negPtr(resultGPR);
    store32(scratchGPR, Address(allocatorGPR, LocalAllocator::offsetOfFreeList() + FreeList::offsetOfRemaining()));
    Address payloadEndAddr = Address(allocatorGPR, LocalAllocator::offsetOfFreeList() + FreeList::offsetOfPayloadEnd());
    addPtr(payloadEndAddr, resultGPR);

    done = jump();
        
    popPath.link(this);
        
    loadPtr(Address(allocatorGPR, LocalAllocator::offsetOfFreeList() + FreeList::offsetOfScrambledHead()), resultGPR);
    xorPtr(Address(allocatorGPR, LocalAllocator::offsetOfFreeList() + FreeList::offsetOfSecret()), resultGPR);
    slowPath.append(branchTestPtr(Zero, resultGPR));
        
    // The object is half-allocated: we have what we know is a fresh object, but
    // it's still on the GC's free list.
    loadPtr(Address(resultGPR, FreeCell::offsetOfScrambledNext()), scratchGPR);
    storePtr(scratchGPR, Address(allocatorGPR, LocalAllocator::offsetOfFreeList() + FreeList::offsetOfScrambledHead()));
        
    done.link(this);
}

void AssemblyHelpers::emitAllocate(GPRReg resultGPR, const JITAllocator& allocator, GPRReg allocatorGPR, GPRReg scratchGPR, JumpList& slowPath)
{
    if (allocator.isConstant()) {
        if (!allocator.allocator()) {
            slowPath.append(jump());
            return;
        }
    } else
        slowPath.append(branchTestPtr(Zero, allocatorGPR));
    emitAllocateWithNonNullAllocator(resultGPR, allocator, allocatorGPR, scratchGPR, slowPath);
}

void AssemblyHelpers::emitAllocateVariableSized(GPRReg resultGPR, CompleteSubspace& subspace, GPRReg allocationSize, GPRReg scratchGPR1, GPRReg scratchGPR2, JumpList& slowPath)
{
    static_assert(!(MarkedSpace::sizeStep & (MarkedSpace::sizeStep - 1)), "MarkedSpace::sizeStep must be a power of two.");
    
    unsigned stepShift = getLSBSet(MarkedSpace::sizeStep);
    
    add32(TrustedImm32(MarkedSpace::sizeStep - 1), allocationSize, scratchGPR1);
    urshift32(TrustedImm32(stepShift), scratchGPR1);
    slowPath.append(branch32(Above, scratchGPR1, TrustedImm32(MarkedSpace::largeCutoff >> stepShift)));
    move(TrustedImmPtr(subspace.allocatorForSizeStep()), scratchGPR2);
    loadPtr(BaseIndex(scratchGPR2, scratchGPR1, ScalePtr), scratchGPR1);
    
    emitAllocate(resultGPR, JITAllocator::variable(), scratchGPR1, scratchGPR2, slowPath);
}

void AssemblyHelpers::restoreCalleeSavesFromEntryFrameCalleeSavesBuffer(EntryFrame*& topEntryFrame)
{
#if NUMBER_OF_CALLEE_SAVES_REGISTERS > 0
    RegisterAtOffsetList* allCalleeSaves = RegisterSet::vmCalleeSaveRegisterOffsets();
    RegisterSet dontRestoreRegisters = RegisterSet::stackRegisters();
    unsigned registerCount = allCalleeSaves->registerCount();

    GPRReg scratch = InvalidGPRReg;
    unsigned scratchGPREntryIndex = 0;
#if CPU(ARM64)
    // We don't need a second scratch GPR, but we'll also defer restoring this
    // GPR (in the next slot after the scratch) so that we can restore them together
    // later using a loadPair64.
    GPRReg unusedNextSlotGPR = InvalidGPRReg;
#endif

    // Use the first GPR entry's register as our baseGPR.
    for (unsigned i = 0; i < registerCount; i++) {
        RegisterAtOffset entry = allCalleeSaves->at(i);
        if (dontRestoreRegisters.contains(entry.reg()))
            continue;
        if (entry.reg().isGPR()) {
#if CPU(ARM64)
            if (i + 1 < registerCount) {
                RegisterAtOffset entry2 = allCalleeSaves->at(i + 1);
                if (!dontRestoreRegisters.contains(entry2.reg())
                    && entry2.reg().isGPR()
                    && entry2.offset() == entry.offset() + static_cast<ptrdiff_t>(sizeof(CPURegister))) {
                    scratchGPREntryIndex = i;
                    scratch = entry.reg().gpr();
                    unusedNextSlotGPR = entry2.reg().gpr();
                    break;
                }
            }
#else
            scratchGPREntryIndex = i;
            scratch = entry.reg().gpr();
            break;
#endif
        }
    }
    ASSERT(scratch != InvalidGPRReg);
    
    RegisterSet skipList;
    skipList.set(dontRestoreRegisters);

    // Skip the scratch register(s). We'll restore them later.
    skipList.add(scratch);
#if CPU(ARM64)
    RELEASE_ASSERT(unusedNextSlotGPR != InvalidGPRReg);
    skipList.add(unusedNextSlotGPR);
#endif

    loadPtr(&topEntryFrame, scratch);
    restoreCalleeSavesFromVMEntryFrameCalleeSavesBufferImpl(scratch, skipList);

    // Restore the callee save value of the scratch.
    RegisterAtOffset entry = allCalleeSaves->at(scratchGPREntryIndex);
    ASSERT(!dontRestoreRegisters.get(entry.reg()));
    ASSERT(entry.reg().isGPR());
    ASSERT(scratch == entry.reg().gpr());
#if CPU(ARM64)
    RegisterAtOffset entry2 = allCalleeSaves->at(scratchGPREntryIndex + 1);
    ASSERT_UNUSED(entry2, !dontRestoreRegisters.get(entry2.reg()));
    ASSERT(entry2.reg().isGPR());
    ASSERT(unusedNextSlotGPR == entry2.reg().gpr());
    loadPair64(scratch, TrustedImm32(entry.offset()), scratch, unusedNextSlotGPR);
#else
    loadPtr(Address(scratch, entry.offset()), scratch);
#endif

#else
    UNUSED_PARAM(topEntryFrame);
#endif // NUMBER_OF_CALLEE_SAVES_REGISTERS > 0
}

void AssemblyHelpers::restoreCalleeSavesFromVMEntryFrameCalleeSavesBuffer(GPRReg vmGPR, GPRReg scratchGPR)
{
#if NUMBER_OF_CALLEE_SAVES_REGISTERS > 0
    loadPtr(Address(vmGPR, VM::topEntryFrameOffset()), scratchGPR);
    restoreCalleeSavesFromVMEntryFrameCalleeSavesBufferImpl(scratchGPR, RegisterSet::stackRegisters());
#else
    UNUSED_PARAM(vmGPR);
#endif // NUMBER_OF_CALLEE_SAVES_REGISTERS > 0
}

void AssemblyHelpers::restoreCalleeSavesFromVMEntryFrameCalleeSavesBufferImpl(GPRReg entryFrameGPR, const RegisterSet& skipList)
{
#if NUMBER_OF_CALLEE_SAVES_REGISTERS > 0
    addPtr(TrustedImm32(EntryFrame::calleeSaveRegistersBufferOffset()), entryFrameGPR);

    RegisterAtOffsetList* allCalleeSaves = RegisterSet::vmCalleeSaveRegisterOffsets();
    unsigned registerCount = allCalleeSaves->registerCount();

    LoadRegSpooler spooler(*this, entryFrameGPR);

    // Restore all callee saves except for the scratch.
    unsigned i = 0;
    for (; i < registerCount; i++) {
        RegisterAtOffset entry = allCalleeSaves->at(i);
        if (skipList.contains(entry.reg()))
            continue;
        if (!entry.reg().isGPR())
            break;
        spooler.loadGPR(entry);
    }
    spooler.finalizeGPR();
    for (; i < registerCount; i++) {
        RegisterAtOffset entry = allCalleeSaves->at(i);
        if (skipList.contains(entry.reg()))
            continue;
        ASSERT(!entry.reg().isGPR());
        spooler.loadFPR(entry);
    }
    spooler.finalizeFPR();

#else
    UNUSED_PARAM(vmGPR);
    UNUSED_PARAM(skipList);
#endif // NUMBER_OF_CALLEE_SAVES_REGISTERS > 0
}

void AssemblyHelpers::emitVirtualCall(VM& vm, JSGlobalObject* globalObject, CallLinkInfo* info)
{
    move(TrustedImmPtr(globalObject), GPRInfo::regT3);
    move(TrustedImmPtr(info), GPRInfo::regT2);
    emitVirtualCallWithoutMovingGlobalObject(vm, GPRInfo::regT2, info->callMode());
}

void AssemblyHelpers::emitVirtualCallWithoutMovingGlobalObject(VM& vm, GPRReg callLinkInfoGPR, CallMode callMode)
{
    move(callLinkInfoGPR, GPRInfo::regT2);
    Call call = nearCall();
    addLinkTask([=, &vm] (LinkBuffer& linkBuffer) {
        auto callLocation = linkBuffer.locationOfNearCall<JITCompilationPtrTag>(call);
        linkBuffer.addMainThreadFinalizationTask([=, &vm] () {
            MacroAssemblerCodeRef<JITStubRoutinePtrTag> virtualThunk = vm.getCTIVirtualCall(callMode);
            MacroAssembler::repatchNearCall(callLocation, CodeLocationLabel<JITStubRoutinePtrTag>(virtualThunk.code()));
        });
    });
}

#if USE(JSVALUE64)
void AssemblyHelpers::wangsInt64Hash(GPRReg inputAndResult, GPRReg scratch)
{
    GPRReg input = inputAndResult;
    // key += ~(key << 32);
    move(input, scratch);
    lshift64(TrustedImm32(32), scratch);
    not64(scratch);
    add64(scratch, input);
    // key ^= (key >> 22);
    move(input, scratch);
    urshift64(TrustedImm32(22), scratch);
    xor64(scratch, input);
    // key += ~(key << 13);
    move(input, scratch);
    lshift64(TrustedImm32(13), scratch);
    not64(scratch);
    add64(scratch, input);
    // key ^= (key >> 8);
    move(input, scratch);
    urshift64(TrustedImm32(8), scratch);
    xor64(scratch, input);
    // key += (key << 3);
    move(input, scratch);
    lshift64(TrustedImm32(3), scratch);
    add64(scratch, input);
    // key ^= (key >> 15);
    move(input, scratch);
    urshift64(TrustedImm32(15), scratch);
    xor64(scratch, input);
    // key += ~(key << 27);
    move(input, scratch);
    lshift64(TrustedImm32(27), scratch);
    not64(scratch);
    add64(scratch, input);
    // key ^= (key >> 31);
    move(input, scratch);
    urshift64(TrustedImm32(31), scratch);
    xor64(scratch, input);

    // return static_cast<unsigned>(result)
    void* mask = bitwise_cast<void*>(static_cast<uintptr_t>(UINT_MAX));
    and64(TrustedImmPtr(mask), inputAndResult);
}
#endif // USE(JSVALUE64)

void AssemblyHelpers::emitConvertValueToBoolean(VM& vm, JSValueRegs value, GPRReg result, GPRReg scratchIfShouldCheckMasqueradesAsUndefined, FPRReg valueAsFPR, FPRReg tempFPR, bool shouldCheckMasqueradesAsUndefined, JSGlobalObject* globalObject, bool invert)
{
    // Implements the following control flow structure:
    // if (value is cell) {
    //     if (value is string or value is HeapBigInt)
    //         result = !!value->length
    //     else {
    //         do evil things for masquerades-as-undefined
    //         result = true
    //     }
    // } else if (value is int32) {
    //     result = !!unboxInt32(value)
    // } else if (value is number) {
    //     result = !!unboxDouble(value)
    // } else if (value is BigInt32) {
    //     result = !!unboxBigInt32(value)
    // } else {
    //     result = value == jsTrue
    // }

    JumpList done;

    auto notCell = branchIfNotCell(value);
    auto isString = branchIfString(value.payloadGPR());
    auto isHeapBigInt = branchIfHeapBigInt(value.payloadGPR());

    if (shouldCheckMasqueradesAsUndefined) {
        ASSERT(scratchIfShouldCheckMasqueradesAsUndefined != InvalidGPRReg);
        JumpList isNotMasqueradesAsUndefined;
        isNotMasqueradesAsUndefined.append(branchTest8(Zero, Address(value.payloadGPR(), JSCell::typeInfoFlagsOffset()), TrustedImm32(MasqueradesAsUndefined)));
        emitLoadStructure(vm, value.payloadGPR(), result);
        move(TrustedImmPtr(globalObject), scratchIfShouldCheckMasqueradesAsUndefined);
        isNotMasqueradesAsUndefined.append(branchPtr(NotEqual, Address(result, Structure::globalObjectOffset()), scratchIfShouldCheckMasqueradesAsUndefined));

        // We act like we are "undefined" here.
        move(invert ? TrustedImm32(1) : TrustedImm32(0), result);
        done.append(jump());
        isNotMasqueradesAsUndefined.link(this);
    }
    move(invert ? TrustedImm32(0) : TrustedImm32(1), result);
    done.append(jump());

    isString.link(this);
    move(TrustedImmPtr(jsEmptyString(vm)), result);
    comparePtr(invert ? Equal : NotEqual, value.payloadGPR(), result, result);
    done.append(jump());

    isHeapBigInt.link(this);
    load32(Address(value.payloadGPR(), JSBigInt::offsetOfLength()), result);
    compare32(invert ? Equal : NotEqual, result, TrustedImm32(0), result);
    done.append(jump());

    notCell.link(this);
    auto notInt32 = branchIfNotInt32(value);
    compare32(invert ? Equal : NotEqual, value.payloadGPR(), TrustedImm32(0), result);
    done.append(jump());

    notInt32.link(this);
    auto notDouble = branchIfNotDoubleKnownNotInt32(value);
#if USE(JSVALUE64)
    unboxDouble(value.gpr(), result, valueAsFPR);
#else
    unboxDouble(value, valueAsFPR);
#endif
    move(invert ? TrustedImm32(1) : TrustedImm32(0), result);
    done.append(branchDoubleZeroOrNaN(valueAsFPR, tempFPR));
    move(invert ? TrustedImm32(0) : TrustedImm32(1), result);
    done.append(jump());

    notDouble.link(this);
#if USE(BIGINT32)
    auto isNotBigInt32 = branchIfNotBigInt32(value.gpr(), result);
    move(value.gpr(), result);
    urshift64(TrustedImm32(16), result);
    compare32(invert ? Equal : NotEqual, result, TrustedImm32(0), result);
    done.append(jump());

    isNotBigInt32.link(this);
#endif // USE(BIGINT32)
#if USE(JSVALUE64)
    compare64(invert ? NotEqual : Equal, value.gpr(), TrustedImm32(JSValue::ValueTrue), result);
#else
    move(invert ? TrustedImm32(1) : TrustedImm32(0), result);
    done.append(branchIfNotBoolean(value, InvalidGPRReg));
    compare32(invert ? Equal : NotEqual, value.payloadGPR(), TrustedImm32(0), result);
#endif

    done.link(this);
}

AssemblyHelpers::JumpList AssemblyHelpers::branchIfValue(VM& vm, JSValueRegs value, GPRReg scratch, GPRReg scratchIfShouldCheckMasqueradesAsUndefined, FPRReg valueAsFPR, FPRReg tempFPR, bool shouldCheckMasqueradesAsUndefined, std::variant<JSGlobalObject*, GPRReg> globalObject, bool invert)
{
    // Implements the following control flow structure:
    // if (value is cell) {
    //     if (value is string or value is HeapBigInt)
    //         result = !!value->length
    //     else {
    //         do evil things for masquerades-as-undefined
    //         result = true
    //     }
    // } else if (value is int32) {
    //     result = !!unboxInt32(value)
    // } else if (value is number) {
    //     result = !!unboxDouble(value)
    // } else if (value is BigInt32) {
    //     result = !!unboxBigInt32(value)
    // } else {
    //     result = value == jsTrue
    // }

    JumpList done;
    JumpList truthy;

    auto notCell = branchIfNotCell(value);
    auto isString = branchIfString(value.payloadGPR());
    auto isHeapBigInt = branchIfHeapBigInt(value.payloadGPR());

    if (shouldCheckMasqueradesAsUndefined) {
        ASSERT(scratchIfShouldCheckMasqueradesAsUndefined != InvalidGPRReg);
        JumpList isNotMasqueradesAsUndefined;
        isNotMasqueradesAsUndefined.append(branchTest8(Zero, Address(value.payloadGPR(), JSCell::typeInfoFlagsOffset()), TrustedImm32(MasqueradesAsUndefined)));
        emitLoadStructure(vm, value.payloadGPR(), scratch);
        if (std::holds_alternative<JSGlobalObject*>(globalObject))
            move(TrustedImmPtr(std::get<JSGlobalObject*>(globalObject)), scratchIfShouldCheckMasqueradesAsUndefined);
        else
            move(std::get<GPRReg>(globalObject), scratchIfShouldCheckMasqueradesAsUndefined);
        isNotMasqueradesAsUndefined.append(branchPtr(NotEqual, Address(scratch, Structure::globalObjectOffset()), scratchIfShouldCheckMasqueradesAsUndefined));

        // We act like we are "undefined" here.
        if (invert)
            truthy.append(jump());
        else
            done.append(jump());

        if (invert)
            done.append(isNotMasqueradesAsUndefined);
        else
            truthy.append(isNotMasqueradesAsUndefined);
    } else {
        if (invert)
            done.append(jump());
        else
            truthy.append(jump());
    }

    isString.link(this);
    truthy.append(branchPtr(invert ? Equal : NotEqual, value.payloadGPR(), TrustedImmPtr(jsEmptyString(vm))));
    done.append(jump());

    isHeapBigInt.link(this);
    truthy.append(branchTest32(invert ? Zero : NonZero, Address(value.payloadGPR(), JSBigInt::offsetOfLength())));
    done.append(jump());

    notCell.link(this);
    auto notInt32 = branchIfNotInt32(value);
    truthy.append(branchTest32(invert ? Zero : NonZero, value.payloadGPR()));
    done.append(jump());

    notInt32.link(this);
    auto notDouble = branchIfNotDoubleKnownNotInt32(value);
#if USE(JSVALUE64)
    unboxDouble(value.gpr(), scratch, valueAsFPR);
#else
    unboxDouble(value, valueAsFPR);
#endif
    if (invert) {
        truthy.append(branchDoubleZeroOrNaN(valueAsFPR, tempFPR));
        done.append(jump());
    } else {
        done.append(branchDoubleZeroOrNaN(valueAsFPR, tempFPR));
        truthy.append(jump());
    }

    notDouble.link(this);
#if USE(BIGINT32)
    auto isNotBigInt32 = branchIfNotBigInt32(value.gpr(), scratch);
    move(value.gpr(), scratch);
    urshift64(TrustedImm32(16), scratch);
    truthy.append(branchTest32(invert ? Zero : NonZero, scratch));
    done.append(jump());

    isNotBigInt32.link(this);
#endif // USE(BIGINT32)
#if USE(JSVALUE64)
    truthy.append(branch64(invert ? NotEqual : Equal, value.gpr(), TrustedImm64(JSValue::encode(jsBoolean(true)))));
#else
    auto notBoolean = branchIfNotBoolean(value, InvalidGPRReg);
    if (invert)
        truthy.append(notBoolean);
    else
        done.append(notBoolean);
    truthy.append(branch32(invert ? Equal : NotEqual, value.payloadGPR(), TrustedImm32(0)));
#endif

    done.link(this);

    return truthy;
}

#if ENABLE(WEBASSEMBLY)
void AssemblyHelpers::loadWasmContextInstance(GPRReg dst)
{
#if ENABLE(FAST_TLS_JIT)
    if (Wasm::Context::useFastTLS()) {
        loadFromTLSPtr(fastTLSOffsetForKey(WTF_WASM_CONTEXT_KEY), dst);
        return;
    }
#endif
    move(Wasm::PinnedRegisterInfo::get().wasmContextInstancePointer, dst);
}

void AssemblyHelpers::storeWasmContextInstance(GPRReg src)
{
#if ENABLE(FAST_TLS_JIT)
    if (Wasm::Context::useFastTLS()) {
        storeToTLSPtr(src, fastTLSOffsetForKey(WTF_WASM_CONTEXT_KEY));
        return;
    }
#endif
    move(src, Wasm::PinnedRegisterInfo::get().wasmContextInstancePointer);
}

bool AssemblyHelpers::loadWasmContextInstanceNeedsMacroScratchRegister()
{
#if ENABLE(FAST_TLS_JIT)
    if (Wasm::Context::useFastTLS())
        return loadFromTLSPtrNeedsMacroScratchRegister();
#endif
    return false;
}

bool AssemblyHelpers::storeWasmContextInstanceNeedsMacroScratchRegister()
{
#if ENABLE(FAST_TLS_JIT)
    if (Wasm::Context::useFastTLS())
        return storeToTLSPtrNeedsMacroScratchRegister();
#endif
    return false;
}

#endif // ENABLE(WEBASSEMBLY)

void AssemblyHelpers::debugCall(VM& vm, V_DebugOperation_EPP function, void* argument)
{
    size_t scratchSize = sizeof(EncodedJSValue) * (GPRInfo::numberOfRegisters + FPRInfo::numberOfRegisters);
    ScratchBuffer* scratchBuffer = vm.scratchBufferForSize(scratchSize);
    EncodedJSValue* buffer = static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer());

    for (unsigned i = 0; i < GPRInfo::numberOfRegisters; ++i) {
#if USE(JSVALUE64)
        store64(GPRInfo::toRegister(i), buffer + i);
#else
        store32(GPRInfo::toRegister(i), buffer + i);
#endif
    }

    for (unsigned i = 0; i < FPRInfo::numberOfRegisters; ++i) {
        move(TrustedImmPtr(buffer + GPRInfo::numberOfRegisters + i), GPRInfo::regT0);
        storeDouble(FPRInfo::toRegister(i), Address(GPRInfo::regT0));
    }

#if CPU(X86_64) || CPU(ARM_THUMB2) || CPU(ARM64) || CPU(MIPS) || CPU(RISCV64)
    move(TrustedImmPtr(buffer), GPRInfo::argumentGPR2);
    move(TrustedImmPtr(argument), GPRInfo::argumentGPR1);
    move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR0);
    GPRReg scratch = selectScratchGPR(GPRInfo::argumentGPR0, GPRInfo::argumentGPR1, GPRInfo::argumentGPR2);
#else
#error "JIT not supported on this platform."
#endif
    prepareCallOperation(vm);
    move(TrustedImmPtr(tagCFunctionPtr<OperationPtrTag>(function)), scratch);
    call(scratch, OperationPtrTag);

    for (unsigned i = 0; i < FPRInfo::numberOfRegisters; ++i) {
        move(TrustedImmPtr(buffer + GPRInfo::numberOfRegisters + i), GPRInfo::regT0);
        loadDouble(Address(GPRInfo::regT0), FPRInfo::toRegister(i));
    }
    for (unsigned i = 0; i < GPRInfo::numberOfRegisters; ++i) {
#if USE(JSVALUE64)
        load64(buffer + i, GPRInfo::toRegister(i));
#else
        load32(buffer + i, GPRInfo::toRegister(i));
#endif
    }
}

void AssemblyHelpers::copyCalleeSavesToEntryFrameCalleeSavesBufferImpl(GPRReg calleeSavesBuffer)
{
#if NUMBER_OF_CALLEE_SAVES_REGISTERS > 0
    addPtr(TrustedImm32(EntryFrame::calleeSaveRegistersBufferOffset()), calleeSavesBuffer);

    RegisterAtOffsetList* allCalleeSaves = RegisterSet::vmCalleeSaveRegisterOffsets();
    RegisterSet dontCopyRegisters = RegisterSet::stackRegisters();
    unsigned registerCount = allCalleeSaves->registerCount();

    StoreRegSpooler spooler(*this, calleeSavesBuffer);

    unsigned i = 0;
    for (; i < registerCount; i++) {
        RegisterAtOffset entry = allCalleeSaves->at(i);
        if (dontCopyRegisters.contains(entry.reg()))
            continue;
        if (!entry.reg().isGPR())
            break;
        spooler.storeGPR(entry);
    }
    spooler.finalizeGPR();
    for (; i < registerCount; i++) {
        RegisterAtOffset entry = allCalleeSaves->at(i);
        if (dontCopyRegisters.contains(entry.reg()))
            continue;
        spooler.storeFPR(entry);
    }
    spooler.finalizeFPR();

#else
    UNUSED_PARAM(calleeSavesBuffer);
#endif
}

void AssemblyHelpers::cageWithoutUntagging(Gigacage::Kind kind, GPRReg storage)
{
#if GIGACAGE_ENABLED
    if (!Gigacage::isEnabled(kind))
        return;

#if CPU(ARM64E)
    RegisterID tempReg = InvalidGPRReg;
    Jump skip;
    if (kind == Gigacage::Primitive) {
        skip = branchPtr(Equal, storage, TrustedImmPtr(JSArrayBufferView::nullVectorPtr()));
        tempReg = getCachedMemoryTempRegisterIDAndInvalidate();
        move(storage, tempReg);
        // Flip the registers since bitFieldInsert only inserts into the low bits.
        std::swap(storage, tempReg);
    }
#endif
    andPtr(TrustedImmPtr(Gigacage::mask(kind)), storage);
    addPtr(TrustedImmPtr(Gigacage::basePtr(kind)), storage);
#if CPU(ARM64E)
    if (kind == Gigacage::Primitive)
        insertBitField64(storage, TrustedImm32(0), TrustedImm32(64 - maxNumberOfAllowedPACBits), tempReg);
    if (skip.isSet())
        skip.link(this);
#endif

#else
    UNUSED_PARAM(kind);
    UNUSED_PARAM(storage);
#endif
}

// length may be the same register as scratch.
void AssemblyHelpers::cageConditionallyAndUntag(Gigacage::Kind kind, GPRReg storage, GPRReg length, GPRReg scratch, bool validateAuth)
{
#if GIGACAGE_ENABLED
    if (Gigacage::isEnabled(kind)) {
        if (kind != Gigacage::Primitive || Gigacage::disablingPrimitiveGigacageIsForbidden())
            cageWithoutUntagging(kind, storage);
        else {
#if CPU(ARM64E)
            if (length == scratch)
                scratch = getCachedMemoryTempRegisterIDAndInvalidate();
#endif
            JumpList done;
#if CPU(ARM64E)
            done.append(branchPtr(Equal, storage, TrustedImmPtr(JSArrayBufferView::nullVectorPtr())));
#endif
            done.append(branchTest8(NonZero, AbsoluteAddress(&Gigacage::disablePrimitiveGigacageRequested)));

            loadPtr(Gigacage::addressOfBasePtr(kind), scratch);
            done.append(branchTest64(Zero, scratch));
#if CPU(ARM64E)
            GPRReg tempReg = getCachedDataTempRegisterIDAndInvalidate();
            move(storage, tempReg);
            ASSERT(LogicalImmediate::create64(Gigacage::mask(kind)).isValid());
            andPtr(TrustedImmPtr(Gigacage::mask(kind)), tempReg);
            addPtr(scratch, tempReg);
            insertBitField64(tempReg, TrustedImm32(0), TrustedImm32(64 - maxNumberOfAllowedPACBits), storage);
#else
            andPtr(TrustedImmPtr(Gigacage::mask(kind)), storage);
            addPtr(scratch, storage);
#endif // CPU(ARM64E)
            done.link(this);
        }
    }
#endif

#if CPU(ARM64E)
    if (kind == Gigacage::Primitive)
        untagArrayPtr(length, storage, validateAuth, scratch);
#endif
    UNUSED_PARAM(validateAuth);
    UNUSED_PARAM(kind);
    UNUSED_PARAM(storage);
    UNUSED_PARAM(length);
    UNUSED_PARAM(scratch);
}

void AssemblyHelpers::emitSave(const RegisterAtOffsetList& list)
{
    StoreRegSpooler spooler(*this, framePointerRegister);

    size_t registerCount = list.registerCount();
    size_t i = 0;
    for (; i < registerCount; i++) {
        auto entry = list.at(i);
        if (!entry.reg().isGPR())
            break;
        spooler.storeGPR(entry);
    }
    spooler.finalizeGPR();

    for (; i < registerCount; i++)
        spooler.storeFPR(list.at(i));
    spooler.finalizeFPR();
}

void AssemblyHelpers::emitRestore(const RegisterAtOffsetList& list)
{
    LoadRegSpooler spooler(*this, framePointerRegister);

    size_t registerCount = list.registerCount();
    size_t i = 0;
    for (; i < registerCount; i++) {
        auto entry = list.at(i);
        if (!entry.reg().isGPR())
            break;
        spooler.loadGPR(entry);
    }
    spooler.finalizeGPR();

    for (; i < registerCount; i++)
        spooler.loadFPR(list.at(i));
    spooler.finalizeFPR();
}

void AssemblyHelpers::emitSaveCalleeSavesFor(const RegisterAtOffsetList* calleeSaves)
{
    RegisterSet dontSaveRegisters = RegisterSet(RegisterSet::stackRegisters());
    unsigned registerCount = calleeSaves->registerCount();

    StoreRegSpooler spooler(*this, framePointerRegister);

    unsigned i = 0;
    for (; i < registerCount; i++) {
        RegisterAtOffset entry = calleeSaves->at(i);
        if (entry.reg().isFPR())
            break;
        if (dontSaveRegisters.contains(entry.reg()))
            continue;
        spooler.storeGPR(entry);
    }
    spooler.finalizeGPR();
    for (; i < registerCount; i++) {
        RegisterAtOffset entry = calleeSaves->at(i);
        if (dontSaveRegisters.contains(entry.reg()))
            continue;
        spooler.storeFPR(entry);
    }
    spooler.finalizeFPR();
}

void AssemblyHelpers::emitRestoreCalleeSavesFor(const RegisterAtOffsetList* calleeSaves)
{
    RegisterSet dontRestoreRegisters = RegisterSet(RegisterSet::stackRegisters());
    unsigned registerCount = calleeSaves->registerCount();
    
    LoadRegSpooler spooler(*this, framePointerRegister);

    unsigned i = 0;
    for (; i < registerCount; i++) {
        RegisterAtOffset entry = calleeSaves->at(i);
        if (entry.reg().isFPR())
            break;
        if (dontRestoreRegisters.get(entry.reg()))
            continue;
        spooler.loadGPR(entry);
    }
    spooler.finalizeGPR();
    for (; i < registerCount; i++) {
        RegisterAtOffset entry = calleeSaves->at(i);
        if (dontRestoreRegisters.get(entry.reg()))
            continue;
        spooler.loadFPR(entry);
    }
    spooler.finalizeFPR();
}

void AssemblyHelpers::copyLLIntBaselineCalleeSavesFromFrameOrRegisterToEntryFrameCalleeSavesBuffer(EntryFrame*& topEntryFrame, const RegisterSet& usedRegisters)
{
#if NUMBER_OF_CALLEE_SAVES_REGISTERS > 0
    // Copy saved calleeSaves on stack or unsaved calleeSaves in register to vm calleeSave buffer
    ScratchRegisterAllocator allocator(usedRegisters);
    GPRReg destBufferGPR = allocator.allocateScratchGPR();
    GPRReg temp1 = allocator.allocateScratchGPR();
    FPRReg fpTemp1 = allocator.allocateScratchFPR();
    GPRReg temp2 = allocator.allocateScratchGPR();
    FPRReg fpTemp2 = allocator.allocateScratchFPR();
    RELEASE_ASSERT(!allocator.didReuseRegisters());

    loadPtr(&topEntryFrame, destBufferGPR);
    addPtr(TrustedImm32(EntryFrame::calleeSaveRegistersBufferOffset()), destBufferGPR);

    CopySpooler spooler(*this, framePointerRegister, destBufferGPR, temp1, temp2, fpTemp1, fpTemp2);

    RegisterAtOffsetList* allCalleeSaves = RegisterSet::vmCalleeSaveRegisterOffsets();
    const RegisterAtOffsetList* currentCalleeSaves = &RegisterAtOffsetList::llintBaselineCalleeSaveRegisters();
    RegisterSet dontCopyRegisters = RegisterSet::stackRegisters();
    unsigned registerCount = allCalleeSaves->registerCount();

    unsigned i = 0;
    for (; i < registerCount; i++) {
        RegisterAtOffset entry = allCalleeSaves->at(i);
        if (dontCopyRegisters.contains(entry.reg()))
            continue;
        RegisterAtOffset* currentFrameEntry = currentCalleeSaves->find(entry.reg());

        if (!entry.reg().isGPR())
            break;
        if (currentFrameEntry)
            spooler.loadGPR(currentFrameEntry->offset());
        else
            spooler.copyGPR(entry.reg().gpr());
        spooler.storeGPR(entry.offset());
    }
    spooler.finalizeGPR();

    for (; i < registerCount; i++) {
        RegisterAtOffset entry = allCalleeSaves->at(i);
        if (dontCopyRegisters.get(entry.reg()))
            continue;
        RegisterAtOffset* currentFrameEntry = currentCalleeSaves->find(entry.reg());

        RELEASE_ASSERT(entry.reg().isFPR());
        if (currentFrameEntry)
            spooler.loadFPR(currentFrameEntry->offset());
        else
            spooler.copyFPR(entry.reg().fpr());
        spooler.storeFPR(entry.offset());
    }
    spooler.finalizeFPR();

#else
    UNUSED_PARAM(topEntryFrame);
    UNUSED_PARAM(usedRegisters);
#endif
}

void AssemblyHelpers::emitSaveOrCopyLLIntBaselineCalleeSavesFor(CodeBlock* codeBlock, VirtualRegister offsetVirtualRegister, RestoreTagRegisterMode tagRegisterMode, GPRReg temp1, GPRReg temp2, GPRReg temp3)
{
    ASSERT_UNUSED(codeBlock, codeBlock);
    ASSERT(JITCode::isBaselineCode(codeBlock->jitType()));
    ASSERT(codeBlock->jitCode()->calleeSaveRegisters() == &RegisterAtOffsetList::llintBaselineCalleeSaveRegisters());

    const RegisterAtOffsetList* calleeSaves = &RegisterAtOffsetList::llintBaselineCalleeSaveRegisters();
    RegisterSet dontSaveRegisters = RegisterSet(RegisterSet::stackRegisters());
    unsigned registerCount = calleeSaves->registerCount();

    GPRReg dstBufferGPR = temp1;
    addPtr(TrustedImm32(offsetVirtualRegister.offsetInBytes()), framePointerRegister, dstBufferGPR);

    CopySpooler spooler(*this, framePointerRegister, dstBufferGPR, temp2, temp3);

    for (unsigned i = 0; i < registerCount; i++) {
        RegisterAtOffset entry = calleeSaves->at(i);
        if (dontSaveRegisters.get(entry.reg()))
            continue;
        RELEASE_ASSERT(entry.reg().isGPR());

#if USE(JSVALUE32_64)
        UNUSED_PARAM(tagRegisterMode);
#else
        if (tagRegisterMode == CopyBaselineCalleeSavedRegistersFromBaseFrame)
            spooler.loadGPR(entry.offset());
        else
#endif
            spooler.copyGPR(entry.reg().gpr());
        spooler.storeGPR(entry.offset());
    }
    spooler.finalizeGPR();
}

} // namespace JSC

#endif // ENABLE(JIT)