/*
 * Copyright (C) 2011-2019 Apple Inc. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
 */

#include "config.h"
#include "DFGOSRExit.h"

#if ENABLE(DFG_JIT)

#include "AssemblyHelpers.h"
#include "BytecodeUseDef.h"
#include "ClonedArguments.h"
#include "DFGGraph.h"
#include "DFGMayExit.h"
#include "DFGOSRExitCompilerCommon.h"
#include "DFGOperations.h"
#include "DFGSpeculativeJIT.h"
#include "DirectArguments.h"
#include "FrameTracers.h"
#include "InlineCallFrame.h"
#include "JSCInlines.h"
#include "JSCJSValue.h"
#include "OperandsInlines.h"
#include "ProbeContext.h"
#include "ProbeFrame.h"

namespace JSC { namespace DFG {

// Probe based OSR Exit.

using CPUState = Probe::CPUState;
using Context = Probe::Context;
using Frame = Probe::Frame;

static void reifyInlinedCallFrames(Probe::Context&, CodeBlock* baselineCodeBlock, const OSRExitBase&);
static void adjustAndJumpToTarget(Probe::Context&, VM&, CodeBlock*, CodeBlock* baselineCodeBlock, OSRExit&);
static void printOSRExit(Context&, uint32_t osrExitIndex, const OSRExit&);

static JSValue jsValueFor(CPUState& cpu, JSValueSource source)
{
    if (source.isAddress()) {
        JSValue result;
        std::memcpy(&result, cpu.gpr<uint8_t*>(source.base()) + source.offset(), sizeof(JSValue));
        return result;
    }
#if USE(JSVALUE64)
    return JSValue::decode(cpu.gpr<EncodedJSValue>(source.gpr()));
#else
    if (source.hasKnownTag())
        return JSValue(source.tag(), cpu.gpr<int32_t>(source.payloadGPR()));
    return JSValue(cpu.gpr<int32_t>(source.tagGPR()), cpu.gpr<int32_t>(source.payloadGPR()));
#endif
}

#if NUMBER_OF_CALLEE_SAVES_REGISTERS > 0

// Based on AssemblyHelpers::emitRestoreCalleeSavesFor().
static void restoreCalleeSavesFor(Context& context, CodeBlock* codeBlock)
{
    ASSERT(codeBlock);

    const RegisterAtOffsetList* calleeSaves = codeBlock->calleeSaveRegisters();
    RegisterSet dontRestoreRegisters = RegisterSet(RegisterSet::stackRegisters(), RegisterSet::allFPRs());
    unsigned registerCount = calleeSaves->size();

    UCPURegister* physicalStackFrame = context.fp<UCPURegister*>();
    for (unsigned i = 0; i < registerCount; i++) {
        RegisterAtOffset entry = calleeSaves->at(i);
        if (dontRestoreRegisters.get(entry.reg()))
            continue;
        // The callee saved values come from the original stack, not the recovered stack.
        // Hence, we read the values directly from the physical stack memory instead of
        // going through context.stack().
        ASSERT(!(entry.offset() % sizeof(UCPURegister)));
        context.gpr(entry.reg().gpr()) = physicalStackFrame[entry.offset() / sizeof(UCPURegister)];
    }
}

// Based on AssemblyHelpers::emitSaveCalleeSavesFor().
static void saveCalleeSavesFor(Context& context, CodeBlock* codeBlock)
{
    auto& stack = context.stack();
    ASSERT(codeBlock);

    const RegisterAtOffsetList* calleeSaves = codeBlock->calleeSaveRegisters();
    RegisterSet dontSaveRegisters = RegisterSet(RegisterSet::stackRegisters(), RegisterSet::allFPRs());
    unsigned registerCount = calleeSaves->size();

    for (unsigned i = 0; i < registerCount; i++) {
        RegisterAtOffset entry = calleeSaves->at(i);
        if (dontSaveRegisters.get(entry.reg()))
            continue;
        stack.set(context.fp(), entry.offset(), context.gpr<UCPURegister>(entry.reg().gpr()));
    }
}

// Based on AssemblyHelpers::restoreCalleeSavesFromVMEntryFrameCalleeSavesBuffer().
static void restoreCalleeSavesFromVMEntryFrameCalleeSavesBuffer(Context& context)
{
    VM& vm = *context.arg<VM*>();

    RegisterAtOffsetList* allCalleeSaves = RegisterSet::vmCalleeSaveRegisterOffsets();
    RegisterSet dontRestoreRegisters = RegisterSet::stackRegisters();
    unsigned registerCount = allCalleeSaves->size();

    VMEntryRecord* entryRecord = vmEntryRecord(vm.topEntryFrame);
    UCPURegister* calleeSaveBuffer = reinterpret_cast<UCPURegister*>(entryRecord->calleeSaveRegistersBuffer);

    // Restore all callee saves.
    for (unsigned i = 0; i < registerCount; i++) {
        RegisterAtOffset entry = allCalleeSaves->at(i);
        if (dontRestoreRegisters.get(entry.reg()))
            continue;
        size_t uintptrOffset = entry.offset() / sizeof(UCPURegister);
        if (entry.reg().isGPR())
            context.gpr(entry.reg().gpr()) = calleeSaveBuffer[uintptrOffset];
        else {
#if USE(JSVALUE64)
            context.fpr(entry.reg().fpr()) = bitwise_cast<double>(calleeSaveBuffer[uintptrOffset]);
#else
            // FIXME: <https://webkit.org/b/193275> support callee-saved floating point registers on 32-bit architectures
            RELEASE_ASSERT_NOT_REACHED();
#endif
        }
    }
}

// Based on AssemblyHelpers::copyCalleeSavesToVMEntryFrameCalleeSavesBuffer().
static void copyCalleeSavesToVMEntryFrameCalleeSavesBuffer(Context& context)
{
    VM& vm = *context.arg<VM*>();
    auto& stack = context.stack();

    VMEntryRecord* entryRecord = vmEntryRecord(vm.topEntryFrame);
    void* calleeSaveBuffer = entryRecord->calleeSaveRegistersBuffer;

    RegisterAtOffsetList* allCalleeSaves = RegisterSet::vmCalleeSaveRegisterOffsets();
    RegisterSet dontCopyRegisters = RegisterSet::stackRegisters();
    unsigned registerCount = allCalleeSaves->size();

    for (unsigned i = 0; i < registerCount; i++) {
        RegisterAtOffset entry = allCalleeSaves->at(i);
        if (dontCopyRegisters.get(entry.reg()))
            continue;
        if (entry.reg().isGPR())
            stack.set(calleeSaveBuffer, entry.offset(), context.gpr<UCPURegister>(entry.reg().gpr()));
        else {
#if USE(JSVALUE64)
            stack.set(calleeSaveBuffer, entry.offset(), context.fpr<UCPURegister>(entry.reg().fpr()));
#else
            // FIXME: <https://webkit.org/b/193275> support callee-saved floating point registers on 32-bit architectures
            RELEASE_ASSERT_NOT_REACHED();
#endif
        }
    }
}

// Based on AssemblyHelpers::emitSaveOrCopyCalleeSavesFor().
static void saveOrCopyCalleeSavesFor(Context& context, CodeBlock* codeBlock, VirtualRegister offsetVirtualRegister, bool wasCalledViaTailCall)
{
    Frame frame(context.fp(), context.stack());
    ASSERT(codeBlock);

    const RegisterAtOffsetList* calleeSaves = codeBlock->calleeSaveRegisters();
    RegisterSet dontSaveRegisters = RegisterSet(RegisterSet::stackRegisters(), RegisterSet::allFPRs());
    unsigned registerCount = calleeSaves->size();

    RegisterSet baselineCalleeSaves = RegisterSet::llintBaselineCalleeSaveRegisters();

    for (unsigned i = 0; i < registerCount; i++) {
        RegisterAtOffset entry = calleeSaves->at(i);
        if (dontSaveRegisters.get(entry.reg()))
            continue;

        uintptr_t savedRegisterValue;

        if (wasCalledViaTailCall && baselineCalleeSaves.get(entry.reg()))
            savedRegisterValue = frame.get<uintptr_t>(entry.offset());
        else
            savedRegisterValue = context.gpr(entry.reg().gpr());

        frame.set(offsetVirtualRegister.offsetInBytes() + entry.offset(), savedRegisterValue);
    }
}
#else // not NUMBER_OF_CALLEE_SAVES_REGISTERS > 0

static void restoreCalleeSavesFor(Context&, CodeBlock*) { }
static void saveCalleeSavesFor(Context&, CodeBlock*) { }
static void restoreCalleeSavesFromVMEntryFrameCalleeSavesBuffer(Context&) { }
static void copyCalleeSavesToVMEntryFrameCalleeSavesBuffer(Context&) { }
static void saveOrCopyCalleeSavesFor(Context&, CodeBlock*, VirtualRegister, bool) { }

#endif // NUMBER_OF_CALLEE_SAVES_REGISTERS > 0

static JSCell* createDirectArgumentsDuringExit(Context& context, CodeBlock* codeBlock, InlineCallFrame* inlineCallFrame, JSFunction* callee, int32_t argumentCount)
{
    VM& vm = *context.arg<VM*>();

    ASSERT(vm.heap.isDeferred());

    if (inlineCallFrame)
        codeBlock = baselineCodeBlockForInlineCallFrame(inlineCallFrame);

    unsigned length = argumentCount - 1;
    unsigned capacity = std::max(length, static_cast<unsigned>(codeBlock->numParameters() - 1));
    DirectArguments* result = DirectArguments::create(
        vm, codeBlock->globalObject()->directArgumentsStructure(), length, capacity);

    result->setCallee(vm, callee);

    void* frameBase = context.fp<Register*>() + (inlineCallFrame ? inlineCallFrame->stackOffset : 0);
    Frame frame(frameBase, context.stack());
    for (unsigned i = length; i--;)
        result->setIndexQuickly(vm, i, frame.argument(i));

    return result;
}

static JSCell* createClonedArgumentsDuringExit(Context& context, CodeBlock* codeBlock, InlineCallFrame* inlineCallFrame, JSFunction* callee, int32_t argumentCount)
{
    VM& vm = *context.arg<VM*>();

    ASSERT(vm.heap.isDeferred());

    if (inlineCallFrame)
        codeBlock = baselineCodeBlockForInlineCallFrame(inlineCallFrame);

    JSGlobalObject* globalObject = codeBlock->globalObject();
    unsigned length = argumentCount - 1;
    ClonedArguments* result = ClonedArguments::createEmpty(
        vm, globalObject->clonedArgumentsStructure(), callee, length);

    void* frameBase = context.fp<Register*>() + (inlineCallFrame ? inlineCallFrame->stackOffset : 0);
    Frame frame(frameBase, context.stack());
    for (unsigned i = length; i--;)
        result->putDirectIndex(globalObject, i, frame.argument(i));
    return result;
}

static void emitRestoreArguments(Context& context, CodeBlock* codeBlock, DFG::JITCode* dfgJITCode, const Operands<ValueRecovery>& operands)
{
    Frame frame(context.fp(), context.stack());

    HashMap<MinifiedID, int> alreadyAllocatedArguments; // Maps phantom arguments node ID to operand.
    for (size_t index = 0; index < operands.size(); ++index) {
        const ValueRecovery& recovery = operands[index];
        int operand = operands.operandForIndex(index);

        if (recovery.technique() != DirectArgumentsThatWereNotCreated
            && recovery.technique() != ClonedArgumentsThatWereNotCreated)
            continue;

        MinifiedID id = recovery.nodeID();
        auto iter = alreadyAllocatedArguments.find(id);
        if (iter != alreadyAllocatedArguments.end()) {
            frame.setOperand(operand, frame.operand(iter->value));
            continue;
        }

        InlineCallFrame* inlineCallFrame =
            dfgJITCode->minifiedDFG.at(id)->inlineCallFrame();

        int stackOffset;
        if (inlineCallFrame)
            stackOffset = inlineCallFrame->stackOffset;
        else
            stackOffset = 0;

        JSFunction* callee;
        if (!inlineCallFrame || inlineCallFrame->isClosureCall)
            callee = jsCast<JSFunction*>(frame.operand(stackOffset + CallFrameSlot::callee).asCell());
        else
            callee = jsCast<JSFunction*>(inlineCallFrame->calleeRecovery.constant().asCell());

        int32_t argumentCount;
        if (!inlineCallFrame || inlineCallFrame->isVarargs())
            argumentCount = frame.operand<int32_t>(stackOffset + CallFrameSlot::argumentCountIncludingThis, PayloadOffset);
        else
            argumentCount = inlineCallFrame->argumentCountIncludingThis;

        JSCell* argumentsObject;
        switch (recovery.technique()) {
        case DirectArgumentsThatWereNotCreated:
            argumentsObject = createDirectArgumentsDuringExit(context, codeBlock, inlineCallFrame, callee, argumentCount);
            break;
        case ClonedArgumentsThatWereNotCreated:
            argumentsObject = createClonedArgumentsDuringExit(context, codeBlock, inlineCallFrame, callee, argumentCount);
            break;
        default:
            RELEASE_ASSERT_NOT_REACHED();
            break;
        }
        frame.setOperand(operand, JSValue(argumentsObject));

        alreadyAllocatedArguments.add(id, operand);
    }
}

// The following is a list of extra initializations that need to be done in order
// of most likely needed (lower enum value) to least likely needed (higher enum value).
// Each level initialization includes the previous lower enum value (see use of the
// extraInitializationLevel value below).
enum class ExtraInitializationLevel {
    None,
    SpeculationRecovery,
    ValueProfileUpdate,
    ArrayProfileUpdate,
    Other
};

void OSRExit::executeOSRExit(Context& context)
{
    VM& vm = *context.arg<VM*>();
    auto scope = DECLARE_THROW_SCOPE(vm);

    CallFrame* callFrame = context.fp<CallFrame*>();
    ASSERT(&callFrame->deprecatedVM() == &vm);
    auto& cpu = context.cpu;

    if (validateDFGDoesGC) {
        // We're about to exit optimized code. So, there's no longer any optimized
        // code running that expects no GC.
        vm.heap.setExpectDoesGC(true);
    }

    if (vm.callFrameForCatch) {
        callFrame = vm.callFrameForCatch;
        context.fp() = callFrame;
    }

    CodeBlock* codeBlock = callFrame->codeBlock();
    ASSERT(codeBlock);
    ASSERT(codeBlock->jitType() == JITType::DFGJIT);

    // It's sort of preferable that we don't GC while in here. Anyways, doing so wouldn't
    // really be profitable.
    DeferGCForAWhile deferGC(vm.heap);

    uint32_t exitIndex = vm.osrExitIndex;
    DFG::JITCode* dfgJITCode = codeBlock->jitCode()->dfg();
    OSRExit& exit = dfgJITCode->osrExit[exitIndex];

    ASSERT(!vm.callFrameForCatch || exit.m_kind == GenericUnwind);
    EXCEPTION_ASSERT_UNUSED(scope, !!scope.exception() || !exit.isExceptionHandler());

    if (UNLIKELY(!exit.exitState)) {
        ExtraInitializationLevel extraInitializationLevel = ExtraInitializationLevel::None;

        // We only need to execute this block once for each OSRExit record. The computed
        // results will be cached in the OSRExitState record for use of the rest of the
        // exit ramp code.

        CodeBlock* baselineCodeBlock = codeBlock->baselineAlternative();
        ASSERT(JITCode::isBaselineCode(baselineCodeBlock->jitType()));

        SpeculationRecovery* recovery = nullptr;
        if (exit.m_recoveryIndex != UINT_MAX) {
            recovery = &dfgJITCode->speculationRecovery[exit.m_recoveryIndex];
            extraInitializationLevel = std::max(extraInitializationLevel, ExtraInitializationLevel::SpeculationRecovery);
        }

        if (UNLIKELY(exit.m_kind == GenericUnwind))
            extraInitializationLevel = std::max(extraInitializationLevel, ExtraInitializationLevel::Other);

        ArrayProfile* arrayProfile = nullptr;
        if (!!exit.m_jsValueSource) {
            if (exit.m_valueProfile)
                extraInitializationLevel = std::max(extraInitializationLevel, ExtraInitializationLevel::ValueProfileUpdate);
            if (exit.m_kind == BadCache || exit.m_kind == BadIndexingType) {
                CodeOrigin codeOrigin = exit.m_codeOriginForExitProfile;
                CodeBlock* profiledCodeBlock = baselineCodeBlockForOriginAndBaselineCodeBlock(codeOrigin, baselineCodeBlock);
                arrayProfile = profiledCodeBlock->getArrayProfile(codeOrigin.bytecodeIndex());
                if (arrayProfile)
                    extraInitializationLevel = std::max(extraInitializationLevel, ExtraInitializationLevel::ArrayProfileUpdate);
            }
        }

        int32_t activeThreshold = baselineCodeBlock->adjustedCounterValue(Options::thresholdForOptimizeAfterLongWarmUp());
        double adjustedThreshold = applyMemoryUsageHeuristicsAndConvertToInt(activeThreshold, baselineCodeBlock);
        ASSERT(adjustedThreshold > 0);
        adjustedThreshold = BaselineExecutionCounter::clippedThreshold(codeBlock->globalObject(), adjustedThreshold);

        CodeBlock* codeBlockForExit = baselineCodeBlockForOriginAndBaselineCodeBlock(exit.m_codeOrigin, baselineCodeBlock);
        bool exitToLLInt = Options::forceOSRExitToLLInt() || codeBlockForExit->jitType() == JITType::InterpreterThunk;
        void* jumpTarget;
        if (exitToLLInt) {
            BytecodeIndex bytecodeOffset = exit.m_codeOrigin.bytecodeIndex();
            const Instruction& currentInstruction = *codeBlockForExit->instructions().at(bytecodeOffset).ptr();
            MacroAssemblerCodePtr<JSEntryPtrTag> destination = LLInt::getCodePtr<JSEntryPtrTag>(currentInstruction);
            jumpTarget = destination.executableAddress();    
        } else {
            const JITCodeMap& codeMap = codeBlockForExit->jitCodeMap();
            CodeLocationLabel<JSEntryPtrTag> codeLocation = codeMap.find(exit.m_codeOrigin.bytecodeIndex());
            ASSERT(codeLocation);
            jumpTarget = codeLocation.executableAddress();
        }

        // Compute the value recoveries.
        Operands<ValueRecovery> operands;
        Vector<UndefinedOperandSpan> undefinedOperandSpans;
        dfgJITCode->variableEventStream.reconstruct(codeBlock, exit.m_codeOrigin, dfgJITCode->minifiedDFG, exit.m_streamIndex, operands, &undefinedOperandSpans);
        ptrdiff_t stackPointerOffset = -static_cast<ptrdiff_t>(codeBlock->jitCode()->dfgCommon()->requiredRegisterCountForExit) * sizeof(Register);

        exit.exitState = adoptRef(new OSRExitState(exit, codeBlock, baselineCodeBlock, operands, WTFMove(undefinedOperandSpans), recovery, stackPointerOffset, activeThreshold, adjustedThreshold, jumpTarget, arrayProfile, exitToLLInt));

        if (UNLIKELY(vm.m_perBytecodeProfiler && codeBlock->jitCode()->dfgCommon()->compilation)) {
            Profiler::Database& database = *vm.m_perBytecodeProfiler;
            Profiler::Compilation* compilation = codeBlock->jitCode()->dfgCommon()->compilation.get();

            Profiler::OSRExit* profilerExit = compilation->addOSRExit(
                exitIndex, Profiler::OriginStack(database, codeBlock, exit.m_codeOrigin),
                exit.m_kind, exit.m_kind == UncountableInvalidation);
            exit.exitState->profilerExit = profilerExit;
            extraInitializationLevel = std::max(extraInitializationLevel, ExtraInitializationLevel::Other);
        }

        if (UNLIKELY(Options::printEachOSRExit()))
            extraInitializationLevel = std::max(extraInitializationLevel, ExtraInitializationLevel::Other);

        exit.exitState->extraInitializationLevel = extraInitializationLevel;

        if (UNLIKELY(Options::verboseOSR() || Options::verboseDFGOSRExit())) {
            dataLogF("DFG OSR exit #%u (%s, %s) from %s, with operands = %s\n",
                exitIndex, toCString(exit.m_codeOrigin).data(),
                exitKindToString(exit.m_kind), toCString(*codeBlock).data(),
                toCString(ignoringContext<DumpContext>(operands)).data());
        }
    }

    OSRExitState& exitState = *exit.exitState.get();
    CodeBlock* baselineCodeBlock = exitState.baselineCodeBlock;
    ASSERT(JITCode::isBaselineCode(baselineCodeBlock->jitType()));

    Operands<ValueRecovery>& operands = exitState.operands;
    Vector<UndefinedOperandSpan>& undefinedOperandSpans = exitState.undefinedOperandSpans;

    context.sp() = context.fp<uint8_t*>() + exitState.stackPointerOffset;

    // The only reason for using this do while loop is so we can break out midway when appropriate.
    do {
        auto extraInitializationLevel = static_cast<ExtraInitializationLevel>(exitState.extraInitializationLevel);

        if (extraInitializationLevel == ExtraInitializationLevel::None)
            break;

        // Begin extra initilization level: SpeculationRecovery

        // We need to do speculation recovery first because array profiling and value profiling
        // may rely on a value that it recovers. However, that doesn't mean that it is likely
        // to have a recovery value. So, we'll decorate it as UNLIKELY.
        SpeculationRecovery* recovery = exitState.recovery;
        if (UNLIKELY(recovery)) {
            switch (recovery->type()) {
            case SpeculativeAdd:
                cpu.gpr(recovery->dest()) = cpu.gpr<uint32_t>(recovery->dest()) - cpu.gpr<uint32_t>(recovery->src());
#if USE(JSVALUE64)
                ASSERT(!(cpu.gpr(recovery->dest()) >> 32));
                cpu.gpr(recovery->dest()) |= JSValue::NumberTag;
#endif
                break;

            case SpeculativeAddSelf:
                cpu.gpr(recovery->dest()) = static_cast<uint32_t>(cpu.gpr<int32_t>(recovery->dest()) >> 1) ^ 0x80000000U;
#if USE(JSVALUE64)
                ASSERT(!(cpu.gpr(recovery->dest()) >> 32));
                cpu.gpr(recovery->dest()) |= JSValue::NumberTag;
#endif
                break;

            case SpeculativeAddImmediate:
                cpu.gpr(recovery->dest()) = (cpu.gpr<uint32_t>(recovery->dest()) - recovery->immediate());
#if USE(JSVALUE64)
                ASSERT(!(cpu.gpr(recovery->dest()) >> 32));
                cpu.gpr(recovery->dest()) |= JSValue::NumberTag;
#endif
                break;

            case BooleanSpeculationCheck:
#if USE(JSVALUE64)
                cpu.gpr(recovery->dest()) = cpu.gpr(recovery->dest()) ^ JSValue::ValueFalse;
#endif
                break;

            default:
                break;
            }
        }
        if (extraInitializationLevel <= ExtraInitializationLevel::SpeculationRecovery)
            break;

        // Begin extra initilization level: ValueProfileUpdate
        JSValue profiledValue;
        if (!!exit.m_jsValueSource) {
            profiledValue = jsValueFor(cpu, exit.m_jsValueSource);
            if (MethodOfGettingAValueProfile profile = exit.m_valueProfile)
                profile.reportValue(profiledValue);
        }
        if (extraInitializationLevel <= ExtraInitializationLevel::ValueProfileUpdate)
            break;

        // Begin extra initilization level: ArrayProfileUpdate
        if (ArrayProfile* arrayProfile = exitState.arrayProfile) {
            ASSERT(!!exit.m_jsValueSource);
            ASSERT(exit.m_kind == BadCache || exit.m_kind == BadIndexingType);
            CodeBlock* profiledCodeBlock = baselineCodeBlockForOriginAndBaselineCodeBlock(exit.m_codeOriginForExitProfile, baselineCodeBlock);
            const Instruction* instruction = profiledCodeBlock->instructions().at(exit.m_codeOriginForExitProfile.bytecodeIndex()).ptr();
            bool doProfile = !instruction->is<OpGetById>() || instruction->as<OpGetById>().metadata(profiledCodeBlock).m_modeMetadata.mode == GetByIdMode::ArrayLength;
            if (doProfile) {
                Structure* structure = profiledValue.asCell()->structure(vm);
                arrayProfile->observeStructure(structure);
                arrayProfile->observeArrayMode(arrayModesFromStructure(structure));
            }
        }
        if (extraInitializationLevel <= ExtraInitializationLevel::ArrayProfileUpdate)
            break;

        // Begin Extra initilization level: Other
        if (UNLIKELY(exit.m_kind == GenericUnwind)) {
            // We are acting as a defacto op_catch because we arrive here from genericUnwind().
            // So, we must restore our call frame and stack pointer.
            restoreCalleeSavesFromVMEntryFrameCalleeSavesBuffer(context);
            ASSERT(context.fp() == vm.callFrameForCatch);
        }

        if (exitState.profilerExit)
            exitState.profilerExit->incCount();

        if (UNLIKELY(Options::printEachOSRExit()))
            printOSRExit(context, vm.osrExitIndex, exit);

    } while (false); // End extra initialization.

    Frame frame(cpu.fp(), context.stack());
    ASSERT(!(context.fp<uintptr_t>() & 0x7));

#if USE(JSVALUE64)
    ASSERT(cpu.gpr<int64_t>(GPRInfo::numberTagRegister) == JSValue::NumberTag);
    ASSERT(cpu.gpr<int64_t>(GPRInfo::notCellMaskRegister) == JSValue::NotCellMask);
#endif

    // Do all data format conversions and store the results into the stack.
    // Note: we need to recover values before restoring callee save registers below
    // because the recovery may rely on values in some of callee save registers.

    int calleeSaveSpaceAsVirtualRegisters = static_cast<int>(baselineCodeBlock->calleeSaveSpaceAsVirtualRegisters());
    size_t numberOfOperands = operands.size();
    size_t numUndefinedOperandSpans = undefinedOperandSpans.size();

    size_t nextUndefinedSpanIndex = 0;
    size_t nextUndefinedOperandIndex = numberOfOperands;
    if (numUndefinedOperandSpans)
        nextUndefinedOperandIndex = undefinedOperandSpans[nextUndefinedSpanIndex].firstIndex;

    JSValue undefined = jsUndefined();
    for (size_t spanIndex = 0; spanIndex < numUndefinedOperandSpans; ++spanIndex) {
        auto& span = undefinedOperandSpans[spanIndex];
        int firstOffset = span.minOffset;
        int lastOffset = firstOffset + span.numberOfRegisters;

        for (int offset = firstOffset; offset < lastOffset; ++offset)
            frame.setOperand(offset, undefined);
    }

    for (size_t index = 0; index < numberOfOperands; ++index) {
        const ValueRecovery& recovery = operands[index];
        VirtualRegister reg = operands.virtualRegisterForIndex(index);

        if (UNLIKELY(index == nextUndefinedOperandIndex)) {
            index += undefinedOperandSpans[nextUndefinedSpanIndex++].numberOfRegisters - 1;
            if (nextUndefinedSpanIndex < numUndefinedOperandSpans)
                nextUndefinedOperandIndex = undefinedOperandSpans[nextUndefinedSpanIndex].firstIndex;
            else
                nextUndefinedOperandIndex = numberOfOperands;
            continue;
        }

        if (reg.isLocal() && reg.toLocal() < calleeSaveSpaceAsVirtualRegisters)
            continue;

        int operand = reg.offset();

        switch (recovery.technique()) {
        case DisplacedInJSStack:
            frame.setOperand(operand, callFrame->r(recovery.virtualRegister()).asanUnsafeJSValue());
            break;

        case InFPR:
            frame.setOperand(operand, cpu.fpr<JSValue>(recovery.fpr()));
            break;

#if USE(JSVALUE64)
        case InGPR:
            frame.setOperand(operand, cpu.gpr<JSValue>(recovery.gpr()));
            break;
#else
        case InPair:
            frame.setOperand(operand, JSValue(cpu.gpr<int32_t>(recovery.tagGPR()), cpu.gpr<int32_t>(recovery.payloadGPR())));
            break;
#endif

        case UnboxedCellInGPR:
            frame.setOperand(operand, JSValue(cpu.gpr<JSCell*>(recovery.gpr())));
            break;

        case CellDisplacedInJSStack:
            frame.setOperand(operand, JSValue(callFrame->r(recovery.virtualRegister()).asanUnsafeUnboxedCell()));
            break;

#if USE(JSVALUE32_64)
        case UnboxedBooleanInGPR:
            frame.setOperand(operand, jsBoolean(cpu.gpr<bool>(recovery.gpr())));
            break;
#endif

        case BooleanDisplacedInJSStack:
#if USE(JSVALUE64)
            frame.setOperand(operand, callFrame->r(recovery.virtualRegister()).asanUnsafeJSValue());
#else
            frame.setOperand(operand, jsBoolean(callFrame->r(recovery.virtualRegister()).asanUnsafeJSValue().payload()));
#endif
            break;

        case UnboxedInt32InGPR:
            frame.setOperand(operand, JSValue(cpu.gpr<int32_t>(recovery.gpr())));
            break;

        case Int32DisplacedInJSStack:
            frame.setOperand(operand, JSValue(callFrame->r(recovery.virtualRegister()).asanUnsafeUnboxedInt32()));
            break;

#if USE(JSVALUE64)
        case UnboxedInt52InGPR:
            frame.setOperand(operand, JSValue(cpu.gpr<int64_t>(recovery.gpr()) >> JSValue::int52ShiftAmount));
            break;

        case Int52DisplacedInJSStack:
            frame.setOperand(operand, JSValue(callFrame->r(recovery.virtualRegister()).asanUnsafeUnboxedInt52()));
            break;

        case UnboxedStrictInt52InGPR:
            frame.setOperand(operand, JSValue(cpu.gpr<int64_t>(recovery.gpr())));
            break;

        case StrictInt52DisplacedInJSStack:
            frame.setOperand(operand, JSValue(callFrame->r(recovery.virtualRegister()).asanUnsafeUnboxedStrictInt52()));
            break;
#endif

        case UnboxedDoubleInFPR:
            frame.setOperand(operand, JSValue(JSValue::EncodeAsDouble, purifyNaN(cpu.fpr(recovery.fpr()))));
            break;

        case DoubleDisplacedInJSStack:
            frame.setOperand(operand, JSValue(JSValue::EncodeAsDouble, purifyNaN(callFrame->r(recovery.virtualRegister()).asanUnsafeUnboxedDouble())));
            break;

        case Constant:
            frame.setOperand(operand, recovery.constant());
            break;

        case DirectArgumentsThatWereNotCreated:
        case ClonedArgumentsThatWereNotCreated:
            // Don't do this, yet.
            break;

        default:
            RELEASE_ASSERT_NOT_REACHED();
            break;
        }
    }

    // Restore the DFG callee saves and then save the ones the baseline JIT uses.
    restoreCalleeSavesFor(context, codeBlock);
    saveCalleeSavesFor(context, baselineCodeBlock);

#if USE(JSVALUE64)
    cpu.gpr(GPRInfo::numberTagRegister) = static_cast<JSC::UCPURegister>(JSValue::NumberTag);
    cpu.gpr(GPRInfo::notCellMaskRegister) = static_cast<JSC::UCPURegister>(JSValue::NotCellMask);
#endif

    if (exit.isExceptionHandler())
        copyCalleeSavesToVMEntryFrameCalleeSavesBuffer(context);

    // Now that things on the stack are recovered, do the arguments recovery. We assume that arguments
    // recoveries don't recursively refer to each other. But, we don't try to assume that they only
    // refer to certain ranges of locals. Hence why we need to do this here, once the stack is sensible.
    // Note that we also roughly assume that the arguments might still be materialized outside of its
    // inline call frame scope - but for now the DFG wouldn't do that.

    DFG::emitRestoreArguments(context, codeBlock, dfgJITCode, operands);

    // Adjust the old JIT's execute counter. Since we are exiting OSR, we know
    // that all new calls into this code will go to the new JIT, so the execute
    // counter only affects call frames that performed OSR exit and call frames
    // that were still executing the old JIT at the time of another call frame's
    // OSR exit. We want to ensure that the following is true:
    //
    // (a) Code the performs an OSR exit gets a chance to reenter optimized
    //     code eventually, since optimized code is faster. But we don't
    //     want to do such reentery too aggressively (see (c) below).
    //
    // (b) If there is code on the call stack that is still running the old
    //     JIT's code and has never OSR'd, then it should get a chance to
    //     perform OSR entry despite the fact that we've exited.
    //
    // (c) Code the performs an OSR exit should not immediately retry OSR
    //     entry, since both forms of OSR are expensive. OSR entry is
    //     particularly expensive.
    //
    // (d) Frequent OSR failures, even those that do not result in the code
    //     running in a hot loop, result in recompilation getting triggered.
    //
    // To ensure (c), we'd like to set the execute counter to
    // counterValueForOptimizeAfterWarmUp(). This seems like it would endanger
    // (a) and (b), since then every OSR exit would delay the opportunity for
    // every call frame to perform OSR entry. Essentially, if OSR exit happens
    // frequently and the function has few loops, then the counter will never
    // become non-negative and OSR entry will never be triggered. OSR entry
    // will only happen if a loop gets hot in the old JIT, which does a pretty
    // good job of ensuring (a) and (b). But that doesn't take care of (d),
    // since each speculation failure would reset the execute counter.
    // So we check here if the number of speculation failures is significantly
    // larger than the number of successes (we want 90% success rate), and if
    // there have been a large enough number of failures. If so, we set the
    // counter to 0; otherwise we set the counter to
    // counterValueForOptimizeAfterWarmUp().

    if (UNLIKELY(codeBlock->updateOSRExitCounterAndCheckIfNeedToReoptimize(exitState) == CodeBlock::OptimizeAction::ReoptimizeNow))
        triggerReoptimizationNow(baselineCodeBlock, codeBlock, &exit);

    reifyInlinedCallFrames(context, baselineCodeBlock, exit);
    adjustAndJumpToTarget(context, vm, codeBlock, baselineCodeBlock, exit);
}

static void reifyInlinedCallFrames(Context& context, CodeBlock* outermostBaselineCodeBlock, const OSRExitBase& exit)
{
    auto& cpu = context.cpu;
    Frame frame(cpu.fp(), context.stack());

    // FIXME: We shouldn't leave holes on the stack when performing an OSR exit
    // in presence of inlined tail calls.
    // https://bugs.webkit.org/show_bug.cgi?id=147511
    ASSERT(JITCode::isBaselineCode(outermostBaselineCodeBlock->jitType()));
    frame.setOperand<CodeBlock*>(CallFrameSlot::codeBlock, outermostBaselineCodeBlock);

    const CodeOrigin* codeOrigin;
    for (codeOrigin = &exit.m_codeOrigin; codeOrigin && codeOrigin->inlineCallFrame(); codeOrigin = codeOrigin->inlineCallFrame()->getCallerSkippingTailCalls()) {
        InlineCallFrame* inlineCallFrame = codeOrigin->inlineCallFrame();
        CodeBlock* baselineCodeBlock = baselineCodeBlockForOriginAndBaselineCodeBlock(*codeOrigin, outermostBaselineCodeBlock);
        InlineCallFrame::Kind trueCallerCallKind;
        CodeOrigin* trueCaller = inlineCallFrame->getCallerSkippingTailCalls(&trueCallerCallKind);
        void* callerFrame = cpu.fp();

        bool callerIsLLInt = false;

        if (!trueCaller) {
            ASSERT(inlineCallFrame->isTail());
            void* returnPC = frame.get<void*>(CallFrame::returnPCOffset());
#if CPU(ARM64E)
            void* oldEntrySP = cpu.fp<uint8_t*>() + sizeof(CallerFrameAndPC);
            void* newEntrySP = cpu.fp<uint8_t*>() + inlineCallFrame->returnPCOffset() + sizeof(void*);
            returnPC = retagCodePtr(returnPC, bitwise_cast<PtrTag>(oldEntrySP), bitwise_cast<PtrTag>(newEntrySP));
#endif
            frame.set<void*>(inlineCallFrame->returnPCOffset(), returnPC);
            callerFrame = frame.get<void*>(CallFrame::callerFrameOffset());
        } else {
            CodeBlock* baselineCodeBlockForCaller = baselineCodeBlockForOriginAndBaselineCodeBlock(*trueCaller, outermostBaselineCodeBlock);
            BytecodeIndex callBytecodeIndex = trueCaller->bytecodeIndex();
            void* jumpTarget = callerReturnPC(baselineCodeBlockForCaller, callBytecodeIndex, trueCallerCallKind, callerIsLLInt).untaggedExecutableAddress();

            if (trueCaller->inlineCallFrame())
                callerFrame = cpu.fp<uint8_t*>() + trueCaller->inlineCallFrame()->stackOffset * sizeof(EncodedJSValue);

#if CPU(ARM64E)
            void* newEntrySP = cpu.fp<uint8_t*>() + inlineCallFrame->returnPCOffset() + sizeof(void*);
            jumpTarget = tagCodePtr(jumpTarget, bitwise_cast<PtrTag>(newEntrySP));
#endif
            frame.set<void*>(inlineCallFrame->returnPCOffset(), jumpTarget);
        }

        frame.setOperand<void*>(inlineCallFrame->stackOffset + CallFrameSlot::codeBlock, baselineCodeBlock);

        // Restore the inline call frame's callee save registers.
        // If this inlined frame is a tail call that will return back to the original caller, we need to
        // copy the prior contents of the tag registers already saved for the outer frame to this frame.
        saveOrCopyCalleeSavesFor(context, baselineCodeBlock, VirtualRegister(inlineCallFrame->stackOffset), !trueCaller);

        if (callerIsLLInt) {
            CodeBlock* baselineCodeBlockForCaller = baselineCodeBlockForOriginAndBaselineCodeBlock(*trueCaller, outermostBaselineCodeBlock);
            frame.set<const void*>(calleeSaveSlot(inlineCallFrame, baselineCodeBlock, LLInt::Registers::metadataTableGPR).offset, baselineCodeBlockForCaller->metadataTable());
#if USE(JSVALUE64)
            frame.set<const void*>(calleeSaveSlot(inlineCallFrame, baselineCodeBlock, LLInt::Registers::pbGPR).offset, baselineCodeBlockForCaller->instructionsRawPointer());
#endif
        }

        if (!inlineCallFrame->isVarargs())
            frame.setOperand<uint32_t>(inlineCallFrame->stackOffset + CallFrameSlot::argumentCountIncludingThis, PayloadOffset, inlineCallFrame->argumentCountIncludingThis);
        ASSERT(callerFrame);
        frame.set<void*>(inlineCallFrame->callerFrameOffset(), callerFrame);
#if USE(JSVALUE64)
        uint32_t locationBits = CallSiteIndex(codeOrigin->bytecodeIndex()).bits();
        frame.setOperand<uint32_t>(inlineCallFrame->stackOffset + CallFrameSlot::argumentCountIncludingThis, TagOffset, locationBits);
        if (!inlineCallFrame->isClosureCall)
            frame.setOperand(inlineCallFrame->stackOffset + CallFrameSlot::callee, JSValue(inlineCallFrame->calleeConstant()));
#else // USE(JSVALUE64) // so this is the 32-bit part
        const Instruction* instruction = baselineCodeBlock->instructions().at(codeOrigin->bytecodeIndex()).ptr();
        uint32_t locationBits = CallSiteIndex(BytecodeIndex(bitwise_cast<uint32_t>(instruction))).bits();
        frame.setOperand<uint32_t>(inlineCallFrame->stackOffset + CallFrameSlot::argumentCountIncludingThis, TagOffset, locationBits);
        frame.setOperand<uint32_t>(inlineCallFrame->stackOffset + CallFrameSlot::callee, TagOffset, static_cast<uint32_t>(JSValue::CellTag));
        if (!inlineCallFrame->isClosureCall)
            frame.setOperand(inlineCallFrame->stackOffset + CallFrameSlot::callee, PayloadOffset, inlineCallFrame->calleeConstant());
#endif // USE(JSVALUE64) // ending the #else part, so directly above is the 32-bit part
    }

    // Don't need to set the toplevel code origin if we only did inline tail calls
    if (codeOrigin) {
#if USE(JSVALUE64)
        uint32_t locationBits = CallSiteIndex(codeOrigin->bytecodeIndex()).bits();
#else
        const Instruction* instruction = outermostBaselineCodeBlock->instructions().at(codeOrigin->bytecodeIndex()).ptr();
        uint32_t locationBits = CallSiteIndex(BytecodeIndex(bitwise_cast<uint32_t>(instruction))).bits();
#endif
        frame.setOperand<uint32_t>(CallFrameSlot::argumentCountIncludingThis, TagOffset, locationBits);
    }
}

static void adjustAndJumpToTarget(Context& context, VM& vm, CodeBlock* codeBlock, CodeBlock* baselineCodeBlock, OSRExit& exit)
{
    OSRExitState* exitState = exit.exitState.get();

    WTF::storeLoadFence(); // The optimizing compiler expects that the OSR exit mechanism will execute this fence.
    vm.heap.writeBarrier(baselineCodeBlock);

    // We barrier all inlined frames -- and not just the current inline stack --
    // because we don't know which inlined function owns the value profile that
    // we'll update when we exit. In the case of "f() { a(); b(); }", if both
    // a and b are inlined, we might exit inside b due to a bad value loaded
    // from a.
    // FIXME: MethodOfGettingAValueProfile should remember which CodeBlock owns
    // the value profile.
    InlineCallFrameSet* inlineCallFrames = codeBlock->jitCode()->dfgCommon()->inlineCallFrames.get();
    if (inlineCallFrames) {
        for (InlineCallFrame* inlineCallFrame : *inlineCallFrames)
            vm.heap.writeBarrier(inlineCallFrame->baselineCodeBlock.get());
    }

    auto* exitInlineCallFrame = exit.m_codeOrigin.inlineCallFrame();
    if (exitInlineCallFrame)
        context.fp() = context.fp<uint8_t*>() + exitInlineCallFrame->stackOffset * sizeof(EncodedJSValue);

    void* jumpTarget = exitState->jumpTarget;
    ASSERT(jumpTarget);

    if (exit.isExceptionHandler()) {
        // Since we're jumping to op_catch, we need to set callFrameForCatch.
        vm.callFrameForCatch = context.fp<CallFrame*>();
    }

    vm.topCallFrame = context.fp<CallFrame*>();

    if (exitState->isJumpToLLInt) {
        CodeBlock* codeBlockForExit = baselineCodeBlockForOriginAndBaselineCodeBlock(exit.m_codeOrigin, baselineCodeBlock);
        BytecodeIndex bytecodeIndex = exit.m_codeOrigin.bytecodeIndex();
        const Instruction& currentInstruction = *codeBlockForExit->instructions().at(bytecodeIndex).ptr();

        context.gpr(LLInt::Registers::metadataTableGPR) = bitwise_cast<uintptr_t>(codeBlockForExit->metadataTable());
#if USE(JSVALUE64)
        context.gpr(LLInt::Registers::pbGPR) = bitwise_cast<uintptr_t>(codeBlockForExit->instructionsRawPointer());
        context.gpr(LLInt::Registers::pcGPR) = static_cast<uintptr_t>(exit.m_codeOrigin.bytecodeIndex().offset());
#else
        context.gpr(LLInt::Registers::pcGPR) = bitwise_cast<uintptr_t>(&currentInstruction);
#endif

        if (exit.isExceptionHandler())
            vm.targetInterpreterPCForThrow = &currentInstruction;
    }

    context.pc() = untagCodePtr<JSEntryPtrTag>(jumpTarget);
}

static void printOSRExit(Context& context, uint32_t osrExitIndex, const OSRExit& exit)
{
    CallFrame* callFrame = context.fp<CallFrame*>();
    CodeBlock* codeBlock = callFrame->codeBlock();
    CodeBlock* alternative = codeBlock->alternative();
    ExitKind kind = exit.m_kind;
    BytecodeIndex bytecodeOffset = exit.m_codeOrigin.bytecodeIndex();

    dataLog("Speculation failure in ", *codeBlock);
    dataLog(" @ exit #", osrExitIndex, " (", bytecodeOffset, ", ", exitKindToString(kind), ") with ");
    if (alternative) {
        dataLog(
            "executeCounter = ", alternative->jitExecuteCounter(),
            ", reoptimizationRetryCounter = ", alternative->reoptimizationRetryCounter(),
            ", optimizationDelayCounter = ", alternative->optimizationDelayCounter());
    } else
        dataLog("no alternative code block (i.e. we've been jettisoned)");
    dataLog(", osrExitCounter = ", codeBlock->osrExitCounter(), "\n");
    dataLog("    GPRs at time of exit:");
    for (unsigned i = 0; i < GPRInfo::numberOfRegisters; ++i) {
        GPRReg gpr = GPRInfo::toRegister(i);
        dataLog(" ", context.gprName(gpr), ":", RawPointer(context.gpr<void*>(gpr)));
    }
    dataLog("\n");
    dataLog("    FPRs at time of exit:");
    for (unsigned i = 0; i < FPRInfo::numberOfRegisters; ++i) {
        FPRReg fpr = FPRInfo::toRegister(i);
        dataLog(" ", context.fprName(fpr), ":");
        uint64_t bits = context.fpr<uint64_t>(fpr);
        double value = context.fpr(fpr);
        dataLogF("%llx:%lf", static_cast<long long>(bits), value);
    }
    dataLog("\n");
}

// JIT based OSR Exit.

OSRExit::OSRExit(ExitKind kind, JSValueSource jsValueSource, MethodOfGettingAValueProfile valueProfile, SpeculativeJIT* jit, unsigned streamIndex, unsigned recoveryIndex)
    : OSRExitBase(kind, jit->m_origin.forExit, jit->m_origin.semantic, jit->m_origin.wasHoisted)
    , m_jsValueSource(jsValueSource)
    , m_valueProfile(valueProfile)
    , m_recoveryIndex(recoveryIndex)
    , m_streamIndex(streamIndex)
{
    bool canExit = jit->m_origin.exitOK;
    if (!canExit && jit->m_currentNode) {
        ExitMode exitMode = mayExit(jit->m_jit.graph(), jit->m_currentNode);
        canExit = exitMode == ExitMode::Exits || exitMode == ExitMode::ExitsForExceptions;
    }
    DFG_ASSERT(jit->m_jit.graph(), jit->m_currentNode, canExit);
}

CodeLocationJump<JSInternalPtrTag> OSRExit::codeLocationForRepatch() const
{
    return CodeLocationJump<JSInternalPtrTag>(m_patchableJumpLocation);
}

void OSRExit::emitRestoreArguments(CCallHelpers& jit, VM& vm, const Operands<ValueRecovery>& operands)
{
    HashMap<MinifiedID, int> alreadyAllocatedArguments; // Maps phantom arguments node ID to operand.
    for (size_t index = 0; index < operands.size(); ++index) {
        const ValueRecovery& recovery = operands[index];
        int operand = operands.operandForIndex(index);

        if (recovery.technique() != DirectArgumentsThatWereNotCreated
            && recovery.technique() != ClonedArgumentsThatWereNotCreated)
            continue;

        MinifiedID id = recovery.nodeID();
        auto iter = alreadyAllocatedArguments.find(id);
        if (iter != alreadyAllocatedArguments.end()) {
            JSValueRegs regs = JSValueRegs::withTwoAvailableRegs(GPRInfo::regT0, GPRInfo::regT1);
            jit.loadValue(CCallHelpers::addressFor(iter->value), regs);
            jit.storeValue(regs, CCallHelpers::addressFor(operand));
            continue;
        }

        InlineCallFrame* inlineCallFrame =
            jit.codeBlock()->jitCode()->dfg()->minifiedDFG.at(id)->inlineCallFrame();

        int stackOffset;
        if (inlineCallFrame)
            stackOffset = inlineCallFrame->stackOffset;
        else
            stackOffset = 0;

        if (!inlineCallFrame || inlineCallFrame->isClosureCall) {
            jit.loadPtr(
                AssemblyHelpers::addressFor(stackOffset + CallFrameSlot::callee),
                GPRInfo::regT0);
        } else {
            jit.move(
                AssemblyHelpers::TrustedImmPtr(inlineCallFrame->calleeRecovery.constant().asCell()),
                GPRInfo::regT0);
        }

        if (!inlineCallFrame || inlineCallFrame->isVarargs()) {
            jit.load32(
                AssemblyHelpers::payloadFor(stackOffset + CallFrameSlot::argumentCountIncludingThis),
                GPRInfo::regT1);
        } else {
            jit.move(
                AssemblyHelpers::TrustedImm32(inlineCallFrame->argumentCountIncludingThis),
                GPRInfo::regT1);
        }

        static_assert(std::is_same<decltype(operationCreateDirectArgumentsDuringExit), decltype(operationCreateClonedArgumentsDuringExit)>::value, "We assume these functions have the same signature below.");
        jit.setupArguments<decltype(operationCreateDirectArgumentsDuringExit)>(
            AssemblyHelpers::TrustedImmPtr(&vm), AssemblyHelpers::TrustedImmPtr(inlineCallFrame), GPRInfo::regT0, GPRInfo::regT1);
        jit.prepareCallOperation(vm);
        switch (recovery.technique()) {
        case DirectArgumentsThatWereNotCreated:
            jit.move(AssemblyHelpers::TrustedImmPtr(tagCFunctionPtr<OperationPtrTag>(operationCreateDirectArgumentsDuringExit)), GPRInfo::nonArgGPR0);
            break;
        case ClonedArgumentsThatWereNotCreated:
            jit.move(AssemblyHelpers::TrustedImmPtr(tagCFunctionPtr<OperationPtrTag>(operationCreateClonedArgumentsDuringExit)), GPRInfo::nonArgGPR0);
            break;
        default:
            RELEASE_ASSERT_NOT_REACHED();
            break;
        }
        jit.call(GPRInfo::nonArgGPR0, OperationPtrTag);
        jit.storeCell(GPRInfo::returnValueGPR, AssemblyHelpers::addressFor(operand));

        alreadyAllocatedArguments.add(id, operand);
    }
}

void JIT_OPERATION operationCompileOSRExit(CallFrame* callFrame)
{
    VM& vm = callFrame->deprecatedVM();
    auto scope = DECLARE_THROW_SCOPE(vm);

    if (validateDFGDoesGC) {
        // We're about to exit optimized code. So, there's no longer any optimized
        // code running that expects no GC.
        vm.heap.setExpectDoesGC(true);
    }

    if (vm.callFrameForCatch)
        RELEASE_ASSERT(vm.callFrameForCatch == callFrame);

    CodeBlock* codeBlock = callFrame->codeBlock();
    ASSERT(codeBlock);
    ASSERT(codeBlock->jitType() == JITType::DFGJIT);

    // It's sort of preferable that we don't GC while in here. Anyways, doing so wouldn't
    // really be profitable.
    DeferGCForAWhile deferGC(vm.heap);

    uint32_t exitIndex = vm.osrExitIndex;
    OSRExit& exit = codeBlock->jitCode()->dfg()->osrExit[exitIndex];

    ASSERT(!vm.callFrameForCatch || exit.m_kind == GenericUnwind);
    EXCEPTION_ASSERT_UNUSED(scope, !!scope.exception() || !exit.isExceptionHandler());
    
    // Compute the value recoveries.
    Operands<ValueRecovery> operands;
    codeBlock->jitCode()->dfg()->variableEventStream.reconstruct(codeBlock, exit.m_codeOrigin, codeBlock->jitCode()->dfg()->minifiedDFG, exit.m_streamIndex, operands);

    SpeculationRecovery* recovery = 0;
    if (exit.m_recoveryIndex != UINT_MAX)
        recovery = &codeBlock->jitCode()->dfg()->speculationRecovery[exit.m_recoveryIndex];

    {
        CCallHelpers jit(codeBlock);

        if (exit.m_kind == GenericUnwind) {
            // We are acting as a defacto op_catch because we arrive here from genericUnwind().
            // So, we must restore our call frame and stack pointer.
            jit.restoreCalleeSavesFromEntryFrameCalleeSavesBuffer(vm.topEntryFrame);
            jit.loadPtr(vm.addressOfCallFrameForCatch(), GPRInfo::callFrameRegister);
        }
        jit.addPtr(
            CCallHelpers::TrustedImm32(codeBlock->stackPointerOffset() * sizeof(Register)),
            GPRInfo::callFrameRegister, CCallHelpers::stackPointerRegister);

        jit.jitAssertHasValidCallFrame();

        if (UNLIKELY(vm.m_perBytecodeProfiler && codeBlock->jitCode()->dfgCommon()->compilation)) {
            Profiler::Database& database = *vm.m_perBytecodeProfiler;
            Profiler::Compilation* compilation = codeBlock->jitCode()->dfgCommon()->compilation.get();

            Profiler::OSRExit* profilerExit = compilation->addOSRExit(
                exitIndex, Profiler::OriginStack(database, codeBlock, exit.m_codeOrigin),
                exit.m_kind, exit.m_kind == UncountableInvalidation);
            jit.add64(CCallHelpers::TrustedImm32(1), CCallHelpers::AbsoluteAddress(profilerExit->counterAddress()));
        }

        OSRExit::compileExit(jit, vm, exit, operands, recovery);

        LinkBuffer patchBuffer(jit, codeBlock);
        exit.m_code = FINALIZE_CODE_IF(
            shouldDumpDisassembly() || Options::verboseOSR() || Options::verboseDFGOSRExit(),
            patchBuffer, OSRExitPtrTag,
            "DFG OSR exit #%u (%s, %s) from %s, with operands = %s",
                exitIndex, toCString(exit.m_codeOrigin).data(),
                exitKindToString(exit.m_kind), toCString(*codeBlock).data(),
                toCString(ignoringContext<DumpContext>(operands)).data());
    }

    MacroAssembler::repatchJump(exit.codeLocationForRepatch(), CodeLocationLabel<OSRExitPtrTag>(exit.m_code.code()));

    vm.osrExitJumpDestination = exit.m_code.code().executableAddress();
}

void OSRExit::compileExit(CCallHelpers& jit, VM& vm, const OSRExit& exit, const Operands<ValueRecovery>& operands, SpeculationRecovery* recovery)
{
    jit.jitAssertTagsInPlace();

    // Pro-forma stuff.
    if (Options::printEachOSRExit()) {
        SpeculationFailureDebugInfo* debugInfo = new SpeculationFailureDebugInfo;
        debugInfo->codeBlock = jit.codeBlock();
        debugInfo->kind = exit.m_kind;
        debugInfo->bytecodeIndex = exit.m_codeOrigin.bytecodeIndex();

        jit.debugCall(vm, operationDebugPrintSpeculationFailure, debugInfo);
    }

    // Perform speculation recovery. This only comes into play when an operation
    // starts mutating state before verifying the speculation it has already made.

    if (recovery) {
        switch (recovery->type()) {
        case SpeculativeAdd:
            jit.sub32(recovery->src(), recovery->dest());
#if USE(JSVALUE64)
            jit.or64(GPRInfo::numberTagRegister, recovery->dest());
#endif
            break;

        case SpeculativeAddSelf:
            // If A + A = A (int32_t) overflows, A can be recovered by ((static_cast<int32_t>(A) >> 1) ^ 0x8000000).
            jit.rshift32(AssemblyHelpers::TrustedImm32(1), recovery->dest());
            jit.xor32(AssemblyHelpers::TrustedImm32(0x80000000), recovery->dest());
#if USE(JSVALUE64)
            jit.or64(GPRInfo::numberTagRegister, recovery->dest());
#endif
            break;

        case SpeculativeAddImmediate:
            jit.sub32(AssemblyHelpers::Imm32(recovery->immediate()), recovery->dest());
#if USE(JSVALUE64)
            jit.or64(GPRInfo::numberTagRegister, recovery->dest());
#endif
            break;

        case BooleanSpeculationCheck:
#if USE(JSVALUE64)
            jit.xor64(AssemblyHelpers::TrustedImm32(JSValue::ValueFalse), recovery->dest());
#endif
            break;

        default:
            break;
        }
    }

    // Refine some array and/or value profile, if appropriate.

    if (!!exit.m_jsValueSource) {
        if (exit.m_kind == BadCache || exit.m_kind == BadIndexingType) {
            // If the instruction that this originated from has an array profile, then
            // refine it. If it doesn't, then do nothing. The latter could happen for
            // hoisted checks, or checks emitted for operations that didn't have array
            // profiling - either ops that aren't array accesses at all, or weren't
            // known to be array acceses in the bytecode. The latter case is a FIXME
            // while the former case is an outcome of a CheckStructure not knowing why
            // it was emitted (could be either due to an inline cache of a property
            // property access, or due to an array profile).

            CodeOrigin codeOrigin = exit.m_codeOriginForExitProfile;
            CodeBlock* codeBlock = jit.baselineCodeBlockFor(codeOrigin);
            if (ArrayProfile* arrayProfile = codeBlock->getArrayProfile(codeOrigin.bytecodeIndex())) {
                const Instruction* instruction = codeBlock->instructions().at(codeOrigin.bytecodeIndex()).ptr();
                CCallHelpers::Jump skipProfile;
                if (instruction->is<OpGetById>()) {
                    auto& metadata = instruction->as<OpGetById>().metadata(codeBlock);
                    skipProfile = jit.branch8(CCallHelpers::NotEqual, CCallHelpers::AbsoluteAddress(&metadata.m_modeMetadata.mode), CCallHelpers::TrustedImm32(static_cast<uint8_t>(GetByIdMode::ArrayLength)));
                }

#if USE(JSVALUE64)
                GPRReg usedRegister;
                if (exit.m_jsValueSource.isAddress())
                    usedRegister = exit.m_jsValueSource.base();
                else
                    usedRegister = exit.m_jsValueSource.gpr();
#else
                GPRReg usedRegister1;
                GPRReg usedRegister2;
                if (exit.m_jsValueSource.isAddress()) {
                    usedRegister1 = exit.m_jsValueSource.base();
                    usedRegister2 = InvalidGPRReg;
                } else {
                    usedRegister1 = exit.m_jsValueSource.payloadGPR();
                    if (exit.m_jsValueSource.hasKnownTag())
                        usedRegister2 = InvalidGPRReg;
                    else
                        usedRegister2 = exit.m_jsValueSource.tagGPR();
                }
#endif

                GPRReg scratch1;
                GPRReg scratch2;
#if USE(JSVALUE64)
                scratch1 = AssemblyHelpers::selectScratchGPR(usedRegister);
                scratch2 = AssemblyHelpers::selectScratchGPR(usedRegister, scratch1);
#else
                scratch1 = AssemblyHelpers::selectScratchGPR(usedRegister1, usedRegister2);
                scratch2 = AssemblyHelpers::selectScratchGPR(usedRegister1, usedRegister2, scratch1);
#endif

                if (isARM64()) {
                    jit.pushToSave(scratch1);
                    jit.pushToSave(scratch2);
                } else {
                    jit.push(scratch1);
                    jit.push(scratch2);
                }

                GPRReg value;
                if (exit.m_jsValueSource.isAddress()) {
                    value = scratch1;
                    jit.loadPtr(AssemblyHelpers::Address(exit.m_jsValueSource.asAddress()), value);
                } else
                    value = exit.m_jsValueSource.payloadGPR();

                jit.load32(AssemblyHelpers::Address(value, JSCell::structureIDOffset()), scratch1);
                jit.store32(scratch1, arrayProfile->addressOfLastSeenStructureID());

                jit.load8(AssemblyHelpers::Address(value, JSCell::typeInfoTypeOffset()), scratch2);
                jit.sub32(AssemblyHelpers::TrustedImm32(FirstTypedArrayType), scratch2);
                auto notTypedArray = jit.branch32(MacroAssembler::AboveOrEqual, scratch2, AssemblyHelpers::TrustedImm32(NumberOfTypedArrayTypesExcludingDataView));
                jit.move(AssemblyHelpers::TrustedImmPtr(typedArrayModes), scratch1);
                jit.load32(AssemblyHelpers::BaseIndex(scratch1, scratch2, AssemblyHelpers::TimesFour), scratch2);
                auto storeArrayModes = jit.jump();

                notTypedArray.link(&jit);
#if USE(JSVALUE64)
                jit.load8(AssemblyHelpers::Address(value, JSCell::indexingTypeAndMiscOffset()), scratch1);
#else
                jit.load8(AssemblyHelpers::Address(scratch1, Structure::indexingModeIncludingHistoryOffset()), scratch1);
#endif
                jit.and32(AssemblyHelpers::TrustedImm32(IndexingModeMask), scratch1);
                jit.move(AssemblyHelpers::TrustedImm32(1), scratch2);
                jit.lshift32(scratch1, scratch2);
                storeArrayModes.link(&jit);
                jit.or32(scratch2, AssemblyHelpers::AbsoluteAddress(arrayProfile->addressOfArrayModes()));

                if (isARM64()) {
                    jit.popToRestore(scratch2);
                    jit.popToRestore(scratch1);
                } else {
                    jit.pop(scratch2);
                    jit.pop(scratch1);
                }

                if (skipProfile.isSet())
                    skipProfile.link(&jit);
            }
        }

        if (MethodOfGettingAValueProfile profile = exit.m_valueProfile) {
#if USE(JSVALUE64)
            if (exit.m_jsValueSource.isAddress()) {
                // We can't be sure that we have a spare register. So use the numberTagRegister,
                // since we know how to restore it.
                jit.load64(AssemblyHelpers::Address(exit.m_jsValueSource.asAddress()), GPRInfo::numberTagRegister);
                profile.emitReportValue(jit, JSValueRegs(GPRInfo::numberTagRegister));
                jit.move(AssemblyHelpers::TrustedImm64(JSValue::NumberTag), GPRInfo::numberTagRegister);
            } else
                profile.emitReportValue(jit, JSValueRegs(exit.m_jsValueSource.gpr()));
#else // not USE(JSVALUE64)
            if (exit.m_jsValueSource.isAddress()) {
                // Save a register so we can use it.
                GPRReg scratchPayload = AssemblyHelpers::selectScratchGPR(exit.m_jsValueSource.base());
                GPRReg scratchTag = AssemblyHelpers::selectScratchGPR(exit.m_jsValueSource.base(), scratchPayload);
                jit.pushToSave(scratchPayload);
                jit.pushToSave(scratchTag);

                JSValueRegs scratch(scratchTag, scratchPayload);
                
                jit.loadValue(exit.m_jsValueSource.asAddress(), scratch);
                profile.emitReportValue(jit, scratch);
                
                jit.popToRestore(scratchTag);
                jit.popToRestore(scratchPayload);
            } else if (exit.m_jsValueSource.hasKnownTag()) {
                GPRReg scratchTag = AssemblyHelpers::selectScratchGPR(exit.m_jsValueSource.payloadGPR());
                jit.pushToSave(scratchTag);
                jit.move(AssemblyHelpers::TrustedImm32(exit.m_jsValueSource.tag()), scratchTag);
                JSValueRegs value(scratchTag, exit.m_jsValueSource.payloadGPR());
                profile.emitReportValue(jit, value);
                jit.popToRestore(scratchTag);
            } else
                profile.emitReportValue(jit, exit.m_jsValueSource.regs());
#endif // USE(JSVALUE64)
        }
    }

    // What follows is an intentionally simple OSR exit implementation that generates
    // fairly poor code but is very easy to hack. In particular, it dumps all state that
    // needs conversion into a scratch buffer so that in step 6, where we actually do the
    // conversions, we know that all temp registers are free to use and the variable is
    // definitely in a well-known spot in the scratch buffer regardless of whether it had
    // originally been in a register or spilled. This allows us to decouple "where was
    // the variable" from "how was it represented". Consider that the
    // Int32DisplacedInJSStack recovery: it tells us that the value is in a
    // particular place and that that place holds an unboxed int32. We have two different
    // places that a value could be (displaced, register) and a bunch of different
    // ways of representing a value. The number of recoveries is two * a bunch. The code
    // below means that we have to have two + a bunch cases rather than two * a bunch.
    // Once we have loaded the value from wherever it was, the reboxing is the same
    // regardless of its location. Likewise, before we do the reboxing, the way we get to
    // the value (i.e. where we load it from) is the same regardless of its type. Because
    // the code below always dumps everything into a scratch buffer first, the two
    // questions become orthogonal, which simplifies adding new types and adding new
    // locations.
    //
    // This raises the question: does using such a suboptimal implementation of OSR exit,
    // where we always emit code to dump all state into a scratch buffer only to then
    // dump it right back into the stack, hurt us in any way? The asnwer is that OSR exits
    // are rare. Our tiering strategy ensures this. This is because if an OSR exit is
    // taken more than ~100 times, we jettison the DFG code block along with all of its
    // exits. It is impossible for an OSR exit - i.e. the code we compile below - to
    // execute frequently enough for the codegen to matter that much. It probably matters
    // enough that we don't want to turn this into some super-slow function call, but so
    // long as we're generating straight-line code, that code can be pretty bad. Also
    // because we tend to exit only along one OSR exit from any DFG code block - that's an
    // empirical result that we're extremely confident about - the code size of this
    // doesn't matter much. Hence any attempt to optimize the codegen here is just purely
    // harmful to the system: it probably won't reduce either net memory usage or net
    // execution time. It will only prevent us from cleanly decoupling "where was the
    // variable" from "how was it represented", which will make it more difficult to add
    // features in the future and it will make it harder to reason about bugs.

    // Save all state from GPRs into the scratch buffer.

    ScratchBuffer* scratchBuffer = vm.scratchBufferForSize(sizeof(EncodedJSValue) * operands.size());
    EncodedJSValue* scratch = scratchBuffer ? static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer()) : 0;

    for (size_t index = 0; index < operands.size(); ++index) {
        const ValueRecovery& recovery = operands[index];

        switch (recovery.technique()) {
        case UnboxedInt32InGPR:
        case UnboxedCellInGPR:
#if USE(JSVALUE64)
        case InGPR:
        case UnboxedInt52InGPR:
        case UnboxedStrictInt52InGPR:
            jit.store64(recovery.gpr(), scratch + index);
            break;
#else
        case UnboxedBooleanInGPR:
            jit.store32(
                recovery.gpr(),
                &bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.payload);
            break;
            
        case InPair:
            jit.store32(
                recovery.tagGPR(),
                &bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.tag);
            jit.store32(
                recovery.payloadGPR(),
                &bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.payload);
            break;
#endif

        default:
            break;
        }
    }

    // And voila, all GPRs are free to reuse.

    // Save all state from FPRs into the scratch buffer.

    for (size_t index = 0; index < operands.size(); ++index) {
        const ValueRecovery& recovery = operands[index];

        switch (recovery.technique()) {
        case UnboxedDoubleInFPR:
        case InFPR:
            jit.move(AssemblyHelpers::TrustedImmPtr(scratch + index), GPRInfo::regT0);
            jit.storeDouble(recovery.fpr(), MacroAssembler::Address(GPRInfo::regT0));
            break;

        default:
            break;
        }
    }

    // Now, all FPRs are also free.

    // Save all state from the stack into the scratch buffer. For simplicity we
    // do this even for state that's already in the right place on the stack.
    // It makes things simpler later.

    for (size_t index = 0; index < operands.size(); ++index) {
        const ValueRecovery& recovery = operands[index];

        switch (recovery.technique()) {
        case DisplacedInJSStack:
        case CellDisplacedInJSStack:
        case BooleanDisplacedInJSStack:
        case Int32DisplacedInJSStack:
        case DoubleDisplacedInJSStack:
#if USE(JSVALUE64)
        case Int52DisplacedInJSStack:
        case StrictInt52DisplacedInJSStack:
            jit.load64(AssemblyHelpers::addressFor(recovery.virtualRegister()), GPRInfo::regT0);
            jit.store64(GPRInfo::regT0, scratch + index);
            break;
#else
            jit.load32(
                AssemblyHelpers::tagFor(recovery.virtualRegister()),
                GPRInfo::regT0);
            jit.load32(
                AssemblyHelpers::payloadFor(recovery.virtualRegister()),
                GPRInfo::regT1);
            jit.store32(
                GPRInfo::regT0,
                &bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.tag);
            jit.store32(
                GPRInfo::regT1,
                &bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.payload);
            break;
#endif

        default:
            break;
        }
    }

    if (validateDFGDoesGC) {
        // We're about to exit optimized code. So, there's no longer any optimized
        // code running that expects no GC. We need to set this before arguments
        // materialization below (see emitRestoreArguments()).

        // Even though we set Heap::m_expectDoesGC in compileOSRExit(), we also need
        // to set it here because compileOSRExit() is only called on the first time
        // we exit from this site, but all subsequent exits will take this compiled
        // ramp without calling compileOSRExit() first.
        jit.store8(CCallHelpers::TrustedImm32(true), vm.heap.addressOfExpectDoesGC());
    }

    // Need to ensure that the stack pointer accounts for the worst-case stack usage at exit. This
    // could toast some stack that the DFG used. We need to do it before storing to stack offsets
    // used by baseline.
    jit.addPtr(
        CCallHelpers::TrustedImm32(
            -jit.codeBlock()->jitCode()->dfgCommon()->requiredRegisterCountForExit * sizeof(Register)),
        CCallHelpers::framePointerRegister, CCallHelpers::stackPointerRegister);

    // Restore the DFG callee saves and then save the ones the baseline JIT uses.
    jit.emitRestoreCalleeSaves();
    jit.emitSaveCalleeSavesFor(jit.baselineCodeBlock());

    // The tag registers are needed to materialize recoveries below.
    jit.emitMaterializeTagCheckRegisters();

    if (exit.isExceptionHandler())
        jit.copyCalleeSavesToEntryFrameCalleeSavesBuffer(vm.topEntryFrame);

    // Do all data format conversions and store the results into the stack.

    for (size_t index = 0; index < operands.size(); ++index) {
        const ValueRecovery& recovery = operands[index];
        VirtualRegister reg = operands.virtualRegisterForIndex(index);

        if (reg.isLocal() && reg.toLocal() < static_cast<int>(jit.baselineCodeBlock()->calleeSaveSpaceAsVirtualRegisters()))
            continue;

        int operand = reg.offset();

        switch (recovery.technique()) {
        case DisplacedInJSStack:
        case InFPR:
#if USE(JSVALUE64)
        case InGPR:
        case UnboxedCellInGPR:
        case CellDisplacedInJSStack:
        case BooleanDisplacedInJSStack:
            jit.load64(scratch + index, GPRInfo::regT0);
            jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(operand));
            break;
#else // not USE(JSVALUE64)
        case InPair:
            jit.load32(
                &bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.tag,
                GPRInfo::regT0);
            jit.load32(
                &bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.payload,
                GPRInfo::regT1);
            jit.store32(
                GPRInfo::regT0,
                AssemblyHelpers::tagFor(operand));
            jit.store32(
                GPRInfo::regT1,
                AssemblyHelpers::payloadFor(operand));
            break;

        case UnboxedCellInGPR:
        case CellDisplacedInJSStack:
            jit.load32(
                &bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.payload,
                GPRInfo::regT0);
            jit.store32(
                AssemblyHelpers::TrustedImm32(JSValue::CellTag),
                AssemblyHelpers::tagFor(operand));
            jit.store32(
                GPRInfo::regT0,
                AssemblyHelpers::payloadFor(operand));
            break;

        case UnboxedBooleanInGPR:
        case BooleanDisplacedInJSStack:
            jit.load32(
                &bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.payload,
                GPRInfo::regT0);
            jit.store32(
                AssemblyHelpers::TrustedImm32(JSValue::BooleanTag),
                AssemblyHelpers::tagFor(operand));
            jit.store32(
                GPRInfo::regT0,
                AssemblyHelpers::payloadFor(operand));
            break;
#endif // USE(JSVALUE64)

        case UnboxedInt32InGPR:
        case Int32DisplacedInJSStack:
#if USE(JSVALUE64)
            jit.load64(scratch + index, GPRInfo::regT0);
            jit.zeroExtend32ToPtr(GPRInfo::regT0, GPRInfo::regT0);
            jit.or64(GPRInfo::numberTagRegister, GPRInfo::regT0);
            jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(operand));
#else
            jit.load32(
                &bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.payload,
                GPRInfo::regT0);
            jit.store32(
                AssemblyHelpers::TrustedImm32(JSValue::Int32Tag),
                AssemblyHelpers::tagFor(operand));
            jit.store32(
                GPRInfo::regT0,
                AssemblyHelpers::payloadFor(operand));
#endif
            break;

#if USE(JSVALUE64)
        case UnboxedInt52InGPR:
        case Int52DisplacedInJSStack:
            jit.load64(scratch + index, GPRInfo::regT0);
            jit.rshift64(
                AssemblyHelpers::TrustedImm32(JSValue::int52ShiftAmount), GPRInfo::regT0);
            jit.boxInt52(GPRInfo::regT0, GPRInfo::regT0, GPRInfo::regT1, FPRInfo::fpRegT0);
            jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(operand));
            break;

        case UnboxedStrictInt52InGPR:
        case StrictInt52DisplacedInJSStack:
            jit.load64(scratch + index, GPRInfo::regT0);
            jit.boxInt52(GPRInfo::regT0, GPRInfo::regT0, GPRInfo::regT1, FPRInfo::fpRegT0);
            jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(operand));
            break;
#endif

        case UnboxedDoubleInFPR:
        case DoubleDisplacedInJSStack:
            jit.move(AssemblyHelpers::TrustedImmPtr(scratch + index), GPRInfo::regT0);
            jit.loadDouble(MacroAssembler::Address(GPRInfo::regT0), FPRInfo::fpRegT0);
            jit.purifyNaN(FPRInfo::fpRegT0);
#if USE(JSVALUE64)
            jit.boxDouble(FPRInfo::fpRegT0, GPRInfo::regT0);
            jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(operand));
#else
            jit.storeDouble(FPRInfo::fpRegT0, AssemblyHelpers::addressFor(operand));
#endif
            break;

        case Constant:
#if USE(JSVALUE64)
            jit.store64(
                AssemblyHelpers::TrustedImm64(JSValue::encode(recovery.constant())),
                AssemblyHelpers::addressFor(operand));
#else
            jit.store32(
                AssemblyHelpers::TrustedImm32(recovery.constant().tag()),
                AssemblyHelpers::tagFor(operand));
            jit.store32(
                AssemblyHelpers::TrustedImm32(recovery.constant().payload()),
                AssemblyHelpers::payloadFor(operand));
#endif
            break;

        case DirectArgumentsThatWereNotCreated:
        case ClonedArgumentsThatWereNotCreated:
            // Don't do this, yet.
            break;

        default:
            RELEASE_ASSERT_NOT_REACHED();
            break;
        }
    }

    // Now that things on the stack are recovered, do the arguments recovery. We assume that arguments
    // recoveries don't recursively refer to each other. But, we don't try to assume that they only
    // refer to certain ranges of locals. Hence why we need to do this here, once the stack is sensible.
    // Note that we also roughly assume that the arguments might still be materialized outside of its
    // inline call frame scope - but for now the DFG wouldn't do that.

    emitRestoreArguments(jit, vm, operands);

    // Adjust the old JIT's execute counter. Since we are exiting OSR, we know
    // that all new calls into this code will go to the new JIT, so the execute
    // counter only affects call frames that performed OSR exit and call frames
    // that were still executing the old JIT at the time of another call frame's
    // OSR exit. We want to ensure that the following is true:
    //
    // (a) Code the performs an OSR exit gets a chance to reenter optimized
    //     code eventually, since optimized code is faster. But we don't
    //     want to do such reentery too aggressively (see (c) below).
    //
    // (b) If there is code on the call stack that is still running the old
    //     JIT's code and has never OSR'd, then it should get a chance to
    //     perform OSR entry despite the fact that we've exited.
    //
    // (c) Code the performs an OSR exit should not immediately retry OSR
    //     entry, since both forms of OSR are expensive. OSR entry is
    //     particularly expensive.
    //
    // (d) Frequent OSR failures, even those that do not result in the code
    //     running in a hot loop, result in recompilation getting triggered.
    //
    // To ensure (c), we'd like to set the execute counter to
    // counterValueForOptimizeAfterWarmUp(). This seems like it would endanger
    // (a) and (b), since then every OSR exit would delay the opportunity for
    // every call frame to perform OSR entry. Essentially, if OSR exit happens
    // frequently and the function has few loops, then the counter will never
    // become non-negative and OSR entry will never be triggered. OSR entry
    // will only happen if a loop gets hot in the old JIT, which does a pretty
    // good job of ensuring (a) and (b). But that doesn't take care of (d),
    // since each speculation failure would reset the execute counter.
    // So we check here if the number of speculation failures is significantly
    // larger than the number of successes (we want 90% success rate), and if
    // there have been a large enough number of failures. If so, we set the
    // counter to 0; otherwise we set the counter to
    // counterValueForOptimizeAfterWarmUp().

    handleExitCounts(vm, jit, exit);

    // Reify inlined call frames.

    reifyInlinedCallFrames(jit, exit);

    // And finish.
    adjustAndJumpToTarget(vm, jit, exit);
}

void JIT_OPERATION operationDebugPrintSpeculationFailure(CallFrame* callFrame, void* debugInfoRaw, void* scratch)
{
    VM& vm = callFrame->deprecatedVM();
    NativeCallFrameTracer tracer(vm, callFrame);

    SpeculationFailureDebugInfo* debugInfo = static_cast<SpeculationFailureDebugInfo*>(debugInfoRaw);
    CodeBlock* codeBlock = debugInfo->codeBlock;
    CodeBlock* alternative = codeBlock->alternative();
    dataLog("Speculation failure in ", *codeBlock);
    dataLog(" @ exit #", vm.osrExitIndex, " (", debugInfo->bytecodeIndex, ", ", exitKindToString(debugInfo->kind), ") with ");
    if (alternative) {
        dataLog(
            "executeCounter = ", alternative->jitExecuteCounter(),
            ", reoptimizationRetryCounter = ", alternative->reoptimizationRetryCounter(),
            ", optimizationDelayCounter = ", alternative->optimizationDelayCounter());
    } else
        dataLog("no alternative code block (i.e. we've been jettisoned)");
    dataLog(", osrExitCounter = ", codeBlock->osrExitCounter(), "\n");
    dataLog("    GPRs at time of exit:");
    char* scratchPointer = static_cast<char*>(scratch);
    for (unsigned i = 0; i < GPRInfo::numberOfRegisters; ++i) {
        GPRReg gpr = GPRInfo::toRegister(i);
        dataLog(" ", GPRInfo::debugName(gpr), ":", RawPointer(*reinterpret_cast_ptr<void**>(scratchPointer)));
        scratchPointer += sizeof(EncodedJSValue);
    }
    dataLog("\n");
    dataLog("    FPRs at time of exit:");
    for (unsigned i = 0; i < FPRInfo::numberOfRegisters; ++i) {
        FPRReg fpr = FPRInfo::toRegister(i);
        dataLog(" ", FPRInfo::debugName(fpr), ":");
        uint64_t bits = *reinterpret_cast_ptr<uint64_t*>(scratchPointer);
        double value = *reinterpret_cast_ptr<double*>(scratchPointer);
        dataLogF("%llx:%lf", static_cast<long long>(bits), value);
        scratchPointer += sizeof(EncodedJSValue);
    }
    dataLog("\n");
}

} } // namespace JSC::DFG

#endif // ENABLE(DFG_JIT)