blob: 0821d68ab9eb77b0ac298d6c1e32d44640bbdee9 [file] [log] [blame]
/*
* Copyright (C) 2013-2021 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "config.h"
#include "FTLOSRExitCompiler.h"
#if ENABLE(FTL_JIT)
#include "AssemblyHelpersSpoolers.h"
#include "BytecodeStructs.h"
#include "CheckpointOSRExitSideState.h"
#include "DFGOSRExitCompilerCommon.h"
#include "FTLJITCode.h"
#include "FTLLocation.h"
#include "FTLOSRExit.h"
#include "FTLOperations.h"
#include "FTLSaveRestore.h"
#include "FTLState.h"
#include "JSCJSValueInlines.h"
#include "LinkBuffer.h"
#include "MaxFrameExtentForSlowPathCall.h"
#include "OperandsInlines.h"
#include "ProbeContext.h"
#include <wtf/Scope.h>
namespace JSC { namespace FTL {
using namespace DFG;
static void reboxAccordingToFormat(
DataFormat format, AssemblyHelpers& jit, GPRReg value, GPRReg scratch1, GPRReg scratch2)
{
switch (format) {
case DataFormatInt32: {
jit.zeroExtend32ToWord(value, value);
jit.or64(GPRInfo::numberTagRegister, value);
break;
}
case DataFormatInt52: {
jit.rshift64(AssemblyHelpers::TrustedImm32(JSValue::int52ShiftAmount), value);
jit.moveDoubleTo64(FPRInfo::fpRegT0, scratch2);
jit.boxInt52(value, value, scratch1, FPRInfo::fpRegT0);
jit.move64ToDouble(scratch2, FPRInfo::fpRegT0);
break;
}
case DataFormatStrictInt52: {
jit.moveDoubleTo64(FPRInfo::fpRegT0, scratch2);
jit.boxInt52(value, value, scratch1, FPRInfo::fpRegT0);
jit.move64ToDouble(scratch2, FPRInfo::fpRegT0);
break;
}
case DataFormatBoolean: {
jit.zeroExtend32ToWord(value, value);
jit.or32(MacroAssembler::TrustedImm32(JSValue::ValueFalse), value);
break;
}
case DataFormatJS: {
// Done already!
break;
}
case DataFormatDouble: {
jit.moveDoubleTo64(FPRInfo::fpRegT0, scratch1);
jit.move64ToDouble(value, FPRInfo::fpRegT0);
jit.purifyNaN(FPRInfo::fpRegT0);
jit.boxDouble(FPRInfo::fpRegT0, value);
jit.move64ToDouble(scratch1, FPRInfo::fpRegT0);
break;
}
default:
RELEASE_ASSERT_NOT_REACHED();
break;
}
}
static void compileRecovery(
CCallHelpers& jit, const ExitValue& value,
const FixedVector<B3::ValueRep>& valueReps,
char* registerScratch,
const HashMap<ExitTimeObjectMaterialization*, EncodedJSValue*>& materializationToPointer)
{
switch (value.kind()) {
case ExitValueDead:
jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsUndefined())), GPRInfo::regT0);
break;
case ExitValueConstant:
jit.move(MacroAssembler::TrustedImm64(JSValue::encode(value.constant())), GPRInfo::regT0);
break;
case ExitValueArgument:
Location::forValueRep(valueReps[value.exitArgument().argument()]).restoreInto(
jit, registerScratch, GPRInfo::regT0);
break;
case ExitValueInJSStack:
case ExitValueInJSStackAsInt32:
case ExitValueInJSStackAsInt52:
case ExitValueInJSStackAsDouble:
jit.load64(AssemblyHelpers::addressFor(value.virtualRegister()), GPRInfo::regT0);
break;
case ExitValueMaterializeNewObject:
jit.loadPtr(materializationToPointer.get(value.objectMaterialization()), GPRInfo::regT0);
break;
default:
RELEASE_ASSERT_NOT_REACHED();
break;
}
reboxAccordingToFormat(
value.dataFormat(), jit, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT2);
}
static void compileStub(VM& vm, unsigned exitID, JITCode* jitCode, OSRExit& exit, CodeBlock* codeBlock)
{
// This code requires framePointerRegister is the same as callFrameRegister
static_assert(MacroAssembler::framePointerRegister == GPRInfo::callFrameRegister, "MacroAssembler::framePointerRegister and GPRInfo::callFrameRegister must be the same");
CCallHelpers jit(codeBlock);
// The first thing we need to do is restablish our frame in the case of an exception.
if (exit.isGenericUnwindHandler()) {
RELEASE_ASSERT(vm.callFrameForCatch); // The first time we hit this exit, like at all other times, this field should be non-null.
jit.restoreCalleeSavesFromEntryFrameCalleeSavesBuffer(vm.topEntryFrame);
jit.loadPtr(vm.addressOfCallFrameForCatch(), MacroAssembler::framePointerRegister);
jit.addPtr(CCallHelpers::TrustedImm32(codeBlock->stackPointerOffset() * sizeof(Register)),
MacroAssembler::framePointerRegister, CCallHelpers::stackPointerRegister);
// Do a pushToSave because that's what the exit compiler below expects the stack
// to look like because that's the last thing the ExitThunkGenerator does. The code
// below doesn't actually use the value that was pushed, but it does rely on the
// general shape of the stack being as it is in the non-exception OSR case.
jit.pushToSaveImmediateWithoutTouchingRegisters(CCallHelpers::TrustedImm32(0xbadbeef));
}
// We need scratch space to save all registers, to build up the JS stack, to deal with unwind
// fixup, pointers to all of the objects we materialize, and the elements inside those objects
// that we materialize.
// Figure out how much space we need for those object allocations.
unsigned numMaterializations = 0;
size_t maxMaterializationNumArguments = 0;
for (ExitTimeObjectMaterialization* materialization : exit.m_descriptor->m_materializations) {
numMaterializations++;
maxMaterializationNumArguments = std::max(
maxMaterializationNumArguments,
materialization->properties().size());
}
ScratchBuffer* scratchBuffer = vm.scratchBufferForSize(
sizeof(EncodedJSValue) * (
exit.m_descriptor->m_values.size() + numMaterializations + maxMaterializationNumArguments) +
requiredScratchMemorySizeInBytes() +
codeBlock->jitCode()->calleeSaveRegisters()->sizeOfAreaInBytes());
EncodedJSValue* scratch = scratchBuffer ? static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer()) : nullptr;
EncodedJSValue* materializationPointers = scratch + exit.m_descriptor->m_values.size();
EncodedJSValue* materializationArguments = materializationPointers + numMaterializations;
char* registerScratch = bitwise_cast<char*>(materializationArguments + maxMaterializationNumArguments);
uint64_t* unwindScratch = bitwise_cast<uint64_t*>(registerScratch + requiredScratchMemorySizeInBytes());
HashMap<ExitTimeObjectMaterialization*, EncodedJSValue*> materializationToPointer;
unsigned materializationCount = 0;
for (ExitTimeObjectMaterialization* materialization : exit.m_descriptor->m_materializations) {
materializationToPointer.add(
materialization, materializationPointers + materializationCount++);
}
auto recoverValue = [&] (const ExitValue& value) {
compileRecovery(
jit, value,
exit.m_valueReps,
registerScratch, materializationToPointer);
};
// Note that we come in here, the stack used to be as B3 left it except that someone called pushToSave().
// We don't care about the value they saved. But, we do appreciate the fact that they did it, because we use
// that slot for saveAllRegisters().
saveAllRegisters(jit, registerScratch);
if constexpr (validateDFGDoesGC) {
if (Options::validateDoesGC()) {
// We're about to exit optimized code. So, there's no longer any optimized
// code running that expects no GC. We need to set this before object
// materialization below.
// Even though we set Heap::m_doesGC in compileFTLOSRExit(), we also need
// to set it here because compileFTLOSRExit() is only called on the first time
// we exit from this site, but all subsequent exits will take this compiled
// ramp without calling compileFTLOSRExit() first.
jit.store64(CCallHelpers::TrustedImm64(DoesGCCheck::encode(true, DoesGCCheck::Special::FTLOSRExit)), vm.addressOfDoesGC());
}
}
// Bring the stack back into a sane form and assert that it's sane.
jit.popToRestore(GPRInfo::regT0);
jit.checkStackPointerAlignment();
if (UNLIKELY(vm.m_perBytecodeProfiler && jitCode->dfgCommon()->compilation)) {
Profiler::Database& database = *vm.m_perBytecodeProfiler;
Profiler::Compilation* compilation = jitCode->dfgCommon()->compilation.get();
Profiler::OSRExit* profilerExit = compilation->addOSRExit(
exitID, Profiler::OriginStack(database, codeBlock, exit.m_codeOrigin),
exit.m_kind, exit.m_kind == UncountableInvalidation);
jit.add64(CCallHelpers::TrustedImm32(1), CCallHelpers::AbsoluteAddress(profilerExit->counterAddress()));
}
// The remaining code assumes that SP/FP are in the same state that they were in the FTL's
// call frame.
// Get the call frame and tag thingies.
// Restore the exiting function's callFrame value into a regT4
jit.emitMaterializeTagCheckRegisters();
// Do some value profiling.
if (exit.m_descriptor->m_profileDataFormat != DataFormatNone) {
Location::forValueRep(exit.m_valueReps[0]).restoreInto(jit, registerScratch, GPRInfo::regT0);
reboxAccordingToFormat(
exit.m_descriptor->m_profileDataFormat, jit, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT2);
if (exit.m_kind == BadCache || exit.m_kind == BadIndexingType) {
CodeOrigin codeOrigin = exit.m_codeOriginForExitProfile;
CodeBlock* codeBlock = jit.baselineCodeBlockFor(codeOrigin);
if (ArrayProfile* arrayProfile = codeBlock->getArrayProfile(codeOrigin.bytecodeIndex())) {
const auto* instruction = codeBlock->instructions().at(codeOrigin.bytecodeIndex()).ptr();
CCallHelpers::Jump skipProfile;
if (instruction->is<OpGetById>()) {
auto& metadata = instruction->as<OpGetById>().metadata(codeBlock);
skipProfile = jit.branch8(CCallHelpers::NotEqual, CCallHelpers::AbsoluteAddress(&metadata.m_modeMetadata.mode), CCallHelpers::TrustedImm32(static_cast<uint8_t>(GetByIdMode::ArrayLength)));
}
jit.load32(MacroAssembler::Address(GPRInfo::regT0, JSCell::structureIDOffset()), GPRInfo::regT1);
jit.store32(GPRInfo::regT1, arrayProfile->addressOfLastSeenStructureID());
jit.load8(MacroAssembler::Address(GPRInfo::regT0, JSCell::typeInfoTypeOffset()), GPRInfo::regT2);
jit.sub32(MacroAssembler::TrustedImm32(FirstTypedArrayType), GPRInfo::regT2);
auto notTypedArray = jit.branch32(MacroAssembler::AboveOrEqual, GPRInfo::regT2, MacroAssembler::TrustedImm32(NumberOfTypedArrayTypesExcludingDataView));
jit.move(MacroAssembler::TrustedImmPtr(typedArrayModes), GPRInfo::regT1);
jit.load32(MacroAssembler::BaseIndex(GPRInfo::regT1, GPRInfo::regT2, MacroAssembler::TimesFour), GPRInfo::regT2);
auto storeArrayModes = jit.jump();
notTypedArray.link(&jit);
jit.load8(MacroAssembler::Address(GPRInfo::regT0, JSCell::indexingTypeAndMiscOffset()), GPRInfo::regT1);
jit.and32(MacroAssembler::TrustedImm32(IndexingModeMask), GPRInfo::regT1);
jit.move(MacroAssembler::TrustedImm32(1), GPRInfo::regT2);
jit.lshift32(GPRInfo::regT1, GPRInfo::regT2);
storeArrayModes.link(&jit);
jit.or32(GPRInfo::regT2, MacroAssembler::AbsoluteAddress(arrayProfile->addressOfArrayModes()));
if (skipProfile.isSet())
skipProfile.link(&jit);
}
}
if (exit.m_descriptor->m_valueProfile)
exit.m_descriptor->m_valueProfile.emitReportValue(jit, jit.codeBlock(), JSValueRegs(GPRInfo::regT0), GPRInfo::regT1);
}
// Materialize all objects. Don't materialize an object until all
// of the objects it needs have been materialized. We break cycles
// by populating objects late - we only consider an object as
// needing another object if the later is needed for the
// allocation of the former.
HashSet<ExitTimeObjectMaterialization*> toMaterialize;
for (ExitTimeObjectMaterialization* materialization : exit.m_descriptor->m_materializations)
toMaterialize.add(materialization);
while (!toMaterialize.isEmpty()) {
unsigned previousToMaterializeSize = toMaterialize.size();
Vector<ExitTimeObjectMaterialization*> worklist;
worklist.appendRange(toMaterialize.begin(), toMaterialize.end());
for (ExitTimeObjectMaterialization* materialization : worklist) {
// Check if we can do anything about this right now.
bool allGood = true;
for (ExitPropertyValue value : materialization->properties()) {
if (!value.value().isObjectMaterialization())
continue;
if (!value.location().neededForMaterialization())
continue;
if (toMaterialize.contains(value.value().objectMaterialization())) {
// Gotta skip this one, since it needs a
// materialization that hasn't been materialized.
allGood = false;
break;
}
}
if (!allGood)
continue;
// All systems go for materializing the object. First we
// recover the values of all of its fields and then we
// call a function to actually allocate the beast.
// We only recover the fields that are needed for the allocation.
for (unsigned propertyIndex = materialization->properties().size(); propertyIndex--;) {
const ExitPropertyValue& property = materialization->properties()[propertyIndex];
if (!property.location().neededForMaterialization())
continue;
recoverValue(property.value());
jit.storePtr(GPRInfo::regT0, materializationArguments + propertyIndex);
}
static_assert(FunctionTraits<decltype(operationMaterializeObjectInOSR)>::arity < GPRInfo::numberOfArgumentRegisters, "This call assumes that we don't pass arguments on the stack.");
jit.setupArguments<decltype(operationMaterializeObjectInOSR)>(
CCallHelpers::TrustedImmPtr(codeBlock->globalObjectFor(materialization->origin())),
CCallHelpers::TrustedImmPtr(materialization),
CCallHelpers::TrustedImmPtr(materializationArguments));
jit.prepareCallOperation(vm);
jit.move(CCallHelpers::TrustedImmPtr(tagCFunction<OperationPtrTag>(operationMaterializeObjectInOSR)), GPRInfo::nonArgGPR0);
jit.call(GPRInfo::nonArgGPR0, OperationPtrTag);
jit.storePtr(GPRInfo::returnValueGPR, materializationToPointer.get(materialization));
// Let everyone know that we're done.
toMaterialize.remove(materialization);
}
// We expect progress! This ensures that we crash rather than looping infinitely if there
// is something broken about this fixpoint. Or, this could happen if we ever violate the
// "materializations form a DAG" rule.
RELEASE_ASSERT(toMaterialize.size() < previousToMaterializeSize);
}
// Now that all the objects have been allocated, we populate them
// with the correct values. This time we can recover all the
// fields, including those that are only needed for the allocation.
for (ExitTimeObjectMaterialization* materialization : exit.m_descriptor->m_materializations) {
for (unsigned propertyIndex = materialization->properties().size(); propertyIndex--;) {
recoverValue(materialization->properties()[propertyIndex].value());
jit.storePtr(GPRInfo::regT0, materializationArguments + propertyIndex);
}
static_assert(FunctionTraits<decltype(operationPopulateObjectInOSR)>::arity < GPRInfo::numberOfArgumentRegisters, "This call assumes that we don't pass arguments on the stack.");
jit.setupArguments<decltype(operationPopulateObjectInOSR)>(
CCallHelpers::TrustedImmPtr(codeBlock->globalObjectFor(materialization->origin())),
CCallHelpers::TrustedImmPtr(materialization),
CCallHelpers::TrustedImmPtr(materializationToPointer.get(materialization)),
CCallHelpers::TrustedImmPtr(materializationArguments));
jit.prepareCallOperation(vm);
jit.move(CCallHelpers::TrustedImmPtr(tagCFunction<OperationPtrTag>(operationPopulateObjectInOSR)), GPRInfo::nonArgGPR0);
jit.call(GPRInfo::nonArgGPR0, OperationPtrTag);
}
// Save all state from wherever the exit data tells us it was, into the appropriate place in
// the scratch buffer. This also does the reboxing.
{
std::optional<GPRReg> undefinedGPR;
jit.move(CCallHelpers::TrustedImmPtr(scratch), GPRInfo::regT3);
CCallHelpers::CopySpooler spooler(jit, CCallHelpers::framePointerRegister, GPRInfo::regT3, GPRInfo::regT0, GPRInfo::regT1);
for (unsigned index = exit.m_descriptor->m_values.size(); index--;) {
auto& value = exit.m_descriptor->m_values[index];
if (value.dataFormat() == DataFormatJS) {
switch (value.kind()) {
case ExitValueDead:
if (UNLIKELY(!undefinedGPR)) {
jit.move(CCallHelpers::TrustedImm64(JSValue::encode(jsUndefined())), GPRInfo::regT4);
undefinedGPR = GPRInfo::regT4;
}
spooler.copyGPR(undefinedGPR.value());
spooler.storeGPR(index * sizeof(EncodedJSValue));
break;
case ExitValueConstant: {
EncodedJSValue currentConstant = JSValue::encode(value.constant());
if (currentConstant == encodedJSUndefined()) {
if (UNLIKELY(!undefinedGPR)) {
jit.move(CCallHelpers::TrustedImm64(JSValue::encode(jsUndefined())), GPRInfo::regT4);
undefinedGPR = GPRInfo::regT4;
}
spooler.copyGPR(undefinedGPR.value());
} else
spooler.moveConstant(currentConstant);
spooler.storeGPR(index * sizeof(EncodedJSValue));
break;
}
case ExitValueArgument:
Location::forValueRep(exit.m_valueReps[value.exitArgument().argument()]).restoreInto(jit, registerScratch, GPRInfo::regT0);
jit.store64(GPRInfo::regT0, CCallHelpers::Address(GPRInfo::regT3, index * sizeof(EncodedJSValue)));
break;
case ExitValueInJSStack:
case ExitValueInJSStackAsInt32:
case ExitValueInJSStackAsInt52:
case ExitValueInJSStackAsDouble:
spooler.loadGPR(value.virtualRegister().offset() * sizeof(EncodedJSValue));
spooler.storeGPR(index * sizeof(EncodedJSValue));
break;
case ExitValueMaterializeNewObject:
jit.loadPtr(materializationToPointer.get(value.objectMaterialization()), GPRInfo::regT0);
jit.store64(GPRInfo::regT0, CCallHelpers::Address(GPRInfo::regT3, index * sizeof(EncodedJSValue)));
break;
default:
RELEASE_ASSERT_NOT_REACHED();
break;
}
} else {
recoverValue(value);
jit.store64(GPRInfo::regT0, CCallHelpers::Address(GPRInfo::regT3, index * sizeof(EncodedJSValue)));
}
}
spooler.finalizeGPR();
}
// Henceforth we make it look like the exiting function was called through a register
// preservation wrapper. This implies that FP must be nudged down by a certain amount. Then
// we restore the various things according to either exit.m_descriptor->m_values or by copying from the
// old frame, and finally we save the various callee-save registers into where the
// restoration thunk would restore them from.
// Before we start messing with the frame, we need to set aside any registers that the
// FTL code was preserving.
{
constexpr GPRReg srcBufferGPR = GPRInfo::regT2;
constexpr GPRReg destBufferGPR = GPRInfo::regT3;
jit.move(CCallHelpers::framePointerRegister, srcBufferGPR);
jit.move(CCallHelpers::TrustedImmPtr(unwindScratch), destBufferGPR);
CCallHelpers::CopySpooler spooler(CCallHelpers::CopySpooler::BufferRegs::AllowModification, jit, srcBufferGPR, destBufferGPR, GPRInfo::regT0, GPRInfo::regT1);
for (unsigned i = codeBlock->jitCode()->calleeSaveRegisters()->registerCount(); i--;) {
RegisterAtOffset entry = codeBlock->jitCode()->calleeSaveRegisters()->at(i);
spooler.loadGPR(entry.offset());
spooler.storeGPR(i * sizeof(uint64_t));
}
spooler.finalizeGPR();
}
CodeBlock* baselineCodeBlock = jit.baselineCodeBlockFor(exit.m_codeOrigin);
// First set up SP so that our data doesn't get clobbered by signals.
unsigned conservativeStackDelta =
(exit.m_descriptor->m_values.numberOfLocals() + CodeBlock::calleeSaveSpaceAsVirtualRegisters(*baselineCodeBlock->jitCode()->calleeSaveRegisters())) * sizeof(Register) +
maxFrameExtentForSlowPathCall;
conservativeStackDelta = WTF::roundUpToMultipleOf(
stackAlignmentBytes(), conservativeStackDelta);
jit.addPtr(
MacroAssembler::TrustedImm32(-conservativeStackDelta),
MacroAssembler::framePointerRegister, MacroAssembler::stackPointerRegister);
jit.checkStackPointerAlignment();
{
RegisterSet allFTLCalleeSaves = RegisterSet::ftlCalleeSaveRegisters();
const RegisterAtOffsetList* baselineCalleeSaves = baselineCodeBlock->jitCode()->calleeSaveRegisters();
auto iterateCalleeSavesImpl = [&](auto check, auto func) {
for (Reg reg = Reg::first(); reg <= Reg::last(); reg = reg.next()) {
if (!allFTLCalleeSaves.get(reg))
continue;
if (!check(reg))
continue;
unsigned unwindIndex = codeBlock->jitCode()->calleeSaveRegisters()->indexOf(reg);
const RegisterAtOffset* baselineRegisterOffset = baselineCalleeSaves->find(reg);
func(reg, unwindIndex, baselineRegisterOffset);
}
};
auto iterateGPRCalleeSaves = [&](auto func) {
iterateCalleeSavesImpl([](Reg reg) { return reg.isGPR(); }, func);
};
auto iterateFPRCalleeSaves = [&](auto func) {
iterateCalleeSavesImpl([](Reg reg) { return reg.isFPR(); }, func);
};
{
// unwindIndex == UINT_MAX indicates that the FTL compilation didn't preserve these registers.
// This means that it also didn't use them. Their values at the beginning of OSR exit should
// be the ones to retain. We saved all registers into the register scratch buffer at the beginning
// of the thunk. So we can restore them from there.
ASSERT(!allFTLCalleeSaves.contains(GPRInfo::regT3));
ASSERT(!allFTLCalleeSaves.contains(GPRInfo::regT0));
ASSERT(!allFTLCalleeSaves.contains(GPRInfo::regT1));
ASSERT(!allFTLCalleeSaves.contains(FPRInfo::fpRegT0));
ASSERT(!allFTLCalleeSaves.contains(FPRInfo::fpRegT1));
jit.move(CCallHelpers::TrustedImmPtr(registerScratch), GPRInfo::regT3);
{
// Load from registerScratch buffer to callee-save registers.
CCallHelpers::LoadRegSpooler spooler(jit, GPRInfo::regT3);
iterateGPRCalleeSaves([&](Reg reg, unsigned unwindIndex, const RegisterAtOffset* baselineRegisterOffset) {
if (unwindIndex == UINT_MAX && !baselineRegisterOffset)
spooler.loadGPR({ reg, static_cast<ptrdiff_t>(offsetOfReg(reg)), });
});
spooler.finalizeGPR();
iterateFPRCalleeSaves([&](Reg reg, unsigned unwindIndex, const RegisterAtOffset* baselineRegisterOffset) {
if (unwindIndex == UINT_MAX && !baselineRegisterOffset)
spooler.loadFPR({ reg, static_cast<ptrdiff_t>(offsetOfReg(reg)), });
});
spooler.finalizeFPR();
}
{
// Copy from registerScratch buffer to call frame.
CCallHelpers::CopySpooler spooler(jit, GPRInfo::regT3, CCallHelpers::framePointerRegister, GPRInfo::regT0, GPRInfo::regT1, FPRInfo::fpRegT0, FPRInfo::fpRegT1);
iterateGPRCalleeSaves([&](Reg reg, unsigned unwindIndex, const RegisterAtOffset* baselineRegisterOffset) {
if (unwindIndex == UINT_MAX && baselineRegisterOffset) {
spooler.loadGPR(offsetOfReg(reg));
spooler.storeGPR(baselineRegisterOffset->offset());
}
});
spooler.finalizeGPR();
iterateFPRCalleeSaves([&](Reg reg, unsigned unwindIndex, const RegisterAtOffset* baselineRegisterOffset) {
if (unwindIndex == UINT_MAX && baselineRegisterOffset) {
spooler.loadFPR(offsetOfReg(reg));
spooler.storeFPR(baselineRegisterOffset->offset());
}
});
spooler.finalizeFPR();
}
}
{
// The FTL compilation preserved these registers. Their new values are therefore irrelevant,
// but we can get their values that were preserved by using the unwind data. We've already
// copied all unwind-able preserved registers into the unwind scratch buffer, so we can get
// the values to restore from there.
ASSERT((bitwise_cast<uintptr_t>(unwindScratch) - bitwise_cast<uintptr_t>(registerScratch)) == requiredScratchMemorySizeInBytes());
jit.addPtr(CCallHelpers::TrustedImm32(requiredScratchMemorySizeInBytes()), GPRInfo::regT3); // Change registerScratch to unwindScratch.
{
// Load from unwindScratch buffer to callee-save registers.
CCallHelpers::LoadRegSpooler spooler(jit, GPRInfo::regT3);
iterateGPRCalleeSaves([&](Reg reg, unsigned unwindIndex, const RegisterAtOffset* baselineRegisterOffset) {
if (unwindIndex != UINT_MAX && !baselineRegisterOffset)
spooler.loadGPR({ reg, static_cast<ptrdiff_t>(unwindIndex * sizeof(uint64_t)), });
});
spooler.finalizeGPR();
iterateFPRCalleeSaves([&](Reg reg, unsigned unwindIndex, const RegisterAtOffset* baselineRegisterOffset) {
if (unwindIndex != UINT_MAX && !baselineRegisterOffset)
spooler.loadFPR({ reg, static_cast<ptrdiff_t>(unwindIndex * sizeof(uint64_t)), });
});
spooler.finalizeFPR();
}
{
// Copy from unwindScratch buffer to call frame.
CCallHelpers::CopySpooler spooler(jit, GPRInfo::regT3, CCallHelpers::framePointerRegister, GPRInfo::regT0, GPRInfo::regT1, FPRInfo::fpRegT0, FPRInfo::fpRegT1);
iterateGPRCalleeSaves([&](Reg, unsigned unwindIndex, const RegisterAtOffset* baselineRegisterOffset) {
if (unwindIndex != UINT_MAX && baselineRegisterOffset) {
spooler.loadGPR(static_cast<ptrdiff_t>(unwindIndex * sizeof(uint64_t)));
spooler.storeGPR(baselineRegisterOffset->offset());
}
});
spooler.finalizeGPR();
iterateFPRCalleeSaves([&](Reg, unsigned unwindIndex, const RegisterAtOffset* baselineRegisterOffset) {
if (unwindIndex != UINT_MAX && baselineRegisterOffset) {
spooler.loadFPR(static_cast<ptrdiff_t>(unwindIndex * sizeof(uint64_t)));
spooler.storeFPR(baselineRegisterOffset->offset());
}
});
spooler.finalizeFPR();
}
}
}
size_t baselineVirtualRegistersForCalleeSaves = CodeBlock::calleeSaveSpaceAsVirtualRegisters(*baselineCodeBlock->jitCode()->calleeSaveRegisters());
if (exit.m_codeOrigin.inlineStackContainsActiveCheckpoint()) {
EncodedJSValue* tmpScratch = scratch + exit.m_descriptor->m_values.tmpIndex(0);
jit.setupArguments<decltype(operationMaterializeOSRExitSideState)>(CCallHelpers::TrustedImmPtr(&vm), CCallHelpers::TrustedImmPtr(&exit), CCallHelpers::TrustedImmPtr(tmpScratch));
jit.prepareCallOperation(vm);
jit.move(AssemblyHelpers::TrustedImmPtr(tagCFunction<OperationPtrTag>(operationMaterializeOSRExitSideState)), GPRInfo::nonArgGPR0);
jit.call(GPRInfo::nonArgGPR0, OperationPtrTag);
}
// Now get state out of the scratch buffer and place it back into the stack. The values are
// already reboxed so we just move them.
{
constexpr GPRReg srcBufferGPR = GPRInfo::regT2;
constexpr GPRReg destBufferGPR = GPRInfo::regT3;
jit.move(CCallHelpers::TrustedImmPtr(scratch), srcBufferGPR);
jit.move(GPRInfo::callFrameRegister, destBufferGPR);
CCallHelpers::CopySpooler spooler(CCallHelpers::CopySpooler::BufferRegs::AllowModification, jit, srcBufferGPR, destBufferGPR, GPRInfo::regT0, GPRInfo::regT1);
for (unsigned index = exit.m_descriptor->m_values.size(); index--;) {
Operand operand = exit.m_descriptor->m_values.operandForIndex(index);
if (operand.isTmp())
continue;
if (operand.isLocal() && operand.toLocal() < static_cast<int>(baselineVirtualRegistersForCalleeSaves))
continue;
spooler.loadGPR(index * sizeof(EncodedJSValue));
spooler.storeGPR(operand.virtualRegister().offset() * sizeof(EncodedJSValue));
}
spooler.finalizeGPR();
}
handleExitCounts(vm, jit, exit);
reifyInlinedCallFrames(jit, exit);
adjustAndJumpToTarget(vm, jit, exit);
LinkBuffer patchBuffer(jit, codeBlock, LinkBuffer::Profile::FTLOSRExit);
exit.m_code = FINALIZE_CODE_IF(
shouldDumpDisassembly() || Options::verboseOSR() || Options::verboseFTLOSRExit(),
patchBuffer, OSRExitPtrTag,
"FTL OSR exit #%u (D@%u, %s, %s) from %s, with operands = %s",
exitID, exit.m_dfgNodeIndex, toCString(exit.m_codeOrigin).data(),
exitKindToString(exit.m_kind), toCString(*codeBlock).data(),
toCString(ignoringContext<DumpContext>(exit.m_descriptor->m_values)).data()
);
}
JSC_DEFINE_JIT_OPERATION(operationCompileFTLOSRExit, void*, (CallFrame* callFrame, unsigned exitID))
{
if (shouldDumpDisassembly() || Options::verboseOSR() || Options::verboseFTLOSRExit())
dataLog("Compiling OSR exit with exitID = ", exitID, "\n");
VM& vm = callFrame->deprecatedVM();
// Don't need an ActiveScratchBufferScope here because we DeferGCForAWhile below.
if constexpr (validateDFGDoesGC) {
// We're about to exit optimized code. So, there's no longer any optimized
// code running that expects no GC.
vm.setDoesGCExpectation(true, DoesGCCheck::Special::FTLOSRExit);
}
if (vm.callFrameForCatch)
RELEASE_ASSERT(vm.callFrameForCatch == callFrame);
CodeBlock* codeBlock = callFrame->codeBlock();
ASSERT(codeBlock);
ASSERT(codeBlock->jitType() == JITType::FTLJIT);
// It's sort of preferable that we don't GC while in here. Anyways, doing so wouldn't
// really be profitable.
DeferGCForAWhile deferGC(vm);
JITCode* jitCode = codeBlock->jitCode()->ftl();
OSRExit& exit = jitCode->m_osrExit[exitID];
if (shouldDumpDisassembly() || Options::verboseOSR() || Options::verboseFTLOSRExit()) {
dataLog(" Owning block: ", pointerDump(codeBlock), "\n");
dataLog(" Origin: ", exit.m_codeOrigin, "\n");
if (exit.m_codeOriginForExitProfile != exit.m_codeOrigin)
dataLog(" Origin for exit profile: ", exit.m_codeOriginForExitProfile, "\n");
dataLog(" Current call site index: ", callFrame->callSiteIndex().bits(), "\n");
dataLog(" Exit is exception handler: ", exit.isExceptionHandler(), "\n");
dataLog(" Is unwind handler: ", exit.isGenericUnwindHandler(), "\n");
dataLog(" Exit values: ", exit.m_descriptor->m_values, "\n");
dataLog(" Value reps: ", listDump(exit.m_valueReps), "\n");
if (!exit.m_descriptor->m_materializations.isEmpty()) {
dataLog(" Materializations:\n");
for (ExitTimeObjectMaterialization* materialization : exit.m_descriptor->m_materializations)
dataLog(" ", pointerDump(materialization), "\n");
}
}
compileStub(vm, exitID, jitCode, exit, codeBlock);
MacroAssembler::repatchJump(
exit.codeLocationForRepatch(codeBlock), CodeLocationLabel<OSRExitPtrTag>(exit.m_code.code()));
return exit.m_code.code().executableAddress();
}
} } // namespace JSC::FTL
#endif // ENABLE(FTL_JIT)