| /* |
| * Copyright (C) 2011-2019 Apple Inc. All rights reserved. |
| * |
| * Redistribution and use in source and binary forms, with or without |
| * modification, are permitted provided that the following conditions |
| * are met: |
| * 1. Redistributions of source code must retain the above copyright |
| * notice, this list of conditions and the following disclaimer. |
| * 2. Redistributions in binary form must reproduce the above copyright |
| * notice, this list of conditions and the following disclaimer in the |
| * documentation and/or other materials provided with the distribution. |
| * |
| * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY |
| * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
| * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
| * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR |
| * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, |
| * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, |
| * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR |
| * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY |
| * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| */ |
| |
| #include "config.h" |
| #include "DFGOSRExit.h" |
| |
| #if ENABLE(DFG_JIT) |
| |
| #include "AssemblyHelpers.h" |
| #include "BytecodeUseDef.h" |
| #include "ClonedArguments.h" |
| #include "DFGGraph.h" |
| #include "DFGMayExit.h" |
| #include "DFGOSRExitCompilerCommon.h" |
| #include "DFGOperations.h" |
| #include "DFGSpeculativeJIT.h" |
| #include "DirectArguments.h" |
| #include "FrameTracers.h" |
| #include "InlineCallFrame.h" |
| #include "JSCInlines.h" |
| #include "JSCJSValue.h" |
| #include "OperandsInlines.h" |
| #include "ProbeContext.h" |
| #include "ProbeFrame.h" |
| |
| namespace JSC { namespace DFG { |
| |
| // Probe based OSR Exit. |
| |
| using CPUState = Probe::CPUState; |
| using Context = Probe::Context; |
| using Frame = Probe::Frame; |
| |
| static void reifyInlinedCallFrames(Probe::Context&, CodeBlock* baselineCodeBlock, const OSRExitBase&); |
| static void adjustAndJumpToTarget(Probe::Context&, VM&, CodeBlock*, CodeBlock* baselineCodeBlock, OSRExit&); |
| static void printOSRExit(Context&, uint32_t osrExitIndex, const OSRExit&); |
| |
| static JSValue jsValueFor(CPUState& cpu, JSValueSource source) |
| { |
| if (source.isAddress()) { |
| JSValue result; |
| std::memcpy(&result, cpu.gpr<uint8_t*>(source.base()) + source.offset(), sizeof(JSValue)); |
| return result; |
| } |
| #if USE(JSVALUE64) |
| return JSValue::decode(cpu.gpr<EncodedJSValue>(source.gpr())); |
| #else |
| if (source.hasKnownTag()) |
| return JSValue(source.tag(), cpu.gpr<int32_t>(source.payloadGPR())); |
| return JSValue(cpu.gpr<int32_t>(source.tagGPR()), cpu.gpr<int32_t>(source.payloadGPR())); |
| #endif |
| } |
| |
| #if NUMBER_OF_CALLEE_SAVES_REGISTERS > 0 |
| |
| // Based on AssemblyHelpers::emitRestoreCalleeSavesFor(). |
| static void restoreCalleeSavesFor(Context& context, CodeBlock* codeBlock) |
| { |
| ASSERT(codeBlock); |
| |
| const RegisterAtOffsetList* calleeSaves = codeBlock->calleeSaveRegisters(); |
| RegisterSet dontRestoreRegisters = RegisterSet(RegisterSet::stackRegisters(), RegisterSet::allFPRs()); |
| unsigned registerCount = calleeSaves->size(); |
| |
| UCPURegister* physicalStackFrame = context.fp<UCPURegister*>(); |
| for (unsigned i = 0; i < registerCount; i++) { |
| RegisterAtOffset entry = calleeSaves->at(i); |
| if (dontRestoreRegisters.get(entry.reg())) |
| continue; |
| // The callee saved values come from the original stack, not the recovered stack. |
| // Hence, we read the values directly from the physical stack memory instead of |
| // going through context.stack(). |
| ASSERT(!(entry.offset() % sizeof(UCPURegister))); |
| context.gpr(entry.reg().gpr()) = physicalStackFrame[entry.offset() / sizeof(UCPURegister)]; |
| } |
| } |
| |
| // Based on AssemblyHelpers::emitSaveCalleeSavesFor(). |
| static void saveCalleeSavesFor(Context& context, CodeBlock* codeBlock) |
| { |
| auto& stack = context.stack(); |
| ASSERT(codeBlock); |
| |
| const RegisterAtOffsetList* calleeSaves = codeBlock->calleeSaveRegisters(); |
| RegisterSet dontSaveRegisters = RegisterSet(RegisterSet::stackRegisters(), RegisterSet::allFPRs()); |
| unsigned registerCount = calleeSaves->size(); |
| |
| for (unsigned i = 0; i < registerCount; i++) { |
| RegisterAtOffset entry = calleeSaves->at(i); |
| if (dontSaveRegisters.get(entry.reg())) |
| continue; |
| stack.set(context.fp(), entry.offset(), context.gpr<UCPURegister>(entry.reg().gpr())); |
| } |
| } |
| |
| // Based on AssemblyHelpers::restoreCalleeSavesFromVMEntryFrameCalleeSavesBuffer(). |
| static void restoreCalleeSavesFromVMEntryFrameCalleeSavesBuffer(Context& context) |
| { |
| VM& vm = *context.arg<VM*>(); |
| |
| RegisterAtOffsetList* allCalleeSaves = RegisterSet::vmCalleeSaveRegisterOffsets(); |
| RegisterSet dontRestoreRegisters = RegisterSet::stackRegisters(); |
| unsigned registerCount = allCalleeSaves->size(); |
| |
| VMEntryRecord* entryRecord = vmEntryRecord(vm.topEntryFrame); |
| UCPURegister* calleeSaveBuffer = reinterpret_cast<UCPURegister*>(entryRecord->calleeSaveRegistersBuffer); |
| |
| // Restore all callee saves. |
| for (unsigned i = 0; i < registerCount; i++) { |
| RegisterAtOffset entry = allCalleeSaves->at(i); |
| if (dontRestoreRegisters.get(entry.reg())) |
| continue; |
| size_t uintptrOffset = entry.offset() / sizeof(UCPURegister); |
| if (entry.reg().isGPR()) |
| context.gpr(entry.reg().gpr()) = calleeSaveBuffer[uintptrOffset]; |
| else { |
| #if USE(JSVALUE64) |
| context.fpr(entry.reg().fpr()) = bitwise_cast<double>(calleeSaveBuffer[uintptrOffset]); |
| #else |
| // FIXME: <https://webkit.org/b/193275> support callee-saved floating point registers on 32-bit architectures |
| RELEASE_ASSERT_NOT_REACHED(); |
| #endif |
| } |
| } |
| } |
| |
| // Based on AssemblyHelpers::copyCalleeSavesToVMEntryFrameCalleeSavesBuffer(). |
| static void copyCalleeSavesToVMEntryFrameCalleeSavesBuffer(Context& context) |
| { |
| VM& vm = *context.arg<VM*>(); |
| auto& stack = context.stack(); |
| |
| VMEntryRecord* entryRecord = vmEntryRecord(vm.topEntryFrame); |
| void* calleeSaveBuffer = entryRecord->calleeSaveRegistersBuffer; |
| |
| RegisterAtOffsetList* allCalleeSaves = RegisterSet::vmCalleeSaveRegisterOffsets(); |
| RegisterSet dontCopyRegisters = RegisterSet::stackRegisters(); |
| unsigned registerCount = allCalleeSaves->size(); |
| |
| for (unsigned i = 0; i < registerCount; i++) { |
| RegisterAtOffset entry = allCalleeSaves->at(i); |
| if (dontCopyRegisters.get(entry.reg())) |
| continue; |
| if (entry.reg().isGPR()) |
| stack.set(calleeSaveBuffer, entry.offset(), context.gpr<UCPURegister>(entry.reg().gpr())); |
| else { |
| #if USE(JSVALUE64) |
| stack.set(calleeSaveBuffer, entry.offset(), context.fpr<UCPURegister>(entry.reg().fpr())); |
| #else |
| // FIXME: <https://webkit.org/b/193275> support callee-saved floating point registers on 32-bit architectures |
| RELEASE_ASSERT_NOT_REACHED(); |
| #endif |
| } |
| } |
| } |
| |
| // Based on AssemblyHelpers::emitSaveOrCopyCalleeSavesFor(). |
| static void saveOrCopyCalleeSavesFor(Context& context, CodeBlock* codeBlock, VirtualRegister offsetVirtualRegister, bool wasCalledViaTailCall) |
| { |
| Frame frame(context.fp(), context.stack()); |
| ASSERT(codeBlock); |
| |
| const RegisterAtOffsetList* calleeSaves = codeBlock->calleeSaveRegisters(); |
| RegisterSet dontSaveRegisters = RegisterSet(RegisterSet::stackRegisters(), RegisterSet::allFPRs()); |
| unsigned registerCount = calleeSaves->size(); |
| |
| RegisterSet baselineCalleeSaves = RegisterSet::llintBaselineCalleeSaveRegisters(); |
| |
| for (unsigned i = 0; i < registerCount; i++) { |
| RegisterAtOffset entry = calleeSaves->at(i); |
| if (dontSaveRegisters.get(entry.reg())) |
| continue; |
| |
| uintptr_t savedRegisterValue; |
| |
| if (wasCalledViaTailCall && baselineCalleeSaves.get(entry.reg())) |
| savedRegisterValue = frame.get<uintptr_t>(entry.offset()); |
| else |
| savedRegisterValue = context.gpr(entry.reg().gpr()); |
| |
| frame.set(offsetVirtualRegister.offsetInBytes() + entry.offset(), savedRegisterValue); |
| } |
| } |
| #else // not NUMBER_OF_CALLEE_SAVES_REGISTERS > 0 |
| |
| static void restoreCalleeSavesFor(Context&, CodeBlock*) { } |
| static void saveCalleeSavesFor(Context&, CodeBlock*) { } |
| static void restoreCalleeSavesFromVMEntryFrameCalleeSavesBuffer(Context&) { } |
| static void copyCalleeSavesToVMEntryFrameCalleeSavesBuffer(Context&) { } |
| static void saveOrCopyCalleeSavesFor(Context&, CodeBlock*, VirtualRegister, bool) { } |
| |
| #endif // NUMBER_OF_CALLEE_SAVES_REGISTERS > 0 |
| |
| static JSCell* createDirectArgumentsDuringExit(Context& context, CodeBlock* codeBlock, InlineCallFrame* inlineCallFrame, JSFunction* callee, int32_t argumentCount) |
| { |
| VM& vm = *context.arg<VM*>(); |
| |
| ASSERT(vm.heap.isDeferred()); |
| |
| if (inlineCallFrame) |
| codeBlock = baselineCodeBlockForInlineCallFrame(inlineCallFrame); |
| |
| unsigned length = argumentCount - 1; |
| unsigned capacity = std::max(length, static_cast<unsigned>(codeBlock->numParameters() - 1)); |
| DirectArguments* result = DirectArguments::create( |
| vm, codeBlock->globalObject()->directArgumentsStructure(), length, capacity); |
| |
| result->setCallee(vm, callee); |
| |
| void* frameBase = context.fp<Register*>() + (inlineCallFrame ? inlineCallFrame->stackOffset : 0); |
| Frame frame(frameBase, context.stack()); |
| for (unsigned i = length; i--;) |
| result->setIndexQuickly(vm, i, frame.argument(i)); |
| |
| return result; |
| } |
| |
| static JSCell* createClonedArgumentsDuringExit(Context& context, CodeBlock* codeBlock, InlineCallFrame* inlineCallFrame, JSFunction* callee, int32_t argumentCount) |
| { |
| VM& vm = *context.arg<VM*>(); |
| |
| ASSERT(vm.heap.isDeferred()); |
| |
| if (inlineCallFrame) |
| codeBlock = baselineCodeBlockForInlineCallFrame(inlineCallFrame); |
| |
| JSGlobalObject* globalObject = codeBlock->globalObject(); |
| unsigned length = argumentCount - 1; |
| ClonedArguments* result = ClonedArguments::createEmpty( |
| vm, globalObject->clonedArgumentsStructure(), callee, length); |
| |
| void* frameBase = context.fp<Register*>() + (inlineCallFrame ? inlineCallFrame->stackOffset : 0); |
| Frame frame(frameBase, context.stack()); |
| for (unsigned i = length; i--;) |
| result->putDirectIndex(globalObject, i, frame.argument(i)); |
| return result; |
| } |
| |
| static void emitRestoreArguments(Context& context, CodeBlock* codeBlock, DFG::JITCode* dfgJITCode, const Operands<ValueRecovery>& operands) |
| { |
| Frame frame(context.fp(), context.stack()); |
| |
| HashMap<MinifiedID, int> alreadyAllocatedArguments; // Maps phantom arguments node ID to operand. |
| for (size_t index = 0; index < operands.size(); ++index) { |
| const ValueRecovery& recovery = operands[index]; |
| int operand = operands.operandForIndex(index); |
| |
| if (recovery.technique() != DirectArgumentsThatWereNotCreated |
| && recovery.technique() != ClonedArgumentsThatWereNotCreated) |
| continue; |
| |
| MinifiedID id = recovery.nodeID(); |
| auto iter = alreadyAllocatedArguments.find(id); |
| if (iter != alreadyAllocatedArguments.end()) { |
| frame.setOperand(operand, frame.operand(iter->value)); |
| continue; |
| } |
| |
| InlineCallFrame* inlineCallFrame = |
| dfgJITCode->minifiedDFG.at(id)->inlineCallFrame(); |
| |
| int stackOffset; |
| if (inlineCallFrame) |
| stackOffset = inlineCallFrame->stackOffset; |
| else |
| stackOffset = 0; |
| |
| JSFunction* callee; |
| if (!inlineCallFrame || inlineCallFrame->isClosureCall) |
| callee = jsCast<JSFunction*>(frame.operand(stackOffset + CallFrameSlot::callee).asCell()); |
| else |
| callee = jsCast<JSFunction*>(inlineCallFrame->calleeRecovery.constant().asCell()); |
| |
| int32_t argumentCount; |
| if (!inlineCallFrame || inlineCallFrame->isVarargs()) |
| argumentCount = frame.operand<int32_t>(stackOffset + CallFrameSlot::argumentCountIncludingThis, PayloadOffset); |
| else |
| argumentCount = inlineCallFrame->argumentCountIncludingThis; |
| |
| JSCell* argumentsObject; |
| switch (recovery.technique()) { |
| case DirectArgumentsThatWereNotCreated: |
| argumentsObject = createDirectArgumentsDuringExit(context, codeBlock, inlineCallFrame, callee, argumentCount); |
| break; |
| case ClonedArgumentsThatWereNotCreated: |
| argumentsObject = createClonedArgumentsDuringExit(context, codeBlock, inlineCallFrame, callee, argumentCount); |
| break; |
| default: |
| RELEASE_ASSERT_NOT_REACHED(); |
| break; |
| } |
| frame.setOperand(operand, JSValue(argumentsObject)); |
| |
| alreadyAllocatedArguments.add(id, operand); |
| } |
| } |
| |
| // The following is a list of extra initializations that need to be done in order |
| // of most likely needed (lower enum value) to least likely needed (higher enum value). |
| // Each level initialization includes the previous lower enum value (see use of the |
| // extraInitializationLevel value below). |
| enum class ExtraInitializationLevel { |
| None, |
| SpeculationRecovery, |
| ValueProfileUpdate, |
| ArrayProfileUpdate, |
| Other |
| }; |
| |
| void OSRExit::executeOSRExit(Context& context) |
| { |
| VM& vm = *context.arg<VM*>(); |
| auto scope = DECLARE_THROW_SCOPE(vm); |
| |
| CallFrame* callFrame = context.fp<CallFrame*>(); |
| ASSERT(&callFrame->deprecatedVM() == &vm); |
| auto& cpu = context.cpu; |
| |
| if (validateDFGDoesGC) { |
| // We're about to exit optimized code. So, there's no longer any optimized |
| // code running that expects no GC. |
| vm.heap.setExpectDoesGC(true); |
| } |
| |
| if (vm.callFrameForCatch) { |
| callFrame = vm.callFrameForCatch; |
| context.fp() = callFrame; |
| } |
| |
| CodeBlock* codeBlock = callFrame->codeBlock(); |
| ASSERT(codeBlock); |
| ASSERT(codeBlock->jitType() == JITType::DFGJIT); |
| |
| // It's sort of preferable that we don't GC while in here. Anyways, doing so wouldn't |
| // really be profitable. |
| DeferGCForAWhile deferGC(vm.heap); |
| |
| uint32_t exitIndex = vm.osrExitIndex; |
| DFG::JITCode* dfgJITCode = codeBlock->jitCode()->dfg(); |
| OSRExit& exit = dfgJITCode->osrExit[exitIndex]; |
| |
| ASSERT(!vm.callFrameForCatch || exit.m_kind == GenericUnwind); |
| EXCEPTION_ASSERT_UNUSED(scope, !!scope.exception() || !exit.isExceptionHandler()); |
| |
| if (UNLIKELY(!exit.exitState)) { |
| ExtraInitializationLevel extraInitializationLevel = ExtraInitializationLevel::None; |
| |
| // We only need to execute this block once for each OSRExit record. The computed |
| // results will be cached in the OSRExitState record for use of the rest of the |
| // exit ramp code. |
| |
| CodeBlock* baselineCodeBlock = codeBlock->baselineAlternative(); |
| ASSERT(JITCode::isBaselineCode(baselineCodeBlock->jitType())); |
| |
| SpeculationRecovery* recovery = nullptr; |
| if (exit.m_recoveryIndex != UINT_MAX) { |
| recovery = &dfgJITCode->speculationRecovery[exit.m_recoveryIndex]; |
| extraInitializationLevel = std::max(extraInitializationLevel, ExtraInitializationLevel::SpeculationRecovery); |
| } |
| |
| if (UNLIKELY(exit.m_kind == GenericUnwind)) |
| extraInitializationLevel = std::max(extraInitializationLevel, ExtraInitializationLevel::Other); |
| |
| ArrayProfile* arrayProfile = nullptr; |
| if (!!exit.m_jsValueSource) { |
| if (exit.m_valueProfile) |
| extraInitializationLevel = std::max(extraInitializationLevel, ExtraInitializationLevel::ValueProfileUpdate); |
| if (exit.m_kind == BadCache || exit.m_kind == BadIndexingType) { |
| CodeOrigin codeOrigin = exit.m_codeOriginForExitProfile; |
| CodeBlock* profiledCodeBlock = baselineCodeBlockForOriginAndBaselineCodeBlock(codeOrigin, baselineCodeBlock); |
| arrayProfile = profiledCodeBlock->getArrayProfile(codeOrigin.bytecodeIndex()); |
| if (arrayProfile) |
| extraInitializationLevel = std::max(extraInitializationLevel, ExtraInitializationLevel::ArrayProfileUpdate); |
| } |
| } |
| |
| int32_t activeThreshold = baselineCodeBlock->adjustedCounterValue(Options::thresholdForOptimizeAfterLongWarmUp()); |
| double adjustedThreshold = applyMemoryUsageHeuristicsAndConvertToInt(activeThreshold, baselineCodeBlock); |
| ASSERT(adjustedThreshold > 0); |
| adjustedThreshold = BaselineExecutionCounter::clippedThreshold(codeBlock->globalObject(), adjustedThreshold); |
| |
| CodeBlock* codeBlockForExit = baselineCodeBlockForOriginAndBaselineCodeBlock(exit.m_codeOrigin, baselineCodeBlock); |
| bool exitToLLInt = Options::forceOSRExitToLLInt() || codeBlockForExit->jitType() == JITType::InterpreterThunk; |
| void* jumpTarget; |
| if (exitToLLInt) { |
| BytecodeIndex bytecodeOffset = exit.m_codeOrigin.bytecodeIndex(); |
| const Instruction& currentInstruction = *codeBlockForExit->instructions().at(bytecodeOffset).ptr(); |
| MacroAssemblerCodePtr<JSEntryPtrTag> destination = LLInt::getCodePtr<JSEntryPtrTag>(currentInstruction); |
| jumpTarget = destination.executableAddress(); |
| } else { |
| const JITCodeMap& codeMap = codeBlockForExit->jitCodeMap(); |
| CodeLocationLabel<JSEntryPtrTag> codeLocation = codeMap.find(exit.m_codeOrigin.bytecodeIndex()); |
| ASSERT(codeLocation); |
| jumpTarget = codeLocation.executableAddress(); |
| } |
| |
| // Compute the value recoveries. |
| Operands<ValueRecovery> operands; |
| Vector<UndefinedOperandSpan> undefinedOperandSpans; |
| dfgJITCode->variableEventStream.reconstruct(codeBlock, exit.m_codeOrigin, dfgJITCode->minifiedDFG, exit.m_streamIndex, operands, &undefinedOperandSpans); |
| ptrdiff_t stackPointerOffset = -static_cast<ptrdiff_t>(codeBlock->jitCode()->dfgCommon()->requiredRegisterCountForExit) * sizeof(Register); |
| |
| exit.exitState = adoptRef(new OSRExitState(exit, codeBlock, baselineCodeBlock, operands, WTFMove(undefinedOperandSpans), recovery, stackPointerOffset, activeThreshold, adjustedThreshold, jumpTarget, arrayProfile, exitToLLInt)); |
| |
| if (UNLIKELY(vm.m_perBytecodeProfiler && codeBlock->jitCode()->dfgCommon()->compilation)) { |
| Profiler::Database& database = *vm.m_perBytecodeProfiler; |
| Profiler::Compilation* compilation = codeBlock->jitCode()->dfgCommon()->compilation.get(); |
| |
| Profiler::OSRExit* profilerExit = compilation->addOSRExit( |
| exitIndex, Profiler::OriginStack(database, codeBlock, exit.m_codeOrigin), |
| exit.m_kind, exit.m_kind == UncountableInvalidation); |
| exit.exitState->profilerExit = profilerExit; |
| extraInitializationLevel = std::max(extraInitializationLevel, ExtraInitializationLevel::Other); |
| } |
| |
| if (UNLIKELY(Options::printEachOSRExit())) |
| extraInitializationLevel = std::max(extraInitializationLevel, ExtraInitializationLevel::Other); |
| |
| exit.exitState->extraInitializationLevel = extraInitializationLevel; |
| |
| if (UNLIKELY(Options::verboseOSR() || Options::verboseDFGOSRExit())) { |
| dataLogF("DFG OSR exit #%u (%s, %s) from %s, with operands = %s\n", |
| exitIndex, toCString(exit.m_codeOrigin).data(), |
| exitKindToString(exit.m_kind), toCString(*codeBlock).data(), |
| toCString(ignoringContext<DumpContext>(operands)).data()); |
| } |
| } |
| |
| OSRExitState& exitState = *exit.exitState.get(); |
| CodeBlock* baselineCodeBlock = exitState.baselineCodeBlock; |
| ASSERT(JITCode::isBaselineCode(baselineCodeBlock->jitType())); |
| |
| Operands<ValueRecovery>& operands = exitState.operands; |
| Vector<UndefinedOperandSpan>& undefinedOperandSpans = exitState.undefinedOperandSpans; |
| |
| context.sp() = context.fp<uint8_t*>() + exitState.stackPointerOffset; |
| |
| // The only reason for using this do while loop is so we can break out midway when appropriate. |
| do { |
| auto extraInitializationLevel = static_cast<ExtraInitializationLevel>(exitState.extraInitializationLevel); |
| |
| if (extraInitializationLevel == ExtraInitializationLevel::None) |
| break; |
| |
| // Begin extra initilization level: SpeculationRecovery |
| |
| // We need to do speculation recovery first because array profiling and value profiling |
| // may rely on a value that it recovers. However, that doesn't mean that it is likely |
| // to have a recovery value. So, we'll decorate it as UNLIKELY. |
| SpeculationRecovery* recovery = exitState.recovery; |
| if (UNLIKELY(recovery)) { |
| switch (recovery->type()) { |
| case SpeculativeAdd: |
| cpu.gpr(recovery->dest()) = cpu.gpr<uint32_t>(recovery->dest()) - cpu.gpr<uint32_t>(recovery->src()); |
| #if USE(JSVALUE64) |
| ASSERT(!(cpu.gpr(recovery->dest()) >> 32)); |
| cpu.gpr(recovery->dest()) |= JSValue::NumberTag; |
| #endif |
| break; |
| |
| case SpeculativeAddSelf: |
| cpu.gpr(recovery->dest()) = static_cast<uint32_t>(cpu.gpr<int32_t>(recovery->dest()) >> 1) ^ 0x80000000U; |
| #if USE(JSVALUE64) |
| ASSERT(!(cpu.gpr(recovery->dest()) >> 32)); |
| cpu.gpr(recovery->dest()) |= JSValue::NumberTag; |
| #endif |
| break; |
| |
| case SpeculativeAddImmediate: |
| cpu.gpr(recovery->dest()) = (cpu.gpr<uint32_t>(recovery->dest()) - recovery->immediate()); |
| #if USE(JSVALUE64) |
| ASSERT(!(cpu.gpr(recovery->dest()) >> 32)); |
| cpu.gpr(recovery->dest()) |= JSValue::NumberTag; |
| #endif |
| break; |
| |
| case BooleanSpeculationCheck: |
| #if USE(JSVALUE64) |
| cpu.gpr(recovery->dest()) = cpu.gpr(recovery->dest()) ^ JSValue::ValueFalse; |
| #endif |
| break; |
| |
| default: |
| break; |
| } |
| } |
| if (extraInitializationLevel <= ExtraInitializationLevel::SpeculationRecovery) |
| break; |
| |
| // Begin extra initilization level: ValueProfileUpdate |
| JSValue profiledValue; |
| if (!!exit.m_jsValueSource) { |
| profiledValue = jsValueFor(cpu, exit.m_jsValueSource); |
| if (MethodOfGettingAValueProfile profile = exit.m_valueProfile) |
| profile.reportValue(profiledValue); |
| } |
| if (extraInitializationLevel <= ExtraInitializationLevel::ValueProfileUpdate) |
| break; |
| |
| // Begin extra initilization level: ArrayProfileUpdate |
| if (ArrayProfile* arrayProfile = exitState.arrayProfile) { |
| ASSERT(!!exit.m_jsValueSource); |
| ASSERT(exit.m_kind == BadCache || exit.m_kind == BadIndexingType); |
| CodeBlock* profiledCodeBlock = baselineCodeBlockForOriginAndBaselineCodeBlock(exit.m_codeOriginForExitProfile, baselineCodeBlock); |
| const Instruction* instruction = profiledCodeBlock->instructions().at(exit.m_codeOriginForExitProfile.bytecodeIndex()).ptr(); |
| bool doProfile = !instruction->is<OpGetById>() || instruction->as<OpGetById>().metadata(profiledCodeBlock).m_modeMetadata.mode == GetByIdMode::ArrayLength; |
| if (doProfile) { |
| Structure* structure = profiledValue.asCell()->structure(vm); |
| arrayProfile->observeStructure(structure); |
| arrayProfile->observeArrayMode(arrayModesFromStructure(structure)); |
| } |
| } |
| if (extraInitializationLevel <= ExtraInitializationLevel::ArrayProfileUpdate) |
| break; |
| |
| // Begin Extra initilization level: Other |
| if (UNLIKELY(exit.m_kind == GenericUnwind)) { |
| // We are acting as a defacto op_catch because we arrive here from genericUnwind(). |
| // So, we must restore our call frame and stack pointer. |
| restoreCalleeSavesFromVMEntryFrameCalleeSavesBuffer(context); |
| ASSERT(context.fp() == vm.callFrameForCatch); |
| } |
| |
| if (exitState.profilerExit) |
| exitState.profilerExit->incCount(); |
| |
| if (UNLIKELY(Options::printEachOSRExit())) |
| printOSRExit(context, vm.osrExitIndex, exit); |
| |
| } while (false); // End extra initialization. |
| |
| Frame frame(cpu.fp(), context.stack()); |
| ASSERT(!(context.fp<uintptr_t>() & 0x7)); |
| |
| #if USE(JSVALUE64) |
| ASSERT(cpu.gpr<int64_t>(GPRInfo::numberTagRegister) == JSValue::NumberTag); |
| ASSERT(cpu.gpr<int64_t>(GPRInfo::notCellMaskRegister) == JSValue::NotCellMask); |
| #endif |
| |
| // Do all data format conversions and store the results into the stack. |
| // Note: we need to recover values before restoring callee save registers below |
| // because the recovery may rely on values in some of callee save registers. |
| |
| int calleeSaveSpaceAsVirtualRegisters = static_cast<int>(baselineCodeBlock->calleeSaveSpaceAsVirtualRegisters()); |
| size_t numberOfOperands = operands.size(); |
| size_t numUndefinedOperandSpans = undefinedOperandSpans.size(); |
| |
| size_t nextUndefinedSpanIndex = 0; |
| size_t nextUndefinedOperandIndex = numberOfOperands; |
| if (numUndefinedOperandSpans) |
| nextUndefinedOperandIndex = undefinedOperandSpans[nextUndefinedSpanIndex].firstIndex; |
| |
| JSValue undefined = jsUndefined(); |
| for (size_t spanIndex = 0; spanIndex < numUndefinedOperandSpans; ++spanIndex) { |
| auto& span = undefinedOperandSpans[spanIndex]; |
| int firstOffset = span.minOffset; |
| int lastOffset = firstOffset + span.numberOfRegisters; |
| |
| for (int offset = firstOffset; offset < lastOffset; ++offset) |
| frame.setOperand(offset, undefined); |
| } |
| |
| for (size_t index = 0; index < numberOfOperands; ++index) { |
| const ValueRecovery& recovery = operands[index]; |
| VirtualRegister reg = operands.virtualRegisterForIndex(index); |
| |
| if (UNLIKELY(index == nextUndefinedOperandIndex)) { |
| index += undefinedOperandSpans[nextUndefinedSpanIndex++].numberOfRegisters - 1; |
| if (nextUndefinedSpanIndex < numUndefinedOperandSpans) |
| nextUndefinedOperandIndex = undefinedOperandSpans[nextUndefinedSpanIndex].firstIndex; |
| else |
| nextUndefinedOperandIndex = numberOfOperands; |
| continue; |
| } |
| |
| if (reg.isLocal() && reg.toLocal() < calleeSaveSpaceAsVirtualRegisters) |
| continue; |
| |
| int operand = reg.offset(); |
| |
| switch (recovery.technique()) { |
| case DisplacedInJSStack: |
| frame.setOperand(operand, callFrame->r(recovery.virtualRegister()).asanUnsafeJSValue()); |
| break; |
| |
| case InFPR: |
| frame.setOperand(operand, cpu.fpr<JSValue>(recovery.fpr())); |
| break; |
| |
| #if USE(JSVALUE64) |
| case InGPR: |
| frame.setOperand(operand, cpu.gpr<JSValue>(recovery.gpr())); |
| break; |
| #else |
| case InPair: |
| frame.setOperand(operand, JSValue(cpu.gpr<int32_t>(recovery.tagGPR()), cpu.gpr<int32_t>(recovery.payloadGPR()))); |
| break; |
| #endif |
| |
| case UnboxedCellInGPR: |
| frame.setOperand(operand, JSValue(cpu.gpr<JSCell*>(recovery.gpr()))); |
| break; |
| |
| case CellDisplacedInJSStack: |
| frame.setOperand(operand, JSValue(callFrame->r(recovery.virtualRegister()).asanUnsafeUnboxedCell())); |
| break; |
| |
| #if USE(JSVALUE32_64) |
| case UnboxedBooleanInGPR: |
| frame.setOperand(operand, jsBoolean(cpu.gpr<bool>(recovery.gpr()))); |
| break; |
| #endif |
| |
| case BooleanDisplacedInJSStack: |
| #if USE(JSVALUE64) |
| frame.setOperand(operand, callFrame->r(recovery.virtualRegister()).asanUnsafeJSValue()); |
| #else |
| frame.setOperand(operand, jsBoolean(callFrame->r(recovery.virtualRegister()).asanUnsafeJSValue().payload())); |
| #endif |
| break; |
| |
| case UnboxedInt32InGPR: |
| frame.setOperand(operand, JSValue(cpu.gpr<int32_t>(recovery.gpr()))); |
| break; |
| |
| case Int32DisplacedInJSStack: |
| frame.setOperand(operand, JSValue(callFrame->r(recovery.virtualRegister()).asanUnsafeUnboxedInt32())); |
| break; |
| |
| #if USE(JSVALUE64) |
| case UnboxedInt52InGPR: |
| frame.setOperand(operand, JSValue(cpu.gpr<int64_t>(recovery.gpr()) >> JSValue::int52ShiftAmount)); |
| break; |
| |
| case Int52DisplacedInJSStack: |
| frame.setOperand(operand, JSValue(callFrame->r(recovery.virtualRegister()).asanUnsafeUnboxedInt52())); |
| break; |
| |
| case UnboxedStrictInt52InGPR: |
| frame.setOperand(operand, JSValue(cpu.gpr<int64_t>(recovery.gpr()))); |
| break; |
| |
| case StrictInt52DisplacedInJSStack: |
| frame.setOperand(operand, JSValue(callFrame->r(recovery.virtualRegister()).asanUnsafeUnboxedStrictInt52())); |
| break; |
| #endif |
| |
| case UnboxedDoubleInFPR: |
| frame.setOperand(operand, JSValue(JSValue::EncodeAsDouble, purifyNaN(cpu.fpr(recovery.fpr())))); |
| break; |
| |
| case DoubleDisplacedInJSStack: |
| frame.setOperand(operand, JSValue(JSValue::EncodeAsDouble, purifyNaN(callFrame->r(recovery.virtualRegister()).asanUnsafeUnboxedDouble()))); |
| break; |
| |
| case Constant: |
| frame.setOperand(operand, recovery.constant()); |
| break; |
| |
| case DirectArgumentsThatWereNotCreated: |
| case ClonedArgumentsThatWereNotCreated: |
| // Don't do this, yet. |
| break; |
| |
| default: |
| RELEASE_ASSERT_NOT_REACHED(); |
| break; |
| } |
| } |
| |
| // Restore the DFG callee saves and then save the ones the baseline JIT uses. |
| restoreCalleeSavesFor(context, codeBlock); |
| saveCalleeSavesFor(context, baselineCodeBlock); |
| |
| #if USE(JSVALUE64) |
| cpu.gpr(GPRInfo::numberTagRegister) = static_cast<JSC::UCPURegister>(JSValue::NumberTag); |
| cpu.gpr(GPRInfo::notCellMaskRegister) = static_cast<JSC::UCPURegister>(JSValue::NotCellMask); |
| #endif |
| |
| if (exit.isExceptionHandler()) |
| copyCalleeSavesToVMEntryFrameCalleeSavesBuffer(context); |
| |
| // Now that things on the stack are recovered, do the arguments recovery. We assume that arguments |
| // recoveries don't recursively refer to each other. But, we don't try to assume that they only |
| // refer to certain ranges of locals. Hence why we need to do this here, once the stack is sensible. |
| // Note that we also roughly assume that the arguments might still be materialized outside of its |
| // inline call frame scope - but for now the DFG wouldn't do that. |
| |
| DFG::emitRestoreArguments(context, codeBlock, dfgJITCode, operands); |
| |
| // Adjust the old JIT's execute counter. Since we are exiting OSR, we know |
| // that all new calls into this code will go to the new JIT, so the execute |
| // counter only affects call frames that performed OSR exit and call frames |
| // that were still executing the old JIT at the time of another call frame's |
| // OSR exit. We want to ensure that the following is true: |
| // |
| // (a) Code the performs an OSR exit gets a chance to reenter optimized |
| // code eventually, since optimized code is faster. But we don't |
| // want to do such reentery too aggressively (see (c) below). |
| // |
| // (b) If there is code on the call stack that is still running the old |
| // JIT's code and has never OSR'd, then it should get a chance to |
| // perform OSR entry despite the fact that we've exited. |
| // |
| // (c) Code the performs an OSR exit should not immediately retry OSR |
| // entry, since both forms of OSR are expensive. OSR entry is |
| // particularly expensive. |
| // |
| // (d) Frequent OSR failures, even those that do not result in the code |
| // running in a hot loop, result in recompilation getting triggered. |
| // |
| // To ensure (c), we'd like to set the execute counter to |
| // counterValueForOptimizeAfterWarmUp(). This seems like it would endanger |
| // (a) and (b), since then every OSR exit would delay the opportunity for |
| // every call frame to perform OSR entry. Essentially, if OSR exit happens |
| // frequently and the function has few loops, then the counter will never |
| // become non-negative and OSR entry will never be triggered. OSR entry |
| // will only happen if a loop gets hot in the old JIT, which does a pretty |
| // good job of ensuring (a) and (b). But that doesn't take care of (d), |
| // since each speculation failure would reset the execute counter. |
| // So we check here if the number of speculation failures is significantly |
| // larger than the number of successes (we want 90% success rate), and if |
| // there have been a large enough number of failures. If so, we set the |
| // counter to 0; otherwise we set the counter to |
| // counterValueForOptimizeAfterWarmUp(). |
| |
| if (UNLIKELY(codeBlock->updateOSRExitCounterAndCheckIfNeedToReoptimize(exitState) == CodeBlock::OptimizeAction::ReoptimizeNow)) |
| triggerReoptimizationNow(baselineCodeBlock, codeBlock, &exit); |
| |
| reifyInlinedCallFrames(context, baselineCodeBlock, exit); |
| adjustAndJumpToTarget(context, vm, codeBlock, baselineCodeBlock, exit); |
| } |
| |
| static void reifyInlinedCallFrames(Context& context, CodeBlock* outermostBaselineCodeBlock, const OSRExitBase& exit) |
| { |
| auto& cpu = context.cpu; |
| Frame frame(cpu.fp(), context.stack()); |
| |
| // FIXME: We shouldn't leave holes on the stack when performing an OSR exit |
| // in presence of inlined tail calls. |
| // https://bugs.webkit.org/show_bug.cgi?id=147511 |
| ASSERT(JITCode::isBaselineCode(outermostBaselineCodeBlock->jitType())); |
| frame.setOperand<CodeBlock*>(CallFrameSlot::codeBlock, outermostBaselineCodeBlock); |
| |
| const CodeOrigin* codeOrigin; |
| for (codeOrigin = &exit.m_codeOrigin; codeOrigin && codeOrigin->inlineCallFrame(); codeOrigin = codeOrigin->inlineCallFrame()->getCallerSkippingTailCalls()) { |
| InlineCallFrame* inlineCallFrame = codeOrigin->inlineCallFrame(); |
| CodeBlock* baselineCodeBlock = baselineCodeBlockForOriginAndBaselineCodeBlock(*codeOrigin, outermostBaselineCodeBlock); |
| InlineCallFrame::Kind trueCallerCallKind; |
| CodeOrigin* trueCaller = inlineCallFrame->getCallerSkippingTailCalls(&trueCallerCallKind); |
| void* callerFrame = cpu.fp(); |
| |
| bool callerIsLLInt = false; |
| |
| if (!trueCaller) { |
| ASSERT(inlineCallFrame->isTail()); |
| void* returnPC = frame.get<void*>(CallFrame::returnPCOffset()); |
| #if CPU(ARM64E) |
| void* oldEntrySP = cpu.fp<uint8_t*>() + sizeof(CallerFrameAndPC); |
| void* newEntrySP = cpu.fp<uint8_t*>() + inlineCallFrame->returnPCOffset() + sizeof(void*); |
| returnPC = retagCodePtr(returnPC, bitwise_cast<PtrTag>(oldEntrySP), bitwise_cast<PtrTag>(newEntrySP)); |
| #endif |
| frame.set<void*>(inlineCallFrame->returnPCOffset(), returnPC); |
| callerFrame = frame.get<void*>(CallFrame::callerFrameOffset()); |
| } else { |
| CodeBlock* baselineCodeBlockForCaller = baselineCodeBlockForOriginAndBaselineCodeBlock(*trueCaller, outermostBaselineCodeBlock); |
| BytecodeIndex callBytecodeIndex = trueCaller->bytecodeIndex(); |
| void* jumpTarget = callerReturnPC(baselineCodeBlockForCaller, callBytecodeIndex, trueCallerCallKind, callerIsLLInt).untaggedExecutableAddress(); |
| |
| if (trueCaller->inlineCallFrame()) |
| callerFrame = cpu.fp<uint8_t*>() + trueCaller->inlineCallFrame()->stackOffset * sizeof(EncodedJSValue); |
| |
| #if CPU(ARM64E) |
| void* newEntrySP = cpu.fp<uint8_t*>() + inlineCallFrame->returnPCOffset() + sizeof(void*); |
| jumpTarget = tagCodePtr(jumpTarget, bitwise_cast<PtrTag>(newEntrySP)); |
| #endif |
| frame.set<void*>(inlineCallFrame->returnPCOffset(), jumpTarget); |
| } |
| |
| frame.setOperand<void*>(inlineCallFrame->stackOffset + CallFrameSlot::codeBlock, baselineCodeBlock); |
| |
| // Restore the inline call frame's callee save registers. |
| // If this inlined frame is a tail call that will return back to the original caller, we need to |
| // copy the prior contents of the tag registers already saved for the outer frame to this frame. |
| saveOrCopyCalleeSavesFor(context, baselineCodeBlock, VirtualRegister(inlineCallFrame->stackOffset), !trueCaller); |
| |
| if (callerIsLLInt) { |
| CodeBlock* baselineCodeBlockForCaller = baselineCodeBlockForOriginAndBaselineCodeBlock(*trueCaller, outermostBaselineCodeBlock); |
| frame.set<const void*>(calleeSaveSlot(inlineCallFrame, baselineCodeBlock, LLInt::Registers::metadataTableGPR).offset, baselineCodeBlockForCaller->metadataTable()); |
| #if USE(JSVALUE64) |
| frame.set<const void*>(calleeSaveSlot(inlineCallFrame, baselineCodeBlock, LLInt::Registers::pbGPR).offset, baselineCodeBlockForCaller->instructionsRawPointer()); |
| #endif |
| } |
| |
| if (!inlineCallFrame->isVarargs()) |
| frame.setOperand<uint32_t>(inlineCallFrame->stackOffset + CallFrameSlot::argumentCountIncludingThis, PayloadOffset, inlineCallFrame->argumentCountIncludingThis); |
| ASSERT(callerFrame); |
| frame.set<void*>(inlineCallFrame->callerFrameOffset(), callerFrame); |
| #if USE(JSVALUE64) |
| uint32_t locationBits = CallSiteIndex(codeOrigin->bytecodeIndex()).bits(); |
| frame.setOperand<uint32_t>(inlineCallFrame->stackOffset + CallFrameSlot::argumentCountIncludingThis, TagOffset, locationBits); |
| if (!inlineCallFrame->isClosureCall) |
| frame.setOperand(inlineCallFrame->stackOffset + CallFrameSlot::callee, JSValue(inlineCallFrame->calleeConstant())); |
| #else // USE(JSVALUE64) // so this is the 32-bit part |
| const Instruction* instruction = baselineCodeBlock->instructions().at(codeOrigin->bytecodeIndex()).ptr(); |
| uint32_t locationBits = CallSiteIndex(BytecodeIndex(bitwise_cast<uint32_t>(instruction))).bits(); |
| frame.setOperand<uint32_t>(inlineCallFrame->stackOffset + CallFrameSlot::argumentCountIncludingThis, TagOffset, locationBits); |
| frame.setOperand<uint32_t>(inlineCallFrame->stackOffset + CallFrameSlot::callee, TagOffset, static_cast<uint32_t>(JSValue::CellTag)); |
| if (!inlineCallFrame->isClosureCall) |
| frame.setOperand(inlineCallFrame->stackOffset + CallFrameSlot::callee, PayloadOffset, inlineCallFrame->calleeConstant()); |
| #endif // USE(JSVALUE64) // ending the #else part, so directly above is the 32-bit part |
| } |
| |
| // Don't need to set the toplevel code origin if we only did inline tail calls |
| if (codeOrigin) { |
| #if USE(JSVALUE64) |
| uint32_t locationBits = CallSiteIndex(codeOrigin->bytecodeIndex()).bits(); |
| #else |
| const Instruction* instruction = outermostBaselineCodeBlock->instructions().at(codeOrigin->bytecodeIndex()).ptr(); |
| uint32_t locationBits = CallSiteIndex(BytecodeIndex(bitwise_cast<uint32_t>(instruction))).bits(); |
| #endif |
| frame.setOperand<uint32_t>(CallFrameSlot::argumentCountIncludingThis, TagOffset, locationBits); |
| } |
| } |
| |
| static void adjustAndJumpToTarget(Context& context, VM& vm, CodeBlock* codeBlock, CodeBlock* baselineCodeBlock, OSRExit& exit) |
| { |
| OSRExitState* exitState = exit.exitState.get(); |
| |
| WTF::storeLoadFence(); // The optimizing compiler expects that the OSR exit mechanism will execute this fence. |
| vm.heap.writeBarrier(baselineCodeBlock); |
| |
| // We barrier all inlined frames -- and not just the current inline stack -- |
| // because we don't know which inlined function owns the value profile that |
| // we'll update when we exit. In the case of "f() { a(); b(); }", if both |
| // a and b are inlined, we might exit inside b due to a bad value loaded |
| // from a. |
| // FIXME: MethodOfGettingAValueProfile should remember which CodeBlock owns |
| // the value profile. |
| InlineCallFrameSet* inlineCallFrames = codeBlock->jitCode()->dfgCommon()->inlineCallFrames.get(); |
| if (inlineCallFrames) { |
| for (InlineCallFrame* inlineCallFrame : *inlineCallFrames) |
| vm.heap.writeBarrier(inlineCallFrame->baselineCodeBlock.get()); |
| } |
| |
| auto* exitInlineCallFrame = exit.m_codeOrigin.inlineCallFrame(); |
| if (exitInlineCallFrame) |
| context.fp() = context.fp<uint8_t*>() + exitInlineCallFrame->stackOffset * sizeof(EncodedJSValue); |
| |
| void* jumpTarget = exitState->jumpTarget; |
| ASSERT(jumpTarget); |
| |
| if (exit.isExceptionHandler()) { |
| // Since we're jumping to op_catch, we need to set callFrameForCatch. |
| vm.callFrameForCatch = context.fp<CallFrame*>(); |
| } |
| |
| vm.topCallFrame = context.fp<CallFrame*>(); |
| |
| if (exitState->isJumpToLLInt) { |
| CodeBlock* codeBlockForExit = baselineCodeBlockForOriginAndBaselineCodeBlock(exit.m_codeOrigin, baselineCodeBlock); |
| BytecodeIndex bytecodeIndex = exit.m_codeOrigin.bytecodeIndex(); |
| const Instruction& currentInstruction = *codeBlockForExit->instructions().at(bytecodeIndex).ptr(); |
| |
| context.gpr(LLInt::Registers::metadataTableGPR) = bitwise_cast<uintptr_t>(codeBlockForExit->metadataTable()); |
| #if USE(JSVALUE64) |
| context.gpr(LLInt::Registers::pbGPR) = bitwise_cast<uintptr_t>(codeBlockForExit->instructionsRawPointer()); |
| context.gpr(LLInt::Registers::pcGPR) = static_cast<uintptr_t>(exit.m_codeOrigin.bytecodeIndex().offset()); |
| #else |
| context.gpr(LLInt::Registers::pcGPR) = bitwise_cast<uintptr_t>(¤tInstruction); |
| #endif |
| |
| if (exit.isExceptionHandler()) |
| vm.targetInterpreterPCForThrow = ¤tInstruction; |
| } |
| |
| context.pc() = untagCodePtr<JSEntryPtrTag>(jumpTarget); |
| } |
| |
| static void printOSRExit(Context& context, uint32_t osrExitIndex, const OSRExit& exit) |
| { |
| CallFrame* callFrame = context.fp<CallFrame*>(); |
| CodeBlock* codeBlock = callFrame->codeBlock(); |
| CodeBlock* alternative = codeBlock->alternative(); |
| ExitKind kind = exit.m_kind; |
| BytecodeIndex bytecodeOffset = exit.m_codeOrigin.bytecodeIndex(); |
| |
| dataLog("Speculation failure in ", *codeBlock); |
| dataLog(" @ exit #", osrExitIndex, " (", bytecodeOffset, ", ", exitKindToString(kind), ") with "); |
| if (alternative) { |
| dataLog( |
| "executeCounter = ", alternative->jitExecuteCounter(), |
| ", reoptimizationRetryCounter = ", alternative->reoptimizationRetryCounter(), |
| ", optimizationDelayCounter = ", alternative->optimizationDelayCounter()); |
| } else |
| dataLog("no alternative code block (i.e. we've been jettisoned)"); |
| dataLog(", osrExitCounter = ", codeBlock->osrExitCounter(), "\n"); |
| dataLog(" GPRs at time of exit:"); |
| for (unsigned i = 0; i < GPRInfo::numberOfRegisters; ++i) { |
| GPRReg gpr = GPRInfo::toRegister(i); |
| dataLog(" ", context.gprName(gpr), ":", RawPointer(context.gpr<void*>(gpr))); |
| } |
| dataLog("\n"); |
| dataLog(" FPRs at time of exit:"); |
| for (unsigned i = 0; i < FPRInfo::numberOfRegisters; ++i) { |
| FPRReg fpr = FPRInfo::toRegister(i); |
| dataLog(" ", context.fprName(fpr), ":"); |
| uint64_t bits = context.fpr<uint64_t>(fpr); |
| double value = context.fpr(fpr); |
| dataLogF("%llx:%lf", static_cast<long long>(bits), value); |
| } |
| dataLog("\n"); |
| } |
| |
| // JIT based OSR Exit. |
| |
| OSRExit::OSRExit(ExitKind kind, JSValueSource jsValueSource, MethodOfGettingAValueProfile valueProfile, SpeculativeJIT* jit, unsigned streamIndex, unsigned recoveryIndex) |
| : OSRExitBase(kind, jit->m_origin.forExit, jit->m_origin.semantic, jit->m_origin.wasHoisted) |
| , m_jsValueSource(jsValueSource) |
| , m_valueProfile(valueProfile) |
| , m_recoveryIndex(recoveryIndex) |
| , m_streamIndex(streamIndex) |
| { |
| bool canExit = jit->m_origin.exitOK; |
| if (!canExit && jit->m_currentNode) { |
| ExitMode exitMode = mayExit(jit->m_jit.graph(), jit->m_currentNode); |
| canExit = exitMode == ExitMode::Exits || exitMode == ExitMode::ExitsForExceptions; |
| } |
| DFG_ASSERT(jit->m_jit.graph(), jit->m_currentNode, canExit); |
| } |
| |
| CodeLocationJump<JSInternalPtrTag> OSRExit::codeLocationForRepatch() const |
| { |
| return CodeLocationJump<JSInternalPtrTag>(m_patchableJumpLocation); |
| } |
| |
| void OSRExit::emitRestoreArguments(CCallHelpers& jit, VM& vm, const Operands<ValueRecovery>& operands) |
| { |
| HashMap<MinifiedID, int> alreadyAllocatedArguments; // Maps phantom arguments node ID to operand. |
| for (size_t index = 0; index < operands.size(); ++index) { |
| const ValueRecovery& recovery = operands[index]; |
| int operand = operands.operandForIndex(index); |
| |
| if (recovery.technique() != DirectArgumentsThatWereNotCreated |
| && recovery.technique() != ClonedArgumentsThatWereNotCreated) |
| continue; |
| |
| MinifiedID id = recovery.nodeID(); |
| auto iter = alreadyAllocatedArguments.find(id); |
| if (iter != alreadyAllocatedArguments.end()) { |
| JSValueRegs regs = JSValueRegs::withTwoAvailableRegs(GPRInfo::regT0, GPRInfo::regT1); |
| jit.loadValue(CCallHelpers::addressFor(iter->value), regs); |
| jit.storeValue(regs, CCallHelpers::addressFor(operand)); |
| continue; |
| } |
| |
| InlineCallFrame* inlineCallFrame = |
| jit.codeBlock()->jitCode()->dfg()->minifiedDFG.at(id)->inlineCallFrame(); |
| |
| int stackOffset; |
| if (inlineCallFrame) |
| stackOffset = inlineCallFrame->stackOffset; |
| else |
| stackOffset = 0; |
| |
| if (!inlineCallFrame || inlineCallFrame->isClosureCall) { |
| jit.loadPtr( |
| AssemblyHelpers::addressFor(stackOffset + CallFrameSlot::callee), |
| GPRInfo::regT0); |
| } else { |
| jit.move( |
| AssemblyHelpers::TrustedImmPtr(inlineCallFrame->calleeRecovery.constant().asCell()), |
| GPRInfo::regT0); |
| } |
| |
| if (!inlineCallFrame || inlineCallFrame->isVarargs()) { |
| jit.load32( |
| AssemblyHelpers::payloadFor(stackOffset + CallFrameSlot::argumentCountIncludingThis), |
| GPRInfo::regT1); |
| } else { |
| jit.move( |
| AssemblyHelpers::TrustedImm32(inlineCallFrame->argumentCountIncludingThis), |
| GPRInfo::regT1); |
| } |
| |
| static_assert(std::is_same<decltype(operationCreateDirectArgumentsDuringExit), decltype(operationCreateClonedArgumentsDuringExit)>::value, "We assume these functions have the same signature below."); |
| jit.setupArguments<decltype(operationCreateDirectArgumentsDuringExit)>( |
| AssemblyHelpers::TrustedImmPtr(&vm), AssemblyHelpers::TrustedImmPtr(inlineCallFrame), GPRInfo::regT0, GPRInfo::regT1); |
| jit.prepareCallOperation(vm); |
| switch (recovery.technique()) { |
| case DirectArgumentsThatWereNotCreated: |
| jit.move(AssemblyHelpers::TrustedImmPtr(tagCFunctionPtr<OperationPtrTag>(operationCreateDirectArgumentsDuringExit)), GPRInfo::nonArgGPR0); |
| break; |
| case ClonedArgumentsThatWereNotCreated: |
| jit.move(AssemblyHelpers::TrustedImmPtr(tagCFunctionPtr<OperationPtrTag>(operationCreateClonedArgumentsDuringExit)), GPRInfo::nonArgGPR0); |
| break; |
| default: |
| RELEASE_ASSERT_NOT_REACHED(); |
| break; |
| } |
| jit.call(GPRInfo::nonArgGPR0, OperationPtrTag); |
| jit.storeCell(GPRInfo::returnValueGPR, AssemblyHelpers::addressFor(operand)); |
| |
| alreadyAllocatedArguments.add(id, operand); |
| } |
| } |
| |
| void JIT_OPERATION operationCompileOSRExit(CallFrame* callFrame) |
| { |
| VM& vm = callFrame->deprecatedVM(); |
| auto scope = DECLARE_THROW_SCOPE(vm); |
| |
| if (validateDFGDoesGC) { |
| // We're about to exit optimized code. So, there's no longer any optimized |
| // code running that expects no GC. |
| vm.heap.setExpectDoesGC(true); |
| } |
| |
| if (vm.callFrameForCatch) |
| RELEASE_ASSERT(vm.callFrameForCatch == callFrame); |
| |
| CodeBlock* codeBlock = callFrame->codeBlock(); |
| ASSERT(codeBlock); |
| ASSERT(codeBlock->jitType() == JITType::DFGJIT); |
| |
| // It's sort of preferable that we don't GC while in here. Anyways, doing so wouldn't |
| // really be profitable. |
| DeferGCForAWhile deferGC(vm.heap); |
| |
| uint32_t exitIndex = vm.osrExitIndex; |
| OSRExit& exit = codeBlock->jitCode()->dfg()->osrExit[exitIndex]; |
| |
| ASSERT(!vm.callFrameForCatch || exit.m_kind == GenericUnwind); |
| EXCEPTION_ASSERT_UNUSED(scope, !!scope.exception() || !exit.isExceptionHandler()); |
| |
| // Compute the value recoveries. |
| Operands<ValueRecovery> operands; |
| codeBlock->jitCode()->dfg()->variableEventStream.reconstruct(codeBlock, exit.m_codeOrigin, codeBlock->jitCode()->dfg()->minifiedDFG, exit.m_streamIndex, operands); |
| |
| SpeculationRecovery* recovery = 0; |
| if (exit.m_recoveryIndex != UINT_MAX) |
| recovery = &codeBlock->jitCode()->dfg()->speculationRecovery[exit.m_recoveryIndex]; |
| |
| { |
| CCallHelpers jit(codeBlock); |
| |
| if (exit.m_kind == GenericUnwind) { |
| // We are acting as a defacto op_catch because we arrive here from genericUnwind(). |
| // So, we must restore our call frame and stack pointer. |
| jit.restoreCalleeSavesFromEntryFrameCalleeSavesBuffer(vm.topEntryFrame); |
| jit.loadPtr(vm.addressOfCallFrameForCatch(), GPRInfo::callFrameRegister); |
| } |
| jit.addPtr( |
| CCallHelpers::TrustedImm32(codeBlock->stackPointerOffset() * sizeof(Register)), |
| GPRInfo::callFrameRegister, CCallHelpers::stackPointerRegister); |
| |
| jit.jitAssertHasValidCallFrame(); |
| |
| if (UNLIKELY(vm.m_perBytecodeProfiler && codeBlock->jitCode()->dfgCommon()->compilation)) { |
| Profiler::Database& database = *vm.m_perBytecodeProfiler; |
| Profiler::Compilation* compilation = codeBlock->jitCode()->dfgCommon()->compilation.get(); |
| |
| Profiler::OSRExit* profilerExit = compilation->addOSRExit( |
| exitIndex, Profiler::OriginStack(database, codeBlock, exit.m_codeOrigin), |
| exit.m_kind, exit.m_kind == UncountableInvalidation); |
| jit.add64(CCallHelpers::TrustedImm32(1), CCallHelpers::AbsoluteAddress(profilerExit->counterAddress())); |
| } |
| |
| OSRExit::compileExit(jit, vm, exit, operands, recovery); |
| |
| LinkBuffer patchBuffer(jit, codeBlock); |
| exit.m_code = FINALIZE_CODE_IF( |
| shouldDumpDisassembly() || Options::verboseOSR() || Options::verboseDFGOSRExit(), |
| patchBuffer, OSRExitPtrTag, |
| "DFG OSR exit #%u (%s, %s) from %s, with operands = %s", |
| exitIndex, toCString(exit.m_codeOrigin).data(), |
| exitKindToString(exit.m_kind), toCString(*codeBlock).data(), |
| toCString(ignoringContext<DumpContext>(operands)).data()); |
| } |
| |
| MacroAssembler::repatchJump(exit.codeLocationForRepatch(), CodeLocationLabel<OSRExitPtrTag>(exit.m_code.code())); |
| |
| vm.osrExitJumpDestination = exit.m_code.code().executableAddress(); |
| } |
| |
| void OSRExit::compileExit(CCallHelpers& jit, VM& vm, const OSRExit& exit, const Operands<ValueRecovery>& operands, SpeculationRecovery* recovery) |
| { |
| jit.jitAssertTagsInPlace(); |
| |
| // Pro-forma stuff. |
| if (UNLIKELY(Options::printEachOSRExit())) { |
| SpeculationFailureDebugInfo* debugInfo = new SpeculationFailureDebugInfo; |
| debugInfo->codeBlock = jit.codeBlock(); |
| debugInfo->kind = exit.m_kind; |
| debugInfo->bytecodeIndex = exit.m_codeOrigin.bytecodeIndex(); |
| |
| jit.debugCall(vm, operationDebugPrintSpeculationFailure, debugInfo); |
| } |
| |
| // Perform speculation recovery. This only comes into play when an operation |
| // starts mutating state before verifying the speculation it has already made. |
| |
| if (recovery) { |
| switch (recovery->type()) { |
| case SpeculativeAdd: |
| jit.sub32(recovery->src(), recovery->dest()); |
| #if USE(JSVALUE64) |
| jit.or64(GPRInfo::numberTagRegister, recovery->dest()); |
| #endif |
| break; |
| |
| case SpeculativeAddSelf: |
| // If A + A = A (int32_t) overflows, A can be recovered by ((static_cast<int32_t>(A) >> 1) ^ 0x8000000). |
| jit.rshift32(AssemblyHelpers::TrustedImm32(1), recovery->dest()); |
| jit.xor32(AssemblyHelpers::TrustedImm32(0x80000000), recovery->dest()); |
| #if USE(JSVALUE64) |
| jit.or64(GPRInfo::numberTagRegister, recovery->dest()); |
| #endif |
| break; |
| |
| case SpeculativeAddImmediate: |
| jit.sub32(AssemblyHelpers::Imm32(recovery->immediate()), recovery->dest()); |
| #if USE(JSVALUE64) |
| jit.or64(GPRInfo::numberTagRegister, recovery->dest()); |
| #endif |
| break; |
| |
| case BooleanSpeculationCheck: |
| #if USE(JSVALUE64) |
| jit.xor64(AssemblyHelpers::TrustedImm32(JSValue::ValueFalse), recovery->dest()); |
| #endif |
| break; |
| |
| default: |
| break; |
| } |
| } |
| |
| // Refine some array and/or value profile, if appropriate. |
| |
| if (!!exit.m_jsValueSource) { |
| if (exit.m_kind == BadCache || exit.m_kind == BadIndexingType) { |
| // If the instruction that this originated from has an array profile, then |
| // refine it. If it doesn't, then do nothing. The latter could happen for |
| // hoisted checks, or checks emitted for operations that didn't have array |
| // profiling - either ops that aren't array accesses at all, or weren't |
| // known to be array acceses in the bytecode. The latter case is a FIXME |
| // while the former case is an outcome of a CheckStructure not knowing why |
| // it was emitted (could be either due to an inline cache of a property |
| // property access, or due to an array profile). |
| |
| CodeOrigin codeOrigin = exit.m_codeOriginForExitProfile; |
| CodeBlock* codeBlock = jit.baselineCodeBlockFor(codeOrigin); |
| if (ArrayProfile* arrayProfile = codeBlock->getArrayProfile(codeOrigin.bytecodeIndex())) { |
| const Instruction* instruction = codeBlock->instructions().at(codeOrigin.bytecodeIndex()).ptr(); |
| CCallHelpers::Jump skipProfile; |
| if (instruction->is<OpGetById>()) { |
| auto& metadata = instruction->as<OpGetById>().metadata(codeBlock); |
| skipProfile = jit.branch8(CCallHelpers::NotEqual, CCallHelpers::AbsoluteAddress(&metadata.m_modeMetadata.mode), CCallHelpers::TrustedImm32(static_cast<uint8_t>(GetByIdMode::ArrayLength))); |
| } |
| |
| #if USE(JSVALUE64) |
| GPRReg usedRegister; |
| if (exit.m_jsValueSource.isAddress()) |
| usedRegister = exit.m_jsValueSource.base(); |
| else |
| usedRegister = exit.m_jsValueSource.gpr(); |
| #else |
| GPRReg usedRegister1; |
| GPRReg usedRegister2; |
| if (exit.m_jsValueSource.isAddress()) { |
| usedRegister1 = exit.m_jsValueSource.base(); |
| usedRegister2 = InvalidGPRReg; |
| } else { |
| usedRegister1 = exit.m_jsValueSource.payloadGPR(); |
| if (exit.m_jsValueSource.hasKnownTag()) |
| usedRegister2 = InvalidGPRReg; |
| else |
| usedRegister2 = exit.m_jsValueSource.tagGPR(); |
| } |
| #endif |
| |
| GPRReg scratch1; |
| GPRReg scratch2; |
| #if USE(JSVALUE64) |
| scratch1 = AssemblyHelpers::selectScratchGPR(usedRegister); |
| scratch2 = AssemblyHelpers::selectScratchGPR(usedRegister, scratch1); |
| #else |
| scratch1 = AssemblyHelpers::selectScratchGPR(usedRegister1, usedRegister2); |
| scratch2 = AssemblyHelpers::selectScratchGPR(usedRegister1, usedRegister2, scratch1); |
| #endif |
| |
| if (isARM64()) { |
| jit.pushToSave(scratch1); |
| jit.pushToSave(scratch2); |
| } else { |
| jit.push(scratch1); |
| jit.push(scratch2); |
| } |
| |
| GPRReg value; |
| if (exit.m_jsValueSource.isAddress()) { |
| value = scratch1; |
| jit.loadPtr(AssemblyHelpers::Address(exit.m_jsValueSource.asAddress()), value); |
| } else |
| value = exit.m_jsValueSource.payloadGPR(); |
| |
| jit.load32(AssemblyHelpers::Address(value, JSCell::structureIDOffset()), scratch1); |
| jit.store32(scratch1, arrayProfile->addressOfLastSeenStructureID()); |
| |
| jit.load8(AssemblyHelpers::Address(value, JSCell::typeInfoTypeOffset()), scratch2); |
| jit.sub32(AssemblyHelpers::TrustedImm32(FirstTypedArrayType), scratch2); |
| auto notTypedArray = jit.branch32(MacroAssembler::AboveOrEqual, scratch2, AssemblyHelpers::TrustedImm32(NumberOfTypedArrayTypesExcludingDataView)); |
| jit.move(AssemblyHelpers::TrustedImmPtr(typedArrayModes), scratch1); |
| jit.load32(AssemblyHelpers::BaseIndex(scratch1, scratch2, AssemblyHelpers::TimesFour), scratch2); |
| auto storeArrayModes = jit.jump(); |
| |
| notTypedArray.link(&jit); |
| #if USE(JSVALUE64) |
| jit.load8(AssemblyHelpers::Address(value, JSCell::indexingTypeAndMiscOffset()), scratch1); |
| #else |
| jit.load8(AssemblyHelpers::Address(scratch1, Structure::indexingModeIncludingHistoryOffset()), scratch1); |
| #endif |
| jit.and32(AssemblyHelpers::TrustedImm32(IndexingModeMask), scratch1); |
| jit.move(AssemblyHelpers::TrustedImm32(1), scratch2); |
| jit.lshift32(scratch1, scratch2); |
| storeArrayModes.link(&jit); |
| jit.or32(scratch2, AssemblyHelpers::AbsoluteAddress(arrayProfile->addressOfArrayModes())); |
| |
| if (isARM64()) { |
| jit.popToRestore(scratch2); |
| jit.popToRestore(scratch1); |
| } else { |
| jit.pop(scratch2); |
| jit.pop(scratch1); |
| } |
| |
| if (skipProfile.isSet()) |
| skipProfile.link(&jit); |
| } |
| } |
| |
| if (MethodOfGettingAValueProfile profile = exit.m_valueProfile) { |
| #if USE(JSVALUE64) |
| if (exit.m_jsValueSource.isAddress()) { |
| // We can't be sure that we have a spare register. So use the numberTagRegister, |
| // since we know how to restore it. |
| jit.load64(AssemblyHelpers::Address(exit.m_jsValueSource.asAddress()), GPRInfo::numberTagRegister); |
| profile.emitReportValue(jit, JSValueRegs(GPRInfo::numberTagRegister)); |
| jit.move(AssemblyHelpers::TrustedImm64(JSValue::NumberTag), GPRInfo::numberTagRegister); |
| } else |
| profile.emitReportValue(jit, JSValueRegs(exit.m_jsValueSource.gpr())); |
| #else // not USE(JSVALUE64) |
| if (exit.m_jsValueSource.isAddress()) { |
| // Save a register so we can use it. |
| GPRReg scratchPayload = AssemblyHelpers::selectScratchGPR(exit.m_jsValueSource.base()); |
| GPRReg scratchTag = AssemblyHelpers::selectScratchGPR(exit.m_jsValueSource.base(), scratchPayload); |
| jit.pushToSave(scratchPayload); |
| jit.pushToSave(scratchTag); |
| |
| JSValueRegs scratch(scratchTag, scratchPayload); |
| |
| jit.loadValue(exit.m_jsValueSource.asAddress(), scratch); |
| profile.emitReportValue(jit, scratch); |
| |
| jit.popToRestore(scratchTag); |
| jit.popToRestore(scratchPayload); |
| } else if (exit.m_jsValueSource.hasKnownTag()) { |
| GPRReg scratchTag = AssemblyHelpers::selectScratchGPR(exit.m_jsValueSource.payloadGPR()); |
| jit.pushToSave(scratchTag); |
| jit.move(AssemblyHelpers::TrustedImm32(exit.m_jsValueSource.tag()), scratchTag); |
| JSValueRegs value(scratchTag, exit.m_jsValueSource.payloadGPR()); |
| profile.emitReportValue(jit, value); |
| jit.popToRestore(scratchTag); |
| } else |
| profile.emitReportValue(jit, exit.m_jsValueSource.regs()); |
| #endif // USE(JSVALUE64) |
| } |
| } |
| |
| // What follows is an intentionally simple OSR exit implementation that generates |
| // fairly poor code but is very easy to hack. In particular, it dumps all state that |
| // needs conversion into a scratch buffer so that in step 6, where we actually do the |
| // conversions, we know that all temp registers are free to use and the variable is |
| // definitely in a well-known spot in the scratch buffer regardless of whether it had |
| // originally been in a register or spilled. This allows us to decouple "where was |
| // the variable" from "how was it represented". Consider that the |
| // Int32DisplacedInJSStack recovery: it tells us that the value is in a |
| // particular place and that that place holds an unboxed int32. We have two different |
| // places that a value could be (displaced, register) and a bunch of different |
| // ways of representing a value. The number of recoveries is two * a bunch. The code |
| // below means that we have to have two + a bunch cases rather than two * a bunch. |
| // Once we have loaded the value from wherever it was, the reboxing is the same |
| // regardless of its location. Likewise, before we do the reboxing, the way we get to |
| // the value (i.e. where we load it from) is the same regardless of its type. Because |
| // the code below always dumps everything into a scratch buffer first, the two |
| // questions become orthogonal, which simplifies adding new types and adding new |
| // locations. |
| // |
| // This raises the question: does using such a suboptimal implementation of OSR exit, |
| // where we always emit code to dump all state into a scratch buffer only to then |
| // dump it right back into the stack, hurt us in any way? The asnwer is that OSR exits |
| // are rare. Our tiering strategy ensures this. This is because if an OSR exit is |
| // taken more than ~100 times, we jettison the DFG code block along with all of its |
| // exits. It is impossible for an OSR exit - i.e. the code we compile below - to |
| // execute frequently enough for the codegen to matter that much. It probably matters |
| // enough that we don't want to turn this into some super-slow function call, but so |
| // long as we're generating straight-line code, that code can be pretty bad. Also |
| // because we tend to exit only along one OSR exit from any DFG code block - that's an |
| // empirical result that we're extremely confident about - the code size of this |
| // doesn't matter much. Hence any attempt to optimize the codegen here is just purely |
| // harmful to the system: it probably won't reduce either net memory usage or net |
| // execution time. It will only prevent us from cleanly decoupling "where was the |
| // variable" from "how was it represented", which will make it more difficult to add |
| // features in the future and it will make it harder to reason about bugs. |
| |
| // Save all state from GPRs into the scratch buffer. |
| |
| ScratchBuffer* scratchBuffer = vm.scratchBufferForSize(sizeof(EncodedJSValue) * operands.size()); |
| EncodedJSValue* scratch = scratchBuffer ? static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer()) : 0; |
| |
| for (size_t index = 0; index < operands.size(); ++index) { |
| const ValueRecovery& recovery = operands[index]; |
| |
| switch (recovery.technique()) { |
| case UnboxedInt32InGPR: |
| case UnboxedCellInGPR: |
| #if USE(JSVALUE64) |
| case InGPR: |
| case UnboxedInt52InGPR: |
| case UnboxedStrictInt52InGPR: |
| jit.store64(recovery.gpr(), scratch + index); |
| break; |
| #else |
| case UnboxedBooleanInGPR: |
| jit.store32( |
| recovery.gpr(), |
| &bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.payload); |
| break; |
| |
| case InPair: |
| jit.store32( |
| recovery.tagGPR(), |
| &bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.tag); |
| jit.store32( |
| recovery.payloadGPR(), |
| &bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.payload); |
| break; |
| #endif |
| |
| default: |
| break; |
| } |
| } |
| |
| // And voila, all GPRs are free to reuse. |
| |
| // Save all state from FPRs into the scratch buffer. |
| |
| for (size_t index = 0; index < operands.size(); ++index) { |
| const ValueRecovery& recovery = operands[index]; |
| |
| switch (recovery.technique()) { |
| case UnboxedDoubleInFPR: |
| case InFPR: |
| jit.move(AssemblyHelpers::TrustedImmPtr(scratch + index), GPRInfo::regT0); |
| jit.storeDouble(recovery.fpr(), MacroAssembler::Address(GPRInfo::regT0)); |
| break; |
| |
| default: |
| break; |
| } |
| } |
| |
| // Now, all FPRs are also free. |
| |
| // Save all state from the stack into the scratch buffer. For simplicity we |
| // do this even for state that's already in the right place on the stack. |
| // It makes things simpler later. |
| |
| for (size_t index = 0; index < operands.size(); ++index) { |
| const ValueRecovery& recovery = operands[index]; |
| |
| switch (recovery.technique()) { |
| case DisplacedInJSStack: |
| case CellDisplacedInJSStack: |
| case BooleanDisplacedInJSStack: |
| case Int32DisplacedInJSStack: |
| case DoubleDisplacedInJSStack: |
| #if USE(JSVALUE64) |
| case Int52DisplacedInJSStack: |
| case StrictInt52DisplacedInJSStack: |
| jit.load64(AssemblyHelpers::addressFor(recovery.virtualRegister()), GPRInfo::regT0); |
| jit.store64(GPRInfo::regT0, scratch + index); |
| break; |
| #else |
| jit.load32( |
| AssemblyHelpers::tagFor(recovery.virtualRegister()), |
| GPRInfo::regT0); |
| jit.load32( |
| AssemblyHelpers::payloadFor(recovery.virtualRegister()), |
| GPRInfo::regT1); |
| jit.store32( |
| GPRInfo::regT0, |
| &bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.tag); |
| jit.store32( |
| GPRInfo::regT1, |
| &bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.payload); |
| break; |
| #endif |
| |
| default: |
| break; |
| } |
| } |
| |
| if (validateDFGDoesGC) { |
| // We're about to exit optimized code. So, there's no longer any optimized |
| // code running that expects no GC. We need to set this before arguments |
| // materialization below (see emitRestoreArguments()). |
| |
| // Even though we set Heap::m_expectDoesGC in compileOSRExit(), we also need |
| // to set it here because compileOSRExit() is only called on the first time |
| // we exit from this site, but all subsequent exits will take this compiled |
| // ramp without calling compileOSRExit() first. |
| jit.store8(CCallHelpers::TrustedImm32(true), vm.heap.addressOfExpectDoesGC()); |
| } |
| |
| // Need to ensure that the stack pointer accounts for the worst-case stack usage at exit. This |
| // could toast some stack that the DFG used. We need to do it before storing to stack offsets |
| // used by baseline. |
| jit.addPtr( |
| CCallHelpers::TrustedImm32( |
| -jit.codeBlock()->jitCode()->dfgCommon()->requiredRegisterCountForExit * sizeof(Register)), |
| CCallHelpers::framePointerRegister, CCallHelpers::stackPointerRegister); |
| |
| // Restore the DFG callee saves and then save the ones the baseline JIT uses. |
| jit.emitRestoreCalleeSaves(); |
| jit.emitSaveCalleeSavesFor(jit.baselineCodeBlock()); |
| |
| // The tag registers are needed to materialize recoveries below. |
| jit.emitMaterializeTagCheckRegisters(); |
| |
| if (exit.isExceptionHandler()) |
| jit.copyCalleeSavesToEntryFrameCalleeSavesBuffer(vm.topEntryFrame); |
| |
| // Do all data format conversions and store the results into the stack. |
| |
| for (size_t index = 0; index < operands.size(); ++index) { |
| const ValueRecovery& recovery = operands[index]; |
| VirtualRegister reg = operands.virtualRegisterForIndex(index); |
| |
| if (reg.isLocal() && reg.toLocal() < static_cast<int>(jit.baselineCodeBlock()->calleeSaveSpaceAsVirtualRegisters())) |
| continue; |
| |
| int operand = reg.offset(); |
| |
| switch (recovery.technique()) { |
| case DisplacedInJSStack: |
| case InFPR: |
| #if USE(JSVALUE64) |
| case InGPR: |
| case UnboxedCellInGPR: |
| case CellDisplacedInJSStack: |
| case BooleanDisplacedInJSStack: |
| jit.load64(scratch + index, GPRInfo::regT0); |
| jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(operand)); |
| break; |
| #else // not USE(JSVALUE64) |
| case InPair: |
| jit.load32( |
| &bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.tag, |
| GPRInfo::regT0); |
| jit.load32( |
| &bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.payload, |
| GPRInfo::regT1); |
| jit.store32( |
| GPRInfo::regT0, |
| AssemblyHelpers::tagFor(operand)); |
| jit.store32( |
| GPRInfo::regT1, |
| AssemblyHelpers::payloadFor(operand)); |
| break; |
| |
| case UnboxedCellInGPR: |
| case CellDisplacedInJSStack: |
| jit.load32( |
| &bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.payload, |
| GPRInfo::regT0); |
| jit.store32( |
| AssemblyHelpers::TrustedImm32(JSValue::CellTag), |
| AssemblyHelpers::tagFor(operand)); |
| jit.store32( |
| GPRInfo::regT0, |
| AssemblyHelpers::payloadFor(operand)); |
| break; |
| |
| case UnboxedBooleanInGPR: |
| case BooleanDisplacedInJSStack: |
| jit.load32( |
| &bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.payload, |
| GPRInfo::regT0); |
| jit.store32( |
| AssemblyHelpers::TrustedImm32(JSValue::BooleanTag), |
| AssemblyHelpers::tagFor(operand)); |
| jit.store32( |
| GPRInfo::regT0, |
| AssemblyHelpers::payloadFor(operand)); |
| break; |
| #endif // USE(JSVALUE64) |
| |
| case UnboxedInt32InGPR: |
| case Int32DisplacedInJSStack: |
| #if USE(JSVALUE64) |
| jit.load64(scratch + index, GPRInfo::regT0); |
| jit.zeroExtend32ToPtr(GPRInfo::regT0, GPRInfo::regT0); |
| jit.or64(GPRInfo::numberTagRegister, GPRInfo::regT0); |
| jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(operand)); |
| #else |
| jit.load32( |
| &bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.payload, |
| GPRInfo::regT0); |
| jit.store32( |
| AssemblyHelpers::TrustedImm32(JSValue::Int32Tag), |
| AssemblyHelpers::tagFor(operand)); |
| jit.store32( |
| GPRInfo::regT0, |
| AssemblyHelpers::payloadFor(operand)); |
| #endif |
| break; |
| |
| #if USE(JSVALUE64) |
| case UnboxedInt52InGPR: |
| case Int52DisplacedInJSStack: |
| jit.load64(scratch + index, GPRInfo::regT0); |
| jit.rshift64( |
| AssemblyHelpers::TrustedImm32(JSValue::int52ShiftAmount), GPRInfo::regT0); |
| jit.boxInt52(GPRInfo::regT0, GPRInfo::regT0, GPRInfo::regT1, FPRInfo::fpRegT0); |
| jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(operand)); |
| break; |
| |
| case UnboxedStrictInt52InGPR: |
| case StrictInt52DisplacedInJSStack: |
| jit.load64(scratch + index, GPRInfo::regT0); |
| jit.boxInt52(GPRInfo::regT0, GPRInfo::regT0, GPRInfo::regT1, FPRInfo::fpRegT0); |
| jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(operand)); |
| break; |
| #endif |
| |
| case UnboxedDoubleInFPR: |
| case DoubleDisplacedInJSStack: |
| jit.move(AssemblyHelpers::TrustedImmPtr(scratch + index), GPRInfo::regT0); |
| jit.loadDouble(MacroAssembler::Address(GPRInfo::regT0), FPRInfo::fpRegT0); |
| jit.purifyNaN(FPRInfo::fpRegT0); |
| #if USE(JSVALUE64) |
| jit.boxDouble(FPRInfo::fpRegT0, GPRInfo::regT0); |
| jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(operand)); |
| #else |
| jit.storeDouble(FPRInfo::fpRegT0, AssemblyHelpers::addressFor(operand)); |
| #endif |
| break; |
| |
| case Constant: |
| #if USE(JSVALUE64) |
| jit.store64( |
| AssemblyHelpers::TrustedImm64(JSValue::encode(recovery.constant())), |
| AssemblyHelpers::addressFor(operand)); |
| #else |
| jit.store32( |
| AssemblyHelpers::TrustedImm32(recovery.constant().tag()), |
| AssemblyHelpers::tagFor(operand)); |
| jit.store32( |
| AssemblyHelpers::TrustedImm32(recovery.constant().payload()), |
| AssemblyHelpers::payloadFor(operand)); |
| #endif |
| break; |
| |
| case DirectArgumentsThatWereNotCreated: |
| case ClonedArgumentsThatWereNotCreated: |
| // Don't do this, yet. |
| break; |
| |
| default: |
| RELEASE_ASSERT_NOT_REACHED(); |
| break; |
| } |
| } |
| |
| // Now that things on the stack are recovered, do the arguments recovery. We assume that arguments |
| // recoveries don't recursively refer to each other. But, we don't try to assume that they only |
| // refer to certain ranges of locals. Hence why we need to do this here, once the stack is sensible. |
| // Note that we also roughly assume that the arguments might still be materialized outside of its |
| // inline call frame scope - but for now the DFG wouldn't do that. |
| |
| emitRestoreArguments(jit, vm, operands); |
| |
| // Adjust the old JIT's execute counter. Since we are exiting OSR, we know |
| // that all new calls into this code will go to the new JIT, so the execute |
| // counter only affects call frames that performed OSR exit and call frames |
| // that were still executing the old JIT at the time of another call frame's |
| // OSR exit. We want to ensure that the following is true: |
| // |
| // (a) Code the performs an OSR exit gets a chance to reenter optimized |
| // code eventually, since optimized code is faster. But we don't |
| // want to do such reentery too aggressively (see (c) below). |
| // |
| // (b) If there is code on the call stack that is still running the old |
| // JIT's code and has never OSR'd, then it should get a chance to |
| // perform OSR entry despite the fact that we've exited. |
| // |
| // (c) Code the performs an OSR exit should not immediately retry OSR |
| // entry, since both forms of OSR are expensive. OSR entry is |
| // particularly expensive. |
| // |
| // (d) Frequent OSR failures, even those that do not result in the code |
| // running in a hot loop, result in recompilation getting triggered. |
| // |
| // To ensure (c), we'd like to set the execute counter to |
| // counterValueForOptimizeAfterWarmUp(). This seems like it would endanger |
| // (a) and (b), since then every OSR exit would delay the opportunity for |
| // every call frame to perform OSR entry. Essentially, if OSR exit happens |
| // frequently and the function has few loops, then the counter will never |
| // become non-negative and OSR entry will never be triggered. OSR entry |
| // will only happen if a loop gets hot in the old JIT, which does a pretty |
| // good job of ensuring (a) and (b). But that doesn't take care of (d), |
| // since each speculation failure would reset the execute counter. |
| // So we check here if the number of speculation failures is significantly |
| // larger than the number of successes (we want 90% success rate), and if |
| // there have been a large enough number of failures. If so, we set the |
| // counter to 0; otherwise we set the counter to |
| // counterValueForOptimizeAfterWarmUp(). |
| |
| handleExitCounts(vm, jit, exit); |
| |
| // Reify inlined call frames. |
| |
| reifyInlinedCallFrames(jit, exit); |
| |
| // And finish. |
| adjustAndJumpToTarget(vm, jit, exit); |
| } |
| |
| void JIT_OPERATION operationDebugPrintSpeculationFailure(CallFrame* callFrame, void* debugInfoRaw, void* scratch) |
| { |
| VM& vm = callFrame->deprecatedVM(); |
| NativeCallFrameTracer tracer(vm, callFrame); |
| |
| SpeculationFailureDebugInfo* debugInfo = static_cast<SpeculationFailureDebugInfo*>(debugInfoRaw); |
| CodeBlock* codeBlock = debugInfo->codeBlock; |
| CodeBlock* alternative = codeBlock->alternative(); |
| dataLog("Speculation failure in ", *codeBlock); |
| dataLog(" @ exit #", vm.osrExitIndex, " (", debugInfo->bytecodeIndex, ", ", exitKindToString(debugInfo->kind), ") with "); |
| if (alternative) { |
| dataLog( |
| "executeCounter = ", alternative->jitExecuteCounter(), |
| ", reoptimizationRetryCounter = ", alternative->reoptimizationRetryCounter(), |
| ", optimizationDelayCounter = ", alternative->optimizationDelayCounter()); |
| } else |
| dataLog("no alternative code block (i.e. we've been jettisoned)"); |
| dataLog(", osrExitCounter = ", codeBlock->osrExitCounter(), "\n"); |
| dataLog(" GPRs at time of exit:"); |
| char* scratchPointer = static_cast<char*>(scratch); |
| for (unsigned i = 0; i < GPRInfo::numberOfRegisters; ++i) { |
| GPRReg gpr = GPRInfo::toRegister(i); |
| dataLog(" ", GPRInfo::debugName(gpr), ":", RawPointer(*reinterpret_cast_ptr<void**>(scratchPointer))); |
| scratchPointer += sizeof(EncodedJSValue); |
| } |
| dataLog("\n"); |
| dataLog(" FPRs at time of exit:"); |
| for (unsigned i = 0; i < FPRInfo::numberOfRegisters; ++i) { |
| FPRReg fpr = FPRInfo::toRegister(i); |
| dataLog(" ", FPRInfo::debugName(fpr), ":"); |
| uint64_t bits = *reinterpret_cast_ptr<uint64_t*>(scratchPointer); |
| double value = *reinterpret_cast_ptr<double*>(scratchPointer); |
| dataLogF("%llx:%lf", static_cast<long long>(bits), value); |
| scratchPointer += sizeof(EncodedJSValue); |
| } |
| dataLog("\n"); |
| } |
| |
| } } // namespace JSC::DFG |
| |
| #endif // ENABLE(DFG_JIT) |
