blob: a7f0506c4f1159d4141b9a4254ea88dfea459d47 [file] [log] [blame]
/*
* Copyright (C) 2011-2019 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "config.h"
#include "DFGOSREntry.h"
#if ENABLE(DFG_JIT)
#include "CallFrame.h"
#include "CodeBlock.h"
#include "DFGJITCode.h"
#include "DFGNode.h"
#include "InterpreterInlines.h"
#include "JIT.h"
#include "JSCInlines.h"
#include "VMInlines.h"
#include <wtf/CommaPrinter.h>
namespace JSC { namespace DFG {
void OSREntryData::dumpInContext(PrintStream& out, DumpContext* context) const
{
out.print("bc#", m_bytecodeIndex, ", machine code = ", RawPointer(m_machineCode.executableAddress()));
out.print(", stack rules = [");
auto printOperand = [&] (VirtualRegister reg) {
out.print(inContext(m_expectedValues.operand(reg), context), " (");
VirtualRegister toReg;
bool overwritten = false;
for (OSREntryReshuffling reshuffling : m_reshufflings) {
if (reg == VirtualRegister(reshuffling.fromOffset)) {
toReg = VirtualRegister(reshuffling.toOffset);
break;
}
if (reg == VirtualRegister(reshuffling.toOffset))
overwritten = true;
}
if (!overwritten && !toReg.isValid())
toReg = reg;
if (toReg.isValid()) {
if (toReg.isLocal() && !m_machineStackUsed.get(toReg.toLocal()))
out.print("ignored");
else
out.print("maps to ", toReg);
} else
out.print("overwritten");
if (reg.isLocal() && m_localsForcedDouble.get(reg.toLocal()))
out.print(", forced double");
if (reg.isLocal() && m_localsForcedAnyInt.get(reg.toLocal()))
out.print(", forced machine int");
out.print(")");
};
CommaPrinter comma;
for (size_t argumentIndex = m_expectedValues.numberOfArguments(); argumentIndex--;) {
out.print(comma, "arg", argumentIndex, ":");
printOperand(virtualRegisterForArgument(argumentIndex));
}
for (size_t localIndex = 0; localIndex < m_expectedValues.numberOfLocals(); ++localIndex) {
out.print(comma, "loc", localIndex, ":");
printOperand(virtualRegisterForLocal(localIndex));
}
out.print("], machine stack used = ", m_machineStackUsed);
}
void OSREntryData::dump(PrintStream& out) const
{
dumpInContext(out, nullptr);
}
SUPPRESS_ASAN
void* prepareOSREntry(ExecState* exec, CodeBlock* codeBlock, unsigned bytecodeIndex)
{
ASSERT(JITCode::isOptimizingJIT(codeBlock->jitType()));
ASSERT(codeBlock->alternative());
ASSERT(codeBlock->alternative()->jitType() == JITType::BaselineJIT);
ASSERT(!codeBlock->jitCodeMap());
ASSERT(codeBlock->jitCode()->dfgCommon()->isStillValid);
if (!Options::useOSREntryToDFG())
return nullptr;
if (Options::verboseOSR()) {
dataLog(
"DFG OSR in ", *codeBlock->alternative(), " -> ", *codeBlock,
" from bc#", bytecodeIndex, "\n");
}
VM* vm = &exec->vm();
sanitizeStackForVM(vm);
if (bytecodeIndex)
codeBlock->ownerExecutable()->setDidTryToEnterInLoop(true);
if (codeBlock->jitType() != JITType::DFGJIT) {
RELEASE_ASSERT(codeBlock->jitType() == JITType::FTLJIT);
// When will this happen? We could have:
//
// - An exit from the FTL JIT into the baseline JIT followed by an attempt
// to reenter. We're fine with allowing this to fail. If it happens
// enough we'll just reoptimize. It basically means that the OSR exit cost
// us dearly and so reoptimizing is the right thing to do.
//
// - We have recursive code with hot loops. Consider that foo has a hot loop
// that calls itself. We have two foo's on the stack, lets call them foo1
// and foo2, with foo1 having called foo2 from foo's hot loop. foo2 gets
// optimized all the way into the FTL. Then it returns into foo1, and then
// foo1 wants to get optimized. It might reach this conclusion from its
// hot loop and attempt to OSR enter. And we'll tell it that it can't. It
// might be worth addressing this case, but I just think this case will
// be super rare. For now, if it does happen, it'll cause some compilation
// thrashing.
if (Options::verboseOSR())
dataLog(" OSR failed because the target code block is not DFG.\n");
return nullptr;
}
JITCode* jitCode = codeBlock->jitCode()->dfg();
OSREntryData* entry = jitCode->osrEntryDataForBytecodeIndex(bytecodeIndex);
if (!entry) {
if (Options::verboseOSR())
dataLogF(" OSR failed because the entrypoint was optimized out.\n");
return nullptr;
}
ASSERT(entry->m_bytecodeIndex == bytecodeIndex);
// The code below checks if it is safe to perform OSR entry. It may find
// that it is unsafe to do so, for any number of reasons, which are documented
// below. If the code decides not to OSR then it returns 0, and it's the caller's
// responsibility to patch up the state in such a way as to ensure that it's
// both safe and efficient to continue executing baseline code for now. This
// should almost certainly include calling either codeBlock->optimizeAfterWarmUp()
// or codeBlock->dontOptimizeAnytimeSoon().
// 1) Verify predictions. If the predictions are inconsistent with the actual
// values, then OSR entry is not possible at this time. It's tempting to
// assume that we could somehow avoid this case. We can certainly avoid it
// for first-time loop OSR - that is, OSR into a CodeBlock that we have just
// compiled. Then we are almost guaranteed that all of the predictions will
// check out. It would be pretty easy to make that a hard guarantee. But
// then there would still be the case where two call frames with the same
// baseline CodeBlock are on the stack at the same time. The top one
// triggers compilation and OSR. In that case, we may no longer have
// accurate value profiles for the one deeper in the stack. Hence, when we
// pop into the CodeBlock that is deeper on the stack, we might OSR and
// realize that the predictions are wrong. Probably, in most cases, this is
// just an anomaly in the sense that the older CodeBlock simply went off
// into a less-likely path. So, the wisest course of action is to simply not
// OSR at this time.
for (size_t argument = 0; argument < entry->m_expectedValues.numberOfArguments(); ++argument) {
if (argument >= exec->argumentCountIncludingThis()) {
if (Options::verboseOSR()) {
dataLogF(" OSR failed because argument %zu was not passed, expected ", argument);
entry->m_expectedValues.argument(argument).dump(WTF::dataFile());
dataLogF(".\n");
}
return nullptr;
}
JSValue value;
if (!argument)
value = exec->thisValue();
else
value = exec->argument(argument - 1);
if (!entry->m_expectedValues.argument(argument).validateOSREntryValue(value, FlushedJSValue)) {
if (Options::verboseOSR()) {
dataLog(
" OSR failed because argument ", argument, " is ", value,
", expected ", entry->m_expectedValues.argument(argument), ".\n");
}
return nullptr;
}
}
for (size_t local = 0; local < entry->m_expectedValues.numberOfLocals(); ++local) {
int localOffset = virtualRegisterForLocal(local).offset();
JSValue value = exec->registers()[localOffset].asanUnsafeJSValue();
FlushFormat format = FlushedJSValue;
if (entry->m_localsForcedAnyInt.get(local)) {
if (!value.isAnyInt()) {
dataLogLnIf(Options::verboseOSR(),
" OSR failed because variable ", localOffset, " is ",
value, ", expected ",
"machine int.");
return nullptr;
}
value = jsDoubleNumber(value.asAnyInt());
format = FlushedInt52;
}
if (entry->m_localsForcedDouble.get(local)) {
if (!value.isNumber()) {
dataLogLnIf(Options::verboseOSR(),
" OSR failed because variable ", localOffset, " is ",
value, ", expected number.");
return nullptr;
}
value = jsDoubleNumber(value.asNumber());
format = FlushedDouble;
}
if (!entry->m_expectedValues.local(local).validateOSREntryValue(value, format)) {
dataLogLnIf(Options::verboseOSR(),
" OSR failed because variable ", VirtualRegister(localOffset), " is ",
value, ", expected ",
entry->m_expectedValues.local(local), ".");
return nullptr;
}
}
// 2) Check the stack height. The DFG JIT may require a taller stack than the
// baseline JIT, in some cases. If we can't grow the stack, then don't do
// OSR right now. That's the only option we have unless we want basic block
// boundaries to start throwing RangeErrors. Although that would be possible,
// it seems silly: you'd be diverting the program to error handling when it
// would have otherwise just kept running albeit less quickly.
unsigned frameSizeForCheck = jitCode->common.requiredRegisterCountForExecutionAndExit();
if (UNLIKELY(!vm->ensureStackCapacityFor(&exec->registers()[virtualRegisterForLocal(frameSizeForCheck - 1).offset()]))) {
if (Options::verboseOSR())
dataLogF(" OSR failed because stack growth failed.\n");
return nullptr;
}
if (Options::verboseOSR())
dataLogF(" OSR should succeed.\n");
// At this point we're committed to entering. We will do some work to set things up,
// but we also rely on our caller recognizing that when we return a non-null pointer,
// that means that we're already past the point of no return and we must succeed at
// entering.
// 3) Set up the data in the scratch buffer and perform data format conversions.
unsigned frameSize = jitCode->common.frameRegisterCount;
unsigned baselineFrameSize = entry->m_expectedValues.numberOfLocals();
unsigned maxFrameSize = std::max(frameSize, baselineFrameSize);
Register* scratch = bitwise_cast<Register*>(vm->scratchBufferForSize(sizeof(Register) * (2 + CallFrame::headerSizeInRegisters + maxFrameSize))->dataBuffer());
*bitwise_cast<size_t*>(scratch + 0) = frameSize;
void* targetPC = entry->m_machineCode.executableAddress();
RELEASE_ASSERT(codeBlock->jitCode()->contains(entry->m_machineCode.untaggedExecutableAddress()));
if (Options::verboseOSR())
dataLogF(" OSR using target PC %p.\n", targetPC);
RELEASE_ASSERT(targetPC);
*bitwise_cast<void**>(scratch + 1) = retagCodePtr(targetPC, OSREntryPtrTag, bitwise_cast<PtrTag>(exec));
Register* pivot = scratch + 2 + CallFrame::headerSizeInRegisters;
for (int index = -CallFrame::headerSizeInRegisters; index < static_cast<int>(baselineFrameSize); ++index) {
VirtualRegister reg(-1 - index);
if (reg.isLocal()) {
if (entry->m_localsForcedDouble.get(reg.toLocal())) {
*bitwise_cast<double*>(pivot + index) = exec->registers()[reg.offset()].asanUnsafeJSValue().asNumber();
continue;
}
if (entry->m_localsForcedAnyInt.get(reg.toLocal())) {
*bitwise_cast<int64_t*>(pivot + index) = exec->registers()[reg.offset()].asanUnsafeJSValue().asAnyInt() << JSValue::int52ShiftAmount;
continue;
}
}
pivot[index] = exec->registers()[reg.offset()].asanUnsafeJSValue();
}
// 4) Reshuffle those registers that need reshuffling.
Vector<JSValue> temporaryLocals(entry->m_reshufflings.size());
for (unsigned i = entry->m_reshufflings.size(); i--;)
temporaryLocals[i] = pivot[VirtualRegister(entry->m_reshufflings[i].fromOffset).toLocal()].asanUnsafeJSValue();
for (unsigned i = entry->m_reshufflings.size(); i--;)
pivot[VirtualRegister(entry->m_reshufflings[i].toOffset).toLocal()] = temporaryLocals[i];
// 5) Clear those parts of the call frame that the DFG ain't using. This helps GC on
// some programs by eliminating some stale pointer pathologies.
for (unsigned i = frameSize; i--;) {
if (entry->m_machineStackUsed.get(i))
continue;
pivot[i] = JSValue();
}
// 6) Copy our callee saves to buffer.
#if NUMBER_OF_CALLEE_SAVES_REGISTERS > 0
const RegisterAtOffsetList* registerSaveLocations = codeBlock->calleeSaveRegisters();
RegisterAtOffsetList* allCalleeSaves = RegisterSet::vmCalleeSaveRegisterOffsets();
RegisterSet dontSaveRegisters = RegisterSet(RegisterSet::stackRegisters(), RegisterSet::allFPRs());
unsigned registerCount = registerSaveLocations->size();
VMEntryRecord* record = vmEntryRecord(vm->topEntryFrame);
for (unsigned i = 0; i < registerCount; i++) {
RegisterAtOffset currentEntry = registerSaveLocations->at(i);
if (dontSaveRegisters.get(currentEntry.reg()))
continue;
RegisterAtOffset* calleeSavesEntry = allCalleeSaves->find(currentEntry.reg());
*(bitwise_cast<intptr_t*>(pivot - 1) - currentEntry.offsetAsIndex()) = record->calleeSaveRegistersBuffer[calleeSavesEntry->offsetAsIndex()];
}
#endif
// 7) Fix the call frame to have the right code block.
*bitwise_cast<CodeBlock**>(pivot - 1 - CallFrameSlot::codeBlock) = codeBlock;
if (Options::verboseOSR())
dataLogF(" OSR returning data buffer %p.\n", scratch);
return scratch;
}
MacroAssemblerCodePtr<ExceptionHandlerPtrTag> prepareCatchOSREntry(ExecState* exec, CodeBlock* codeBlock, unsigned bytecodeIndex)
{
ASSERT(codeBlock->jitType() == JITType::DFGJIT || codeBlock->jitType() == JITType::FTLJIT);
ASSERT(codeBlock->jitCode()->dfgCommon()->isStillValid);
if (!Options::useOSREntryToDFG() && codeBlock->jitCode()->jitType() == JITType::DFGJIT)
return nullptr;
if (!Options::useOSREntryToFTL() && codeBlock->jitCode()->jitType() == JITType::FTLJIT)
return nullptr;
VM& vm = exec->vm();
CommonData* dfgCommon = codeBlock->jitCode()->dfgCommon();
RELEASE_ASSERT(dfgCommon);
DFG::CatchEntrypointData* catchEntrypoint = dfgCommon->catchOSREntryDataForBytecodeIndex(bytecodeIndex);
if (!catchEntrypoint) {
// This can be null under some circumstances. The most common is that we didn't
// compile this op_catch as an entrypoint since it had never executed when starting
// the compilation.
return nullptr;
}
// We're only allowed to OSR enter if we've proven we have compatible argument types.
for (unsigned argument = 0; argument < catchEntrypoint->argumentFormats.size(); ++argument) {
JSValue value = exec->uncheckedR(virtualRegisterForArgument(argument)).jsValue();
switch (catchEntrypoint->argumentFormats[argument]) {
case DFG::FlushedInt32:
if (!value.isInt32())
return nullptr;
break;
case DFG::FlushedCell:
if (!value.isCell())
return nullptr;
break;
case DFG::FlushedBoolean:
if (!value.isBoolean())
return nullptr;
break;
case DFG::DeadFlush:
// This means the argument is not alive. Therefore, it's allowed to be any type.
break;
case DFG::FlushedJSValue:
// An argument is trivially a JSValue.
break;
default:
RELEASE_ASSERT_NOT_REACHED();
}
}
unsigned frameSizeForCheck = dfgCommon->requiredRegisterCountForExecutionAndExit();
if (UNLIKELY(!vm.ensureStackCapacityFor(&exec->registers()[virtualRegisterForLocal(frameSizeForCheck).offset()])))
return nullptr;
auto instruction = exec->codeBlock()->instructions().at(exec->bytecodeOffset());
ASSERT(instruction->is<OpCatch>());
ValueProfileAndOperandBuffer* buffer = instruction->as<OpCatch>().metadata(exec).m_buffer;
JSValue* dataBuffer = reinterpret_cast<JSValue*>(dfgCommon->catchOSREntryBuffer->dataBuffer());
unsigned index = 0;
buffer->forEach([&] (ValueProfileAndOperand& profile) {
if (!VirtualRegister(profile.m_operand).isLocal())
return;
dataBuffer[index] = exec->uncheckedR(profile.m_operand).jsValue();
++index;
});
// The active length of catchOSREntryBuffer will be zeroed by ClearCatchLocals node.
dfgCommon->catchOSREntryBuffer->setActiveLength(sizeof(JSValue) * index);
return catchEntrypoint->machineCode;
}
} } // namespace JSC::DFG
#endif // ENABLE(DFG_JIT)