blob: 97a559f0ba330c41fe6ee429e9d4e170366ccb83 [file] [log] [blame]
/*
* Copyright (C) 2012-2019 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "config.h"
#include "CallLinkStatus.h"
#include "BytecodeStructs.h"
#include "CallLinkInfo.h"
#include "CodeBlock.h"
#include "JSCInlines.h"
#include "LLIntCallLinkInfo.h"
#include <wtf/CommaPrinter.h>
#include <wtf/ListDump.h>
namespace JSC {
namespace CallLinkStatusInternal {
static constexpr bool verbose = false;
}
CallLinkStatus::CallLinkStatus(JSValue value)
: m_couldTakeSlowPath(false)
, m_isProved(false)
{
if (!value || !value.isCell()) {
m_couldTakeSlowPath = true;
return;
}
m_variants.append(CallVariant(value.asCell()));
}
CallLinkStatus CallLinkStatus::computeFromLLInt(const ConcurrentJSLocker&, CodeBlock* profiledBlock, BytecodeIndex bytecodeIndex)
{
UNUSED_PARAM(profiledBlock);
UNUSED_PARAM(bytecodeIndex);
#if ENABLE(DFG_JIT)
if (profiledBlock->unlinkedCodeBlock()->hasExitSite(DFG::FrequentExitSite(bytecodeIndex, BadConstantValue))) {
// We could force this to be a closure call, but instead we'll just assume that it
// takes slow path.
return takesSlowPath();
}
#endif
auto instruction = profiledBlock->instructions().at(bytecodeIndex.offset());
OpcodeID op = instruction->opcodeID();
LLIntCallLinkInfo* callLinkInfo;
switch (op) {
case op_call:
callLinkInfo = &instruction->as<OpCall>().metadata(profiledBlock).m_callLinkInfo;
break;
case op_construct:
callLinkInfo = &instruction->as<OpConstruct>().metadata(profiledBlock).m_callLinkInfo;
break;
case op_tail_call:
callLinkInfo = &instruction->as<OpTailCall>().metadata(profiledBlock).m_callLinkInfo;
break;
case op_iterator_open:
callLinkInfo = &instruction->as<OpIteratorOpen>().metadata(profiledBlock).m_callLinkInfo;
break;
case op_iterator_next:
callLinkInfo = &instruction->as<OpIteratorNext>().metadata(profiledBlock).m_callLinkInfo;
break;
default:
return CallLinkStatus();
}
return CallLinkStatus(callLinkInfo->lastSeenCallee());
}
CallLinkStatus CallLinkStatus::computeFor(
CodeBlock* profiledBlock, BytecodeIndex bytecodeIndex, const ICStatusMap& map,
ExitSiteData exitSiteData)
{
ConcurrentJSLocker locker(profiledBlock->m_lock);
UNUSED_PARAM(profiledBlock);
UNUSED_PARAM(bytecodeIndex);
UNUSED_PARAM(map);
UNUSED_PARAM(exitSiteData);
#if ENABLE(DFG_JIT)
CallLinkInfo* callLinkInfo = map.get(CodeOrigin(bytecodeIndex)).callLinkInfo;
if (!callLinkInfo) {
if (exitSiteData.takesSlowPath)
return takesSlowPath();
return computeFromLLInt(locker, profiledBlock, bytecodeIndex);
}
return computeFor(locker, profiledBlock, *callLinkInfo, exitSiteData);
#else
return CallLinkStatus();
#endif
}
CallLinkStatus CallLinkStatus::computeFor(
CodeBlock* profiledBlock, BytecodeIndex bytecodeIndex, const ICStatusMap& map)
{
return computeFor(profiledBlock, bytecodeIndex, map, computeExitSiteData(profiledBlock, bytecodeIndex));
}
CallLinkStatus::ExitSiteData CallLinkStatus::computeExitSiteData(CodeBlock* profiledBlock, BytecodeIndex bytecodeIndex)
{
ExitSiteData exitSiteData;
#if ENABLE(DFG_JIT)
UnlinkedCodeBlock* codeBlock = profiledBlock->unlinkedCodeBlock();
ConcurrentJSLocker locker(codeBlock->m_lock);
auto takesSlowPath = [&] (ExitingInlineKind inlineKind) -> ExitFlag {
return ExitFlag(
codeBlock->hasExitSite(locker, DFG::FrequentExitSite(bytecodeIndex, BadType, ExitFromAnything, inlineKind))
|| codeBlock->hasExitSite(locker, DFG::FrequentExitSite(bytecodeIndex, BadExecutable, ExitFromAnything, inlineKind)),
inlineKind);
};
auto badFunction = [&] (ExitingInlineKind inlineKind) -> ExitFlag {
return ExitFlag(codeBlock->hasExitSite(locker, DFG::FrequentExitSite(bytecodeIndex, BadConstantValue, ExitFromAnything, inlineKind)), inlineKind);
};
exitSiteData.takesSlowPath |= takesSlowPath(ExitFromNotInlined);
exitSiteData.takesSlowPath |= takesSlowPath(ExitFromInlined);
exitSiteData.badFunction |= badFunction(ExitFromNotInlined);
exitSiteData.badFunction |= badFunction(ExitFromInlined);
#else
UNUSED_PARAM(profiledBlock);
UNUSED_PARAM(bytecodeIndex);
#endif
return exitSiteData;
}
#if ENABLE(JIT)
CallLinkStatus CallLinkStatus::computeFor(
const ConcurrentJSLocker& locker, CodeBlock* profiledBlock, CallLinkInfo& callLinkInfo)
{
// We don't really need this, but anytime we have to debug this code, it becomes indispensable.
UNUSED_PARAM(profiledBlock);
CallLinkStatus result = computeFromCallLinkInfo(locker, callLinkInfo);
result.m_maxArgumentCountIncludingThis = callLinkInfo.maxArgumentCountIncludingThis();
return result;
}
CallLinkStatus CallLinkStatus::computeFromCallLinkInfo(
const ConcurrentJSLocker&, CallLinkInfo& callLinkInfo)
{
// We cannot tell you anything about direct calls.
if (callLinkInfo.isDirect())
return CallLinkStatus();
if (callLinkInfo.clearedByGC() || callLinkInfo.clearedByVirtual())
return takesSlowPath();
// Note that despite requiring that the locker is held, this code is racy with respect
// to the CallLinkInfo: it may get cleared while this code runs! This is because
// CallLinkInfo::unlink() may be called from a different CodeBlock than the one that owns
// the CallLinkInfo and currently we save space by not having CallLinkInfos know who owns
// them. So, there is no way for either the caller of CallLinkInfo::unlock() or unlock()
// itself to figure out which lock to lock.
//
// Fortunately, that doesn't matter. The only things we ask of CallLinkInfo - the slow
// path count, the stub, and the target - can all be asked racily. Stubs and targets can
// only be deleted at next GC, so if we load a non-null one, then it must contain data
// that is still marginally valid (i.e. the pointers ain't stale). This kind of raciness
// is probably OK for now.
// PolymorphicCallStubRoutine is a GCAwareJITStubRoutine, so if non-null, it will stay alive
// until next GC even if the CallLinkInfo is concurrently cleared. Also, the variants list is
// never mutated after the PolymorphicCallStubRoutine is instantiated. We have some conservative
// fencing in place to make sure that we see the variants list after construction.
if (PolymorphicCallStubRoutine* stub = callLinkInfo.stub()) {
WTF::loadLoadFence();
if (!stub->hasEdges()) {
// This means we have an FTL profile, which has incomplete information.
//
// It's not clear if takesSlowPath() or CallLinkStatus() are appropriate here, but I
// think that takesSlowPath() is a narrow winner.
//
// Either way, this is telling us that the FTL had been led to believe that it's
// profitable not to inline anything.
return takesSlowPath();
}
CallEdgeList edges = stub->edges();
// Now that we've loaded the edges list, there are no further concurrency concerns. We will
// just manipulate and prune this list to our liking - mostly removing entries that are too
// infrequent and ensuring that it's sorted in descending order of frequency.
RELEASE_ASSERT(edges.size());
std::sort(
edges.begin(), edges.end(),
[] (CallEdge a, CallEdge b) {
return a.count() > b.count();
});
RELEASE_ASSERT(edges.first().count() >= edges.last().count());
double totalCallsToKnown = 0;
double totalCallsToUnknown = callLinkInfo.slowPathCount();
CallVariantList variants;
for (size_t i = 0; i < edges.size(); ++i) {
CallEdge edge = edges[i];
// If the call is at the tail of the distribution, then we don't optimize it and we
// treat it as if it was a call to something unknown. We define the tail as being either
// a call that doesn't belong to the N most frequent callees (N =
// maxPolymorphicCallVariantsForInlining) or that has a total call count that is too
// small.
if (i >= Options::maxPolymorphicCallVariantsForInlining()
|| edge.count() < Options::frequentCallThreshold())
totalCallsToUnknown += edge.count();
else {
totalCallsToKnown += edge.count();
variants.append(edge.callee());
}
}
// Bail if we didn't find any calls that qualified.
RELEASE_ASSERT(!!totalCallsToKnown == !!variants.size());
if (variants.isEmpty())
return takesSlowPath();
// We require that the distribution of callees is skewed towards a handful of common ones.
if (totalCallsToKnown / totalCallsToUnknown < Options::minimumCallToKnownRate())
return takesSlowPath();
RELEASE_ASSERT(totalCallsToKnown);
RELEASE_ASSERT(variants.size());
CallLinkStatus result;
result.m_variants = variants;
result.m_couldTakeSlowPath = !!totalCallsToUnknown;
result.m_isBasedOnStub = true;
return result;
}
CallLinkStatus result;
if (JSObject* target = callLinkInfo.lastSeenCallee()) {
CallVariant variant(target);
if (callLinkInfo.hasSeenClosure())
variant = variant.despecifiedClosure();
result.m_variants.append(variant);
}
result.m_couldTakeSlowPath = !!callLinkInfo.slowPathCount();
return result;
}
CallLinkStatus CallLinkStatus::computeFor(
const ConcurrentJSLocker& locker, CodeBlock* profiledBlock, CallLinkInfo& callLinkInfo,
ExitSiteData exitSiteData, ExitingInlineKind inlineKind)
{
CallLinkStatus result = computeFor(locker, profiledBlock, callLinkInfo);
result.accountForExits(exitSiteData, inlineKind);
return result;
}
void CallLinkStatus::accountForExits(ExitSiteData exitSiteData, ExitingInlineKind inlineKind)
{
if (exitSiteData.badFunction.isSet(inlineKind)) {
if (isBasedOnStub()) {
// If we have a polymorphic stub, then having an exit site is not quite so useful. In
// most cases, the information in the stub has higher fidelity.
makeClosureCall();
} else {
// We might not have a polymorphic stub for any number of reasons. When this happens, we
// are in less certain territory, so exit sites mean a lot.
m_couldTakeSlowPath = true;
}
}
if (exitSiteData.takesSlowPath.isSet(inlineKind))
m_couldTakeSlowPath = true;
}
CallLinkStatus CallLinkStatus::computeFor(
CodeBlock* profiledBlock, CodeOrigin codeOrigin,
const ICStatusMap& baselineMap, const ICStatusContextStack& optimizedStack)
{
if (CallLinkStatusInternal::verbose)
dataLog("Figuring out call profiling for ", codeOrigin, "\n");
ExitSiteData exitSiteData = computeExitSiteData(profiledBlock, codeOrigin.bytecodeIndex());
if (CallLinkStatusInternal::verbose) {
dataLog("takesSlowPath = ", exitSiteData.takesSlowPath, "\n");
dataLog("badFunction = ", exitSiteData.badFunction, "\n");
}
for (const ICStatusContext* context : optimizedStack) {
if (CallLinkStatusInternal::verbose)
dataLog("Examining status in ", pointerDump(context->optimizedCodeBlock), "\n");
ICStatus status = context->get(codeOrigin);
// If we have both a CallLinkStatus and a callLinkInfo for the same origin, then it means
// one of two things:
//
// 1) We initially thought that we'd be able to inline this call so we recorded a status
// but then we realized that it was pointless and gave up and emitted a normal call
// anyway.
//
// 2) We did a polymorphic call inline that had a slow path case.
//
// In the first case, it's essential that we use the callLinkInfo, since the status may
// be polymorphic but the link info benefits from polyvariant profiling.
//
// In the second case, it's essential that we use the status, since the callLinkInfo
// corresponds to the slow case.
//
// It would be annoying to distinguish those two cases. However, we know that:
//
// If the first case happens in the FTL, then it means that even with polyvariant
// profiling, we failed to do anything useful. That suggests that in the FTL, it's OK to
// prioritize the status. That status will again tell us to not do anything useful.
//
// The second case only happens in the FTL.
//
// Therefore, we prefer the status in the FTL and the info in the DFG.
//
// Luckily, this case doesn't matter for the other ICStatuses, since they never do the
// fast-path-slow-path control-flow-diamond style of IC inlining. It's either all fast
// path or it's a full IC. So, for them, if there is an IC status then it means case (1).
bool checkStatusFirst = context->optimizedCodeBlock->jitType() == JITType::FTLJIT;
auto bless = [&] (CallLinkStatus& result) {
if (!context->isInlined(codeOrigin))
result.merge(computeFor(profiledBlock, codeOrigin.bytecodeIndex(), baselineMap, exitSiteData));
};
auto checkInfo = [&] () -> CallLinkStatus {
if (!status.callLinkInfo)
return CallLinkStatus();
if (CallLinkStatusInternal::verbose)
dataLog("Have CallLinkInfo with CodeOrigin = ", status.callLinkInfo->codeOrigin(), "\n");
CallLinkStatus result;
{
ConcurrentJSLocker locker(context->optimizedCodeBlock->m_lock);
result = computeFor(
locker, context->optimizedCodeBlock, *status.callLinkInfo, exitSiteData,
context->inlineKind(codeOrigin));
if (CallLinkStatusInternal::verbose)
dataLog("Got result: ", result, "\n");
}
bless(result);
return result;
};
auto checkStatus = [&] () -> CallLinkStatus {
if (!status.callStatus)
return CallLinkStatus();
CallLinkStatus result = *status.callStatus;
if (CallLinkStatusInternal::verbose)
dataLog("Have callStatus: ", result, "\n");
result.accountForExits(exitSiteData, context->inlineKind(codeOrigin));
bless(result);
return result;
};
if (checkStatusFirst) {
if (CallLinkStatus result = checkStatus())
return result;
if (CallLinkStatus result = checkInfo())
return result;
continue;
}
if (CallLinkStatus result = checkInfo())
return result;
if (CallLinkStatus result = checkStatus())
return result;
}
return computeFor(profiledBlock, codeOrigin.bytecodeIndex(), baselineMap, exitSiteData);
}
#endif
void CallLinkStatus::setProvenConstantCallee(CallVariant variant)
{
m_variants = CallVariantList{ variant };
m_couldTakeSlowPath = false;
m_isProved = true;
}
bool CallLinkStatus::isClosureCall() const
{
for (unsigned i = m_variants.size(); i--;) {
if (m_variants[i].isClosureCall())
return true;
}
return false;
}
void CallLinkStatus::makeClosureCall()
{
m_variants = despecifiedVariantList(m_variants);
}
bool CallLinkStatus::finalize(VM& vm)
{
for (CallVariant& variant : m_variants) {
if (!variant.finalize(vm))
return false;
}
return true;
}
void CallLinkStatus::merge(const CallLinkStatus& other)
{
m_couldTakeSlowPath |= other.m_couldTakeSlowPath;
for (const CallVariant& otherVariant : other.m_variants) {
bool found = false;
for (CallVariant& thisVariant : m_variants) {
if (thisVariant.merge(otherVariant)) {
found = true;
break;
}
}
if (!found)
m_variants.append(otherVariant);
}
}
void CallLinkStatus::filter(VM& vm, JSValue value)
{
m_variants.removeAllMatching(
[&] (CallVariant& variant) -> bool {
variant.filter(vm, value);
return !variant;
});
}
void CallLinkStatus::dump(PrintStream& out) const
{
if (!isSet()) {
out.print("Not Set");
return;
}
CommaPrinter comma;
if (m_isProved)
out.print(comma, "Statically Proved");
if (m_couldTakeSlowPath)
out.print(comma, "Could Take Slow Path");
if (m_isBasedOnStub)
out.print(comma, "Based On Stub");
if (!m_variants.isEmpty())
out.print(comma, listDump(m_variants));
if (m_maxArgumentCountIncludingThis)
out.print(comma, "maxArgumentCountIncludingThis = ", m_maxArgumentCountIncludingThis);
}
} // namespace JSC