blob: 2b5e5547a1999ee88a1608defde30ceee178dce8 [file] [log] [blame]
/*
* Copyright (C) 2018-2020 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "config.h"
#include "LocalAllocator.h"
#include "AllocatingScope.h"
#include "FreeListInlines.h"
#include "GCDeferralContext.h"
#include "LocalAllocatorInlines.h"
#include "Options.h"
#include "SuperSampler.h"
namespace JSC {
LocalAllocator::LocalAllocator(BlockDirectory* directory)
: m_directory(directory)
, m_freeList(directory->m_cellSize)
{
auto locker = holdLock(directory->m_localAllocatorsLock);
directory->m_localAllocators.append(this);
}
void LocalAllocator::reset()
{
m_freeList.clear();
m_currentBlock = nullptr;
m_lastActiveBlock = nullptr;
m_allocationCursor = 0;
}
LocalAllocator::~LocalAllocator()
{
if (isOnList()) {
auto locker = holdLock(m_directory->m_localAllocatorsLock);
remove();
}
bool ok = true;
if (!m_freeList.allocationWillFail()) {
dataLog("FATAL: ", RawPointer(this), "->~LocalAllocator has non-empty free-list.\n");
ok = false;
}
if (m_currentBlock) {
dataLog("FATAL: ", RawPointer(this), "->~LocalAllocator has non-null current block.\n");
ok = false;
}
if (m_lastActiveBlock) {
dataLog("FATAL: ", RawPointer(this), "->~LocalAllocator has non-null last active block.\n");
ok = false;
}
RELEASE_ASSERT(ok);
}
void LocalAllocator::stopAllocating()
{
ASSERT(!m_lastActiveBlock);
if (!m_currentBlock) {
ASSERT(m_freeList.allocationWillFail());
return;
}
m_currentBlock->stopAllocating(m_freeList);
m_lastActiveBlock = m_currentBlock;
m_currentBlock = nullptr;
m_freeList.clear();
}
void LocalAllocator::resumeAllocating()
{
if (!m_lastActiveBlock)
return;
m_lastActiveBlock->resumeAllocating(m_freeList);
m_currentBlock = m_lastActiveBlock;
m_lastActiveBlock = nullptr;
}
void LocalAllocator::prepareForAllocation()
{
reset();
}
void LocalAllocator::stopAllocatingForGood()
{
stopAllocating();
reset();
}
void* LocalAllocator::allocateSlowCase(Heap& heap, GCDeferralContext* deferralContext, AllocationFailureMode failureMode)
{
SuperSamplerScope superSamplerScope(false);
ASSERT(heap.vm().currentThreadIsHoldingAPILock());
doTestCollectionsIfNeeded(heap, deferralContext);
ASSERT(!m_directory->markedSpace().isIterating());
heap.didAllocate(m_freeList.originalSize());
didConsumeFreeList();
AllocatingScope helpingHeap(heap);
heap.collectIfNecessaryOrDefer(deferralContext);
// Goofy corner case: the GC called a callback and now this directory has a currentBlock. This only
// happens when running WebKit tests, which inject a callback into the GC's finalization.
if (UNLIKELY(m_currentBlock))
return allocate(heap, deferralContext, failureMode);
void* result = tryAllocateWithoutCollecting();
if (LIKELY(result != nullptr))
return result;
Subspace* subspace = m_directory->m_subspace;
if (subspace->isIsoSubspace()) {
if (void* result = static_cast<IsoSubspace*>(subspace)->tryAllocateFromLowerTier())
return result;
}
MarkedBlock::Handle* block = m_directory->tryAllocateBlock(heap);
if (!block) {
if (failureMode == AllocationFailureMode::Assert)
RELEASE_ASSERT_NOT_REACHED();
else
return nullptr;
}
m_directory->addBlock(block);
result = allocateIn(block);
ASSERT(result);
return result;
}
void LocalAllocator::didConsumeFreeList()
{
if (m_currentBlock)
m_currentBlock->didConsumeFreeList();
m_freeList.clear();
m_currentBlock = nullptr;
}
void* LocalAllocator::tryAllocateWithoutCollecting()
{
// FIXME: If we wanted this to be used for real multi-threaded allocations then we would have to
// come up with some concurrency protocol here. That protocol would need to be able to handle:
//
// - The basic case of multiple LocalAllocators trying to do an allocationCursor search on the
// same bitvector. That probably needs the bitvector lock at least.
//
// - The harder case of some LocalAllocator triggering a steal from a different BlockDirectory
// via a search in the AlignedMemoryAllocator's list. Who knows what locks that needs.
//
// One way to make this work is to have a single per-Heap lock that protects all mutator lock
// allocation slow paths. That would probably be scalable enough for years. It would certainly be
// for using TLC allocation from JIT threads.
// https://bugs.webkit.org/show_bug.cgi?id=181635
SuperSamplerScope superSamplerScope(false);
ASSERT(!m_currentBlock);
ASSERT(m_freeList.allocationWillFail());
for (;;) {
MarkedBlock::Handle* block = m_directory->findBlockForAllocation(*this);
if (!block)
break;
if (void* result = tryAllocateIn(block))
return result;
}
if (Options::stealEmptyBlocksFromOtherAllocators()) {
if (MarkedBlock::Handle* block = m_directory->m_subspace->findEmptyBlockToSteal()) {
RELEASE_ASSERT(block->alignedMemoryAllocator() == m_directory->m_subspace->alignedMemoryAllocator());
block->sweep(nullptr);
// It's good that this clears canAllocateButNotEmpty as well as all other bits,
// because there is a remote chance that a block may have both canAllocateButNotEmpty
// and empty set at the same time.
block->removeFromDirectory();
m_directory->addBlock(block);
return allocateIn(block);
}
}
return nullptr;
}
void* LocalAllocator::allocateIn(MarkedBlock::Handle* block)
{
void* result = tryAllocateIn(block);
RELEASE_ASSERT(result);
return result;
}
void* LocalAllocator::tryAllocateIn(MarkedBlock::Handle* block)
{
ASSERT(block);
ASSERT(!block->isFreeListed());
block->sweep(&m_freeList);
// It's possible to stumble on a completely full block. Marking tries to retire these, but
// that algorithm is racy and may forget to do it sometimes.
if (m_freeList.allocationWillFail()) {
ASSERT(block->isFreeListed());
block->unsweepWithNoNewlyAllocated();
ASSERT(!block->isFreeListed());
ASSERT(!m_directory->isEmpty(NoLockingNecessary, block));
ASSERT(!m_directory->isCanAllocateButNotEmpty(NoLockingNecessary, block));
return nullptr;
}
m_currentBlock = block;
void* result = m_freeList.allocate(
[] () -> HeapCell* {
RELEASE_ASSERT_NOT_REACHED();
return nullptr;
});
m_directory->setIsEden(NoLockingNecessary, m_currentBlock, true);
m_directory->markedSpace().didAllocateInBlock(m_currentBlock);
return result;
}
void LocalAllocator::doTestCollectionsIfNeeded(Heap& heap, GCDeferralContext* deferralContext)
{
if (!Options::slowPathAllocsBetweenGCs())
return;
static unsigned allocationCount = 0;
if (!allocationCount) {
if (!heap.isDeferred()) {
if (deferralContext)
deferralContext->m_shouldGC = true;
else
heap.collectNow(Sync, CollectionScope::Full);
}
}
if (++allocationCount >= Options::slowPathAllocsBetweenGCs())
allocationCount = 0;
}
bool LocalAllocator::isFreeListedCell(const void* target) const
{
// This abomination exists to detect when an object is in the dead-but-not-destructed state.
// Therefore, it's not even clear that this needs to do anything beyond returning "false", since
// if we know that the block owning the object is free-listed, then it's impossible for any
// objects to be in the dead-but-not-destructed state.
// FIXME: Get rid of this abomination. https://bugs.webkit.org/show_bug.cgi?id=181655
return m_freeList.contains(bitwise_cast<HeapCell*>(target), m_currentBlock);
}
} // namespace JSC