blob: cd25cced516553d8b8f637f8cd26fe5b1aa8545a [file] [log] [blame]
/*
* Copyright (c) 2019-2022 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "TestHarness.h"
#if PAS_ENABLE_ISO
#include "bmalloc_heap.h"
#include "bmalloc_heap_config.h"
#include "bmalloc_heap_innards.h"
#include "hotbit_heap.h"
#include "hotbit_heap_config.h"
#include "hotbit_heap_innards.h"
#include "iso_heap.h"
#include "iso_heap_config.h"
#include "iso_heap_innards.h"
#include "iso_test_heap.h"
#include "iso_test_heap_config.h"
#include "jit_heap.h"
#include "jit_heap_config.h"
#include <map>
#include "minalign32_heap.h"
#include "minalign32_heap_config.h"
#include <mutex>
#include "pagesize64k_heap.h"
#include "pagesize64k_heap_config.h"
#include "pas_all_heaps.h"
#include "pas_baseline_allocator_table.h"
#include "pas_enumerable_page_malloc.h"
#include "pas_epoch.h"
#include "pas_get_page_base.h"
#include "pas_heap.h"
#include "pas_root.h"
#include "pas_segregated_page.h"
#include "pas_thread_local_cache.h"
#include "pas_utility_heap.h"
#include <set>
#include <sstream>
#include <string.h>
#include <sys/mman.h>
#include <vector>
#include <thread>
#if PAS_OS(DARWIN)
#include <mach/thread_act.h>
#endif
using namespace std;
namespace {
constexpr bool verbose = false;
pas_heap_config* selectedHeapConfig;
void* (*selectedAllocateCommonPrimitive)(size_t size);
void* (*selectedAllocate)(pas_heap_ref* heapRef);
void* (*selectedAllocateArray)(pas_heap_ref* heapRef, size_t count, size_t alignment);
void (*selectedShrink)(void* ptr, size_t newSize);
void (*selectedDeallocate)(void* ptr);
pas_heap* selectedCommonPrimitiveHeap;
pas_heap* (*selectedHeapRefGetHeap)(pas_heap_ref* heapRef);
pas_heap_ref* (*selectedHeapRefCreateForSize)(size_t size);
pas_heap_ref* createIsoHeapRefForSize(size_t size)
{
return new pas_heap_ref(ISO_HEAP_REF_INITIALIZER_WITH_ALIGNMENT(
deterministicRandomNumber(100), 1));
}
#if PAS_ENABLE_BMALLOC
pas_primitive_heap_ref gigacageHeapRef;
void* gigacageAllocate(size_t size)
{
return bmalloc_allocate_auxiliary(&gigacageHeapRef, size);
}
pas_heap_ref* createBmallocHeapRefForSize(size_t size)
{
PAS_ASSERT((unsigned)size == size);
ostringstream stringOut;
stringOut << "Test Type With Size = " << size;
return new pas_heap_ref(
BMALLOC_HEAP_REF_INITIALIZER(
new bmalloc_type(BMALLOC_TYPE_INITIALIZER((unsigned)size, 1, strdup(stringOut.str().c_str())))));
}
#endif // PAS_ENABLE_BMALLOC
void flushDeallocationLogAndStopAllocators()
{
#if TLC
pas_thread_local_cache_shrink(pas_thread_local_cache_try_get(), pas_lock_is_not_held);
pas_baseline_allocator_table_for_all(pas_allocator_scavenge_force_stop_action);
pas_utility_heap_for_all_allocators(pas_allocator_scavenge_force_stop_action,
pas_lock_is_not_held);
#endif
}
template<typename ObjectVerificationFunc>
void verifyObjectSet(const set<void*>& expectedObjects,
const vector<pas_heap*>& heaps,
const ObjectVerificationFunc& objectVerificationFunc)
{
flushDeallocationLogAndStopAllocators();
if (verbose) {
cout << "Expected objects:\n";
dumpObjectSet(expectedObjects);
dumpObjectsInHeaps(heaps);
}
set<void*> foundObjects;
bool expectedAllFoundObjects = true;
bool foundFirstUnexpectedObjectInBitfit = false;
for (pas_heap* heap : heaps) {
forEachLiveObject(
heap,
[&] (void* object, size_t size) -> bool {
if (!expectedObjects.count(object)) {
cout << "Found unexpected object: " << object << " of size " << size << "\n";
expectedAllFoundObjects = false;
if (!foundFirstUnexpectedObjectInBitfit) {
pas_page_base* page_base = pas_get_page_base(object, *selectedHeapConfig);
if (pas_page_base_is_bitfit(page_base)) {
pas_bitfit_page* page = pas_page_base_get_bitfit(page_base);
uintptr_t offset = reinterpret_cast<uintptr_t>(object)
- reinterpret_cast<uintptr_t>(
pas_bitfit_page_boundary(
page,
*pas_heap_config_bitfit_page_config_ptr_for_variant(
selectedHeapConfig,
pas_page_kind_get_bitfit_variant(
pas_page_base_get_kind(page_base)))));
pas_bitfit_page_log_bits(page, offset, offset + 1);
foundFirstUnexpectedObjectInBitfit = true;
}
}
}
CHECK(!foundObjects.count(object));
objectVerificationFunc(object, size);
foundObjects.insert(object);
return true;
});
}
if (foundObjects.size() != expectedObjects.size() || !expectedAllFoundObjects) {
for (void* object : expectedObjects) {
if (!foundObjects.count(object))
cout << "Did not find expected object: " << object << "\n";
}
CHECK_EQUAL(foundObjects.size(), expectedObjects.size());
CHECK(expectedAllFoundObjects);
}
}
mutex lock;
set<pthread_t> runningThreads;
void scavengerDidStart()
{
lock_guard<mutex> locker(lock);
runningThreads.insert(pthread_self());
}
void scavengerWillShutDown()
{
lock_guard<mutex> locker(lock);
runningThreads.erase(pthread_self());
}
struct PageRange {
PageRange() = default;
PageRange(void* base, size_t size)
: base(base)
, size(size)
{
PAS_ASSERT(pas_is_aligned(reinterpret_cast<uintptr_t>(base), pas_page_malloc_alignment()));
PAS_ASSERT(pas_is_aligned(size, pas_page_malloc_alignment()));
PAS_ASSERT(!!base == !!size);
}
bool operator<(PageRange range) const
{
return base < range.base;
}
void* end() const
{
return reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(base) + size);
}
void* base { nullptr };
size_t size { 0 };
};
set<PageRange> pageRanges;
void addPageRange(pas_range range)
{
if (verbose) {
cout << "libpas page range: " << reinterpret_cast<void*>(range.begin) << "..."
<< reinterpret_cast<void*>(range.end) << "\n";
}
pageRanges.insert(PageRange(reinterpret_cast<void*>(range.begin), pas_range_size(range)));
}
bool addPageRangeCallback(pas_range range, void* arg)
{
PAS_ASSERT(!arg);
addPageRange(range);
return true;
}
struct RecordedRange {
RecordedRange() = default;
RecordedRange(void* base, size_t size)
: base(base)
, size(size)
{
PAS_ASSERT(base);
}
bool operator<(RecordedRange other) const
{
return base < other.base;
}
void* end() const
{
return reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(base) + size);
}
void* base { nullptr };
size_t size { 0 };
};
struct ReaderRange {
ReaderRange() = default;
ReaderRange(void* base, size_t size)
: base(base)
, size(size)
{
PAS_ASSERT(base);
PAS_ASSERT(size);
}
bool operator<(ReaderRange other) const
{
if (base != other.base)
return base < other.base;
return size < other.size;
}
void* base { nullptr };
size_t size { 0 };
};
map<pas_enumerator_record_kind, set<RecordedRange>> recordedRanges;
map<ReaderRange, void*> readerCache;
void* enumeratorReader(pas_enumerator* enumerator,
void* address,
size_t size,
void* arg)
{
CHECK(!arg);
CHECK(size);
ReaderRange range = ReaderRange(address, size);
auto readerCacheIter = readerCache.find(range);
if (readerCacheIter != readerCache.end())
return readerCacheIter->second;
void* result = pas_enumerator_allocate(enumerator, size);
void* pageAddress = reinterpret_cast<void*>(
pas_round_down_to_power_of_2(
reinterpret_cast<uintptr_t>(address),
pas_page_malloc_alignment()));
size_t pagesSize =
pas_round_up_to_power_of_2(
reinterpret_cast<uintptr_t>(address) + size,
pas_page_malloc_alignment())
- reinterpret_cast<uintptr_t>(pageAddress);
void* pageEndAddress = reinterpret_cast<void*>(
reinterpret_cast<uintptr_t>(pageAddress) + pagesSize);
bool areProtectedPages = false;
if (!pageRanges.empty()) {
auto pageRangeIter = pageRanges.upper_bound(PageRange(pageAddress, pagesSize));
if (pageRangeIter != pageRanges.begin()) {
--pageRangeIter;
areProtectedPages =
pageRangeIter->base <= pageAddress
&& pageRangeIter->end() > pageAddress;
}
}
if (verbose) {
cout << "address = " << address << "..."
<< reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(address) + size)
<< ", pageAddress = " << pageAddress << "..." << pageEndAddress
<< ", areProtectedPages = " << areProtectedPages << "\n";
}
if (areProtectedPages) {
int systemResult = mprotect(pageAddress, pagesSize, PROT_READ);
PAS_ASSERT(!systemResult);
}
memcpy(result, address, size);
if (areProtectedPages) {
int systemResult = mprotect(pageAddress, pagesSize, PROT_NONE);
PAS_ASSERT(!systemResult);
}
readerCache[range] = result;
return result;
}
void enumeratorRecorder(pas_enumerator* enumerator,
void* address,
size_t size,
pas_enumerator_record_kind kind,
void* arg)
{
PAS_UNUSED_PARAM(enumerator);
CHECK(size);
CHECK(!arg);
if (verbose) {
cout << "Recording " << pas_enumerator_record_kind_get_string(kind) << ":" << address
<< "..." << reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(address) + size) << "\n";
}
RecordedRange range = RecordedRange(address, size);
CHECK(!recordedRanges[kind].count(range));
recordedRanges[kind].insert(range);
}
unsigned uniformlyRandomUpTo5000()
{
return deterministicRandomNumber(5000);
}
unsigned sometimesSmallSometimesBig()
{
if (deterministicRandomNumber(300))
return deterministicRandomNumber(5000);
unsigned result = deterministicRandomNumber(1000000);
return result;
}
void testAllocationChaos(unsigned numThreads, unsigned numIsolatedHeaps,
unsigned numInitialAllocations, unsigned numActions,
unsigned (*chooseSize)(),
unsigned maxTotalSize, bool testEnumerator)
{
PAS_ASSERT(!pas_epoch_is_counter); // We don't want to run these tests in the fake scavenger mode.
if (verbose)
cout << "Starting.\n";
struct OptionalObject {
OptionalObject() = default;
OptionalObject(void* ptr, size_t size)
: isValid(true)
, ptr(ptr)
, size(size)
{
}
bool isValid { false };
void* ptr { nullptr };
size_t size { 0 };
};
struct Object {
Object() = default;
Object(void* ptr, size_t size, uint8_t startValue)
: ptr(ptr)
, size(size)
, startValue(startValue)
{
}
void* ptr { nullptr };
size_t size { 0 };
uint8_t startValue { 0 };
};
vector<Object> objects;
vector<thread> threads;
unsigned totalSize;
vector<OptionalObject> optionalObjects;
if (testEnumerator) {
pas_scavenger_did_start_callback = scavengerDidStart;
pas_scavenger_will_shut_down_callback = scavengerWillShutDown;
for (unsigned threadIndex = numThreads; threadIndex--;)
optionalObjects.push_back(OptionalObject());
}
if (!selectedAllocate && numIsolatedHeaps) {
cout << "Skipping because test requires isolated heaps and we don't support them in this configuration.\n";
return;
}
vector<pas_heap_ref*> isolatedHeaps;
for (unsigned isolatedHeapIndex = numIsolatedHeaps; isolatedHeapIndex--;)
isolatedHeaps.push_back(selectedHeapRefCreateForSize(deterministicRandomNumber(100)));
auto addObject =
[&] (unsigned threadIndex, void* ptr, unsigned size) {
CHECK(ptr);
uint8_t startValue = static_cast<uint8_t>(deterministicRandomNumber(255));
auto populateObject = [&] () {
for (unsigned i = 0; i < size; ++i)
static_cast<uint8_t*>(ptr)[i] = static_cast<uint8_t>(startValue + i);
};
if (!testEnumerator)
populateObject();
{
lock_guard<mutex> locker(lock);
if (testEnumerator)
populateObject();
if (verbose)
cout << "Adding object " << ptr << "\n";
if (testEnumerator)
optionalObjects[threadIndex] = OptionalObject();
objects.push_back(Object(ptr, size, startValue));
totalSize += size;
}
};
auto logOptionalObject =
[&] (unsigned threadIndex, unsigned size) {
if (testEnumerator) {
lock_guard<mutex> locker(lock);
if (verbose)
cout << "Allocating object of size = " << size << "\n";
optionalObjects[threadIndex] = OptionalObject(nullptr, size);
}
};
auto allocateSomething = [&] (unsigned threadIndex) {
unsigned heapIndex = deterministicRandomNumber(static_cast<unsigned>(isolatedHeaps.size() + 1));
unsigned size = chooseSize();
if (heapIndex == isolatedHeaps.size()) {
logOptionalObject(threadIndex, size);
void* ptr = selectedAllocateCommonPrimitive(size);
addObject(threadIndex, ptr, size);
return;
}
PAS_ASSERT(selectedAllocate);
pas_heap_ref* heap = isolatedHeaps[heapIndex];
unsigned count;
if (selectedAllocateArray)
count = size / pas_simple_type_size(reinterpret_cast<pas_simple_type>(heap->type));
else
count = 1;
size = static_cast<unsigned>(
count * pas_simple_type_size(reinterpret_cast<pas_simple_type>(heap->type)));
logOptionalObject(threadIndex, size);
void* ptr;
if (count <= 1)
ptr = selectedAllocate(heap);
else
ptr = selectedAllocateArray(heap, count, 1);
addObject(threadIndex, ptr, size);
};
auto freeSomething = [&] (unsigned threadIndex) {
Object object;
auto validateObject = [&] () {
for (unsigned i = 0; i < object.size; ++i) {
CHECK_EQUAL(static_cast<unsigned>(static_cast<uint8_t*>(object.ptr)[i]),
static_cast<unsigned>(static_cast<uint8_t>(object.startValue + i)));
}
};
{
lock_guard<mutex> locker(lock);
if (objects.empty())
return;
unsigned index = deterministicRandomNumber(static_cast<unsigned>(objects.size()));
object = objects[index];
objects[index] = objects.back();
objects.pop_back();
totalSize -= object.size;
if (verbose)
cout << "Freeing object " << object.ptr << "\n";
if (testEnumerator) {
optionalObjects[threadIndex] = OptionalObject(object.ptr, object.size);
validateObject();
}
}
if (!testEnumerator)
validateObject();
selectedDeallocate(object.ptr);
if (testEnumerator) {
lock_guard<mutex> locker(lock);
if (verbose)
cout << "Freed object " << object.ptr << "\n";
optionalObjects[threadIndex] = OptionalObject();
}
};
auto shrinkSomething = [&] (unsigned threadIndex) {
// We could actually test the enumerator here, but we don't because that would require adding
// hacks to be tolerant of the weirdness the enumerator might see during the shrink. So for
// now we just don't test it. FIXME: At least test that the enumerator doesn't crash when
// dealing with shrink.
PAS_ASSERT(!testEnumerator);
Object object;
size_t newSize;
{
lock_guard<mutex> locker(lock);
if (objects.empty())
return;
unsigned index = deterministicRandomNumber(static_cast<unsigned>(objects.size()));
object = objects[index];
objects[index] = objects.back();
objects.pop_back();
totalSize -= object.size;
if (verbose)
cout << "Shrinking object " << object.ptr << "\n";
}
if (object.size)
newSize = deterministicRandomNumber(static_cast<unsigned>(object.size));
else
newSize = 0;
selectedShrink(object.ptr, newSize);
object.size = newSize;
{
lock_guard<mutex> locker(lock);
objects.push_back(object);
}
};
pas_heap_lock_lock();
pas_root* root = pas_root_create();
pas_heap_lock_unlock();
unsigned numEnumerations = 0;
auto runEnumerator =
[&] () {
lock.lock();
if (verbose) {
cout << "Beginning enumeration, optionalObjects = " << "\n";
for (size_t index = 0; index < optionalObjects.size(); ++index) {
if (!optionalObjects[index].isValid)
continue;
cout << " " << index << ": ptr = " << optionalObjects[index].ptr
<< ", size " << optionalObjects[index].size << "\n";
}
}
#if PAS_OS(DARWIN)
for (pthread_t thread : runningThreads) {
kern_return_t result = thread_suspend(pthread_mach_thread_np(thread));
PAS_ASSERT(result == KERN_SUCCESS);
}
#endif
pageRanges.clear();
readerCache.clear();
recordedRanges.clear();
addPageRange(
pas_range_create(
pas_compact_heap_reservation_base + pas_compact_heap_reservation_guard_size,
pas_compact_heap_reservation_base + pas_compact_heap_reservation_guard_size
+ pas_compact_heap_reservation_size));
bool enumerateResult = pas_enumerable_range_list_iterate(
&pas_enumerable_page_malloc_page_list, addPageRangeCallback, nullptr);
PAS_ASSERT(enumerateResult);
for (PageRange range : pageRanges) {
int result = mprotect(range.base, range.size, PROT_NONE);
PAS_ASSERT(!result);
}
pas_enumerator* enumerator = pas_enumerator_create(root,
enumeratorReader,
nullptr,
enumeratorRecorder,
nullptr,
pas_enumerator_record_meta_records,
pas_enumerator_record_payload_records,
pas_enumerator_record_object_records);
pas_enumerator_enumerate_all(enumerator);
if (verbose) {
for (auto& entry : recordedRanges) {
pas_enumerator_record_kind kind = entry.first;
for (auto& range : entry.second)
cout << pas_enumerator_record_kind_get_string(kind) << ": "
<< range.base << "..." << range.end() << "\n";
}
}
auto checkNonOverlapping =
[&] (const char* aSetName, set<RecordedRange>& aSet,
const char* bSetName, set<RecordedRange>& bSet,
bool allowIdentical) {
for (RecordedRange range : aSet) {
auto iter = bSet.lower_bound(range);
if (iter == bSet.end())
continue;
if (allowIdentical
&& iter->base == range.base
&& iter->size == range.size)
continue;
if (iter->base < range.end()) {
cout << "Recorded " << aSetName
<< " range " << range.base << "..." << range.end()
<< " overlaps with " << bSetName << " "<< iter->base << "..."
<< iter->end() << "\n";
CHECK_GREATER_EQUAL(iter->base, range.end());
}
}
};
// Check that the ranges are non-overlapping.
for (auto& setEntry : recordedRanges) {
checkNonOverlapping(pas_enumerator_record_kind_get_string(setEntry.first),
setEntry.second,
pas_enumerator_record_kind_get_string(setEntry.first),
setEntry.second,
true);
}
checkNonOverlapping("meta",
recordedRanges[pas_enumerator_meta_record],
"payload",
recordedRanges[pas_enumerator_payload_record],
false);
checkNonOverlapping("payload",
recordedRanges[pas_enumerator_payload_record],
"meta",
recordedRanges[pas_enumerator_meta_record],
false);
// Check that each object range is contained within a payload range.
for (RecordedRange objectRange : recordedRanges[pas_enumerator_object_record]) {
if (recordedRanges[pas_enumerator_payload_record].empty()) {
cout << "Recorded object range " << objectRange.base << "..." << objectRange.end()
<< " is not within any payload range (no payload ranges)\n";
CHECK(!"Object range but no payload ranges");
}
auto iter = recordedRanges[pas_enumerator_payload_record].upper_bound(objectRange);
if (iter == recordedRanges[pas_enumerator_payload_record].begin()) {
cout << "Recorded object range " << objectRange.base << "..." << objectRange.end()
<< " is not within any payload range (no range to the left, range to the "
<< "right is " << iter->base << "..." << iter->end() << ")\n";
CHECK(!"Object range not in any payload range");
}
--iter;
PAS_ASSERT(iter->base <= objectRange.base);
if (iter->end() < objectRange.base) {
cout << "Recorded object range " << objectRange.base << "..." << objectRange.end()
<< " is not within any payload range (range to the left is " << iter->base
<< "..." << iter->end() << ", ";
++iter;
if (iter == recordedRanges[pas_enumerator_payload_record].end())
cout << "no range to the right";
else
cout << "range to the right is " << iter->base << "..." << iter->end();
cout << ")\n";
CHECK_GREATER_EQUAL(iter->end(), objectRange.base);
}
if (iter->end() < objectRange.end()) {
cout << "Recorded object range " << objectRange.base << "..." << objectRange.end()
<< " is not entirely within the closest payload range " << iter->base
<< "..." << iter->end() << "\n";
CHECK_GREATER_EQUAL(iter->end(), objectRange.end());
}
}
// Make sure that the enumerator found all of the expected objects.
set<void*> expectedObjects;
for (Object object : objects) {
expectedObjects.insert(object.ptr);
auto iter = recordedRanges[pas_enumerator_object_record].find(
RecordedRange(object.ptr, object.size));
if (iter == recordedRanges[pas_enumerator_object_record].end()) {
cout << "Expected object " << object.ptr << " with size " << object.size
<< " not found by enumerator.\n";
CHECK(iter != recordedRanges[pas_enumerator_object_record].end());
}
if (iter->size < object.size) {
cout << "Expected object " << object.ptr << " with size " << object.size
<< " was found by enumerator, but with size " << iter->size << ".\n";
CHECK_GREATER_EQUAL(iter->size, object.size);
}
}
// Make sure that the enumerator only found objects that we expected, plus optional objects.
vector<bool> foundOptionalObjects;
for (size_t count = optionalObjects.size(); count--;)
foundOptionalObjects.push_back(false);
for (RecordedRange objectRange : recordedRanges[pas_enumerator_object_record]) {
if (expectedObjects.count(objectRange.base))
continue;
if (verbose) {
cout << "Found object " << objectRange.base << " with size " << objectRange.size
<< " is either unexpected or optional.\n";
}
// Now check if the object is something we could be allocating or freeing. Note that if
// we think that we're freeing an object, then we cannot really treat it differently, since
// athough we know the address of freed objects, that address could be reused for
// allocation just before we do the enumeration.
size_t bestIndex = SIZE_MAX;
for (size_t i = optionalObjects.size(); i--;) {
if (foundOptionalObjects[i])
continue;
if (optionalObjects[i].isValid
&& optionalObjects[i].size <= objectRange.size) {
if (bestIndex == SIZE_MAX
|| optionalObjects[i].size > optionalObjects[bestIndex].size)
bestIndex = i;
}
}
if (bestIndex == SIZE_MAX) {
cout << "Found object " << objectRange.base << " is not expected or optional.\n";
CHECK(bestIndex != SIZE_MAX);
}
if (verbose)
cout << "Checking off as optional index = " << bestIndex << "\n";
// Don't allow the same allocating object to be found twice.
foundOptionalObjects[bestIndex] = true;
}
pas_enumerator_destroy(enumerator);
for (PageRange range : pageRanges) {
int result = mprotect(range.base, range.size, PROT_READ | PROT_WRITE);
PAS_ASSERT(!result);
}
if (verbose)
cout << "Done enumerating!\n";
++numEnumerations;
if (!(numEnumerations % 50))
cout << " Did " << numEnumerations << " enumerations.\n";
#if PAS_OS(DARWIN)
for (pthread_t thread : runningThreads) {
kern_return_t result = thread_resume(pthread_mach_thread_np(thread));
PAS_ASSERT(result == KERN_SUCCESS);
}
#endif
lock.unlock();
};
if (testEnumerator)
runEnumerator();
for (unsigned allocationIndex = numInitialAllocations; allocationIndex--;)
allocateSomething(0);
if (testEnumerator)
runEnumerator();
unsigned numThreadsStopped = 0;
bool didStartThreads = false;
auto threadFunc = [&] (unsigned threadIndex) {
if (verbose)
cout << " Thread " << (void*)pthread_self() << " starting.\n";
{
lock_guard<mutex> locker(lock);
PAS_ASSERT(threads[threadIndex].get_id() == this_thread::get_id());
runningThreads.insert(pthread_self());
}
for (unsigned actionIndex = numActions; actionIndex--;) {
if (totalSize >= maxTotalSize) {
freeSomething(threadIndex);
continue;
}
switch (deterministicRandomNumber(selectedShrink ? 3 : 2)) {
case 0:
allocateSomething(threadIndex);
break;
case 1:
freeSomething(threadIndex);
break;
case 2:
PAS_ASSERT(selectedShrink);
shrinkSomething(threadIndex);
break;
default:
PAS_ASSERT(!"bad random number");
break;
}
}
if (verbose)
cout << " Thread " << (void*)pthread_self() << " stopping.\n";
if (testEnumerator)
pas_thread_local_cache_destroy(pas_lock_is_not_held);
{
lock_guard<mutex> locker(lock);
runningThreads.erase(pthread_self());
numThreadsStopped++;
}
};
thread enumerationThread;
if (testEnumerator) {
enumerationThread = thread(
[&] () {
while (!didStartThreads || numThreadsStopped < threads.size()) {
// Sleep for a tiny bit.
for (unsigned count = 1000; count--;)
sched_yield();
runEnumerator();
}
});
}
{
lock_guard<mutex> locker(lock);
for (unsigned threadIndex = 0; threadIndex < numThreads; ++threadIndex)
threads.push_back(thread(threadFunc, threadIndex));
didStartThreads = true;
}
if (verbose)
cout << "Joining.\n";
for (thread& thread : threads)
thread.join();
if (testEnumerator)
enumerationThread.join();
if (verbose)
cout << "Done.\n";
if (testEnumerator)
cout << " Test done; did " << numEnumerations << " enumerations.\n";
CHECK_EQUAL(numThreadsStopped, threads.size());
vector<pas_heap*> heaps;
heaps.push_back(selectedCommonPrimitiveHeap);
for (pas_heap_ref* heapRef : isolatedHeaps)
heaps.push_back(selectedHeapRefGetHeap(heapRef));
set<void*> expectedObjects;
for (Object object : objects)
expectedObjects.insert(object.ptr);
verifyObjectSet(expectedObjects, heaps, [] (void*, size_t) { });
pas_scavenger_run_synchronously_now();
if ((false))
printStatusReport();
pas_heap_lock_lock();
pas_all_heaps_verify_in_steady_state();
pas_heap_lock_unlock();
}
void addTheTests(unsigned multiplier, bool testEnumerator)
{
#if PAS_OS(LINUX)
// FIXME: thread suspension/resume in libpas, required for enumerator tests, is missing on Linux
// http://webkit.org/b/234071
testEnumerator = false;
#endif
ADD_TEST(testAllocationChaos(1, 0, 1000 * multiplier, 1000000 * multiplier, uniformlyRandomUpTo5000, 10000000 * multiplier, false));
ADD_TEST(testAllocationChaos(1, 10, 1000 * multiplier, 500000 * multiplier, sometimesSmallSometimesBig, 10000000 * multiplier, false));
ADD_TEST(testAllocationChaos(10, 0, 1000 * multiplier, 200000 * multiplier, sometimesSmallSometimesBig, 10000000 * multiplier, false));
ADD_TEST(testAllocationChaos(10, 10, 1000 * multiplier, 200000 * multiplier, uniformlyRandomUpTo5000, 10000000 * multiplier, false));
if (testEnumerator) {
ADD_TEST(testAllocationChaos(1, 0, 1000, 7000, sometimesSmallSometimesBig, 10000000, testEnumerator));
ADD_TEST(testAllocationChaos(1, 10, 1000, 7000, uniformlyRandomUpTo5000, 10000000, testEnumerator));
ADD_TEST(testAllocationChaos(10, 0, 1000, 4000, uniformlyRandomUpTo5000, 10000000, testEnumerator));
ADD_TEST(testAllocationChaos(10, 10, 1000, 4000, sometimesSmallSometimesBig, 10000000, testEnumerator));
}
}
void addSpotTests(bool testEnumerator)
{
ADD_GROUP(addTheTests(10, testEnumerator));
{
ForceExclusives forceExclusives;
DisableBitfit disableBitfit;
ADD_GROUP(addTheTests(1, testEnumerator));
}
{
ForceTLAs forceTLAs;
ForcePartials forcePartials;
ADD_GROUP(addTheTests(1, false));
}
{
ForceBitfit forceBitfit;
ADD_GROUP(addTheTests(1, false));
}
}
void addIsoTests()
{
#if PAS_ENABLE_ISO
{
TestScope iso(
"iso",
[] () {
selectedHeapConfig = &iso_heap_config;
selectedAllocateCommonPrimitive = iso_try_allocate_common_primitive;
selectedAllocate = iso_try_allocate;
selectedAllocateArray = iso_try_allocate_array_by_count;
selectedDeallocate = iso_deallocate;
selectedCommonPrimitiveHeap = &iso_common_primitive_heap;
selectedHeapRefGetHeap = iso_heap_ref_get_heap;
selectedHeapRefCreateForSize = createIsoHeapRefForSize;
});
ADD_GROUP(addTheTests(10, true));
{
DisableBitfit disableBitfit;
ADD_GROUP(addTheTests(1, false));
}
{
ForceExclusives forceExclusives;
ADD_GROUP(addTheTests(1, false));
}
{
ForceExclusives forceExclusives;
DisableBitfit disableBitfit;
ADD_GROUP(addTheTests(1, true)); // This tests enumeration with exclusives on for sure.
}
{
ForceTLAs forceTLAs;
ForcePartials forcePartials;
ADD_GROUP(addTheTests(1, true)); // This tests enumeration with partials and TLAs on for sure.
}
{
ForceBitfit forceBitfit;
ADD_GROUP(addTheTests(1, true)); // This tests enumeration with bitfit on for sure.
}
{
ForceBaselines forceBaselines;
DisableBitfit disableBitfit;
ADD_GROUP(addTheTests(1, true)); // This tests enumeration with baselines and segheaps on for sure.
}
{
ForceTLAs forceTLAs;
ADD_GROUP(addTheTests(1, false));
}
{
ForceBaselines forceBaselines;
ForcePartials forcePartials;
ADD_GROUP(addTheTests(1, false));
}
{
TestScope frequentScavenging(
"frequent-scavenging",
[] () {
pas_scavenger_period_in_milliseconds = 1.;
pas_scavenger_max_epoch_delta = -1ll * 1000ll * 1000ll;
});
addSpotTests(false);
}
}
#endif // PAS_ENABLE_ISO
}
void addAllTests()
{
addIsoTests();
#if PAS_ENABLE_ISO_TEST
{
TestScope iso(
"iso_test",
[] () {
selectedHeapConfig = &iso_test_heap_config;
selectedAllocateCommonPrimitive = iso_test_allocate_common_primitive;
selectedAllocate = iso_test_allocate;
selectedAllocateArray = iso_test_allocate_array_by_count;
selectedDeallocate = iso_test_deallocate;
selectedCommonPrimitiveHeap = &iso_test_common_primitive_heap;
selectedHeapRefGetHeap = iso_test_heap_ref_get_heap;
selectedHeapRefCreateForSize = createIsoHeapRefForSize;
});
addSpotTests(false);
}
#endif // PAS_ENABLE_ISO_TEST
#if PAS_ENABLE_MINALIGN32
{
TestScope iso(
"minalign32",
[] () {
selectedHeapConfig = &minalign32_heap_config;
selectedAllocateCommonPrimitive = minalign32_allocate_common_primitive;
selectedAllocate = minalign32_allocate;
selectedAllocateArray = minalign32_allocate_array_by_count;
selectedDeallocate = minalign32_deallocate;
selectedCommonPrimitiveHeap = &minalign32_common_primitive_heap;
selectedHeapRefGetHeap = minalign32_heap_ref_get_heap;
selectedHeapRefCreateForSize = createIsoHeapRefForSize;
});
addSpotTests(false);
}
#endif // PAS_ENABLE_MINALIGN32
#if PAS_ENABLE_PAGESIZE64K
{
TestScope iso(
"pagesize64k",
[] () {
selectedHeapConfig = &pagesize64k_heap_config;
selectedAllocateCommonPrimitive = pagesize64k_allocate_common_primitive;
selectedAllocate = pagesize64k_allocate;
selectedAllocateArray = pagesize64k_allocate_array_by_count;
selectedDeallocate = pagesize64k_deallocate;
selectedCommonPrimitiveHeap = &pagesize64k_common_primitive_heap;
selectedHeapRefGetHeap = pagesize64k_heap_ref_get_heap;
selectedHeapRefCreateForSize = createIsoHeapRefForSize;
});
addSpotTests(true);
}
#endif // PAS_ENABLE_PAGESIZE64K
#if PAS_ENABLE_BMALLOC
{
TestScope iso(
"bmalloc",
[] () {
selectedHeapConfig = &bmalloc_heap_config;
selectedAllocateCommonPrimitive = bmalloc_allocate;
selectedAllocate = bmalloc_iso_allocate;
selectedAllocateArray = bmalloc_iso_allocate_array_by_count_with_alignment;
selectedDeallocate = bmalloc_deallocate;
selectedCommonPrimitiveHeap = &bmalloc_common_primitive_heap;
selectedHeapRefGetHeap = bmalloc_heap_ref_get_heap;
selectedHeapRefCreateForSize = createBmallocHeapRefForSize;
});
addSpotTests(true);
}
{
TestScope iso(
"bmalloc-gigacage",
[] () {
static const bmalloc_type gigacageType = BMALLOC_TYPE_INITIALIZER(1, 1, "Gigacage");
gigacageHeapRef = BMALLOC_AUXILIARY_HEAP_REF_INITIALIZER(&gigacageType);
size_t reservationSize = 1000000000;
void* reservation = malloc(reservationSize);
PAS_ASSERT(reservation);
pas_heap* heap = bmalloc_force_auxiliary_heap_into_reserved_memory(
&gigacageHeapRef,
reinterpret_cast<uintptr_t>(reservation),
reinterpret_cast<uintptr_t>(reservation) + reservationSize);
selectedHeapConfig = &bmalloc_heap_config;
selectedAllocateCommonPrimitive = gigacageAllocate;
selectedAllocate = nullptr;
selectedAllocateArray = nullptr;
selectedDeallocate = bmalloc_deallocate;
selectedCommonPrimitiveHeap = heap;
selectedHeapRefGetHeap = nullptr;
selectedHeapRefCreateForSize = nullptr;
});
addSpotTests(false);
{
TestScope frequentScavenging(
"frequent-scavenging",
[] () {
pas_scavenger_period_in_milliseconds = 1.;
pas_scavenger_max_epoch_delta = -1ll * 1000ll * 1000ll;
});
addSpotTests(false);
}
}
#endif // PAS_ENABLE_BMALLOC
#if PAS_ENABLE_HOTBIT
{
TestScope iso(
"hotbit",
[] () {
selectedHeapConfig = &hotbit_heap_config;
selectedAllocateCommonPrimitive = hotbit_try_allocate;
selectedAllocate = nullptr;
selectedAllocateArray = nullptr;
selectedDeallocate = hotbit_deallocate;
selectedCommonPrimitiveHeap = &hotbit_common_primitive_heap;
selectedHeapRefGetHeap = nullptr;
});
addSpotTests(false);
}
#endif // PAS_ENABLE_HOTBIT
#if PAS_ENABLE_JIT
{
TestScope iso(
"jit",
[] () {
selectedHeapConfig = &jit_heap_config;
selectedAllocateCommonPrimitive = jit_heap_try_allocate;
selectedAllocate = nullptr;
selectedAllocateArray = nullptr;
selectedDeallocate = jit_heap_deallocate;
selectedCommonPrimitiveHeap = &jit_common_primitive_heap;
selectedHeapRefGetHeap = nullptr;
selectedHeapRefCreateForSize = nullptr;
});
BootJITHeap bootJITHeap;
ADD_GROUP(addTheTests(1, true));
{
ForceBitfit forceBitfit;
ADD_GROUP(addTheTests(1, true));
}
}
{
TestScope iso(
"jit-with-shrink",
[] () {
selectedHeapConfig = &jit_heap_config;
selectedAllocateCommonPrimitive = jit_heap_try_allocate;
selectedAllocate = nullptr;
selectedAllocateArray = nullptr;
selectedShrink = jit_heap_shrink;
selectedDeallocate = jit_heap_deallocate;
selectedCommonPrimitiveHeap = &jit_common_primitive_heap;
selectedHeapRefGetHeap = nullptr;
selectedHeapRefCreateForSize = nullptr;
});
BootJITHeap bootJITHeap;
ADD_GROUP(addTheTests(1, false));
{
ForceBitfit forceBitfit;
ADD_GROUP(addTheTests(1, false));
}
}
#endif // PAS_ENABLE_HOTBIT
}
} // anonymous namespace
#endif // PAS_ENABLE_ISO
void addIsoHeapChaosTests()
{
{
EnablePageBalancing enablePageBalancing;
addIsoTests();
}
{
DisablePageBalancing disablePageBalancing;
addAllTests();
}
}