[bmalloc] IsoHeap's initial setup should be small
https://bugs.webkit.org/show_bug.cgi?id=206214
Reviewed by Michael Saboff.
Source/bmalloc:
Keep IsoHeap related data structures small by using Packed technique. We start using IsoHeap for many classes,
then it is important that we keep metadata for IsoHeap small because these data persistently exists.
1. We pass IsoHeapImpl<> as a parameter instead of holding it unnecessarily.
2. We make some of pointers Packed so that we can keep sizeof(IsoHeapImpl<Config>) small.
3. One of the drawback of PackedPtr is that loading and storing are not atomic. And we pass `const std::lock_guard<Mutex>&`
to functions if functions need to be called with lock so that we ensure that PackedPtr are accessed only when lock is
held correctly.
* CMakeLists.txt:
* bmalloc.xcodeproj/project.pbxproj:
* bmalloc/Algorithm.h:
(bmalloc::ctzConstexpr):
(bmalloc::getLSBSetNonZeroConstexpr):
* bmalloc/BPlatform.h:
* bmalloc/DebugHeap.cpp:
(bmalloc::DebugHeap::DebugHeap):
* bmalloc/DebugHeap.h:
* bmalloc/DeferredTrigger.h:
* bmalloc/DeferredTriggerInlines.h:
(bmalloc::DeferredTrigger<trigger>::didBecome):
(bmalloc::DeferredTrigger<trigger>::handleDeferral):
* bmalloc/Environment.cpp:
(bmalloc::Environment::Environment):
* bmalloc/Environment.h:
* bmalloc/Gigacage.cpp:
(bmalloc::PrimitiveDisableCallbacks::PrimitiveDisableCallbacks):
* bmalloc/Heap.cpp:
(bmalloc::Heap::freeableMemory):
(bmalloc::Heap::markAllLargeAsEligibile):
(bmalloc::Heap::decommitLargeRange):
(bmalloc::Heap::scavenge):
(bmalloc::Heap::scavengeToHighWatermark):
* bmalloc/Heap.h:
* bmalloc/HeapConstants.cpp:
(bmalloc::HeapConstants::HeapConstants):
* bmalloc/HeapConstants.h:
* bmalloc/IsoAllocator.h:
* bmalloc/IsoAllocatorInlines.h:
(bmalloc::IsoAllocator<Config>::IsoAllocator):
(bmalloc::IsoAllocator<Config>::allocate):
(bmalloc::IsoAllocator<Config>::allocateSlow):
(bmalloc::IsoAllocator<Config>::scavenge):
* bmalloc/IsoDeallocatorInlines.h:
(bmalloc::IsoDeallocator<Config>::scavenge):
* bmalloc/IsoDirectory.h:
* bmalloc/IsoDirectoryInlines.h:
(bmalloc::passedNumPages>::IsoDirectory):
(bmalloc::passedNumPages>::takeFirstEligible):
(bmalloc::passedNumPages>::didBecome):
(bmalloc::passedNumPages>::didDecommit):
(bmalloc::passedNumPages>::scavengePage):
(bmalloc::passedNumPages>::scavenge):
(bmalloc::passedNumPages>::scavengeToHighWatermark):
(bmalloc::passedNumPages>::forEachCommittedPage):
* bmalloc/IsoHeapImpl.cpp:
(bmalloc::IsoHeapImplBase::IsoHeapImplBase):
* bmalloc/IsoHeapImpl.h:
* bmalloc/IsoHeapImplInlines.h:
(bmalloc::IsoHeapImpl<Config>::IsoHeapImpl):
(bmalloc::IsoHeapImpl<Config>::takeFirstEligible):
(bmalloc::IsoHeapImpl<Config>::didBecomeEligibleOrDecommited):
(bmalloc::IsoHeapImpl<Config>::scavenge):
(bmalloc::IsoHeapImpl<Config>::scavengeToHighWatermark):
(bmalloc::IsoHeapImplBase::freeableMemory):
(bmalloc::IsoHeapImpl<Config>::numLiveObjects):
(bmalloc::IsoHeapImpl<Config>::numCommittedPages):
(bmalloc::IsoHeapImpl<Config>::forEachDirectory):
(bmalloc::IsoHeapImpl<Config>::forEachCommittedPage):
(bmalloc::IsoHeapImpl<Config>::forEachLiveObject):
(bmalloc::IsoHeapImplBase::footprint):
(bmalloc::IsoHeapImplBase::didCommit):
(bmalloc::IsoHeapImplBase::didDecommit):
(bmalloc::IsoHeapImplBase::isNowFreeable):
(bmalloc::IsoHeapImplBase::isNoLongerFreeable):
(bmalloc::IsoHeapImpl<Config>::allocateFromShared):
(bmalloc::IsoHeapImpl<Config>::freeableMemory): Deleted.
(bmalloc::IsoHeapImpl<Config>::footprint): Deleted.
(bmalloc::IsoHeapImpl<Config>::didCommit): Deleted.
(bmalloc::IsoHeapImpl<Config>::didDecommit): Deleted.
(bmalloc::IsoHeapImpl<Config>::isNowFreeable): Deleted.
(bmalloc::IsoHeapImpl<Config>::isNoLongerFreeable): Deleted.
* bmalloc/IsoPage.h:
(bmalloc::IsoPageBase::IsoPageBase):
* bmalloc/IsoPageInlines.h:
(bmalloc::IsoPage<Config>::IsoPage):
(bmalloc::IsoPage<Config>::free):
(bmalloc::IsoPage<Config>::startAllocating):
(bmalloc::IsoPage<Config>::stopAllocating):
(bmalloc::IsoPage<Config>::forEachLiveObject):
* bmalloc/IsoSharedHeap.h:
(bmalloc::IsoSharedHeap::IsoSharedHeap):
* bmalloc/IsoSharedHeapInlines.h:
(bmalloc::IsoSharedHeap::allocateNew):
(bmalloc::IsoSharedHeap::allocateSlow):
* bmalloc/IsoSharedPage.h:
* bmalloc/IsoSharedPageInlines.h:
(bmalloc::IsoSharedPage::free):
(bmalloc::IsoSharedPage::startAllocating):
(bmalloc::IsoSharedPage::stopAllocating):
* bmalloc/IsoTLS.h:
* bmalloc/IsoTLSAllocatorEntry.h:
* bmalloc/IsoTLSAllocatorEntryInlines.h:
(bmalloc::IsoTLSAllocatorEntry<Config>::scavenge):
* bmalloc/IsoTLSDeallocatorEntry.h:
* bmalloc/IsoTLSDeallocatorEntryInlines.h:
(bmalloc::IsoTLSDeallocatorEntry<Config>::scavenge):
* bmalloc/IsoTLSEntry.cpp:
(bmalloc::IsoTLSEntry::IsoTLSEntry):
* bmalloc/IsoTLSEntry.h:
* bmalloc/IsoTLSEntryInlines.h:
(bmalloc::DefaultIsoTLSEntry<EntryType>::DefaultIsoTLSEntry):
(bmalloc::DefaultIsoTLSEntry<EntryType>::~DefaultIsoTLSEntry): Deleted.
(bmalloc::DefaultIsoTLSEntry<EntryType>::scavenge): Deleted.
* bmalloc/IsoTLSInlines.h:
(bmalloc::IsoTLS::scavenge):
(bmalloc::IsoTLS::allocateImpl):
(bmalloc::IsoTLS::allocateFast):
(bmalloc::IsoTLS::allocateSlow):
* bmalloc/IsoTLSLayout.cpp:
(bmalloc::IsoTLSLayout::add):
* bmalloc/Packed.h: Added.
(bmalloc::Packed::Packed):
(bmalloc::Packed::get const):
(bmalloc::Packed::set):
(bmalloc::Packed::operator=):
(bmalloc::Packed::exchange):
(bmalloc::Packed::swap):
(bmalloc::alignof):
(bmalloc::PackedPtrTraits::exchange):
(bmalloc::PackedPtrTraits::swap):
(bmalloc::PackedPtrTraits::unwrap):
* bmalloc/Scavenger.cpp:
(bmalloc::Scavenger::Scavenger):
* bmalloc/Scavenger.h:
* bmalloc/VMHeap.cpp:
(bmalloc::VMHeap::VMHeap):
* bmalloc/VMHeap.h:
* bmalloc/Zone.cpp:
(bmalloc::Zone::Zone):
* bmalloc/Zone.h:
Tools:
* TestWebKitAPI/Tests/WTF/bmalloc/IsoHeap.cpp:
(assertHasObjects):
(assertHasOnlyObjects):
(assertClean):
(TEST):
git-svn-id: http://svn.webkit.org/repository/webkit/trunk@254708 268f45cc-cd09-0410-ab3c-d52691b4dbfc
diff --git a/Source/bmalloc/CMakeLists.txt b/Source/bmalloc/CMakeLists.txt
index 30dbc34..6c6b7a5 100644
--- a/Source/bmalloc/CMakeLists.txt
+++ b/Source/bmalloc/CMakeLists.txt
@@ -112,6 +112,7 @@
bmalloc/Mutex.h
bmalloc/Object.h
bmalloc/ObjectType.h
+ bmalloc/Packed.h
bmalloc/PerHeapKind.h
bmalloc/PerProcess.h
bmalloc/PerThread.h
diff --git a/Source/bmalloc/ChangeLog b/Source/bmalloc/ChangeLog
index 7d8a167..229febe 100644
--- a/Source/bmalloc/ChangeLog
+++ b/Source/bmalloc/ChangeLog
@@ -1,3 +1,152 @@
+2020-01-16 Yusuke Suzuki <ysuzuki@apple.com>
+
+ [bmalloc] IsoHeap's initial setup should be small
+ https://bugs.webkit.org/show_bug.cgi?id=206214
+
+ Reviewed by Michael Saboff.
+
+ Keep IsoHeap related data structures small by using Packed technique. We start using IsoHeap for many classes,
+ then it is important that we keep metadata for IsoHeap small because these data persistently exists.
+
+ 1. We pass IsoHeapImpl<> as a parameter instead of holding it unnecessarily.
+ 2. We make some of pointers Packed so that we can keep sizeof(IsoHeapImpl<Config>) small.
+ 3. One of the drawback of PackedPtr is that loading and storing are not atomic. And we pass `const std::lock_guard<Mutex>&`
+ to functions if functions need to be called with lock so that we ensure that PackedPtr are accessed only when lock is
+ held correctly.
+
+ * CMakeLists.txt:
+ * bmalloc.xcodeproj/project.pbxproj:
+ * bmalloc/Algorithm.h:
+ (bmalloc::ctzConstexpr):
+ (bmalloc::getLSBSetNonZeroConstexpr):
+ * bmalloc/BPlatform.h:
+ * bmalloc/DebugHeap.cpp:
+ (bmalloc::DebugHeap::DebugHeap):
+ * bmalloc/DebugHeap.h:
+ * bmalloc/DeferredTrigger.h:
+ * bmalloc/DeferredTriggerInlines.h:
+ (bmalloc::DeferredTrigger<trigger>::didBecome):
+ (bmalloc::DeferredTrigger<trigger>::handleDeferral):
+ * bmalloc/Environment.cpp:
+ (bmalloc::Environment::Environment):
+ * bmalloc/Environment.h:
+ * bmalloc/Gigacage.cpp:
+ (bmalloc::PrimitiveDisableCallbacks::PrimitiveDisableCallbacks):
+ * bmalloc/Heap.cpp:
+ (bmalloc::Heap::freeableMemory):
+ (bmalloc::Heap::markAllLargeAsEligibile):
+ (bmalloc::Heap::decommitLargeRange):
+ (bmalloc::Heap::scavenge):
+ (bmalloc::Heap::scavengeToHighWatermark):
+ * bmalloc/Heap.h:
+ * bmalloc/HeapConstants.cpp:
+ (bmalloc::HeapConstants::HeapConstants):
+ * bmalloc/HeapConstants.h:
+ * bmalloc/IsoAllocator.h:
+ * bmalloc/IsoAllocatorInlines.h:
+ (bmalloc::IsoAllocator<Config>::IsoAllocator):
+ (bmalloc::IsoAllocator<Config>::allocate):
+ (bmalloc::IsoAllocator<Config>::allocateSlow):
+ (bmalloc::IsoAllocator<Config>::scavenge):
+ * bmalloc/IsoDeallocatorInlines.h:
+ (bmalloc::IsoDeallocator<Config>::scavenge):
+ * bmalloc/IsoDirectory.h:
+ * bmalloc/IsoDirectoryInlines.h:
+ (bmalloc::passedNumPages>::IsoDirectory):
+ (bmalloc::passedNumPages>::takeFirstEligible):
+ (bmalloc::passedNumPages>::didBecome):
+ (bmalloc::passedNumPages>::didDecommit):
+ (bmalloc::passedNumPages>::scavengePage):
+ (bmalloc::passedNumPages>::scavenge):
+ (bmalloc::passedNumPages>::scavengeToHighWatermark):
+ (bmalloc::passedNumPages>::forEachCommittedPage):
+ * bmalloc/IsoHeapImpl.cpp:
+ (bmalloc::IsoHeapImplBase::IsoHeapImplBase):
+ * bmalloc/IsoHeapImpl.h:
+ * bmalloc/IsoHeapImplInlines.h:
+ (bmalloc::IsoHeapImpl<Config>::IsoHeapImpl):
+ (bmalloc::IsoHeapImpl<Config>::takeFirstEligible):
+ (bmalloc::IsoHeapImpl<Config>::didBecomeEligibleOrDecommited):
+ (bmalloc::IsoHeapImpl<Config>::scavenge):
+ (bmalloc::IsoHeapImpl<Config>::scavengeToHighWatermark):
+ (bmalloc::IsoHeapImplBase::freeableMemory):
+ (bmalloc::IsoHeapImpl<Config>::numLiveObjects):
+ (bmalloc::IsoHeapImpl<Config>::numCommittedPages):
+ (bmalloc::IsoHeapImpl<Config>::forEachDirectory):
+ (bmalloc::IsoHeapImpl<Config>::forEachCommittedPage):
+ (bmalloc::IsoHeapImpl<Config>::forEachLiveObject):
+ (bmalloc::IsoHeapImplBase::footprint):
+ (bmalloc::IsoHeapImplBase::didCommit):
+ (bmalloc::IsoHeapImplBase::didDecommit):
+ (bmalloc::IsoHeapImplBase::isNowFreeable):
+ (bmalloc::IsoHeapImplBase::isNoLongerFreeable):
+ (bmalloc::IsoHeapImpl<Config>::allocateFromShared):
+ (bmalloc::IsoHeapImpl<Config>::freeableMemory): Deleted.
+ (bmalloc::IsoHeapImpl<Config>::footprint): Deleted.
+ (bmalloc::IsoHeapImpl<Config>::didCommit): Deleted.
+ (bmalloc::IsoHeapImpl<Config>::didDecommit): Deleted.
+ (bmalloc::IsoHeapImpl<Config>::isNowFreeable): Deleted.
+ (bmalloc::IsoHeapImpl<Config>::isNoLongerFreeable): Deleted.
+ * bmalloc/IsoPage.h:
+ (bmalloc::IsoPageBase::IsoPageBase):
+ * bmalloc/IsoPageInlines.h:
+ (bmalloc::IsoPage<Config>::IsoPage):
+ (bmalloc::IsoPage<Config>::free):
+ (bmalloc::IsoPage<Config>::startAllocating):
+ (bmalloc::IsoPage<Config>::stopAllocating):
+ (bmalloc::IsoPage<Config>::forEachLiveObject):
+ * bmalloc/IsoSharedHeap.h:
+ (bmalloc::IsoSharedHeap::IsoSharedHeap):
+ * bmalloc/IsoSharedHeapInlines.h:
+ (bmalloc::IsoSharedHeap::allocateNew):
+ (bmalloc::IsoSharedHeap::allocateSlow):
+ * bmalloc/IsoSharedPage.h:
+ * bmalloc/IsoSharedPageInlines.h:
+ (bmalloc::IsoSharedPage::free):
+ (bmalloc::IsoSharedPage::startAllocating):
+ (bmalloc::IsoSharedPage::stopAllocating):
+ * bmalloc/IsoTLS.h:
+ * bmalloc/IsoTLSAllocatorEntry.h:
+ * bmalloc/IsoTLSAllocatorEntryInlines.h:
+ (bmalloc::IsoTLSAllocatorEntry<Config>::scavenge):
+ * bmalloc/IsoTLSDeallocatorEntry.h:
+ * bmalloc/IsoTLSDeallocatorEntryInlines.h:
+ (bmalloc::IsoTLSDeallocatorEntry<Config>::scavenge):
+ * bmalloc/IsoTLSEntry.cpp:
+ (bmalloc::IsoTLSEntry::IsoTLSEntry):
+ * bmalloc/IsoTLSEntry.h:
+ * bmalloc/IsoTLSEntryInlines.h:
+ (bmalloc::DefaultIsoTLSEntry<EntryType>::DefaultIsoTLSEntry):
+ (bmalloc::DefaultIsoTLSEntry<EntryType>::~DefaultIsoTLSEntry): Deleted.
+ (bmalloc::DefaultIsoTLSEntry<EntryType>::scavenge): Deleted.
+ * bmalloc/IsoTLSInlines.h:
+ (bmalloc::IsoTLS::scavenge):
+ (bmalloc::IsoTLS::allocateImpl):
+ (bmalloc::IsoTLS::allocateFast):
+ (bmalloc::IsoTLS::allocateSlow):
+ * bmalloc/IsoTLSLayout.cpp:
+ (bmalloc::IsoTLSLayout::add):
+ * bmalloc/Packed.h: Added.
+ (bmalloc::Packed::Packed):
+ (bmalloc::Packed::get const):
+ (bmalloc::Packed::set):
+ (bmalloc::Packed::operator=):
+ (bmalloc::Packed::exchange):
+ (bmalloc::Packed::swap):
+ (bmalloc::alignof):
+ (bmalloc::PackedPtrTraits::exchange):
+ (bmalloc::PackedPtrTraits::swap):
+ (bmalloc::PackedPtrTraits::unwrap):
+ * bmalloc/Scavenger.cpp:
+ (bmalloc::Scavenger::Scavenger):
+ * bmalloc/Scavenger.h:
+ * bmalloc/VMHeap.cpp:
+ (bmalloc::VMHeap::VMHeap):
+ * bmalloc/VMHeap.h:
+ * bmalloc/Zone.cpp:
+ (bmalloc::Zone::Zone):
+ * bmalloc/Zone.h:
+
2020-01-14 Basuke Suzuki <basuke.suzuki@sony.com>
[bmalloc] Calculate LineMetadata for specific VM page size in compile time
diff --git a/Source/bmalloc/bmalloc.xcodeproj/project.pbxproj b/Source/bmalloc/bmalloc.xcodeproj/project.pbxproj
index b173a97..3b85e77 100644
--- a/Source/bmalloc/bmalloc.xcodeproj/project.pbxproj
+++ b/Source/bmalloc/bmalloc.xcodeproj/project.pbxproj
@@ -140,6 +140,7 @@
AD14AD2A202529C700890E3B /* ProcessCheck.mm in Sources */ = {isa = PBXBuildFile; fileRef = AD14AD28202529B000890E3B /* ProcessCheck.mm */; };
DE8B13B321CC5D9F00A63FCD /* BVMTags.h in Headers */ = {isa = PBXBuildFile; fileRef = DE8B13B221CC5D9F00A63FCD /* BVMTags.h */; settings = {ATTRIBUTES = (Private, ); }; };
E31E74802238CA5C005D084A /* StaticPerProcess.h in Headers */ = {isa = PBXBuildFile; fileRef = E31E747F2238CA5B005D084A /* StaticPerProcess.h */; settings = {ATTRIBUTES = (Private, ); }; };
+ E328D84D23CEB38900545B18 /* Packed.h in Headers */ = {isa = PBXBuildFile; fileRef = E328D84C23CEB38900545B18 /* Packed.h */; settings = {ATTRIBUTES = (Private, ); }; };
E3A413C9226061140037F470 /* IsoSharedPageInlines.h in Headers */ = {isa = PBXBuildFile; fileRef = E3A413C8226061140037F470 /* IsoSharedPageInlines.h */; settings = {ATTRIBUTES = (Private, ); }; };
E3F24402225D2C0100A0E0C3 /* IsoSharedPage.h in Headers */ = {isa = PBXBuildFile; fileRef = E3F24401225D2C0100A0E0C3 /* IsoSharedPage.h */; settings = {ATTRIBUTES = (Private, ); }; };
E3F24404225D2C7600A0E0C3 /* IsoSharedPage.cpp in Sources */ = {isa = PBXBuildFile; fileRef = E3F24403225D2C7600A0E0C3 /* IsoSharedPage.cpp */; };
@@ -295,6 +296,7 @@
AD14AD28202529B000890E3B /* ProcessCheck.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; name = ProcessCheck.mm; path = bmalloc/ProcessCheck.mm; sourceTree = "<group>"; };
DE8B13B221CC5D9F00A63FCD /* BVMTags.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = BVMTags.h; path = bmalloc/BVMTags.h; sourceTree = "<group>"; };
E31E747F2238CA5B005D084A /* StaticPerProcess.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = StaticPerProcess.h; path = bmalloc/StaticPerProcess.h; sourceTree = "<group>"; };
+ E328D84C23CEB38900545B18 /* Packed.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = Packed.h; path = bmalloc/Packed.h; sourceTree = "<group>"; };
E3A413C8226061140037F470 /* IsoSharedPageInlines.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = IsoSharedPageInlines.h; path = bmalloc/IsoSharedPageInlines.h; sourceTree = "<group>"; };
E3F24401225D2C0100A0E0C3 /* IsoSharedPage.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = IsoSharedPage.h; path = bmalloc/IsoSharedPage.h; sourceTree = "<group>"; };
E3F24403225D2C7600A0E0C3 /* IsoSharedPage.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = IsoSharedPage.cpp; path = bmalloc/IsoSharedPage.cpp; sourceTree = "<group>"; };
@@ -532,6 +534,7 @@
14C8992A1CC485E70027A057 /* Map.h */,
143CB81A19022BC900B16A45 /* Mutex.cpp */,
143CB81B19022BC900B16A45 /* Mutex.h */,
+ E328D84C23CEB38900545B18 /* Packed.h */,
0F5BF1481F22A8D80029D91D /* PerHeapKind.h */,
0F26A7A42054830D0090A141 /* PerProcess.cpp */,
14446A0717A61FA400F9EA1D /* PerProcess.h */,
@@ -648,6 +651,7 @@
143CB81D19022BC900B16A45 /* Mutex.h in Headers */,
144BE11F1CA346520099C8C0 /* Object.h in Headers */,
14DD789318F48D0F00950702 /* ObjectType.h in Headers */,
+ E328D84D23CEB38900545B18 /* Packed.h in Headers */,
0F5BF1491F22A8D80029D91D /* PerHeapKind.h in Headers */,
14DD78CB18F48D7500950702 /* PerProcess.h in Headers */,
14DD78CC18F48D7500950702 /* PerThread.h in Headers */,
diff --git a/Source/bmalloc/bmalloc/Algorithm.h b/Source/bmalloc/bmalloc/Algorithm.h
index 90a255a..d22f3ce 100644
--- a/Source/bmalloc/bmalloc/Algorithm.h
+++ b/Source/bmalloc/bmalloc/Algorithm.h
@@ -28,6 +28,7 @@
#include "BAssert.h"
#include <algorithm>
+#include <climits>
#include <cstdint>
#include <cstddef>
#include <limits>
@@ -193,6 +194,31 @@
return false;
}
+template <typename T>
+constexpr unsigned ctzConstexpr(T value)
+{
+ constexpr unsigned bitSize = sizeof(T) * CHAR_BIT;
+
+ using UT = typename std::make_unsigned<T>::type;
+ UT uValue = value;
+
+ unsigned zeroCount = 0;
+ for (unsigned i = 0; i < bitSize; i++) {
+ if (uValue & 1)
+ break;
+
+ zeroCount++;
+ uValue >>= 1;
+ }
+ return zeroCount;
+}
+
+template<typename T>
+constexpr unsigned getLSBSetNonZeroConstexpr(T t)
+{
+ return ctzConstexpr(t);
+}
+
} // namespace bmalloc
#endif // Algorithm_h
diff --git a/Source/bmalloc/bmalloc/BPlatform.h b/Source/bmalloc/bmalloc/BPlatform.h
index afe9e20..d784cf1 100644
--- a/Source/bmalloc/bmalloc/BPlatform.h
+++ b/Source/bmalloc/bmalloc/BPlatform.h
@@ -230,6 +230,45 @@
#endif /* ARM */
+
+#if BCOMPILER(GCC_COMPATIBLE)
+/* __LP64__ is not defined on 64bit Windows since it uses LLP64. Using __SIZEOF_POINTER__ is simpler. */
+#if __SIZEOF_POINTER__ == 8
+#define BCPU_ADDRESS64 1
+#elif __SIZEOF_POINTER__ == 4
+#define BCPU_ADDRESS32 1
+#else
+#error "Unsupported pointer width"
+#endif
+#else
+#error "Unsupported compiler for bmalloc"
+#endif
+
+#if BCOMPILER(GCC_COMPATIBLE)
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+#define BCPU_BIG_ENDIAN 1
+#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+#define BCPU_LITTLE_ENDIAN 1
+#elif __BYTE_ORDER__ == __ORDER_PDP_ENDIAN__
+#define BCPU_MIDDLE_ENDIAN 1
+#else
+#error "Unknown endian"
+#endif
+#else
+#error "Unsupported compiler for bmalloc"
+#endif
+
+#if BCPU(ADDRESS64)
+#if BOS(DARWIN) && BCPU(ARM64)
+#define BOS_EFFECTIVE_ADDRESS_WIDTH 36
+#else
+/* We strongly assume that effective address width is <= 48 in 64bit architectures (e.g. NaN boxing). */
+#define BOS_EFFECTIVE_ADDRESS_WIDTH 48
+#endif
+#else
+#define BOS_EFFECTIVE_ADDRESS_WIDTH 32
+#endif
+
#define BATTRIBUTE_PRINTF(formatStringArgument, extraArguments) __attribute__((__format__(printf, formatStringArgument, extraArguments)))
#if BPLATFORM(MAC) || BPLATFORM(IOS_FAMILY)
diff --git a/Source/bmalloc/bmalloc/DebugHeap.cpp b/Source/bmalloc/bmalloc/DebugHeap.cpp
index 7a667a7..2c9f5b4 100644
--- a/Source/bmalloc/bmalloc/DebugHeap.cpp
+++ b/Source/bmalloc/bmalloc/DebugHeap.cpp
@@ -40,7 +40,7 @@
#if BOS(DARWIN)
-DebugHeap::DebugHeap(std::lock_guard<Mutex>&)
+DebugHeap::DebugHeap(const std::lock_guard<Mutex>&)
: m_zone(malloc_create_zone(0, 0))
, m_pageSize(vmPageSize())
{
@@ -88,7 +88,7 @@
#else
-DebugHeap::DebugHeap(std::lock_guard<Mutex>&)
+DebugHeap::DebugHeap(const std::lock_guard<Mutex>&)
: m_pageSize(vmPageSize())
{
}
diff --git a/Source/bmalloc/bmalloc/DebugHeap.h b/Source/bmalloc/bmalloc/DebugHeap.h
index 958a220..9db8552 100644
--- a/Source/bmalloc/bmalloc/DebugHeap.h
+++ b/Source/bmalloc/bmalloc/DebugHeap.h
@@ -40,7 +40,7 @@
class DebugHeap : private StaticPerProcess<DebugHeap> {
public:
- DebugHeap(std::lock_guard<Mutex>&);
+ DebugHeap(const std::lock_guard<Mutex>&);
void* malloc(size_t, FailureAction);
void* memalign(size_t alignment, size_t, FailureAction);
diff --git a/Source/bmalloc/bmalloc/DeferredTrigger.h b/Source/bmalloc/bmalloc/DeferredTrigger.h
index b276a2c..29fb197 100644
--- a/Source/bmalloc/bmalloc/DeferredTrigger.h
+++ b/Source/bmalloc/bmalloc/DeferredTrigger.h
@@ -26,6 +26,8 @@
#pragma once
#include "IsoPageTrigger.h"
+#include "Mutex.h"
+#include <mutex>
namespace bmalloc {
@@ -37,10 +39,10 @@
DeferredTrigger() { }
template<typename Config>
- void didBecome(IsoPage<Config>&);
+ void didBecome(const std::lock_guard<Mutex>&, IsoPage<Config>&);
template<typename Config>
- void handleDeferral(IsoPage<Config>&);
+ void handleDeferral(const std::lock_guard<Mutex>&, IsoPage<Config>&);
private:
bool m_hasBeenDeferred { false };
diff --git a/Source/bmalloc/bmalloc/DeferredTriggerInlines.h b/Source/bmalloc/bmalloc/DeferredTriggerInlines.h
index 09cfc85..71ea726 100644
--- a/Source/bmalloc/bmalloc/DeferredTriggerInlines.h
+++ b/Source/bmalloc/bmalloc/DeferredTriggerInlines.h
@@ -32,22 +32,22 @@
template<IsoPageTrigger trigger>
template<typename Config>
-void DeferredTrigger<trigger>::didBecome(IsoPage<Config>& page)
+void DeferredTrigger<trigger>::didBecome(const std::lock_guard<Mutex>& locker, IsoPage<Config>& page)
{
if (page.isInUseForAllocation())
m_hasBeenDeferred = true;
else
- page.directory().didBecome(&page, trigger);
+ page.directory().didBecome(locker, &page, trigger);
}
template<IsoPageTrigger trigger>
template<typename Config>
-void DeferredTrigger<trigger>::handleDeferral(IsoPage<Config>& page)
+void DeferredTrigger<trigger>::handleDeferral(const std::lock_guard<Mutex>& locker, IsoPage<Config>& page)
{
RELEASE_BASSERT(!page.isInUseForAllocation());
if (m_hasBeenDeferred) {
- page.directory().didBecome(&page, trigger);
+ page.directory().didBecome(locker, &page, trigger);
m_hasBeenDeferred = false;
}
}
diff --git a/Source/bmalloc/bmalloc/Environment.cpp b/Source/bmalloc/bmalloc/Environment.cpp
index 2574b23..877a6c2 100644
--- a/Source/bmalloc/bmalloc/Environment.cpp
+++ b/Source/bmalloc/bmalloc/Environment.cpp
@@ -127,7 +127,7 @@
DEFINE_STATIC_PER_PROCESS_STORAGE(Environment);
-Environment::Environment(std::lock_guard<Mutex>&)
+Environment::Environment(const std::lock_guard<Mutex>&)
: m_isDebugHeapEnabled(computeIsDebugHeapEnabled())
{
}
diff --git a/Source/bmalloc/bmalloc/Environment.h b/Source/bmalloc/bmalloc/Environment.h
index a56256e..7b28de9 100644
--- a/Source/bmalloc/bmalloc/Environment.h
+++ b/Source/bmalloc/bmalloc/Environment.h
@@ -33,7 +33,7 @@
class Environment : public StaticPerProcess<Environment> {
public:
- BEXPORT Environment(std::lock_guard<Mutex>&);
+ BEXPORT Environment(const std::lock_guard<Mutex>&);
bool isDebugHeapEnabled() { return m_isDebugHeapEnabled; }
diff --git a/Source/bmalloc/bmalloc/Gigacage.cpp b/Source/bmalloc/bmalloc/Gigacage.cpp
index 3cca2d5..4201959f 100644
--- a/Source/bmalloc/bmalloc/Gigacage.cpp
+++ b/Source/bmalloc/bmalloc/Gigacage.cpp
@@ -61,7 +61,7 @@
namespace bmalloc {
struct PrimitiveDisableCallbacks : public StaticPerProcess<PrimitiveDisableCallbacks> {
- PrimitiveDisableCallbacks(std::lock_guard<Mutex>&) { }
+ PrimitiveDisableCallbacks(const std::lock_guard<Mutex>&) { }
Vector<Gigacage::Callback> callbacks;
};
diff --git a/Source/bmalloc/bmalloc/Heap.cpp b/Source/bmalloc/bmalloc/Heap.cpp
index 1189b87..3aae43a 100644
--- a/Source/bmalloc/bmalloc/Heap.cpp
+++ b/Source/bmalloc/bmalloc/Heap.cpp
@@ -81,7 +81,7 @@
return Gigacage::size(gigacageKind(m_kind));
}
-size_t Heap::freeableMemory(std::lock_guard<Mutex>&)
+size_t Heap::freeableMemory(const std::lock_guard<Mutex>&)
{
return m_freeableMemory;
}
@@ -91,14 +91,14 @@
return m_footprint;
}
-void Heap::markAllLargeAsEligibile(std::lock_guard<Mutex>&)
+void Heap::markAllLargeAsEligibile(const std::lock_guard<Mutex>&)
{
m_largeFree.markAllAsEligibile();
m_hasPendingDecommits = false;
m_condition.notify_all();
}
-void Heap::decommitLargeRange(std::lock_guard<Mutex>&, LargeRange& range, BulkDecommit& decommitter)
+void Heap::decommitLargeRange(const std::lock_guard<Mutex>&, LargeRange& range, BulkDecommit& decommitter)
{
m_footprint -= range.totalPhysicalSize();
m_freeableMemory -= range.totalPhysicalSize();
@@ -114,9 +114,9 @@
}
#if BUSE(PARTIAL_SCAVENGE)
-void Heap::scavenge(std::lock_guard<Mutex>& lock, BulkDecommit& decommitter)
+void Heap::scavenge(const std::lock_guard<Mutex>& lock, BulkDecommit& decommitter)
#else
-void Heap::scavenge(std::lock_guard<Mutex>& lock, BulkDecommit& decommitter, size_t& deferredDecommits)
+void Heap::scavenge(const std::lock_guard<Mutex>& lock, BulkDecommit& decommitter, size_t& deferredDecommits)
#endif
{
for (auto& list : m_freePages) {
@@ -169,7 +169,7 @@
}
#if BUSE(PARTIAL_SCAVENGE)
-void Heap::scavengeToHighWatermark(std::lock_guard<Mutex>& lock, BulkDecommit& decommitter)
+void Heap::scavengeToHighWatermark(const std::lock_guard<Mutex>& lock, BulkDecommit& decommitter)
{
void* newHighWaterMark = nullptr;
for (LargeRange& range : m_largeFree) {
diff --git a/Source/bmalloc/bmalloc/Heap.h b/Source/bmalloc/bmalloc/Heap.h
index db32453..59c75f2 100644
--- a/Source/bmalloc/bmalloc/Heap.h
+++ b/Source/bmalloc/bmalloc/Heap.h
@@ -76,14 +76,14 @@
void shrinkLarge(std::unique_lock<Mutex>&, const Range&, size_t);
#if BUSE(PARTIAL_SCAVENGE)
- void scavengeToHighWatermark(std::lock_guard<Mutex>&, BulkDecommit&);
- void scavenge(std::lock_guard<Mutex>&, BulkDecommit&);
+ void scavengeToHighWatermark(const std::lock_guard<Mutex>&, BulkDecommit&);
+ void scavenge(const std::lock_guard<Mutex>&, BulkDecommit&);
#else
- void scavenge(std::lock_guard<Mutex>&, BulkDecommit&, size_t& deferredDecommits);
+ void scavenge(const std::lock_guard<Mutex>&, BulkDecommit&, size_t& deferredDecommits);
#endif
- void scavenge(std::lock_guard<Mutex>&, BulkDecommit&, size_t& freed, size_t goal);
+ void scavenge(const std::lock_guard<Mutex>&, BulkDecommit&, size_t& freed, size_t goal);
- size_t freeableMemory(std::lock_guard<Mutex>&);
+ size_t freeableMemory(const std::lock_guard<Mutex>&);
size_t footprint();
void externalDecommit(void* ptr, size_t);
@@ -91,10 +91,10 @@
void externalCommit(void* ptr, size_t);
void externalCommit(std::unique_lock<Mutex>&, void* ptr, size_t);
- void markAllLargeAsEligibile(std::lock_guard<Mutex>&);
+ void markAllLargeAsEligibile(const std::lock_guard<Mutex>&);
private:
- void decommitLargeRange(std::lock_guard<Mutex>&, LargeRange&, BulkDecommit&);
+ void decommitLargeRange(const std::lock_guard<Mutex>&, LargeRange&, BulkDecommit&);
struct LargeObjectHash {
static unsigned hash(void* key)
diff --git a/Source/bmalloc/bmalloc/HeapConstants.cpp b/Source/bmalloc/bmalloc/HeapConstants.cpp
index fca9fcf..f384a54 100644
--- a/Source/bmalloc/bmalloc/HeapConstants.cpp
+++ b/Source/bmalloc/bmalloc/HeapConstants.cpp
@@ -30,7 +30,7 @@
DEFINE_STATIC_PER_PROCESS_STORAGE(HeapConstants);
-HeapConstants::HeapConstants(std::lock_guard<Mutex>&)
+HeapConstants::HeapConstants(const std::lock_guard<Mutex>&)
: m_vmPageSizePhysical { vmPageSizePhysical() }
{
RELEASE_BASSERT(m_vmPageSizePhysical >= smallPageSize);
diff --git a/Source/bmalloc/bmalloc/HeapConstants.h b/Source/bmalloc/bmalloc/HeapConstants.h
index 04e73d0..0ebc950 100644
--- a/Source/bmalloc/bmalloc/HeapConstants.h
+++ b/Source/bmalloc/bmalloc/HeapConstants.h
@@ -37,7 +37,7 @@
class HeapConstants : public StaticPerProcess<HeapConstants> {
public:
- HeapConstants(std::lock_guard<Mutex>&);
+ HeapConstants(const std::lock_guard<Mutex>&);
~HeapConstants() = delete;
inline size_t pageClass(size_t sizeClass) const { return m_pageClasses[sizeClass]; }
diff --git a/Source/bmalloc/bmalloc/IsoAllocator.h b/Source/bmalloc/bmalloc/IsoAllocator.h
index a0f7080..3ba08aa 100644
--- a/Source/bmalloc/bmalloc/IsoAllocator.h
+++ b/Source/bmalloc/bmalloc/IsoAllocator.h
@@ -40,13 +40,12 @@
IsoAllocator(IsoHeapImpl<Config>&);
~IsoAllocator();
- void* allocate(bool abortOnFailure);
- void scavenge();
+ void* allocate(IsoHeapImpl<Config>&, bool abortOnFailure);
+ void scavenge(IsoHeapImpl<Config>&);
private:
- void* allocateSlow(bool abortOnFailure);
+ void* allocateSlow(IsoHeapImpl<Config>&, bool abortOnFailure);
- IsoHeapImpl<Config>* m_heap { nullptr };
FreeList m_freeList;
IsoPage<Config>* m_currentPage { nullptr };
};
diff --git a/Source/bmalloc/bmalloc/IsoAllocatorInlines.h b/Source/bmalloc/bmalloc/IsoAllocatorInlines.h
index 5455ffd..d79f424 100644
--- a/Source/bmalloc/bmalloc/IsoAllocatorInlines.h
+++ b/Source/bmalloc/bmalloc/IsoAllocatorInlines.h
@@ -34,8 +34,7 @@
namespace bmalloc {
template<typename Config>
-IsoAllocator<Config>::IsoAllocator(IsoHeapImpl<Config>& heap)
- : m_heap(&heap)
+IsoAllocator<Config>::IsoAllocator(IsoHeapImpl<Config>&)
{
}
@@ -45,36 +44,36 @@
}
template<typename Config>
-void* IsoAllocator<Config>::allocate(bool abortOnFailure)
+void* IsoAllocator<Config>::allocate(IsoHeapImpl<Config>& heap, bool abortOnFailure)
{
static constexpr bool verbose = false;
void* result = m_freeList.allocate<Config>(
[&] () -> void* {
- return allocateSlow(abortOnFailure);
+ return allocateSlow(heap, abortOnFailure);
});
if (verbose)
- fprintf(stderr, "%p: allocated %p of size %u\n", m_heap, result, Config::objectSize);
+ fprintf(stderr, "%p: allocated %p of size %u\n", &heap, result, Config::objectSize);
return result;
}
template<typename Config>
-BNO_INLINE void* IsoAllocator<Config>::allocateSlow(bool abortOnFailure)
+BNO_INLINE void* IsoAllocator<Config>::allocateSlow(IsoHeapImpl<Config>& heap, bool abortOnFailure)
{
- std::lock_guard<Mutex> locker(m_heap->lock);
+ std::lock_guard<Mutex> locker(heap.lock);
- AllocationMode allocationMode = m_heap->updateAllocationMode();
+ AllocationMode allocationMode = heap.updateAllocationMode();
if (allocationMode == AllocationMode::Shared) {
if (m_currentPage) {
- m_currentPage->stopAllocating(m_freeList);
+ m_currentPage->stopAllocating(locker, m_freeList);
m_currentPage = nullptr;
m_freeList.clear();
}
- return m_heap->allocateFromShared(locker, abortOnFailure);
+ return heap.allocateFromShared(locker, abortOnFailure);
}
BASSERT(allocationMode == AllocationMode::Fast);
- EligibilityResult<Config> result = m_heap->takeFirstEligible();
+ EligibilityResult<Config> result = heap.takeFirstEligible(locker);
if (result.kind != EligibilityKind::Success) {
RELEASE_BASSERT(result.kind == EligibilityKind::OutOfMemory);
RELEASE_BASSERT(!abortOnFailure);
@@ -82,20 +81,20 @@
}
if (m_currentPage)
- m_currentPage->stopAllocating(m_freeList);
+ m_currentPage->stopAllocating(locker, m_freeList);
m_currentPage = result.page;
- m_freeList = m_currentPage->startAllocating();
+ m_freeList = m_currentPage->startAllocating(locker);
return m_freeList.allocate<Config>([] () { BCRASH(); return nullptr; });
}
template<typename Config>
-void IsoAllocator<Config>::scavenge()
+void IsoAllocator<Config>::scavenge(IsoHeapImpl<Config>& heap)
{
if (m_currentPage) {
- std::lock_guard<Mutex> locker(m_heap->lock);
- m_currentPage->stopAllocating(m_freeList);
+ std::lock_guard<Mutex> locker(heap.lock);
+ m_currentPage->stopAllocating(locker, m_freeList);
m_currentPage = nullptr;
m_freeList.clear();
}
diff --git a/Source/bmalloc/bmalloc/IsoDeallocatorInlines.h b/Source/bmalloc/bmalloc/IsoDeallocatorInlines.h
index 23f32a3..032fb77 100644
--- a/Source/bmalloc/bmalloc/IsoDeallocatorInlines.h
+++ b/Source/bmalloc/bmalloc/IsoDeallocatorInlines.h
@@ -76,7 +76,7 @@
std::lock_guard<Mutex> locker(*m_lock);
for (void* ptr : m_objectLog)
- IsoPage<Config>::pageFor(ptr)->free(ptr);
+ IsoPage<Config>::pageFor(ptr)->free(locker, ptr);
m_objectLog.clear();
}
diff --git a/Source/bmalloc/bmalloc/IsoDirectory.h b/Source/bmalloc/bmalloc/IsoDirectory.h
index bafeb3e..e72b2d1 100644
--- a/Source/bmalloc/bmalloc/IsoDirectory.h
+++ b/Source/bmalloc/bmalloc/IsoDirectory.h
@@ -28,6 +28,7 @@
#include "Bits.h"
#include "EligibilityResult.h"
#include "IsoPage.h"
+#include "Packed.h"
#include "Vector.h"
namespace bmalloc {
@@ -49,7 +50,7 @@
IsoHeapImpl<Config>& heap() { return m_heap; }
- virtual void didBecome(IsoPage<Config>*, IsoPageTrigger) = 0;
+ virtual void didBecome(const std::lock_guard<Mutex>&, IsoPage<Config>*, IsoPageTrigger) = 0;
protected:
IsoHeapImpl<Config>& m_heap;
@@ -64,9 +65,9 @@
// Find the first page that is eligible for allocation and return it. May return null if there is no
// such thing. May allocate a new page if we have an uncommitted page.
- EligibilityResult<Config> takeFirstEligible();
+ EligibilityResult<Config> takeFirstEligible(const std::lock_guard<Mutex>&);
- void didBecome(IsoPage<Config>*, IsoPageTrigger) override;
+ void didBecome(const std::lock_guard<Mutex>&, IsoPage<Config>*, IsoPageTrigger) override;
// This gets called from a bulk decommit function in the Scavenger, so no locks are held. This function
// needs to get the heap lock.
@@ -74,23 +75,23 @@
// Iterate over all empty and committed pages, and put them into the vector. This also records the
// pages as being decommitted. It's the caller's job to do the actual decommitting.
- void scavenge(Vector<DeferredDecommit>&);
+ void scavenge(const std::lock_guard<Mutex>&, Vector<DeferredDecommit>&);
#if BUSE(PARTIAL_SCAVENGE)
- void scavengeToHighWatermark(Vector<DeferredDecommit>&);
+ void scavengeToHighWatermark(const std::lock_guard<Mutex>&, Vector<DeferredDecommit>&);
#endif
template<typename Func>
- void forEachCommittedPage(const Func&);
+ void forEachCommittedPage(const std::lock_guard<Mutex>&, const Func&);
private:
- void scavengePage(size_t, Vector<DeferredDecommit>&);
+ void scavengePage(const std::lock_guard<Mutex>&, size_t, Vector<DeferredDecommit>&);
+ std::array<PackedAlignedPtr<IsoPage<Config>, IsoPage<Config>::pageSize>, numPages> m_pages { };
// NOTE: I suppose that this could be two bitvectors. But from working on the GC, I found that the
// number of bitvectors does not matter as much as whether or not they make intuitive sense.
Bits<numPages> m_eligible;
Bits<numPages> m_empty;
Bits<numPages> m_committed;
- std::array<IsoPage<Config>*, numPages> m_pages;
unsigned m_firstEligibleOrDecommitted { 0 };
#if BUSE(PARTIAL_SCAVENGE)
unsigned m_highWatermark { 0 };
diff --git a/Source/bmalloc/bmalloc/IsoDirectoryInlines.h b/Source/bmalloc/bmalloc/IsoDirectoryInlines.h
index 0b1a1ec..85542df 100644
--- a/Source/bmalloc/bmalloc/IsoDirectoryInlines.h
+++ b/Source/bmalloc/bmalloc/IsoDirectoryInlines.h
@@ -39,12 +39,10 @@
IsoDirectory<Config, passedNumPages>::IsoDirectory(IsoHeapImpl<Config>& heap)
: IsoDirectoryBase<Config>(heap)
{
- for (unsigned i = numPages; i--;)
- m_pages[i] = nullptr;
}
template<typename Config, unsigned passedNumPages>
-EligibilityResult<Config> IsoDirectory<Config, passedNumPages>::takeFirstEligible()
+EligibilityResult<Config> IsoDirectory<Config, passedNumPages>::takeFirstEligible(const std::lock_guard<Mutex>&)
{
unsigned pageIndex = (m_eligible | ~m_committed).findBit(m_firstEligibleOrDecommitted, true);
m_firstEligibleOrDecommitted = pageIndex;
@@ -59,7 +57,7 @@
Scavenger& scavenger = *Scavenger::get();
scavenger.didStartGrowing();
- IsoPage<Config>* page = m_pages[pageIndex];
+ IsoPage<Config>* page = m_pages[pageIndex].get();
if (!m_committed[pageIndex]) {
scavenger.scheduleIfUnderMemoryPressure(IsoPageBase::pageSize);
@@ -93,7 +91,7 @@
}
template<typename Config, unsigned passedNumPages>
-void IsoDirectory<Config, passedNumPages>::didBecome(IsoPage<Config>* page, IsoPageTrigger trigger)
+void IsoDirectory<Config, passedNumPages>::didBecome(const std::lock_guard<Mutex>& locker, IsoPage<Config>* page, IsoPageTrigger trigger)
{
static constexpr bool verbose = false;
unsigned pageIndex = page->index();
@@ -103,7 +101,7 @@
fprintf(stderr, "%p: %p did become eligible.\n", this, page);
m_eligible[pageIndex] = true;
m_firstEligibleOrDecommitted = std::min(m_firstEligibleOrDecommitted, pageIndex);
- this->m_heap.didBecomeEligibleOrDecommited(this);
+ this->m_heap.didBecomeEligibleOrDecommited(locker, this);
return;
case IsoPageTrigger::Empty:
if (verbose)
@@ -125,28 +123,28 @@
// syscall itself (which has to do many hard things).
std::lock_guard<Mutex> locker(this->m_heap.lock);
BASSERT(!!m_committed[index]);
- this->m_heap.isNoLongerFreeable(m_pages[index], IsoPageBase::pageSize);
+ this->m_heap.isNoLongerFreeable(m_pages[index].get(), IsoPageBase::pageSize);
m_committed[index] = false;
m_firstEligibleOrDecommitted = std::min(m_firstEligibleOrDecommitted, index);
- this->m_heap.didBecomeEligibleOrDecommited(this);
- this->m_heap.didDecommit(m_pages[index], IsoPageBase::pageSize);
+ this->m_heap.didBecomeEligibleOrDecommited(locker, this);
+ this->m_heap.didDecommit(m_pages[index].get(), IsoPageBase::pageSize);
}
template<typename Config, unsigned passedNumPages>
-void IsoDirectory<Config, passedNumPages>::scavengePage(size_t index, Vector<DeferredDecommit>& decommits)
+void IsoDirectory<Config, passedNumPages>::scavengePage(const std::lock_guard<Mutex>&, size_t index, Vector<DeferredDecommit>& decommits)
{
// Make sure that this page is now off limits.
m_empty[index] = false;
m_eligible[index] = false;
- decommits.push(DeferredDecommit(this, m_pages[index], index));
+ decommits.push(DeferredDecommit(this, m_pages[index].get(), index));
}
template<typename Config, unsigned passedNumPages>
-void IsoDirectory<Config, passedNumPages>::scavenge(Vector<DeferredDecommit>& decommits)
+void IsoDirectory<Config, passedNumPages>::scavenge(const std::lock_guard<Mutex>& locker, Vector<DeferredDecommit>& decommits)
{
(m_empty & m_committed).forEachSetBit(
[&] (size_t index) {
- scavengePage(index, decommits);
+ scavengePage(locker, index, decommits);
});
#if BUSE(PARTIAL_SCAVENGE)
m_highWatermark = 0;
@@ -155,12 +153,12 @@
#if BUSE(PARTIAL_SCAVENGE)
template<typename Config, unsigned passedNumPages>
-void IsoDirectory<Config, passedNumPages>::scavengeToHighWatermark(Vector<DeferredDecommit>& decommits)
+void IsoDirectory<Config, passedNumPages>::scavengeToHighWatermark(const std::lock_guard<Mutex>& locker, Vector<DeferredDecommit>& decommits)
{
(m_empty & m_committed).forEachSetBit(
[&] (size_t index) {
if (index > m_highWatermark)
- scavengePage(index, decommits);
+ scavengePage(locker, index, decommits);
});
m_highWatermark = 0;
}
@@ -168,11 +166,11 @@
template<typename Config, unsigned passedNumPages>
template<typename Func>
-void IsoDirectory<Config, passedNumPages>::forEachCommittedPage(const Func& func)
+void IsoDirectory<Config, passedNumPages>::forEachCommittedPage(const std::lock_guard<Mutex>&, const Func& func)
{
m_committed.forEachSetBit(
[&] (size_t index) {
- func(*m_pages[index]);
+ func(*(m_pages[index].get()));
});
}
diff --git a/Source/bmalloc/bmalloc/IsoHeapImpl.cpp b/Source/bmalloc/bmalloc/IsoHeapImpl.cpp
index 696d22e..dd089a1 100644
--- a/Source/bmalloc/bmalloc/IsoHeapImpl.cpp
+++ b/Source/bmalloc/bmalloc/IsoHeapImpl.cpp
@@ -31,7 +31,8 @@
namespace bmalloc {
-IsoHeapImplBase::IsoHeapImplBase()
+IsoHeapImplBase::IsoHeapImplBase(Mutex& lock)
+ : lock(lock)
{
}
diff --git a/Source/bmalloc/bmalloc/IsoHeapImpl.h b/Source/bmalloc/bmalloc/IsoHeapImpl.h
index 09511e2..0baa808 100644
--- a/Source/bmalloc/bmalloc/IsoHeapImpl.h
+++ b/Source/bmalloc/bmalloc/IsoHeapImpl.h
@@ -29,6 +29,7 @@
#include "IsoAllocator.h"
#include "IsoDirectoryPage.h"
#include "IsoTLSAllocatorEntry.h"
+#include "Packed.h"
#include "PhysicalPageMap.h"
namespace bmalloc {
@@ -51,27 +52,48 @@
#if BUSE(PARTIAL_SCAVENGE)
virtual void scavengeToHighWatermark(Vector<DeferredDecommit>&) = 0;
#endif
- virtual size_t freeableMemory() = 0;
- virtual size_t footprint() = 0;
void scavengeNow();
static void finishScavenging(Vector<DeferredDecommit>&);
+ void didCommit(void* ptr, size_t bytes);
+ void didDecommit(void* ptr, size_t bytes);
+
+ void isNowFreeable(void* ptr, size_t bytes);
+ void isNoLongerFreeable(void* ptr, size_t bytes);
+
+ size_t freeableMemory();
+ size_t footprint();
+
void addToAllIsoHeaps();
protected:
- IsoHeapImplBase();
+ IsoHeapImplBase(Mutex&);
friend class IsoSharedPage;
friend class AllIsoHeaps;
+public:
+ // It's almost always the caller's responsibility to grab the lock. This lock comes from the
+ // (*PerProcess<IsoTLSEntryHolder<IsoTLSDeallocatorEntry<Config>>>::get())->lock. That's pretty weird, and we don't
+ // try to disguise the fact that it's weird. We only do that because heaps in the same size class
+ // share the same deallocator log, so it makes sense for them to also share the same lock to
+ // amortize lock acquisition costs.
+ Mutex& lock;
+protected:
IsoHeapImplBase* m_next { nullptr };
std::chrono::steady_clock::time_point m_lastSlowPathTime;
- std::array<void*, maxAllocationFromShared> m_sharedCells { };
+ size_t m_footprint { 0 };
+ size_t m_freeableMemory { 0 };
+#if ENABLE_PHYSICAL_PAGE_MAP
+ PhysicalPageMap m_physicalPageMap;
+#endif
+ std::array<PackedAlignedPtr<uint8_t, bmalloc::alignment>, maxAllocationFromShared> m_sharedCells { };
+protected:
unsigned m_numberOfAllocationsFromSharedInOneCycle { 0 };
unsigned m_availableShared { maxAllocationFromSharedMask };
AllocationMode m_allocationMode { AllocationMode::Init };
-
+ bool m_isInlineDirectoryEligibleOrDecommitted { true };
static_assert(sizeof(m_availableShared) * 8 >= maxAllocationFromShared, "");
};
@@ -83,21 +105,17 @@
public:
IsoHeapImpl();
- EligibilityResult<Config> takeFirstEligible();
+ EligibilityResult<Config> takeFirstEligible(const std::lock_guard<Mutex>&);
// Callbacks from directory.
- void didBecomeEligibleOrDecommited(IsoDirectory<Config, numPagesInInlineDirectory>*);
- void didBecomeEligibleOrDecommited(IsoDirectory<Config, IsoDirectoryPage<Config>::numPages>*);
+ void didBecomeEligibleOrDecommited(const std::lock_guard<Mutex>&, IsoDirectory<Config, numPagesInInlineDirectory>*);
+ void didBecomeEligibleOrDecommited(const std::lock_guard<Mutex>&, IsoDirectory<Config, IsoDirectoryPage<Config>::numPages>*);
void scavenge(Vector<DeferredDecommit>&) override;
#if BUSE(PARTIAL_SCAVENGE)
void scavengeToHighWatermark(Vector<DeferredDecommit>&) override;
#endif
- size_t freeableMemory() override;
-
- size_t footprint() override;
-
unsigned allocatorOffset();
unsigned deallocatorOffset();
@@ -106,46 +124,25 @@
unsigned numCommittedPages();
template<typename Func>
- void forEachDirectory(const Func&);
+ void forEachDirectory(const std::lock_guard<Mutex>&, const Func&);
template<typename Func>
- void forEachCommittedPage(const Func&);
+ void forEachCommittedPage(const std::lock_guard<Mutex>&, const Func&);
// This is only accurate when all threads are scavenged. Otherwise it will overestimate.
template<typename Func>
- void forEachLiveObject(const Func&);
-
- void didCommit(void* ptr, size_t bytes);
- void didDecommit(void* ptr, size_t bytes);
-
- void isNowFreeable(void* ptr, size_t bytes);
- void isNoLongerFreeable(void* ptr, size_t bytes);
+ void forEachLiveObject(const std::lock_guard<Mutex>&, const Func&);
AllocationMode updateAllocationMode();
void* allocateFromShared(const std::lock_guard<Mutex>&, bool abortOnFailure);
-
- // It's almost always the caller's responsibility to grab the lock. This lock comes from the
- // (*PerProcess<IsoTLSEntryHolder<IsoTLSDeallocatorEntry<Config>>>::get())->lock. That's pretty weird, and we don't
- // try to disguise the fact that it's weird. We only do that because heaps in the same size class
- // share the same deallocator log, so it makes sense for them to also share the same lock to
- // amortize lock acquisition costs.
- Mutex& lock;
private:
+ PackedPtr<IsoDirectoryPage<Config>> m_headDirectory { nullptr };
+ PackedPtr<IsoDirectoryPage<Config>> m_tailDirectory { nullptr };
+ PackedPtr<IsoDirectoryPage<Config>> m_firstEligibleOrDecommitedDirectory { nullptr };
IsoDirectory<Config, numPagesInInlineDirectory> m_inlineDirectory;
- IsoDirectoryPage<Config>* m_headDirectory { nullptr };
- IsoDirectoryPage<Config>* m_tailDirectory { nullptr };
- size_t m_footprint { 0 };
- size_t m_freeableMemory { 0 };
-#if ENABLE_PHYSICAL_PAGE_MAP
- PhysicalPageMap m_physicalPageMap;
-#endif
unsigned m_nextDirectoryPageIndex { 1 }; // We start at 1 so that the high water mark being zero means we've only allocated in the inline directory since the last scavenge.
unsigned m_directoryHighWatermark { 0 };
-
- bool m_isInlineDirectoryEligibleOrDecommitted { true };
- IsoDirectoryPage<Config>* m_firstEligibleOrDecommitedDirectory { nullptr };
-
IsoTLSEntryHolder<IsoTLSAllocatorEntry<Config>> m_allocator;
};
diff --git a/Source/bmalloc/bmalloc/IsoHeapImplInlines.h b/Source/bmalloc/bmalloc/IsoHeapImplInlines.h
index 2a4fb28..94eb3f4 100644
--- a/Source/bmalloc/bmalloc/IsoHeapImplInlines.h
+++ b/Source/bmalloc/bmalloc/IsoHeapImplInlines.h
@@ -34,64 +34,70 @@
template<typename Config>
IsoHeapImpl<Config>::IsoHeapImpl()
- : lock((*PerProcess<IsoTLSEntryHolder<IsoTLSDeallocatorEntry<Config>>>::get())->lock)
+ : IsoHeapImplBase((*PerProcess<IsoTLSEntryHolder<IsoTLSDeallocatorEntry<Config>>>::get())->lock)
, m_inlineDirectory(*this)
, m_allocator(*this)
{
}
template<typename Config>
-EligibilityResult<Config> IsoHeapImpl<Config>::takeFirstEligible()
+EligibilityResult<Config> IsoHeapImpl<Config>::takeFirstEligible(const std::lock_guard<Mutex>& locker)
{
if (m_isInlineDirectoryEligibleOrDecommitted) {
- EligibilityResult<Config> result = m_inlineDirectory.takeFirstEligible();
+ EligibilityResult<Config> result = m_inlineDirectory.takeFirstEligible(locker);
if (result.kind == EligibilityKind::Full)
m_isInlineDirectoryEligibleOrDecommitted = false;
else
return result;
}
- if (!m_firstEligibleOrDecommitedDirectory) {
- // If nothing is eligible, it can only be because we have no directories. It wouldn't be the end
- // of the world if we broke this invariant. It would only mean that didBecomeEligibleOrDecommited() would need
- // a null check.
- RELEASE_BASSERT(!m_headDirectory);
- RELEASE_BASSERT(!m_tailDirectory);
- }
-
- for (; m_firstEligibleOrDecommitedDirectory; m_firstEligibleOrDecommitedDirectory = m_firstEligibleOrDecommitedDirectory->next) {
- EligibilityResult<Config> result = m_firstEligibleOrDecommitedDirectory->payload.takeFirstEligible();
- if (result.kind != EligibilityKind::Full) {
- m_directoryHighWatermark = std::max(m_directoryHighWatermark, m_firstEligibleOrDecommitedDirectory->index());
- return result;
+ {
+ auto* cursor = m_firstEligibleOrDecommitedDirectory.get();
+ if (!cursor) {
+ // If nothing is eligible, it can only be because we have no directories. It wouldn't be the end
+ // of the world if we broke this invariant. It would only mean that didBecomeEligibleOrDecommited() would need
+ // a null check.
+ RELEASE_BASSERT(!m_headDirectory.get());
+ RELEASE_BASSERT(!m_tailDirectory.get());
+ } else {
+ for (; cursor; cursor = cursor->next) {
+ EligibilityResult<Config> result = cursor->payload.takeFirstEligible(locker);
+ ASSERT(m_firstEligibleOrDecommitedDirectory.get() == cursor);
+ if (result.kind != EligibilityKind::Full) {
+ m_directoryHighWatermark = std::max(m_directoryHighWatermark, cursor->index());
+ m_firstEligibleOrDecommitedDirectory = cursor;
+ return result;
+ }
+ }
+ m_firstEligibleOrDecommitedDirectory = nullptr;
}
}
auto* newDirectory = new IsoDirectoryPage<Config>(*this, m_nextDirectoryPageIndex++);
- if (m_headDirectory) {
+ if (m_headDirectory.get()) {
m_tailDirectory->next = newDirectory;
m_tailDirectory = newDirectory;
} else {
- RELEASE_BASSERT(!m_tailDirectory);
+ RELEASE_BASSERT(!m_tailDirectory.get());
m_headDirectory = newDirectory;
m_tailDirectory = newDirectory;
}
m_directoryHighWatermark = newDirectory->index();
m_firstEligibleOrDecommitedDirectory = newDirectory;
- EligibilityResult<Config> result = newDirectory->payload.takeFirstEligible();
+ EligibilityResult<Config> result = newDirectory->payload.takeFirstEligible(locker);
RELEASE_BASSERT(result.kind != EligibilityKind::Full);
return result;
}
template<typename Config>
-void IsoHeapImpl<Config>::didBecomeEligibleOrDecommited(IsoDirectory<Config, numPagesInInlineDirectory>* directory)
+void IsoHeapImpl<Config>::didBecomeEligibleOrDecommited(const std::lock_guard<Mutex>&, IsoDirectory<Config, numPagesInInlineDirectory>* directory)
{
RELEASE_BASSERT(directory == &m_inlineDirectory);
m_isInlineDirectoryEligibleOrDecommitted = true;
}
template<typename Config>
-void IsoHeapImpl<Config>::didBecomeEligibleOrDecommited(IsoDirectory<Config, IsoDirectoryPage<Config>::numPages>* directory)
+void IsoHeapImpl<Config>::didBecomeEligibleOrDecommited(const std::lock_guard<Mutex>&, IsoDirectory<Config, IsoDirectoryPage<Config>::numPages>* directory)
{
RELEASE_BASSERT(m_firstEligibleOrDecommitedDirectory);
auto* directoryPage = IsoDirectoryPage<Config>::pageFor(directory);
@@ -104,8 +110,9 @@
{
std::lock_guard<Mutex> locker(this->lock);
forEachDirectory(
+ locker,
[&] (auto& directory) {
- directory.scavenge(decommits);
+ directory.scavenge(locker, decommits);
});
m_directoryHighWatermark = 0;
}
@@ -116,17 +123,16 @@
{
std::lock_guard<Mutex> locker(this->lock);
if (!m_directoryHighWatermark)
- m_inlineDirectory.scavengeToHighWatermark(decommits);
- for (IsoDirectoryPage<Config>* page = m_headDirectory; page; page = page->next) {
+ m_inlineDirectory.scavengeToHighWatermark(locker, decommits);
+ for (IsoDirectoryPage<Config>* page = m_headDirectory.get(); page; page = page->next) {
if (page->index() >= m_directoryHighWatermark)
- page->payload.scavengeToHighWatermark(decommits);
+ page->payload.scavengeToHighWatermark(locker, decommits);
}
m_directoryHighWatermark = 0;
}
#endif
-template<typename Config>
-size_t IsoHeapImpl<Config>::freeableMemory()
+inline size_t IsoHeapImplBase::freeableMemory()
{
return m_freeableMemory;
}
@@ -146,8 +152,10 @@
template<typename Config>
unsigned IsoHeapImpl<Config>::numLiveObjects()
{
+ std::lock_guard<Mutex> locker(this->lock);
unsigned result = 0;
forEachLiveObject(
+ locker,
[&] (void*) {
result++;
});
@@ -157,8 +165,10 @@
template<typename Config>
unsigned IsoHeapImpl<Config>::numCommittedPages()
{
+ std::lock_guard<Mutex> locker(this->lock);
unsigned result = 0;
forEachCommittedPage(
+ locker,
[&] (IsoPage<Config>&) {
result++;
});
@@ -167,40 +177,41 @@
template<typename Config>
template<typename Func>
-void IsoHeapImpl<Config>::forEachDirectory(const Func& func)
+void IsoHeapImpl<Config>::forEachDirectory(const std::lock_guard<Mutex>&, const Func& func)
{
func(m_inlineDirectory);
- for (IsoDirectoryPage<Config>* page = m_headDirectory; page; page = page->next)
+ for (IsoDirectoryPage<Config>* page = m_headDirectory.get(); page; page = page->next)
func(page->payload);
}
template<typename Config>
template<typename Func>
-void IsoHeapImpl<Config>::forEachCommittedPage(const Func& func)
+void IsoHeapImpl<Config>::forEachCommittedPage(const std::lock_guard<Mutex>& locker, const Func& func)
{
forEachDirectory(
+ locker,
[&] (auto& directory) {
- directory.forEachCommittedPage(func);
+ directory.forEachCommittedPage(locker, func);
});
}
template<typename Config>
template<typename Func>
-void IsoHeapImpl<Config>::forEachLiveObject(const Func& func)
+void IsoHeapImpl<Config>::forEachLiveObject(const std::lock_guard<Mutex>& locker, const Func& func)
{
forEachCommittedPage(
+ locker,
[&] (IsoPage<Config>& page) {
- page.forEachLiveObject(func);
+ page.forEachLiveObject(locker, func);
});
for (unsigned index = 0; index < maxAllocationFromShared; ++index) {
- void* pointer = m_sharedCells[index];
+ void* pointer = m_sharedCells[index].get();
if (pointer && !(m_availableShared & (1U << index)))
func(pointer);
}
}
-template<typename Config>
-size_t IsoHeapImpl<Config>::footprint()
+inline size_t IsoHeapImplBase::footprint()
{
#if ENABLE_PHYSICAL_PAGE_MAP
RELEASE_BASSERT(m_footprint == m_physicalPageMap.footprint());
@@ -208,8 +219,7 @@
return m_footprint;
}
-template<typename Config>
-void IsoHeapImpl<Config>::didCommit(void* ptr, size_t bytes)
+inline void IsoHeapImplBase::didCommit(void* ptr, size_t bytes)
{
BUNUSED_PARAM(ptr);
m_footprint += bytes;
@@ -218,8 +228,7 @@
#endif
}
-template<typename Config>
-void IsoHeapImpl<Config>::didDecommit(void* ptr, size_t bytes)
+inline void IsoHeapImplBase::didDecommit(void* ptr, size_t bytes)
{
BUNUSED_PARAM(ptr);
m_footprint -= bytes;
@@ -228,15 +237,13 @@
#endif
}
-template<typename Config>
-void IsoHeapImpl<Config>::isNowFreeable(void* ptr, size_t bytes)
+inline void IsoHeapImplBase::isNowFreeable(void* ptr, size_t bytes)
{
BUNUSED_PARAM(ptr);
m_freeableMemory += bytes;
}
-template<typename Config>
-void IsoHeapImpl<Config>::isNoLongerFreeable(void* ptr, size_t bytes)
+inline void IsoHeapImplBase::isNoLongerFreeable(void* ptr, size_t bytes)
{
BUNUSED_PARAM(ptr);
m_freeableMemory -= bytes;
@@ -301,7 +308,7 @@
unsigned indexPlusOne = __builtin_ffs(m_availableShared);
BASSERT(indexPlusOne);
unsigned index = indexPlusOne - 1;
- void* result = m_sharedCells[index];
+ void* result = m_sharedCells[index].get();
if (result) {
if (verbose)
fprintf(stderr, "%p: allocated %p from shared again of size %u\n", this, result, Config::objectSize);
@@ -314,7 +321,7 @@
fprintf(stderr, "%p: allocated %p from shared of size %u\n", this, result, Config::objectSize);
BASSERT(index < IsoHeapImplBase::maxAllocationFromShared);
*indexSlotFor<Config>(result) = index;
- m_sharedCells[index] = result;
+ m_sharedCells[index] = bitwise_cast<uint8_t*>(result);
}
BASSERT(result);
m_availableShared &= ~(1U << index);
diff --git a/Source/bmalloc/bmalloc/IsoPage.h b/Source/bmalloc/bmalloc/IsoPage.h
index 5425b88..a151cf98 100644
--- a/Source/bmalloc/bmalloc/IsoPage.h
+++ b/Source/bmalloc/bmalloc/IsoPage.h
@@ -28,7 +28,9 @@
#include "Bits.h"
#include "DeferredTrigger.h"
#include "FreeList.h"
+#include "Mutex.h"
#include <climits>
+#include <mutex>
namespace bmalloc {
@@ -42,6 +44,8 @@
explicit IsoPageBase(bool isShared)
: m_isShared(isShared)
+ , m_eligibilityHasBeenNoted(true)
+ , m_isInUseForAllocation(false)
{
}
@@ -52,7 +56,9 @@
protected:
BEXPORT static void* allocatePageMemory();
- bool m_isShared { false };
+ bool m_isShared : 1;
+ bool m_eligibilityHasBeenNoted : 1;
+ bool m_isInUseForAllocation : 1;
};
template<typename Config>
@@ -71,19 +77,19 @@
unsigned index() const { return m_index; }
- void free(void*);
+ void free(const std::lock_guard<Mutex>&, void*);
// Called after this page is already selected for allocation.
- FreeList startAllocating();
+ FreeList startAllocating(const std::lock_guard<Mutex>&);
// Called after the allocator picks another page to replace this one.
- void stopAllocating(FreeList freeList);
+ void stopAllocating(const std::lock_guard<Mutex>&, FreeList);
IsoDirectoryBase<Config>& directory() { return m_directory; }
bool isInUseForAllocation() const { return m_isInUseForAllocation; }
template<typename Func>
- void forEachLiveObject(const Func&);
+ void forEachLiveObject(const std::lock_guard<Mutex>&, const Func&);
IsoHeapImpl<Config>& heap();
@@ -111,16 +117,13 @@
// This must have a trivial destructor.
- bool m_eligibilityHasBeenNoted { true };
- bool m_isInUseForAllocation { false };
DeferredTrigger<IsoPageTrigger::Eligible> m_eligibilityTrigger;
DeferredTrigger<IsoPageTrigger::Empty> m_emptyTrigger;
-
- IsoDirectoryBase<Config>& m_directory;
+ uint8_t m_numNonEmptyWords { 0 };
+ static_assert(bitsArrayLength(numObjects) <= UINT8_MAX);
unsigned m_index { UINT_MAX };
-
+ IsoDirectoryBase<Config>& m_directory;
unsigned m_allocBits[bitsArrayLength(numObjects)];
- unsigned m_numNonEmptyWords { 0 };
};
} // namespace bmalloc
diff --git a/Source/bmalloc/bmalloc/IsoPageInlines.h b/Source/bmalloc/bmalloc/IsoPageInlines.h
index f37b349..c5ce871 100644
--- a/Source/bmalloc/bmalloc/IsoPageInlines.h
+++ b/Source/bmalloc/bmalloc/IsoPageInlines.h
@@ -47,8 +47,8 @@
template<typename Config>
IsoPage<Config>::IsoPage(IsoDirectoryBase<Config>& directory, unsigned index)
: IsoPageBase(false)
- , m_directory(directory)
, m_index(index)
+ , m_directory(directory)
{
memset(m_allocBits, 0, sizeof(m_allocBits));
}
@@ -65,14 +65,14 @@
}
template<typename Config>
-void IsoPage<Config>::free(void* passedPtr)
+void IsoPage<Config>::free(const std::lock_guard<Mutex>& locker, void* passedPtr)
{
BASSERT(!m_isShared);
unsigned offset = static_cast<char*>(passedPtr) - reinterpret_cast<char*>(this);
unsigned index = offset / Config::objectSize;
if (!m_eligibilityHasBeenNoted) {
- m_eligibilityTrigger.didBecome(*this);
+ m_eligibilityTrigger.didBecome(locker, *this);
m_eligibilityHasBeenNoted = true;
}
@@ -82,12 +82,12 @@
unsigned newWord = m_allocBits[wordIndex] &= ~(1 << bitIndex);
if (!newWord) {
if (!--m_numNonEmptyWords)
- m_emptyTrigger.didBecome(*this);
+ m_emptyTrigger.didBecome(locker, *this);
}
}
template<typename Config>
-FreeList IsoPage<Config>::startAllocating()
+FreeList IsoPage<Config>::startAllocating(const std::lock_guard<Mutex>&)
{
static constexpr bool verbose = false;
@@ -208,7 +208,7 @@
}
template<typename Config>
-void IsoPage<Config>::stopAllocating(FreeList freeList)
+void IsoPage<Config>::stopAllocating(const std::lock_guard<Mutex>& locker, FreeList freeList)
{
static constexpr bool verbose = false;
@@ -217,19 +217,19 @@
freeList.forEach<Config>(
[&] (void* ptr) {
- free(ptr);
+ free(locker, ptr);
});
RELEASE_BASSERT(m_isInUseForAllocation);
m_isInUseForAllocation = false;
- m_eligibilityTrigger.handleDeferral(*this);
- m_emptyTrigger.handleDeferral(*this);
+ m_eligibilityTrigger.handleDeferral(locker, *this);
+ m_emptyTrigger.handleDeferral(locker, *this);
}
template<typename Config>
template<typename Func>
-void IsoPage<Config>::forEachLiveObject(const Func& func)
+void IsoPage<Config>::forEachLiveObject(const std::lock_guard<Mutex>&, const Func& func)
{
for (unsigned wordIndex = 0; wordIndex < bitsArrayLength(numObjects); ++wordIndex) {
unsigned word = m_allocBits[wordIndex];
diff --git a/Source/bmalloc/bmalloc/IsoSharedHeap.h b/Source/bmalloc/bmalloc/IsoSharedHeap.h
index c9d04ee..5ea51e5 100644
--- a/Source/bmalloc/bmalloc/IsoSharedHeap.h
+++ b/Source/bmalloc/bmalloc/IsoSharedHeap.h
@@ -53,7 +53,7 @@
class IsoSharedHeap : public StaticPerProcess<IsoSharedHeap> {
public:
- IsoSharedHeap(std::lock_guard<Mutex>&)
+ IsoSharedHeap(const std::lock_guard<Mutex>&)
{
}
@@ -62,7 +62,7 @@
private:
template<unsigned>
- void* allocateSlow(bool abortOnFailure);
+ void* allocateSlow(const std::lock_guard<Mutex>&, bool abortOnFailure);
IsoSharedPage* m_currentPage { nullptr };
VariadicBumpAllocator m_allocator;
diff --git a/Source/bmalloc/bmalloc/IsoSharedHeapInlines.h b/Source/bmalloc/bmalloc/IsoSharedHeapInlines.h
index d9f7f11..da18e6e7 100644
--- a/Source/bmalloc/bmalloc/IsoSharedHeapInlines.h
+++ b/Source/bmalloc/bmalloc/IsoSharedHeapInlines.h
@@ -55,12 +55,12 @@
constexpr unsigned objectSize = computeObjectSizeForSharedCell(passedObjectSize);
return m_allocator.template allocate<objectSize>(
[&] () -> void* {
- return allocateSlow<passedObjectSize>(abortOnFailure);
+ return allocateSlow<passedObjectSize>(locker, abortOnFailure);
});
}
template<unsigned passedObjectSize>
-BNO_INLINE void* IsoSharedHeap::allocateSlow(bool abortOnFailure)
+BNO_INLINE void* IsoSharedHeap::allocateSlow(const std::lock_guard<Mutex>& locker, bool abortOnFailure)
{
Scavenger& scavenger = *Scavenger::get();
scavenger.didStartGrowing();
@@ -73,10 +73,10 @@
}
if (m_currentPage)
- m_currentPage->stopAllocating();
+ m_currentPage->stopAllocating(locker);
m_currentPage = page;
- m_allocator = m_currentPage->startAllocating();
+ m_allocator = m_currentPage->startAllocating(locker);
constexpr unsigned objectSize = computeObjectSizeForSharedCell(passedObjectSize);
return m_allocator.allocate<objectSize>([] () { BCRASH(); return nullptr; });
diff --git a/Source/bmalloc/bmalloc/IsoSharedPage.h b/Source/bmalloc/bmalloc/IsoSharedPage.h
index a9e137e..bff70c2 100644
--- a/Source/bmalloc/bmalloc/IsoSharedPage.h
+++ b/Source/bmalloc/bmalloc/IsoSharedPage.h
@@ -39,8 +39,8 @@
template<typename Config, typename Type>
void free(const std::lock_guard<Mutex>&, api::IsoHeap<Type>&, void*);
- VariadicBumpAllocator startAllocating();
- void stopAllocating();
+ VariadicBumpAllocator startAllocating(const std::lock_guard<Mutex>&);
+ void stopAllocating(const std::lock_guard<Mutex>&);
private:
IsoSharedPage()
diff --git a/Source/bmalloc/bmalloc/IsoSharedPageInlines.h b/Source/bmalloc/bmalloc/IsoSharedPageInlines.h
index c4216b5..e4410ba 100644
--- a/Source/bmalloc/bmalloc/IsoSharedPageInlines.h
+++ b/Source/bmalloc/bmalloc/IsoSharedPageInlines.h
@@ -42,11 +42,11 @@
// IsoDeallocator::deallocate is called from delete operator. This is dispatched by vtable if virtual destructor exists.
// If vptr is replaced to the other vptr, we may accidentally chain this pointer to the incorrect HeapImplBase, which totally breaks the IsoHeap's goal.
// To harden that, we validate that this pointer is actually allocated for a specific HeapImplBase here by checking whether this pointer is listed in HeapImplBase's shared cells.
- RELEASE_BASSERT(heapImpl.m_sharedCells[index] == ptr);
+ RELEASE_BASSERT(heapImpl.m_sharedCells[index].get() == ptr);
heapImpl.m_availableShared |= (1U << index);
}
-inline VariadicBumpAllocator IsoSharedPage::startAllocating()
+inline VariadicBumpAllocator IsoSharedPage::startAllocating(const std::lock_guard<Mutex>&)
{
static constexpr bool verbose = false;
@@ -61,7 +61,7 @@
return VariadicBumpAllocator(payloadEnd, remaining);
}
-inline void IsoSharedPage::stopAllocating()
+inline void IsoSharedPage::stopAllocating(const std::lock_guard<Mutex>&)
{
static constexpr bool verbose = false;
diff --git a/Source/bmalloc/bmalloc/IsoTLS.h b/Source/bmalloc/bmalloc/IsoTLS.h
index de2867e..babbb30 100644
--- a/Source/bmalloc/bmalloc/IsoTLS.h
+++ b/Source/bmalloc/bmalloc/IsoTLS.h
@@ -60,8 +60,8 @@
template<typename Config, typename Type>
static void* allocateImpl(api::IsoHeap<Type>&, bool abortOnFailure);
- template<typename Config>
- void* allocateFast(unsigned offset, bool abortOnFailure);
+ template<typename Config, typename Type>
+ void* allocateFast(api::IsoHeap<Type>&, unsigned offset, bool abortOnFailure);
template<typename Config, typename Type>
static void* allocateSlow(api::IsoHeap<Type>&, bool abortOnFailure);
diff --git a/Source/bmalloc/bmalloc/IsoTLSAllocatorEntry.h b/Source/bmalloc/bmalloc/IsoTLSAllocatorEntry.h
index 35db871..e1d8aac 100644
--- a/Source/bmalloc/bmalloc/IsoTLSAllocatorEntry.h
+++ b/Source/bmalloc/bmalloc/IsoTLSAllocatorEntry.h
@@ -42,6 +42,8 @@
IsoTLSAllocatorEntry(IsoHeapImpl<Config>&);
void construct(void* dst) override;
+
+ void scavenge(void* entry) override;
IsoHeapImpl<Config>& m_heap;
};
diff --git a/Source/bmalloc/bmalloc/IsoTLSAllocatorEntryInlines.h b/Source/bmalloc/bmalloc/IsoTLSAllocatorEntryInlines.h
index 1ae1e6a..3e557ad 100644
--- a/Source/bmalloc/bmalloc/IsoTLSAllocatorEntryInlines.h
+++ b/Source/bmalloc/bmalloc/IsoTLSAllocatorEntryInlines.h
@@ -46,5 +46,11 @@
new (dst) IsoAllocator<Config>(m_heap);
}
+template<typename Config>
+void IsoTLSAllocatorEntry<Config>::scavenge(void* entry)
+{
+ static_cast<IsoAllocator<Config>*>(entry)->scavenge(m_heap);
+}
+
} // namespace bmalloc
diff --git a/Source/bmalloc/bmalloc/IsoTLSDeallocatorEntry.h b/Source/bmalloc/bmalloc/IsoTLSDeallocatorEntry.h
index 7384ce0..a0dbc30 100644
--- a/Source/bmalloc/bmalloc/IsoTLSDeallocatorEntry.h
+++ b/Source/bmalloc/bmalloc/IsoTLSDeallocatorEntry.h
@@ -46,6 +46,7 @@
IsoTLSDeallocatorEntry(const std::lock_guard<Mutex>&);
void construct(void* entry) override;
+ void scavenge(void* entry) override;
};
} // namespace bmalloc
diff --git a/Source/bmalloc/bmalloc/IsoTLSDeallocatorEntryInlines.h b/Source/bmalloc/bmalloc/IsoTLSDeallocatorEntryInlines.h
index cbd5f9d..c953428 100644
--- a/Source/bmalloc/bmalloc/IsoTLSDeallocatorEntryInlines.h
+++ b/Source/bmalloc/bmalloc/IsoTLSDeallocatorEntryInlines.h
@@ -43,5 +43,11 @@
new (entry) IsoDeallocator<Config>(lock);
}
+template<typename Config>
+void IsoTLSDeallocatorEntry<Config>::scavenge(void* entry)
+{
+ static_cast<IsoDeallocator<Config>*>(entry)->scavenge();
+}
+
} // namespace bmalloc
diff --git a/Source/bmalloc/bmalloc/IsoTLSEntry.cpp b/Source/bmalloc/bmalloc/IsoTLSEntry.cpp
index cb03bde..830a511 100644
--- a/Source/bmalloc/bmalloc/IsoTLSEntry.cpp
+++ b/Source/bmalloc/bmalloc/IsoTLSEntry.cpp
@@ -31,10 +31,8 @@
namespace bmalloc {
-IsoTLSEntry::IsoTLSEntry(size_t alignment, size_t size)
- : m_offset(UINT_MAX)
- , m_alignment(alignment)
- , m_size(size)
+IsoTLSEntry::IsoTLSEntry(size_t size)
+ : m_size(size)
{
}
diff --git a/Source/bmalloc/bmalloc/IsoTLSEntry.h b/Source/bmalloc/bmalloc/IsoTLSEntry.h
index 74f449b..a478e6a 100644
--- a/Source/bmalloc/bmalloc/IsoTLSEntry.h
+++ b/Source/bmalloc/bmalloc/IsoTLSEntry.h
@@ -65,7 +65,7 @@
virtual ~IsoTLSEntry();
size_t offset() const { return m_offset; }
- size_t alignment() const { return m_alignment; }
+ size_t alignment() const { return sizeof(void*); }
size_t size() const { return m_size; }
size_t extent() const { return m_offset + m_size; }
@@ -78,7 +78,7 @@
void walkUpToInclusive(IsoTLSEntry*, const Func&);
protected:
- IsoTLSEntry(size_t alignment, size_t size);
+ IsoTLSEntry(size_t size);
private:
friend class IsoTLS;
@@ -86,15 +86,14 @@
IsoTLSEntry* m_next { nullptr };
- size_t m_offset; // Computed in constructor.
- size_t m_alignment;
- size_t m_size;
+ unsigned m_offset { UINT_MAX }; // Computed in constructor.
+ unsigned m_size;
};
template<typename EntryType>
class DefaultIsoTLSEntry : public IsoTLSEntry {
public:
- ~DefaultIsoTLSEntry();
+ ~DefaultIsoTLSEntry() = default;
protected:
DefaultIsoTLSEntry();
@@ -106,8 +105,6 @@
// Likewise, this is separate from scavenging. When the TLS is shutting down, we will be asked to
// scavenge and then we will be asked to destruct.
void destruct(void* entry) override;
-
- void scavenge(void* entry) override;
};
} // namespace bmalloc
diff --git a/Source/bmalloc/bmalloc/IsoTLSEntryInlines.h b/Source/bmalloc/bmalloc/IsoTLSEntryInlines.h
index 6f615f95..c2a727e 100644
--- a/Source/bmalloc/bmalloc/IsoTLSEntryInlines.h
+++ b/Source/bmalloc/bmalloc/IsoTLSEntryInlines.h
@@ -43,13 +43,10 @@
template<typename EntryType>
DefaultIsoTLSEntry<EntryType>::DefaultIsoTLSEntry()
- : IsoTLSEntry(alignof(EntryType), sizeof(EntryType))
+ : IsoTLSEntry(sizeof(EntryType))
{
-}
-
-template<typename EntryType>
-DefaultIsoTLSEntry<EntryType>::~DefaultIsoTLSEntry()
-{
+ static_assert(sizeof(EntryType) <= UINT32_MAX);
+ static_assert(sizeof(void*) == alignof(EntryType), "Because IsoTLSEntry includes vtable, it should be the same to the pointer");
}
template<typename EntryType>
@@ -67,12 +64,5 @@
entry->~EntryType();
}
-template<typename EntryType>
-void DefaultIsoTLSEntry<EntryType>::scavenge(void* passedEntry)
-{
- EntryType* entry = static_cast<EntryType*>(passedEntry);
- entry->scavenge();
-}
-
} // namespace bmalloc
diff --git a/Source/bmalloc/bmalloc/IsoTLSInlines.h b/Source/bmalloc/bmalloc/IsoTLSInlines.h
index 1eab271..0d4b201 100644
--- a/Source/bmalloc/bmalloc/IsoTLSInlines.h
+++ b/Source/bmalloc/bmalloc/IsoTLSInlines.h
@@ -60,7 +60,7 @@
return;
unsigned offset = handle.allocatorOffset();
if (offset < tls->m_extent)
- reinterpret_cast<IsoAllocator<typename api::IsoHeap<Type>::Config>*>(tls->m_data + offset)->scavenge();
+ reinterpret_cast<IsoAllocator<typename api::IsoHeap<Type>::Config>*>(tls->m_data + offset)->scavenge(handle.impl());
offset = handle.deallocatorOffset();
if (offset < tls->m_extent)
reinterpret_cast<IsoDeallocator<typename api::IsoHeap<Type>::Config>*>(tls->m_data + offset)->scavenge();
@@ -74,13 +74,13 @@
IsoTLS* tls = get();
if (!tls || offset >= tls->m_extent)
return allocateSlow<Config>(handle, abortOnFailure);
- return tls->allocateFast<Config>(offset, abortOnFailure);
+ return tls->allocateFast<Config>(handle, offset, abortOnFailure);
}
-template<typename Config>
-void* IsoTLS::allocateFast(unsigned offset, bool abortOnFailure)
+template<typename Config, typename Type>
+void* IsoTLS::allocateFast(api::IsoHeap<Type>& handle, unsigned offset, bool abortOnFailure)
{
- return reinterpret_cast<IsoAllocator<Config>*>(m_data + offset)->allocate(abortOnFailure);
+ return reinterpret_cast<IsoAllocator<Config>*>(m_data + offset)->allocate(handle.impl(), abortOnFailure);
}
template<typename Config, typename Type>
@@ -108,7 +108,7 @@
IsoTLS* tls = ensureHeapAndEntries(handle);
- return tls->allocateFast<Config>(handle.allocatorOffset(), abortOnFailure);
+ return tls->allocateFast<Config>(handle, handle.allocatorOffset(), abortOnFailure);
}
template<typename Config, typename Type>
diff --git a/Source/bmalloc/bmalloc/IsoTLSLayout.cpp b/Source/bmalloc/bmalloc/IsoTLSLayout.cpp
index 4b335ea..0bae607 100644
--- a/Source/bmalloc/bmalloc/IsoTLSLayout.cpp
+++ b/Source/bmalloc/bmalloc/IsoTLSLayout.cpp
@@ -44,7 +44,9 @@
std::lock_guard<Mutex> locking(addingMutex);
if (m_head) {
RELEASE_BASSERT(m_tail);
- entry->m_offset = roundUpToMultipleOf(entry->alignment(), m_tail->extent());
+ size_t offset = roundUpToMultipleOf(entry->alignment(), m_tail->extent());
+ RELEASE_BASSERT(offset < UINT_MAX);
+ entry->m_offset = offset;
std::atomic_thread_fence(std::memory_order_seq_cst);
m_tail->m_next = entry;
m_tail = entry;
diff --git a/Source/bmalloc/bmalloc/Packed.h b/Source/bmalloc/bmalloc/Packed.h
new file mode 100644
index 0000000..c59e0c1
--- /dev/null
+++ b/Source/bmalloc/bmalloc/Packed.h
@@ -0,0 +1,236 @@
+/*
+ * Copyright (C) 2019-2020 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#include "Algorithm.h"
+#include "StdLibExtras.h"
+#include <array>
+
+namespace bmalloc {
+
+template<typename T>
+class Packed {
+public:
+ static_assert(std::is_trivial<T>::value);
+ static constexpr bool isPackedType = true;
+
+ Packed()
+ : Packed(T { })
+ {
+ }
+
+ Packed(const T& value)
+ {
+ memcpy(m_storage.data(), &value, sizeof(T));
+ }
+
+ T get() const
+ {
+ T value { };
+ memcpy(&value, m_storage.data(), sizeof(T));
+ return value;
+ }
+
+ void set(const T& value)
+ {
+ memcpy(m_storage.data(), &value, sizeof(T));
+ }
+
+ Packed<T>& operator=(const T& value)
+ {
+ set(value);
+ return *this;
+ }
+
+ template<class U>
+ T exchange(U&& newValue)
+ {
+ T oldValue = get();
+ set(std::forward<U>(newValue));
+ return oldValue;
+ }
+
+ void swap(Packed& other)
+ {
+ m_storage.swap(other.m_storage);
+ }
+
+ template<typename Other, typename = std::enable_if_t<Other::isPackedType>>
+ void swap(Other& other)
+ {
+ T t1 = get();
+ T t2 = other.get();
+ set(t2);
+ other.set(t1);
+ }
+
+ void swap(T& t2)
+ {
+ T t1 = get();
+ std::swap(t1, t2);
+ set(t1);
+ }
+
+private:
+ std::array<uint8_t, sizeof(T)> m_storage;
+};
+
+// PackedAlignedPtr can take alignment parameter too. PackedAlignedPtr only uses this alignment information if it is profitable: we use
+// alignment information only when we can reduce the size of the storage. Since the pointer width is 36 bits and JSCells are aligned to 16 bytes,
+// we can use 4 bits in Darwin ARM64, we can compact cell pointer into 4 bytes (32 bits).
+template<typename T, size_t alignment = alignof(T)>
+class PackedAlignedPtr {
+public:
+ static_assert(isPowerOfTwo(alignment), "Alignment needs to be power-of-two");
+ static constexpr bool isPackedType = true;
+ static constexpr unsigned alignmentShiftSizeIfProfitable = getLSBSetNonZeroConstexpr(alignment);
+ static constexpr unsigned storageSizeWithoutAlignmentShift = roundUpToMultipleOf<8, uintptr_t>(BOS_EFFECTIVE_ADDRESS_WIDTH) / 8;
+ static constexpr unsigned storageSizeWithAlignmentShift = roundUpToMultipleOf<8, uintptr_t>(BOS_EFFECTIVE_ADDRESS_WIDTH - alignmentShiftSizeIfProfitable) / 8;
+ static constexpr bool isAlignmentShiftProfitable = storageSizeWithoutAlignmentShift > storageSizeWithAlignmentShift;
+ static constexpr unsigned alignmentShiftSize = isAlignmentShiftProfitable ? alignmentShiftSizeIfProfitable : 0;
+ static constexpr unsigned storageSize = storageSizeWithAlignmentShift;
+
+ constexpr PackedAlignedPtr()
+ : m_storage()
+ {
+ }
+
+ constexpr PackedAlignedPtr(std::nullptr_t)
+ : m_storage()
+ {
+ }
+
+ PackedAlignedPtr(T* value)
+ {
+ set(value);
+ }
+
+ T* get() const
+ {
+ // FIXME: PackedPtr<> can load memory with one mov by checking page boundary.
+ // https://bugs.webkit.org/show_bug.cgi?id=197754
+ uintptr_t value = 0;
+#if BCPU(LITTLE_ENDIAN)
+ memcpy(&value, m_storage.data(), storageSize);
+#else
+ memcpy(bitwise_cast<uint8_t*>(&value) + (sizeof(void*) - storageSize), m_storage.data(), storageSize);
+#endif
+ if (isAlignmentShiftProfitable)
+ value <<= alignmentShiftSize;
+ return bitwise_cast<T*>(value);
+ }
+
+ void set(T* passedValue)
+ {
+ uintptr_t value = bitwise_cast<uintptr_t>(passedValue);
+ if (isAlignmentShiftProfitable)
+ value >>= alignmentShiftSize;
+#if BCPU(LITTLE_ENDIAN)
+ memcpy(m_storage.data(), &value, storageSize);
+#else
+ memcpy(m_storage.data(), bitwise_cast<uint8_t*>(&value) + (sizeof(void*) - storageSize), storageSize);
+#endif
+ }
+
+ void clear()
+ {
+ set(nullptr);
+ }
+
+ T* operator->() const { return get(); }
+ T& operator*() const { return *get(); }
+ bool operator!() const { return !get(); }
+
+ // This conversion operator allows implicit conversion to bool but not to other integer types.
+ typedef T* (PackedAlignedPtr::*UnspecifiedBoolType);
+ operator UnspecifiedBoolType() const { return get() ? &PackedAlignedPtr::m_storage : nullptr; }
+ explicit operator bool() const { return get(); }
+
+ PackedAlignedPtr& operator=(T* value)
+ {
+ set(value);
+ return *this;
+ }
+
+ template<class U>
+ T* exchange(U&& newValue)
+ {
+ T* oldValue = get();
+ set(std::forward<U>(newValue));
+ return oldValue;
+ }
+
+ void swap(std::nullptr_t) { clear(); }
+
+ void swap(PackedAlignedPtr& other)
+ {
+ m_storage.swap(other.m_storage);
+ }
+
+ template<typename Other, typename = std::enable_if_t<Other::isPackedType>>
+ void swap(Other& other)
+ {
+ T* t1 = get();
+ T* t2 = other.get();
+ set(t2);
+ other.set(t1);
+ }
+
+ void swap(T* t2)
+ {
+ T* t1 = get();
+ std::swap(t1, t2);
+ set(t1);
+ }
+
+private:
+ std::array<uint8_t, storageSize> m_storage;
+};
+
+template<typename T>
+class Packed<T*> : public PackedAlignedPtr<T, 1> {
+public:
+ using Base = PackedAlignedPtr<T, 1>;
+ using Base::Base;
+};
+
+template<typename T>
+using PackedPtr = Packed<T*>;
+
+template<typename T>
+struct PackedPtrTraits {
+ template<typename U> using RebindTraits = PackedPtrTraits<U>;
+
+ using StorageType = PackedPtr<T>;
+
+ template<class U> static T* exchange(StorageType& ptr, U&& newValue) { return ptr.exchange(newValue); }
+
+ template<typename Other> static void swap(PackedPtr<T>& a, Other& b) { a.swap(b); }
+
+ static T* unwrap(const StorageType& ptr) { return ptr.get(); }
+};
+
+} // namespace bmalloc
diff --git a/Source/bmalloc/bmalloc/Scavenger.cpp b/Source/bmalloc/bmalloc/Scavenger.cpp
index eb15b31..60f0676 100644
--- a/Source/bmalloc/bmalloc/Scavenger.cpp
+++ b/Source/bmalloc/bmalloc/Scavenger.cpp
@@ -30,6 +30,7 @@
#include "BulkDecommit.h"
#include "Environment.h"
#include "Heap.h"
+#include "IsoHeapImplInlines.h"
#if BOS(DARWIN)
#import <dispatch/dispatch.h>
#import <mach/host_info.h>
@@ -67,7 +68,7 @@
DEFINE_STATIC_PER_PROCESS_STORAGE(Scavenger);
-Scavenger::Scavenger(std::lock_guard<Mutex>&)
+Scavenger::Scavenger(const std::lock_guard<Mutex>&)
{
BASSERT(!Environment::get()->isDebugHeapEnabled());
diff --git a/Source/bmalloc/bmalloc/Scavenger.h b/Source/bmalloc/bmalloc/Scavenger.h
index f48934b..728a75e 100644
--- a/Source/bmalloc/bmalloc/Scavenger.h
+++ b/Source/bmalloc/bmalloc/Scavenger.h
@@ -42,7 +42,7 @@
class Scavenger : public StaticPerProcess<Scavenger> {
public:
- BEXPORT Scavenger(std::lock_guard<Mutex>&);
+ BEXPORT Scavenger(const std::lock_guard<Mutex>&);
~Scavenger() = delete;
diff --git a/Source/bmalloc/bmalloc/VMHeap.cpp b/Source/bmalloc/bmalloc/VMHeap.cpp
index 5c8e28a..adf2167 100644
--- a/Source/bmalloc/bmalloc/VMHeap.cpp
+++ b/Source/bmalloc/bmalloc/VMHeap.cpp
@@ -31,7 +31,7 @@
DEFINE_STATIC_PER_PROCESS_STORAGE(VMHeap);
-VMHeap::VMHeap(std::lock_guard<Mutex>&)
+VMHeap::VMHeap(const std::lock_guard<Mutex>&)
{
}
diff --git a/Source/bmalloc/bmalloc/VMHeap.h b/Source/bmalloc/bmalloc/VMHeap.h
index 47a113d..68f0abe 100644
--- a/Source/bmalloc/bmalloc/VMHeap.h
+++ b/Source/bmalloc/bmalloc/VMHeap.h
@@ -47,7 +47,7 @@
class VMHeap : public StaticPerProcess<VMHeap> {
public:
- VMHeap(std::lock_guard<Mutex>&);
+ VMHeap(const std::lock_guard<Mutex>&);
LargeRange tryAllocateLargeChunk(size_t alignment, size_t);
};
diff --git a/Source/bmalloc/bmalloc/Zone.cpp b/Source/bmalloc/bmalloc/Zone.cpp
index f11b517..0bb8296 100644
--- a/Source/bmalloc/bmalloc/Zone.cpp
+++ b/Source/bmalloc/bmalloc/Zone.cpp
@@ -115,7 +115,7 @@
.statistics = bmalloc::statistics
};
-Zone::Zone(std::lock_guard<Mutex>&)
+Zone::Zone(const std::lock_guard<Mutex>&)
{
malloc_zone_t::size = &bmalloc::zoneSize;
malloc_zone_t::zone_name = "WebKit Malloc";
diff --git a/Source/bmalloc/bmalloc/Zone.h b/Source/bmalloc/bmalloc/Zone.h
index 009d722..f5d6495 100644
--- a/Source/bmalloc/bmalloc/Zone.h
+++ b/Source/bmalloc/bmalloc/Zone.h
@@ -42,7 +42,7 @@
// Enough capacity to track a 64GB heap, so probably enough for anything.
static constexpr size_t capacity = 2048;
- Zone(std::lock_guard<Mutex>&);
+ Zone(const std::lock_guard<Mutex>&);
Zone(task_t, memory_reader_t, vm_address_t);
void addRange(Range);
diff --git a/Tools/ChangeLog b/Tools/ChangeLog
index 4f170f5..1c81d3f 100644
--- a/Tools/ChangeLog
+++ b/Tools/ChangeLog
@@ -1,3 +1,16 @@
+2020-01-16 Yusuke Suzuki <ysuzuki@apple.com>
+
+ [bmalloc] IsoHeap's initial setup should be small
+ https://bugs.webkit.org/show_bug.cgi?id=206214
+
+ Reviewed by Michael Saboff.
+
+ * TestWebKitAPI/Tests/WTF/bmalloc/IsoHeap.cpp:
+ (assertHasObjects):
+ (assertHasOnlyObjects):
+ (assertClean):
+ (TEST):
+
2020-01-16 Brady Eidson <beidson@apple.com>
Make the callAsyncJavaScriptFunction function actually be async (so await works).
diff --git a/Tools/TestWebKitAPI/Tests/WTF/bmalloc/IsoHeap.cpp b/Tools/TestWebKitAPI/Tests/WTF/bmalloc/IsoHeap.cpp
index 4263f11..08b72aa 100644
--- a/Tools/TestWebKitAPI/Tests/WTF/bmalloc/IsoHeap.cpp
+++ b/Tools/TestWebKitAPI/Tests/WTF/bmalloc/IsoHeap.cpp
@@ -80,6 +80,7 @@
auto& impl = heap.impl();
std::lock_guard<bmalloc::Mutex> locker(impl.lock);
impl.forEachLiveObject(
+ locker,
[&] (void* object) {
pointers.erase(object);
});
@@ -96,6 +97,7 @@
auto& impl = heap.impl();
std::lock_guard<bmalloc::Mutex> locker(impl.lock);
impl.forEachLiveObject(
+ locker,
[&] (void* object) {
EXPECT_EQ(pointers.erase(object), 1U);
});
@@ -108,15 +110,11 @@
scavengeThisThread();
if (!Environment::get()->isDebugHeapEnabled()) {
auto& impl = heap.impl();
- {
- std::lock_guard<bmalloc::Mutex> locker(impl.lock);
- EXPECT_FALSE(impl.numLiveObjects());
- }
+ EXPECT_FALSE(impl.numLiveObjects());
}
heap.scavenge();
if (!Environment::get()->isDebugHeapEnabled()) {
auto& impl = heap.impl();
- std::lock_guard<bmalloc::Mutex> locker(impl.lock);
EXPECT_FALSE(impl.numCommittedPages());
}
}
@@ -187,19 +185,12 @@
ptrs[i] = nullptr;
}
heap.scavenge();
- unsigned numCommittedPagesBefore;
auto& impl = heap.impl();
- {
- std::lock_guard<bmalloc::Mutex> locker(impl.lock);
- numCommittedPagesBefore = impl.numCommittedPages();
- }
+ unsigned numCommittedPagesBefore = impl.numCommittedPages();
assertHasOnlyObjects(heap, toptrset(ptrs));
for (unsigned i = ptrs.size() / 2; i--;)
ptrs.push_back(heap.allocate());
- {
- std::lock_guard<bmalloc::Mutex> locker(impl.lock);
- EXPECT_EQ(numCommittedPagesBefore, impl.numCommittedPages());
- }
+ EXPECT_EQ(numCommittedPagesBefore, impl.numCommittedPages());
for (void* ptr : ptrs)
heap.deallocate(ptr);
assertClean(heap);
@@ -220,19 +211,12 @@
ptrs[i] = nullptr;
}
heap.scavenge();
- unsigned numCommittedPagesBefore;
auto& impl = heap.impl();
- {
- std::lock_guard<bmalloc::Mutex> locker(impl.lock);
- numCommittedPagesBefore = impl.numCommittedPages();
- }
+ unsigned numCommittedPagesBefore = impl.numCommittedPages();
assertHasOnlyObjects(heap, toptrset(ptrs));
for (unsigned i = ptrs.size() / 2; i--;)
ptrs.push_back(heap.allocate());
- {
- std::lock_guard<bmalloc::Mutex> locker(impl.lock);
- EXPECT_EQ(numCommittedPagesBefore, impl.numCommittedPages());
- }
+ EXPECT_EQ(numCommittedPagesBefore, impl.numCommittedPages());
for (void* ptr : ptrs)
heap.deallocate(ptr);
assertClean(heap);
