[JSC] Optimize Map iteration with intrinsic
https://bugs.webkit.org/show_bug.cgi?id=174355
Reviewed by Saam Barati.
JSTests:
* stress/map-iterator-result-should-have-expected-shape.js: Added.
(shouldBe):
(throw.new.Error):
* stress/set-iterator-result-should-have-expected-shape.js: Added.
(shouldBe):
(throw.new.Error.let.iterator.set Symbol):
(throw.new.Error.set add):
(let.iterator.set Symbol):
Source/JavaScriptCore:
This patch optimizes Map/Set iteration by taking the approach similar to Array iteration.
We create a simple iterator object instead of JSMapIterator and JSSetIterator. And we
directly handles Map/Set buckets in JS builtins. We carefully create mapIteratorNext and
setIteratorNext functions which should be inlined. This leads significant performance boost
when they are inlined in for-of iteration.
This patch changes how DFG and FTL handles MapBucket if the bucket is not found.
Previously, we use nullptr for that, and DFG and FTL specially handle this nullptr as bucket.
Instead, this patch introduces sentinel buckets. They are marked as deleted, and not linked
to any hash maps. And its key and value fields are filled with Undefined. By returning this
sentinel bucket instead of returning nullptr, we simplify DFG and FTL's LoadXXXFromMapBucket
code.
We still keep JSMapIterator and JSSetIterator because they are useful to serialize Map and Set
in WebCore. So they are not used in user observable JS. We change them from JS objects to JS cells.
Existing microbenchmarks shows performance improvements.
large-map-iteration 164.1622+-4.1618 ^ 56.6284+-1.5355 ^ definitely 2.8989x faster
set-for-of 15.4369+-1.0631 ^ 9.2955+-0.5979 ^ definitely 1.6607x faster
map-for-each 7.5889+-0.5792 ^ 6.3011+-0.4816 ^ definitely 1.2044x faster
map-for-of 32.3904+-1.3003 ^ 12.6907+-0.6118 ^ definitely 2.5523x faster
map-rehash 13.9275+-0.9187 ^ 11.5367+-0.6430 ^ definitely 1.2072x faster
* CMakeLists.txt:
* DerivedSources.make:
* builtins/ArrayPrototype.js:
(globalPrivate.createArrayIterator):
* builtins/BuiltinNames.h:
* builtins/MapIteratorPrototype.js: Copied from Source/JavaScriptCore/builtins/MapPrototype.js.
(globalPrivate.mapIteratorNext):
(next):
* builtins/MapPrototype.js:
(globalPrivate.createMapIterator):
(values):
(keys):
(entries):
(forEach):
* builtins/SetIteratorPrototype.js: Copied from Source/JavaScriptCore/builtins/MapPrototype.js.
(globalPrivate.setIteratorNext):
(next):
* builtins/SetPrototype.js:
(globalPrivate.createSetIterator):
(values):
(entries):
(forEach):
* bytecode/BytecodeIntrinsicRegistry.cpp:
(JSC::BytecodeIntrinsicRegistry::BytecodeIntrinsicRegistry):
* bytecode/BytecodeIntrinsicRegistry.h:
* bytecode/SpeculatedType.h:
* dfg/DFGAbstractInterpreterInlines.h:
(JSC::DFG::AbstractInterpreter<AbstractStateType>::executeEffects):
* dfg/DFGByteCodeParser.cpp:
(JSC::DFG::ByteCodeParser::handleIntrinsicCall):
* dfg/DFGClobberize.h:
(JSC::DFG::clobberize):
* dfg/DFGDoesGC.cpp:
(JSC::DFG::doesGC):
* dfg/DFGFixupPhase.cpp:
(JSC::DFG::FixupPhase::fixupNode):
* dfg/DFGHeapLocation.cpp:
(WTF::printInternal):
* dfg/DFGHeapLocation.h:
* dfg/DFGNode.h:
(JSC::DFG::Node::hasHeapPrediction):
(JSC::DFG::Node::hasBucketOwnerType):
(JSC::DFG::Node::bucketOwnerType):
(JSC::DFG::Node::OpInfoWrapper::as const):
* dfg/DFGNodeType.h:
* dfg/DFGOperations.cpp:
* dfg/DFGPredictionPropagationPhase.cpp:
* dfg/DFGSafeToExecute.h:
(JSC::DFG::safeToExecute):
* dfg/DFGSpeculativeJIT.cpp:
(JSC::DFG::SpeculativeJIT::compileGetMapBucketHead):
(JSC::DFG::SpeculativeJIT::compileGetMapBucketNext):
(JSC::DFG::SpeculativeJIT::compileLoadKeyFromMapBucket):
(JSC::DFG::SpeculativeJIT::compileLoadValueFromMapBucket):
(JSC::DFG::SpeculativeJIT::compileCompareEqPtr): Deleted.
* dfg/DFGSpeculativeJIT.h:
* dfg/DFGSpeculativeJIT32_64.cpp:
(JSC::DFG::SpeculativeJIT::compileCompareEqPtr):
(JSC::DFG::SpeculativeJIT::compile):
* dfg/DFGSpeculativeJIT64.cpp:
(JSC::DFG::SpeculativeJIT::compileCompareEqPtr):
(JSC::DFG::SpeculativeJIT::compile):
* ftl/FTLAbstractHeapRepository.h:
* ftl/FTLCapabilities.cpp:
(JSC::FTL::canCompile):
* ftl/FTLLowerDFGToB3.cpp:
(JSC::FTL::DFG::LowerDFGToB3::compileNode):
(JSC::FTL::DFG::LowerDFGToB3::compileGetMapBucket):
(JSC::FTL::DFG::LowerDFGToB3::compileGetMapBucketHead):
(JSC::FTL::DFG::LowerDFGToB3::compileGetMapBucketNext):
(JSC::FTL::DFG::LowerDFGToB3::compileLoadValueFromMapBucket):
(JSC::FTL::DFG::LowerDFGToB3::compileLoadKeyFromMapBucket):
(JSC::FTL::DFG::LowerDFGToB3::setStorage):
(JSC::FTL::DFG::LowerDFGToB3::compileLoadFromJSMapBucket): Deleted.
(JSC::FTL::DFG::LowerDFGToB3::compileIsNonEmptyMapBucket): Deleted.
(JSC::FTL::DFG::LowerDFGToB3::lowMapBucket): Deleted.
(JSC::FTL::DFG::LowerDFGToB3::setMapBucket): Deleted.
* inspector/JSInjectedScriptHost.cpp:
(Inspector::JSInjectedScriptHost::subtype):
(Inspector::JSInjectedScriptHost::getInternalProperties):
(Inspector::cloneMapIteratorObject):
(Inspector::cloneSetIteratorObject):
(Inspector::JSInjectedScriptHost::iteratorEntries):
* runtime/HashMapImpl.h:
(JSC::HashMapBucket::createSentinel):
(JSC::HashMapBucket::offsetOfNext):
(JSC::HashMapBucket::offsetOfDeleted):
(JSC::HashMapImpl::offsetOfHead):
* runtime/Intrinsic.cpp:
(JSC::intrinsicName):
* runtime/Intrinsic.h:
* runtime/JSGlobalObject.cpp:
(JSC::JSGlobalObject::init):
* runtime/JSGlobalObject.h:
* runtime/JSMap.h:
* runtime/JSMapIterator.cpp:
(JSC::JSMapIterator::clone): Deleted.
* runtime/JSMapIterator.h:
(JSC::JSMapIterator::iteratedValue const):
* runtime/JSSet.h:
* runtime/JSSetIterator.cpp:
(JSC::JSSetIterator::clone): Deleted.
* runtime/JSSetIterator.h:
(JSC::JSSetIterator::iteratedValue const):
* runtime/MapConstructor.cpp:
(JSC::mapPrivateFuncMapBucketHead):
(JSC::mapPrivateFuncMapBucketNext):
(JSC::mapPrivateFuncMapBucketKey):
(JSC::mapPrivateFuncMapBucketValue):
* runtime/MapConstructor.h:
* runtime/MapIteratorPrototype.cpp:
(JSC::MapIteratorPrototype::finishCreation):
(JSC::MapIteratorPrototypeFuncNext): Deleted.
* runtime/MapPrototype.cpp:
(JSC::MapPrototype::finishCreation):
(JSC::mapProtoFuncValues): Deleted.
(JSC::mapProtoFuncEntries): Deleted.
(JSC::mapProtoFuncKeys): Deleted.
(JSC::privateFuncMapIterator): Deleted.
(JSC::privateFuncMapIteratorNext): Deleted.
* runtime/MapPrototype.h:
* runtime/SetConstructor.cpp:
(JSC::setPrivateFuncSetBucketHead):
(JSC::setPrivateFuncSetBucketNext):
(JSC::setPrivateFuncSetBucketKey):
* runtime/SetConstructor.h:
* runtime/SetIteratorPrototype.cpp:
(JSC::SetIteratorPrototype::finishCreation):
(JSC::SetIteratorPrototypeFuncNext): Deleted.
* runtime/SetPrototype.cpp:
(JSC::SetPrototype::finishCreation):
(JSC::setProtoFuncSize):
(JSC::setProtoFuncValues): Deleted.
(JSC::setProtoFuncEntries): Deleted.
(JSC::privateFuncSetIterator): Deleted.
(JSC::privateFuncSetIteratorNext): Deleted.
* runtime/SetPrototype.h:
* runtime/VM.cpp:
(JSC::VM::VM):
* runtime/VM.h:
Source/WebCore:
* bindings/js/SerializedScriptValue.cpp:
(WebCore::CloneSerializer::serialize):
git-svn-id: http://svn.webkit.org/repository/webkit/trunk@221110 268f45cc-cd09-0410-ab3c-d52691b4dbfc
diff --git a/Source/JavaScriptCore/dfg/DFGAbstractInterpreterInlines.h b/Source/JavaScriptCore/dfg/DFGAbstractInterpreterInlines.h
index f603c74..40eea94 100644
--- a/Source/JavaScriptCore/dfg/DFGAbstractInterpreterInlines.h
+++ b/Source/JavaScriptCore/dfg/DFGAbstractInterpreterInlines.h
@@ -1048,16 +1048,28 @@
break;
}
- case LoadFromJSMapBucket:
+ case LoadKeyFromMapBucket:
+ case LoadValueFromMapBucket:
forNode(node).makeHeapTop();
break;
case GetMapBucket:
- forNode(node).setType(m_graph, SpecCellOther);
+ case GetMapBucketHead:
+ if (node->child1().useKind() == MapObjectUse)
+ forNode(node).set(m_graph, m_vm.hashMapBucketMapStructure.get());
+ else {
+ ASSERT(node->child1().useKind() == SetObjectUse);
+ forNode(node).set(m_graph, m_vm.hashMapBucketSetStructure.get());
+ }
break;
- case IsNonEmptyMapBucket:
- forNode(node).setType(SpecBoolean);
+ case GetMapBucketNext:
+ if (node->bucketOwnerType() == BucketOwnerType::Map)
+ forNode(node).set(m_graph, m_vm.hashMapBucketMapStructure.get());
+ else {
+ ASSERT(node->bucketOwnerType() == BucketOwnerType::Set);
+ forNode(node).set(m_graph, m_vm.hashMapBucketSetStructure.get());
+ }
break;
case IsEmpty:
diff --git a/Source/JavaScriptCore/dfg/DFGByteCodeParser.cpp b/Source/JavaScriptCore/dfg/DFGByteCodeParser.cpp
index 54740f3..a3cbb20 100644
--- a/Source/JavaScriptCore/dfg/DFGByteCodeParser.cpp
+++ b/Source/JavaScriptCore/dfg/DFGByteCodeParser.cpp
@@ -2868,7 +2868,7 @@
Node* key = get(virtualRegisterForArgument(1, registerOffset));
Node* hash = addToGraph(MapHash, key);
Node* bucket = addToGraph(GetMapBucket, Edge(map, MapObjectUse), Edge(key), Edge(hash));
- Node* result = addToGraph(LoadFromJSMapBucket, OpInfo(), OpInfo(prediction), bucket);
+ Node* result = addToGraph(LoadValueFromMapBucket, OpInfo(), OpInfo(prediction), bucket);
set(VirtualRegister(resultOperand), result);
return true;
}
@@ -2884,7 +2884,60 @@
Node* hash = addToGraph(MapHash, key);
UseKind useKind = intrinsic == JSSetHasIntrinsic ? SetObjectUse : MapObjectUse;
Node* bucket = addToGraph(GetMapBucket, OpInfo(0), Edge(mapOrSet, useKind), Edge(key), Edge(hash));
- Node* result = addToGraph(IsNonEmptyMapBucket, bucket);
+ JSCell* sentinel = nullptr;
+ if (intrinsic == JSMapHasIntrinsic)
+ sentinel = m_vm->sentinelMapBucket.get();
+ else
+ sentinel = m_vm->sentinelSetBucket.get();
+
+ FrozenValue* frozenPointer = m_graph.freeze(sentinel);
+ Node* invertedResult = addToGraph(CompareEqPtr, OpInfo(frozenPointer), bucket);
+ Node* result = addToGraph(LogicalNot, invertedResult);
+ set(VirtualRegister(resultOperand), result);
+ return true;
+ }
+
+ case JSSetBucketHeadIntrinsic:
+ case JSMapBucketHeadIntrinsic: {
+ ASSERT(argumentCountIncludingThis == 2);
+
+ insertChecks();
+ Node* map = get(virtualRegisterForArgument(1, registerOffset));
+ UseKind useKind = intrinsic == JSSetBucketHeadIntrinsic ? SetObjectUse : MapObjectUse;
+ Node* result = addToGraph(GetMapBucketHead, Edge(map, useKind));
+ set(VirtualRegister(resultOperand), result);
+ return true;
+ }
+
+ case JSSetBucketNextIntrinsic:
+ case JSMapBucketNextIntrinsic: {
+ ASSERT(argumentCountIncludingThis == 2);
+
+ insertChecks();
+ Node* bucket = get(virtualRegisterForArgument(1, registerOffset));
+ BucketOwnerType type = intrinsic == JSSetBucketNextIntrinsic ? BucketOwnerType::Set : BucketOwnerType::Map;
+ Node* result = addToGraph(GetMapBucketNext, OpInfo(type), bucket);
+ set(VirtualRegister(resultOperand), result);
+ return true;
+ }
+
+ case JSSetBucketKeyIntrinsic:
+ case JSMapBucketKeyIntrinsic: {
+ ASSERT(argumentCountIncludingThis == 2);
+
+ insertChecks();
+ Node* bucket = get(virtualRegisterForArgument(1, registerOffset));
+ Node* result = addToGraph(LoadKeyFromMapBucket, OpInfo(), OpInfo(prediction), bucket);
+ set(VirtualRegister(resultOperand), result);
+ return true;
+ }
+
+ case JSMapBucketValueIntrinsic: {
+ ASSERT(argumentCountIncludingThis == 2);
+
+ insertChecks();
+ Node* bucket = get(virtualRegisterForArgument(1, registerOffset));
+ Node* result = addToGraph(LoadValueFromMapBucket, OpInfo(), OpInfo(prediction), bucket);
set(VirtualRegister(resultOperand), result);
return true;
}
diff --git a/Source/JavaScriptCore/dfg/DFGClobberize.h b/Source/JavaScriptCore/dfg/DFGClobberize.h
index 4dbe4c5..9a4bd38 100644
--- a/Source/JavaScriptCore/dfg/DFGClobberize.h
+++ b/Source/JavaScriptCore/dfg/DFGClobberize.h
@@ -148,6 +148,10 @@
// We should enable CSE of LazyJSConstant. It's a little annoying since LazyJSValue has
// more bits than we currently have in PureValue.
return;
+
+ case CompareEqPtr:
+ def(PureValue(node, node->cellOperand()->cell()));
+ return;
case ArithIMul:
case ArithMin:
@@ -158,7 +162,6 @@
case GetGlobalObject:
case StringCharCodeAt:
case CompareStrictEq:
- case CompareEqPtr:
case IsEmpty:
case IsUndefined:
case IsBoolean:
@@ -1551,16 +1554,33 @@
def(HeapLocation(MapBucketLoc, MiscFields, mapEdge, keyEdge), LazyNode(node));
return;
}
- case LoadFromJSMapBucket: {
+ case GetMapBucketHead: {
read(MiscFields);
- Edge& bucketEdge = node->child1();
- def(HeapLocation(JSMapGetLoc, MiscFields, bucketEdge), LazyNode(node));
+ Edge& mapEdge = node->child1();
+ def(HeapLocation(MapBucketHeadLoc, MiscFields, mapEdge), LazyNode(node));
return;
}
- case IsNonEmptyMapBucket:
+ case GetMapBucketNext: {
read(MiscFields);
- def(HeapLocation(MapHasLoc, MiscFields, node->child1()), LazyNode(node));
+ LocationKind locationKind = MapBucketMapNextLoc;
+ if (node->bucketOwnerType() == BucketOwnerType::Set)
+ locationKind = MapBucketSetNextLoc;
+ Edge& bucketEdge = node->child1();
+ def(HeapLocation(locationKind, MiscFields, bucketEdge), LazyNode(node));
return;
+ }
+ case LoadKeyFromMapBucket: {
+ read(MiscFields);
+ Edge& bucketEdge = node->child1();
+ def(HeapLocation(MapBucketKeyLoc, MiscFields, bucketEdge), LazyNode(node));
+ return;
+ }
+ case LoadValueFromMapBucket: {
+ read(MiscFields);
+ Edge& bucketEdge = node->child1();
+ def(HeapLocation(MapBucketValueLoc, MiscFields, bucketEdge), LazyNode(node));
+ return;
+ }
case ToLowerCase:
def(PureValue(node));
diff --git a/Source/JavaScriptCore/dfg/DFGDoesGC.cpp b/Source/JavaScriptCore/dfg/DFGDoesGC.cpp
index 4a23de7..07dca04 100644
--- a/Source/JavaScriptCore/dfg/DFGDoesGC.cpp
+++ b/Source/JavaScriptCore/dfg/DFGDoesGC.cpp
@@ -194,8 +194,10 @@
case StringFromCharCode:
case MapHash:
case GetMapBucket:
- case LoadFromJSMapBucket:
- case IsNonEmptyMapBucket:
+ case GetMapBucketHead:
+ case GetMapBucketNext:
+ case LoadKeyFromMapBucket:
+ case LoadValueFromMapBucket:
case Unreachable:
case ExtractOSREntryLocal:
case CheckTierUpInLoop:
diff --git a/Source/JavaScriptCore/dfg/DFGFixupPhase.cpp b/Source/JavaScriptCore/dfg/DFGFixupPhase.cpp
index a450fad..daf0bd2 100644
--- a/Source/JavaScriptCore/dfg/DFGFixupPhase.cpp
+++ b/Source/JavaScriptCore/dfg/DFGFixupPhase.cpp
@@ -1795,12 +1795,19 @@
fixEdge<Int32Use>(node->child3());
break;
- case LoadFromJSMapBucket:
- fixEdge<KnownCellUse>(node->child1());
+ case GetMapBucketHead:
+ if (node->child1().useKind() == MapObjectUse)
+ fixEdge<MapObjectUse>(node->child1());
+ else if (node->child1().useKind() == SetObjectUse)
+ fixEdge<SetObjectUse>(node->child1());
+ else
+ RELEASE_ASSERT_NOT_REACHED();
break;
- case IsNonEmptyMapBucket:
- fixEdge<KnownCellUse>(node->child1());
+ case GetMapBucketNext:
+ case LoadKeyFromMapBucket:
+ case LoadValueFromMapBucket:
+ fixEdge<CellUse>(node->child1());
break;
case MapHash: {
diff --git a/Source/JavaScriptCore/dfg/DFGHeapLocation.cpp b/Source/JavaScriptCore/dfg/DFGHeapLocation.cpp
index 6a5c950..60bd572 100644
--- a/Source/JavaScriptCore/dfg/DFGHeapLocation.cpp
+++ b/Source/JavaScriptCore/dfg/DFGHeapLocation.cpp
@@ -162,11 +162,20 @@
case MapBucketLoc:
out.print("MapBucketLoc");
return;
- case JSMapGetLoc:
- out.print("JSMapGetLoc");
+ case MapBucketHeadLoc:
+ out.print("MapBucketHeadLoc");
return;
- case MapHasLoc:
- out.print("MapHasLoc");
+ case MapBucketKeyLoc:
+ out.print("MapBucketKeyLoc");
+ return;
+ case MapBucketValueLoc:
+ out.print("MapBucketValueLoc");
+ return;
+ case MapBucketMapNextLoc:
+ out.print("MapBucketMapNextLoc");
+ return;
+ case MapBucketSetNextLoc:
+ out.print("MapBucketSetNextLoc");
return;
case DOMStateLoc:
out.print("DOMStateLoc");
diff --git a/Source/JavaScriptCore/dfg/DFGHeapLocation.h b/Source/JavaScriptCore/dfg/DFGHeapLocation.h
index 7c87f20..b82c922 100644
--- a/Source/JavaScriptCore/dfg/DFGHeapLocation.h
+++ b/Source/JavaScriptCore/dfg/DFGHeapLocation.h
@@ -63,8 +63,11 @@
StackLoc,
StackPayloadLoc,
MapBucketLoc,
- JSMapGetLoc,
- MapHasLoc,
+ MapBucketHeadLoc,
+ MapBucketValueLoc,
+ MapBucketKeyLoc,
+ MapBucketMapNextLoc,
+ MapBucketSetNextLoc,
DOMStateLoc,
};
diff --git a/Source/JavaScriptCore/dfg/DFGNode.h b/Source/JavaScriptCore/dfg/DFGNode.h
index 1b3ad6b..cf4cb04 100644
--- a/Source/JavaScriptCore/dfg/DFGNode.h
+++ b/Source/JavaScriptCore/dfg/DFGNode.h
@@ -243,6 +243,11 @@
unsigned identifierNumber { 0 };
};
+enum class BucketOwnerType : uint32_t {
+ Map,
+ Set
+};
+
// === Node ===
//
// Node represents a single operation in the data flow graph.
@@ -1515,7 +1520,8 @@
case StringReplace:
case StringReplaceRegExp:
case ToNumber:
- case LoadFromJSMapBucket:
+ case LoadKeyFromMapBucket:
+ case LoadValueFromMapBucket:
case CallDOMGetter:
case CallDOM:
case ParseInt:
@@ -2544,6 +2550,17 @@
return m_opInfo.as<unsigned>();
}
+ bool hasBucketOwnerType()
+ {
+ return op() == GetMapBucketNext;
+ }
+
+ BucketOwnerType bucketOwnerType()
+ {
+ ASSERT(hasBucketOwnerType());
+ return m_opInfo.as<BucketOwnerType>();
+ }
+
void dumpChildren(PrintStream& out)
{
if (!child1())
@@ -2652,14 +2669,14 @@
return static_cast<T>(u.constPointer);
}
template <typename T>
- ALWAYS_INLINE auto as() const -> typename std::enable_if<std::is_integral<T>::value && sizeof(T) == 4, T>::type
+ ALWAYS_INLINE auto as() const -> typename std::enable_if<(std::is_integral<T>::value || std::is_enum<T>::value) && sizeof(T) == 4, T>::type
{
- return u.int32;
+ return static_cast<T>(u.int32);
}
template <typename T>
- ALWAYS_INLINE auto as() const -> typename std::enable_if<std::is_integral<T>::value && sizeof(T) == 8, T>::type
+ ALWAYS_INLINE auto as() const -> typename std::enable_if<(std::is_integral<T>::value || std::is_enum<T>::value) && sizeof(T) == 8, T>::type
{
- return u.int64;
+ return static_cast<T>(u.int64);
}
ALWAYS_INLINE RegisteredStructure asRegisteredStructure() const
{
diff --git a/Source/JavaScriptCore/dfg/DFGNodeType.h b/Source/JavaScriptCore/dfg/DFGNodeType.h
index 277d4c3..867a92c 100644
--- a/Source/JavaScriptCore/dfg/DFGNodeType.h
+++ b/Source/JavaScriptCore/dfg/DFGNodeType.h
@@ -431,8 +431,10 @@
/* Nodes for JSMap and JSSet */ \
macro(MapHash, NodeResultInt32) \
macro(GetMapBucket, NodeResultJS) \
- macro(LoadFromJSMapBucket, NodeResultJS) \
- macro(IsNonEmptyMapBucket, NodeResultBoolean) \
+ macro(GetMapBucketHead, NodeResultJS) \
+ macro(GetMapBucketNext, NodeResultJS) \
+ macro(LoadKeyFromMapBucket, NodeResultJS) \
+ macro(LoadValueFromMapBucket, NodeResultJS) \
\
macro(ToLowerCase, NodeResultJS) \
/* Nodes for DOM JIT */\
diff --git a/Source/JavaScriptCore/dfg/DFGOperations.cpp b/Source/JavaScriptCore/dfg/DFGOperations.cpp
index cbb7f4c..5c41c83 100644
--- a/Source/JavaScriptCore/dfg/DFGOperations.cpp
+++ b/Source/JavaScriptCore/dfg/DFGOperations.cpp
@@ -2299,7 +2299,7 @@
NativeCallFrameTracer tracer(&vm, exec);
JSMap::BucketType** bucket = jsCast<JSMap*>(map)->findBucket(exec, normalizeMapKey(JSValue::decode(key)), hash);
if (!bucket)
- return nullptr;
+ return vm.sentinelMapBucket.get();
return *bucket;
}
@@ -2309,7 +2309,7 @@
NativeCallFrameTracer tracer(&vm, exec);
JSSet::BucketType** bucket = jsCast<JSSet*>(map)->findBucket(exec, normalizeMapKey(JSValue::decode(key)), hash);
if (!bucket)
- return nullptr;
+ return vm.sentinelSetBucket.get();
return *bucket;
}
diff --git a/Source/JavaScriptCore/dfg/DFGPredictionPropagationPhase.cpp b/Source/JavaScriptCore/dfg/DFGPredictionPropagationPhase.cpp
index 084f726..891d5e0 100644
--- a/Source/JavaScriptCore/dfg/DFGPredictionPropagationPhase.cpp
+++ b/Source/JavaScriptCore/dfg/DFGPredictionPropagationPhase.cpp
@@ -717,7 +717,8 @@
case GetGlobalLexicalVariable:
case GetClosureVar:
case GetFromArguments:
- case LoadFromJSMapBucket:
+ case LoadKeyFromMapBucket:
+ case LoadValueFromMapBucket:
case ToNumber:
case GetArgument:
case CallDOMGetter:
@@ -756,12 +757,12 @@
case MapHash:
setPrediction(SpecInt32Only);
break;
+
case GetMapBucket:
+ case GetMapBucketHead:
+ case GetMapBucketNext:
setPrediction(SpecCellOther);
break;
- case IsNonEmptyMapBucket:
- setPrediction(SpecBoolean);
- break;
case GetRestLength:
case ArrayIndexOf: {
diff --git a/Source/JavaScriptCore/dfg/DFGSafeToExecute.h b/Source/JavaScriptCore/dfg/DFGSafeToExecute.h
index 1ab1724..1bdc59e 100644
--- a/Source/JavaScriptCore/dfg/DFGSafeToExecute.h
+++ b/Source/JavaScriptCore/dfg/DFGSafeToExecute.h
@@ -383,8 +383,10 @@
case MapHash:
case ToLowerCase:
case GetMapBucket:
- case LoadFromJSMapBucket:
- case IsNonEmptyMapBucket:
+ case GetMapBucketHead:
+ case GetMapBucketNext:
+ case LoadKeyFromMapBucket:
+ case LoadValueFromMapBucket:
case AtomicsAdd:
case AtomicsAnd:
case AtomicsCompareExchange:
diff --git a/Source/JavaScriptCore/dfg/DFGSpeculativeJIT.cpp b/Source/JavaScriptCore/dfg/DFGSpeculativeJIT.cpp
index 186a5f0..7c2c86b 100644
--- a/Source/JavaScriptCore/dfg/DFGSpeculativeJIT.cpp
+++ b/Source/JavaScriptCore/dfg/DFGSpeculativeJIT.cpp
@@ -10008,19 +10008,6 @@
noResult(node);
}
-void SpeculativeJIT::compileCompareEqPtr(Node* node)
-{
- JSValueOperand operand(this, node->child1());
- GPRTemporary result(this);
- JSValueRegs regs = operand.jsValueRegs();
- GPRReg resultGPR = result.gpr();
- m_jit.boxBooleanPayload(false, resultGPR);
- JITCompiler::JumpList notEqual = m_jit.branchIfNotEqual(regs, node->cellOperand()->value());
- m_jit.boxBooleanPayload(true, resultGPR);
- notEqual.link(&m_jit);
- blessedBooleanResult(resultGPR, node);
-}
-
void SpeculativeJIT::compileDefineDataProperty(Node* node)
{
#if USE(JSVALUE64)
@@ -10195,6 +10182,82 @@
m_jit.store32(sizeGPR, MacroAssembler::Address(storageResultGPR, Butterfly::offsetOfVectorLength()));
}
+void SpeculativeJIT::compileGetMapBucketHead(Node* node)
+{
+ SpeculateCellOperand map(this, node->child1());
+ GPRTemporary bucket(this);
+
+ GPRReg mapGPR = map.gpr();
+ GPRReg bucketGPR = bucket.gpr();
+
+ if (node->child1().useKind() == MapObjectUse)
+ speculateMapObject(node->child1(), mapGPR);
+ else if (node->child1().useKind() == SetObjectUse)
+ speculateSetObject(node->child1(), mapGPR);
+ else
+ RELEASE_ASSERT_NOT_REACHED();
+
+ ASSERT(HashMapImpl<HashMapBucket<HashMapBucketDataKey>>::offsetOfHead() == HashMapImpl<HashMapBucket<HashMapBucketDataKeyValue>>::offsetOfHead());
+ m_jit.loadPtr(MacroAssembler::Address(mapGPR, HashMapImpl<HashMapBucket<HashMapBucketDataKey>>::offsetOfHead()), bucketGPR);
+ cellResult(bucketGPR, node);
+}
+
+void SpeculativeJIT::compileGetMapBucketNext(Node* node)
+{
+ SpeculateCellOperand bucket(this, node->child1());
+ GPRTemporary result(this);
+
+ GPRReg bucketGPR = bucket.gpr();
+ GPRReg resultGPR = result.gpr();
+
+ ASSERT(HashMapBucket<HashMapBucketDataKey>::offsetOfNext() == HashMapBucket<HashMapBucketDataKeyValue>::offsetOfNext());
+ ASSERT(HashMapBucket<HashMapBucketDataKey>::offsetOfDeleted() == HashMapBucket<HashMapBucketDataKeyValue>::offsetOfDeleted());
+ m_jit.loadPtr(MacroAssembler::Address(bucketGPR, HashMapBucket<HashMapBucketDataKeyValue>::offsetOfNext()), resultGPR);
+
+ MacroAssembler::Label loop = m_jit.label();
+ auto notBucket = m_jit.branchTestPtr(MacroAssembler::Zero, resultGPR);
+ auto done = m_jit.branchTest32(MacroAssembler::Zero, MacroAssembler::Address(resultGPR, HashMapBucket<HashMapBucketDataKeyValue>::offsetOfDeleted()));
+ m_jit.loadPtr(MacroAssembler::Address(resultGPR, HashMapBucket<HashMapBucketDataKeyValue>::offsetOfNext()), resultGPR);
+ m_jit.jump().linkTo(loop, &m_jit);
+
+ notBucket.link(&m_jit);
+ JSCell* sentinel = nullptr;
+ if (node->bucketOwnerType() == BucketOwnerType::Map)
+ sentinel = m_jit.vm()->sentinelMapBucket.get();
+ else {
+ ASSERT(node->bucketOwnerType() == BucketOwnerType::Set);
+ sentinel = m_jit.vm()->sentinelSetBucket.get();
+ }
+ m_jit.move(TrustedImmPtr::weakPointer(m_jit.graph(), sentinel), resultGPR);
+ done.link(&m_jit);
+
+ cellResult(resultGPR, node);
+}
+
+void SpeculativeJIT::compileLoadKeyFromMapBucket(Node* node)
+{
+ SpeculateCellOperand bucket(this, node->child1());
+ JSValueRegsTemporary result(this);
+
+ GPRReg bucketGPR = bucket.gpr();
+ JSValueRegs resultRegs = result.regs();
+
+ m_jit.loadValue(MacroAssembler::Address(bucketGPR, HashMapBucket<HashMapBucketDataKeyValue>::offsetOfKey()), resultRegs);
+ jsValueResult(resultRegs, node);
+}
+
+void SpeculativeJIT::compileLoadValueFromMapBucket(Node* node)
+{
+ SpeculateCellOperand bucket(this, node->child1());
+ JSValueRegsTemporary result(this);
+
+ GPRReg bucketGPR = bucket.gpr();
+ JSValueRegs resultRegs = result.regs();
+
+ m_jit.loadValue(MacroAssembler::Address(bucketGPR, HashMapBucket<HashMapBucketDataKeyValue>::offsetOfValue()), resultRegs);
+ jsValueResult(resultRegs, node);
+}
+
} } // namespace JSC::DFG
#endif
diff --git a/Source/JavaScriptCore/dfg/DFGSpeculativeJIT.h b/Source/JavaScriptCore/dfg/DFGSpeculativeJIT.h
index b651328..b6122ef 100644
--- a/Source/JavaScriptCore/dfg/DFGSpeculativeJIT.h
+++ b/Source/JavaScriptCore/dfg/DFGSpeculativeJIT.h
@@ -2779,6 +2779,10 @@
void compileCallDOMGetter(Node*);
void compileCallDOM(Node*);
void compileCheckSubClass(Node*);
+ void compileGetMapBucketHead(Node*);
+ void compileGetMapBucketNext(Node*);
+ void compileLoadKeyFromMapBucket(Node*);
+ void compileLoadValueFromMapBucket(Node*);
#if USE(JSVALUE32_64)
template<typename BaseOperandType, typename PropertyOperandType, typename ValueOperandType, typename TagType>
diff --git a/Source/JavaScriptCore/dfg/DFGSpeculativeJIT32_64.cpp b/Source/JavaScriptCore/dfg/DFGSpeculativeJIT32_64.cpp
index 35501aa..1031652 100644
--- a/Source/JavaScriptCore/dfg/DFGSpeculativeJIT32_64.cpp
+++ b/Source/JavaScriptCore/dfg/DFGSpeculativeJIT32_64.cpp
@@ -692,6 +692,19 @@
booleanResult(result.gpr(), node);
}
+void SpeculativeJIT::compileCompareEqPtr(Node* node)
+{
+ JSValueOperand operand(this, node->child1());
+ GPRTemporary result(this);
+ JSValueRegs regs = operand.jsValueRegs();
+ GPRReg resultGPR = result.gpr();
+ m_jit.boxBooleanPayload(false, resultGPR);
+ JITCompiler::JumpList notEqual = m_jit.branchIfNotEqual(regs, node->cellOperand()->value());
+ m_jit.boxBooleanPayload(true, resultGPR);
+ notEqual.link(&m_jit);
+ blessedBooleanResult(resultGPR, node);
+}
+
void SpeculativeJIT::emitCall(Node* node)
{
CallLinkInfo::CallType callType;
@@ -4902,38 +4915,21 @@
break;
}
- case LoadFromJSMapBucket: {
- SpeculateCellOperand bucket(this, node->child1());
- GPRTemporary resultPayload(this);
- GPRTemporary resultTag(this);
-
- GPRReg bucketGPR = bucket.gpr();
- GPRReg resultPayloadGPR = resultPayload.gpr();
- GPRReg resultTagGPR = resultTag.gpr();
-
- auto notBucket = m_jit.branchTestPtr(MacroAssembler::Zero, bucketGPR);
- m_jit.loadValue(MacroAssembler::Address(bucketGPR, HashMapBucket<HashMapBucketDataKeyValue>::offsetOfValue()), JSValueRegs(resultTagGPR, resultPayloadGPR));
- auto done = m_jit.jump();
-
- notBucket.link(&m_jit);
- m_jit.move(TrustedImm32(JSValue::UndefinedTag), resultTagGPR);
- m_jit.move(TrustedImm32(0), resultPayloadGPR);
- done.link(&m_jit);
- jsValueResult(resultTagGPR, resultPayloadGPR, node);
+ case GetMapBucketHead:
+ compileGetMapBucketHead(node);
break;
- }
- case IsNonEmptyMapBucket: {
- SpeculateCellOperand bucket(this, node->child1());
- GPRTemporary result(this);
-
- GPRReg bucketGPR = bucket.gpr();
- GPRReg resultGPR = result.gpr();
-
- m_jit.comparePtr(MacroAssembler::NotEqual, bucketGPR, TrustedImm32(0), resultGPR);
- booleanResult(resultGPR, node);
+ case GetMapBucketNext:
+ compileGetMapBucketNext(node);
break;
- }
+
+ case LoadKeyFromMapBucket:
+ compileLoadKeyFromMapBucket(node);
+ break;
+
+ case LoadValueFromMapBucket:
+ compileLoadValueFromMapBucket(node);
+ break;
case Flush:
break;
diff --git a/Source/JavaScriptCore/dfg/DFGSpeculativeJIT64.cpp b/Source/JavaScriptCore/dfg/DFGSpeculativeJIT64.cpp
index fa63608..8326241 100644
--- a/Source/JavaScriptCore/dfg/DFGSpeculativeJIT64.cpp
+++ b/Source/JavaScriptCore/dfg/DFGSpeculativeJIT64.cpp
@@ -1836,6 +1836,18 @@
jsValueResult(result.gpr(), node, DataFormatJSBoolean);
}
+void SpeculativeJIT::compileCompareEqPtr(Node* node)
+{
+ JSValueOperand value(this, node->child1());
+ GPRTemporary result(this);
+ GPRReg valueGPR = value.gpr();
+ GPRReg resultGPR = result.gpr();
+
+ m_jit.move(TrustedImmPtr::weakPointer(m_jit.graph(), node->cellOperand()->cell()), resultGPR);
+ m_jit.compare64(MacroAssembler::Equal, valueGPR, resultGPR, resultGPR);
+ unblessedBooleanResult(resultGPR, node);
+}
+
void SpeculativeJIT::compileObjectOrOtherLogicalNot(Edge nodeUse)
{
JSValueOperand value(this, nodeUse, ManualOperandSpeculation);
@@ -5254,42 +5266,30 @@
}
notPresentInTable.link(&m_jit);
- m_jit.move(TrustedImmPtr(nullptr), resultGPR);
+ if (node->child1().useKind() == MapObjectUse)
+ m_jit.move(TrustedImmPtr::weakPointer(m_jit.graph(), m_jit.vm()->sentinelMapBucket.get()), resultGPR);
+ else
+ m_jit.move(TrustedImmPtr::weakPointer(m_jit.graph(), m_jit.vm()->sentinelSetBucket.get()), resultGPR);
done.link(&m_jit);
cellResult(resultGPR, node);
break;
}
- case LoadFromJSMapBucket: {
- SpeculateCellOperand bucket(this, node->child1());
- GPRTemporary result(this);
-
- GPRReg bucketGPR = bucket.gpr();
- GPRReg resultGPR = result.gpr();
-
- auto notBucket = m_jit.branchTestPtr(MacroAssembler::Zero, bucketGPR);
- m_jit.load64(MacroAssembler::Address(bucketGPR, HashMapBucket<HashMapBucketDataKeyValue>::offsetOfValue()), resultGPR);
- auto done = m_jit.jump();
-
- notBucket.link(&m_jit);
- m_jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsUndefined())), resultGPR);
- done.link(&m_jit);
- jsValueResult(resultGPR, node);
+ case GetMapBucketHead:
+ compileGetMapBucketHead(node);
break;
- }
- case IsNonEmptyMapBucket: {
- SpeculateCellOperand bucket(this, node->child1());
- GPRTemporary result(this);
-
- GPRReg bucketGPR = bucket.gpr();
- GPRReg resultGPR = result.gpr();
-
- m_jit.comparePtr(MacroAssembler::NotEqual, bucketGPR, TrustedImm32(0), resultGPR);
- m_jit.or32(TrustedImm32(ValueFalse), resultGPR);
- jsValueResult(resultGPR, node, DataFormatJSBoolean);
+ case GetMapBucketNext:
+ compileGetMapBucketNext(node);
break;
- }
+
+ case LoadKeyFromMapBucket:
+ compileLoadKeyFromMapBucket(node);
+ break;
+
+ case LoadValueFromMapBucket:
+ compileLoadValueFromMapBucket(node);
+ break;
case ToLowerCase: {
compileToLowerCase(node);
