[ES6] Implement tail calls in the LLInt and Baseline JIT
https://bugs.webkit.org/show_bug.cgi?id=148661
Fix for the breakage of Speedometer/Full.html (https://bugs.webkit.org/show_bug.cgi?id=149162).
Reviewed by Filip Pizlo.
Changed SetupVarargsFrame.cpp::emitSetVarargsFrame to align the callframe size to be a
multiple of stackAlignmentRegisters() in addition to the location of the new frame.
Fixed Reviewed by Filip Pizlo.
* CMakeLists.txt:
* JavaScriptCore.vcxproj/JavaScriptCore.vcxproj:
* JavaScriptCore.vcxproj/JavaScriptCore.vcxproj.filters:
* JavaScriptCore.xcodeproj/project.pbxproj:
* assembler/AbortReason.h:
* assembler/AbstractMacroAssembler.h:
(JSC::AbstractMacroAssembler::Call::Call):
(JSC::AbstractMacroAssembler::repatchNearCall):
(JSC::AbstractMacroAssembler::repatchCompact):
* assembler/CodeLocation.h:
(JSC::CodeLocationNearCall::CodeLocationNearCall):
(JSC::CodeLocationNearCall::callMode):
(JSC::CodeLocationCommon::callAtOffset):
(JSC::CodeLocationCommon::nearCallAtOffset):
(JSC::CodeLocationCommon::dataLabelPtrAtOffset):
* assembler/LinkBuffer.h:
(JSC::LinkBuffer::locationOfNearCall):
(JSC::LinkBuffer::locationOf):
* assembler/MacroAssemblerARM.h:
(JSC::MacroAssemblerARM::nearCall):
(JSC::MacroAssemblerARM::nearTailCall):
(JSC::MacroAssemblerARM::call):
(JSC::MacroAssemblerARM::linkCall):
* assembler/MacroAssemblerARM64.h:
(JSC::MacroAssemblerARM64::nearCall):
(JSC::MacroAssemblerARM64::nearTailCall):
(JSC::MacroAssemblerARM64::ret):
(JSC::MacroAssemblerARM64::linkCall):
* assembler/MacroAssemblerARMv7.h:
(JSC::MacroAssemblerARMv7::nearCall):
(JSC::MacroAssemblerARMv7::nearTailCall):
(JSC::MacroAssemblerARMv7::call):
(JSC::MacroAssemblerARMv7::linkCall):
* assembler/MacroAssemblerMIPS.h:
(JSC::MacroAssemblerMIPS::nearCall):
(JSC::MacroAssemblerMIPS::nearTailCall):
(JSC::MacroAssemblerMIPS::call):
(JSC::MacroAssemblerMIPS::linkCall):
(JSC::MacroAssemblerMIPS::repatchCall):
* assembler/MacroAssemblerSH4.h:
(JSC::MacroAssemblerSH4::call):
(JSC::MacroAssemblerSH4::nearTailCall):
(JSC::MacroAssemblerSH4::nearCall):
(JSC::MacroAssemblerSH4::linkCall):
(JSC::MacroAssemblerSH4::repatchCall):
* assembler/MacroAssemblerX86.h:
(JSC::MacroAssemblerX86::linkCall):
* assembler/MacroAssemblerX86Common.h:
(JSC::MacroAssemblerX86Common::breakpoint):
(JSC::MacroAssemblerX86Common::nearTailCall):
(JSC::MacroAssemblerX86Common::nearCall):
* assembler/MacroAssemblerX86_64.h:
(JSC::MacroAssemblerX86_64::linkCall):
* bytecode/BytecodeList.json:
* bytecode/BytecodeUseDef.h:
(JSC::computeUsesForBytecodeOffset):
(JSC::computeDefsForBytecodeOffset):
* bytecode/CallLinkInfo.h:
(JSC::CallLinkInfo::callTypeFor):
(JSC::CallLinkInfo::isVarargsCallType):
(JSC::CallLinkInfo::CallLinkInfo):
(JSC::CallLinkInfo::specializationKind):
(JSC::CallLinkInfo::callModeFor):
(JSC::CallLinkInfo::callMode):
(JSC::CallLinkInfo::isTailCall):
(JSC::CallLinkInfo::isVarargs):
(JSC::CallLinkInfo::registerPreservationMode):
* bytecode/CallLinkStatus.cpp:
(JSC::CallLinkStatus::computeFromLLInt):
* bytecode/CodeBlock.cpp:
(JSC::CodeBlock::dumpBytecode):
(JSC::CodeBlock::CodeBlock):
* bytecompiler/BytecodeGenerator.cpp:
(JSC::BytecodeGenerator::BytecodeGenerator):
(JSC::BytecodeGenerator::emitCallInTailPosition):
(JSC::BytecodeGenerator::emitCallEval):
(JSC::BytecodeGenerator::emitCall):
(JSC::BytecodeGenerator::emitCallVarargsInTailPosition):
(JSC::BytecodeGenerator::emitConstructVarargs):
* bytecompiler/NodesCodegen.cpp:
(JSC::CallArguments::CallArguments):
(JSC::LabelNode::emitBytecode):
* dfg/DFGByteCodeParser.cpp:
(JSC::DFG::ByteCodeParser::addCallWithoutSettingResult):
* ftl/FTLLowerDFGToLLVM.cpp:
(JSC::FTL::DFG::LowerDFGToLLVM::compileCallOrConstruct):
* interpreter/Interpreter.h:
(JSC::Interpreter::isCallBytecode):
(JSC::calleeFrameForVarargs):
* jit/CCallHelpers.h:
(JSC::CCallHelpers::jumpToExceptionHandler):
(JSC::CCallHelpers::prepareForTailCallSlow):
* jit/JIT.cpp:
(JSC::JIT::privateCompileMainPass):
(JSC::JIT::privateCompileSlowCases):
* jit/JIT.h:
* jit/JITCall.cpp:
(JSC::JIT::compileOpCall):
(JSC::JIT::compileOpCallSlowCase):
(JSC::JIT::emit_op_call):
(JSC::JIT::emit_op_tail_call):
(JSC::JIT::emit_op_call_eval):
(JSC::JIT::emit_op_call_varargs):
(JSC::JIT::emit_op_tail_call_varargs):
(JSC::JIT::emit_op_construct_varargs):
(JSC::JIT::emitSlow_op_call):
(JSC::JIT::emitSlow_op_tail_call):
(JSC::JIT::emitSlow_op_call_eval):
(JSC::JIT::emitSlow_op_call_varargs):
(JSC::JIT::emitSlow_op_tail_call_varargs):
(JSC::JIT::emitSlow_op_construct_varargs):
* jit/JITCall32_64.cpp:
(JSC::JIT::emitSlow_op_call):
(JSC::JIT::emitSlow_op_tail_call):
(JSC::JIT::emitSlow_op_call_eval):
(JSC::JIT::emitSlow_op_call_varargs):
(JSC::JIT::emitSlow_op_tail_call_varargs):
(JSC::JIT::emitSlow_op_construct_varargs):
(JSC::JIT::emit_op_call):
(JSC::JIT::emit_op_tail_call):
(JSC::JIT::emit_op_call_eval):
(JSC::JIT::emit_op_call_varargs):
(JSC::JIT::emit_op_tail_call_varargs):
(JSC::JIT::emit_op_construct_varargs):
(JSC::JIT::compileOpCall):
(JSC::JIT::compileOpCallSlowCase):
* jit/JITInlines.h:
(JSC::JIT::emitNakedCall):
(JSC::JIT::emitNakedTailCall):
(JSC::JIT::updateTopCallFrame):
* jit/JITOperations.cpp:
* jit/JITOperations.h:
* jit/Repatch.cpp:
(JSC::linkVirtualFor):
(JSC::linkPolymorphicCall):
* jit/SetupVarargsFrame.cpp:
(JSC::emitSetVarargsFrame):
* jit/ThunkGenerators.cpp:
(JSC::throwExceptionFromCallSlowPathGenerator):
(JSC::slowPathFor):
(JSC::linkCallThunkGenerator):
(JSC::virtualThunkFor):
(JSC::arityFixupGenerator):
(JSC::unreachableGenerator):
(JSC::baselineGetterReturnThunkGenerator):
* jit/ThunkGenerators.h:
* llint/LowLevelInterpreter.asm:
* llint/LowLevelInterpreter32_64.asm:
* llint/LowLevelInterpreter64.asm:
* runtime/CommonSlowPaths.h:
(JSC::CommonSlowPaths::arityCheckFor):
(JSC::CommonSlowPaths::opIn):
git-svn-id: http://svn.webkit.org/repository/webkit/trunk@189884 268f45cc-cd09-0410-ab3c-d52691b4dbfc
diff --git a/Source/JavaScriptCore/CMakeLists.txt b/Source/JavaScriptCore/CMakeLists.txt
index a6ce71c..adb3f3c 100644
--- a/Source/JavaScriptCore/CMakeLists.txt
+++ b/Source/JavaScriptCore/CMakeLists.txt
@@ -80,6 +80,7 @@
bytecode/CallEdge.cpp
bytecode/CallLinkInfo.cpp
bytecode/CallLinkStatus.cpp
+ bytecode/CallMode.cpp
bytecode/CallVariant.cpp
bytecode/CodeBlock.cpp
bytecode/CodeBlockHash.cpp
diff --git a/Source/JavaScriptCore/ChangeLog b/Source/JavaScriptCore/ChangeLog
index 1688088..71a2e9d 100644
--- a/Source/JavaScriptCore/ChangeLog
+++ b/Source/JavaScriptCore/ChangeLog
@@ -1,3 +1,169 @@
+2015-09-16 Michael Saboff <msaboff@apple.com>
+
+ [ES6] Implement tail calls in the LLInt and Baseline JIT
+ https://bugs.webkit.org/show_bug.cgi?id=148661
+
+ Fix for the breakage of Speedometer/Full.html (https://bugs.webkit.org/show_bug.cgi?id=149162).
+
+ Reviewed by Filip Pizlo.
+ Changed SetupVarargsFrame.cpp::emitSetVarargsFrame to align the callframe size to be a
+ multiple of stackAlignmentRegisters() in addition to the location of the new frame.
+
+ Fixed Reviewed by Filip Pizlo.
+
+ * CMakeLists.txt:
+ * JavaScriptCore.vcxproj/JavaScriptCore.vcxproj:
+ * JavaScriptCore.vcxproj/JavaScriptCore.vcxproj.filters:
+ * JavaScriptCore.xcodeproj/project.pbxproj:
+ * assembler/AbortReason.h:
+ * assembler/AbstractMacroAssembler.h:
+ (JSC::AbstractMacroAssembler::Call::Call):
+ (JSC::AbstractMacroAssembler::repatchNearCall):
+ (JSC::AbstractMacroAssembler::repatchCompact):
+ * assembler/CodeLocation.h:
+ (JSC::CodeLocationNearCall::CodeLocationNearCall):
+ (JSC::CodeLocationNearCall::callMode):
+ (JSC::CodeLocationCommon::callAtOffset):
+ (JSC::CodeLocationCommon::nearCallAtOffset):
+ (JSC::CodeLocationCommon::dataLabelPtrAtOffset):
+ * assembler/LinkBuffer.h:
+ (JSC::LinkBuffer::locationOfNearCall):
+ (JSC::LinkBuffer::locationOf):
+ * assembler/MacroAssemblerARM.h:
+ (JSC::MacroAssemblerARM::nearCall):
+ (JSC::MacroAssemblerARM::nearTailCall):
+ (JSC::MacroAssemblerARM::call):
+ (JSC::MacroAssemblerARM::linkCall):
+ * assembler/MacroAssemblerARM64.h:
+ (JSC::MacroAssemblerARM64::nearCall):
+ (JSC::MacroAssemblerARM64::nearTailCall):
+ (JSC::MacroAssemblerARM64::ret):
+ (JSC::MacroAssemblerARM64::linkCall):
+ * assembler/MacroAssemblerARMv7.h:
+ (JSC::MacroAssemblerARMv7::nearCall):
+ (JSC::MacroAssemblerARMv7::nearTailCall):
+ (JSC::MacroAssemblerARMv7::call):
+ (JSC::MacroAssemblerARMv7::linkCall):
+ * assembler/MacroAssemblerMIPS.h:
+ (JSC::MacroAssemblerMIPS::nearCall):
+ (JSC::MacroAssemblerMIPS::nearTailCall):
+ (JSC::MacroAssemblerMIPS::call):
+ (JSC::MacroAssemblerMIPS::linkCall):
+ (JSC::MacroAssemblerMIPS::repatchCall):
+ * assembler/MacroAssemblerSH4.h:
+ (JSC::MacroAssemblerSH4::call):
+ (JSC::MacroAssemblerSH4::nearTailCall):
+ (JSC::MacroAssemblerSH4::nearCall):
+ (JSC::MacroAssemblerSH4::linkCall):
+ (JSC::MacroAssemblerSH4::repatchCall):
+ * assembler/MacroAssemblerX86.h:
+ (JSC::MacroAssemblerX86::linkCall):
+ * assembler/MacroAssemblerX86Common.h:
+ (JSC::MacroAssemblerX86Common::breakpoint):
+ (JSC::MacroAssemblerX86Common::nearTailCall):
+ (JSC::MacroAssemblerX86Common::nearCall):
+ * assembler/MacroAssemblerX86_64.h:
+ (JSC::MacroAssemblerX86_64::linkCall):
+ * bytecode/BytecodeList.json:
+ * bytecode/BytecodeUseDef.h:
+ (JSC::computeUsesForBytecodeOffset):
+ (JSC::computeDefsForBytecodeOffset):
+ * bytecode/CallLinkInfo.h:
+ (JSC::CallLinkInfo::callTypeFor):
+ (JSC::CallLinkInfo::isVarargsCallType):
+ (JSC::CallLinkInfo::CallLinkInfo):
+ (JSC::CallLinkInfo::specializationKind):
+ (JSC::CallLinkInfo::callModeFor):
+ (JSC::CallLinkInfo::callMode):
+ (JSC::CallLinkInfo::isTailCall):
+ (JSC::CallLinkInfo::isVarargs):
+ (JSC::CallLinkInfo::registerPreservationMode):
+ * bytecode/CallLinkStatus.cpp:
+ (JSC::CallLinkStatus::computeFromLLInt):
+ * bytecode/CodeBlock.cpp:
+ (JSC::CodeBlock::dumpBytecode):
+ (JSC::CodeBlock::CodeBlock):
+ * bytecompiler/BytecodeGenerator.cpp:
+ (JSC::BytecodeGenerator::BytecodeGenerator):
+ (JSC::BytecodeGenerator::emitCallInTailPosition):
+ (JSC::BytecodeGenerator::emitCallEval):
+ (JSC::BytecodeGenerator::emitCall):
+ (JSC::BytecodeGenerator::emitCallVarargsInTailPosition):
+ (JSC::BytecodeGenerator::emitConstructVarargs):
+ * bytecompiler/NodesCodegen.cpp:
+ (JSC::CallArguments::CallArguments):
+ (JSC::LabelNode::emitBytecode):
+ * dfg/DFGByteCodeParser.cpp:
+ (JSC::DFG::ByteCodeParser::addCallWithoutSettingResult):
+ * ftl/FTLLowerDFGToLLVM.cpp:
+ (JSC::FTL::DFG::LowerDFGToLLVM::compileCallOrConstruct):
+ * interpreter/Interpreter.h:
+ (JSC::Interpreter::isCallBytecode):
+ (JSC::calleeFrameForVarargs):
+ * jit/CCallHelpers.h:
+ (JSC::CCallHelpers::jumpToExceptionHandler):
+ (JSC::CCallHelpers::prepareForTailCallSlow):
+ * jit/JIT.cpp:
+ (JSC::JIT::privateCompileMainPass):
+ (JSC::JIT::privateCompileSlowCases):
+ * jit/JIT.h:
+ * jit/JITCall.cpp:
+ (JSC::JIT::compileOpCall):
+ (JSC::JIT::compileOpCallSlowCase):
+ (JSC::JIT::emit_op_call):
+ (JSC::JIT::emit_op_tail_call):
+ (JSC::JIT::emit_op_call_eval):
+ (JSC::JIT::emit_op_call_varargs):
+ (JSC::JIT::emit_op_tail_call_varargs):
+ (JSC::JIT::emit_op_construct_varargs):
+ (JSC::JIT::emitSlow_op_call):
+ (JSC::JIT::emitSlow_op_tail_call):
+ (JSC::JIT::emitSlow_op_call_eval):
+ (JSC::JIT::emitSlow_op_call_varargs):
+ (JSC::JIT::emitSlow_op_tail_call_varargs):
+ (JSC::JIT::emitSlow_op_construct_varargs):
+ * jit/JITCall32_64.cpp:
+ (JSC::JIT::emitSlow_op_call):
+ (JSC::JIT::emitSlow_op_tail_call):
+ (JSC::JIT::emitSlow_op_call_eval):
+ (JSC::JIT::emitSlow_op_call_varargs):
+ (JSC::JIT::emitSlow_op_tail_call_varargs):
+ (JSC::JIT::emitSlow_op_construct_varargs):
+ (JSC::JIT::emit_op_call):
+ (JSC::JIT::emit_op_tail_call):
+ (JSC::JIT::emit_op_call_eval):
+ (JSC::JIT::emit_op_call_varargs):
+ (JSC::JIT::emit_op_tail_call_varargs):
+ (JSC::JIT::emit_op_construct_varargs):
+ (JSC::JIT::compileOpCall):
+ (JSC::JIT::compileOpCallSlowCase):
+ * jit/JITInlines.h:
+ (JSC::JIT::emitNakedCall):
+ (JSC::JIT::emitNakedTailCall):
+ (JSC::JIT::updateTopCallFrame):
+ * jit/JITOperations.cpp:
+ * jit/JITOperations.h:
+ * jit/Repatch.cpp:
+ (JSC::linkVirtualFor):
+ (JSC::linkPolymorphicCall):
+ * jit/SetupVarargsFrame.cpp:
+ (JSC::emitSetVarargsFrame):
+ * jit/ThunkGenerators.cpp:
+ (JSC::throwExceptionFromCallSlowPathGenerator):
+ (JSC::slowPathFor):
+ (JSC::linkCallThunkGenerator):
+ (JSC::virtualThunkFor):
+ (JSC::arityFixupGenerator):
+ (JSC::unreachableGenerator):
+ (JSC::baselineGetterReturnThunkGenerator):
+ * jit/ThunkGenerators.h:
+ * llint/LowLevelInterpreter.asm:
+ * llint/LowLevelInterpreter32_64.asm:
+ * llint/LowLevelInterpreter64.asm:
+ * runtime/CommonSlowPaths.h:
+ (JSC::CommonSlowPaths::arityCheckFor):
+ (JSC::CommonSlowPaths::opIn):
+
2015-09-15 Michael Saboff <msaboff@apple.com>
Rollout r189774 and 189818.
diff --git a/Source/JavaScriptCore/JavaScriptCore.vcxproj/JavaScriptCore.vcxproj b/Source/JavaScriptCore/JavaScriptCore.vcxproj/JavaScriptCore.vcxproj
index bc7d07e..7ebdfc7 100644
--- a/Source/JavaScriptCore/JavaScriptCore.vcxproj/JavaScriptCore.vcxproj
+++ b/Source/JavaScriptCore/JavaScriptCore.vcxproj/JavaScriptCore.vcxproj
@@ -319,6 +319,7 @@
<ClCompile Include="..\bytecode\CallEdge.cpp" />
<ClCompile Include="..\bytecode\CallLinkInfo.cpp" />
<ClCompile Include="..\bytecode\CallLinkStatus.cpp" />
+ <ClCompile Include="..\bytecode\CallMode.cpp" />
<ClCompile Include="..\bytecode\CallVariant.cpp" />
<ClCompile Include="..\bytecode\CodeBlock.cpp" />
<ClCompile Include="..\bytecode\CodeBlockHash.cpp" />
@@ -1012,6 +1013,7 @@
<ClInclude Include="..\bytecode\CallEdge.h" />
<ClInclude Include="..\bytecode\CallLinkInfo.h" />
<ClInclude Include="..\bytecode\CallLinkStatus.h" />
+ <ClInclude Include="..\bytecode\CallMode.h" />
<ClInclude Include="..\bytecode\CallReturnOffsetToBytecodeOffset.h" />
<ClInclude Include="..\bytecode\CallVariant.h" />
<ClInclude Include="..\bytecode\CodeBlock.h" />
diff --git a/Source/JavaScriptCore/JavaScriptCore.vcxproj/JavaScriptCore.vcxproj.filters b/Source/JavaScriptCore/JavaScriptCore.vcxproj/JavaScriptCore.vcxproj.filters
index 56816ed..d5c6e28 100644
--- a/Source/JavaScriptCore/JavaScriptCore.vcxproj/JavaScriptCore.vcxproj.filters
+++ b/Source/JavaScriptCore/JavaScriptCore.vcxproj/JavaScriptCore.vcxproj.filters
@@ -162,6 +162,9 @@
<ClCompile Include="..\bytecode\CallLinkStatus.cpp">
<Filter>bytecode</Filter>
</ClCompile>
+ <ClCompile Include="..\bytecode\CallMode.cpp">
+ <Filter>bytecode</Filter>
+ </ClCompile>
<ClCompile Include="..\bytecode\CodeBlock.cpp">
<Filter>bytecode</Filter>
</ClCompile>
@@ -2034,6 +2037,9 @@
<ClInclude Include="..\bytecode\CallLinkStatus.h">
<Filter>bytecode</Filter>
</ClInclude>
+ <ClInclude Include="..\bytecode\CallMode.h">
+ <Filter>bytecode</Filter>
+ </ClInclude>
<ClInclude Include="..\bytecode\CallReturnOffsetToBytecodeOffset.h">
<Filter>bytecode</Filter>
</ClInclude>
diff --git a/Source/JavaScriptCore/JavaScriptCore.xcodeproj/project.pbxproj b/Source/JavaScriptCore/JavaScriptCore.xcodeproj/project.pbxproj
index c7a254c..ac2ea4b 100644
--- a/Source/JavaScriptCore/JavaScriptCore.xcodeproj/project.pbxproj
+++ b/Source/JavaScriptCore/JavaScriptCore.xcodeproj/project.pbxproj
@@ -969,6 +969,8 @@
5DBB1525131D0BD70056AD36 /* minidom.js in Copy Support Script */ = {isa = PBXBuildFile; fileRef = 1412110D0A48788700480255 /* minidom.js */; };
5DE6E5B30E1728EC00180407 /* create_hash_table in Headers */ = {isa = PBXBuildFile; fileRef = F692A8540255597D01FF60F7 /* create_hash_table */; settings = {ATTRIBUTES = (); }; };
623A37EC1B87A7C000754209 /* RegisterMap.h in Headers */ = {isa = PBXBuildFile; fileRef = 623A37EB1B87A7BD00754209 /* RegisterMap.h */; };
+ 627673231B680C1E00FD9F2E /* CallMode.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 627673211B680C1E00FD9F2E /* CallMode.cpp */; };
+ 627673241B680C1E00FD9F2E /* CallMode.h in Headers */ = {isa = PBXBuildFile; fileRef = 627673221B680C1E00FD9F2E /* CallMode.h */; settings = {ATTRIBUTES = (Private, ); }; };
62D2D38F1ADF103F000206C1 /* FunctionRareData.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 62D2D38D1ADF103F000206C1 /* FunctionRareData.cpp */; };
62D2D3901ADF103F000206C1 /* FunctionRareData.h in Headers */ = {isa = PBXBuildFile; fileRef = 62D2D38E1ADF103F000206C1 /* FunctionRareData.h */; settings = {ATTRIBUTES = (Private, ); }; };
62E3D5F01B8D0B7300B868BB /* DataFormat.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 62E3D5EF1B8D0B7300B868BB /* DataFormat.cpp */; };
@@ -2772,6 +2774,8 @@
5DDDF44614FEE72200B4FB4D /* LLIntDesiredOffsets.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = LLIntDesiredOffsets.h; path = LLIntOffsets/LLIntDesiredOffsets.h; sourceTree = BUILT_PRODUCTS_DIR; };
5DE3D0F40DD8DDFB00468714 /* WebKitAvailability.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = WebKitAvailability.h; sourceTree = "<group>"; };
623A37EB1B87A7BD00754209 /* RegisterMap.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = RegisterMap.h; sourceTree = "<group>"; };
+ 627673211B680C1E00FD9F2E /* CallMode.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = CallMode.cpp; sourceTree = "<group>"; };
+ 627673221B680C1E00FD9F2E /* CallMode.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = CallMode.h; sourceTree = "<group>"; };
62A9A29E1B0BED4800BD54CA /* DFGLazyNode.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = DFGLazyNode.cpp; path = dfg/DFGLazyNode.cpp; sourceTree = "<group>"; };
62A9A29F1B0BED4800BD54CA /* DFGLazyNode.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = DFGLazyNode.h; path = dfg/DFGLazyNode.h; sourceTree = "<group>"; };
62D2D38D1ADF103F000206C1 /* FunctionRareData.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = FunctionRareData.cpp; sourceTree = "<group>"; };
@@ -5538,6 +5542,8 @@
0F0B83AF14BCF71400885B4F /* CallLinkInfo.h */,
0F93329314CA7DC10085F3C6 /* CallLinkStatus.cpp */,
0F93329414CA7DC10085F3C6 /* CallLinkStatus.h */,
+ 627673211B680C1E00FD9F2E /* CallMode.cpp */,
+ 627673221B680C1E00FD9F2E /* CallMode.h */,
0F0B83B814BCF95B00885B4F /* CallReturnOffsetToBytecodeOffset.h */,
0F3B7E2419A11B8000D9BC56 /* CallVariant.cpp */,
0F3B7E2519A11B8000D9BC56 /* CallVariant.h */,
@@ -6190,6 +6196,7 @@
E33B3E261B7ABD750048DB2E /* InspectorInstrumentationObject.lut.h in Headers */,
A7D89CFE17A0B8CC00773AD8 /* DFGOSRAvailabilityAnalysisPhase.h in Headers */,
0FD82E57141DAF1000179C94 /* DFGOSREntry.h in Headers */,
+ 627673241B680C1E00FD9F2E /* CallMode.h in Headers */,
0FD8A32617D51F5700CA2C40 /* DFGOSREntrypointCreationPhase.h in Headers */,
62F2AA381B0BEDE300610C7A /* DFGLazyNode.h in Headers */,
0FC0976A1468A6F700CF2442 /* DFGOSRExit.h in Headers */,
@@ -7971,6 +7978,7 @@
0F2D4DEF19832DD3007D4B19 /* TypeSet.cpp in Sources */,
9335F24D12E6765B002B5553 /* StringRecursionChecker.cpp in Sources */,
BCDE3B430E6C832D001453A7 /* Structure.cpp in Sources */,
+ 627673231B680C1E00FD9F2E /* CallMode.cpp in Sources */,
7E4EE70F0EBB7A5B005934AA /* StructureChain.cpp in Sources */,
C2F0F2D116BAEEE900187C19 /* StructureRareData.cpp in Sources */,
0FD3E40B1B618B6600C80E1E /* ObjectPropertyConditionSet.cpp in Sources */,
diff --git a/Source/JavaScriptCore/assembler/AbortReason.h b/Source/JavaScriptCore/assembler/AbortReason.h
index cba1f54..692ad6e 100644
--- a/Source/JavaScriptCore/assembler/AbortReason.h
+++ b/Source/JavaScriptCore/assembler/AbortReason.h
@@ -58,6 +58,7 @@
DFGUnreachableBasicBlock = 220,
DFGUnreasonableOSREntryJumpDestination = 230,
DFGVarargsThrowingPathDidNotThrow = 235,
+ JITDidReturnFromTailCall = 237,
JITDivOperandsAreNotNumbers = 240,
JITGetByValResultIsNotEmpty = 250,
JITNotSupported = 260,
diff --git a/Source/JavaScriptCore/assembler/AbstractMacroAssembler.h b/Source/JavaScriptCore/assembler/AbstractMacroAssembler.h
index 703ca03..01974ef 100644
--- a/Source/JavaScriptCore/assembler/AbstractMacroAssembler.h
+++ b/Source/JavaScriptCore/assembler/AbstractMacroAssembler.h
@@ -507,7 +507,9 @@
None = 0x0,
Linkable = 0x1,
Near = 0x2,
+ Tail = 0x4,
LinkableNear = 0x3,
+ LinkableNearTail = 0x7,
};
Call()
@@ -962,7 +964,15 @@
static void repatchNearCall(CodeLocationNearCall nearCall, CodeLocationLabel destination)
{
- AssemblerType::relinkCall(nearCall.dataLocation(), destination.executableAddress());
+ switch (nearCall.callMode()) {
+ case NearCallMode::Tail:
+ AssemblerType::relinkJump(nearCall.dataLocation(), destination.executableAddress());
+ return;
+ case NearCallMode::Regular:
+ AssemblerType::relinkCall(nearCall.dataLocation(), destination.executableAddress());
+ return;
+ }
+ RELEASE_ASSERT_NOT_REACHED();
}
static void repatchCompact(CodeLocationDataLabelCompact dataLabelCompact, int32_t value)
diff --git a/Source/JavaScriptCore/assembler/CodeLocation.h b/Source/JavaScriptCore/assembler/CodeLocation.h
index 86d1f2b..3116e06 100644
--- a/Source/JavaScriptCore/assembler/CodeLocation.h
+++ b/Source/JavaScriptCore/assembler/CodeLocation.h
@@ -32,6 +32,8 @@
namespace JSC {
+enum NearCallMode { Regular, Tail };
+
class CodeLocationInstruction;
class CodeLocationLabel;
class CodeLocationJump;
@@ -59,7 +61,7 @@
CodeLocationLabel labelAtOffset(int offset);
CodeLocationJump jumpAtOffset(int offset);
CodeLocationCall callAtOffset(int offset);
- CodeLocationNearCall nearCallAtOffset(int offset);
+ CodeLocationNearCall nearCallAtOffset(int offset, NearCallMode);
CodeLocationDataLabelPtr dataLabelPtrAtOffset(int offset);
CodeLocationDataLabel32 dataLabel32AtOffset(int offset);
CodeLocationDataLabelCompact dataLabelCompactAtOffset(int offset);
@@ -115,10 +117,13 @@
class CodeLocationNearCall : public CodeLocationCommon {
public:
CodeLocationNearCall() {}
- explicit CodeLocationNearCall(MacroAssemblerCodePtr location)
- : CodeLocationCommon(location) {}
- explicit CodeLocationNearCall(void* location)
- : CodeLocationCommon(MacroAssemblerCodePtr(location)) {}
+ explicit CodeLocationNearCall(MacroAssemblerCodePtr location, NearCallMode callMode)
+ : CodeLocationCommon(location), m_callMode(callMode) { }
+ explicit CodeLocationNearCall(void* location, NearCallMode callMode)
+ : CodeLocationCommon(MacroAssemblerCodePtr(location)), m_callMode(callMode) { }
+ NearCallMode callMode() { return m_callMode; }
+private:
+ NearCallMode m_callMode = NearCallMode::Regular;
};
class CodeLocationDataLabel32 : public CodeLocationCommon {
@@ -181,10 +186,10 @@
return CodeLocationCall(reinterpret_cast<char*>(dataLocation()) + offset);
}
-inline CodeLocationNearCall CodeLocationCommon::nearCallAtOffset(int offset)
+inline CodeLocationNearCall CodeLocationCommon::nearCallAtOffset(int offset, NearCallMode callMode)
{
ASSERT_VALID_CODE_OFFSET(offset);
- return CodeLocationNearCall(reinterpret_cast<char*>(dataLocation()) + offset);
+ return CodeLocationNearCall(reinterpret_cast<char*>(dataLocation()) + offset, callMode);
}
inline CodeLocationDataLabelPtr CodeLocationCommon::dataLabelPtrAtOffset(int offset)
diff --git a/Source/JavaScriptCore/assembler/LinkBuffer.h b/Source/JavaScriptCore/assembler/LinkBuffer.h
index 9b0d4c4..b34a6f1 100644
--- a/Source/JavaScriptCore/assembler/LinkBuffer.h
+++ b/Source/JavaScriptCore/assembler/LinkBuffer.h
@@ -180,7 +180,8 @@
{
ASSERT(call.isFlagSet(Call::Linkable));
ASSERT(call.isFlagSet(Call::Near));
- return CodeLocationNearCall(MacroAssembler::getLinkerAddress(code(), applyOffset(call.m_label)));
+ return CodeLocationNearCall(MacroAssembler::getLinkerAddress(code(), applyOffset(call.m_label)),
+ call.isFlagSet(Call::Tail) ? NearCallMode::Tail : NearCallMode::Regular);
}
CodeLocationLabel locationOf(PatchableJump jump)
diff --git a/Source/JavaScriptCore/assembler/MacroAssemblerARM.h b/Source/JavaScriptCore/assembler/MacroAssemblerARM.h
index 91fef4d..d2b411f 100644
--- a/Source/JavaScriptCore/assembler/MacroAssemblerARM.h
+++ b/Source/JavaScriptCore/assembler/MacroAssemblerARM.h
@@ -904,6 +904,11 @@
return Call(m_assembler.blx(ARMRegisters::S1), Call::LinkableNear);
}
+ Call nearTailCall()
+ {
+ return Call(m_assembler.jmp(), Call::LinkableNearTail);
+ }
+
Call call(RegisterID target)
{
return Call(m_assembler.blx(target), Call::None);
@@ -1488,7 +1493,10 @@
static void linkCall(void* code, Call call, FunctionPtr function)
{
- ARMAssembler::linkCall(code, call.m_label, function.value());
+ if (call.isFlagSet(Call::Tail))
+ ARMAssembler::linkJump(code, call.m_label, function.value());
+ else
+ ARMAssembler::linkCall(code, call.m_label, function.value());
}
diff --git a/Source/JavaScriptCore/assembler/MacroAssemblerARM64.h b/Source/JavaScriptCore/assembler/MacroAssemblerARM64.h
index 2cdcd179..7b17207 100644
--- a/Source/JavaScriptCore/assembler/MacroAssemblerARM64.h
+++ b/Source/JavaScriptCore/assembler/MacroAssemblerARM64.h
@@ -2204,6 +2204,13 @@
return Call(m_assembler.label(), Call::LinkableNear);
}
+ ALWAYS_INLINE Call nearTailCall()
+ {
+ AssemblerLabel label = m_assembler.label();
+ m_assembler.b();
+ return Call(label, Call::LinkableNearTail);
+ }
+
ALWAYS_INLINE void ret()
{
m_assembler.ret();
@@ -2882,10 +2889,12 @@
static void linkCall(void* code, Call call, FunctionPtr function)
{
- if (call.isFlagSet(Call::Near))
- ARM64Assembler::linkCall(code, call.m_label, function.value());
- else
+ if (!call.isFlagSet(Call::Near))
ARM64Assembler::linkPointer(code, call.m_label.labelAtOffset(REPATCH_OFFSET_CALL_TO_POINTER), function.value());
+ else if (call.isFlagSet(Call::Tail))
+ ARM64Assembler::linkJump(code, call.m_label, function.value());
+ else
+ ARM64Assembler::linkCall(code, call.m_label, function.value());
}
CachedTempRegister m_dataMemoryTempRegister;
diff --git a/Source/JavaScriptCore/assembler/MacroAssemblerARMv7.h b/Source/JavaScriptCore/assembler/MacroAssemblerARMv7.h
index dc1c8c4..e807f79 100644
--- a/Source/JavaScriptCore/assembler/MacroAssemblerARMv7.h
+++ b/Source/JavaScriptCore/assembler/MacroAssemblerARMv7.h
@@ -1677,6 +1677,12 @@
return Call(m_assembler.blx(dataTempRegister), Call::LinkableNear);
}
+ ALWAYS_INLINE Call nearTailCall()
+ {
+ moveFixedWidthEncoding(TrustedImm32(0), dataTempRegister);
+ return Call(m_assembler.bx(dataTempRegister), Call::LinkableNearTail);
+ }
+
ALWAYS_INLINE Call call()
{
moveFixedWidthEncoding(TrustedImm32(0), dataTempRegister);
@@ -2012,7 +2018,10 @@
static void linkCall(void* code, Call call, FunctionPtr function)
{
- ARMv7Assembler::linkCall(code, call.m_label, function.value());
+ if (call.isFlagSet(Call::Tail))
+ ARMv7Assembler::linkJump(code, call.m_label, function.value());
+ else
+ ARMv7Assembler::linkCall(code, call.m_label, function.value());
}
#if ENABLE(MASM_PROBE)
diff --git a/Source/JavaScriptCore/assembler/MacroAssemblerMIPS.h b/Source/JavaScriptCore/assembler/MacroAssemblerMIPS.h
index 1a93128..48e3401 100644
--- a/Source/JavaScriptCore/assembler/MacroAssemblerMIPS.h
+++ b/Source/JavaScriptCore/assembler/MacroAssemblerMIPS.h
@@ -1975,6 +1975,16 @@
return Call(m_assembler.label(), Call::LinkableNear);
}
+ Call nearTailCall()
+ {
+ m_assembler.nop();
+ m_assembler.nop();
+ m_assembler.beq(MIPSRegisters::zero, MIPSRegisters::zero, 0);
+ m_assembler.nop();
+ insertRelaxationWords();
+ return Call(m_assembler.label(), Call::LinkableNearTail);
+ }
+
Call call()
{
m_assembler.lui(MIPSRegisters::t9, 0);
@@ -2800,7 +2810,10 @@
static void linkCall(void* code, Call call, FunctionPtr function)
{
- MIPSAssembler::linkCall(code, call.m_label, function.value());
+ if (call.isFlagSet(Call::Tail))
+ MIPSAssembler::linkJump(code, call.m_label, function.value());
+ else
+ MIPSAssembler::linkCall(code, call.m_label, function.value());
}
static void repatchCall(CodeLocationCall call, CodeLocationLabel destination)
diff --git a/Source/JavaScriptCore/assembler/MacroAssemblerSH4.h b/Source/JavaScriptCore/assembler/MacroAssemblerSH4.h
index 6857c60..5cfb05b 100644
--- a/Source/JavaScriptCore/assembler/MacroAssemblerSH4.h
+++ b/Source/JavaScriptCore/assembler/MacroAssemblerSH4.h
@@ -2403,6 +2403,11 @@
return Call(m_assembler.call(), Call::Linkable);
}
+ Call nearTailCall()
+ {
+ return Call(m_assembler.jump(), Call::LinkableNearTail);
+ }
+
Call nearCall()
{
return Call(m_assembler.call(), Call::LinkableNear);
@@ -2608,7 +2613,10 @@
static void linkCall(void* code, Call call, FunctionPtr function)
{
- SH4Assembler::linkCall(code, call.m_label, function.value());
+ if (call.isFlagSet(Call::Tail))
+ SH4Assembler::linkJump(code, call.m_label, function.value());
+ else
+ SH4Assembler::linkCall(code, call.m_label, function.value());
}
static void repatchCall(CodeLocationCall call, CodeLocationLabel destination)
diff --git a/Source/JavaScriptCore/assembler/MacroAssemblerX86.h b/Source/JavaScriptCore/assembler/MacroAssemblerX86.h
index 108f511..7ea3b5d 100644
--- a/Source/JavaScriptCore/assembler/MacroAssemblerX86.h
+++ b/Source/JavaScriptCore/assembler/MacroAssemblerX86.h
@@ -361,7 +361,10 @@
static void linkCall(void* code, Call call, FunctionPtr function)
{
- X86Assembler::linkCall(code, call.m_label, function.value());
+ if (call.isFlagSet(Call::Tail))
+ X86Assembler::linkJump(code, call.m_label, function.value());
+ else
+ X86Assembler::linkCall(code, call.m_label, function.value());
}
};
diff --git a/Source/JavaScriptCore/assembler/MacroAssemblerX86Common.h b/Source/JavaScriptCore/assembler/MacroAssemblerX86Common.h
index 84a4f52..deba962 100644
--- a/Source/JavaScriptCore/assembler/MacroAssemblerX86Common.h
+++ b/Source/JavaScriptCore/assembler/MacroAssemblerX86Common.h
@@ -1403,6 +1403,11 @@
m_assembler.int3();
}
+ Call nearTailCall()
+ {
+ return Call(m_assembler.jmp(), Call::LinkableNearTail);
+ }
+
Call nearCall()
{
return Call(m_assembler.call(), Call::LinkableNear);
diff --git a/Source/JavaScriptCore/assembler/MacroAssemblerX86_64.h b/Source/JavaScriptCore/assembler/MacroAssemblerX86_64.h
index ac1912b..8d071f6 100644
--- a/Source/JavaScriptCore/assembler/MacroAssemblerX86_64.h
+++ b/Source/JavaScriptCore/assembler/MacroAssemblerX86_64.h
@@ -872,6 +872,8 @@
{
if (!call.isFlagSet(Call::Near))
X86Assembler::linkPointer(code, call.m_label.labelAtOffset(-REPATCH_OFFSET_CALL_R11), function.value());
+ else if (call.isFlagSet(Call::Tail))
+ X86Assembler::linkJump(code, call.m_label, function.value());
else
X86Assembler::linkCall(code, call.m_label, function.value());
}
diff --git a/Source/JavaScriptCore/bytecode/BytecodeList.json b/Source/JavaScriptCore/bytecode/BytecodeList.json
index 6c07463..854bf61 100644
--- a/Source/JavaScriptCore/bytecode/BytecodeList.json
+++ b/Source/JavaScriptCore/bytecode/BytecodeList.json
@@ -92,8 +92,10 @@
{ "name" : "op_new_func_exp", "length" : 4 },
{ "name" : "op_new_arrow_func_exp", "length" : 5 },
{ "name" : "op_call", "length" : 9 },
+ { "name" : "op_tail_call", "length" : 9 },
{ "name" : "op_call_eval", "length" : 9 },
{ "name" : "op_call_varargs", "length" : 9 },
+ { "name" : "op_tail_call_varargs", "length" : 9 },
{ "name" : "op_ret", "length" : 2 },
{ "name" : "op_construct", "length" : 9 },
{ "name" : "op_construct_varargs", "length" : 9 },
@@ -144,7 +146,10 @@
{ "name" : "llint_cloop_did_return_from_js_5" },
{ "name" : "llint_cloop_did_return_from_js_6" },
{ "name" : "llint_cloop_did_return_from_js_7" },
- { "name" : "llint_cloop_did_return_from_js_8" }
+ { "name" : "llint_cloop_did_return_from_js_8" },
+ { "name" : "llint_cloop_did_return_from_js_9" },
+ { "name" : "llint_cloop_did_return_from_js_10" },
+ { "name" : "llint_cloop_did_return_from_js_11" }
]
},
{
diff --git a/Source/JavaScriptCore/bytecode/BytecodeUseDef.h b/Source/JavaScriptCore/bytecode/BytecodeUseDef.h
index d839867..f77a872 100644
--- a/Source/JavaScriptCore/bytecode/BytecodeUseDef.h
+++ b/Source/JavaScriptCore/bytecode/BytecodeUseDef.h
@@ -191,7 +191,8 @@
}
case op_has_structure_property:
case op_construct_varargs:
- case op_call_varargs: {
+ case op_call_varargs:
+ case op_tail_call_varargs: {
functor(codeBlock, instruction, opcodeID, instruction[2].u.operand);
functor(codeBlock, instruction, opcodeID, instruction[3].u.operand);
functor(codeBlock, instruction, opcodeID, instruction[4].u.operand);
@@ -220,7 +221,8 @@
}
case op_construct:
case op_call_eval:
- case op_call: {
+ case op_call:
+ case op_tail_call: {
functor(codeBlock, instruction, opcodeID, instruction[2].u.operand);
int argCount = instruction[3].u.operand;
int registerOffset = -instruction[4].u.operand;
@@ -311,9 +313,11 @@
case op_new_func_exp:
case op_new_arrow_func_exp:
case op_call_varargs:
+ case op_tail_call_varargs:
case op_construct_varargs:
case op_get_from_scope:
case op_call:
+ case op_tail_call:
case op_call_eval:
case op_construct:
case op_get_by_id:
diff --git a/Source/JavaScriptCore/bytecode/CallLinkInfo.h b/Source/JavaScriptCore/bytecode/CallLinkInfo.h
index 5f5c6b4..7326d13 100644
--- a/Source/JavaScriptCore/bytecode/CallLinkInfo.h
+++ b/Source/JavaScriptCore/bytecode/CallLinkInfo.h
@@ -26,6 +26,7 @@
#ifndef CallLinkInfo_h
#define CallLinkInfo_h
+#include "CallMode.h"
#include "CodeLocation.h"
#include "CodeSpecializationKind.h"
#include "JITWriteBarrier.h"
@@ -41,19 +42,36 @@
class CallLinkInfo : public BasicRawSentinelNode<CallLinkInfo> {
public:
- enum CallType { None, Call, CallVarargs, Construct, ConstructVarargs };
+ enum CallType { None, Call, CallVarargs, Construct, ConstructVarargs, TailCall, TailCallVarargs };
static CallType callTypeFor(OpcodeID opcodeID)
{
if (opcodeID == op_call || opcodeID == op_call_eval)
return Call;
+ if (opcodeID == op_call_varargs)
+ return CallVarargs;
if (opcodeID == op_construct)
return Construct;
if (opcodeID == op_construct_varargs)
return ConstructVarargs;
- ASSERT(opcodeID == op_call_varargs);
- return CallVarargs;
+ if (opcodeID == op_tail_call)
+ return TailCall;
+ ASSERT(opcodeID == op_tail_call_varargs);
+ return TailCallVarargs;
}
-
+
+ static bool isVarargsCallType(CallType callType)
+ {
+ switch (callType) {
+ case CallVarargs:
+ case ConstructVarargs:
+ case TailCallVarargs:
+ return true;
+
+ default:
+ return false;
+ }
+ }
+
CallLinkInfo()
: m_registerPreservationMode(static_cast<unsigned>(RegisterPreservationNotRequired))
, m_hasSeenShouldRepatch(false)
@@ -83,6 +101,40 @@
return specializationKindFor(static_cast<CallType>(m_callType));
}
+ static CallMode callModeFor(CallType callType)
+ {
+ switch (callType) {
+ case Call:
+ case CallVarargs:
+ return CallMode::Regular;
+ case TailCall:
+ case TailCallVarargs:
+ return CallMode::Tail;
+ case Construct:
+ case ConstructVarargs:
+ return CallMode::Construct;
+ case None:
+ RELEASE_ASSERT_NOT_REACHED();
+ }
+
+ RELEASE_ASSERT_NOT_REACHED();
+ }
+
+ CallMode callMode() const
+ {
+ return callModeFor(static_cast<CallType>(m_callType));
+ }
+
+ bool isTailCall() const
+ {
+ return callMode() == CallMode::Tail;
+ }
+
+ bool isVarargs() const
+ {
+ return isVarargsCallType(static_cast<CallType>(m_callType));
+ }
+
RegisterPreservationMode registerPreservationMode() const
{
return static_cast<RegisterPreservationMode>(m_registerPreservationMode);
diff --git a/Source/JavaScriptCore/bytecode/CallLinkStatus.cpp b/Source/JavaScriptCore/bytecode/CallLinkStatus.cpp
index 68eaff9..687a24c 100644
--- a/Source/JavaScriptCore/bytecode/CallLinkStatus.cpp
+++ b/Source/JavaScriptCore/bytecode/CallLinkStatus.cpp
@@ -69,7 +69,7 @@
Instruction* instruction = profiledBlock->instructions().begin() + bytecodeIndex;
OpcodeID op = vm.interpreter->getOpcodeID(instruction[0].u.opcode);
- if (op != op_call && op != op_construct)
+ if (op != op_call && op != op_construct && op != op_tail_call)
return CallLinkStatus();
LLIntCallLinkInfo* callLinkInfo = instruction[5].u.callLinkInfo;
diff --git a/Source/JavaScriptCore/bytecode/CodeBlock.cpp b/Source/JavaScriptCore/bytecode/CodeBlock.cpp
index cb48868..788c799 100644
--- a/Source/JavaScriptCore/bytecode/CodeBlock.cpp
+++ b/Source/JavaScriptCore/bytecode/CodeBlock.cpp
@@ -1248,13 +1248,18 @@
printCallOp(out, exec, location, it, "call", DumpCaches, hasPrintedProfiling, callLinkInfos);
break;
}
+ case op_tail_call: {
+ printCallOp(out, exec, location, it, "tail_call", DumpCaches, hasPrintedProfiling, callLinkInfos);
+ break;
+ }
case op_call_eval: {
printCallOp(out, exec, location, it, "call_eval", DontDumpCaches, hasPrintedProfiling, callLinkInfos);
break;
}
case op_construct_varargs:
- case op_call_varargs: {
+ case op_call_varargs:
+ case op_tail_call_varargs: {
int result = (++it)->u.operand;
int callee = (++it)->u.operand;
int thisValue = (++it)->u.operand;
@@ -1262,7 +1267,7 @@
int firstFreeRegister = (++it)->u.operand;
int varArgOffset = (++it)->u.operand;
++it;
- printLocationAndOp(out, exec, location, it, opcode == op_call_varargs ? "call_varargs" : "construct_varargs");
+ printLocationAndOp(out, exec, location, it, opcode == op_call_varargs ? "call_varargs" : opcode == op_construct_varargs ? "construct_varargs" : "tail_call_varargs");
out.printf("%s, %s, %s, %s, %d, %d", registerName(result).data(), registerName(callee).data(), registerName(thisValue).data(), registerName(arguments).data(), firstFreeRegister, varArgOffset);
dumpValueProfiling(out, it, hasPrintedProfiling);
break;
@@ -1829,6 +1834,7 @@
break;
}
case op_call_varargs:
+ case op_tail_call_varargs:
case op_construct_varargs:
case op_get_by_val: {
int arrayProfileIndex = pc[opLength - 2].u.operand;
@@ -1878,6 +1884,7 @@
}
case op_call:
+ case op_tail_call:
case op_call_eval: {
ValueProfile* profile = &m_valueProfiles[pc[opLength - 1].u.operand];
ASSERT(profile->m_bytecodeOffset == -1);
diff --git a/Source/JavaScriptCore/bytecompiler/BytecodeGenerator.cpp b/Source/JavaScriptCore/bytecompiler/BytecodeGenerator.cpp
index f54d8db..ab45d52 100644
--- a/Source/JavaScriptCore/bytecompiler/BytecodeGenerator.cpp
+++ b/Source/JavaScriptCore/bytecompiler/BytecodeGenerator.cpp
@@ -193,7 +193,12 @@
, m_vm(&vm)
, m_isBuiltinFunction(codeBlock->isBuiltinFunction())
, m_usesNonStrictEval(codeBlock->usesEval() && !codeBlock->isStrictMode())
- , m_inTailPosition(Options::enableTailCalls() && constructorKind() == ConstructorKind::None && isStrictMode())
+ // FIXME: We should be able to have tail call elimination with the profiler
+ // enabled. This is currently not possible because the profiler expects
+ // op_will_call / op_did_call pairs before and after a call, which are not
+ // compatible with tail calls (we have no way of emitting op_did_call).
+ // https://bugs.webkit.org/show_bug.cgi?id=148819
+ , m_inTailPosition(Options::enableTailCalls() && constructorKind() == ConstructorKind::None && isStrictMode() && !m_shouldEmitProfileHooks)
{
for (auto& constantRegister : m_linkTimeConstantRegisters)
constantRegister = nullptr;
@@ -2513,10 +2518,7 @@
RegisterID* BytecodeGenerator::emitCallInTailPosition(RegisterID* dst, RegisterID* func, ExpectedFunction expectedFunction, CallArguments& callArguments, const JSTextPosition& divot, const JSTextPosition& divotStart, const JSTextPosition& divotEnd)
{
- // FIXME: We should be emitting a new op_tail_call here when
- // m_inTailPosition is false
- // https://bugs.webkit.org/show_bug.cgi?id=148661
- return emitCall(op_call, dst, func, expectedFunction, callArguments, divot, divotStart, divotEnd);
+ return emitCall(m_inTailPosition ? op_tail_call : op_call, dst, func, expectedFunction, callArguments, divot, divotStart, divotEnd);
}
RegisterID* BytecodeGenerator::emitCallEval(RegisterID* dst, RegisterID* func, CallArguments& callArguments, const JSTextPosition& divot, const JSTextPosition& divotStart, const JSTextPosition& divotEnd)
@@ -2601,7 +2603,7 @@
RegisterID* BytecodeGenerator::emitCall(OpcodeID opcodeID, RegisterID* dst, RegisterID* func, ExpectedFunction expectedFunction, CallArguments& callArguments, const JSTextPosition& divot, const JSTextPosition& divotStart, const JSTextPosition& divotEnd)
{
- ASSERT(opcodeID == op_call || opcodeID == op_call_eval);
+ ASSERT(opcodeID == op_call || opcodeID == op_call_eval || opcodeID == op_tail_call);
ASSERT(func->refCount());
if (m_shouldEmitProfileHooks)
@@ -2617,7 +2619,7 @@
RefPtr<RegisterID> argumentRegister;
argumentRegister = expression->emitBytecode(*this, callArguments.argumentRegister(0));
RefPtr<RegisterID> thisRegister = emitMove(newTemporary(), callArguments.thisRegister());
- return emitCallVarargs(dst, func, callArguments.thisRegister(), argumentRegister.get(), newTemporary(), 0, callArguments.profileHookRegister(), divot, divotStart, divotEnd);
+ return emitCallVarargs(opcodeID == op_tail_call ? op_tail_call_varargs : op_call_varargs, dst, func, callArguments.thisRegister(), argumentRegister.get(), newTemporary(), 0, callArguments.profileHookRegister(), divot, divotStart, divotEnd);
}
for (; n; n = n->m_next)
emitNode(callArguments.argumentRegister(argument++), n);
@@ -2670,10 +2672,7 @@
RegisterID* BytecodeGenerator::emitCallVarargsInTailPosition(RegisterID* dst, RegisterID* func, RegisterID* thisRegister, RegisterID* arguments, RegisterID* firstFreeRegister, int32_t firstVarArgOffset, RegisterID* profileHookRegister, const JSTextPosition& divot, const JSTextPosition& divotStart, const JSTextPosition& divotEnd)
{
- // FIXME: We should be emitting a new op_tail_call here when
- // m_inTailPosition is false
- // https://bugs.webkit.org/show_bug.cgi?id=148661
- return emitCallVarargs(op_call_varargs, dst, func, thisRegister, arguments, firstFreeRegister, firstVarArgOffset, profileHookRegister, divot, divotStart, divotEnd);
+ return emitCallVarargs(m_inTailPosition ? op_tail_call_varargs : op_call_varargs, dst, func, thisRegister, arguments, firstFreeRegister, firstVarArgOffset, profileHookRegister, divot, divotStart, divotEnd);
}
RegisterID* BytecodeGenerator::emitConstructVarargs(RegisterID* dst, RegisterID* func, RegisterID* thisRegister, RegisterID* arguments, RegisterID* firstFreeRegister, int32_t firstVarArgOffset, RegisterID* profileHookRegister, const JSTextPosition& divot, const JSTextPosition& divotStart, const JSTextPosition& divotEnd)
diff --git a/Source/JavaScriptCore/bytecompiler/NodesCodegen.cpp b/Source/JavaScriptCore/bytecompiler/NodesCodegen.cpp
index 0529e32..48647a3 100644
--- a/Source/JavaScriptCore/bytecompiler/NodesCodegen.cpp
+++ b/Source/JavaScriptCore/bytecompiler/NodesCodegen.cpp
@@ -678,6 +678,12 @@
m_argv[i] = generator.newTemporary();
ASSERT(static_cast<size_t>(i) == m_argv.size() - 1 || m_argv[i]->index() == m_argv[i + 1]->index() - 1);
}
+
+ // We need to ensure that the frame size is stack-aligned
+ while ((JSStack::CallFrameHeaderSize + m_argv.size()) % stackAlignmentRegisters()) {
+ m_argv.insert(0, generator.newTemporary());
+ m_padding++;
+ }
while (stackOffset() % stackAlignmentRegisters()) {
m_argv.insert(0, generator.newTemporary());
@@ -2786,7 +2792,7 @@
ASSERT(!generator.breakTarget(m_name));
LabelScopePtr scope = generator.newLabelScope(LabelScope::NamedLabel, &m_name);
- generator.emitNode(dst, m_statement);
+ generator.emitNodeInTailPosition(dst, m_statement);
generator.emitLabel(scope->breakTarget());
}
diff --git a/Source/JavaScriptCore/dfg/DFGByteCodeParser.cpp b/Source/JavaScriptCore/dfg/DFGByteCodeParser.cpp
index 7bd3b37..9fbd80c 100644
--- a/Source/JavaScriptCore/dfg/DFGByteCodeParser.cpp
+++ b/Source/JavaScriptCore/dfg/DFGByteCodeParser.cpp
@@ -741,7 +741,10 @@
SpeculatedType prediction)
{
addVarArgChild(callee);
- size_t parameterSlots = JSStack::CallFrameHeaderSize - JSStack::CallerFrameAndPCSize + argCount;
+ size_t frameSize = JSStack::CallFrameHeaderSize + argCount;
+ size_t alignedFrameSize = WTF::roundUpToMultipleOf(stackAlignmentRegisters(), frameSize);
+ size_t parameterSlots = alignedFrameSize - JSStack::CallerFrameAndPCSize;
+
if (parameterSlots > m_parameterSlots)
m_parameterSlots = parameterSlots;
diff --git a/Source/JavaScriptCore/ftl/FTLLowerDFGToLLVM.cpp b/Source/JavaScriptCore/ftl/FTLLowerDFGToLLVM.cpp
index e74139c..568bdf4 100644
--- a/Source/JavaScriptCore/ftl/FTLLowerDFGToLLVM.cpp
+++ b/Source/JavaScriptCore/ftl/FTLLowerDFGToLLVM.cpp
@@ -4329,24 +4329,29 @@
void compileCallOrConstruct()
{
- int numPassedArgs = m_node->numChildren() - 1;
- int numArgs = numPassedArgs;
+ int numArgs = m_node->numChildren() - 1;
LValue jsCallee = lowJSValue(m_graph.varArgChild(m_node, 0));
unsigned stackmapID = m_stackmapIDs++;
-
+
+ unsigned frameSize = JSStack::CallFrameHeaderSize + numArgs;
+ unsigned alignedFrameSize = WTF::roundUpToMultipleOf(stackAlignmentRegisters(), frameSize);
+ unsigned padding = alignedFrameSize - frameSize;
+
Vector<LValue> arguments;
arguments.append(m_out.constInt64(stackmapID));
arguments.append(m_out.constInt32(sizeOfCall()));
arguments.append(constNull(m_out.ref8));
- arguments.append(m_out.constInt32(1 + JSStack::CallFrameHeaderSize - JSStack::CallerFrameAndPCSize + numArgs));
+ arguments.append(m_out.constInt32(1 + alignedFrameSize - JSStack::CallerFrameAndPCSize));
arguments.append(jsCallee); // callee -> %rax
arguments.append(getUndef(m_out.int64)); // code block
arguments.append(jsCallee); // callee -> stack
arguments.append(m_out.constInt64(numArgs)); // argument count and zeros for the tag
- for (int i = 0; i < numPassedArgs; ++i)
+ for (int i = 0; i < numArgs; ++i)
arguments.append(lowJSValue(m_graph.varArgChild(m_node, 1 + i)));
+ for (unsigned i = 0; i < padding; ++i)
+ arguments.append(getUndef(m_out.int64));
callPreflight();
diff --git a/Source/JavaScriptCore/interpreter/Interpreter.h b/Source/JavaScriptCore/interpreter/Interpreter.h
index a96efb7..a4fb2ec 100644
--- a/Source/JavaScriptCore/interpreter/Interpreter.h
+++ b/Source/JavaScriptCore/interpreter/Interpreter.h
@@ -253,7 +253,7 @@
void dumpRegisters(CallFrame*);
- bool isCallBytecode(Opcode opcode) { return opcode == getOpcode(op_call) || opcode == getOpcode(op_construct) || opcode == getOpcode(op_call_eval); }
+ bool isCallBytecode(Opcode opcode) { return opcode == getOpcode(op_call) || opcode == getOpcode(op_construct) || opcode == getOpcode(op_call_eval) || opcode == getOpcode(op_tail_call); }
void enableSampler();
int m_sampleEntryDepth;
@@ -277,6 +277,15 @@
inline CallFrame* calleeFrameForVarargs(CallFrame* callFrame, unsigned numUsedStackSlots, unsigned argumentCountIncludingThis)
{
+#if 1
+ // We want the new frame to be allocated on a stack aligned offset with a stack
+ // aligned size. Align the size here.
+ argumentCountIncludingThis = WTF::roundUpToMultipleOf(
+ stackAlignmentRegisters(),
+ argumentCountIncludingThis + JSStack::CallFrameHeaderSize) - JSStack::CallFrameHeaderSize;
+
+ // Align the frame offset here.
+#endif
unsigned paddedCalleeFrameOffset = WTF::roundUpToMultipleOf(
stackAlignmentRegisters(),
numUsedStackSlots + argumentCountIncludingThis + JSStack::CallFrameHeaderSize);
diff --git a/Source/JavaScriptCore/jit/CCallHelpers.h b/Source/JavaScriptCore/jit/CCallHelpers.h
index 7198bb5..40b37df 100644
--- a/Source/JavaScriptCore/jit/CCallHelpers.h
+++ b/Source/JavaScriptCore/jit/CCallHelpers.h
@@ -30,6 +30,7 @@
#include "AssemblyHelpers.h"
#include "GPRInfo.h"
+#include "StackAlignment.h"
namespace JSC {
@@ -2082,6 +2083,95 @@
loadPtr(&vm()->targetMachinePCForThrow, GPRInfo::regT1);
jump(GPRInfo::regT1);
}
+
+ void prepareForTailCallSlow(GPRReg calleeGPR = InvalidGPRReg)
+ {
+ GPRReg temp1 = calleeGPR == GPRInfo::regT0 ? GPRInfo::regT3 : GPRInfo::regT0;
+ GPRReg temp2 = calleeGPR == GPRInfo::regT1 ? GPRInfo::regT3 : GPRInfo::regT1;
+ GPRReg temp3 = calleeGPR == GPRInfo::regT2 ? GPRInfo::regT3 : GPRInfo::regT2;
+
+ GPRReg newFramePointer = temp1;
+ GPRReg newFrameSizeGPR = temp2;
+ {
+ // The old frame size is its number of arguments (or number of
+ // parameters in case of arity fixup), plus the frame header size,
+ // aligned
+ GPRReg oldFrameSizeGPR = temp2;
+ {
+ GPRReg argCountGPR = oldFrameSizeGPR;
+ load32(Address(framePointerRegister, JSStack::ArgumentCount * static_cast<int>(sizeof(Register)) + PayloadOffset), argCountGPR);
+
+ {
+ GPRReg numParametersGPR = temp1;
+ {
+ GPRReg codeBlockGPR = numParametersGPR;
+ loadPtr(Address(framePointerRegister, JSStack::CodeBlock * static_cast<int>(sizeof(Register))), codeBlockGPR);
+ load32(Address(codeBlockGPR, CodeBlock::offsetOfNumParameters()), numParametersGPR);
+ }
+
+ ASSERT(numParametersGPR != argCountGPR);
+ Jump argumentCountWasNotFixedUp = branch32(BelowOrEqual, numParametersGPR, argCountGPR);
+ move(numParametersGPR, argCountGPR);
+ argumentCountWasNotFixedUp.link(this);
+ }
+
+ add32(TrustedImm32(stackAlignmentRegisters() + JSStack::CallFrameHeaderSize - 1), argCountGPR, oldFrameSizeGPR);
+ and32(TrustedImm32(-stackAlignmentRegisters()), oldFrameSizeGPR);
+ // We assume < 2^28 arguments
+ mul32(TrustedImm32(sizeof(Register)), oldFrameSizeGPR, oldFrameSizeGPR);
+ }
+
+ // The new frame pointer is at framePointer + oldFrameSize - newFrameSize
+ ASSERT(newFramePointer != oldFrameSizeGPR);
+ move(framePointerRegister, newFramePointer);
+ addPtr(oldFrameSizeGPR, newFramePointer);
+
+ // The new frame size is just the number of arguments plus the
+ // frame header size, aligned
+ ASSERT(newFrameSizeGPR != newFramePointer);
+ load32(Address(stackPointerRegister, JSStack::ArgumentCount * static_cast<int>(sizeof(Register)) + PayloadOffset - sizeof(CallerFrameAndPC)),
+ newFrameSizeGPR);
+ add32(TrustedImm32(stackAlignmentRegisters() + JSStack::CallFrameHeaderSize - 1), newFrameSizeGPR);
+ and32(TrustedImm32(-stackAlignmentRegisters()), newFrameSizeGPR);
+ // We assume < 2^28 arguments
+ mul32(TrustedImm32(sizeof(Register)), newFrameSizeGPR, newFrameSizeGPR);
+ }
+
+ GPRReg tempGPR = temp3;
+ ASSERT(tempGPR != newFramePointer && tempGPR != newFrameSizeGPR);
+
+ // We don't need the current frame beyond this point. Masquerade as our
+ // caller.
+#if CPU(ARM) || CPU(SH4) || CPU(ARM64)
+ loadPtr(Address(framePointerRegister, sizeof(void*)), linkRegister);
+ subPtr(TrustedImm32(2 * sizeof(void*)), newFrameSizeGPR);
+#elif CPU(MIPS)
+ loadPtr(Address(framePointerRegister, sizeof(void*)), returnAddressRegister);
+ subPtr(TrustedImm32(2 * sizeof(void*)), newFrameSizeGPR);
+#elif CPU(X86) || CPU(X86_64)
+ loadPtr(Address(framePointerRegister, sizeof(void*)), tempGPR);
+ push(tempGPR);
+ subPtr(TrustedImm32(sizeof(void*)), newFrameSizeGPR);
+#else
+ UNREACHABLE_FOR_PLATFORM();
+#endif
+ subPtr(newFrameSizeGPR, newFramePointer);
+ loadPtr(Address(framePointerRegister), framePointerRegister);
+
+
+ // We need to move the newFrameSizeGPR slots above the stack pointer by
+ // newFramePointer registers. We use pointer-sized chunks.
+ MacroAssembler::Label copyLoop(label());
+
+ subPtr(TrustedImm32(sizeof(void*)), newFrameSizeGPR);
+ loadPtr(BaseIndex(stackPointerRegister, newFrameSizeGPR, TimesOne), tempGPR);
+ storePtr(tempGPR, BaseIndex(newFramePointer, newFrameSizeGPR, TimesOne));
+
+ branchTest32(MacroAssembler::NonZero, newFrameSizeGPR).linkTo(copyLoop, this);
+
+ // Ready for a jump!
+ move(newFramePointer, stackPointerRegister);
+ }
};
} // namespace JSC
diff --git a/Source/JavaScriptCore/jit/JIT.cpp b/Source/JavaScriptCore/jit/JIT.cpp
index 6a9827c..ad8c04d 100644
--- a/Source/JavaScriptCore/jit/JIT.cpp
+++ b/Source/JavaScriptCore/jit/JIT.cpp
@@ -197,8 +197,10 @@
DEFINE_OP(op_bitor)
DEFINE_OP(op_bitxor)
DEFINE_OP(op_call)
+ DEFINE_OP(op_tail_call)
DEFINE_OP(op_call_eval)
DEFINE_OP(op_call_varargs)
+ DEFINE_OP(op_tail_call_varargs)
DEFINE_OP(op_construct_varargs)
DEFINE_OP(op_catch)
DEFINE_OP(op_construct)
@@ -371,8 +373,10 @@
DEFINE_SLOWCASE_OP(op_bitor)
DEFINE_SLOWCASE_OP(op_bitxor)
DEFINE_SLOWCASE_OP(op_call)
+ DEFINE_SLOWCASE_OP(op_tail_call)
DEFINE_SLOWCASE_OP(op_call_eval)
DEFINE_SLOWCASE_OP(op_call_varargs)
+ DEFINE_SLOWCASE_OP(op_tail_call_varargs)
DEFINE_SLOWCASE_OP(op_construct_varargs)
DEFINE_SLOWCASE_OP(op_construct)
DEFINE_SLOWCASE_OP(op_to_this)
diff --git a/Source/JavaScriptCore/jit/JIT.h b/Source/JavaScriptCore/jit/JIT.h
index 35902b1..7fab237 100644
--- a/Source/JavaScriptCore/jit/JIT.h
+++ b/Source/JavaScriptCore/jit/JIT.h
@@ -487,8 +487,10 @@
void emit_op_bitor(Instruction*);
void emit_op_bitxor(Instruction*);
void emit_op_call(Instruction*);
+ void emit_op_tail_call(Instruction*);
void emit_op_call_eval(Instruction*);
void emit_op_call_varargs(Instruction*);
+ void emit_op_tail_call_varargs(Instruction*);
void emit_op_construct_varargs(Instruction*);
void emit_op_catch(Instruction*);
void emit_op_construct(Instruction*);
@@ -600,8 +602,10 @@
void emitSlow_op_bitor(Instruction*, Vector<SlowCaseEntry>::iterator&);
void emitSlow_op_bitxor(Instruction*, Vector<SlowCaseEntry>::iterator&);
void emitSlow_op_call(Instruction*, Vector<SlowCaseEntry>::iterator&);
+ void emitSlow_op_tail_call(Instruction*, Vector<SlowCaseEntry>::iterator&);
void emitSlow_op_call_eval(Instruction*, Vector<SlowCaseEntry>::iterator&);
void emitSlow_op_call_varargs(Instruction*, Vector<SlowCaseEntry>::iterator&);
+ void emitSlow_op_tail_call_varargs(Instruction*, Vector<SlowCaseEntry>::iterator&);
void emitSlow_op_construct_varargs(Instruction*, Vector<SlowCaseEntry>::iterator&);
void emitSlow_op_construct(Instruction*, Vector<SlowCaseEntry>::iterator&);
void emitSlow_op_to_this(Instruction*, Vector<SlowCaseEntry>::iterator&);
@@ -824,6 +828,7 @@
void updateTopCallFrame();
Call emitNakedCall(CodePtr function = CodePtr());
+ Call emitNakedTailCall(CodePtr function = CodePtr());
// Loads the character value of a single character string into dst.
void emitLoadCharacterString(RegisterID src, RegisterID dst, JumpList& failures);
diff --git a/Source/JavaScriptCore/jit/JITCall.cpp b/Source/JavaScriptCore/jit/JITCall.cpp
index 1af51ad..5b5099a 100644
--- a/Source/JavaScriptCore/jit/JITCall.cpp
+++ b/Source/JavaScriptCore/jit/JITCall.cpp
@@ -145,10 +145,12 @@
COMPILE_ASSERT(OPCODE_LENGTH(op_call) == OPCODE_LENGTH(op_construct), call_and_construct_opcodes_must_be_same_length);
COMPILE_ASSERT(OPCODE_LENGTH(op_call) == OPCODE_LENGTH(op_call_varargs), call_and_call_varargs_opcodes_must_be_same_length);
COMPILE_ASSERT(OPCODE_LENGTH(op_call) == OPCODE_LENGTH(op_construct_varargs), call_and_construct_varargs_opcodes_must_be_same_length);
+ COMPILE_ASSERT(OPCODE_LENGTH(op_call) == OPCODE_LENGTH(op_tail_call), call_and_tail_call_opcodes_must_be_same_length);
+ COMPILE_ASSERT(OPCODE_LENGTH(op_call) == OPCODE_LENGTH(op_tail_call_varargs), call_and_tail_call_varargs_opcodes_must_be_same_length);
CallLinkInfo* info;
if (opcodeID != op_call_eval)
info = m_codeBlock->addCallLinkInfo();
- if (opcodeID == op_call_varargs || opcodeID == op_construct_varargs)
+ if (opcodeID == op_call_varargs || opcodeID == op_construct_varargs || opcodeID == op_tail_call_varargs)
compileSetupVarargsFrame(instruction, info);
else {
int argCount = instruction[3].u.operand;
@@ -172,12 +174,15 @@
emitGetVirtualRegister(callee, regT0); // regT0 holds callee.
store64(regT0, Address(stackPointerRegister, JSStack::Callee * static_cast<int>(sizeof(Register)) - sizeof(CallerFrameAndPC)));
-
+
if (opcodeID == op_call_eval) {
compileCallEval(instruction);
return;
}
+ if (opcodeID == op_tail_call || opcodeID == op_tail_call_varargs)
+ emitRestoreCalleeSaves();
+
DataLabelPtr addressOfLinkedFunctionCheck;
Jump slowCase = branchPtrWithPatch(NotEqual, regT0, addressOfLinkedFunctionCheck, TrustedImmPtr(0));
addSlowCase(slowCase);
@@ -188,6 +193,12 @@
m_callCompilationInfo[callLinkInfoIndex].hotPathBegin = addressOfLinkedFunctionCheck;
m_callCompilationInfo[callLinkInfoIndex].callLinkInfo = info;
+ if (opcodeID == op_tail_call || opcodeID == op_tail_call_varargs) {
+ prepareForTailCallSlow();
+ m_callCompilationInfo[callLinkInfoIndex].hotPathOther = emitNakedTailCall();
+ return;
+ }
+
m_callCompilationInfo[callLinkInfoIndex].hotPathOther = emitNakedCall();
addPtr(TrustedImm32(stackPointerOffsetFor(m_codeBlock) * sizeof(Register)), callFrameRegister, stackPointerRegister);
@@ -208,8 +219,14 @@
linkSlowCase(iter);
move(TrustedImmPtr(m_callCompilationInfo[callLinkInfoIndex].callLinkInfo), regT2);
+
m_callCompilationInfo[callLinkInfoIndex].callReturnLocation = emitNakedCall(m_vm->getCTIStub(linkCallThunkGenerator).code());
+ if (opcodeID == op_tail_call || opcodeID == op_tail_call_varargs) {
+ abortWithReason(JITDidReturnFromTailCall);
+ return;
+ }
+
addPtr(TrustedImm32(stackPointerOffsetFor(m_codeBlock) * sizeof(Register)), callFrameRegister, stackPointerRegister);
checkStackPointerAlignment();
@@ -223,6 +240,11 @@
compileOpCall(op_call, currentInstruction, m_callLinkInfoIndex++);
}
+void JIT::emit_op_tail_call(Instruction* currentInstruction)
+{
+ compileOpCall(op_tail_call, currentInstruction, m_callLinkInfoIndex++);
+}
+
void JIT::emit_op_call_eval(Instruction* currentInstruction)
{
compileOpCall(op_call_eval, currentInstruction, m_callLinkInfoIndex);
@@ -232,7 +254,12 @@
{
compileOpCall(op_call_varargs, currentInstruction, m_callLinkInfoIndex++);
}
-
+
+void JIT::emit_op_tail_call_varargs(Instruction* currentInstruction)
+{
+ compileOpCall(op_tail_call_varargs, currentInstruction, m_callLinkInfoIndex++);
+}
+
void JIT::emit_op_construct_varargs(Instruction* currentInstruction)
{
compileOpCall(op_construct_varargs, currentInstruction, m_callLinkInfoIndex++);
@@ -248,6 +275,11 @@
compileOpCallSlowCase(op_call, currentInstruction, iter, m_callLinkInfoIndex++);
}
+void JIT::emitSlow_op_tail_call(Instruction* currentInstruction, Vector<SlowCaseEntry>::iterator& iter)
+{
+ compileOpCallSlowCase(op_tail_call, currentInstruction, iter, m_callLinkInfoIndex++);
+}
+
void JIT::emitSlow_op_call_eval(Instruction* currentInstruction, Vector<SlowCaseEntry>::iterator& iter)
{
compileOpCallSlowCase(op_call_eval, currentInstruction, iter, m_callLinkInfoIndex);
@@ -257,6 +289,11 @@
{
compileOpCallSlowCase(op_call_varargs, currentInstruction, iter, m_callLinkInfoIndex++);
}
+
+void JIT::emitSlow_op_tail_call_varargs(Instruction* currentInstruction, Vector<SlowCaseEntry>::iterator& iter)
+{
+ compileOpCallSlowCase(op_tail_call_varargs, currentInstruction, iter, m_callLinkInfoIndex++);
+}
void JIT::emitSlow_op_construct_varargs(Instruction* currentInstruction, Vector<SlowCaseEntry>::iterator& iter)
{
diff --git a/Source/JavaScriptCore/jit/JITCall32_64.cpp b/Source/JavaScriptCore/jit/JITCall32_64.cpp
index 89c3bac..4d163c2 100644
--- a/Source/JavaScriptCore/jit/JITCall32_64.cpp
+++ b/Source/JavaScriptCore/jit/JITCall32_64.cpp
@@ -69,6 +69,11 @@
compileOpCallSlowCase(op_call, currentInstruction, iter, m_callLinkInfoIndex++);
}
+void JIT::emitSlow_op_tail_call(Instruction* currentInstruction, Vector<SlowCaseEntry>::iterator& iter)
+{
+ compileOpCallSlowCase(op_tail_call, currentInstruction, iter, m_callLinkInfoIndex++);
+}
+
void JIT::emitSlow_op_call_eval(Instruction* currentInstruction, Vector<SlowCaseEntry>::iterator& iter)
{
compileOpCallSlowCase(op_call_eval, currentInstruction, iter, m_callLinkInfoIndex);
@@ -78,6 +83,11 @@
{
compileOpCallSlowCase(op_call_varargs, currentInstruction, iter, m_callLinkInfoIndex++);
}
+
+void JIT::emitSlow_op_tail_call_varargs(Instruction* currentInstruction, Vector<SlowCaseEntry>::iterator& iter)
+{
+ compileOpCallSlowCase(op_tail_call_varargs, currentInstruction, iter, m_callLinkInfoIndex++);
+}
void JIT::emitSlow_op_construct_varargs(Instruction* currentInstruction, Vector<SlowCaseEntry>::iterator& iter)
{
@@ -94,6 +104,11 @@
compileOpCall(op_call, currentInstruction, m_callLinkInfoIndex++);
}
+void JIT::emit_op_tail_call(Instruction* currentInstruction)
+{
+ compileOpCall(op_tail_call, currentInstruction, m_callLinkInfoIndex++);
+}
+
void JIT::emit_op_call_eval(Instruction* currentInstruction)
{
compileOpCall(op_call_eval, currentInstruction, m_callLinkInfoIndex);
@@ -103,6 +118,11 @@
{
compileOpCall(op_call_varargs, currentInstruction, m_callLinkInfoIndex++);
}
+
+void JIT::emit_op_tail_call_varargs(Instruction* currentInstruction)
+{
+ compileOpCall(op_tail_call_varargs, currentInstruction, m_callLinkInfoIndex++);
+}
void JIT::emit_op_construct_varargs(Instruction* currentInstruction)
{
@@ -210,7 +230,7 @@
CallLinkInfo* info;
if (opcodeID != op_call_eval)
info = m_codeBlock->addCallLinkInfo();
- if (opcodeID == op_call_varargs || opcodeID == op_construct_varargs)
+ if (opcodeID == op_call_varargs || opcodeID == op_construct_varargs || opcodeID == op_tail_call_varargs)
compileSetupVarargsFrame(instruction, info);
else {
int argCount = instruction[3].u.operand;
@@ -241,6 +261,9 @@
return;
}
+ if (opcodeID == op_tail_call || opcodeID == op_tail_call_varargs)
+ emitRestoreCalleeSaves();
+
addSlowCase(branch32(NotEqual, regT1, TrustedImm32(JSValue::CellTag)));
DataLabelPtr addressOfLinkedFunctionCheck;
@@ -255,6 +278,12 @@
m_callCompilationInfo[callLinkInfoIndex].callLinkInfo = info;
checkStackPointerAlignment();
+ if (opcodeID == op_tail_call || opcodeID == op_tail_call_varargs) {
+ prepareForTailCallSlow();
+ m_callCompilationInfo[callLinkInfoIndex].hotPathOther = emitNakedTailCall();
+ return;
+ }
+
m_callCompilationInfo[callLinkInfoIndex].hotPathOther = emitNakedCall();
addPtr(TrustedImm32(stackPointerOffsetFor(m_codeBlock) * sizeof(Register)), callFrameRegister, stackPointerRegister);
@@ -275,8 +304,17 @@
linkSlowCase(iter);
move(TrustedImmPtr(m_callCompilationInfo[callLinkInfoIndex].callLinkInfo), regT2);
+
+ if (opcodeID == op_tail_call || opcodeID == op_tail_call_varargs)
+ emitRestoreCalleeSaves();
+
m_callCompilationInfo[callLinkInfoIndex].callReturnLocation = emitNakedCall(m_vm->getCTIStub(linkCallThunkGenerator).code());
+ if (opcodeID == op_tail_call || opcodeID == op_tail_call_varargs) {
+ abortWithReason(JITDidReturnFromTailCall);
+ return;
+ }
+
addPtr(TrustedImm32(stackPointerOffsetFor(m_codeBlock) * sizeof(Register)), callFrameRegister, stackPointerRegister);
checkStackPointerAlignment();
diff --git a/Source/JavaScriptCore/jit/JITInlines.h b/Source/JavaScriptCore/jit/JITInlines.h
index d327673..386f0fa 100644
--- a/Source/JavaScriptCore/jit/JITInlines.h
+++ b/Source/JavaScriptCore/jit/JITInlines.h
@@ -125,6 +125,14 @@
return nakedCall;
}
+ALWAYS_INLINE JIT::Call JIT::emitNakedTailCall(CodePtr function)
+{
+ ASSERT(m_bytecodeOffset != std::numeric_limits<unsigned>::max()); // This method should only be called during hot/cold path generation, so that m_bytecodeOffset is set.
+ Call nakedCall = nearTailCall();
+ m_calls.append(CallRecord(nakedCall, m_bytecodeOffset, function.executableAddress()));
+ return nakedCall;
+}
+
ALWAYS_INLINE void JIT::updateTopCallFrame()
{
ASSERT(static_cast<int>(m_bytecodeOffset) >= 0);
diff --git a/Source/JavaScriptCore/jit/JITOperations.cpp b/Source/JavaScriptCore/jit/JITOperations.cpp
index cf2d109..d703fda 100644
--- a/Source/JavaScriptCore/jit/JITOperations.cpp
+++ b/Source/JavaScriptCore/jit/JITOperations.cpp
@@ -682,14 +682,14 @@
return JSValue::encode(result);
}
-static void* handleHostCall(ExecState* execCallee, JSValue callee, CodeSpecializationKind kind)
+static SlowPathReturnType handleHostCall(ExecState* execCallee, JSValue callee, CallLinkInfo* callLinkInfo)
{
ExecState* exec = execCallee->callerFrame();
VM* vm = &exec->vm();
execCallee->setCodeBlock(0);
- if (kind == CodeForCall) {
+ if (callLinkInfo->specializationKind() == CodeForCall) {
CallData callData;
CallType callType = getCallData(callee, callData);
@@ -699,18 +699,25 @@
NativeCallFrameTracer tracer(vm, execCallee);
execCallee->setCallee(asObject(callee));
vm->hostCallReturnValue = JSValue::decode(callData.native.function(execCallee));
- if (vm->exception())
- return vm->getCTIStub(throwExceptionFromCallSlowPathGenerator).code().executableAddress();
+ if (vm->exception()) {
+ return encodeResult(
+ vm->getCTIStub(throwExceptionFromCallSlowPathGenerator).code().executableAddress(),
+ reinterpret_cast<void*>(KeepTheFrame));
+ }
- return reinterpret_cast<void*>(getHostCallReturnValue);
+ return encodeResult(
+ bitwise_cast<void*>(getHostCallReturnValue),
+ reinterpret_cast<void*>(callLinkInfo->callMode() == CallMode::Tail ? ReuseTheFrame : KeepTheFrame));
}
ASSERT(callType == CallTypeNone);
exec->vm().throwException(exec, createNotAFunctionError(exec, callee));
- return vm->getCTIStub(throwExceptionFromCallSlowPathGenerator).code().executableAddress();
+ return encodeResult(
+ vm->getCTIStub(throwExceptionFromCallSlowPathGenerator).code().executableAddress(),
+ reinterpret_cast<void*>(KeepTheFrame));
}
- ASSERT(kind == CodeForConstruct);
+ ASSERT(callLinkInfo->specializationKind() == CodeForConstruct);
ConstructData constructData;
ConstructType constructType = getConstructData(callee, constructData);
@@ -721,18 +728,23 @@
NativeCallFrameTracer tracer(vm, execCallee);
execCallee->setCallee(asObject(callee));
vm->hostCallReturnValue = JSValue::decode(constructData.native.function(execCallee));
- if (vm->exception())
- return vm->getCTIStub(throwExceptionFromCallSlowPathGenerator).code().executableAddress();
+ if (vm->exception()) {
+ return encodeResult(
+ vm->getCTIStub(throwExceptionFromCallSlowPathGenerator).code().executableAddress(),
+ reinterpret_cast<void*>(KeepTheFrame));
+ }
- return reinterpret_cast<void*>(getHostCallReturnValue);
+ return encodeResult(bitwise_cast<void*>(getHostCallReturnValue), reinterpret_cast<void*>(KeepTheFrame));
}
ASSERT(constructType == ConstructTypeNone);
exec->vm().throwException(exec, createNotAConstructorError(exec, callee));
- return vm->getCTIStub(throwExceptionFromCallSlowPathGenerator).code().executableAddress();
+ return encodeResult(
+ vm->getCTIStub(throwExceptionFromCallSlowPathGenerator).code().executableAddress(),
+ reinterpret_cast<void*>(KeepTheFrame));
}
-char* JIT_OPERATION operationLinkCall(ExecState* execCallee, CallLinkInfo* callLinkInfo)
+SlowPathReturnType JIT_OPERATION operationLinkCall(ExecState* execCallee, CallLinkInfo* callLinkInfo)
{
ExecState* exec = execCallee->callerFrame();
VM* vm = &exec->vm();
@@ -745,7 +757,7 @@
// FIXME: We should cache these kinds of calls. They can be common and currently they are
// expensive.
// https://bugs.webkit.org/show_bug.cgi?id=144458
- return reinterpret_cast<char*>(handleHostCall(execCallee, calleeAsValue, kind));
+ return handleHostCall(execCallee, calleeAsValue, callLinkInfo);
}
JSFunction* callee = jsCast<JSFunction*>(calleeAsFunctionCell);
@@ -774,17 +786,21 @@
if (!isCall(kind) && functionExecutable->constructAbility() == ConstructAbility::CannotConstruct) {
exec->vm().throwException(exec, createNotAConstructorError(exec, callee));
- return reinterpret_cast<char*>(vm->getCTIStub(throwExceptionFromCallSlowPathGenerator).code().executableAddress());
+ return encodeResult(
+ vm->getCTIStub(throwExceptionFromCallSlowPathGenerator).code().executableAddress(),
+ reinterpret_cast<void*>(KeepTheFrame));
}
JSObject* error = functionExecutable->prepareForExecution(execCallee, callee, scope, kind);
if (error) {
exec->vm().throwException(exec, error);
- return reinterpret_cast<char*>(vm->getCTIStub(throwExceptionFromCallSlowPathGenerator).code().executableAddress());
+ return encodeResult(
+ vm->getCTIStub(throwExceptionFromCallSlowPathGenerator).code().executableAddress(),
+ reinterpret_cast<void*>(KeepTheFrame));
}
codeBlock = functionExecutable->codeBlockFor(kind);
ArityCheckMode arity;
- if (execCallee->argumentCountIncludingThis() < static_cast<size_t>(codeBlock->numParameters()) || callLinkInfo->callType() == CallLinkInfo::CallVarargs || callLinkInfo->callType() == CallLinkInfo::ConstructVarargs)
+ if (execCallee->argumentCountIncludingThis() < static_cast<size_t>(codeBlock->numParameters()) || callLinkInfo->isVarargs())
arity = MustCheckArity;
else
arity = ArityCheckNotRequired;
@@ -795,10 +811,10 @@
else
linkFor(execCallee, *callLinkInfo, codeBlock, callee, codePtr);
- return reinterpret_cast<char*>(codePtr.executableAddress());
+ return encodeResult(codePtr.executableAddress(), reinterpret_cast<void*>(callLinkInfo->callMode() == CallMode::Tail ? ReuseTheFrame : KeepTheFrame));
}
-inline char* virtualForWithFunction(
+inline SlowPathReturnType virtualForWithFunction(
ExecState* execCallee, CallLinkInfo* callLinkInfo, JSCell*& calleeAsFunctionCell)
{
ExecState* exec = execCallee->callerFrame();
@@ -809,7 +825,7 @@
JSValue calleeAsValue = execCallee->calleeAsValue();
calleeAsFunctionCell = getJSFunction(calleeAsValue);
if (UNLIKELY(!calleeAsFunctionCell))
- return reinterpret_cast<char*>(handleHostCall(execCallee, calleeAsValue, kind));
+ return handleHostCall(execCallee, calleeAsValue, callLinkInfo);
JSFunction* function = jsCast<JSFunction*>(calleeAsFunctionCell);
JSScope* scope = function->scopeUnchecked();
@@ -824,13 +840,17 @@
if (!isCall(kind) && functionExecutable->constructAbility() == ConstructAbility::CannotConstruct) {
exec->vm().throwException(exec, createNotAConstructorError(exec, function));
- return reinterpret_cast<char*>(vm->getCTIStub(throwExceptionFromCallSlowPathGenerator).code().executableAddress());
+ return encodeResult(
+ vm->getCTIStub(throwExceptionFromCallSlowPathGenerator).code().executableAddress(),
+ reinterpret_cast<void*>(KeepTheFrame));
}
JSObject* error = functionExecutable->prepareForExecution(execCallee, function, scope, kind);
if (error) {
exec->vm().throwException(exec, error);
- return reinterpret_cast<char*>(vm->getCTIStub(throwExceptionFromCallSlowPathGenerator).code().executableAddress());
+ return encodeResult(
+ vm->getCTIStub(throwExceptionFromCallSlowPathGenerator).code().executableAddress(),
+ reinterpret_cast<void*>(KeepTheFrame));
}
} else {
#if ENABLE(WEBASSEMBLY)
@@ -844,22 +864,23 @@
#endif
}
}
- return reinterpret_cast<char*>(executable->entrypointFor(
- *vm, kind, MustCheckArity, callLinkInfo->registerPreservationMode()).executableAddress());
+ return encodeResult(executable->entrypointFor(
+ *vm, kind, MustCheckArity, callLinkInfo->registerPreservationMode()).executableAddress(),
+ reinterpret_cast<void*>(callLinkInfo->callMode() == CallMode::Tail ? ReuseTheFrame : KeepTheFrame));
}
-char* JIT_OPERATION operationLinkPolymorphicCall(ExecState* execCallee, CallLinkInfo* callLinkInfo)
+SlowPathReturnType JIT_OPERATION operationLinkPolymorphicCall(ExecState* execCallee, CallLinkInfo* callLinkInfo)
{
ASSERT(callLinkInfo->specializationKind() == CodeForCall);
JSCell* calleeAsFunctionCell;
- char* result = virtualForWithFunction(execCallee, callLinkInfo, calleeAsFunctionCell);
+ SlowPathReturnType result = virtualForWithFunction(execCallee, callLinkInfo, calleeAsFunctionCell);
linkPolymorphicCall(execCallee, *callLinkInfo, CallVariant(calleeAsFunctionCell));
return result;
}
-char* JIT_OPERATION operationVirtualCall(ExecState* execCallee, CallLinkInfo* callLinkInfo)
+SlowPathReturnType JIT_OPERATION operationVirtualCall(ExecState* execCallee, CallLinkInfo* callLinkInfo)
{
JSCell* calleeAsFunctionCellIgnored;
return virtualForWithFunction(execCallee, callLinkInfo, calleeAsFunctionCellIgnored);
diff --git a/Source/JavaScriptCore/jit/JITOperations.h b/Source/JavaScriptCore/jit/JITOperations.h
index d708bba..82c95f6 100644
--- a/Source/JavaScriptCore/jit/JITOperations.h
+++ b/Source/JavaScriptCore/jit/JITOperations.h
@@ -237,6 +237,7 @@
typedef char* JIT_OPERATION (*P_JITOperation_EStSS)(ExecState*, Structure*, size_t, size_t);
typedef char* JIT_OPERATION (*P_JITOperation_EStZ)(ExecState*, Structure*, int32_t);
typedef char* JIT_OPERATION (*P_JITOperation_EZZ)(ExecState*, int32_t, int32_t);
+typedef SlowPathReturnType JIT_OPERATION (*Sprt_JITOperation_ECli)(ExecState*, CallLinkInfo*);
typedef StringImpl* JIT_OPERATION (*T_JITOperation_EJss)(ExecState*, JSString*);
typedef JSString* JIT_OPERATION (*Jss_JITOperation_EZ)(ExecState*, int32_t);
@@ -278,9 +279,9 @@
void JIT_OPERATION operationPutByValGeneric(ExecState*, EncodedJSValue, EncodedJSValue, EncodedJSValue, ByValInfo*) WTF_INTERNAL;
void JIT_OPERATION operationDirectPutByValGeneric(ExecState*, EncodedJSValue, EncodedJSValue, EncodedJSValue, ByValInfo*) WTF_INTERNAL;
EncodedJSValue JIT_OPERATION operationCallEval(ExecState*, ExecState*) WTF_INTERNAL;
-char* JIT_OPERATION operationLinkCall(ExecState*, CallLinkInfo*) WTF_INTERNAL;
-char* JIT_OPERATION operationLinkPolymorphicCall(ExecState*, CallLinkInfo*) WTF_INTERNAL;
-char* JIT_OPERATION operationVirtualCall(ExecState*, CallLinkInfo*) WTF_INTERNAL;
+SlowPathReturnType JIT_OPERATION operationLinkCall(ExecState*, CallLinkInfo*) WTF_INTERNAL;
+SlowPathReturnType JIT_OPERATION operationLinkPolymorphicCall(ExecState*, CallLinkInfo*) WTF_INTERNAL;
+SlowPathReturnType JIT_OPERATION operationVirtualCall(ExecState*, CallLinkInfo*) WTF_INTERNAL;
size_t JIT_OPERATION operationCompareLess(ExecState*, EncodedJSValue, EncodedJSValue) WTF_INTERNAL;
size_t JIT_OPERATION operationCompareLessEq(ExecState*, EncodedJSValue, EncodedJSValue) WTF_INTERNAL;
diff --git a/Source/JavaScriptCore/jit/Repatch.cpp b/Source/JavaScriptCore/jit/Repatch.cpp
index 6bf4a07..90efb10 100644
--- a/Source/JavaScriptCore/jit/Repatch.cpp
+++ b/Source/JavaScriptCore/jit/Repatch.cpp
@@ -609,7 +609,7 @@
{
CodeBlock* callerCodeBlock = exec->callerFrame()->codeBlock();
VM* vm = callerCodeBlock->vm();
-
+
if (shouldShowDisassemblyFor(callerCodeBlock))
dataLog("Linking virtual call at ", *callerCodeBlock, " ", exec->callerFrame()->codeOrigin(), "\n");
@@ -680,7 +680,7 @@
codeBlock = jsCast<FunctionExecutable*>(executable)->codeBlockForCall();
// If we cannot handle a callee, assume that it's better for this whole thing to be a
// virtual call.
- if (exec->argumentCountIncludingThis() < static_cast<size_t>(codeBlock->numParameters()) || callLinkInfo.callType() == CallLinkInfo::CallVarargs || callLinkInfo.callType() == CallLinkInfo::ConstructVarargs) {
+ if (exec->argumentCountIncludingThis() < static_cast<size_t>(codeBlock->numParameters()) || callLinkInfo.isVarargs()) {
linkVirtualFor(exec, callLinkInfo);
return;
}
@@ -803,7 +803,11 @@
CCallHelpers::TrustedImm32(1),
CCallHelpers::Address(fastCountsBaseGPR, caseIndex * sizeof(uint32_t)));
}
- calls[caseIndex].call = stubJit.nearCall();
+ if (callLinkInfo.isTailCall()) {
+ stubJit.prepareForTailCallSlow();
+ calls[caseIndex].call = stubJit.nearTailCall();
+ } else
+ calls[caseIndex].call = stubJit.nearCall();
calls[caseIndex].codePtr = codePtr;
done.append(stubJit.jump());
}
diff --git a/Source/JavaScriptCore/jit/SetupVarargsFrame.cpp b/Source/JavaScriptCore/jit/SetupVarargsFrame.cpp
index 9e133cc..f43551e 100644
--- a/Source/JavaScriptCore/jit/SetupVarargsFrame.cpp
+++ b/Source/JavaScriptCore/jit/SetupVarargsFrame.cpp
@@ -37,6 +37,15 @@
void emitSetVarargsFrame(CCallHelpers& jit, GPRReg lengthGPR, bool lengthIncludesThis, GPRReg numUsedSlotsGPR, GPRReg resultGPR)
{
jit.move(numUsedSlotsGPR, resultGPR);
+ // We really want to make sure the size of the new call frame is a multiple of
+ // stackAlignmentRegisters(), however it is easier to accomplish this by
+ // rounding numUsedSlotsGPR to the next multiple of stackAlignmentRegisters().
+ // Together with the rounding below, we will assure that the new call frame is
+ // located on a stackAlignmentRegisters() boundary and a multiple of
+ // stackAlignmentRegisters() in size.
+ jit.addPtr(CCallHelpers::TrustedImm32(stackAlignmentRegisters() - 1), resultGPR);
+ jit.andPtr(CCallHelpers::TrustedImm32(~(stackAlignmentRegisters() - 1)), resultGPR);
+
jit.addPtr(lengthGPR, resultGPR);
jit.addPtr(CCallHelpers::TrustedImm32(JSStack::CallFrameHeaderSize + (lengthIncludesThis? 0 : 1)), resultGPR);
diff --git a/Source/JavaScriptCore/jit/ThunkGenerators.cpp b/Source/JavaScriptCore/jit/ThunkGenerators.cpp
index cb648bb..11289a5 100644
--- a/Source/JavaScriptCore/jit/ThunkGenerators.cpp
+++ b/Source/JavaScriptCore/jit/ThunkGenerators.cpp
@@ -79,10 +79,26 @@
}
static void slowPathFor(
- CCallHelpers& jit, VM* vm, P_JITOperation_ECli slowPathFunction)
+ CCallHelpers& jit, VM* vm, Sprt_JITOperation_ECli slowPathFunction)
{
jit.emitFunctionPrologue();
jit.storePtr(GPRInfo::callFrameRegister, &vm->topCallFrame);
+#if OS(WINDOWS) && CPU(X86_64)
+ // Windows X86_64 needs some space pointed to by arg0 for return types larger than 64 bits.
+ // Other argument values are shift by 1. Use space on the stack for our two return values.
+ // Moving the stack down maxFrameExtentForSlowPathCall bytes gives us room for our 3 arguments
+ // and space for the 16 byte return area.
+ jit.addPtr(CCallHelpers::TrustedImm32(-maxFrameExtentForSlowPathCall), CCallHelpers::stackPointerRegister);
+ jit.move(GPRInfo::regT2, GPRInfo::argumentGPR2);
+ jit.addPtr(CCallHelpers::TrustedImm32(32), CCallHelpers::stackPointerRegister, GPRInfo::argumentGPR0);
+ jit.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR1);
+ jit.move(CCallHelpers::TrustedImmPtr(bitwise_cast<void*>(slowPathFunction)), GPRInfo::nonArgGPR0);
+ emitPointerValidation(jit, GPRInfo::nonArgGPR0);
+ jit.call(GPRInfo::nonArgGPR0);
+ jit.loadPtr(CCallHelpers::Address(GPRInfo::returnValueGPR, 8), GPRInfo::returnValueGPR2);
+ jit.loadPtr(CCallHelpers::Address(GPRInfo::returnValueGPR), GPRInfo::returnValueGPR);
+ jit.addPtr(CCallHelpers::TrustedImm32(maxFrameExtentForSlowPathCall), CCallHelpers::stackPointerRegister);
+#else
if (maxFrameExtentForSlowPathCall)
jit.addPtr(CCallHelpers::TrustedImm32(-maxFrameExtentForSlowPathCall), CCallHelpers::stackPointerRegister);
jit.setupArgumentsWithExecState(GPRInfo::regT2);
@@ -91,13 +107,24 @@
jit.call(GPRInfo::nonArgGPR0);
if (maxFrameExtentForSlowPathCall)
jit.addPtr(CCallHelpers::TrustedImm32(maxFrameExtentForSlowPathCall), CCallHelpers::stackPointerRegister);
-
+#endif
+
// This slow call will return the address of one of the following:
// 1) Exception throwing thunk.
// 2) Host call return value returner thingy.
// 3) The function to call.
+ // The second return value GPR will hold a non-zero value for tail calls.
+
emitPointerValidation(jit, GPRInfo::returnValueGPR);
jit.emitFunctionEpilogue();
+
+ RELEASE_ASSERT(reinterpret_cast<void*>(KeepTheFrame) == reinterpret_cast<void*>(0));
+ CCallHelpers::Jump doNotTrash = jit.branchTestPtr(CCallHelpers::Zero, GPRInfo::returnValueGPR2);
+
+ jit.preserveReturnAddressAfterCall(GPRInfo::nonPreservedNonReturnGPR);
+ jit.prepareForTailCallSlow(GPRInfo::returnValueGPR);
+
+ doNotTrash.link(&jit);
jit.jump(GPRInfo::returnValueGPR);
}
@@ -108,7 +135,6 @@
// to perform linking and lazy compilation if necessary. We expect the callee
// to be in regT0/regT1 (payload/tag), the CallFrame to have already
// been adjusted, and all other registers to be available for use.
-
CCallHelpers jit(vm);
slowPathFor(jit, vm, operationLinkCall);
@@ -185,6 +211,10 @@
// Make a tail call. This will return back to JIT code.
emitPointerValidation(jit, GPRInfo::regT4);
+ if (callLinkInfo.isTailCall()) {
+ jit.preserveReturnAddressAfterCall(GPRInfo::regT0);
+ jit.prepareForTailCallSlow(GPRInfo::regT4);
+ }
jit.jump(GPRInfo::regT4);
slowCase.link(&jit);
@@ -196,9 +226,8 @@
LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(
patchBuffer,
- ("Virtual %s%s slow path thunk",
- callLinkInfo.specializationKind() == CodeForCall ? "call" : "construct",
- callLinkInfo.registerPreservationMode() == MustPreserveRegisters ? " that preserves registers" : ""));
+ ("Virtual %s slow path thunk",
+ callLinkInfo.callMode() == CallMode::Regular ? "call" : callLinkInfo.callMode() == CallMode::Tail ? "tail call" : "construct"));
}
enum ThunkEntryType { EnterViaCall, EnterViaJump };
@@ -367,7 +396,7 @@
{
JSInterfaceJIT jit(vm);
- // We enter with fixup count, in aligned stack units, in argumentGPR0 and the return thunk in argumentGPR1
+ // We enter with fixup count in argumentGPR0
// We have the guarantee that a0, a1, a2, t3, t4 and t5 (or t0 for Windows) are all distinct :-)
#if USE(JSVALUE64)
#if OS(WINDOWS)
@@ -378,12 +407,25 @@
# if CPU(X86_64)
jit.pop(JSInterfaceJIT::regT4);
# endif
- jit.lshift32(JSInterfaceJIT::TrustedImm32(logStackAlignmentRegisters()), JSInterfaceJIT::argumentGPR0);
- jit.neg64(JSInterfaceJIT::argumentGPR0);
jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT3);
jit.load32(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, JSStack::ArgumentCount * sizeof(Register)), JSInterfaceJIT::argumentGPR2);
jit.add32(JSInterfaceJIT::TrustedImm32(JSStack::CallFrameHeaderSize), JSInterfaceJIT::argumentGPR2);
+ // Check to see if we have extra slots we can use
+ jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::argumentGPR1);
+ jit.and32(JSInterfaceJIT::TrustedImm32(stackAlignmentRegisters() - 1), JSInterfaceJIT::argumentGPR1);
+ JSInterfaceJIT::Jump noExtraSlot = jit.branchTest32(MacroAssembler::Zero, JSInterfaceJIT::argumentGPR1);
+ jit.move(JSInterfaceJIT::TrustedImm64(ValueUndefined), extraTemp);
+ JSInterfaceJIT::Label fillExtraSlots(jit.label());
+ jit.store64(extraTemp, MacroAssembler::BaseIndex(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR2, JSInterfaceJIT::TimesEight));
+ jit.add32(JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2);
+ jit.branchSub32(JSInterfaceJIT::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR1).linkTo(fillExtraSlots, &jit);
+ jit.and32(JSInterfaceJIT::TrustedImm32(-stackAlignmentRegisters()), JSInterfaceJIT::argumentGPR0);
+ JSInterfaceJIT::Jump done = jit.branchTest32(MacroAssembler::Zero, JSInterfaceJIT::argumentGPR0);
+ noExtraSlot.link(&jit);
+
+ jit.neg64(JSInterfaceJIT::argumentGPR0);
+
// Move current frame down argumentGPR0 number of slots
JSInterfaceJIT::Label copyLoop(jit.label());
jit.load64(JSInterfaceJIT::regT3, extraTemp);
@@ -391,10 +433,9 @@
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2).linkTo(copyLoop, &jit);
- // Fill in argumentGPR0 - 1 missing arg slots with undefined
+ // Fill in argumentGPR0 missing arg slots with undefined
jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::argumentGPR2);
jit.move(JSInterfaceJIT::TrustedImm64(ValueUndefined), extraTemp);
- jit.add32(JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2);
JSInterfaceJIT::Label fillUndefinedLoop(jit.label());
jit.store64(extraTemp, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
@@ -406,6 +447,8 @@
jit.addPtr(extraTemp, JSInterfaceJIT::callFrameRegister);
jit.addPtr(extraTemp, JSInterfaceJIT::stackPointerRegister);
+ done.link(&jit);
+
# if CPU(X86_64)
jit.push(JSInterfaceJIT::regT4);
# endif
@@ -414,29 +457,43 @@
# if CPU(X86)
jit.pop(JSInterfaceJIT::regT4);
# endif
- jit.lshift32(JSInterfaceJIT::TrustedImm32(logStackAlignmentRegisters()), JSInterfaceJIT::argumentGPR0);
- jit.neg32(JSInterfaceJIT::argumentGPR0);
jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT3);
jit.load32(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, JSStack::ArgumentCount * sizeof(Register)), JSInterfaceJIT::argumentGPR2);
jit.add32(JSInterfaceJIT::TrustedImm32(JSStack::CallFrameHeaderSize), JSInterfaceJIT::argumentGPR2);
+ // Check to see if we have extra slots we can use
+ jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::argumentGPR1);
+ jit.and32(JSInterfaceJIT::TrustedImm32(stackAlignmentRegisters() - 1), JSInterfaceJIT::argumentGPR1);
+ JSInterfaceJIT::Jump noExtraSlot = jit.branchTest32(MacroAssembler::Zero, JSInterfaceJIT::argumentGPR1);
+ JSInterfaceJIT::Label fillExtraSlots(jit.label());
+ jit.move(JSInterfaceJIT::TrustedImm32(0), JSInterfaceJIT::regT5);
+ jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR2, JSInterfaceJIT::TimesEight, PayloadOffset));
+ jit.move(JSInterfaceJIT::TrustedImm32(JSValue::UndefinedTag), JSInterfaceJIT::regT5);
+ jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR2, JSInterfaceJIT::TimesEight, TagOffset));
+ jit.add32(JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2);
+ jit.branchSub32(JSInterfaceJIT::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR1).linkTo(fillExtraSlots, &jit);
+ jit.and32(JSInterfaceJIT::TrustedImm32(-stackAlignmentRegisters()), JSInterfaceJIT::argumentGPR0);
+ JSInterfaceJIT::Jump done = jit.branchTest32(MacroAssembler::Zero, JSInterfaceJIT::argumentGPR0);
+ noExtraSlot.link(&jit);
+
+ jit.neg32(JSInterfaceJIT::argumentGPR0);
+
// Move current frame down argumentGPR0 number of slots
JSInterfaceJIT::Label copyLoop(jit.label());
- jit.load32(JSInterfaceJIT::regT3, JSInterfaceJIT::regT5);
- jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight));
- jit.load32(MacroAssembler::Address(JSInterfaceJIT::regT3, 4), JSInterfaceJIT::regT5);
- jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, 4));
+ jit.load32(MacroAssembler::Address(JSInterfaceJIT::regT3, PayloadOffset), JSInterfaceJIT::regT5);
+ jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, PayloadOffset));
+ jit.load32(MacroAssembler::Address(JSInterfaceJIT::regT3, TagOffset), JSInterfaceJIT::regT5);
+ jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, TagOffset));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2).linkTo(copyLoop, &jit);
- // Fill in argumentGPR0 - 1 missing arg slots with undefined
+ // Fill in argumentGPR0 missing arg slots with undefined
jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::argumentGPR2);
- jit.add32(JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2);
JSInterfaceJIT::Label fillUndefinedLoop(jit.label());
jit.move(JSInterfaceJIT::TrustedImm32(0), JSInterfaceJIT::regT5);
- jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight));
+ jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, PayloadOffset));
jit.move(JSInterfaceJIT::TrustedImm32(JSValue::UndefinedTag), JSInterfaceJIT::regT5);
- jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, 4));
+ jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, TagOffset));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchAdd32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2).linkTo(fillUndefinedLoop, &jit);
@@ -447,6 +504,8 @@
jit.addPtr(JSInterfaceJIT::regT5, JSInterfaceJIT::callFrameRegister);
jit.addPtr(JSInterfaceJIT::regT5, JSInterfaceJIT::stackPointerRegister);
+ done.link(&jit);
+
# if CPU(X86)
jit.push(JSInterfaceJIT::regT4);
# endif
@@ -457,6 +516,16 @@
return FINALIZE_CODE(patchBuffer, ("fixup arity"));
}
+MacroAssemblerCodeRef unreachableGenerator(VM* vm)
+{
+ JSInterfaceJIT jit(vm);
+
+ jit.breakpoint();
+
+ LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID);
+ return FINALIZE_CODE(patchBuffer, ("unreachable thunk"));
+}
+
MacroAssemblerCodeRef baselineGetterReturnThunkGenerator(VM* vm)
{
JSInterfaceJIT jit(vm);
diff --git a/Source/JavaScriptCore/jit/ThunkGenerators.h b/Source/JavaScriptCore/jit/ThunkGenerators.h
index 01c2df6..52f79c6 100644
--- a/Source/JavaScriptCore/jit/ThunkGenerators.h
+++ b/Source/JavaScriptCore/jit/ThunkGenerators.h
@@ -34,9 +34,11 @@
namespace JSC {
class CallLinkInfo;
+class CCallHelpers;
MacroAssemblerCodeRef throwExceptionFromCallSlowPathGenerator(VM*);
+MacroAssemblerCodeRef linkCallThunk(VM*, CallLinkInfo&, CodeSpecializationKind, RegisterPreservationMode);
MacroAssemblerCodeRef linkCallThunkGenerator(VM*);
MacroAssemblerCodeRef linkPolymorphicCallThunkGenerator(VM*);
@@ -46,6 +48,7 @@
MacroAssemblerCodeRef nativeConstructGenerator(VM*);
MacroAssemblerCodeRef nativeTailCallGenerator(VM*);
MacroAssemblerCodeRef arityFixupGenerator(VM*);
+MacroAssemblerCodeRef unreachableGenerator(VM*);
MacroAssemblerCodeRef baselineGetterReturnThunkGenerator(VM* vm);
MacroAssemblerCodeRef baselineSetterReturnThunkGenerator(VM* vm);
diff --git a/Source/JavaScriptCore/llint/LowLevelInterpreter.asm b/Source/JavaScriptCore/llint/LowLevelInterpreter.asm
index 5b669e3..dc01126 100644
--- a/Source/JavaScriptCore/llint/LowLevelInterpreter.asm
+++ b/Source/JavaScriptCore/llint/LowLevelInterpreter.asm
@@ -167,6 +167,7 @@
const DirectArguments_storage = (sizeof DirectArguments + SlotSize - 1) & ~(SlotSize - 1)
const StackAlignment = 16
+const StackAlignmentSlots = 2
const StackAlignmentMask = StackAlignment - 1
const CallerFrameAndPCSize = 2 * PtrSize
@@ -697,36 +698,79 @@
end
end
-macro callTargetFunction(callLinkInfo, calleeFramePtr)
- move calleeFramePtr, sp
+macro callTargetFunction(callee)
if C_LOOP
- cloopCallJSFunction LLIntCallLinkInfo::machineCodeTarget[callLinkInfo]
+ cloopCallJSFunction callee
else
- call LLIntCallLinkInfo::machineCodeTarget[callLinkInfo]
+ call callee
end
restoreStackPointerAfterCall()
dispatchAfterCall()
end
-macro slowPathForCall(slowPath)
+macro prepareForRegularCall(callee, temp1, temp2, temp3)
+ addp CallerFrameAndPCSize, sp
+end
+
+# sp points to the new frame
+macro prepareForTailCall(callee, temp1, temp2, temp3)
+ restoreCalleeSavesUsedByLLInt()
+
+ loadi PayloadOffset + ArgumentCount[cfr], temp2
+ loadp CodeBlock[cfr], temp1
+ loadp CodeBlock::m_numParameters[temp1], temp1
+ bilteq temp1, temp2, .noArityFixup
+ move temp1, temp2
+
+.noArityFixup:
+ # We assume < 2^28 arguments
+ muli SlotSize, temp2
+ addi StackAlignment - 1 + CallFrameHeaderSize, temp2
+ andi ~StackAlignmentMask, temp2
+
+ move cfr, temp1
+ addp temp2, temp1
+
+ loadi PayloadOffset + ArgumentCount[sp], temp2
+ # We assume < 2^28 arguments
+ muli SlotSize, temp2
+ addi StackAlignment - 1 + CallFrameHeaderSize, temp2
+ andi ~StackAlignmentMask, temp2
+
+ if ARM or SH4 or ARM64 or C_LOOP or MIPS
+ addp 2 * PtrSize, sp
+ subi 2 * PtrSize, temp2
+ loadp PtrSize[cfr], lr
+ else
+ addp PtrSize, sp
+ subi PtrSize, temp2
+ loadp PtrSize[cfr], temp3
+ storep temp3, [sp]
+ end
+
+ subp temp2, temp1
+ loadp [cfr], cfr
+
+.copyLoop:
+ subi PtrSize, temp2
+ loadp [sp, temp2, 1], temp3
+ storep temp3, [temp1, temp2, 1]
+ btinz temp2, .copyLoop
+
+ move temp1, sp
+ jmp callee
+end
+
+macro slowPathForCall(slowPath, prepareCall)
callCallSlowPath(
slowPath,
- macro (callee, calleeFrame)
- btpz calleeFrame, .dontUpdateSP
- if ARMv7
- addp CallerFrameAndPCSize, calleeFrame, calleeFrame
- move calleeFrame, sp
- else
- addp CallerFrameAndPCSize, calleeFrame, sp
- end
+ # Those are r0 and r1
+ macro (callee, calleeFramePtr)
+ btpz calleeFramePtr, .dontUpdateSP
+ move calleeFramePtr, sp
+ prepareCall(callee, t2, t3, t4)
.dontUpdateSP:
- if C_LOOP
- cloopCallJSFunction callee
- else
- call callee
- end
- restoreStackPointerAfterCall()
- dispatchAfterCall()
+ callTargetFunction(callee)
end)
end
@@ -1415,49 +1459,50 @@
_llint_op_call:
traceExecution()
arrayProfileForCall()
- doCall(_llint_slow_path_call)
+ doCall(_llint_slow_path_call, prepareForRegularCall)
+_llint_op_tail_call:
+ traceExecution()
+ arrayProfileForCall()
+ checkSwitchToJITForEpilogue()
+ doCall(_llint_slow_path_call, prepareForTailCall)
_llint_op_construct:
traceExecution()
- doCall(_llint_slow_path_construct)
+ doCall(_llint_slow_path_construct, prepareForRegularCall)
+macro doCallVarargs(slowPath, prepareCall)
+ callSlowPath(_llint_slow_path_size_frame_for_varargs)
+ branchIfException(_llint_throw_from_slow_path_trampoline)
+ # calleeFrame in r1
+ if JSVALUE64
+ move r1, sp
+ else
+ # The calleeFrame is not stack aligned, move down by CallerFrameAndPCSize to align
+ if ARMv7
+ subp r1, CallerFrameAndPCSize, t2
+ move t2, sp
+ else
+ subp r1, CallerFrameAndPCSize, sp
+ end
+ end
+ slowPathForCall(slowPath, prepareCall)
+end
_llint_op_call_varargs:
traceExecution()
- callSlowPath(_llint_slow_path_size_frame_for_varargs)
- branchIfException(_llint_throw_from_slow_path_trampoline)
- # calleeFrame in r1
- if JSVALUE64
- move r1, sp
- else
- # The calleeFrame is not stack aligned, move down by CallerFrameAndPCSize to align
- if ARMv7
- subp r1, CallerFrameAndPCSize, t2
- move t2, sp
- else
- subp r1, CallerFrameAndPCSize, sp
- end
- end
- slowPathForCall(_llint_slow_path_call_varargs)
+ doCallVarargs(_llint_slow_path_call_varargs, prepareForRegularCall)
+
+_llint_op_tail_call_varargs:
+ traceExecution()
+ checkSwitchToJITForEpilogue()
+ # We lie and perform the tail call instead of preparing it since we can't
+ # prepare the frame for a call opcode
+ doCallVarargs(_llint_slow_path_call_varargs, prepareForTailCall)
_llint_op_construct_varargs:
traceExecution()
- callSlowPath(_llint_slow_path_size_frame_for_varargs)
- branchIfException(_llint_throw_from_slow_path_trampoline)
- # calleeFrame in r1
- if JSVALUE64
- move r1, sp
- else
- # The calleeFrame is not stack aligned, move down by CallerFrameAndPCSize to align
- if ARMv7
- subp r1, CallerFrameAndPCSize, t2
- move t2, sp
- else
- subp r1, CallerFrameAndPCSize, sp
- end
- end
- slowPathForCall(_llint_slow_path_construct_varargs)
+ doCallVarargs(_llint_slow_path_construct_varargs, prepareForRegularCall)
_llint_op_call_eval:
@@ -1496,7 +1541,7 @@
# and a PC to call, and that PC may be a dummy thunk that just
# returns the JS value that the eval returned.
- slowPathForCall(_llint_slow_path_call_eval)
+ slowPathForCall(_llint_slow_path_call_eval, prepareForRegularCall)
_llint_generic_return_point:
diff --git a/Source/JavaScriptCore/llint/LowLevelInterpreter32_64.asm b/Source/JavaScriptCore/llint/LowLevelInterpreter32_64.asm
index 13d7df6..7c51137 100644
--- a/Source/JavaScriptCore/llint/LowLevelInterpreter32_64.asm
+++ b/Source/JavaScriptCore/llint/LowLevelInterpreter32_64.asm
@@ -602,13 +602,27 @@
.proceedInline:
loadi CommonSlowPaths::ArityCheckData::paddedStackSpace[r1], t1
btiz t1, .continue
-
- // Move frame up "t1 * 2" slots
- lshiftp 1, t1
- negi t1
- move cfr, t3
loadi PayloadOffset + ArgumentCount[cfr], t2
addi CallFrameHeaderSlots, t2
+
+ // Check if there are some unaligned slots we can use
+ move t1, t3
+ andi StackAlignmentSlots - 1, t3
+ btiz t3, .noExtraSlot
+.fillExtraSlots:
+ move 0, t0
+ storei t0, PayloadOffset[cfr, t2, 8]
+ move UndefinedTag, t0
+ storei t0, TagOffset[cfr, t2, 8]
+ addi 1, t2
+ bsubinz 1, t3, .fillExtraSlots
+ andi ~(StackAlignmentSlots - 1), t1
+ btiz t1, .continue
+
+.noExtraSlot:
+ // Move frame up t1 slots
+ negi t1
+ move cfr, t3
.copyLoop:
loadi PayloadOffset[t3], t0
storei t0, PayloadOffset[t3, t1, 8]
@@ -1764,7 +1778,7 @@
.done:
end
-macro doCall(slowPath)
+macro doCall(slowPath, prepareCall)
loadi 8[PC], t0
loadi 20[PC], t1
loadp LLIntCallLinkInfo::callee[t1], t2
@@ -1779,14 +1793,14 @@
storei PC, ArgumentCount + TagOffset[cfr]
storei t2, ArgumentCount + PayloadOffset[t3]
storei CellTag, Callee + TagOffset[t3]
- addp CallerFrameAndPCSize, t3
- callTargetFunction(t1, t3)
+ move t3, sp
+ prepareCall(LLIntCallLinkInfo::machineCodeTarget[t1], t2, t3, t4)
+ callTargetFunction(LLIntCallLinkInfo::machineCodeTarget[t1])
.opCallSlow:
- slowPathForCall(slowPath)
+ slowPathForCall(slowPath, prepareCall)
end
-
_llint_op_ret:
traceExecution()
checkSwitchToJITForEpilogue()
diff --git a/Source/JavaScriptCore/llint/LowLevelInterpreter64.asm b/Source/JavaScriptCore/llint/LowLevelInterpreter64.asm
index 8ae2a0b..18d0ed1 100644
--- a/Source/JavaScriptCore/llint/LowLevelInterpreter64.asm
+++ b/Source/JavaScriptCore/llint/LowLevelInterpreter64.asm
@@ -510,14 +510,27 @@
.noError:
loadi CommonSlowPaths::ArityCheckData::paddedStackSpace[r1], t1
btiz t1, .continue
+ loadi PayloadOffset + ArgumentCount[cfr], t2
+ addi CallFrameHeaderSlots, t2
- // Move frame up "t1 * 2" slots
- lshiftp 1, t1
+ // Check if there are some unaligned slots we can use
+ move t1, t3
+ andi StackAlignmentSlots - 1, t3
+ btiz t3, .noExtraSlot
+ move ValueUndefined, t0
+.fillExtraSlots:
+ storeq t0, [cfr, t2, 8]
+ addi 1, t2
+ bsubinz 1, t3, .fillExtraSlots
+ andi ~(StackAlignmentSlots - 1), t1
+ btiz t1, .continue
+
+.noExtraSlot:
+ // Move frame up t1 slots
negq t1
move cfr, t3
subp CalleeSaveSpaceAsVirtualRegisters * 8, t3
- loadi PayloadOffset + ArgumentCount[cfr], t2
- addi CallFrameHeaderSlots + CalleeSaveSpaceAsVirtualRegisters, t2
+ addi CalleeSaveSpaceAsVirtualRegisters, t2
.copyLoop:
loadq [t3], t0
storeq t0, [t3, t1, 8]
@@ -1653,7 +1666,7 @@
.done:
end
-macro doCall(slowPath)
+macro doCall(slowPath, prepareCall)
loadisFromInstruction(2, t0)
loadpFromInstruction(5, t1)
loadp LLIntCallLinkInfo::callee[t1], t2
@@ -1667,14 +1680,14 @@
loadisFromInstruction(3, t2)
storei PC, ArgumentCount + TagOffset[cfr]
storei t2, ArgumentCount + PayloadOffset[t3]
- addp CallerFrameAndPCSize, t3
- callTargetFunction(t1, t3)
+ move t3, sp
+ prepareCall(LLIntCallLinkInfo::machineCodeTarget[t1], t2, t3, t4)
+ callTargetFunction(LLIntCallLinkInfo::machineCodeTarget[t1])
.opCallSlow:
- slowPathForCall(slowPath)
+ slowPathForCall(slowPath, prepareCall)
end
-
_llint_op_ret:
traceExecution()
checkSwitchToJITForEpilogue()
diff --git a/Source/JavaScriptCore/runtime/CommonSlowPaths.h b/Source/JavaScriptCore/runtime/CommonSlowPaths.h
index 92af43b..05762e5 100644
--- a/Source/JavaScriptCore/runtime/CommonSlowPaths.h
+++ b/Source/JavaScriptCore/runtime/CommonSlowPaths.h
@@ -59,13 +59,14 @@
int argumentCountIncludingThis = exec->argumentCountIncludingThis();
ASSERT(argumentCountIncludingThis < newCodeBlock->numParameters());
- int missingArgumentCount = newCodeBlock->numParameters() - argumentCountIncludingThis;
- int neededStackSpace = missingArgumentCount + 1; // Allow space to save the original return PC.
- int paddedStackSpace = WTF::roundUpToMultipleOf(stackAlignmentRegisters(), neededStackSpace);
+ int frameSize = argumentCountIncludingThis + JSStack::CallFrameHeaderSize;
+ int alignedFrameSizeForParameters = WTF::roundUpToMultipleOf(stackAlignmentRegisters(),
+ newCodeBlock->numParameters() + JSStack::CallFrameHeaderSize);
+ int paddedStackSpace = alignedFrameSizeForParameters - frameSize;
- if (!stack->ensureCapacityFor(exec->registers() - paddedStackSpace))
+ if (!stack->ensureCapacityFor(exec->registers() - paddedStackSpace % stackAlignmentRegisters()))
return -1;
- return paddedStackSpace / stackAlignmentRegisters();
+ return paddedStackSpace;
}
inline bool opIn(ExecState* exec, JSValue propName, JSValue baseVal)
