WebAssembly: running out of executable memory should throw OoM
https://bugs.webkit.org/show_bug.cgi?id=171537
<rdar://problem/32963338>

Reviewed by Saam Barati.

JSTests:

* wasm.yaml:
* wasm/lowExecutableMemory/executable-memory-oom.js: Added.
(const.invoke):
(failCount.0.catch):
(failCount.0.module.undefined.catch):
* wasm/lowExecutableMemory/exports-oom.js: Added.
(const.type):
(const.params):
(const.randomProgram):
(failCount.0.catch):
(failCount.0.module.undefined.catch):
* wasm/lowExecutableMemory/imports-oom.js: Added.
(const.type):
(const.params):
(const.randomProgram):
(f.imports.push):
(failCount.0.catch):
(failCount.0.module.undefined.catch):

Source/JavaScriptCore:

Both on first compile with BBQ as well as on tier-up with OMG,
running out of X memory shouldn't cause the entire program to
terminate. An exception will do when compiling initial code (since
we don't have any other fallback at the moment), and refusal to
tier up will do as well (it'll just be slower).

This is useful because programs which generate huge amounts of
code simply look like crashes, which developers report to
us. Getting a JavaScript exception instead is much clearer.

* jit/ExecutableAllocator.cpp:
(JSC::ExecutableAllocator::allocate):
* llint/LLIntSlowPaths.cpp:
(JSC::LLInt::shouldJIT):
* runtime/Options.h:
* wasm/WasmBBQPlan.cpp:
(JSC::Wasm::BBQPlan::prepare):
(JSC::Wasm::BBQPlan::complete):
* wasm/WasmBinding.cpp:
(JSC::Wasm::wasmToJs):
(JSC::Wasm::wasmToWasm):
* wasm/WasmBinding.h:
* wasm/WasmOMGPlan.cpp:
(JSC::Wasm::OMGPlan::work):
* wasm/js/JSWebAssemblyCodeBlock.cpp:
(JSC::JSWebAssemblyCodeBlock::JSWebAssemblyCodeBlock):
* wasm/js/JSWebAssemblyCodeBlock.h:
* wasm/js/JSWebAssemblyInstance.cpp:
(JSC::JSWebAssemblyInstance::finalizeCreation):

Tools:

* Scripts/run-jsc-stress-tests: add a configuration which runs the
tests under limited executable memory and avoids non-WebAssembly
code generation so that we more reliably run out of executable
memory in WebAssembly.


git-svn-id: http://svn.webkit.org/repository/webkit/trunk@218868 268f45cc-cd09-0410-ab3c-d52691b4dbfc
diff --git a/Source/JavaScriptCore/wasm/WasmBinding.cpp b/Source/JavaScriptCore/wasm/WasmBinding.cpp
index 36d8b15..8f9aa0c 100644
--- a/Source/JavaScriptCore/wasm/WasmBinding.cpp
+++ b/Source/JavaScriptCore/wasm/WasmBinding.cpp
@@ -51,7 +51,7 @@
     jit.loadPtr(JIT::Address(result, JSWebAssemblyInstance::offsetOfImportFunction(importIndex)), result);
 }
 
-MacroAssemblerCodeRef wasmToJs(VM* vm, Bag<CallLinkInfo>& callLinkInfos, SignatureIndex signatureIndex, unsigned importIndex)
+Expected<MacroAssemblerCodeRef, BindingFailure> wasmToJs(VM* vm, Bag<CallLinkInfo>& callLinkInfos, SignatureIndex signatureIndex, unsigned importIndex)
 {
     // FIXME: This function doesn't properly abstract away the calling convention.
     // It'd be super easy to do so: https://bugs.webkit.org/show_bug.cgi?id=169401
@@ -119,7 +119,10 @@
                 ASSERT(!!vm->callFrameForCatch);
             };
 
-            LinkBuffer linkBuffer(jit, GLOBAL_THUNK_ID);
+            LinkBuffer linkBuffer(jit, GLOBAL_THUNK_ID, JITCompilationCanFail);
+            if (UNLIKELY(linkBuffer.didFailToAllocate()))
+                return makeUnexpected(BindingFailure::OutOfMemory);
+
             linkBuffer.link(call, throwBadI64);
             return FINALIZE_CODE(linkBuffer, ("WebAssembly->JavaScript invalid i64 use in import[%i]", importIndex));
         }
@@ -303,7 +306,10 @@
         jit.emitFunctionEpilogue();
         jit.ret();
 
-        LinkBuffer linkBuffer(jit, GLOBAL_THUNK_ID);
+        LinkBuffer linkBuffer(jit, GLOBAL_THUNK_ID, JITCompilationCanFail);
+        if (UNLIKELY(linkBuffer.didFailToAllocate()))
+            return makeUnexpected(BindingFailure::OutOfMemory);
+
         linkBuffer.link(call, callFunc);
         linkBuffer.link(exceptionCall, doUnwinding);
 
@@ -600,7 +606,10 @@
         });
     }
 
-    LinkBuffer patchBuffer(jit, GLOBAL_THUNK_ID);
+    LinkBuffer patchBuffer(jit, GLOBAL_THUNK_ID, JITCompilationCanFail);
+    if (UNLIKELY(patchBuffer.didFailToAllocate()))
+        return makeUnexpected(BindingFailure::OutOfMemory);
+
     patchBuffer.link(slowCall, FunctionPtr(vm->getCTIStub(linkCallThunkGenerator).code().executableAddress()));
     CodeLocationLabel callReturnLocation(patchBuffer.locationOfNearCall(slowCall));
     CodeLocationLabel hotPathBegin(patchBuffer.locationOf(targetToCheck));
@@ -610,7 +619,7 @@
     return FINALIZE_CODE(patchBuffer, ("WebAssembly->JavaScript import[%i] %s", importIndex, signature.toString().ascii().data()));
 }
 
-MacroAssemblerCodeRef wasmToWasm(unsigned importIndex)
+Expected<MacroAssemblerCodeRef, BindingFailure> wasmToWasm(unsigned importIndex)
 {
     const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
     JIT jit;
@@ -653,7 +662,10 @@
     jit.loadPtr(scratch, scratch);
     jit.jump(scratch);
 
-    LinkBuffer patchBuffer(jit, GLOBAL_THUNK_ID);
+    LinkBuffer patchBuffer(jit, GLOBAL_THUNK_ID, JITCompilationCanFail);
+    if (UNLIKELY(patchBuffer.didFailToAllocate()))
+        return makeUnexpected(BindingFailure::OutOfMemory);
+
     return FINALIZE_CODE(patchBuffer, ("WebAssembly->WebAssembly import[%i]", importIndex));
 }