Web Automation: Add Automation.screenshot
https://bugs.webkit.org/show_bug.cgi?id=156073
<rdar://problem/25468646>
Reviewed by Timothy Hatcher and Brian Burg.
* UIProcess/Automation/Automation.json:
Add Automation.screenshot which returns base64 encoded PNG data.
* WebProcess/Automation/WebAutomationSessionProxy.h:
* WebProcess/Automation/WebAutomationSessionProxy.messages.in:
* UIProcess/Automation/WebAutomationSession.cpp:
(WebKit::WebAutomationSession::screenshot):
(WebKit::WebAutomationSession::didTakeScreenshot):
(WebKit::WebAutomationSession::platformGetBase64EncodedPNGData):
Send a message to the WebPage to get an Image of the page
and handle the response.
* UIProcess/Cocoa/WebAutomationSessionCocoa.mm:
(WebKit::WebAutomationSession::platformGetBase64EncodedPNGData):
Convert the bitmap data to a base64 encoded PNG.
* UIProcess/Automation/WebAutomationSession.h:
* UIProcess/Automation/WebAutomationSession.messages.in:
* WebProcess/Automation/WebAutomationSessionProxy.cpp:
(WebKit::evaluateJavaScriptCallback):
(WebKit::WebAutomationSessionProxy::didClearWindowObjectForFrame):
(WebKit::WebAutomationSessionProxy::resolveChildFrameWithOrdinal):
(WebKit::WebAutomationSessionProxy::resolveChildFrameWithNodeHandle):
(WebKit::WebAutomationSessionProxy::resolveChildFrameWithName):
(WebKit::WebAutomationSessionProxy::resolveParentFrame):
(WebKit::WebAutomationSessionProxy::computeElementLayout):
Use null string where possible for efficiency.
(WebKit::WebAutomationSessionProxy::takeScreenshot):
Take an image of the entire page and pass back to the UIProcess.
git-svn-id: http://svn.webkit.org/repository/webkit/trunk@198907 268f45cc-cd09-0410-ab3c-d52691b4dbfc
diff --git a/Source/WebKit2/ChangeLog b/Source/WebKit2/ChangeLog
index cd2811c..1885df7 100644
--- a/Source/WebKit2/ChangeLog
+++ b/Source/WebKit2/ChangeLog
@@ -1,3 +1,42 @@
+2016-03-31 Joseph Pecoraro <pecoraro@apple.com>
+
+ Web Automation: Add Automation.screenshot
+ https://bugs.webkit.org/show_bug.cgi?id=156073
+ <rdar://problem/25468646>
+
+ Reviewed by Timothy Hatcher and Brian Burg.
+
+ * UIProcess/Automation/Automation.json:
+ Add Automation.screenshot which returns base64 encoded PNG data.
+
+ * WebProcess/Automation/WebAutomationSessionProxy.h:
+ * WebProcess/Automation/WebAutomationSessionProxy.messages.in:
+ * UIProcess/Automation/WebAutomationSession.cpp:
+ (WebKit::WebAutomationSession::screenshot):
+ (WebKit::WebAutomationSession::didTakeScreenshot):
+ (WebKit::WebAutomationSession::platformGetBase64EncodedPNGData):
+ Send a message to the WebPage to get an Image of the page
+ and handle the response.
+
+ * UIProcess/Cocoa/WebAutomationSessionCocoa.mm:
+ (WebKit::WebAutomationSession::platformGetBase64EncodedPNGData):
+ Convert the bitmap data to a base64 encoded PNG.
+
+ * UIProcess/Automation/WebAutomationSession.h:
+ * UIProcess/Automation/WebAutomationSession.messages.in:
+ * WebProcess/Automation/WebAutomationSessionProxy.cpp:
+ (WebKit::evaluateJavaScriptCallback):
+ (WebKit::WebAutomationSessionProxy::didClearWindowObjectForFrame):
+ (WebKit::WebAutomationSessionProxy::resolveChildFrameWithOrdinal):
+ (WebKit::WebAutomationSessionProxy::resolveChildFrameWithNodeHandle):
+ (WebKit::WebAutomationSessionProxy::resolveChildFrameWithName):
+ (WebKit::WebAutomationSessionProxy::resolveParentFrame):
+ (WebKit::WebAutomationSessionProxy::computeElementLayout):
+ Use null string where possible for efficiency.
+
+ (WebKit::WebAutomationSessionProxy::takeScreenshot):
+ Take an image of the entire page and pass back to the UIProcess.
+
2016-03-31 Timothy Hatcher <timothy@apple.com>
Web Automation: Handle undefined when calling a function that has no result
diff --git a/Source/WebKit2/UIProcess/Automation/Automation.json b/Source/WebKit2/UIProcess/Automation/Automation.json
index 2fc931d..bfb18fc2 100644
--- a/Source/WebKit2/UIProcess/Automation/Automation.json
+++ b/Source/WebKit2/UIProcess/Automation/Automation.json
@@ -305,11 +305,22 @@
"name": "performKeyboardInteractions",
"description": "Simulates delivering the results of pressing one or more keyboard keys together or successively.",
"parameters": [
- { "name": "handle", "$ref": "BrowsingContextHandle", "description": "The handle for the browsing context which should recieve key." },
+ { "name": "handle", "$ref": "BrowsingContextHandle", "description": "The handle for the browsing context which should receive key." },
{ "name": "interactions", "type": "array", "items": { "$ref": "KeyboardInteraction" }, "description": "An ordered list of key sequences to be delivered using native key events." }
]
},
{
+ "name": "takeScreenshot",
+ "description": "Take a screenshot of the current page in a browsing context.",
+ "parameters": [
+ { "name": "handle", "$ref": "BrowsingContextHandle", "description": "The handle for the browsing context to take a screenshot of." }
+ ],
+ "returns": [
+ { "name": "data", "type": "string", "description": "Base64-encoded image data (PNG)." }
+ ],
+ "async": true
+ },
+ {
"name": "resolveChildFrameHandle",
"description": "Determines the <code>FrameHandle</code> based on the ordinal, name or node handle of a child frame.",
"parameters": [
diff --git a/Source/WebKit2/UIProcess/Automation/WebAutomationSession.cpp b/Source/WebKit2/UIProcess/Automation/WebAutomationSession.cpp
index 3028a7e..14949c2 100644
--- a/Source/WebKit2/UIProcess/Automation/WebAutomationSession.cpp
+++ b/Source/WebKit2/UIProcess/Automation/WebAutomationSession.cpp
@@ -771,10 +771,41 @@
for (auto& action : actionsToPerform)
action();
}
-
#endif // USE(APPKIT)
}
+void WebAutomationSession::takeScreenshot(ErrorString& errorString, const String& handle, Ref<TakeScreenshotCallback>&& callback)
+{
+ WebPageProxy* page = webPageProxyForHandle(handle);
+ if (!page)
+ FAIL_WITH_PREDEFINED_ERROR_MESSAGE(WindowNotFound);
+
+ uint64_t callbackID = m_nextScreenshotCallbackID++;
+ m_screenshotCallbacks.set(callbackID, WTFMove(callback));
+
+ page->process().send(Messages::WebAutomationSessionProxy::TakeScreenshot(page->pageID(), callbackID), 0);
+}
+
+void WebAutomationSession::didTakeScreenshot(uint64_t callbackID, const ShareableBitmap::Handle& imageDataHandle, const String& errorType)
+{
+ auto callback = m_screenshotCallbacks.take(callbackID);
+ if (!callback)
+ return;
+
+ if (!errorType.isEmpty()) {
+ callback->sendFailure(errorType);
+ return;
+ }
+
+ String base64EncodedData = platformGetBase64EncodedPNGData(imageDataHandle);
+ if (base64EncodedData.isEmpty()) {
+ callback->sendFailure(Inspector::Protocol::getEnumConstantValue(Inspector::Protocol::Automation::ErrorMessage::InternalError));
+ return;
+ }
+
+ callback->sendSuccess(base64EncodedData);
+}
+
#if !USE(APPKIT)
void WebAutomationSession::platformSimulateMouseInteraction(WebKit::WebPageProxy&, const WebCore::IntPoint&, Inspector::Protocol::Automation::MouseInteraction, Inspector::Protocol::Automation::MouseButton, WebEvent::Modifiers)
{
@@ -787,6 +818,11 @@
void WebAutomationSession::platformSimulateKeySequence(WebPageProxy&, const String&)
{
}
+
+String WebAutomationSession::platformGetBase64EncodedPNGData(const ShareableBitmap::Handle&)
+{
+ return String();
+}
#endif // !USE(APPKIT)
} // namespace WebKit
diff --git a/Source/WebKit2/UIProcess/Automation/WebAutomationSession.h b/Source/WebKit2/UIProcess/Automation/WebAutomationSession.h
index f937d7a..f0e5225 100644
--- a/Source/WebKit2/UIProcess/Automation/WebAutomationSession.h
+++ b/Source/WebKit2/UIProcess/Automation/WebAutomationSession.h
@@ -28,6 +28,7 @@
#include "APIObject.h"
#include "AutomationBackendDispatchers.h"
#include "Connection.h"
+#include "ShareableBitmap.h"
#include "WebEvent.h"
#include <wtf/Forward.h>
@@ -105,6 +106,7 @@
void evaluateJavaScriptFunction(Inspector::ErrorString&, const String& browsingContextHandle, const String* optionalFrameHandle, const String& function, const Inspector::InspectorArray& arguments, bool expectsImplicitCallbackArgument, Ref<Inspector::AutomationBackendDispatcherHandler::EvaluateJavaScriptFunctionCallback>&&) override;
void performMouseInteraction(Inspector::ErrorString&, const String& handle, const Inspector::InspectorObject& requestedPosition, const String& mouseButton, const String& mouseInteraction, const Inspector::InspectorArray& keyModifiers, RefPtr<Inspector::Protocol::Automation::Point>& updatedPosition) override;
void performKeyboardInteractions(Inspector::ErrorString&, const String& handle, const Inspector::InspectorArray& interactions) override;
+ void takeScreenshot(Inspector::ErrorString&, const String& handle, Ref<Inspector::AutomationBackendDispatcherHandler::TakeScreenshotCallback>&&) override;
void resolveChildFrameHandle(Inspector::ErrorString&, const String& browsingContextHandle, const String* optionalFrameHandle, const int* optionalOrdinal, const String* optionalName, const String* optionalNodeHandle, Ref<ResolveChildFrameHandleCallback>&&) override;
void resolveParentFrameHandle(Inspector::ErrorString&, const String& browsingContextHandle, const String& frameHandle, Ref<ResolveParentFrameHandleCallback>&&) override;
void computeElementLayout(Inspector::ErrorString&, const String& browsingContextHandle, const String& frameHandle, const String& nodeHandle, const bool* optionalScrollIntoViewIfNeeded, const bool* useViewportCoordinates, Ref<Inspector::AutomationBackendDispatcherHandler::ComputeElementLayoutCallback>&&) override;
@@ -135,6 +137,7 @@
void didResolveChildFrame(uint64_t callbackID, uint64_t frameID, const String& errorType);
void didResolveParentFrame(uint64_t callbackID, uint64_t frameID, const String& errorType);
void didComputeElementLayout(uint64_t callbackID, WebCore::IntRect, const String& errorType);
+ void didTakeScreenshot(uint64_t callbackID, const ShareableBitmap::Handle&, const String& errorType);
// Platform-specific helper methods.
void platformSimulateMouseInteraction(WebPageProxy&, const WebCore::IntPoint& viewPosition, Inspector::Protocol::Automation::MouseInteraction, Inspector::Protocol::Automation::MouseButton, WebEvent::Modifiers);
@@ -142,6 +145,8 @@
void platformSimulateKeyStroke(WebPageProxy&, Inspector::Protocol::Automation::KeyboardInteractionType, Inspector::Protocol::Automation::VirtualKey);
// Simulates key presses to produce the codepoints in a string. One or more code points are delivered atomically at grapheme cluster boundaries.
void platformSimulateKeySequence(WebPageProxy&, const String&);
+ // Get base64 encoded PNG data from a bitmap.
+ String platformGetBase64EncodedPNGData(const ShareableBitmap::Handle&);
#if USE(APPKIT)
void sendSynthesizedEventsToPage(WebPageProxy&, NSArray *eventsToSend);
@@ -174,6 +179,9 @@
uint64_t m_nextComputeElementLayoutCallbackID { 1 };
HashMap<uint64_t, RefPtr<Inspector::AutomationBackendDispatcherHandler::ComputeElementLayoutCallback>> m_computeElementLayoutCallbacks;
+ uint64_t m_nextScreenshotCallbackID { 1 };
+ HashMap<uint64_t, RefPtr<Inspector::AutomationBackendDispatcherHandler::TakeScreenshotCallback>> m_screenshotCallbacks;
+
#if ENABLE(REMOTE_INSPECTOR)
Inspector::FrontendChannel* m_remoteChannel { nullptr };
#endif
diff --git a/Source/WebKit2/UIProcess/Automation/WebAutomationSession.messages.in b/Source/WebKit2/UIProcess/Automation/WebAutomationSession.messages.in
index 72d9a93..5c3b7a0e 100644
--- a/Source/WebKit2/UIProcess/Automation/WebAutomationSession.messages.in
+++ b/Source/WebKit2/UIProcess/Automation/WebAutomationSession.messages.in
@@ -27,4 +27,6 @@
DidResolveParentFrame(uint64_t callbackID, uint64_t frameID, String errorType)
DidComputeElementLayout(uint64_t callbackID, WebCore::IntRect rect, String errorType)
+
+ DidTakeScreenshot(uint64_t callbackID, WebKit::ShareableBitmap::Handle imageDataHandle, String errorType)
}
diff --git a/Source/WebKit2/UIProcess/Cocoa/WebAutomationSessionCocoa.mm b/Source/WebKit2/UIProcess/Cocoa/WebAutomationSessionCocoa.mm
index 5bca6bfb..c3ab665 100644
--- a/Source/WebKit2/UIProcess/Cocoa/WebAutomationSessionCocoa.mm
+++ b/Source/WebKit2/UIProcess/Cocoa/WebAutomationSessionCocoa.mm
@@ -454,6 +454,21 @@
sendSynthesizedEventsToPage(page, eventsToBeSent.get());
}
+String WebAutomationSession::platformGetBase64EncodedPNGData(const ShareableBitmap::Handle& imageDataHandle)
+{
+ RefPtr<ShareableBitmap> bitmap = ShareableBitmap::create(imageDataHandle, SharedMemory::Protection::ReadOnly);
+ RetainPtr<CGImageRef> cgImage = bitmap->makeCGImage();
+ RetainPtr<NSMutableData> imageData = adoptNS([[NSMutableData alloc] init]);
+ RetainPtr<CGImageDestinationRef> destination = adoptCF(CGImageDestinationCreateWithData((CFMutableDataRef)imageData.get(), kUTTypePNG, 1, 0));
+ if (!destination)
+ return String();
+
+ CGImageDestinationAddImage(destination.get(), cgImage.get(), 0);
+ CGImageDestinationFinalize(destination.get());
+
+ return [imageData base64EncodedStringWithOptions:0];
+}
+
#endif // USE(APPKIT)
} // namespace WebKit
diff --git a/Source/WebKit2/WebProcess/Automation/WebAutomationSessionProxy.cpp b/Source/WebKit2/WebProcess/Automation/WebAutomationSessionProxy.cpp
index 7bd11d3..8c3e5fc 100644
--- a/Source/WebKit2/WebProcess/Automation/WebAutomationSessionProxy.cpp
+++ b/Source/WebKit2/WebProcess/Automation/WebAutomationSessionProxy.cpp
@@ -31,6 +31,7 @@
#include "WebAutomationSessionProxyMessages.h"
#include "WebAutomationSessionProxyScriptSource.h"
#include "WebFrame.h"
+#include "WebImage.h"
#include "WebPage.h"
#include "WebProcess.h"
#include <JavaScriptCore/APICast.h>
@@ -138,7 +139,7 @@
uint64_t callbackID = JSValueToNumber(context, arguments[1], exception);
JSRetainPtr<JSStringRef> result(Adopt, JSValueToStringCopy(context, arguments[2], exception));
- automationSessionProxy->didEvaluateJavaScriptFunction(frameID, callbackID, result->string(), emptyString());
+ automationSessionProxy->didEvaluateJavaScriptFunction(frameID, callbackID, result->string(), String());
return JSValueMakeUndefined(context);
}
@@ -208,7 +209,7 @@
auto pendingFrameCallbacks = m_webFramePendingEvaluateJavaScriptCallbacksMap.take(frameID);
for (uint64_t callbackID : pendingFrameCallbacks)
- WebProcess::singleton().parentProcessConnection()->send(Messages::WebAutomationSession::DidEvaluateJavaScriptFunction(callbackID, emptyString(), errorType), 0);
+ WebProcess::singleton().parentProcessConnection()->send(Messages::WebAutomationSession::DidEvaluateJavaScriptFunction(callbackID, String(), errorType), 0);
}
void WebAutomationSessionProxy::evaluateJavaScriptFunction(uint64_t frameID, const String& function, Vector<String> arguments, bool expectsImplicitCallbackArgument, uint64_t callbackID)
@@ -303,7 +304,7 @@
return;
}
- WebProcess::singleton().parentProcessConnection()->send(Messages::WebAutomationSession::DidResolveChildFrame(callbackID, childFrame->frameID(), emptyString()), 0);
+ WebProcess::singleton().parentProcessConnection()->send(Messages::WebAutomationSession::DidResolveChildFrame(callbackID, childFrame->frameID(), String()), 0);
}
void WebAutomationSessionProxy::resolveChildFrameWithNodeHandle(uint64_t frameID, const String& nodeHandle, uint64_t callbackID)
@@ -334,7 +335,7 @@
return;
}
- WebProcess::singleton().parentProcessConnection()->send(Messages::WebAutomationSession::DidResolveChildFrame(callbackID, frameFromElement->frameID(), emptyString()), 0);
+ WebProcess::singleton().parentProcessConnection()->send(Messages::WebAutomationSession::DidResolveChildFrame(callbackID, frameFromElement->frameID(), String()), 0);
}
void WebAutomationSessionProxy::resolveChildFrameWithName(uint64_t frameID, const String& name, uint64_t callbackID)
@@ -365,7 +366,7 @@
return;
}
- WebProcess::singleton().parentProcessConnection()->send(Messages::WebAutomationSession::DidResolveChildFrame(callbackID, childFrame->frameID(), emptyString()), 0);
+ WebProcess::singleton().parentProcessConnection()->send(Messages::WebAutomationSession::DidResolveChildFrame(callbackID, childFrame->frameID(), String()), 0);
}
void WebAutomationSessionProxy::resolveParentFrame(uint64_t frameID, uint64_t callbackID)
@@ -384,7 +385,7 @@
return;
}
- WebProcess::singleton().parentProcessConnection()->send(Messages::WebAutomationSession::DidResolveParentFrame(callbackID, parentFrame->frameID(), emptyString()), 0);
+ WebProcess::singleton().parentProcessConnection()->send(Messages::WebAutomationSession::DidResolveParentFrame(callbackID, parentFrame->frameID(), String()), 0);
}
void WebAutomationSessionProxy::focusFrame(uint64_t frameID)
@@ -446,7 +447,37 @@
rect = coreFrameView->rootViewToContents(rect);
}
- WebProcess::singleton().parentProcessConnection()->send(Messages::WebAutomationSession::DidComputeElementLayout(callbackID, rect, emptyString()), 0);
+ WebProcess::singleton().parentProcessConnection()->send(Messages::WebAutomationSession::DidComputeElementLayout(callbackID, rect, String()), 0);
+}
+
+void WebAutomationSessionProxy::takeScreenshot(uint64_t pageID, uint64_t callbackID)
+{
+ ShareableBitmap::Handle handle;
+ String windowNotFoundErrorType = Inspector::Protocol::getEnumConstantValue(Inspector::Protocol::Automation::ErrorMessage::WindowNotFound);
+
+ WebPage* page = WebProcess::singleton().webPage(pageID);
+ if (!page) {
+ WebProcess::singleton().parentProcessConnection()->send(Messages::WebAutomationSession::DidTakeScreenshot(callbackID, handle, windowNotFoundErrorType), 0);
+ return;
+ }
+
+ WebCore::FrameView* frameView = page->mainFrameView();
+ if (!frameView) {
+ WebProcess::singleton().parentProcessConnection()->send(Messages::WebAutomationSession::DidTakeScreenshot(callbackID, handle, String()), 0);
+ return;
+ }
+
+ WebCore::IntRect snapshotRect = WebCore::IntRect(WebCore::IntPoint(0, 0), frameView->contentsSize());
+ if (snapshotRect.isEmpty()) {
+ WebProcess::singleton().parentProcessConnection()->send(Messages::WebAutomationSession::DidTakeScreenshot(callbackID, handle, String()), 0);
+ return;
+ }
+
+ RefPtr<WebImage> image = page->scaledSnapshotWithOptions(snapshotRect, 1, SnapshotOptionsShareable);
+ if (image)
+ image->bitmap()->createHandle(handle, SharedMemory::Protection::ReadOnly);
+
+ WebProcess::singleton().parentProcessConnection()->send(Messages::WebAutomationSession::DidTakeScreenshot(callbackID, handle, String()), 0);
}
} // namespace WebKit
diff --git a/Source/WebKit2/WebProcess/Automation/WebAutomationSessionProxy.h b/Source/WebKit2/WebProcess/Automation/WebAutomationSessionProxy.h
index 15cdfd0..ab83a3d 100644
--- a/Source/WebKit2/WebProcess/Automation/WebAutomationSessionProxy.h
+++ b/Source/WebKit2/WebProcess/Automation/WebAutomationSessionProxy.h
@@ -65,6 +65,7 @@
void resolveParentFrame(uint64_t frameID, uint64_t callbackID);
void focusFrame(uint64_t frameID);
void computeElementLayout(uint64_t frameID, String nodeHandle, bool scrollIntoViewIfNeeded, bool useViewportCoordinates, uint64_t callbackID);
+ void takeScreenshot(uint64_t pageID, uint64_t callbackID);
String m_sessionIdentifier;
diff --git a/Source/WebKit2/WebProcess/Automation/WebAutomationSessionProxy.messages.in b/Source/WebKit2/WebProcess/Automation/WebAutomationSessionProxy.messages.in
index f7231e7..62acee8 100644
--- a/Source/WebKit2/WebProcess/Automation/WebAutomationSessionProxy.messages.in
+++ b/Source/WebKit2/WebProcess/Automation/WebAutomationSessionProxy.messages.in
@@ -31,4 +31,6 @@
FocusFrame(uint64_t frameID)
ComputeElementLayout(uint64_t frameID, String nodeHandle, bool scrollIntoViewIfNeeded, bool useViewportCoordinates, uint64_t callbackID)
+
+ TakeScreenshot(uint64_t pageID, uint64_t callbackID)
}
