Mute capture when disconnected from hardware console
rdar://87794804
Reviewed by Brent Fulgham
* Source/WebCore/PAL/pal/spi/cg/CoreGraphicsSPI.h
* Source/WebCore/page/ActivityState.cpp
* Source/WebCore/page/ActivityState.h
* Source/WebKit/UIProcess/API/Cocoa/WKWebViewPrivateForTesting.h
* Source/WebKit/UIProcess/API/Cocoa/WKWebViewTesting.mm
* Source/WebKit/UIProcess/Cocoa/WebProcessPoolCocoa.mm
* Source/WebKit/UIProcess/Cocoa/WebProcessProxyCocoa.mm
* Source/WebKit/UIProcess/WebPageProxy.cpp
* Source/WebKit/UIProcess/WebPageProxy.h
* Source/WebKit/UIProcess/WebProcessPool.h
* Source/WebKit/UIProcess/WebProcessProxy.h
* Source/WebKit/UIProcess/mac/WindowServerConnection.h
* Source/WebKit/UIProcess/mac/WindowServerConnection.mm
* Tools/TestWebKitAPI/Tests/WebKit/GetUserMedia.mm
Canonical link: https://commits.webkit.org/251761@main
git-svn-id: http://svn.webkit.org/repository/webkit/trunk@295757 268f45cc-cd09-0410-ab3c-d52691b4dbfc
diff --git a/Source/WebCore/PAL/pal/spi/cg/CoreGraphicsSPI.h b/Source/WebCore/PAL/pal/spi/cg/CoreGraphicsSPI.h
index f56189e..d5dda4c 100644
--- a/Source/WebCore/PAL/pal/spi/cg/CoreGraphicsSPI.h
+++ b/Source/WebCore/PAL/pal/spi/cg/CoreGraphicsSPI.h
@@ -206,6 +206,11 @@
static const CGSNotificationType kCGSConnectionWindowModificationsStopped = (CGSNotificationType)(kCGSFirstConnectionNotification + 7);
static const CGSNotificationType kCGSessionConsoleConnect = kCGSFirstSessionNotification;
static const CGSNotificationType kCGSessionConsoleDisconnect = (CGSNotificationType)(kCGSessionConsoleConnect + 1);
+static const CGSNotificationType kCGSessionRemoteConnect = (CGSNotificationType)(kCGSessionConsoleDisconnect + 1);
+static const CGSNotificationType kCGSessionRemoteDisconnect = (CGSNotificationType)(kCGSessionRemoteConnect + 1);
+static const CGSNotificationType kCGSessionLoggedOn = (CGSNotificationType)(kCGSessionRemoteDisconnect + 1);
+static const CGSNotificationType kCGSessionLoggedOff = (CGSNotificationType)(kCGSessionLoggedOn + 1);
+static const CGSNotificationType kCGSessionConsoleWillDisconnect = (CGSNotificationType)(kCGSessionLoggedOff + 1);
#endif // PLATFORM(MAC)
diff --git a/Source/WebCore/page/ActivityState.cpp b/Source/WebCore/page/ActivityState.cpp
index 0b67a62..d5f5c0a 100644
--- a/Source/WebCore/page/ActivityState.cpp
+++ b/Source/WebCore/page/ActivityState.cpp
@@ -52,6 +52,7 @@
appendIf(ActivityState::IsAudible, "audible");
appendIf(ActivityState::IsLoading, "loading");
appendIf(ActivityState::IsCapturingMedia, "capturing media");
+ appendIf(ActivityState::IsConnectedToHardwareConsole, "attached to hardware console");
return ts;
}
diff --git a/Source/WebCore/page/ActivityState.h b/Source/WebCore/page/ActivityState.h
index 09c4a2b..9aa239d 100644
--- a/Source/WebCore/page/ActivityState.h
+++ b/Source/WebCore/page/ActivityState.h
@@ -44,9 +44,10 @@
IsAudible = 1 << 6,
IsLoading = 1 << 7,
IsCapturingMedia = 1 << 8,
+ IsConnectedToHardwareConsole = 1 << 9,
};
- static constexpr OptionSet<Flag> allFlags() { return { WindowIsActive, IsFocused, IsVisible, IsVisibleOrOccluded, IsInWindow, IsVisuallyIdle, IsAudible, IsLoading, IsCapturingMedia }; }
+ static constexpr OptionSet<Flag> allFlags() { return { WindowIsActive, IsFocused, IsVisible, IsVisibleOrOccluded, IsInWindow, IsVisuallyIdle, IsAudible, IsLoading, IsCapturingMedia, IsConnectedToHardwareConsole }; }
};
enum class ActivityStateForCPUSampling {
@@ -72,7 +73,8 @@
WebCore::ActivityState::Flag::IsVisuallyIdle,
WebCore::ActivityState::Flag::IsAudible,
WebCore::ActivityState::Flag::IsLoading,
- WebCore::ActivityState::Flag::IsCapturingMedia
+ WebCore::ActivityState::Flag::IsCapturingMedia,
+ WebCore::ActivityState::Flag::IsConnectedToHardwareConsole
>;
};
diff --git a/Source/WebKit/UIProcess/API/Cocoa/WKWebViewPrivateForTesting.h b/Source/WebKit/UIProcess/API/Cocoa/WKWebViewPrivateForTesting.h
index 9d59e83..68974d0 100644
--- a/Source/WebKit/UIProcess/API/Cocoa/WKWebViewPrivateForTesting.h
+++ b/Source/WebKit/UIProcess/API/Cocoa/WKWebViewPrivateForTesting.h
@@ -121,6 +121,8 @@
- (void)_computePagesForPrinting:(_WKFrameHandle *)handle completionHandler:(void(^)(void))completionHandler WK_API_AVAILABLE(macos(13.0), ios(16.0));
+- (void)_setConnectedToHardwareConsoleForTesting:(BOOL)connected;
+
@end
typedef NS_ENUM(NSInteger, _WKMediaSessionReadyState) {
diff --git a/Source/WebKit/UIProcess/API/Cocoa/WKWebViewTesting.mm b/Source/WebKit/UIProcess/API/Cocoa/WKWebViewTesting.mm
index 8ae54da..e688a4a 100644
--- a/Source/WebKit/UIProcess/API/Cocoa/WKWebViewTesting.mm
+++ b/Source/WebKit/UIProcess/API/Cocoa/WKWebViewTesting.mm
@@ -44,6 +44,7 @@
#import <wtf/RetainPtr.h>
#if PLATFORM(MAC)
+#import "WindowServerConnection.h"
#import "WKWebViewMac.h"
#endif
@@ -455,6 +456,13 @@
});
}
+- (void)_setConnectedToHardwareConsoleForTesting:(BOOL)connected
+{
+#if PLATFORM(MAC) || PLATFORM(MACCATALYST)
+ WebKit::WindowServerConnection::singleton().hardwareConsoleStateChanged(connected ? WebKit::WindowServerConnection::HardwareConsoleState::Connected : WebKit::WindowServerConnection::HardwareConsoleState::Disconnected);
+#endif
+}
+
- (void)_createMediaSessionCoordinatorForTesting:(id <_WKMediaSessionCoordinator>)privateCoordinator completionHandler:(void(^)(BOOL))completionHandler
{
#if ENABLE(MEDIA_SESSION_COORDINATOR)
diff --git a/Source/WebKit/UIProcess/Cocoa/WebProcessPoolCocoa.mm b/Source/WebKit/UIProcess/Cocoa/WebProcessPoolCocoa.mm
index 63d3013..eda0b0c 100644
--- a/Source/WebKit/UIProcess/Cocoa/WebProcessPoolCocoa.mm
+++ b/Source/WebKit/UIProcess/Cocoa/WebProcessPoolCocoa.mm
@@ -534,6 +534,13 @@
return;
pool->sendToAllProcesses(Messages::WebProcess::ColorPreferencesDidChange());
}
+
+void WebProcessPool::hardwareConsoleStateChanged()
+{
+ for (auto& process : m_processes)
+ process->hardwareConsoleStateChanged();
+}
+
#endif
#if ENABLE(REMOTE_INSPECTOR) && PLATFORM(IOS_FAMILY) && !PLATFORM(MACCATALYST)
diff --git a/Source/WebKit/UIProcess/Cocoa/WebProcessProxyCocoa.mm b/Source/WebKit/UIProcess/Cocoa/WebProcessProxyCocoa.mm
index 5c52a39..0428712 100644
--- a/Source/WebKit/UIProcess/Cocoa/WebProcessProxyCocoa.mm
+++ b/Source/WebKit/UIProcess/Cocoa/WebProcessProxyCocoa.mm
@@ -58,11 +58,12 @@
#endif
#if HAVE(MEDIA_ACCESSIBILITY_FRAMEWORK)
-#include <WebCore/CaptionUserPreferencesMediaAF.h>
+#import <WebCore/CaptionUserPreferencesMediaAF.h>
#endif
#if PLATFORM(MAC)
-#include "TCCSoftLink.h"
+#import "WindowServerConnection.h"
+#import "TCCSoftLink.h"
#endif
namespace WebKit {
@@ -265,6 +266,13 @@
auto authenticated = TCCAccessCheckAuditToken(get_TCC_kTCCServiceAccessibility(), auditToken, nullptr);
completionHandler(authenticated);
}
+
+void WebProcessProxy::hardwareConsoleStateChanged()
+{
+ m_isConnectedToHardwareConsole = WindowServerConnection::singleton().hardwareConsoleState() == WindowServerConnection::HardwareConsoleState::Connected;
+ for (const auto& page : m_pageMap.values())
+ page->activityStateDidChange(ActivityState::IsConnectedToHardwareConsole);
+}
#endif
#if HAVE(AUDIO_COMPONENT_SERVER_REGISTRATIONS)
@@ -329,4 +337,6 @@
return SandboxExtension::createHandleForMachLookup("com.apple.fonts"_s, auditToken(), machBootstrapOptions).value_or(SandboxExtension::Handle { });
}
+
}
+
diff --git a/Source/WebKit/UIProcess/WebPageProxy.cpp b/Source/WebKit/UIProcess/WebPageProxy.cpp
index 1bdea3e..b8c1355 100644
--- a/Source/WebKit/UIProcess/WebPageProxy.cpp
+++ b/Source/WebKit/UIProcess/WebPageProxy.cpp
@@ -2253,6 +2253,9 @@
m_process->pageIsBecomingInvisible(m_webPageID);
}
+ if (m_potentiallyChangedActivityStateFlags & ActivityState::IsConnectedToHardwareConsole)
+ isConnectedToHardwareConsoleDidChange();
+
bool isNowInWindow = (changed & ActivityState::IsInWindow) && isInWindow();
// We always want to wait for the Web process to reply if we've been in-window before and are coming back in-window.
if (m_viewWasEverInWindow && isNowInWindow) {
@@ -2638,6 +2641,26 @@
setMuted(state);
}
+void WebPageProxy::isConnectedToHardwareConsoleDidChange()
+{
+ SetForScope<bool> isProcessing(m_isProcessingIsConnectedToHardwareConsoleDidChangeNotification, true);
+ if (m_process->isConnectedToHardwareConsole()) {
+ if (!m_captureWasMutedWhenHardwareConsoleDisconnected)
+ setMediaStreamCaptureMuted(false);
+
+ m_captureWasMutedWhenHardwareConsoleDisconnected = false;
+ return;
+ }
+
+ m_captureWasMutedWhenHardwareConsoleDisconnected = m_mutedState.containsAny(WebCore::MediaProducer::MediaStreamCaptureIsMuted);
+ setMediaStreamCaptureMuted(true);
+}
+
+bool WebPageProxy::isAllowedToChangeMuteState() const
+{
+ return m_isProcessingIsConnectedToHardwareConsoleDidChangeNotification || m_process->isConnectedToHardwareConsole();
+}
+
void WebPageProxy::activateMediaStreamCaptureInPage()
{
#if ENABLE(MEDIA_STREAM)
@@ -6461,6 +6484,9 @@
void WebPageProxy::setMuted(WebCore::MediaProducerMutedStateFlags state, CompletionHandler<void()>&& completionHandler)
{
+ if (!isAllowedToChangeMuteState())
+ state.add(WebCore::MediaProducer::MediaStreamCaptureIsMuted);
+
m_mutedState = state;
if (!hasRunningProcess())
diff --git a/Source/WebKit/UIProcess/WebPageProxy.h b/Source/WebKit/UIProcess/WebPageProxy.h
index 5518f5d..d37d6a2 100644
--- a/Source/WebKit/UIProcess/WebPageProxy.h
+++ b/Source/WebKit/UIProcess/WebPageProxy.h
@@ -808,6 +808,8 @@
void activateMediaStreamCaptureInPage();
bool isMediaStreamCaptureMuted() const { return m_mutedState.containsAny(WebCore::MediaProducer::MediaStreamCaptureIsMuted); }
void setMediaStreamCaptureMuted(bool);
+ void isConnectedToHardwareConsoleDidChange();
+ bool isAllowedToChangeMuteState() const;
void requestFontAttributesAtSelectionStart(CompletionHandler<void(const WebCore::FontAttributes&)>&&);
@@ -3055,6 +3057,8 @@
bool m_mayStartMediaWhenInWindow { true };
bool m_mediaPlaybackIsSuspended { false };
bool m_mediaCaptureEnabled { true };
+ bool m_isProcessingIsConnectedToHardwareConsoleDidChangeNotification { false };
+ bool m_captureWasMutedWhenHardwareConsoleDisconnected { false };
bool m_waitingForDidUpdateActivityState { false };
diff --git a/Source/WebKit/UIProcess/WebProcessPool.h b/Source/WebKit/UIProcess/WebProcessPool.h
index a0adde3..c155f15 100644
--- a/Source/WebKit/UIProcess/WebProcessPool.h
+++ b/Source/WebKit/UIProcess/WebProcessPool.h
@@ -519,6 +519,10 @@
bool processesShouldSuspend() const { return m_processesShouldSuspend; }
#endif
+#if PLATFORM(MAC)
+ void hardwareConsoleStateChanged();
+#endif
+
private:
void platformInitialize();
diff --git a/Source/WebKit/UIProcess/WebProcessProxy.h b/Source/WebKit/UIProcess/WebProcessProxy.h
index 5678995..55097ac 100644
--- a/Source/WebKit/UIProcess/WebProcessProxy.h
+++ b/Source/WebKit/UIProcess/WebProcessProxy.h
@@ -438,6 +438,12 @@
SandboxExtension::Handle fontdMachExtensionHandle(SandboxExtension::MachBootstrapOptions) const;
#endif
+ bool isConnectedToHardwareConsole() const { return m_isConnectedToHardwareConsole; }
+
+#if PLATFORM(MAC)
+ void hardwareConsoleStateChanged();
+#endif
+
protected:
WebProcessProxy(WebProcessPool&, WebsiteDataStore*, IsPrewarmed, WebCore::CrossOriginMode, CaptivePortalMode);
@@ -675,6 +681,7 @@
std::unique_ptr<SpeechRecognitionRemoteRealtimeMediaSourceManager> m_speechRecognitionRemoteRealtimeMediaSourceManager;
#endif
std::unique_ptr<WebLockRegistryProxy> m_webLockRegistry;
+ bool m_isConnectedToHardwareConsole { true };
};
} // namespace WebKit
diff --git a/Source/WebKit/UIProcess/mac/WindowServerConnection.h b/Source/WebKit/UIProcess/mac/WindowServerConnection.h
index 9d02108..4ffdeb2 100644
--- a/Source/WebKit/UIProcess/mac/WindowServerConnection.h
+++ b/Source/WebKit/UIProcess/mac/WindowServerConnection.h
@@ -1,5 +1,5 @@
/*
- * Copyright (C) 2014-2017 Apple Inc. All rights reserved.
+ * Copyright (C) 2014-2022 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -31,12 +31,17 @@
bool applicationWindowModificationsHaveStopped() const { return m_applicationWindowModificationsHaveStopped; }
+ enum class HardwareConsoleState : uint8_t { Connected, Disconnected };
+ void hardwareConsoleStateChanged(HardwareConsoleState);
+ HardwareConsoleState hardwareConsoleState() const { return m_connectionState; }
+
private:
WindowServerConnection();
void windowServerConnectionStateChanged();
void applicationWindowModificationsStopped(bool stopped);
+ HardwareConsoleState m_connectionState { HardwareConsoleState::Connected };
bool m_applicationWindowModificationsHaveStopped;
};
diff --git a/Source/WebKit/UIProcess/mac/WindowServerConnection.mm b/Source/WebKit/UIProcess/mac/WindowServerConnection.mm
index 2053c17..56e5fcd 100644
--- a/Source/WebKit/UIProcess/mac/WindowServerConnection.mm
+++ b/Source/WebKit/UIProcess/mac/WindowServerConnection.mm
@@ -1,5 +1,5 @@
/*
- * Copyright (C) 2010-2017 Apple Inc. All rights reserved.
+ * Copyright (C) 2010-2022 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -26,13 +26,14 @@
#import "config.h"
#import "WindowServerConnection.h"
-#if PLATFORM(MAC)
+#if PLATFORM(MAC) || PLATFORM(MACCATALYST)
#import "WebProcessPool.h"
#import <pal/spi/cg/CoreGraphicsSPI.h>
namespace WebKit {
+#if PLATFORM(MAC)
void WindowServerConnection::applicationWindowModificationsStopped(bool stopped)
{
if (m_applicationWindowModificationsHaveStopped == stopped)
@@ -46,6 +47,17 @@
for (auto& processPool : WebProcessPool::allProcessPools())
processPool->windowServerConnectionStateChanged();
}
+#endif
+
+void WindowServerConnection::hardwareConsoleStateChanged(HardwareConsoleState state)
+{
+ if (state == m_connectionState)
+ return;
+
+ m_connectionState = state;
+ for (auto& processPool : WebProcessPool::allProcessPools())
+ processPool->hardwareConsoleStateChanged();
+}
WindowServerConnection& WindowServerConnection::singleton()
{
@@ -53,6 +65,7 @@
return windowServerConnection;
}
+#if PLATFORM(MAC)
static bool registerOcclusionNotificationHandler(CGSNotificationType type, CGSNotifyConnectionProcPtr handler)
{
CGSConnectionID mainConnection = CGSMainConnectionID();
@@ -67,10 +80,12 @@
return CGSRegisterConnectionNotifyProc(mainConnection, handler, type, nullptr) == kCGErrorSuccess;
}
+#endif
WindowServerConnection::WindowServerConnection()
: m_applicationWindowModificationsHaveStopped(false)
{
+#if PLATFORM(MAC)
struct OcclusionNotificationHandler {
CGSNotificationType notificationType;
CGSNotifyConnectionProcPtr handler;
@@ -95,8 +110,33 @@
UNUSED_PARAM(result);
ASSERT_WITH_MESSAGE(result, "Registration of \"%s\" notification handler failed.\n", occlusionNotificationHandler.name);
}
+#endif
+
+ static auto consoleWillDisconnect = [](CGSNotificationType, void*, uint32_t, void*) {
+ WindowServerConnection::singleton().hardwareConsoleStateChanged(HardwareConsoleState::Disconnected);
+ };
+ static auto consoleWillConnect = [](CGSNotificationType, void*, uint32_t, void*) {
+ WindowServerConnection::singleton().hardwareConsoleStateChanged(HardwareConsoleState::Connected);
+ };
+
+ struct ConnectionStateNotificationHandler {
+ CGSNotificationType notificationType;
+ CGSNotifyProcPtr handler;
+ const char* name;
+ };
+
+ static const ConnectionStateNotificationHandler connectionStateNotificationHandlers[] = {
+ { kCGSessionConsoleWillDisconnect, consoleWillDisconnect, "Console Disconnected" },
+ { kCGSessionConsoleConnect, consoleWillConnect, "Console Connected" },
+ };
+
+ for (const auto& connectionStateNotificationHandler : connectionStateNotificationHandlers) {
+ auto error = CGSRegisterNotifyProc(connectionStateNotificationHandler.handler, connectionStateNotificationHandler.notificationType, nullptr);
+ UNUSED_PARAM(error);
+ ASSERT_WITH_MESSAGE(!error, "Registration of \"%s\" notification handler failed.\n", connectionStateNotificationHandler.name);
+ }
}
} // namespace WebKit
-#endif
+#endif // PLATFORM(MAC) || PLATFORM(MACCATALYST)
diff --git a/Tools/TestWebKitAPI/Tests/WebKit/GetUserMedia.mm b/Tools/TestWebKitAPI/Tests/WebKit/GetUserMedia.mm
index f4d8fb5..e9b7e11 100644
--- a/Tools/TestWebKitAPI/Tests/WebKit/GetUserMedia.mm
+++ b/Tools/TestWebKitAPI/Tests/WebKit/GetUserMedia.mm
@@ -1126,6 +1126,63 @@
}
#endif
+#if PLATFORM(MAC)
+TEST(WebKit2, ConnectedToHardwareConsole)
+{
+ auto configuration = adoptNS([[WKWebViewConfiguration alloc] init]);
+ auto processPoolConfig = adoptNS([[_WKProcessPoolConfiguration alloc] init]);
+ initializeMediaCaptureConfiguration(configuration.get());
+ auto webView = adoptNS([[TestWKWebView alloc] initWithFrame:CGRectMake(0, 0, 320, 500) configuration:configuration.get() processPoolConfiguration:processPoolConfig.get()]);
+ auto delegate = adoptNS([[UserMediaCaptureUIDelegate alloc] init]);
+ [webView setUIDelegate:delegate.get()];
+ [webView _setMediaCaptureReportingDelayForTesting:0];
+
+ auto observer = adoptNS([[MediaCaptureObserver alloc] init]);
+ [webView addObserver:observer.get() forKeyPath:@"microphoneCaptureState" options:NSKeyValueObservingOptionNew context:nil];
+ [webView addObserver:observer.get() forKeyPath:@"cameraCaptureState" options:NSKeyValueObservingOptionNew context:nil];
+
+ cameraCaptureStateChange = false;
+ [webView loadTestPageNamed:@"getUserMedia"];
+ EXPECT_TRUE(waitUntilCameraState(webView.get(), WKMediaCaptureStateActive));
+
+ cameraCaptureStateChange = false;
+ [webView _setConnectedToHardwareConsoleForTesting:NO];
+ EXPECT_TRUE(waitUntilCameraState(webView.get(), WKMediaCaptureStateMuted));
+
+ // It should not be possible to unmute while detached.
+ cameraCaptureStateChange = false;
+ [webView setCameraCaptureState:WKMediaCaptureStateActive completionHandler:nil];
+ int retryCount = 1000;
+ while (--retryCount && cameraCaptureState == WKMediaCaptureStateMuted)
+ TestWebKitAPI::Util::spinRunLoop(10);
+ EXPECT_TRUE(!!retryCount);
+ EXPECT_TRUE(cameraCaptureState == WKMediaCaptureStateMuted);
+
+ // Capture should be unmuted if it was active when the disconnect happened.
+ cameraCaptureStateChange = false;
+ [webView _setConnectedToHardwareConsoleForTesting:YES];
+ EXPECT_TRUE(waitUntilCameraState(webView.get(), WKMediaCaptureStateActive));
+
+ cameraCaptureStateChange = false;
+ [webView setCameraCaptureState:WKMediaCaptureStateMuted completionHandler:nil];
+ EXPECT_TRUE(waitUntilCameraState(webView.get(), WKMediaCaptureStateMuted));
+
+ // Reconnecting should not unmute if capture if it was already muted when the disconnect happened.
+ [webView _setConnectedToHardwareConsoleForTesting:NO];
+ retryCount = 1000;
+ while (--retryCount)
+ TestWebKitAPI::Util::spinRunLoop(10);
+ EXPECT_TRUE(cameraCaptureState == WKMediaCaptureStateMuted);
+
+ [webView _setConnectedToHardwareConsoleForTesting:YES];
+ retryCount = 1000;
+ while (--retryCount)
+ TestWebKitAPI::Util::spinRunLoop(10);
+ EXPECT_TRUE(cameraCaptureState == WKMediaCaptureStateMuted);
+}
+
+#endif
+
} // namespace TestWebKitAPI
#endif // ENABLE(MEDIA_STREAM)
