Diff
Modified: trunk/LayoutTests/ChangeLog (285992 => 285993)
--- trunk/LayoutTests/ChangeLog 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/LayoutTests/ChangeLog 2021-11-18 10:34:22 UTC (rev 285993)
@@ -1,3 +1,16 @@
+2021-11-18 Youenn Fablet <[email protected]>
+
+ MediaPlayerAVFoundation should support rvfc
+ https://bugs.webkit.org/show_bug.cgi?id=231945
+
+ Reviewed by Eric Carlson.
+
+ Unskipped some tests.
+
+ * platform/ios-wk2/TestExpectations:
+ * platform/mac-wk1/TestExpectations:
+ * platform/mac-wk2/TestExpectations:
+
2021-11-18 Sergio Villar Senin <[email protected]>
[css-flexbox] Import WPT tests. November 2021 edition.
Modified: trunk/LayoutTests/imported/w3c/ChangeLog (285992 => 285993)
--- trunk/LayoutTests/imported/w3c/ChangeLog 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/LayoutTests/imported/w3c/ChangeLog 2021-11-18 10:34:22 UTC (rev 285993)
@@ -1,3 +1,22 @@
+2021-11-18 Youenn Fablet <[email protected]>
+
+ MediaPlayerAVFoundation should support rvfc
+ https://bugs.webkit.org/show_bug.cgi?id=231945
+ <rdar://problem/84657372>
+
+ Reviewed by Eric Carlson.
+
+ Transforming tests from async to promise based so that there is only one video playing at a time, to make it work smoothly in iOS.
+
+ * web-platform-tests/video-rvfc/request-video-frame-callback-dom-expected.txt:
+ * web-platform-tests/video-rvfc/request-video-frame-callback-dom.html:
+ * web-platform-tests/video-rvfc/request-video-frame-callback-parallel-expected.txt:
+ * web-platform-tests/video-rvfc/request-video-frame-callback-parallel.html:
+ * web-platform-tests/video-rvfc/request-video-frame-callback-repeating-expected.txt:
+ * web-platform-tests/video-rvfc/request-video-frame-callback-repeating.html:
+ * web-platform-tests/video-rvfc/request-video-frame-callback-expected.txt:
+ * web-platform-tests/video-rvfc/request-video-frame-callback.html:
+
2021-11-18 Sergio Villar Senin <[email protected]>
[css-flexbox] Import WPT tests. November 2021 edition.
Modified: trunk/LayoutTests/imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback-dom-expected.txt (285992 => 285993)
--- trunk/LayoutTests/imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback-dom-expected.txt 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/LayoutTests/imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback-dom-expected.txt 2021-11-18 10:34:22 UTC (rev 285993)
@@ -1,7 +1,5 @@
-Harness Error (TIMEOUT), message = null
+PASS Test a video outside of the DOM can still use video.rVFC.
+PASS Test video.rVFC works with "display:none".
+PASS Test video.rVFC works with "visibility:hidden".
-TIMEOUT Test a video outside of the DOM can still use video.rVFC. Test timed out
-TIMEOUT Test video.rVFC works with "display:none". Test timed out
-TIMEOUT Test video.rVFC works with "visibility:hidden". Test timed out
-
Modified: trunk/LayoutTests/imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback-dom.html (285992 => 285993)
--- trunk/LayoutTests/imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback-dom.html 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/LayoutTests/imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback-dom.html 2021-11-18 10:34:22 UTC (rev 285993)
@@ -13,17 +13,24 @@
width: 320,
}
-async_test(function(t) {
+promise_test(async function(t) {
+ let done;
+ const promise = new Promise(resolve => done = resolve);
+
let video = document.createElement('video');
- video.requestVideoFrameCallback(t.step_func_done());
+ video.requestVideoFrameCallback(done);
video.src = ""
- video.play();
+ await video.play();
+ return promise;
}, 'Test a video outside of the DOM can still use video.rVFC.');
function rvfcStyleTest(applyStyle, description) {
- async_test(function(t) {
+ promise_test(async function(t) {
+ let done;
+ const promise = new Promise(resolve => done = resolve);
+
let video = document.createElement('video');
document.body.appendChild(video);
applyStyle(video);
@@ -31,12 +38,14 @@
video.requestVideoFrameCallback(
t.step_func( _ => {
// Make sure we can receive more than one callback.
- video.requestVideoFrameCallback(t.step_func_done());
+ video.requestVideoFrameCallback(done);
})
);
video.src = ""
- video.play();
+ await video.play();
+
+ return promise;
}, description);
}
Modified: trunk/LayoutTests/imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback-expected.txt (285992 => 285993)
--- trunk/LayoutTests/imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback-expected.txt 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/LayoutTests/imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback-expected.txt 2021-11-18 10:34:22 UTC (rev 285993)
@@ -1,10 +1,8 @@
-Harness Error (TIMEOUT), message = null
-
-TIMEOUT Test we can register a video.rVFC callback. Test timed out
-TIMEOUT Test video.rVFC callbacks run before window.rAF callbacks. Test timed out
-TIMEOUT Test we can cancel a video.rVFC request. Test timed out
+PASS Test we can register a video.rVFC callback.
+PASS Test video.rVFC callbacks run before window.rAF callbacks.
+PASS Test we can cancel a video.rVFC request.
PASS Test invalid calls to the video.rVFC API.
-TIMEOUT Test video.rVFC does not stop when switching sources. Test timed out
+PASS Test video.rVFC does not stop when switching sources.
Modified: trunk/LayoutTests/imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback-parallel-expected.txt (285992 => 285993)
--- trunk/LayoutTests/imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback-parallel-expected.txt 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/LayoutTests/imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback-parallel-expected.txt 2021-11-18 10:34:22 UTC (rev 285993)
@@ -1,7 +1,5 @@
-Harness Error (TIMEOUT), message = null
+PASS Test callbacks get the same information.
+PASS Test we can cancel callbacks from callbacks.
-TIMEOUT Test callbacks get the same information. Test timed out
-TIMEOUT Test we can cancel callbacks from callbacks. Test timed out
-
Modified: trunk/LayoutTests/imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback-parallel.html (285992 => 285993)
--- trunk/LayoutTests/imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback-parallel.html 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/LayoutTests/imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback-parallel.html 2021-11-18 10:34:22 UTC (rev 285993)
@@ -7,7 +7,10 @@
<script src=""
<script>
-async_test(function(t) {
+promise_test(async function(t) {
+ let done;
+ const promise = new Promise(resolve => done = resolve);
+
let video = document.createElement('video');
document.body.appendChild(video);
@@ -19,17 +22,22 @@
firstMetadata = metadata;
}));
- video.requestVideoFrameCallback(t.step_func_done((time, metadata) => {
+ video.requestVideoFrameCallback(t.step_func((time, metadata) => {
assert_equals(firstTime, time);
assert_object_equals(firstMetadata, metadata);
+ done();
}));
video.src = ""
video.play();
+ return promise;
}, 'Test callbacks get the same information.');
-async_test(function(t) {
+promise_test(async function(t) {
+ let done;
+ const promise = new Promise(resolve => done = resolve);
+
let video = document.createElement('video');
document.body.appendChild(video);
@@ -46,12 +54,12 @@
);
// NOTE: This callback should be executed last.
- video.requestVideoFrameCallback(
- t.step_func_done()
- );
+ video.requestVideoFrameCallback(done);
video.src = ""
video.play();
+
+ return promise;
}, 'Test we can cancel callbacks from callbacks.');
</script>
</html>
Modified: trunk/LayoutTests/imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback-repeating-expected.txt (285992 => 285993)
--- trunk/LayoutTests/imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback-repeating-expected.txt 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/LayoutTests/imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback-repeating-expected.txt 2021-11-18 10:34:22 UTC (rev 285993)
@@ -1,7 +1,5 @@
-Harness Error (TIMEOUT), message = null
+PASS Test new callbacks are only called on the next frame.
+PASS Test chaining calls to video.rVFC, and verify the required parameters.
-TIMEOUT Test new callbacks are only called on the next frame. Test timed out
-TIMEOUT Test chaining calls to video.rVFC, and verify the required parameters. Test timed out
-
Modified: trunk/LayoutTests/imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback-repeating.html (285992 => 285993)
--- trunk/LayoutTests/imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback-repeating.html 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/LayoutTests/imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback-repeating.html 2021-11-18 10:34:22 UTC (rev 285993)
@@ -7,7 +7,10 @@
<script src=""
<script>
-async_test(function(t) {
+promise_test(async function(t) {
+ let done;
+ const promise = new Promise(resolve => done = resolve);
+
let video = document.createElement('video');
document.body.appendChild(video);
@@ -24,18 +27,23 @@
// Queue up a second callback, and make sure it's called at the same time
// as the one we just queued up.
- video.requestVideoFrameCallback(t.step_func_done((time) => {
+ video.requestVideoFrameCallback(t.step_func((time) => {
assert_equals(time, secondTime, "Callbacks queued together should be called at the same time");
+ done();
}))
}));
video.src = ""
- video.play();
+ await video.play();
+ return promise;
}, 'Test new callbacks are only called on the next frame.');
-async_test(function(t) {
+promise_test(async function(t) {
+ let done;
+ const promise = new Promise(resolve => done = resolve);
+
let video = document.createElement('video');
document.body.appendChild(video);
@@ -64,7 +72,7 @@
lastMetadata = metadata;
if (++currentCallNumber > maxNumberOfCalls) {
- t.done()
+ done()
} else {
video.requestVideoFrameCallback(t.step_func(repeatingCallback));
}
@@ -73,8 +81,9 @@
video.requestVideoFrameCallback(t.step_func(repeatingCallback));
video.src = ""
- video.play();
+ await video.play();
+ return promise;
}, 'Test chaining calls to video.rVFC, and verify the required parameters.');
</script>
</html>
Modified: trunk/LayoutTests/imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback.html (285992 => 285993)
--- trunk/LayoutTests/imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback.html 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/LayoutTests/imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback.html 2021-11-18 10:34:22 UTC (rev 285993)
@@ -18,15 +18,19 @@
width: 320,
}
-async_test(function(t) {
+promise_test(async function(t) {
+ let done;
+ const promise = new Promise(resolve => done = resolve);
+
let video = document.createElement('video');
document.body.appendChild(video);
let id = video.requestVideoFrameCallback(
- t.step_func_done((time, metadata) => {
+ t.step_func((time, metadata) => {
assert_true(time > 0);
assert_equals(metadata.height, testVideo.height);
assert_equals(metadata.width, testVideo.width);
+ done();
})
);
@@ -33,11 +37,15 @@
assert_true(id > 0);
video.src = ""
- video.play();
+ await video.play();
+ return promise;
}, 'Test we can register a video.rVFC callback.');
-async_test(function(t) {
+promise_test(async function(t) {
+ let done;
+ const promise = new Promise(resolve => done = resolve);
+
let video = document.createElement('video');
document.body.appendChild(video);
@@ -45,19 +53,24 @@
t.step_func(video_now => {
// Queue a call to window.rAF, and make sure it is executed within the
// same turn of the event loop (with the same 'time' parameter).
- window.requestAnimationFrame( t.step_func_done( window_now => {
+ window.requestAnimationFrame( t.step_func( window_now => {
assert_equals(video_now, window_now);
+ done();
}));
})
);
video.src = ""
- video.play();
+ await video.play();
+ return promise;
}, 'Test video.rVFC callbacks run before window.rAF callbacks.');
-async_test(function(t) {
+promise_test(async function(t) {
+ let done;
+ const promise = new Promise(resolve => done = resolve);
+
let video = document.createElement('video');
document.body.appendChild(video);
@@ -74,12 +87,14 @@
// At this point, the other callback shouldn't have fired, but
// give it some more time and really make sure it doesn't, by going
// throught the event loop once more.
- t.step_timeout(() => { t.done(); }, 0);
+ t.step_timeout(() => { done(); }, 0);
})
);
video.src = ""
- video.play();
+ await video.play();
+
+ return promise;
}, 'Test we can cancel a video.rVFC request.');
test(function(t) {
@@ -105,6 +120,7 @@
promise_test(async function(t) {
let video = document.createElement('video');
+ video.autoplay = true;
document.body.appendChild(video);
let first_width = 0;
Modified: trunk/LayoutTests/platform/ios-wk2/TestExpectations (285992 => 285993)
--- trunk/LayoutTests/platform/ios-wk2/TestExpectations 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/LayoutTests/platform/ios-wk2/TestExpectations 2021-11-18 10:34:22 UTC (rev 285993)
@@ -143,13 +143,9 @@
imported/w3c/web-platform-tests/video-rvfc [ Pass ]
fast/mediastream/getUserMedia-rvfc.html [ Pass ]
webrtc/peerConnection-rvfc.html [ Pass ]
-# Timing out tests until we add regular video streaming backend support.
+# Timing out tests until we add XR session.
imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback-before-xr-session.https.html [ Skip ]
-imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback-dom.html [ Skip ]
imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback-during-xr-session.https.html [ Skip ]
-imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback-parallel.html [ Skip ]
-imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback-repeating.html [ Skip ]
-imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback.html [ Skip ]
#//////////////////////////////////////////////////////////////////////////////////////////
# End platform-specific directories.
Modified: trunk/LayoutTests/platform/mac-wk1/TestExpectations (285992 => 285993)
--- trunk/LayoutTests/platform/mac-wk1/TestExpectations 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/LayoutTests/platform/mac-wk1/TestExpectations 2021-11-18 10:34:22 UTC (rev 285993)
@@ -33,6 +33,11 @@
# rdar://problem/61793884 : Enabling this test for WK1 only.
fast/shapes/shape-outside-floats/shape-outside-imagedata-overflow.html [ Pass ]
+imported/w3c/web-platform-tests/video-rvfc [ Pass ]
+# Timing out tests until we add XR session.
+imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback-before-xr-session.https.html [ Skip ]
+imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback-during-xr-session.https.html [ Skip ]
+
#//////////////////////////////////////////////////////////////////////////////////////////
# End platform-specific directories.
#//////////////////////////////////////////////////////////////////////////////////////////
Modified: trunk/LayoutTests/platform/mac-wk2/TestExpectations (285992 => 285993)
--- trunk/LayoutTests/platform/mac-wk2/TestExpectations 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/LayoutTests/platform/mac-wk2/TestExpectations 2021-11-18 10:34:22 UTC (rev 285993)
@@ -122,13 +122,9 @@
imported/w3c/web-platform-tests/video-rvfc [ Pass ]
fast/mediastream/getUserMedia-rvfc.html [ Pass ]
webrtc/peerConnection-rvfc.html [ Pass ]
-# Timing out tests until we add regular video streaming backend support.
+# Timing out tests until we add XR session.
imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback-before-xr-session.https.html [ Skip ]
-imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback-dom.html [ Skip ]
imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback-during-xr-session.https.html [ Skip ]
-imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback-parallel.html [ Skip ]
-imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback-repeating.html [ Skip ]
-imported/w3c/web-platform-tests/video-rvfc/request-video-frame-callback.html [ Skip ]
#//////////////////////////////////////////////////////////////////////////////////////////
# End platform-specific directories.
Modified: trunk/Source/WebCore/ChangeLog (285992 => 285993)
--- trunk/Source/WebCore/ChangeLog 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/Source/WebCore/ChangeLog 2021-11-18 10:34:22 UTC (rev 285993)
@@ -1,3 +1,39 @@
+2021-11-18 Youenn Fablet <[email protected]>
+
+ MediaPlayerAVFoundation should support rvfc
+ https://bugs.webkit.org/show_bug.cgi?id=231945
+ <rdar://problem/84657372>
+
+ Reviewed by Eric Carlson.
+
+ Add API to start/stop gathering metadata.
+ This goes from video element down to media player private.
+ Conversely, add a callback to get the metadata, including the pixel buffer.
+
+ Implement support in MediaPlayerPrivateAVFoundationObjC by adding a specific timeline observer.
+ We reuse the same video output and the same pixel buffer as when painting the video element in the canvas.
+ To make sure painting in a canvas does not have side effects with gathering video metadata, we do not update the pixel buffer
+ when painting on a canvas in case video metadata is gathered.
+ Instead, we consider that pixel buffer is as up to date as possible in that case.
+ To make it work in WK1, we implement the synchronous video frame callback getter.
+
+ Round the millisecond now time given to the callback, like done for RAF.
+
+ Covered by enabled tests.
+
+ * WebCore.xcodeproj/project.pbxproj:
+ * html/HTMLMediaElement.h:
+ * html/HTMLVideoElement.cpp:
+ * html/HTMLVideoElement.h:
+ * platform/VideoFrameMetadata.h:
+ * platform/cocoa/CVPixelBufferRef.h: Added.
+ * platform/graphics/MediaPlayer.cpp:
+ * platform/graphics/MediaPlayer.h:
+ * platform/graphics/MediaPlayerPrivate.h:
+ * platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.h:
+ * platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.mm:
+ * platform/graphics/cv/PixelBufferConformerCV.h:
+
2021-11-18 Chris Lord <[email protected]>
[GLIB] twitch.tv forces synchronous scrolling
Modified: trunk/Source/WebCore/html/HTMLMediaElement.h (285992 => 285993)
--- trunk/Source/WebCore/html/HTMLMediaElement.h 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/Source/WebCore/html/HTMLMediaElement.h 2021-11-18 10:34:22 UTC (rev 285993)
@@ -641,6 +641,9 @@
void setChangingVideoFullscreenMode(bool value) { m_changingVideoFullscreenMode = value; }
bool isChangingVideoFullscreenMode() const { return m_changingVideoFullscreenMode; }
+protected:
+ void mediaPlayerEngineUpdated() override;
+
private:
friend class Internals;
@@ -692,7 +695,6 @@
void mediaPlayerSizeChanged() final;
void mediaPlayerRenderingModeChanged() final;
bool mediaPlayerAcceleratedCompositingEnabled() final;
- void mediaPlayerEngineUpdated() final;
void mediaPlayerWillInitializeMediaEngine() final;
void mediaPlayerDidInitializeMediaEngine() final;
void mediaPlayerReloadAndResumePlaybackIfNeeded() final;
Modified: trunk/Source/WebCore/html/HTMLVideoElement.cpp (285992 => 285993)
--- trunk/Source/WebCore/html/HTMLVideoElement.cpp 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/Source/WebCore/html/HTMLVideoElement.cpp 2021-11-18 10:34:22 UTC (rev 285993)
@@ -589,6 +589,9 @@
unsigned HTMLVideoElement::requestVideoFrameCallback(Ref<VideoFrameRequestCallback>&& callback)
{
+ if (m_videoFrameRequests.isEmpty() && player())
+ player()->startVideoFrameMetadataGathering();
+
auto identifier = ++m_nextVideoFrameRequestIndex;
m_videoFrameRequests.append(makeUniqueRef<VideoFrameRequest>(identifier, WTFMove(callback)));
@@ -608,6 +611,9 @@
return;
}
m_videoFrameRequests.remove(index);
+
+ if (m_videoFrameRequests.isEmpty() && player())
+ player()->stopVideoFrameMetadataGathering();
}
static void processVideoFrameMetadataTimestamps(VideoFrameMetadata& metadata, Performance& performance)
@@ -640,7 +646,7 @@
for (size_t index = 0; index < callbackCount; ++index) {
auto& request = m_videoFrameRequests[index];
if (!request->cancelled) {
- request->callback->handleEvent(now.value(), *videoFrameMetadata);
+ request->callback->handleEvent(std::round(now.milliseconds()), *videoFrameMetadata);
request->cancelled = true;
}
}
@@ -647,8 +653,18 @@
m_isRunningVideoFrameRequests = false;
m_videoFrameRequests.removeAllMatching([](auto& callback) { return callback->cancelled; });
+
+ if (m_videoFrameRequests.isEmpty() && player())
+ player()->stopVideoFrameMetadataGathering();
}
+void HTMLVideoElement::mediaPlayerEngineUpdated()
+{
+ HTMLMediaElement::mediaPlayerEngineUpdated();
+ if (!m_videoFrameRequests.isEmpty() && player())
+ player()->startVideoFrameMetadataGathering();
}
+}
+
#endif
Modified: trunk/Source/WebCore/html/HTMLVideoElement.h (285992 => 285993)
--- trunk/Source/WebCore/html/HTMLVideoElement.h 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/Source/WebCore/html/HTMLVideoElement.h 2021-11-18 10:34:22 UTC (rev 285993)
@@ -141,6 +141,8 @@
PlatformMediaSession::MediaType presentationType() const final { return PlatformMediaSession::MediaType::Video; }
+ void mediaPlayerEngineUpdated() final;
+
std::unique_ptr<HTMLImageLoader> m_imageLoader;
AtomString m_defaultPosterURL;
Modified: trunk/Source/WebCore/platform/VideoFrameMetadata.h (285992 => 285993)
--- trunk/Source/WebCore/platform/VideoFrameMetadata.h 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/Source/WebCore/platform/VideoFrameMetadata.h 2021-11-18 10:34:22 UTC (rev 285993)
@@ -43,8 +43,73 @@
std::optional<double> captureTime;
std::optional<double> receiveTime;
std::optional<unsigned> rtpTimestamp;
+
+ template<class Encoder> void encode(Encoder&) const;
+ template<class Decoder> static std::optional<VideoFrameMetadata> decode(Decoder&);
};
+template<class Encoder>
+inline void VideoFrameMetadata::encode(Encoder& encoder) const
+{
+ encoder << presentationTime << expectedDisplayTime << width << height << mediaTime << presentedFrames << processingDuration << captureTime << receiveTime << rtpTimestamp;
}
+template<class Decoder>
+inline std::optional<VideoFrameMetadata> VideoFrameMetadata::decode(Decoder& decoder)
+{
+ std::optional<double> presentationTime;
+ decoder >> presentationTime;
+ if (!presentationTime)
+ return std::nullopt;
+
+ std::optional<double> expectedDisplayTime;
+ decoder >> expectedDisplayTime;
+ if (!expectedDisplayTime)
+ return std::nullopt;
+
+ std::optional<unsigned> width;
+ decoder >> width;
+ if (!width)
+ return std::nullopt;
+
+ std::optional<unsigned> height;
+ decoder >> height;
+ if (!height)
+ return std::nullopt;
+
+ std::optional<double> mediaTime;
+ decoder >> mediaTime;
+ if (!mediaTime)
+ return std::nullopt;
+
+ std::optional<unsigned> presentedFrames;
+ decoder >> presentedFrames;
+ if (!presentedFrames)
+ return std::nullopt;
+
+ std::optional<std::optional<double>> processingDuration;
+ decoder >> processingDuration;
+ if (!processingDuration)
+ return std::nullopt;
+
+ std::optional<std::optional<double>> captureTime;
+ decoder >> captureTime;
+ if (!captureTime)
+ return std::nullopt;
+
+ std::optional<std::optional<double>> receiveTime;
+ decoder >> receiveTime;
+ if (!receiveTime)
+ return std::nullopt;
+
+ std::optional<std::optional<unsigned>> rtpTimestamp;
+ decoder >> rtpTimestamp;
+ if (!rtpTimestamp)
+ return std::nullopt;
+
+ return VideoFrameMetadata { *presentationTime, *expectedDisplayTime, *width, *height, *mediaTime, *presentedFrames, *processingDuration, *captureTime, *receiveTime, *rtpTimestamp };
+}
+
+}
+
#endif // ENABLE(VIDEO)
Modified: trunk/Source/WebCore/platform/graphics/MediaPlayer.cpp (285992 => 285993)
--- trunk/Source/WebCore/platform/graphics/MediaPlayer.cpp 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/Source/WebCore/platform/graphics/MediaPlayer.cpp 2021-11-18 10:34:22 UTC (rev 285993)
@@ -590,6 +590,8 @@
m_private->setPageIsVisible(m_pageIsVisible);
if (m_visibleInViewport)
m_private->setVisibleInViewport(m_visibleInViewport);
+ if (m_isGatheringVideoFrameMetadata)
+ m_private->startVideoFrameMetadataGathering();
m_private->prepareForPlayback(m_privateBrowsing, m_preload, m_preservesPitch, m_shouldPrepareToRender);
}
}
@@ -1721,6 +1723,25 @@
return m_private->videoFrameMetadata();
}
+void MediaPlayer::startVideoFrameMetadataGathering()
+{
+ m_isGatheringVideoFrameMetadata = true;
+ m_private->startVideoFrameMetadataGathering();
+}
+
+void MediaPlayer::stopVideoFrameMetadataGathering()
+{
+ m_isGatheringVideoFrameMetadata = false;
+ m_private->stopVideoFrameMetadataGathering();
+}
+
+#if PLATFORM(COCOA)
+void MediaPlayer::onNewVideoFrameMetadata(VideoFrameMetadata&& metadata, RetainPtr<CVPixelBufferRef>&& buffer)
+{
+ client().mediaPlayerOnNewVideoFrameMetadata(WTFMove(metadata), WTFMove(buffer));
+}
+#endif
+
String MediaPlayer::elementId() const
{
return client().mediaPlayerElementId();
Modified: trunk/Source/WebCore/platform/graphics/MediaPlayer.h (285992 => 285993)
--- trunk/Source/WebCore/platform/graphics/MediaPlayer.h 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/Source/WebCore/platform/graphics/MediaPlayer.h 2021-11-18 10:34:22 UTC (rev 285993)
@@ -286,6 +286,10 @@
virtual void mediaPlayerQueueTaskOnEventLoop(Function<void()>&& task) { callOnMainThread(WTFMove(task)); }
+#if PLATFORM(COCOA)
+ virtual void mediaPlayerOnNewVideoFrameMetadata(VideoFrameMetadata&&, RetainPtr<CVPixelBufferRef>&&) { }
+#endif
+
#if !RELEASE_LOG_DISABLED
virtual const void* mediaPlayerLogIdentifier() { return nullptr; }
virtual const Logger& mediaPlayerLogger() = 0;
@@ -584,6 +588,10 @@
void removeTextTrack(InbandTextTrackPrivate&);
void removeVideoTrack(VideoTrackPrivate&);
+#if PLATFORM(COCOA)
+ void onNewVideoFrameMetadata(VideoFrameMetadata&&, RetainPtr<CVPixelBufferRef>&&);
+#endif
+
bool requiresTextTrackRepresentation() const;
void setTextTrackRepresentation(TextTrackRepresentation*);
void syncTextTrackBounds();
@@ -681,6 +689,8 @@
bool hasMediaEngine() const;
std::optional<VideoFrameMetadata> videoFrameMetadata();
+ void startVideoFrameMetadataGathering();
+ void stopVideoFrameMetadataGathering();
private:
MediaPlayer(MediaPlayerClient&);
@@ -725,6 +735,7 @@
#if ENABLE(LEGACY_ENCRYPTED_MEDIA) && ENABLE(ENCRYPTED_MEDIA)
bool m_shouldContinueAfterKeyNeeded { false };
#endif
+ bool m_isGatheringVideoFrameMetadata { false };
};
class MediaPlayerFactory {
Modified: trunk/Source/WebCore/platform/graphics/MediaPlayerPrivate.h (285992 => 285993)
--- trunk/Source/WebCore/platform/graphics/MediaPlayerPrivate.h 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/Source/WebCore/platform/graphics/MediaPlayerPrivate.h 2021-11-18 10:34:22 UTC (rev 285993)
@@ -323,6 +323,8 @@
virtual bool pauseAtHostTime(const MonotonicTime&) { return false; }
virtual std::optional<VideoFrameMetadata> videoFrameMetadata() { return { }; }
+ virtual void startVideoFrameMetadataGathering() { }
+ virtual void stopVideoFrameMetadataGathering() { }
};
}
Modified: trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.h (285992 => 285993)
--- trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.h 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.h 2021-11-18 10:34:22 UTC (rev 285993)
@@ -28,6 +28,7 @@
#if ENABLE(VIDEO) && USE(AVFOUNDATION)
#include "MediaPlayerPrivateAVFoundation.h"
+#include <CoreMedia/CMTime.h>
#include <wtf/Function.h>
#include <wtf/HashMap.h>
@@ -342,6 +343,11 @@
bool pauseAtHostTime(const MonotonicTime&) final;
bool haveBeenAskedToPaint() const { return m_haveBeenAskedToPaint; }
+ void startVideoFrameMetadataGathering() final;
+ void stopVideoFrameMetadataGathering() final;
+ std::optional<VideoFrameMetadata> videoFrameMetadata() final { return std::exchange(m_videoFrameMetadata, { }); }
+ void checkNewVideoFrameMetadata(CMTime);
+
RetainPtr<AVURLAsset> m_avAsset;
RetainPtr<AVPlayer> m_avPlayer;
RetainPtr<AVPlayerItem> m_avPlayerItem;
@@ -451,6 +457,10 @@
bool m_haveProcessedChapterTracks { false };
bool m_waitForVideoOutputMediaDataWillChangeTimedOut { false };
bool m_haveBeenAskedToPaint { false };
+ uint64_t m_sampleCount { 0 };
+ RetainPtr<id> m_videoFrameMetadataGatheringObserver;
+ bool m_isGatheringVideoFrameMetadata { false };
+ std::optional<VideoFrameMetadata> m_videoFrameMetadata;
};
}
Modified: trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.mm (285992 => 285993)
--- trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.mm 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.mm 2021-11-18 10:34:22 UTC (rev 285993)
@@ -536,6 +536,11 @@
[m_avPlayer removeTimeObserver:m_currentTimeObserver.get()];
m_currentTimeObserver = nil;
+ if (m_videoFrameMetadataGatheringObserver) {
+ [m_avPlayer removeTimeObserver:m_videoFrameMetadataGatheringObserver.get()];
+ m_videoFrameMetadataGatheringObserver = nil;
+ }
+
m_avPlayer = nil;
}
@@ -1108,6 +1113,8 @@
});
}];
+ if (m_isGatheringVideoFrameMetadata)
+ startVideoFrameMetadataGathering();
setDelayCallbacks(false);
}
@@ -1433,6 +1440,49 @@
return m_cachedTimeControlStatus == AVPlayerTimeControlStatusPaused;
}
+void MediaPlayerPrivateAVFoundationObjC::startVideoFrameMetadataGathering()
+{
+ ASSERT(!m_videoFrameMetadataGatheringObserver || m_avPlayer);
+ m_isGatheringVideoFrameMetadata = true;
+
+ // FIXME: We should use a CADisplayLink to get updates on rendering, for now we emulate with addPeriodicTimeObserverForInterval.
+ m_videoFrameMetadataGatheringObserver = [m_avPlayer addPeriodicTimeObserverForInterval:PAL::CMTimeMake(1, 60) queue:dispatch_get_main_queue() usingBlock:[weakThis = WeakPtr { *this }](CMTime currentTime) {
+ ensureOnMainThread([weakThis, currentTime] {
+ if (weakThis)
+ weakThis->checkNewVideoFrameMetadata(currentTime);
+ });
+ }];
+}
+
+void MediaPlayerPrivateAVFoundationObjC::checkNewVideoFrameMetadata(CMTime currentTime)
+{
+ if (!updateLastPixelBuffer())
+ return;
+
+ VideoFrameMetadata metadata;
+ metadata.width = m_cachedPresentationSize.width();
+ metadata.height = m_cachedPresentationSize.height();
+ metadata.presentedFrames = ++m_sampleCount;
+ metadata.mediaTime = PAL::CMTimeGetSeconds(currentTime);
+ // FIXME: presentationTime and expectedDisplayTime might not always have the same value, we should try getting more precise values.
+ metadata.presentationTime = MonotonicTime::now().secondsSinceEpoch().seconds();
+ metadata.expectedDisplayTime = metadata.presentationTime;
+
+ m_videoFrameMetadata = metadata;
+ player()->onNewVideoFrameMetadata(WTFMove(metadata), m_lastPixelBuffer.get());
+}
+
+void MediaPlayerPrivateAVFoundationObjC::stopVideoFrameMetadataGathering()
+{
+ m_isGatheringVideoFrameMetadata = false;
+ m_videoFrameMetadata = { };
+
+ if (m_videoFrameMetadataGatheringObserver) {
+ [m_avPlayer removeTimeObserver:m_videoFrameMetadataGatheringObserver.get()];
+ m_videoFrameMetadataGatheringObserver = nil;
+ }
+}
+
MediaTime MediaPlayerPrivateAVFoundationObjC::platformDuration() const
{
// Do not ask the asset for duration before it has been loaded or it will fetch the
@@ -2548,7 +2598,7 @@
// Calls to copyPixelBufferForItemTime:itemTimeForDisplay: may return nil if the pixel buffer
// for the requested time has already been retrieved. In this case, the last valid image (if any)
// should be displayed.
- if (!updateLastPixelBuffer() && (m_lastImage || !m_lastPixelBuffer))
+ if ((m_isGatheringVideoFrameMetadata || !updateLastPixelBuffer()) && (m_lastImage || !m_lastPixelBuffer))
return;
if (!m_pixelBufferConformer) {
@@ -2583,9 +2633,8 @@
RetainPtr<CVPixelBufferRef> MediaPlayerPrivateAVFoundationObjC::pixelBufferForCurrentTime()
{
- updateLastPixelBuffer();
- if (!m_lastPixelBuffer)
- return nullptr;
+ if (!m_isGatheringVideoFrameMetadata)
+ updateLastPixelBuffer();
return m_lastPixelBuffer;
}
Modified: trunk/Source/WebCore/platform/graphics/cv/PixelBufferConformerCV.h (285992 => 285993)
--- trunk/Source/WebCore/platform/graphics/cv/PixelBufferConformerCV.h 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/Source/WebCore/platform/graphics/cv/PixelBufferConformerCV.h 2021-11-18 10:34:22 UTC (rev 285993)
@@ -38,7 +38,7 @@
public:
WEBCORE_EXPORT PixelBufferConformerCV(CFDictionaryRef attributes);
WEBCORE_EXPORT RetainPtr<CVPixelBufferRef> convert(CVPixelBufferRef);
- RetainPtr<CGImageRef> createImageFromPixelBuffer(CVPixelBufferRef);
+ WEBCORE_EXPORT RetainPtr<CGImageRef> createImageFromPixelBuffer(CVPixelBufferRef);
private:
RetainPtr<VTPixelBufferConformerRef> m_pixelConformer;
Modified: trunk/Source/WebKit/ChangeLog (285992 => 285993)
--- trunk/Source/WebKit/ChangeLog 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/Source/WebKit/ChangeLog 2021-11-18 10:34:22 UTC (rev 285993)
@@ -1,3 +1,28 @@
+2021-11-18 Youenn Fablet <[email protected]>
+
+ MediaPlayerAVFoundation should support rvfc
+ https://bugs.webkit.org/show_bug.cgi?id=231945
+ <rdar://problem/84657372>
+
+ Reviewed by Eric Carlson.
+
+ Add IPC messaging to start/stop metadata gathering from WebProcess to GPUProcess.
+ Add IPC messaging to push metadata to WebProcess, including sending the corresponding CVPixelBuffer for the current metadata.
+ MediaPlayerPrivateRemote will then serve that buffer when being requested the current image.
+ This ensures that painting on a canvas does not mark the current buffer as acquired, which would then skip the video frame metadata gathering.
+ This also ensures the same buffer is used if a video element is painted several times during the same rvfc callback.
+ This has the side benefit of removing the synchronous IPC to paint the video element in a canvas.
+
+ * GPUProcess/media/RemoteMediaPlayerProxy.cpp:
+ * GPUProcess/media/RemoteMediaPlayerProxy.h:
+ * GPUProcess/media/RemoteMediaPlayerProxy.messages.in:
+ * GPUProcess/media/cocoa/RemoteMediaPlayerProxyCocoa.mm:
+ * Scripts/webkit/messages.py:
+ * WebProcess/GPU/media/MediaPlayerPrivateRemote.cpp:
+ * WebProcess/GPU/media/MediaPlayerPrivateRemote.h:
+ * WebProcess/GPU/media/MediaPlayerPrivateRemote.messages.in:
+ * WebProcess/GPU/media/cocoa/MediaPlayerPrivateRemoteCocoa.mm:
+
2021-11-18 Chris Lord <[email protected]>
[GLIB] twitch.tv forces synchronous scrolling
Modified: trunk/Source/WebKit/GPUProcess/media/RemoteMediaPlayerProxy.cpp (285992 => 285993)
--- trunk/Source/WebKit/GPUProcess/media/RemoteMediaPlayerProxy.cpp 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/Source/WebKit/GPUProcess/media/RemoteMediaPlayerProxy.cpp 2021-11-18 10:34:22 UTC (rev 285993)
@@ -1098,6 +1098,18 @@
m_player->pauseAtHostTime(time);
}
+void RemoteMediaPlayerProxy::startVideoFrameMetadataGathering()
+{
+ if (m_player)
+ m_player->startVideoFrameMetadataGathering();
+}
+
+void RemoteMediaPlayerProxy::stopVideoFrameMetadataGathering()
+{
+ if (m_player)
+ m_player->startVideoFrameMetadataGathering();
+}
+
#if !RELEASE_LOG_DISABLED
WTFLogChannel& RemoteMediaPlayerProxy::logChannel() const
{
Modified: trunk/Source/WebKit/GPUProcess/media/RemoteMediaPlayerProxy.h (285992 => 285993)
--- trunk/Source/WebKit/GPUProcess/media/RemoteMediaPlayerProxy.h 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/Source/WebKit/GPUProcess/media/RemoteMediaPlayerProxy.h 2021-11-18 10:34:22 UTC (rev 285993)
@@ -307,6 +307,12 @@
void playAtHostTime(MonotonicTime);
void pauseAtHostTime(MonotonicTime);
+ void startVideoFrameMetadataGathering();
+ void stopVideoFrameMetadataGathering();
+#if PLATFORM(COCOA)
+ void mediaPlayerOnNewVideoFrameMetadata(WebCore::VideoFrameMetadata&&, RetainPtr<CVPixelBufferRef>&&);
+#endif
+
bool mediaPlayerPausedOrStalled() const;
void currentTimeChanged(const MediaTime&);
Modified: trunk/Source/WebKit/GPUProcess/media/RemoteMediaPlayerProxy.messages.in (285992 => 285993)
--- trunk/Source/WebKit/GPUProcess/media/RemoteMediaPlayerProxy.messages.in 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/Source/WebKit/GPUProcess/media/RemoteMediaPlayerProxy.messages.in 2021-11-18 10:34:22 UTC (rev 285993)
@@ -132,6 +132,9 @@
PlayAtHostTime(MonotonicTime time)
PauseAtHostTime(MonotonicTime time)
+
+ StartVideoFrameMetadataGathering()
+ StopVideoFrameMetadataGathering()
}
#endif
Modified: trunk/Source/WebKit/GPUProcess/media/cocoa/RemoteMediaPlayerProxyCocoa.mm (285992 => 285993)
--- trunk/Source/WebKit/GPUProcess/media/cocoa/RemoteMediaPlayerProxyCocoa.mm 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/Source/WebKit/GPUProcess/media/cocoa/RemoteMediaPlayerProxyCocoa.mm 2021-11-18 10:34:22 UTC (rev 285993)
@@ -30,6 +30,7 @@
#import "LayerHostingContext.h"
#import "MediaPlayerPrivateRemoteMessages.h"
+#import "WebCoreArgumentCoders.h"
#import <QuartzCore/QuartzCore.h>
#import <WebCore/FloatSize.h>
#import <WebCore/IOSurface.h>
@@ -83,6 +84,11 @@
setVideoInlineSizeIfPossible(*m_inlineLayerHostingContext, size);
}
+void RemoteMediaPlayerProxy::mediaPlayerOnNewVideoFrameMetadata(VideoFrameMetadata&& metadata, RetainPtr<CVPixelBufferRef>&& buffer)
+{
+ m_webProcessConnection->send(Messages::MediaPlayerPrivateRemote::PushVideoFrameMetadata(metadata, buffer), m_id);
+}
+
void RemoteMediaPlayerProxy::nativeImageForCurrentTime(CompletionHandler<void(std::optional<WTF::MachSendRight>&&, WebCore::DestinationColorSpace)>&& completionHandler)
{
if (!m_player) {
Modified: trunk/Source/WebKit/Scripts/webkit/messages.py (285992 => 285993)
--- trunk/Source/WebKit/Scripts/webkit/messages.py 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/Source/WebKit/Scripts/webkit/messages.py 2021-11-18 10:34:22 UTC (rev 285993)
@@ -371,8 +371,9 @@
'"LayerHostingContext.h"': ["PLATFORM(COCOA)", ],
'"GestureTypes.h"': ["PLATFORM(IOS_FAMILY)"],
'"WCLayerTreeHostIdentifier.h"': ["USE(GRAPHICS_LAYER_WC)"],
+ '<WebCore/CVUtilities.h>': ["PLATFORM(COCOA)", ],
+ '<WebCore/DataDetectorType.h>': ["ENABLE(DATA_DETECTION)"],
'<WebCore/MediaPlaybackTargetContext.h>': ["ENABLE(WIRELESS_PLAYBACK_TARGET)"],
- '<WebCore/DataDetectorType.h>': ["ENABLE(DATA_DETECTION)"],
}
if not header in conditions:
return None
@@ -679,6 +680,7 @@
header_infos_and_types = class_template_headers(type)
special_cases = {
+ 'CVPixelBufferRef': ['<WebCore/CVUtilities.h>'],
'IPC::Semaphore': ['"IPCSemaphore.h"'],
'Inspector::ExtensionError': ['"InspectorExtensionTypes.h"'],
'Inspector::FrontendChannel::ConnectionType': ['<_javascript_Core/InspectorFrontendChannel.h>'],
Modified: trunk/Source/WebKit/WebProcess/GPU/media/MediaPlayerPrivateRemote.cpp (285992 => 285993)
--- trunk/Source/WebKit/WebProcess/GPU/media/MediaPlayerPrivateRemote.cpp 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/Source/WebKit/WebProcess/GPU/media/MediaPlayerPrivateRemote.cpp 2021-11-18 10:34:22 UTC (rev 285993)
@@ -80,6 +80,7 @@
#endif
#if PLATFORM(COCOA)
+#import <WebCore/PixelBufferConformerCV.h>
#import <WebCore/VideoLayerManagerObjC.h>
#endif
@@ -1377,6 +1378,27 @@
return true;
}
+std::optional<VideoFrameMetadata> MediaPlayerPrivateRemote::videoFrameMetadata()
+{
+ auto videoFrameMetadata = std::exchange(m_videoFrameMetadata, { });
+ return videoFrameMetadata;
+}
+
+void MediaPlayerPrivateRemote::startVideoFrameMetadataGathering()
+{
+ m_isGatheringVideoFrameMetadata = true;
+ connection().send(Messages::RemoteMediaPlayerProxy::StartVideoFrameMetadataGathering(), m_id);
+}
+
+void MediaPlayerPrivateRemote::stopVideoFrameMetadataGathering()
+{
+ m_isGatheringVideoFrameMetadata = false;
+#if PLATFORM(COCOA)
+ m_pixelBufferGatheredWithVideoFrameMetadata = nullptr;
+#endif
+ connection().send(Messages::RemoteMediaPlayerProxy::StopVideoFrameMetadataGathering(), m_id);
+}
+
void MediaPlayerPrivateRemote::requestResource(RemoteMediaResourceIdentifier remoteMediaResourceIdentifier, WebCore::ResourceRequest&& request, WebCore::PlatformMediaResourceLoader::LoadOptions options, CompletionHandler<void()>&& completionHandler)
{
ASSERT(!m_mediaResources.contains(remoteMediaResourceIdentifier));
Modified: trunk/Source/WebKit/WebProcess/GPU/media/MediaPlayerPrivateRemote.h (285992 => 285993)
--- trunk/Source/WebKit/WebProcess/GPU/media/MediaPlayerPrivateRemote.h 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/Source/WebKit/WebProcess/GPU/media/MediaPlayerPrivateRemote.h 2021-11-18 10:34:22 UTC (rev 285993)
@@ -39,6 +39,7 @@
#include "VideoTrackPrivateRemote.h"
#include <WebCore/MediaPlayerPrivate.h>
#include <WebCore/SecurityOriginData.h>
+#include <WebCore/VideoFrameMetadata.h>
#include <wtf/LoggerHelper.h>
#include <wtf/MediaTime.h>
#include <wtf/WeakPtr.h>
@@ -56,6 +57,10 @@
class ISOWebVTTCue;
class SerializedPlatformDataCueValue;
class VideoLayerManager;
+
+#if PLATFORM(COCOA)
+class PixelBufferConformerCV;
+#endif
}
namespace WebKit {
@@ -400,6 +405,14 @@
bool pauseAtHostTime(const MonotonicTime&) final;
void updateConfiguration(RemoteMediaPlayerConfiguration&&);
+ std::optional<WebCore::VideoFrameMetadata> videoFrameMetadata() final;
+ void startVideoFrameMetadataGathering() final;
+ void stopVideoFrameMetadataGathering() final;
+
+#if PLATFORM(COCOA)
+ void pushVideoFrameMetadata(WebCore::VideoFrameMetadata&&, RetainPtr<CVPixelBufferRef>&&);
+#endif
+
WeakPtr<WebCore::MediaPlayer> m_player;
Ref<WebCore::PlatformMediaResourceLoader> m_mediaResourceLoader;
#if PLATFORM(COCOA)
@@ -450,6 +463,12 @@
#if USE(AVFOUNDATION)
RetainPtr<CVPixelBufferRef> m_pixelBufferForCurrentTime;
#endif
+#if PLATFORM(COCOA)
+ RetainPtr<CVPixelBufferRef> m_pixelBufferGatheredWithVideoFrameMetadata;
+ std::unique_ptr<WebCore::PixelBufferConformerCV> m_pixelBufferConformer;
+#endif
+ std::optional<WebCore::VideoFrameMetadata> m_videoFrameMetadata;
+ bool m_isGatheringVideoFrameMetadata { false };
};
} // namespace WebKit
Modified: trunk/Source/WebKit/WebProcess/GPU/media/MediaPlayerPrivateRemote.messages.in (285992 => 285993)
--- trunk/Source/WebKit/WebProcess/GPU/media/MediaPlayerPrivateRemote.messages.in 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/Source/WebKit/WebProcess/GPU/media/MediaPlayerPrivateRemote.messages.in 2021-11-18 10:34:22 UTC (rev 285993)
@@ -91,6 +91,10 @@
#if PLATFORM(IOS_FAMILY)
GetRawCookies(URL url) -> (Vector<WebCore::Cookie> result) Async
#endif
+
+#if PLATFORM(COCOA)
+ PushVideoFrameMetadata(struct WebCore::VideoFrameMetadata metadata, RetainPtr<CVPixelBufferRef> buffer);
+#endif
}
#endif
Modified: trunk/Source/WebKit/WebProcess/GPU/media/cocoa/MediaPlayerPrivateRemoteCocoa.mm (285992 => 285993)
--- trunk/Source/WebKit/WebProcess/GPU/media/cocoa/MediaPlayerPrivateRemoteCocoa.mm 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/Source/WebKit/WebProcess/GPU/media/cocoa/MediaPlayerPrivateRemoteCocoa.mm 2021-11-18 10:34:22 UTC (rev 285993)
@@ -33,9 +33,12 @@
#import "WebCoreArgumentCoders.h"
#import <WebCore/ColorSpaceCG.h>
#import <WebCore/IOSurface.h>
+#import <WebCore/PixelBufferConformerCV.h>
#import <pal/spi/cocoa/QuartzCoreSPI.h>
#import <wtf/MachSendRight.h>
+#import <WebCore/CoreVideoSoftLink.h>
+
namespace WebKit {
using namespace WebCore;
@@ -46,8 +49,25 @@
}
#endif
+void MediaPlayerPrivateRemote::pushVideoFrameMetadata(WebCore::VideoFrameMetadata&& videoFrameMetadata, RetainPtr<CVPixelBufferRef>&& buffer)
+{
+ if (!m_isGatheringVideoFrameMetadata)
+ return;
+ m_videoFrameMetadata = WTFMove(videoFrameMetadata);
+ m_pixelBufferGatheredWithVideoFrameMetadata = WTFMove(buffer);
+}
+
RefPtr<NativeImage> MediaPlayerPrivateRemote::nativeImageForCurrentTime()
{
+ if (m_pixelBufferGatheredWithVideoFrameMetadata) {
+ if (!m_pixelBufferConformer)
+ m_pixelBufferConformer = makeUnique<PixelBufferConformerCV>((__bridge CFDictionaryRef)@{ (__bridge NSString *)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA) });
+ ASSERT(m_pixelBufferConformer);
+ if (!m_pixelBufferConformer)
+ return nullptr;
+ return NativeImage::create(m_pixelBufferConformer->createImageFromPixelBuffer(m_pixelBufferGatheredWithVideoFrameMetadata.get()));
+ }
+
std::optional<MachSendRight> sendRight;
auto colorSpace = DestinationColorSpace::SRGB();
if (!connection().sendSync(Messages::RemoteMediaPlayerProxy::NativeImageForCurrentTime(), Messages::RemoteMediaPlayerProxy::NativeImageForCurrentTime::Reply(sendRight, colorSpace), m_id))
Modified: trunk/Tools/ChangeLog (285992 => 285993)
--- trunk/Tools/ChangeLog 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/Tools/ChangeLog 2021-11-18 10:34:22 UTC (rev 285993)
@@ -1,3 +1,15 @@
+2021-11-18 Youenn Fablet <[email protected]>
+
+ MediaPlayerAVFoundation should support rvfc
+ https://bugs.webkit.org/show_bug.cgi?id=231945
+
+ Reviewed by Eric Carlson.
+
+ Enable rvfc by default in test runner.
+
+ * DumpRenderTree/mac/DumpRenderTree.mm:
+ * WebKitTestRunner/TestController.cpp:
+
2021-11-18 Carlos Garcia Campos <[email protected]>
[GLIB] jsc_value_object_define_property_accessor() throws an exception when called on a value without a wrapper instance
Modified: trunk/Tools/DumpRenderTree/mac/DumpRenderTree.mm (285992 => 285993)
--- trunk/Tools/DumpRenderTree/mac/DumpRenderTree.mm 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/Tools/DumpRenderTree/mac/DumpRenderTree.mm 2021-11-18 10:34:22 UTC (rev 285993)
@@ -863,11 +863,8 @@
[preferences _resetForTesting];
if (enableAllExperimentalFeatures) {
- for (WebFeature *feature in [WebPreferences _experimentalFeatures]) {
- // FIXME: We disable rvfc by default. Enable it when the video backend support is good enough.
- auto enabled = [feature.name isEqual:@"RequestVideoFrameCallback"] ? NO : YES;
- [preferences _setEnabled:enabled forFeature:feature];
- }
+ for (WebFeature *feature in [WebPreferences _experimentalFeatures])
+ [preferences _setEnabled:YES forFeature:feature];
}
if (persistentUserStyleSheetLocation()) {
Modified: trunk/Tools/WebKitTestRunner/TestController.cpp (285992 => 285993)
--- trunk/Tools/WebKitTestRunner/TestController.cpp 2021-11-18 10:06:53 UTC (rev 285992)
+++ trunk/Tools/WebKitTestRunner/TestController.cpp 2021-11-18 10:34:22 UTC (rev 285993)
@@ -905,9 +905,6 @@
if (enableAllExperimentalFeatures)
WKPreferencesEnableAllExperimentalFeatures(preferences);
- // FIXME: We disable rvfc by default. Enable it when the video backend support is good enough.
- WKPreferencesSetRequestVideoFrameCallbackEnabled(preferences, false);
-
WKPreferencesResetAllInternalDebugFeatures(preferences);
WKPreferencesSetProcessSwapOnNavigationEnabled(preferences, options.shouldEnableProcessSwapOnNavigation());
