| /* |
| * Copyright (C) 2020 Apple Inc. All rights reserved. |
| * |
| * Redistribution and use in source and binary forms, with or without |
| * modification, are permitted provided that the following conditions |
| * are met: |
| * 1. Redistributions of source code must retain the above copyright |
| * notice, this list of conditions and the following disclaimer. |
| * 2. Redistributions in binary form must reproduce the above copyright |
| * notice, this list of conditions and the following disclaimer in the |
| * documentation and/or other materials provided with the distribution. |
| * |
| * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND ANY |
| * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
| * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
| * DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY |
| * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES |
| * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
| * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON |
| * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| */ |
| |
| #import "config.h" |
| #import "VideoSampleBufferCompressor.h" |
| |
| #if ENABLE(MEDIA_RECORDER) && USE(AVFOUNDATION) |
| |
| #import "ContentType.h" |
| #import "Logging.h" |
| #import <CoreMedia/CoreMedia.h> |
| #import <Foundation/Foundation.h> |
| #import <VideoToolbox/VTCompressionSession.h> |
| #import <wtf/SoftLinking.h> |
| |
| #import <pal/cf/CoreMediaSoftLink.h> |
| #import <pal/cf/VideoToolboxSoftLink.h> |
| |
| namespace WebCore { |
| |
| std::unique_ptr<VideoSampleBufferCompressor> VideoSampleBufferCompressor::create(String mimeType, CMBufferQueueTriggerCallback callback, void* callbackObject) |
| { |
| auto profile = Profile::Baseline; |
| for (auto codec : ContentType(mimeType).codecs()) { |
| if (startsWithLettersIgnoringASCIICase(codec, "avc1."_s) && codec.length() >= 11) { |
| if (codec[5] == '6' && codec[6] == '4') |
| profile = Profile::High; |
| else if (codec[5] == '4' && (codec[6] == 'd' || codec[6] == 'D')) |
| profile = Profile::Main; |
| break; |
| } |
| } |
| |
| auto compressor = std::unique_ptr<VideoSampleBufferCompressor>(new VideoSampleBufferCompressor(kCMVideoCodecType_H264, profile)); |
| if (!compressor->initialize(callback, callbackObject)) |
| return nullptr; |
| return compressor; |
| } |
| |
| VideoSampleBufferCompressor::VideoSampleBufferCompressor(CMVideoCodecType outputCodecType, Profile profile) |
| : m_serialDispatchQueue { WorkQueue::create("com.apple.VideoSampleBufferCompressor") } |
| , m_outputCodecType { outputCodecType } |
| , m_profile { profile } |
| { |
| } |
| |
| VideoSampleBufferCompressor::~VideoSampleBufferCompressor() |
| { |
| if (m_vtSession) { |
| PAL::VTCompressionSessionInvalidate(m_vtSession.get()); |
| m_vtSession = nullptr; |
| } |
| } |
| |
| bool VideoSampleBufferCompressor::initialize(CMBufferQueueTriggerCallback callback, void* callbackObject) |
| { |
| CMBufferQueueRef outputBufferQueue; |
| if (auto error = PAL::CMBufferQueueCreate(kCFAllocatorDefault, 0, PAL::CMBufferQueueGetCallbacksForUnsortedSampleBuffers(), &outputBufferQueue)) { |
| RELEASE_LOG_ERROR(MediaStream, "VideoSampleBufferCompressor unable to create buffer queue %d", error); |
| return false; |
| } |
| m_outputBufferQueue = adoptCF(outputBufferQueue); |
| PAL::CMBufferQueueInstallTrigger(m_outputBufferQueue.get(), callback, callbackObject, kCMBufferQueueTrigger_WhenDataBecomesReady, PAL::kCMTimeZero, NULL); |
| |
| m_isEncoding = true; |
| return true; |
| } |
| |
| void VideoSampleBufferCompressor::setBitsPerSecond(unsigned bitRate) |
| { |
| m_outputBitRate = bitRate; |
| } |
| |
| void VideoSampleBufferCompressor::finish() |
| { |
| m_serialDispatchQueue->dispatchSync([this] { |
| auto error = PAL::VTCompressionSessionCompleteFrames(m_vtSession.get(), PAL::kCMTimeInvalid); |
| RELEASE_LOG_ERROR_IF(error, MediaStream, "VideoSampleBufferCompressor VTCompressionSessionCompleteFrames failed with %d", error); |
| |
| error = PAL::CMBufferQueueMarkEndOfData(m_outputBufferQueue.get()); |
| RELEASE_LOG_ERROR_IF(error, MediaStream, "VideoSampleBufferCompressor CMBufferQueueMarkEndOfData failed with %d", error); |
| |
| m_isEncoding = false; |
| }); |
| } |
| |
| void VideoSampleBufferCompressor::videoCompressionCallback(void *refCon, void*, OSStatus status, VTEncodeInfoFlags, CMSampleBufferRef buffer) |
| { |
| RELEASE_LOG_ERROR_IF(status, MediaStream, "VideoSampleBufferCompressor videoCompressionCallback status is %d", status); |
| if (status != noErr) |
| return; |
| |
| VideoSampleBufferCompressor *compressor = static_cast<VideoSampleBufferCompressor*>(refCon); |
| |
| auto error = PAL::CMBufferQueueEnqueue(compressor->m_outputBufferQueue.get(), buffer); |
| RELEASE_LOG_ERROR_IF(error, MediaStream, "VideoSampleBufferCompressor CMBufferQueueEnqueue failed with %d", error); |
| } |
| |
| static inline OSStatus setCompressionSessionProperty(VTCompressionSessionRef vtSession, CFStringRef key, uint32_t value) |
| { |
| int64_t value64 = value; |
| auto cfValue = adoptCF(CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &value64)); |
| OSStatus status = VTSessionSetProperty(vtSession, key, cfValue.get()); |
| return status; |
| } |
| |
| CFStringRef VideoSampleBufferCompressor::vtProfileLevel() const |
| { |
| switch (m_profile) { |
| case Profile::Baseline: |
| return PAL::kVTProfileLevel_H264_Baseline_AutoLevel; |
| case Profile::High: |
| return PAL::kVTProfileLevel_H264_High_AutoLevel; |
| case Profile::Main: |
| return PAL::kVTProfileLevel_H264_Main_AutoLevel; |
| } |
| } |
| |
| bool VideoSampleBufferCompressor::initCompressionSession(CMVideoFormatDescriptionRef formatDescription) |
| { |
| CMVideoDimensions dimensions = PAL::CMVideoFormatDescriptionGetDimensions(formatDescription); |
| #if PLATFORM(IOS) |
| NSDictionary *encoderSpecifications = nil; |
| #else |
| NSDictionary *encoderSpecifications = @{(__bridge NSString *)PAL::kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder:@YES}; |
| #endif |
| |
| VTCompressionSessionRef vtSession; |
| auto error = PAL::VTCompressionSessionCreate(kCFAllocatorDefault, dimensions.width, dimensions.height, m_outputCodecType, (__bridge CFDictionaryRef)encoderSpecifications, NULL, NULL, videoCompressionCallback, this, &vtSession); |
| if (error) { |
| RELEASE_LOG_ERROR(MediaStream, "VideoSampleBufferCompressor VTCompressionSessionCreate failed with %d", error); |
| return NO; |
| } |
| m_vtSession = adoptCF(vtSession); |
| |
| error = VTSessionSetProperty(m_vtSession.get(), PAL::kVTCompressionPropertyKey_RealTime, kCFBooleanTrue); |
| RELEASE_LOG_ERROR_IF(error, MediaStream, "VideoSampleBufferCompressor VTSessionSetProperty kVTCompressionPropertyKey_RealTime failed with %d", error); |
| error = setCompressionSessionProperty(m_vtSession.get(), PAL::kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, m_maxKeyFrameIntervalDuration); |
| RELEASE_LOG_ERROR_IF(error, MediaStream, "VideoSampleBufferCompressor VTSessionSetProperty kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration failed with %d", error); |
| error = setCompressionSessionProperty(m_vtSession.get(), PAL::kVTCompressionPropertyKey_MaxKeyFrameInterval, m_maxKeyFrameIntervalDuration * m_expectedFrameRate); |
| RELEASE_LOG_ERROR_IF(error, MediaStream, "VideoSampleBufferCompressor VTSessionSetProperty kVTCompressionPropertyKey_MaxKeyFrameInterval failed with %d", error); |
| error = setCompressionSessionProperty(m_vtSession.get(), PAL::kVTCompressionPropertyKey_ExpectedFrameRate, m_expectedFrameRate); |
| RELEASE_LOG_ERROR_IF(error, MediaStream, "VideoSampleBufferCompressor VTSessionSetProperty kVTCompressionPropertyKey_ExpectedFrameRate failed with %d", error); |
| |
| error = VTSessionSetProperty(m_vtSession.get(), PAL::kVTCompressionPropertyKey_ProfileLevel, vtProfileLevel()); |
| if (error) { |
| RELEASE_LOG_ERROR(MediaStream, "VideoSampleBufferCompressor VTSessionSetProperty kVTCompressionPropertyKey_ProfileLevel failed with %d for profile %d", error, m_profile); |
| if (m_profile != Profile::Baseline) { |
| error = VTSessionSetProperty(m_vtSession.get(), PAL::kVTCompressionPropertyKey_ProfileLevel, PAL::kVTProfileLevel_H264_Baseline_AutoLevel); |
| RELEASE_LOG_ERROR_IF(error, MediaStream, "VideoSampleBufferCompressor VTSessionSetProperty kVTCompressionPropertyKey_ProfileLevel failed with %d for default profile", error); |
| } |
| } |
| |
| if (m_outputBitRate) { |
| error = setCompressionSessionProperty(m_vtSession.get(), PAL::kVTCompressionPropertyKey_AverageBitRate, *m_outputBitRate); |
| RELEASE_LOG_ERROR_IF(error, MediaStream, "VideoSampleBufferCompressor VTSessionSetProperty kVTCompressionPropertyKey_AverageBitRate failed with %d", error); |
| } |
| |
| error = PAL::VTCompressionSessionPrepareToEncodeFrames(m_vtSession.get()); |
| RELEASE_LOG_ERROR_IF(error, MediaStream, "VideoSampleBufferCompressor VTCompressionSessionPrepareToEncodeFrames failed with %d", error); |
| |
| return YES; |
| } |
| |
| void VideoSampleBufferCompressor::processSampleBuffer(CMSampleBufferRef buffer) |
| { |
| if (!m_vtSession) { |
| if (!initCompressionSession(PAL::CMSampleBufferGetFormatDescription(buffer))) |
| return; |
| } |
| |
| auto imageBuffer = PAL::CMSampleBufferGetImageBuffer(buffer); |
| auto presentationTimeStamp = PAL::CMSampleBufferGetPresentationTimeStamp(buffer); |
| auto duration = PAL::CMSampleBufferGetDuration(buffer); |
| auto error = PAL::VTCompressionSessionEncodeFrame(m_vtSession.get(), imageBuffer, presentationTimeStamp, duration, NULL, this, NULL); |
| RELEASE_LOG_ERROR_IF(error, MediaStream, "VideoSampleBufferCompressor VTCompressionSessionEncodeFrame failed with %d", error); |
| } |
| |
| void VideoSampleBufferCompressor::addSampleBuffer(CMSampleBufferRef buffer) |
| { |
| m_serialDispatchQueue->dispatchSync([this, buffer] { |
| if (!m_isEncoding) |
| return; |
| |
| processSampleBuffer(buffer); |
| }); |
| } |
| |
| CMSampleBufferRef VideoSampleBufferCompressor::getOutputSampleBuffer() |
| { |
| return (CMSampleBufferRef)(const_cast<void*>(PAL::CMBufferQueueGetHead(m_outputBufferQueue.get()))); |
| } |
| |
| RetainPtr<CMSampleBufferRef> VideoSampleBufferCompressor::takeOutputSampleBuffer() |
| { |
| return adoptCF((CMSampleBufferRef)(const_cast<void*>(PAL::CMBufferQueueDequeueAndRetain(m_outputBufferQueue.get())))); |
| } |
| |
| unsigned VideoSampleBufferCompressor::bitRate() const |
| { |
| return m_outputBitRate.value_or(0); |
| } |
| |
| } |
| #endif |