Source/WebCore/platform/mediastream/libwebrtc/GStreamerVideoDecoderFactory.cpp

/*
 * Copyright (C) 2018 Metrological Group B.V.
 * Copyright (C) 2018 Igalia S.L. All rights reserved.
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public License
 * aint with this library; see the file COPYING.LIB.  If not, write to
 * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
 * Boston, MA 02110-1301, USA.
 */

#include "config.h"

#if ENABLE(VIDEO) && ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER)
#include "GStreamerVideoDecoderFactory.h"

#include "GStreamerVideoFrameLibWebRTC.h"
#include "webrtc/common_video/h264/h264_common.h"
#include "webrtc/common_video/h264/profile_level_id.h"
#include "webrtc/media/base/codec.h"
#include "webrtc/modules/video_coding/codecs/h264/include/h264.h"
#include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h"
#include "webrtc/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.h"
#include "webrtc/modules/video_coding/include/video_codec_interface.h"
#include <gst/app/gstappsink.h>
#include <gst/app/gstappsrc.h>
#include <gst/video/video.h>
#include <mutex>
#include <wtf/Lock.h>
#include <wtf/StdMap.h>
#include <wtf/glib/RunLoopSourcePriority.h>
#include <wtf/text/WTFString.h>

GST_DEBUG_CATEGORY(webkit_webrtcdec_debug);
#define GST_CAT_DEFAULT webkit_webrtcdec_debug

namespace WebCore {

typedef struct {
    uint64_t timestamp;
    int64_t renderTimeMs;
} InputTimestamps;

class GStreamerVideoDecoder : public webrtc::VideoDecoder {
public:
    GStreamerVideoDecoder()
        : m_pictureId(0)
        , m_width(0)
        , m_height(0)
        , m_requireParse(false)
        , m_needsKeyframe(true)
        , m_firstBufferPts(GST_CLOCK_TIME_NONE)
        , m_firstBufferDts(GST_CLOCK_TIME_NONE)
    {
    }

    static void decodebinPadAddedCb(GstElement*,
        GstPad* srcpad,
        GstPad* sinkpad)
    {
        GST_INFO_OBJECT(srcpad, "connecting pad with %" GST_PTR_FORMAT, sinkpad);
        if (gst_pad_link(srcpad, sinkpad) != GST_PAD_LINK_OK)
            ASSERT_NOT_REACHED();
    }

    GstElement* pipeline()
    {
        return m_pipeline.get();
    }

    GstElement* makeElement(const gchar* factoryName)
    {
        GUniquePtr<char> name(g_strdup_printf("%s_dec_%s_%p", Name(), factoryName, this));

        return gst_element_factory_make(factoryName, name.get());
    }

    int32_t InitDecode(const webrtc::VideoCodec* codecSettings, int32_t) override
    {
        m_src = makeElement("appsrc");

        GRefPtr<GstCaps> caps = nullptr;
        auto capsfilter = CreateFilter();
        auto decoder = makeElement("decodebin");

        if (codecSettings) {
            m_width = codecSettings->width;
            m_height = codecSettings->height;
        }

        m_pipeline = makeElement("pipeline");
        connectSimpleBusMessageCallback(m_pipeline.get());

        auto sinkpad = adoptGRef(gst_element_get_static_pad(capsfilter, "sink"));
        g_signal_connect(decoder, "pad-added", G_CALLBACK(decodebinPadAddedCb), sinkpad.get());
        // Make the decoder output "parsed" frames only and let the main decodebin
        // do the real decoding. This allows us to have optimized decoding/rendering
        // happening in the main pipeline.
        if (m_requireParse) {
            caps = gst_caps_new_simple(Caps(), "parsed", G_TYPE_BOOLEAN, TRUE, nullptr);
            GRefPtr<GstBus> bus = adoptGRef(gst_pipeline_get_bus(GST_PIPELINE(m_pipeline.get())));

            gst_bus_enable_sync_message_emission(bus.get());
            g_signal_connect(bus.get(), "sync-message::warning",
                G_CALLBACK(+[](GstBus*, GstMessage* message, GStreamerVideoDecoder* justThis) {
                GUniqueOutPtr<GError> err;

                switch (GST_MESSAGE_TYPE(message)) {
                case GST_MESSAGE_WARNING: {
                    gst_message_parse_warning(message, &err.outPtr(), nullptr);
                    FALLTHROUGH;
                }
                case GST_MESSAGE_ERROR: {
                    if (!err)
                        gst_message_parse_error(message, &err.outPtr(), nullptr);

                    if (g_error_matches(err.get(), GST_STREAM_ERROR, GST_STREAM_ERROR_DECODE)) {
                        GST_INFO_OBJECT(justThis->pipeline(), "--> needs keyframe (%s)",
                            err->message);
                        justThis->m_needsKeyframe = true;
                    }
                    break;
                }
                default:
                    break;
                }
                }), this);
        } else {
            /* FIXME - How could we handle missing keyframes case we do not plug parsers ? */
            caps = gst_caps_new_empty_simple(Caps());
        }
        g_object_set(decoder, "caps", caps.get(), nullptr);

        m_sink = makeElement("appsink");
        gst_app_sink_set_emit_signals(GST_APP_SINK(m_sink), true);
        // This is an decoder, everything should happen as fast as possible and not
        // be synced on the clock.
        g_object_set(m_sink, "sync", false, nullptr);

        gst_bin_add_many(GST_BIN(pipeline()), m_src, decoder, capsfilter, m_sink, nullptr);
        if (!gst_element_link(m_src, decoder)) {
            GST_ERROR_OBJECT(pipeline(), "Could not link src to decoder.");
            return WEBRTC_VIDEO_CODEC_ERROR;
        }

        if (!gst_element_link(capsfilter, m_sink)) {
            GST_ERROR_OBJECT(pipeline(), "Could not link capsfilter to sink.");
            return WEBRTC_VIDEO_CODEC_ERROR;
        }

        if (gst_element_set_state(pipeline(), GST_STATE_PLAYING) == GST_STATE_CHANGE_FAILURE) {
            GST_ERROR_OBJECT(pipeline(), "Could not set state to PLAYING.");
            return WEBRTC_VIDEO_CODEC_ERROR;
        }

        return WEBRTC_VIDEO_CODEC_OK;
    }

    int32_t RegisterDecodeCompleteCallback(webrtc::DecodedImageCallback* callback)
    {
        m_imageReadyCb = callback;

        return WEBRTC_VIDEO_CODEC_OK;
    }

    virtual GstElement* CreateFilter()
    {
        return makeElement("identity");
    }

    int32_t Release() final
    {
        if (m_pipeline.get()) {
            GRefPtr<GstBus> bus = adoptGRef(gst_pipeline_get_bus(GST_PIPELINE(m_pipeline.get())));
            gst_bus_set_sync_handler(bus.get(), nullptr, nullptr, nullptr);

            gst_element_set_state(m_pipeline.get(), GST_STATE_NULL);
            m_src = nullptr;
            m_sink = nullptr;
            m_pipeline = nullptr;
        }

        return WEBRTC_VIDEO_CODEC_OK;
    }

    int32_t Decode(const webrtc::EncodedImage& inputImage,
        bool,
        const webrtc::CodecSpecificInfo*,
        int64_t renderTimeMs) override
    {
        if (m_needsKeyframe) {
            if (inputImage._frameType != webrtc::kVideoFrameKey) {
                GST_ERROR("Waiting for keyframe but got a delta unit... asking for keyframe");
                return WEBRTC_VIDEO_CODEC_ERROR;
            }
            if (inputImage._completeFrame)
                m_needsKeyframe = false;
            else {
                GST_ERROR("Waiting for keyframe but didn't get full frame, getting out");
                return WEBRTC_VIDEO_CODEC_ERROR;
            }
        }


        if (!m_src) {
            GST_ERROR("No source set, can't decode.");

            return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
        }

        if (!GST_CLOCK_TIME_IS_VALID(m_firstBufferPts)) {
            GRefPtr<GstPad> srcpad = adoptGRef(gst_element_get_static_pad(m_src, "src"));
            m_firstBufferPts = (static_cast<guint64>(renderTimeMs)) * GST_MSECOND;
            m_firstBufferDts = (static_cast<guint64>(inputImage.Timestamp())) * GST_MSECOND;
        }

        // FIXME- Use a GstBufferPool.
        auto buffer = adoptGRef(gst_buffer_new_wrapped(g_memdup(inputImage._buffer, inputImage._size),
            inputImage._size));
        GST_BUFFER_DTS(buffer.get()) = (static_cast<guint64>(inputImage.Timestamp()) * GST_MSECOND) - m_firstBufferDts;
        GST_BUFFER_PTS(buffer.get()) = (static_cast<guint64>(renderTimeMs) * GST_MSECOND) - m_firstBufferPts;
        InputTimestamps timestamps = {inputImage.Timestamp(), renderTimeMs};
        m_dtsPtsMap[GST_BUFFER_PTS(buffer.get())] = timestamps;

        GST_LOG_OBJECT(pipeline(), "%" G_GINT64_FORMAT " Decoding: %" GST_PTR_FORMAT, renderTimeMs, buffer.get());
        auto sample = adoptGRef(gst_sample_new(buffer.get(), GetCapsForFrame(inputImage), nullptr, nullptr));
        switch (gst_app_src_push_sample(GST_APP_SRC(m_src), sample.get())) {
        case GST_FLOW_OK:
            break;
        case GST_FLOW_FLUSHING:
            return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
        default:
            return WEBRTC_VIDEO_CODEC_ERROR;
        }

        return pullSample();
    }

    int32_t pullSample()
    {
        auto sample = gst_app_sink_try_pull_sample(GST_APP_SINK(m_sink), GST_SECOND / 30);
        if (!sample) {
            GST_ERROR("Needs more data");
            return WEBRTC_VIDEO_CODEC_OK;
        }
        auto buffer = gst_sample_get_buffer(sample);

        // Make sure that the frame.timestamp == previsouly input_frame._timeStamp
        // as it is required by the VideoDecoder baseclass.
        auto timestamps = m_dtsPtsMap[GST_BUFFER_PTS(buffer)];
        m_dtsPtsMap.erase(GST_BUFFER_PTS(buffer));

        auto frame(LibWebRTCVideoFrameFromGStreamerSample(sample, webrtc::kVideoRotation_0,
            timestamps.timestamp, timestamps.renderTimeMs));

        GST_BUFFER_DTS(buffer) = GST_CLOCK_TIME_NONE;
        GST_LOG_OBJECT(pipeline(), "Output decoded frame! %d -> %" GST_PTR_FORMAT,
            frame->timestamp(), buffer);

        m_imageReadyCb->Decoded(*frame.get(), absl::optional<int32_t>(), absl::optional<uint8_t>());

        return WEBRTC_VIDEO_CODEC_OK;
    }

    virtual GstCaps* GetCapsForFrame(const webrtc::EncodedImage& image)
    {
        if (!m_caps) {
            m_caps = adoptGRef(gst_caps_new_simple(Caps(),
                "width", G_TYPE_INT, image._encodedWidth ? image._encodedWidth : m_width,
                "height", G_TYPE_INT, image._encodedHeight ? image._encodedHeight : m_height,
                nullptr));
        }

        return m_caps.get();
    }

    void AddDecoderIfSupported(std::vector<webrtc::SdpVideoFormat> codecList)
    {
        if (HasGstDecoder()) {
            webrtc::SdpVideoFormat format = ConfigureSupportedDecoder();

            codecList.push_back(format);
        }
    }

    virtual webrtc::SdpVideoFormat ConfigureSupportedDecoder()
    {
        return webrtc::SdpVideoFormat(Name());
    }

    static GRefPtr<GstElementFactory> GstDecoderFactory(const char *capsStr)
    {
        auto all_decoders = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_DECODER,
            GST_RANK_MARGINAL);
        auto caps = adoptGRef(gst_caps_from_string(capsStr));
        auto decoders = gst_element_factory_list_filter(all_decoders,
            caps.get(), GST_PAD_SINK, FALSE);

        gst_plugin_feature_list_free(all_decoders);
        GRefPtr<GstElementFactory> res;
        if (decoders)
            res = GST_ELEMENT_FACTORY(decoders->data);
        gst_plugin_feature_list_free(decoders);

        return res;
    }

    bool HasGstDecoder()
    {
        return GstDecoderFactory(Caps());
    }

    virtual const gchar* Caps() = 0;
    virtual webrtc::VideoCodecType CodecType() = 0;
    const char* ImplementationName() const { return "GStreamer"; }
    virtual const gchar* Name() = 0;

protected:
    GRefPtr<GstCaps> m_caps;
    gint m_pictureId;
    gint m_width;
    gint m_height;
    bool m_requireParse = false;
    bool m_needsKeyframe;

private:
    GRefPtr<GstElement> m_pipeline;
    GstElement* m_sink;
    GstElement* m_src;

    GstVideoInfo m_info;
    webrtc::DecodedImageCallback* m_imageReadyCb;

    StdMap<GstClockTime, InputTimestamps> m_dtsPtsMap;
    GstClockTime m_firstBufferPts;
    GstClockTime m_firstBufferDts;
};

class H264Decoder : public GStreamerVideoDecoder {
public:
    H264Decoder() { m_requireParse = true; }

    int32_t InitDecode(const webrtc::VideoCodec* codecInfo, int32_t nCores) final
    {
        if (codecInfo && codecInfo->codecType != webrtc::kVideoCodecH264)
            return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;

        m_profile = nullptr;
        if (codecInfo) {
            auto h264Info = codecInfo->H264();

            switch (h264Info.profile) {
            case webrtc::H264::kProfileConstrainedBaseline:
                m_profile = "constrained-baseline";
                break;
            case webrtc::H264::kProfileBaseline:
                m_profile = "baseline";
                break;
            case webrtc::H264::kProfileMain:
                m_profile = "main";
                break;
            case webrtc::H264::kProfileConstrainedHigh:
                m_profile = "constrained-high";
                break;
            case webrtc::H264::kProfileHigh:
                m_profile = "high";
                break;
            }
        }

        return GStreamerVideoDecoder::InitDecode(codecInfo, nCores);
    }

    GstCaps* GetCapsForFrame(const webrtc::EncodedImage& image) final
    {
        if (!m_caps) {
            m_caps = adoptGRef(gst_caps_new_simple(Caps(),
                "width", G_TYPE_INT, image._encodedWidth ? image._encodedWidth : m_width,
                "height", G_TYPE_INT, image._encodedHeight ? image._encodedHeight : m_height,
                "alignment", G_TYPE_STRING, "au",
                nullptr));
        }

        return m_caps.get();
    }
    const gchar* Caps() final { return "video/x-h264"; }
    const gchar* Name() final { return cricket::kH264CodecName; }
    webrtc::VideoCodecType CodecType() final { return webrtc::kVideoCodecH264; }

private:
    const gchar* m_profile;
};

class VP8Decoder : public GStreamerVideoDecoder {
public:
    VP8Decoder() { }
    const gchar* Caps() final { return "video/x-vp8"; }
    const gchar* Name() final { return cricket::kVp8CodecName; }
    webrtc::VideoCodecType CodecType() final { return webrtc::kVideoCodecVP8; }
    static std::unique_ptr<webrtc::VideoDecoder> Create()
    {
        auto factory = GstDecoderFactory("video/x-vp8");

        if (factory && !g_strcmp0(GST_OBJECT_NAME(GST_OBJECT(factory.get())), "vp8dec")) {
            GST_INFO("Our best GStreamer VP8 decoder is vp8dec, better use the one from LibWebRTC");

            return std::unique_ptr<webrtc::VideoDecoder>(new webrtc::LibvpxVp8Decoder());
        }

        return std::unique_ptr<webrtc::VideoDecoder>(new VP8Decoder());
    }
};

std::unique_ptr<webrtc::VideoDecoder> GStreamerVideoDecoderFactory::CreateVideoDecoder(const webrtc::SdpVideoFormat& format)
{
    webrtc::VideoDecoder* dec;

    if (format.name == cricket::kH264CodecName)
        dec = new H264Decoder();
    else if (format.name == cricket::kVp8CodecName)
        return VP8Decoder::Create();
    else {
        GST_ERROR("Could not create decoder for %s", format.name.c_str());

        return nullptr;
    }

    return std::unique_ptr<webrtc::VideoDecoder>(dec);
}

GStreamerVideoDecoderFactory::GStreamerVideoDecoderFactory()
{
    static std::once_flag debugRegisteredFlag;

    std::call_once(debugRegisteredFlag, [] {
        GST_DEBUG_CATEGORY_INIT(webkit_webrtcdec_debug, "webkitlibwebrtcvideodecoder", 0, "WebKit WebRTC video decoder");
    });
}
std::vector<webrtc::SdpVideoFormat> GStreamerVideoDecoderFactory::GetSupportedFormats() const
{
    std::vector<webrtc::SdpVideoFormat> formats;

    VP8Decoder().AddDecoderIfSupported(formats);
    H264Decoder().AddDecoderIfSupported(formats);

    return formats;
}
}
#endif