blob: dd10f1a2e5326a5ce15db169617a2a89c706e3e8 [file] [log] [blame]
/*
* Copyright (C) 2013 Cable Television Laboratories, Inc.
* Copyright (C) 2021 Igalia S.L.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "config.h"
#include "TextCombinerGStreamer.h"
#if ENABLE(VIDEO) && USE(GSTREAMER)
#include "GStreamerCommon.h"
#include "TextCombinerPadGStreamer.h"
#include <wtf/glib/WTFGType.h>
static GstStaticPadTemplate sinkTemplate = GST_STATIC_PAD_TEMPLATE("sink_%u", GST_PAD_SINK, GST_PAD_REQUEST, GST_STATIC_CAPS_ANY);
static GstStaticPadTemplate srcTemplate = GST_STATIC_PAD_TEMPLATE("src", GST_PAD_SRC, GST_PAD_ALWAYS, GST_STATIC_CAPS_ANY);
GST_DEBUG_CATEGORY_STATIC(webkitTextCombinerDebug);
#define GST_CAT_DEFAULT webkitTextCombinerDebug
struct _WebKitTextCombinerPrivate {
GRefPtr<GstElement> combinerElement;
};
#define webkit_text_combiner_parent_class parent_class
WEBKIT_DEFINE_TYPE_WITH_CODE(WebKitTextCombiner, webkit_text_combiner, GST_TYPE_BIN,
GST_DEBUG_CATEGORY_INIT(webkitTextCombinerDebug, "webkittextcombiner", 0, "webkit text combiner"))
using namespace WebCore;
void webKitTextCombinerHandleCaps(WebKitTextCombiner* combiner, GstPad* pad, const GstCaps* caps)
{
ASSERT(caps);
GST_DEBUG_OBJECT(combiner, "Handling caps %" GST_PTR_FORMAT, caps);
auto target = adoptGRef(gst_ghost_pad_get_target(GST_GHOST_PAD(pad)));
auto targetParent = target ? adoptGRef(gst_pad_get_parent_element(target.get())) : nullptr;
auto* combinerPad = WEBKIT_TEXT_COMBINER_PAD(pad);
GRefPtr<GstPad> internalPad;
g_object_get(combinerPad, "inner-combiner-pad", &internalPad.outPtr(), nullptr);
auto cea608Caps = adoptGRef(gst_caps_new_empty_simple("closedcaption/x-cea-608"));
auto textCaps = adoptGRef(gst_caps_new_empty_simple("text/x-raw"));
if (gst_caps_can_intersect(textCaps.get(), caps)) {
// Caps are plain text, we want a WebVTT encoder between the ghostpad and the combinerElement.
if (!target || gstElementFactoryEquals(targetParent.get(), "webvttenc"_s)) {
GST_DEBUG_OBJECT(combiner, "Setting up a WebVTT encoder");
auto* encoder = makeGStreamerElement("webvttenc", nullptr);
ASSERT(encoder);
gst_bin_add(GST_BIN_CAST(combiner), encoder);
gst_element_sync_state_with_parent(encoder);
// Switch the ghostpad to target the WebVTT encoder.
auto sinkPad = adoptGRef(gst_element_get_static_pad(encoder, "sink"));
ASSERT(sinkPad);
gst_ghost_pad_set_target(GST_GHOST_PAD(pad), sinkPad.get());
// Connect the WebVTT encoder to the combinerElement.
auto srcPad = adoptGRef(gst_element_get_static_pad(encoder, "src"));
ASSERT(srcPad);
gst_pad_link(srcPad.get(), internalPad.get());
} // Else: pipeline is already correct.
} else if (gst_caps_can_intersect(cea608Caps.get(), caps)) {
if (!isGStreamerPluginAvailable("rsclosedcaption") || !isGStreamerPluginAvailable("closedcaption")) {
WTFLogAlways("GStreamer closedcaption plugins are missing. Please install gst-plugins-bad and gst-plugins-rs");
return;
}
GST_DEBUG_OBJECT(combiner, "Converting CEA-608 closed captions to WebVTT.");
auto* encoder = gst_bin_new(nullptr);
auto* queue = gst_element_factory_make("queue", nullptr);
auto* converter = makeGStreamerElement("ccconverter", nullptr);
auto* rawCapsFilter = gst_element_factory_make("capsfilter", nullptr);
auto* webvttEncoder = makeGStreamerElement("cea608tott", nullptr);
auto* vttCapsFilter = gst_element_factory_make("capsfilter", nullptr);
auto rawCaps = adoptGRef(gst_caps_new_simple("closedcaption/x-cea-608", "format", G_TYPE_STRING, "raw", nullptr));
g_object_set(rawCapsFilter, "caps", rawCaps.get(), nullptr);
auto vttCaps = adoptGRef(gst_caps_new_empty_simple("application/x-subtitle-vtt"));
g_object_set(vttCapsFilter, "caps", vttCaps.get(), nullptr);
gst_bin_add_many(GST_BIN_CAST(encoder), queue, converter, rawCapsFilter, webvttEncoder, vttCapsFilter, nullptr);
gst_element_link_many(queue, converter, rawCapsFilter, webvttEncoder, vttCapsFilter, nullptr);
auto encoderSinkPad = adoptGRef(gst_element_get_static_pad(queue, "sink"));
auto* ghostSinkPad = gst_ghost_pad_new("sink", encoderSinkPad.get());
gst_element_add_pad(encoder, ghostSinkPad);
auto encoderSrcPad = adoptGRef(gst_element_get_static_pad(vttCapsFilter, "src"));
auto* ghostSrcPad = gst_ghost_pad_new("src", encoderSrcPad.get());
gst_element_add_pad(encoder, ghostSrcPad);
gst_bin_add(GST_BIN_CAST(combiner), encoder);
gst_element_sync_state_with_parent(encoder);
gst_ghost_pad_set_target(GST_GHOST_PAD(pad), ghostSinkPad);
gst_pad_link(ghostSrcPad, internalPad.get());
} else {
// Caps are not plain text or CEA-608, we assume it's WebVTT.
// Remove the WebVTT encoder if present.
if (target && targetParent) {
GST_DEBUG_OBJECT(combiner, "Removing WebVTT encoder");
gst_element_set_state(targetParent.get(), GST_STATE_NULL);
gst_bin_remove(GST_BIN_CAST(combiner), targetParent.get());
target = nullptr;
targetParent = nullptr;
}
// Link the pad to the combinerElement.
if (!target) {
GST_DEBUG_OBJECT(combiner, "Associating %" GST_PTR_FORMAT " to %" GST_PTR_FORMAT, internalPad.get(), pad);
gst_ghost_pad_set_target(GST_GHOST_PAD(pad), internalPad.get());
} // Else: pipeline is already correct.
}
}
static GstPad* webkitTextCombinerRequestNewPad(GstElement* element, GstPadTemplate* templ, const char* name, const GstCaps* caps)
{
auto* combiner = WEBKIT_TEXT_COMBINER(element);
ASSERT(combiner);
GST_DEBUG_OBJECT(element, "Requesting new sink pad");
auto* ghostPad = GST_PAD_CAST(g_object_new(WEBKIT_TYPE_TEXT_COMBINER_PAD, "direction", GST_PAD_SINK, nullptr));
ASSERT(ghostPad);
auto* internalPad = gst_element_request_pad(combiner->priv->combinerElement.get(), templ, name, caps);
g_object_set(WEBKIT_TEXT_COMBINER_PAD(ghostPad), "inner-combiner-pad", internalPad, nullptr);
gst_pad_set_active(ghostPad, true);
gst_element_add_pad(GST_ELEMENT_CAST(combiner), ghostPad);
return ghostPad;
}
static void webkitTextCombinerReleasePad(GstElement* element, GstPad* pad)
{
auto* combiner = WEBKIT_TEXT_COMBINER(element);
auto* combinerPad = WEBKIT_TEXT_COMBINER_PAD(pad);
if (auto target = adoptGRef(gst_ghost_pad_get_target(GST_GHOST_PAD(pad)))) {
auto parent = adoptGRef(gst_pad_get_parent_element(target.get()));
ASSERT(parent);
if (parent) {
gst_element_set_state(parent.get(), GST_STATE_NULL);
gst_bin_remove(GST_BIN_CAST(combiner), parent.get());
}
}
auto internalPad = adoptGRef(webKitTextCombinerPadLeakInternalPadRef(combinerPad));
gst_element_release_request_pad(combiner->priv->combinerElement.get(), internalPad.get());
gst_element_remove_pad(element, pad);
}
static void webKitTextCombinerConstructed(GObject* object)
{
GST_CALL_PARENT(G_OBJECT_CLASS, constructed, (object));
auto* combiner = WEBKIT_TEXT_COMBINER(object);
auto* priv = combiner->priv;
priv->combinerElement = gst_element_factory_make("concat", nullptr);
ASSERT(priv->combinerElement);
g_object_set(priv->combinerElement.get(), "adjust-base", FALSE, nullptr);
gst_bin_add(GST_BIN_CAST(combiner), priv->combinerElement.get());
auto pad = adoptGRef(gst_element_get_static_pad(priv->combinerElement.get(), "src"));
ASSERT(pad);
gst_element_add_pad(GST_ELEMENT_CAST(combiner), gst_ghost_pad_new("src", pad.get()));
}
static void webkit_text_combiner_class_init(WebKitTextCombinerClass* klass)
{
auto* elementClass = GST_ELEMENT_CLASS(klass);
auto* objectClass = G_OBJECT_CLASS(klass);
objectClass->constructed = webKitTextCombinerConstructed;
gst_element_class_add_pad_template(elementClass, gst_static_pad_template_get(&sinkTemplate));
gst_element_class_add_pad_template(elementClass, gst_static_pad_template_get(&srcTemplate));
gst_element_class_set_metadata(elementClass, "WebKit text combiner", "Generic",
"A combiner that accepts any caps, but converts plain text to WebVTT",
"Brendan Long <b.long@cablelabs.com>");
elementClass->request_new_pad = GST_DEBUG_FUNCPTR(webkitTextCombinerRequestNewPad);
elementClass->release_pad = GST_DEBUG_FUNCPTR(webkitTextCombinerReleasePad);
}
GstElement* webkitTextCombinerNew()
{
// The combiner relies on webvttenc, fail early if it's not there.
if (!isGStreamerPluginAvailable("subenc")) {
WTFLogAlways("WebKit wasn't able to find a WebVTT encoder. Not continuing without platform support for subtitles.");
return nullptr;
}
return GST_ELEMENT(g_object_new(WEBKIT_TYPE_TEXT_COMBINER, nullptr));
}
#endif // ENABLE(VIDEO) && USE(GSTREAMER)