1/*
2 * Copyright (C) 2007 OpenedHand
3 * Copyright (C) 2007 Alp Toker <alp@atoker.com>
4 * Copyright (C) 2009, 2010, 2011, 2012, 2015, 2016 Igalia S.L
5 * Copyright (C) 2015, 2016 Metrological Group B.V.
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2 of the License, or (at your option) any later version.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 */
21
22/*
23 *
24 * WebKitVideoSink is a GStreamer sink element that triggers
25 * repaints in the WebKit GStreamer media player for the
26 * current video buffer.
27 */
28
29#include "config.h"
30#include "VideoSinkGStreamer.h"
31
32#if ENABLE(VIDEO) && USE(GSTREAMER)
33#include "GStreamerCommon.h"
34#include "IntSize.h"
35#include <glib.h>
36#include <gst/gst.h>
37#include <gst/video/gstvideometa.h>
38#include <wtf/Condition.h>
39#include <wtf/RunLoop.h>
40
41using namespace WebCore;
42
43// CAIRO_FORMAT_RGB24 used to render the video buffers is little/big endian dependant.
44#if G_BYTE_ORDER == G_LITTLE_ENDIAN
45#define GST_CAPS_FORMAT "{ BGRx, BGRA }"
46#else
47#define GST_CAPS_FORMAT "{ xRGB, ARGB }"
48#endif
49
50#define WEBKIT_VIDEO_SINK_PAD_CAPS GST_VIDEO_CAPS_MAKE_WITH_FEATURES(GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, GST_CAPS_FORMAT) ";" GST_VIDEO_CAPS_MAKE(GST_CAPS_FORMAT)
51
52static GstStaticPadTemplate s_sinkTemplate = GST_STATIC_PAD_TEMPLATE("sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS(WEBKIT_VIDEO_SINK_PAD_CAPS));
53
54
55GST_DEBUG_CATEGORY_STATIC(webkitVideoSinkDebug);
56#define GST_CAT_DEFAULT webkitVideoSinkDebug
57
58enum {
59 REPAINT_REQUESTED,
60 REPAINT_CANCELLED,
61 LAST_SIGNAL
62};
63
64static guint webkitVideoSinkSignals[LAST_SIGNAL] = { 0, };
65
66static void webkitVideoSinkRepaintRequested(WebKitVideoSink*, GstSample*);
67static GRefPtr<GstSample> webkitVideoSinkRequestRender(WebKitVideoSink*, GstBuffer*);
68
69class VideoRenderRequestScheduler {
70public:
71 void start()
72 {
73 LockHolder locker(m_sampleMutex);
74 m_unlocked = false;
75 }
76
77 void stop()
78 {
79 LockHolder locker(m_sampleMutex);
80 m_sample = nullptr;
81 m_unlocked = true;
82 }
83
84 void drain()
85 {
86 LockHolder locker(m_sampleMutex);
87 m_sample = nullptr;
88 }
89
90 bool requestRender(WebKitVideoSink* sink, GstBuffer* buffer)
91 {
92 LockHolder locker(m_sampleMutex);
93 if (m_unlocked)
94 return true;
95
96 m_sample = webkitVideoSinkRequestRender(sink, buffer);
97 if (!m_sample)
98 return false;
99
100 auto sample = WTFMove(m_sample);
101 locker.unlockEarly();
102 if (LIKELY(GST_IS_SAMPLE(sample.get())))
103 webkitVideoSinkRepaintRequested(sink, sample.get());
104
105 return true;
106 }
107
108private:
109 Lock m_sampleMutex;
110 GRefPtr<GstSample> m_sample;
111
112 // If this is true all processing should finish ASAP
113 // This is necessary because there could be a race between
114 // unlock() and render(), where unlock() wins, signals the
115 // Condition, then render() tries to render a frame although
116 // everything else isn't running anymore. This will lead
117 // to deadlocks because render() holds the stream lock.
118 //
119 // Protected by the sample mutex
120 bool m_unlocked { false };
121};
122
123struct _WebKitVideoSinkPrivate {
124 _WebKitVideoSinkPrivate()
125 {
126 gst_video_info_init(&info);
127 }
128
129 ~_WebKitVideoSinkPrivate()
130 {
131 if (currentCaps)
132 gst_caps_unref(currentCaps);
133 }
134
135 VideoRenderRequestScheduler scheduler;
136 GstVideoInfo info;
137 GstCaps* currentCaps { nullptr };
138};
139
140#define webkit_video_sink_parent_class parent_class
141G_DEFINE_TYPE_WITH_CODE(WebKitVideoSink, webkit_video_sink, GST_TYPE_VIDEO_SINK, GST_DEBUG_CATEGORY_INIT(webkitVideoSinkDebug, "webkitsink", 0, "webkit video sink"));
142
143
144static void webkit_video_sink_init(WebKitVideoSink* sink)
145{
146 sink->priv = G_TYPE_INSTANCE_GET_PRIVATE(sink, WEBKIT_TYPE_VIDEO_SINK, WebKitVideoSinkPrivate);
147 g_object_set(GST_BASE_SINK(sink), "enable-last-sample", FALSE, nullptr);
148 new (sink->priv) WebKitVideoSinkPrivate();
149}
150
151static void webkitVideoSinkRepaintRequested(WebKitVideoSink* sink, GstSample* sample)
152{
153 g_signal_emit(sink, webkitVideoSinkSignals[REPAINT_REQUESTED], 0, sample);
154}
155
156static void webkitVideoSinkRepaintCancelled(WebKitVideoSink* sink)
157{
158 g_signal_emit(sink, webkitVideoSinkSignals[REPAINT_CANCELLED], 0);
159}
160
161static GRefPtr<GstSample> webkitVideoSinkRequestRender(WebKitVideoSink* sink, GstBuffer* buffer)
162{
163 WebKitVideoSinkPrivate* priv = sink->priv;
164 GRefPtr<GstSample> sample = adoptGRef(gst_sample_new(buffer, priv->currentCaps, nullptr, nullptr));
165
166 // The video info structure is valid only if the sink handled an allocation query.
167 GstVideoFormat format = GST_VIDEO_INFO_FORMAT(&priv->info);
168 if (format == GST_VIDEO_FORMAT_UNKNOWN)
169 return nullptr;
170
171 return sample;
172}
173
174static GstFlowReturn webkitVideoSinkRender(GstBaseSink* baseSink, GstBuffer* buffer)
175{
176 WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(baseSink);
177 return sink->priv->scheduler.requestRender(sink, buffer) ? GST_FLOW_OK : GST_FLOW_ERROR;
178}
179
180static void webkitVideoSinkFinalize(GObject* object)
181{
182 WEBKIT_VIDEO_SINK(object)->priv->~WebKitVideoSinkPrivate();
183 G_OBJECT_CLASS(parent_class)->finalize(object);
184}
185
186static gboolean webkitVideoSinkUnlock(GstBaseSink* baseSink)
187{
188 WebKitVideoSinkPrivate* priv = WEBKIT_VIDEO_SINK(baseSink)->priv;
189
190 priv->scheduler.stop();
191 webkitVideoSinkRepaintCancelled(WEBKIT_VIDEO_SINK(baseSink));
192
193 return GST_CALL_PARENT_WITH_DEFAULT(GST_BASE_SINK_CLASS, unlock, (baseSink), TRUE);
194}
195
196static gboolean webkitVideoSinkUnlockStop(GstBaseSink* baseSink)
197{
198 WebKitVideoSinkPrivate* priv = WEBKIT_VIDEO_SINK(baseSink)->priv;
199
200 priv->scheduler.start();
201
202 return GST_CALL_PARENT_WITH_DEFAULT(GST_BASE_SINK_CLASS, unlock_stop, (baseSink), TRUE);
203}
204
205static gboolean webkitVideoSinkStop(GstBaseSink* baseSink)
206{
207 WebKitVideoSinkPrivate* priv = WEBKIT_VIDEO_SINK(baseSink)->priv;
208
209 priv->scheduler.stop();
210 webkitVideoSinkRepaintCancelled(WEBKIT_VIDEO_SINK(baseSink));
211 if (priv->currentCaps) {
212 gst_caps_unref(priv->currentCaps);
213 priv->currentCaps = nullptr;
214 }
215
216 return TRUE;
217}
218
219static gboolean webkitVideoSinkStart(GstBaseSink* baseSink)
220{
221 WebKitVideoSinkPrivate* priv = WEBKIT_VIDEO_SINK(baseSink)->priv;
222
223 priv->scheduler.start();
224
225 return TRUE;
226}
227
228static gboolean webkitVideoSinkSetCaps(GstBaseSink* baseSink, GstCaps* caps)
229{
230 WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(baseSink);
231 WebKitVideoSinkPrivate* priv = sink->priv;
232
233 GST_DEBUG_OBJECT(sink, "Current caps %" GST_PTR_FORMAT ", setting caps %" GST_PTR_FORMAT, priv->currentCaps, caps);
234
235 GstVideoInfo videoInfo;
236 gst_video_info_init(&videoInfo);
237 if (!gst_video_info_from_caps(&videoInfo, caps)) {
238 GST_ERROR_OBJECT(sink, "Invalid caps %" GST_PTR_FORMAT, caps);
239 return FALSE;
240 }
241
242 priv->info = videoInfo;
243 gst_caps_replace(&priv->currentCaps, caps);
244 return TRUE;
245}
246
247static gboolean webkitVideoSinkProposeAllocation(GstBaseSink* baseSink, GstQuery* query)
248{
249 GstCaps* caps;
250 gst_query_parse_allocation(query, &caps, nullptr);
251 if (!caps)
252 return FALSE;
253
254 WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(baseSink);
255 if (!gst_video_info_from_caps(&sink->priv->info, caps))
256 return FALSE;
257
258 gst_query_add_allocation_meta(query, GST_VIDEO_META_API_TYPE, nullptr);
259 gst_query_add_allocation_meta(query, GST_VIDEO_CROP_META_API_TYPE, nullptr);
260 gst_query_add_allocation_meta(query, GST_VIDEO_GL_TEXTURE_UPLOAD_META_API_TYPE, nullptr);
261 return TRUE;
262}
263
264static gboolean webkitVideoSinkEvent(GstBaseSink* baseSink, GstEvent* event)
265{
266 switch (GST_EVENT_TYPE(event)) {
267 case GST_EVENT_FLUSH_START: {
268 WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(baseSink);
269 sink->priv->scheduler.drain();
270
271 GST_DEBUG_OBJECT(sink, "Flush-start, releasing m_sample");
272 }
273 FALLTHROUGH;
274 default:
275 return GST_CALL_PARENT_WITH_DEFAULT(GST_BASE_SINK_CLASS, event, (baseSink, event), TRUE);
276 }
277}
278
279static void webkit_video_sink_class_init(WebKitVideoSinkClass* klass)
280{
281 GObjectClass* gobjectClass = G_OBJECT_CLASS(klass);
282 GstBaseSinkClass* baseSinkClass = GST_BASE_SINK_CLASS(klass);
283 GstElementClass* elementClass = GST_ELEMENT_CLASS(klass);
284
285 gst_element_class_add_pad_template(elementClass, gst_static_pad_template_get(&s_sinkTemplate));
286 gst_element_class_set_metadata(elementClass, "WebKit video sink", "Sink/Video", "Sends video data from a GStreamer pipeline to WebKit", "Igalia, Alp Toker <alp@atoker.com>");
287
288 g_type_class_add_private(klass, sizeof(WebKitVideoSinkPrivate));
289
290 gobjectClass->finalize = webkitVideoSinkFinalize;
291
292 baseSinkClass->unlock = webkitVideoSinkUnlock;
293 baseSinkClass->unlock_stop = webkitVideoSinkUnlockStop;
294 baseSinkClass->render = webkitVideoSinkRender;
295 baseSinkClass->preroll = webkitVideoSinkRender;
296 baseSinkClass->stop = webkitVideoSinkStop;
297 baseSinkClass->start = webkitVideoSinkStart;
298 baseSinkClass->set_caps = webkitVideoSinkSetCaps;
299 baseSinkClass->propose_allocation = webkitVideoSinkProposeAllocation;
300 baseSinkClass->event = webkitVideoSinkEvent;
301
302 webkitVideoSinkSignals[REPAINT_REQUESTED] = g_signal_new("repaint-requested",
303 G_TYPE_FROM_CLASS(klass),
304 static_cast<GSignalFlags>(G_SIGNAL_RUN_LAST | G_SIGNAL_ACTION),
305 0, // Class offset
306 0, // Accumulator
307 0, // Accumulator data
308 g_cclosure_marshal_generic,
309 G_TYPE_NONE, // Return type
310 1, // Only one parameter
311 GST_TYPE_SAMPLE);
312 webkitVideoSinkSignals[REPAINT_CANCELLED] = g_signal_new("repaint-cancelled",
313 G_TYPE_FROM_CLASS(klass),
314 G_SIGNAL_RUN_LAST,
315 0, // Class offset
316 nullptr, // Accumulator
317 nullptr, // Accumulator data
318 g_cclosure_marshal_generic,
319 G_TYPE_NONE, // Return type
320 0, // No parameters
321 G_TYPE_NONE);
322}
323
324
325GstElement* webkitVideoSinkNew()
326{
327 return GST_ELEMENT(g_object_new(WEBKIT_TYPE_VIDEO_SINK, nullptr));
328}
329
330#endif // ENABLE(VIDEO) && USE(GSTREAMER)
331