1/*
2 * Copyright (C) 2011, 2012 Igalia S.L
3 * Copyright (C) 2014 Sebastian Dröge <sebastian@centricular.com>
4 *
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Lesser General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
9 *
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Lesser General Public License for more details.
14 *
15 * You should have received a copy of the GNU Lesser General Public
16 * License along with this library; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
18 */
19
20#include "config.h"
21
22#include "WebKitWebAudioSourceGStreamer.h"
23
24#if ENABLE(WEB_AUDIO) && USE(GSTREAMER)
25
26#include "AudioBus.h"
27#include "AudioIOCallback.h"
28#include "GStreamerCommon.h"
29#include <gst/app/gstappsrc.h>
30#include <gst/audio/audio-info.h>
31#include <gst/pbutils/missing-plugins.h>
32#include <wtf/glib/GUniquePtr.h>
33
34using namespace WebCore;
35
36typedef struct _WebKitWebAudioSrcClass WebKitWebAudioSrcClass;
37typedef struct _WebKitWebAudioSourcePrivate WebKitWebAudioSourcePrivate;
38
39struct _WebKitWebAudioSrc {
40 GstBin parent;
41
42 WebKitWebAudioSourcePrivate* priv;
43};
44
45struct _WebKitWebAudioSrcClass {
46 GstBinClass parentClass;
47};
48
49#define WEBKIT_WEB_AUDIO_SRC_GET_PRIVATE(obj) (G_TYPE_INSTANCE_GET_PRIVATE((obj), WEBKIT_TYPE_WEBAUDIO_SRC, WebKitWebAudioSourcePrivate))
50struct _WebKitWebAudioSourcePrivate {
51 gfloat sampleRate;
52 AudioBus* bus;
53 AudioIOCallback* provider;
54 guint framesToPull;
55 guint bufferSize;
56
57 GRefPtr<GstElement> interleave;
58
59 GRefPtr<GstTask> task;
60 GRecMutex mutex;
61
62 // List of appsrc. One appsrc for each planar audio channel.
63 Vector<GRefPtr<GstElement>> sources;
64
65 // src pad of the element, interleaved wav data is pushed to it.
66 GstPad* sourcePad;
67
68 guint64 numberOfSamples;
69
70 GRefPtr<GstBufferPool> pool;
71
72 bool enableGapBufferSupport;
73};
74
75enum {
76 PROP_RATE = 1,
77 PROP_BUS,
78 PROP_PROVIDER,
79 PROP_FRAMES
80};
81
82static GstStaticPadTemplate srcTemplate = GST_STATIC_PAD_TEMPLATE("src",
83 GST_PAD_SRC,
84 GST_PAD_ALWAYS,
85 GST_STATIC_CAPS(GST_AUDIO_CAPS_MAKE(GST_AUDIO_NE(F32))));
86
87GST_DEBUG_CATEGORY_STATIC(webkit_web_audio_src_debug);
88#define GST_CAT_DEFAULT webkit_web_audio_src_debug
89
90static void webKitWebAudioSrcConstructed(GObject*);
91static void webKitWebAudioSrcFinalize(GObject*);
92static void webKitWebAudioSrcSetProperty(GObject*, guint propertyId, const GValue*, GParamSpec*);
93static void webKitWebAudioSrcGetProperty(GObject*, guint propertyId, GValue*, GParamSpec*);
94static GstStateChangeReturn webKitWebAudioSrcChangeState(GstElement*, GstStateChange);
95static void webKitWebAudioSrcLoop(WebKitWebAudioSrc*);
96
97static GstCaps* getGStreamerMonoAudioCaps(float sampleRate)
98{
99 return gst_caps_new_simple("audio/x-raw", "rate", G_TYPE_INT, static_cast<int>(sampleRate),
100 "channels", G_TYPE_INT, 1,
101 "format", G_TYPE_STRING, GST_AUDIO_NE(F32),
102 "layout", G_TYPE_STRING, "interleaved", nullptr);
103}
104
105static GstAudioChannelPosition webKitWebAudioGStreamerChannelPosition(int channelIndex)
106{
107 GstAudioChannelPosition position = GST_AUDIO_CHANNEL_POSITION_NONE;
108
109 switch (channelIndex) {
110 case AudioBus::ChannelLeft:
111 position = GST_AUDIO_CHANNEL_POSITION_FRONT_LEFT;
112 break;
113 case AudioBus::ChannelRight:
114 position = GST_AUDIO_CHANNEL_POSITION_FRONT_RIGHT;
115 break;
116 case AudioBus::ChannelCenter:
117 position = GST_AUDIO_CHANNEL_POSITION_FRONT_CENTER;
118 break;
119 case AudioBus::ChannelLFE:
120 position = GST_AUDIO_CHANNEL_POSITION_LFE1;
121 break;
122 case AudioBus::ChannelSurroundLeft:
123 position = GST_AUDIO_CHANNEL_POSITION_REAR_LEFT;
124 break;
125 case AudioBus::ChannelSurroundRight:
126 position = GST_AUDIO_CHANNEL_POSITION_REAR_RIGHT;
127 break;
128 default:
129 break;
130 };
131
132 return position;
133}
134
135#define webkit_web_audio_src_parent_class parent_class
136G_DEFINE_TYPE_WITH_CODE(WebKitWebAudioSrc, webkit_web_audio_src, GST_TYPE_BIN, GST_DEBUG_CATEGORY_INIT(webkit_web_audio_src_debug, \
137 "webkitwebaudiosrc", \
138 0, \
139 "webaudiosrc element"));
140
141static void webkit_web_audio_src_class_init(WebKitWebAudioSrcClass* webKitWebAudioSrcClass)
142{
143 GObjectClass* objectClass = G_OBJECT_CLASS(webKitWebAudioSrcClass);
144 GstElementClass* elementClass = GST_ELEMENT_CLASS(webKitWebAudioSrcClass);
145
146 gst_element_class_add_pad_template(elementClass, gst_static_pad_template_get(&srcTemplate));
147 gst_element_class_set_metadata(elementClass, "WebKit WebAudio source element", "Source", "Handles WebAudio data from WebCore", "Philippe Normand <pnormand@igalia.com>");
148
149 objectClass->constructed = webKitWebAudioSrcConstructed;
150 objectClass->finalize = webKitWebAudioSrcFinalize;
151 elementClass->change_state = webKitWebAudioSrcChangeState;
152
153 objectClass->set_property = webKitWebAudioSrcSetProperty;
154 objectClass->get_property = webKitWebAudioSrcGetProperty;
155
156 GParamFlags flags = static_cast<GParamFlags>(G_PARAM_CONSTRUCT_ONLY | G_PARAM_READWRITE);
157 g_object_class_install_property(objectClass,
158 PROP_RATE,
159 g_param_spec_float("rate", "rate",
160 "Sample rate", G_MINDOUBLE, G_MAXDOUBLE,
161 44100.0, flags));
162
163 g_object_class_install_property(objectClass,
164 PROP_BUS,
165 g_param_spec_pointer("bus", "bus",
166 "Bus", flags));
167
168 g_object_class_install_property(objectClass,
169 PROP_PROVIDER,
170 g_param_spec_pointer("provider", "provider",
171 "Provider", flags));
172
173 g_object_class_install_property(objectClass,
174 PROP_FRAMES,
175 g_param_spec_uint("frames", "frames",
176 "Number of audio frames to pull at each iteration",
177 0, G_MAXUINT8, 128, flags));
178
179 g_type_class_add_private(webKitWebAudioSrcClass, sizeof(WebKitWebAudioSourcePrivate));
180}
181
182static void webkit_web_audio_src_init(WebKitWebAudioSrc* src)
183{
184 WebKitWebAudioSourcePrivate* priv = G_TYPE_INSTANCE_GET_PRIVATE(src, WEBKIT_TYPE_WEB_AUDIO_SRC, WebKitWebAudioSourcePrivate);
185 src->priv = priv;
186 new (priv) WebKitWebAudioSourcePrivate();
187
188 priv->sourcePad = webkitGstGhostPadFromStaticTemplate(&srcTemplate, "src", nullptr);
189 gst_element_add_pad(GST_ELEMENT(src), priv->sourcePad);
190
191 priv->provider = nullptr;
192 priv->bus = nullptr;
193
194 g_rec_mutex_init(&priv->mutex);
195 priv->task = adoptGRef(gst_task_new(reinterpret_cast<GstTaskFunction>(webKitWebAudioSrcLoop), src, nullptr));
196
197 // GAP buffer support is enabled only for GStreamer 1.12.5 because of a
198 // memory leak that was fixed in that version.
199 // https://bugzilla.gnome.org/show_bug.cgi?id=793067
200 priv->enableGapBufferSupport = webkitGstCheckVersion(1, 12, 5);
201
202 gst_task_set_lock(priv->task.get(), &priv->mutex);
203}
204
205static void webKitWebAudioSrcConstructed(GObject* object)
206{
207 WebKitWebAudioSrc* src = WEBKIT_WEB_AUDIO_SRC(object);
208 WebKitWebAudioSourcePrivate* priv = src->priv;
209
210 ASSERT(priv->bus);
211 ASSERT(priv->provider);
212 ASSERT(priv->sampleRate);
213
214 priv->interleave = gst_element_factory_make("interleave", nullptr);
215
216 if (!priv->interleave) {
217 GST_ERROR_OBJECT(src, "Failed to create interleave");
218 return;
219 }
220
221 gst_bin_add(GST_BIN(src), priv->interleave.get());
222
223 // For each channel of the bus create a new upstream branch for interleave, like:
224 // appsrc ! . which is plugged to a new interleave request sinkpad.
225 for (unsigned channelIndex = 0; channelIndex < priv->bus->numberOfChannels(); channelIndex++) {
226 GUniquePtr<gchar> appsrcName(g_strdup_printf("webaudioSrc%u", channelIndex));
227 GRefPtr<GstElement> appsrc = gst_element_factory_make("appsrc", appsrcName.get());
228 GRefPtr<GstCaps> monoCaps = adoptGRef(getGStreamerMonoAudioCaps(priv->sampleRate));
229
230 GstAudioInfo info;
231 gst_audio_info_from_caps(&info, monoCaps.get());
232 GST_AUDIO_INFO_POSITION(&info, 0) = webKitWebAudioGStreamerChannelPosition(channelIndex);
233 GRefPtr<GstCaps> caps = adoptGRef(gst_audio_info_to_caps(&info));
234
235 // Configure the appsrc for minimal latency.
236 g_object_set(appsrc.get(), "max-bytes", static_cast<guint64>(2 * priv->bufferSize), "block", TRUE,
237 "blocksize", priv->bufferSize,
238 "format", GST_FORMAT_TIME, "caps", caps.get(), nullptr);
239
240 priv->sources.append(appsrc);
241
242 gst_bin_add(GST_BIN(src), appsrc.get());
243 gst_element_link_pads_full(appsrc.get(), "src", priv->interleave.get(), "sink_%u", GST_PAD_LINK_CHECK_NOTHING);
244 }
245
246 // interleave's src pad is the only visible pad of our element.
247 GRefPtr<GstPad> targetPad = adoptGRef(gst_element_get_static_pad(priv->interleave.get(), "src"));
248 gst_ghost_pad_set_target(GST_GHOST_PAD(priv->sourcePad), targetPad.get());
249}
250
251static void webKitWebAudioSrcFinalize(GObject* object)
252{
253 WebKitWebAudioSrc* src = WEBKIT_WEB_AUDIO_SRC(object);
254 WebKitWebAudioSourcePrivate* priv = src->priv;
255
256 g_rec_mutex_clear(&priv->mutex);
257
258 priv->~WebKitWebAudioSourcePrivate();
259 GST_CALL_PARENT(G_OBJECT_CLASS, finalize, ((GObject* )(src)));
260}
261
262static void webKitWebAudioSrcSetProperty(GObject* object, guint propertyId, const GValue* value, GParamSpec* pspec)
263{
264 WebKitWebAudioSrc* src = WEBKIT_WEB_AUDIO_SRC(object);
265 WebKitWebAudioSourcePrivate* priv = src->priv;
266
267 switch (propertyId) {
268 case PROP_RATE:
269 priv->sampleRate = g_value_get_float(value);
270 break;
271 case PROP_BUS:
272 priv->bus = static_cast<AudioBus*>(g_value_get_pointer(value));
273 break;
274 case PROP_PROVIDER:
275 priv->provider = static_cast<AudioIOCallback*>(g_value_get_pointer(value));
276 break;
277 case PROP_FRAMES:
278 priv->framesToPull = g_value_get_uint(value);
279 priv->bufferSize = sizeof(float) * priv->framesToPull;
280 break;
281 default:
282 G_OBJECT_WARN_INVALID_PROPERTY_ID(object, propertyId, pspec);
283 break;
284 }
285}
286
287static void webKitWebAudioSrcGetProperty(GObject* object, guint propertyId, GValue* value, GParamSpec* pspec)
288{
289 WebKitWebAudioSrc* src = WEBKIT_WEB_AUDIO_SRC(object);
290 WebKitWebAudioSourcePrivate* priv = src->priv;
291
292 switch (propertyId) {
293 case PROP_RATE:
294 g_value_set_float(value, priv->sampleRate);
295 break;
296 case PROP_BUS:
297 g_value_set_pointer(value, priv->bus);
298 break;
299 case PROP_PROVIDER:
300 g_value_set_pointer(value, priv->provider);
301 break;
302 case PROP_FRAMES:
303 g_value_set_uint(value, priv->framesToPull);
304 break;
305 default:
306 G_OBJECT_WARN_INVALID_PROPERTY_ID(object, propertyId, pspec);
307 break;
308 }
309}
310
311static Optional<Vector<GRefPtr<GstBuffer>>> webKitWebAudioSrcAllocateBuffersAndRenderAudio(WebKitWebAudioSrc* src)
312{
313 WebKitWebAudioSourcePrivate* priv = src->priv;
314
315 ASSERT(priv->bus);
316 ASSERT(priv->provider);
317 if (!priv->provider || !priv->bus) {
318 GST_ELEMENT_ERROR(src, CORE, FAILED, ("Internal WebAudioSrc error"), ("Can't start without provider or bus"));
319 gst_task_stop(src->priv->task.get());
320 return WTF::nullopt;
321 }
322
323 ASSERT(priv->pool);
324 GstClockTime timestamp = gst_util_uint64_scale(priv->numberOfSamples, GST_SECOND, priv->sampleRate);
325 priv->numberOfSamples += priv->framesToPull;
326 GstClockTime duration = gst_util_uint64_scale(priv->numberOfSamples, GST_SECOND, priv->sampleRate) - timestamp;
327
328 Vector<GRefPtr<GstBuffer>> channelBufferList;
329 channelBufferList.reserveInitialCapacity(priv->sources.size());
330 Vector<RefPtr<GstMappedBuffer>> mappedBuffers;
331 mappedBuffers.reserveInitialCapacity(priv->sources.size());
332 for (unsigned i = 0; i < priv->sources.size(); ++i) {
333 GRefPtr<GstBuffer> buffer;
334 GstFlowReturn ret = gst_buffer_pool_acquire_buffer(priv->pool.get(), &buffer.outPtr(), nullptr);
335 if (ret != GST_FLOW_OK) {
336 // FLUSHING and EOS are not errors.
337 if (ret < GST_FLOW_EOS || ret == GST_FLOW_NOT_LINKED)
338 GST_ELEMENT_ERROR(src, CORE, PAD, ("Internal WebAudioSrc error"), ("Failed to allocate buffer for flow: %s", gst_flow_get_name(ret)));
339 return WTF::nullopt;
340 }
341
342 ASSERT(buffer);
343 GST_BUFFER_TIMESTAMP(buffer.get()) = timestamp;
344 GST_BUFFER_DURATION(buffer.get()) = duration;
345 auto mappedBuffer = GstMappedBuffer::create(buffer.get(), GST_MAP_READWRITE);
346 ASSERT(mappedBuffer);
347 mappedBuffers.uncheckedAppend(WTFMove(mappedBuffer));
348 priv->bus->setChannelMemory(i, reinterpret_cast<float*>(mappedBuffers[i]->data()), priv->framesToPull);
349 channelBufferList.uncheckedAppend(WTFMove(buffer));
350 }
351
352 // FIXME: Add support for local/live audio input.
353 priv->provider->render(nullptr, priv->bus, priv->framesToPull);
354
355 return makeOptional(channelBufferList);
356}
357
358static void webKitWebAudioSrcLoop(WebKitWebAudioSrc* src)
359{
360 WebKitWebAudioSourcePrivate* priv = src->priv;
361
362 Optional<Vector<GRefPtr<GstBuffer>>> channelBufferList = webKitWebAudioSrcAllocateBuffersAndRenderAudio(src);
363 if (!channelBufferList) {
364 gst_task_stop(src->priv->task.get());
365 return;
366 }
367
368 ASSERT(channelBufferList->size() == priv->sources.size());
369
370 bool failed = false;
371 for (unsigned i = 0; i < priv->sources.size(); ++i) {
372 auto& buffer = channelBufferList.value()[i];
373
374 if (priv->enableGapBufferSupport && priv->bus->channel(i)->isSilent())
375 GST_BUFFER_FLAG_SET(buffer.get(), GST_BUFFER_FLAG_GAP);
376
377 if (failed)
378 continue;
379
380 auto& appsrc = priv->sources[i];
381 // Leak the buffer ref, because gst_app_src_push_buffer steals it.
382 GstFlowReturn ret = gst_app_src_push_buffer(GST_APP_SRC(appsrc.get()), buffer.leakRef());
383 if (ret != GST_FLOW_OK) {
384 // FLUSHING and EOS are not errors.
385 if (ret < GST_FLOW_EOS || ret == GST_FLOW_NOT_LINKED)
386 GST_ELEMENT_ERROR(src, CORE, PAD, ("Internal WebAudioSrc error"), ("Failed to push buffer on %s flow: %s", GST_OBJECT_NAME(appsrc.get()), gst_flow_get_name(ret)));
387 gst_task_stop(src->priv->task.get());
388 failed = true;
389 }
390 }
391}
392
393static GstStateChangeReturn webKitWebAudioSrcChangeState(GstElement* element, GstStateChange transition)
394{
395 GstStateChangeReturn returnValue = GST_STATE_CHANGE_SUCCESS;
396 WebKitWebAudioSrc* src = WEBKIT_WEB_AUDIO_SRC(element);
397
398 switch (transition) {
399 case GST_STATE_CHANGE_NULL_TO_READY:
400 if (!src->priv->interleave) {
401 gst_element_post_message(element, gst_missing_element_message_new(element, "interleave"));
402 GST_ELEMENT_ERROR(src, CORE, MISSING_PLUGIN, (nullptr), ("no interleave"));
403 return GST_STATE_CHANGE_FAILURE;
404 }
405 src->priv->numberOfSamples = 0;
406 break;
407 default:
408 break;
409 }
410
411 returnValue = GST_ELEMENT_CLASS(parent_class)->change_state(element, transition);
412 if (UNLIKELY(returnValue == GST_STATE_CHANGE_FAILURE)) {
413 GST_DEBUG_OBJECT(src, "State change failed");
414 return returnValue;
415 }
416
417 switch (transition) {
418 case GST_STATE_CHANGE_READY_TO_PAUSED: {
419 GST_DEBUG_OBJECT(src, "READY->PAUSED");
420
421 src->priv->pool = gst_buffer_pool_new();
422 GstStructure* config = gst_buffer_pool_get_config(src->priv->pool.get());
423 gst_buffer_pool_config_set_params(config, nullptr, src->priv->bufferSize, 0, 0);
424 gst_buffer_pool_set_config(src->priv->pool.get(), config);
425 if (!gst_buffer_pool_set_active(src->priv->pool.get(), TRUE))
426 returnValue = GST_STATE_CHANGE_FAILURE;
427 else if (!gst_task_start(src->priv->task.get()))
428 returnValue = GST_STATE_CHANGE_FAILURE;
429 break;
430 }
431 case GST_STATE_CHANGE_PAUSED_TO_READY:
432 GST_DEBUG_OBJECT(src, "PAUSED->READY");
433
434 gst_buffer_pool_set_flushing(src->priv->pool.get(), TRUE);
435 if (!gst_task_join(src->priv->task.get()))
436 returnValue = GST_STATE_CHANGE_FAILURE;
437 gst_buffer_pool_set_active(src->priv->pool.get(), FALSE);
438 src->priv->pool = nullptr;
439 break;
440 default:
441 break;
442 }
443
444 return returnValue;
445}
446
447#endif // ENABLE(WEB_AUDIO) && USE(GSTREAMER)
448