1/*
2 * Copyright (C) 2019 Igalia S.L
3 *
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Library General Public
6 * License as published by the Free Software Foundation; either
7 * version 2 of the License, or (at your option) any later version.
8 *
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Library General Public License for more details.
13 *
14 * You should have received a copy of the GNU Library General Public License
15 * aint with this library; see the file COPYING.LIB. If not, write to
16 * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
17 * Boston, MA 02110-1301, USA.
18 */
19
20#include "config.h"
21#include "GStreamerRegistryScanner.h"
22
23#if USE(GSTREAMER)
24#include "ContentType.h"
25#include "GStreamerCommon.h"
26#include <fnmatch.h>
27#include <wtf/PrintStream.h>
28
29namespace WebCore {
30
31GST_DEBUG_CATEGORY_STATIC(webkit_media_gst_registry_scanner_debug);
32#define GST_CAT_DEFAULT webkit_media_gst_registry_scanner_debug
33
34GStreamerRegistryScanner& GStreamerRegistryScanner::singleton()
35{
36 static NeverDestroyed<GStreamerRegistryScanner> sharedInstance;
37 return sharedInstance;
38}
39
40GStreamerRegistryScanner::GStreamerRegistryScanner(bool isMediaSource)
41 : m_isMediaSource(isMediaSource)
42{
43 GST_DEBUG_CATEGORY_INIT(webkit_media_gst_registry_scanner_debug, "webkitregistryscanner", 0, "WebKit GStreamer registry scanner");
44 m_audioDecoderFactories = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_DECODER | GST_ELEMENT_FACTORY_TYPE_MEDIA_AUDIO, GST_RANK_MARGINAL);
45 m_audioParserFactories = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_PARSER | GST_ELEMENT_FACTORY_TYPE_MEDIA_AUDIO, GST_RANK_NONE);
46 m_videoDecoderFactories = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_DECODER | GST_ELEMENT_FACTORY_TYPE_MEDIA_VIDEO, GST_RANK_MARGINAL);
47 m_videoParserFactories = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_PARSER | GST_ELEMENT_FACTORY_TYPE_MEDIA_VIDEO, GST_RANK_MARGINAL);
48 m_demuxerFactories = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_DEMUXER, GST_RANK_MARGINAL);
49
50 initialize();
51#ifndef GST_DISABLE_GST_DEBUG
52 GST_DEBUG("%s registry scanner initialized", m_isMediaSource ? "MSE" : "Regular playback");
53 for (auto& mimeType : m_mimeTypeSet)
54 GST_DEBUG("Mime-type registered: %s", mimeType.utf8().data());
55 for (auto& item : m_codecMap)
56 GST_DEBUG("%s codec pattern registered: %s", item.value ? "Hardware" : "Software", item.key.string().utf8().data());
57#endif
58}
59
60GStreamerRegistryScanner::~GStreamerRegistryScanner()
61{
62 gst_plugin_feature_list_free(m_audioDecoderFactories);
63 gst_plugin_feature_list_free(m_audioParserFactories);
64 gst_plugin_feature_list_free(m_videoDecoderFactories);
65 gst_plugin_feature_list_free(m_videoParserFactories);
66 gst_plugin_feature_list_free(m_demuxerFactories);
67}
68
69GStreamerRegistryScanner::RegistryLookupResult GStreamerRegistryScanner::hasElementForMediaType(GList* elementFactories, const char* capsString, bool shouldCheckHardwareClassifier)
70{
71 GRefPtr<GstCaps> caps = adoptGRef(gst_caps_from_string(capsString));
72 GList* candidates = gst_element_factory_list_filter(elementFactories, caps.get(), GST_PAD_SINK, false);
73 bool isSupported = candidates;
74 bool isUsingHardware = false;
75
76 if (shouldCheckHardwareClassifier) {
77 for (GList* factories = candidates; factories; factories = g_list_next(factories)) {
78 auto* factory = reinterpret_cast<GstElementFactory*>(factories->data);
79 String metadata = gst_element_factory_get_metadata(factory, GST_ELEMENT_METADATA_KLASS);
80 auto components = metadata.split('/');
81 if (components.contains("Hardware")) {
82 isUsingHardware = true;
83 break;
84 }
85 }
86 }
87
88 gst_plugin_feature_list_free(candidates);
89#ifndef GST_DISABLE_GST_DEBUG
90 const char* elementType = "";
91 if (elementFactories == m_audioParserFactories)
92 elementType = "Audio parser";
93 else if (elementFactories == m_audioDecoderFactories)
94 elementType = "Audio decoder";
95 else if (elementFactories == m_videoParserFactories)
96 elementType = "Video parser";
97 else if (elementFactories == m_videoDecoderFactories)
98 elementType = "Video decoder";
99 else if (elementFactories == m_demuxerFactories)
100 elementType = "Demuxer";
101 else
102 ASSERT_NOT_REACHED();
103 GST_LOG("%s lookup result for caps %" GST_PTR_FORMAT " : isSupported=%s, isUsingHardware=%s", elementType, caps.get(), boolForPrinting(isSupported), boolForPrinting(isUsingHardware));
104#endif
105 return GStreamerRegistryScanner::RegistryLookupResult { isSupported, isUsingHardware };
106}
107
108void GStreamerRegistryScanner::fillMimeTypeSetFromCapsMapping(Vector<GstCapsWebKitMapping>& mapping)
109{
110 for (auto& current : mapping) {
111 GList* factories;
112 switch (current.elementType) {
113 case Demuxer:
114 factories = m_demuxerFactories;
115 break;
116 case AudioDecoder:
117 factories = m_audioDecoderFactories;
118 break;
119 case VideoDecoder:
120 factories = m_videoDecoderFactories;
121 break;
122 }
123
124 if (hasElementForMediaType(factories, current.capsString)) {
125 if (!current.webkitCodecPatterns.isEmpty()) {
126 for (const auto& pattern : current.webkitCodecPatterns)
127 m_codecMap.add(pattern, false);
128 }
129 if (!current.webkitMimeTypes.isEmpty()) {
130 for (const auto& mimeType : current.webkitMimeTypes)
131 m_mimeTypeSet.add(mimeType);
132 } else
133 m_mimeTypeSet.add(AtomicString(current.capsString));
134 }
135 }
136}
137
138void GStreamerRegistryScanner::initialize()
139{
140 if (hasElementForMediaType(m_audioDecoderFactories, "audio/mpeg, mpegversion=(int)4")) {
141 m_mimeTypeSet.add(AtomicString("audio/aac"));
142 m_mimeTypeSet.add(AtomicString("audio/mp4"));
143 m_mimeTypeSet.add(AtomicString("audio/x-m4a"));
144 m_codecMap.add(AtomicString("mpeg"), false);
145 m_codecMap.add(AtomicString("mp4a*"), false);
146 }
147
148 auto opusSupported = hasElementForMediaType(m_audioDecoderFactories, "audio/x-opus");
149 if (opusSupported && (!m_isMediaSource || hasElementForMediaType(m_audioParserFactories, "audio/x-opus"))) {
150 m_mimeTypeSet.add(AtomicString("audio/opus"));
151 m_codecMap.add(AtomicString("opus"), false);
152 m_codecMap.add(AtomicString("x-opus"), false);
153 }
154
155 auto vorbisSupported = hasElementForMediaType(m_audioDecoderFactories, "audio/x-vorbis");
156 if (vorbisSupported && (!m_isMediaSource || hasElementForMediaType(m_audioParserFactories, "audio/x-vorbis"))) {
157 m_codecMap.add(AtomicString("vorbis"), false);
158 m_codecMap.add(AtomicString("x-vorbis"), false);
159 }
160
161 if (hasElementForMediaType(m_demuxerFactories, "video/x-matroska")) {
162 auto vp8DecoderAvailable = hasElementForMediaType(m_videoDecoderFactories, "video/x-vp8", true);
163 auto vp9DecoderAvailable = hasElementForMediaType(m_videoDecoderFactories, "video/x-vp9", true);
164
165 if (vp8DecoderAvailable || vp9DecoderAvailable)
166 m_mimeTypeSet.add(AtomicString("video/webm"));
167
168 if (vp8DecoderAvailable) {
169 m_codecMap.add(AtomicString("vp8"), vp8DecoderAvailable.isUsingHardware);
170 m_codecMap.add(AtomicString("x-vp8"), vp8DecoderAvailable.isUsingHardware);
171 m_codecMap.add(AtomicString("vp8.0"), vp8DecoderAvailable.isUsingHardware);
172 }
173 if (vp9DecoderAvailable) {
174 m_codecMap.add(AtomicString("vp9"), vp9DecoderAvailable.isUsingHardware);
175 m_codecMap.add(AtomicString("x-vp9"), vp9DecoderAvailable.isUsingHardware);
176 m_codecMap.add(AtomicString("vp9.0"), vp9DecoderAvailable.isUsingHardware);
177 }
178 if (opusSupported)
179 m_mimeTypeSet.add(AtomicString("audio/webm"));
180 }
181
182 auto h264DecoderAvailable = hasElementForMediaType(m_videoDecoderFactories, "video/x-h264, profile=(string){ constrained-baseline, baseline, high }", true);
183 if (h264DecoderAvailable && (!m_isMediaSource || hasElementForMediaType(m_videoParserFactories, "video/x-h264"))) {
184 m_mimeTypeSet.add(AtomicString("video/mp4"));
185 m_mimeTypeSet.add(AtomicString("video/x-m4v"));
186 m_codecMap.add(AtomicString("x-h264"), h264DecoderAvailable.isUsingHardware);
187 m_codecMap.add(AtomicString("avc*"), h264DecoderAvailable.isUsingHardware);
188 m_codecMap.add(AtomicString("mp4v*"), h264DecoderAvailable.isUsingHardware);
189 }
190
191 if (m_isMediaSource)
192 return;
193
194 // The mime-types initialized below are not supported by the MSE backend.
195
196 Vector<GstCapsWebKitMapping> mapping = {
197 {AudioDecoder, "audio/midi", {"audio/midi", "audio/riff-midi"}, { }},
198 {AudioDecoder, "audio/x-ac3", { }, { }},
199 {AudioDecoder, "audio/x-dts", { }, { }},
200 {AudioDecoder, "audio/x-eac3", {"audio/x-ac3"}, { }},
201 {AudioDecoder, "audio/x-flac", {"audio/x-flac", "audio/flac"}, { }},
202 {AudioDecoder, "audio/x-sbc", { }, { }},
203 {AudioDecoder, "audio/x-sid", { }, { }},
204 {AudioDecoder, "audio/x-speex", {"audio/speex", "audio/x-speex"}, { }},
205 {AudioDecoder, "audio/x-wavpack", {"audio/x-wavpack"}, { }},
206 {VideoDecoder, "video/mpeg, mpegversion=(int){1,2}, systemstream=(boolean)false", {"video/mpeg"}, {"mpeg"}},
207 {VideoDecoder, "video/mpegts", { }, { }},
208 {VideoDecoder, "video/x-dirac", { }, { }},
209 {VideoDecoder, "video/x-flash-video", {"video/flv", "video/x-flv"}, { }},
210 {VideoDecoder, "video/x-h263", { }, { }},
211 {VideoDecoder, "video/x-msvideocodec", {"video/x-msvideo"}, { }},
212 {Demuxer, "application/vnd.rn-realmedia", { }, { }},
213 {Demuxer, "application/x-3gp", { }, { }},
214 {Demuxer, "application/x-hls", {"application/vnd.apple.mpegurl", "application/x-mpegurl"}, { }},
215 {Demuxer, "application/x-pn-realaudio", { }, { }},
216 {Demuxer, "audio/x-aiff", { }, { }},
217 {Demuxer, "audio/x-wav", {"audio/x-wav", "audio/wav", "audio/vnd.wave"}, {"1"}},
218 {Demuxer, "video/quicktime", { }, { }},
219 {Demuxer, "video/quicktime, variant=(string)3gpp", {"video/3gpp"}, { }},
220 {Demuxer, "video/x-ms-asf", { }, { }},
221 };
222 fillMimeTypeSetFromCapsMapping(mapping);
223
224 if (hasElementForMediaType(m_demuxerFactories, "application/ogg")) {
225 m_mimeTypeSet.add(AtomicString("application/ogg"));
226
227 if (vorbisSupported) {
228 m_mimeTypeSet.add(AtomicString("audio/ogg"));
229 m_mimeTypeSet.add(AtomicString("audio/x-vorbis+ogg"));
230 }
231
232 if (hasElementForMediaType(m_audioDecoderFactories, "audio/x-speex")) {
233 m_mimeTypeSet.add(AtomicString("audio/ogg"));
234 m_codecMap.add(AtomicString("speex"), false);
235 }
236
237 if (hasElementForMediaType(m_videoDecoderFactories, "video/x-theora")) {
238 m_mimeTypeSet.add(AtomicString("video/ogg"));
239 m_codecMap.add(AtomicString("theora"), false);
240 }
241 }
242
243 bool audioMpegSupported = false;
244 if (hasElementForMediaType(m_audioDecoderFactories, "audio/mpeg, mpegversion=(int)1, layer=(int)[1, 3]")) {
245 audioMpegSupported = true;
246 m_mimeTypeSet.add(AtomicString("audio/mp1"));
247 m_mimeTypeSet.add(AtomicString("audio/mp3"));
248 m_mimeTypeSet.add(AtomicString("audio/x-mp3"));
249 m_codecMap.add(AtomicString("audio/mp3"), false);
250 }
251
252 if (hasElementForMediaType(m_audioDecoderFactories, "audio/mpeg, mpegversion=(int)2")) {
253 audioMpegSupported = true;
254 m_mimeTypeSet.add(AtomicString("audio/mp2"));
255 }
256
257 audioMpegSupported |= isContainerTypeSupported("audio/mp4");
258 if (audioMpegSupported) {
259 m_mimeTypeSet.add(AtomicString("audio/mpeg"));
260 m_mimeTypeSet.add(AtomicString("audio/x-mpeg"));
261 }
262
263 bool matroskaSupported = hasElementForMediaType(m_demuxerFactories, "video/x-matroska");
264 if (matroskaSupported) {
265 m_mimeTypeSet.add(AtomicString("video/x-matroska"));
266
267 if (hasElementForMediaType(m_videoDecoderFactories, "video/x-vp10"))
268 m_mimeTypeSet.add(AtomicString("video/webm"));
269 }
270
271 if ((matroskaSupported || isContainerTypeSupported("video/mp4")) && hasElementForMediaType(m_videoDecoderFactories, "video/x-av1"))
272 m_codecMap.add(AtomicString("av01*"), false);
273}
274
275bool GStreamerRegistryScanner::isCodecSupported(String codec, bool shouldCheckForHardwareUse) const
276{
277 // If the codec is named like a mimetype (eg: video/avc) remove the "video/" part.
278 size_t slashIndex = codec.find('/');
279 if (slashIndex != WTF::notFound)
280 codec = codec.substring(slashIndex + 1);
281
282 bool supported = false;
283 for (const auto& item : m_codecMap) {
284 if (!fnmatch(item.key.string().utf8().data(), codec.utf8().data(), 0)) {
285 supported = shouldCheckForHardwareUse ? item.value : true;
286 if (supported)
287 break;
288 }
289 }
290
291 GST_LOG("Checked %s codec \"%s\" supported %s", shouldCheckForHardwareUse ? "hardware" : "software", codec.utf8().data(), boolForPrinting(supported));
292 return supported;
293}
294
295bool GStreamerRegistryScanner::areAllCodecsSupported(const Vector<String>& codecs, bool shouldCheckForHardwareUse) const
296{
297 for (String codec : codecs) {
298 if (!isCodecSupported(codec, shouldCheckForHardwareUse))
299 return false;
300 }
301
302 return true;
303}
304
305GStreamerRegistryScanner::RegistryLookupResult GStreamerRegistryScanner::isDecodingSupported(MediaConfiguration& configuration) const
306{
307 bool isSupported = false;
308 bool isUsingHardware = false;
309
310 if (configuration.video) {
311 auto& videoConfiguration = configuration.video.value();
312 GST_DEBUG("Checking support for video configuration: \"%s\" size: %ux%u bitrate: %" G_GUINT64_FORMAT " framerate: %f",
313 videoConfiguration.contentType.utf8().data(),
314 videoConfiguration.width, videoConfiguration.height,
315 videoConfiguration.bitrate, videoConfiguration.framerate);
316
317 auto contentType = ContentType(videoConfiguration.contentType);
318 isSupported = isContainerTypeSupported(contentType.containerType());
319 auto codecs = contentType.codecs();
320 if (!codecs.isEmpty())
321 isUsingHardware = areAllCodecsSupported(codecs, true);
322 }
323
324 if (configuration.audio) {
325 auto& audioConfiguration = configuration.audio.value();
326 GST_DEBUG("Checking support for audio configuration: \"%s\" %s channels, bitrate: %" G_GUINT64_FORMAT " samplerate: %u",
327 audioConfiguration.contentType.utf8().data(), audioConfiguration.channels.utf8().data(),
328 audioConfiguration.bitrate, audioConfiguration.samplerate);
329 auto contentType = ContentType(audioConfiguration.contentType);
330 isSupported = isContainerTypeSupported(contentType.containerType());
331 }
332
333 return GStreamerRegistryScanner::RegistryLookupResult { isSupported, isUsingHardware };
334}
335
336}
337#endif
338