1/*
2 * Copyright (C) 2018-2019 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27#include "RealtimeVideoSource.h"
28
29#if ENABLE(MEDIA_STREAM)
30#include "CaptureDevice.h"
31#include "Logging.h"
32#include "RealtimeMediaSourceCenter.h"
33#include "RealtimeMediaSourceSettings.h"
34#include "RemoteVideoSample.h"
35#include <wtf/JSONValues.h>
36
37#if PLATFORM(COCOA)
38#include "ImageTransferSessionVT.h"
39#endif
40
41namespace WebCore {
42
43RealtimeVideoSource::RealtimeVideoSource(String&& name, String&& id, String&& hashSalt)
44 : RealtimeMediaSource(Type::Video, WTFMove(name), WTFMove(id), WTFMove(hashSalt))
45{
46}
47
48RealtimeVideoSource::~RealtimeVideoSource()
49{
50#if PLATFORM(IOS_FAMILY)
51 RealtimeMediaSourceCenter::singleton().videoCaptureFactory().unsetActiveSource(*this);
52#endif
53}
54
55void RealtimeVideoSource::prepareToProduceData()
56{
57 ASSERT(frameRate());
58
59#if PLATFORM(IOS_FAMILY)
60 RealtimeMediaSourceCenter::singleton().videoCaptureFactory().setActiveSource(*this);
61#endif
62
63 if (size().isEmpty() && !m_defaultSize.isEmpty())
64 setSize(m_defaultSize);
65}
66
67const Vector<Ref<VideoPreset>>& RealtimeVideoSource::presets()
68{
69 if (m_presets.isEmpty())
70 generatePresets();
71
72 ASSERT(!m_presets.isEmpty());
73 return m_presets;
74}
75
76void RealtimeVideoSource::setSupportedPresets(Vector<VideoPresetData>&& presetData)
77{
78 Vector<Ref<VideoPreset>> presets;
79
80 for (auto& data : presetData)
81 presets.append(VideoPreset::create(WTFMove(data)));
82
83 setSupportedPresets(WTFMove(presets));
84}
85
86void RealtimeVideoSource::setSupportedPresets(const Vector<Ref<VideoPreset>>& presets)
87{
88 m_presets = WTF::map(presets, [](auto& preset) {
89 return preset.copyRef();
90 });
91
92 for (auto& preset : m_presets) {
93 std::sort(preset->frameRateRanges.begin(), preset->frameRateRanges.end(),
94 [&] (const auto& a, const auto& b) -> bool {
95 return a.minimum < b.minimum;
96 });
97 }
98}
99
100const Vector<IntSize>& RealtimeVideoSource::standardVideoSizes()
101{
102 static const auto sizes = makeNeverDestroyed([] {
103 static IntSize videoSizes[] = {
104 { 112, 112 },
105 { 160, 160 },
106 { 160, 120 }, // 4:3, QQVGA
107 { 176, 144 }, // 4:3, QCIF
108 { 192, 192 },
109 { 192, 112 }, // 16:9
110 { 192, 144 }, // 3:4
111 { 240, 240 },
112 { 240, 160 }, // 3:2, HQVGA
113 { 320, 320 },
114 { 320, 180 }, // 16:9
115 { 320, 240 }, // 4:3, QVGA
116 { 352, 288 }, // CIF
117 { 480, 272 }, // 16:9
118 { 480, 360 }, // 4:3
119 { 480, 480 },
120 { 640, 640 },
121 { 640, 360 }, // 16:9, 360p nHD
122 { 640, 480 }, // 4:3
123 { 720, 720 },
124 { 800, 600 }, // 4:3, SVGA
125 { 960, 540 }, // 16:9, qHD
126 { 1024, 600 }, // 16:9, WSVGA
127 { 1024, 768 }, // 4:3, XGA
128 { 1280, 960 }, // 4:3
129 { 1280, 1024 }, // 5:4, SXGA
130 { 1280, 720 }, // 16:9, WXGA
131 { 1366, 768 }, // 16:9, HD
132 { 1600, 1200}, // 4:3, UXGA
133 { 1920, 1080 }, // 16:9, 1080p FHD
134 { 2560, 1440 }, // 16:9, QHD
135 { 2592, 1936 },
136 { 3264, 2448 }, // 3:4
137 { 3840, 2160 }, // 16:9, 4K UHD
138 };
139 Vector<IntSize> sizes;
140 for (auto& size : videoSizes)
141 sizes.append(size);
142
143 return sizes;
144 }());
145
146 return sizes.get();
147}
148template <typename ValueType>
149static void updateMinMax(ValueType& min, ValueType& max, ValueType value)
150{
151 min = std::min<ValueType>(min, value);
152 max = std::max<ValueType>(max, value);
153}
154
155void RealtimeVideoSource::updateCapabilities(RealtimeMediaSourceCapabilities& capabilities)
156{
157 ASSERT(!presets().isEmpty());
158
159 int minimumWidth = std::numeric_limits<int>::max();
160 int maximumWidth = 0;
161 int minimumHeight = std::numeric_limits<int>::max();
162 int maximumHeight = 0;
163 double minimumAspectRatio = std::numeric_limits<double>::max();
164 double maximumAspectRatio = 0;
165 double minimumFrameRate = std::numeric_limits<double>::max();
166 double maximumFrameRate = 0;
167 for (const auto& preset : presets()) {
168 const auto& size = preset->size;
169 updateMinMax(minimumWidth, maximumWidth, size.width());
170 updateMinMax(minimumHeight, maximumHeight, size.height());
171 updateMinMax(minimumAspectRatio, maximumAspectRatio, static_cast<double>(size.width()) / size.height());
172
173 for (const auto& rate : preset->frameRateRanges) {
174 updateMinMax(minimumFrameRate, maximumFrameRate, rate.minimum);
175 updateMinMax(minimumFrameRate, maximumFrameRate, rate.maximum);
176 }
177 }
178
179 if (canResizeVideoFrames()) {
180 for (auto& size : standardVideoSizes()) {
181 if (size.width() < minimumWidth || size.height() < minimumHeight) {
182 minimumWidth = std::min(minimumWidth, size.width());
183 minimumHeight = std::min(minimumHeight, size.height());
184 minimumAspectRatio = std::min(minimumAspectRatio, static_cast<double>(size.width()) / size.height());
185 }
186 }
187 }
188
189 capabilities.setWidth({ minimumWidth, maximumWidth });
190 capabilities.setHeight({ minimumHeight, maximumHeight });
191 capabilities.setAspectRatio({ minimumAspectRatio, maximumAspectRatio });
192 capabilities.setFrameRate({ minimumFrameRate, maximumFrameRate });
193}
194
195bool RealtimeVideoSource::supportsSizeAndFrameRate(Optional<int> width, Optional<int> height, Optional<double> frameRate)
196{
197 if (!width && !height && !frameRate)
198 return true;
199
200 return !!bestSupportedSizeAndFrameRate(width, height, frameRate);
201}
202
203bool RealtimeVideoSource::frameRateRangeIncludesRate(const FrameRateRange& range, double frameRate)
204{
205 const double epsilon = 0.001;
206 return frameRate + epsilon >= range.minimum && frameRate - epsilon <= range.maximum;
207}
208
209bool RealtimeVideoSource::presetSupportsFrameRate(RefPtr<VideoPreset> preset, double frameRate)
210{
211 for (const auto& range : preset->frameRateRanges) {
212 if (frameRateRangeIncludesRate(range, frameRate))
213 return true;
214 }
215
216 return false;
217}
218
219bool RealtimeVideoSource::supportsCaptureSize(Optional<int> width, Optional<int> height, const Function<bool(const IntSize&)>&& function)
220{
221 if (width && height)
222 return function({ width.value(), height.value() });
223
224 if (width) {
225 for (auto& size : standardVideoSizes()) {
226 if (width.value() == size.width() && function({ size.width(), size.height() }))
227 return true;
228 }
229
230 return false;
231 }
232
233 for (auto& size : standardVideoSizes()) {
234 if (height.value() == size.height() && function({ size.width(), size.height() }))
235 return true;
236 }
237
238 return false;
239}
240
241bool RealtimeVideoSource::shouldUsePreset(VideoPreset& current, VideoPreset& candidate)
242{
243 return candidate.size.width() <= current.size.width() && candidate.size.height() <= current.size.height() && prefersPreset(candidate);
244}
245
246Optional<RealtimeVideoSource::CaptureSizeAndFrameRate> RealtimeVideoSource::bestSupportedSizeAndFrameRate(Optional<int> requestedWidth, Optional<int> requestedHeight, Optional<double> requestedFrameRate)
247{
248 if (!requestedWidth && !requestedHeight && !requestedFrameRate)
249 return { };
250
251 if (!requestedWidth && !requestedHeight && !size().isEmpty()) {
252 requestedWidth = size().width();
253 requestedHeight = size().height();
254 }
255 if (!requestedFrameRate)
256 requestedFrameRate = frameRate();
257
258 CaptureSizeAndFrameRate result;
259 RefPtr<VideoPreset> exactSizePreset;
260 RefPtr<VideoPreset> aspectRatioPreset;
261 IntSize aspectRatioMatchSize;
262 RefPtr<VideoPreset> resizePreset;
263 IntSize resizeSize;
264
265 for (const auto& preset : presets()) {
266 const auto& presetSize = preset->size;
267
268 if (!presetSupportsFrameRate(&preset.get(), requestedFrameRate.value()))
269 continue;
270
271 if (!requestedWidth && !requestedHeight) {
272 result.requestedFrameRate = requestedFrameRate.value();
273 return result;
274 }
275
276 // Don't look at presets smaller than the requested resolution because we never want to resize larger.
277 if ((requestedWidth && presetSize.width() < requestedWidth.value()) || (requestedHeight && presetSize.height() < requestedHeight.value()))
278 continue;
279
280 auto lookForExactSizeMatch = [&] (const IntSize& size) -> bool {
281 return preset->size == size;
282 };
283 if (supportsCaptureSize(requestedWidth, requestedHeight, WTFMove(lookForExactSizeMatch))) {
284 if (!exactSizePreset || prefersPreset(preset))
285 exactSizePreset = &preset.get();
286 continue;
287 }
288
289 IntSize encodingSize;
290 auto lookForAspectRatioMatch = [this, &preset, &encodingSize] (const IntSize& size) -> bool {
291 auto aspectRatio = [] (const IntSize size) -> double {
292 return size.width() / static_cast<double>(size.height());
293 };
294 if (std::abs(aspectRatio(preset->size) - aspectRatio(size)) > 10e-7 || !canResizeVideoFrames())
295 return false;
296
297 encodingSize = size;
298 return true;
299 };
300 if (supportsCaptureSize(requestedWidth, requestedHeight, WTFMove(lookForAspectRatioMatch))) {
301 if (!aspectRatioPreset || shouldUsePreset(*aspectRatioPreset, preset)) {
302 aspectRatioPreset = &preset.get();
303 aspectRatioMatchSize = encodingSize;
304 }
305 }
306
307 if (exactSizePreset || aspectRatioPreset)
308 continue;
309
310 if (requestedWidth && requestedHeight) {
311 const auto& minStandardSize = standardVideoSizes()[0];
312 if (requestedWidth.value() >= minStandardSize.width() && requestedHeight.value() >= minStandardSize.height()) {
313 if (!resizePreset || shouldUsePreset(*resizePreset, preset)) {
314 resizePreset = &preset.get();
315 resizeSize = { requestedWidth.value(), requestedHeight.value() };
316 }
317 }
318 } else {
319 for (auto& standardSize : standardVideoSizes()) {
320 if (standardSize.width() > preset->size.width() || standardSize.height() > preset->size.height())
321 break;
322 if ((requestedWidth && requestedWidth.value() != standardSize.width()) || (requestedHeight && requestedHeight.value() != standardSize.height()))
323 continue;
324
325 if (!resizePreset || shouldUsePreset(*resizePreset, preset)) {
326 resizePreset = &preset.get();
327 resizeSize = standardSize;
328 }
329 }
330 }
331 }
332
333 if (!exactSizePreset && !aspectRatioPreset && !resizePreset)
334 return { };
335
336 result.requestedFrameRate = requestedFrameRate.value();
337 if (exactSizePreset) {
338 result.encodingPreset = exactSizePreset;
339 result.requestedSize = exactSizePreset->size;
340 return result;
341 }
342
343 if (aspectRatioPreset) {
344 result.encodingPreset = aspectRatioPreset;
345 result.requestedSize = aspectRatioMatchSize;
346 return result;
347 }
348
349 result.encodingPreset = resizePreset;
350 result.requestedSize = resizeSize;
351 return result;
352}
353
354void RealtimeVideoSource::setSizeAndFrameRate(Optional<int> width, Optional<int> height, Optional<double> frameRate)
355{
356 ALWAYS_LOG_IF(loggerPtr(), LOGIDENTIFIER, SizeAndFrameRate { width, height, frameRate });
357
358 auto size = this->size();
359 if (!width && !height && !size.isEmpty()) {
360 width = size.width();
361 height = size.height();
362 }
363
364 Optional<RealtimeVideoSource::CaptureSizeAndFrameRate> match = bestSupportedSizeAndFrameRate(width, height, frameRate);
365 ASSERT(match);
366 if (!match)
367 return;
368
369 setFrameRateWithPreset(match->requestedFrameRate, match->encodingPreset);
370
371 if (!match->requestedSize.isEmpty())
372 setSize(match->requestedSize);
373 setFrameRate(match->requestedFrameRate);
374}
375
376void RealtimeVideoSource::dispatchMediaSampleToObservers(MediaSample& sample)
377{
378 MediaTime sampleTime = sample.outputPresentationTime();
379 if (!sampleTime || !sampleTime.isValid())
380 sampleTime = sample.presentationTime();
381
382 auto frameTime = sampleTime.toDouble();
383 m_observedFrameTimeStamps.append(frameTime);
384 m_observedFrameTimeStamps.removeAllMatching([&](auto time) {
385 return time <= frameTime - 2;
386 });
387
388 auto interval = m_observedFrameTimeStamps.last() - m_observedFrameTimeStamps.first();
389 if (interval > 1)
390 m_observedFrameRate = (m_observedFrameTimeStamps.size() / interval);
391
392 auto mediaSample = makeRefPtr(&sample);
393#if PLATFORM(COCOA)
394 if (!isRemote()) {
395 auto size = this->size();
396 if (!size.isEmpty() && size != expandedIntSize(sample.presentationSize())) {
397
398 if (!m_imageTransferSession || m_imageTransferSession->pixelFormat() != sample.videoPixelFormat())
399 m_imageTransferSession = ImageTransferSessionVT::create(sample.videoPixelFormat());
400
401 if (m_imageTransferSession) {
402 mediaSample = m_imageTransferSession->convertMediaSample(sample, size);
403 if (!mediaSample) {
404 ASSERT_NOT_REACHED();
405 return;
406 }
407 }
408 }
409 }
410#endif
411
412 videoSampleAvailable(mediaSample.releaseNonNull());
413}
414
415#if !RELEASE_LOG_DISABLED
416Ref<JSON::Object> SizeAndFrameRate::toJSONObject() const
417{
418 auto object = JSON::Object::create();
419
420 object->setDouble("width"_s, width ? width.value() : 0);
421 object->setDouble("height"_s, height ? height.value() : 0);
422 object->setDouble("frameRate"_s, frameRate ? frameRate.value() : 0);
423
424 return object;
425}
426
427String SizeAndFrameRate::toJSONString() const
428{
429 return toJSONObject()->toJSONString();
430}
431#endif
432
433} // namespace WebCore
434
435#endif // ENABLE(MEDIA_STREAM)
436