1/*
2 * Copyright (C) 2012 Google Inc. All rights reserved.
3 * Copyright (C) 2013-2019 Apple Inc. All rights reserved.
4 * Copyright (C) 2013 Nokia Corporation and/or its subsidiary(-ies).
5 * Copyright (C) 2015 Ericsson AB. All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer
15 * in the documentation and/or other materials provided with the
16 * distribution.
17 * 3. Neither the name of Google Inc. nor the names of its contributors
18 * may be used to endorse or promote products derived from this
19 * software without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34#include "config.h"
35
36#if ENABLE(MEDIA_STREAM)
37#include "RealtimeMediaSource.h"
38
39#include "Logging.h"
40#include "MediaConstraints.h"
41#include "NotImplemented.h"
42#include "RealtimeMediaSourceCapabilities.h"
43#include "RealtimeMediaSourceCenter.h"
44#include <wtf/CompletionHandler.h>
45#include <wtf/MainThread.h>
46#include <wtf/UUID.h>
47#include <wtf/text/StringHash.h>
48
49namespace WebCore {
50
51RealtimeMediaSource::RealtimeMediaSource(Type type, String&& name, String&& deviceID, String&& hashSalt)
52 : m_idHashSalt(WTFMove(hashSalt))
53 , m_persistentID(WTFMove(deviceID))
54 , m_type(type)
55 , m_name(WTFMove(name))
56{
57 if (m_persistentID.isEmpty())
58 m_persistentID = createCanonicalUUIDString();
59
60 m_hashedID = RealtimeMediaSourceCenter::singleton().hashStringWithSalt(m_persistentID, m_idHashSalt);
61}
62
63void RealtimeMediaSource::addObserver(RealtimeMediaSource::Observer& observer)
64{
65 auto locker = holdLock(m_observersLock);
66 m_observers.add(&observer);
67}
68
69void RealtimeMediaSource::removeObserver(RealtimeMediaSource::Observer& observer)
70{
71 auto locker = holdLock(m_observersLock);
72
73 m_observers.remove(&observer);
74 if (m_observers.isEmpty())
75 stop();
76}
77
78void RealtimeMediaSource::setInterrupted(bool interrupted, bool pageMuted)
79{
80 if (interrupted == m_interrupted)
81 return;
82
83 ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER, interrupted, ", page muted : ", pageMuted);
84
85 m_interrupted = interrupted;
86 if (!interrupted && pageMuted)
87 return;
88
89 setMuted(interrupted);
90}
91
92void RealtimeMediaSource::setMuted(bool muted)
93{
94 ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER, muted);
95
96 if (muted)
97 stop();
98 else {
99 if (interrupted())
100 return;
101
102 start();
103 }
104
105 notifyMutedChange(muted);
106}
107
108void RealtimeMediaSource::notifyMutedChange(bool muted)
109{
110 if (m_muted == muted)
111 return;
112
113 ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER, muted);
114 m_muted = muted;
115
116 notifyMutedObservers();
117}
118
119void RealtimeMediaSource::setInterruptedForTesting(bool interrupted)
120{
121 notifyMutedChange(interrupted);
122}
123
124void RealtimeMediaSource::forEachObserver(const WTF::Function<void(Observer&)>& apply) const
125{
126 Vector<Observer*> observersCopy;
127 {
128 auto locker = holdLock(m_observersLock);
129 observersCopy = copyToVector(m_observers);
130 }
131 for (auto* observer : observersCopy) {
132 auto locker = holdLock(m_observersLock);
133 // Make sure the observer has not been destroyed.
134 if (!m_observers.contains(observer))
135 continue;
136 apply(*observer);
137 }
138}
139
140void RealtimeMediaSource::notifyMutedObservers() const
141{
142 forEachObserver([](auto& observer) {
143 observer.sourceMutedChanged();
144 });
145}
146
147void RealtimeMediaSource::notifySettingsDidChangeObservers(OptionSet<RealtimeMediaSourceSettings::Flag> flags)
148{
149 ASSERT(isMainThread());
150
151 settingsDidChange(flags);
152
153 if (m_pendingSettingsDidChangeNotification)
154 return;
155 m_pendingSettingsDidChangeNotification = true;
156
157 ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER, flags);
158
159 scheduleDeferredTask([this] {
160 m_pendingSettingsDidChangeNotification = false;
161 forEachObserver([](auto& observer) {
162 observer.sourceSettingsChanged();
163 });
164 });
165}
166
167void RealtimeMediaSource::videoSampleAvailable(MediaSample& mediaSample)
168{
169#if !RELEASE_LOG_DISABLED
170 ++m_frameCount;
171
172 auto timestamp = MonotonicTime::now();
173 auto delta = timestamp - m_lastFrameLogTime;
174 if (!m_lastFrameLogTime || delta >= 1_s) {
175 if (m_lastFrameLogTime) {
176 INFO_LOG_IF(loggerPtr(), LOGIDENTIFIER, m_frameCount, " frames sent in ", delta.value(), " seconds");
177 m_frameCount = 0;
178 }
179 m_lastFrameLogTime = timestamp;
180 }
181#endif
182
183 forEachObserver([&](auto& observer) {
184 observer.videoSampleAvailable(mediaSample);
185 });
186}
187
188void RealtimeMediaSource::audioSamplesAvailable(const MediaTime& time, const PlatformAudioData& audioData, const AudioStreamDescription& description, size_t numberOfFrames)
189{
190 forEachObserver([&](auto& observer) {
191 observer.audioSamplesAvailable(time, audioData, description, numberOfFrames);
192 });
193}
194
195void RealtimeMediaSource::start()
196{
197 if (m_isProducingData || m_isEnded)
198 return;
199
200 ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER);
201
202 m_isProducingData = true;
203 startProducingData();
204
205 if (!m_isProducingData)
206 return;
207
208 forEachObserver([](auto& observer) {
209 observer.sourceStarted();
210 });
211}
212
213void RealtimeMediaSource::stop()
214{
215 if (!m_isProducingData)
216 return;
217
218 ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER);
219
220 m_isProducingData = false;
221 stopProducingData();
222}
223
224void RealtimeMediaSource::requestToEnd(Observer& callingObserver)
225{
226 if (!m_isProducingData)
227 return;
228
229 bool hasObserverPreventingStopping = false;
230 forEachObserver([&](auto& observer) {
231 if (observer.preventSourceFromStopping())
232 hasObserverPreventingStopping = true;
233 });
234 if (hasObserverPreventingStopping)
235 return;
236
237 auto protectedThis = makeRef(*this);
238
239 stop();
240 m_isEnded = true;
241 hasEnded();
242
243 forEachObserver([callingObserver](auto& observer) {
244 if (&observer != &callingObserver)
245 observer.sourceStopped();
246 });
247}
248
249void RealtimeMediaSource::captureFailed()
250{
251 ERROR_LOG_IF(m_logger, LOGIDENTIFIER);
252
253 m_isProducingData = false;
254 m_captureDidFailed = true;
255
256 forEachObserver([](auto& observer) {
257 observer.sourceStopped();
258 });
259}
260
261bool RealtimeMediaSource::supportsSizeAndFrameRate(Optional<int>, Optional<int>, Optional<double>)
262{
263 // The size and frame rate are within the capability limits, so they are supported.
264 return true;
265}
266
267bool RealtimeMediaSource::supportsSizeAndFrameRate(Optional<IntConstraint> widthConstraint, Optional<IntConstraint> heightConstraint, Optional<DoubleConstraint> frameRateConstraint, String& badConstraint, double& distance)
268{
269 if (!widthConstraint && !heightConstraint && !frameRateConstraint)
270 return true;
271
272 auto& capabilities = this->capabilities();
273
274 distance = std::numeric_limits<double>::infinity();
275
276 Optional<int> width;
277 if (widthConstraint && capabilities.supportsWidth()) {
278 double constraintDistance = fitnessDistance(*widthConstraint);
279 if (std::isinf(constraintDistance)) {
280 badConstraint = widthConstraint->name();
281 return false;
282 }
283
284 distance = std::min(distance, constraintDistance);
285 if (widthConstraint->isMandatory()) {
286 auto range = capabilities.width();
287 width = widthConstraint->valueForCapabilityRange(size().width(), range.rangeMin().asInt, range.rangeMax().asInt);
288 }
289 }
290
291 Optional<int> height;
292 if (heightConstraint && capabilities.supportsHeight()) {
293 double constraintDistance = fitnessDistance(*heightConstraint);
294 if (std::isinf(constraintDistance)) {
295 badConstraint = heightConstraint->name();
296 return false;
297 }
298
299 distance = std::min(distance, constraintDistance);
300 if (heightConstraint->isMandatory()) {
301 auto range = capabilities.height();
302 height = heightConstraint->valueForCapabilityRange(size().height(), range.rangeMin().asInt, range.rangeMax().asInt);
303 }
304 }
305
306 Optional<double> frameRate;
307 if (frameRateConstraint && capabilities.supportsFrameRate()) {
308 double constraintDistance = fitnessDistance(*frameRateConstraint);
309 if (std::isinf(constraintDistance)) {
310 badConstraint = frameRateConstraint->name();
311 return false;
312 }
313
314 distance = std::min(distance, constraintDistance);
315 if (frameRateConstraint->isMandatory()) {
316 auto range = capabilities.frameRate();
317 frameRate = frameRateConstraint->valueForCapabilityRange(this->frameRate(), range.rangeMin().asDouble, range.rangeMax().asDouble);
318 }
319 }
320
321 // Each of the non-null values is supported individually, see if they all can be applied at the same time.
322 if (!supportsSizeAndFrameRate(WTFMove(width), WTFMove(height), WTFMove(frameRate))) {
323 if (widthConstraint)
324 badConstraint = widthConstraint->name();
325 else if (heightConstraint)
326 badConstraint = heightConstraint->name();
327 else
328 badConstraint = frameRateConstraint->name();
329 return false;
330 }
331
332 return true;
333}
334
335double RealtimeMediaSource::fitnessDistance(const MediaConstraint& constraint)
336{
337 auto& capabilities = this->capabilities();
338
339 switch (constraint.constraintType()) {
340 case MediaConstraintType::Width: {
341 ASSERT(constraint.isInt());
342 if (!capabilities.supportsWidth())
343 return 0;
344
345 auto range = capabilities.width();
346 return downcast<IntConstraint>(constraint).fitnessDistance(range.rangeMin().asInt, range.rangeMax().asInt);
347 break;
348 }
349
350 case MediaConstraintType::Height: {
351 ASSERT(constraint.isInt());
352 if (!capabilities.supportsHeight())
353 return 0;
354
355 auto range = capabilities.height();
356 return downcast<IntConstraint>(constraint).fitnessDistance(range.rangeMin().asInt, range.rangeMax().asInt);
357 break;
358 }
359
360 case MediaConstraintType::FrameRate: {
361 ASSERT(constraint.isDouble());
362 if (!capabilities.supportsFrameRate())
363 return 0;
364
365 auto range = capabilities.frameRate();
366 return downcast<DoubleConstraint>(constraint).fitnessDistance(range.rangeMin().asDouble, range.rangeMax().asDouble);
367 break;
368 }
369
370 case MediaConstraintType::AspectRatio: {
371 ASSERT(constraint.isDouble());
372 if (!capabilities.supportsAspectRatio())
373 return 0;
374
375 auto range = capabilities.aspectRatio();
376 return downcast<DoubleConstraint>(constraint).fitnessDistance(range.rangeMin().asDouble, range.rangeMax().asDouble);
377 break;
378 }
379
380 case MediaConstraintType::Volume: {
381 ASSERT(constraint.isDouble());
382 if (!capabilities.supportsVolume())
383 return 0;
384
385 auto range = capabilities.volume();
386 return downcast<DoubleConstraint>(constraint).fitnessDistance(range.rangeMin().asDouble, range.rangeMax().asDouble);
387 break;
388 }
389
390 case MediaConstraintType::SampleRate: {
391 ASSERT(constraint.isInt());
392 if (!capabilities.supportsSampleRate())
393 return 0;
394
395 if (auto discreteRates = discreteSampleRates())
396 return downcast<IntConstraint>(constraint).fitnessDistance(*discreteRates);
397
398 auto range = capabilities.sampleRate();
399 return downcast<IntConstraint>(constraint).fitnessDistance(range.rangeMin().asInt, range.rangeMax().asInt);
400 break;
401 }
402
403 case MediaConstraintType::SampleSize: {
404 ASSERT(constraint.isInt());
405 if (!capabilities.supportsSampleSize())
406 return 0;
407
408 if (auto discreteSizes = discreteSampleSizes())
409 return downcast<IntConstraint>(constraint).fitnessDistance(*discreteSizes);
410
411 auto range = capabilities.sampleSize();
412 return downcast<IntConstraint>(constraint).fitnessDistance(range.rangeMin().asInt, range.rangeMax().asInt);
413 break;
414 }
415
416 case MediaConstraintType::FacingMode: {
417 ASSERT(constraint.isString());
418 if (!capabilities.supportsFacingMode())
419 return 0;
420
421 auto& modes = capabilities.facingMode();
422 Vector<String> supportedModes;
423 supportedModes.reserveInitialCapacity(modes.size());
424 for (auto& mode : modes)
425 supportedModes.uncheckedAppend(RealtimeMediaSourceSettings::facingMode(mode));
426 return downcast<StringConstraint>(constraint).fitnessDistance(supportedModes);
427 break;
428 }
429
430 case MediaConstraintType::EchoCancellation: {
431 ASSERT(constraint.isBoolean());
432 if (!capabilities.supportsEchoCancellation())
433 return 0;
434
435 bool echoCancellationReadWrite = capabilities.echoCancellation() == RealtimeMediaSourceCapabilities::EchoCancellation::ReadWrite;
436 return downcast<BooleanConstraint>(constraint).fitnessDistance(echoCancellationReadWrite);
437 break;
438 }
439
440 case MediaConstraintType::DeviceId:
441 ASSERT(!m_hashedID.isEmpty());
442 return downcast<StringConstraint>(constraint).fitnessDistance(m_hashedID);
443 break;
444
445 case MediaConstraintType::GroupId: {
446 ASSERT(constraint.isString());
447 if (!capabilities.supportsDeviceId())
448 return 0;
449
450 return downcast<StringConstraint>(constraint).fitnessDistance(settings().groupId());
451 break;
452 }
453
454 case MediaConstraintType::DisplaySurface:
455 case MediaConstraintType::LogicalSurface:
456 break;
457
458 case MediaConstraintType::Unknown:
459 // Unknown (or unsupported) constraints should be ignored.
460 break;
461 }
462
463 return 0;
464}
465
466template <typename ValueType>
467static void applyNumericConstraint(const NumericConstraint<ValueType>& constraint, ValueType current, Optional<Vector<ValueType>> discreteCapabilityValues, ValueType capabilityMin, ValueType capabilityMax, RealtimeMediaSource& source, void (RealtimeMediaSource::*applier)(ValueType))
468{
469 if (discreteCapabilityValues) {
470 int value = constraint.valueForDiscreteCapabilityValues(current, *discreteCapabilityValues);
471 if (value != current)
472 (source.*applier)(value);
473 return;
474 }
475
476 ValueType value = constraint.valueForCapabilityRange(current, capabilityMin, capabilityMax);
477 if (value != current)
478 (source.*applier)(value);
479}
480
481void RealtimeMediaSource::setSizeAndFrameRate(Optional<int> width, Optional<int> height, Optional<double> frameRate)
482{
483 IntSize size;
484 if (width)
485 size.setWidth(width.value());
486 if (height)
487 size.setHeight(height.value());
488 setSize(size);
489 if (frameRate)
490 setFrameRate(frameRate.value());
491}
492
493void RealtimeMediaSource::applyConstraint(const MediaConstraint& constraint)
494{
495 ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER, constraint.name());
496
497 auto& capabilities = this->capabilities();
498 switch (constraint.constraintType()) {
499 case MediaConstraintType::Width:
500 ASSERT_NOT_REACHED();
501 break;
502
503 case MediaConstraintType::Height:
504 ASSERT_NOT_REACHED();
505 break;
506
507 case MediaConstraintType::FrameRate:
508 ASSERT_NOT_REACHED();
509 break;
510
511 case MediaConstraintType::AspectRatio: {
512 ASSERT(constraint.isDouble());
513 if (!capabilities.supportsAspectRatio())
514 return;
515
516 auto range = capabilities.aspectRatio();
517 applyNumericConstraint(downcast<DoubleConstraint>(constraint), aspectRatio(), { }, range.rangeMin().asDouble, range.rangeMax().asDouble, *this, &RealtimeMediaSource::setAspectRatio);
518 break;
519 }
520
521 case MediaConstraintType::Volume: {
522 ASSERT(constraint.isDouble());
523 if (!capabilities.supportsVolume())
524 return;
525
526 auto range = capabilities.volume();
527 applyNumericConstraint(downcast<DoubleConstraint>(constraint), volume(), { }, range.rangeMin().asDouble, range.rangeMax().asDouble, *this, &RealtimeMediaSource::setVolume);
528 break;
529 }
530
531 case MediaConstraintType::SampleRate: {
532 ASSERT(constraint.isInt());
533 if (!capabilities.supportsSampleRate())
534 return;
535
536 auto range = capabilities.sampleRate();
537 applyNumericConstraint(downcast<IntConstraint>(constraint), sampleRate(), discreteSampleRates(), range.rangeMin().asInt, range.rangeMax().asInt, *this, &RealtimeMediaSource::setSampleRate);
538 break;
539 }
540
541 case MediaConstraintType::SampleSize: {
542 ASSERT(constraint.isInt());
543 if (!capabilities.supportsSampleSize())
544 return;
545
546 auto range = capabilities.sampleSize();
547 applyNumericConstraint(downcast<IntConstraint>(constraint), sampleSize(), { }, range.rangeMin().asInt, range.rangeMax().asInt, *this, &RealtimeMediaSource::setSampleSize);
548 break;
549 }
550
551 case MediaConstraintType::EchoCancellation: {
552 ASSERT(constraint.isBoolean());
553 if (!capabilities.supportsEchoCancellation())
554 return;
555
556 bool setting;
557 const BooleanConstraint& boolConstraint = downcast<BooleanConstraint>(constraint);
558 if (boolConstraint.getExact(setting) || boolConstraint.getIdeal(setting))
559 setEchoCancellation(setting);
560 break;
561 }
562
563 case MediaConstraintType::FacingMode: {
564 ASSERT(constraint.isString());
565 if (!capabilities.supportsFacingMode())
566 return;
567
568 auto& supportedModes = capabilities.facingMode();
569 auto filter = [supportedModes](const String& modeString) {
570 auto mode = RealtimeMediaSourceSettings::videoFacingModeEnum(modeString);
571 for (auto& supportedMode : supportedModes) {
572 if (mode == supportedMode)
573 return true;
574 }
575 return false;
576 };
577
578 auto modeString = downcast<StringConstraint>(constraint).find(WTFMove(filter));
579 if (!modeString.isEmpty())
580 setFacingMode(RealtimeMediaSourceSettings::videoFacingModeEnum(modeString));
581 break;
582 }
583
584 case MediaConstraintType::DeviceId:
585 case MediaConstraintType::GroupId:
586 ASSERT(constraint.isString());
587 // There is nothing to do here, neither can be changed.
588 break;
589
590 case MediaConstraintType::DisplaySurface:
591 case MediaConstraintType::LogicalSurface:
592 ASSERT(constraint.isBoolean());
593 break;
594
595 case MediaConstraintType::Unknown:
596 break;
597 }
598}
599
600bool RealtimeMediaSource::selectSettings(const MediaConstraints& constraints, FlattenedConstraint& candidates, String& failedConstraint)
601{
602 double minimumDistance = std::numeric_limits<double>::infinity();
603
604 // https://w3c.github.io/mediacapture-main/#dfn-selectsettings
605 //
606 // 1. Each constraint specifies one or more values (or a range of values) for its property.
607 // A property may appear more than once in the list of 'advanced' ConstraintSets. If an
608 // empty object or list has been given as the value for a constraint, it must be interpreted
609 // as if the constraint were not specified (in other words, an empty constraint == no constraint).
610 //
611 // Note that unknown properties are discarded by WebIDL, which means that unknown/unsupported required
612 // constraints will silently disappear. To avoid this being a surprise, application authors are
613 // expected to first use the getSupportedConstraints() method as shown in the Examples below.
614
615 // 2. Let object be the ConstrainablePattern object on which this algorithm is applied. Let copy be an
616 // unconstrained copy of object (i.e., copy should behave as if it were object with all ConstraintSets
617 // removed.)
618
619 // 3. For every possible settings dictionary of copy compute its fitness distance, treating bare values of
620 // properties as ideal values. Let candidates be the set of settings dictionaries for which the fitness
621 // distance is finite.
622
623 failedConstraint = emptyString();
624
625 // Check width, height and frame rate jointly, because while they may be supported individually the combination may not be supported.
626 double distance = std::numeric_limits<double>::infinity();
627 if (!supportsSizeAndFrameRate(constraints.mandatoryConstraints.width(), constraints.mandatoryConstraints.height(), constraints.mandatoryConstraints.frameRate(), failedConstraint, minimumDistance))
628 return false;
629
630 constraints.mandatoryConstraints.filter([&](const MediaConstraint& constraint) {
631 if (!supportsConstraint(constraint))
632 return false;
633
634 if (constraint.constraintType() == MediaConstraintType::Width || constraint.constraintType() == MediaConstraintType::Height || constraint.constraintType() == MediaConstraintType::FrameRate) {
635 candidates.set(constraint);
636 return false;
637 }
638
639 double constraintDistance = fitnessDistance(constraint);
640 if (std::isinf(constraintDistance)) {
641 failedConstraint = constraint.name();
642 return true;
643 }
644
645 distance = std::min(distance, constraintDistance);
646 candidates.set(constraint);
647 return false;
648 });
649
650 if (!failedConstraint.isEmpty())
651 return false;
652
653 minimumDistance = distance;
654
655 // 4. If candidates is empty, return undefined as the result of the SelectSettings() algorithm.
656 if (candidates.isEmpty())
657 return true;
658
659 // 5. Iterate over the 'advanced' ConstraintSets in newConstraints in the order in which they were specified.
660 // For each ConstraintSet:
661
662 // 5.1 compute the fitness distance between it and each settings dictionary in candidates, treating bare
663 // values of properties as exact.
664 Vector<std::pair<double, MediaTrackConstraintSetMap>> supportedConstraints;
665
666 for (const auto& advancedConstraint : constraints.advancedConstraints) {
667 double constraintDistance = 0;
668 bool supported = false;
669
670 if (advancedConstraint.width() || advancedConstraint.height() || advancedConstraint.frameRate()) {
671 String dummy;
672 if (!supportsSizeAndFrameRate(advancedConstraint.width(), advancedConstraint.height(), advancedConstraint.frameRate(), dummy, constraintDistance))
673 continue;
674
675 supported = true;
676 }
677
678 advancedConstraint.forEach([&](const MediaConstraint& constraint) {
679
680 if (constraint.constraintType() == MediaConstraintType::Width || constraint.constraintType() == MediaConstraintType::Height || constraint.constraintType() == MediaConstraintType::FrameRate)
681 return;
682
683 distance = fitnessDistance(constraint);
684 constraintDistance += distance;
685 if (!std::isinf(distance))
686 supported = true;
687 });
688
689 minimumDistance = std::min(minimumDistance, constraintDistance);
690
691 // 5.2 If the fitness distance is finite for one or more settings dictionaries in candidates, keep those
692 // settings dictionaries in candidates, discarding others.
693 // If the fitness distance is infinite for all settings dictionaries in candidates, ignore this ConstraintSet.
694 if (supported)
695 supportedConstraints.append({constraintDistance, advancedConstraint});
696 }
697
698 // 6. Select one settings dictionary from candidates, and return it as the result of the SelectSettings() algorithm.
699 // The UA should use the one with the smallest fitness distance, as calculated in step 3.
700 if (!supportedConstraints.isEmpty()) {
701 supportedConstraints.removeAllMatching([&](const std::pair<double, MediaTrackConstraintSetMap>& pair) -> bool {
702 return std::isinf(pair.first) || pair.first > minimumDistance;
703 });
704
705 if (!supportedConstraints.isEmpty()) {
706 auto& advancedConstraint = supportedConstraints[0].second;
707 advancedConstraint.forEach([&](const MediaConstraint& constraint) {
708 candidates.merge(constraint);
709 });
710
711 minimumDistance = std::min(minimumDistance, supportedConstraints[0].first);
712 }
713 }
714
715 return true;
716}
717
718bool RealtimeMediaSource::supportsConstraint(const MediaConstraint& constraint)
719{
720 auto& capabilities = this->capabilities();
721
722 switch (constraint.constraintType()) {
723 case MediaConstraintType::Width:
724 ASSERT(constraint.isInt());
725 return capabilities.supportsWidth();
726 break;
727
728 case MediaConstraintType::Height:
729 ASSERT(constraint.isInt());
730 return capabilities.supportsHeight();
731 break;
732
733 case MediaConstraintType::FrameRate:
734 ASSERT(constraint.isDouble());
735 return capabilities.supportsFrameRate();
736 break;
737
738 case MediaConstraintType::AspectRatio:
739 ASSERT(constraint.isDouble());
740 return capabilities.supportsAspectRatio();
741 break;
742
743 case MediaConstraintType::Volume:
744 ASSERT(constraint.isDouble());
745 return capabilities.supportsVolume();
746 break;
747
748 case MediaConstraintType::SampleRate:
749 ASSERT(constraint.isInt());
750 return capabilities.supportsSampleRate();
751 break;
752
753 case MediaConstraintType::SampleSize:
754 ASSERT(constraint.isInt());
755 return capabilities.supportsSampleSize();
756 break;
757
758 case MediaConstraintType::FacingMode:
759 ASSERT(constraint.isString());
760 return capabilities.supportsFacingMode();
761 break;
762
763 case MediaConstraintType::EchoCancellation:
764 ASSERT(constraint.isBoolean());
765 return capabilities.supportsEchoCancellation();
766 break;
767
768 case MediaConstraintType::DeviceId:
769 ASSERT(constraint.isString());
770 return capabilities.supportsDeviceId();
771 break;
772
773 case MediaConstraintType::GroupId:
774 ASSERT(constraint.isString());
775 return capabilities.supportsDeviceId();
776 break;
777
778 case MediaConstraintType::DisplaySurface:
779 case MediaConstraintType::LogicalSurface:
780 // https://www.w3.org/TR/screen-capture/#new-constraints-for-captured-display-surfaces
781 // 5.2.1 New Constraints for Captured Display Surfaces
782 // Since the source of media cannot be changed after a MediaStreamTrack has been returned,
783 // these constraints cannot be changed by an application.
784 return false;
785 break;
786
787 case MediaConstraintType::Unknown:
788 // Unknown (or unsupported) constraints should be ignored.
789 break;
790 }
791
792 return false;
793}
794
795bool RealtimeMediaSource::supportsConstraints(const MediaConstraints& constraints, String& invalidConstraint)
796{
797 ASSERT(constraints.isValid);
798
799 ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER);
800
801 FlattenedConstraint candidates;
802 if (!selectSettings(constraints, candidates, invalidConstraint))
803 return false;
804
805 m_fitnessScore = 0;
806 for (auto& variant : candidates) {
807 double distance = fitnessDistance(variant);
808 switch (variant.constraintType()) {
809 case MediaConstraintType::DeviceId:
810 case MediaConstraintType::FacingMode:
811 m_fitnessScore += distance ? 1 : 32;
812 break;
813
814 case MediaConstraintType::Width:
815 case MediaConstraintType::Height:
816 case MediaConstraintType::FrameRate:
817 case MediaConstraintType::AspectRatio:
818 case MediaConstraintType::Volume:
819 case MediaConstraintType::SampleRate:
820 case MediaConstraintType::SampleSize:
821 case MediaConstraintType::EchoCancellation:
822 case MediaConstraintType::GroupId:
823 case MediaConstraintType::DisplaySurface:
824 case MediaConstraintType::LogicalSurface:
825 case MediaConstraintType::Unknown:
826 m_fitnessScore += distance ? 1 : 2;
827 break;
828 }
829 }
830
831 ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER, "fitness distance : ", m_fitnessScore);
832
833 return true;
834}
835
836void RealtimeMediaSource::applyConstraints(const FlattenedConstraint& constraints)
837{
838 if (constraints.isEmpty())
839 return;
840
841 beginConfiguration();
842
843 auto& capabilities = this->capabilities();
844
845 Optional<int> width;
846 if (const MediaConstraint* constraint = constraints.find(MediaConstraintType::Width)) {
847 ASSERT(constraint->isInt());
848 if (capabilities.supportsWidth()) {
849 auto range = capabilities.width();
850 width = downcast<IntConstraint>(*constraint).valueForCapabilityRange(size().width(), range.rangeMin().asInt, range.rangeMax().asInt);
851 }
852 }
853
854 Optional<int> height;
855 if (const MediaConstraint* constraint = constraints.find(MediaConstraintType::Height)) {
856 ASSERT(constraint->isInt());
857 if (capabilities.supportsHeight()) {
858 auto range = capabilities.height();
859 height = downcast<IntConstraint>(*constraint).valueForCapabilityRange(size().height(), range.rangeMin().asInt, range.rangeMax().asInt);
860 }
861 }
862
863 Optional<double> frameRate;
864 if (const MediaConstraint* constraint = constraints.find(MediaConstraintType::FrameRate)) {
865 ASSERT(constraint->isDouble());
866 if (capabilities.supportsFrameRate()) {
867 auto range = capabilities.frameRate();
868 frameRate = downcast<DoubleConstraint>(*constraint).valueForCapabilityRange(this->frameRate(), range.rangeMin().asDouble, range.rangeMax().asDouble);
869 }
870 }
871
872 if (width || height || frameRate)
873 setSizeAndFrameRate(WTFMove(width), WTFMove(height), WTFMove(frameRate));
874
875 for (auto& variant : constraints) {
876 if (variant.constraintType() == MediaConstraintType::Width || variant.constraintType() == MediaConstraintType::Height || variant.constraintType() == MediaConstraintType::FrameRate)
877 continue;
878
879 applyConstraint(variant);
880 }
881
882 commitConfiguration();
883}
884
885Optional<RealtimeMediaSource::ApplyConstraintsError> RealtimeMediaSource::applyConstraints(const MediaConstraints& constraints)
886{
887 ASSERT(constraints.isValid);
888
889 ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER);
890
891 FlattenedConstraint candidates;
892 String failedConstraint;
893 if (!selectSettings(constraints, candidates, failedConstraint))
894 return ApplyConstraintsError { failedConstraint, "Constraint not supported"_s };
895
896 applyConstraints(candidates);
897 return { };
898}
899
900void RealtimeMediaSource::applyConstraints(const MediaConstraints& constraints, ApplyConstraintsHandler&& completionHandler)
901{
902 completionHandler(applyConstraints(constraints));
903}
904
905void RealtimeMediaSource::setSize(const IntSize& size)
906{
907 if (size == m_size)
908 return;
909
910 ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER, size);
911
912 m_size = size;
913 notifySettingsDidChangeObservers({ RealtimeMediaSourceSettings::Flag::Width, RealtimeMediaSourceSettings::Flag::Height });
914}
915
916const IntSize RealtimeMediaSource::size() const
917{
918 auto size = m_size;
919
920 if (size.isEmpty() && !m_intrinsicSize.isEmpty()) {
921 if (size.isZero())
922 size = m_intrinsicSize;
923 else if (size.width())
924 size.setHeight(size.width() * (m_intrinsicSize.height() / static_cast<double>(m_intrinsicSize.width())));
925 else if (size.height())
926 size.setWidth(size.height() * (m_intrinsicSize.width() / static_cast<double>(m_intrinsicSize.height())));
927 }
928
929 return size;
930}
931
932void RealtimeMediaSource::setIntrinsicSize(const IntSize& size)
933{
934 if (m_intrinsicSize == size)
935 return;
936
937 ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER, size);
938
939 auto currentSize = this->size();
940 m_intrinsicSize = size;
941
942 if (currentSize != this->size())
943 notifySettingsDidChangeObservers({ RealtimeMediaSourceSettings::Flag::Width, RealtimeMediaSourceSettings::Flag::Height });
944}
945
946const IntSize RealtimeMediaSource::intrinsicSize() const
947{
948 return m_intrinsicSize;
949}
950
951void RealtimeMediaSource::setFrameRate(double rate)
952{
953 if (m_frameRate == rate)
954 return;
955
956 ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER, rate);
957
958 m_frameRate = rate;
959 notifySettingsDidChangeObservers(RealtimeMediaSourceSettings::Flag::FrameRate);
960}
961
962void RealtimeMediaSource::setAspectRatio(double ratio)
963{
964 if (m_aspectRatio == ratio)
965 return;
966
967 ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER, ratio);
968
969 m_aspectRatio = ratio;
970 m_size.setHeight(m_size.width() / ratio);
971 notifySettingsDidChangeObservers({ RealtimeMediaSourceSettings::Flag::AspectRatio, RealtimeMediaSourceSettings::Flag::Height });
972}
973
974void RealtimeMediaSource::setFacingMode(RealtimeMediaSourceSettings::VideoFacingMode mode)
975{
976 if (m_facingMode == mode)
977 return;
978
979 ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER, mode);
980
981 m_facingMode = mode;
982 notifySettingsDidChangeObservers(RealtimeMediaSourceSettings::Flag::FacingMode);
983}
984
985void RealtimeMediaSource::setVolume(double volume)
986{
987 if (m_volume == volume)
988 return;
989
990 ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER, volume);
991
992 m_volume = volume;
993 notifySettingsDidChangeObservers(RealtimeMediaSourceSettings::Flag::Volume);
994}
995
996void RealtimeMediaSource::setSampleRate(int rate)
997{
998 if (m_sampleRate == rate)
999 return;
1000
1001 ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER, rate);
1002
1003 m_sampleRate = rate;
1004 notifySettingsDidChangeObservers(RealtimeMediaSourceSettings::Flag::SampleRate);
1005}
1006
1007Optional<Vector<int>> RealtimeMediaSource::discreteSampleRates() const
1008{
1009 return WTF::nullopt;
1010}
1011
1012void RealtimeMediaSource::setSampleSize(int size)
1013{
1014 if (m_sampleSize == size)
1015 return;
1016
1017 ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER, size);
1018
1019 m_sampleSize = size;
1020 notifySettingsDidChangeObservers(RealtimeMediaSourceSettings::Flag::SampleSize);
1021}
1022
1023Optional<Vector<int>> RealtimeMediaSource::discreteSampleSizes() const
1024{
1025 return WTF::nullopt;
1026}
1027
1028void RealtimeMediaSource::setEchoCancellation(bool echoCancellation)
1029{
1030 if (m_echoCancellation == echoCancellation)
1031 return;
1032
1033 ALWAYS_LOG_IF(m_logger, LOGIDENTIFIER, echoCancellation);
1034 m_echoCancellation = echoCancellation;
1035 notifySettingsDidChangeObservers(RealtimeMediaSourceSettings::Flag::EchoCancellation);
1036}
1037
1038void RealtimeMediaSource::scheduleDeferredTask(WTF::Function<void()>&& function)
1039{
1040 ASSERT(function);
1041 callOnMainThread([weakThis = makeWeakPtr(*this), function = WTFMove(function)] {
1042 if (!weakThis)
1043 return;
1044
1045 function();
1046 });
1047}
1048
1049const String& RealtimeMediaSource::hashedId() const
1050{
1051 ASSERT(!m_hashedID.isEmpty());
1052 return m_hashedID;
1053}
1054
1055String RealtimeMediaSource::deviceIDHashSalt() const
1056{
1057 return m_idHashSalt;
1058}
1059
1060RealtimeMediaSource::Observer::~Observer()
1061{
1062}
1063
1064#if !RELEASE_LOG_DISABLED
1065void RealtimeMediaSource::setLogger(const Logger& newLogger, const void* newLogIdentifier)
1066{
1067 m_logger = &newLogger;
1068 m_logIdentifier = newLogIdentifier;
1069 ALWAYS_LOG(LOGIDENTIFIER, m_type, ", ", m_name, ", ", m_hashedID);
1070}
1071
1072WTFLogChannel& RealtimeMediaSource::logChannel() const
1073{
1074 return LogWebRTC;
1075}
1076#endif
1077
1078String convertEnumerationToString(RealtimeMediaSource::Type enumerationValue)
1079{
1080 static const NeverDestroyed<String> values[] = {
1081 MAKE_STATIC_STRING_IMPL("None"),
1082 MAKE_STATIC_STRING_IMPL("Audio"),
1083 MAKE_STATIC_STRING_IMPL("Video"),
1084 };
1085 static_assert(static_cast<size_t>(RealtimeMediaSource::Type::None) == 0, "RealtimeMediaSource::Type::None is not 0 as expected");
1086 static_assert(static_cast<size_t>(RealtimeMediaSource::Type::Audio) == 1, "RealtimeMediaSource::Type::Audio is not 1 as expected");
1087 static_assert(static_cast<size_t>(RealtimeMediaSource::Type::Video) == 2, "RealtimeMediaSource::Type::Video is not 2 as expected");
1088 ASSERT(static_cast<size_t>(enumerationValue) < WTF_ARRAY_LENGTH(values));
1089 return values[static_cast<size_t>(enumerationValue)];
1090}
1091
1092} // namespace WebCore
1093
1094#endif // ENABLE(MEDIA_STREAM)
1095