1/*
2 * Copyright (C) 2011 Ericsson AB. All rights reserved.
3 * Copyright (C) 2012 Google Inc. All rights reserved.
4 * Copyright (C) 2013-2019 Apple Inc. All rights reserved.
5 * Copyright (C) 2013 Nokia Corporation and/or its subsidiary(-ies).
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer
15 * in the documentation and/or other materials provided with the
16 * distribution.
17 * 3. Neither the name of Ericsson nor the names of its contributors
18 * may be used to endorse or promote products derived from this
19 * software without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34#pragma once
35
36#if ENABLE(MEDIA_STREAM)
37
38#include "CaptureDevice.h"
39#include "Image.h"
40#include "MediaConstraints.h"
41#include "MediaSample.h"
42#include "PlatformLayer.h"
43#include "RealtimeMediaSourceCapabilities.h"
44#include "RealtimeMediaSourceFactory.h"
45#include <wtf/LoggerHelper.h>
46#include <wtf/RecursiveLockAdapter.h>
47#include <wtf/ThreadSafeRefCounted.h>
48#include <wtf/Vector.h>
49#include <wtf/WeakPtr.h>
50#include <wtf/text/WTFString.h>
51
52namespace WTF {
53class MediaTime;
54}
55
56namespace WebCore {
57
58class AudioStreamDescription;
59class FloatRect;
60class GraphicsContext;
61class MediaStreamPrivate;
62class OrientationNotifier;
63class PlatformAudioData;
64class RealtimeMediaSourceSettings;
65class RemoteVideoSample;
66
67struct CaptureSourceOrError;
68
69class WEBCORE_EXPORT RealtimeMediaSource
70 : public ThreadSafeRefCounted<RealtimeMediaSource>
71 , public CanMakeWeakPtr<RealtimeMediaSource>
72#if !RELEASE_LOG_DISABLED
73 , private LoggerHelper
74#endif
75{
76public:
77 class Observer {
78 public:
79 virtual ~Observer();
80
81 // Source state changes.
82 virtual void sourceStarted() { }
83 virtual void sourceStopped() { }
84 virtual void sourceMutedChanged() { }
85 virtual void sourceSettingsChanged() { }
86
87 // Observer state queries.
88 virtual bool preventSourceFromStopping() { return false; }
89
90 // Called on the main thread.
91 virtual void videoSampleAvailable(MediaSample&) { }
92
93 // May be called on a background thread.
94 virtual void audioSamplesAvailable(const MediaTime&, const PlatformAudioData&, const AudioStreamDescription&, size_t /*numberOfFrames*/) { }
95 };
96
97 virtual ~RealtimeMediaSource() = default;
98
99 const String& hashedId() const;
100 String deviceIDHashSalt() const;
101
102 const String& persistentID() const { return m_persistentID; }
103
104 enum class Type { None, Audio, Video };
105 Type type() const { return m_type; }
106
107 bool isProducingData() const { return m_isProducingData; }
108 void start();
109 void stop();
110 void requestToEnd(Observer& callingObserver);
111
112 bool muted() const { return m_muted; }
113 void setMuted(bool);
114
115 bool captureDidFail() const { return m_captureDidFailed; }
116
117 virtual bool interrupted() const { return m_interrupted; }
118 virtual void setInterrupted(bool, bool);
119
120 const String& name() const { return m_name; }
121 void setName(String&& name) { m_name = WTFMove(name); }
122
123 unsigned fitnessScore() const { return m_fitnessScore; }
124
125 WEBCORE_EXPORT void addObserver(Observer&);
126 WEBCORE_EXPORT void removeObserver(Observer&);
127
128 const IntSize size() const;
129 void setSize(const IntSize&);
130
131 const IntSize intrinsicSize() const;
132 void setIntrinsicSize(const IntSize&);
133
134 double frameRate() const { return m_frameRate; }
135 void setFrameRate(double);
136
137 double aspectRatio() const { return m_aspectRatio; }
138 void setAspectRatio(double);
139
140 RealtimeMediaSourceSettings::VideoFacingMode facingMode() const { return m_facingMode; }
141 void setFacingMode(RealtimeMediaSourceSettings::VideoFacingMode);
142
143 double volume() const { return m_volume; }
144 void setVolume(double);
145
146 int sampleRate() const { return m_sampleRate; }
147 void setSampleRate(int);
148 virtual Optional<Vector<int>> discreteSampleRates() const;
149
150 int sampleSize() const { return m_sampleSize; }
151 void setSampleSize(int);
152 virtual Optional<Vector<int>> discreteSampleSizes() const;
153
154 bool echoCancellation() const { return m_echoCancellation; }
155 void setEchoCancellation(bool);
156
157 virtual const RealtimeMediaSourceCapabilities& capabilities() = 0;
158 virtual const RealtimeMediaSourceSettings& settings() = 0;
159
160 struct ApplyConstraintsError {
161 String badConstraint;
162 String message;
163 };
164 using ApplyConstraintsHandler = CompletionHandler<void(Optional<ApplyConstraintsError>&&)>;
165 virtual void applyConstraints(const MediaConstraints&, ApplyConstraintsHandler&&);
166 Optional<ApplyConstraintsError> applyConstraints(const MediaConstraints&);
167
168 bool supportsConstraints(const MediaConstraints&, String&);
169 bool supportsConstraint(const MediaConstraint&);
170
171 virtual bool isIsolated() const { return false; }
172
173 virtual bool isCaptureSource() const { return false; }
174 virtual CaptureDevice::DeviceType deviceType() const { return CaptureDevice::DeviceType::Unknown; }
175
176 virtual void monitorOrientation(OrientationNotifier&) { }
177
178 virtual void captureFailed();
179
180 virtual bool isIncomingAudioSource() const { return false; }
181 virtual bool isIncomingVideoSource() const { return false; }
182
183 void setIsRemote(bool isRemote) { m_isRemote = isRemote; }
184 bool isRemote() const { return m_isRemote; }
185
186#if !RELEASE_LOG_DISABLED
187 void setLogger(const Logger&, const void*);
188 const Logger* loggerPtr() const { return m_logger.get(); }
189 const Logger& logger() const final { ASSERT(m_logger); return *m_logger.get(); }
190 const void* logIdentifier() const final { return m_logIdentifier; }
191 const char* logClassName() const override { return "RealtimeMediaSource"; }
192 WTFLogChannel& logChannel() const final;
193#endif
194
195 // Testing only
196 virtual void delaySamples(Seconds) { };
197 void setInterruptedForTesting(bool);
198
199protected:
200 RealtimeMediaSource(Type, String&& name, String&& deviceID = { }, String&& hashSalt = { });
201
202 void scheduleDeferredTask(WTF::Function<void()>&&);
203
204 virtual void beginConfiguration() { }
205 virtual void commitConfiguration() { }
206
207 bool selectSettings(const MediaConstraints&, FlattenedConstraint&, String&);
208 double fitnessDistance(const MediaConstraint&);
209 void applyConstraint(const MediaConstraint&);
210 void applyConstraints(const FlattenedConstraint&);
211 bool supportsSizeAndFrameRate(Optional<IntConstraint> width, Optional<IntConstraint> height, Optional<DoubleConstraint>, String&, double& fitnessDistance);
212
213 virtual bool supportsSizeAndFrameRate(Optional<int> width, Optional<int> height, Optional<double>);
214 virtual void setSizeAndFrameRate(Optional<int> width, Optional<int> height, Optional<double>);
215
216 void notifyMutedObservers() const;
217 void notifyMutedChange(bool muted);
218 void notifySettingsDidChangeObservers(OptionSet<RealtimeMediaSourceSettings::Flag>);
219
220 void initializeVolume(double volume) { m_volume = volume; }
221 void initializeSampleRate(int sampleRate) { m_sampleRate = sampleRate; }
222 void initializeEchoCancellation(bool echoCancellation) { m_echoCancellation = echoCancellation; }
223
224 void videoSampleAvailable(MediaSample&);
225 void audioSamplesAvailable(const MediaTime&, const PlatformAudioData&, const AudioStreamDescription&, size_t);
226
227private:
228 virtual void startProducingData() { }
229 virtual void stopProducingData() { }
230 virtual void settingsDidChange(OptionSet<RealtimeMediaSourceSettings::Flag>) { }
231
232 virtual void hasEnded() { }
233
234 void forEachObserver(const WTF::Function<void(Observer&)>&) const;
235
236#if !RELEASE_LOG_DISABLED
237 RefPtr<const Logger> m_logger;
238 const void* m_logIdentifier;
239 MonotonicTime m_lastFrameLogTime;
240 unsigned m_frameCount { 0 };
241#endif
242
243 String m_idHashSalt;
244 String m_hashedID;
245 String m_persistentID;
246 Type m_type;
247 String m_name;
248 mutable RecursiveLock m_observersLock;
249 HashSet<Observer*> m_observers;
250 IntSize m_size;
251 IntSize m_intrinsicSize;
252 double m_frameRate { 30 };
253 double m_aspectRatio { 0 };
254 double m_volume { 1 };
255 double m_sampleRate { 0 };
256 double m_sampleSize { 0 };
257 double m_fitnessScore { 0 };
258 RealtimeMediaSourceSettings::VideoFacingMode m_facingMode { RealtimeMediaSourceSettings::User};
259
260 bool m_muted { false };
261 bool m_pendingSettingsDidChangeNotification { false };
262 bool m_echoCancellation { false };
263 bool m_isProducingData { false };
264 bool m_interrupted { false };
265 bool m_captureDidFailed { false };
266 bool m_isRemote { false };
267 bool m_isEnded { false };
268};
269
270struct CaptureSourceOrError {
271 CaptureSourceOrError() = default;
272 CaptureSourceOrError(Ref<RealtimeMediaSource>&& source) : captureSource(WTFMove(source)) { }
273 CaptureSourceOrError(String&& message) : errorMessage(WTFMove(message)) { }
274
275 operator bool() const { return !!captureSource; }
276 Ref<RealtimeMediaSource> source() { return captureSource.releaseNonNull(); }
277
278 RefPtr<RealtimeMediaSource> captureSource;
279 String errorMessage;
280};
281
282String convertEnumerationToString(RealtimeMediaSource::Type);
283
284} // namespace WebCore
285
286namespace WTF {
287
288template<typename Type>
289struct LogArgument;
290
291template <>
292struct LogArgument<WebCore::RealtimeMediaSource::Type> {
293 static String toString(const WebCore::RealtimeMediaSource::Type type)
294 {
295 return convertEnumerationToString(type);
296 }
297};
298
299}; // namespace WTF
300
301#endif // ENABLE(MEDIA_STREAM)
302