1/*
2 * Copyright (C) 2017 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
15 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
16 * DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY
17 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
18 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
19 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
20 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
21 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
22 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
23 */
24
25#include "config.h"
26#include "CanvasCaptureMediaStreamTrack.h"
27
28#include "GraphicsContext.h"
29#include "HTMLCanvasElement.h"
30#include "WebGLRenderingContextBase.h"
31#include <wtf/IsoMallocInlines.h>
32
33#if ENABLE(MEDIA_STREAM)
34
35namespace WebCore {
36
37WTF_MAKE_ISO_ALLOCATED_IMPL(CanvasCaptureMediaStreamTrack);
38
39Ref<CanvasCaptureMediaStreamTrack> CanvasCaptureMediaStreamTrack::create(ScriptExecutionContext& context, Ref<HTMLCanvasElement>&& canvas, Optional<double>&& frameRequestRate)
40{
41 auto source = CanvasCaptureMediaStreamTrack::Source::create(canvas.get(), WTFMove(frameRequestRate));
42 return adoptRef(*new CanvasCaptureMediaStreamTrack(context, WTFMove(canvas), WTFMove(source)));
43}
44
45CanvasCaptureMediaStreamTrack::CanvasCaptureMediaStreamTrack(ScriptExecutionContext& context, Ref<HTMLCanvasElement>&& canvas, Ref<CanvasCaptureMediaStreamTrack::Source>&& source)
46 : MediaStreamTrack(context, MediaStreamTrackPrivate::create(source.copyRef()))
47 , m_canvas(WTFMove(canvas))
48{
49}
50
51CanvasCaptureMediaStreamTrack::CanvasCaptureMediaStreamTrack(ScriptExecutionContext& context, Ref<HTMLCanvasElement>&& canvas, Ref<MediaStreamTrackPrivate>&& privateTrack)
52 : MediaStreamTrack(context, WTFMove(privateTrack))
53 , m_canvas(WTFMove(canvas))
54{
55}
56
57Ref<CanvasCaptureMediaStreamTrack::Source> CanvasCaptureMediaStreamTrack::Source::create(HTMLCanvasElement& canvas, Optional<double>&& frameRequestRate)
58{
59 auto source = adoptRef(*new Source(canvas, WTFMove(frameRequestRate)));
60 source->start();
61
62 callOnMainThread([source = source.copyRef()] {
63 if (!source->m_canvas)
64 return;
65 source->captureCanvas();
66 });
67 return source;
68}
69
70// FIXME: Give source id and name
71CanvasCaptureMediaStreamTrack::Source::Source(HTMLCanvasElement& canvas, Optional<double>&& frameRequestRate)
72 : RealtimeMediaSource(Type::Video, "CanvasCaptureMediaStreamTrack"_s)
73 , m_frameRequestRate(WTFMove(frameRequestRate))
74 , m_requestFrameTimer(*this, &Source::requestFrameTimerFired)
75 , m_canvasChangedTimer(*this, &Source::captureCanvas)
76 , m_canvas(&canvas)
77{
78}
79
80void CanvasCaptureMediaStreamTrack::Source::startProducingData()
81{
82 if (!m_canvas)
83 return;
84 m_canvas->addObserver(*this);
85
86 if (!m_frameRequestRate)
87 return;
88
89 if (m_frameRequestRate.value())
90 m_requestFrameTimer.startRepeating(1_s / m_frameRequestRate.value());
91}
92
93void CanvasCaptureMediaStreamTrack::Source::stopProducingData()
94{
95 m_requestFrameTimer.stop();
96
97 if (!m_canvas)
98 return;
99 m_canvas->removeObserver(*this);
100}
101
102void CanvasCaptureMediaStreamTrack::Source::requestFrameTimerFired()
103{
104 requestFrame();
105}
106
107void CanvasCaptureMediaStreamTrack::Source::canvasDestroyed(CanvasBase& canvas)
108{
109 ASSERT_UNUSED(canvas, m_canvas == &canvas);
110
111 stop();
112 m_canvas = nullptr;
113}
114
115const RealtimeMediaSourceSettings& CanvasCaptureMediaStreamTrack::Source::settings()
116{
117 if (m_currentSettings)
118 return m_currentSettings.value();
119
120 RealtimeMediaSourceSupportedConstraints constraints;
121 constraints.setSupportsWidth(true);
122 constraints.setSupportsHeight(true);
123
124 RealtimeMediaSourceSettings settings;
125 settings.setWidth(m_canvas->width());
126 settings.setHeight(m_canvas->height());
127 settings.setSupportedConstraints(constraints);
128
129 m_currentSettings = WTFMove(settings);
130 return m_currentSettings.value();
131}
132
133void CanvasCaptureMediaStreamTrack::Source::settingsDidChange(OptionSet<RealtimeMediaSourceSettings::Flag> settings)
134{
135 if (settings.containsAny({ RealtimeMediaSourceSettings::Flag::Width, RealtimeMediaSourceSettings::Flag::Height }))
136 m_currentSettings = WTF::nullopt;
137}
138
139void CanvasCaptureMediaStreamTrack::Source::canvasResized(CanvasBase& canvas)
140{
141 ASSERT_UNUSED(canvas, m_canvas == &canvas);
142 setSize(IntSize(m_canvas->width(), m_canvas->height()));
143}
144
145void CanvasCaptureMediaStreamTrack::Source::canvasChanged(CanvasBase& canvas, const FloatRect&)
146{
147 ASSERT_UNUSED(canvas, m_canvas == &canvas);
148
149#if ENABLE(WEBGL)
150 // FIXME: We need to preserve drawing buffer as we are currently grabbing frames asynchronously.
151 // We should instead add an anchor point for both 2d and 3d contexts where canvas will actually paint.
152 // And call canvas observers from that point.
153 if (is<WebGLRenderingContextBase>(canvas.renderingContext())) {
154 auto& context = downcast<WebGLRenderingContextBase>(*canvas.renderingContext());
155 if (!context.isPreservingDrawingBuffer()) {
156 canvas.scriptExecutionContext()->addConsoleMessage(MessageSource::JS, MessageLevel::Warning, "Turning drawing buffer preservation for the WebGL canvas being captured"_s);
157 context.setPreserveDrawingBuffer(true);
158 }
159 }
160#endif
161
162 // FIXME: We should try to generate the frame at the time the screen is being updated.
163 if (m_canvasChangedTimer.isActive())
164 return;
165 m_canvasChangedTimer.startOneShot(0_s);
166}
167
168void CanvasCaptureMediaStreamTrack::Source::captureCanvas()
169{
170 ASSERT(m_canvas);
171
172 if (!isProducingData())
173 return;
174
175 if (m_frameRequestRate) {
176 if (!m_shouldEmitFrame)
177 return;
178 m_shouldEmitFrame = false;
179 }
180
181 if (!m_canvas->originClean())
182 return;
183
184 auto sample = m_canvas->toMediaSample();
185 if (!sample)
186 return;
187
188 videoSampleAvailable(*sample);
189}
190
191RefPtr<MediaStreamTrack> CanvasCaptureMediaStreamTrack::clone()
192{
193 if (!scriptExecutionContext())
194 return nullptr;
195
196 return adoptRef(*new CanvasCaptureMediaStreamTrack(*scriptExecutionContext(), m_canvas.copyRef(), m_private->clone()));
197}
198
199}
200
201#endif // ENABLE(MEDIA_STREAM)
202