1/*
2 * Copyright (C) 2014 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23 * THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27
28#if ENABLE(VIDEO)
29
30#include "MediaElementSession.h"
31
32#include "Document.h"
33#include "DocumentLoader.h"
34#include "Frame.h"
35#include "FrameView.h"
36#include "FullscreenManager.h"
37#include "HTMLAudioElement.h"
38#include "HTMLMediaElement.h"
39#include "HTMLNames.h"
40#include "HTMLVideoElement.h"
41#include "HitTestResult.h"
42#include "Logging.h"
43#include "Page.h"
44#include "PlatformMediaSessionManager.h"
45#include "Quirks.h"
46#include "RenderMedia.h"
47#include "RenderView.h"
48#include "ScriptController.h"
49#include "Settings.h"
50#include "SourceBuffer.h"
51#include <wtf/text/StringBuilder.h>
52
53#if PLATFORM(IOS_FAMILY)
54#include "AudioSession.h"
55#include "RuntimeApplicationChecks.h"
56#include <wtf/spi/darwin/dyldSPI.h>
57#endif
58
59namespace WebCore {
60
61static const Seconds clientDataBufferingTimerThrottleDelay { 100_ms };
62static const Seconds elementMainContentCheckInterval { 250_ms };
63
64static bool isElementRectMostlyInMainFrame(const HTMLMediaElement&);
65static bool isElementLargeEnoughForMainContent(const HTMLMediaElement&, MediaSessionMainContentPurpose);
66static bool isElementMainContentForPurposesOfAutoplay(const HTMLMediaElement&, bool shouldHitTestMainFrame);
67
68#if !RELEASE_LOG_DISABLED
69static String restrictionNames(MediaElementSession::BehaviorRestrictions restriction)
70{
71 StringBuilder restrictionBuilder;
72#define CASE(restrictionType) \
73 if (restriction & MediaElementSession::restrictionType) { \
74 if (!restrictionBuilder.isEmpty()) \
75 restrictionBuilder.appendLiteral(", "); \
76 restrictionBuilder.append(#restrictionType); \
77 } \
78
79 CASE(NoRestrictions)
80 CASE(RequireUserGestureForLoad)
81 CASE(RequireUserGestureForVideoRateChange)
82 CASE(RequireUserGestureForAudioRateChange)
83 CASE(RequireUserGestureForFullscreen)
84 CASE(RequirePageConsentToLoadMedia)
85 CASE(RequirePageConsentToResumeMedia)
86 CASE(RequireUserGestureToShowPlaybackTargetPicker)
87 CASE(WirelessVideoPlaybackDisabled)
88 CASE(RequireUserGestureToAutoplayToExternalDevice)
89 CASE(AutoPreloadingNotPermitted)
90 CASE(InvisibleAutoplayNotPermitted)
91 CASE(OverrideUserGestureRequirementForMainContent)
92 CASE(RequireUserGestureToControlControlsManager)
93 CASE(RequirePlaybackToControlControlsManager)
94 CASE(RequireUserGestureForVideoDueToLowPowerMode)
95
96 return restrictionBuilder.toString();
97}
98#endif
99
100static bool pageExplicitlyAllowsElementToAutoplayInline(const HTMLMediaElement& element)
101{
102 Document& document = element.document();
103 Page* page = document.page();
104 return document.isMediaDocument() && !document.ownerElement() && page && page->allowsMediaDocumentInlinePlayback();
105}
106
107MediaElementSession::MediaElementSession(HTMLMediaElement& element)
108 : PlatformMediaSession(element)
109 , m_element(element)
110 , m_restrictions(NoRestrictions)
111#if ENABLE(WIRELESS_PLAYBACK_TARGET)
112 , m_targetAvailabilityChangedTimer(*this, &MediaElementSession::targetAvailabilityChangedTimerFired)
113 , m_hasPlaybackTargets(PlatformMediaSessionManager::sharedManager().hasWirelessTargetsAvailable())
114#endif
115 , m_mainContentCheckTimer(*this, &MediaElementSession::mainContentCheckTimerFired)
116 , m_clientDataBufferingTimer(*this, &MediaElementSession::clientDataBufferingTimerFired)
117#if !RELEASE_LOG_DISABLED
118 , m_logIdentifier(element.logIdentifier())
119#endif
120{
121}
122
123void MediaElementSession::registerWithDocument(Document& document)
124{
125#if ENABLE(WIRELESS_PLAYBACK_TARGET)
126 document.addPlaybackTargetPickerClient(*this);
127#else
128 UNUSED_PARAM(document);
129#endif
130}
131
132void MediaElementSession::unregisterWithDocument(Document& document)
133{
134#if ENABLE(WIRELESS_PLAYBACK_TARGET)
135 document.removePlaybackTargetPickerClient(*this);
136#else
137 UNUSED_PARAM(document);
138#endif
139}
140
141void MediaElementSession::clientWillBeginAutoplaying()
142{
143 PlatformMediaSession::clientWillBeginAutoplaying();
144 m_elementIsHiddenBecauseItWasRemovedFromDOM = false;
145 updateClientDataBuffering();
146}
147
148bool MediaElementSession::clientWillBeginPlayback()
149{
150 if (!PlatformMediaSession::clientWillBeginPlayback())
151 return false;
152
153 m_elementIsHiddenBecauseItWasRemovedFromDOM = false;
154 updateClientDataBuffering();
155 return true;
156}
157
158bool MediaElementSession::clientWillPausePlayback()
159{
160 if (!PlatformMediaSession::clientWillPausePlayback())
161 return false;
162
163 updateClientDataBuffering();
164 return true;
165}
166
167void MediaElementSession::visibilityChanged()
168{
169 scheduleClientDataBufferingCheck();
170
171 if (m_element.elementIsHidden() && !m_element.isFullscreen())
172 m_elementIsHiddenUntilVisibleInViewport = true;
173 else if (m_element.isVisibleInViewport())
174 m_elementIsHiddenUntilVisibleInViewport = false;
175}
176
177void MediaElementSession::isVisibleInViewportChanged()
178{
179 scheduleClientDataBufferingCheck();
180
181 if (m_element.isFullscreen() || m_element.isVisibleInViewport())
182 m_elementIsHiddenUntilVisibleInViewport = false;
183}
184
185void MediaElementSession::inActiveDocumentChanged()
186{
187 m_elementIsHiddenBecauseItWasRemovedFromDOM = !m_element.inActiveDocument();
188 scheduleClientDataBufferingCheck();
189}
190
191void MediaElementSession::scheduleClientDataBufferingCheck()
192{
193 if (!m_clientDataBufferingTimer.isActive())
194 m_clientDataBufferingTimer.startOneShot(clientDataBufferingTimerThrottleDelay);
195}
196
197void MediaElementSession::clientDataBufferingTimerFired()
198{
199 INFO_LOG(LOGIDENTIFIER, "visible = ", m_element.elementIsHidden());
200
201 updateClientDataBuffering();
202
203#if PLATFORM(IOS_FAMILY)
204 PlatformMediaSessionManager::sharedManager().configureWireLessTargetMonitoring();
205#endif
206
207 if (state() != Playing || !m_element.elementIsHidden())
208 return;
209
210 PlatformMediaSessionManager::SessionRestrictions restrictions = PlatformMediaSessionManager::sharedManager().restrictions(mediaType());
211 if ((restrictions & PlatformMediaSessionManager::BackgroundTabPlaybackRestricted) == PlatformMediaSessionManager::BackgroundTabPlaybackRestricted)
212 pauseSession();
213}
214
215void MediaElementSession::updateClientDataBuffering()
216{
217 if (m_clientDataBufferingTimer.isActive())
218 m_clientDataBufferingTimer.stop();
219
220 m_element.setBufferingPolicy(preferredBufferingPolicy());
221}
222
223void MediaElementSession::addBehaviorRestriction(BehaviorRestrictions restrictions)
224{
225 if (restrictions & ~m_restrictions)
226 INFO_LOG(LOGIDENTIFIER, "adding ", restrictionNames(restrictions & ~m_restrictions));
227
228 m_restrictions |= restrictions;
229
230 if (restrictions & OverrideUserGestureRequirementForMainContent)
231 m_mainContentCheckTimer.startRepeating(elementMainContentCheckInterval);
232}
233
234void MediaElementSession::removeBehaviorRestriction(BehaviorRestrictions restriction)
235{
236 if (restriction & RequireUserGestureToControlControlsManager) {
237 m_mostRecentUserInteractionTime = MonotonicTime::now();
238 if (auto page = m_element.document().page())
239 page->setAllowsPlaybackControlsForAutoplayingAudio(true);
240 }
241
242 if (!(m_restrictions & restriction))
243 return;
244
245 INFO_LOG(LOGIDENTIFIER, "removed ", restrictionNames(m_restrictions & restriction));
246 m_restrictions &= ~restriction;
247}
248
249SuccessOr<MediaPlaybackDenialReason> MediaElementSession::playbackPermitted() const
250{
251 if (m_element.isSuspended()) {
252 ALWAYS_LOG(LOGIDENTIFIER, "Returning FALSE because element is suspended");
253 return MediaPlaybackDenialReason::InvalidState;
254 }
255
256 auto& document = m_element.document();
257 auto* page = document.page();
258 if (!page || page->mediaPlaybackIsSuspended()) {
259 ALWAYS_LOG(LOGIDENTIFIER, "Returning FALSE because media playback is suspended");
260 return MediaPlaybackDenialReason::PageConsentRequired;
261 }
262
263 if (document.isMediaDocument() && !document.ownerElement())
264 return { };
265
266 if (pageExplicitlyAllowsElementToAutoplayInline(m_element))
267 return { };
268
269 if (requiresFullscreenForVideoPlayback() && !fullscreenPermitted()) {
270 ALWAYS_LOG(LOGIDENTIFIER, "Returning FALSE because of fullscreen restriction");
271 return MediaPlaybackDenialReason::FullscreenRequired;
272 }
273
274 if (m_restrictions & OverrideUserGestureRequirementForMainContent && updateIsMainContent())
275 return { };
276
277#if ENABLE(MEDIA_STREAM)
278 if (m_element.hasMediaStreamSrcObject()) {
279 if (document.isCapturing())
280 return { };
281 if (document.mediaState() & MediaProducer::IsPlayingAudio)
282 return { };
283 }
284#endif
285
286 // FIXME: Why are we checking top-level document only for PerDocumentAutoplayBehavior?
287 const auto& topDocument = document.topDocument();
288 if (topDocument.mediaState() & MediaProducer::HasUserInteractedWithMediaElement && topDocument.quirks().needsPerDocumentAutoplayBehavior())
289 return { };
290
291 if (document.hasHadUserInteraction() && document.quirks().shouldAutoplayForArbitraryUserGesture())
292 return { };
293
294 if (m_restrictions & RequireUserGestureForVideoRateChange && m_element.isVideo() && !document.processingUserGestureForMedia()) {
295 ALWAYS_LOG(LOGIDENTIFIER, "Returning FALSE because a user gesture is required for video rate change restriction");
296 return MediaPlaybackDenialReason::UserGestureRequired;
297 }
298
299 if (m_restrictions & RequireUserGestureForAudioRateChange && (!m_element.isVideo() || m_element.hasAudio()) && !m_element.muted() && m_element.volume() && !document.processingUserGestureForMedia()) {
300 ALWAYS_LOG(LOGIDENTIFIER, "Returning FALSE because a user gesture is required for audio rate change restriction");
301 return MediaPlaybackDenialReason::UserGestureRequired;
302 }
303
304 if (m_restrictions & RequireUserGestureForVideoDueToLowPowerMode && m_element.isVideo() && !document.processingUserGestureForMedia()) {
305 ALWAYS_LOG(LOGIDENTIFIER, "Returning FALSE because of video low power mode restriction");
306 return MediaPlaybackDenialReason::UserGestureRequired;
307 }
308
309 return { };
310}
311
312bool MediaElementSession::autoplayPermitted() const
313{
314 const Document& document = m_element.document();
315 if (document.pageCacheState() != Document::NotInPageCache)
316 return false;
317 if (document.activeDOMObjectsAreSuspended())
318 return false;
319
320 if (!hasBehaviorRestriction(MediaElementSession::InvisibleAutoplayNotPermitted))
321 return true;
322
323 // If the media element is audible, allow autoplay even when not visible as pausing it would be observable by the user.
324 if ((!m_element.isVideo() || m_element.hasAudio()) && !m_element.muted() && m_element.volume())
325 return true;
326
327 auto* renderer = m_element.renderer();
328 if (!renderer) {
329 ALWAYS_LOG(LOGIDENTIFIER, "Returning FALSE because element has no renderer");
330 return false;
331 }
332 if (renderer->style().visibility() != Visibility::Visible) {
333 ALWAYS_LOG(LOGIDENTIFIER, "Returning FALSE because element is not visible");
334 return false;
335 }
336 if (renderer->view().frameView().isOffscreen()) {
337 ALWAYS_LOG(LOGIDENTIFIER, "Returning FALSE because frame is offscreen");
338 return false;
339 }
340 if (renderer->visibleInViewportState() != VisibleInViewportState::Yes) {
341 ALWAYS_LOG(LOGIDENTIFIER, "Returning FALSE because element is not visible in the viewport");
342 return false;
343 }
344 return true;
345}
346
347bool MediaElementSession::dataLoadingPermitted() const
348{
349 if (m_restrictions & OverrideUserGestureRequirementForMainContent && updateIsMainContent())
350 return true;
351
352 if (m_restrictions & RequireUserGestureForLoad && !m_element.document().processingUserGestureForMedia()) {
353 INFO_LOG(LOGIDENTIFIER, "returning FALSE");
354 return false;
355 }
356
357 return true;
358}
359
360MediaPlayer::BufferingPolicy MediaElementSession::preferredBufferingPolicy() const
361{
362 if (isSuspended())
363 return MediaPlayer::BufferingPolicy::MakeResourcesPurgeable;
364
365 if (bufferingSuspended())
366 return MediaPlayer::BufferingPolicy::LimitReadAhead;
367
368 if (state() == PlatformMediaSession::Playing)
369 return MediaPlayer::BufferingPolicy::Default;
370
371 if (shouldOverrideBackgroundLoadingRestriction())
372 return MediaPlayer::BufferingPolicy::Default;
373
374#if ENABLE(WIRELESS_PLAYBACK_TARGET)
375 if (m_shouldPlayToPlaybackTarget)
376 return MediaPlayer::BufferingPolicy::Default;
377#endif
378
379 if (m_elementIsHiddenUntilVisibleInViewport || m_elementIsHiddenBecauseItWasRemovedFromDOM || m_element.elementIsHidden())
380 return MediaPlayer::BufferingPolicy::MakeResourcesPurgeable;
381
382 return MediaPlayer::BufferingPolicy::Default;
383}
384
385bool MediaElementSession::fullscreenPermitted() const
386{
387 if (m_restrictions & RequireUserGestureForFullscreen && !m_element.document().processingUserGestureForMedia()) {
388 INFO_LOG(LOGIDENTIFIER, "returning FALSE");
389 return false;
390 }
391
392 return true;
393}
394
395bool MediaElementSession::pageAllowsDataLoading() const
396{
397 Page* page = m_element.document().page();
398 if (m_restrictions & RequirePageConsentToLoadMedia && page && !page->canStartMedia()) {
399 INFO_LOG(LOGIDENTIFIER, "returning FALSE");
400 return false;
401 }
402
403 return true;
404}
405
406bool MediaElementSession::pageAllowsPlaybackAfterResuming() const
407{
408 Page* page = m_element.document().page();
409 if (m_restrictions & RequirePageConsentToResumeMedia && page && !page->canStartMedia()) {
410 INFO_LOG(LOGIDENTIFIER, "returning FALSE");
411 return false;
412 }
413
414 return true;
415}
416
417bool MediaElementSession::canShowControlsManager(PlaybackControlsPurpose purpose) const
418{
419 if (m_element.isSuspended() || !m_element.inActiveDocument()) {
420 INFO_LOG(LOGIDENTIFIER, "returning FALSE: isSuspended()");
421 return false;
422 }
423
424 if (m_element.isFullscreen()) {
425 INFO_LOG(LOGIDENTIFIER, "returning TRUE: is fullscreen");
426 return true;
427 }
428
429 if (m_element.muted()) {
430 INFO_LOG(LOGIDENTIFIER, "returning FALSE: muted");
431 return false;
432 }
433
434 if (m_element.document().isMediaDocument() && (m_element.document().frame() && m_element.document().frame()->isMainFrame())) {
435 INFO_LOG(LOGIDENTIFIER, "returning TRUE: is media document");
436 return true;
437 }
438
439 if (client().presentationType() == Audio) {
440 if (!hasBehaviorRestriction(RequireUserGestureToControlControlsManager) || m_element.document().processingUserGestureForMedia()) {
441 INFO_LOG(LOGIDENTIFIER, "returning TRUE: audio element with user gesture");
442 return true;
443 }
444
445 if (m_element.isPlaying() && allowsPlaybackControlsForAutoplayingAudio()) {
446 INFO_LOG(LOGIDENTIFIER, "returning TRUE: user has played media before");
447 return true;
448 }
449
450 INFO_LOG(LOGIDENTIFIER, "returning FALSE: audio element is not suitable");
451 return false;
452 }
453
454 if (purpose == PlaybackControlsPurpose::ControlsManager && !isElementRectMostlyInMainFrame(m_element)) {
455 INFO_LOG(LOGIDENTIFIER, "returning FALSE: not in main frame");
456 return false;
457 }
458
459 if (!m_element.hasAudio() && !m_element.hasEverHadAudio()) {
460 INFO_LOG(LOGIDENTIFIER, "returning FALSE: no audio");
461 return false;
462 }
463
464 if (!playbackPermitted()) {
465 INFO_LOG(LOGIDENTIFIER, "returning FALSE: playback not permitted");
466 return false;
467 }
468
469 if (!hasBehaviorRestriction(RequireUserGestureToControlControlsManager) || m_element.document().processingUserGestureForMedia()) {
470 INFO_LOG(LOGIDENTIFIER, "returning TRUE: no user gesture required");
471 return true;
472 }
473
474 if (purpose == PlaybackControlsPurpose::ControlsManager && hasBehaviorRestriction(RequirePlaybackToControlControlsManager) && !m_element.isPlaying()) {
475 INFO_LOG(LOGIDENTIFIER, "returning FALSE: needs to be playing");
476 return false;
477 }
478
479 if (!m_element.hasEverNotifiedAboutPlaying()) {
480 INFO_LOG(LOGIDENTIFIER, "returning FALSE: hasn't fired playing notification");
481 return false;
482 }
483
484#if ENABLE(FULLSCREEN_API)
485 // Elements which are not descendents of the current fullscreen element cannot be main content.
486 auto* fullscreenElement = m_element.document().fullscreenManager().currentFullscreenElement();
487 if (fullscreenElement && !m_element.isDescendantOf(*fullscreenElement)) {
488 INFO_LOG(LOGIDENTIFIER, "returning FALSE: outside of full screen");
489 return false;
490 }
491#endif
492
493 // Only allow the main content heuristic to forbid videos from showing up if our purpose is the controls manager.
494 if (purpose == PlaybackControlsPurpose::ControlsManager && m_element.isVideo()) {
495 if (!m_element.renderer()) {
496 INFO_LOG(LOGIDENTIFIER, "returning FALSE: no renderer");
497 return false;
498 }
499
500 if (!m_element.hasVideo() && !m_element.hasEverHadVideo()) {
501 INFO_LOG(LOGIDENTIFIER, "returning FALSE: no video");
502 return false;
503 }
504
505 if (isLargeEnoughForMainContent(MediaSessionMainContentPurpose::MediaControls)) {
506 INFO_LOG(LOGIDENTIFIER, "returning TRUE: is main content");
507 return true;
508 }
509 }
510
511 if (purpose == PlaybackControlsPurpose::NowPlaying) {
512 INFO_LOG(LOGIDENTIFIER, "returning TRUE: potentially plays audio");
513 return true;
514 }
515
516 INFO_LOG(LOGIDENTIFIER, "returning FALSE: no user gesture");
517 return false;
518}
519
520bool MediaElementSession::isLargeEnoughForMainContent(MediaSessionMainContentPurpose purpose) const
521{
522 return isElementLargeEnoughForMainContent(m_element, purpose);
523}
524
525bool MediaElementSession::isMainContentForPurposesOfAutoplayEvents() const
526{
527 return isElementMainContentForPurposesOfAutoplay(m_element, false);
528}
529
530MonotonicTime MediaElementSession::mostRecentUserInteractionTime() const
531{
532 return m_mostRecentUserInteractionTime;
533}
534
535bool MediaElementSession::wantsToObserveViewportVisibilityForMediaControls() const
536{
537 return isLargeEnoughForMainContent(MediaSessionMainContentPurpose::MediaControls);
538}
539
540bool MediaElementSession::wantsToObserveViewportVisibilityForAutoplay() const
541{
542 return m_element.isVideo();
543}
544
545#if ENABLE(WIRELESS_PLAYBACK_TARGET)
546void MediaElementSession::showPlaybackTargetPicker()
547{
548 INFO_LOG(LOGIDENTIFIER);
549
550 auto& document = m_element.document();
551 if (m_restrictions & RequireUserGestureToShowPlaybackTargetPicker && !document.processingUserGestureForMedia()) {
552 INFO_LOG(LOGIDENTIFIER, "returning early because of permissions");
553 return;
554 }
555
556 if (!document.page()) {
557 INFO_LOG(LOGIDENTIFIER, "returning early because page is NULL");
558 return;
559 }
560
561#if !PLATFORM(IOS_FAMILY)
562 if (m_element.readyState() < HTMLMediaElementEnums::HAVE_METADATA) {
563 INFO_LOG(LOGIDENTIFIER, "returning early because element is not playable");
564 return;
565 }
566#endif
567
568 auto& audioSession = AudioSession::sharedSession();
569 document.showPlaybackTargetPicker(*this, is<HTMLVideoElement>(m_element), audioSession.routeSharingPolicy(), audioSession.routingContextUID());
570}
571
572bool MediaElementSession::hasWirelessPlaybackTargets() const
573{
574 INFO_LOG(LOGIDENTIFIER, "returning ", m_hasPlaybackTargets);
575
576 return m_hasPlaybackTargets;
577}
578
579bool MediaElementSession::wirelessVideoPlaybackDisabled() const
580{
581 if (!m_element.document().settings().allowsAirPlayForMediaPlayback()) {
582 INFO_LOG(LOGIDENTIFIER, "returning TRUE because of settings");
583 return true;
584 }
585
586 if (m_element.hasAttributeWithoutSynchronization(HTMLNames::webkitwirelessvideoplaybackdisabledAttr)) {
587 INFO_LOG(LOGIDENTIFIER, "returning TRUE because of attribute");
588 return true;
589 }
590
591#if PLATFORM(IOS_FAMILY)
592 auto& legacyAirplayAttributeValue = m_element.attributeWithoutSynchronization(HTMLNames::webkitairplayAttr);
593 if (equalLettersIgnoringASCIICase(legacyAirplayAttributeValue, "deny")) {
594 INFO_LOG(LOGIDENTIFIER, "returning TRUE because of legacy attribute");
595 return true;
596 }
597 if (equalLettersIgnoringASCIICase(legacyAirplayAttributeValue, "allow")) {
598 INFO_LOG(LOGIDENTIFIER, "returning FALSE because of legacy attribute");
599 return false;
600 }
601#endif
602
603 auto player = m_element.player();
604 if (!player)
605 return true;
606
607 bool disabled = player->wirelessVideoPlaybackDisabled();
608 INFO_LOG(LOGIDENTIFIER, "returning ", disabled, " because media engine says so");
609
610 return disabled;
611}
612
613void MediaElementSession::setWirelessVideoPlaybackDisabled(bool disabled)
614{
615 if (disabled)
616 addBehaviorRestriction(WirelessVideoPlaybackDisabled);
617 else
618 removeBehaviorRestriction(WirelessVideoPlaybackDisabled);
619
620 auto player = m_element.player();
621 if (!player)
622 return;
623
624 INFO_LOG(LOGIDENTIFIER, disabled);
625 player->setWirelessVideoPlaybackDisabled(disabled);
626}
627
628void MediaElementSession::setHasPlaybackTargetAvailabilityListeners(bool hasListeners)
629{
630 INFO_LOG(LOGIDENTIFIER, hasListeners);
631
632#if PLATFORM(IOS_FAMILY)
633 m_hasPlaybackTargetAvailabilityListeners = hasListeners;
634 PlatformMediaSessionManager::sharedManager().configureWireLessTargetMonitoring();
635#else
636 UNUSED_PARAM(hasListeners);
637 m_element.document().playbackTargetPickerClientStateDidChange(*this, m_element.mediaState());
638#endif
639}
640
641void MediaElementSession::setPlaybackTarget(Ref<MediaPlaybackTarget>&& device)
642{
643 m_playbackTarget = WTFMove(device);
644 client().setWirelessPlaybackTarget(*m_playbackTarget.copyRef());
645}
646
647void MediaElementSession::targetAvailabilityChangedTimerFired()
648{
649 client().wirelessRoutesAvailableDidChange();
650}
651
652void MediaElementSession::externalOutputDeviceAvailableDidChange(bool hasTargets)
653{
654 if (m_hasPlaybackTargets == hasTargets)
655 return;
656
657 INFO_LOG(LOGIDENTIFIER, hasTargets);
658
659 m_hasPlaybackTargets = hasTargets;
660 m_targetAvailabilityChangedTimer.startOneShot(0_s);
661}
662
663bool MediaElementSession::isPlayingToWirelessPlaybackTarget() const
664{
665#if !PLATFORM(IOS_FAMILY)
666 if (!m_playbackTarget || !m_playbackTarget->hasActiveRoute())
667 return false;
668#endif
669
670 return client().isPlayingToWirelessPlaybackTarget();
671}
672
673void MediaElementSession::setShouldPlayToPlaybackTarget(bool shouldPlay)
674{
675 INFO_LOG(LOGIDENTIFIER, shouldPlay);
676 m_shouldPlayToPlaybackTarget = shouldPlay;
677 updateClientDataBuffering();
678 client().setShouldPlayToPlaybackTarget(shouldPlay);
679}
680
681void MediaElementSession::mediaStateDidChange(MediaProducer::MediaStateFlags state)
682{
683 m_element.document().playbackTargetPickerClientStateDidChange(*this, state);
684}
685#endif
686
687MediaPlayer::Preload MediaElementSession::effectivePreloadForElement() const
688{
689 MediaPlayer::Preload preload = m_element.preloadValue();
690
691 if (pageExplicitlyAllowsElementToAutoplayInline(m_element))
692 return preload;
693
694 if (m_restrictions & AutoPreloadingNotPermitted) {
695 if (preload > MediaPlayer::MetaData)
696 return MediaPlayer::MetaData;
697 }
698
699 return preload;
700}
701
702bool MediaElementSession::requiresFullscreenForVideoPlayback() const
703{
704 if (pageExplicitlyAllowsElementToAutoplayInline(m_element))
705 return false;
706
707 if (is<HTMLAudioElement>(m_element))
708 return false;
709
710 if (m_element.document().isMediaDocument()) {
711 ASSERT(is<HTMLVideoElement>(m_element));
712 const HTMLVideoElement& videoElement = *downcast<const HTMLVideoElement>(&m_element);
713 if (m_element.readyState() < HTMLVideoElement::HAVE_METADATA || !videoElement.hasEverHadVideo())
714 return false;
715 }
716
717 if (m_element.isTemporarilyAllowingInlinePlaybackAfterFullscreen())
718 return false;
719
720 if (!m_element.document().settings().allowsInlineMediaPlayback())
721 return true;
722
723 if (!m_element.document().settings().inlineMediaPlaybackRequiresPlaysInlineAttribute())
724 return false;
725
726#if PLATFORM(IOS_FAMILY)
727 if (IOSApplication::isIBooks())
728 return !m_element.hasAttributeWithoutSynchronization(HTMLNames::webkit_playsinlineAttr) && !m_element.hasAttributeWithoutSynchronization(HTMLNames::playsinlineAttr);
729 if (dyld_get_program_sdk_version() < DYLD_IOS_VERSION_10_0)
730 return !m_element.hasAttributeWithoutSynchronization(HTMLNames::webkit_playsinlineAttr);
731#endif
732
733 if (m_element.document().isMediaDocument() && m_element.document().ownerElement())
734 return false;
735
736 return !m_element.hasAttributeWithoutSynchronization(HTMLNames::playsinlineAttr);
737}
738
739bool MediaElementSession::allowsAutomaticMediaDataLoading() const
740{
741 if (pageExplicitlyAllowsElementToAutoplayInline(m_element))
742 return true;
743
744 if (m_element.document().settings().mediaDataLoadsAutomatically())
745 return true;
746
747 return false;
748}
749
750void MediaElementSession::mediaEngineUpdated()
751{
752 INFO_LOG(LOGIDENTIFIER);
753
754#if ENABLE(WIRELESS_PLAYBACK_TARGET)
755 if (m_restrictions & WirelessVideoPlaybackDisabled)
756 setWirelessVideoPlaybackDisabled(true);
757 if (m_playbackTarget)
758 client().setWirelessPlaybackTarget(*m_playbackTarget.copyRef());
759 if (m_shouldPlayToPlaybackTarget)
760 client().setShouldPlayToPlaybackTarget(true);
761#endif
762
763}
764
765void MediaElementSession::resetPlaybackSessionState()
766{
767 m_mostRecentUserInteractionTime = MonotonicTime();
768 addBehaviorRestriction(RequireUserGestureToControlControlsManager | RequirePlaybackToControlControlsManager);
769}
770
771void MediaElementSession::suspendBuffering()
772{
773 updateClientDataBuffering();
774}
775
776void MediaElementSession::resumeBuffering()
777{
778 updateClientDataBuffering();
779}
780
781bool MediaElementSession::bufferingSuspended() const
782{
783 if (auto* page = m_element.document().page())
784 return page->mediaBufferingIsSuspended();
785 return true;
786}
787
788bool MediaElementSession::allowsPictureInPicture() const
789{
790 return m_element.document().settings().allowsPictureInPictureMediaPlayback();
791}
792
793#if PLATFORM(IOS_FAMILY)
794bool MediaElementSession::requiresPlaybackTargetRouteMonitoring() const
795{
796 return m_hasPlaybackTargetAvailabilityListeners && !m_element.elementIsHidden();
797}
798#endif
799
800#if ENABLE(MEDIA_SOURCE)
801size_t MediaElementSession::maximumMediaSourceBufferSize(const SourceBuffer& buffer) const
802{
803 // A good quality 1080p video uses 8,000 kbps and stereo audio uses 384 kbps, so assume 95% for video and 5% for audio.
804 const float bufferBudgetPercentageForVideo = .95;
805 const float bufferBudgetPercentageForAudio = .05;
806
807 size_t maximum = buffer.document().settings().maximumSourceBufferSize();
808
809 // Allow a SourceBuffer to buffer as though it is audio-only even if it doesn't have any active tracks (yet).
810 size_t bufferSize = static_cast<size_t>(maximum * bufferBudgetPercentageForAudio);
811 if (buffer.hasVideo())
812 bufferSize += static_cast<size_t>(maximum * bufferBudgetPercentageForVideo);
813
814 // FIXME: we might want to modify this algorithm to:
815 // - decrease the maximum size for background tabs
816 // - decrease the maximum size allowed for inactive elements when a process has more than one
817 // element, eg. so a page with many elements which are played one at a time doesn't keep
818 // everything buffered after an element has finished playing.
819
820 return bufferSize;
821}
822#endif
823
824static bool isElementMainContentForPurposesOfAutoplay(const HTMLMediaElement& element, bool shouldHitTestMainFrame)
825{
826 Document& document = element.document();
827 if (!document.hasLivingRenderTree() || document.activeDOMObjectsAreStopped() || element.isSuspended() || !element.hasAudio() || !element.hasVideo())
828 return false;
829
830 // Elements which have not yet been laid out, or which are not yet in the DOM, cannot be main content.
831 auto* renderer = element.renderer();
832 if (!renderer)
833 return false;
834
835 if (!isElementLargeEnoughForMainContent(element, MediaSessionMainContentPurpose::Autoplay))
836 return false;
837
838 // Elements which are hidden by style, or have been scrolled out of view, cannot be main content.
839 // But elements which have audio & video and are already playing should not stop playing because
840 // they are scrolled off the page.
841 if (renderer->style().visibility() != Visibility::Visible)
842 return false;
843 if (renderer->visibleInViewportState() != VisibleInViewportState::Yes && !element.isPlaying())
844 return false;
845
846 // Main content elements must be in the main frame.
847 if (!document.frame() || !document.frame()->isMainFrame())
848 return false;
849
850 auto& mainFrame = document.frame()->mainFrame();
851 if (!mainFrame.view() || !mainFrame.view()->renderView())
852 return false;
853
854 if (!shouldHitTestMainFrame)
855 return true;
856
857 RenderView& mainRenderView = *mainFrame.view()->renderView();
858
859 // Hit test the area of the main frame where the element appears, to determine if the element is being obscured.
860 IntRect rectRelativeToView = element.clientRect();
861 ScrollPosition scrollPosition = mainFrame.view()->documentScrollPositionRelativeToViewOrigin();
862 IntRect rectRelativeToTopDocument(rectRelativeToView.location() + scrollPosition, rectRelativeToView.size());
863 HitTestRequest request(HitTestRequest::ReadOnly | HitTestRequest::Active | HitTestRequest::AllowChildFrameContent | HitTestRequest::IgnoreClipping | HitTestRequest::DisallowUserAgentShadowContent);
864 HitTestResult result(rectRelativeToTopDocument.center());
865
866 // Elements which are obscured by other elements cannot be main content.
867 mainRenderView.hitTest(request, result);
868 result.setToNonUserAgentShadowAncestor();
869 RefPtr<Element> hitElement = result.targetElement();
870 if (hitElement != &element)
871 return false;
872
873 return true;
874}
875
876static bool isElementRectMostlyInMainFrame(const HTMLMediaElement& element)
877{
878 if (!element.renderer())
879 return false;
880
881 auto documentFrame = makeRefPtr(element.document().frame());
882 if (!documentFrame)
883 return false;
884
885 auto mainFrameView = documentFrame->mainFrame().view();
886 if (!mainFrameView)
887 return false;
888
889 IntRect mainFrameRectAdjustedForScrollPosition = IntRect(-mainFrameView->documentScrollPositionRelativeToViewOrigin(), mainFrameView->contentsSize());
890 IntRect elementRectInMainFrame = element.clientRect();
891 auto totalElementArea = elementRectInMainFrame.area<RecordOverflow>();
892 if (totalElementArea.hasOverflowed())
893 return false;
894
895 elementRectInMainFrame.intersect(mainFrameRectAdjustedForScrollPosition);
896
897 return elementRectInMainFrame.area().unsafeGet() > totalElementArea.unsafeGet() / 2;
898}
899
900static bool isElementLargeRelativeToMainFrame(const HTMLMediaElement& element)
901{
902 static const double minimumPercentageOfMainFrameAreaForMainContent = 0.9;
903 auto* renderer = element.renderer();
904 if (!renderer)
905 return false;
906
907 auto documentFrame = makeRefPtr(element.document().frame());
908 if (!documentFrame)
909 return false;
910
911 if (!documentFrame->mainFrame().view())
912 return false;
913
914 auto& mainFrameView = *documentFrame->mainFrame().view();
915 auto maxVisibleClientWidth = std::min(renderer->clientWidth().toInt(), mainFrameView.visibleWidth());
916 auto maxVisibleClientHeight = std::min(renderer->clientHeight().toInt(), mainFrameView.visibleHeight());
917
918 return maxVisibleClientWidth * maxVisibleClientHeight > minimumPercentageOfMainFrameAreaForMainContent * mainFrameView.visibleWidth() * mainFrameView.visibleHeight();
919}
920
921static bool isElementLargeEnoughForMainContent(const HTMLMediaElement& element, MediaSessionMainContentPurpose purpose)
922{
923 static const double elementMainContentAreaMinimum = 400 * 300;
924 static const double maximumAspectRatio = purpose == MediaSessionMainContentPurpose::MediaControls ? 3 : 1.8;
925 static const double minimumAspectRatio = .5; // Slightly smaller than 9:16.
926
927 // Elements which have not yet been laid out, or which are not yet in the DOM, cannot be main content.
928 auto* renderer = element.renderer();
929 if (!renderer)
930 return false;
931
932 double width = renderer->clientWidth();
933 double height = renderer->clientHeight();
934 double area = width * height;
935 double aspectRatio = width / height;
936
937 if (area < elementMainContentAreaMinimum)
938 return false;
939
940 if (aspectRatio >= minimumAspectRatio && aspectRatio <= maximumAspectRatio)
941 return true;
942
943 return isElementLargeRelativeToMainFrame(element);
944}
945
946void MediaElementSession::mainContentCheckTimerFired()
947{
948 if (!hasBehaviorRestriction(OverrideUserGestureRequirementForMainContent))
949 return;
950
951 updateIsMainContent();
952}
953
954bool MediaElementSession::updateIsMainContent() const
955{
956 if (m_element.isSuspended())
957 return false;
958
959 bool wasMainContent = m_isMainContent;
960 m_isMainContent = isElementMainContentForPurposesOfAutoplay(m_element, true);
961
962 if (m_isMainContent != wasMainContent)
963 m_element.updateShouldPlay();
964
965 return m_isMainContent;
966}
967
968bool MediaElementSession::allowsNowPlayingControlsVisibility() const
969{
970 auto page = m_element.document().page();
971 return page && !page->isVisibleAndActive();
972}
973
974bool MediaElementSession::allowsPlaybackControlsForAutoplayingAudio() const
975{
976 auto page = m_element.document().page();
977 return page && page->allowsPlaybackControlsForAutoplayingAudio();
978}
979
980String convertEnumerationToString(const MediaPlaybackDenialReason enumerationValue)
981{
982 static const NeverDestroyed<String> values[] = {
983 MAKE_STATIC_STRING_IMPL("UserGestureRequired"),
984 MAKE_STATIC_STRING_IMPL("FullscreenRequired"),
985 MAKE_STATIC_STRING_IMPL("PageConsentRequired"),
986 MAKE_STATIC_STRING_IMPL("InvalidState"),
987 };
988 static_assert(static_cast<size_t>(MediaPlaybackDenialReason::UserGestureRequired) == 0, "MediaPlaybackDenialReason::UserGestureRequired is not 0 as expected");
989 static_assert(static_cast<size_t>(MediaPlaybackDenialReason::FullscreenRequired) == 1, "MediaPlaybackDenialReason::FullscreenRequired is not 1 as expected");
990 static_assert(static_cast<size_t>(MediaPlaybackDenialReason::PageConsentRequired) == 2, "MediaPlaybackDenialReason::PageConsentRequired is not 2 as expected");
991 static_assert(static_cast<size_t>(MediaPlaybackDenialReason::InvalidState) == 3, "MediaPlaybackDenialReason::InvalidState is not 3 as expected");
992 ASSERT(static_cast<size_t>(enumerationValue) < WTF_ARRAY_LENGTH(values));
993 return values[static_cast<size_t>(enumerationValue)];
994}
995
996}
997
998#endif // ENABLE(VIDEO)
999