1/*
2 * Copyright (C) 2011, 2013 Google Inc. All rights reserved.
3 * Copyright (C) 2011-2017 Apple Inc. All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are
7 * met:
8 *
9 * * Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * * Redistributions in binary form must reproduce the above
12 * copyright notice, this list of conditions and the following disclaimer
13 * in the documentation and/or other materials provided with the
14 * distribution.
15 * * Neither the name of Google Inc. nor the names of its
16 * contributors may be used to endorse or promote products derived from
17 * this software without specific prior written permission.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 */
31
32#include "config.h"
33#include "TextTrack.h"
34
35#if ENABLE(VIDEO_TRACK)
36
37#include "Document.h"
38#include "Event.h"
39#include "HTMLMediaElement.h"
40#include "SourceBuffer.h"
41#include "TextTrackCueList.h"
42#include "TextTrackList.h"
43#include "VTTRegion.h"
44#include "VTTRegionList.h"
45#include <wtf/IsoMallocInlines.h>
46#include <wtf/NeverDestroyed.h>
47
48namespace WebCore {
49
50WTF_MAKE_ISO_ALLOCATED_IMPL(TextTrack);
51
52const AtomicString& TextTrack::subtitlesKeyword()
53{
54 static NeverDestroyed<const AtomicString> subtitles("subtitles", AtomicString::ConstructFromLiteral);
55 return subtitles;
56}
57
58static const AtomicString& captionsKeyword()
59{
60 static NeverDestroyed<const AtomicString> captions("captions", AtomicString::ConstructFromLiteral);
61 return captions;
62}
63
64static const AtomicString& descriptionsKeyword()
65{
66 static NeverDestroyed<const AtomicString> descriptions("descriptions", AtomicString::ConstructFromLiteral);
67 return descriptions;
68}
69
70static const AtomicString& chaptersKeyword()
71{
72 static NeverDestroyed<const AtomicString> chapters("chapters", AtomicString::ConstructFromLiteral);
73 return chapters;
74}
75
76static const AtomicString& metadataKeyword()
77{
78 static NeverDestroyed<const AtomicString> metadata("metadata", AtomicString::ConstructFromLiteral);
79 return metadata;
80}
81
82static const AtomicString& forcedKeyword()
83{
84 static NeverDestroyed<const AtomicString> forced("forced", AtomicString::ConstructFromLiteral);
85 return forced;
86}
87
88TextTrack* TextTrack::captionMenuOffItem()
89{
90 static TextTrack& off = TextTrack::create(nullptr, nullptr, "off menu item", emptyAtom(), emptyAtom(), emptyAtom()).leakRef();
91 return &off;
92}
93
94TextTrack* TextTrack::captionMenuAutomaticItem()
95{
96 static TextTrack& automatic = TextTrack::create(nullptr, nullptr, "automatic menu item", emptyAtom(), emptyAtom(), emptyAtom()).leakRef();
97 return &automatic;
98}
99
100TextTrack::TextTrack(ScriptExecutionContext* context, TextTrackClient* client, const AtomicString& kind, const AtomicString& id, const AtomicString& label, const AtomicString& language, TextTrackType type)
101 : TrackBase(TrackBase::TextTrack, id, label, language)
102 , ContextDestructionObserver(context)
103 , m_client(client)
104 , m_trackType(type)
105{
106 if (kind == captionsKeyword())
107 m_kind = Kind::Captions;
108 else if (kind == chaptersKeyword())
109 m_kind = Kind::Chapters;
110 else if (kind == descriptionsKeyword())
111 m_kind = Kind::Descriptions;
112 else if (kind == forcedKeyword())
113 m_kind = Kind::Forced;
114 else if (kind == metadataKeyword())
115 m_kind = Kind::Metadata;
116}
117
118TextTrack::~TextTrack()
119{
120 if (m_cues) {
121 if (m_client)
122 m_client->textTrackRemoveCues(*this, *m_cues);
123 for (size_t i = 0; i < m_cues->length(); ++i)
124 m_cues->item(i)->setTrack(nullptr);
125 }
126 if (m_regions) {
127 for (size_t i = 0; i < m_regions->length(); ++i)
128 m_regions->item(i)->setTrack(nullptr);
129 }
130}
131
132bool TextTrack::enabled() const
133{
134 return m_mode != Mode::Disabled;
135}
136
137bool TextTrack::isValidKindKeyword(const AtomicString& value)
138{
139 if (value == subtitlesKeyword())
140 return true;
141 if (value == captionsKeyword())
142 return true;
143 if (value == descriptionsKeyword())
144 return true;
145 if (value == chaptersKeyword())
146 return true;
147 if (value == metadataKeyword())
148 return true;
149 if (value == forcedKeyword())
150 return true;
151
152 return false;
153}
154
155const AtomicString& TextTrack::kindKeyword() const
156{
157 switch (m_kind) {
158 case Kind::Captions:
159 return captionsKeyword();
160 case Kind::Chapters:
161 return chaptersKeyword();
162 case Kind::Descriptions:
163 return descriptionsKeyword();
164 case Kind::Forced:
165 return forcedKeyword();
166 case Kind::Metadata:
167 return metadataKeyword();
168 case Kind::Subtitles:
169 return subtitlesKeyword();
170 }
171 ASSERT_NOT_REACHED();
172 return subtitlesKeyword();
173}
174
175void TextTrack::setKind(Kind newKind)
176{
177 auto oldKind = m_kind;
178
179 // 10.1 kind, on setting:
180 // 1. If the value being assigned to this attribute does not match one of the text track kinds,
181 // then abort these steps.
182
183 // 2. Update this attribute to the new value.
184 m_kind = newKind;
185
186#if ENABLE(MEDIA_SOURCE)
187 // 3. If the sourceBuffer attribute on this track is not null, then queue a task to fire a simple
188 // event named change at sourceBuffer.textTracks.
189 if (m_sourceBuffer)
190 m_sourceBuffer->textTracks().scheduleChangeEvent();
191
192 // 4. Queue a task to fire a simple event named change at the TextTrackList object referenced by
193 // the textTracks attribute on the HTMLMediaElement.
194 if (mediaElement())
195 mediaElement()->ensureTextTracks().scheduleChangeEvent();
196#endif
197
198 if (m_client && oldKind != m_kind)
199 m_client->textTrackKindChanged(*this);
200}
201
202void TextTrack::setKindKeywordIgnoringASCIICase(StringView keyword)
203{
204 if (keyword.isNull()) {
205 // The missing value default is the subtitles state.
206 setKind(Kind::Subtitles);
207 return;
208 }
209 if (equalLettersIgnoringASCIICase(keyword, "captions"))
210 setKind(Kind::Captions);
211 else if (equalLettersIgnoringASCIICase(keyword, "chapters"))
212 setKind(Kind::Chapters);
213 else if (equalLettersIgnoringASCIICase(keyword, "descriptions"))
214 setKind(Kind::Descriptions);
215 else if (equalLettersIgnoringASCIICase(keyword, "forced"))
216 setKind(Kind::Forced);
217 else if (equalLettersIgnoringASCIICase(keyword, "metadata"))
218 setKind(Kind::Metadata);
219 else if (equalLettersIgnoringASCIICase(keyword, "subtitles"))
220 setKind(Kind::Subtitles);
221 else {
222 // The invalid value default is the metadata state.
223 setKind(Kind::Metadata);
224 }
225}
226
227void TextTrack::setMode(Mode mode)
228{
229 // On setting, if the new value isn't equal to what the attribute would currently
230 // return, the new value must be processed as follows ...
231 if (m_mode == mode)
232 return;
233
234 // If mode changes to disabled, remove this track's cues from the client
235 // because they will no longer be accessible from the cues() function.
236 if (mode == Mode::Disabled && m_client && m_cues)
237 m_client->textTrackRemoveCues(*this, *m_cues);
238
239 if (mode != Mode::Showing && m_cues) {
240 for (size_t i = 0; i < m_cues->length(); ++i) {
241 RefPtr<TextTrackCue> cue = m_cues->item(i);
242 if (cue->isRenderable())
243 toVTTCue(cue.get())->removeDisplayTree();
244 }
245 }
246
247 m_mode = mode;
248
249 if (m_client)
250 m_client->textTrackModeChanged(*this);
251}
252
253TextTrackCueList* TextTrack::cues()
254{
255 // 4.8.10.12.5 If the text track mode ... is not the text track disabled mode,
256 // then the cues attribute must return a live TextTrackCueList object ...
257 // Otherwise, it must return null. When an object is returned, the
258 // same object must be returned each time.
259 // http://www.whatwg.org/specs/web-apps/current-work/#dom-texttrack-cues
260 if (m_mode == Mode::Disabled)
261 return nullptr;
262 return &ensureTextTrackCueList();
263}
264
265void TextTrack::removeAllCues()
266{
267 if (!m_cues)
268 return;
269
270 if (m_client)
271 m_client->textTrackRemoveCues(*this, *m_cues);
272
273 for (size_t i = 0; i < m_cues->length(); ++i)
274 m_cues->item(i)->setTrack(nullptr);
275
276 m_cues->clear();
277}
278
279TextTrackCueList* TextTrack::activeCues() const
280{
281 // 4.8.10.12.5 If the text track mode ... is not the text track disabled mode,
282 // then the activeCues attribute must return a live TextTrackCueList object ...
283 // ... whose active flag was set when the script started, in text track cue
284 // order. Otherwise, it must return null. When an object is returned, the
285 // same object must be returned each time.
286 // http://www.whatwg.org/specs/web-apps/current-work/#dom-texttrack-activecues
287 if (!m_cues || m_mode == Mode::Disabled)
288 return nullptr;
289 return &m_cues->activeCues();
290}
291
292ExceptionOr<void> TextTrack::addCue(Ref<TextTrackCue>&& cue)
293{
294 // 4.7.10.12.6 Text tracks exposing in-band metadata
295 // The UA will use DataCue to expose only text track cue objects that belong to a text track that has a text
296 // track kind of metadata.
297 // If a DataCue is added to a TextTrack via the addCue() method but the text track does not have its text
298 // track kind set to metadata, throw a InvalidNodeTypeError exception and don't add the cue to the TextTrackList
299 // of the TextTrack.
300 if (cue->cueType() == TextTrackCue::Data && m_kind != Kind::Metadata)
301 return Exception { InvalidNodeTypeError };
302
303 // TODO(93143): Add spec-compliant behavior for negative time values.
304 if (!cue->startMediaTime().isValid() || !cue->endMediaTime().isValid() || cue->startMediaTime() < MediaTime::zeroTime() || cue->endMediaTime() < MediaTime::zeroTime())
305 return { };
306
307 // 4.8.10.12.5 Text track API
308
309 // The addCue(cue) method of TextTrack objects, when invoked, must run the following steps:
310
311 auto cueTrack = makeRefPtr(cue->track());
312 if (cueTrack == this)
313 return { };
314
315 // 1. If the given cue is in a text track list of cues, then remove cue from that text track
316 // list of cues.
317 if (cueTrack)
318 cueTrack->removeCue(cue.get());
319
320 // 2. Add cue to the method's TextTrack object's text track's text track list of cues.
321 cue->setTrack(this);
322 ensureTextTrackCueList().add(cue.copyRef());
323
324 if (m_client)
325 m_client->textTrackAddCue(*this, cue);
326
327 return { };
328}
329
330ExceptionOr<void> TextTrack::removeCue(TextTrackCue& cue)
331{
332 // 4.8.10.12.5 Text track API
333
334 // The removeCue(cue) method of TextTrack objects, when invoked, must run the following steps:
335
336 // 1. If the given cue is not currently listed in the method's TextTrack
337 // object's text track's text track list of cues, then throw a NotFoundError exception.
338 if (cue.track() != this)
339 return Exception { NotFoundError };
340 if (!m_cues)
341 return Exception { InvalidStateError };
342
343 INFO_LOG(LOGIDENTIFIER, cue);
344
345 // 2. Remove cue from the method's TextTrack object's text track's text track list of cues.
346 m_cues->remove(cue);
347 cue.setIsActive(false);
348 cue.setTrack(nullptr);
349 if (m_client)
350 m_client->textTrackRemoveCue(*this, cue);
351
352 return { };
353}
354
355VTTRegionList& TextTrack::ensureVTTRegionList()
356{
357 if (!m_regions)
358 m_regions = VTTRegionList::create();
359
360 return *m_regions;
361}
362
363VTTRegionList* TextTrack::regions()
364{
365 // If the text track mode of the text track that the TextTrack object
366 // represents is not the text track disabled mode, then the regions
367 // attribute must return a live VTTRegionList object that represents
368 // the text track list of regions of the text track. Otherwise, it must
369 // return null. When an object is returned, the same object must be returned
370 // each time.
371 if (m_mode == Mode::Disabled)
372 return nullptr;
373 return &ensureVTTRegionList();
374}
375
376void TextTrack::addRegion(RefPtr<VTTRegion>&& region)
377{
378 if (!region)
379 return;
380
381 auto& regionList = ensureVTTRegionList();
382
383 // 1. If the given region is in a text track list of regions, then remove
384 // region from that text track list of regions.
385 auto regionTrack = makeRefPtr(region->track());
386 if (regionTrack && regionTrack != this)
387 regionTrack->removeRegion(region.get());
388
389 // 2. If the method's TextTrack object's text track list of regions contains
390 // a region with the same identifier as region replace the values of that
391 // region's width, height, anchor point, viewport anchor point and scroll
392 // attributes with those of region.
393 auto existingRegion = makeRefPtr(regionList.getRegionById(region->id()));
394 if (existingRegion) {
395 existingRegion->updateParametersFromRegion(*region);
396 return;
397 }
398
399 // Otherwise: add region to the method's TextTrack object's text track list of regions.
400 region->setTrack(this);
401 regionList.add(region.releaseNonNull());
402}
403
404ExceptionOr<void> TextTrack::removeRegion(VTTRegion* region)
405{
406 if (!region)
407 return { };
408
409 // 1. If the given region is not currently listed in the method's TextTrack
410 // object's text track list of regions, then throw a NotFoundError exception.
411 if (region->track() != this)
412 return Exception { NotFoundError };
413
414 ASSERT(m_regions);
415 m_regions->remove(*region);
416 region->setTrack(nullptr);
417 return { };
418}
419
420void TextTrack::cueWillChange(TextTrackCue* cue)
421{
422 if (!m_client)
423 return;
424
425 // The cue may need to be repositioned in the media element's interval tree, may need to
426 // be re-rendered, etc, so remove it before the modification...
427 m_client->textTrackRemoveCue(*this, *cue);
428}
429
430void TextTrack::cueDidChange(TextTrackCue* cue)
431{
432 if (!m_client)
433 return;
434
435 // Make sure the TextTrackCueList order is up-to-date.
436 ensureTextTrackCueList().updateCueIndex(*cue);
437
438 // ... and add it back again.
439 m_client->textTrackAddCue(*this, *cue);
440}
441
442int TextTrack::trackIndex()
443{
444 ASSERT(m_mediaElement);
445 if (!m_trackIndex)
446 m_trackIndex = m_mediaElement->ensureTextTracks().getTrackIndex(*this);
447 return m_trackIndex.value();
448}
449
450void TextTrack::invalidateTrackIndex()
451{
452 m_trackIndex = WTF::nullopt;
453 m_renderedTrackIndex = WTF::nullopt;
454}
455
456bool TextTrack::isRendered()
457{
458 return (m_kind == Kind::Captions || m_kind == Kind::Subtitles || m_kind == Kind::Forced)
459 && m_mode == Mode::Showing;
460}
461
462TextTrackCueList& TextTrack::ensureTextTrackCueList()
463{
464 if (!m_cues)
465 m_cues = TextTrackCueList::create();
466 return *m_cues;
467}
468
469int TextTrack::trackIndexRelativeToRenderedTracks()
470{
471 ASSERT(m_mediaElement);
472 if (!m_renderedTrackIndex)
473 m_renderedTrackIndex = m_mediaElement->ensureTextTracks().getTrackIndexRelativeToRenderedTracks(*this);
474 return m_renderedTrackIndex.value();
475}
476
477bool TextTrack::hasCue(TextTrackCue* cue, TextTrackCue::CueMatchRules match)
478{
479 if (cue->startMediaTime() < MediaTime::zeroTime() || cue->endMediaTime() < MediaTime::zeroTime())
480 return false;
481
482 if (!m_cues || !m_cues->length())
483 return false;
484
485 size_t searchStart = 0;
486 size_t searchEnd = m_cues->length();
487
488 while (1) {
489 ASSERT(searchStart <= m_cues->length());
490 ASSERT(searchEnd <= m_cues->length());
491
492 RefPtr<TextTrackCue> existingCue;
493
494 // Cues in the TextTrackCueList are maintained in start time order.
495 if (searchStart == searchEnd) {
496 if (!searchStart)
497 return false;
498
499 // If there is more than one cue with the same start time, back up to first one so we
500 // consider all of them.
501 while (searchStart >= 2 && cue->hasEquivalentStartTime(*m_cues->item(searchStart - 2)))
502 --searchStart;
503
504 bool firstCompare = true;
505 while (1) {
506 if (!firstCompare)
507 ++searchStart;
508 firstCompare = false;
509 if (searchStart > m_cues->length())
510 return false;
511
512 existingCue = m_cues->item(searchStart - 1);
513 if (!existingCue)
514 return false;
515
516 if (cue->startMediaTime() > (existingCue->startMediaTime() + startTimeVariance()))
517 return false;
518
519 if (existingCue->isEqual(*cue, match))
520 return true;
521 }
522 }
523
524 size_t index = (searchStart + searchEnd) / 2;
525 existingCue = m_cues->item(index);
526 if ((cue->startMediaTime() + startTimeVariance()) < existingCue->startMediaTime() || (match != TextTrackCue::IgnoreDuration && cue->hasEquivalentStartTime(*existingCue) && cue->endMediaTime() > existingCue->endMediaTime()))
527 searchEnd = index;
528 else
529 searchStart = index + 1;
530 }
531
532 ASSERT_NOT_REACHED();
533 return false;
534}
535
536bool TextTrack::isMainProgramContent() const
537{
538 // "Main program" content is intrinsic to the presentation of the media file, regardless of locale. Content such as
539 // directors commentary is not "main program" because it is not essential for the presentation. HTML5 doesn't have
540 // a way to express this in a machine-reable form, it is typically done with the track label, so we assume that caption
541 // tracks are main content and all other track types are not.
542 return m_kind == Kind::Captions;
543}
544
545bool TextTrack::containsOnlyForcedSubtitles() const
546{
547 return m_kind == Kind::Forced;
548}
549
550#if ENABLE(MEDIA_SOURCE)
551void TextTrack::setLanguage(const AtomicString& language)
552{
553 // 11.1 language, on setting:
554 // 1. If the value being assigned to this attribute is not an empty string or a BCP 47 language
555 // tag[BCP47], then abort these steps.
556 // BCP 47 validation is done in TrackBase::setLanguage() which is
557 // shared between all tracks that support setting language.
558
559 // 2. Update this attribute to the new value.
560 TrackBase::setLanguage(language);
561
562 // 3. If the sourceBuffer attribute on this track is not null, then queue a task to fire a simple
563 // event named change at sourceBuffer.textTracks.
564 if (m_sourceBuffer)
565 m_sourceBuffer->textTracks().scheduleChangeEvent();
566
567 // 4. Queue a task to fire a simple event named change at the TextTrackList object referenced by
568 // the textTracks attribute on the HTMLMediaElement.
569 if (mediaElement())
570 mediaElement()->ensureTextTracks().scheduleChangeEvent();
571}
572#endif
573
574} // namespace WebCore
575
576#endif
577