1/*
2 * Copyright (C) 2017-2019 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23 * THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27#include "ResourceLoadStatisticsMemoryStore.h"
28
29#if ENABLE(RESOURCE_LOAD_STATISTICS)
30
31#include "Logging.h"
32#include "NetworkSession.h"
33#include "PluginProcessManager.h"
34#include "PluginProcessProxy.h"
35#include "ResourceLoadStatisticsPersistentStorage.h"
36#include "StorageAccessStatus.h"
37#include "WebProcessProxy.h"
38#include "WebResourceLoadStatisticsTelemetry.h"
39#include "WebsiteDataStore.h"
40#include <WebCore/DocumentStorageAccess.h>
41#include <WebCore/KeyedCoding.h>
42#include <WebCore/NetworkStorageSession.h>
43#include <WebCore/ResourceLoadStatistics.h>
44#include <WebCore/RuntimeEnabledFeatures.h>
45#include <WebCore/UserGestureIndicator.h>
46#include <wtf/CallbackAggregator.h>
47#include <wtf/DateMath.h>
48#include <wtf/MathExtras.h>
49#include <wtf/text/StringBuilder.h>
50
51namespace WebKit {
52using namespace WebCore;
53
54constexpr unsigned statisticsModelVersion { 16 };
55
56struct StatisticsLastSeen {
57 RegistrableDomain domain;
58 WallTime lastSeen;
59};
60
61static void pruneResources(HashMap<RegistrableDomain, ResourceLoadStatistics>& statisticsMap, Vector<StatisticsLastSeen>& statisticsToPrune, size_t& numberOfEntriesToPrune)
62{
63 if (statisticsToPrune.size() > numberOfEntriesToPrune) {
64 std::sort(statisticsToPrune.begin(), statisticsToPrune.end(), [](const StatisticsLastSeen& a, const StatisticsLastSeen& b) {
65 return a.lastSeen < b.lastSeen;
66 });
67 }
68
69 for (size_t i = 0, end = std::min(numberOfEntriesToPrune, statisticsToPrune.size()); i != end; ++i, --numberOfEntriesToPrune)
70 statisticsMap.remove(statisticsToPrune[i].domain);
71}
72
73ResourceLoadStatisticsMemoryStore::ResourceLoadStatisticsMemoryStore(WebResourceLoadStatisticsStore& store, WorkQueue& workQueue, ShouldIncludeLocalhost shouldIncludeLocalhost)
74 : ResourceLoadStatisticsStore(store, workQueue, shouldIncludeLocalhost)
75{
76 ASSERT(!RunLoop::isMain());
77
78 workQueue.dispatchAfter(5_s, [weakThis = makeWeakPtr(*this)] {
79 if (weakThis)
80 weakThis->calculateAndSubmitTelemetry();
81 });
82}
83
84bool ResourceLoadStatisticsMemoryStore::isEmpty() const
85{
86 return m_resourceStatisticsMap.isEmpty();
87}
88
89void ResourceLoadStatisticsMemoryStore::setPersistentStorage(ResourceLoadStatisticsPersistentStorage& persistentStorage)
90{
91 m_persistentStorage = makeWeakPtr(persistentStorage);
92}
93
94void ResourceLoadStatisticsMemoryStore::calculateAndSubmitTelemetry() const
95{
96 ASSERT(!RunLoop::isMain());
97
98 if (parameters().shouldSubmitTelemetry)
99 WebResourceLoadStatisticsTelemetry::calculateAndSubmit(*this);
100}
101
102void ResourceLoadStatisticsMemoryStore::incrementRecordsDeletedCountForDomains(HashSet<RegistrableDomain>&& domainsWithDeletedWebsiteData)
103{
104 for (auto& domain : domainsWithDeletedWebsiteData) {
105 auto& statistic = ensureResourceStatisticsForRegistrableDomain(domain);
106 ++statistic.dataRecordsRemoved;
107 }
108}
109
110unsigned ResourceLoadStatisticsMemoryStore::recursivelyGetAllDomainsThatHaveRedirectedToThisDomain(const ResourceLoadStatistics& resourceStatistic, HashSet<RegistrableDomain>& domainsThatHaveRedirectedTo, unsigned numberOfRecursiveCalls) const
111{
112 ASSERT(!RunLoop::isMain());
113
114 if (numberOfRecursiveCalls >= maxNumberOfRecursiveCallsInRedirectTraceBack) {
115 // Model version 14 invokes a deliberate re-classification of the whole set.
116 if (statisticsModelVersion != 14)
117 ASSERT_NOT_REACHED();
118 RELEASE_LOG(ResourceLoadStatistics, "Hit %u recursive calls in redirect backtrace. Returning early.", maxNumberOfRecursiveCallsInRedirectTraceBack);
119 return numberOfRecursiveCalls;
120 }
121
122 numberOfRecursiveCalls++;
123
124 for (auto& subresourceUniqueRedirectFromDomain : resourceStatistic.subresourceUniqueRedirectsFrom) {
125 auto mapEntry = m_resourceStatisticsMap.find(RegistrableDomain { subresourceUniqueRedirectFromDomain });
126 if (mapEntry == m_resourceStatisticsMap.end() || mapEntry->value.isPrevalentResource)
127 continue;
128 if (domainsThatHaveRedirectedTo.add(mapEntry->value.registrableDomain).isNewEntry)
129 numberOfRecursiveCalls = recursivelyGetAllDomainsThatHaveRedirectedToThisDomain(mapEntry->value, domainsThatHaveRedirectedTo, numberOfRecursiveCalls);
130 }
131 for (auto& topFrameUniqueRedirectFromDomain : resourceStatistic.topFrameUniqueRedirectsFrom) {
132 auto mapEntry = m_resourceStatisticsMap.find(RegistrableDomain { topFrameUniqueRedirectFromDomain });
133 if (mapEntry == m_resourceStatisticsMap.end() || mapEntry->value.isPrevalentResource)
134 continue;
135 if (domainsThatHaveRedirectedTo.add(mapEntry->value.registrableDomain).isNewEntry)
136 numberOfRecursiveCalls = recursivelyGetAllDomainsThatHaveRedirectedToThisDomain(mapEntry->value, domainsThatHaveRedirectedTo, numberOfRecursiveCalls);
137 }
138
139 return numberOfRecursiveCalls;
140}
141
142void ResourceLoadStatisticsMemoryStore::markAsPrevalentIfHasRedirectedToPrevalent(ResourceLoadStatistics& resourceStatistic)
143{
144 ASSERT(!RunLoop::isMain());
145
146 if (resourceStatistic.isPrevalentResource)
147 return;
148
149 for (auto& subresourceDomainRedirectedTo : resourceStatistic.subresourceUniqueRedirectsTo) {
150 auto mapEntry = m_resourceStatisticsMap.find(RegistrableDomain { subresourceDomainRedirectedTo });
151 if (mapEntry != m_resourceStatisticsMap.end() && mapEntry->value.isPrevalentResource) {
152 setPrevalentResource(resourceStatistic, ResourceLoadPrevalence::High);
153 return;
154 }
155 }
156
157 for (auto& topFrameDomainRedirectedTo : resourceStatistic.topFrameUniqueRedirectsTo) {
158 auto mapEntry = m_resourceStatisticsMap.find(RegistrableDomain { topFrameDomainRedirectedTo });
159 if (mapEntry != m_resourceStatisticsMap.end() && mapEntry->value.isPrevalentResource) {
160 setPrevalentResource(resourceStatistic, ResourceLoadPrevalence::High);
161 return;
162 }
163 }
164}
165
166bool ResourceLoadStatisticsMemoryStore::isPrevalentDueToDebugMode(ResourceLoadStatistics& resourceStatistic)
167{
168 if (!debugModeEnabled())
169 return false;
170
171 return resourceStatistic.registrableDomain == debugStaticPrevalentResource() || resourceStatistic.registrableDomain == debugManualPrevalentResource();
172}
173
174void ResourceLoadStatisticsMemoryStore::classifyPrevalentResources()
175{
176 for (auto& resourceStatistic : m_resourceStatisticsMap.values()) {
177 if (shouldSkip(resourceStatistic.registrableDomain))
178 continue;
179 if (isPrevalentDueToDebugMode(resourceStatistic))
180 setPrevalentResource(resourceStatistic, ResourceLoadPrevalence::High);
181 else if (!resourceStatistic.isVeryPrevalentResource) {
182 markAsPrevalentIfHasRedirectedToPrevalent(resourceStatistic);
183 auto currentPrevalence = resourceStatistic.isPrevalentResource ? ResourceLoadPrevalence::High : ResourceLoadPrevalence::Low;
184 auto newPrevalence = classifier().calculateResourcePrevalence(resourceStatistic, currentPrevalence);
185 if (newPrevalence != currentPrevalence)
186 setPrevalentResource(resourceStatistic, newPrevalence);
187 }
188 }
189}
190
191void ResourceLoadStatisticsMemoryStore::syncStorageIfNeeded()
192{
193 if (m_persistentStorage)
194 m_persistentStorage->scheduleOrWriteMemoryStore(ResourceLoadStatisticsPersistentStorage::ForceImmediateWrite::No);
195}
196
197void ResourceLoadStatisticsMemoryStore::syncStorageImmediately()
198{
199 if (m_persistentStorage)
200 m_persistentStorage->scheduleOrWriteMemoryStore(ResourceLoadStatisticsPersistentStorage::ForceImmediateWrite::Yes);
201}
202
203void ResourceLoadStatisticsMemoryStore::hasStorageAccess(const SubFrameDomain& subFrameDomain, const TopFrameDomain& topFrameDomain, Optional<FrameID> frameID, PageID pageID, CompletionHandler<void(bool)>&& completionHandler)
204{
205 ASSERT(!RunLoop::isMain());
206
207 auto& subFrameStatistic = ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
208 if (shouldBlockAndPurgeCookies(subFrameStatistic)) {
209 completionHandler(false);
210 return;
211 }
212
213 if (!shouldBlockAndKeepCookies(subFrameStatistic)) {
214 completionHandler(true);
215 return;
216 }
217
218 RunLoop::main().dispatch([store = makeRef(store()), subFrameDomain = subFrameDomain.isolatedCopy(), topFrameDomain = topFrameDomain.isolatedCopy(), frameID, pageID, completionHandler = WTFMove(completionHandler)]() mutable {
219 store->callHasStorageAccessForFrameHandler(subFrameDomain, topFrameDomain, frameID.value(), pageID, [store = store.copyRef(), completionHandler = WTFMove(completionHandler)](bool result) mutable {
220 store->statisticsQueue().dispatch([completionHandler = WTFMove(completionHandler), result] () mutable {
221 completionHandler(result);
222 });
223 });
224 });
225}
226
227void ResourceLoadStatisticsMemoryStore::requestStorageAccess(SubFrameDomain&& subFrameDomain, TopFrameDomain&& topFrameDomain, FrameID frameID, uint64_t pageID, CompletionHandler<void(StorageAccessStatus)>&& completionHandler)
228{
229 ASSERT(!RunLoop::isMain());
230
231 auto& subFrameStatistic = ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
232 if (shouldBlockAndPurgeCookies(subFrameStatistic)) {
233#if !RELEASE_LOG_DISABLED
234 RELEASE_LOG_INFO_IF(debugLoggingEnabled(), ResourceLoadStatisticsDebug, "Cannot grant storage access to %{public}s since its cookies are blocked in third-party contexts and it has not received user interaction as first-party.", subFrameDomain.string().utf8().data());
235#endif
236 completionHandler(StorageAccessStatus::CannotRequestAccess);
237 return;
238 }
239
240 if (!shouldBlockAndKeepCookies(subFrameStatistic)) {
241#if !RELEASE_LOG_DISABLED
242 RELEASE_LOG_INFO_IF(debugLoggingEnabled(), ResourceLoadStatisticsDebug, "No need to grant storage access to %{public}s since its cookies are not blocked in third-party contexts.", subFrameDomain.string().utf8().data());
243#endif
244 completionHandler(StorageAccessStatus::HasAccess);
245 return;
246 }
247
248 auto userWasPromptedEarlier = hasUserGrantedStorageAccessThroughPrompt(subFrameStatistic, topFrameDomain);
249 if (userWasPromptedEarlier == StorageAccessPromptWasShown::No) {
250#if !RELEASE_LOG_DISABLED
251 RELEASE_LOG_INFO_IF(debugLoggingEnabled(), ResourceLoadStatisticsDebug, "About to ask the user whether they want to grant storage access to %{public}s under %{public}s or not.", subFrameDomain.string().utf8().data(), topFrameDomain.string().utf8().data());
252#endif
253 completionHandler(StorageAccessStatus::RequiresUserPrompt);
254 return;
255 }
256
257#if !RELEASE_LOG_DISABLED
258 if (userWasPromptedEarlier == StorageAccessPromptWasShown::Yes)
259 RELEASE_LOG_INFO_IF(debugLoggingEnabled(), ResourceLoadStatisticsDebug, "Storage access was granted to %{public}s under %{public}s.", subFrameDomain.string().utf8().data(), topFrameDomain.string().utf8().data());
260#endif
261
262 subFrameStatistic.timesAccessedAsFirstPartyDueToStorageAccessAPI++;
263
264 grantStorageAccessInternal(WTFMove(subFrameDomain), WTFMove(topFrameDomain), frameID, pageID, userWasPromptedEarlier, [completionHandler = WTFMove(completionHandler)] (StorageAccessWasGranted wasGranted) mutable {
265 completionHandler(wasGranted == StorageAccessWasGranted::Yes ? StorageAccessStatus::HasAccess : StorageAccessStatus::CannotRequestAccess);
266 });
267}
268
269void ResourceLoadStatisticsMemoryStore::requestStorageAccessUnderOpener(DomainInNeedOfStorageAccess&& domainInNeedOfStorageAccess, OpenerPageID openerPageID, OpenerDomain&& openerDomain)
270{
271 ASSERT(domainInNeedOfStorageAccess != openerDomain);
272 ASSERT(!RunLoop::isMain());
273
274 if (domainInNeedOfStorageAccess == openerDomain)
275 return;
276
277 auto& domainInNeedOfStorageAccessStatistic = ensureResourceStatisticsForRegistrableDomain(domainInNeedOfStorageAccess);
278 auto cookiesBlockedAndPurged = shouldBlockAndPurgeCookies(domainInNeedOfStorageAccessStatistic);
279
280 // The domain already has access if its cookies are not blocked.
281 if (!cookiesBlockedAndPurged && !shouldBlockAndKeepCookies(domainInNeedOfStorageAccessStatistic))
282 return;
283
284#if !RELEASE_LOG_DISABLED
285 RELEASE_LOG_INFO_IF(debugLoggingEnabled(), ResourceLoadStatisticsDebug, "[Temporary combatibility fix] Storage access was granted for %{public}s under opener page from %{public}s, with user interaction in the opened window.", domainInNeedOfStorageAccess.string().utf8().data(), openerDomain.string().utf8().data());
286#endif
287 grantStorageAccessInternal(WTFMove(domainInNeedOfStorageAccess), WTFMove(openerDomain), WTF::nullopt, openerPageID, StorageAccessPromptWasShown::No, [](StorageAccessWasGranted) { });
288}
289
290void ResourceLoadStatisticsMemoryStore::grantStorageAccess(SubFrameDomain&& subFrameDomain, TopFrameDomain&& topFrameDomain, uint64_t frameID, uint64_t pageID, StorageAccessPromptWasShown promptWasShown, CompletionHandler<void(StorageAccessWasGranted)>&& completionHandler)
291{
292 ASSERT(!RunLoop::isMain());
293
294 if (promptWasShown == StorageAccessPromptWasShown::Yes) {
295 auto& subFrameStatistic = ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
296 ASSERT(subFrameStatistic.hadUserInteraction);
297 subFrameStatistic.storageAccessUnderTopFrameDomains.add(topFrameDomain);
298 }
299 grantStorageAccessInternal(WTFMove(subFrameDomain), WTFMove(topFrameDomain), frameID, pageID, promptWasShown, WTFMove(completionHandler));
300}
301
302void ResourceLoadStatisticsMemoryStore::grantStorageAccessInternal(SubFrameDomain&& subFrameDomain, TopFrameDomain&& topFrameDomain, Optional<FrameID> frameID, PageID pageID, StorageAccessPromptWasShown promptWasShownNowOrEarlier, CompletionHandler<void(StorageAccessWasGranted)>&& completionHandler)
303{
304 ASSERT(!RunLoop::isMain());
305
306 if (subFrameDomain == topFrameDomain) {
307 completionHandler(StorageAccessWasGranted::Yes);
308 return;
309 }
310
311 if (promptWasShownNowOrEarlier == StorageAccessPromptWasShown::Yes) {
312 auto& subFrameStatistic = ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
313 ASSERT(subFrameStatistic.hadUserInteraction);
314 ASSERT(subFrameStatistic.storageAccessUnderTopFrameDomains.contains(topFrameDomain));
315 subFrameStatistic.mostRecentUserInteractionTime = WallTime::now();
316 }
317
318 RunLoop::main().dispatch([subFrameDomain = subFrameDomain.isolatedCopy(), topFrameDomain = topFrameDomain.isolatedCopy(), frameID, pageID, store = makeRef(store()), completionHandler = WTFMove(completionHandler)]() mutable {
319 store->callGrantStorageAccessHandler(subFrameDomain, topFrameDomain, frameID, pageID, [completionHandler = WTFMove(completionHandler), store = store.copyRef()](StorageAccessWasGranted wasGranted) mutable {
320 store->statisticsQueue().dispatch([wasGranted, completionHandler = WTFMove(completionHandler)] () mutable {
321 completionHandler(wasGranted);
322 });
323 });
324 });
325}
326
327void ResourceLoadStatisticsMemoryStore::grandfatherDataForDomains(const HashSet<RegistrableDomain>& domains)
328{
329 for (auto& domain : domains) {
330 auto& statistic = ensureResourceStatisticsForRegistrableDomain(domain);
331 statistic.grandfathered = true;
332 }
333}
334
335Vector<RegistrableDomain> ResourceLoadStatisticsMemoryStore::ensurePrevalentResourcesForDebugMode()
336{
337 if (!debugModeEnabled())
338 return { };
339
340 Vector<RegistrableDomain> domainsToBlock;
341 domainsToBlock.reserveInitialCapacity(2);
342
343 auto& staticSesourceStatistic = ensureResourceStatisticsForRegistrableDomain(debugStaticPrevalentResource());
344 setPrevalentResource(staticSesourceStatistic, ResourceLoadPrevalence::High);
345 domainsToBlock.uncheckedAppend(debugStaticPrevalentResource());
346
347 if (!debugManualPrevalentResource().isEmpty()) {
348 auto& manualResourceStatistic = ensureResourceStatisticsForRegistrableDomain(debugManualPrevalentResource());
349 setPrevalentResource(manualResourceStatistic, ResourceLoadPrevalence::High);
350 domainsToBlock.uncheckedAppend(debugManualPrevalentResource());
351#if !RELEASE_LOG_DISABLED
352 RELEASE_LOG_INFO(ResourceLoadStatisticsDebug, "Did set %{public}s as prevalent resource for the purposes of ITP Debug Mode.", debugManualPrevalentResource().string().utf8().data());
353#endif
354 }
355
356 return domainsToBlock;
357}
358
359void ResourceLoadStatisticsMemoryStore::logFrameNavigation(const RegistrableDomain& targetDomain, const RegistrableDomain& topFrameDomain, const RegistrableDomain& sourceDomain, bool isRedirect, bool isMainFrame)
360{
361 ASSERT(!RunLoop::isMain());
362
363 bool areTargetAndTopFrameDomainsSameSite = targetDomain == topFrameDomain;
364 bool areTargetAndSourceDomainsSameSite = targetDomain == sourceDomain;
365
366 bool statisticsWereUpdated = false;
367 if (!isMainFrame && !(areTargetAndTopFrameDomainsSameSite || areTargetAndSourceDomainsSameSite)) {
368 auto& targetStatistics = ensureResourceStatisticsForRegistrableDomain(targetDomain);
369 targetStatistics.lastSeen = ResourceLoadStatistics::reduceTimeResolution(WallTime::now());
370 if (targetStatistics.subframeUnderTopFrameDomains.add(topFrameDomain).isNewEntry)
371 statisticsWereUpdated = true;
372 }
373
374 if (isRedirect && !areTargetAndSourceDomainsSameSite) {
375 if (isMainFrame) {
376 auto& redirectingDomainStatistics = ensureResourceStatisticsForRegistrableDomain(sourceDomain);
377 if (redirectingDomainStatistics.topFrameUniqueRedirectsTo.add(targetDomain).isNewEntry)
378 statisticsWereUpdated = true;
379 auto& targetStatistics = ensureResourceStatisticsForRegistrableDomain(targetDomain);
380 if (targetStatistics.topFrameUniqueRedirectsFrom.add(sourceDomain).isNewEntry)
381 statisticsWereUpdated = true;
382 } else {
383 auto& redirectingDomainStatistics = ensureResourceStatisticsForRegistrableDomain(sourceDomain);
384 if (redirectingDomainStatistics.subresourceUniqueRedirectsTo.add(targetDomain).isNewEntry)
385 statisticsWereUpdated = true;
386 auto& targetStatistics = ensureResourceStatisticsForRegistrableDomain(targetDomain);
387 if (targetStatistics.subresourceUniqueRedirectsFrom.add(sourceDomain).isNewEntry)
388 statisticsWereUpdated = true;
389 }
390 }
391
392 if (statisticsWereUpdated)
393 scheduleStatisticsProcessingRequestIfNecessary();
394}
395
396void ResourceLoadStatisticsMemoryStore::logSubresourceLoading(const SubResourceDomain& targetDomain, const TopFrameDomain& topFrameDomain, WallTime lastSeen)
397{
398 ASSERT(!RunLoop::isMain());
399
400 auto& targetStatistics = ensureResourceStatisticsForRegistrableDomain(targetDomain);
401 targetStatistics.lastSeen = lastSeen;
402 if (targetStatistics.subresourceUnderTopFrameDomains.add(topFrameDomain).isNewEntry)
403 scheduleStatisticsProcessingRequestIfNecessary();
404}
405
406void ResourceLoadStatisticsMemoryStore::logSubresourceRedirect(const RedirectedFromDomain& sourceDomain, const RedirectedToDomain& targetDomain)
407{
408 ASSERT(!RunLoop::isMain());
409
410 auto& redirectingDomainStatistics = ensureResourceStatisticsForRegistrableDomain(sourceDomain);
411 bool isNewRedirectToEntry = redirectingDomainStatistics.subresourceUniqueRedirectsTo.add(targetDomain).isNewEntry;
412 auto& targetStatistics = ensureResourceStatisticsForRegistrableDomain(targetDomain);
413 bool isNewRedirectFromEntry = targetStatistics.subresourceUniqueRedirectsFrom.add(sourceDomain).isNewEntry;
414
415 if (isNewRedirectToEntry || isNewRedirectFromEntry)
416 scheduleStatisticsProcessingRequestIfNecessary();
417}
418
419void ResourceLoadStatisticsMemoryStore::logUserInteraction(const TopFrameDomain& domain)
420{
421 ASSERT(!RunLoop::isMain());
422
423 auto& statistics = ensureResourceStatisticsForRegistrableDomain(domain);
424 statistics.hadUserInteraction = true;
425 statistics.mostRecentUserInteractionTime = WallTime::now();
426}
427
428void ResourceLoadStatisticsMemoryStore::logCrossSiteLoadWithLinkDecoration(const NavigatedFromDomain& fromDomain, const NavigatedToDomain& toDomain)
429{
430 ASSERT(!RunLoop::isMain());
431 ASSERT(fromDomain != toDomain);
432
433 auto& toStatistics = ensureResourceStatisticsForRegistrableDomain(toDomain);
434 toStatistics.topFrameLinkDecorationsFrom.add(fromDomain);
435
436 auto& fromStatistics = ensureResourceStatisticsForRegistrableDomain(fromDomain);
437 if (fromStatistics.isPrevalentResource)
438 toStatistics.gotLinkDecorationFromPrevalentResource = true;
439}
440
441void ResourceLoadStatisticsMemoryStore::clearUserInteraction(const RegistrableDomain& domain)
442{
443 ASSERT(!RunLoop::isMain());
444
445 auto& statistics = ensureResourceStatisticsForRegistrableDomain(domain);
446 statistics.hadUserInteraction = false;
447 statistics.mostRecentUserInteractionTime = { };
448}
449
450bool ResourceLoadStatisticsMemoryStore::hasHadUserInteraction(const RegistrableDomain& domain, OperatingDatesWindow operatingDatesWindow)
451{
452 ASSERT(!RunLoop::isMain());
453
454 auto mapEntry = m_resourceStatisticsMap.find(domain);
455 return mapEntry == m_resourceStatisticsMap.end() ? false: hasHadUnexpiredRecentUserInteraction(mapEntry->value, operatingDatesWindow);
456}
457
458void ResourceLoadStatisticsMemoryStore::setPrevalentResource(ResourceLoadStatistics& resourceStatistic, ResourceLoadPrevalence newPrevalence)
459{
460 ASSERT(!RunLoop::isMain());
461
462 if (shouldSkip(resourceStatistic.registrableDomain))
463 return;
464
465 resourceStatistic.isPrevalentResource = true;
466 resourceStatistic.isVeryPrevalentResource = newPrevalence == ResourceLoadPrevalence::VeryHigh;
467 HashSet<RegistrableDomain> domainsThatHaveRedirectedTo;
468 recursivelyGetAllDomainsThatHaveRedirectedToThisDomain(resourceStatistic, domainsThatHaveRedirectedTo, 0);
469 for (auto& domain : domainsThatHaveRedirectedTo) {
470 auto mapEntry = m_resourceStatisticsMap.find(domain);
471 if (mapEntry == m_resourceStatisticsMap.end())
472 continue;
473 ASSERT(!mapEntry->value.isPrevalentResource);
474 mapEntry->value.isPrevalentResource = true;
475 }
476}
477
478String ResourceLoadStatisticsMemoryStore::dumpResourceLoadStatistics() const
479{
480 ASSERT(!RunLoop::isMain());
481
482 StringBuilder result;
483 result.appendLiteral("Resource load statistics:\n\n");
484 for (auto& mapEntry : m_resourceStatisticsMap.values())
485 result.append(mapEntry.toString());
486 return result.toString();
487}
488
489bool ResourceLoadStatisticsMemoryStore::isPrevalentResource(const RegistrableDomain& domain) const
490{
491 ASSERT(!RunLoop::isMain());
492
493 if (shouldSkip(domain))
494 return false;
495
496 auto mapEntry = m_resourceStatisticsMap.find(domain);
497 return mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.isPrevalentResource;
498}
499
500bool ResourceLoadStatisticsMemoryStore::isVeryPrevalentResource(const RegistrableDomain& domain) const
501{
502 ASSERT(!RunLoop::isMain());
503
504 if (shouldSkip(domain))
505 return false;
506
507 auto mapEntry = m_resourceStatisticsMap.find(domain);
508 return mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.isPrevalentResource && mapEntry->value.isVeryPrevalentResource;
509}
510
511bool ResourceLoadStatisticsMemoryStore::isRegisteredAsSubresourceUnder(const SubResourceDomain& subresourceDomain, const TopFrameDomain& topFrameDomain) const
512{
513 ASSERT(!RunLoop::isMain());
514
515 auto mapEntry = m_resourceStatisticsMap.find(subresourceDomain);
516 return mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.subresourceUnderTopFrameDomains.contains(topFrameDomain);
517}
518
519bool ResourceLoadStatisticsMemoryStore::isRegisteredAsSubFrameUnder(const SubFrameDomain& subFrameDomain, const TopFrameDomain& topFrameDomain) const
520{
521 ASSERT(!RunLoop::isMain());
522
523 auto mapEntry = m_resourceStatisticsMap.find(subFrameDomain);
524 return mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.subframeUnderTopFrameDomains.contains(topFrameDomain);
525}
526
527bool ResourceLoadStatisticsMemoryStore::isRegisteredAsRedirectingTo(const RedirectedFromDomain& redirectedFromDomain, const RedirectedToDomain& redirectedToDomain) const
528{
529 ASSERT(!RunLoop::isMain());
530
531 auto mapEntry = m_resourceStatisticsMap.find(redirectedFromDomain);
532 return mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.subresourceUniqueRedirectsTo.contains(redirectedToDomain);
533}
534
535void ResourceLoadStatisticsMemoryStore::clearPrevalentResource(const RegistrableDomain& domain)
536{
537 ASSERT(!RunLoop::isMain());
538
539 auto& statistics = ensureResourceStatisticsForRegistrableDomain(domain);
540 statistics.isPrevalentResource = false;
541 statistics.isVeryPrevalentResource = false;
542}
543
544void ResourceLoadStatisticsMemoryStore::setGrandfathered(const RegistrableDomain& domain, bool value)
545{
546 ASSERT(!RunLoop::isMain());
547
548 auto& statistics = ensureResourceStatisticsForRegistrableDomain(domain);
549 statistics.grandfathered = value;
550}
551
552bool ResourceLoadStatisticsMemoryStore::isGrandfathered(const RegistrableDomain& domain) const
553{
554 ASSERT(!RunLoop::isMain());
555
556 auto mapEntry = m_resourceStatisticsMap.find(domain);
557 return mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.grandfathered;
558}
559
560void ResourceLoadStatisticsMemoryStore::setSubframeUnderTopFrameDomain(const SubFrameDomain& subFrameDomain, const TopFrameDomain& topFrameDomain)
561{
562 ASSERT(!RunLoop::isMain());
563
564 auto& statistics = ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
565 statistics.subframeUnderTopFrameDomains.add(topFrameDomain);
566 // For consistency, make sure we also have a statistics entry for the top frame domain.
567 ensureResourceStatisticsForRegistrableDomain(topFrameDomain);
568}
569
570void ResourceLoadStatisticsMemoryStore::setSubresourceUnderTopFrameDomain(const SubResourceDomain& subresourceDomain, const RegistrableDomain& topFrameDomain)
571{
572 ASSERT(!RunLoop::isMain());
573
574 auto& statistics = ensureResourceStatisticsForRegistrableDomain(subresourceDomain);
575 statistics.subresourceUnderTopFrameDomains.add(topFrameDomain);
576 // For consistency, make sure we also have a statistics entry for the top frame domain.
577 ensureResourceStatisticsForRegistrableDomain(topFrameDomain);
578}
579
580void ResourceLoadStatisticsMemoryStore::setSubresourceUniqueRedirectTo(const SubResourceDomain& subresourceDomain, const RedirectDomain& redirectDomain)
581{
582 ASSERT(!RunLoop::isMain());
583
584 auto& statistics = ensureResourceStatisticsForRegistrableDomain(subresourceDomain);
585 statistics.subresourceUniqueRedirectsTo.add(redirectDomain);
586 // For consistency, make sure we also have a statistics entry for the redirect domain.
587 ensureResourceStatisticsForRegistrableDomain(redirectDomain);
588}
589
590void ResourceLoadStatisticsMemoryStore::setSubresourceUniqueRedirectFrom(const SubResourceDomain& subresourceDomain, const RedirectDomain& redirectDomain)
591{
592 ASSERT(!RunLoop::isMain());
593
594 auto& statistics = ensureResourceStatisticsForRegistrableDomain(subresourceDomain);
595 statistics.subresourceUniqueRedirectsFrom.add(redirectDomain);
596 // For consistency, make sure we also have a statistics entry for the redirect domain.
597 ensureResourceStatisticsForRegistrableDomain(redirectDomain);
598}
599
600void ResourceLoadStatisticsMemoryStore::setTopFrameUniqueRedirectTo(const TopFrameDomain& topFrameDomain, const RedirectDomain& redirectDomain)
601{
602 ASSERT(!RunLoop::isMain());
603
604 auto& statistics = ensureResourceStatisticsForRegistrableDomain(topFrameDomain);
605 statistics.topFrameUniqueRedirectsTo.add(redirectDomain);
606 // For consistency, make sure we also have a statistics entry for the redirect domain.
607 ensureResourceStatisticsForRegistrableDomain(redirectDomain);
608}
609
610void ResourceLoadStatisticsMemoryStore::setTopFrameUniqueRedirectFrom(const TopFrameDomain& topFrameDomain, const RedirectDomain& redirectDomain)
611{
612 ASSERT(!RunLoop::isMain());
613
614 auto& statistics = ensureResourceStatisticsForRegistrableDomain(topFrameDomain);
615 statistics.topFrameUniqueRedirectsFrom.add(redirectDomain);
616 // For consistency, make sure we also have a statistics entry for the redirect domain.
617 ensureResourceStatisticsForRegistrableDomain(redirectDomain);
618}
619
620ResourceLoadStatistics& ResourceLoadStatisticsMemoryStore::ensureResourceStatisticsForRegistrableDomain(const RegistrableDomain& domain)
621{
622 ASSERT(!RunLoop::isMain());
623
624 return m_resourceStatisticsMap.ensure(domain, [&domain] {
625 return ResourceLoadStatistics(domain);
626 }).iterator->value;
627}
628
629std::unique_ptr<KeyedEncoder> ResourceLoadStatisticsMemoryStore::createEncoderFromData() const
630{
631 ASSERT(!RunLoop::isMain());
632
633 auto encoder = KeyedEncoder::encoder();
634 encoder->encodeUInt32("version", statisticsModelVersion);
635 encoder->encodeDouble("endOfGrandfatheringTimestamp", endOfGrandfatheringTimestamp().secondsSinceEpoch().value());
636
637 encoder->encodeObjects("browsingStatistics", m_resourceStatisticsMap.begin(), m_resourceStatisticsMap.end(), [](KeyedEncoder& encoderInner, const auto& domain) {
638 domain.value.encode(encoderInner);
639 });
640
641 auto& operatingDates = this->operatingDates();
642 encoder->encodeObjects("operatingDates", operatingDates.begin(), operatingDates.end(), [](KeyedEncoder& encoderInner, OperatingDate date) {
643 encoderInner.encodeDouble("date", date.secondsSinceEpoch().value());
644 });
645
646 return encoder;
647}
648
649void ResourceLoadStatisticsMemoryStore::mergeWithDataFromDecoder(KeyedDecoder& decoder)
650{
651 ASSERT(!RunLoop::isMain());
652
653 unsigned versionOnDisk;
654 if (!decoder.decodeUInt32("version", versionOnDisk))
655 return;
656
657 if (versionOnDisk > statisticsModelVersion) {
658 WTFLogAlways("Found resource load statistics on disk with model version %u whereas the highest supported version is %u. Resetting.", versionOnDisk, statisticsModelVersion);
659 return;
660 }
661
662 double endOfGrandfatheringTimestamp;
663 if (decoder.decodeDouble("endOfGrandfatheringTimestamp", endOfGrandfatheringTimestamp))
664 setEndOfGrandfatheringTimestamp(WallTime::fromRawSeconds(endOfGrandfatheringTimestamp));
665 else
666 clearEndOfGrandfatheringTimeStamp();
667
668 Vector<ResourceLoadStatistics> loadedStatistics;
669 bool succeeded = decoder.decodeObjects("browsingStatistics", loadedStatistics, [versionOnDisk](KeyedDecoder& decoderInner, ResourceLoadStatistics& statistics) {
670 return statistics.decode(decoderInner, versionOnDisk);
671 });
672
673 if (!succeeded)
674 return;
675
676 mergeStatistics(WTFMove(loadedStatistics));
677 updateCookieBlocking([]() { });
678
679 Vector<OperatingDate> operatingDates;
680 succeeded = decoder.decodeObjects("operatingDates", operatingDates, [](KeyedDecoder& decoder, OperatingDate& date) {
681 double value;
682 if (!decoder.decodeDouble("date", value))
683 return false;
684
685 date = OperatingDate::fromWallTime(WallTime::fromRawSeconds(value));
686 return true;
687 });
688
689 if (!succeeded)
690 return;
691
692 mergeOperatingDates(WTFMove(operatingDates));
693}
694
695void ResourceLoadStatisticsMemoryStore::clear(CompletionHandler<void()>&& completionHandler)
696{
697 ASSERT(!RunLoop::isMain());
698
699 m_resourceStatisticsMap.clear();
700 clearOperatingDates();
701
702 auto callbackAggregator = CallbackAggregator::create(WTFMove(completionHandler));
703
704 removeAllStorageAccess([callbackAggregator = callbackAggregator.copyRef()] { });
705
706 auto primaryDomainsToBlock = ensurePrevalentResourcesForDebugMode();
707 updateCookieBlockingForDomains(primaryDomainsToBlock, [callbackAggregator = callbackAggregator.copyRef()] { });
708}
709
710bool ResourceLoadStatisticsMemoryStore::wasAccessedAsFirstPartyDueToUserInteraction(const ResourceLoadStatistics& current, const ResourceLoadStatistics& updated) const
711{
712 if (!current.hadUserInteraction && !updated.hadUserInteraction)
713 return false;
714
715 auto mostRecentUserInteractionTime = std::max(current.mostRecentUserInteractionTime, updated.mostRecentUserInteractionTime);
716
717 return updated.lastSeen <= mostRecentUserInteractionTime + 24_h;
718}
719
720void ResourceLoadStatisticsMemoryStore::mergeStatistics(Vector<ResourceLoadStatistics>&& statistics)
721{
722 ASSERT(!RunLoop::isMain());
723
724 for (auto& statistic : statistics) {
725 auto result = m_resourceStatisticsMap.ensure(statistic.registrableDomain, [&statistic] {
726 return WTFMove(statistic);
727 });
728 if (!result.isNewEntry) {
729 if (wasAccessedAsFirstPartyDueToUserInteraction(result.iterator->value, statistic))
730 result.iterator->value.timesAccessedAsFirstPartyDueToUserInteraction++;
731 result.iterator->value.merge(statistic);
732 }
733 }
734}
735
736bool ResourceLoadStatisticsMemoryStore::shouldBlockAndKeepCookies(const ResourceLoadStatistics& statistic)
737{
738 return statistic.isPrevalentResource && statistic.hadUserInteraction;
739}
740
741bool ResourceLoadStatisticsMemoryStore::shouldBlockAndPurgeCookies(const ResourceLoadStatistics& statistic)
742{
743 return statistic.isPrevalentResource && !statistic.hadUserInteraction;
744}
745
746StorageAccessPromptWasShown ResourceLoadStatisticsMemoryStore::hasUserGrantedStorageAccessThroughPrompt(const ResourceLoadStatistics& statistic, const RegistrableDomain& firstPartyDomain)
747{
748 return statistic.storageAccessUnderTopFrameDomains.contains(firstPartyDomain) ? StorageAccessPromptWasShown::Yes : StorageAccessPromptWasShown::No;
749}
750
751void ResourceLoadStatisticsMemoryStore::updateCookieBlocking(CompletionHandler<void()>&& completionHandler)
752{
753 ASSERT(!RunLoop::isMain());
754
755 Vector<RegistrableDomain> domainsToBlock;
756 for (auto& resourceStatistic : m_resourceStatisticsMap.values()) {
757 if (resourceStatistic.isPrevalentResource)
758 domainsToBlock.append(resourceStatistic.registrableDomain);
759 }
760
761 if (domainsToBlock.isEmpty()) {
762 completionHandler();
763 return;
764 }
765
766 if (debugLoggingEnabled() && !domainsToBlock.isEmpty())
767 debugLogDomainsInBatches("block", domainsToBlock);
768
769 RunLoop::main().dispatch([weakThis = makeWeakPtr(*this), store = makeRef(store()), domainsToBlock = crossThreadCopy(domainsToBlock), completionHandler = WTFMove(completionHandler)] () mutable {
770 store->callUpdatePrevalentDomainsToBlockCookiesForHandler(domainsToBlock, [weakThis = WTFMove(weakThis), store = store.copyRef(), completionHandler = WTFMove(completionHandler)]() mutable {
771 store->statisticsQueue().dispatch([weakThis = WTFMove(weakThis), completionHandler = WTFMove(completionHandler)]() mutable {
772 completionHandler();
773 if (!weakThis)
774 return;
775#if !RELEASE_LOG_DISABLED
776 RELEASE_LOG_INFO_IF(weakThis->debugLoggingEnabled(), ResourceLoadStatisticsDebug, "Done updating cookie blocking.");
777#endif
778 });
779 });
780 });
781}
782
783void ResourceLoadStatisticsMemoryStore::processStatistics(const Function<void(const ResourceLoadStatistics&)>& processFunction) const
784{
785 ASSERT(!RunLoop::isMain());
786
787 for (auto& resourceStatistic : m_resourceStatisticsMap.values())
788 processFunction(resourceStatistic);
789}
790
791bool ResourceLoadStatisticsMemoryStore::hasHadUnexpiredRecentUserInteraction(ResourceLoadStatistics& resourceStatistic, OperatingDatesWindow operatingDatesWindow) const
792{
793 ASSERT(!RunLoop::isMain());
794
795 if (resourceStatistic.hadUserInteraction && hasStatisticsExpired(resourceStatistic, operatingDatesWindow)) {
796 // Drop privacy sensitive data because we no longer need it.
797 // Set timestamp to 0 so that statistics merge will know
798 // it has been reset as opposed to its default -1.
799 resourceStatistic.mostRecentUserInteractionTime = { };
800 resourceStatistic.storageAccessUnderTopFrameDomains.clear();
801 resourceStatistic.hadUserInteraction = false;
802 }
803
804 return resourceStatistic.hadUserInteraction;
805}
806
807bool ResourceLoadStatisticsMemoryStore::shouldRemoveAllWebsiteDataFor(ResourceLoadStatistics& resourceStatistic, bool shouldCheckForGrandfathering) const
808{
809 return resourceStatistic.isPrevalentResource && !hasHadUnexpiredRecentUserInteraction(resourceStatistic, OperatingDatesWindow::Long) && (!shouldCheckForGrandfathering || !resourceStatistic.grandfathered);
810}
811
812bool ResourceLoadStatisticsMemoryStore::shouldRemoveAllButCookiesFor(ResourceLoadStatistics& resourceStatistic, bool shouldCheckForGrandfathering) const
813{
814 return RuntimeEnabledFeatures::sharedFeatures().isITPFirstPartyWebsiteDataRemovalEnabled() && resourceStatistic.gotLinkDecorationFromPrevalentResource && !hasHadUnexpiredRecentUserInteraction(resourceStatistic, OperatingDatesWindow::Short) && (!shouldCheckForGrandfathering || !resourceStatistic.grandfathered);
815}
816
817HashMap<RegistrableDomain, WebsiteDataToRemove> ResourceLoadStatisticsMemoryStore::registrableDomainsToRemoveWebsiteDataFor()
818{
819 ASSERT(!RunLoop::isMain());
820
821 bool shouldCheckForGrandfathering = endOfGrandfatheringTimestamp() > WallTime::now();
822 bool shouldClearGrandfathering = !shouldCheckForGrandfathering && endOfGrandfatheringTimestamp();
823
824 if (shouldClearGrandfathering)
825 clearEndOfGrandfatheringTimeStamp();
826
827 HashMap<RegistrableDomain, WebsiteDataToRemove> domainsToRemoveWebsiteDataFor;
828 for (auto& statistic : m_resourceStatisticsMap.values()) {
829 if (shouldRemoveAllWebsiteDataFor(statistic, shouldCheckForGrandfathering))
830 domainsToRemoveWebsiteDataFor.add(statistic.registrableDomain, WebsiteDataToRemove::All);
831 else if (shouldRemoveAllButCookiesFor(statistic, shouldCheckForGrandfathering)) {
832 domainsToRemoveWebsiteDataFor.add(statistic.registrableDomain, WebsiteDataToRemove::AllButCookies);
833 statistic.gotLinkDecorationFromPrevalentResource = false;
834 }
835
836 if (shouldClearGrandfathering && statistic.grandfathered)
837 statistic.grandfathered = false;
838 }
839
840 return domainsToRemoveWebsiteDataFor;
841}
842
843void ResourceLoadStatisticsMemoryStore::pruneStatisticsIfNeeded()
844{
845 ASSERT(!RunLoop::isMain());
846
847 if (m_resourceStatisticsMap.size() <= parameters().maxStatisticsEntries)
848 return;
849
850 ASSERT(parameters().pruneEntriesDownTo <= parameters().maxStatisticsEntries);
851
852 size_t numberOfEntriesLeftToPrune = m_resourceStatisticsMap.size() - parameters().pruneEntriesDownTo;
853 ASSERT(numberOfEntriesLeftToPrune);
854
855 Vector<StatisticsLastSeen> resourcesToPrunePerImportance[maxImportance + 1];
856 for (auto& resourceStatistic : m_resourceStatisticsMap.values())
857 resourcesToPrunePerImportance[computeImportance(resourceStatistic)].append({ resourceStatistic.registrableDomain, resourceStatistic.lastSeen });
858
859 for (unsigned importance = 0; numberOfEntriesLeftToPrune && importance <= maxImportance; ++importance)
860 pruneResources(m_resourceStatisticsMap, resourcesToPrunePerImportance[importance], numberOfEntriesLeftToPrune);
861
862 ASSERT(!numberOfEntriesLeftToPrune);
863}
864
865void ResourceLoadStatisticsMemoryStore::setLastSeen(const RegistrableDomain& domain, Seconds seconds)
866{
867 ASSERT(!RunLoop::isMain());
868
869 auto& statistics = ensureResourceStatisticsForRegistrableDomain(domain);
870 statistics.lastSeen = WallTime::fromRawSeconds(seconds.seconds());
871}
872
873void ResourceLoadStatisticsMemoryStore::setPrevalentResource(const RegistrableDomain& domain)
874{
875 ASSERT(!RunLoop::isMain());
876
877 if (shouldSkip(domain))
878 return;
879
880 auto& resourceStatistic = ensureResourceStatisticsForRegistrableDomain(domain);
881 setPrevalentResource(resourceStatistic, ResourceLoadPrevalence::High);
882}
883
884void ResourceLoadStatisticsMemoryStore::setVeryPrevalentResource(const RegistrableDomain& domain)
885{
886 ASSERT(!RunLoop::isMain());
887
888 if (shouldSkip(domain))
889 return;
890
891 auto& resourceStatistic = ensureResourceStatisticsForRegistrableDomain(domain);
892 setPrevalentResource(resourceStatistic, ResourceLoadPrevalence::VeryHigh);
893}
894
895} // namespace WebKit
896
897#endif
898