1/*
2 * Copyright (C) 2017 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27#include "CacheStorageEngine.h"
28
29#include "CacheStorageEngineCaches.h"
30#include "NetworkCacheCoders.h"
31#include "NetworkCacheIOChannel.h"
32#include "NetworkCacheKey.h"
33#include "NetworkProcess.h"
34#include "WebCoreArgumentCoders.h"
35#include <WebCore/CacheQueryOptions.h>
36#include <WebCore/HTTPParsers.h>
37#include <pal/SessionID.h>
38#include <wtf/MainThread.h>
39#include <wtf/NeverDestroyed.h>
40#include <wtf/UUID.h>
41#include <wtf/persistence/PersistentCoders.h>
42#include <wtf/persistence/PersistentDecoder.h>
43#include <wtf/persistence/PersistentEncoder.h>
44#include <wtf/text/StringBuilder.h>
45#include <wtf/text/StringHash.h>
46
47namespace WebKit {
48
49namespace CacheStorage {
50
51using namespace WebCore;
52using namespace WebCore::DOMCacheEngine;
53using namespace NetworkCache;
54
55static inline String computeKeyURL(const URL& url)
56{
57 URL keyURL { url };
58 keyURL.removeQueryAndFragmentIdentifier();
59 return keyURL.string();
60}
61
62static inline Vector<uint64_t> queryCache(const Vector<RecordInformation>* records, const ResourceRequest& request, const CacheQueryOptions& options)
63{
64 if (!records)
65 return { };
66
67 if (!options.ignoreMethod && request.httpMethod() != "GET")
68 return { };
69
70 Vector<uint64_t> results;
71 for (const auto& record : *records) {
72 if (WebCore::DOMCacheEngine::queryCacheMatch(request, record.url, record.hasVaryStar, record.varyHeaders, options))
73 results.append(record.identifier);
74 }
75 return results;
76}
77
78static inline void updateVaryInformation(RecordInformation& recordInformation, const ResourceRequest& request, const ResourceResponse& response)
79{
80 auto varyValue = response.httpHeaderField(WebCore::HTTPHeaderName::Vary);
81 if (varyValue.isNull()) {
82 recordInformation.hasVaryStar = false;
83 recordInformation.varyHeaders = { };
84 return;
85 }
86
87 varyValue.split(',', [&](StringView view) {
88 if (!recordInformation.hasVaryStar && stripLeadingAndTrailingHTTPSpaces(view) == "*")
89 recordInformation.hasVaryStar = true;
90 String headerName = view.toString();
91 recordInformation.varyHeaders.add(headerName, request.httpHeaderField(headerName));
92 });
93
94 if (recordInformation.hasVaryStar)
95 recordInformation.varyHeaders = { };
96}
97
98RecordInformation Cache::toRecordInformation(const Record& record)
99{
100 Key key { "record"_s, m_uniqueName, { }, createCanonicalUUIDString(), m_caches.salt() };
101 RecordInformation recordInformation { WTFMove(key), MonotonicTime::now().secondsSinceEpoch().milliseconds(), record.identifier, 0 , record.responseBodySize, record.request.url(), false, { } };
102
103 updateVaryInformation(recordInformation, record.request, record.response);
104
105 return recordInformation;
106}
107
108Cache::Cache(Caches& caches, uint64_t identifier, State state, String&& name, String&& uniqueName)
109 : m_caches(caches)
110 , m_state(state)
111 , m_identifier(identifier)
112 , m_name(WTFMove(name))
113 , m_uniqueName(WTFMove(uniqueName))
114{
115}
116
117void Cache::dispose()
118{
119 m_caches.dispose(*this);
120}
121
122void Cache::clearMemoryRepresentation()
123{
124 m_records = { };
125 m_nextRecordIdentifier = 0;
126 m_state = State::Uninitialized;
127}
128
129static RecordInformation isolatedCopy(const RecordInformation& information)
130{
131 auto result = RecordInformation { information.key, information.insertionTime, information.identifier, information.updateResponseCounter, information.size, information.url.isolatedCopy(), information.hasVaryStar, { } };
132 HashMap<String, String> varyHeaders;
133 for (const auto& keyValue : information.varyHeaders)
134 varyHeaders.set(keyValue.key.isolatedCopy(), keyValue.value.isolatedCopy());
135 result.varyHeaders = WTFMove(varyHeaders);
136 return result;
137}
138
139struct TraversalResult {
140 uint64_t cacheIdentifier;
141 HashMap<String, Vector<RecordInformation>> records;
142 Vector<Key> failedRecords;
143};
144
145static TraversalResult isolatedCopy(TraversalResult&& result)
146{
147 HashMap<String, Vector<RecordInformation>> isolatedRecords;
148 for (auto& keyValue : result.records) {
149 auto& recordVector = keyValue.value;
150 for (size_t cptr = 0; cptr < recordVector.size(); cptr++)
151 recordVector[cptr] = isolatedCopy(recordVector[cptr]);
152
153 isolatedRecords.set(keyValue.key.isolatedCopy(), WTFMove(recordVector));
154 }
155
156 // No need to isolate keys since they are isolated through the copy constructor
157 return TraversalResult { result.cacheIdentifier, WTFMove(isolatedRecords), WTFMove(result.failedRecords) };
158}
159
160void Cache::open(CompletionCallback&& callback)
161{
162 if (m_state == State::Open) {
163 callback(WTF::nullopt);
164 return;
165 }
166 if (m_state == State::Opening) {
167 m_pendingOpeningCallbacks.append(WTFMove(callback));
168 return;
169 }
170 m_state = State::Opening;
171 TraversalResult traversalResult { m_identifier, { }, { } };
172 m_caches.readRecordsList(*this, [caches = makeRef(m_caches), callback = WTFMove(callback), traversalResult = WTFMove(traversalResult)](const auto* storageRecord, const auto&) mutable {
173 if (!storageRecord) {
174 RunLoop::main().dispatch([caches = WTFMove(caches), callback = WTFMove(callback), traversalResult = isolatedCopy(WTFMove(traversalResult)) ]() mutable {
175 for (auto& key : traversalResult.failedRecords)
176 caches->removeCacheEntry(key);
177
178 auto* cache = caches->find(traversalResult.cacheIdentifier);
179 if (!cache) {
180 callback(Error::Internal);
181 return;
182 }
183 cache->m_records = WTFMove(traversalResult.records);
184 cache->finishOpening(WTFMove(callback), WTF::nullopt);
185 });
186 return;
187 }
188
189 auto decoded = decodeRecordHeader(*storageRecord);
190 if (!decoded) {
191 traversalResult.failedRecords.append(storageRecord->key);
192 return;
193 }
194
195 auto& record = decoded->record;
196 auto insertionTime = decoded->insertionTime;
197
198 RecordInformation recordInformation { storageRecord->key, insertionTime, 0, 0, record.responseBodySize, record.request.url(), false, { } };
199 updateVaryInformation(recordInformation, record.request, record.response);
200
201 auto& sameURLRecords = traversalResult.records.ensure(computeKeyURL(recordInformation.url), [] { return Vector<RecordInformation> { }; }).iterator->value;
202 sameURLRecords.append(WTFMove(recordInformation));
203 });
204}
205
206void Cache::finishOpening(CompletionCallback&& callback, Optional<Error>&& error)
207{
208 Vector<std::reference_wrapper<RecordInformation>> records;
209 for (auto& value : m_records.values()) {
210 for (auto& record : value)
211 records.append(record);
212 }
213 std::sort(records.begin(), records.end(), [&](const auto& a, const auto& b) {
214 return a.get().insertionTime < b.get().insertionTime;
215 });
216 for (auto& record : records)
217 record.get().identifier = ++m_nextRecordIdentifier;
218
219 if (error) {
220 m_state = State::Uninitialized;
221 callback(error.value());
222 auto callbacks = WTFMove(m_pendingOpeningCallbacks);
223 for (auto& callback : callbacks)
224 callback(error.value());
225 return;
226 }
227 m_state = State::Open;
228
229 callback(WTF::nullopt);
230 auto callbacks = WTFMove(m_pendingOpeningCallbacks);
231 for (auto& callback : callbacks)
232 callback(WTF::nullopt);
233}
234
235class ReadRecordTaskCounter : public RefCounted<ReadRecordTaskCounter> {
236public:
237 using ReadRecordsCallback = WTF::Function<void(Vector<Record>&&, Vector<uint64_t>&&)>;
238 static Ref<ReadRecordTaskCounter> create(ReadRecordsCallback&& callback) { return adoptRef(*new ReadRecordTaskCounter(WTFMove(callback))); }
239
240 ~ReadRecordTaskCounter()
241 {
242 ASSERT(RunLoop::isMain());
243 if (!m_callback)
244 return;
245 std::sort(m_records.begin(), m_records.end(), [&] (const auto& a, const auto& b) {
246 return a.identifier < b.identifier;
247 });
248 m_callback(WTFMove(m_records), WTFMove(m_failedRecords));
249 }
250
251 void appendRecord(Expected<Record, Error>&& result, uint64_t recordIdentifier, uint64_t updateCounter)
252 {
253 ASSERT(RunLoop::isMain());
254 if (!result.has_value()) {
255 m_failedRecords.append(recordIdentifier);
256 return;
257 }
258 result.value().identifier = recordIdentifier;
259 result.value().updateResponseCounter = updateCounter;
260 m_records.append(WTFMove(result.value()));
261 }
262
263private:
264 explicit ReadRecordTaskCounter(ReadRecordsCallback&& callback)
265 : m_callback(WTFMove(callback))
266 {
267 }
268
269 ReadRecordsCallback m_callback;
270 Vector<Record> m_records;
271 Vector<uint64_t> m_failedRecords;
272};
273
274void Cache::retrieveRecord(const RecordInformation& record, Ref<ReadRecordTaskCounter>&& taskCounter)
275{
276 readRecordFromDisk(record, [caches = makeRef(m_caches), identifier = m_identifier, recordIdentifier = record.identifier, updateCounter = record.updateResponseCounter, taskCounter = WTFMove(taskCounter)](Expected<Record, Error>&& result) mutable {
277 auto* cache = caches->find(identifier);
278 if (!cache)
279 return;
280 taskCounter->appendRecord(WTFMove(result), recordIdentifier, updateCounter);
281 });
282}
283
284void Cache::retrieveRecords(const URL& url, RecordsCallback&& callback)
285{
286 ASSERT(m_state == State::Open);
287
288 auto taskCounter = ReadRecordTaskCounter::create([caches = makeRef(m_caches), identifier = m_identifier, callback = WTFMove(callback)](Vector<Record>&& records, Vector<uint64_t>&& failedRecordIdentifiers) mutable {
289 auto* cache = caches->find(identifier);
290 if (cache)
291 cache->removeFromRecordList(failedRecordIdentifiers);
292 callback(WTFMove(records));
293 });
294
295 if (url.isNull()) {
296 for (auto& records : m_records.values()) {
297 for (auto& record : records)
298 retrieveRecord(record, taskCounter.copyRef());
299 }
300 return;
301 }
302
303 auto* records = recordsFromURL(url);
304 if (!records)
305 return;
306
307 for (auto& record : *records)
308 retrieveRecord(record, taskCounter.copyRef());
309}
310
311RecordInformation& Cache::addRecord(Vector<RecordInformation>* records, const Record& record)
312{
313 if (!records) {
314 auto key = computeKeyURL(record.request.url());
315 ASSERT(!m_records.contains(key));
316 records = &m_records.set(key, Vector<RecordInformation> { }).iterator->value;
317 }
318 records->append(toRecordInformation(record));
319 return records->last();
320}
321
322Vector<RecordInformation>* Cache::recordsFromURL(const URL& url)
323{
324 auto iterator = m_records.find(computeKeyURL(url));
325 if (iterator == m_records.end())
326 return nullptr;
327 return &iterator->value;
328}
329
330const Vector<RecordInformation>* Cache::recordsFromURL(const URL& url) const
331{
332 auto iterator = m_records.find(computeKeyURL(url));
333 if (iterator == m_records.end())
334 return nullptr;
335 return &iterator->value;
336}
337
338class AsynchronousPutTaskCounter : public RefCounted<AsynchronousPutTaskCounter> {
339public:
340 static Ref<AsynchronousPutTaskCounter> create(RecordIdentifiersCallback&& callback) { return adoptRef(*new AsynchronousPutTaskCounter(WTFMove(callback))); }
341 ~AsynchronousPutTaskCounter()
342 {
343 ASSERT(RunLoop::isMain());
344 if (!m_callback)
345 return;
346 if (m_error) {
347 m_callback(makeUnexpected(m_error.value()));
348 return;
349 }
350 m_callback(WTFMove(m_recordIdentifiers));
351 }
352
353 void setError(Error error)
354 {
355 ASSERT(RunLoop::isMain());
356 if (m_error)
357 return;
358
359 m_error = error;
360 }
361
362 void addRecordIdentifier(uint64_t identifier)
363 {
364 m_recordIdentifiers.append(identifier);
365 }
366
367private:
368 explicit AsynchronousPutTaskCounter(RecordIdentifiersCallback&& callback)
369 : m_callback(WTFMove(callback))
370 {
371 }
372
373 Optional<Error> m_error;
374 RecordIdentifiersCallback m_callback;
375 Vector<uint64_t> m_recordIdentifiers;
376};
377
378void Cache::storeRecords(Vector<Record>&& records, RecordIdentifiersCallback&& callback)
379{
380 auto taskCounter = AsynchronousPutTaskCounter::create(WTFMove(callback));
381
382 WebCore::CacheQueryOptions options;
383 for (auto& record : records) {
384 auto* sameURLRecords = recordsFromURL(record.request.url());
385 auto matchingRecords = queryCache(sameURLRecords, record.request, options);
386
387 auto position = !matchingRecords.isEmpty() ? sameURLRecords->findMatching([&](const auto& item) { return item.identifier == matchingRecords[0]; }) : notFound;
388
389 if (position == notFound) {
390 record.identifier = ++m_nextRecordIdentifier;
391 taskCounter->addRecordIdentifier(record.identifier);
392
393 auto& recordToWrite = addRecord(sameURLRecords, record);
394 writeRecordToDisk(recordToWrite, WTFMove(record), taskCounter.copyRef(), 0);
395 } else {
396 auto& existingRecord = sameURLRecords->at(position);
397 taskCounter->addRecordIdentifier(existingRecord.identifier);
398 updateRecordToDisk(existingRecord, WTFMove(record), taskCounter.copyRef());
399 }
400 }
401}
402
403void Cache::put(Vector<Record>&& records, RecordIdentifiersCallback&& callback)
404{
405 ASSERT(m_state == State::Open);
406
407 WebCore::CacheQueryOptions options;
408 uint64_t spaceRequired = 0;
409
410 for (auto& record : records) {
411 auto* sameURLRecords = recordsFromURL(record.request.url());
412 auto matchingRecords = queryCache(sameURLRecords, record.request, options);
413
414 auto position = (sameURLRecords && !matchingRecords.isEmpty()) ? sameURLRecords->findMatching([&](const auto& item) { return item.identifier == matchingRecords[0]; }) : notFound;
415
416 spaceRequired += record.responseBodySize;
417 if (position != notFound)
418 spaceRequired -= sameURLRecords->at(position).size;
419 }
420
421 m_caches.requestSpace(spaceRequired, [caches = makeRef(m_caches), identifier = m_identifier, records = WTFMove(records), callback = WTFMove(callback)](Optional<DOMCacheEngine::Error>&& error) mutable {
422 if (error) {
423 callback(makeUnexpected(error.value()));
424 return;
425 }
426 auto* cache = caches->find(identifier);
427 if (!cache) {
428 callback(makeUnexpected(DOMCacheEngine::Error::Internal));
429 return;
430 }
431 cache->storeRecords(WTFMove(records), WTFMove(callback));
432 });
433}
434
435void Cache::remove(WebCore::ResourceRequest&& request, WebCore::CacheQueryOptions&& options, RecordIdentifiersCallback&& callback)
436{
437 ASSERT(m_state == State::Open);
438
439 auto* records = recordsFromURL(request.url());
440 auto recordIdentifiers = queryCache(records, request, options);
441 if (recordIdentifiers.isEmpty()) {
442 callback({ });
443 return;
444 }
445
446 records->removeAllMatching([this, &recordIdentifiers](auto& item) {
447 bool shouldRemove = recordIdentifiers.findMatching([&item](auto identifier) { return identifier == item.identifier; }) != notFound;
448 if (shouldRemove)
449 this->removeRecordFromDisk(item);
450 return shouldRemove;
451 });
452
453 callback(WTFMove(recordIdentifiers));
454}
455
456void Cache::removeFromRecordList(const Vector<uint64_t>& recordIdentifiers)
457{
458 if (recordIdentifiers.isEmpty())
459 return;
460
461 for (auto& records : m_records.values()) {
462 auto* cache = this;
463 records.removeAllMatching([cache, &recordIdentifiers](const auto& item) {
464 return notFound != recordIdentifiers.findMatching([cache, &item](const auto& identifier) {
465 if (item.identifier != identifier)
466 return false;
467 cache->removeRecordFromDisk(item);
468 return true;
469 });
470 });
471 }
472}
473
474void Cache::writeRecordToDisk(const RecordInformation& recordInformation, Record&& record, Ref<AsynchronousPutTaskCounter>&& taskCounter, uint64_t previousRecordSize)
475{
476 m_caches.writeRecord(*this, recordInformation, WTFMove(record), previousRecordSize, [taskCounter = WTFMove(taskCounter)](Optional<Error>&& error) {
477 if (error)
478 taskCounter->setError(error.value());
479 });
480}
481
482void Cache::updateRecordToDisk(RecordInformation& existingRecord, Record&& record, Ref<AsynchronousPutTaskCounter>&& taskCounter)
483{
484 ++existingRecord.updateResponseCounter;
485 readRecordFromDisk(existingRecord, [caches = makeRef(m_caches), identifier = m_identifier, recordIdentifier = existingRecord.identifier, record = WTFMove(record), taskCounter = WTFMove(taskCounter)](Expected<Record, Error>&& result) mutable {
486 if (!result.has_value())
487 return;
488
489 auto* cache = caches->find(identifier);
490 if (!cache)
491 return;
492
493 auto* sameURLRecords = cache->recordsFromURL(result.value().request.url());
494 if (!sameURLRecords)
495 return;
496
497 auto position = sameURLRecords->findMatching([&] (const auto& item) { return item.identifier == recordIdentifier; });
498 if (position == notFound)
499 return;
500 auto& recordInfo = sameURLRecords->at(position);
501 auto previousSize = recordInfo.size;
502 recordInfo.size = record.responseBodySize;
503
504 auto& recordFromDisk = result.value();
505 record.requestHeadersGuard = recordFromDisk.requestHeadersGuard;
506 record.request = WTFMove(recordFromDisk.request);
507 record.options = WTFMove(recordFromDisk.options);
508 record.referrer = WTFMove(recordFromDisk.referrer);
509
510 updateVaryInformation(recordInfo, record.request, record.response);
511
512 cache->writeRecordToDisk(recordInfo, WTFMove(record), WTFMove(taskCounter), previousSize);
513 });
514}
515
516void Cache::readRecordFromDisk(const RecordInformation& record, WTF::Function<void(Expected<Record, Error>&&)>&& callback)
517{
518 m_caches.readRecord(record.key, WTFMove(callback));
519}
520
521void Cache::removeRecordFromDisk(const RecordInformation& record)
522{
523 m_caches.removeRecord(record);
524}
525
526Storage::Record Cache::encode(const RecordInformation& recordInformation, const Record& record)
527{
528 WTF::Persistence::Encoder encoder;
529 encoder << recordInformation.insertionTime;
530 encoder << recordInformation.size;
531 encoder << record.requestHeadersGuard;
532 record.request.encodeWithoutPlatformData(encoder);
533 record.options.encodePersistent(encoder);
534 encoder << record.referrer;
535
536 encoder << record.responseHeadersGuard;
537 encoder << record.response;
538 encoder << record.responseBodySize;
539
540 encoder.encodeChecksum();
541
542 Data header(encoder.buffer(), encoder.bufferSize());
543 Data body;
544 WTF::switchOn(record.responseBody, [](const Ref<WebCore::FormData>& formData) {
545 // FIXME: Store form data body.
546 }, [&](const Ref<WebCore::SharedBuffer>& buffer) {
547 body = { reinterpret_cast<const uint8_t*>(buffer->data()), buffer->size() };
548 }, [](const std::nullptr_t&) {
549 });
550
551 return { recordInformation.key, { }, header, body, { } };
552}
553
554Optional<Cache::DecodedRecord> Cache::decodeRecordHeader(const Storage::Record& storage)
555{
556 WTF::Persistence::Decoder decoder(storage.header.data(), storage.header.size());
557
558 Record record;
559
560 double insertionTime;
561 if (!decoder.decode(insertionTime))
562 return WTF::nullopt;
563
564 uint64_t size;
565 if (!decoder.decode(size))
566 return WTF::nullopt;
567
568 if (!decoder.decode(record.requestHeadersGuard))
569 return WTF::nullopt;
570
571 if (!record.request.decodeWithoutPlatformData(decoder))
572 return WTF::nullopt;
573
574 if (!FetchOptions::decodePersistent(decoder, record.options))
575 return WTF::nullopt;
576
577 if (!decoder.decode(record.referrer))
578 return WTF::nullopt;
579
580 if (!decoder.decode(record.responseHeadersGuard))
581 return WTF::nullopt;
582
583 if (!decoder.decode(record.response))
584 return WTF::nullopt;
585
586 if (!decoder.decode(record.responseBodySize))
587 return WTF::nullopt;
588
589 if (!decoder.verifyChecksum())
590 return WTF::nullopt;
591
592 return DecodedRecord { insertionTime, size, WTFMove(record) };
593}
594
595Optional<Record> Cache::decode(const Storage::Record& storage)
596{
597 auto result = decodeRecordHeader(storage);
598
599 if (!result)
600 return WTF::nullopt;
601
602 auto record = WTFMove(result->record);
603 record.responseBody = WebCore::SharedBuffer::create(storage.body.data(), storage.body.size());
604
605 return record;
606}
607
608Vector<Key> Cache::keys() const
609{
610 Vector<Key> keys;
611 for (auto& records : m_records.values()) {
612 for (auto& record : records)
613 keys.append(record.key);
614 }
615 return keys;
616}
617
618} // namespace CacheStorage
619
620} // namespace WebKit
621