diff options
Diffstat (limited to 'Source/WebKit2/NetworkProcess/cache')
32 files changed, 6353 insertions, 0 deletions
diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCache.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCache.cpp new file mode 100644 index 000000000..9ea0d3f41 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCache.cpp @@ -0,0 +1,656 @@ +/* + * Copyright (C) 2014-2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCache.h" + +#if ENABLE(NETWORK_CACHE) + +#include "Logging.h" +#include "NetworkCacheSpeculativeLoadManager.h" +#include "NetworkCacheStatistics.h" +#include "NetworkCacheStorage.h" +#include <WebCore/CacheValidation.h> +#include <WebCore/FileSystem.h> +#include <WebCore/HTTPHeaderNames.h> +#include <WebCore/NetworkStorageSession.h> +#include <WebCore/PlatformCookieJar.h> +#include <WebCore/ResourceRequest.h> +#include <WebCore/ResourceResponse.h> +#include <WebCore/SharedBuffer.h> +#include <wtf/MainThread.h> +#include <wtf/NeverDestroyed.h> +#include <wtf/RunLoop.h> +#include <wtf/text/StringBuilder.h> + +#if PLATFORM(COCOA) +#include <notify.h> +#endif + +namespace WebKit { +namespace NetworkCache { + +static const AtomicString& resourceType() +{ + ASSERT(WTF::isMainThread()); + static NeverDestroyed<const AtomicString> resource("resource", AtomicString::ConstructFromLiteral); + return resource; +} + +Cache& singleton() +{ + static NeverDestroyed<Cache> instance; + return instance; +} + +#if PLATFORM(GTK) +static void dumpFileChanged(Cache* cache) +{ + cache->dumpContentsToFile(); +} +#endif + +bool Cache::initialize(const String& cachePath, const Parameters& parameters) +{ + m_storage = Storage::open(cachePath); + +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) + if (parameters.enableNetworkCacheSpeculativeRevalidation) + m_speculativeLoadManager = std::make_unique<SpeculativeLoadManager>(*m_storage); +#endif + + if (parameters.enableEfficacyLogging) + m_statistics = Statistics::open(cachePath); + +#if PLATFORM(COCOA) + // Triggers with "notifyutil -p com.apple.WebKit.Cache.dump". + if (m_storage) { + int token; + notify_register_dispatch("com.apple.WebKit.Cache.dump", &token, dispatch_get_main_queue(), ^(int) { + dumpContentsToFile(); + }); + } +#endif +#if PLATFORM(GTK) + // Triggers with "touch $cachePath/dump". + if (m_storage) { + CString dumpFilePath = WebCore::fileSystemRepresentation(WebCore::pathByAppendingComponent(m_storage->basePath(), "dump")); + GRefPtr<GFile> dumpFile = adoptGRef(g_file_new_for_path(dumpFilePath.data())); + GFileMonitor* monitor = g_file_monitor_file(dumpFile.get(), G_FILE_MONITOR_NONE, nullptr, nullptr); + g_signal_connect_swapped(monitor, "changed", G_CALLBACK(dumpFileChanged), this); + } +#endif + + LOG(NetworkCache, "(NetworkProcess) opened cache storage, success %d", !!m_storage); + return !!m_storage; +} + +void Cache::setCapacity(size_t maximumSize) +{ + if (!m_storage) + return; + m_storage->setCapacity(maximumSize); +} + +static Key makeCacheKey(const WebCore::ResourceRequest& request) +{ +#if ENABLE(CACHE_PARTITIONING) + String partition = request.cachePartition(); +#else + String partition; +#endif + if (partition.isEmpty()) + partition = ASCIILiteral("No partition"); + + // FIXME: This implements minimal Range header disk cache support. We don't parse + // ranges so only the same exact range request will be served from the cache. + String range = request.httpHeaderField(WebCore::HTTPHeaderName::Range); + return { partition, resourceType(), range, request.url().string() }; +} + +static String headerValueForVary(const WebCore::ResourceRequest& request, const String& headerName) +{ + // Explicit handling for cookies is needed because they are added magically by the networking layer. + // FIXME: The value might have changed between making the request and retrieving the cookie here. + // We could fetch the cookie when making the request but that seems overkill as the case is very rare and it + // is a blocking operation. This should be sufficient to cover reasonable cases. + if (headerName == httpHeaderNameString(WebCore::HTTPHeaderName::Cookie)) + return WebCore::cookieRequestHeaderFieldValue(WebCore::NetworkStorageSession::defaultStorageSession(), request.firstPartyForCookies(), request.url()); + return request.httpHeaderField(headerName); +} + +static Vector<std::pair<String, String>> collectVaryingRequestHeaders(const WebCore::ResourceRequest& request, const WebCore::ResourceResponse& response) +{ + String varyValue = response.httpHeaderField(WebCore::HTTPHeaderName::Vary); + if (varyValue.isEmpty()) + return { }; + Vector<String> varyingHeaderNames; + varyValue.split(',', /*allowEmptyEntries*/ false, varyingHeaderNames); + Vector<std::pair<String, String>> varyingRequestHeaders; + varyingRequestHeaders.reserveCapacity(varyingHeaderNames.size()); + for (auto& varyHeaderName : varyingHeaderNames) { + String headerName = varyHeaderName.stripWhiteSpace(); + String headerValue = headerValueForVary(request, headerName); + varyingRequestHeaders.append(std::make_pair(headerName, headerValue)); + } + return varyingRequestHeaders; +} + +static bool verifyVaryingRequestHeaders(const Vector<std::pair<String, String>>& varyingRequestHeaders, const WebCore::ResourceRequest& request) +{ + for (auto& varyingRequestHeader : varyingRequestHeaders) { + // FIXME: Vary: * in response would ideally trigger a cache delete instead of a store. + if (varyingRequestHeader.first == "*") + return false; + String headerValue = headerValueForVary(request, varyingRequestHeader.first); + if (headerValue != varyingRequestHeader.second) + return false; + } + return true; +} + +static bool cachePolicyAllowsExpired(WebCore::ResourceRequestCachePolicy policy) +{ + switch (policy) { + case WebCore::ReturnCacheDataElseLoad: + case WebCore::ReturnCacheDataDontLoad: + return true; + case WebCore::UseProtocolCachePolicy: + case WebCore::ReloadIgnoringCacheData: + return false; + } + ASSERT_NOT_REACHED(); + return false; +} + +static bool responseHasExpired(const WebCore::ResourceResponse& response, std::chrono::system_clock::time_point timestamp, Optional<std::chrono::microseconds> maxStale) +{ + if (response.cacheControlContainsNoCache()) + return true; + + auto age = WebCore::computeCurrentAge(response, timestamp); + auto lifetime = WebCore::computeFreshnessLifetimeForHTTPFamily(response, timestamp); + + auto maximumStaleness = maxStale ? maxStale.value() : 0_ms; + bool hasExpired = age - lifetime > maximumStaleness; + +#ifndef LOG_DISABLED + if (hasExpired) + LOG(NetworkCache, "(NetworkProcess) needsRevalidation hasExpired age=%f lifetime=%f max-stale=%g", age, lifetime, maxStale); +#endif + + return hasExpired; +} + +static bool responseNeedsRevalidation(const WebCore::ResourceResponse& response, const WebCore::ResourceRequest& request, std::chrono::system_clock::time_point timestamp) +{ + auto requestDirectives = WebCore::parseCacheControlDirectives(request.httpHeaderFields()); + if (requestDirectives.noCache) + return true; + // For requests we ignore max-age values other than zero. + if (requestDirectives.maxAge && requestDirectives.maxAge.value() == 0_ms) + return true; + + return responseHasExpired(response, timestamp, requestDirectives.maxStale); +} + +static UseDecision makeUseDecision(const Entry& entry, const WebCore::ResourceRequest& request) +{ + // The request is conditional so we force revalidation from the network. We merely check the disk cache + // so we can update the cache entry. + if (request.isConditional() && !entry.redirectRequest()) + return UseDecision::Validate; + + if (!verifyVaryingRequestHeaders(entry.varyingRequestHeaders(), request)) + return UseDecision::NoDueToVaryingHeaderMismatch; + + // We never revalidate in the case of a history navigation. + if (cachePolicyAllowsExpired(request.cachePolicy())) + return UseDecision::Use; + + if (!responseNeedsRevalidation(entry.response(), request, entry.timeStamp())) + return UseDecision::Use; + + if (!entry.response().hasCacheValidatorFields()) + return UseDecision::NoDueToMissingValidatorFields; + + return entry.redirectRequest() ? UseDecision::NoDueToExpiredRedirect : UseDecision::Validate; +} + +static RetrieveDecision makeRetrieveDecision(const WebCore::ResourceRequest& request) +{ + // FIXME: Support HEAD requests. + if (request.httpMethod() != "GET") + return RetrieveDecision::NoDueToHTTPMethod; + if (request.cachePolicy() == WebCore::ReloadIgnoringCacheData && !request.isConditional()) + return RetrieveDecision::NoDueToReloadIgnoringCache; + + return RetrieveDecision::Yes; +} + +// http://tools.ietf.org/html/rfc7231#page-48 +static bool isStatusCodeCacheableByDefault(int statusCode) +{ + switch (statusCode) { + case 200: // OK + case 203: // Non-Authoritative Information + case 204: // No Content + case 206: // Partial Content + case 300: // Multiple Choices + case 301: // Moved Permanently + case 404: // Not Found + case 405: // Method Not Allowed + case 410: // Gone + case 414: // URI Too Long + case 501: // Not Implemented + return true; + default: + return false; + } +} + +static bool isStatusCodePotentiallyCacheable(int statusCode) +{ + switch (statusCode) { + case 201: // Created + case 202: // Accepted + case 205: // Reset Content + case 302: // Found + case 303: // See Other + case 307: // Temporary redirect + case 403: // Forbidden + case 406: // Not Acceptable + case 415: // Unsupported Media Type + return true; + default: + return false; + } +} + +static bool isMediaMIMEType(const String& mimeType) +{ + if (mimeType.startsWith("video/", /*caseSensitive*/ false)) + return true; + if (mimeType.startsWith("audio/", /*caseSensitive*/ false)) + return true; + return false; +} + +static StoreDecision makeStoreDecision(const WebCore::ResourceRequest& originalRequest, const WebCore::ResourceResponse& response) +{ + if (!originalRequest.url().protocolIsInHTTPFamily() || !response.isHTTP()) + return StoreDecision::NoDueToProtocol; + + if (originalRequest.httpMethod() != "GET") + return StoreDecision::NoDueToHTTPMethod; + + auto requestDirectives = WebCore::parseCacheControlDirectives(originalRequest.httpHeaderFields()); + if (requestDirectives.noStore) + return StoreDecision::NoDueToNoStoreRequest; + + if (response.cacheControlContainsNoStore()) + return StoreDecision::NoDueToNoStoreResponse; + + if (!isStatusCodeCacheableByDefault(response.httpStatusCode())) { + // http://tools.ietf.org/html/rfc7234#section-4.3.2 + bool hasExpirationHeaders = response.expires() || response.cacheControlMaxAge(); + bool expirationHeadersAllowCaching = isStatusCodePotentiallyCacheable(response.httpStatusCode()) && hasExpirationHeaders; + if (!expirationHeadersAllowCaching) + return StoreDecision::NoDueToHTTPStatusCode; + } + + bool isMainResource = originalRequest.requester() == WebCore::ResourceRequest::Requester::Main; + bool storeUnconditionallyForHistoryNavigation = isMainResource || originalRequest.priority() == WebCore::ResourceLoadPriority::VeryHigh; + if (!storeUnconditionallyForHistoryNavigation) { + auto now = std::chrono::system_clock::now(); + bool hasNonZeroLifetime = !response.cacheControlContainsNoCache() && WebCore::computeFreshnessLifetimeForHTTPFamily(response, now) > 0_ms; + + bool possiblyReusable = response.hasCacheValidatorFields() || hasNonZeroLifetime; + if (!possiblyReusable) + return StoreDecision::NoDueToUnlikelyToReuse; + } + + // Media loaded via XHR is likely being used for MSE streaming (YouTube and Netflix for example). + // Streaming media fills the cache quickly and is unlikely to be reused. + // FIXME: We should introduce a separate media cache partition that doesn't affect other resources. + // FIXME: We should also make sure make the MSE paths are copy-free so we can use mapped buffers from disk effectively. + bool isLikelyStreamingMedia = originalRequest.requester() == WebCore::ResourceRequest::Requester::XHR && isMediaMIMEType(response.mimeType()); + if (isLikelyStreamingMedia) + return StoreDecision::NoDueToStreamingMedia; + + return StoreDecision::Yes; +} + +void Cache::retrieve(const WebCore::ResourceRequest& request, const GlobalFrameID& frameID, std::function<void (std::unique_ptr<Entry>)> completionHandler) +{ + ASSERT(isEnabled()); + ASSERT(request.url().protocolIsInHTTPFamily()); + + LOG(NetworkCache, "(NetworkProcess) retrieving %s priority %d", request.url().string().ascii().data(), static_cast<int>(request.priority())); + + if (m_statistics) + m_statistics->recordRetrievalRequest(frameID.first); + + Key storageKey = makeCacheKey(request); + +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) + if (m_speculativeLoadManager) + m_speculativeLoadManager->registerLoad(frameID, request, storageKey); +#endif + + auto retrieveDecision = makeRetrieveDecision(request); + if (retrieveDecision != RetrieveDecision::Yes) { + if (m_statistics) + m_statistics->recordNotUsingCacheForRequest(frameID.first, storageKey, request, retrieveDecision); + + completionHandler(nullptr); + return; + } + +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) + if (m_speculativeLoadManager && m_speculativeLoadManager->retrieve(frameID, storageKey, [request, completionHandler](std::unique_ptr<Entry> entry) { + if (entry && verifyVaryingRequestHeaders(entry->varyingRequestHeaders(), request)) + completionHandler(WTFMove(entry)); + else + completionHandler(nullptr); + })) + return; +#endif + + auto startTime = std::chrono::system_clock::now(); + auto priority = static_cast<unsigned>(request.priority()); + + m_storage->retrieve(storageKey, priority, [this, request, completionHandler, startTime, storageKey, frameID](std::unique_ptr<Storage::Record> record) { + if (!record) { + LOG(NetworkCache, "(NetworkProcess) not found in storage"); + + if (m_statistics) + m_statistics->recordRetrievalFailure(frameID.first, storageKey, request); + + completionHandler(nullptr); + return false; + } + + ASSERT(record->key == storageKey); + + auto entry = Entry::decodeStorageRecord(*record); + + auto useDecision = entry ? makeUseDecision(*entry, request) : UseDecision::NoDueToDecodeFailure; + switch (useDecision) { + case UseDecision::Use: + break; + case UseDecision::Validate: + entry->setNeedsValidation(); + break; + default: + entry = nullptr; + }; + +#if !LOG_DISABLED + auto elapsedMS = static_cast<int64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now() - startTime).count()); + LOG(NetworkCache, "(NetworkProcess) retrieve complete useDecision=%d priority=%d time=%" PRIi64 "ms", static_cast<int>(useDecision), static_cast<int>(request.priority()), elapsedMS); +#endif + completionHandler(WTFMove(entry)); + + if (m_statistics) + m_statistics->recordRetrievedCachedEntry(frameID.first, storageKey, request, useDecision); + return useDecision != UseDecision::NoDueToDecodeFailure; + }); +} + +std::unique_ptr<Entry> Cache::store(const WebCore::ResourceRequest& request, const WebCore::ResourceResponse& response, RefPtr<WebCore::SharedBuffer>&& responseData, std::function<void (MappedBody&)> completionHandler) +{ + ASSERT(isEnabled()); + ASSERT(responseData); + + LOG(NetworkCache, "(NetworkProcess) storing %s, partition %s", request.url().string().latin1().data(), makeCacheKey(request).partition().latin1().data()); + + StoreDecision storeDecision = makeStoreDecision(request, response); + if (storeDecision != StoreDecision::Yes) { + LOG(NetworkCache, "(NetworkProcess) didn't store, storeDecision=%d", static_cast<int>(storeDecision)); + auto key = makeCacheKey(request); + + auto isSuccessfulRevalidation = response.httpStatusCode() == 304; + if (!isSuccessfulRevalidation) { + // Make sure we don't keep a stale entry in the cache. + remove(key); + } + + if (m_statistics) + m_statistics->recordNotCachingResponse(key, storeDecision); + + return nullptr; + } + + std::unique_ptr<Entry> cacheEntry = std::make_unique<Entry>(makeCacheKey(request), response, WTFMove(responseData), collectVaryingRequestHeaders(request, response)); + + auto record = cacheEntry->encodeAsStorageRecord(); + + m_storage->store(record, [completionHandler](const Data& bodyData) { + MappedBody mappedBody; +#if ENABLE(SHAREABLE_RESOURCE) + if (RefPtr<SharedMemory> sharedMemory = bodyData.tryCreateSharedMemory()) { + mappedBody.shareableResource = ShareableResource::create(WTFMove(sharedMemory), 0, bodyData.size()); + ASSERT(mappedBody.shareableResource); + mappedBody.shareableResource->createHandle(mappedBody.shareableResourceHandle); + } +#endif + completionHandler(mappedBody); + LOG(NetworkCache, "(NetworkProcess) stored"); + }); + + return cacheEntry; +} + +std::unique_ptr<Entry> Cache::storeRedirect(const WebCore::ResourceRequest& request, const WebCore::ResourceResponse& response, const WebCore::ResourceRequest& redirectRequest) +{ + ASSERT(isEnabled()); + + LOG(NetworkCache, "(NetworkProcess) storing redirect %s -> %s", request.url().string().latin1().data(), redirectRequest.url().string().latin1().data()); + + StoreDecision storeDecision = makeStoreDecision(request, response); + if (storeDecision != StoreDecision::Yes) { + LOG(NetworkCache, "(NetworkProcess) didn't store redirect, storeDecision=%d", static_cast<int>(storeDecision)); + auto key = makeCacheKey(request); + if (m_statistics) + m_statistics->recordNotCachingResponse(key, storeDecision); + + return nullptr; + } + + std::unique_ptr<Entry> cacheEntry = std::make_unique<Entry>(makeCacheKey(request), response, redirectRequest, collectVaryingRequestHeaders(request, response)); + + auto record = cacheEntry->encodeAsStorageRecord(); + + m_storage->store(record, nullptr); + + return cacheEntry; +} + +std::unique_ptr<Entry> Cache::update(const WebCore::ResourceRequest& originalRequest, const GlobalFrameID& frameID, const Entry& existingEntry, const WebCore::ResourceResponse& validatingResponse) +{ + LOG(NetworkCache, "(NetworkProcess) updating %s", originalRequest.url().string().latin1().data()); + + WebCore::ResourceResponse response = existingEntry.response(); + WebCore::updateResponseHeadersAfterRevalidation(response, validatingResponse); + response.setSource(WebCore::ResourceResponse::Source::DiskCache); + + auto updateEntry = std::make_unique<Entry>(existingEntry.key(), response, existingEntry.buffer(), collectVaryingRequestHeaders(originalRequest, response)); + auto updateRecord = updateEntry->encodeAsStorageRecord(); + + m_storage->store(updateRecord, { }); + + if (m_statistics) + m_statistics->recordRevalidationSuccess(frameID.first, existingEntry.key(), originalRequest); + + return updateEntry; +} + +void Cache::remove(const Key& key) +{ + ASSERT(isEnabled()); + + m_storage->remove(key); +} + +void Cache::remove(const WebCore::ResourceRequest& request) +{ + remove(makeCacheKey(request)); +} + +void Cache::traverse(const std::function<void (const TraversalEntry*)>& traverseHandler) +{ + ASSERT(isEnabled()); + + // Protect against clients making excessive traversal requests. + const unsigned maximumTraverseCount = 3; + if (m_traverseCount >= maximumTraverseCount) { + WTFLogAlways("Maximum parallel cache traverse count exceeded. Ignoring traversal request."); + + RunLoop::main().dispatch([traverseHandler] { + traverseHandler(nullptr); + }); + return; + } + + ++m_traverseCount; + + m_storage->traverse(resourceType(), 0, [this, traverseHandler](const Storage::Record* record, const Storage::RecordInfo& recordInfo) { + if (!record) { + --m_traverseCount; + traverseHandler(nullptr); + return; + } + + auto entry = Entry::decodeStorageRecord(*record); + if (!entry) + return; + + TraversalEntry traversalEntry { *entry, recordInfo }; + traverseHandler(&traversalEntry); + }); +} + +String Cache::dumpFilePath() const +{ + return WebCore::pathByAppendingComponent(m_storage->versionPath(), "dump.json"); +} + +void Cache::dumpContentsToFile() +{ + if (!m_storage) + return; + auto fd = WebCore::openFile(dumpFilePath(), WebCore::OpenForWrite); + if (!WebCore::isHandleValid(fd)) + return; + auto prologue = String("{\n\"entries\": [\n").utf8(); + WebCore::writeToFile(fd, prologue.data(), prologue.length()); + + struct Totals { + unsigned count { 0 }; + double worth { 0 }; + size_t bodySize { 0 }; + }; + Totals totals; + auto flags = Storage::TraverseFlag::ComputeWorth | Storage::TraverseFlag::ShareCount; + size_t capacity = m_storage->capacity(); + m_storage->traverse(resourceType(), flags, [fd, totals, capacity](const Storage::Record* record, const Storage::RecordInfo& info) mutable { + if (!record) { + StringBuilder epilogue; + epilogue.appendLiteral("{}\n],\n"); + epilogue.appendLiteral("\"totals\": {\n"); + epilogue.appendLiteral("\"capacity\": "); + epilogue.appendNumber(capacity); + epilogue.appendLiteral(",\n"); + epilogue.appendLiteral("\"count\": "); + epilogue.appendNumber(totals.count); + epilogue.appendLiteral(",\n"); + epilogue.appendLiteral("\"bodySize\": "); + epilogue.appendNumber(totals.bodySize); + epilogue.appendLiteral(",\n"); + epilogue.appendLiteral("\"averageWorth\": "); + epilogue.appendNumber(totals.count ? totals.worth / totals.count : 0); + epilogue.appendLiteral("\n"); + epilogue.appendLiteral("}\n}\n"); + auto writeData = epilogue.toString().utf8(); + WebCore::writeToFile(fd, writeData.data(), writeData.length()); + WebCore::closeFile(fd); + return; + } + auto entry = Entry::decodeStorageRecord(*record); + if (!entry) + return; + ++totals.count; + totals.worth += info.worth; + totals.bodySize += info.bodySize; + + StringBuilder json; + entry->asJSON(json, info); + json.appendLiteral(",\n"); + auto writeData = json.toString().utf8(); + WebCore::writeToFile(fd, writeData.data(), writeData.length()); + }); +} + +void Cache::deleteDumpFile() +{ + auto queue = WorkQueue::create("com.apple.WebKit.Cache.delete"); + StringCapture dumpFilePathCapture(dumpFilePath()); + queue->dispatch([dumpFilePathCapture] { + WebCore::deleteFile(dumpFilePathCapture.string()); + }); +} + +void Cache::clear(std::chrono::system_clock::time_point modifiedSince, std::function<void ()>&& completionHandler) +{ + LOG(NetworkCache, "(NetworkProcess) clearing cache"); + + if (m_statistics) + m_statistics->clear(); + + if (!m_storage) { + RunLoop::main().dispatch(completionHandler); + return; + } + String anyType; + m_storage->clear(anyType, modifiedSince, WTFMove(completionHandler)); + + deleteDumpFile(); +} + +void Cache::clear() +{ + clear(std::chrono::system_clock::time_point::min(), nullptr); +} + +String Cache::recordsPath() const +{ + return m_storage ? m_storage->recordsPath() : String(); +} + +} +} + +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCache.h b/Source/WebKit2/NetworkProcess/cache/NetworkCache.h new file mode 100644 index 000000000..38db1fab8 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCache.h @@ -0,0 +1,145 @@ +/* + * Copyright (C) 2014-2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCache_h +#define NetworkCache_h + +#if ENABLE(NETWORK_CACHE) + +#include "NetworkCacheEntry.h" +#include "NetworkCacheStorage.h" +#include "ShareableResource.h" +#include <WebCore/ResourceResponse.h> +#include <wtf/text/WTFString.h> + +namespace WebCore { +class ResourceRequest; +class SharedBuffer; +class URL; +} + +namespace WebKit { +namespace NetworkCache { + +class Cache; +class SpeculativeLoadManager; +class Statistics; + +Cache& singleton(); + +struct MappedBody { +#if ENABLE(SHAREABLE_RESOURCE) + RefPtr<ShareableResource> shareableResource; + ShareableResource::Handle shareableResourceHandle; +#endif +}; + +enum class RetrieveDecision { + Yes, + NoDueToHTTPMethod, + NoDueToConditionalRequest, + NoDueToReloadIgnoringCache +}; + +// FIXME: This enum is used in the Statistics code in a way that prevents removing or reordering anything. +enum class StoreDecision { + Yes, + NoDueToProtocol, + NoDueToHTTPMethod, + NoDueToAttachmentResponse, // Unused. + NoDueToNoStoreResponse, + NoDueToHTTPStatusCode, + NoDueToNoStoreRequest, + NoDueToUnlikelyToReuse, + NoDueToStreamingMedia +}; + +enum class UseDecision { + Use, + Validate, + NoDueToVaryingHeaderMismatch, + NoDueToMissingValidatorFields, + NoDueToDecodeFailure, + NoDueToExpiredRedirect, +}; + +using GlobalFrameID = std::pair<uint64_t /*webPageID*/, uint64_t /*webFrameID*/>; + +class Cache { + WTF_MAKE_NONCOPYABLE(Cache); + friend class WTF::NeverDestroyed<Cache>; +public: + struct Parameters { + bool enableEfficacyLogging; +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) + bool enableNetworkCacheSpeculativeRevalidation; +#endif + }; + bool initialize(const String& cachePath, const Parameters&); + void setCapacity(size_t); + + bool isEnabled() const { return !!m_storage; } + + // Completion handler may get called back synchronously on failure. + void retrieve(const WebCore::ResourceRequest&, const GlobalFrameID&, std::function<void (std::unique_ptr<Entry>)>); + std::unique_ptr<Entry> store(const WebCore::ResourceRequest&, const WebCore::ResourceResponse&, RefPtr<WebCore::SharedBuffer>&&, std::function<void (MappedBody&)>); + std::unique_ptr<Entry> storeRedirect(const WebCore::ResourceRequest&, const WebCore::ResourceResponse&, const WebCore::ResourceRequest& redirectRequest); + std::unique_ptr<Entry> update(const WebCore::ResourceRequest&, const GlobalFrameID&, const Entry&, const WebCore::ResourceResponse& validatingResponse); + + struct TraversalEntry { + const Entry& entry; + const Storage::RecordInfo& recordInfo; + }; + void traverse(const std::function<void (const TraversalEntry*)>&); + void remove(const Key&); + void remove(const WebCore::ResourceRequest&); + + void clear(); + void clear(std::chrono::system_clock::time_point modifiedSince, std::function<void ()>&& completionHandler); + + void dumpContentsToFile(); + + String recordsPath() const; + +private: + Cache() = default; + ~Cache() = delete; + + String dumpFilePath() const; + void deleteDumpFile(); + + std::unique_ptr<Storage> m_storage; +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) + std::unique_ptr<SpeculativeLoadManager> m_speculativeLoadManager; +#endif + std::unique_ptr<Statistics> m_statistics; + + unsigned m_traverseCount { 0 }; +}; + +} +} +#endif +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheBlobStorage.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheBlobStorage.cpp new file mode 100644 index 000000000..32c0a1ec5 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheBlobStorage.cpp @@ -0,0 +1,151 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCacheBlobStorage.h" + +#if ENABLE(NETWORK_CACHE) + +#include "Logging.h" +#include "NetworkCacheFileSystem.h" +#include <WebCore/FileSystem.h> +#include <fcntl.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <wtf/RunLoop.h> +#include <wtf/SHA1.h> +#include <wtf/text/StringBuilder.h> + +namespace WebKit { +namespace NetworkCache { + +BlobStorage::BlobStorage(const String& blobDirectoryPath) + : m_blobDirectoryPath(blobDirectoryPath) +{ +} + +String BlobStorage::blobDirectoryPath() const +{ + return m_blobDirectoryPath.isolatedCopy(); +} + +void BlobStorage::synchronize() +{ + ASSERT(!RunLoop::isMain()); + + WebCore::makeAllDirectories(blobDirectoryPath()); + + m_approximateSize = 0; + auto blobDirectory = blobDirectoryPath(); + traverseDirectory(blobDirectory, [this, &blobDirectory](const String& name, DirectoryEntryType type) { + if (type != DirectoryEntryType::File) + return; + auto path = WebCore::pathByAppendingComponent(blobDirectory, name); + auto filePath = WebCore::fileSystemRepresentation(path); + struct stat stat; + ::stat(filePath.data(), &stat); + // No clients left for this blob. + if (stat.st_nlink == 1) + unlink(filePath.data()); + else + m_approximateSize += stat.st_size; + }); + + LOG(NetworkCacheStorage, "(NetworkProcess) blob synchronization completed approximateSize=%zu", approximateSize()); +} + +String BlobStorage::blobPathForHash(const SHA1::Digest& hash) const +{ + auto hashAsString = SHA1::hexDigest(hash); + return WebCore::pathByAppendingComponent(blobDirectoryPath(), String::fromUTF8(hashAsString)); +} + +BlobStorage::Blob BlobStorage::add(const String& path, const Data& data) +{ + ASSERT(!RunLoop::isMain()); + + auto hash = computeSHA1(data); + if (data.isEmpty()) + return { data, hash }; + + auto blobPath = WebCore::fileSystemRepresentation(blobPathForHash(hash)); + auto linkPath = WebCore::fileSystemRepresentation(path); + unlink(linkPath.data()); + + bool blobExists = access(blobPath.data(), F_OK) != -1; + if (blobExists) { + auto existingData = mapFile(blobPath.data()); + if (bytesEqual(existingData, data)) { + link(blobPath.data(), linkPath.data()); + return { existingData, hash }; + } + unlink(blobPath.data()); + } + + auto mappedData = data.mapToFile(blobPath.data()); + if (mappedData.isNull()) + return { }; + + link(blobPath.data(), linkPath.data()); + + m_approximateSize += mappedData.size(); + + return { mappedData, hash }; +} + +BlobStorage::Blob BlobStorage::get(const String& path) +{ + ASSERT(!RunLoop::isMain()); + + auto linkPath = WebCore::fileSystemRepresentation(path); + auto data = mapFile(linkPath.data()); + + return { data, computeSHA1(data) }; +} + +void BlobStorage::remove(const String& path) +{ + ASSERT(!RunLoop::isMain()); + + auto linkPath = WebCore::fileSystemRepresentation(path); + unlink(linkPath.data()); +} + +unsigned BlobStorage::shareCount(const String& path) +{ + ASSERT(!RunLoop::isMain()); + + auto linkPath = WebCore::fileSystemRepresentation(path); + struct stat stat; + if (::stat(linkPath.data(), &stat) < 0) + return 0; + // Link count is 2 in the single client case (the blob file and a link). + return stat.st_nlink - 1; +} + +} +} + +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheBlobStorage.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheBlobStorage.h new file mode 100644 index 000000000..e4b494cf9 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheBlobStorage.h @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheBlobStorage_h +#define NetworkCacheBlobStorage_h + +#if ENABLE(NETWORK_CACHE) + +#include "NetworkCacheData.h" +#include "NetworkCacheKey.h" +#include <wtf/SHA1.h> + +namespace WebKit { +namespace NetworkCache { + +// BlobStorage deduplicates the data using SHA1 hash computed over the blob bytes. +class BlobStorage { + WTF_MAKE_NONCOPYABLE(BlobStorage); +public: + BlobStorage(const String& blobDirectoryPath); + + struct Blob { + Data data; + SHA1::Digest hash; + }; + // These are all synchronous and should not be used from the main thread. + Blob add(const String& path, const Data&); + Blob get(const String& path); + + // Blob won't be removed until synchronization. + void remove(const String& path); + + unsigned shareCount(const String& path); + + size_t approximateSize() const { return m_approximateSize; } + + void synchronize(); + +private: + String blobDirectoryPath() const; + String blobPathForHash(const SHA1::Digest&) const; + + const String m_blobDirectoryPath; + + std::atomic<size_t> m_approximateSize { 0 }; +}; + +} +} + +#endif +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheCoder.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheCoder.h new file mode 100644 index 000000000..4b4be369d --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheCoder.h @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2010, 2014 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheCoder_h +#define NetworkCacheCoder_h + +#if ENABLE(NETWORK_CACHE) + +namespace WebKit { +namespace NetworkCache { + +class Decoder; +class Encoder; + +template<typename T> struct Coder { + static void encode(Encoder& encoder, const T& t) + { + t.encode(encoder); + } + + static bool decode(Decoder& decoder, T& t) + { + return T::decode(decoder, t); + } +}; + +} +} + +#endif +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheCoders.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheCoders.cpp new file mode 100644 index 000000000..868d7c3ed --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheCoders.cpp @@ -0,0 +1,182 @@ +/* + * Copyright (C) 2011, 2014-2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCacheCoders.h" + +#if ENABLE(NETWORK_CACHE) + +#include "WebCoreArgumentCoders.h" +#include <wtf/text/CString.h> +#include <wtf/text/WTFString.h> + +namespace WebKit { +namespace NetworkCache { + +void Coder<AtomicString>::encode(Encoder& encoder, const AtomicString& atomicString) +{ + encoder << atomicString.string(); +} + +bool Coder<AtomicString>::decode(Decoder& decoder, AtomicString& atomicString) +{ + String string; + if (!decoder.decode(string)) + return false; + + atomicString = string; + return true; +} + +void Coder<CString>::encode(Encoder& encoder, const CString& string) +{ + // Special case the null string. + if (string.isNull()) { + encoder << std::numeric_limits<uint32_t>::max(); + return; + } + + uint32_t length = string.length(); + encoder << length; + encoder.encodeFixedLengthData(reinterpret_cast<const uint8_t*>(string.data()), length); +} + +bool Coder<CString>::decode(Decoder& decoder, CString& result) +{ + uint32_t length; + if (!decoder.decode(length)) + return false; + + if (length == std::numeric_limits<uint32_t>::max()) { + // This is the null string. + result = CString(); + return true; + } + + // Before allocating the string, make sure that the decoder buffer is big enough. + if (!decoder.bufferIsLargeEnoughToContain<char>(length)) + return false; + + char* buffer; + CString string = CString::newUninitialized(length, buffer); + if (!decoder.decodeFixedLengthData(reinterpret_cast<uint8_t*>(buffer), length)) + return false; + + result = string; + return true; +} + + +void Coder<String>::encode(Encoder& encoder, const String& string) +{ + // Special case the null string. + if (string.isNull()) { + encoder << std::numeric_limits<uint32_t>::max(); + return; + } + + uint32_t length = string.length(); + bool is8Bit = string.is8Bit(); + + encoder << length << is8Bit; + + if (is8Bit) + encoder.encodeFixedLengthData(reinterpret_cast<const uint8_t*>(string.characters8()), length * sizeof(LChar)); + else + encoder.encodeFixedLengthData(reinterpret_cast<const uint8_t*>(string.characters16()), length * sizeof(UChar)); +} + +template <typename CharacterType> +static inline bool decodeStringText(Decoder& decoder, uint32_t length, String& result) +{ + // Before allocating the string, make sure that the decoder buffer is big enough. + if (!decoder.bufferIsLargeEnoughToContain<CharacterType>(length)) + return false; + + CharacterType* buffer; + String string = String::createUninitialized(length, buffer); + if (!decoder.decodeFixedLengthData(reinterpret_cast<uint8_t*>(buffer), length * sizeof(CharacterType))) + return false; + + result = string; + return true; +} + +bool Coder<String>::decode(Decoder& decoder, String& result) +{ + uint32_t length; + if (!decoder.decode(length)) + return false; + + if (length == std::numeric_limits<uint32_t>::max()) { + // This is the null string. + result = String(); + return true; + } + + bool is8Bit; + if (!decoder.decode(is8Bit)) + return false; + + if (is8Bit) + return decodeStringText<LChar>(decoder, length, result); + return decodeStringText<UChar>(decoder, length, result); +} + +void Coder<WebCore::CertificateInfo>::encode(Encoder& encoder, const WebCore::CertificateInfo& certificateInfo) +{ + // FIXME: Cocoa CertificateInfo is a CF object tree. Generalize CF type coding so we don't need to use ArgumentCoder here. + IPC::ArgumentEncoder argumentEncoder; + argumentEncoder << certificateInfo; + encoder << static_cast<uint64_t>(argumentEncoder.bufferSize()); + encoder.encodeFixedLengthData(argumentEncoder.buffer(), argumentEncoder.bufferSize()); +} + +bool Coder<WebCore::CertificateInfo>::decode(Decoder& decoder, WebCore::CertificateInfo& certificateInfo) +{ + uint64_t certificateSize; + if (!decoder.decode(certificateSize)) + return false; + Vector<uint8_t> data(certificateSize); + if (!decoder.decodeFixedLengthData(data.data(), data.size())) + return false; + IPC::ArgumentDecoder argumentDecoder(data.data(), data.size()); + return argumentDecoder.decode(certificateInfo); +} + +void Coder<SHA1::Digest>::encode(Encoder& encoder, const SHA1::Digest& digest) +{ + encoder.encodeFixedLengthData(digest.data(), sizeof(digest)); +} + +bool Coder<SHA1::Digest>::decode(Decoder& decoder, SHA1::Digest& digest) +{ + return decoder.decodeFixedLengthData(digest.data(), sizeof(digest)); +} + +} +} + +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheCoders.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheCoders.h new file mode 100644 index 000000000..13aefbf4f --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheCoders.h @@ -0,0 +1,264 @@ +/* + * Copyright (C) 2010, 2014-2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheCoders_h +#define NetworkCacheCoders_h + +#if ENABLE(NETWORK_CACHE) + +#include "NetworkCacheDecoder.h" +#include "NetworkCacheEncoder.h" +#include <WebCore/CertificateInfo.h> +#include <utility> +#include <wtf/Forward.h> +#include <wtf/HashMap.h> +#include <wtf/HashSet.h> +#include <wtf/SHA1.h> +#include <wtf/Vector.h> + +namespace WebKit { +namespace NetworkCache { + +template<typename T, typename U> struct Coder<std::pair<T, U>> { + static void encode(Encoder& encoder, const std::pair<T, U>& pair) + { + encoder << pair.first << pair.second; + } + + static bool decode(Decoder& decoder, std::pair<T, U>& pair) + { + T first; + if (!decoder.decode(first)) + return false; + + U second; + if (!decoder.decode(second)) + return false; + + pair.first = first; + pair.second = second; + return true; + } +}; + +template<typename Rep, typename Period> struct Coder<std::chrono::duration<Rep, Period>> { + static void encode(Encoder& encoder, const std::chrono::duration<Rep, Period>& duration) + { + static_assert(std::is_integral<Rep>::value && std::is_signed<Rep>::value && sizeof(Rep) <= sizeof(int64_t), "Serialization of this Rep type is not supported yet. Only signed integer type which can be fit in an int64_t is currently supported."); + encoder << static_cast<int64_t>(duration.count()); + } + + static bool decode(Decoder& decoder, std::chrono::duration<Rep, Period>& result) + { + int64_t count; + if (!decoder.decode(count)) + return false; + result = std::chrono::duration<Rep, Period>(static_cast<Rep>(count)); + return true; + } +}; + +template<typename KeyType, typename ValueType> struct Coder<WTF::KeyValuePair<KeyType, ValueType>> { + static void encode(Encoder& encoder, const WTF::KeyValuePair<KeyType, ValueType>& pair) + { + encoder << pair.key << pair.value; + } + + static bool decode(Decoder& decoder, WTF::KeyValuePair<KeyType, ValueType>& pair) + { + KeyType key; + if (!decoder.decode(key)) + return false; + + ValueType value; + if (!decoder.decode(value)) + return false; + + pair.key = key; + pair.value = value; + return true; + } +}; + +template<bool fixedSizeElements, typename T, size_t inlineCapacity> struct VectorCoder; + +template<typename T, size_t inlineCapacity> struct VectorCoder<false, T, inlineCapacity> { + static void encode(Encoder& encoder, const Vector<T, inlineCapacity>& vector) + { + encoder << static_cast<uint64_t>(vector.size()); + for (size_t i = 0; i < vector.size(); ++i) + encoder << vector[i]; + } + + static bool decode(Decoder& decoder, Vector<T, inlineCapacity>& vector) + { + uint64_t size; + if (!decoder.decode(size)) + return false; + + Vector<T, inlineCapacity> tmp; + for (size_t i = 0; i < size; ++i) { + T element; + if (!decoder.decode(element)) + return false; + + tmp.append(WTFMove(element)); + } + + tmp.shrinkToFit(); + vector.swap(tmp); + return true; + } +}; + +template<typename T, size_t inlineCapacity> struct VectorCoder<true, T, inlineCapacity> { + static void encode(Encoder& encoder, const Vector<T, inlineCapacity>& vector) + { + encoder << static_cast<uint64_t>(vector.size()); + encoder.encodeFixedLengthData(reinterpret_cast<const uint8_t*>(vector.data()), vector.size() * sizeof(T), alignof(T)); + } + + static bool decode(Decoder& decoder, Vector<T, inlineCapacity>& vector) + { + uint64_t size; + if (!decoder.decode(size)) + return false; + + // Since we know the total size of the elements, we can allocate the vector in + // one fell swoop. Before allocating we must however make sure that the decoder buffer + // is big enough. + if (!decoder.bufferIsLargeEnoughToContain<T>(size)) + return false; + + Vector<T, inlineCapacity> temp; + temp.resize(size); + + decoder.decodeFixedLengthData(reinterpret_cast<uint8_t*>(temp.data()), size * sizeof(T)); + + vector.swap(temp); + return true; + } +}; + +template<typename T, size_t inlineCapacity> struct Coder<Vector<T, inlineCapacity>> : VectorCoder<std::is_arithmetic<T>::value, T, inlineCapacity> { }; + +template<typename KeyArg, typename MappedArg, typename HashArg, typename KeyTraitsArg, typename MappedTraitsArg> struct Coder<HashMap<KeyArg, MappedArg, HashArg, KeyTraitsArg, MappedTraitsArg>> { + typedef HashMap<KeyArg, MappedArg, HashArg, KeyTraitsArg, MappedTraitsArg> HashMapType; + + static void encode(Encoder& encoder, const HashMapType& hashMap) + { + encoder << static_cast<uint64_t>(hashMap.size()); + for (typename HashMapType::const_iterator it = hashMap.begin(), end = hashMap.end(); it != end; ++it) + encoder << *it; + } + + static bool decode(Decoder& decoder, HashMapType& hashMap) + { + uint64_t hashMapSize; + if (!decoder.decode(hashMapSize)) + return false; + + HashMapType tempHashMap; + for (uint64_t i = 0; i < hashMapSize; ++i) { + KeyArg key; + MappedArg value; + if (!decoder.decode(key)) + return false; + if (!decoder.decode(value)) + return false; + + if (!tempHashMap.add(key, value).isNewEntry) { + // The hash map already has the specified key, bail. + return false; + } + } + + hashMap.swap(tempHashMap); + return true; + } +}; + +template<typename KeyArg, typename HashArg, typename KeyTraitsArg> struct Coder<HashSet<KeyArg, HashArg, KeyTraitsArg>> { + typedef HashSet<KeyArg, HashArg, KeyTraitsArg> HashSetType; + + static void encode(Encoder& encoder, const HashSetType& hashSet) + { + encoder << static_cast<uint64_t>(hashSet.size()); + for (typename HashSetType::const_iterator it = hashSet.begin(), end = hashSet.end(); it != end; ++it) + encoder << *it; + } + + static bool decode(Decoder& decoder, HashSetType& hashSet) + { + uint64_t hashSetSize; + if (!decoder.decode(hashSetSize)) + return false; + + HashSetType tempHashSet; + for (uint64_t i = 0; i < hashSetSize; ++i) { + KeyArg key; + if (!decoder.decode(key)) + return false; + + if (!tempHashSet.add(key).isNewEntry) { + // The hash map already has the specified key, bail. + return false; + } + } + + hashSet.swap(tempHashSet); + return true; + } +}; + +template<> struct Coder<AtomicString> { + static void encode(Encoder&, const AtomicString&); + static bool decode(Decoder&, AtomicString&); +}; + +template<> struct Coder<CString> { + static void encode(Encoder&, const CString&); + static bool decode(Decoder&, CString&); +}; + +template<> struct Coder<String> { + static void encode(Encoder&, const String&); + static bool decode(Decoder&, String&); +}; + +template<> struct Coder<WebCore::CertificateInfo> { + static void encode(Encoder&, const WebCore::CertificateInfo&); + static bool decode(Decoder&, WebCore::CertificateInfo&); +}; + +template<> struct Coder<SHA1::Digest> { + static void encode(Encoder&, const SHA1::Digest&); + static bool decode(Decoder&, SHA1::Digest&); +}; + +} +} +#endif +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheData.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheData.cpp new file mode 100644 index 000000000..f950d69ed --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheData.cpp @@ -0,0 +1,130 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCacheData.h" + +#if ENABLE(NETWORK_CACHE) + +#include <fcntl.h> +#include <sys/mman.h> +#include <sys/stat.h> + +namespace WebKit { +namespace NetworkCache { + +Data Data::mapToFile(const char* path) const +{ + int fd = open(path, O_CREAT | O_EXCL | O_RDWR , S_IRUSR | S_IWUSR); + if (fd < 0) + return { }; + + if (ftruncate(fd, m_size) < 0) { + close(fd); + return { }; + } + + void* map = mmap(nullptr, m_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (map == MAP_FAILED) { + close(fd); + return { }; + } + + uint8_t* mapData = static_cast<uint8_t*>(map); + apply([&mapData](const uint8_t* bytes, size_t bytesSize) { + memcpy(mapData, bytes, bytesSize); + mapData += bytesSize; + return true; + }); + + // Drop the write permission. + mprotect(map, m_size, PROT_READ); + + // Flush (asynchronously) to file, turning this into clean memory. + msync(map, m_size, MS_ASYNC); + + return Data::adoptMap(map, m_size, fd); +} + +Data mapFile(const char* path) +{ + int fd = open(path, O_RDONLY, 0); + if (fd < 0) + return { }; + struct stat stat; + if (fstat(fd, &stat) < 0) { + close(fd); + return { }; + } + size_t size = stat.st_size; + if (!size) { + close(fd); + return Data::empty(); + } + + return adoptAndMapFile(fd, 0, size); +} + +Data adoptAndMapFile(int fd, size_t offset, size_t size) +{ + if (!size) { + close(fd); + return Data::empty(); + } + + void* map = mmap(nullptr, size, PROT_READ, MAP_PRIVATE, fd, offset); + if (map == MAP_FAILED) { + close(fd); + return { }; + } + + return Data::adoptMap(map, size, fd); +} + +SHA1::Digest computeSHA1(const Data& data) +{ + SHA1 sha1; + data.apply([&sha1](const uint8_t* data, size_t size) { + sha1.addBytes(data, size); + return true; + }); + SHA1::Digest digest; + sha1.computeHash(digest); + return digest; +} + +bool bytesEqual(const Data& a, const Data& b) +{ + if (a.isNull() || b.isNull()) + return false; + if (a.size() != b.size()) + return false; + return !memcmp(a.data(), b.data(), a.size()); +} + +} // namespace NetworkCache +} // namespace WebKit + +#endif // #if ENABLE(NETWORK_CACHE) diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheData.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheData.h new file mode 100644 index 000000000..225589854 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheData.h @@ -0,0 +1,163 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheData_h +#define NetworkCacheData_h + +#if ENABLE(NETWORK_CACHE) + +#include <functional> +#include <wtf/FunctionDispatcher.h> +#include <wtf/SHA1.h> +#include <wtf/ThreadSafeRefCounted.h> +#include <wtf/text/WTFString.h> + +#if USE(SOUP) +#include <WebCore/GRefPtrSoup.h> +#endif + +namespace WebKit { + +class SharedMemory; + +namespace NetworkCache { + +#if PLATFORM(COCOA) +template <typename T> class DispatchPtr; +template <typename T> DispatchPtr<T> adoptDispatch(T dispatchObject); + +// FIXME: Use OSObjectPtr instead when it works with dispatch_data_t on all platforms. +template<typename T> class DispatchPtr { +public: + DispatchPtr() + : m_ptr(nullptr) + { + } + explicit DispatchPtr(T ptr) + : m_ptr(ptr) + { + if (m_ptr) + dispatch_retain(m_ptr); + } + DispatchPtr(const DispatchPtr& other) + : m_ptr(other.m_ptr) + { + if (m_ptr) + dispatch_retain(m_ptr); + } + ~DispatchPtr() + { + if (m_ptr) + dispatch_release(m_ptr); + } + + DispatchPtr& operator=(const DispatchPtr& other) + { + auto copy = other; + std::swap(m_ptr, copy.m_ptr); + return *this; + } + + T get() const { return m_ptr; } + explicit operator bool() const { return m_ptr; } + + friend DispatchPtr adoptDispatch<T>(T); + +private: + struct Adopt { }; + DispatchPtr(Adopt, T data) + : m_ptr(data) + { + } + + T m_ptr; +}; + +template <typename T> DispatchPtr<T> adoptDispatch(T dispatchObject) +{ + return DispatchPtr<T>(typename DispatchPtr<T>::Adopt { }, dispatchObject); +} +#endif + +class Data { +public: + Data() { } + Data(const uint8_t*, size_t); + + static Data empty(); + static Data adoptMap(void* map, size_t, int fd); + +#if PLATFORM(COCOA) + enum class Backing { Buffer, Map }; + Data(DispatchPtr<dispatch_data_t>, Backing = Backing::Buffer); +#endif +#if USE(SOUP) + Data(GRefPtr<SoupBuffer>&&, int fd = -1); +#endif + bool isNull() const; + bool isEmpty() const { return !m_size; } + + const uint8_t* data() const; + size_t size() const { return m_size; } + bool isMap() const { return m_isMap; } + RefPtr<SharedMemory> tryCreateSharedMemory() const; + + Data subrange(size_t offset, size_t) const; + + bool apply(const std::function<bool (const uint8_t*, size_t)>&&) const; + + Data mapToFile(const char* path) const; + +#if PLATFORM(COCOA) + dispatch_data_t dispatchData() const { return m_dispatchData.get(); } +#endif + +#if USE(SOUP) + SoupBuffer* soupBuffer() const { return m_buffer.get(); } +#endif +private: +#if PLATFORM(COCOA) + mutable DispatchPtr<dispatch_data_t> m_dispatchData; +#endif +#if USE(SOUP) + mutable GRefPtr<SoupBuffer> m_buffer; + int m_fileDescriptor { -1 }; +#endif + mutable const uint8_t* m_data { nullptr }; + size_t m_size { 0 }; + bool m_isMap { false }; +}; + +Data concatenate(const Data&, const Data&); +bool bytesEqual(const Data&, const Data&); +Data adoptAndMapFile(int fd, size_t offset, size_t); +Data mapFile(const char* path); +SHA1::Digest computeSHA1(const Data&); + +} +} + +#endif +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheDataSoup.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheDataSoup.cpp new file mode 100644 index 000000000..644c38e30 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheDataSoup.cpp @@ -0,0 +1,142 @@ +/* + * Copyright (C) 2015 Igalia S.L + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCacheData.h" + +#if ENABLE(NETWORK_CACHE) + +#include "SharedMemory.h" +#include <fcntl.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> + +namespace WebKit { +namespace NetworkCache { + +Data::Data(const uint8_t* data, size_t size) + : m_size(size) +{ + uint8_t* copiedData = static_cast<uint8_t*>(fastMalloc(size)); + memcpy(copiedData, data, size); + m_buffer = adoptGRef(soup_buffer_new_with_owner(copiedData, size, copiedData, fastFree)); +} + +Data::Data(GRefPtr<SoupBuffer>&& buffer, int fd) + : m_buffer(buffer) + , m_fileDescriptor(fd) + , m_size(buffer ? buffer->length : 0) + , m_isMap(m_size && fd != -1) +{ +} + +Data Data::empty() +{ + GRefPtr<SoupBuffer> buffer = adoptGRef(soup_buffer_new(SOUP_MEMORY_TAKE, nullptr, 0)); + return { WTFMove(buffer) }; +} + +const uint8_t* Data::data() const +{ + return m_buffer ? reinterpret_cast<const uint8_t*>(m_buffer->data) : nullptr; +} + +bool Data::isNull() const +{ + return !m_buffer; +} + +bool Data::apply(const std::function<bool (const uint8_t*, size_t)>&& applier) const +{ + if (!m_size) + return false; + + return applier(reinterpret_cast<const uint8_t*>(m_buffer->data), m_buffer->length); +} + +Data Data::subrange(size_t offset, size_t size) const +{ + if (!m_buffer) + return { }; + + GRefPtr<SoupBuffer> subBuffer = adoptGRef(soup_buffer_new_subbuffer(m_buffer.get(), offset, size)); + return { WTFMove(subBuffer) }; +} + +Data concatenate(const Data& a, const Data& b) +{ + if (a.isNull()) + return b; + if (b.isNull()) + return a; + + size_t size = a.size() + b.size(); + uint8_t* data = static_cast<uint8_t*>(fastMalloc(size)); + memcpy(data, a.soupBuffer()->data, a.soupBuffer()->length); + memcpy(data + a.soupBuffer()->length, b.soupBuffer()->data, b.soupBuffer()->length); + GRefPtr<SoupBuffer> buffer = adoptGRef(soup_buffer_new_with_owner(data, size, data, fastFree)); + return { WTFMove(buffer) }; +} + +struct MapWrapper { + ~MapWrapper() + { + munmap(map, size); + close(fileDescriptor); + } + + void* map; + size_t size; + int fileDescriptor; +}; + +static void deleteMapWrapper(MapWrapper* wrapper) +{ + delete wrapper; +} + +Data Data::adoptMap(void* map, size_t size, int fd) +{ + ASSERT(map); + ASSERT(map != MAP_FAILED); + MapWrapper* wrapper = new MapWrapper { map, size, fd }; + GRefPtr<SoupBuffer> buffer = adoptGRef(soup_buffer_new_with_owner(map, size, wrapper, reinterpret_cast<GDestroyNotify>(deleteMapWrapper))); + return { WTFMove(buffer), fd }; +} + +RefPtr<SharedMemory> Data::tryCreateSharedMemory() const +{ + if (isNull() || !isMap()) + return nullptr; + + return SharedMemory::wrapMap(const_cast<char*>(m_buffer->data), m_buffer->length, m_fileDescriptor); +} + +} // namespace NetworkCache +} // namespace WebKit + +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheDecoder.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheDecoder.cpp new file mode 100644 index 000000000..80e657578 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheDecoder.cpp @@ -0,0 +1,137 @@ +/* + * Copyright (C) 2010, 2011, 2014 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCacheDecoder.h" + +#if ENABLE(NETWORK_CACHE) + +#include "NetworkCacheEncoder.h" + +namespace WebKit { +namespace NetworkCache { + +Decoder::Decoder(const uint8_t* buffer, size_t bufferSize) + : m_buffer(buffer) + , m_bufferPosition(buffer) + , m_bufferEnd(buffer + bufferSize) +{ +} + +Decoder::~Decoder() +{ +} + +bool Decoder::bufferIsLargeEnoughToContain(size_t size) const +{ + return size <= static_cast<size_t>(m_bufferEnd - m_bufferPosition); +} + +bool Decoder::decodeFixedLengthData(uint8_t* data, size_t size) +{ + if (!bufferIsLargeEnoughToContain(size)) + return false; + + memcpy(data, m_bufferPosition, size); + m_bufferPosition += size; + + Encoder::updateChecksumForData(m_sha1, data, size); + return true; +} + +template<typename Type> +bool Decoder::decodeNumber(Type& value) +{ + if (!bufferIsLargeEnoughToContain(sizeof(value))) + return false; + + memcpy(&value, m_bufferPosition, sizeof(value)); + m_bufferPosition += sizeof(Type); + + Encoder::updateChecksumForNumber(m_sha1, value); + return true; +} + +bool Decoder::decode(bool& result) +{ + return decodeNumber(result); +} + +bool Decoder::decode(uint8_t& result) +{ + return decodeNumber(result); +} + +bool Decoder::decode(uint16_t& result) +{ + return decodeNumber(result); +} + +bool Decoder::decode(uint32_t& result) +{ + return decodeNumber(result); +} + +bool Decoder::decode(uint64_t& result) +{ + return decodeNumber(result); +} + +bool Decoder::decode(int32_t& result) +{ + return decodeNumber(result); +} + +bool Decoder::decode(int64_t& result) +{ + return decodeNumber(result); +} + +bool Decoder::decode(float& result) +{ + return decodeNumber(result); +} + +bool Decoder::decode(double& result) +{ + return decodeNumber(result); +} + +bool Decoder::verifyChecksum() +{ + SHA1::Digest computedHash; + m_sha1.computeHash(computedHash); + + SHA1::Digest savedHash; + if (!decodeFixedLengthData(savedHash.data(), sizeof(savedHash))) + return false; + + return computedHash == savedHash; +} + +} +} + +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheDecoder.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheDecoder.h new file mode 100644 index 000000000..12e3d03dc --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheDecoder.h @@ -0,0 +1,103 @@ +/* + * Copyright (C) 2014 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheDecoder_h +#define NetworkCacheDecoder_h + +#if ENABLE(NETWORK_CACHE) + +#include "NetworkCacheCoder.h" +#include <wtf/SHA1.h> + +namespace WebKit { +namespace NetworkCache { + +class Decoder { + WTF_MAKE_FAST_ALLOCATED; +public: + Decoder(const uint8_t* buffer, size_t bufferSize); + virtual ~Decoder(); + + size_t length() const { return m_bufferEnd - m_buffer; } + size_t currentOffset() const { return m_bufferPosition - m_buffer; } + + bool verifyChecksum(); + + bool decodeFixedLengthData(uint8_t*, size_t); + + bool decode(bool&); + bool decode(uint8_t&); + bool decode(uint16_t&); + bool decode(uint32_t&); + bool decode(uint64_t&); + bool decode(int32_t&); + bool decode(int64_t&); + bool decode(float&); + bool decode(double&); + + template<typename T> bool decodeEnum(T& result) + { + static_assert(sizeof(T) <= 8, "Enum type T must not be larger than 64 bits!"); + + uint64_t value; + if (!decode(value)) + return false; + + result = static_cast<T>(value); + return true; + } + + template<typename T> bool decode(T& t) + { + return Coder<T>::decode(*this, t); + } + + template<typename T> + bool bufferIsLargeEnoughToContain(size_t numElements) const + { + static_assert(std::is_arithmetic<T>::value, "Type T must have a fixed, known encoded size!"); + + if (numElements > std::numeric_limits<size_t>::max() / sizeof(T)) + return false; + + return bufferIsLargeEnoughToContain(numElements * sizeof(T)); + } + +private: + bool bufferIsLargeEnoughToContain(size_t) const; + template<typename Type> bool decodeNumber(Type&); + + const uint8_t* m_buffer; + const uint8_t* m_bufferPosition; + const uint8_t* m_bufferEnd; + + SHA1 m_sha1; +}; + +} +} + +#endif +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheEncoder.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheEncoder.cpp new file mode 100644 index 000000000..0c2d7191d --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheEncoder.cpp @@ -0,0 +1,128 @@ +/* + * Copyright (C) 2010, 2014 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCacheEncoder.h" + +#if ENABLE(NETWORK_CACHE) + +namespace WebKit { +namespace NetworkCache { + +Encoder::Encoder() +{ +} + +Encoder::~Encoder() +{ +} + +uint8_t* Encoder::grow(size_t size) +{ + size_t newPosition = m_buffer.size(); + m_buffer.grow(m_buffer.size() + size); + return m_buffer.data() + newPosition; +} + +void Encoder::updateChecksumForData(SHA1& sha1, const uint8_t* data, size_t size) +{ + auto typeSalt = Salt<uint8_t*>::value; + sha1.addBytes(reinterpret_cast<uint8_t*>(&typeSalt), sizeof(typeSalt)); + sha1.addBytes(data, size); +} + +void Encoder::encodeFixedLengthData(const uint8_t* data, size_t size) +{ + updateChecksumForData(m_sha1, data, size); + + uint8_t* buffer = grow(size); + memcpy(buffer, data, size); +} + +template<typename Type> +void Encoder::encodeNumber(Type value) +{ + Encoder::updateChecksumForNumber(m_sha1, value); + + uint8_t* buffer = grow(sizeof(Type)); + memcpy(buffer, &value, sizeof(Type)); +} + +void Encoder::encode(bool value) +{ + encodeNumber(value); +} + +void Encoder::encode(uint8_t value) +{ + encodeNumber(value); +} + +void Encoder::encode(uint16_t value) +{ + encodeNumber(value); +} + +void Encoder::encode(uint32_t value) +{ + encodeNumber(value); +} + +void Encoder::encode(uint64_t value) +{ + encodeNumber(value); +} + +void Encoder::encode(int32_t value) +{ + encodeNumber(value); +} + +void Encoder::encode(int64_t value) +{ + encodeNumber(value); +} + +void Encoder::encode(float value) +{ + encodeNumber(value); +} + +void Encoder::encode(double value) +{ + encodeNumber(value); +} + +void Encoder::encodeChecksum() +{ + SHA1::Digest hash; + m_sha1.computeHash(hash); + encodeFixedLengthData(hash.data(), hash.size()); +} + +} +} + +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheEncoder.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheEncoder.h new file mode 100644 index 000000000..469c06c59 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheEncoder.h @@ -0,0 +1,118 @@ +/* + * Copyright (C) 2010 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheEncoder_h +#define NetworkCacheEncoder_h + +#if ENABLE(NETWORK_CACHE) + +#include "NetworkCacheCoder.h" +#include <wtf/SHA1.h> +#include <wtf/Vector.h> + +namespace WebKit { +namespace NetworkCache { + +class Encoder; +class DataReference; + +class Encoder { + WTF_MAKE_FAST_ALLOCATED; +public: + Encoder(); + virtual ~Encoder(); + + void encodeChecksum(); + void encodeFixedLengthData(const uint8_t*, size_t); + + template<typename T> void encodeEnum(T t) + { + COMPILE_ASSERT(sizeof(T) <= sizeof(uint64_t), enum_type_must_not_be_larger_than_64_bits); + + encode(static_cast<uint64_t>(t)); + } + + template<typename T> void encode(const T& t) + { + Coder<T>::encode(*this, t); + } + + template<typename T> Encoder& operator<<(const T& t) + { + encode(t); + return *this; + } + + const uint8_t* buffer() const { return m_buffer.data(); } + size_t bufferSize() const { return m_buffer.size(); } + + static void updateChecksumForData(SHA1&, const uint8_t*, size_t); + template <typename Type> static void updateChecksumForNumber(SHA1&, Type); + +private: + void encode(bool); + void encode(uint8_t); + void encode(uint16_t); + void encode(uint32_t); + void encode(uint64_t); + void encode(int32_t); + void encode(int64_t); + void encode(float); + void encode(double); + + template<typename Type> void encodeNumber(Type); + + uint8_t* grow(size_t); + + template <typename Type> struct Salt; + + Vector<uint8_t, 4096> m_buffer; + SHA1 m_sha1; +}; + +template <> struct Encoder::Salt<bool> { static const unsigned value = 3; }; +template <> struct Encoder::Salt<uint8_t> { static const unsigned value = 5; }; +template <> struct Encoder::Salt<uint16_t> { static const unsigned value = 7; }; +template <> struct Encoder::Salt<uint32_t> { static const unsigned value = 11; }; +template <> struct Encoder::Salt<uint64_t> { static const unsigned value = 13; }; +template <> struct Encoder::Salt<int32_t> { static const unsigned value = 17; }; +template <> struct Encoder::Salt<int64_t> { static const unsigned value = 19; }; +template <> struct Encoder::Salt<float> { static const unsigned value = 23; }; +template <> struct Encoder::Salt<double> { static const unsigned value = 29; }; +template <> struct Encoder::Salt<uint8_t*> { static const unsigned value = 101; }; + +template <typename Type> +void Encoder::updateChecksumForNumber(SHA1& sha1, Type value) +{ + auto typeSalt = Salt<Type>::value; + sha1.addBytes(reinterpret_cast<uint8_t*>(&typeSalt), sizeof(typeSalt)); + sha1.addBytes(reinterpret_cast<uint8_t*>(&value), sizeof(value)); +} + +} +} + +#endif +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheEntry.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheEntry.cpp new file mode 100644 index 000000000..faa4f04d1 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheEntry.cpp @@ -0,0 +1,242 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCache.h" + +#include "Logging.h" +#include "NetworkCacheCoders.h" +#include "NetworkCacheDecoder.h" +#include "NetworkCacheEncoder.h" +#include <WebCore/ResourceRequest.h> +#include <WebCore/SharedBuffer.h> +#include <wtf/text/StringBuilder.h> + +#if ENABLE(NETWORK_CACHE) + +namespace WebKit { +namespace NetworkCache { + +Entry::Entry(const Key& key, const WebCore::ResourceResponse& response, RefPtr<WebCore::SharedBuffer>&& buffer, const Vector<std::pair<String, String>>& varyingRequestHeaders) + : m_key(key) + , m_timeStamp(std::chrono::system_clock::now()) + , m_response(response) + , m_varyingRequestHeaders(varyingRequestHeaders) + , m_buffer(WTFMove(buffer)) +{ + ASSERT(m_key.type() == "resource"); +} + +Entry::Entry(const Key& key, const WebCore::ResourceResponse& response, const WebCore::ResourceRequest& redirectRequest, const Vector<std::pair<String, String>>& varyingRequestHeaders) + : m_key(key) + , m_timeStamp(std::chrono::system_clock::now()) + , m_response(response) + , m_varyingRequestHeaders(varyingRequestHeaders) + , m_redirectRequest(WebCore::ResourceRequest::adopt(redirectRequest.copyData())) // Don't include the underlying platform request object. +{ + ASSERT(m_key.type() == "resource"); + // Redirect body is not needed even if exists. + m_redirectRequest->setHTTPBody(nullptr); +} + +Entry::Entry(const Entry& other) + : m_key(other.m_key) + , m_timeStamp(other.m_timeStamp) + , m_response(other.m_response) + , m_varyingRequestHeaders(other.m_varyingRequestHeaders) + , m_buffer(other.m_buffer) + , m_sourceStorageRecord(other.m_sourceStorageRecord) +{ +} + +Entry::Entry(const Storage::Record& storageEntry) + : m_key(storageEntry.key) + , m_timeStamp(storageEntry.timeStamp) + , m_sourceStorageRecord(storageEntry) +{ + ASSERT(m_key.type() == "resource"); +} + +Storage::Record Entry::encodeAsStorageRecord() const +{ + Encoder encoder; + encoder << m_response; + + bool hasVaryingRequestHeaders = !m_varyingRequestHeaders.isEmpty(); + encoder << hasVaryingRequestHeaders; + if (hasVaryingRequestHeaders) + encoder << m_varyingRequestHeaders; + + bool isRedirect = !!m_redirectRequest; + encoder << isRedirect; + if (isRedirect) + m_redirectRequest->encodeWithoutPlatformData(encoder); + + encoder.encodeChecksum(); + + Data header(encoder.buffer(), encoder.bufferSize()); + Data body; + if (m_buffer) + body = { reinterpret_cast<const uint8_t*>(m_buffer->data()), m_buffer->size() }; + + return { m_key, m_timeStamp, header, body }; +} + +std::unique_ptr<Entry> Entry::decodeStorageRecord(const Storage::Record& storageEntry) +{ + auto entry = std::make_unique<Entry>(storageEntry); + + Decoder decoder(storageEntry.header.data(), storageEntry.header.size()); + if (!decoder.decode(entry->m_response)) + return nullptr; + entry->m_response.setSource(WebCore::ResourceResponse::Source::DiskCache); + + bool hasVaryingRequestHeaders; + if (!decoder.decode(hasVaryingRequestHeaders)) + return nullptr; + + if (hasVaryingRequestHeaders) { + if (!decoder.decode(entry->m_varyingRequestHeaders)) + return nullptr; + } + + bool isRedirect; + if (!decoder.decode(isRedirect)) + return nullptr; + + if (isRedirect) { + entry->m_redirectRequest = std::make_unique<WebCore::ResourceRequest>(); + if (!entry->m_redirectRequest->decodeWithoutPlatformData(decoder)) + return nullptr; + } + + if (!decoder.verifyChecksum()) { + LOG(NetworkCache, "(NetworkProcess) checksum verification failure\n"); + return nullptr; + } + + return entry; +} + +#if ENABLE(SHAREABLE_RESOURCE) +void Entry::initializeShareableResourceHandleFromStorageRecord() const +{ + RefPtr<SharedMemory> sharedMemory = m_sourceStorageRecord.body.tryCreateSharedMemory(); + if (!sharedMemory) + return; + + RefPtr<ShareableResource> shareableResource = ShareableResource::create(sharedMemory.release(), 0, m_sourceStorageRecord.body.size()); + ASSERT(shareableResource); + shareableResource->createHandle(m_shareableResourceHandle); +} +#endif + +void Entry::initializeBufferFromStorageRecord() const +{ +#if ENABLE(SHAREABLE_RESOURCE) + if (!shareableResourceHandle().isNull()) { + m_buffer = m_shareableResourceHandle.tryWrapInSharedBuffer(); + if (m_buffer) + return; + } +#endif + m_buffer = WebCore::SharedBuffer::create(m_sourceStorageRecord.body.data(), m_sourceStorageRecord.body.size()); +} + +WebCore::SharedBuffer* Entry::buffer() const +{ + if (!m_buffer) + initializeBufferFromStorageRecord(); + + return m_buffer.get(); +} + +#if ENABLE(SHAREABLE_RESOURCE) +ShareableResource::Handle& Entry::shareableResourceHandle() const +{ + if (m_shareableResourceHandle.isNull()) + initializeShareableResourceHandleFromStorageRecord(); + + return m_shareableResourceHandle; +} +#endif + +bool Entry::needsValidation() const +{ + return m_response.source() == WebCore::ResourceResponse::Source::DiskCacheAfterValidation; +} + +void Entry::setNeedsValidation() +{ + ASSERT(m_response.source() == WebCore::ResourceResponse::Source::DiskCache); + m_response.setSource(WebCore::ResourceResponse::Source::DiskCacheAfterValidation); +} + +void Entry::asJSON(StringBuilder& json, const Storage::RecordInfo& info) const +{ + json.appendLiteral("{\n"); + json.appendLiteral("\"hash\": "); + json.appendQuotedJSONString(m_key.hashAsString()); + json.appendLiteral(",\n"); + json.appendLiteral("\"bodySize\": "); + json.appendNumber(info.bodySize); + json.appendLiteral(",\n"); + json.appendLiteral("\"worth\": "); + json.appendNumber(info.worth); + json.appendLiteral(",\n"); + json.appendLiteral("\"partition\": "); + json.appendQuotedJSONString(m_key.partition()); + json.appendLiteral(",\n"); + json.appendLiteral("\"timestamp\": "); + json.appendNumber(std::chrono::duration_cast<std::chrono::milliseconds>(m_timeStamp.time_since_epoch()).count()); + json.appendLiteral(",\n"); + json.appendLiteral("\"URL\": "); + json.appendQuotedJSONString(m_response.url().string()); + json.appendLiteral(",\n"); + json.appendLiteral("\"bodyHash\": "); + json.appendQuotedJSONString(info.bodyHash); + json.appendLiteral(",\n"); + json.appendLiteral("\"bodyShareCount\": "); + json.appendNumber(info.bodyShareCount); + json.appendLiteral(",\n"); + json.appendLiteral("\"headers\": {\n"); + bool firstHeader = true; + for (auto& header : m_response.httpHeaderFields()) { + if (!firstHeader) + json.appendLiteral(",\n"); + firstHeader = false; + json.appendLiteral(" "); + json.appendQuotedJSONString(header.key); + json.appendLiteral(": "); + json.appendQuotedJSONString(header.value); + } + json.appendLiteral("\n}\n"); + json.appendLiteral("}"); +} + +} +} + +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheEntry.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheEntry.h new file mode 100644 index 000000000..0855a4efe --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheEntry.h @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheEntry_h +#define NetworkCacheEntry_h + +#if ENABLE(NETWORK_CACHE) + +#include "NetworkCacheStorage.h" +#include "ShareableResource.h" +#include <WebCore/ResourceRequest.h> +#include <WebCore/ResourceResponse.h> +#include <wtf/Noncopyable.h> +#include <wtf/text/WTFString.h> + +namespace WebCore { +class SharedBuffer; +} + +namespace WebKit { +namespace NetworkCache { + +class Entry { + WTF_MAKE_FAST_ALLOCATED; +public: + Entry(const Key&, const WebCore::ResourceResponse&, RefPtr<WebCore::SharedBuffer>&&, const Vector<std::pair<String, String>>& varyingRequestHeaders); + Entry(const Key&, const WebCore::ResourceResponse&, const WebCore::ResourceRequest& redirectRequest, const Vector<std::pair<String, String>>& varyingRequestHeaders); + explicit Entry(const Storage::Record&); + Entry(const Entry&); + + Storage::Record encodeAsStorageRecord() const; + static std::unique_ptr<Entry> decodeStorageRecord(const Storage::Record&); + + const Key& key() const { return m_key; } + std::chrono::system_clock::time_point timeStamp() const { return m_timeStamp; } + const WebCore::ResourceResponse& response() const { return m_response; } + const Vector<std::pair<String, String>>& varyingRequestHeaders() const { return m_varyingRequestHeaders; } + + WebCore::SharedBuffer* buffer() const; + const WebCore::ResourceRequest* redirectRequest() const { return m_redirectRequest.get(); } + +#if ENABLE(SHAREABLE_RESOURCE) + ShareableResource::Handle& shareableResourceHandle() const; +#endif + + bool needsValidation() const; + void setNeedsValidation(); + + const Storage::Record& sourceStorageRecord() const { return m_sourceStorageRecord; } + + void asJSON(StringBuilder&, const Storage::RecordInfo&) const; + +private: + void initializeBufferFromStorageRecord() const; +#if ENABLE(SHAREABLE_RESOURCE) + void initializeShareableResourceHandleFromStorageRecord() const; +#endif + + Key m_key; + std::chrono::system_clock::time_point m_timeStamp; + WebCore::ResourceResponse m_response; + Vector<std::pair<String, String>> m_varyingRequestHeaders; + + std::unique_ptr<WebCore::ResourceRequest> m_redirectRequest; + mutable RefPtr<WebCore::SharedBuffer> m_buffer; +#if ENABLE(SHAREABLE_RESOURCE) + mutable ShareableResource::Handle m_shareableResourceHandle; +#endif + + Storage::Record m_sourceStorageRecord { }; +}; + +} +} + +#endif +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheFileSystem.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheFileSystem.cpp new file mode 100644 index 000000000..93d6a58d6 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheFileSystem.cpp @@ -0,0 +1,131 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCacheFileSystem.h" + +#if ENABLE(NETWORK_CACHE) + +#include <WebCore/FileSystem.h> +#include <dirent.h> +#include <sys/stat.h> +#include <sys/time.h> +#include <wtf/text/CString.h> + +#if USE(SOUP) +#include <gio/gio.h> +#include <wtf/glib/GRefPtr.h> +#endif + +namespace WebKit { +namespace NetworkCache { + +static DirectoryEntryType directoryEntryType(uint8_t dtype) +{ + switch (dtype) { + case DT_DIR: + return DirectoryEntryType::Directory; + case DT_REG: + return DirectoryEntryType::File; + default: + ASSERT_NOT_REACHED(); + return DirectoryEntryType::File; + } +} + +void traverseDirectory(const String& path, const std::function<void (const String&, DirectoryEntryType)>& function) +{ + DIR* dir = opendir(WebCore::fileSystemRepresentation(path).data()); + if (!dir) + return; + dirent* dp; + while ((dp = readdir(dir))) { + if (dp->d_type != DT_DIR && dp->d_type != DT_REG) + continue; + const char* name = dp->d_name; + if (!strcmp(name, ".") || !strcmp(name, "..")) + continue; + auto nameString = String::fromUTF8(name); + if (nameString.isNull()) + continue; + function(nameString, directoryEntryType(dp->d_type)); + } + closedir(dir); +} + +void deleteDirectoryRecursively(const String& path) +{ + traverseDirectory(path, [&path](const String& name, DirectoryEntryType type) { + String entryPath = WebCore::pathByAppendingComponent(path, name); + switch (type) { + case DirectoryEntryType::File: + WebCore::deleteFile(entryPath); + break; + case DirectoryEntryType::Directory: + deleteDirectoryRecursively(entryPath); + break; + // This doesn't follow symlinks. + } + }); + WebCore::deleteEmptyDirectory(path); +} + +FileTimes fileTimes(const String& path) +{ +#if HAVE(STAT_BIRTHTIME) + struct stat fileInfo; + if (stat(WebCore::fileSystemRepresentation(path).data(), &fileInfo)) + return { }; + return { std::chrono::system_clock::from_time_t(fileInfo.st_birthtime), std::chrono::system_clock::from_time_t(fileInfo.st_mtime) }; +#elif USE(SOUP) + // There's no st_birthtime in some operating systems like Linux, so we use xattrs to set/get the creation time. + GRefPtr<GFile> file = adoptGRef(g_file_new_for_path(WebCore::fileSystemRepresentation(path).data())); + GRefPtr<GFileInfo> fileInfo = adoptGRef(g_file_query_info(file.get(), "xattr::birthtime,time::modified", G_FILE_QUERY_INFO_NONE, nullptr, nullptr)); + if (!fileInfo) + return { }; + const char* birthtimeString = g_file_info_get_attribute_string(fileInfo.get(), "xattr::birthtime"); + if (!birthtimeString) + return { }; + return { std::chrono::system_clock::from_time_t(g_ascii_strtoull(birthtimeString, nullptr, 10)), + std::chrono::system_clock::from_time_t(g_file_info_get_attribute_uint64(fileInfo.get(), "time::modified")) }; +#endif +} + +void updateFileModificationTimeIfNeeded(const String& path) +{ + auto times = fileTimes(path); + if (times.creation != times.modification) { + // Don't update more than once per hour. + if (std::chrono::system_clock::now() - times.modification < std::chrono::hours(1)) + return; + } + // This really updates both the access time and the modification time. + utimes(WebCore::fileSystemRepresentation(path).data(), nullptr); +} + +} +} + +#endif // ENABLE(NETWORK_CACHE) diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheFileSystem.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheFileSystem.h new file mode 100644 index 000000000..17407e8ab --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheFileSystem.h @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheFileSystem_h +#define NetworkCacheFileSystem_h + +#if ENABLE(NETWORK_CACHE) + +#include <WebCore/FileSystem.h> +#include <functional> + +namespace WebKit { +namespace NetworkCache { + +enum class DirectoryEntryType { Directory, File }; +void traverseDirectory(const String& path, const std::function<void (const String& fileName, DirectoryEntryType)>&); + +void deleteDirectoryRecursively(const String& path); + +struct FileTimes { + std::chrono::system_clock::time_point creation; + std::chrono::system_clock::time_point modification; +}; +FileTimes fileTimes(const String& path); +void updateFileModificationTimeIfNeeded(const String& path); + +} +} + +#endif + +#endif + diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheIOChannel.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheIOChannel.h new file mode 100644 index 000000000..c5edaa8f2 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheIOChannel.h @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheIOChannel_h +#define NetworkCacheIOChannel_h + +#if ENABLE(NETWORK_CACHE) + +#include "NetworkCacheData.h" +#include <functional> +#include <wtf/ThreadSafeRefCounted.h> +#include <wtf/WorkQueue.h> +#include <wtf/text/WTFString.h> + +#if USE(SOUP) +#include <wtf/glib/GRefPtr.h> +#endif + +namespace WebKit { +namespace NetworkCache { + +class IOChannel : public ThreadSafeRefCounted<IOChannel> { +public: + enum class Type { Read, Write, Create }; + static Ref<IOChannel> open(const String& file, Type); + + // Using nullptr as queue submits the result to the main queue. + // FIXME: We should add WorkQueue::main() instead. + void read(size_t offset, size_t, WorkQueue*, std::function<void (Data&, int error)>); + void write(size_t offset, const Data&, WorkQueue*, std::function<void (int error)>); + + const String& path() const { return m_path; } + Type type() const { return m_type; } + + int fileDescriptor() const { return m_fileDescriptor; } + +private: + IOChannel(const String& filePath, IOChannel::Type); + +#if USE(SOUP) + void readSyncInThread(size_t offset, size_t, WorkQueue*, std::function<void (Data&, int error)>); +#endif + + String m_path; + Type m_type; + + int m_fileDescriptor { 0 }; +#if PLATFORM(COCOA) + DispatchPtr<dispatch_io_t> m_dispatchIO; +#endif +#if USE(SOUP) + GRefPtr<GInputStream> m_inputStream; + GRefPtr<GOutputStream> m_outputStream; + GRefPtr<GFileIOStream> m_ioStream; +#endif +}; + +} +} + +#endif +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheIOChannelSoup.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheIOChannelSoup.cpp new file mode 100644 index 000000000..080ce256d --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheIOChannelSoup.cpp @@ -0,0 +1,290 @@ +/* + * Copyright (C) 2015 Igalia S.L. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCacheIOChannel.h" + +#if ENABLE(NETWORK_CACHE) + +#include "NetworkCacheFileSystem.h" +#include <wtf/MainThread.h> +#include <wtf/RunLoop.h> +#include <wtf/glib/GUniquePtr.h> + +namespace WebKit { +namespace NetworkCache { + +static const size_t gDefaultReadBufferSize = 4096; + +IOChannel::IOChannel(const String& filePath, Type type) + : m_path(filePath) + , m_type(type) +{ + auto path = WebCore::fileSystemRepresentation(filePath); + GRefPtr<GFile> file = adoptGRef(g_file_new_for_path(path.data())); + switch (m_type) { + case Type::Create: { + g_file_delete(file.get(), nullptr, nullptr); + m_outputStream = adoptGRef(G_OUTPUT_STREAM(g_file_create(file.get(), static_cast<GFileCreateFlags>(G_FILE_CREATE_PRIVATE), nullptr, nullptr))); +#if !HAVE(STAT_BIRTHTIME) + GUniquePtr<char> birthtimeString(g_strdup_printf("%" G_GUINT64_FORMAT, std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()))); + g_file_set_attribute_string(file.get(), "xattr::birthtime", birthtimeString.get(), G_FILE_QUERY_INFO_NONE, nullptr, nullptr); +#endif + break; + } + case Type::Write: { + m_ioStream = adoptGRef(g_file_open_readwrite(file.get(), nullptr, nullptr)); + break; + } + case Type::Read: + m_inputStream = adoptGRef(G_INPUT_STREAM(g_file_read(file.get(), nullptr, nullptr))); + break; + } +} + +Ref<IOChannel> IOChannel::open(const String& filePath, IOChannel::Type type) +{ + return adoptRef(*new IOChannel(filePath, type)); +} + +static inline void runTaskInQueue(std::function<void ()> task, WorkQueue* queue) +{ + if (queue) { + queue->dispatch(task); + return; + } + + // Using nullptr as queue submits the result to the main context. + RunLoop::main().dispatch(WTFMove(task)); +} + +static void fillDataFromReadBuffer(SoupBuffer* readBuffer, size_t size, Data& data) +{ + GRefPtr<SoupBuffer> buffer; + if (size != readBuffer->length) { + // The subbuffer does not copy the data. + buffer = adoptGRef(soup_buffer_new_subbuffer(readBuffer, 0, size)); + } else + buffer = readBuffer; + + if (data.isNull()) { + // First chunk, we need to force the data to be copied. + data = { reinterpret_cast<const uint8_t*>(buffer->data), size }; + } else { + Data dataRead(WTFMove(buffer)); + // Concatenate will copy the data. + data = concatenate(data, dataRead); + } +} + +struct ReadAsyncData { + RefPtr<IOChannel> channel; + GRefPtr<SoupBuffer> buffer; + RefPtr<WorkQueue> queue; + size_t bytesToRead; + std::function<void (Data&, int error)> completionHandler; + Data data; +}; + +static void inputStreamReadReadyCallback(GInputStream* stream, GAsyncResult* result, gpointer userData) +{ + std::unique_ptr<ReadAsyncData> asyncData(static_cast<ReadAsyncData*>(userData)); + gssize bytesRead = g_input_stream_read_finish(stream, result, nullptr); + if (bytesRead == -1) { + WorkQueue* queue = asyncData->queue.get(); + auto* asyncDataPtr = asyncData.release(); + runTaskInQueue([asyncDataPtr] { + std::unique_ptr<ReadAsyncData> asyncData(asyncDataPtr); + asyncData->completionHandler(asyncData->data, -1); + }, queue); + return; + } + + if (!bytesRead) { + WorkQueue* queue = asyncData->queue.get(); + auto* asyncDataPtr = asyncData.release(); + runTaskInQueue([asyncDataPtr] { + std::unique_ptr<ReadAsyncData> asyncData(asyncDataPtr); + asyncData->completionHandler(asyncData->data, 0); + }, queue); + return; + } + + ASSERT(bytesRead > 0); + fillDataFromReadBuffer(asyncData->buffer.get(), static_cast<size_t>(bytesRead), asyncData->data); + + size_t pendingBytesToRead = asyncData->bytesToRead - asyncData->data.size(); + if (!pendingBytesToRead) { + WorkQueue* queue = asyncData->queue.get(); + auto* asyncDataPtr = asyncData.release(); + runTaskInQueue([asyncDataPtr] { + std::unique_ptr<ReadAsyncData> asyncData(asyncDataPtr); + asyncData->completionHandler(asyncData->data, 0); + }, queue); + return; + } + + size_t bytesToRead = std::min(pendingBytesToRead, asyncData->buffer->length); + // Use a local variable for the data buffer to pass it to g_input_stream_read_async(), because ReadAsyncData is released. + auto data = const_cast<char*>(asyncData->buffer->data); + g_input_stream_read_async(stream, data, bytesToRead, G_PRIORITY_DEFAULT, nullptr, + reinterpret_cast<GAsyncReadyCallback>(inputStreamReadReadyCallback), asyncData.release()); +} + +void IOChannel::read(size_t offset, size_t size, WorkQueue* queue, std::function<void (Data&, int error)> completionHandler) +{ + RefPtr<IOChannel> channel(this); + if (!m_inputStream) { + runTaskInQueue([channel, completionHandler] { + Data data; + completionHandler(data, -1); + }, queue); + return; + } + + if (!isMainThread()) { + readSyncInThread(offset, size, queue, completionHandler); + return; + } + + size_t bufferSize = std::min(size, gDefaultReadBufferSize); + uint8_t* bufferData = static_cast<uint8_t*>(fastMalloc(bufferSize)); + GRefPtr<SoupBuffer> buffer = adoptGRef(soup_buffer_new_with_owner(bufferData, bufferSize, bufferData, fastFree)); + ReadAsyncData* asyncData = new ReadAsyncData { this, buffer.get(), queue, size, completionHandler, { } }; + + // FIXME: implement offset. + g_input_stream_read_async(m_inputStream.get(), const_cast<char*>(buffer->data), bufferSize, G_PRIORITY_DEFAULT, nullptr, + reinterpret_cast<GAsyncReadyCallback>(inputStreamReadReadyCallback), asyncData); +} + +void IOChannel::readSyncInThread(size_t offset, size_t size, WorkQueue* queue, std::function<void (Data&, int error)> completionHandler) +{ + ASSERT(!isMainThread()); + + RefPtr<IOChannel> channel(this); + detachThread(createThread("IOChannel::readSync", [channel, size, queue, completionHandler] { + size_t bufferSize = std::min(size, gDefaultReadBufferSize); + uint8_t* bufferData = static_cast<uint8_t*>(fastMalloc(bufferSize)); + GRefPtr<SoupBuffer> readBuffer = adoptGRef(soup_buffer_new_with_owner(bufferData, bufferSize, bufferData, fastFree)); + Data data; + size_t pendingBytesToRead = size; + size_t bytesToRead = bufferSize; + do { + // FIXME: implement offset. + gssize bytesRead = g_input_stream_read(channel->m_inputStream.get(), const_cast<char*>(readBuffer->data), bytesToRead, nullptr, nullptr); + if (bytesRead == -1) { + runTaskInQueue([channel, completionHandler] { + Data data; + completionHandler(data, -1); + }, queue); + return; + } + + if (!bytesRead) + break; + + ASSERT(bytesRead > 0); + fillDataFromReadBuffer(readBuffer.get(), static_cast<size_t>(bytesRead), data); + + pendingBytesToRead = size - data.size(); + bytesToRead = std::min(pendingBytesToRead, readBuffer->length); + } while (pendingBytesToRead); + + GRefPtr<SoupBuffer> bufferCapture = data.soupBuffer(); + runTaskInQueue([channel, bufferCapture, completionHandler] { + GRefPtr<SoupBuffer> buffer = bufferCapture; + Data data = { WTFMove(buffer) }; + completionHandler(data, 0); + }, queue); + })); +} + +struct WriteAsyncData { + RefPtr<IOChannel> channel; + GRefPtr<SoupBuffer> buffer; + RefPtr<WorkQueue> queue; + std::function<void (int error)> completionHandler; +}; + +static void outputStreamWriteReadyCallback(GOutputStream* stream, GAsyncResult* result, gpointer userData) +{ + std::unique_ptr<WriteAsyncData> asyncData(static_cast<WriteAsyncData*>(userData)); + gssize bytesWritten = g_output_stream_write_finish(stream, result, nullptr); + if (bytesWritten == -1) { + WorkQueue* queue = asyncData->queue.get(); + auto* asyncDataPtr = asyncData.release(); + runTaskInQueue([asyncDataPtr] { + std::unique_ptr<WriteAsyncData> asyncData(asyncDataPtr); + asyncData->completionHandler(-1); + }, queue); + return; + } + + gssize pendingBytesToWrite = asyncData->buffer->length - bytesWritten; + if (!pendingBytesToWrite) { + WorkQueue* queue = asyncData->queue.get(); + auto* asyncDataPtr = asyncData.release(); + runTaskInQueue([asyncDataPtr] { + std::unique_ptr<WriteAsyncData> asyncData(asyncDataPtr); + asyncData->completionHandler(0); + }, queue); + return; + } + + asyncData->buffer = adoptGRef(soup_buffer_new_subbuffer(asyncData->buffer.get(), bytesWritten, pendingBytesToWrite)); + // Use a local variable for the data buffer to pass it to g_output_stream_write_async(), because WriteAsyncData is released. + auto data = asyncData->buffer->data; + g_output_stream_write_async(stream, data, pendingBytesToWrite, G_PRIORITY_DEFAULT_IDLE, nullptr, + reinterpret_cast<GAsyncReadyCallback>(outputStreamWriteReadyCallback), asyncData.release()); +} + +void IOChannel::write(size_t offset, const Data& data, WorkQueue* queue, std::function<void (int error)> completionHandler) +{ + RefPtr<IOChannel> channel(this); + if (!m_outputStream && !m_ioStream) { + runTaskInQueue([channel, completionHandler] { + completionHandler(-1); + }, queue); + return; + } + + GOutputStream* stream = m_outputStream ? m_outputStream.get() : g_io_stream_get_output_stream(G_IO_STREAM(m_ioStream.get())); + if (!stream) { + runTaskInQueue([channel, completionHandler] { + completionHandler(-1); + }, queue); + return; + } + + WriteAsyncData* asyncData = new WriteAsyncData { this, data.soupBuffer(), queue, completionHandler }; + // FIXME: implement offset. + g_output_stream_write_async(stream, asyncData->buffer->data, data.size(), G_PRIORITY_DEFAULT_IDLE, nullptr, + reinterpret_cast<GAsyncReadyCallback>(outputStreamWriteReadyCallback), asyncData); +} + +} // namespace NetworkCache +} // namespace WebKit + +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheKey.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheKey.cpp new file mode 100644 index 000000000..1f06a65bb --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheKey.cpp @@ -0,0 +1,156 @@ +/* + * Copyright (C) 2014-2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCacheKey.h" + +#if ENABLE(NETWORK_CACHE) + +#include "NetworkCacheCoders.h" +#include <wtf/ASCIICType.h> +#include <wtf/text/CString.h> +#include <wtf/text/StringBuilder.h> + +namespace WebKit { +namespace NetworkCache { + +Key::Key(const Key& o) + : m_partition(o.m_partition.isolatedCopy()) + , m_type(o.m_type.isolatedCopy()) + , m_identifier(o.m_identifier.isolatedCopy()) + , m_range(o.m_range.isolatedCopy()) + , m_hash(o.m_hash) +{ +} + +Key::Key(const String& partition, const String& type, const String& range, const String& identifier) + : m_partition(partition.isolatedCopy()) + , m_type(type.isolatedCopy()) + , m_identifier(identifier.isolatedCopy()) + , m_range(range.isolatedCopy()) + , m_hash(computeHash()) +{ +} + +Key::Key(WTF::HashTableDeletedValueType) + : m_identifier(WTF::HashTableDeletedValue) +{ +} + +Key& Key::operator=(const Key& other) +{ + m_partition = other.m_partition.isolatedCopy(); + m_type = other.m_type.isolatedCopy(); + m_identifier = other.m_identifier.isolatedCopy(); + m_range = other.m_range.isolatedCopy(); + m_hash = other.m_hash; + return *this; +} + +static void hashString(SHA1& sha1, const String& string) +{ + if (string.isNull()) + return; + + if (string.is8Bit() && string.containsOnlyASCII()) { + const uint8_t nullByte = 0; + sha1.addBytes(string.characters8(), string.length()); + sha1.addBytes(&nullByte, 1); + return; + } + auto cString = string.utf8(); + // Include terminating null byte. + sha1.addBytes(reinterpret_cast<const uint8_t*>(cString.data()), cString.length() + 1); +} + +Key::HashType Key::computeHash() const +{ + // We don't really need a cryptographic hash. The key is always verified against the entry header. + // SHA1 just happens to be suitably sized, fast and available. + SHA1 sha1; + hashString(sha1, m_partition); + hashString(sha1, m_type); + hashString(sha1, m_identifier); + hashString(sha1, m_range); + SHA1::Digest hash; + sha1.computeHash(hash); + return hash; +} + +String Key::hashAsString() const +{ + StringBuilder builder; + builder.reserveCapacity(hashStringLength()); + for (auto byte : m_hash) { + builder.append(upperNibbleToASCIIHexDigit(byte)); + builder.append(lowerNibbleToASCIIHexDigit(byte)); + } + return builder.toString(); +} + +template <typename CharType> bool hexDigitsToHash(CharType* characters, Key::HashType& hash) +{ + for (unsigned i = 0; i < sizeof(hash); ++i) { + auto high = characters[2 * i]; + auto low = characters[2 * i + 1]; + if (!isASCIIHexDigit(high) || !isASCIIHexDigit(low)) + return false; + hash[i] = toASCIIHexValue(high, low); + } + return true; +} + +bool Key::stringToHash(const String& string, HashType& hash) +{ + if (string.length() != hashStringLength()) + return false; + if (string.is8Bit()) + return hexDigitsToHash(string.characters8(), hash); + return hexDigitsToHash(string.characters16(), hash); +} + +bool Key::operator==(const Key& other) const +{ + return m_hash == other.m_hash && m_partition == other.m_partition && m_type == other.m_type && m_identifier == other.m_identifier && m_range == other.m_range; +} + +void Key::encode(Encoder& encoder) const +{ + encoder << m_partition; + encoder << m_type; + encoder << m_identifier; + encoder << m_range; + encoder << m_hash; +} + +bool Key::decode(Decoder& decoder, Key& key) +{ + return decoder.decode(key.m_partition) && decoder.decode(key.m_type) && decoder.decode(key.m_identifier) && decoder.decode(key.m_range) && decoder.decode(key.m_hash); +} + +} +} + +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheKey.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheKey.h new file mode 100644 index 000000000..12a7007fd --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheKey.h @@ -0,0 +1,120 @@ +/* + * Copyright (C) 2014-2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheKey_h +#define NetworkCacheKey_h + +#if ENABLE(NETWORK_CACHE) + +#include <wtf/SHA1.h> +#include <wtf/text/WTFString.h> + +namespace WebKit { +namespace NetworkCache { + +class Encoder; +class Decoder; + +class Key { +public: + typedef SHA1::Digest HashType; + + Key() { } + Key(const Key&); + Key(Key&&) = default; + Key(const String& partition, const String& type, const String& range, const String& identifier); + + Key& operator=(const Key&); + Key& operator=(Key&&) = default; + + Key(WTF::HashTableDeletedValueType); + bool isHashTableDeletedValue() const { return m_identifier.isHashTableDeletedValue(); } + + bool isNull() const { return m_identifier.isNull(); } + + const String& partition() const { return m_partition; } + const String& identifier() const { return m_identifier; } + const String& type() const { return m_type; } + const String& range() const { return m_range; } + + HashType hash() const { return m_hash; } + + static bool stringToHash(const String&, HashType&); + + static size_t hashStringLength() { return 2 * sizeof(m_hash); } + String hashAsString() const; + + void encode(Encoder&) const; + static bool decode(Decoder&, Key&); + + bool operator==(const Key&) const; + bool operator!=(const Key& other) const { return !(*this == other); } + +private: + HashType computeHash() const; + + String m_partition; + String m_type; + String m_identifier; + String m_range; + HashType m_hash; +}; + +} +} + +namespace WTF { + +struct NetworkCacheKeyHash { + static unsigned hash(const WebKit::NetworkCache::Key& key) + { + static_assert(SHA1::hashSize >= sizeof(unsigned), "Hash size must be greater than sizeof(unsigned)"); + return *reinterpret_cast<const unsigned*>(key.hash().data()); + } + + static bool equal(const WebKit::NetworkCache::Key& a, const WebKit::NetworkCache::Key& b) + { + return a == b; + } + + static const bool safeToCompareToEmptyOrDeleted = false; +}; + +template<typename T> struct DefaultHash; +template<> struct DefaultHash<WebKit::NetworkCache::Key> { + typedef NetworkCacheKeyHash Hash; +}; + +template<> struct HashTraits<WebKit::NetworkCache::Key> : SimpleClassHashTraits<WebKit::NetworkCache::Key> { + static const bool emptyValueIsZero = false; + + static const bool hasIsEmptyValueFunction = true; + static bool isEmptyValue(const WebKit::NetworkCache::Key& key) { return key.isNull(); } +}; + +} // namespace WTF + +#endif +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoad.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoad.cpp new file mode 100644 index 000000000..f6f594030 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoad.cpp @@ -0,0 +1,148 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) +#include "NetworkCacheSpeculativeLoad.h" + +#include "Logging.h" +#include "NetworkCache.h" +#include "NetworkLoad.h" +#include <WebCore/SessionID.h> +#include <wtf/CurrentTime.h> +#include <wtf/RunLoop.h> + +namespace WebKit { +namespace NetworkCache { + +using namespace WebCore; + +SpeculativeLoad::SpeculativeLoad(const GlobalFrameID& frameID, const ResourceRequest& request, std::unique_ptr<NetworkCache::Entry> cacheEntryForValidation, RevalidationCompletionHandler&& completionHandler) + : m_frameID(frameID) + , m_completionHandler(WTFMove(completionHandler)) + , m_originalRequest(request) + , m_bufferedDataForCache(SharedBuffer::create()) + , m_cacheEntryForValidation(WTFMove(cacheEntryForValidation)) +{ + ASSERT(m_cacheEntryForValidation); + ASSERT(m_cacheEntryForValidation->needsValidation()); + + NetworkLoadParameters parameters; + parameters.sessionID = SessionID::defaultSessionID(); + parameters.allowStoredCredentials = AllowStoredCredentials; + parameters.contentSniffingPolicy = DoNotSniffContent; + parameters.request = m_originalRequest; + m_networkLoad = std::make_unique<NetworkLoad>(*this, parameters); +} + +SpeculativeLoad::~SpeculativeLoad() +{ + ASSERT(!m_networkLoad); +} + +void SpeculativeLoad::willSendRedirectedRequest(const ResourceRequest& request, const ResourceRequest& redirectRequest, const ResourceResponse& redirectResponse) +{ + updateRedirectChainStatus(m_redirectChainCacheStatus, redirectResponse); +} + +auto SpeculativeLoad::didReceiveResponse(const ResourceResponse& receivedResponse) -> ShouldContinueDidReceiveResponse +{ + m_response = receivedResponse; + + if (m_response.isMultipart()) + m_bufferedDataForCache = nullptr; + + ASSERT(m_cacheEntryForValidation); + + bool validationSucceeded = m_response.httpStatusCode() == 304; // 304 Not Modified + if (validationSucceeded) { + m_cacheEntryForValidation = NetworkCache::singleton().update(m_originalRequest, m_frameID, *m_cacheEntryForValidation, m_response); + didComplete(); + return ShouldContinueDidReceiveResponse::No; + } + + m_cacheEntryForValidation = nullptr; + + return ShouldContinueDidReceiveResponse::Yes; +} + +void SpeculativeLoad::didReceiveBuffer(RefPtr<SharedBuffer>&& buffer, int reportedEncodedDataLength) +{ + ASSERT(!m_cacheEntryForValidation); + + if (m_bufferedDataForCache) { + // Prevent memory growth in case of streaming data. + const size_t maximumCacheBufferSize = 10 * 1024 * 1024; + if (m_bufferedDataForCache->size() + buffer->size() <= maximumCacheBufferSize) + m_bufferedDataForCache->append(buffer.get()); + else + m_bufferedDataForCache = nullptr; + } +} + +void SpeculativeLoad::didFinishLoading(double finishTime) +{ + ASSERT(!m_cacheEntryForValidation); + + bool allowStale = m_originalRequest.cachePolicy() >= ReturnCacheDataElseLoad; + bool hasCacheableRedirect = m_response.isHTTP() && redirectChainAllowsReuse(m_redirectChainCacheStatus, allowStale ? ReuseExpiredRedirection : DoNotReuseExpiredRedirection); + if (hasCacheableRedirect && m_redirectChainCacheStatus.status == RedirectChainCacheStatus::CachedRedirection) { + // Maybe we should cache the actual redirects instead of the end result? + auto now = std::chrono::system_clock::now(); + auto responseEndOfValidity = now + computeFreshnessLifetimeForHTTPFamily(m_response, now) - computeCurrentAge(m_response, now); + hasCacheableRedirect = responseEndOfValidity <= m_redirectChainCacheStatus.endOfValidity; + } + + if (m_bufferedDataForCache && hasCacheableRedirect) + m_cacheEntryForValidation = NetworkCache::singleton().store(m_originalRequest, m_response, WTFMove(m_bufferedDataForCache), [](NetworkCache::MappedBody& mappedBody) { }); + else if (!hasCacheableRedirect) { + // Make sure we don't keep a stale entry in the cache. + NetworkCache::singleton().remove(m_originalRequest); + } + + didComplete(); +} + +void SpeculativeLoad::didFailLoading(const ResourceError&) +{ + m_cacheEntryForValidation = nullptr; + + didComplete(); +} + +void SpeculativeLoad::didComplete() +{ + RELEASE_ASSERT(RunLoop::isMain()); + + m_networkLoad = nullptr; + + m_completionHandler(WTFMove(m_cacheEntryForValidation)); +} + +} // namespace NetworkCache +} // namespace WebKit + +#endif // ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoad.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoad.h new file mode 100644 index 000000000..3ce3b63bf --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoad.h @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheSpeculativeLoad_h +#define NetworkCacheSpeculativeLoad_h + +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) + +#include "NetworkCache.h" +#include "NetworkCacheEntry.h" +#include "NetworkLoadClient.h" +#include <WebCore/ResourceRequest.h> +#include <WebCore/ResourceResponse.h> +#include <WebCore/SharedBuffer.h> + +namespace WebKit { + +class NetworkLoad; + +namespace NetworkCache { + +class SpeculativeLoad final : public NetworkLoadClient { + WTF_MAKE_FAST_ALLOCATED; +public: + typedef std::function<void (std::unique_ptr<NetworkCache::Entry>)> RevalidationCompletionHandler; + SpeculativeLoad(const GlobalFrameID&, const WebCore::ResourceRequest&, std::unique_ptr<NetworkCache::Entry>, RevalidationCompletionHandler&&); + + virtual ~SpeculativeLoad(); + +private: + // NetworkLoadClient. + virtual void didSendData(unsigned long long bytesSent, unsigned long long totalBytesToBeSent) override { } + virtual void canAuthenticateAgainstProtectionSpaceAsync(const WebCore::ProtectionSpace&) override { } + virtual bool isSynchronous() const override { return false; } + virtual void willSendRedirectedRequest(const WebCore::ResourceRequest&, const WebCore::ResourceRequest& redirectRequest, const WebCore::ResourceResponse& redirectResponse) override; + virtual ShouldContinueDidReceiveResponse didReceiveResponse(const WebCore::ResourceResponse&) override; + virtual void didReceiveBuffer(RefPtr<WebCore::SharedBuffer>&&, int reportedEncodedDataLength) override; + virtual void didFinishLoading(double finishTime) override; + virtual void didFailLoading(const WebCore::ResourceError&) override; + virtual void didConvertToDownload() override { ASSERT_NOT_REACHED(); } +#if PLATFORM(COCOA) + virtual void willCacheResponseAsync(CFCachedURLResponseRef) override { } +#endif + + void didComplete(); + + GlobalFrameID m_frameID; + RevalidationCompletionHandler m_completionHandler; + WebCore::ResourceRequest m_originalRequest; + + std::unique_ptr<NetworkLoad> m_networkLoad; + + WebCore::ResourceResponse m_response; + + RefPtr<WebCore::SharedBuffer> m_bufferedDataForCache; + std::unique_ptr<NetworkCache::Entry> m_cacheEntryForValidation; + + WebCore::RedirectChainCacheStatus m_redirectChainCacheStatus; +}; + +} // namespace NetworkCache +} // namespace WebKit + +#endif // ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) + +#endif // NetworkCacheSpeculativeLoad_h diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoadManager.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoadManager.cpp new file mode 100644 index 000000000..027591100 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoadManager.cpp @@ -0,0 +1,453 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) +#include "NetworkCacheSpeculativeLoadManager.h" + +#include "Logging.h" +#include "NetworkCacheEntry.h" +#include "NetworkCacheSpeculativeLoad.h" +#include "NetworkCacheSubresourcesEntry.h" +#include "NetworkProcess.h" +#include <WebCore/DiagnosticLoggingKeys.h> +#include <WebCore/HysteresisActivity.h> +#include <wtf/HashCountedSet.h> +#include <wtf/NeverDestroyed.h> +#include <wtf/RefCounted.h> +#include <wtf/RunLoop.h> + +namespace WebKit { + +namespace NetworkCache { + +using namespace WebCore; + +static const auto preloadedEntryLifetime = 10_s; + +#if !LOG_DISABLED +static HashCountedSet<String>& allSpeculativeLoadingDiagnosticMessages() +{ + static NeverDestroyed<HashCountedSet<String>> messages; + return messages; +} + +static void printSpeculativeLoadingDiagnosticMessageCounts() +{ + LOG(NetworkCacheSpeculativePreloading, "-- Speculative loading statistics --"); + for (auto& pair : allSpeculativeLoadingDiagnosticMessages()) + LOG(NetworkCacheSpeculativePreloading, "%s: %u", pair.key.utf8().data(), pair.value); +} +#endif + +static void logSpeculativeLoadingDiagnosticMessage(const GlobalFrameID& frameID, const String& message) +{ +#if !LOG_DISABLED + if (WebKit2LogNetworkCacheSpeculativePreloading.state == WTFLogChannelOn) + allSpeculativeLoadingDiagnosticMessages().add(message); +#endif + NetworkProcess::singleton().logDiagnosticMessage(frameID.first, WebCore::DiagnosticLoggingKeys::networkCacheKey(), message, WebCore::ShouldSample::Yes); +} + +static const AtomicString& subresourcesType() +{ + ASSERT(RunLoop::isMain()); + static NeverDestroyed<const AtomicString> resource("subresources", AtomicString::ConstructFromLiteral); + return resource; +} + +static inline Key makeSubresourcesKey(const Key& resourceKey) +{ + return Key(resourceKey.partition(), subresourcesType(), resourceKey.range(), resourceKey.identifier()); +} + +static inline ResourceRequest constructRevalidationRequest(const Entry& entry) +{ + ResourceRequest revalidationRequest(entry.key().identifier()); + + String eTag = entry.response().httpHeaderField(HTTPHeaderName::ETag); + if (!eTag.isEmpty()) + revalidationRequest.setHTTPHeaderField(HTTPHeaderName::IfNoneMatch, eTag); + + String lastModified = entry.response().httpHeaderField(HTTPHeaderName::LastModified); + if (!lastModified.isEmpty()) + revalidationRequest.setHTTPHeaderField(HTTPHeaderName::IfModifiedSince, lastModified); + + return revalidationRequest; +} + +static bool responseNeedsRevalidation(const ResourceResponse& response, std::chrono::system_clock::time_point timestamp) +{ + if (response.cacheControlContainsNoCache()) + return true; + + auto age = computeCurrentAge(response, timestamp); + auto lifetime = computeFreshnessLifetimeForHTTPFamily(response, timestamp); + return age - lifetime > 0_ms; +} + +class SpeculativeLoadManager::ExpiringEntry { + WTF_MAKE_FAST_ALLOCATED; +public: + explicit ExpiringEntry(std::function<void()>&& expirationHandler) + : m_lifetimeTimer(WTFMove(expirationHandler)) + { + m_lifetimeTimer.startOneShot(preloadedEntryLifetime); + } + +private: + Timer m_lifetimeTimer; +}; + +class SpeculativeLoadManager::PreloadedEntry : private ExpiringEntry { + WTF_MAKE_FAST_ALLOCATED; +public: + PreloadedEntry(std::unique_ptr<Entry> entry, WasRevalidated wasRevalidated, std::function<void()>&& lifetimeReachedHandler) + : ExpiringEntry(WTFMove(lifetimeReachedHandler)) + , m_entry(WTFMove(entry)) + , m_wasRevalidated(wasRevalidated == WasRevalidated::Yes) + { } + + std::unique_ptr<Entry> takeCacheEntry() + { + ASSERT(m_entry); + return WTFMove(m_entry); + } + + bool wasRevalidated() const { return m_wasRevalidated; } + +private: + std::unique_ptr<Entry> m_entry; + bool m_wasRevalidated; +}; + +class SpeculativeLoadManager::PendingFrameLoad : public RefCounted<PendingFrameLoad> { +public: + static Ref<PendingFrameLoad> create(Storage& storage, const Key& mainResourceKey, std::function<void()>&& loadCompletionHandler) + { + return adoptRef(*new PendingFrameLoad(storage, mainResourceKey, WTFMove(loadCompletionHandler))); + } + + ~PendingFrameLoad() + { + ASSERT(m_didFinishLoad); + ASSERT(m_didRetrieveExistingEntry); + } + + void registerSubresource(const Key& subresourceKey) + { + ASSERT(RunLoop::isMain()); + m_subresourceKeys.append(subresourceKey); + m_loadHysteresisActivity.impulse(); + } + + void markLoadAsCompleted() + { + ASSERT(RunLoop::isMain()); + if (m_didFinishLoad) + return; + +#if !LOG_DISABLED + printSpeculativeLoadingDiagnosticMessageCounts(); +#endif + + m_didFinishLoad = true; + saveToDiskIfReady(); + m_loadCompletionHandler(); + } + + void setExistingSubresourcesEntry(std::unique_ptr<SubresourcesEntry> entry) + { + ASSERT(!m_existingEntry); + ASSERT(!m_didRetrieveExistingEntry); + + m_existingEntry = WTFMove(entry); + m_didRetrieveExistingEntry = true; + saveToDiskIfReady(); + } + +private: + PendingFrameLoad(Storage& storage, const Key& mainResourceKey, std::function<void()>&& loadCompletionHandler) + : m_storage(storage) + , m_mainResourceKey(mainResourceKey) + , m_loadCompletionHandler(WTFMove(loadCompletionHandler)) + , m_loadHysteresisActivity([this](HysteresisState state) { if (state == HysteresisState::Stopped) markLoadAsCompleted(); }) + { } + + void saveToDiskIfReady() + { + if (!m_didFinishLoad || !m_didRetrieveExistingEntry) + return; + + if (m_subresourceKeys.isEmpty()) + return; + +#if !LOG_DISABLED + LOG(NetworkCacheSpeculativePreloading, "(NetworkProcess) Saving to disk list of subresources for '%s':", m_mainResourceKey.identifier().utf8().data()); + for (auto& subresourceKey : m_subresourceKeys) + LOG(NetworkCacheSpeculativePreloading, "(NetworkProcess) * Subresource: '%s'.", subresourceKey.identifier().utf8().data()); +#endif + + if (m_existingEntry) { + m_existingEntry->updateSubresourceKeys(m_subresourceKeys); + m_storage.store(m_existingEntry->encodeAsStorageRecord(), [](const Data&) { }); + } else { + SubresourcesEntry entry(makeSubresourcesKey(m_mainResourceKey), m_subresourceKeys); + m_storage.store(entry.encodeAsStorageRecord(), [](const Data&) { }); + } + } + + Storage& m_storage; + Key m_mainResourceKey; + Vector<Key> m_subresourceKeys; + std::function<void()> m_loadCompletionHandler; + HysteresisActivity m_loadHysteresisActivity; + std::unique_ptr<SubresourcesEntry> m_existingEntry; + bool m_didFinishLoad { false }; + bool m_didRetrieveExistingEntry { false }; +}; + +SpeculativeLoadManager::SpeculativeLoadManager(Storage& storage) + : m_storage(storage) +{ +} + +SpeculativeLoadManager::~SpeculativeLoadManager() +{ +} + +bool SpeculativeLoadManager::retrieve(const GlobalFrameID& frameID, const Key& storageKey, const RetrieveCompletionHandler& completionHandler) +{ + // Check already preloaded entries. + if (auto preloadedEntry = m_preloadedEntries.take(storageKey)) { + LOG(NetworkCacheSpeculativePreloading, "(NetworkProcess) Retrieval: Using preloaded entry to satisfy request for '%s':", storageKey.identifier().utf8().data()); + if (preloadedEntry->wasRevalidated()) + logSpeculativeLoadingDiagnosticMessage(frameID, DiagnosticLoggingKeys::successfulSpeculativeWarmupWithRevalidationKey()); + else + logSpeculativeLoadingDiagnosticMessage(frameID, DiagnosticLoggingKeys::successfulSpeculativeWarmupWithoutRevalidationKey()); + + completionHandler(preloadedEntry->takeCacheEntry()); + return true; + } + + // Check pending speculative revalidations. + if (!m_pendingPreloads.contains(storageKey)) { + if (m_notPreloadedEntries.remove(storageKey)) + logSpeculativeLoadingDiagnosticMessage(frameID, DiagnosticLoggingKeys::entryWronglyNotWarmedUpKey()); + else + logSpeculativeLoadingDiagnosticMessage(frameID, DiagnosticLoggingKeys::unknownEntryRequestKey()); + + return false; + } + + LOG(NetworkCacheSpeculativePreloading, "(NetworkProcess) Retrieval: revalidation already in progress for '%s':", storageKey.identifier().utf8().data()); + + // FIXME: This breaks incremental loading when the revalidation is not successful. + auto addResult = m_pendingRetrieveRequests.add(storageKey, nullptr); + if (addResult.isNewEntry) + addResult.iterator->value = std::make_unique<Vector<RetrieveCompletionHandler>>(); + addResult.iterator->value->append(completionHandler); + return true; +} + +void SpeculativeLoadManager::registerLoad(const GlobalFrameID& frameID, const ResourceRequest& request, const Key& resourceKey) +{ + ASSERT(RunLoop::isMain()); + + if (!request.url().protocolIsInHTTPFamily() || request.httpMethod() != "GET") + return; + + auto isMainResource = request.requester() == ResourceRequest::Requester::Main; + if (isMainResource) { + // Mark previous load in this frame as completed if necessary. + if (auto* pendingFrameLoad = m_pendingFrameLoads.get(frameID)) + pendingFrameLoad->markLoadAsCompleted(); + + ASSERT(!m_pendingFrameLoads.contains(frameID)); + + // Start tracking loads in this frame. + RefPtr<PendingFrameLoad> pendingFrameLoad = PendingFrameLoad::create(m_storage, resourceKey, [this, frameID] { + bool wasRemoved = m_pendingFrameLoads.remove(frameID); + ASSERT_UNUSED(wasRemoved, wasRemoved); + }); + m_pendingFrameLoads.add(frameID, pendingFrameLoad); + + // Retrieve the subresources entry if it exists to start speculative revalidation and to update it. + retrieveSubresourcesEntry(resourceKey, [this, frameID, pendingFrameLoad](std::unique_ptr<SubresourcesEntry> entry) { + if (entry) + startSpeculativeRevalidation(frameID, *entry); + + pendingFrameLoad->setExistingSubresourcesEntry(WTFMove(entry)); + }); + return; + } + + if (auto* pendingFrameLoad = m_pendingFrameLoads.get(frameID)) + pendingFrameLoad->registerSubresource(resourceKey); +} + +void SpeculativeLoadManager::addPreloadedEntry(std::unique_ptr<Entry> entry, const GlobalFrameID& frameID, WasRevalidated wasRevalidated) +{ + ASSERT(entry); + ASSERT(!entry->needsValidation()); + auto key = entry->key(); + m_preloadedEntries.add(key, std::make_unique<PreloadedEntry>(WTFMove(entry), wasRevalidated, [this, key, frameID] { + auto preloadedEntry = m_preloadedEntries.take(key); + ASSERT(preloadedEntry); + if (preloadedEntry->wasRevalidated()) + logSpeculativeLoadingDiagnosticMessage(frameID, DiagnosticLoggingKeys::wastedSpeculativeWarmupWithRevalidationKey()); + else + logSpeculativeLoadingDiagnosticMessage(frameID, DiagnosticLoggingKeys::wastedSpeculativeWarmupWithoutRevalidationKey()); + })); +} + +void SpeculativeLoadManager::retrieveEntryFromStorage(const Key& key, const RetrieveCompletionHandler& completionHandler) +{ + m_storage.retrieve(key, static_cast<unsigned>(ResourceLoadPriority::Medium), [completionHandler](std::unique_ptr<Storage::Record> record) { + if (!record) { + completionHandler(nullptr); + return false; + } + auto entry = Entry::decodeStorageRecord(*record); + if (!entry) { + completionHandler(nullptr); + return false; + } + + auto& response = entry->response(); + if (!response.hasCacheValidatorFields()) { + completionHandler(nullptr); + return true; + } + + if (responseNeedsRevalidation(response, entry->timeStamp())) + entry->setNeedsValidation(); + + completionHandler(WTFMove(entry)); + return true; + }); +} + +bool SpeculativeLoadManager::satisfyPendingRequests(const Key& key, Entry* entry) +{ + auto completionHandlers = m_pendingRetrieveRequests.take(key); + if (!completionHandlers) + return false; + + for (auto& completionHandler : *completionHandlers) + completionHandler(entry ? std::make_unique<Entry>(*entry) : nullptr); + + return true; +} + +void SpeculativeLoadManager::revalidateEntry(std::unique_ptr<Entry> entry, const GlobalFrameID& frameID) +{ + ASSERT(entry); + ASSERT(entry->needsValidation()); + + auto key = entry->key(); + LOG(NetworkCacheSpeculativePreloading, "(NetworkProcess) Speculatively revalidating '%s':", key.identifier().utf8().data()); + auto revalidator = std::make_unique<SpeculativeLoad>(frameID, constructRevalidationRequest(*entry), WTFMove(entry), [this, key, frameID](std::unique_ptr<Entry> revalidatedEntry) { + ASSERT(!revalidatedEntry || !revalidatedEntry->needsValidation()); + auto protectRevalidator = m_pendingPreloads.take(key); + LOG(NetworkCacheSpeculativePreloading, "(NetworkProcess) Speculative revalidation completed for '%s':", key.identifier().utf8().data()); + + if (satisfyPendingRequests(key, revalidatedEntry.get())) { + if (revalidatedEntry) + logSpeculativeLoadingDiagnosticMessage(frameID, DiagnosticLoggingKeys::successfulSpeculativeWarmupWithRevalidationKey()); + return; + } + + if (revalidatedEntry) + addPreloadedEntry(WTFMove(revalidatedEntry), frameID, WasRevalidated::Yes); + }); + m_pendingPreloads.add(key, WTFMove(revalidator)); +} + +void SpeculativeLoadManager::preloadEntry(const Key& key, const GlobalFrameID& frameID) +{ + m_pendingPreloads.add(key, nullptr); + retrieveEntryFromStorage(key, [this, key, frameID](std::unique_ptr<Entry> entry) { + m_pendingPreloads.remove(key); + + if (satisfyPendingRequests(key, entry.get())) { + if (entry) + logSpeculativeLoadingDiagnosticMessage(frameID, DiagnosticLoggingKeys::successfulSpeculativeWarmupWithoutRevalidationKey()); + return; + } + + if (!entry) + return; + + if (entry->needsValidation()) + revalidateEntry(WTFMove(entry), frameID); + else + addPreloadedEntry(WTFMove(entry), frameID, WasRevalidated::No); + }); +} + +void SpeculativeLoadManager::startSpeculativeRevalidation(const GlobalFrameID& frameID, SubresourcesEntry& entry) +{ + for (auto& subresource : entry.subresources()) { + auto key = subresource.key; + if (!subresource.value.isTransient) + preloadEntry(key, frameID); + else { + LOG(NetworkCacheSpeculativePreloading, "(NetworkProcess) Not preloading '%s' because it is marked as transient", subresource.key.identifier().utf8().data()); + m_notPreloadedEntries.add(key, std::make_unique<ExpiringEntry>([this, key, frameID] { + logSpeculativeLoadingDiagnosticMessage(frameID, DiagnosticLoggingKeys::entryRightlyNotWarmedUpKey()); + m_notPreloadedEntries.remove(key); + })); + } + } +} + +void SpeculativeLoadManager::retrieveSubresourcesEntry(const Key& storageKey, std::function<void (std::unique_ptr<SubresourcesEntry>)> completionHandler) +{ + ASSERT(storageKey.type() == "resource"); + auto subresourcesStorageKey = makeSubresourcesKey(storageKey); + m_storage.retrieve(subresourcesStorageKey, static_cast<unsigned>(ResourceLoadPriority::Medium), [completionHandler](std::unique_ptr<Storage::Record> record) { + if (!record) { + completionHandler(nullptr); + return false; + } + + auto subresourcesEntry = SubresourcesEntry::decodeStorageRecord(*record); + if (!subresourcesEntry) { + completionHandler(nullptr); + return false; + } + + completionHandler(WTFMove(subresourcesEntry)); + return true; + }); +} + +} // namespace NetworkCache + +} // namespace WebKit + +#endif // ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoadManager.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoadManager.h new file mode 100644 index 000000000..4c1de7200 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSpeculativeLoadManager.h @@ -0,0 +1,86 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheSpeculativeLoadManager_h +#define NetworkCacheSpeculativeLoadManager_h + +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) + +#include "NetworkCache.h" +#include "NetworkCacheStorage.h" +#include <WebCore/ResourceRequest.h> +#include <wtf/HashMap.h> +#include <wtf/Vector.h> + +namespace WebKit { + +namespace NetworkCache { + +class Entry; +class SpeculativeLoad; +class SubresourcesEntry; + +class SpeculativeLoadManager { +public: + explicit SpeculativeLoadManager(Storage&); + ~SpeculativeLoadManager(); + + void registerLoad(const GlobalFrameID&, const WebCore::ResourceRequest&, const Key& resourceKey); + + typedef std::function<void (std::unique_ptr<Entry>)> RetrieveCompletionHandler; + bool retrieve(const GlobalFrameID&, const Key& storageKey, const RetrieveCompletionHandler&); + +private: + enum class WasRevalidated { No, Yes }; + void addPreloadedEntry(std::unique_ptr<Entry>, const GlobalFrameID&, WasRevalidated); + void preloadEntry(const Key&, const GlobalFrameID&); + void retrieveEntryFromStorage(const Key&, const RetrieveCompletionHandler&); + void revalidateEntry(std::unique_ptr<Entry>, const GlobalFrameID&); + bool satisfyPendingRequests(const Key&, Entry*); + void retrieveSubresourcesEntry(const Key& storageKey, std::function<void (std::unique_ptr<SubresourcesEntry>)>); + void startSpeculativeRevalidation(const GlobalFrameID&, SubresourcesEntry&); + + Storage& m_storage; + + class PendingFrameLoad; + HashMap<GlobalFrameID, RefPtr<PendingFrameLoad>> m_pendingFrameLoads; + + HashMap<Key, std::unique_ptr<SpeculativeLoad>> m_pendingPreloads; + HashMap<Key, std::unique_ptr<Vector<RetrieveCompletionHandler>>> m_pendingRetrieveRequests; + + class PreloadedEntry; + HashMap<Key, std::unique_ptr<PreloadedEntry>> m_preloadedEntries; + + class ExpiringEntry; + HashMap<Key, std::unique_ptr<ExpiringEntry>> m_notPreloadedEntries; // For logging. +}; + +} // namespace NetworkCache + +} // namespace WebKit + +#endif // ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) + +#endif // NetworkCacheSpeculativeLoadManager_h diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheStatistics.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheStatistics.cpp new file mode 100644 index 000000000..263393db7 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheStatistics.cpp @@ -0,0 +1,462 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#if ENABLE(NETWORK_CACHE) +#include "NetworkCacheStatistics.h" + +#include "Logging.h" +#include "NetworkCache.h" +#include "NetworkCacheFileSystem.h" +#include "NetworkProcess.h" +#include <WebCore/DiagnosticLoggingKeys.h> +#include <WebCore/DiagnosticLoggingResultType.h> +#include <WebCore/ResourceRequest.h> +#include <WebCore/SQLiteDatabaseTracker.h> +#include <WebCore/SQLiteStatement.h> +#include <WebCore/SQLiteTransaction.h> +#include <wtf/RunLoop.h> + +namespace WebKit { +namespace NetworkCache { + +static const char* StatisticsDatabaseName = "WebKitCacheStatistics.db"; +static const std::chrono::milliseconds mininumWriteInterval = std::chrono::milliseconds(10000); + +static bool executeSQLCommand(WebCore::SQLiteDatabase& database, const String& sql) +{ + ASSERT(!RunLoop::isMain()); + ASSERT(WebCore::SQLiteDatabaseTracker::hasTransactionInProgress()); + ASSERT(database.isOpen()); + + bool result = database.executeCommand(sql); + if (!result) + LOG_ERROR("Network cache statistics: failed to execute statement \"%s\" error \"%s\"", sql.utf8().data(), database.lastErrorMsg()); + + return result; +} + +static bool executeSQLStatement(WebCore::SQLiteStatement& statement) +{ + ASSERT(!RunLoop::isMain()); + ASSERT(WebCore::SQLiteDatabaseTracker::hasTransactionInProgress()); + ASSERT(statement.database().isOpen()); + + if (statement.step() != SQLITE_DONE) { + LOG_ERROR("Network cache statistics: failed to execute statement \"%s\" error \"%s\"", statement.query().utf8().data(), statement.database().lastErrorMsg()); + return false; + } + + return true; +} + +std::unique_ptr<Statistics> Statistics::open(const String& cachePath) +{ + ASSERT(RunLoop::isMain()); + + String databasePath = WebCore::pathByAppendingComponent(cachePath, StatisticsDatabaseName); + return std::unique_ptr<Statistics>(new Statistics(databasePath)); +} + +Statistics::Statistics(const String& databasePath) + : m_serialBackgroundIOQueue(WorkQueue::create("com.apple.WebKit.Cache.Statistics.Background", WorkQueue::Type::Serial, WorkQueue::QOS::Background)) + , m_writeTimer(*this, &Statistics::writeTimerFired) +{ + initialize(databasePath); +} + +void Statistics::initialize(const String& databasePath) +{ + ASSERT(RunLoop::isMain()); + + auto startTime = std::chrono::system_clock::now(); + + StringCapture databasePathCapture(databasePath); + StringCapture networkCachePathCapture(singleton().recordsPath()); + serialBackgroundIOQueue().dispatch([this, databasePathCapture, networkCachePathCapture, startTime] { + WebCore::SQLiteTransactionInProgressAutoCounter transactionCounter; + + String databasePath = databasePathCapture.string(); + if (!WebCore::makeAllDirectories(WebCore::directoryName(databasePath))) + return; + + LOG(NetworkCache, "(NetworkProcess) Opening network cache statistics database at %s...", databasePath.utf8().data()); + m_database.open(databasePath); + m_database.disableThreadingChecks(); + if (!m_database.isOpen()) { + LOG_ERROR("Network cache statistics: Failed to open / create the network cache statistics database"); + return; + } + + executeSQLCommand(m_database, ASCIILiteral("CREATE TABLE IF NOT EXISTS AlreadyRequested (hash TEXT PRIMARY KEY)")); + executeSQLCommand(m_database, ASCIILiteral("CREATE TABLE IF NOT EXISTS UncachedReason (hash TEXT PRIMARY KEY, reason INTEGER)")); + + WebCore::SQLiteStatement statement(m_database, ASCIILiteral("SELECT count(*) FROM AlreadyRequested")); + if (statement.prepareAndStep() != SQLITE_ROW) { + LOG_ERROR("Network cache statistics: Failed to count the number of rows in AlreadyRequested table"); + return; + } + + m_approximateEntryCount = statement.getColumnInt(0); + +#if !LOG_DISABLED + auto elapsedMS = static_cast<int64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now() - startTime).count()); +#endif + LOG(NetworkCache, "(NetworkProcess) Network cache statistics database load complete, entries=%lu time=%" PRIi64 "ms", static_cast<size_t>(m_approximateEntryCount), elapsedMS); + + if (!m_approximateEntryCount) { + bootstrapFromNetworkCache(networkCachePathCapture.string()); +#if !LOG_DISABLED + elapsedMS = static_cast<int64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now() - startTime).count()); +#endif + LOG(NetworkCache, "(NetworkProcess) Network cache statistics database bootstrapping complete, entries=%lu time=%" PRIi64 "ms", static_cast<size_t>(m_approximateEntryCount), elapsedMS); + } + }); +} + +void Statistics::bootstrapFromNetworkCache(const String& networkCachePath) +{ + ASSERT(!RunLoop::isMain()); + + LOG(NetworkCache, "(NetworkProcess) Bootstrapping the network cache statistics database from the network cache..."); + + Vector<StringCapture> hashes; + traverseRecordsFiles(networkCachePath, ASCIILiteral("resource"), [&hashes](const String& fileName, const String& hashString, const String& type, bool isBodyBlob, const String& recordDirectoryPath) { + if (isBodyBlob) + return; + + Key::HashType hash; + if (!Key::stringToHash(hashString, hash)) + return; + + hashes.append(hashString); + }); + + WebCore::SQLiteTransactionInProgressAutoCounter transactionCounter; + WebCore::SQLiteTransaction writeTransaction(m_database); + writeTransaction.begin(); + + addHashesToDatabase(hashes); + + writeTransaction.commit(); +} + +void Statistics::shrinkIfNeeded() +{ + ASSERT(RunLoop::isMain()); + const size_t maxEntries = 100000; + + if (m_approximateEntryCount < maxEntries) + return; + + LOG(NetworkCache, "(NetworkProcess) shrinking statistics cache m_approximateEntryCount=%lu, maxEntries=%lu", static_cast<size_t>(m_approximateEntryCount), maxEntries); + + clear(); + + StringCapture networkCachePathCapture(singleton().recordsPath()); + serialBackgroundIOQueue().dispatch([this, networkCachePathCapture] { + bootstrapFromNetworkCache(networkCachePathCapture.string()); + LOG(NetworkCache, "(NetworkProcess) statistics cache shrink completed m_approximateEntryCount=%lu", static_cast<size_t>(m_approximateEntryCount)); + }); +} + +void Statistics::recordRetrievalRequest(uint64_t webPageID) +{ + NetworkProcess::singleton().logDiagnosticMessage(webPageID, WebCore::DiagnosticLoggingKeys::networkCacheKey(), WebCore::DiagnosticLoggingKeys::retrievalRequestKey(), WebCore::ShouldSample::Yes); +} + +void Statistics::recordNotCachingResponse(const Key& key, StoreDecision storeDecision) +{ + ASSERT(storeDecision != StoreDecision::Yes); + + m_storeDecisionsToAdd.set(key.hashAsString(), storeDecision); + if (!m_writeTimer.isActive()) + m_writeTimer.startOneShot(mininumWriteInterval); +} + +static String retrieveDecisionToDiagnosticKey(RetrieveDecision retrieveDecision) +{ + switch (retrieveDecision) { + case RetrieveDecision::NoDueToHTTPMethod: + return WebCore::DiagnosticLoggingKeys::unsupportedHTTPMethodKey(); + case RetrieveDecision::NoDueToConditionalRequest: + return WebCore::DiagnosticLoggingKeys::isConditionalRequestKey(); + case RetrieveDecision::NoDueToReloadIgnoringCache: + return WebCore::DiagnosticLoggingKeys::isReloadIgnoringCacheDataKey(); + case RetrieveDecision::Yes: + ASSERT_NOT_REACHED(); + break; + } + return emptyString(); +} + +void Statistics::recordNotUsingCacheForRequest(uint64_t webPageID, const Key& key, const WebCore::ResourceRequest& request, RetrieveDecision retrieveDecision) +{ + ASSERT(retrieveDecision != RetrieveDecision::Yes); + + String hash = key.hashAsString(); + WebCore::URL requestURL = request.url(); + queryWasEverRequested(hash, NeedUncachedReason::No, [this, hash, requestURL, webPageID, retrieveDecision](bool wasEverRequested, const Optional<StoreDecision>&) { + if (wasEverRequested) { + String diagnosticKey = retrieveDecisionToDiagnosticKey(retrieveDecision); + LOG(NetworkCache, "(NetworkProcess) webPageID %" PRIu64 ": %s was previously requested but we are not using the cache, reason: %s", webPageID, requestURL.string().ascii().data(), diagnosticKey.utf8().data()); + NetworkProcess::singleton().logDiagnosticMessageWithValue(webPageID, WebCore::DiagnosticLoggingKeys::networkCacheKey(), WebCore::DiagnosticLoggingKeys::unusedKey(), diagnosticKey, WebCore::ShouldSample::Yes); + } else { + NetworkProcess::singleton().logDiagnosticMessageWithValue(webPageID, WebCore::DiagnosticLoggingKeys::networkCacheKey(), WebCore::DiagnosticLoggingKeys::requestKey(), WebCore::DiagnosticLoggingKeys::neverSeenBeforeKey(), WebCore::ShouldSample::Yes); + markAsRequested(hash); + } + }); +} + +static String storeDecisionToDiagnosticKey(StoreDecision storeDecision) +{ + switch (storeDecision) { + case StoreDecision::NoDueToProtocol: + return WebCore::DiagnosticLoggingKeys::notHTTPFamilyKey(); + case StoreDecision::NoDueToHTTPMethod: + return WebCore::DiagnosticLoggingKeys::unsupportedHTTPMethodKey(); + case StoreDecision::NoDueToAttachmentResponse: + return WebCore::DiagnosticLoggingKeys::isAttachmentKey(); + case StoreDecision::NoDueToNoStoreResponse: + case StoreDecision::NoDueToNoStoreRequest: + return WebCore::DiagnosticLoggingKeys::cacheControlNoStoreKey(); + case StoreDecision::NoDueToHTTPStatusCode: + return WebCore::DiagnosticLoggingKeys::uncacheableStatusCodeKey(); + case StoreDecision::NoDueToUnlikelyToReuse: + return WebCore::DiagnosticLoggingKeys::unlikelyToReuseKey(); + case StoreDecision::NoDueToStreamingMedia: + return WebCore::DiagnosticLoggingKeys::streamingMedia(); + case StoreDecision::Yes: + // It was stored but could not be retrieved so it must have been pruned from the cache. + return WebCore::DiagnosticLoggingKeys::noLongerInCacheKey(); + } + return String(); +} + +void Statistics::recordRetrievalFailure(uint64_t webPageID, const Key& key, const WebCore::ResourceRequest& request) +{ + String hash = key.hashAsString(); + WebCore::URL requestURL = request.url(); + queryWasEverRequested(hash, NeedUncachedReason::Yes, [this, hash, requestURL, webPageID](bool wasPreviouslyRequested, const Optional<StoreDecision>& storeDecision) { + if (wasPreviouslyRequested) { + String diagnosticKey = storeDecisionToDiagnosticKey(storeDecision.value()); + LOG(NetworkCache, "(NetworkProcess) webPageID %" PRIu64 ": %s was previously request but is not in the cache, reason: %s", webPageID, requestURL.string().ascii().data(), diagnosticKey.utf8().data()); + NetworkProcess::singleton().logDiagnosticMessageWithValue(webPageID, WebCore::DiagnosticLoggingKeys::networkCacheKey(), WebCore::DiagnosticLoggingKeys::notInCacheKey(), diagnosticKey, WebCore::ShouldSample::Yes); + } else { + NetworkProcess::singleton().logDiagnosticMessageWithValue(webPageID, WebCore::DiagnosticLoggingKeys::networkCacheKey(), WebCore::DiagnosticLoggingKeys::requestKey(), WebCore::DiagnosticLoggingKeys::neverSeenBeforeKey(), WebCore::ShouldSample::Yes); + markAsRequested(hash); + } + }); +} + +static String cachedEntryReuseFailureToDiagnosticKey(UseDecision decision) +{ + switch (decision) { + case UseDecision::NoDueToVaryingHeaderMismatch: + return WebCore::DiagnosticLoggingKeys::varyingHeaderMismatchKey(); + case UseDecision::NoDueToMissingValidatorFields: + return WebCore::DiagnosticLoggingKeys::missingValidatorFieldsKey(); + case UseDecision::NoDueToDecodeFailure: + case UseDecision::NoDueToExpiredRedirect: + return WebCore::DiagnosticLoggingKeys::otherKey(); + case UseDecision::Use: + case UseDecision::Validate: + ASSERT_NOT_REACHED(); + break; + } + return emptyString(); +} + +void Statistics::recordRetrievedCachedEntry(uint64_t webPageID, const Key& key, const WebCore::ResourceRequest& request, UseDecision decision) +{ + WebCore::URL requestURL = request.url(); + if (decision == UseDecision::Use) { + LOG(NetworkCache, "(NetworkProcess) webPageID %" PRIu64 ": %s is in the cache and is used", webPageID, requestURL.string().ascii().data()); + NetworkProcess::singleton().logDiagnosticMessageWithResult(webPageID, WebCore::DiagnosticLoggingKeys::networkCacheKey(), WebCore::DiagnosticLoggingKeys::retrievalKey(), WebCore::DiagnosticLoggingResultPass, WebCore::ShouldSample::Yes); + return; + } + + if (decision == UseDecision::Validate) { + LOG(NetworkCache, "(NetworkProcess) webPageID %" PRIu64 ": %s is in the cache but needs revalidation", webPageID, requestURL.string().ascii().data()); + NetworkProcess::singleton().logDiagnosticMessageWithValue(webPageID, WebCore::DiagnosticLoggingKeys::networkCacheKey(), WebCore::DiagnosticLoggingKeys::retrievalKey(), WebCore::DiagnosticLoggingKeys::needsRevalidationKey(), WebCore::ShouldSample::Yes); + return; + } + + String diagnosticKey = cachedEntryReuseFailureToDiagnosticKey(decision); + LOG(NetworkCache, "(NetworkProcess) webPageID %" PRIu64 ": %s is in the cache but wasn't used, reason: %s", webPageID, requestURL.string().ascii().data(), diagnosticKey.utf8().data()); + NetworkProcess::singleton().logDiagnosticMessageWithValue(webPageID, WebCore::DiagnosticLoggingKeys::networkCacheKey(), WebCore::DiagnosticLoggingKeys::unusableCachedEntryKey(), diagnosticKey, WebCore::ShouldSample::Yes); +} + +void Statistics::recordRevalidationSuccess(uint64_t webPageID, const Key& key, const WebCore::ResourceRequest& request) +{ + WebCore::URL requestURL = request.url(); + LOG(NetworkCache, "(NetworkProcess) webPageID %" PRIu64 ": %s was successfully revalidated", webPageID, requestURL.string().ascii().data()); + + NetworkProcess::singleton().logDiagnosticMessageWithResult(webPageID, WebCore::DiagnosticLoggingKeys::networkCacheKey(), WebCore::DiagnosticLoggingKeys::revalidatingKey(), WebCore::DiagnosticLoggingResultPass, WebCore::ShouldSample::Yes); +} + +void Statistics::markAsRequested(const String& hash) +{ + ASSERT(RunLoop::isMain()); + + m_hashesToAdd.add(hash); + if (!m_writeTimer.isActive()) + m_writeTimer.startOneShot(mininumWriteInterval); +} + +void Statistics::writeTimerFired() +{ + ASSERT(RunLoop::isMain()); + + Vector<StringCapture> hashesToAdd; + copyToVector(m_hashesToAdd, hashesToAdd); + m_hashesToAdd.clear(); + + Vector<std::pair<StringCapture, StoreDecision>> storeDecisionsToAdd; + copyToVector(m_storeDecisionsToAdd, storeDecisionsToAdd); + m_storeDecisionsToAdd.clear(); + + shrinkIfNeeded(); + + serialBackgroundIOQueue().dispatch([this, hashesToAdd, storeDecisionsToAdd] { + if (!m_database.isOpen()) + return; + + WebCore::SQLiteTransactionInProgressAutoCounter transactionCounter; + WebCore::SQLiteTransaction writeTransaction(m_database); + writeTransaction.begin(); + + addHashesToDatabase(hashesToAdd); + addStoreDecisionsToDatabase(storeDecisionsToAdd); + + writeTransaction.commit(); + }); +} + +void Statistics::queryWasEverRequested(const String& hash, NeedUncachedReason needUncachedReason, const RequestedCompletionHandler& completionHandler) +{ + ASSERT(RunLoop::isMain()); + + // Query pending writes first. + bool wasAlreadyRequested = m_hashesToAdd.contains(hash); + if (wasAlreadyRequested && needUncachedReason == NeedUncachedReason::No) { + completionHandler(true, Nullopt); + return; + } + if (needUncachedReason == NeedUncachedReason::Yes && m_storeDecisionsToAdd.contains(hash)) { + completionHandler(true, m_storeDecisionsToAdd.get(hash)); + return; + } + + // Query the database. + auto everRequestedQuery = std::make_unique<EverRequestedQuery>(EverRequestedQuery { hash, needUncachedReason == NeedUncachedReason::Yes, completionHandler }); + auto& query = *everRequestedQuery; + m_activeQueries.add(WTFMove(everRequestedQuery)); + serialBackgroundIOQueue().dispatch([this, wasAlreadyRequested, &query] () mutable { + WebCore::SQLiteTransactionInProgressAutoCounter transactionCounter; + Optional<StoreDecision> storeDecision; + if (m_database.isOpen()) { + if (!wasAlreadyRequested) { + WebCore::SQLiteStatement statement(m_database, ASCIILiteral("SELECT hash FROM AlreadyRequested WHERE hash=?")); + if (statement.prepare() == SQLITE_OK) { + statement.bindText(1, query.hash); + wasAlreadyRequested = (statement.step() == SQLITE_ROW); + } + } + if (wasAlreadyRequested && query.needUncachedReason) { + WebCore::SQLiteStatement statement(m_database, ASCIILiteral("SELECT reason FROM UncachedReason WHERE hash=?")); + storeDecision = StoreDecision::Yes; + if (statement.prepare() == SQLITE_OK) { + statement.bindText(1, query.hash); + if (statement.step() == SQLITE_ROW) + storeDecision = static_cast<StoreDecision>(statement.getColumnInt(0)); + } + } + } + RunLoop::main().dispatch([this, &query, wasAlreadyRequested, storeDecision] { + query.completionHandler(wasAlreadyRequested, storeDecision); + m_activeQueries.remove(&query); + }); + }); +} + +void Statistics::clear() +{ + ASSERT(RunLoop::isMain()); + + serialBackgroundIOQueue().dispatch([this] { + if (m_database.isOpen()) { + WebCore::SQLiteTransactionInProgressAutoCounter transactionCounter; + WebCore::SQLiteTransaction deleteTransaction(m_database); + deleteTransaction.begin(); + executeSQLCommand(m_database, ASCIILiteral("DELETE FROM AlreadyRequested")); + executeSQLCommand(m_database, ASCIILiteral("DELETE FROM UncachedReason")); + deleteTransaction.commit(); + m_approximateEntryCount = 0; + } + }); +} + +void Statistics::addHashesToDatabase(const Vector<StringCapture>& hashes) +{ + ASSERT(!RunLoop::isMain()); + ASSERT(WebCore::SQLiteDatabaseTracker::hasTransactionInProgress()); + ASSERT(m_database.isOpen()); + + WebCore::SQLiteStatement statement(m_database, ASCIILiteral("INSERT OR IGNORE INTO AlreadyRequested (hash) VALUES (?)")); + if (statement.prepare() != SQLITE_OK) + return; + + for (auto& hash : hashes) { + statement.bindText(1, hash.string()); + if (executeSQLStatement(statement)) + ++m_approximateEntryCount; + statement.reset(); + } +} + +void Statistics::addStoreDecisionsToDatabase(const Vector<std::pair<StringCapture, StoreDecision>>& storeDecisions) +{ + ASSERT(!RunLoop::isMain()); + ASSERT(WebCore::SQLiteDatabaseTracker::hasTransactionInProgress()); + ASSERT(m_database.isOpen()); + + WebCore::SQLiteStatement statement(m_database, ASCIILiteral("INSERT OR REPLACE INTO UncachedReason (hash, reason) VALUES (?, ?)")); + if (statement.prepare() != SQLITE_OK) + return; + + for (auto& pair : storeDecisions) { + statement.bindText(1, pair.first.string()); + statement.bindInt(2, static_cast<int>(pair.second)); + executeSQLStatement(statement); + statement.reset(); + } +} + +} +} + +#endif // ENABLE(NETWORK_CACHE) diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheStatistics.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheStatistics.h new file mode 100644 index 000000000..9f048e722 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheStatistics.h @@ -0,0 +1,96 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheStatistics_h +#define NetworkCacheStatistics_h + +#if ENABLE(NETWORK_CACHE) + +#include "NetworkCache.h" +#include "NetworkCacheKey.h" +#include <WebCore/SQLiteDatabase.h> +#include <WebCore/Timer.h> +#include <wtf/WorkQueue.h> + +namespace WebCore { +class ResourceRequest; +} + +namespace WebKit { +namespace NetworkCache { + +class Statistics { +public: + static std::unique_ptr<Statistics> open(const String& cachePath); + + void clear(); + + void recordRetrievalRequest(uint64_t webPageID); + void recordNotCachingResponse(const Key&, StoreDecision); + void recordNotUsingCacheForRequest(uint64_t webPageID, const Key&, const WebCore::ResourceRequest&, RetrieveDecision); + void recordRetrievalFailure(uint64_t webPageID, const Key&, const WebCore::ResourceRequest&); + void recordRetrievedCachedEntry(uint64_t webPageID, const Key&, const WebCore::ResourceRequest&, UseDecision); + void recordRevalidationSuccess(uint64_t webPageID, const Key&, const WebCore::ResourceRequest&); + +private: + explicit Statistics(const String& databasePath); + + WorkQueue& serialBackgroundIOQueue() { return m_serialBackgroundIOQueue.get(); } + + void initialize(const String& databasePath); + void bootstrapFromNetworkCache(const String& networkCachePath); + void shrinkIfNeeded(); + + void addHashesToDatabase(const Vector<StringCapture>& hashes); + void addStoreDecisionsToDatabase(const Vector<std::pair<StringCapture, NetworkCache::StoreDecision>>&); + void writeTimerFired(); + + typedef std::function<void (bool wasEverRequested, const Optional<StoreDecision>&)> RequestedCompletionHandler; + enum class NeedUncachedReason { No, Yes }; + void queryWasEverRequested(const String&, NeedUncachedReason, const RequestedCompletionHandler&); + void markAsRequested(const String& hash); + + struct EverRequestedQuery { + String hash; + bool needUncachedReason; + RequestedCompletionHandler completionHandler; + }; + + std::atomic<size_t> m_approximateEntryCount { 0 }; + + mutable Ref<WorkQueue> m_serialBackgroundIOQueue; + mutable HashSet<std::unique_ptr<const EverRequestedQuery>> m_activeQueries; + WebCore::SQLiteDatabase m_database; + HashSet<String> m_hashesToAdd; + HashMap<String, NetworkCache::StoreDecision> m_storeDecisionsToAdd; + WebCore::Timer m_writeTimer; +}; + +} +} + +#endif // ENABLE(NETWORK_CACHE) + +#endif // NetworkCacheStatistics_h diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.cpp new file mode 100644 index 000000000..1f650bfd1 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.cpp @@ -0,0 +1,1051 @@ +/* + * Copyright (C) 2014-2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "NetworkCacheStorage.h" + +#if ENABLE(NETWORK_CACHE) + +#include "Logging.h" +#include "NetworkCacheCoders.h" +#include "NetworkCacheFileSystem.h" +#include "NetworkCacheIOChannel.h" +#include <mutex> +#include <wtf/Condition.h> +#include <wtf/Lock.h> +#include <wtf/RandomNumber.h> +#include <wtf/RunLoop.h> +#include <wtf/text/CString.h> +#include <wtf/text/StringBuilder.h> + +namespace WebKit { +namespace NetworkCache { + +static const char versionDirectoryPrefix[] = "Version "; +static const char recordsDirectoryName[] = "Records"; +static const char blobsDirectoryName[] = "Blobs"; +static const char blobSuffix[] = "-blob"; + +static double computeRecordWorth(FileTimes); + +struct Storage::ReadOperation { + WTF_MAKE_FAST_ALLOCATED; +public: + ReadOperation(const Key& key, const RetrieveCompletionHandler& completionHandler) + : key(key) + , completionHandler(completionHandler) + { } + + void cancel(); + bool finish(); + + const Key key; + const RetrieveCompletionHandler completionHandler; + + std::unique_ptr<Record> resultRecord; + SHA1::Digest expectedBodyHash; + BlobStorage::Blob resultBodyBlob; + std::atomic<unsigned> activeCount { 0 }; + bool isCanceled { false }; +}; + +void Storage::ReadOperation::cancel() +{ + ASSERT(RunLoop::isMain()); + + if (isCanceled) + return; + isCanceled = true; + completionHandler(nullptr); +} + +bool Storage::ReadOperation::finish() +{ + ASSERT(RunLoop::isMain()); + + if (isCanceled) + return false; + if (resultRecord && resultRecord->body.isNull()) { + if (resultBodyBlob.hash == expectedBodyHash) + resultRecord->body = resultBodyBlob.data; + else + resultRecord = nullptr; + } + return completionHandler(WTFMove(resultRecord)); +} + +struct Storage::WriteOperation { + WTF_MAKE_FAST_ALLOCATED; +public: + WriteOperation(const Record& record, const MappedBodyHandler& mappedBodyHandler) + : record(record) + , mappedBodyHandler(mappedBodyHandler) + { } + + const Record record; + const MappedBodyHandler mappedBodyHandler; + + std::atomic<unsigned> activeCount { 0 }; +}; + +struct Storage::TraverseOperation { + WTF_MAKE_FAST_ALLOCATED; +public: + TraverseOperation(const String& type, TraverseFlags flags, const TraverseHandler& handler) + : type(type) + , flags(flags) + , handler(handler) + { } + + const String type; + const TraverseFlags flags; + const TraverseHandler handler; + + Lock activeMutex; + Condition activeCondition; + unsigned activeCount { 0 }; +}; + +std::unique_ptr<Storage> Storage::open(const String& cachePath) +{ + ASSERT(RunLoop::isMain()); + + if (!WebCore::makeAllDirectories(cachePath)) + return nullptr; + return std::unique_ptr<Storage>(new Storage(cachePath)); +} + +static String makeVersionedDirectoryPath(const String& baseDirectoryPath) +{ + String versionSubdirectory = versionDirectoryPrefix + String::number(Storage::version); + return WebCore::pathByAppendingComponent(baseDirectoryPath, versionSubdirectory); +} + +static String makeRecordsDirectoryPath(const String& baseDirectoryPath) +{ + return WebCore::pathByAppendingComponent(makeVersionedDirectoryPath(baseDirectoryPath), recordsDirectoryName); +} + +static String makeBlobDirectoryPath(const String& baseDirectoryPath) +{ + return WebCore::pathByAppendingComponent(makeVersionedDirectoryPath(baseDirectoryPath), blobsDirectoryName); +} + +void traverseRecordsFiles(const String& recordsPath, const String& expectedType, const std::function<void (const String& fileName, const String& hashString, const String& type, bool isBlob, const String& recordDirectoryPath)>& function) +{ + traverseDirectory(recordsPath, [&recordsPath, &function, &expectedType](const String& partitionName, DirectoryEntryType entryType) { + if (entryType != DirectoryEntryType::Directory) + return; + String partitionPath = WebCore::pathByAppendingComponent(recordsPath, partitionName); + traverseDirectory(partitionPath, [&function, &partitionPath, &expectedType](const String& actualType, DirectoryEntryType entryType) { + if (entryType != DirectoryEntryType::Directory) + return; + if (!expectedType.isEmpty() && expectedType != actualType) + return; + String recordDirectoryPath = WebCore::pathByAppendingComponent(partitionPath, actualType); + traverseDirectory(recordDirectoryPath, [&function, &recordDirectoryPath, &actualType](const String& fileName, DirectoryEntryType entryType) { + if (entryType != DirectoryEntryType::File || fileName.length() < Key::hashStringLength()) + return; + + String hashString = fileName.substring(0, Key::hashStringLength()); + auto isBlob = fileName.length() > Key::hashStringLength() && fileName.endsWith(blobSuffix); + function(fileName, hashString, actualType, isBlob, recordDirectoryPath); + }); + }); + }); +} + +static void deleteEmptyRecordsDirectories(const String& recordsPath) +{ + traverseDirectory(recordsPath, [&recordsPath](const String& partitionName, DirectoryEntryType type) { + if (type != DirectoryEntryType::Directory) + return; + + // Delete [type] sub-folders. + String partitionPath = WebCore::pathByAppendingComponent(recordsPath, partitionName); + traverseDirectory(partitionPath, [&partitionPath](const String& subdirName, DirectoryEntryType entryType) { + if (entryType != DirectoryEntryType::Directory) + return; + + // Let system figure out if it is really empty. + WebCore::deleteEmptyDirectory(WebCore::pathByAppendingComponent(partitionPath, subdirName)); + }); + + // Delete [Partition] folders. + // Let system figure out if it is really empty. + WebCore::deleteEmptyDirectory(WebCore::pathByAppendingComponent(recordsPath, partitionName)); + }); +} + +Storage::Storage(const String& baseDirectoryPath) + : m_basePath(baseDirectoryPath) + , m_recordsPath(makeRecordsDirectoryPath(baseDirectoryPath)) + , m_readOperationTimeoutTimer(*this, &Storage::cancelAllReadOperations) + , m_writeOperationDispatchTimer(*this, &Storage::dispatchPendingWriteOperations) + , m_ioQueue(WorkQueue::create("com.apple.WebKit.Cache.Storage", WorkQueue::Type::Concurrent)) + , m_backgroundIOQueue(WorkQueue::create("com.apple.WebKit.Cache.Storage.background", WorkQueue::Type::Concurrent, WorkQueue::QOS::Background)) + , m_serialBackgroundIOQueue(WorkQueue::create("com.apple.WebKit.Cache.Storage.serialBackground", WorkQueue::Type::Serial, WorkQueue::QOS::Background)) + , m_blobStorage(makeBlobDirectoryPath(baseDirectoryPath)) +{ + deleteOldVersions(); + synchronize(); +} + +Storage::~Storage() +{ +} + +String Storage::basePath() const +{ + return m_basePath.isolatedCopy(); +} + +String Storage::versionPath() const +{ + return makeVersionedDirectoryPath(basePath()); +} + +String Storage::recordsPath() const +{ + return m_recordsPath.isolatedCopy(); +} + +size_t Storage::approximateSize() const +{ + return m_approximateRecordsSize + m_blobStorage.approximateSize(); +} + +void Storage::synchronize() +{ + ASSERT(RunLoop::isMain()); + + if (m_synchronizationInProgress || m_shrinkInProgress) + return; + m_synchronizationInProgress = true; + + LOG(NetworkCacheStorage, "(NetworkProcess) synchronizing cache"); + + backgroundIOQueue().dispatch([this] { + auto recordFilter = std::make_unique<ContentsFilter>(); + auto blobFilter = std::make_unique<ContentsFilter>(); + size_t recordsSize = 0; + unsigned count = 0; + String anyType; + traverseRecordsFiles(recordsPath(), anyType, [&recordFilter, &blobFilter, &recordsSize, &count](const String& fileName, const String& hashString, const String& type, bool isBlob, const String& recordDirectoryPath) { + auto filePath = WebCore::pathByAppendingComponent(recordDirectoryPath, fileName); + + Key::HashType hash; + if (!Key::stringToHash(hashString, hash)) { + WebCore::deleteFile(filePath); + return; + } + long long fileSize = 0; + WebCore::getFileSize(filePath, fileSize); + if (!fileSize) { + WebCore::deleteFile(filePath); + return; + } + + if (isBlob) { + blobFilter->add(hash); + return; + } + + recordFilter->add(hash); + recordsSize += fileSize; + ++count; + }); + + auto* recordFilterPtr = recordFilter.release(); + auto* blobFilterPtr = blobFilter.release(); + RunLoop::main().dispatch([this, recordFilterPtr, blobFilterPtr, recordsSize] { + auto recordFilter = std::unique_ptr<ContentsFilter>(recordFilterPtr); + auto blobFilter = std::unique_ptr<ContentsFilter>(blobFilterPtr); + + for (auto& recordFilterKey : m_recordFilterHashesAddedDuringSynchronization) + recordFilter->add(recordFilterKey); + m_recordFilterHashesAddedDuringSynchronization.clear(); + + for (auto& hash : m_blobFilterHashesAddedDuringSynchronization) + blobFilter->add(hash); + m_blobFilterHashesAddedDuringSynchronization.clear(); + + m_recordFilter = WTFMove(recordFilter); + m_blobFilter = WTFMove(blobFilter); + m_approximateRecordsSize = recordsSize; + m_synchronizationInProgress = false; + }); + + m_blobStorage.synchronize(); + + deleteEmptyRecordsDirectories(recordsPath()); + + LOG(NetworkCacheStorage, "(NetworkProcess) cache synchronization completed size=%zu count=%u", recordsSize, count); + }); +} + +void Storage::addToRecordFilter(const Key& key) +{ + ASSERT(RunLoop::isMain()); + + if (m_recordFilter) + m_recordFilter->add(key.hash()); + + // If we get new entries during filter synchronization take care to add them to the new filter as well. + if (m_synchronizationInProgress) + m_recordFilterHashesAddedDuringSynchronization.append(key.hash()); +} + +bool Storage::mayContain(const Key& key) const +{ + ASSERT(RunLoop::isMain()); + return !m_recordFilter || m_recordFilter->mayContain(key.hash()); +} + +bool Storage::mayContainBlob(const Key& key) const +{ + ASSERT(RunLoop::isMain()); + return !m_blobFilter || m_blobFilter->mayContain(key.hash()); +} + +String Storage::recordDirectoryPathForKey(const Key& key) const +{ + ASSERT(!key.partition().isEmpty()); + ASSERT(!key.type().isEmpty()); + return WebCore::pathByAppendingComponent(WebCore::pathByAppendingComponent(recordsPath(), key.partition()), key.type()); +} + +String Storage::recordPathForKey(const Key& key) const +{ + return WebCore::pathByAppendingComponent(recordDirectoryPathForKey(key), key.hashAsString()); +} + +static String blobPathForRecordPath(const String& recordPath) +{ + return recordPath + blobSuffix; +} + +String Storage::blobPathForKey(const Key& key) const +{ + return blobPathForRecordPath(recordPathForKey(key)); +} + +struct RecordMetaData { + RecordMetaData() { } + explicit RecordMetaData(const Key& key) + : cacheStorageVersion(Storage::version) + , key(key) + { } + + unsigned cacheStorageVersion; + Key key; + // FIXME: Add encoder/decoder for time_point. + std::chrono::milliseconds epochRelativeTimeStamp; + SHA1::Digest headerHash; + uint64_t headerSize; + SHA1::Digest bodyHash; + uint64_t bodySize; + bool isBodyInline; + + // Not encoded as a field. Header starts immediately after meta data. + uint64_t headerOffset; +}; + +static bool decodeRecordMetaData(RecordMetaData& metaData, const Data& fileData) +{ + bool success = false; + fileData.apply([&metaData, &success](const uint8_t* data, size_t size) { + Decoder decoder(data, size); + if (!decoder.decode(metaData.cacheStorageVersion)) + return false; + if (!decoder.decode(metaData.key)) + return false; + if (!decoder.decode(metaData.epochRelativeTimeStamp)) + return false; + if (!decoder.decode(metaData.headerHash)) + return false; + if (!decoder.decode(metaData.headerSize)) + return false; + if (!decoder.decode(metaData.bodyHash)) + return false; + if (!decoder.decode(metaData.bodySize)) + return false; + if (!decoder.decode(metaData.isBodyInline)) + return false; + if (!decoder.verifyChecksum()) + return false; + metaData.headerOffset = decoder.currentOffset(); + success = true; + return false; + }); + return success; +} + +static bool decodeRecordHeader(const Data& fileData, RecordMetaData& metaData, Data& headerData) +{ + if (!decodeRecordMetaData(metaData, fileData)) { + LOG(NetworkCacheStorage, "(NetworkProcess) meta data decode failure"); + return false; + } + + if (metaData.cacheStorageVersion != Storage::version) { + LOG(NetworkCacheStorage, "(NetworkProcess) version mismatch"); + return false; + } + + headerData = fileData.subrange(metaData.headerOffset, metaData.headerSize); + if (metaData.headerHash != computeSHA1(headerData)) { + LOG(NetworkCacheStorage, "(NetworkProcess) header checksum mismatch"); + return false; + } + return true; +} + +void Storage::readRecord(ReadOperation& readOperation, const Data& recordData) +{ + ASSERT(!RunLoop::isMain()); + + RecordMetaData metaData; + Data headerData; + if (!decodeRecordHeader(recordData, metaData, headerData)) + return; + + if (metaData.key != readOperation.key) + return; + + // Sanity check against time stamps in future. + auto timeStamp = std::chrono::system_clock::time_point(metaData.epochRelativeTimeStamp); + if (timeStamp > std::chrono::system_clock::now()) + return; + + Data bodyData; + if (metaData.isBodyInline) { + size_t bodyOffset = metaData.headerOffset + headerData.size(); + if (bodyOffset + metaData.bodySize != recordData.size()) + return; + bodyData = recordData.subrange(bodyOffset, metaData.bodySize); + if (metaData.bodyHash != computeSHA1(bodyData)) + return; + } + + readOperation.expectedBodyHash = metaData.bodyHash; + readOperation.resultRecord = std::make_unique<Storage::Record>(Storage::Record { + metaData.key, + timeStamp, + headerData, + bodyData + }); +} + +static Data encodeRecordMetaData(const RecordMetaData& metaData) +{ + Encoder encoder; + + encoder << metaData.cacheStorageVersion; + encoder << metaData.key; + encoder << metaData.epochRelativeTimeStamp; + encoder << metaData.headerHash; + encoder << metaData.headerSize; + encoder << metaData.bodyHash; + encoder << metaData.bodySize; + encoder << metaData.isBodyInline; + + encoder.encodeChecksum(); + + return Data(encoder.buffer(), encoder.bufferSize()); +} + +Optional<BlobStorage::Blob> Storage::storeBodyAsBlob(WriteOperation& writeOperation) +{ + auto blobPath = blobPathForKey(writeOperation.record.key); + + // Store the body. + auto blob = m_blobStorage.add(blobPath, writeOperation.record.body); + if (blob.data.isNull()) + return { }; + + ++writeOperation.activeCount; + + RunLoop::main().dispatch([this, blob, &writeOperation] { + if (m_blobFilter) + m_blobFilter->add(writeOperation.record.key.hash()); + if (m_synchronizationInProgress) + m_blobFilterHashesAddedDuringSynchronization.append(writeOperation.record.key.hash()); + + if (writeOperation.mappedBodyHandler) + writeOperation.mappedBodyHandler(blob.data); + + finishWriteOperation(writeOperation); + }); + return blob; +} + +Data Storage::encodeRecord(const Record& record, Optional<BlobStorage::Blob> blob) +{ + ASSERT(!blob || bytesEqual(blob.value().data, record.body)); + + RecordMetaData metaData(record.key); + metaData.epochRelativeTimeStamp = std::chrono::duration_cast<std::chrono::milliseconds>(record.timeStamp.time_since_epoch()); + metaData.headerHash = computeSHA1(record.header); + metaData.headerSize = record.header.size(); + metaData.bodyHash = blob ? blob.value().hash : computeSHA1(record.body); + metaData.bodySize = record.body.size(); + metaData.isBodyInline = !blob; + + auto encodedMetaData = encodeRecordMetaData(metaData); + auto headerData = concatenate(encodedMetaData, record.header); + + if (metaData.isBodyInline) + return concatenate(headerData, record.body); + + return { headerData }; +} + +void Storage::removeFromPendingWriteOperations(const Key& key) +{ + while (true) { + auto found = m_pendingWriteOperations.findIf([&key](const std::unique_ptr<WriteOperation>& operation) { + return operation->record.key == key; + }); + + if (found == m_pendingWriteOperations.end()) + break; + + m_pendingWriteOperations.remove(found); + } +} + +void Storage::remove(const Key& key) +{ + ASSERT(RunLoop::isMain()); + + if (!mayContain(key)) + return; + + // We can't remove the key from the Bloom filter (but some false positives are expected anyway). + // For simplicity we also don't reduce m_approximateSize on removals. + // The next synchronization will update everything. + + removeFromPendingWriteOperations(key); + + serialBackgroundIOQueue().dispatch([this, key] { + WebCore::deleteFile(recordPathForKey(key)); + m_blobStorage.remove(blobPathForKey(key)); + }); +} + +void Storage::updateFileModificationTime(const String& path) +{ + StringCapture filePathCapture(path); + serialBackgroundIOQueue().dispatch([filePathCapture] { + updateFileModificationTimeIfNeeded(filePathCapture.string()); + }); +} + +void Storage::dispatchReadOperation(std::unique_ptr<ReadOperation> readOperationPtr) +{ + ASSERT(RunLoop::isMain()); + + auto& readOperation = *readOperationPtr; + m_activeReadOperations.add(WTFMove(readOperationPtr)); + + // I/O pressure may make disk operations slow. If they start taking very long time we rather go to network. + const auto readTimeout = 1500_ms; + m_readOperationTimeoutTimer.startOneShot(readTimeout); + + bool shouldGetBodyBlob = mayContainBlob(readOperation.key); + + ioQueue().dispatch([this, &readOperation, shouldGetBodyBlob] { + auto recordPath = recordPathForKey(readOperation.key); + + ++readOperation.activeCount; + if (shouldGetBodyBlob) + ++readOperation.activeCount; + + auto channel = IOChannel::open(recordPath, IOChannel::Type::Read); + channel->read(0, std::numeric_limits<size_t>::max(), &ioQueue(), [this, &readOperation](const Data& fileData, int error) { + if (!error) + readRecord(readOperation, fileData); + finishReadOperation(readOperation); + }); + + if (shouldGetBodyBlob) { + // Read the blob in parallel with the record read. + auto blobPath = blobPathForKey(readOperation.key); + readOperation.resultBodyBlob = m_blobStorage.get(blobPath); + finishReadOperation(readOperation); + } + }); +} + +void Storage::finishReadOperation(ReadOperation& readOperation) +{ + ASSERT(readOperation.activeCount); + // Record and blob reads must finish. + if (--readOperation.activeCount) + return; + + RunLoop::main().dispatch([this, &readOperation] { + bool success = readOperation.finish(); + if (success) + updateFileModificationTime(recordPathForKey(readOperation.key)); + else if (!readOperation.isCanceled) + remove(readOperation.key); + + ASSERT(m_activeReadOperations.contains(&readOperation)); + m_activeReadOperations.remove(&readOperation); + + if (m_activeReadOperations.isEmpty()) + m_readOperationTimeoutTimer.stop(); + + dispatchPendingReadOperations(); + + LOG(NetworkCacheStorage, "(NetworkProcess) read complete success=%d", success); + }); +} + +void Storage::cancelAllReadOperations() +{ + ASSERT(RunLoop::isMain()); + + for (auto& readOperation : m_activeReadOperations) + readOperation->cancel(); + + size_t pendingCount = 0; + for (int priority = maximumRetrievePriority; priority >= 0; --priority) { + auto& pendingRetrieveQueue = m_pendingReadOperationsByPriority[priority]; + pendingCount += pendingRetrieveQueue.size(); + for (auto it = pendingRetrieveQueue.rbegin(), end = pendingRetrieveQueue.rend(); it != end; ++it) + (*it)->cancel(); + pendingRetrieveQueue.clear(); + } + + LOG(NetworkCacheStorage, "(NetworkProcess) retrieve timeout, canceled %u active and %zu pending", m_activeReadOperations.size(), pendingCount); +} + +void Storage::dispatchPendingReadOperations() +{ + ASSERT(RunLoop::isMain()); + + const int maximumActiveReadOperationCount = 5; + + for (int priority = maximumRetrievePriority; priority >= 0; --priority) { + if (m_activeReadOperations.size() > maximumActiveReadOperationCount) { + LOG(NetworkCacheStorage, "(NetworkProcess) limiting parallel retrieves"); + return; + } + auto& pendingRetrieveQueue = m_pendingReadOperationsByPriority[priority]; + if (pendingRetrieveQueue.isEmpty()) + continue; + dispatchReadOperation(pendingRetrieveQueue.takeLast()); + } +} + +template <class T> bool retrieveFromMemory(const T& operations, const Key& key, Storage::RetrieveCompletionHandler& completionHandler) +{ + for (auto& operation : operations) { + if (operation->record.key == key) { + LOG(NetworkCacheStorage, "(NetworkProcess) found write operation in progress"); + auto record = operation->record; + RunLoop::main().dispatch([record, completionHandler] { + completionHandler(std::make_unique<Storage::Record>(record)); + }); + return true; + } + } + return false; +} + +void Storage::dispatchPendingWriteOperations() +{ + ASSERT(RunLoop::isMain()); + + const int maximumActiveWriteOperationCount { 1 }; + + while (!m_pendingWriteOperations.isEmpty()) { + if (m_activeWriteOperations.size() >= maximumActiveWriteOperationCount) { + LOG(NetworkCacheStorage, "(NetworkProcess) limiting parallel writes"); + return; + } + dispatchWriteOperation(m_pendingWriteOperations.takeLast()); + } +} + +static bool shouldStoreBodyAsBlob(const Data& bodyData) +{ + const size_t maximumInlineBodySize { 16 * 1024 }; + return bodyData.size() > maximumInlineBodySize; +} + +void Storage::dispatchWriteOperation(std::unique_ptr<WriteOperation> writeOperationPtr) +{ + ASSERT(RunLoop::isMain()); + + auto& writeOperation = *writeOperationPtr; + m_activeWriteOperations.add(WTFMove(writeOperationPtr)); + + // This was added already when starting the store but filter might have been wiped. + addToRecordFilter(writeOperation.record.key); + + backgroundIOQueue().dispatch([this, &writeOperation] { + auto recordDirectorPath = recordDirectoryPathForKey(writeOperation.record.key); + auto recordPath = recordPathForKey(writeOperation.record.key); + + WebCore::makeAllDirectories(recordDirectorPath); + + ++writeOperation.activeCount; + + bool shouldStoreAsBlob = shouldStoreBodyAsBlob(writeOperation.record.body); + auto blob = shouldStoreAsBlob ? storeBodyAsBlob(writeOperation) : Nullopt; + + auto recordData = encodeRecord(writeOperation.record, blob); + + auto channel = IOChannel::open(recordPath, IOChannel::Type::Create); + size_t recordSize = recordData.size(); + channel->write(0, recordData, nullptr, [this, &writeOperation, recordSize](int error) { + // On error the entry still stays in the contents filter until next synchronization. + m_approximateRecordsSize += recordSize; + finishWriteOperation(writeOperation); + + LOG(NetworkCacheStorage, "(NetworkProcess) write complete error=%d", error); + }); + }); +} + +void Storage::finishWriteOperation(WriteOperation& writeOperation) +{ + ASSERT(RunLoop::isMain()); + ASSERT(writeOperation.activeCount); + ASSERT(m_activeWriteOperations.contains(&writeOperation)); + + if (--writeOperation.activeCount) + return; + + m_activeWriteOperations.remove(&writeOperation); + dispatchPendingWriteOperations(); + + shrinkIfNeeded(); +} + +void Storage::retrieve(const Key& key, unsigned priority, RetrieveCompletionHandler&& completionHandler) +{ + ASSERT(RunLoop::isMain()); + ASSERT(priority <= maximumRetrievePriority); + ASSERT(!key.isNull()); + + if (!m_capacity) { + completionHandler(nullptr); + return; + } + + if (!mayContain(key)) { + completionHandler(nullptr); + return; + } + + if (retrieveFromMemory(m_pendingWriteOperations, key, completionHandler)) + return; + if (retrieveFromMemory(m_activeWriteOperations, key, completionHandler)) + return; + + auto readOperation = std::make_unique<ReadOperation>(key, WTFMove(completionHandler)); + m_pendingReadOperationsByPriority[priority].prepend(WTFMove(readOperation)); + dispatchPendingReadOperations(); +} + +void Storage::store(const Record& record, MappedBodyHandler&& mappedBodyHandler) +{ + ASSERT(RunLoop::isMain()); + ASSERT(!record.key.isNull()); + + if (!m_capacity) + return; + + auto writeOperation = std::make_unique<WriteOperation>(record, WTFMove(mappedBodyHandler)); + m_pendingWriteOperations.prepend(WTFMove(writeOperation)); + + // Add key to the filter already here as we do lookups from the pending operations too. + addToRecordFilter(record.key); + + bool isInitialWrite = m_pendingWriteOperations.size() == 1; + if (!isInitialWrite) + return; + + // Delay the start of writes a bit to avoid affecting early page load. + // Completing writes will dispatch more writes without delay. + static const auto initialWriteDelay = 1_s; + m_writeOperationDispatchTimer.startOneShot(initialWriteDelay); +} + +void Storage::traverse(const String& type, TraverseFlags flags, TraverseHandler&& traverseHandler) +{ + ASSERT(RunLoop::isMain()); + ASSERT(traverseHandler); + // Avoid non-thread safe std::function copies. + + auto traverseOperationPtr = std::make_unique<TraverseOperation>(type, flags, WTFMove(traverseHandler)); + auto& traverseOperation = *traverseOperationPtr; + m_activeTraverseOperations.add(WTFMove(traverseOperationPtr)); + + ioQueue().dispatch([this, &traverseOperation] { + traverseRecordsFiles(recordsPath(), traverseOperation.type, [this, &traverseOperation](const String& fileName, const String& hashString, const String& type, bool isBlob, const String& recordDirectoryPath) { + ASSERT(type == traverseOperation.type); + if (isBlob) + return; + + auto recordPath = WebCore::pathByAppendingComponent(recordDirectoryPath, fileName); + + double worth = -1; + if (traverseOperation.flags & TraverseFlag::ComputeWorth) + worth = computeRecordWorth(fileTimes(recordPath)); + unsigned bodyShareCount = 0; + if (traverseOperation.flags & TraverseFlag::ShareCount) + bodyShareCount = m_blobStorage.shareCount(blobPathForRecordPath(recordPath)); + + std::unique_lock<Lock> lock(traverseOperation.activeMutex); + ++traverseOperation.activeCount; + + auto channel = IOChannel::open(recordPath, IOChannel::Type::Read); + channel->read(0, std::numeric_limits<size_t>::max(), nullptr, [this, &traverseOperation, worth, bodyShareCount](Data& fileData, int) { + RecordMetaData metaData; + Data headerData; + if (decodeRecordHeader(fileData, metaData, headerData)) { + Record record { + metaData.key, + std::chrono::system_clock::time_point(metaData.epochRelativeTimeStamp), + headerData, + { } + }; + RecordInfo info { + static_cast<size_t>(metaData.bodySize), + worth, + bodyShareCount, + String::fromUTF8(SHA1::hexDigest(metaData.bodyHash)) + }; + traverseOperation.handler(&record, info); + } + + std::lock_guard<Lock> lock(traverseOperation.activeMutex); + --traverseOperation.activeCount; + traverseOperation.activeCondition.notifyOne(); + }); + + const unsigned maximumParallelReadCount = 5; + traverseOperation.activeCondition.wait(lock, [&traverseOperation] { + return traverseOperation.activeCount <= maximumParallelReadCount; + }); + }); + // Wait for all reads to finish. + std::unique_lock<Lock> lock(traverseOperation.activeMutex); + traverseOperation.activeCondition.wait(lock, [&traverseOperation] { + return !traverseOperation.activeCount; + }); + RunLoop::main().dispatch([this, &traverseOperation] { + traverseOperation.handler(nullptr, { }); + m_activeTraverseOperations.remove(&traverseOperation); + }); + }); +} + +void Storage::setCapacity(size_t capacity) +{ + ASSERT(RunLoop::isMain()); + +#if !ASSERT_DISABLED + const size_t assumedAverageRecordSize = 50 << 10; + size_t maximumRecordCount = capacity / assumedAverageRecordSize; + // ~10 bits per element are required for <1% false positive rate. + size_t effectiveBloomFilterCapacity = ContentsFilter::tableSize / 10; + // If this gets hit it might be time to increase the filter size. + ASSERT(maximumRecordCount < effectiveBloomFilterCapacity); +#endif + + m_capacity = capacity; + + shrinkIfNeeded(); +} + +void Storage::clear(const String& type, std::chrono::system_clock::time_point modifiedSinceTime, std::function<void ()>&& completionHandler) +{ + ASSERT(RunLoop::isMain()); + LOG(NetworkCacheStorage, "(NetworkProcess) clearing cache"); + + if (m_recordFilter) + m_recordFilter->clear(); + if (m_blobFilter) + m_blobFilter->clear(); + m_approximateRecordsSize = 0; + + // Avoid non-thread safe std::function copies. + auto* completionHandlerPtr = completionHandler ? new std::function<void ()>(WTFMove(completionHandler)) : nullptr; + StringCapture typeCapture(type); + ioQueue().dispatch([this, modifiedSinceTime, completionHandlerPtr, typeCapture] { + auto recordsPath = this->recordsPath(); + traverseRecordsFiles(recordsPath, typeCapture.string(), [modifiedSinceTime](const String& fileName, const String& hashString, const String& type, bool isBlob, const String& recordDirectoryPath) { + auto filePath = WebCore::pathByAppendingComponent(recordDirectoryPath, fileName); + if (modifiedSinceTime > std::chrono::system_clock::time_point::min()) { + auto times = fileTimes(filePath); + if (times.modification < modifiedSinceTime) + return; + } + WebCore::deleteFile(filePath); + }); + + deleteEmptyRecordsDirectories(recordsPath); + + // This cleans unreferenced blobs. + m_blobStorage.synchronize(); + + if (completionHandlerPtr) { + RunLoop::main().dispatch([completionHandlerPtr] { + (*completionHandlerPtr)(); + delete completionHandlerPtr; + }); + } + }); +} + +static double computeRecordWorth(FileTimes times) +{ + using namespace std::chrono; + auto age = system_clock::now() - times.creation; + // File modification time is updated manually on cache read. We don't use access time since OS may update it automatically. + auto accessAge = times.modification - times.creation; + + // For sanity. + if (age <= 0_s || accessAge < 0_s || accessAge > age) + return 0; + + // We like old entries that have been accessed recently. + return duration<double>(accessAge) / age; +} + +static double deletionProbability(FileTimes times, unsigned bodyShareCount) +{ + static const double maximumProbability { 0.33 }; + static const unsigned maximumEffectiveShareCount { 5 }; + + auto worth = computeRecordWorth(times); + + // Adjust a bit so the most valuable entries don't get deleted at all. + auto effectiveWorth = std::min(1.1 * worth, 1.); + + auto probability = (1 - effectiveWorth) * maximumProbability; + + // It is less useful to remove an entry that shares its body data. + if (bodyShareCount) + probability /= std::min(bodyShareCount, maximumEffectiveShareCount); + + return probability; +} + +void Storage::shrinkIfNeeded() +{ + ASSERT(RunLoop::isMain()); + + if (approximateSize() > m_capacity) + shrink(); +} + +void Storage::shrink() +{ + ASSERT(RunLoop::isMain()); + + if (m_shrinkInProgress || m_synchronizationInProgress) + return; + m_shrinkInProgress = true; + + LOG(NetworkCacheStorage, "(NetworkProcess) shrinking cache approximateSize=%zu capacity=%zu", approximateSize(), m_capacity); + + backgroundIOQueue().dispatch([this] { + auto recordsPath = this->recordsPath(); + String anyType; + traverseRecordsFiles(recordsPath, anyType, [this](const String& fileName, const String& hashString, const String& type, bool isBlob, const String& recordDirectoryPath) { + if (isBlob) + return; + + auto recordPath = WebCore::pathByAppendingComponent(recordDirectoryPath, fileName); + auto blobPath = blobPathForRecordPath(recordPath); + + auto times = fileTimes(recordPath); + unsigned bodyShareCount = m_blobStorage.shareCount(blobPath); + auto probability = deletionProbability(times, bodyShareCount); + + bool shouldDelete = randomNumber() < probability; + + LOG(NetworkCacheStorage, "Deletion probability=%f bodyLinkCount=%d shouldDelete=%d", probability, bodyShareCount, shouldDelete); + + if (shouldDelete) { + WebCore::deleteFile(recordPath); + m_blobStorage.remove(blobPath); + } + }); + + RunLoop::main().dispatch([this] { + m_shrinkInProgress = false; + // We could synchronize during the shrink traversal. However this is fast and it is better to have just one code path. + synchronize(); + }); + + LOG(NetworkCacheStorage, "(NetworkProcess) cache shrink completed"); + }); +} + +void Storage::deleteOldVersions() +{ + backgroundIOQueue().dispatch([this] { + auto cachePath = basePath(); + traverseDirectory(cachePath, [&cachePath](const String& subdirName, DirectoryEntryType type) { + if (type != DirectoryEntryType::Directory) + return; + if (!subdirName.startsWith(versionDirectoryPrefix)) + return; + auto versionString = subdirName.substring(strlen(versionDirectoryPrefix)); + bool success; + unsigned directoryVersion = versionString.toUIntStrict(&success); + if (!success) + return; + if (directoryVersion >= version) + return; + +#if PLATFORM(MAC) + // Allow the last stable version of the cache to co-exist with the latest development one on Mac. + const unsigned lastStableVersion = 4; + if (directoryVersion == lastStableVersion) + return; +#endif + + auto oldVersionPath = WebCore::pathByAppendingComponent(cachePath, subdirName); + LOG(NetworkCacheStorage, "(NetworkProcess) deleting old cache version, path %s", oldVersionPath.utf8().data()); + + deleteDirectoryRecursively(oldVersionPath); + }); + }); +} + +} +} + +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.h new file mode 100644 index 000000000..4762a9cbf --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheStorage.h @@ -0,0 +1,178 @@ +/* + * Copyright (C) 2014-2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheStorage_h +#define NetworkCacheStorage_h + +#if ENABLE(NETWORK_CACHE) + +#include "NetworkCacheBlobStorage.h" +#include "NetworkCacheData.h" +#include "NetworkCacheKey.h" +#include <WebCore/Timer.h> +#include <wtf/BloomFilter.h> +#include <wtf/Deque.h> +#include <wtf/HashSet.h> +#include <wtf/Optional.h> +#include <wtf/WorkQueue.h> +#include <wtf/text/WTFString.h> + +namespace WebKit { +namespace NetworkCache { + +class IOChannel; + +class Storage { + WTF_MAKE_NONCOPYABLE(Storage); +public: + static std::unique_ptr<Storage> open(const String& cachePath); + + struct Record { + WTF_MAKE_FAST_ALLOCATED; + public: + Key key; + std::chrono::system_clock::time_point timeStamp; + Data header; + Data body; + }; + // This may call completion handler synchronously on failure. + typedef std::function<bool (std::unique_ptr<Record>)> RetrieveCompletionHandler; + void retrieve(const Key&, unsigned priority, RetrieveCompletionHandler&&); + + typedef std::function<void (const Data& mappedBody)> MappedBodyHandler; + void store(const Record&, MappedBodyHandler&&); + + void remove(const Key&); + void clear(const String& type, std::chrono::system_clock::time_point modifiedSinceTime, std::function<void ()>&& completionHandler); + + struct RecordInfo { + size_t bodySize; + double worth; // 0-1 where 1 is the most valuable. + unsigned bodyShareCount; + String bodyHash; + }; + enum TraverseFlag { + ComputeWorth = 1 << 0, + ShareCount = 1 << 1, + }; + typedef unsigned TraverseFlags; + typedef std::function<void (const Record*, const RecordInfo&)> TraverseHandler; + // Null record signals end. + void traverse(const String& type, TraverseFlags, TraverseHandler&&); + + void setCapacity(size_t); + size_t capacity() const { return m_capacity; } + size_t approximateSize() const; + + static const unsigned version = 5; + + String basePath() const; + String versionPath() const; + String recordsPath() const; + + ~Storage(); + +private: + Storage(const String& directoryPath); + + String recordDirectoryPathForKey(const Key&) const; + String recordPathForKey(const Key&) const; + String blobPathForKey(const Key&) const; + + void synchronize(); + void deleteOldVersions(); + void shrinkIfNeeded(); + void shrink(); + + struct ReadOperation; + void dispatchReadOperation(std::unique_ptr<ReadOperation>); + void dispatchPendingReadOperations(); + void finishReadOperation(ReadOperation&); + void cancelAllReadOperations(); + + struct WriteOperation; + void dispatchWriteOperation(std::unique_ptr<WriteOperation>); + void dispatchPendingWriteOperations(); + void finishWriteOperation(WriteOperation&); + + Optional<BlobStorage::Blob> storeBodyAsBlob(WriteOperation&); + Data encodeRecord(const Record&, Optional<BlobStorage::Blob>); + void readRecord(ReadOperation&, const Data&); + + void updateFileModificationTime(const String& path); + void removeFromPendingWriteOperations(const Key&); + + WorkQueue& ioQueue() { return m_ioQueue.get(); } + WorkQueue& backgroundIOQueue() { return m_backgroundIOQueue.get(); } + WorkQueue& serialBackgroundIOQueue() { return m_serialBackgroundIOQueue.get(); } + + bool mayContain(const Key&) const; + bool mayContainBlob(const Key&) const; + + void addToRecordFilter(const Key&); + + const String m_basePath; + const String m_recordsPath; + + size_t m_capacity { std::numeric_limits<size_t>::max() }; + size_t m_approximateRecordsSize { 0 }; + + // 2^18 bit filter can support up to 26000 entries with false positive rate < 1%. + using ContentsFilter = BloomFilter<18>; + std::unique_ptr<ContentsFilter> m_recordFilter; + std::unique_ptr<ContentsFilter> m_blobFilter; + + bool m_synchronizationInProgress { false }; + bool m_shrinkInProgress { false }; + + Vector<Key::HashType> m_recordFilterHashesAddedDuringSynchronization; + Vector<Key::HashType> m_blobFilterHashesAddedDuringSynchronization; + + static const int maximumRetrievePriority = 4; + Deque<std::unique_ptr<ReadOperation>> m_pendingReadOperationsByPriority[maximumRetrievePriority + 1]; + HashSet<std::unique_ptr<ReadOperation>> m_activeReadOperations; + WebCore::Timer m_readOperationTimeoutTimer; + + Deque<std::unique_ptr<WriteOperation>> m_pendingWriteOperations; + HashSet<std::unique_ptr<WriteOperation>> m_activeWriteOperations; + WebCore::Timer m_writeOperationDispatchTimer; + + struct TraverseOperation; + HashSet<std::unique_ptr<TraverseOperation>> m_activeTraverseOperations; + + Ref<WorkQueue> m_ioQueue; + Ref<WorkQueue> m_backgroundIOQueue; + Ref<WorkQueue> m_serialBackgroundIOQueue; + + BlobStorage m_blobStorage; +}; + +// FIXME: Remove, used by NetworkCacheStatistics only. +void traverseRecordsFiles(const String& recordsPath, const String& type, const std::function<void (const String& fileName, const String& hashString, const String& type, bool isBodyBlob, const String& recordDirectoryPath)>&); + +} +} +#endif +#endif diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheSubresourcesEntry.cpp b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSubresourcesEntry.cpp new file mode 100644 index 000000000..e762c3b33 --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSubresourcesEntry.cpp @@ -0,0 +1,95 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) +#include "NetworkCacheSubresourcesEntry.h" + +#include "Logging.h" +#include "NetworkCacheCoders.h" +#include "NetworkCacheDecoder.h" +#include "NetworkCacheEncoder.h" + +namespace WebKit { +namespace NetworkCache { + +Storage::Record SubresourcesEntry::encodeAsStorageRecord() const +{ + Encoder encoder; + encoder << m_subresources; + + encoder.encodeChecksum(); + + return { m_key, m_timeStamp, { encoder.buffer(), encoder.bufferSize() } , { } }; +} + +std::unique_ptr<SubresourcesEntry> SubresourcesEntry::decodeStorageRecord(const Storage::Record& storageEntry) +{ + auto entry = std::make_unique<SubresourcesEntry>(storageEntry); + + Decoder decoder(storageEntry.header.data(), storageEntry.header.size()); + if (!decoder.decode(entry->m_subresources)) + return nullptr; + + if (!decoder.verifyChecksum()) { + LOG(NetworkCache, "(NetworkProcess) checksum verification failure\n"); + return nullptr; + } + + return entry; +} + +SubresourcesEntry::SubresourcesEntry(const Storage::Record& storageEntry) + : m_key(storageEntry.key) + , m_timeStamp(storageEntry.timeStamp) +{ + ASSERT(m_key.type() == "subresources"); +} + +SubresourcesEntry::SubresourcesEntry(Key&& key, const Vector<Key>& subresourceKeys) + : m_key(WTFMove(key)) + , m_timeStamp(std::chrono::system_clock::now()) +{ + ASSERT(m_key.type() == "subresources"); + for (auto& key : subresourceKeys) + m_subresources.add(key, SubresourceInfo()); +} + +void SubresourcesEntry::updateSubresourceKeys(const Vector<Key>& subresourceKeys) +{ + auto oldSubresources = WTFMove(m_subresources); + + // Mark keys that are common with last load as non-Transient. + for (auto& key : subresourceKeys) { + bool isTransient = !oldSubresources.contains(key); + m_subresources.add(key, SubresourceInfo(isTransient)); + } +} + +} // namespace WebKit +} // namespace NetworkCache + +#endif // ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) diff --git a/Source/WebKit2/NetworkProcess/cache/NetworkCacheSubresourcesEntry.h b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSubresourcesEntry.h new file mode 100644 index 000000000..d12282ffa --- /dev/null +++ b/Source/WebKit2/NetworkProcess/cache/NetworkCacheSubresourcesEntry.h @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2015 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef NetworkCacheSubresourcesEntry_h +#define NetworkCacheSubresourcesEntry_h + +#if ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) + +#include "NetworkCacheDecoder.h" +#include "NetworkCacheEncoder.h" +#include "NetworkCacheStorage.h" +#include <wtf/HashMap.h> + +namespace WebKit { +namespace NetworkCache { + +class SubresourcesEntry { + WTF_MAKE_NONCOPYABLE(SubresourcesEntry); WTF_MAKE_FAST_ALLOCATED; +public: + struct SubresourceInfo { + void encode(Encoder& encoder) const { encoder << isTransient; } + static bool decode(Decoder& decoder, SubresourceInfo& info) { return decoder.decode(info.isTransient); } + + SubresourceInfo() = default; + SubresourceInfo(bool isTransient) : isTransient(isTransient) { } + + bool isTransient { false }; + }; + SubresourcesEntry(Key&&, const Vector<Key>& subresourceKeys); + explicit SubresourcesEntry(const Storage::Record&); + + Storage::Record encodeAsStorageRecord() const; + static std::unique_ptr<SubresourcesEntry> decodeStorageRecord(const Storage::Record&); + + const Key& key() const { return m_key; } + std::chrono::system_clock::time_point timeStamp() const { return m_timeStamp; } + const HashMap<Key, SubresourceInfo>& subresources() const { return m_subresources; } + + void updateSubresourceKeys(const Vector<Key>&); + +private: + Key m_key; + std::chrono::system_clock::time_point m_timeStamp; + HashMap<Key, SubresourceInfo> m_subresources; +}; + +} // namespace WebKit +} // namespace NetworkCache + +#endif // ENABLE(NETWORK_CACHE_SPECULATIVE_REVALIDATION) +#endif // NetworkCacheSubresourcesEntry_h |