[core] introduce Blob for compressed and uncompressed dataupstream/blob

- Blob is a wrapper type for a shared_ptr<const string> that has accessor functions for getting compressed and uncompressed data
- Moved util::writeFile, util::readFile, util::compress, util::uncompress, decodeImage, and encodePNG to the Blob interface
- Added Blob support to Request and file sources
- Added Blob support to VectorTile objects
- Added support for gzip decoding to util::uncompress
- We're no longer compressing WebP, PNG, and JPEG data when storing in the OfflineDatabase
- Android's HTTPRequest returns compressed Blobs by default

One caveat is that our previous decompress function didn't support gzip, so once users upgrade to this version, their offline cache may contain both zlib-compressed data and gzip-compressed data, but older versions won't be able to decompress gzip data. On the other hand, we don't support downgrading SDKs anyway, so this shouldn't be a problem. To be on the safe side, we could bump the user_version of the SQLite DB.

10 files changed, 104 insertions, 45 deletions

diff --git a/platform/default/asset_file_source.cpp b/platform/default/asset_file_source.cpp
index 3063bf88a0..0b57a6dcef 100644
--- a/platform/default/asset_file_source.cpp
+++ b/platform/default/asset_file_source.cpp
@@ -44,12 +44,10 @@ public:
         } else if (result == -1 && errno == ENOENT) {
             response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound);
         } else {
-            try {
-                response.data = std::make_shared<std::string>(util::read_file(path));
-            } catch (...) {
+            response.data = util::readFile(path);
+            if (!response.data) {
                 response.error = std::make_unique<Response::Error>(
-                    Response::Error::Reason::Other,
-                    util::toString(std::current_exception()));
+                    Response::Error::Reason::Other, "Cannot read file " + path);
             }
         }
 
diff --git a/platform/default/default_file_source.cpp b/platform/default/default_file_source.cpp
index cb602995a4..5dcd28facd 100644
--- a/platform/default/default_file_source.cpp
+++ b/platform/default/default_file_source.cpp
@@ -151,8 +151,23 @@ public:
 
             // Get from the online file source
             if (resource.hasLoadingMethod(Resource::LoadingMethod::Network)) {
+                // Always solicit a compressed response so that we can insert it into the database
+                // while still compressed to save on CPU time.
+                const auto compression = resource.compression;
+                resource.compression = Resource::Compression::PreferCompressed;
                 tasks[req] = onlineFileSource.request(resource, [=] (Response onlineResponse) mutable {
                     this->offlineDatabase->put(resource, onlineResponse);
+                    // If the original request expects an uncompressed response, uncompress before
+                    // handing it back.
+                    if (onlineResponse.data && onlineResponse.data.isCompressed() &&
+                        compression == Resource::Compression::Uncompressed) {
+                        try {
+                            onlineResponse.data.uncompress();
+                        } catch (std::exception& ex) {
+                            onlineResponse.error = std::make_unique<Response::Error>(
+                                Response::Error::Reason::Other, ex.what());
+                        }
+                    }
                     callback(onlineResponse);
                 });
             }
diff --git a/platform/default/http_file_source.cpp b/platform/default/http_file_source.cpp
index a9c442c2de..1a0ae577b8 100644
--- a/platform/default/http_file_source.cpp
+++ b/platform/default/http_file_source.cpp
@@ -371,9 +371,9 @@ void HTTPRequest::handleResult(CURLcode code) {
 
         if (responseCode == 200) {
             if (data) {
-                response->data = std::move(data);
+                response->data = Blob{ std::move(data), false };
             } else {
-                response->data = std::make_shared<std::string>();
+                response->data = Blob{ "", false };
             }
         } else if (responseCode == 204 || (responseCode == 404 && resource.kind == Resource::Kind::Tile)) {
             response->noContent = true;
diff --git a/platform/default/image.cpp b/platform/default/image.cpp
index 447c6bcd66..4fde1898c6 100644
--- a/platform/default/image.cpp
+++ b/platform/default/image.cpp
@@ -11,9 +11,10 @@ PremultipliedImage decodeWebP(const uint8_t*, size_t);
 PremultipliedImage decodePNG(const uint8_t*, size_t);
 PremultipliedImage decodeJPEG(const uint8_t*, size_t);
 
-PremultipliedImage decodeImage(const std::string& string) {
-    const auto* data = reinterpret_cast<const uint8_t*>(string.data());
-    const size_t size = string.size();
+PremultipliedImage decodeImage(Blob blob) {
+    const auto uncompressed = blob.uncompressedData();
+    const auto* data = reinterpret_cast<const uint8_t*>(uncompressed->data());
+    const size_t size = uncompressed->size();
 
 #if !defined(__ANDROID__) && !defined(__APPLE__)
     if (size >= 12) {
diff --git a/platform/default/local_file_source.cpp b/platform/default/local_file_source.cpp
index 0635e86d80..9a8f5ae51b 100644
--- a/platform/default/local_file_source.cpp
+++ b/platform/default/local_file_source.cpp
@@ -46,12 +46,10 @@ public:
         } else if (result == -1 && errno == ENOENT) {
             response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound);
         } else {
-            try {
-                response.data = std::make_shared<std::string>(util::read_file(path));
-            } catch (...) {
+            response.data = util::readFile(path);
+            if (!response.data) {
                 response.error = std::make_unique<Response::Error>(
-                    Response::Error::Reason::Other,
-                    util::toString(std::current_exception()));
+                    Response::Error::Reason::Other, "Cannot read file " + path);
             }
         }
 
diff --git a/platform/default/mbgl/storage/offline_database.cpp b/platform/default/mbgl/storage/offline_database.cpp
index 65c2097182..d7022f1c80 100644
--- a/platform/default/mbgl/storage/offline_database.cpp
+++ b/platform/default/mbgl/storage/offline_database.cpp
@@ -1,10 +1,10 @@
 #include <mbgl/storage/offline_database.hpp>
 #include <mbgl/storage/response.hpp>
-#include <mbgl/util/compression.hpp>
 #include <mbgl/util/io.hpp>
 #include <mbgl/util/string.hpp>
 #include <mbgl/util/chrono.hpp>
 #include <mbgl/util/logging.hpp>
+#include <mbgl/util/compression.hpp>
 
 #include "sqlite3.hpp"
 
@@ -153,7 +153,7 @@ optional<Response> OfflineDatabase::get(const Resource& resource) {
 optional<std::pair<Response, uint64_t>> OfflineDatabase::getInternal(const Resource& resource) {
     if (resource.kind == Resource::Kind::Tile) {
         assert(resource.tileData);
-        return getTile(*resource.tileData);
+        return getTile(resource);
     } else {
         return getResource(resource);
     }
@@ -177,14 +177,31 @@ std::pair<bool, uint64_t> OfflineDatabase::putInternal(const Resource& resource,
         return { false, 0 };
     }
 
-    std::string compressedData;
+    std::shared_ptr<const std::string> data;
     bool compressed = false;
     uint64_t size = 0;
 
     if (response.data) {
-        compressedData = util::compress(*response.data);
-        compressed = compressedData.size() < response.data->size();
-        size = compressed ? compressedData.size() : response.data->size();
+        if (response.data.isCompressed()) {
+            // The response is already compressed; don't try to compare it against the uncompressed size.
+            compressed = true;
+            data = response.data.compressedData();
+        } else {
+            data = response.data.uncompressedData();
+
+            // Only try to compress the data when we have a good chance that the data can actually
+            // be considerably compressed.
+            if (util::isCompressible(*data)) {
+                const auto compressedData = response.data.compressedData();
+                if (compressedData->size() < data->size()) {
+                    compressed = true;
+                    data = compressedData;
+                }
+            }
+        }
+        size = data->size();
+    } else {
+        data = std::make_shared<const std::string>();
     }
 
     if (evict_ && !evict(size)) {
@@ -196,13 +213,9 @@ std::pair<bool, uint64_t> OfflineDatabase::putInternal(const Resource& resource,
 
     if (resource.kind == Resource::Kind::Tile) {
         assert(resource.tileData);
-        inserted = putTile(*resource.tileData, response,
-                compressed ? compressedData : response.data ? *response.data : "",
-                compressed);
+        inserted = putTile(*resource.tileData, response, *data, compressed);
     } else {
-        inserted = putResource(resource, response,
-                compressed ? compressedData : response.data ? *response.data : "",
-                compressed);
+        inserted = putResource(resource, response, *data, compressed);
     }
 
     return { inserted, size };
@@ -243,12 +256,20 @@ optional<std::pair<Response, uint64_t>> OfflineDatabase::getResource(const Resou
     optional<std::string> data = stmt->get<optional<std::string>>(4);
     if (!data) {
         response.noContent = true;
-    } else if (stmt->get<bool>(5)) {
-        response.data = std::make_shared<std::string>(util::decompress(*data));
-        size = data->length();
     } else {
-        response.data = std::make_shared<std::string>(*data);
+        response.data = { std::move(*data), stmt->get<bool>(5) };
         size = data->length();
+
+        // Make sure the data is decompressed when the user explicitly requested uncompressed data.
+        if (response.data.isCompressed() &&
+            resource.compression == Resource::Compression::Uncompressed) {
+            try {
+                response.data.uncompress();
+            } catch (std::exception& ex) {
+                response.error =
+                    std::make_unique<Response::Error>(Response::Error::Reason::Other, ex.what());
+            }
+        }
     }
 
     return std::make_pair(response, size);
@@ -359,7 +380,9 @@ bool OfflineDatabase::putResource(const Resource& resource,
     return true;
 }
 
-optional<std::pair<Response, uint64_t>> OfflineDatabase::getTile(const Resource::TileData& tile) {
+optional<std::pair<Response, uint64_t>> OfflineDatabase::getTile(const Resource& resource) {
+    const auto& tile = *resource.tileData;
+
     // clang-format off
     Statement accessedStmt = getStatement(
         "UPDATE tiles "
@@ -412,12 +435,20 @@ optional<std::pair<Response, uint64_t>> OfflineDatabase::getTile(const Resource:
     optional<std::string> data = stmt->get<optional<std::string>>(4);
     if (!data) {
         response.noContent = true;
-    } else if (stmt->get<bool>(5)) {
-        response.data = std::make_shared<std::string>(util::decompress(*data));
-        size = data->length();
     } else {
-        response.data = std::make_shared<std::string>(*data);
+        response.data = { std::move(*data), stmt->get<bool>(5) };
         size = data->length();
+
+        // Make sure the data is decompressed when the user explicitly requested uncompressed data.
+        if (response.data.isCompressed() &&
+            resource.compression == Resource::Compression::Uncompressed) {
+            try {
+                response.data.uncompress();
+            } catch (std::exception& ex) {
+                response.error =
+                    std::make_unique<Response::Error>(Response::Error::Reason::Other, ex.what());
+            }
+        }
     }
 
     return std::make_pair(response, size);
diff --git a/platform/default/mbgl/storage/offline_database.hpp b/platform/default/mbgl/storage/offline_database.hpp
index 91b544a9e0..fba1a3c230 100644
--- a/platform/default/mbgl/storage/offline_database.hpp
+++ b/platform/default/mbgl/storage/offline_database.hpp
@@ -81,7 +81,7 @@ private:
 
     Statement getStatement(const char *);
 
-    optional<std::pair<Response, uint64_t>> getTile(const Resource::TileData&);
+    optional<std::pair<Response, uint64_t>> getTile(const Resource&);
     optional<int64_t> hasTile(const Resource::TileData&);
     bool putTile(const Resource::TileData&, const Response&,
                  const std::string&, bool compressed);
diff --git a/platform/default/mbgl/storage/offline_download.cpp b/platform/default/mbgl/storage/offline_download.cpp
index ba504c1f9b..d8fe8c646b 100644
--- a/platform/default/mbgl/storage/offline_download.cpp
+++ b/platform/default/mbgl/storage/offline_download.cpp
@@ -71,7 +71,7 @@ OfflineRegionStatus OfflineDownload::getStatus() const {
     }
 
     style::Parser parser;
-    parser.parse(*styleResponse->data);
+    parser.parse(*styleResponse->data.uncompressedData());
 
     result.requiredResourceCountIsPrecise = true;
 
@@ -88,7 +88,8 @@ OfflineRegionStatus OfflineDownload::getStatus() const {
                 optional<Response> sourceResponse = offlineDatabase.get(Resource::source(url));
                 if (sourceResponse) {
                     style::conversion::Error error;
-                    optional<Tileset> tileset = style::conversion::convertJSON<Tileset>(*sourceResponse->data, error);
+                    optional<Tileset> tileset = style::conversion::convertJSON<Tileset>(
+                        *sourceResponse->data.uncompressedData(), error);
                     if (tileset) {
                         result.requiredResourceCount +=
                             definition.tileCount(type, tileSize, (*tileset).zoomRange);
@@ -160,7 +161,7 @@ void OfflineDownload::activateDownload() {
         status.requiredResourceCountIsPrecise = true;
 
         style::Parser parser;
-        parser.parse(*styleResponse.data);
+        parser.parse(*styleResponse.data.uncompressedData());
 
         for (const auto& source : parser.sources) {
             SourceType type = source->getType();
@@ -176,7 +177,8 @@ void OfflineDownload::activateDownload() {
 
                     ensureResource(Resource::source(url), [=](Response sourceResponse) {
                         style::conversion::Error error;
-                        optional<Tileset> tileset = style::conversion::convertJSON<Tileset>(*sourceResponse.data, error);
+                        optional<Tileset> tileset = style::conversion::convertJSON<Tileset>(
+                            *sourceResponse.data.uncompressedData(), error);
                         if (tileset) {
                             util::mapbox::canonicalizeTileset(*tileset, url, type, tileSize);
                             queueTiles(type, tileSize, *tileset);
@@ -236,14 +238,18 @@ void OfflineDownload::activateDownload() {
         if (!parser.glyphURL.empty()) {
             for (const auto& fontStack : parser.fontStacks()) {
                 for (char16_t i = 0; i < GLYPH_RANGES_PER_FONT_STACK; i++) {
-                    queueResource(Resource::glyphs(parser.glyphURL, fontStack, getGlyphRange(i * GLYPHS_PER_GLYPH_RANGE)));
+                    auto resource = Resource::glyphs(parser.glyphURL, fontStack, getGlyphRange(i * GLYPHS_PER_GLYPH_RANGE));
+                    resource.compression = Resource::Compression::PreferCompressed;
+                    queueResource(resource);
                 }
             }
         }
 
         if (!parser.spriteURL.empty()) {
             queueResource(Resource::spriteImage(parser.spriteURL, definition.pixelRatio));
-            queueResource(Resource::spriteJSON(parser.spriteURL, definition.pixelRatio));
+            auto spriteJSON = Resource::spriteJSON(parser.spriteURL, definition.pixelRatio);
+            spriteJSON.compression = Resource::Compression::PreferCompressed;
+            queueResource(spriteJSON);
         }
 
         continueDownload();
diff --git a/platform/default/online_file_source.cpp b/platform/default/online_file_source.cpp
index d685109b95..fe77df8c02 100644
--- a/platform/default/online_file_source.cpp
+++ b/platform/default/online_file_source.cpp
@@ -374,6 +374,16 @@ void OnlineFileRequest::completed(Response response) {
         failedRequestReason = Response::Error::Reason::Success;
     }
 
+    // Make sure the data is decompressed when the user explicitly requested uncompressed data.
+    if (response.data && response.data.isCompressed() &&
+        resource.compression == Resource::Compression::Uncompressed) {
+        try {
+            response.data.uncompress();
+        } catch (std::exception& ex) {
+            response.error = std::make_unique<Response::Error>(Response::Error::Reason::Other, ex.what());
+        }
+    }
+
     schedule(response.expires);
 
     // Calling the callback may result in `this` being deleted. It needs to be done last,
diff --git a/platform/default/png_writer.cpp b/platform/default/png_writer.cpp
index 9ef9052158..d3297b8ebf 100644
--- a/platform/default/png_writer.cpp
+++ b/platform/default/png_writer.cpp
@@ -38,7 +38,7 @@ void addChunk(std::string& png, const char* type, const char* data = "", const u
 namespace mbgl {
 
 // Encode PNGs without libpng.
-std::string encodePNG(const PremultipliedImage& pre) {
+Blob encodePNG(const PremultipliedImage& pre) {
     // Make copy of the image so that we can unpremultiply it.
     const auto src = util::unpremultiply(pre.clone());
 
@@ -74,7 +74,7 @@ std::string encodePNG(const PremultipliedImage& pre) {
     addChunk(png, "IHDR", ihdr, 13);
     addChunk(png, "IDAT", idat.data(), static_cast<uint32_t>(idat.size()));
     addChunk(png, "IEND");
-    return png;
+    return { std::move(png), false };
 }
 
 } // namespace mbgl