feat: Improve statistics for remote hits/misses

ccache collects statistics about local and remote storage layer get/put
operations and describes them as "local/remote hits/misses" in the
output of "ccache -s". However, since "hits" and "misses" mean "result
hit/miss" in the "cacheable calls" section, it's easy to think that they
measure the same thing.

This commit improves the situation by:

- Adding new "local/remote hits/misses" counters that mean "result
  hit/miss". These are shown if remote storage is used (since they
  otherwise are redundant and equal to the normal "hits/misses"
  counters) or if the -v/--verbose option is given.
- Presenting the previous "local/remote hits/misses" counters as
  "local/remote reads". Only shown in verbose mode.
- Adding "local/remote writes" counters. Only shown in verbose mode.

Closes #1016.

11 files changed, 322 insertions, 106 deletions

diff --git a/src/core/Statistic.hpp b/src/core/Statistic.hpp
index 747caf73..1f4fb921 100644
--- a/src/core/Statistic.hpp
+++ b/src/core/Statistic.hpp
@@ -57,14 +57,20 @@ enum class Statistic {
   could_not_use_modules = 32,
   direct_cache_miss = 33,
   preprocessed_cache_miss = 34,
-  local_storage_hit = 35,
-  local_storage_miss = 36,
-  remote_storage_hit = 37,
-  remote_storage_miss = 38,
+  local_storage_read_hit = 35,
+  local_storage_read_miss = 36,
+  remote_storage_read_hit = 37,
+  remote_storage_read_miss = 38,
   remote_storage_error = 39,
   remote_storage_timeout = 40,
   recache = 41,
   unsupported_environment_variable = 42,
+  local_storage_write = 43,
+  local_storage_hit = 44,
+  local_storage_miss = 45,
+  remote_storage_write = 46,
+  remote_storage_hit = 47,
+  remote_storage_miss = 48,
 
   END
 };
diff --git a/src/core/Statistics.cpp b/src/core/Statistics.cpp
index 399bcb67..4952ce20 100644
--- a/src/core/Statistics.cpp
+++ b/src/core/Statistics.cpp
@@ -95,6 +95,9 @@ const StatisticsField k_statistics_fields[] = {
   FIELD(internal_error, "Internal error", FLAG_ERROR),
   FIELD(local_storage_hit, nullptr),
   FIELD(local_storage_miss, nullptr),
+  FIELD(local_storage_read_hit, nullptr),
+  FIELD(local_storage_read_miss, nullptr),
+  FIELD(local_storage_write, nullptr),
   FIELD(missing_cache_file, "Missing cache file", FLAG_ERROR),
   FIELD(multiple_source_files, "Multiple source files", FLAG_UNCACHEABLE),
   FIELD(no_input_file, "No input file", FLAG_UNCACHEABLE),
@@ -108,6 +111,9 @@ const StatisticsField k_statistics_fields[] = {
   FIELD(remote_storage_error, nullptr),
   FIELD(remote_storage_hit, nullptr),
   FIELD(remote_storage_miss, nullptr),
+  FIELD(remote_storage_read_hit, nullptr),
+  FIELD(remote_storage_read_miss, nullptr),
+  FIELD(remote_storage_write, nullptr),
   FIELD(remote_storage_timeout, nullptr),
   FIELD(stats_zeroed_timestamp, nullptr),
   FIELD(
@@ -295,12 +301,20 @@ Statistics::format_human_readable(const Config& config,
   const uint64_t g = 1'000'000'000;
   const uint64_t local_hits = S(local_storage_hit);
   const uint64_t local_misses = S(local_storage_miss);
+  const uint64_t local_reads =
+    S(local_storage_read_hit) + S(local_storage_read_miss);
+  const uint64_t local_writes = S(local_storage_write);
   const uint64_t local_size = S(cache_size_kibibyte) * 1024;
   const uint64_t cleanups = S(cleanups_performed);
-  table.add_heading("Local storage:");
-  add_ratio_row(table, "  Hits:", local_hits, local_hits + local_misses);
-  add_ratio_row(table, "  Misses:", local_misses, local_hits + local_misses);
+  const uint64_t remote_hits = S(remote_storage_hit);
+  const uint64_t remote_misses = S(remote_storage_miss);
+  const uint64_t remote_reads =
+    S(remote_storage_read_hit) + S(remote_storage_read_miss);
+  const uint64_t remote_writes = S(remote_storage_write);
+  const uint64_t remote_errors = S(remote_storage_error);
+  const uint64_t remote_timeouts = S(remote_storage_timeout);
 
+  table.add_heading("Local storage:");
   if (!from_log) {
     std::vector<C> size_cells{
       "  Cache size (GB):",
@@ -328,11 +342,14 @@ Statistics::format_human_readable(const Config& config,
       table.add_row({"  Cleanups:", cleanups});
     }
   }
-
-  const uint64_t remote_hits = S(remote_storage_hit);
-  const uint64_t remote_misses = S(remote_storage_miss);
-  const uint64_t remote_errors = S(remote_storage_error);
-  const uint64_t remote_timeouts = S(remote_storage_timeout);
+  if (verbosity > 0 || (remote_hits + remote_misses) > 0) {
+    add_ratio_row(table, "  Hits:", local_hits, local_hits + local_misses);
+    add_ratio_row(table, "  Misses:", local_misses, local_hits + local_misses);
+  }
+  if (verbosity > 0) {
+    table.add_row({"  Reads:", local_reads});
+    table.add_row({"  Writes:", local_writes});
+  }
 
   if (verbosity > 1
       || remote_hits + remote_misses + remote_errors + remote_timeouts > 0) {
@@ -340,6 +357,10 @@ Statistics::format_human_readable(const Config& config,
     add_ratio_row(table, "  Hits:", remote_hits, remote_hits + remote_misses);
     add_ratio_row(
       table, "  Misses:", remote_misses, remote_hits + remote_misses);
+    if (verbosity > 0) {
+      table.add_row({"  Reads:", remote_reads});
+      table.add_row({"  Writes:", remote_writes});
+    }
     if (verbosity > 1 || remote_errors > 0) {
       table.add_row({"  Errors:", remote_errors});
     }
diff --git a/src/storage/Storage.cpp b/src/storage/Storage.cpp
index a607be50..5c843db2 100644
--- a/src/storage/Storage.cpp
+++ b/src/storage/Storage.cpp
@@ -239,8 +239,6 @@ Storage::get(const Digest& key,
 
   if (!m_config.remote_only()) {
     auto value = local.get(key, type);
-    local.increment_statistic(value ? core::Statistic::local_storage_hit
-                                    : core::Statistic::local_storage_miss);
     if (value) {
       if (m_config.reshare()) {
         put_in_remote_storage(key, *value, true);
@@ -251,7 +249,7 @@ Storage::get(const Digest& key,
     }
   }
 
-  get_from_remote_storage(key, [&](util::Bytes&& data) {
+  get_from_remote_storage(key, type, [&](util::Bytes&& data) {
     if (!m_config.remote_only()) {
       local.put(key, type, data, true);
     }
@@ -425,6 +423,7 @@ Storage::get_backend(RemoteStorageEntry& entry,
 
 void
 Storage::get_from_remote_storage(const Digest& key,
+                                 const core::CacheEntryType type,
                                  const EntryReceiver& entry_receiver)
 {
   MTR_SCOPE("remote_storage", "get");
@@ -449,7 +448,10 @@ Storage::get_from_remote_storage(const Digest& key,
           key.to_string(),
           backend->url_for_logging,
           ms);
-      local.increment_statistic(core::Statistic::remote_storage_hit);
+      local.increment_statistic(core::Statistic::remote_storage_read_hit);
+      if (type == core::CacheEntryType::result) {
+        local.increment_statistic(core::Statistic::remote_storage_hit);
+      }
       if (entry_receiver(std::move(*value))) {
         return;
       }
@@ -458,7 +460,10 @@ Storage::get_from_remote_storage(const Digest& key,
           key.to_string(),
           backend->url_for_logging,
           ms);
-      local.increment_statistic(core::Statistic::remote_storage_miss);
+      local.increment_statistic(core::Statistic::remote_storage_read_miss);
+      if (type == core::CacheEntryType::result) {
+        local.increment_statistic(core::Statistic::remote_storage_miss);
+      }
     }
   }
 }
@@ -497,6 +502,7 @@ Storage::put_in_remote_storage(const Digest& key,
         key.to_string(),
         backend->url_for_logging,
         ms);
+    local.increment_statistic(core::Statistic::remote_storage_write);
   }
 }
 
@@ -531,6 +537,8 @@ Storage::remove_from_remote_storage(const Digest& key)
           backend->url_for_logging,
           ms);
     }
+
+    local.increment_statistic(core::Statistic::remote_storage_write);
   }
 }
 
diff --git a/src/storage/Storage.hpp b/src/storage/Storage.hpp
index f3eeba56..1b72106e 100644
--- a/src/storage/Storage.hpp
+++ b/src/storage/Storage.hpp
@@ -82,6 +82,7 @@ private:
                                          const bool for_writing);
 
   void get_from_remote_storage(const Digest& key,
+                               core::CacheEntryType type,
                                const EntryReceiver& entry_receiver);
 
   void put_in_remote_storage(const Digest& key,
diff --git a/src/storage/local/LocalStorage.cpp b/src/storage/local/LocalStorage.cpp
index 4c1f3996..fdc48dc6 100644
--- a/src/storage/local/LocalStorage.cpp
+++ b/src/storage/local/LocalStorage.cpp
@@ -178,27 +178,39 @@ LocalStorage::finalize()
 }
 
 std::optional<util::Bytes>
-LocalStorage::get(const Digest& key, const core::CacheEntryType type) const
+LocalStorage::get(const Digest& key, const core::CacheEntryType type)
 {
   MTR_SCOPE("local_storage", "get");
 
+  std::optional<util::Bytes> return_value;
+
   const auto cache_file = look_up_cache_file(key, type);
-  if (!cache_file.stat) {
+  if (cache_file.stat) {
+    const auto value = util::read_file<util::Bytes>(cache_file.path);
+    if (value) {
+      LOG("Retrieved {} from local storage ({})",
+          key.to_string(),
+          cache_file.path);
+
+      // Update modification timestamp to save file from LRU cleanup.
+      util::set_timestamps(cache_file.path);
+
+      return_value = *value;
+    } else {
+      LOG("Failed to read {}: {}", cache_file.path, value.error());
+    }
+  } else {
     LOG("No {} in local storage", key.to_string());
-    return std::nullopt;
   }
-  const auto value = util::read_file<util::Bytes>(cache_file.path);
-  if (!value) {
-    LOG("Failed to read {}: {}", cache_file.path, value.error());
-    return std::nullopt;
-  }
-
-  LOG("Retrieved {} from local storage ({})", key.to_string(), cache_file.path);
 
-  // Update modification timestamp to save file from LRU cleanup.
-  util::set_timestamps(cache_file.path);
+  increment_statistic(return_value ? core::Statistic::local_storage_read_hit
+                                   : core::Statistic::local_storage_read_miss);
+  if (type == core::CacheEntryType::result) {
+    increment_statistic(return_value ? core::Statistic::local_storage_hit
+                                     : core::Statistic::local_storage_miss);
+  }
 
-  return *value;
+  return return_value;
 }
 
 void
@@ -229,12 +241,11 @@ LocalStorage::put(const Digest& key,
   }
 
   try {
+    increment_statistic(core::Statistic::local_storage_write);
     AtomicFile result_file(cache_file.path, AtomicFile::Mode::binary);
     result_file.write(value);
     result_file.commit();
-  }
-
-  catch (core::Error& e) {
+  } catch (core::Error& e) {
     LOG("Failed to write to {}: {}", cache_file.path, e.what());
     return;
   }
@@ -269,6 +280,7 @@ LocalStorage::remove(const Digest& key, const core::CacheEntryType type)
 
   const auto cache_file = look_up_cache_file(key, type);
   if (cache_file.stat) {
+    increment_statistic(core::Statistic::local_storage_write);
     Util::unlink_safe(cache_file.path);
     LOG("Removed {} from local storage ({})", key.to_string(), cache_file.path);
   } else {
diff --git a/src/storage/local/LocalStorage.hpp b/src/storage/local/LocalStorage.hpp
index e2635645..6251b3e4 100644
--- a/src/storage/local/LocalStorage.hpp
+++ b/src/storage/local/LocalStorage.hpp
@@ -55,8 +55,7 @@ public:
 
   // --- Cache entry handling ---
 
-  std::optional<util::Bytes> get(const Digest& key,
-                                 core::CacheEntryType type) const;
+  std::optional<util::Bytes> get(const Digest& key, core::CacheEntryType type);
 
   void put(const Digest& key,
            core::CacheEntryType type,
diff --git a/test/suites/base.bash b/test/suites/base.bash
index 93753073..db3389dc 100644
--- a/test/suites/base.bash
+++ b/test/suites/base.bash
@@ -11,8 +11,14 @@ base_tests() {
     expect_stat preprocessed_cache_miss 1
     expect_stat local_storage_hit 0
     expect_stat local_storage_miss 1
+    expect_stat local_storage_read_hit 0
+    expect_stat local_storage_read_miss 1
+    expect_stat local_storage_write 1
     expect_stat remote_storage_hit 0
     expect_stat remote_storage_miss 0
+    expect_stat remote_storage_read_hit 0
+    expect_stat remote_storage_read_miss 0
+    expect_stat remote_storage_write 0
     expect_stat files_in_cache 1
     expect_equal_object_files reference_test1.o test1.o
 
@@ -23,8 +29,14 @@ base_tests() {
     expect_stat preprocessed_cache_miss 1
     expect_stat local_storage_hit 1
     expect_stat local_storage_miss 1
+    expect_stat local_storage_read_hit 1
+    expect_stat local_storage_read_miss 1
+    expect_stat local_storage_write 1
     expect_stat remote_storage_hit 0
     expect_stat remote_storage_miss 0
+    expect_stat remote_storage_read_hit 0
+    expect_stat remote_storage_read_miss 0
+    expect_stat remote_storage_write 0
     expect_stat files_in_cache 1
     expect_equal_object_files reference_test1.o test1.o
 
diff --git a/test/suites/direct.bash b/test/suites/direct.bash
index 65969bac..5fea53b1 100644
--- a/test/suites/direct.bash
+++ b/test/suites/direct.bash
@@ -36,9 +36,15 @@ SUITE_direct() {
     expect_stat direct_cache_miss 1
     expect_stat preprocessed_cache_miss 1
     expect_stat local_storage_hit 0
-    expect_stat local_storage_miss 2 # result + manifest
+    expect_stat local_storage_miss 1
+    expect_stat local_storage_read_hit 0
+    expect_stat local_storage_read_miss 2 # result + manifest
+    expect_stat local_storage_write 2 # result + manifest
     expect_stat remote_storage_hit 0
     expect_stat remote_storage_miss 0
+    expect_stat remote_storage_read_hit 0
+    expect_stat remote_storage_read_miss 0
+    expect_stat remote_storage_write 0
     expect_stat files_in_cache 2 # result + manifest
     expect_equal_object_files reference_test.o test.o
 
@@ -51,10 +57,16 @@ SUITE_direct() {
     expect_stat cache_miss 1
     expect_stat direct_cache_miss 1
     expect_stat preprocessed_cache_miss 1
-    expect_stat local_storage_hit 2 # result + manifest
-    expect_stat local_storage_miss 2 # result + manifest
+    expect_stat local_storage_hit 1
+    expect_stat local_storage_miss 1
+    expect_stat local_storage_read_hit 2 # result + manifest
+    expect_stat local_storage_read_miss 2 # result + manifest
+    expect_stat local_storage_write 2 # result + manifest
     expect_stat remote_storage_hit 0
     expect_stat remote_storage_miss 0
+    expect_stat remote_storage_read_hit 0
+    expect_stat remote_storage_read_miss 0
+    expect_stat remote_storage_write 0
     expect_stat files_in_cache 2
     expect_equal_object_files reference_test.o test.o
     expect_newer_than $manifest_file test.c
diff --git a/test/suites/remote_file.bash b/test/suites/remote_file.bash
index af37b724..3e23b57e 100644
--- a/test/suites/remote_file.bash
+++ b/test/suites/remote_file.bash
@@ -18,9 +18,15 @@ SUITE_remote_file() {
     expect_stat cache_miss 1
     expect_stat files_in_cache 2
     expect_stat local_storage_hit 0
-    expect_stat local_storage_miss 2 # result + manifest
+    expect_stat local_storage_miss 1
+    expect_stat local_storage_read_hit 0
+    expect_stat local_storage_read_miss 2 # result + manifest
+    expect_stat local_storage_write 2 # result + manifest
     expect_stat remote_storage_hit 0
-    expect_stat remote_storage_miss 2 # result + manifest
+    expect_stat remote_storage_miss 1
+    expect_stat remote_storage_read_hit 0
+    expect_stat remote_storage_read_miss 2 # result + manifest
+    expect_stat remote_storage_write 2 # result + manifest
     expect_exists remote/CACHEDIR.TAG
     subdirs=$(find remote -type d | wc -l)
     if [ "${subdirs}" -lt 2 ]; then # "remote" itself counts as one
@@ -32,10 +38,16 @@ SUITE_remote_file() {
     $CCACHE_COMPILE -c test.c
     expect_stat direct_cache_hit 1
     expect_stat cache_miss 1
-    expect_stat local_storage_hit 2 # result + manifest
-    expect_stat local_storage_miss 2 # result + manifest
+    expect_stat local_storage_hit 1
+    expect_stat local_storage_miss 1
+    expect_stat local_storage_read_hit 2 # result + manifest
+    expect_stat local_storage_read_miss 2 # result + manifest
+    expect_stat local_storage_write 2 # result + manifest
     expect_stat remote_storage_hit 0
-    expect_stat remote_storage_miss 2 # result + manifest
+    expect_stat remote_storage_miss 1
+    expect_stat remote_storage_read_hit 0
+    expect_stat remote_storage_read_miss 2
+    expect_stat remote_storage_write 2
     expect_stat files_in_cache 2
     expect_file_count 3 '*' remote # CACHEDIR.TAG + result + manifest
 
@@ -48,10 +60,16 @@ SUITE_remote_file() {
     $CCACHE_COMPILE -c test.c
     expect_stat direct_cache_hit 2
     expect_stat cache_miss 1
-    expect_stat local_storage_hit 2
-    expect_stat local_storage_miss 4 # 2 * (result + manifest)
-    expect_stat remote_storage_hit 2 # result + manifest
-    expect_stat remote_storage_miss 2 # result + manifest
+    expect_stat local_storage_hit 1
+    expect_stat local_storage_miss 2
+    expect_stat local_storage_read_hit 2 # result + manifest
+    expect_stat local_storage_read_miss 4 # 2 * (result + manifest)
+    expect_stat local_storage_write 4 # 2 * (result + manifest)
+    expect_stat remote_storage_hit 1
+    expect_stat remote_storage_miss 1
+    expect_stat remote_storage_read_hit 2 # result + manifest
+    expect_stat remote_storage_read_miss 2 # result + manifest
+    expect_stat remote_storage_write 2 # result + manifest
     expect_stat files_in_cache 2 # fetched from remote
     expect_file_count 3 '*' remote # CACHEDIR.TAG + result + manifest
 
@@ -59,10 +77,16 @@ SUITE_remote_file() {
     $CCACHE_COMPILE -c test.c
     expect_stat direct_cache_hit 3
     expect_stat cache_miss 1
-    expect_stat local_storage_hit 4
-    expect_stat local_storage_miss 4 # 2 * (result + manifest)
-    expect_stat remote_storage_hit 2 # result + manifest
-    expect_stat remote_storage_miss 2 # result + manifest
+    expect_stat local_storage_hit 2
+    expect_stat local_storage_miss 2
+    expect_stat local_storage_read_hit 4 # 2 * (result + manifest)
+    expect_stat local_storage_read_miss 4 # 2 * (result + manifest)
+    expect_stat local_storage_write 4 # 2 * (result + manifest)
+    expect_stat remote_storage_hit 1
+    expect_stat remote_storage_miss 1
+    expect_stat remote_storage_read_hit 2 # result + manifest
+    expect_stat remote_storage_read_miss 2 # result + manifest
+    expect_stat remote_storage_write 2 # result + manifest
     expect_stat files_in_cache 2 # fetched from remote
     expect_file_count 3 '*' remote # CACHEDIR.TAG + result + manifest
 
@@ -205,27 +229,45 @@ SUITE_remote_file() {
     expect_stat cache_miss 1
     expect_stat files_in_cache 2
     expect_stat local_storage_hit 0
-    expect_stat local_storage_miss 2
+    expect_stat local_storage_miss 1
+    expect_stat local_storage_read_hit 0
+    expect_stat local_storage_read_miss 2
+    expect_stat local_storage_write 2
     expect_stat remote_storage_hit 0
     expect_stat remote_storage_miss 0
+    expect_stat remote_storage_read_hit 0
+    expect_stat remote_storage_read_miss 0
+    expect_stat remote_storage_write 0
     expect_missing remote
 
     $CCACHE_COMPILE -c test.c
     expect_stat direct_cache_hit 1
     expect_stat cache_miss 1
-    expect_stat local_storage_hit 2
-    expect_stat local_storage_miss 2
+    expect_stat local_storage_hit 1
+    expect_stat local_storage_miss 1
+    expect_stat local_storage_read_hit 2
+    expect_stat local_storage_read_miss 2
+    expect_stat local_storage_write 2
     expect_stat remote_storage_hit 0
     expect_stat remote_storage_miss 0
+    expect_stat remote_storage_read_hit 0
+    expect_stat remote_storage_read_miss 0
+    expect_stat remote_storage_write 0
     expect_missing remote
 
     CCACHE_RESHARE=1 $CCACHE_COMPILE -c test.c
     expect_stat direct_cache_hit 2
     expect_stat cache_miss 1
-    expect_stat local_storage_hit 4
-    expect_stat local_storage_miss 2
+    expect_stat local_storage_hit 2
+    expect_stat local_storage_miss 1
+    expect_stat local_storage_read_hit 4
+    expect_stat local_storage_read_miss 2
+    expect_stat local_storage_write 2
     expect_stat remote_storage_hit 0
     expect_stat remote_storage_miss 0
+    expect_stat remote_storage_read_hit 0
+    expect_stat remote_storage_read_miss 0
+    expect_stat remote_storage_write 2
     expect_file_count 3 '*' remote # CACHEDIR.TAG + result + manifest
 
     $CCACHE -C >/dev/null
@@ -233,10 +275,16 @@ SUITE_remote_file() {
     $CCACHE_COMPILE -c test.c
     expect_stat direct_cache_hit 3
     expect_stat cache_miss 1
-    expect_stat local_storage_hit 4
-    expect_stat local_storage_miss 4
-    expect_stat remote_storage_hit 2
+    expect_stat local_storage_hit 2
+    expect_stat local_storage_miss 2
+    expect_stat local_storage_read_hit 4
+    expect_stat local_storage_read_miss 4
+    expect_stat local_storage_write 4
+    expect_stat remote_storage_hit 1
     expect_stat remote_storage_miss 0
+    expect_stat remote_storage_read_hit 2
+    expect_stat remote_storage_read_miss 0
+    expect_stat remote_storage_write 2
     expect_file_count 3 '*' remote # CACHEDIR.TAG + result + manifest
 
     # -------------------------------------------------------------------------
@@ -245,13 +293,20 @@ SUITE_remote_file() {
     CCACHE_RECACHE=1 $CCACHE_COMPILE -c test.c
     expect_stat direct_cache_hit 0
     expect_stat direct_cache_miss 0
+    expect_stat preprocessed_cache_miss 0
     expect_stat cache_miss 0
     expect_stat recache 1
     expect_stat files_in_cache 2
     expect_stat local_storage_hit 0
-    expect_stat local_storage_miss 1 # Try to read manifest for updating
+    expect_stat local_storage_miss 0
+    expect_stat local_storage_read_hit 0
+    expect_stat local_storage_read_miss 1 # Try to read manifest for updating
+    expect_stat local_storage_write 2
     expect_stat remote_storage_hit 0
-    expect_stat remote_storage_miss 1 # Try to read manifest for updating
+    expect_stat remote_storage_miss 0
+    expect_stat remote_storage_read_hit 0
+    expect_stat remote_storage_read_miss 1 # Try to read manifest for updating
+    expect_stat remote_storage_write 2
     expect_file_count 3 '*' remote # CACHEDIR.TAG + result + manifest
 
     $CCACHE -C >/dev/null
@@ -260,14 +315,22 @@ SUITE_remote_file() {
 
     CCACHE_RECACHE=1 $CCACHE_COMPILE -c test.c
     expect_stat direct_cache_hit 0
+
     expect_stat direct_cache_miss 0
+    expect_stat preprocessed_cache_miss 0
     expect_stat cache_miss 0
     expect_stat recache 2
     expect_stat files_in_cache 2
     expect_stat local_storage_hit 0
-    expect_stat local_storage_miss 2 # Try to read manifest for updating
-    expect_stat remote_storage_hit 1 # Read manifest for updating
-    expect_stat remote_storage_miss 1
+    expect_stat local_storage_miss 0
+    expect_stat local_storage_read_hit 0
+    expect_stat local_storage_read_miss 2 # Try to read manifest for updating
+    expect_stat local_storage_write 4
+    expect_stat remote_storage_hit 0
+    expect_stat remote_storage_miss 0
+    expect_stat remote_storage_read_hit 1 # Read manifest for updating
+    expect_stat remote_storage_read_miss 1
+    expect_stat remote_storage_write 3 # Not 4 since result key already present
 
     # -------------------------------------------------------------------------
     if touch test.c && ln test.c test-if-fs-supports-hard-links.c 2>/dev/null; then
@@ -278,18 +341,30 @@ SUITE_remote_file() {
         expect_stat cache_miss 1
         expect_stat files_in_cache 3
         expect_stat local_storage_hit 0
-        expect_stat local_storage_miss 2 # result + manifest
+        expect_stat local_storage_miss 1
+        expect_stat local_storage_read_hit 0
+        expect_stat local_storage_read_miss 2
+        expect_stat local_storage_write 2
         expect_stat remote_storage_hit 0
         expect_stat remote_storage_miss 0
+        expect_stat remote_storage_read_hit 0
+        expect_stat remote_storage_read_miss 0
+        expect_stat remote_storage_write 0
 
         CCACHE_RESHARE=1 $CCACHE_COMPILE -c test.c
         expect_stat direct_cache_hit 1
         expect_stat cache_miss 1
         expect_stat files_in_cache 3
-        expect_stat local_storage_hit 2
-        expect_stat local_storage_miss 2 # result + manifest
+        expect_stat local_storage_hit 1
+        expect_stat local_storage_miss 1
+        expect_stat local_storage_read_hit 2
+        expect_stat local_storage_read_miss 2
+        expect_stat local_storage_write 2
         expect_stat remote_storage_hit 0
         expect_stat remote_storage_miss 0
+        expect_stat remote_storage_read_hit 0
+        expect_stat remote_storage_read_miss 0
+        expect_stat remote_storage_write 1 # result not saved since not self-contained
         expect_file_count 2 '*' remote # CACHEDIR.TAG + manifest, not result
     fi
 
@@ -304,9 +379,15 @@ SUITE_remote_file() {
     expect_stat direct_cache_hit 0
     expect_stat cache_miss 1
     expect_stat local_storage_hit 0
-    expect_stat local_storage_miss 2 # miss: manifest + result
+    expect_stat local_storage_miss 1
+    expect_stat local_storage_read_hit 0
+    expect_stat local_storage_read_miss 2 # miss: manifest + result
+    expect_stat local_storage_write 2 # manifest + result
     expect_stat remote_storage_hit 0
-    expect_stat remote_storage_miss 2 # miss: manifest + result
+    expect_stat remote_storage_miss 1
+    expect_stat remote_storage_read_hit 0
+    expect_stat remote_storage_read_miss 2
+    expect_stat remote_storage_write 2 # miss: manifest + result
 
     # Both local and remote now have an "int x;" key in the manifest.
 
@@ -316,10 +397,16 @@ SUITE_remote_file() {
     $CCACHE_COMPILE -c test.c
     expect_stat direct_cache_hit 0
     expect_stat cache_miss 2
-    expect_stat local_storage_hit 1 # hit: manifest without key
-    expect_stat local_storage_miss 3 # miss: result
-    expect_stat remote_storage_hit 1 # his: manifest without key
-    expect_stat remote_storage_miss 3 # miss: result
+    expect_stat local_storage_hit 0
+    expect_stat local_storage_miss 2
+    expect_stat local_storage_read_hit 1 # hit: manifest without key
+    expect_stat local_storage_read_miss 3 # miss: result
+    expect_stat local_storage_write 5 # miss: merged manifest + new manifest entry + result
+    expect_stat remote_storage_hit 0
+    expect_stat remote_storage_miss 2
+    expect_stat remote_storage_read_hit 1 # # hit: manifest without key
+    expect_stat remote_storage_read_miss 3 # miss: result
+    expect_stat remote_storage_write 4 # miss: manifest + result
 
     # Both local and remote now have "int x;" and "int y;" keys in the manifest.
 
@@ -334,10 +421,16 @@ SUITE_remote_file() {
     $CCACHE_COMPILE -c test.c
     expect_stat direct_cache_hit 1
     expect_stat cache_miss 2
-    expect_stat local_storage_hit 1
-    expect_stat local_storage_miss 5 # miss: manifest + result
-    expect_stat remote_storage_hit 3 # hit: manifest + result
-    expect_stat remote_storage_miss 3
+    expect_stat local_storage_hit 0
+    expect_stat local_storage_miss 3
+    expect_stat local_storage_read_hit 1
+    expect_stat local_storage_read_miss 5 # miss: manifest + result
+    expect_stat local_storage_write 7 # miss: manifest + result
+    expect_stat remote_storage_hit 1
+    expect_stat remote_storage_miss 2
+    expect_stat remote_storage_read_hit 3
+    expect_stat remote_storage_read_miss 3
+    expect_stat remote_storage_write 4
 
     # Should be able to get remote hit without involving local.
 
@@ -347,10 +440,16 @@ SUITE_remote_file() {
     $CCACHE_COMPILE -c test.c
     expect_stat direct_cache_hit 2
     expect_stat cache_miss 2
-    expect_stat local_storage_hit 2 # hit: manifest with key (downloaded from previous step)
-    expect_stat local_storage_miss 6 # miss: result
-    expect_stat remote_storage_hit 4 # hit: result
-    expect_stat remote_storage_miss 3
+    expect_stat local_storage_hit 0
+    expect_stat local_storage_miss 4
+    expect_stat local_storage_read_hit 2 # hit: manifest with key (downloaded from previous step)
+    expect_stat local_storage_read_miss 6 # miss: manifest + result
+    expect_stat local_storage_write 8 # miss: result
+    expect_stat remote_storage_hit 2
+    expect_stat remote_storage_miss 2
+    expect_stat remote_storage_read_hit 4 # hit: result
+    expect_stat remote_storage_read_miss 3
+    expect_stat remote_storage_write 4
 
     # -------------------------------------------------------------------------
     TEST "Manifest merging"
@@ -363,9 +462,15 @@ SUITE_remote_file() {
     expect_stat direct_cache_hit 0
     expect_stat cache_miss 1
     expect_stat local_storage_hit 0
-    expect_stat local_storage_miss 2 # miss: manifest + result
+    expect_stat local_storage_miss 1
+    expect_stat local_storage_read_hit 0
+    expect_stat local_storage_read_miss 2 # miss: manifest + result
+    expect_stat local_storage_write 2
     expect_stat remote_storage_hit 0
-    expect_stat remote_storage_miss 2 # miss: manifest + result
+    expect_stat remote_storage_miss 1
+    expect_stat remote_storage_read_hit 0
+    expect_stat remote_storage_read_miss 2 # miss: manifest + result 3
+    expect_stat remote_storage_write 2
 
     $CCACHE -C >/dev/null
 
@@ -378,9 +483,15 @@ SUITE_remote_file() {
     expect_stat direct_cache_hit 0
     expect_stat cache_miss 2
     expect_stat local_storage_hit 0
-    expect_stat local_storage_miss 4 # miss: manifest + result
+    expect_stat local_storage_miss 2
+    expect_stat local_storage_read_hit 0
+    expect_stat local_storage_read_miss 4 # miss: manifest + result
+    expect_stat local_storage_write 4
     expect_stat remote_storage_hit 0
-    expect_stat remote_storage_miss 2
+    expect_stat remote_storage_miss 1
+    expect_stat remote_storage_read_hit 0
+    expect_stat remote_storage_read_miss 2
+    expect_stat remote_storage_write 2
 
     # Now local has "int y;" while remote still has "int x;".
 
@@ -390,10 +501,16 @@ SUITE_remote_file() {
     $CCACHE_COMPILE -c test.c
     expect_stat direct_cache_hit 1
     expect_stat cache_miss 2
-    expect_stat local_storage_hit 1 # hit: manifest without key
-    expect_stat local_storage_miss 5 # miss: result
-    expect_stat remote_storage_hit 2 # hit: manifest + result
-    expect_stat remote_storage_miss 2
+    expect_stat local_storage_hit 0
+    expect_stat local_storage_miss 3
+    expect_stat local_storage_read_hit 1 # hit: manifest without key
+    expect_stat local_storage_read_miss 5 # miss: result
+    expect_stat local_storage_write 6
+    expect_stat remote_storage_hit 1
+    expect_stat remote_storage_miss 1
+    expect_stat remote_storage_read_hit 2 # hit: manifest + result
+    expect_stat remote_storage_read_miss 2
+    expect_stat remote_storage_write 2
 
     # Local manifest with "int y;" was merged with remote's "int x;" above, so
     # we should now be able to get "int x;" and "int y;" hits locally.
@@ -404,10 +521,16 @@ SUITE_remote_file() {
     CCACHE_REMOTE_STORAGE= $CCACHE_COMPILE -c test.c
     expect_stat direct_cache_hit 2
     expect_stat cache_miss 2
-    expect_stat local_storage_hit 3 # hit: manifest + result
-    expect_stat local_storage_miss 5
-    expect_stat remote_storage_hit 2
-    expect_stat remote_storage_miss 2
+    expect_stat local_storage_hit 1
+    expect_stat local_storage_miss 3
+    expect_stat local_storage_read_hit 3 # hit: manifest + result
+    expect_stat local_storage_read_miss 5 # miss: result
+    expect_stat local_storage_write 6
+    expect_stat remote_storage_hit 1
+    expect_stat remote_storage_miss 1
+    expect_stat remote_storage_read_hit 2
+    expect_stat remote_storage_read_miss 2
+    expect_stat remote_storage_write 2
 
     echo 'int x;' >test.h
     backdate test.h
@@ -415,8 +538,14 @@ SUITE_remote_file() {
     CCACHE_REMOTE_STORAGE= $CCACHE_COMPILE -c test.c
     expect_stat direct_cache_hit 3
     expect_stat cache_miss 2
-    expect_stat local_storage_hit 5 # hit: manifest + result
-    expect_stat local_storage_miss 5
-    expect_stat remote_storage_hit 2
-    expect_stat remote_storage_miss 2
+    expect_stat local_storage_hit 2
+    expect_stat local_storage_miss 3
+    expect_stat local_storage_read_hit 5 # hit: manifest + result
+    expect_stat local_storage_read_miss 5 # miss: result
+    expect_stat local_storage_write 6
+    expect_stat remote_storage_hit 1
+    expect_stat remote_storage_miss 1
+    expect_stat remote_storage_read_hit 2
+    expect_stat remote_storage_read_miss 2
+    expect_stat remote_storage_write 2
 }
diff --git a/test/suites/remote_only.bash b/test/suites/remote_only.bash
index 3dea32ac..44e1e8b1 100644
--- a/test/suites/remote_only.bash
+++ b/test/suites/remote_only.bash
@@ -16,8 +16,14 @@ SUITE_remote_only() {
     expect_stat files_in_cache 0
     expect_stat local_storage_hit 0
     expect_stat local_storage_miss 0
+    expect_stat local_storage_read_hit 0
+    expect_stat local_storage_read_miss 0
+    expect_stat local_storage_write 0
     expect_stat remote_storage_hit 0
-    expect_stat remote_storage_miss 2 # result + manifest
+    expect_stat remote_storage_miss 1
+    expect_stat remote_storage_read_hit 0
+    expect_stat remote_storage_read_miss 2
+    expect_stat remote_storage_write 2
     expect_file_count 3 '*' remote # CACHEDIR.TAG + result + manifest
 
     $CCACHE_COMPILE -c test.c
@@ -26,8 +32,14 @@ SUITE_remote_only() {
     expect_stat files_in_cache 0
     expect_stat local_storage_hit 0
     expect_stat local_storage_miss 0
-    expect_stat remote_storage_hit 2
-    expect_stat remote_storage_miss 2
+    expect_stat local_storage_read_hit 0
+    expect_stat local_storage_read_miss 0
+    expect_stat local_storage_write 0
+    expect_stat remote_storage_hit 1
+    expect_stat remote_storage_miss 1
+    expect_stat remote_storage_read_hit 2
+    expect_stat remote_storage_read_miss 2
+    expect_stat remote_storage_write 2
     expect_stat files_in_cache 0
     expect_file_count 3 '*' remote # CACHEDIR.TAG + result + manifest
 }
diff --git a/test/suites/stats_log.bash b/test/suites/stats_log.bash
index 9decaa05..7fa1be74 100644
--- a/test/suites/stats_log.bash
+++ b/test/suites/stats_log.bash
@@ -12,24 +12,28 @@ SUITE_stats_log() {
     expect_stat direct_cache_hit 0
     expect_stat preprocessed_cache_miss 1
     expect_stat cache_miss 1
-    expect_stat local_storage_hit 0
-    expect_stat local_storage_miss 2
+    expect_stat local_storage_read_hit 0
+    expect_stat local_storage_read_miss 2
 
     $CCACHE_COMPILE -c test.c
     expect_stat direct_cache_hit 1
     expect_stat preprocessed_cache_miss 1
     expect_stat cache_miss 1
-    expect_stat local_storage_hit 2
-    expect_stat local_storage_miss 2
+    expect_stat local_storage_read_hit 2
+    expect_stat local_storage_read_miss 2
 
     expect_content stats.log "# test.c
 cache_miss
 direct_cache_miss
 local_storage_miss
-local_storage_miss
+local_storage_read_miss
+local_storage_read_miss
+local_storage_write
+local_storage_write
 preprocessed_cache_miss
 # test.c
 direct_cache_hit
 local_storage_hit
-local_storage_hit"
+local_storage_read_hit
+local_storage_read_hit"
 }