feat: Do clean/clear/evict-style operations per level 2 directory

Progress bars will now be smoother since the operations are now divided
into 256 instead of 16 "read files + act on files" steps. This is also
in preparation for future improvements related to cache cleanup.

6 files changed, 353 insertions, 384 deletions

diff --git a/src/storage/local/LocalStorage.cpp b/src/storage/local/LocalStorage.cpp
index 4c04e860..50f6b1c9 100644
--- a/src/storage/local/LocalStorage.cpp
+++ b/src/storage/local/LocalStorage.cpp
@@ -87,6 +87,37 @@ const uint8_t k_min_cache_levels = 2;
 // k_max_cache_files_per_directory.
 const uint8_t k_max_cache_levels = 4;
 
+namespace {
+
+struct SubdirCounters
+{
+  uint64_t files = 0;
+  uint64_t size = 0;
+  uint64_t cleanups_performed = 0;
+
+  SubdirCounters&
+  operator+=(const SubdirCounters& other)
+  {
+    files += other.files;
+    size += other.size;
+    cleanups_performed += other.cleanups_performed;
+    return *this;
+  }
+};
+
+} // namespace
+
+static void
+set_counters(const std::string& level_1_dir, const SubdirCounters& level_1_cs)
+{
+  const std::string stats_file = level_1_dir + "/stats";
+  StatsFile(stats_file).update([&](auto& cs) {
+    cs.increment(Statistic::cleanups_performed, level_1_cs.cleanups_performed);
+    cs.set(Statistic::files_in_cache, level_1_cs.files);
+    cs.set(Statistic::cache_size_kibibyte, level_1_cs.size / 1024);
+  });
+}
+
 static std::string
 suffix_from_type(const core::CacheEntryType type)
 {
@@ -114,6 +145,138 @@ calculate_wanted_cache_level(const uint64_t files_in_level_1)
   return k_max_cache_levels;
 }
 
+static void
+delete_file(const std::string& path,
+            const uint64_t size,
+            uint64_t* cache_size,
+            uint64_t* files_in_cache)
+{
+  const bool deleted = Util::unlink_safe(path, Util::UnlinkLog::ignore_failure);
+  if (!deleted && errno != ENOENT && errno != ESTALE) {
+    LOG("Failed to unlink {} ({})", path, strerror(errno));
+  } else if (cache_size && files_in_cache) {
+    // The counters are intentionally subtracted even if there was no file to
+    // delete since the final cache size calculation will be incorrect if they
+    // aren't. (This can happen when there are several parallel ongoing
+    // cleanups of the same directory.)
+    *cache_size -= size;
+    --*files_in_cache;
+  }
+}
+
+static SubdirCounters
+clean_dir(const std::string& subdir,
+          const uint64_t max_size,
+          const uint64_t max_files,
+          const std::optional<uint64_t> max_age,
+          const std::optional<std::string> namespace_,
+          const ProgressReceiver& progress_receiver)
+{
+  LOG("Cleaning up cache directory {}", subdir);
+
+  auto files = get_cache_dir_files(subdir);
+  progress_receiver(1.0 / 3);
+
+  uint64_t cache_size = 0;
+  uint64_t files_in_cache = 0;
+  auto current_time = util::TimePoint::now();
+  std::unordered_map<std::string /*result_file*/,
+                     std::vector<std::string> /*associated_raw_files*/>
+    raw_files_map;
+
+  for (size_t i = 0; i < files.size();
+       ++i, progress_receiver(1.0 / 3 + 1.0 * i / files.size() / 3)) {
+    const auto& file = files[i];
+
+    if (!file.is_regular()) {
+      // Not a file or missing file.
+      continue;
+    }
+
+    // Delete any tmp files older than 1 hour right away.
+    if (file.mtime() + util::Duration(3600) < current_time
+        && TemporaryFile::is_tmp_file(file.path())) {
+      Util::unlink_tmp(file.path());
+      continue;
+    }
+
+    if (namespace_ && file_type_from_path(file.path()) == FileType::raw) {
+      const auto result_filename =
+        FMT("{}R", file.path().substr(0, file.path().length() - 2));
+      raw_files_map[result_filename].push_back(file.path());
+    }
+
+    cache_size += file.size_on_disk();
+    files_in_cache += 1;
+  }
+
+  // Sort according to modification time, oldest first.
+  std::sort(files.begin(), files.end(), [](const auto& f1, const auto& f2) {
+    return f1.mtime() < f2.mtime();
+  });
+
+  LOG("Before cleanup: {:.0f} KiB, {:.0f} files",
+      static_cast<double>(cache_size) / 1024,
+      static_cast<double>(files_in_cache));
+
+  bool cleaned = false;
+  for (size_t i = 0; i < files.size();
+       ++i, progress_receiver(2.0 / 3 + 1.0 * i / files.size() / 3)) {
+    const auto& file = files[i];
+
+    if (!file || file.is_directory()) {
+      continue;
+    }
+
+    if ((max_size == 0 || cache_size <= max_size)
+        && (max_files == 0 || files_in_cache <= max_files)
+        && (!max_age
+            || file.mtime() > (current_time - util::Duration(*max_age)))
+        && (!namespace_ || max_age)) {
+      break;
+    }
+
+    if (namespace_) {
+      try {
+        core::CacheEntry::Header header(file.path());
+        if (header.namespace_ != *namespace_) {
+          continue;
+        }
+      } catch (core::Error&) {
+        // Failed to read header: ignore.
+        continue;
+      }
+
+      // For namespace eviction we need to remove raw files based on result
+      // filename since they don't have a header.
+      if (file_type_from_path(file.path()) == FileType::result) {
+        const auto entry = raw_files_map.find(file.path());
+        if (entry != raw_files_map.end()) {
+          for (const auto& raw_file : entry->second) {
+            delete_file(raw_file,
+                        Stat::lstat(raw_file).size_on_disk(),
+                        &cache_size,
+                        &files_in_cache);
+          }
+        }
+      }
+    }
+
+    delete_file(file.path(), file.size_on_disk(), &cache_size, &files_in_cache);
+    cleaned = true;
+  }
+
+  LOG("After cleanup: {:.0f} KiB, {:.0f} files",
+      static_cast<double>(cache_size) / 1024,
+      static_cast<double>(files_in_cache));
+
+  if (cleaned) {
+    LOG("Cleaned up cache directory {}", subdir);
+  }
+
+  return SubdirCounters{files_in_cache, cache_size, cleaned ? 1U : 0U};
+}
+
 FileType
 file_type_from_path(std::string_view path)
 {
@@ -204,13 +367,14 @@ LocalStorage::finalize()
   if (need_cleanup) {
     const double factor = m_config.limit_multiple() / 16;
     const uint64_t max_size = round(m_config.max_size() * factor);
-    const uint32_t max_files = round(m_config.max_files() * factor);
-    clean_dir(subdir,
-              max_size,
-              max_files,
-              std::nullopt,
-              std::nullopt,
-              [](double /*progress*/) {});
+    const uint64_t max_files = round(m_config.max_files() * factor);
+    auto level_1_cs = clean_dir(subdir,
+                                max_size,
+                                max_files,
+                                std::nullopt,
+                                std::nullopt,
+                                [](double /*progress*/) {});
+    set_counters(subdir, level_1_cs);
   }
 }
 
@@ -432,213 +596,86 @@ LocalStorage::get_all_statistics() const
   return std::make_pair(counters, last_updated);
 }
 
-static void
-delete_file(const std::string& path,
-            const uint64_t size,
-            uint64_t* cache_size,
-            uint64_t* files_in_cache)
-{
-  const bool deleted = Util::unlink_safe(path, Util::UnlinkLog::ignore_failure);
-  if (!deleted && errno != ENOENT && errno != ESTALE) {
-    LOG("Failed to unlink {} ({})", path, strerror(errno));
-  } else if (cache_size && files_in_cache) {
-    // The counters are intentionally subtracted even if there was no file to
-    // delete since the final cache size calculation will be incorrect if they
-    // aren't. (This can happen when there are several parallel ongoing
-    // cleanups of the same directory.)
-    *cache_size -= size;
-    --*files_in_cache;
-  }
-}
-
-static void
-update_counters(const std::string& dir,
-                const uint64_t files_in_cache,
-                const uint64_t cache_size,
-                const bool cleanup_performed)
-{
-  const std::string stats_file = dir + "/stats";
-  StatsFile(stats_file).update([=](auto& cs) {
-    if (cleanup_performed) {
-      cs.increment(Statistic::cleanups_performed);
-    }
-    cs.set(Statistic::files_in_cache, files_in_cache);
-    cs.set(Statistic::cache_size_kibibyte, cache_size / 1024);
-  });
-}
-
 void
 LocalStorage::evict(const ProgressReceiver& progress_receiver,
                     std::optional<uint64_t> max_age,
                     std::optional<std::string> namespace_)
 {
-  for_each_level_1_subdir(
+  for_each_cache_subdir(
     m_config.cache_dir(),
-    [&](const std::string& subdir,
-        const ProgressReceiver& sub_progress_receiver) {
-      clean_dir(subdir, 0, 0, max_age, namespace_, sub_progress_receiver);
-    },
-    progress_receiver);
-}
-
-// Clean up one cache subdirectory.
-void
-LocalStorage::clean_dir(const std::string& subdir,
-                        const uint64_t max_size,
-                        const uint64_t max_files,
-                        const std::optional<uint64_t> max_age,
-                        const std::optional<std::string> namespace_,
-                        const ProgressReceiver& progress_receiver)
-{
-  LOG("Cleaning up cache directory {}", subdir);
-
-  auto files = get_level_1_files(
-    subdir, [&](double progress) { progress_receiver(progress / 3); });
-
-  uint64_t cache_size = 0;
-  uint64_t files_in_cache = 0;
-  auto current_time = util::TimePoint::now();
-  std::unordered_map<std::string /*result_file*/,
-                     std::vector<std::string> /*associated_raw_files*/>
-    raw_files_map;
-
-  for (size_t i = 0; i < files.size();
-       ++i, progress_receiver(1.0 / 3 + 1.0 * i / files.size() / 3)) {
-    const auto& file = files[i];
-
-    if (!file.is_regular()) {
-      // Not a file or missing file.
-      continue;
-    }
-
-    // Delete any tmp files older than 1 hour right away.
-    if (file.mtime() + util::Duration(3600) < current_time
-        && TemporaryFile::is_tmp_file(file.path())) {
-      Util::unlink_tmp(file.path());
-      continue;
-    }
-
-    if (namespace_ && file_type_from_path(file.path()) == FileType::raw) {
-      const auto result_filename =
-        FMT("{}R", file.path().substr(0, file.path().length() - 2));
-      raw_files_map[result_filename].push_back(file.path());
-    }
-
-    cache_size += file.size_on_disk();
-    files_in_cache += 1;
-  }
-
-  // Sort according to modification time, oldest first.
-  std::sort(files.begin(), files.end(), [](const auto& f1, const auto& f2) {
-    return f1.mtime() < f2.mtime();
-  });
-
-  LOG("Before cleanup: {:.0f} KiB, {:.0f} files",
-      static_cast<double>(cache_size) / 1024,
-      static_cast<double>(files_in_cache));
-
-  bool cleaned = false;
-  for (size_t i = 0; i < files.size();
-       ++i, progress_receiver(2.0 / 3 + 1.0 * i / files.size() / 3)) {
-    const auto& file = files[i];
-
-    if (!file || file.is_directory()) {
-      continue;
-    }
-
-    if ((max_size == 0 || cache_size <= max_size)
-        && (max_files == 0 || files_in_cache <= max_files)
-        && (!max_age
-            || file.mtime() > (current_time - util::Duration(*max_age)))
-        && (!namespace_ || max_age)) {
-      break;
-    }
-
-    if (namespace_) {
-      try {
-        core::CacheEntry::Header header(file.path());
-        if (header.namespace_ != *namespace_) {
-          continue;
-        }
-      } catch (core::Error&) {
-        // Failed to read header: ignore.
-        continue;
-      }
-
-      // For namespace eviction we need to remove raw files based on result
-      // filename since they don't have a header.
-      if (file_type_from_path(file.path()) == FileType::result) {
-        const auto entry = raw_files_map.find(file.path());
-        if (entry != raw_files_map.end()) {
-          for (const auto& raw_file : entry->second) {
-            delete_file(raw_file,
-                        Stat::lstat(raw_file).size_on_disk(),
-                        &cache_size,
-                        &files_in_cache);
-          }
-        }
-      }
-    }
-
-    delete_file(file.path(), file.size_on_disk(), &cache_size, &files_in_cache);
-    cleaned = true;
-  }
-
-  LOG("After cleanup: {:.0f} KiB, {:.0f} files",
-      static_cast<double>(cache_size) / 1024,
-      static_cast<double>(files_in_cache));
-
-  if (cleaned) {
-    LOG("Cleaned up cache directory {}", subdir);
-  }
-
-  update_counters(subdir, files_in_cache, cache_size, cleaned);
+    progress_receiver,
+    [&](const auto& level_1_dir, const auto& level_1_progress_receiver) {
+      SubdirCounters counters;
+      for_each_cache_subdir(
+        level_1_dir,
+        level_1_progress_receiver,
+        [&](const std::string& level_2_dir,
+            const ProgressReceiver& level_2_progress_receiver) {
+          counters += clean_dir(
+            level_2_dir, 0, 0, max_age, namespace_, level_2_progress_receiver);
+        });
+
+      set_counters(level_1_dir, counters);
+    });
 }
 
 // Clean up all cache subdirectories.
 void
 LocalStorage::clean_all(const ProgressReceiver& progress_receiver)
 {
-  for_each_level_1_subdir(
+  for_each_cache_subdir(
     m_config.cache_dir(),
-    [&](const std::string& subdir,
-        const ProgressReceiver& sub_progress_receiver) {
-      clean_dir(subdir,
-                m_config.max_size() / 16,
-                m_config.max_files() / 16,
-                std::nullopt,
-                std::nullopt,
-                sub_progress_receiver);
-    },
-    progress_receiver);
-}
-
-// Wipe one cache subdirectory.
-static void
-wipe_dir(const std::string& subdir, const ProgressReceiver& progress_receiver)
-{
-  LOG("Clearing out cache directory {}", subdir);
-
-  const auto files = get_level_1_files(
-    subdir, [&](double progress) { progress_receiver(progress / 2); });
-
-  for (size_t i = 0; i < files.size(); ++i) {
-    Util::unlink_safe(files[i].path());
-    progress_receiver(0.5 + 0.5 * i / files.size());
-  }
-
-  const bool cleared = !files.empty();
-  if (cleared) {
-    LOG("Cleared out cache directory {}", subdir);
-  }
-  update_counters(subdir, 0, 0, cleared);
+    progress_receiver,
+    [&](const auto& level_1_dir, const auto& level_1_progress_receiver) {
+      SubdirCounters counters;
+      for_each_cache_subdir(
+        level_1_dir,
+        level_1_progress_receiver,
+        [&](const std::string& level_2_dir,
+            const ProgressReceiver& level_2_progress_receiver) {
+          counters += clean_dir(level_2_dir,
+                                m_config.max_size() / 256,
+                                m_config.max_files() / 16,
+                                std::nullopt,
+                                std::nullopt,
+                                level_2_progress_receiver);
+        });
+      set_counters(level_1_dir, counters);
+    });
 }
 
 // Wipe all cached files in all subdirectories.
 void
 LocalStorage::wipe_all(const ProgressReceiver& progress_receiver)
 {
-  for_each_level_1_subdir(m_config.cache_dir(), wipe_dir, progress_receiver);
+  for_each_cache_subdir(
+    m_config.cache_dir(),
+    progress_receiver,
+    [&](const auto& level_1_dir, const auto& level_1_progress_receiver) {
+      uint64_t cleanups = 0;
+      for_each_cache_subdir(
+        level_1_dir,
+        level_1_progress_receiver,
+        [&](const std::string& level_2_dir,
+            const ProgressReceiver& level_2_progress_receiver) {
+          LOG("Clearing out cache directory {}", level_2_dir);
+
+          const auto files = get_cache_dir_files(level_2_dir);
+          level_2_progress_receiver(0.5);
+
+          for (size_t i = 0; i < files.size(); ++i) {
+            Util::unlink_safe(files[i].path());
+            level_2_progress_receiver(0.5 + 0.5 * i / files.size());
+          }
+
+          if (!files.empty()) {
+            ++cleanups;
+            LOG("Cleared out cache directory {}", level_2_dir);
+          }
+        });
+
+      set_counters(level_1_dir, SubdirCounters{0, 0, cleanups});
+    });
 }
 
 CompressionStatistics
@@ -647,28 +684,31 @@ LocalStorage::get_compression_statistics(
 {
   CompressionStatistics cs{};
 
-  for_each_level_1_subdir(
+  for_each_cache_subdir(
     m_config.cache_dir(),
-    [&](const auto& subdir, const auto& sub_progress_receiver) {
-      const auto files = get_level_1_files(
-        subdir, [&](double progress) { sub_progress_receiver(progress / 2); });
-
-      for (size_t i = 0; i < files.size(); ++i) {
-        const auto& cache_file = files[i];
-        cs.on_disk_size += cache_file.size_on_disk();
-
-        try {
-          core::CacheEntry::Header header(cache_file.path());
-          cs.compr_size += cache_file.size();
-          cs.content_size += header.entry_size;
-        } catch (core::Error&) {
-          cs.incompr_size += cache_file.size();
-        }
-
-        sub_progress_receiver(1.0 / 2 + 1.0 * i / files.size() / 2);
-      }
-    },
-    progress_receiver);
+    progress_receiver,
+    [&](const auto& level_1_dir, const auto& level_1_progress_receiver) {
+      for_each_cache_subdir(
+        level_1_dir,
+        level_1_progress_receiver,
+        [&](const auto& level_2_dir, const auto& level_2_progress_receiver) {
+          const auto files = get_cache_dir_files(level_2_dir);
+          level_2_progress_receiver(0.2);
+
+          for (size_t i = 0; i < files.size(); ++i) {
+            const auto& cache_file = files[i];
+            cs.on_disk_size += cache_file.size_on_disk();
+            try {
+              core::CacheEntry::Header header(cache_file.path());
+              cs.compr_size += cache_file.size();
+              cs.content_size += header.entry_size;
+            } catch (core::Error&) {
+              cs.incompr_size += cache_file.size();
+            }
+            level_2_progress_receiver(0.2 + 0.8 * i / files.size());
+          }
+        });
+    });
 
   return cs;
 }
@@ -685,53 +725,59 @@ LocalStorage::recompress(const std::optional<int8_t> level,
 
   std::atomic<uint64_t> incompressible_size = 0;
 
-  for_each_level_1_subdir(
+  for_each_cache_subdir(
     m_config.cache_dir(),
-    [&](const auto& subdir, const auto& sub_progress_receiver) {
-      auto files = get_level_1_files(subdir, [&](double progress) {
-        sub_progress_receiver(0.1 * progress);
-      });
-
-      auto stats_file = subdir + "/stats";
-
-      for (size_t i = 0; i < files.size(); ++i) {
-        const auto& file = files[i];
-
-        if (file_type_from_path(file.path()) != FileType::unknown) {
-          thread_pool.enqueue(
-            [&recompressor, &incompressible_size, level, stats_file, file] {
-              try {
-                Stat new_stat = recompressor.recompress(
-                  file, level, core::FileRecompressor::KeepAtime::no);
-                auto size_change_kibibyte =
-                  Util::size_change_kibibyte(file, new_stat);
-                if (size_change_kibibyte != 0) {
-                  StatsFile(stats_file).update([=](auto& cs) {
-                    cs.increment(core::Statistic::cache_size_kibibyte,
-                                 size_change_kibibyte);
-                  });
-                }
-              } catch (core::Error&) {
-                // Ignore for now.
-                incompressible_size += file.size_on_disk();
-              }
-            });
-        } else if (!TemporaryFile::is_tmp_file(file.path())) {
-          incompressible_size += file.size_on_disk();
-        }
-
-        sub_progress_receiver(0.1 + 0.9 * i / files.size());
-      }
+    progress_receiver,
+    [&](const auto& level_1_dir, const auto& level_1_progress_receiver) {
+      for_each_cache_subdir(
+        level_1_dir,
+        level_1_progress_receiver,
+        [&](const auto& level_2_dir, const auto& level_2_progress_receiver) {
+          (void)level_2_progress_receiver;
+          (void)level_2_dir;
+          auto files = get_cache_dir_files(level_2_dir);
+          level_2_progress_receiver(0.1);
+
+          auto stats_file = FMT("{}/stats", Util::dir_name(level_1_dir));
+
+          for (size_t i = 0; i < files.size(); ++i) {
+            const auto& file = files[i];
+
+            if (file_type_from_path(file.path()) != FileType::unknown) {
+              thread_pool.enqueue(
+                [&recompressor, &incompressible_size, level, stats_file, file] {
+                  try {
+                    Stat new_stat = recompressor.recompress(
+                      file, level, core::FileRecompressor::KeepAtime::no);
+                    auto size_change_kibibyte =
+                      Util::size_change_kibibyte(file, new_stat);
+                    if (size_change_kibibyte != 0) {
+                      StatsFile(stats_file).update([=](auto& cs) {
+                        cs.increment(core::Statistic::cache_size_kibibyte,
+                                     size_change_kibibyte);
+                      });
+                    }
+                  } catch (core::Error&) {
+                    // Ignore for now.
+                    incompressible_size += file.size_on_disk();
+                  }
+                });
+            } else if (!TemporaryFile::is_tmp_file(file.path())) {
+              incompressible_size += file.size_on_disk();
+            }
+
+            level_2_progress_receiver(0.1 + 0.9 * i / files.size());
+          }
 
-      if (util::ends_with(subdir, "f")) {
-        // Wait here instead of after for_each_level_1_subdir to avoid
-        // updating the progress bar to 100% before all work is done.
-        thread_pool.shut_down();
-      }
-    },
-    progress_receiver);
+          if (util::ends_with(level_2_dir, "f/f")) {
+            // Wait here instead of after for_each_cache_subdir to avoid
+            // updating the progress bar to 100% before all work is done.
+            thread_pool.shut_down();
+          }
+        });
+    });
 
-  // In case there was no f subdir, shut down the thread pool now.
+  // In case there was no f/f subdir, shut down the thread pool now.
   thread_pool.shut_down();
 
   if (isatty(STDOUT_FILENO)) {
diff --git a/src/storage/local/LocalStorage.hpp b/src/storage/local/LocalStorage.hpp
index 7ba28707..e79e6478 100644
--- a/src/storage/local/LocalStorage.hpp
+++ b/src/storage/local/LocalStorage.hpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2021-2022 Joel Rosdahl and other contributors
+// Copyright (C) 2021-2023 Joel Rosdahl and other contributors
 //
 // See doc/AUTHORS.adoc for a complete list of contributors.
 //
@@ -156,13 +156,6 @@ private:
   // it. Additionally, `level` single-character, '/'-separated subpaths are
   // split from the beginning of `name` before joining them all.
   std::string get_path_in_cache(uint8_t level, std::string_view name) const;
-
-  static void clean_dir(const std::string& subdir,
-                        uint64_t max_size,
-                        uint64_t max_files,
-                        std::optional<uint64_t> max_age,
-                        std::optional<std::string> namespace_,
-                        const ProgressReceiver& progress_receiver);
 };
 
 // --- Inline implementations ---
diff --git a/src/storage/local/util.cpp b/src/storage/local/util.cpp
index 457c312a..8f2c7eca 100644
--- a/src/storage/local/util.cpp
+++ b/src/storage/local/util.cpp
@@ -25,12 +25,12 @@
 namespace storage::local {
 
 void
-for_each_level_1_subdir(const std::string& cache_dir,
-                        const SubdirVisitor& visitor,
-                        const ProgressReceiver& progress_receiver)
+for_each_cache_subdir(const std::string& cache_dir,
+                      const ProgressReceiver& progress_receiver,
+                      const SubdirVisitor& visitor)
 {
-  for (int i = 0; i <= 0xF; i++) {
-    double progress = 1.0 * i / 16;
+  for (uint8_t i = 0; i < 16; ++i) {
+    double progress = i / 16.0;
     progress_receiver(progress);
     std::string subdir_path = FMT("{}/{:x}", cache_dir, i);
     visitor(subdir_path, [&](double inner_progress) {
@@ -54,8 +54,7 @@ for_each_level_1_and_2_stats_file(
 }
 
 std::vector<Stat>
-get_level_1_files(const std::string& dir,
-                  const ProgressReceiver& progress_receiver)
+get_cache_dir_files(const std::string& dir)
 {
   std::vector<Stat> files;
 
@@ -63,8 +62,6 @@ get_level_1_files(const std::string& dir,
     return files;
   }
 
-  size_t level_2_directories = 0;
-
   Util::traverse(dir, [&](const std::string& path, bool is_dir) {
     auto name = Util::base_name(path);
     if (name == "CACHEDIR.TAG" || name == "stats"
@@ -74,14 +71,9 @@ get_level_1_files(const std::string& dir,
 
     if (!is_dir) {
       files.emplace_back(Stat::lstat(path));
-    } else if (path != dir
-               && path.find('/', dir.size() + 1) == std::string::npos) {
-      ++level_2_directories;
-      progress_receiver(level_2_directories / 16.0);
     }
   });
 
-  progress_receiver(1.0);
   return files;
 }
 
diff --git a/src/storage/local/util.hpp b/src/storage/local/util.hpp
index dd441b5b..084d7255 100644
--- a/src/storage/local/util.hpp
+++ b/src/storage/local/util.hpp
@@ -30,22 +30,16 @@ using ProgressReceiver = std::function<void(double progress)>;
 using SubdirVisitor = std::function<void(
   const std::string& dir_path, const ProgressReceiver& progress_receiver)>;
 
-// Call a function for each subdir (0-9a-f) in the cache.
-//
-// Parameters:
-// - cache_dir: Path to the cache directory.
-// - visitor: Function to call with directory path and progress_receiver as
-//   arguments.
-// - progress_receiver: Function that will be called for progress updates.
-void for_each_level_1_subdir(const std::string& cache_dir,
-                             const SubdirVisitor& visitor,
-                             const ProgressReceiver& progress_receiver);
+// Call `visitor` for each subdirectory (0-9a-f) in `cache_dir`.
+void for_each_cache_subdir(const std::string& cache_dir,
+                           const ProgressReceiver& progress_receiver,
+                           const SubdirVisitor& visitor);
 
 void for_each_level_1_and_2_stats_file(
   const std::string& cache_dir,
   const std::function<void(const std::string& path)> function);
 
-// Get a list of files in a level 1 subdirectory of the cache.
+// Get a list of files in a subdirectory of the cache.
 //
 // The function works under the assumption that directory entries with one
 // character names (except ".") are subdirectories and that there are no other
@@ -55,11 +49,6 @@ void for_each_level_1_and_2_stats_file(
 // - CACHEDIR.TAG
 // - stats
 // - .nfs* (temporary NFS files that may be left for open but deleted files).
-//
-// Parameters:
-// - dir: The directory to traverse recursively.
-// - progress_receiver: Function that will be called for progress updates.
-std::vector<Stat> get_level_1_files(const std::string& dir,
-                                    const ProgressReceiver& progress_receiver);
+std::vector<Stat> get_cache_dir_files(const std::string& dir);
 
 } // namespace storage::local
diff --git a/test/suites/cleanup.bash b/test/suites/cleanup.bash
index 18c2055f..a765bfca 100644
--- a/test/suites/cleanup.bash
+++ b/test/suites/cleanup.bash
@@ -1,23 +1,22 @@
-prepare_cleanup_test_dir() {
-    local dir=$1
+SUITE_cleanup_SETUP() {
+    local l1_dir=$CCACHE_DIR/a
+    local l2_dir=$l1_dir/b
     local i
 
-    rm -rf $dir
-    mkdir -p $dir
+    rm -rf $l2_dir
+    mkdir -p $l2_dir
     for ((i = 0; i < 10; ++i)); do
-        printf 'A%.0s' {1..4017} >$dir/result${i}R
-        backdate $((3 * i + 1)) $dir/result${i}R
+        printf 'A%.0s' {1..4017} >$l2_dir/result${i}R
+        backdate $((3 * i + 1)) $l2_dir/result${i}R
     done
     # NUMFILES: 10, TOTALSIZE: 13 KiB, MAXFILES: 0, MAXSIZE: 0
-    echo "0 0 0 0 0 0 0 0 0 0 0 10 13 0 0" >$dir/stats
+    echo "0 0 0 0 0 0 0 0 0 0 0 10 13 0 0" >$l1_dir/stats
 }
 
 SUITE_cleanup() {
     # -------------------------------------------------------------------------
     TEST "Clear cache"
 
-    prepare_cleanup_test_dir $CCACHE_DIR/a
-
     $CCACHE -C >/dev/null
     expect_file_count 0 '*R' $CCACHE_DIR
     expect_stat files_in_cache 0
@@ -26,8 +25,6 @@ SUITE_cleanup() {
     # -------------------------------------------------------------------------
     TEST "Forced cache cleanup, no limits"
 
-    prepare_cleanup_test_dir $CCACHE_DIR/a
-
     $CCACHE -F 0 -M 0 >/dev/null
     $CCACHE -c >/dev/null
     expect_file_count 10 '*R' $CCACHE_DIR
@@ -37,8 +34,6 @@ SUITE_cleanup() {
     # -------------------------------------------------------------------------
     TEST "Forced cache cleanup, file limit"
 
-    prepare_cleanup_test_dir $CCACHE_DIR/a
-
     # No cleanup needed.
     #
     # 10 * 16 = 160
@@ -57,11 +52,11 @@ SUITE_cleanup() {
     expect_stat files_in_cache 7
     expect_stat cleanups_performed 1
     for i in 0 1 2; do
-        file=$CCACHE_DIR/a/result${i}R
+        file=$CCACHE_DIR/a/b/result${i}R
         expect_missing $CCACHE_DIR/a/result${i}R
     done
     for i in 3 4 5 6 7 8 9; do
-        file=$CCACHE_DIR/a/result${i}R
+        file=$CCACHE_DIR/a/b/result${i}R
         expect_exists $file
     done
 
@@ -76,105 +71,61 @@ SUITE_cleanup() {
         #     cd /tmp
         #     CCACHE=$DIR/ccache $DIR/test.sh
 
-        prepare_cleanup_test_dir $CCACHE_DIR/a
-
-        $CCACHE -F 0 -M 256K >/dev/null
+        $CCACHE -F 0 -M 4096K >/dev/null
         $CCACHE -c >/dev/null
         expect_file_count 3 '*R' $CCACHE_DIR
         expect_stat files_in_cache 3
         expect_stat cleanups_performed 1
         for i in 0 1 2 3 4 5 6; do
-            file=$CCACHE_DIR/a/result${i}R
+            file=$CCACHE_DIR/a/b/result${i}R
             expect_missing $file
         done
         for i in 7 8 9; do
-            file=$CCACHE_DIR/a/result${i}R
+            file=$CCACHE_DIR/a/b/result${i}R
             expect_exists $file
         done
     fi
-    # -------------------------------------------------------------------------
-    TEST "Automatic cache cleanup, limit_multiple 0.9"
-
-    for x in 0 1 2 3 4 5 6 7 8 9 a b c d e f; do
-        prepare_cleanup_test_dir $CCACHE_DIR/$x
-    done
-
-    $CCACHE -F 160 -M 0 >/dev/null
-
-    expect_file_count 160 '*R' $CCACHE_DIR
-    expect_stat files_in_cache 160
-    expect_stat cleanups_performed 0
-
-    touch empty.c
-    CCACHE_LIMIT_MULTIPLE=0.9 $CCACHE_COMPILE -c empty.c -o empty.o
-    expect_file_count 159 '*R' $CCACHE_DIR
-    expect_stat files_in_cache 159
-    expect_stat cleanups_performed 1
-
-    # -------------------------------------------------------------------------
-    TEST "Automatic cache cleanup, limit_multiple 0.7"
-
-    for x in 0 1 2 3 4 5 6 7 8 9 a b c d e f; do
-        prepare_cleanup_test_dir $CCACHE_DIR/$x
-    done
-
-    $CCACHE -F 160 -M 0 >/dev/null
-
-    expect_file_count 160 '*R' $CCACHE_DIR
-    expect_stat files_in_cache 160
-    expect_stat cleanups_performed 0
-
-    touch empty.c
-    CCACHE_LIMIT_MULTIPLE=0.7 $CCACHE_COMPILE -c empty.c -o empty.o
-    expect_file_count 157 '*R' $CCACHE_DIR
-    expect_stat files_in_cache 157
-    expect_stat cleanups_performed 1
 
     # -------------------------------------------------------------------------
     TEST "No cleanup of new unknown file"
 
-    prepare_cleanup_test_dir $CCACHE_DIR/a
-
-    touch $CCACHE_DIR/a/abcd.unknown
+    touch $CCACHE_DIR/a/b/abcd.unknown
     $CCACHE -F 0 -M 0 -c >/dev/null # update counters
     expect_stat files_in_cache 11
 
     $CCACHE -F 160 -M 0 >/dev/null
     $CCACHE -c >/dev/null
-    expect_exists $CCACHE_DIR/a/abcd.unknown
+    expect_exists $CCACHE_DIR/a/b/abcd.unknown
     expect_stat files_in_cache 10
 
     # -------------------------------------------------------------------------
     TEST "Cleanup of old unknown file"
 
-    prepare_cleanup_test_dir $CCACHE_DIR/a
     $CCACHE -F 160 -M 0 >/dev/null
-    touch $CCACHE_DIR/a/abcd.unknown
-    backdate $CCACHE_DIR/a/abcd.unknown
+    touch $CCACHE_DIR/a/b/abcd.unknown
+    backdate $CCACHE_DIR/a/b/abcd.unknown
     $CCACHE -F 0 -M 0 -c >/dev/null # update counters
     expect_stat files_in_cache 11
 
     $CCACHE -F 160 -M 0 -c >/dev/null
-    expect_missing $CCACHE_DIR/a/abcd.unknown
+    expect_missing $CCACHE_DIR/a/b/abcd.unknown
     expect_stat files_in_cache 10
 
     # -------------------------------------------------------------------------
     TEST "Cleanup of tmp file"
 
     mkdir -p $CCACHE_DIR/a
-    touch $CCACHE_DIR/a/abcd.tmp.efgh
+    touch $CCACHE_DIR/a/b/abcd.tmp.efgh
     $CCACHE -c >/dev/null # update counters
-    expect_stat files_in_cache 1
-    backdate $CCACHE_DIR/a/abcd.tmp.efgh
+    expect_stat files_in_cache 11
+    backdate $CCACHE_DIR/a/b/abcd.tmp.efgh
     $CCACHE -c >/dev/null
-    expect_missing $CCACHE_DIR/a/abcd.tmp.efgh
-    expect_stat files_in_cache 0
+    expect_missing $CCACHE_DIR/a/b/abcd.tmp.efgh
+    expect_stat files_in_cache 10
 
     # -------------------------------------------------------------------------
     TEST "No cleanup of .nfs* files"
 
-    prepare_cleanup_test_dir $CCACHE_DIR/a
-
     touch $CCACHE_DIR/a/.nfs0123456789
     $CCACHE -F 0 -M 0 >/dev/null
     $CCACHE -c >/dev/null
@@ -184,8 +135,7 @@ SUITE_cleanup() {
     # -------------------------------------------------------------------------
     TEST "Cleanup of old files by age"
 
-    prepare_cleanup_test_dir $CCACHE_DIR/a
-    touch $CCACHE_DIR/a/nowR
+    touch $CCACHE_DIR/a/b/nowR
     $CCACHE -F 0 -M 0 >/dev/null
 
     $CCACHE --evict-older-than 1d >/dev/null
@@ -196,7 +146,7 @@ SUITE_cleanup() {
     expect_file_count 1 '*R' $CCACHE_DIR
     expect_stat files_in_cache 1
 
-    backdate $CCACHE_DIR/a/nowR
+    backdate $CCACHE_DIR/a/b/nowR
     $CCACHE --evict-older-than 10s  >/dev/null
     expect_stat files_in_cache 0
 }
diff --git a/unittest/test_storage_local_util.cpp b/unittest/test_storage_local_util.cpp
index efc5ff46..d9c86d18 100644
--- a/unittest/test_storage_local_util.cpp
+++ b/unittest/test_storage_local_util.cpp
@@ -19,6 +19,7 @@
 #include "TestUtil.hpp"
 
 #include <Util.hpp>
+#include <fmtmacros.hpp>
 #include <storage/local/util.hpp>
 #include <util/file.hpp>
 
@@ -41,13 +42,13 @@ os_path(std::string path)
 
 TEST_SUITE_BEGIN("storage::local::util");
 
-TEST_CASE("storage::local::for_each_level_1_subdir")
+TEST_CASE("storage::local::for_each_cache_subdir")
 {
   std::vector<std::string> actual;
-  storage::local::for_each_level_1_subdir(
+  storage::local::for_each_cache_subdir(
     "cache_dir",
-    [&](const auto& subdir, const auto&) { actual.push_back(subdir); },
-    [](double) {});
+    [](double) {},
+    [&](const auto& subdir, const auto&) { actual.push_back(subdir); });
 
   std::vector<std::string> expected = {
     "cache_dir/0",
@@ -70,7 +71,7 @@ TEST_CASE("storage::local::for_each_level_1_subdir")
   CHECK(actual == expected);
 }
 
-TEST_CASE("storage::local::get_level_1_files")
+TEST_CASE("storage::local::get_cache_dir_files")
 {
   TestContext test_context;
 
@@ -83,23 +84,21 @@ TEST_CASE("storage::local::get_level_1_files")
   util::write_file("0/1/file_c", "12");
   util::write_file("0/f/c/file_d", "123");
 
-  auto null_receiver = [](double) {};
-
   SUBCASE("nonexistent subdirectory")
   {
-    const auto files = storage::local::get_level_1_files("2", null_receiver);
+    const auto files = storage::local::get_cache_dir_files("2");
     CHECK(files.empty());
   }
 
   SUBCASE("empty subdirectory")
   {
-    const auto files = storage::local::get_level_1_files("e", null_receiver);
+    const auto files = storage::local::get_cache_dir_files("e");
     CHECK(files.empty());
   }
 
   SUBCASE("simple case")
   {
-    auto files = storage::local::get_level_1_files("0", null_receiver);
+    auto files = storage::local::get_cache_dir_files("0");
     REQUIRE(files.size() == 4);
 
     // Files within a level are in arbitrary order, sort them to be able to