summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJoel Rosdahl <joel@rosdahl.net>2022-09-25 20:28:40 +0200
committerJoel Rosdahl <joel@rosdahl.net>2022-09-27 22:33:42 +0200
commit0cd6f70b8d6d74427d6a60958a3bdc1ba0127a2e (patch)
tree2731f23c79cbf5210b4a0c333335b4ab824b3155
parenta2e7f91070a46a31914dae6c2ba9815af5fab823 (diff)
downloadccache-0cd6f70b8d6d74427d6a60958a3bdc1ba0127a2e.tar.gz
chore: Rename primary/secondary storage to local/remote storage
There is a feature request to be able not to use a local cache at all, only a network cache. With such a feature, the names "primary storage" and "secondary storage" make less sense since ccache would be operating in "secondary only" mode, but then that storage would of course become the primary (and only). Let's rename "primary storage" to "local storage" and "secondary storage" to "remote storage" – operating in "remote only" mode then makes sense. One of the original motivations to call networked storage "secondary" is that the file storage can be used for local file systems as well, making such storage "not quite remote", but in practice I guess the file storage backend used primarily for network file systems.
-rw-r--r--ARCHITECTURE.md4
-rw-r--r--CMakeLists.txt2
-rw-r--r--doc/MANUAL.adoc98
-rwxr-xr-xmisc/download-redis2
-rwxr-xr-xmisc/upload-redis3
-rw-r--r--src/Config.cpp128
-rw-r--r--src/Config.hpp8
-rw-r--r--src/Context.hpp2
-rw-r--r--src/ccache.cpp28
-rw-r--r--src/core/CacheEntry.hpp2
-rw-r--r--src/core/Result.hpp2
-rw-r--r--src/core/ResultExtractor.hpp2
-rw-r--r--src/core/ResultRetriever.cpp2
-rw-r--r--src/core/ResultRetriever.hpp2
-rw-r--r--src/core/Statistic.hpp12
-rw-r--r--src/core/Statistics.cpp56
-rw-r--r--src/core/mainoptions.cpp30
-rw-r--r--src/storage/CMakeLists.txt4
-rw-r--r--src/storage/Storage.cpp158
-rw-r--r--src/storage/Storage.hpp44
-rw-r--r--src/storage/local/CMakeLists.txt (renamed from src/storage/primary/CMakeLists.txt)8
-rw-r--r--src/storage/local/CacheFile.cpp (renamed from src/storage/primary/CacheFile.cpp)0
-rw-r--r--src/storage/local/CacheFile.hpp (renamed from src/storage/primary/CacheFile.hpp)0
-rw-r--r--src/storage/local/LocalStorage.cpp (renamed from src/storage/primary/PrimaryStorage.cpp)78
-rw-r--r--src/storage/local/LocalStorage.hpp (renamed from src/storage/primary/PrimaryStorage.hpp)14
-rw-r--r--src/storage/local/LocalStorage_cleanup.cpp (renamed from src/storage/primary/PrimaryStorage_cleanup.cpp)34
-rw-r--r--src/storage/local/LocalStorage_compress.cpp (renamed from src/storage/primary/PrimaryStorage_compress.cpp)14
-rw-r--r--src/storage/local/LocalStorage_statistics.cpp (renamed from src/storage/primary/PrimaryStorage_statistics.cpp)14
-rw-r--r--src/storage/local/StatsFile.cpp (renamed from src/storage/primary/StatsFile.cpp)4
-rw-r--r--src/storage/local/StatsFile.hpp (renamed from src/storage/primary/StatsFile.hpp)4
-rw-r--r--src/storage/local/util.cpp (renamed from src/storage/primary/util.cpp)4
-rw-r--r--src/storage/local/util.hpp (renamed from src/storage/primary/util.hpp)10
-rw-r--r--src/storage/remote/CMakeLists.txt (renamed from src/storage/secondary/CMakeLists.txt)2
-rw-r--r--src/storage/remote/FileStorage.cpp (renamed from src/storage/secondary/FileStorage.cpp)14
-rw-r--r--src/storage/remote/FileStorage.hpp (renamed from src/storage/secondary/FileStorage.hpp)12
-rw-r--r--src/storage/remote/HttpStorage.cpp (renamed from src/storage/secondary/HttpStorage.cpp)14
-rw-r--r--src/storage/remote/HttpStorage.hpp (renamed from src/storage/secondary/HttpStorage.hpp)8
-rw-r--r--src/storage/remote/RedisStorage.cpp (renamed from src/storage/secondary/RedisStorage.cpp)18
-rw-r--r--src/storage/remote/RedisStorage.hpp (renamed from src/storage/secondary/RedisStorage.hpp)8
-rw-r--r--src/storage/remote/RemoteStorage.cpp (renamed from src/storage/secondary/SecondaryStorage.cpp)10
-rw-r--r--src/storage/remote/RemoteStorage.hpp (renamed from src/storage/secondary/SecondaryStorage.hpp)25
-rw-r--r--test/CMakeLists.txt10
-rw-r--r--test/suites/base.bash16
-rw-r--r--test/suites/direct.bash16
-rw-r--r--test/suites/remote_file.bash422
-rw-r--r--test/suites/remote_http.bash (renamed from test/suites/secondary_http.bash)96
-rw-r--r--test/suites/remote_redis.bash (renamed from test/suites/secondary_redis.bash)18
-rw-r--r--test/suites/remote_redis_unix.bash (renamed from test/suites/secondary_redis_unix.bash)18
-rw-r--r--test/suites/remote_url.bash (renamed from test/suites/secondary_url.bash)16
-rw-r--r--test/suites/secondary_file.bash422
-rw-r--r--test/suites/stats_log.bash16
-rw-r--r--test/suites/trim_dir.bash40
-rw-r--r--unittest/CMakeLists.txt4
-rw-r--r--unittest/test_Config.cpp5
-rw-r--r--unittest/test_storage_local_StatsFile.cpp (renamed from unittest/test_storage_primary_StatsFile.cpp)6
-rw-r--r--unittest/test_storage_local_util.cpp (renamed from unittest/test_storage_primary_util.cpp)16
56 files changed, 1007 insertions, 998 deletions
diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md
index 25ed9a20..26ecdd84 100644
--- a/ARCHITECTURE.md
+++ b/ARCHITECTURE.md
@@ -27,8 +27,8 @@ to understand and work with. In other words, this is work in progress.
etc.
* `core`: Everything not part of other directories.
* `storage`: Storage backends.
-* `storage/primary`: Code for the primary storage backend.
-* `storage/secondary`: Code for secondary storage backends.
+* `storage/local`: Code for the local storage backend.
+* `storage/remote`: Code for remote storage backends.
* `third_party`: Bundled third party code.
* `util`: Generic utility functionality that does not depend on ccache-specific
things.
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 30b18a51..6407690e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -119,7 +119,7 @@ option(HIREDIS_FROM_INTERNET
find_package(zstd 1.1.2 MODULE REQUIRED)
-option(REDIS_STORAGE_BACKEND "Enable Redis secondary storage" ON)
+option(REDIS_STORAGE_BACKEND "Enable Redis remote storage" ON)
if(REDIS_STORAGE_BACKEND)
find_package(hiredis 0.13.3 MODULE REQUIRED)
diff --git a/doc/MANUAL.adoc b/doc/MANUAL.adoc
index 10aa9fea..0e59898f 100644
--- a/doc/MANUAL.adoc
+++ b/doc/MANUAL.adoc
@@ -187,15 +187,15 @@ documentation.
Zero the cache statistics (but not the configuration options).
-=== Options for secondary storage
+=== Options for remote file-based storage
*--trim-dir* _PATH_::
Remove old files from directory _PATH_ until it is at most the size specified
by `--trim-max-size`.
+
-WARNING: Don't use this option to trim the primary cache. To trim the primary
-cache directory to a certain size, use `CCACHE_MAXSIZE=_SIZE_ ccache -c`.
+WARNING: Don't use this option to trim the local cache. To trim the local cache
+directory to a certain size, use `CCACHE_MAXSIZE=_SIZE_ ccache -c`.
*--trim-max-size* _SIZE_::
@@ -814,11 +814,11 @@ file in `/etc/rsyslog.d`:
If set, the namespace string will be added to the hashed data for each
compilation. This will make the associated cache entries logically separate
from cache entries with other namespaces, but they will still share the same
- storage space. Cache entries can also be selectively removed from the
- primary cache with the command line option `--evict-namespace`, potentially
- in combination with `--evict-older-than`.
+ storage space. Cache entries can also be selectively removed from the local
+ cache with the command line option `--evict-namespace`, potentially in
+ combination with `--evict-older-than`.
+
-For instance, if you use the same primary cache for several disparate projects,
+For instance, if you use the same local cache for several disparate projects,
you can use a unique namespace string for each one. This allows you to remove
cache entries that belong to a certain project if you stop working with that
project.
@@ -881,11 +881,28 @@ temporary files otherwise. You may also want to set <<config_stats,*stats*>> to
If true, ccache will not use any previously stored result. New results will
still be cached, possibly overwriting any pre-existing results.
+[#config_remote_storage]
+*remote_storage* (*CCACHE_REMOTE_STORAGE*)::
+
+ This option specifies one or several storage backends (separated by space)
+ to query after checking the local cache. See _<<Remote storage backends>>_
+ for documentation of syntax and available backends.
++
+Examples:
++
+* `+file:/shared/nfs/directory+`
+* `+file:///shared/nfs/one|read-only file:///shared/nfs/two+`
+* `+http://example.com/cache+`
+* `+redis://example.com+`
++
+NOTE: In previous ccache versions this option was called *secondary_storage*
+(*CCACHE_SECONDARY_STORAGE*), which can still be used as an alias.
+
[#config_reshare]
*reshare* (*CCACHE_RESHARE* or *CCACHE_NORESHARE*, see _<<Boolean values>>_ above)::
- If true, ccache will write results to secondary storage even for primary
- storage cache hits. The default is false.
+ If true, ccache will write results to remote storage even for local storage
+ cache hits. The default is false.
[#config_run_second_cpp]
*run_second_cpp* (*CCACHE_CPP2* or *CCACHE_NOCPP2*, see _<<Boolean values>>_ above)::
@@ -912,21 +929,6 @@ still has to do _some_ preprocessing (like macros).
This option is ignored with MSVC, as there is no way to make it compile without
preprocessing first.
-[#config_secondary_storage]
-*secondary_storage* (*CCACHE_SECONDARY_STORAGE*)::
-
- This option specifies one or several storage backends (separated by space)
- to query after the primary cache storage. See
- _<<Secondary storage backends>>_ for documentation of syntax and available
- backends.
-+
-Examples:
-+
-* `+file:/shared/nfs/directory+`
-* `+file:///shared/nfs/one|read-only file:///shared/nfs/two+`
-* `+http://example.com/cache+`
-* `+redis://example.com+`
-
[#config_sloppiness]
*sloppiness* (*CCACHE_SLOPPINESS*)::
@@ -1029,23 +1031,23 @@ filesystem as the `CCACHE_DIR` path, but this requirement has been relaxed.
cache with other users.
-== Secondary storage backends
+== Remote storage backends
-The <<config_secondary_storage,*secondary_storage*>> option lets you configure
-ccache to use one or several other storage backends in addition to the primary
-cache storage located in <<config_cache_dir,*cache_dir*>>. Note that cache
-statistics counters will still be kept in the primary cache directory --
-secondary storage backends only store cache results and manifests.
+The <<config_remote_storage,*remote_storage*>> option lets you configure ccache
+to use one or several other storage backends in addition to the local cache
+storage located in <<config_cache_dir,*cache_dir*>>. Note that cache statistics
+counters will still be kept in the local cache directory -- remote storage
+backends only store compilation results and manifests.
-A secondary storage backend is specified with a URL, optionally followed by a
-pipe (`|`) and a pipe-separated list of attributes. An attribute is
-_key_=_value_ or just _key_ as a short form of _key_=*true*. Attribute values
-must be https://en.wikipedia.org/wiki/Percent-encoding[percent-encoded] if they
-contain percent, pipe or space characters.
+A remote storage backend is specified with a URL, optionally followed by a pipe
+(`|`) and a pipe-separated list of attributes. An attribute is _key_=_value_ or
+just _key_ as a short form of _key_=*true*. Attribute values must be
+https://en.wikipedia.org/wiki/Percent-encoding[percent-encoded] if they contain
+percent, pipe or space characters.
=== Attributes for all backends
-These optional attributes are available for all secondary storage backends:
+These optional attributes are available for all remote storage backends:
* *read-only*: If *true*, only read from this backend, don't write. The default
is *false*.
@@ -1075,21 +1077,21 @@ Examples:
=== Storage interaction
-The table below describes the interaction between primary and secondary storage
-on cache hits and misses:
+The table below describes the interaction between local and remote storage on
+cache hits and misses:
[options="header",cols="20%,20%,60%"]
|==============================================================================
-| *Primary storage* | *Secondary storage* | *What happens*
+| *Local storage* | *Remote storage* | *What happens*
-| miss | miss | Compile, write to primary, write to secondary^[1]^
-| miss | hit | Read from secondary, write to primary
-| hit | - | Read from primary, don't write to secondary^[2]^
+| miss | miss | Compile, write to local, write to remote^[1]^
+| miss | hit | Read from remote, write to local
+| hit | - | Read from local, don't write to remote^[2]^
|==============================================================================
-^[1]^ Unless secondary storage has attribute `read-only=true`. +
-^[2]^ Unless primary storage is set to share its cache hits with the
+^[1]^ Unless remote storage has attribute `read-only=true`. +
+^[2]^ Unless local storage is set to share its cache hits with the
<<config_reshare,*reshare*>> option.
@@ -1098,7 +1100,7 @@ on cache hits and misses:
URL format: `+file:DIRECTORY+` or `+file://[HOST]DIRECTORY+`
This backend stores data as separate files in a directory structure below
-*DIRECTORY*, similar (but not identical) to the primary cache storage. A typical
+*DIRECTORY*, similar (but not identical) to the local cache storage. A typical
use case for this backend would be sharing a cache on an NFS directory.
*DIRECTORY* must start with a slash. *HOST* can be the empty string or
localhost. On Windows, *HOST* can also be the name of a server hosting a shared
@@ -1196,7 +1198,7 @@ TIP: See https://ccache.dev/howto/redis-storage.html[How to set up Redis
storage] for hints on setting up a Redis server for use with ccache.
TIP: You can set up a cluster of Redis servers using the `shards` attribute
-described in _<<Secondary storage backends>>_.
+described in _<<Remote storage backends>>_.
Examples:
@@ -1317,7 +1319,7 @@ recompressed.
cleanups (number of performed cleanups, either implicitly due to a cache size
limit being reached or due to explicit `ccache -c` calls), overall hit rate, hit
rate for <<The direct mode,direct>>/<<The preprocessor mode,preprocessed>> modes
-and hit rate for primary and <<config_secondary_storage,secondary>> storage.
+and hit rate for local and <<config_remote_storage,remote storage>>.
The summary also includes counters called "`Errors`" and "`Uncacheable`", which
are sums of more detailed counters. To see those detailed counters, use the
@@ -1770,7 +1772,7 @@ systems. One way of improving cache hit rate in that case is to set
headers.
An alternative to putting the main cache directory on NFS is to set up a
-<<config_secondary_storage,secondary storage>> file cache.
+<<config_remote_storage,remote storage>> file cache.
== Using ccache with other compiler wrappers
diff --git a/misc/download-redis b/misc/download-redis
index 05e0d033..0352b476 100755
--- a/misc/download-redis
+++ b/misc/download-redis
@@ -1,6 +1,6 @@
#!/usr/bin/env python3
-# This script downloads the contents of the cache from a Redis secondary
+# This script downloads the contents of the local cache from a Redis remote
# storage.
import redis
diff --git a/misc/upload-redis b/misc/upload-redis
index 4de656a9..b3595c5a 100755
--- a/misc/upload-redis
+++ b/misc/upload-redis
@@ -1,7 +1,6 @@
#!/usr/bin/env python3
-# This script uploads the contents of the cache from primary storage to a Redis
-# secondary storage.
+# This script uploads the contents of the local cache to a Redis remote storage.
import redis
import os
diff --git a/src/Config.cpp b/src/Config.cpp
index bd42718d..de5b5247 100644
--- a/src/Config.cpp
+++ b/src/Config.cpp
@@ -91,9 +91,9 @@ enum class ConfigItem {
read_only,
read_only_direct,
recache,
+ remote_storage,
reshare,
run_second_cpp,
- secondary_storage,
sloppiness,
stats,
stats_log,
@@ -101,49 +101,59 @@ enum class ConfigItem {
umask,
};
-const std::unordered_map<std::string, ConfigItem> k_config_key_table = {
- {"absolute_paths_in_stderr", ConfigItem::absolute_paths_in_stderr},
- {"base_dir", ConfigItem::base_dir},
- {"cache_dir", ConfigItem::cache_dir},
- {"compiler", ConfigItem::compiler},
- {"compiler_check", ConfigItem::compiler_check},
- {"compiler_type", ConfigItem::compiler_type},
- {"compression", ConfigItem::compression},
- {"compression_level", ConfigItem::compression_level},
- {"cpp_extension", ConfigItem::cpp_extension},
- {"debug", ConfigItem::debug},
- {"debug_dir", ConfigItem::debug_dir},
- {"depend_mode", ConfigItem::depend_mode},
- {"direct_mode", ConfigItem::direct_mode},
- {"disable", ConfigItem::disable},
- {"extra_files_to_hash", ConfigItem::extra_files_to_hash},
- {"file_clone", ConfigItem::file_clone},
- {"hard_link", ConfigItem::hard_link},
- {"hash_dir", ConfigItem::hash_dir},
- {"ignore_headers_in_manifest", ConfigItem::ignore_headers_in_manifest},
- {"ignore_options", ConfigItem::ignore_options},
- {"inode_cache", ConfigItem::inode_cache},
- {"keep_comments_cpp", ConfigItem::keep_comments_cpp},
- {"limit_multiple", ConfigItem::limit_multiple},
- {"log_file", ConfigItem::log_file},
- {"max_files", ConfigItem::max_files},
- {"max_size", ConfigItem::max_size},
- {"namespace", ConfigItem::namespace_},
- {"path", ConfigItem::path},
- {"pch_external_checksum", ConfigItem::pch_external_checksum},
- {"prefix_command", ConfigItem::prefix_command},
- {"prefix_command_cpp", ConfigItem::prefix_command_cpp},
- {"read_only", ConfigItem::read_only},
- {"read_only_direct", ConfigItem::read_only_direct},
- {"recache", ConfigItem::recache},
- {"reshare", ConfigItem::reshare},
- {"run_second_cpp", ConfigItem::run_second_cpp},
- {"secondary_storage", ConfigItem::secondary_storage},
- {"sloppiness", ConfigItem::sloppiness},
- {"stats", ConfigItem::stats},
- {"stats_log", ConfigItem::stats_log},
- {"temporary_dir", ConfigItem::temporary_dir},
- {"umask", ConfigItem::umask},
+enum class ConfigKeyType { normal, alias };
+
+struct ConfigKeyTableEntry
+{
+ ConfigItem item;
+ std::optional<std::string> alias = std::nullopt;
+};
+
+const std::unordered_map<std::string, ConfigKeyTableEntry> k_config_key_table =
+ {
+ {"absolute_paths_in_stderr", {ConfigItem::absolute_paths_in_stderr}},
+ {"base_dir", {ConfigItem::base_dir}},
+ {"cache_dir", {ConfigItem::cache_dir}},
+ {"compiler", {ConfigItem::compiler}},
+ {"compiler_check", {ConfigItem::compiler_check}},
+ {"compiler_type", {ConfigItem::compiler_type}},
+ {"compression", {ConfigItem::compression}},
+ {"compression_level", {ConfigItem::compression_level}},
+ {"cpp_extension", {ConfigItem::cpp_extension}},
+ {"debug", {ConfigItem::debug}},
+ {"debug_dir", {ConfigItem::debug_dir}},
+ {"depend_mode", {ConfigItem::depend_mode}},
+ {"direct_mode", {ConfigItem::direct_mode}},
+ {"disable", {ConfigItem::disable}},
+ {"extra_files_to_hash", {ConfigItem::extra_files_to_hash}},
+ {"file_clone", {ConfigItem::file_clone}},
+ {"hard_link", {ConfigItem::hard_link}},
+ {"hash_dir", {ConfigItem::hash_dir}},
+ {"ignore_headers_in_manifest", {ConfigItem::ignore_headers_in_manifest}},
+ {"ignore_options", {ConfigItem::ignore_options}},
+ {"inode_cache", {ConfigItem::inode_cache}},
+ {"keep_comments_cpp", {ConfigItem::keep_comments_cpp}},
+ {"limit_multiple", {ConfigItem::limit_multiple}},
+ {"log_file", {ConfigItem::log_file}},
+ {"max_files", {ConfigItem::max_files}},
+ {"max_size", {ConfigItem::max_size}},
+ {"namespace", {ConfigItem::namespace_}},
+ {"path", {ConfigItem::path}},
+ {"pch_external_checksum", {ConfigItem::pch_external_checksum}},
+ {"prefix_command", {ConfigItem::prefix_command}},
+ {"prefix_command_cpp", {ConfigItem::prefix_command_cpp}},
+ {"read_only", {ConfigItem::read_only}},
+ {"read_only_direct", {ConfigItem::read_only_direct}},
+ {"recache", {ConfigItem::recache}},
+ {"remote_storage", {ConfigItem::remote_storage}},
+ {"reshare", {ConfigItem::reshare}},
+ {"run_second_cpp", {ConfigItem::run_second_cpp}},
+ {"secondary_storage", {ConfigItem::remote_storage, "remote_storage"}},
+ {"sloppiness", {ConfigItem::sloppiness}},
+ {"stats", {ConfigItem::stats}},
+ {"stats_log", {ConfigItem::stats_log}},
+ {"temporary_dir", {ConfigItem::temporary_dir}},
+ {"umask", {ConfigItem::umask}},
};
const std::unordered_map<std::string, std::string> k_env_variable_table = {
@@ -183,8 +193,9 @@ const std::unordered_map<std::string, std::string> k_env_variable_table = {
{"READONLY", "read_only"},
{"READONLY_DIRECT", "read_only_direct"},
{"RECACHE", "recache"},
+ {"REMOTE_STORAGE", "remote_storage"},
{"RESHARE", "reshare"},
- {"SECONDARY_STORAGE", "secondary_storage"},
+ {"SECONDARY_STORAGE", "remote_storage"}, // Alias for CCACHE_REMOTE_STORAGE
{"SLOPPINESS", "sloppiness"},
{"STATS", "stats"},
{"STATSLOG", "stats_log"},
@@ -661,7 +672,7 @@ Config::get_string_value(const std::string& key) const
throw core::Error(FMT("unknown configuration option \"{}\"", key));
}
- switch (it->second) {
+ switch (it->second.item) {
case ConfigItem::absolute_paths_in_stderr:
return format_bool(m_absolute_paths_in_stderr);
@@ -764,15 +775,15 @@ Config::get_string_value(const std::string& key) const
case ConfigItem::recache:
return format_bool(m_recache);
+ case ConfigItem::remote_storage:
+ return m_remote_storage;
+
case ConfigItem::reshare:
return format_bool(m_reshare);
case ConfigItem::run_second_cpp:
return format_bool(m_run_second_cpp);
- case ConfigItem::secondary_storage:
- return m_secondary_storage;
-
case ConfigItem::sloppiness:
return format_sloppiness(m_sloppiness);
@@ -847,8 +858,10 @@ Config::visit_items(const ItemVisitor& item_visitor) const
std::vector<std::string> keys;
keys.reserve(k_config_key_table.size());
- for (const auto& [key, value] : k_config_key_table) {
- keys.emplace_back(key);
+ for (const auto& [key, entry] : k_config_key_table) {
+ if (!entry.alias) {
+ keys.emplace_back(key);
+ }
}
std::sort(keys.begin(), keys.end());
for (const auto& key : keys) {
@@ -871,7 +884,7 @@ Config::set_item(const std::string& key,
return;
}
- switch (it->second) {
+ switch (it->second.item) {
case ConfigItem::absolute_paths_in_stderr:
m_absolute_paths_in_stderr = parse_bool(value, env_var_key, negate);
break;
@@ -1015,6 +1028,10 @@ Config::set_item(const std::string& key,
m_recache = parse_bool(value, env_var_key, negate);
break;
+ case ConfigItem::remote_storage:
+ m_remote_storage = Util::expand_environment_variables(value);
+ break;
+
case ConfigItem::reshare:
m_reshare = parse_bool(value, env_var_key, negate);
break;
@@ -1023,10 +1040,6 @@ Config::set_item(const std::string& key,
m_run_second_cpp = parse_bool(value, env_var_key, negate);
break;
- case ConfigItem::secondary_storage:
- m_secondary_storage = Util::expand_environment_variables(value);
- break;
-
case ConfigItem::sloppiness:
m_sloppiness = parse_sloppiness(value);
break;
@@ -1055,7 +1068,8 @@ Config::set_item(const std::string& key,
break;
}
- const auto& [element, inserted] = m_origins.emplace(key, origin);
+ const std::string canonical_key = it->second.alias ? *it->second.alias : key;
+ const auto& [element, inserted] = m_origins.emplace(canonical_key, origin);
if (!inserted) {
element->second = origin;
}
diff --git a/src/Config.hpp b/src/Config.hpp
index b58d97dd..7ee59921 100644
--- a/src/Config.hpp
+++ b/src/Config.hpp
@@ -84,9 +84,9 @@ public:
bool read_only() const;
bool read_only_direct() const;
bool recache() const;
+ const std::string& remote_storage() const;
bool reshare() const;
bool run_second_cpp() const;
- const std::string& secondary_storage() const;
core::Sloppiness sloppiness() const;
bool stats() const;
const std::string& stats_log() const;
@@ -193,7 +193,7 @@ private:
bool m_recache = false;
bool m_reshare = false;
bool m_run_second_cpp = true;
- std::string m_secondary_storage;
+ std::string m_remote_storage;
core::Sloppiness m_sloppiness;
bool m_stats = true;
std::string m_stats_log;
@@ -438,9 +438,9 @@ Config::run_second_cpp() const
}
inline const std::string&
-Config::secondary_storage() const
+Config::remote_storage() const
{
- return m_secondary_storage;
+ return m_remote_storage;
}
inline core::Sloppiness
diff --git a/src/Context.hpp b/src/Context.hpp
index 63844489..b7624811 100644
--- a/src/Context.hpp
+++ b/src/Context.hpp
@@ -87,7 +87,7 @@ public:
// Headers (or directories with headers) to ignore in manifest mode.
std::vector<std::string> ignore_header_paths;
- // Storage (fronting primary and secondary storage backends).
+ // Storage (fronting local and remote storage backends).
storage::Storage storage;
// Direct mode manifest.
diff --git a/src/ccache.cpp b/src/ccache.cpp
index a52dd494..6cf1c53a 100644
--- a/src/ccache.cpp
+++ b/src/ccache.cpp
@@ -903,7 +903,7 @@ write_result(Context& ctx,
const auto cache_entry_data = core::CacheEntry::serialize(header, serializer);
const auto raw_files = serializer.get_raw_files();
if (!raw_files.empty()) {
- ctx.storage.primary.put_raw_files(result_key, raw_files);
+ ctx.storage.local.put_raw_files(result_key, raw_files);
}
ctx.storage.put(result_key, core::CacheEntryType::result, cache_entry_data);
@@ -1746,9 +1746,9 @@ hash_direct_mode_specific_data(Context& ctx,
MTR_SCOPE("manifest", "merge");
LOG("Storing merged manifest {} locally", manifest_key->to_string());
core::CacheEntry::Header header(ctx.config, core::CacheEntryType::manifest);
- ctx.storage.primary.put(*manifest_key,
- core::CacheEntryType::manifest,
- core::CacheEntry::serialize(header, ctx.manifest));
+ ctx.storage.local.put(*manifest_key,
+ core::CacheEntryType::manifest,
+ core::CacheEntry::serialize(header, ctx.manifest));
}
return {};
@@ -2069,7 +2069,7 @@ log_result_to_debug_log(Context& ctx)
return;
}
- core::Statistics statistics(ctx.storage.primary.get_statistics_updates());
+ core::Statistics statistics(ctx.storage.local.get_statistics_updates());
for (const auto& message : statistics.get_statistics_ids()) {
LOG("Result: {}", message);
}
@@ -2082,7 +2082,7 @@ log_result_to_stats_log(Context& ctx)
return;
}
- core::Statistics statistics(ctx.storage.primary.get_statistics_updates());
+ core::Statistics statistics(ctx.storage.local.get_statistics_updates());
const auto ids = statistics.get_statistics_ids();
if (ids.empty()) {
return;
@@ -2145,7 +2145,7 @@ cache_compilation(int argc, const char* const* argv)
const auto result = do_cache_compilation(ctx, argv);
const auto& counters = result ? *result : result.error().counters();
- ctx.storage.primary.increment_statistics(counters);
+ ctx.storage.local.increment_statistics(counters);
if (!result) {
if (result.error().exit_code()) {
return *result.error().exit_code();
@@ -2197,8 +2197,8 @@ do_cache_compilation(Context& ctx, const char* const* argv)
const std::string& value,
const std::string& origin) {
const auto& log_value =
- key == "secondary_storage"
- ? ctx.storage.get_secondary_storage_config_for_logging()
+ key == "remote_storage"
+ ? ctx.storage.get_remote_storage_config_for_logging()
: value;
BULK_LOG("Config: ({}) {} = {}", origin, key, log_value);
});
@@ -2271,13 +2271,13 @@ do_cache_compilation(Context& ctx, const char* const* argv)
ctx.config.set_depend_mode(false);
}
- if (ctx.storage.has_secondary_storage()) {
+ if (ctx.storage.has_remote_storage()) {
if (ctx.config.file_clone()) {
- LOG_RAW("Disabling file clone mode since secondary storage is enabled");
+ LOG_RAW("Disabling file clone mode since remote storage is enabled");
ctx.config.set_file_clone(false);
}
if (ctx.config.hard_link()) {
- LOG_RAW("Disabling hard link mode since secondary storage is enabled");
+ LOG_RAW("Disabling hard link mode since remote storage is enabled");
ctx.config.set_hard_link(false);
}
}
@@ -2377,7 +2377,7 @@ do_cache_compilation(Context& ctx, const char* const* argv)
}
if (!ctx.config.recache()) {
- ctx.storage.primary.increment_statistic(Statistic::direct_cache_miss);
+ ctx.storage.local.increment_statistic(Statistic::direct_cache_miss);
}
}
@@ -2439,7 +2439,7 @@ do_cache_compilation(Context& ctx, const char* const* argv)
return Statistic::preprocessed_cache_hit;
}
- ctx.storage.primary.increment_statistic(Statistic::preprocessed_cache_miss);
+ ctx.storage.local.increment_statistic(Statistic::preprocessed_cache_miss);
}
if (ctx.config.read_only()) {
diff --git a/src/core/CacheEntry.hpp b/src/core/CacheEntry.hpp
index 0402f3a9..f3c958a1 100644
--- a/src/core/CacheEntry.hpp
+++ b/src/core/CacheEntry.hpp
@@ -43,7 +43,7 @@
// <entry_type> ::= <result_entry> | <manifest_entry>
// <result_entry> ::= 0 (uint8_t)
// <manifest_entry> ::= 1 (uint8_t)
-// <self_contained> ::= 0/1 (uint8_t) ; whether suitable for secondary storage
+// <self_contained> ::= 0/1 (uint8_t) ; whether suitable for remote storage
// <compr_type> ::= <compr_none> | <compr_zstd>
// <compr_none> ::= 0 (uint8_t)
// <compr_zstd> ::= 1 (uint8_t)
diff --git a/src/core/Result.hpp b/src/core/Result.hpp
index b1d0fcc7..970211f4 100644
--- a/src/core/Result.hpp
+++ b/src/core/Result.hpp
@@ -156,7 +156,7 @@ public:
std::string path;
};
- // Get raw files to store in primary storage.
+ // Get raw files to store in local storage.
const std::vector<RawFile>& get_raw_files() const;
private:
diff --git a/src/core/ResultExtractor.hpp b/src/core/ResultExtractor.hpp
index 204835bb..916bea75 100644
--- a/src/core/ResultExtractor.hpp
+++ b/src/core/ResultExtractor.hpp
@@ -35,7 +35,7 @@ public:
using GetRawFilePathFunction = std::function<std::string(uint8_t)>;
//`result_path` should be the path to the local result entry file if the
- // result comes from primary storage.
+ // result comes from local storage.
ResultExtractor(
const std::string& output_directory,
std::optional<GetRawFilePathFunction> get_raw_file_path = std::nullopt);
diff --git a/src/core/ResultRetriever.cpp b/src/core/ResultRetriever.cpp
index 89f4389d..bdefc19f 100644
--- a/src/core/ResultRetriever.cpp
+++ b/src/core/ResultRetriever.cpp
@@ -97,7 +97,7 @@ ResultRetriever::on_raw_file(uint8_t file_number,
throw core::Error("Raw entry for non-local result");
}
const auto raw_file_path =
- m_ctx.storage.primary.get_raw_file_path(*m_result_key, file_number);
+ m_ctx.storage.local.get_raw_file_path(*m_result_key, file_number);
const auto st = Stat::stat(raw_file_path, Stat::OnError::throw_error);
if (st.size() != file_size) {
throw core::Error(
diff --git a/src/core/ResultRetriever.hpp b/src/core/ResultRetriever.hpp
index 640a761a..92a3af88 100644
--- a/src/core/ResultRetriever.hpp
+++ b/src/core/ResultRetriever.hpp
@@ -40,7 +40,7 @@ public:
};
//`path` should be the path to the local result entry file if the result comes
- // from primary storage.
+ // from local storage.
ResultRetriever(const Context& ctx,
std::optional<Digest> result_key = std::nullopt);
diff --git a/src/core/Statistic.hpp b/src/core/Statistic.hpp
index 7adce5cc..747caf73 100644
--- a/src/core/Statistic.hpp
+++ b/src/core/Statistic.hpp
@@ -57,12 +57,12 @@ enum class Statistic {
could_not_use_modules = 32,
direct_cache_miss = 33,
preprocessed_cache_miss = 34,
- primary_storage_hit = 35,
- primary_storage_miss = 36,
- secondary_storage_hit = 37,
- secondary_storage_miss = 38,
- secondary_storage_error = 39,
- secondary_storage_timeout = 40,
+ local_storage_hit = 35,
+ local_storage_miss = 36,
+ remote_storage_hit = 37,
+ remote_storage_miss = 38,
+ remote_storage_error = 39,
+ remote_storage_timeout = 40,
recache = 41,
unsupported_environment_variable = 42,
diff --git a/src/core/Statistics.cpp b/src/core/Statistics.cpp
index b7d23acb..416f9c61 100644
--- a/src/core/Statistics.cpp
+++ b/src/core/Statistics.cpp
@@ -102,13 +102,13 @@ const StatisticsField k_statistics_fields[] = {
FIELD(preprocessed_cache_hit, nullptr),
FIELD(preprocessed_cache_miss, nullptr),
FIELD(preprocessor_error, "Preprocessing failed", FLAG_UNCACHEABLE),
- FIELD(primary_storage_hit, nullptr),
- FIELD(primary_storage_miss, nullptr),
+ FIELD(local_storage_hit, nullptr),
+ FIELD(local_storage_miss, nullptr),
FIELD(recache, "Forced recache", FLAG_UNCACHEABLE),
- FIELD(secondary_storage_error, nullptr),
- FIELD(secondary_storage_hit, nullptr),
- FIELD(secondary_storage_miss, nullptr),
- FIELD(secondary_storage_timeout, nullptr),
+ FIELD(remote_storage_error, nullptr),
+ FIELD(remote_storage_hit, nullptr),
+ FIELD(remote_storage_miss, nullptr),
+ FIELD(remote_storage_timeout, nullptr),
FIELD(stats_zeroed_timestamp, nullptr),
FIELD(
unsupported_code_directive, "Unsupported code directive", FLAG_UNCACHEABLE),
@@ -293,24 +293,24 @@ Statistics::format_human_readable(const Config& config,
}
const uint64_t g = 1'000'000'000;
- const uint64_t pri_hits = S(primary_storage_hit);
- const uint64_t pri_misses = S(primary_storage_miss);
- const uint64_t pri_size = S(cache_size_kibibyte) * 1024;
+ const uint64_t local_hits = S(local_storage_hit);
+ const uint64_t local_misses = S(local_storage_miss);
+ const uint64_t local_size = S(cache_size_kibibyte) * 1024;
const uint64_t cleanups = S(cleanups_performed);
- table.add_heading("Primary storage:");
- add_ratio_row(table, " Hits:", pri_hits, pri_hits + pri_misses);
- add_ratio_row(table, " Misses:", pri_misses, pri_hits + pri_misses);
+ table.add_heading("Local storage:");
+ add_ratio_row(table, " Hits:", local_hits, local_hits + local_misses);
+ add_ratio_row(table, " Misses:", local_misses, local_hits + local_misses);
if (!from_log) {
std::vector<C> size_cells{
" Cache size (GB):",
- C(FMT("{:.2f}", static_cast<double>(pri_size) / g)).right_align()};
+ C(FMT("{:.2f}", static_cast<double>(local_size) / g)).right_align()};
if (config.max_size() != 0) {
size_cells.emplace_back("/");
size_cells.emplace_back(
C(FMT("{:.2f}", static_cast<double>(config.max_size()) / g))
.right_align());
- size_cells.emplace_back(percent(pri_size, config.max_size()));
+ size_cells.emplace_back(percent(local_size, config.max_size()));
}
table.add_row(size_cells);
@@ -329,20 +329,22 @@ Statistics::format_human_readable(const Config& config,
}
}
- const uint64_t sec_hits = S(secondary_storage_hit);
- const uint64_t sec_misses = S(secondary_storage_miss);
- const uint64_t sec_errors = S(secondary_storage_error);
- const uint64_t sec_timeouts = S(secondary_storage_timeout);
-
- if (verbosity > 1 || sec_hits + sec_misses + sec_errors + sec_timeouts > 0) {
- table.add_heading("Secondary storage:");
- add_ratio_row(table, " Hits:", sec_hits, sec_hits + sec_misses);
- add_ratio_row(table, " Misses:", sec_misses, sec_hits + sec_misses);
- if (verbosity > 1 || sec_errors > 0) {
- table.add_row({" Errors:", sec_errors});
+ const uint64_t remote_hits = S(remote_storage_hit);
+ const uint64_t remote_misses = S(remote_storage_miss);
+ const uint64_t remote_errors = S(remote_storage_error);
+ const uint64_t remote_timeouts = S(remote_storage_timeout);
+
+ if (verbosity > 1
+ || remote_hits + remote_misses + remote_errors + remote_timeouts > 0) {
+ table.add_heading("Remote storage:");
+ add_ratio_row(table, " Hits:", remote_hits, remote_hits + remote_misses);
+ add_ratio_row(
+ table, " Misses:", remote_misses, remote_hits + remote_misses);
+ if (verbosity > 1 || remote_errors > 0) {
+ table.add_row({" Errors:", remote_errors});
}
- if (verbosity > 1 || sec_timeouts > 0) {
- table.add_row({" Timeouts:", sec_timeouts});
+ if (verbosity > 1 || remote_timeouts > 0) {
+ table.add_row({" Timeouts:", remote_timeouts});
}
}
diff --git a/src/core/mainoptions.cpp b/src/core/mainoptions.cpp
index 8b291d8a..30afaa48 100644
--- a/src/core/mainoptions.cpp
+++ b/src/core/mainoptions.cpp
@@ -35,7 +35,7 @@
#include <core/exceptions.hpp>
#include <fmtmacros.hpp>
#include <storage/Storage.hpp>
-#include <storage/primary/PrimaryStorage.hpp>
+#include <storage/local/LocalStorage.hpp>
#include <util/TextTable.hpp>
#include <util/XXH3_128.hpp>
#include <util/expected.hpp>
@@ -121,10 +121,10 @@ Common options:
-h, --help print this help text
-V, --version print version and copyright information
-Options for secondary storage:
+Options for remote file-based storage:
--trim-dir PATH remove old files from directory PATH until it is
at most the size specified by --trim-max-size
- (note: don't use this option to trim the primary
+ (note: don't use this option to trim the local
cache)
--trim-max-size SIZE specify the maximum size for --trim-dir;
available suffixes: k, M, G, T (decimal) and Ki,
@@ -214,7 +214,7 @@ inspect_path(const std::string& path)
}
static void
-print_compression_statistics(const storage::primary::CompressionStatistics& cs)
+print_compression_statistics(const storage::local::CompressionStatistics& cs)
{
const double ratio = cs.compr_size > 0
? static_cast<double>(cs.content_size) / cs.compr_size
@@ -276,7 +276,7 @@ trim_dir(const std::string& dir,
const auto name = Util::base_name(path);
if (name == "ccache.conf" || name == "stats") {
throw Fatal(
- FMT("this looks like a primary cache directory (found {})", path));
+ FMT("this looks like a local cache directory (found {})", path));
}
files.push_back({path, stat});
}
@@ -470,8 +470,8 @@ process_main_options(int argc, const char* const* argv)
std::optional<ResultExtractor::GetRawFilePathFunction> get_raw_file_path;
if (arg != "-") {
get_raw_file_path = [&](uint8_t file_number) {
- return storage::primary::PrimaryStorage::get_raw_file_path(
- arg, file_number);
+ return storage::local::LocalStorage::get_raw_file_path(arg,
+ file_number);
};
}
ResultExtractor result_extractor(".", get_raw_file_path);
@@ -504,7 +504,7 @@ process_main_options(int argc, const char* const* argv)
case PRINT_STATS: {
const auto [counters, last_updated] =
- storage::primary::PrimaryStorage(config).get_all_statistics();
+ storage::local::LocalStorage(config).get_all_statistics();
Statistics statistics(counters);
PRINT_RAW(stdout, statistics.format_machine_readable(last_updated));
break;
@@ -513,7 +513,7 @@ process_main_options(int argc, const char* const* argv)
case 'c': // --cleanup
{
ProgressBar progress_bar("Cleaning...");
- storage::primary::PrimaryStorage(config).clean_all(
+ storage::local::LocalStorage(config).clean_all(
[&](double progress) { progress_bar.update(progress); });
if (isatty(STDOUT_FILENO)) {
PRINT_RAW(stdout, "\n");
@@ -524,7 +524,7 @@ process_main_options(int argc, const char* const* argv)
case 'C': // --clear
{
ProgressBar progress_bar("Clearing...");
- storage::primary::PrimaryStorage(config).wipe_all(
+ storage::local::LocalStorage(config).wipe_all(
[&](double progress) { progress_bar.update(progress); });
if (isatty(STDOUT_FILENO)) {
PRINT_RAW(stdout, "\n");
@@ -596,7 +596,7 @@ process_main_options(int argc, const char* const* argv)
case 's': { // --show-stats
const auto [counters, last_updated] =
- storage::primary::PrimaryStorage(config).get_all_statistics();
+ storage::local::LocalStorage(config).get_all_statistics();
Statistics statistics(counters);
PRINT_RAW(stdout,
statistics.format_human_readable(
@@ -625,7 +625,7 @@ process_main_options(int argc, const char* const* argv)
{
ProgressBar progress_bar("Scanning...");
const auto compression_statistics =
- storage::primary::PrimaryStorage(config).get_compression_statistics(
+ storage::local::LocalStorage(config).get_compression_statistics(
[&](double progress) { progress_bar.update(progress); });
if (isatty(STDOUT_FILENO)) {
PRINT_RAW(stdout, "\n\n");
@@ -645,13 +645,13 @@ process_main_options(int argc, const char* const* argv)
}
ProgressBar progress_bar("Recompressing...");
- storage::primary::PrimaryStorage(config).recompress(
+ storage::local::LocalStorage(config).recompress(
wanted_level, [&](double progress) { progress_bar.update(progress); });
break;
}
case 'z': // --zero-stats
- storage::primary::PrimaryStorage(config).zero_all_statistics();
+ storage::local::LocalStorage(config).zero_all_statistics();
PRINT_RAW(stdout, "Statistics zeroed\n");
break;
@@ -666,7 +666,7 @@ process_main_options(int argc, const char* const* argv)
config.read();
ProgressBar progress_bar("Evicting...");
- storage::primary::PrimaryStorage(config).evict(
+ storage::local::LocalStorage(config).evict(
[&](double progress) { progress_bar.update(progress); },
evict_max_age,
evict_namespace);
diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt
index 4607427f..e5476cc3 100644
--- a/src/storage/CMakeLists.txt
+++ b/src/storage/CMakeLists.txt
@@ -1,5 +1,5 @@
-add_subdirectory(primary)
-add_subdirectory(secondary)
+add_subdirectory(local)
+add_subdirectory(remote)
set(
sources
diff --git a/src/storage/Storage.cpp b/src/storage/Storage.cpp
index 9e617fb3..11490edc 100644
--- a/src/storage/Storage.cpp
+++ b/src/storage/Storage.cpp
@@ -27,10 +27,10 @@
#include <core/Statistic.hpp>
#include <core/exceptions.hpp>
#include <fmtmacros.hpp>
-#include <storage/secondary/FileStorage.hpp>
-#include <storage/secondary/HttpStorage.hpp>
+#include <storage/remote/FileStorage.hpp>
+#include <storage/remote/HttpStorage.hpp>
#ifdef HAVE_REDIS_STORAGE_BACKEND
-# include <storage/secondary/RedisStorage.hpp>
+# include <storage/remote/RedisStorage.hpp>
#endif
#include <core/CacheEntry.hpp>
#include <util/Bytes.hpp>
@@ -52,13 +52,13 @@
namespace storage {
const std::unordered_map<std::string /*scheme*/,
- std::shared_ptr<secondary::SecondaryStorage>>
- k_secondary_storage_implementations = {
- {"file", std::make_shared<secondary::FileStorage>()},
- {"http", std::make_shared<secondary::HttpStorage>()},
+ std::shared_ptr<remote::RemoteStorage>>
+ k_remote_storage_implementations = {
+ {"file", std::make_shared<remote::FileStorage>()},
+ {"http", std::make_shared<remote::HttpStorage>()},
#ifdef HAVE_REDIS_STORAGE_BACKEND
- {"redis", std::make_shared<secondary::RedisStorage>()},
- {"redis+unix", std::make_shared<secondary::RedisStorage>()},
+ {"redis", std::make_shared<remote::RedisStorage>()},
+ {"redis+unix", std::make_shared<remote::RedisStorage>()},
#endif
};
@@ -66,46 +66,46 @@ std::string
get_features()
{
std::vector<std::string> features;
- features.reserve(k_secondary_storage_implementations.size());
- std::transform(k_secondary_storage_implementations.begin(),
- k_secondary_storage_implementations.end(),
+ features.reserve(k_remote_storage_implementations.size());
+ std::transform(k_remote_storage_implementations.begin(),
+ k_remote_storage_implementations.end(),
std::back_inserter(features),
[](auto& entry) { return FMT("{}-storage", entry.first); });
std::sort(features.begin(), features.end());
return util::join(features, " ");
}
-struct SecondaryStorageShardConfig
+struct RemoteStorageShardConfig
{
std::string name;
double weight;
};
-struct SecondaryStorageConfig
+struct RemoteStorageConfig
{
- std::vector<SecondaryStorageShardConfig> shards;
- secondary::SecondaryStorage::Backend::Params params;
+ std::vector<RemoteStorageShardConfig> shards;
+ remote::RemoteStorage::Backend::Params params;
bool read_only = false;
};
-struct SecondaryStorageBackendEntry
+struct RemoteStorageBackendEntry
{
Url url; // With expanded "*".
std::string url_for_logging; // With expanded "*".
- std::unique_ptr<secondary::SecondaryStorage::Backend> impl;
+ std::unique_ptr<remote::RemoteStorage::Backend> impl;
bool failed = false;
};
-struct SecondaryStorageEntry
+struct RemoteStorageEntry
{
- SecondaryStorageConfig config;
+ RemoteStorageConfig config;
std::string url_for_logging; // With unexpanded "*".
- std::shared_ptr<secondary::SecondaryStorage> storage;
- std::vector<SecondaryStorageBackendEntry> backends;
+ std::shared_ptr<remote::RemoteStorage> storage;
+ std::vector<RemoteStorageBackendEntry> backends;
};
static std::string
-to_string(const SecondaryStorageConfig& entry)
+to_string(const RemoteStorageConfig& entry)
{
std::string result = entry.params.url.str();
for (const auto& attr : entry.params.attributes) {
@@ -114,7 +114,7 @@ to_string(const SecondaryStorageConfig& entry)
return result;
}
-static SecondaryStorageConfig
+static RemoteStorageConfig
parse_storage_config(const std::string_view entry)
{
const auto parts =
@@ -122,10 +122,10 @@ parse_storage_config(const std::string_view entry)
if (parts.empty() || parts.front().empty()) {
throw core::Error(
- FMT("secondary storage config must provide a URL: {}", entry));
+ FMT("remote storage config must provide a URL: {}", entry));
}
- SecondaryStorageConfig result;
+ RemoteStorageConfig result;
const auto url_str = std::string(parts[0]);
result.params.url = url_str;
@@ -186,33 +186,33 @@ parse_storage_config(const std::string_view entry)
return result;
}
-static std::vector<SecondaryStorageConfig>
+static std::vector<RemoteStorageConfig>
parse_storage_configs(const std::string_view& configs)
{
- std::vector<SecondaryStorageConfig> result;
+ std::vector<RemoteStorageConfig> result;
for (const auto& config : util::Tokenizer(configs, " ")) {
result.push_back(parse_storage_config(config));
}
return result;
}
-static std::shared_ptr<secondary::SecondaryStorage>
+static std::shared_ptr<remote::RemoteStorage>
get_storage(const Url& url)
{
- const auto it = k_secondary_storage_implementations.find(url.scheme());
- if (it != k_secondary_storage_implementations.end()) {
+ const auto it = k_remote_storage_implementations.find(url.scheme());
+ if (it != k_remote_storage_implementations.end()) {
return it->second;
} else {
return {};
}
}
-Storage::Storage(const Config& config) : primary(config), m_config(config)
+Storage::Storage(const Config& config) : local(config), m_config(config)
{
}
// Define the destructor in the implementation file to avoid having to declare
-// SecondaryStorageEntry and its constituents in the header file.
+// RemoteStorageEntry and its constituents in the header file.
// NOLINTNEXTLINE(modernize-use-equals-default)
Storage::~Storage()
{
@@ -221,13 +221,13 @@ Storage::~Storage()
void
Storage::initialize()
{
- add_secondary_storages();
+ add_remote_storages();
}
void
Storage::finalize()
{
- primary.finalize();
+ local.finalize();
}
void
@@ -237,20 +237,20 @@ Storage::get(const Digest& key,
{
MTR_SCOPE("storage", "get");
- auto value = primary.get(key, type);
- primary.increment_statistic(value ? core::Statistic::primary_storage_hit
- : core::Statistic::primary_storage_miss);
+ auto value = local.get(key, type);
+ local.increment_statistic(value ? core::Statistic::local_storage_hit
+ : core::Statistic::local_storage_miss);
if (value) {
if (m_config.reshare()) {
- put_in_secondary_storage(key, *value, true);
+ put_in_remote_storage(key, *value, true);
}
if (entry_receiver(std::move(*value))) {
return;
}
}
- get_from_secondary_storage(key, [&](util::Bytes&& data) {
- primary.put(key, type, data, true);
+ get_from_remote_storage(key, [&](util::Bytes&& data) {
+ local.put(key, type, data, true);
return entry_receiver(std::move(data));
});
}
@@ -262,8 +262,8 @@ Storage::put(const Digest& key,
{
MTR_SCOPE("storage", "put");
- primary.put(key, type, value);
- put_in_secondary_storage(key, value, false);
+ local.put(key, type, value);
+ put_in_remote_storage(key, value, false);
}
void
@@ -271,20 +271,20 @@ Storage::remove(const Digest& key, const core::CacheEntryType type)
{
MTR_SCOPE("storage", "remove");
- primary.remove(key, type);
- remove_from_secondary_storage(key);
+ local.remove(key, type);
+ remove_from_remote_storage(key);
}
bool
-Storage::has_secondary_storage() const
+Storage::has_remote_storage() const
{
- return !m_secondary_storages.empty();
+ return !m_remote_storages.empty();
}
std::string
-Storage::get_secondary_storage_config_for_logging() const
+Storage::get_remote_storage_config_for_logging() const
{
- auto configs = parse_storage_configs(m_config.secondary_storage());
+ auto configs = parse_storage_configs(m_config.remote_storage());
for (auto& config : configs) {
const auto storage = get_storage(config.params.url);
if (storage) {
@@ -301,33 +301,33 @@ redact_url_for_logging(Url& url_for_logging)
}
void
-Storage::add_secondary_storages()
+Storage::add_remote_storages()
{
- const auto configs = parse_storage_configs(m_config.secondary_storage());
+ const auto configs = parse_storage_configs(m_config.remote_storage());
for (const auto& config : configs) {
auto url_for_logging = config.params.url;
redact_url_for_logging(url_for_logging);
const auto storage = get_storage(config.params.url);
if (!storage) {
throw core::Error(
- FMT("unknown secondary storage URL: {}", url_for_logging.str()));
+ FMT("unknown remote storage URL: {}", url_for_logging.str()));
}
- m_secondary_storages.push_back(std::make_unique<SecondaryStorageEntry>(
- SecondaryStorageEntry{config, url_for_logging.str(), storage, {}}));
+ m_remote_storages.push_back(std::make_unique<RemoteStorageEntry>(
+ RemoteStorageEntry{config, url_for_logging.str(), storage, {}}));
}
}
void
Storage::mark_backend_as_failed(
- SecondaryStorageBackendEntry& backend_entry,
- const secondary::SecondaryStorage::Backend::Failure failure)
+ RemoteStorageBackendEntry& backend_entry,
+ const remote::RemoteStorage::Backend::Failure failure)
{
// The backend is expected to log details about the error.
backend_entry.failed = true;
- primary.increment_statistic(
- failure == secondary::SecondaryStorage::Backend::Failure::timeout
- ? core::Statistic::secondary_storage_timeout
- : core::Statistic::secondary_storage_error);
+ local.increment_statistic(
+ failure == remote::RemoteStorage::Backend::Failure::timeout
+ ? core::Statistic::remote_storage_timeout
+ : core::Statistic::remote_storage_error);
}
static double
@@ -342,7 +342,7 @@ to_half_open_unit_interval(uint64_t value)
static Url
get_shard_url(const Digest& key,
const std::string& url,
- const std::vector<SecondaryStorageShardConfig>& shards)
+ const std::vector<RemoteStorageShardConfig>& shards)
{
ASSERT(!shards.empty());
@@ -366,8 +366,8 @@ get_shard_url(const Digest& key,
return util::replace_first(url, "*", best_shard);
}
-SecondaryStorageBackendEntry*
-Storage::get_backend(SecondaryStorageEntry& entry,
+RemoteStorageBackendEntry*
+Storage::get_backend(RemoteStorageEntry& entry,
const Digest& key,
const std::string_view operation_description,
const bool for_writing)
@@ -397,7 +397,7 @@ Storage::get_backend(SecondaryStorageEntry& entry,
shard_params.url = shard_url;
try {
entry.backends.back().impl = entry.storage->create_backend(shard_params);
- } catch (const secondary::SecondaryStorage::Backend::Failed& e) {
+ } catch (const remote::RemoteStorage::Backend::Failed& e) {
LOG("Failed to construct backend for {}{}",
entry.url_for_logging,
std::string_view(e.what()).empty() ? "" : FMT(": {}", e.what()));
@@ -416,12 +416,12 @@ Storage::get_backend(SecondaryStorageEntry& entry,
}
void
-Storage::get_from_secondary_storage(const Digest& key,
- const EntryReceiver& entry_receiver)
+Storage::get_from_remote_storage(const Digest& key,
+ const EntryReceiver& entry_receiver)
{
- MTR_SCOPE("secondary_storage", "get");
+ MTR_SCOPE("remote_storage", "get");
- for (const auto& entry : m_secondary_storages) {
+ for (const auto& entry : m_remote_storages) {
auto backend = get_backend(*entry, key, "getting from", false);
if (!backend) {
continue;
@@ -441,7 +441,7 @@ Storage::get_from_secondary_storage(const Digest& key,
key.to_string(),
backend->url_for_logging,
ms);
- primary.increment_statistic(core::Statistic::secondary_storage_hit);
+ local.increment_statistic(core::Statistic::remote_storage_hit);
if (entry_receiver(std::move(*value))) {
return;
}
@@ -450,25 +450,25 @@ Storage::get_from_secondary_storage(const Digest& key,
key.to_string(),
backend->url_for_logging,
ms);
- primary.increment_statistic(core::Statistic::secondary_storage_miss);
+ local.increment_statistic(core::Statistic::remote_storage_miss);
}
}
}
void
-Storage::put_in_secondary_storage(const Digest& key,
- nonstd::span<const uint8_t> value,
- bool only_if_missing)
+Storage::put_in_remote_storage(const Digest& key,
+ nonstd::span<const uint8_t> value,
+ bool only_if_missing)
{
- MTR_SCOPE("secondary_storage", "put");
+ MTR_SCOPE("remote_storage", "put");
if (!core::CacheEntry::Header(value).self_contained) {
- LOG("Not putting {} in secondary storage since it's not self-contained",
+ LOG("Not putting {} in remote storage since it's not self-contained",
key.to_string());
return;
}
- for (const auto& entry : m_secondary_storages) {
+ for (const auto& entry : m_remote_storages) {
auto backend = get_backend(*entry, key, "putting in", true);
if (!backend) {
continue;
@@ -493,11 +493,11 @@ Storage::put_in_secondary_storage(const Digest& key,
}
void
-Storage::remove_from_secondary_storage(const Digest& key)
+Storage::remove_from_remote_storage(const Digest& key)
{
- MTR_SCOPE("secondary_storage", "remove");
+ MTR_SCOPE("remote_storage", "remove");
- for (const auto& entry : m_secondary_storages) {
+ for (const auto& entry : m_remote_storages) {
auto backend = get_backend(*entry, key, "removing from", true);
if (!backend) {
continue;
diff --git a/src/storage/Storage.hpp b/src/storage/Storage.hpp
index 152c8349..f3eeba56 100644
--- a/src/storage/Storage.hpp
+++ b/src/storage/Storage.hpp
@@ -19,8 +19,8 @@
#pragma once
#include <core/types.hpp>
-#include <storage/primary/PrimaryStorage.hpp>
-#include <storage/secondary/SecondaryStorage.hpp>
+#include <storage/local/LocalStorage.hpp>
+#include <storage/remote/RemoteStorage.hpp>
#include <storage/types.hpp>
#include <util/Bytes.hpp>
@@ -38,8 +38,8 @@ namespace storage {
std::string get_features();
-struct SecondaryStorageBackendEntry;
-struct SecondaryStorageEntry;
+struct RemoteStorageBackendEntry;
+struct RemoteStorageEntry;
class Storage
{
@@ -50,7 +50,7 @@ public:
void initialize();
void finalize();
- primary::PrimaryStorage primary;
+ local::LocalStorage local;
using EntryReceiver = std::function<bool(util::Bytes&&)>;
@@ -64,33 +64,31 @@ public:
void remove(const Digest& key, core::CacheEntryType type);
- bool has_secondary_storage() const;
- std::string get_secondary_storage_config_for_logging() const;
+ bool has_remote_storage() const;
+ std::string get_remote_storage_config_for_logging() const;
private:
const Config& m_config;
- std::vector<std::unique_ptr<SecondaryStorageEntry>> m_secondary_storages;
+ std::vector<std::unique_ptr<RemoteStorageEntry>> m_remote_storages;
- void add_secondary_storages();
+ void add_remote_storages();
- void
- mark_backend_as_failed(SecondaryStorageBackendEntry& backend_entry,
- secondary::SecondaryStorage::Backend::Failure failure);
+ void mark_backend_as_failed(RemoteStorageBackendEntry& backend_entry,
+ remote::RemoteStorage::Backend::Failure failure);
- SecondaryStorageBackendEntry*
- get_backend(SecondaryStorageEntry& entry,
- const Digest& key,
- std::string_view operation_description,
- const bool for_writing);
+ RemoteStorageBackendEntry* get_backend(RemoteStorageEntry& entry,
+ const Digest& key,
+ std::string_view operation_description,
+ const bool for_writing);
- void get_from_secondary_storage(const Digest& key,
- const EntryReceiver& entry_receiver);
+ void get_from_remote_storage(const Digest& key,
+ const EntryReceiver& entry_receiver);
- void put_in_secondary_storage(const Digest& key,
- nonstd::span<const uint8_t> value,
- bool only_if_missing);
+ void put_in_remote_storage(const Digest& key,
+ nonstd::span<const uint8_t> value,
+ bool only_if_missing);
- void remove_from_secondary_storage(const Digest& key);
+ void remove_from_remote_storage(const Digest& key);
};
} // namespace storage
diff --git a/src/storage/primary/CMakeLists.txt b/src/storage/local/CMakeLists.txt
index 67abaa93..88703781 100644
--- a/src/storage/primary/CMakeLists.txt
+++ b/src/storage/local/CMakeLists.txt
@@ -1,10 +1,10 @@
set(
sources
CacheFile.cpp
- PrimaryStorage.cpp
- PrimaryStorage_cleanup.cpp
- PrimaryStorage_compress.cpp
- PrimaryStorage_statistics.cpp
+ LocalStorage.cpp
+ LocalStorage_cleanup.cpp
+ LocalStorage_compress.cpp
+ LocalStorage_statistics.cpp
StatsFile.cpp
util.cpp
)
diff --git a/src/storage/primary/CacheFile.cpp b/src/storage/local/CacheFile.cpp
index a2716e8e..a2716e8e 100644
--- a/src/storage/primary/CacheFile.cpp
+++ b/src/storage/local/CacheFile.cpp
diff --git a/src/storage/primary/CacheFile.hpp b/src/storage/local/CacheFile.hpp
index e653363b..e653363b 100644
--- a/src/storage/primary/CacheFile.hpp
+++ b/src/storage/local/CacheFile.hpp
diff --git a/src/storage/primary/PrimaryStorage.cpp b/src/storage/local/LocalStorage.cpp
index 88154bff..3b34f70f 100644
--- a/src/storage/primary/PrimaryStorage.cpp
+++ b/src/storage/local/LocalStorage.cpp
@@ -16,7 +16,7 @@
// this program; if not, write to the Free Software Foundation, Inc., 51
// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-#include "PrimaryStorage.hpp"
+#include "LocalStorage.hpp"
#include <AtomicFile.hpp>
#include <Config.hpp>
@@ -27,7 +27,7 @@
#include <core/exceptions.hpp>
#include <core/wincompat.hpp>
#include <fmtmacros.hpp>
-#include <storage/primary/StatsFile.hpp>
+#include <storage/local/StatsFile.hpp>
#include <util/Duration.hpp>
#include <util/file.hpp>
@@ -37,7 +37,7 @@
using core::Statistic;
-namespace storage::primary {
+namespace storage::local {
// How often (in seconds) to scan $CCACHE_DIR/tmp for left-over temporary
// files.
@@ -91,12 +91,12 @@ calculate_wanted_cache_level(const uint64_t files_in_level_1)
return k_max_cache_levels;
}
-PrimaryStorage::PrimaryStorage(const Config& config) : m_config(config)
+LocalStorage::LocalStorage(const Config& config) : m_config(config)
{
}
void
-PrimaryStorage::finalize()
+LocalStorage::finalize()
{
if (m_config.temporary_dir() == m_config.default_temporary_dir()) {
clean_internal_tempdir();
@@ -178,13 +178,13 @@ PrimaryStorage::finalize()
}
std::optional<util::Bytes>
-PrimaryStorage::get(const Digest& key, const core::CacheEntryType type) const
+LocalStorage::get(const Digest& key, const core::CacheEntryType type) const
{
- MTR_SCOPE("primary_storage", "get");
+ MTR_SCOPE("local_storage", "get");
const auto cache_file = look_up_cache_file(key, type);
if (!cache_file.stat) {
- LOG("No {} {} in primary storage", key.to_string(), core::to_string(type));
+ LOG("No {} {} in local storage", key.to_string(), core::to_string(type));
return std::nullopt;
}
const auto value = util::read_file<util::Bytes>(cache_file.path);
@@ -193,8 +193,7 @@ PrimaryStorage::get(const Digest& key, const core::CacheEntryType type) const
return std::nullopt;
}
- LOG(
- "Retrieved {} from primary storage ({})", key.to_string(), cache_file.path);
+ LOG("Retrieved {} from local storage ({})", key.to_string(), cache_file.path);
// Update modification timestamp to save file from LRU cleanup.
util::set_timestamps(cache_file.path);
@@ -203,16 +202,16 @@ PrimaryStorage::get(const Digest& key, const core::CacheEntryType type) const
}
void
-PrimaryStorage::put(const Digest& key,
- const core::CacheEntryType type,
- nonstd::span<const uint8_t> value,
- bool only_if_missing)
+LocalStorage::put(const Digest& key,
+ const core::CacheEntryType type,
+ nonstd::span<const uint8_t> value,
+ bool only_if_missing)
{
- MTR_SCOPE("primary_storage", "put");
+ MTR_SCOPE("local_storage", "put");
const auto cache_file = look_up_cache_file(key, type);
if (only_if_missing && cache_file.stat) {
- LOG("Not storing {} in primary storage since it already exists",
+ LOG("Not storing {} in local storage since it already exists",
cache_file.path);
return;
}
@@ -246,7 +245,7 @@ PrimaryStorage::put(const Digest& key,
return;
}
- LOG("Stored {} in primary storage ({})", key.to_string(), cache_file.path);
+ LOG("Stored {} in local storage ({})", key.to_string(), cache_file.path);
auto& counter_updates = (type == core::CacheEntryType::manifest)
? m_manifest_counter_updates
@@ -264,27 +263,26 @@ PrimaryStorage::put(const Digest& key,
}
void
-PrimaryStorage::remove(const Digest& key, const core::CacheEntryType type)
+LocalStorage::remove(const Digest& key, const core::CacheEntryType type)
{
- MTR_SCOPE("primary_storage", "remove");
+ MTR_SCOPE("local_storage", "remove");
const auto cache_file = look_up_cache_file(key, type);
if (cache_file.stat) {
Util::unlink_safe(cache_file.path);
- LOG(
- "Removed {} from primary storage ({})", key.to_string(), cache_file.path);
+ LOG("Removed {} from local storage ({})", key.to_string(), cache_file.path);
} else {
- LOG("No {} to remove from primary storage", key.to_string());
+ LOG("No {} to remove from local storage", key.to_string());
}
}
std::string
-PrimaryStorage::get_raw_file_path(std::string_view result_path,
- uint8_t file_number)
+LocalStorage::get_raw_file_path(std::string_view result_path,
+ uint8_t file_number)
{
if (file_number >= 10) {
// To support more entries in the future, encode to [0-9a-z]. Note that
- // PrimaryStorage::evict currently assumes that the entry number is
+ // LocalStorage::evict currently assumes that the entry number is
// represented as one character.
throw core::Error(FMT("Too high raw file entry number: {}", file_number));
}
@@ -294,8 +292,8 @@ PrimaryStorage::get_raw_file_path(std::string_view result_path,
}
std::string
-PrimaryStorage::get_raw_file_path(const Digest& result_key,
- uint8_t file_number) const
+LocalStorage::get_raw_file_path(const Digest& result_key,
+ uint8_t file_number) const
{
const auto cache_file =
look_up_cache_file(result_key, core::CacheEntryType::result);
@@ -303,7 +301,7 @@ PrimaryStorage::get_raw_file_path(const Digest& result_key,
}
void
-PrimaryStorage::put_raw_files(
+LocalStorage::put_raw_files(
const Digest& key,
const std::vector<core::Result::Serializer::RawFile> raw_files)
{
@@ -333,23 +331,23 @@ PrimaryStorage::put_raw_files(
}
void
-PrimaryStorage::increment_statistic(const Statistic statistic,
- const int64_t value)
+LocalStorage::increment_statistic(const Statistic statistic,
+ const int64_t value)
{
m_result_counter_updates.increment(statistic, value);
}
void
-PrimaryStorage::increment_statistics(const core::StatisticsCounters& statistics)
+LocalStorage::increment_statistics(const core::StatisticsCounters& statistics)
{
m_result_counter_updates.increment(statistics);
}
// Private methods
-PrimaryStorage::LookUpCacheFileResult
-PrimaryStorage::look_up_cache_file(const Digest& key,
- const core::CacheEntryType type) const
+LocalStorage::LookUpCacheFileResult
+LocalStorage::look_up_cache_file(const Digest& key,
+ const core::CacheEntryType type) const
{
const auto key_string = FMT("{}{}", key.to_string(), suffix_from_type(type));
@@ -368,9 +366,9 @@ PrimaryStorage::look_up_cache_file(const Digest& key,
}
void
-PrimaryStorage::clean_internal_tempdir()
+LocalStorage::clean_internal_tempdir()
{
- MTR_SCOPE("primary_storage", "clean_internal_tempdir");
+ MTR_SCOPE("local_storage", "clean_internal_tempdir");
const auto now = util::TimePoint::now();
const auto cleaned_stamp = FMT("{}/.cleaned", m_config.temporary_dir());
@@ -398,7 +396,7 @@ PrimaryStorage::clean_internal_tempdir()
}
std::optional<core::StatisticsCounters>
-PrimaryStorage::update_stats_and_maybe_move_cache_file(
+LocalStorage::update_stats_and_maybe_move_cache_file(
const Digest& key,
const std::string& current_path,
const core::StatisticsCounters& counter_updates,
@@ -462,8 +460,8 @@ PrimaryStorage::update_stats_and_maybe_move_cache_file(
}
std::string
-PrimaryStorage::get_path_in_cache(const uint8_t level,
- const std::string_view name) const
+LocalStorage::get_path_in_cache(const uint8_t level,
+ const std::string_view name) const
{
ASSERT(level >= 1 && level <= 8);
ASSERT(name.length() >= level);
@@ -483,4 +481,4 @@ PrimaryStorage::get_path_in_cache(const uint8_t level,
return path;
}
-} // namespace storage::primary
+} // namespace storage::local
diff --git a/src/storage/primary/PrimaryStorage.hpp b/src/storage/local/LocalStorage.hpp
index 966f04e1..e2635645 100644
--- a/src/storage/primary/PrimaryStorage.hpp
+++ b/src/storage/local/LocalStorage.hpp
@@ -22,7 +22,7 @@
#include <core/Result.hpp>
#include <core/StatisticsCounters.hpp>
#include <core/types.hpp>
-#include <storage/primary/util.hpp>
+#include <storage/local/util.hpp>
#include <storage/types.hpp>
#include <util/Bytes.hpp>
#include <util/TimePoint.hpp>
@@ -36,7 +36,7 @@
class Config;
namespace storage {
-namespace primary {
+namespace local {
struct CompressionStatistics
{
@@ -46,10 +46,10 @@ struct CompressionStatistics
uint64_t on_disk_size;
};
-class PrimaryStorage
+class LocalStorage
{
public:
- PrimaryStorage(const Config& config);
+ LocalStorage(const Config& config);
void finalize();
@@ -85,7 +85,7 @@ public:
// files in the cache.
void zero_all_statistics();
- // Get statistics and last time of update for the whole primary storage cache.
+ // Get statistics and last time of update for the whole local storage cache.
std::pair<core::StatisticsCounters, util::TimePoint>
get_all_statistics() const;
@@ -163,10 +163,10 @@ private:
// --- Inline implementations ---
inline const core::StatisticsCounters&
-PrimaryStorage::get_statistics_updates() const
+LocalStorage::get_statistics_updates() const
{
return m_result_counter_updates;
}
-} // namespace primary
+} // namespace local
} // namespace storage
diff --git a/src/storage/primary/PrimaryStorage_cleanup.cpp b/src/storage/local/LocalStorage_cleanup.cpp
index 060720c3..3039e838 100644
--- a/src/storage/primary/PrimaryStorage_cleanup.cpp
+++ b/src/storage/local/LocalStorage_cleanup.cpp
@@ -17,7 +17,7 @@
// this program; if not, write to the Free Software Foundation, Inc., 51
// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-#include "PrimaryStorage.hpp"
+#include "LocalStorage.hpp"
#include <Config.hpp>
#include <Context.hpp>
@@ -28,9 +28,9 @@
#include <core/CacheEntry.hpp>
#include <core/exceptions.hpp>
#include <fmtmacros.hpp>
-#include <storage/primary/CacheFile.hpp>
-#include <storage/primary/StatsFile.hpp>
-#include <storage/primary/util.hpp>
+#include <storage/local/CacheFile.hpp>
+#include <storage/local/StatsFile.hpp>
+#include <storage/local/util.hpp>
#include <util/file.hpp>
#include <util/string.hpp>
@@ -42,7 +42,7 @@
using core::Statistic;
-namespace storage::primary {
+namespace storage::local {
static void
delete_file(const std::string& path,
@@ -80,9 +80,9 @@ update_counters(const std::string& dir,
}
void
-PrimaryStorage::evict(const ProgressReceiver& progress_receiver,
- std::optional<uint64_t> max_age,
- std::optional<std::string> namespace_)
+LocalStorage::evict(const ProgressReceiver& progress_receiver,
+ std::optional<uint64_t> max_age,
+ std::optional<std::string> namespace_)
{
for_each_level_1_subdir(
m_config.cache_dir(),
@@ -95,12 +95,12 @@ PrimaryStorage::evict(const ProgressReceiver& progress_receiver,
// Clean up one cache subdirectory.
void
-PrimaryStorage::clean_dir(const std::string& subdir,
- const uint64_t max_size,
- const uint64_t max_files,
- const std::optional<uint64_t> max_age,
- const std::optional<std::string> namespace_,
- const ProgressReceiver& progress_receiver)
+LocalStorage::clean_dir(const std::string& subdir,
+ const uint64_t max_size,
+ const uint64_t max_files,
+ const std::optional<uint64_t> max_age,
+ const std::optional<std::string> namespace_,
+ const ProgressReceiver& progress_receiver)
{
LOG("Cleaning up cache directory {}", subdir);
@@ -228,7 +228,7 @@ PrimaryStorage::clean_dir(const std::string& subdir,
// Clean up all cache subdirectories.
void
-PrimaryStorage::clean_all(const ProgressReceiver& progress_receiver)
+LocalStorage::clean_all(const ProgressReceiver& progress_receiver)
{
for_each_level_1_subdir(
m_config.cache_dir(),
@@ -267,9 +267,9 @@ wipe_dir(const std::string& subdir, const ProgressReceiver& progress_receiver)
// Wipe all cached files in all subdirectories.
void
-PrimaryStorage::wipe_all(const ProgressReceiver& progress_receiver)
+LocalStorage::wipe_all(const ProgressReceiver& progress_receiver)
{
for_each_level_1_subdir(m_config.cache_dir(), wipe_dir, progress_receiver);
}
-} // namespace storage::primary
+} // namespace storage::local
diff --git a/src/storage/primary/PrimaryStorage_compress.cpp b/src/storage/local/LocalStorage_compress.cpp
index b9ee286f..ecb325bc 100644
--- a/src/storage/primary/PrimaryStorage_compress.cpp
+++ b/src/storage/local/LocalStorage_compress.cpp
@@ -16,7 +16,7 @@
// this program; if not, write to the Free Software Foundation, Inc., 51
// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-#include "PrimaryStorage.hpp"
+#include "LocalStorage.hpp"
#include <AtomicFile.hpp>
#include <Context.hpp>
@@ -30,7 +30,7 @@
#include <core/exceptions.hpp>
#include <core/wincompat.hpp>
#include <fmtmacros.hpp>
-#include <storage/primary/StatsFile.hpp>
+#include <storage/local/StatsFile.hpp>
#include <util/expected.hpp>
#include <util/file.hpp>
#include <util/string.hpp>
@@ -45,7 +45,7 @@
#include <string>
#include <thread>
-namespace storage::primary {
+namespace storage::local {
namespace {
@@ -158,7 +158,7 @@ recompress_file(RecompressionStatistics& statistics,
}
CompressionStatistics
-PrimaryStorage::get_compression_statistics(
+LocalStorage::get_compression_statistics(
const ProgressReceiver& progress_receiver) const
{
CompressionStatistics cs{};
@@ -190,8 +190,8 @@ PrimaryStorage::get_compression_statistics(
}
void
-PrimaryStorage::recompress(const std::optional<int8_t> level,
- const ProgressReceiver& progress_receiver)
+LocalStorage::recompress(const std::optional<int8_t> level,
+ const ProgressReceiver& progress_receiver)
{
const size_t threads = std::thread::hardware_concurrency();
const size_t read_ahead = 2 * threads;
@@ -287,4 +287,4 @@ PrimaryStorage::recompress(const std::optional<int8_t> level,
PRINT(stdout, "Size change: {:>9s}\n", size_difference_str);
}
-} // namespace storage::primary
+} // namespace storage::local
diff --git a/src/storage/primary/PrimaryStorage_statistics.cpp b/src/storage/local/LocalStorage_statistics.cpp
index 833e13dd..b801313e 100644
--- a/src/storage/primary/PrimaryStorage_statistics.cpp
+++ b/src/storage/local/LocalStorage_statistics.cpp
@@ -16,16 +16,16 @@
// this program; if not, write to the Free Software Foundation, Inc., 51
// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-#include "PrimaryStorage.hpp"
+#include "LocalStorage.hpp"
#include <Config.hpp>
#include <core/Statistics.hpp>
#include <fmtmacros.hpp>
-#include <storage/primary/StatsFile.hpp>
+#include <storage/local/StatsFile.hpp>
#include <algorithm>
-namespace storage::primary {
+namespace storage::local {
static void
for_each_level_1_and_2_stats_file(
@@ -43,7 +43,7 @@ for_each_level_1_and_2_stats_file(
// Zero all statistics counters except those tracking cache size and number of
// files in the cache.
void
-PrimaryStorage::zero_all_statistics()
+LocalStorage::zero_all_statistics()
{
const auto now = util::TimePoint::now();
const auto zeroable_fields = core::Statistics::get_zeroable_fields();
@@ -59,9 +59,9 @@ PrimaryStorage::zero_all_statistics()
});
}
-// Get statistics and last time of update for the whole primary storage cache.
+// Get statistics and last time of update for the whole local storage cache.
std::pair<core::StatisticsCounters, util::TimePoint>
-PrimaryStorage::get_all_statistics() const
+LocalStorage::get_all_statistics() const
{
core::StatisticsCounters counters;
uint64_t zero_timestamp = 0;
@@ -81,4 +81,4 @@ PrimaryStorage::get_all_statistics() const
return std::make_pair(counters, last_updated);
}
-} // namespace storage::primary
+} // namespace storage::local
diff --git a/src/storage/primary/StatsFile.cpp b/src/storage/local/StatsFile.cpp
index 4d51b787..b00fc82e 100644
--- a/src/storage/primary/StatsFile.cpp
+++ b/src/storage/local/StatsFile.cpp
@@ -25,7 +25,7 @@
#include <util/LockFile.hpp>
#include <util/file.hpp>
-namespace storage::primary {
+namespace storage::local {
StatsFile::StatsFile(const std::string& path) : m_path(path)
{
@@ -88,4 +88,4 @@ StatsFile::update(
return counters;
}
-} // namespace storage::primary
+} // namespace storage::local
diff --git a/src/storage/primary/StatsFile.hpp b/src/storage/local/StatsFile.hpp
index 09716672..2ac4026c 100644
--- a/src/storage/primary/StatsFile.hpp
+++ b/src/storage/local/StatsFile.hpp
@@ -24,7 +24,7 @@
#include <optional>
#include <string>
-namespace storage::primary {
+namespace storage::local {
class StatsFile
{
@@ -45,4 +45,4 @@ private:
const std::string m_path;
};
-} // namespace storage::primary
+} // namespace storage::local
diff --git a/src/storage/primary/util.cpp b/src/storage/local/util.cpp
index 32679ebf..1035925d 100644
--- a/src/storage/primary/util.cpp
+++ b/src/storage/local/util.cpp
@@ -22,7 +22,7 @@
#include <fmtmacros.hpp>
#include <util/string.hpp>
-namespace storage::primary {
+namespace storage::local {
void
for_each_level_1_subdir(const std::string& cache_dir,
@@ -72,4 +72,4 @@ get_level_1_files(const std::string& dir,
return files;
}
-} // namespace storage::primary
+} // namespace storage::local
diff --git a/src/storage/primary/util.hpp b/src/storage/local/util.hpp
index f4e12011..8aa9b873 100644
--- a/src/storage/primary/util.hpp
+++ b/src/storage/local/util.hpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2021 Joel Rosdahl and other contributors
+// Copyright (C) 2021-2022 Joel Rosdahl and other contributors
//
// See doc/AUTHORS.adoc for a complete list of contributors.
//
@@ -18,14 +18,13 @@
#pragma once
-#include <storage/primary/CacheFile.hpp>
+#include <storage/local/CacheFile.hpp>
#include <functional>
#include <string>
#include <vector>
-namespace storage {
-namespace primary {
+namespace storage::local {
using ProgressReceiver = std::function<void(double progress)>;
using SubdirVisitor = std::function<void(
@@ -60,5 +59,4 @@ std::vector<CacheFile>
get_level_1_files(const std::string& dir,
const ProgressReceiver& progress_receiver);
-} // namespace primary
-} // namespace storage
+} // namespace storage::local
diff --git a/src/storage/secondary/CMakeLists.txt b/src/storage/remote/CMakeLists.txt
index 2c593faa..8d14eca1 100644
--- a/src/storage/secondary/CMakeLists.txt
+++ b/src/storage/remote/CMakeLists.txt
@@ -2,7 +2,7 @@ set(
sources
FileStorage.cpp
HttpStorage.cpp
- SecondaryStorage.cpp
+ RemoteStorage.cpp
)
if(REDIS_STORAGE_BACKEND)
diff --git a/src/storage/secondary/FileStorage.cpp b/src/storage/remote/FileStorage.cpp
index 91f301a1..b1c8a338 100644
--- a/src/storage/secondary/FileStorage.cpp
+++ b/src/storage/remote/FileStorage.cpp
@@ -35,11 +35,11 @@
#include <string_view>
-namespace storage::secondary {
+namespace storage::remote {
namespace {
-class FileStorageBackend : public SecondaryStorage::Backend
+class FileStorageBackend : public RemoteStorage::Backend
{
public:
FileStorageBackend(const Params& params);
@@ -104,7 +104,7 @@ FileStorageBackend::FileStorageBackend(const Params& params)
}
}
-nonstd::expected<std::optional<util::Bytes>, SecondaryStorage::Backend::Failure>
+nonstd::expected<std::optional<util::Bytes>, RemoteStorage::Backend::Failure>
FileStorageBackend::get(const Digest& key)
{
const auto path = get_entry_path(key);
@@ -129,7 +129,7 @@ FileStorageBackend::get(const Digest& key)
return std::move(*value);
}
-nonstd::expected<bool, SecondaryStorage::Backend::Failure>
+nonstd::expected<bool, RemoteStorage::Backend::Failure>
FileStorageBackend::put(const Digest& key,
const nonstd::span<const uint8_t> value,
const bool only_if_missing)
@@ -165,7 +165,7 @@ FileStorageBackend::put(const Digest& key,
}
}
-nonstd::expected<bool, SecondaryStorage::Backend::Failure>
+nonstd::expected<bool, RemoteStorage::Backend::Failure>
FileStorageBackend::remove(const Digest& key)
{
return Util::unlink_safe(get_entry_path(key));
@@ -191,10 +191,10 @@ FileStorageBackend::get_entry_path(const Digest& key) const
} // namespace
-std::unique_ptr<SecondaryStorage::Backend>
+std::unique_ptr<RemoteStorage::Backend>
FileStorage::create_backend(const Backend::Params& params) const
{
return std::make_unique<FileStorageBackend>(params);
}
-} // namespace storage::secondary
+} // namespace storage::remote
diff --git a/src/storage/secondary/FileStorage.hpp b/src/storage/remote/FileStorage.hpp
index 771941f9..ae72cfdd 100644
--- a/src/storage/secondary/FileStorage.hpp
+++ b/src/storage/remote/FileStorage.hpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2021 Joel Rosdahl and other contributors
+// Copyright (C) 2021-2022 Joel Rosdahl and other contributors
//
// See doc/AUTHORS.adoc for a complete list of contributors.
//
@@ -18,17 +18,15 @@
#pragma once
-#include <storage/secondary/SecondaryStorage.hpp>
+#include <storage/remote/RemoteStorage.hpp>
-namespace storage {
-namespace secondary {
+namespace storage::remote {
-class FileStorage : public SecondaryStorage
+class FileStorage : public RemoteStorage
{
public:
std::unique_ptr<Backend>
create_backend(const Backend::Params& params) const override;
};
-} // namespace secondary
-} // namespace storage
+} // namespace storage::remote
diff --git a/src/storage/secondary/HttpStorage.cpp b/src/storage/remote/HttpStorage.cpp
index 98748c9a..fce8270f 100644
--- a/src/storage/secondary/HttpStorage.cpp
+++ b/src/storage/remote/HttpStorage.cpp
@@ -32,11 +32,11 @@
#include <string_view>
-namespace storage::secondary {
+namespace storage::remote {
namespace {
-class HttpStorageBackend : public SecondaryStorage::Backend
+class HttpStorageBackend : public RemoteStorage::Backend
{
public:
HttpStorageBackend(const Params& params);
@@ -144,7 +144,7 @@ HttpStorageBackend::HttpStorageBackend(const Params& params)
m_http_client.set_write_timeout(operation_timeout);
}
-nonstd::expected<std::optional<util::Bytes>, SecondaryStorage::Backend::Failure>
+nonstd::expected<std::optional<util::Bytes>, RemoteStorage::Backend::Failure>
HttpStorageBackend::get(const Digest& key)
{
const auto url_path = get_entry_path(key);
@@ -166,7 +166,7 @@ HttpStorageBackend::get(const Digest& key)
return util::Bytes(result->body.data(), result->body.size());
}
-nonstd::expected<bool, SecondaryStorage::Backend::Failure>
+nonstd::expected<bool, RemoteStorage::Backend::Failure>
HttpStorageBackend::put(const Digest& key,
const nonstd::span<const uint8_t> value,
const bool only_if_missing)
@@ -217,7 +217,7 @@ HttpStorageBackend::put(const Digest& key,
return true;
}
-nonstd::expected<bool, SecondaryStorage::Backend::Failure>
+nonstd::expected<bool, RemoteStorage::Backend::Failure>
HttpStorageBackend::remove(const Digest& key)
{
const auto url_path = get_entry_path(key);
@@ -271,7 +271,7 @@ HttpStorageBackend::get_entry_path(const Digest& key) const
} // namespace
-std::unique_ptr<SecondaryStorage::Backend>
+std::unique_ptr<RemoteStorage::Backend>
HttpStorage::create_backend(const Backend::Params& params) const
{
return std::make_unique<HttpStorageBackend>(params);
@@ -296,4 +296,4 @@ HttpStorage::redact_secrets(Backend::Params& params) const
}
}
-} // namespace storage::secondary
+} // namespace storage::remote
diff --git a/src/storage/secondary/HttpStorage.hpp b/src/storage/remote/HttpStorage.hpp
index 32f6345a..dbe45c9d 100644
--- a/src/storage/secondary/HttpStorage.hpp
+++ b/src/storage/remote/HttpStorage.hpp
@@ -18,11 +18,11 @@
#pragma once
-#include <storage/secondary/SecondaryStorage.hpp>
+#include <storage/remote/RemoteStorage.hpp>
-namespace storage::secondary {
+namespace storage::remote {
-class HttpStorage : public SecondaryStorage
+class HttpStorage : public RemoteStorage
{
public:
std::unique_ptr<Backend>
@@ -31,4 +31,4 @@ public:
void redact_secrets(Backend::Params& params) const override;
};
-} // namespace storage::secondary
+} // namespace storage::remote
diff --git a/src/storage/secondary/RedisStorage.cpp b/src/storage/remote/RedisStorage.cpp
index 43928386..e6c8ce3b 100644
--- a/src/storage/secondary/RedisStorage.cpp
+++ b/src/storage/remote/RedisStorage.cpp
@@ -46,7 +46,7 @@
#include <map>
#include <memory>
-namespace storage::secondary {
+namespace storage::remote {
namespace {
@@ -55,10 +55,10 @@ using RedisReply = std::unique_ptr<redisReply, decltype(&freeReplyObject)>;
const uint32_t DEFAULT_PORT = 6379;
-class RedisStorageBackend : public SecondaryStorage::Backend
+class RedisStorageBackend : public RemoteStorage::Backend
{
public:
- RedisStorageBackend(const SecondaryStorage::Backend::Params& params);
+ RedisStorageBackend(const RemoteStorage::Backend::Params& params);
nonstd::expected<std::optional<util::Bytes>, Failure>
get(const Digest& key) override;
@@ -157,7 +157,7 @@ is_timeout(int err)
#endif
}
-nonstd::expected<std::optional<util::Bytes>, SecondaryStorage::Backend::Failure>
+nonstd::expected<std::optional<util::Bytes>, RemoteStorage::Backend::Failure>
RedisStorageBackend::get(const Digest& key)
{
const auto key_string = get_key_string(key);
@@ -175,7 +175,7 @@ RedisStorageBackend::get(const Digest& key)
}
}
-nonstd::expected<bool, SecondaryStorage::Backend::Failure>
+nonstd::expected<bool, RemoteStorage::Backend::Failure>
RedisStorageBackend::put(const Digest& key,
nonstd::span<const uint8_t> value,
bool only_if_missing)
@@ -208,7 +208,7 @@ RedisStorageBackend::put(const Digest& key,
}
}
-nonstd::expected<bool, SecondaryStorage::Backend::Failure>
+nonstd::expected<bool, RemoteStorage::Backend::Failure>
RedisStorageBackend::remove(const Digest& key)
{
const auto key_string = get_key_string(key);
@@ -314,7 +314,7 @@ RedisStorageBackend::authenticate(const Url& url)
}
}
-nonstd::expected<RedisReply, SecondaryStorage::Backend::Failure>
+nonstd::expected<RedisReply, RemoteStorage::Backend::Failure>
RedisStorageBackend::redis_command(const char* format, ...)
{
va_list ap;
@@ -342,7 +342,7 @@ RedisStorageBackend::get_key_string(const Digest& digest) const
} // namespace
-std::unique_ptr<SecondaryStorage::Backend>
+std::unique_ptr<RemoteStorage::Backend>
RedisStorage::create_backend(const Backend::Params& params) const
{
return std::make_unique<RedisStorageBackend>(params);
@@ -364,4 +364,4 @@ RedisStorage::redact_secrets(Backend::Params& params) const
}
}
-} // namespace storage::secondary
+} // namespace storage::remote
diff --git a/src/storage/secondary/RedisStorage.hpp b/src/storage/remote/RedisStorage.hpp
index 01759df7..530cb431 100644
--- a/src/storage/secondary/RedisStorage.hpp
+++ b/src/storage/remote/RedisStorage.hpp
@@ -18,11 +18,11 @@
#pragma once
-#include <storage/secondary/SecondaryStorage.hpp>
+#include <storage/remote/RemoteStorage.hpp>
-namespace storage::secondary {
+namespace storage::remote {
-class RedisStorage : public SecondaryStorage
+class RedisStorage : public RemoteStorage
{
public:
std::unique_ptr<Backend>
@@ -31,4 +31,4 @@ public:
void redact_secrets(Backend::Params& params) const override;
};
-} // namespace storage::secondary
+} // namespace storage::remote
diff --git a/src/storage/secondary/SecondaryStorage.cpp b/src/storage/remote/RemoteStorage.cpp
index a9ae934f..e3103901 100644
--- a/src/storage/secondary/SecondaryStorage.cpp
+++ b/src/storage/remote/RemoteStorage.cpp
@@ -16,24 +16,24 @@
// this program; if not, write to the Free Software Foundation, Inc., 51
// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-#include "SecondaryStorage.hpp"
+#include "RemoteStorage.hpp"
#include <util/expected.hpp>
#include <util/string.hpp>
-namespace storage::secondary {
+namespace storage::remote {
bool
-SecondaryStorage::Backend::is_framework_attribute(const std::string& name)
+RemoteStorage::Backend::is_framework_attribute(const std::string& name)
{
return name == "read-only" || name == "shards";
}
std::chrono::milliseconds
-SecondaryStorage::Backend::parse_timeout_attribute(const std::string& value)
+RemoteStorage::Backend::parse_timeout_attribute(const std::string& value)
{
return std::chrono::milliseconds(util::value_or_throw<Failed>(
util::parse_unsigned(value, 1, 60 * 1000, "timeout")));
}
-} // namespace storage::secondary
+} // namespace storage::remote
diff --git a/src/storage/secondary/SecondaryStorage.hpp b/src/storage/remote/RemoteStorage.hpp
index ffa3f972..a80237ba 100644
--- a/src/storage/secondary/SecondaryStorage.hpp
+++ b/src/storage/remote/RemoteStorage.hpp
@@ -33,14 +33,14 @@
class Digest;
-namespace storage::secondary {
+namespace storage::remote {
constexpr auto k_redacted_password = "********";
const auto k_default_connect_timeout = std::chrono::milliseconds{100};
const auto k_default_operation_timeout = std::chrono::milliseconds{10000};
-// This class defines the API that a secondary storage must implement.
-class SecondaryStorage
+// This class defines the API that a remote storage must implement.
+class RemoteStorage
{
public:
class Backend
@@ -95,7 +95,7 @@ public:
// removed, otherwise false.
virtual nonstd::expected<bool, Failure> remove(const Digest& key) = 0;
- // Determine whether an attribute is handled by the secondary storage
+ // Determine whether an attribute is handled by the remote storage
// framework itself.
static bool is_framework_attribute(const std::string& name);
@@ -104,7 +104,7 @@ public:
parse_timeout_attribute(const std::string& value);
};
- virtual ~SecondaryStorage() = default;
+ virtual ~RemoteStorage() = default;
// Create an instance of the backend. The instance is created just before the
// first call to a backend method, so the backend constructor can open a
@@ -121,27 +121,26 @@ public:
// --- Inline implementations ---
inline void
-SecondaryStorage::redact_secrets(
- SecondaryStorage::Backend::Params& /*config*/) const
+RemoteStorage::redact_secrets(RemoteStorage::Backend::Params& /*config*/) const
{
}
-inline SecondaryStorage::Backend::Failed::Failed(Failure failure)
+inline RemoteStorage::Backend::Failed::Failed(Failure failure)
: Failed("", failure)
{
}
-inline SecondaryStorage::Backend::Failed::Failed(const std::string& message,
- Failure failure)
+inline RemoteStorage::Backend::Failed::Failed(const std::string& message,
+ Failure failure)
: std::runtime_error::runtime_error(message),
m_failure(failure)
{
}
-inline SecondaryStorage::Backend::Failure
-SecondaryStorage::Backend::Failed::failure() const
+inline RemoteStorage::Backend::Failure
+RemoteStorage::Backend::Failed::failure() const
{
return m_failure;
}
-} // namespace storage::secondary
+} // namespace storage::remote
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 1cacb26d..4cd38a38 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -52,12 +52,12 @@ addtest(profiling_gcc)
addtest(profiling_hip_clang)
addtest(readonly)
addtest(readonly_direct)
+addtest(remote_file)
+addtest(remote_http)
+addtest(remote_redis)
+addtest(remote_redis_unix)
+addtest(remote_url)
addtest(sanitize_blacklist)
-addtest(secondary_file)
-addtest(secondary_http)
-addtest(secondary_redis)
-addtest(secondary_redis_unix)
-addtest(secondary_url)
addtest(serialize_diagnostics)
addtest(source_date_epoch)
addtest(split_dwarf)
diff --git a/test/suites/base.bash b/test/suites/base.bash
index c7272363..39dcdf04 100644
--- a/test/suites/base.bash
+++ b/test/suites/base.bash
@@ -9,10 +9,10 @@ base_tests() {
expect_stat cache_miss 1
expect_stat direct_cache_miss 0
expect_stat preprocessed_cache_miss 1
- expect_stat primary_storage_hit 0
- expect_stat primary_storage_miss 1
- expect_stat secondary_storage_hit 0
- expect_stat secondary_storage_miss 0
+ expect_stat local_storage_hit 0
+ expect_stat local_storage_miss 1
+ expect_stat remote_storage_hit 0
+ expect_stat remote_storage_miss 0
expect_stat files_in_cache 1
expect_equal_object_files reference_test1.o test1.o
@@ -21,10 +21,10 @@ base_tests() {
expect_stat cache_miss 1
expect_stat direct_cache_miss 0
expect_stat preprocessed_cache_miss 1
- expect_stat primary_storage_hit 1
- expect_stat primary_storage_miss 1
- expect_stat secondary_storage_hit 0
- expect_stat secondary_storage_miss 0
+ expect_stat local_storage_hit 1
+ expect_stat local_storage_miss 1
+ expect_stat remote_storage_hit 0
+ expect_stat remote_storage_miss 0
expect_stat files_in_cache 1
expect_equal_object_files reference_test1.o test1.o
diff --git a/test/suites/direct.bash b/test/suites/direct.bash
index a0cecf37..65969bac 100644
--- a/test/suites/direct.bash
+++ b/test/suites/direct.bash
@@ -35,10 +35,10 @@ SUITE_direct() {
expect_stat cache_miss 1
expect_stat direct_cache_miss 1
expect_stat preprocessed_cache_miss 1
- expect_stat primary_storage_hit 0
- expect_stat primary_storage_miss 2 # result + manifest
- expect_stat secondary_storage_hit 0
- expect_stat secondary_storage_miss 0
+ expect_stat local_storage_hit 0
+ expect_stat local_storage_miss 2 # result + manifest
+ expect_stat remote_storage_hit 0
+ expect_stat remote_storage_miss 0
expect_stat files_in_cache 2 # result + manifest
expect_equal_object_files reference_test.o test.o
@@ -51,10 +51,10 @@ SUITE_direct() {
expect_stat cache_miss 1
expect_stat direct_cache_miss 1
expect_stat preprocessed_cache_miss 1
- expect_stat primary_storage_hit 2 # result + manifest
- expect_stat primary_storage_miss 2 # result + manifest
- expect_stat secondary_storage_hit 0
- expect_stat secondary_storage_miss 0
+ expect_stat local_storage_hit 2 # result + manifest
+ expect_stat local_storage_miss 2 # result + manifest
+ expect_stat remote_storage_hit 0
+ expect_stat remote_storage_miss 0
expect_stat files_in_cache 2
expect_equal_object_files reference_test.o test.o
expect_newer_than $manifest_file test.c
diff --git a/test/suites/remote_file.bash b/test/suites/remote_file.bash
new file mode 100644
index 00000000..af37b724
--- /dev/null
+++ b/test/suites/remote_file.bash
@@ -0,0 +1,422 @@
+# This test suite verified both the file storage backend and the remote
+# storage framework itself.
+
+SUITE_remote_file_SETUP() {
+ unset CCACHE_NODIRECT
+ export CCACHE_REMOTE_STORAGE="file:$PWD/remote"
+
+ generate_code 1 test.c
+}
+
+SUITE_remote_file() {
+ # -------------------------------------------------------------------------
+ TEST "Base case"
+
+ # Compile and send result to local and remote storage.
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 0
+ expect_stat cache_miss 1
+ expect_stat files_in_cache 2
+ expect_stat local_storage_hit 0
+ expect_stat local_storage_miss 2 # result + manifest
+ expect_stat remote_storage_hit 0
+ expect_stat remote_storage_miss 2 # result + manifest
+ expect_exists remote/CACHEDIR.TAG
+ subdirs=$(find remote -type d | wc -l)
+ if [ "${subdirs}" -lt 2 ]; then # "remote" itself counts as one
+ test_failed "Expected subdirectories in remote"
+ fi
+ expect_file_count 3 '*' remote # CACHEDIR.TAG + result + manifest
+
+ # Get result from local storage.
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 1
+ expect_stat cache_miss 1
+ expect_stat local_storage_hit 2 # result + manifest
+ expect_stat local_storage_miss 2 # result + manifest
+ expect_stat remote_storage_hit 0
+ expect_stat remote_storage_miss 2 # result + manifest
+ expect_stat files_in_cache 2
+ expect_file_count 3 '*' remote # CACHEDIR.TAG + result + manifest
+
+ # Clear local storage.
+ $CCACHE -C >/dev/null
+ expect_stat files_in_cache 0
+ expect_file_count 3 '*' remote # CACHEDIR.TAG + result + manifest
+
+ # Get result from remote storage, copying it to local storage.
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 2
+ expect_stat cache_miss 1
+ expect_stat local_storage_hit 2
+ expect_stat local_storage_miss 4 # 2 * (result + manifest)
+ expect_stat remote_storage_hit 2 # result + manifest
+ expect_stat remote_storage_miss 2 # result + manifest
+ expect_stat files_in_cache 2 # fetched from remote
+ expect_file_count 3 '*' remote # CACHEDIR.TAG + result + manifest
+
+ # Get result from local storage again.
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 3
+ expect_stat cache_miss 1
+ expect_stat local_storage_hit 4
+ expect_stat local_storage_miss 4 # 2 * (result + manifest)
+ expect_stat remote_storage_hit 2 # result + manifest
+ expect_stat remote_storage_miss 2 # result + manifest
+ expect_stat files_in_cache 2 # fetched from remote
+ expect_file_count 3 '*' remote # CACHEDIR.TAG + result + manifest
+
+ # -------------------------------------------------------------------------
+ TEST "Flat layout"
+
+ CCACHE_REMOTE_STORAGE+="|layout=flat"
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 0
+ expect_stat cache_miss 1
+ expect_stat files_in_cache 2
+ expect_exists remote/CACHEDIR.TAG
+ subdirs=$(find remote -type d | wc -l)
+ if [ "${subdirs}" -ne 1 ]; then # "remote" itself counts as one
+ test_failed "Expected no subdirectories in remote"
+ fi
+ expect_file_count 3 '*' remote # CACHEDIR.TAG + result + manifest
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 1
+ expect_stat cache_miss 1
+ expect_stat files_in_cache 2
+ expect_file_count 3 '*' remote # CACHEDIR.TAG + result + manifest
+
+ $CCACHE -C >/dev/null
+ expect_stat files_in_cache 0
+ expect_file_count 3 '*' remote # CACHEDIR.TAG + result + manifest
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 2
+ expect_stat cache_miss 1
+ expect_stat files_in_cache 2 # fetched from remote
+ expect_file_count 3 '*' remote # CACHEDIR.TAG + result + manifest
+
+ # -------------------------------------------------------------------------
+ TEST "Two directories"
+
+ CCACHE_REMOTE_STORAGE+=" file://$PWD/remote_2"
+ mkdir remote_2
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 0
+ expect_stat cache_miss 1
+ expect_stat files_in_cache 2
+ expect_file_count 3 '*' remote # CACHEDIR.TAG + result + manifest
+ expect_file_count 3 '*' remote_2 # CACHEDIR.TAG + result + manifest
+
+ $CCACHE -C >/dev/null
+ expect_stat files_in_cache 0
+ expect_file_count 3 '*' remote # CACHEDIR.TAG + result + manifest
+ expect_file_count 3 '*' remote_2 # CACHEDIR.TAG + result + manifest
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 1
+ expect_stat cache_miss 1
+ expect_stat files_in_cache 2 # fetched from remote
+ expect_file_count 3 '*' remote # CACHEDIR.TAG + result + manifest
+ expect_file_count 3 '*' remote_2 # CACHEDIR.TAG + result + manifest
+
+ $CCACHE -C >/dev/null
+ expect_stat files_in_cache 0
+
+ rm -r remote/??
+ expect_file_count 1 '*' remote # CACHEDIR.TAG
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 2
+ expect_stat cache_miss 1
+ expect_stat files_in_cache 2 # fetched from remote_2
+ expect_file_count 1 '*' remote # CACHEDIR.TAG
+ expect_file_count 3 '*' remote_2 # CACHEDIR.TAG + result + manifest
+
+ # -------------------------------------------------------------------------
+ TEST "Read-only"
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 0
+ expect_stat cache_miss 1
+ expect_stat files_in_cache 2
+ expect_file_count 3 '*' remote # CACHEDIR.TAG + result + manifest
+
+ $CCACHE -C >/dev/null
+ expect_stat files_in_cache 0
+ expect_file_count 3 '*' remote # CACHEDIR.TAG + result + manifest
+
+ CCACHE_REMOTE_STORAGE+="|read-only"
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 1
+ expect_stat cache_miss 1
+ expect_stat files_in_cache 2 # fetched from remote
+ expect_file_count 3 '*' remote # CACHEDIR.TAG + result + manifest
+
+ echo 'int x;' >> test.c
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 1
+ expect_stat cache_miss 2
+ expect_stat files_in_cache 4
+ expect_file_count 3 '*' remote # CACHEDIR.TAG + result + manifest
+
+ # -------------------------------------------------------------------------
+ TEST "umask"
+
+ export CCACHE_UMASK=042
+ CCACHE_REMOTE_STORAGE="file://$PWD/remote|umask=024"
+ rm -rf remote
+ $CCACHE_COMPILE -c test.c
+ expect_perm remote drwxr-x-wx # 777 & 024
+ expect_perm remote/CACHEDIR.TAG -rw-r---w- # 666 & 024
+ result_file=$(find $CCACHE_DIR -name '*R')
+ expect_perm "$(dirname "${result_file}")" drwx-wxr-x # 777 & 042
+ expect_perm "${result_file}" -rw--w-r-- # 666 & 042
+
+ CCACHE_REMOTE_STORAGE="file://$PWD/remote|umask=026"
+ $CCACHE -C >/dev/null
+ rm -rf remote
+ $CCACHE_COMPILE -c test.c
+ expect_perm remote drwxr-x--x # 777 & 026
+ expect_perm remote/CACHEDIR.TAG -rw-r----- # 666 & 026
+
+ # -------------------------------------------------------------------------
+ TEST "Sharding"
+
+ CCACHE_REMOTE_STORAGE="file://$PWD/remote/*|shards=a,b(2)"
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 0
+ expect_stat cache_miss 1
+ expect_stat files_in_cache 2
+ if [ ! -d remote/a ] && [ ! -d remote/b ]; then
+ test_failed "Expected remote/a or remote/b to exist"
+ fi
+
+ # -------------------------------------------------------------------------
+ TEST "Reshare"
+
+ CCACHE_REMOTE_STORAGE="" $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 0
+ expect_stat cache_miss 1
+ expect_stat files_in_cache 2
+ expect_stat local_storage_hit 0
+ expect_stat local_storage_miss 2
+ expect_stat remote_storage_hit 0
+ expect_stat remote_storage_miss 0
+ expect_missing remote
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 1
+ expect_stat cache_miss 1
+ expect_stat local_storage_hit 2
+ expect_stat local_storage_miss 2
+ expect_stat remote_storage_hit 0
+ expect_stat remote_storage_miss 0
+ expect_missing remote
+
+ CCACHE_RESHARE=1 $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 2
+ expect_stat cache_miss 1
+ expect_stat local_storage_hit 4
+ expect_stat local_storage_miss 2
+ expect_stat remote_storage_hit 0
+ expect_stat remote_storage_miss 0
+ expect_file_count 3 '*' remote # CACHEDIR.TAG + result + manifest
+
+ $CCACHE -C >/dev/null
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 3
+ expect_stat cache_miss 1
+ expect_stat local_storage_hit 4
+ expect_stat local_storage_miss 4
+ expect_stat remote_storage_hit 2
+ expect_stat remote_storage_miss 0
+ expect_file_count 3 '*' remote # CACHEDIR.TAG + result + manifest
+
+ # -------------------------------------------------------------------------
+ TEST "Recache"
+
+ CCACHE_RECACHE=1 $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 0
+ expect_stat direct_cache_miss 0
+ expect_stat cache_miss 0
+ expect_stat recache 1
+ expect_stat files_in_cache 2
+ expect_stat local_storage_hit 0
+ expect_stat local_storage_miss 1 # Try to read manifest for updating
+ expect_stat remote_storage_hit 0
+ expect_stat remote_storage_miss 1 # Try to read manifest for updating
+ expect_file_count 3 '*' remote # CACHEDIR.TAG + result + manifest
+
+ $CCACHE -C >/dev/null
+ expect_stat files_in_cache 0
+ expect_file_count 3 '*' remote # CACHEDIR.TAG + result + manifest
+
+ CCACHE_RECACHE=1 $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 0
+ expect_stat direct_cache_miss 0
+ expect_stat cache_miss 0
+ expect_stat recache 2
+ expect_stat files_in_cache 2
+ expect_stat local_storage_hit 0
+ expect_stat local_storage_miss 2 # Try to read manifest for updating
+ expect_stat remote_storage_hit 1 # Read manifest for updating
+ expect_stat remote_storage_miss 1
+
+ # -------------------------------------------------------------------------
+ if touch test.c && ln test.c test-if-fs-supports-hard-links.c 2>/dev/null; then
+ TEST "Don't reshare results with raw files"
+
+ CCACHE_REMOTE_STORAGE= CCACHE_HARDLINK=1 $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 0
+ expect_stat cache_miss 1
+ expect_stat files_in_cache 3
+ expect_stat local_storage_hit 0
+ expect_stat local_storage_miss 2 # result + manifest
+ expect_stat remote_storage_hit 0
+ expect_stat remote_storage_miss 0
+
+ CCACHE_RESHARE=1 $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 1
+ expect_stat cache_miss 1
+ expect_stat files_in_cache 3
+ expect_stat local_storage_hit 2
+ expect_stat local_storage_miss 2 # result + manifest
+ expect_stat remote_storage_hit 0
+ expect_stat remote_storage_miss 0
+ expect_file_count 2 '*' remote # CACHEDIR.TAG + manifest, not result
+ fi
+
+ # -------------------------------------------------------------------------
+ TEST "Manifest handling"
+
+ echo 'int x;' >test.h
+ backdate test.h
+ echo '#include "test.h"' >test.c
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 0
+ expect_stat cache_miss 1
+ expect_stat local_storage_hit 0
+ expect_stat local_storage_miss 2 # miss: manifest + result
+ expect_stat remote_storage_hit 0
+ expect_stat remote_storage_miss 2 # miss: manifest + result
+
+ # Both local and remote now have an "int x;" key in the manifest.
+
+ echo 'int y;' >test.h
+ backdate test.h
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 0
+ expect_stat cache_miss 2
+ expect_stat local_storage_hit 1 # hit: manifest without key
+ expect_stat local_storage_miss 3 # miss: result
+ expect_stat remote_storage_hit 1 # his: manifest without key
+ expect_stat remote_storage_miss 3 # miss: result
+
+ # Both local and remote now have "int x;" and "int y;" keys in the manifest.
+
+ $CCACHE -C >/dev/null
+
+ # Now only remote has "int x;" and "int y;" keys in the manifest. We
+ # should now be able to get remote hit without involving local.
+
+ echo 'int x;' >test.h
+ backdate test.h
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 1
+ expect_stat cache_miss 2
+ expect_stat local_storage_hit 1
+ expect_stat local_storage_miss 5 # miss: manifest + result
+ expect_stat remote_storage_hit 3 # hit: manifest + result
+ expect_stat remote_storage_miss 3
+
+ # Should be able to get remote hit without involving local.
+
+ echo 'int y;' >test.h
+ backdate test.h
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 2
+ expect_stat cache_miss 2
+ expect_stat local_storage_hit 2 # hit: manifest with key (downloaded from previous step)
+ expect_stat local_storage_miss 6 # miss: result
+ expect_stat remote_storage_hit 4 # hit: result
+ expect_stat remote_storage_miss 3
+
+ # -------------------------------------------------------------------------
+ TEST "Manifest merging"
+
+ echo 'int x;' >test.h
+ backdate test.h
+ echo '#include "test.h"' >test.c
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 0
+ expect_stat cache_miss 1
+ expect_stat local_storage_hit 0
+ expect_stat local_storage_miss 2 # miss: manifest + result
+ expect_stat remote_storage_hit 0
+ expect_stat remote_storage_miss 2 # miss: manifest + result
+
+ $CCACHE -C >/dev/null
+
+ # Now remote has an "int x;" key in the manifest and local has none.
+
+ echo 'int y;' >test.h
+ backdate test.h
+
+ CCACHE_REMOTE_STORAGE= $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 0
+ expect_stat cache_miss 2
+ expect_stat local_storage_hit 0
+ expect_stat local_storage_miss 4 # miss: manifest + result
+ expect_stat remote_storage_hit 0
+ expect_stat remote_storage_miss 2
+
+ # Now local has "int y;" while remote still has "int x;".
+
+ echo 'int x;' >test.h
+ backdate test.h
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 1
+ expect_stat cache_miss 2
+ expect_stat local_storage_hit 1 # hit: manifest without key
+ expect_stat local_storage_miss 5 # miss: result
+ expect_stat remote_storage_hit 2 # hit: manifest + result
+ expect_stat remote_storage_miss 2
+
+ # Local manifest with "int y;" was merged with remote's "int x;" above, so
+ # we should now be able to get "int x;" and "int y;" hits locally.
+
+ echo 'int y;' >test.h
+ backdate test.h
+
+ CCACHE_REMOTE_STORAGE= $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 2
+ expect_stat cache_miss 2
+ expect_stat local_storage_hit 3 # hit: manifest + result
+ expect_stat local_storage_miss 5
+ expect_stat remote_storage_hit 2
+ expect_stat remote_storage_miss 2
+
+ echo 'int x;' >test.h
+ backdate test.h
+
+ CCACHE_REMOTE_STORAGE= $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 3
+ expect_stat cache_miss 2
+ expect_stat local_storage_hit 5 # hit: manifest + result
+ expect_stat local_storage_miss 5
+ expect_stat remote_storage_hit 2
+ expect_stat remote_storage_miss 2
+}
diff --git a/test/suites/secondary_http.bash b/test/suites/remote_http.bash
index 825b290f..4815c402 100644
--- a/test/suites/secondary_http.bash
+++ b/test/suites/remote_http.bash
@@ -26,96 +26,96 @@ maybe_start_ipv6_http_server() {
|| return 1
}
-SUITE_secondary_http_PROBE() {
+SUITE_remote_http_PROBE() {
if ! "${HTTP_SERVER}" --help >/dev/null 2>&1; then
echo "cannot execute ${HTTP_SERVER} - Python 3 might be missing"
fi
}
-SUITE_secondary_http_SETUP() {
+SUITE_remote_http_SETUP() {
unset CCACHE_NODIRECT
generate_code 1 test.c
}
-SUITE_secondary_http() {
+SUITE_remote_http() {
# -------------------------------------------------------------------------
TEST "Subdirs layout"
- start_http_server 12780 secondary
- export CCACHE_SECONDARY_STORAGE="http://localhost:12780"
+ start_http_server 12780 remote
+ export CCACHE_REMOTE_STORAGE="http://localhost:12780"
$CCACHE_COMPILE -c test.c
expect_stat direct_cache_hit 0
expect_stat cache_miss 1
expect_stat files_in_cache 2
- expect_file_count 2 '*' secondary # result + manifest
- subdirs=$(find secondary -type d | wc -l)
- if [ "${subdirs}" -lt 2 ]; then # "secondary" itself counts as one
- test_failed "Expected subdirectories in secondary"
+ expect_file_count 2 '*' remote # result + manifest
+ subdirs=$(find remote -type d | wc -l)
+ if [ "${subdirs}" -lt 2 ]; then # "remote" itself counts as one
+ test_failed "Expected subdirectories in remote"
fi
$CCACHE_COMPILE -c test.c
expect_stat direct_cache_hit 1
expect_stat cache_miss 1
expect_stat files_in_cache 2
- expect_file_count 2 '*' secondary # result + manifest
+ expect_file_count 2 '*' remote # result + manifest
$CCACHE -C >/dev/null
expect_stat files_in_cache 0
- expect_file_count 2 '*' secondary # result + manifest
+ expect_file_count 2 '*' remote # result + manifest
$CCACHE_COMPILE -c test.c
expect_stat direct_cache_hit 2
expect_stat cache_miss 1
- expect_stat files_in_cache 2 # fetched from secondary
- expect_file_count 2 '*' secondary # result + manifest
+ expect_stat files_in_cache 2 # fetched from remote
+ expect_file_count 2 '*' remote # result + manifest
# -------------------------------------------------------------------------
TEST "Flat layout"
- start_http_server 12780 secondary
- export CCACHE_SECONDARY_STORAGE="http://localhost:12780|layout=flat"
+ start_http_server 12780 remote
+ export CCACHE_REMOTE_STORAGE="http://localhost:12780|layout=flat"
$CCACHE_COMPILE -c test.c
expect_stat direct_cache_hit 0
expect_stat cache_miss 1
expect_stat files_in_cache 2
- expect_file_count 2 '*' secondary # result + manifest
- subdirs=$(find secondary -type d | wc -l)
- if [ "${subdirs}" -ne 1 ]; then # "secondary" itself counts as one
- test_failed "Expected no subdirectories in secondary"
+ expect_file_count 2 '*' remote # result + manifest
+ subdirs=$(find remote -type d | wc -l)
+ if [ "${subdirs}" -ne 1 ]; then # "remote" itself counts as one
+ test_failed "Expected no subdirectories in remote"
fi
$CCACHE_COMPILE -c test.c
expect_stat direct_cache_hit 1
expect_stat cache_miss 1
expect_stat files_in_cache 2
- expect_file_count 2 '*' secondary # result + manifest
+ expect_file_count 2 '*' remote # result + manifest
$CCACHE -C >/dev/null
expect_stat files_in_cache 0
- expect_file_count 2 '*' secondary # result + manifest
+ expect_file_count 2 '*' remote # result + manifest
$CCACHE_COMPILE -c test.c
expect_stat direct_cache_hit 2
expect_stat cache_miss 1
- expect_stat files_in_cache 2 # fetched from secondary
- expect_file_count 2 '*' secondary # result + manifest
+ expect_stat files_in_cache 2 # fetched from remote
+ expect_file_count 2 '*' remote # result + manifest
# -------------------------------------------------------------------------
TEST "Bazel layout"
- start_http_server 12780 secondary
- mkdir secondary/ac
- export CCACHE_SECONDARY_STORAGE="http://localhost:12780|layout=bazel"
+ start_http_server 12780 remote
+ mkdir remote/ac
+ export CCACHE_REMOTE_STORAGE="http://localhost:12780|layout=bazel"
$CCACHE_COMPILE -c test.c
expect_stat direct_cache_hit 0
expect_stat cache_miss 1
expect_stat files_in_cache 2
- expect_file_count 2 '*' secondary/ac # result + manifest
- if [ "$(ls secondary/ac | grep -Ec '^[0-9a-f]{64}$')" -ne 2 ]; then
+ expect_file_count 2 '*' remote/ac # result + manifest
+ if [ "$(ls remote/ac | grep -Ec '^[0-9a-f]{64}$')" -ne 2 ]; then
test_failed "Bazel layout filenames not as expected"
fi
@@ -123,85 +123,85 @@ SUITE_secondary_http() {
expect_stat direct_cache_hit 1
expect_stat cache_miss 1
expect_stat files_in_cache 2
- expect_file_count 2 '*' secondary/ac # result + manifest
+ expect_file_count 2 '*' remote/ac # result + manifest
$CCACHE -C >/dev/null
expect_stat files_in_cache 0
- expect_file_count 2 '*' secondary/ac # result + manifest
+ expect_file_count 2 '*' remote/ac # result + manifest
$CCACHE_COMPILE -c test.c
expect_stat direct_cache_hit 2
expect_stat cache_miss 1
- expect_stat files_in_cache 2 # fetched from secondary
- expect_file_count 2 '*' secondary/ac # result + manifest
+ expect_stat files_in_cache 2 # fetched from remote
+ expect_file_count 2 '*' remote/ac # result + manifest
# -------------------------------------------------------------------------
TEST "Basic auth"
- start_http_server 12780 secondary "somebody:secret123"
- export CCACHE_SECONDARY_STORAGE="http://somebody:secret123@localhost:12780"
+ start_http_server 12780 remote "somebody:secret123"
+ export CCACHE_REMOTE_STORAGE="http://somebody:secret123@localhost:12780"
CCACHE_DEBUG=1 $CCACHE_COMPILE -c test.c
expect_stat direct_cache_hit 0
expect_stat cache_miss 1
expect_stat files_in_cache 2
- expect_file_count 2 '*' secondary # result + manifest
+ expect_file_count 2 '*' remote # result + manifest
expect_not_contains test.o.*.ccache-log secret123
# -------------------------------------------------------------------------
TEST "Basic auth required"
- start_http_server 12780 secondary "somebody:secret123"
+ start_http_server 12780 remote "somebody:secret123"
# no authentication configured on client
- export CCACHE_SECONDARY_STORAGE="http://localhost:12780"
+ export CCACHE_REMOTE_STORAGE="http://localhost:12780"
CCACHE_DEBUG=1 $CCACHE_COMPILE -c test.c
expect_stat direct_cache_hit 0
expect_stat cache_miss 1
expect_stat files_in_cache 2
- expect_file_count 0 '*' secondary # result + manifest
+ expect_file_count 0 '*' remote # result + manifest
expect_contains test.o.*.ccache-log "status code: 401"
# -------------------------------------------------------------------------
TEST "Basic auth failed"
- start_http_server 12780 secondary "somebody:secret123"
- export CCACHE_SECONDARY_STORAGE="http://somebody:wrong@localhost:12780"
+ start_http_server 12780 remote "somebody:secret123"
+ export CCACHE_REMOTE_STORAGE="http://somebody:wrong@localhost:12780"
CCACHE_DEBUG=1 $CCACHE_COMPILE -c test.c
expect_stat direct_cache_hit 0
expect_stat cache_miss 1
expect_stat files_in_cache 2
- expect_file_count 0 '*' secondary # result + manifest
+ expect_file_count 0 '*' remote # result + manifest
expect_not_contains test.o.*.ccache-log secret123
expect_contains test.o.*.ccache-log "status code: 401"
# -------------------------------------------------------------------------
TEST "IPv6 address"
- if maybe_start_ipv6_http_server 12780 secondary; then
- export CCACHE_SECONDARY_STORAGE="http://[::1]:12780"
+ if maybe_start_ipv6_http_server 12780 remote; then
+ export CCACHE_REMOTE_STORAGE="http://[::1]:12780"
$CCACHE_COMPILE -c test.c
expect_stat direct_cache_hit 0
expect_stat cache_miss 1
expect_stat files_in_cache 2
- expect_file_count 2 '*' secondary # result + manifest
+ expect_file_count 2 '*' remote # result + manifest
$CCACHE_COMPILE -c test.c
expect_stat direct_cache_hit 1
expect_stat cache_miss 1
expect_stat files_in_cache 2
- expect_file_count 2 '*' secondary # result + manifest
+ expect_file_count 2 '*' remote # result + manifest
$CCACHE -C >/dev/null
expect_stat files_in_cache 0
- expect_file_count 2 '*' secondary # result + manifest
+ expect_file_count 2 '*' remote # result + manifest
$CCACHE_COMPILE -c test.c
expect_stat direct_cache_hit 2
expect_stat cache_miss 1
- expect_stat files_in_cache 2 # fetched from secondary
- expect_file_count 2 '*' secondary # result + manifest
+ expect_stat files_in_cache 2 # fetched from remote
+ expect_file_count 2 '*' remote # result + manifest
fi
}
diff --git a/test/suites/secondary_redis.bash b/test/suites/remote_redis.bash
index 4033b314..84351a05 100644
--- a/test/suites/secondary_redis.bash
+++ b/test/suites/remote_redis.bash
@@ -1,4 +1,4 @@
-SUITE_secondary_redis_PROBE() {
+SUITE_remote_redis_PROBE() {
if ! $CCACHE --version | fgrep -q -- redis-storage &> /dev/null; then
echo "redis-storage not available"
return
@@ -30,7 +30,7 @@ start_redis_server() {
fi
}
-SUITE_secondary_redis_SETUP() {
+SUITE_remote_redis_SETUP() {
unset CCACHE_NODIRECT
generate_code 1 test.c
@@ -47,13 +47,13 @@ expect_number_of_redis_cache_entries() {
fi
}
-SUITE_secondary_redis() {
+SUITE_remote_redis() {
# -------------------------------------------------------------------------
TEST "Base case"
port=7777
redis_url="redis://localhost:${port}"
- export CCACHE_SECONDARY_STORAGE="${redis_url}"
+ export CCACHE_REMOTE_STORAGE="${redis_url}"
start_redis_server "${port}"
@@ -76,7 +76,7 @@ SUITE_secondary_redis() {
$CCACHE_COMPILE -c test.c
expect_stat direct_cache_hit 2
expect_stat cache_miss 1
- expect_stat files_in_cache 2 # fetched from secondary
+ expect_stat files_in_cache 2 # fetched from remote
expect_number_of_redis_cache_entries 2 "$redis_url" # result + manifest
# -------------------------------------------------------------------------
@@ -85,7 +85,7 @@ SUITE_secondary_redis() {
port=7777
password=secret123
redis_url="redis://${password}@localhost:${port}"
- export CCACHE_SECONDARY_STORAGE="${redis_url}"
+ export CCACHE_REMOTE_STORAGE="${redis_url}"
start_redis_server "${port}" "${password}"
@@ -109,17 +109,17 @@ SUITE_secondary_redis() {
$CCACHE_COMPILE -c test.c
expect_stat direct_cache_hit 2
expect_stat cache_miss 1
- expect_stat files_in_cache 2 # fetched from secondary
+ expect_stat files_in_cache 2 # fetched from remote
expect_number_of_redis_cache_entries 2 "$redis_url" # result + manifest
# -------------------------------------------------------------------------
TEST "Unreachable server"
- export CCACHE_SECONDARY_STORAGE="redis://localhost:1"
+ export CCACHE_REMOTE_STORAGE="redis://localhost:1"
$CCACHE_COMPILE -c test.c
expect_stat direct_cache_hit 0
expect_stat cache_miss 1
expect_stat files_in_cache 2
- expect_stat secondary_storage_error 1
+ expect_stat remote_storage_error 1
}
diff --git a/test/suites/secondary_redis_unix.bash b/test/suites/remote_redis_unix.bash
index 9a6bfa83..601e17c5 100644
--- a/test/suites/secondary_redis_unix.bash
+++ b/test/suites/remote_redis_unix.bash
@@ -1,4 +1,4 @@
-SUITE_secondary_redis_unix_PROBE() {
+SUITE_remote_redis_unix_PROBE() {
if ! $CCACHE --version | fgrep -q -- redis-storage &> /dev/null; then
echo "redis-storage not available"
return
@@ -40,7 +40,7 @@ start_redis_unix_server() {
fi
}
-SUITE_secondary_redis_unix_SETUP() {
+SUITE_remote_redis_unix_SETUP() {
unset CCACHE_NODIRECT
generate_code 1 test.c
@@ -57,13 +57,13 @@ expect_number_of_redis_unix_cache_entries() {
fi
}
-SUITE_secondary_redis_unix() {
+SUITE_remote_redis_unix() {
# -------------------------------------------------------------------------
TEST "Base case"
socket=$(mktemp)
redis_url="redis+unix:${socket}"
- export CCACHE_SECONDARY_STORAGE="${redis_url}"
+ export CCACHE_REMOTE_STORAGE="${redis_url}"
start_redis_unix_server "${socket}"
@@ -86,7 +86,7 @@ SUITE_secondary_redis_unix() {
$CCACHE_COMPILE -c test.c
expect_stat direct_cache_hit 2
expect_stat cache_miss 1
- expect_stat files_in_cache 2 # fetched from secondary
+ expect_stat files_in_cache 2 # fetched from remote
expect_number_of_redis_unix_cache_entries 2 "${socket}" # result + manifest
# -------------------------------------------------------------------------
@@ -95,7 +95,7 @@ SUITE_secondary_redis_unix() {
socket=$(mktemp)
password=secret123
redis_url="redis+unix://${password}@localhost${socket}"
- export CCACHE_SECONDARY_STORAGE="${redis_url}"
+ export CCACHE_REMOTE_STORAGE="${redis_url}"
start_redis_unix_server "${socket}" "${password}"
@@ -119,17 +119,17 @@ SUITE_secondary_redis_unix() {
$CCACHE_COMPILE -c test.c
expect_stat direct_cache_hit 2
expect_stat cache_miss 1
- expect_stat files_in_cache 2 # fetched from secondary
+ expect_stat files_in_cache 2 # fetched from remote
expect_number_of_redis_unix_cache_entries 2 "${socket}" # result + manifest
# -------------------------------------------------------------------------
TEST "Unreachable server"
- export CCACHE_SECONDARY_STORAGE="redis+unix:///foo"
+ export CCACHE_REMOTE_STORAGE="redis+unix:///foo"
$CCACHE_COMPILE -c test.c
expect_stat direct_cache_hit 0
expect_stat cache_miss 1
expect_stat files_in_cache 2
- expect_stat secondary_storage_error 1
+ expect_stat remote_storage_error 1
}
diff --git a/test/suites/secondary_url.bash b/test/suites/remote_url.bash
index 9c68645d..9eba7ea5 100644
--- a/test/suites/secondary_url.bash
+++ b/test/suites/remote_url.bash
@@ -1,47 +1,47 @@
-SUITE_secondary_url_SETUP() {
+SUITE_remote_url_SETUP() {
generate_code 1 test.c
}
-SUITE_secondary_url() {
+SUITE_remote_url() {
# -------------------------------------------------------------------------
TEST "Reject empty url (without config attributes)"
- export CCACHE_SECONDARY_STORAGE="|"
+ export CCACHE_REMOTE_STORAGE="|"
$CCACHE_COMPILE -c test.c 2>stderr.log
expect_contains stderr.log "must provide a URL"
# -------------------------------------------------------------------------
TEST "Reject empty url (but with config attributes)"
- export CCACHE_SECONDARY_STORAGE="|key=value"
+ export CCACHE_REMOTE_STORAGE="|key=value"
$CCACHE_COMPILE -c test.c 2>stderr.log
expect_contains stderr.log "must provide a URL"
# -------------------------------------------------------------------------
TEST "Reject invalid url"
- export CCACHE_SECONDARY_STORAGE="://qwerty"
+ export CCACHE_REMOTE_STORAGE="://qwerty"
$CCACHE_COMPILE -c test.c 2>stderr.log
expect_contains stderr.log "Cannot parse URL"
# -------------------------------------------------------------------------
TEST "Reject missing scheme"
- export CCACHE_SECONDARY_STORAGE="/qwerty"
+ export CCACHE_REMOTE_STORAGE="/qwerty"
$CCACHE_COMPILE -c test.c 2>stderr.log
expect_contains stderr.log "URL scheme must not be empty"
# -------------------------------------------------------------------------
TEST "Reject user info defined but no host"
- export CCACHE_SECONDARY_STORAGE="http://foo@"
+ export CCACHE_REMOTE_STORAGE="http://foo@"
$CCACHE_COMPILE -c test.c 2>stderr.log
expect_contains stderr.log "User info defined, but host is empty"
# -------------------------------------------------------------------------
TEST "Reject relative path with colon in first part"
- export CCACHE_SECONDARY_STORAGE="file:foo:bar"
+ export CCACHE_REMOTE_STORAGE="file:foo:bar"
$CCACHE_COMPILE -c test.c 2>stderr.log
expect_contains stderr.log "The first segment of the relative path can't contain ':'"
}
diff --git a/test/suites/secondary_file.bash b/test/suites/secondary_file.bash
deleted file mode 100644
index b67bcf66..00000000
--- a/test/suites/secondary_file.bash
+++ /dev/null
@@ -1,422 +0,0 @@
-# This test suite verified both the file storage backend and the secondary
-# storage framework itself.
-
-SUITE_secondary_file_SETUP() {
- unset CCACHE_NODIRECT
- export CCACHE_SECONDARY_STORAGE="file:$PWD/secondary"
-
- generate_code 1 test.c
-}
-
-SUITE_secondary_file() {
- # -------------------------------------------------------------------------
- TEST "Base case"
-
- # Compile and send result to primary and secondary storage.
- $CCACHE_COMPILE -c test.c
- expect_stat direct_cache_hit 0
- expect_stat cache_miss 1
- expect_stat files_in_cache 2
- expect_stat primary_storage_hit 0
- expect_stat primary_storage_miss 2 # result + manifest
- expect_stat secondary_storage_hit 0
- expect_stat secondary_storage_miss 2 # result + manifest
- expect_exists secondary/CACHEDIR.TAG
- subdirs=$(find secondary -type d | wc -l)
- if [ "${subdirs}" -lt 2 ]; then # "secondary" itself counts as one
- test_failed "Expected subdirectories in secondary"
- fi
- expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
-
- # Get result from primary storage.
- $CCACHE_COMPILE -c test.c
- expect_stat direct_cache_hit 1
- expect_stat cache_miss 1
- expect_stat primary_storage_hit 2 # result + manifest
- expect_stat primary_storage_miss 2 # result + manifest
- expect_stat secondary_storage_hit 0
- expect_stat secondary_storage_miss 2 # result + manifest
- expect_stat files_in_cache 2
- expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
-
- # Clear primary storage.
- $CCACHE -C >/dev/null
- expect_stat files_in_cache 0
- expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
-
- # Get result from secondary storage, copying it to primary storage.
- $CCACHE_COMPILE -c test.c
- expect_stat direct_cache_hit 2
- expect_stat cache_miss 1
- expect_stat primary_storage_hit 2
- expect_stat primary_storage_miss 4 # 2 * (result + manifest)
- expect_stat secondary_storage_hit 2 # result + manifest
- expect_stat secondary_storage_miss 2 # result + manifest
- expect_stat files_in_cache 2 # fetched from secondary
- expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
-
- # Get result from primary storage again.
- $CCACHE_COMPILE -c test.c
- expect_stat direct_cache_hit 3
- expect_stat cache_miss 1
- expect_stat primary_storage_hit 4
- expect_stat primary_storage_miss 4 # 2 * (result + manifest)
- expect_stat secondary_storage_hit 2 # result + manifest
- expect_stat secondary_storage_miss 2 # result + manifest
- expect_stat files_in_cache 2 # fetched from secondary
- expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
-
- # -------------------------------------------------------------------------
- TEST "Flat layout"
-
- CCACHE_SECONDARY_STORAGE+="|layout=flat"
-
- $CCACHE_COMPILE -c test.c
- expect_stat direct_cache_hit 0
- expect_stat cache_miss 1
- expect_stat files_in_cache 2
- expect_exists secondary/CACHEDIR.TAG
- subdirs=$(find secondary -type d | wc -l)
- if [ "${subdirs}" -ne 1 ]; then # "secondary" itself counts as one
- test_failed "Expected no subdirectories in secondary"
- fi
- expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
-
- $CCACHE_COMPILE -c test.c
- expect_stat direct_cache_hit 1
- expect_stat cache_miss 1
- expect_stat files_in_cache 2
- expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
-
- $CCACHE -C >/dev/null
- expect_stat files_in_cache 0
- expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
-
- $CCACHE_COMPILE -c test.c
- expect_stat direct_cache_hit 2
- expect_stat cache_miss 1
- expect_stat files_in_cache 2 # fetched from secondary
- expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
-
- # -------------------------------------------------------------------------
- TEST "Two directories"
-
- CCACHE_SECONDARY_STORAGE+=" file://$PWD/secondary_2"
- mkdir secondary_2
-
- $CCACHE_COMPILE -c test.c
- expect_stat direct_cache_hit 0
- expect_stat cache_miss 1
- expect_stat files_in_cache 2
- expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
- expect_file_count 3 '*' secondary_2 # CACHEDIR.TAG + result + manifest
-
- $CCACHE -C >/dev/null
- expect_stat files_in_cache 0
- expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
- expect_file_count 3 '*' secondary_2 # CACHEDIR.TAG + result + manifest
-
- $CCACHE_COMPILE -c test.c
- expect_stat direct_cache_hit 1
- expect_stat cache_miss 1
- expect_stat files_in_cache 2 # fetched from secondary
- expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
- expect_file_count 3 '*' secondary_2 # CACHEDIR.TAG + result + manifest
-
- $CCACHE -C >/dev/null
- expect_stat files_in_cache 0
-
- rm -r secondary/??
- expect_file_count 1 '*' secondary # CACHEDIR.TAG
-
- $CCACHE_COMPILE -c test.c
- expect_stat direct_cache_hit 2
- expect_stat cache_miss 1
- expect_stat files_in_cache 2 # fetched from secondary_2
- expect_file_count 1 '*' secondary # CACHEDIR.TAG
- expect_file_count 3 '*' secondary_2 # CACHEDIR.TAG + result + manifest
-
- # -------------------------------------------------------------------------
- TEST "Read-only"
-
- $CCACHE_COMPILE -c test.c
- expect_stat direct_cache_hit 0
- expect_stat cache_miss 1
- expect_stat files_in_cache 2
- expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
-
- $CCACHE -C >/dev/null
- expect_stat files_in_cache 0
- expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
-
- CCACHE_SECONDARY_STORAGE+="|read-only"
-
- $CCACHE_COMPILE -c test.c
- expect_stat direct_cache_hit 1
- expect_stat cache_miss 1
- expect_stat files_in_cache 2 # fetched from secondary
- expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
-
- echo 'int x;' >> test.c
- $CCACHE_COMPILE -c test.c
- expect_stat direct_cache_hit 1
- expect_stat cache_miss 2
- expect_stat files_in_cache 4
- expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
-
- # -------------------------------------------------------------------------
- TEST "umask"
-
- export CCACHE_UMASK=042
- CCACHE_SECONDARY_STORAGE="file://$PWD/secondary|umask=024"
- rm -rf secondary
- $CCACHE_COMPILE -c test.c
- expect_perm secondary drwxr-x-wx # 777 & 024
- expect_perm secondary/CACHEDIR.TAG -rw-r---w- # 666 & 024
- result_file=$(find $CCACHE_DIR -name '*R')
- expect_perm "$(dirname "${result_file}")" drwx-wxr-x # 777 & 042
- expect_perm "${result_file}" -rw--w-r-- # 666 & 042
-
- CCACHE_SECONDARY_STORAGE="file://$PWD/secondary|umask=026"
- $CCACHE -C >/dev/null
- rm -rf secondary
- $CCACHE_COMPILE -c test.c
- expect_perm secondary drwxr-x--x # 777 & 026
- expect_perm secondary/CACHEDIR.TAG -rw-r----- # 666 & 026
-
- # -------------------------------------------------------------------------
- TEST "Sharding"
-
- CCACHE_SECONDARY_STORAGE="file://$PWD/secondary/*|shards=a,b(2)"
-
- $CCACHE_COMPILE -c test.c
- expect_stat direct_cache_hit 0
- expect_stat cache_miss 1
- expect_stat files_in_cache 2
- if [ ! -d secondary/a ] && [ ! -d secondary/b ]; then
- test_failed "Expected secondary/a or secondary/b to exist"
- fi
-
- # -------------------------------------------------------------------------
- TEST "Reshare"
-
- CCACHE_SECONDARY_STORAGE="" $CCACHE_COMPILE -c test.c
- expect_stat direct_cache_hit 0
- expect_stat cache_miss 1
- expect_stat files_in_cache 2
- expect_stat primary_storage_hit 0
- expect_stat primary_storage_miss 2
- expect_stat secondary_storage_hit 0
- expect_stat secondary_storage_miss 0
- expect_missing secondary
-
- $CCACHE_COMPILE -c test.c
- expect_stat direct_cache_hit 1
- expect_stat cache_miss 1
- expect_stat primary_storage_hit 2
- expect_stat primary_storage_miss 2
- expect_stat secondary_storage_hit 0
- expect_stat secondary_storage_miss 0
- expect_missing secondary
-
- CCACHE_RESHARE=1 $CCACHE_COMPILE -c test.c
- expect_stat direct_cache_hit 2
- expect_stat cache_miss 1
- expect_stat primary_storage_hit 4
- expect_stat primary_storage_miss 2
- expect_stat secondary_storage_hit 0
- expect_stat secondary_storage_miss 0
- expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
-
- $CCACHE -C >/dev/null
-
- $CCACHE_COMPILE -c test.c
- expect_stat direct_cache_hit 3
- expect_stat cache_miss 1
- expect_stat primary_storage_hit 4
- expect_stat primary_storage_miss 4
- expect_stat secondary_storage_hit 2
- expect_stat secondary_storage_miss 0
- expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
-
- # -------------------------------------------------------------------------
- TEST "Recache"
-
- CCACHE_RECACHE=1 $CCACHE_COMPILE -c test.c
- expect_stat direct_cache_hit 0
- expect_stat direct_cache_miss 0
- expect_stat cache_miss 0
- expect_stat recache 1
- expect_stat files_in_cache 2
- expect_stat primary_storage_hit 0
- expect_stat primary_storage_miss 1 # Try to read manifest for updating
- expect_stat secondary_storage_hit 0
- expect_stat secondary_storage_miss 1 # Try to read manifest for updating
- expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
-
- $CCACHE -C >/dev/null
- expect_stat files_in_cache 0
- expect_file_count 3 '*' secondary # CACHEDIR.TAG + result + manifest
-
- CCACHE_RECACHE=1 $CCACHE_COMPILE -c test.c
- expect_stat direct_cache_hit 0
- expect_stat direct_cache_miss 0
- expect_stat cache_miss 0
- expect_stat recache 2
- expect_stat files_in_cache 2
- expect_stat primary_storage_hit 0
- expect_stat primary_storage_miss 2 # Try to read manifest for updating
- expect_stat secondary_storage_hit 1 # Read manifest for updating
- expect_stat secondary_storage_miss 1
-
- # -------------------------------------------------------------------------
- if touch test.c && ln test.c test-if-fs-supports-hard-links.c 2>/dev/null; then
- TEST "Don't reshare results with raw files"
-
- CCACHE_SECONDARY_STORAGE= CCACHE_HARDLINK=1 $CCACHE_COMPILE -c test.c
- expect_stat direct_cache_hit 0
- expect_stat cache_miss 1
- expect_stat files_in_cache 3
- expect_stat primary_storage_hit 0
- expect_stat primary_storage_miss 2 # result + manifest
- expect_stat secondary_storage_hit 0
- expect_stat secondary_storage_miss 0
-
- CCACHE_RESHARE=1 $CCACHE_COMPILE -c test.c
- expect_stat direct_cache_hit 1
- expect_stat cache_miss 1
- expect_stat files_in_cache 3
- expect_stat primary_storage_hit 2
- expect_stat primary_storage_miss 2 # result + manifest
- expect_stat secondary_storage_hit 0
- expect_stat secondary_storage_miss 0
- expect_file_count 2 '*' secondary # CACHEDIR.TAG + manifest, not result
- fi
-
- # -------------------------------------------------------------------------
- TEST "Manifest handling"
-
- echo 'int x;' >test.h
- backdate test.h
- echo '#include "test.h"' >test.c
-
- $CCACHE_COMPILE -c test.c
- expect_stat direct_cache_hit 0
- expect_stat cache_miss 1
- expect_stat primary_storage_hit 0
- expect_stat primary_storage_miss 2 # miss: manifest + result
- expect_stat secondary_storage_hit 0
- expect_stat secondary_storage_miss 2 # miss: manifest + result
-
- # Both primary and secondary now have an "int x;" key in the manifest.
-
- echo 'int y;' >test.h
- backdate test.h
-
- $CCACHE_COMPILE -c test.c
- expect_stat direct_cache_hit 0
- expect_stat cache_miss 2
- expect_stat primary_storage_hit 1 # hit: manifest without key
- expect_stat primary_storage_miss 3 # miss: result
- expect_stat secondary_storage_hit 1 # his: manifest without key
- expect_stat secondary_storage_miss 3 # miss: result
-
- # Both primary and secondary now have "int x;" and "int y;" keys in the manifest.
-
- $CCACHE -C >/dev/null
-
- # Now only secondary has "int x;" and "int y;" keys in the manifest. We
- # should now be able to get secondary hit without involving primary.
-
- echo 'int x;' >test.h
- backdate test.h
-
- $CCACHE_COMPILE -c test.c
- expect_stat direct_cache_hit 1
- expect_stat cache_miss 2
- expect_stat primary_storage_hit 1
- expect_stat primary_storage_miss 5 # miss: manifest + result
- expect_stat secondary_storage_hit 3 # hit: manifest + result
- expect_stat secondary_storage_miss 3
-
- # Should be able to get secondary hit without involving primary.
-
- echo 'int y;' >test.h
- backdate test.h
-
- $CCACHE_COMPILE -c test.c
- expect_stat direct_cache_hit 2
- expect_stat cache_miss 2
- expect_stat primary_storage_hit 2 # hit: manifest with key (downloaded from previous step)
- expect_stat primary_storage_miss 6 # miss: result
- expect_stat secondary_storage_hit 4 # hit: result
- expect_stat secondary_storage_miss 3
-
- # -------------------------------------------------------------------------
- TEST "Manifest merging"
-
- echo 'int x;' >test.h
- backdate test.h
- echo '#include "test.h"' >test.c
-
- $CCACHE_COMPILE -c test.c
- expect_stat direct_cache_hit 0
- expect_stat cache_miss 1
- expect_stat primary_storage_hit 0
- expect_stat primary_storage_miss 2 # miss: manifest + result
- expect_stat secondary_storage_hit 0
- expect_stat secondary_storage_miss 2 # miss: manifest + result
-
- $CCACHE -C >/dev/null
-
- # Now secondary has an "int x;" key in the manifest and primary has none.
-
- echo 'int y;' >test.h
- backdate test.h
-
- CCACHE_SECONDARY_STORAGE= $CCACHE_COMPILE -c test.c
- expect_stat direct_cache_hit 0
- expect_stat cache_miss 2
- expect_stat primary_storage_hit 0
- expect_stat primary_storage_miss 4 # miss: manifest + result
- expect_stat secondary_storage_hit 0
- expect_stat secondary_storage_miss 2
-
- # Now primary has "int y;" while secondary still has "int x;".
-
- echo 'int x;' >test.h
- backdate test.h
-
- $CCACHE_COMPILE -c test.c
- expect_stat direct_cache_hit 1
- expect_stat cache_miss 2
- expect_stat primary_storage_hit 1 # hit: manifest without key
- expect_stat primary_storage_miss 5 # miss: result
- expect_stat secondary_storage_hit 2 # hit: manifest + result
- expect_stat secondary_storage_miss 2
-
- # Primary's manifest with "int y;" was merged with secondary's "int x;"
- # above, so we should now be able to get "int x;" and "int y;" hits locally.
-
- echo 'int y;' >test.h
- backdate test.h
-
- CCACHE_SECONDARY_STORAGE= $CCACHE_COMPILE -c test.c
- expect_stat direct_cache_hit 2
- expect_stat cache_miss 2
- expect_stat primary_storage_hit 3 # hit: manifest + result
- expect_stat primary_storage_miss 5
- expect_stat secondary_storage_hit 2
- expect_stat secondary_storage_miss 2
-
- echo 'int x;' >test.h
- backdate test.h
-
- CCACHE_SECONDARY_STORAGE= $CCACHE_COMPILE -c test.c
- expect_stat direct_cache_hit 3
- expect_stat cache_miss 2
- expect_stat primary_storage_hit 5 # hit: manifest + result
- expect_stat primary_storage_miss 5
- expect_stat secondary_storage_hit 2
- expect_stat secondary_storage_miss 2
-}
diff --git a/test/suites/stats_log.bash b/test/suites/stats_log.bash
index 1ad3b930..9decaa05 100644
--- a/test/suites/stats_log.bash
+++ b/test/suites/stats_log.bash
@@ -12,24 +12,24 @@ SUITE_stats_log() {
expect_stat direct_cache_hit 0
expect_stat preprocessed_cache_miss 1
expect_stat cache_miss 1
- expect_stat primary_storage_hit 0
- expect_stat primary_storage_miss 2
+ expect_stat local_storage_hit 0
+ expect_stat local_storage_miss 2
$CCACHE_COMPILE -c test.c
expect_stat direct_cache_hit 1
expect_stat preprocessed_cache_miss 1
expect_stat cache_miss 1
- expect_stat primary_storage_hit 2
- expect_stat primary_storage_miss 2
+ expect_stat local_storage_hit 2
+ expect_stat local_storage_miss 2
expect_content stats.log "# test.c
cache_miss
direct_cache_miss
+local_storage_miss
+local_storage_miss
preprocessed_cache_miss
-primary_storage_miss
-primary_storage_miss
# test.c
direct_cache_hit
-primary_storage_hit
-primary_storage_hit"
+local_storage_hit
+local_storage_hit"
}
diff --git a/test/suites/trim_dir.bash b/test/suites/trim_dir.bash
index ba8dd3cf..3297930d 100644
--- a/test/suites/trim_dir.bash
+++ b/test/suites/trim_dir.bash
@@ -6,7 +6,7 @@ SUITE_trim_dir_PROBE() {
SUITE_trim_dir() {
# -------------------------------------------------------------------------
- TEST "Trim secondary cache directory"
+ TEST "Trim remote cache directory"
if $HOST_OS_APPLE; then
one_mb=1m
@@ -14,35 +14,35 @@ SUITE_trim_dir() {
one_mb=1M
fi
for subdir in aa bb cc; do
- mkdir -p secondary/$subdir
- dd if=/dev/zero of=secondary/$subdir/1 count=1 bs=$one_mb 2>/dev/null
- dd if=/dev/zero of=secondary/$subdir/2 count=1 bs=$one_mb 2>/dev/null
+ mkdir -p remote/$subdir
+ dd if=/dev/zero of=remote/$subdir/1 count=1 bs=$one_mb 2>/dev/null
+ dd if=/dev/zero of=remote/$subdir/2 count=1 bs=$one_mb 2>/dev/null
done
- backdate secondary/bb/2 secondary/cc/1
- $CCACHE --trim-dir secondary --trim-max-size 4.5M --trim-method mtime \
+ backdate remote/bb/2 remote/cc/1
+ $CCACHE --trim-dir remote --trim-max-size 4.5M --trim-method mtime \
>/dev/null
- expect_exists secondary/aa/1
- expect_exists secondary/aa/2
- expect_exists secondary/bb/1
- expect_missing secondary/bb/2
- expect_missing secondary/cc/1
- expect_exists secondary/cc/2
+ expect_exists remote/aa/1
+ expect_exists remote/aa/2
+ expect_exists remote/bb/1
+ expect_missing remote/bb/2
+ expect_missing remote/cc/1
+ expect_exists remote/cc/2
# -------------------------------------------------------------------------
- TEST "Trim primary cache directory"
+ TEST "Trim local cache directory"
- mkdir -p primary/0
- touch primary/0/stats
- if $CCACHE --trim-dir primary --trim-max-size 0 &>/dev/null; then
+ mkdir -p local/0
+ touch local/0/stats
+ if $CCACHE --trim-dir local --trim-max-size 0 &>/dev/null; then
test_failed "Expected failure"
fi
- rm -rf primary
- mkdir primary
- touch primary/ccache.conf
- if $CCACHE --trim-dir primary --trim-max-size 0 &>/dev/null; then
+ rm -rf local
+ mkdir local
+ touch local/ccache.conf
+ if $CCACHE --trim-dir local --trim-max-size 0 &>/dev/null; then
test_failed "Expected failure"
fi
}
diff --git a/unittest/CMakeLists.txt b/unittest/CMakeLists.txt
index 854d41cb..8b7c8d30 100644
--- a/unittest/CMakeLists.txt
+++ b/unittest/CMakeLists.txt
@@ -17,8 +17,8 @@ set(
test_core_StatisticsCounters.cpp
test_core_StatsLog.cpp
test_hashutil.cpp
- test_storage_primary_StatsFile.cpp
- test_storage_primary_util.cpp
+ test_storage_local_StatsFile.cpp
+ test_storage_local_util.cpp
test_util_Bytes.cpp
test_util_LockFile.cpp
test_util_TextTable.cpp
diff --git a/unittest/test_Config.cpp b/unittest/test_Config.cpp
index ac8241cf..14456811 100644
--- a/unittest/test_Config.cpp
+++ b/unittest/test_Config.cpp
@@ -71,6 +71,7 @@ TEST_CASE("Config: default values")
CHECK_FALSE(config.read_only());
CHECK_FALSE(config.read_only_direct());
CHECK_FALSE(config.recache());
+ CHECK(config.remote_storage().empty());
CHECK_FALSE(config.reshare());
CHECK(config.run_second_cpp());
CHECK(config.sloppiness().to_bitmask() == 0);
@@ -412,9 +413,9 @@ TEST_CASE("Config::visit_items")
"read_only = true\n"
"read_only_direct = true\n"
"recache = true\n"
+ "remote_storage = rs\n"
"reshare = true\n"
"run_second_cpp = false\n"
- "secondary_storage = ss\n"
"sloppiness = include_file_mtime, include_file_ctime, time_macros,"
" file_stat_matches, file_stat_matches_ctime, pch_defines, system_headers,"
" clang_index_store, ivfsoverlay, gcno_cwd\n"
@@ -472,9 +473,9 @@ TEST_CASE("Config::visit_items")
"(test.conf) read_only = true",
"(test.conf) read_only_direct = true",
"(test.conf) recache = true",
+ "(test.conf) remote_storage = rs",
"(test.conf) reshare = true",
"(test.conf) run_second_cpp = false",
- "(test.conf) secondary_storage = ss",
"(test.conf) sloppiness = clang_index_store, file_stat_matches,"
" file_stat_matches_ctime, gcno_cwd, include_file_ctime,"
" include_file_mtime, ivfsoverlay, pch_defines, system_headers,"
diff --git a/unittest/test_storage_primary_StatsFile.cpp b/unittest/test_storage_local_StatsFile.cpp
index 374d60c7..3135b09e 100644
--- a/unittest/test_storage_primary_StatsFile.cpp
+++ b/unittest/test_storage_local_StatsFile.cpp
@@ -21,16 +21,16 @@
#include <Util.hpp>
#include <core/Statistic.hpp>
#include <fmtmacros.hpp>
-#include <storage/primary/StatsFile.hpp>
+#include <storage/local/StatsFile.hpp>
#include <util/file.hpp>
#include <third_party/doctest.h>
using core::Statistic;
-using storage::primary::StatsFile;
+using storage::local::StatsFile;
using TestUtil::TestContext;
-TEST_SUITE_BEGIN("storage::primary::StatsFile");
+TEST_SUITE_BEGIN("storage::local::StatsFile");
TEST_CASE("Read nonexistent")
{
diff --git a/unittest/test_storage_primary_util.cpp b/unittest/test_storage_local_util.cpp
index f56cd33f..3ed23c55 100644
--- a/unittest/test_storage_primary_util.cpp
+++ b/unittest/test_storage_local_util.cpp
@@ -19,7 +19,7 @@
#include "TestUtil.hpp"
#include <Util.hpp>
-#include <storage/primary/util.hpp>
+#include <storage/local/util.hpp>
#include <util/file.hpp>
#include <third_party/doctest.h>
@@ -39,12 +39,12 @@ os_path(std::string path)
return path;
}
-TEST_SUITE_BEGIN("storage::primary::util");
+TEST_SUITE_BEGIN("storage::local::util");
-TEST_CASE("storage::primary::for_each_level_1_subdir")
+TEST_CASE("storage::local::for_each_level_1_subdir")
{
std::vector<std::string> actual;
- storage::primary::for_each_level_1_subdir(
+ storage::local::for_each_level_1_subdir(
"cache_dir",
[&](const auto& subdir, const auto&) { actual.push_back(subdir); },
[](double) {});
@@ -70,7 +70,7 @@ TEST_CASE("storage::primary::for_each_level_1_subdir")
CHECK(actual == expected);
}
-TEST_CASE("storage::primary::get_level_1_files")
+TEST_CASE("storage::local::get_level_1_files")
{
TestContext test_context;
@@ -87,19 +87,19 @@ TEST_CASE("storage::primary::get_level_1_files")
SUBCASE("nonexistent subdirectory")
{
- const auto files = storage::primary::get_level_1_files("2", null_receiver);
+ const auto files = storage::local::get_level_1_files("2", null_receiver);
CHECK(files.empty());
}
SUBCASE("empty subdirectory")
{
- const auto files = storage::primary::get_level_1_files("e", null_receiver);
+ const auto files = storage::local::get_level_1_files("e", null_receiver);
CHECK(files.empty());
}
SUBCASE("simple case")
{
- auto files = storage::primary::get_level_1_files("0", null_receiver);
+ auto files = storage::local::get_level_1_files("0", null_receiver);
REQUIRE(files.size() == 4);
// Files within a level are in arbitrary order, sort them to be able to