summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJoel Rosdahl <joel@rosdahl.net>2022-09-27 20:54:29 +0200
committerJoel Rosdahl <joel@rosdahl.net>2022-09-28 21:35:02 +0200
commitd5782080d6f12a6b246cd24493c76cf07f626413 (patch)
tree486fd464f02ad7ca8207ce725756ac43e76c8652
parent0cd6f70b8d6d74427d6a60958a3bdc1ba0127a2e (diff)
downloadccache-d5782080d6f12a6b246cd24493c76cf07f626413.tar.gz
feat: Implement "remote only" mode
Closes #1010.
-rw-r--r--doc/MANUAL.adoc35
-rw-r--r--src/Config.cpp10
-rw-r--r--src/Config.hpp8
-rw-r--r--src/ccache.cpp12
-rw-r--r--src/storage/Storage.cpp32
-rw-r--r--test/CMakeLists.txt1
-rw-r--r--test/suites/remote_only.bash33
-rw-r--r--unittest/test_Config.cpp3
8 files changed, 111 insertions, 23 deletions
diff --git a/doc/MANUAL.adoc b/doc/MANUAL.adoc
index 0e59898f..8577ac1f 100644
--- a/doc/MANUAL.adoc
+++ b/doc/MANUAL.adoc
@@ -881,12 +881,20 @@ temporary files otherwise. You may also want to set <<config_stats,*stats*>> to
If true, ccache will not use any previously stored result. New results will
still be cached, possibly overwriting any pre-existing results.
+[#config_remote_only]
+*remote_only* (*CCACHE_REMOTE_ONLY* or *CCACHE_NOREMOTE_ONLY*, see _<<Boolean values>>_ above)::
+
+ If true, ccache will only use <<config_remote_storage,remote storage>>. The
+ default is false. Note that cache statistics counters will still be kept in
+ the local cache directory unless <<config_stats,*stats*>> is false.
+
[#config_remote_storage]
*remote_storage* (*CCACHE_REMOTE_STORAGE*)::
This option specifies one or several storage backends (separated by space)
- to query after checking the local cache. See _<<Remote storage backends>>_
- for documentation of syntax and available backends.
+ to query after checking the local cache (unless
+ <<config_remote_only,*remote_only*>> is true). See _<<Remote storage
+ backends>>_ for documentation of syntax and available backends.
+
Examples:
+
@@ -1034,10 +1042,12 @@ filesystem as the `CCACHE_DIR` path, but this requirement has been relaxed.
== Remote storage backends
The <<config_remote_storage,*remote_storage*>> option lets you configure ccache
-to use one or several other storage backends in addition to the local cache
-storage located in <<config_cache_dir,*cache_dir*>>. Note that cache statistics
-counters will still be kept in the local cache directory -- remote storage
-backends only store compilation results and manifests.
+to use one or several remote storage backends. By default, the local cache
+directory located in <<config_cache_dir,*cache_dir*>> will be queried first and
+remote storage second, but <<config_remote_only,*remote_only*>> can be set to
+true to disable local storage. Note that cache statistics counters will still be
+kept in the local cache directory -- remote storage backends only store
+compilation results and manifests.
A remote storage backend is specified with a URL, optionally followed by a pipe
(`|`) and a pipe-separated list of attributes. An attribute is _key_=_value_ or
@@ -1078,7 +1088,8 @@ Examples:
=== Storage interaction
The table below describes the interaction between local and remote storage on
-cache hits and misses:
+cache hits and misses if <<config_remote_only,*remote_only*>> is false (which is
+the default):
[options="header",cols="20%,20%,60%"]
|==============================================================================
@@ -1094,6 +1105,16 @@ cache hits and misses:
^[2]^ Unless local storage is set to share its cache hits with the
<<config_reshare,*reshare*>> option.
+If <<config_remote_only,*remote_only*>> is true:
+
+[options="header",cols="20%,20%,60%"]
+|==============================================================================
+| *Local storage* | *Remote storage* | *What happens*
+
+| - | miss | Compile, write to remote, don't write to local
+| - | hit | Read from remote, don't write to local
+
+|==============================================================================
=== File storage backend
diff --git a/src/Config.cpp b/src/Config.cpp
index de5b5247..9d2b3ae4 100644
--- a/src/Config.cpp
+++ b/src/Config.cpp
@@ -91,6 +91,7 @@ enum class ConfigItem {
read_only,
read_only_direct,
recache,
+ remote_only,
remote_storage,
reshare,
run_second_cpp,
@@ -145,6 +146,7 @@ const std::unordered_map<std::string, ConfigKeyTableEntry> k_config_key_table =
{"read_only", {ConfigItem::read_only}},
{"read_only_direct", {ConfigItem::read_only_direct}},
{"recache", {ConfigItem::recache}},
+ {"remote_only", {ConfigItem::remote_only}},
{"remote_storage", {ConfigItem::remote_storage}},
{"reshare", {ConfigItem::reshare}},
{"run_second_cpp", {ConfigItem::run_second_cpp}},
@@ -193,6 +195,7 @@ const std::unordered_map<std::string, std::string> k_env_variable_table = {
{"READONLY", "read_only"},
{"READONLY_DIRECT", "read_only_direct"},
{"RECACHE", "recache"},
+ {"REMOTE_ONLY", "remote_only"},
{"REMOTE_STORAGE", "remote_storage"},
{"RESHARE", "reshare"},
{"SECONDARY_STORAGE", "remote_storage"}, // Alias for CCACHE_REMOTE_STORAGE
@@ -775,6 +778,9 @@ Config::get_string_value(const std::string& key) const
case ConfigItem::recache:
return format_bool(m_recache);
+ case ConfigItem::remote_only:
+ return format_bool(m_remote_only);
+
case ConfigItem::remote_storage:
return m_remote_storage;
@@ -1028,6 +1034,10 @@ Config::set_item(const std::string& key,
m_recache = parse_bool(value, env_var_key, negate);
break;
+ case ConfigItem::remote_only:
+ m_remote_only = parse_bool(value, env_var_key, negate);
+ break;
+
case ConfigItem::remote_storage:
m_remote_storage = Util::expand_environment_variables(value);
break;
diff --git a/src/Config.hpp b/src/Config.hpp
index 7ee59921..7a01b35e 100644
--- a/src/Config.hpp
+++ b/src/Config.hpp
@@ -84,6 +84,7 @@ public:
bool read_only() const;
bool read_only_direct() const;
bool recache() const;
+ bool remote_only() const;
const std::string& remote_storage() const;
bool reshare() const;
bool run_second_cpp() const;
@@ -193,6 +194,7 @@ private:
bool m_recache = false;
bool m_reshare = false;
bool m_run_second_cpp = true;
+ bool m_remote_only = false;
std::string m_remote_storage;
core::Sloppiness m_sloppiness;
bool m_stats = true;
@@ -437,6 +439,12 @@ Config::run_second_cpp() const
return m_run_second_cpp;
}
+inline bool
+Config::remote_only() const
+{
+ return m_remote_only;
+}
+
inline const std::string&
Config::remote_storage() const
{
diff --git a/src/ccache.cpp b/src/ccache.cpp
index 6cf1c53a..26b4d9fd 100644
--- a/src/ccache.cpp
+++ b/src/ccache.cpp
@@ -901,10 +901,14 @@ write_result(Context& ctx,
core::CacheEntry::Header header(ctx.config, core::CacheEntryType::result);
const auto cache_entry_data = core::CacheEntry::serialize(header, serializer);
- const auto raw_files = serializer.get_raw_files();
- if (!raw_files.empty()) {
- ctx.storage.local.put_raw_files(result_key, raw_files);
+
+ if (!ctx.config.remote_only()) {
+ const auto& raw_files = serializer.get_raw_files();
+ if (!raw_files.empty()) {
+ ctx.storage.local.put_raw_files(result_key, raw_files);
+ }
}
+
ctx.storage.put(result_key, core::CacheEntryType::result, cache_entry_data);
return true;
@@ -1742,7 +1746,7 @@ hash_direct_mode_specific_data(Context& ctx,
}
});
MTR_END("manifest", "manifest_get");
- if (read_manifests > 1) {
+ if (read_manifests > 1 && !ctx.config.remote_only()) {
MTR_SCOPE("manifest", "merge");
LOG("Storing merged manifest {} locally", manifest_key->to_string());
core::CacheEntry::Header header(ctx.config, core::CacheEntryType::manifest);
diff --git a/src/storage/Storage.cpp b/src/storage/Storage.cpp
index 11490edc..a607be50 100644
--- a/src/storage/Storage.cpp
+++ b/src/storage/Storage.cpp
@@ -237,20 +237,24 @@ Storage::get(const Digest& key,
{
MTR_SCOPE("storage", "get");
- auto value = local.get(key, type);
- local.increment_statistic(value ? core::Statistic::local_storage_hit
- : core::Statistic::local_storage_miss);
- if (value) {
- if (m_config.reshare()) {
- put_in_remote_storage(key, *value, true);
- }
- if (entry_receiver(std::move(*value))) {
- return;
+ if (!m_config.remote_only()) {
+ auto value = local.get(key, type);
+ local.increment_statistic(value ? core::Statistic::local_storage_hit
+ : core::Statistic::local_storage_miss);
+ if (value) {
+ if (m_config.reshare()) {
+ put_in_remote_storage(key, *value, true);
+ }
+ if (entry_receiver(std::move(*value))) {
+ return;
+ }
}
}
get_from_remote_storage(key, [&](util::Bytes&& data) {
- local.put(key, type, data, true);
+ if (!m_config.remote_only()) {
+ local.put(key, type, data, true);
+ }
return entry_receiver(std::move(data));
});
}
@@ -262,7 +266,9 @@ Storage::put(const Digest& key,
{
MTR_SCOPE("storage", "put");
- local.put(key, type, value);
+ if (!m_config.remote_only()) {
+ local.put(key, type, value);
+ }
put_in_remote_storage(key, value, false);
}
@@ -271,7 +277,9 @@ Storage::remove(const Digest& key, const core::CacheEntryType type)
{
MTR_SCOPE("storage", "remove");
- local.remove(key, type);
+ if (!m_config.remote_only()) {
+ local.remove(key, type);
+ }
remove_from_remote_storage(key);
}
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 4cd38a38..939a220c 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -54,6 +54,7 @@ addtest(readonly)
addtest(readonly_direct)
addtest(remote_file)
addtest(remote_http)
+addtest(remote_only)
addtest(remote_redis)
addtest(remote_redis_unix)
addtest(remote_url)
diff --git a/test/suites/remote_only.bash b/test/suites/remote_only.bash
new file mode 100644
index 00000000..3dea32ac
--- /dev/null
+++ b/test/suites/remote_only.bash
@@ -0,0 +1,33 @@
+SUITE_remote_only_SETUP() {
+ unset CCACHE_NODIRECT
+ export CCACHE_REMOTE_STORAGE="file:$PWD/remote"
+ export CCACHE_REMOTE_ONLY=1
+
+ generate_code 1 test.c
+}
+
+SUITE_remote_only() {
+ # -------------------------------------------------------------------------
+ TEST "Base case"
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 0
+ expect_stat cache_miss 1
+ expect_stat files_in_cache 0
+ expect_stat local_storage_hit 0
+ expect_stat local_storage_miss 0
+ expect_stat remote_storage_hit 0
+ expect_stat remote_storage_miss 2 # result + manifest
+ expect_file_count 3 '*' remote # CACHEDIR.TAG + result + manifest
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat direct_cache_hit 1
+ expect_stat cache_miss 1
+ expect_stat files_in_cache 0
+ expect_stat local_storage_hit 0
+ expect_stat local_storage_miss 0
+ expect_stat remote_storage_hit 2
+ expect_stat remote_storage_miss 2
+ expect_stat files_in_cache 0
+ expect_file_count 3 '*' remote # CACHEDIR.TAG + result + manifest
+}
diff --git a/unittest/test_Config.cpp b/unittest/test_Config.cpp
index 14456811..0226101f 100644
--- a/unittest/test_Config.cpp
+++ b/unittest/test_Config.cpp
@@ -71,6 +71,7 @@ TEST_CASE("Config: default values")
CHECK_FALSE(config.read_only());
CHECK_FALSE(config.read_only_direct());
CHECK_FALSE(config.recache());
+ CHECK_FALSE(config.remote_only());
CHECK(config.remote_storage().empty());
CHECK_FALSE(config.reshare());
CHECK(config.run_second_cpp());
@@ -413,6 +414,7 @@ TEST_CASE("Config::visit_items")
"read_only = true\n"
"read_only_direct = true\n"
"recache = true\n"
+ "remote_only = true\n"
"remote_storage = rs\n"
"reshare = true\n"
"run_second_cpp = false\n"
@@ -473,6 +475,7 @@ TEST_CASE("Config::visit_items")
"(test.conf) read_only = true",
"(test.conf) read_only_direct = true",
"(test.conf) recache = true",
+ "(test.conf) remote_only = true",
"(test.conf) remote_storage = rs",
"(test.conf) reshare = true",
"(test.conf) run_second_cpp = false",