diff options
author | Joel Rosdahl <joel@rosdahl.net> | 2022-09-27 20:54:29 +0200 |
---|---|---|
committer | Joel Rosdahl <joel@rosdahl.net> | 2022-09-28 21:35:02 +0200 |
commit | d5782080d6f12a6b246cd24493c76cf07f626413 (patch) | |
tree | 486fd464f02ad7ca8207ce725756ac43e76c8652 | |
parent | 0cd6f70b8d6d74427d6a60958a3bdc1ba0127a2e (diff) | |
download | ccache-d5782080d6f12a6b246cd24493c76cf07f626413.tar.gz |
feat: Implement "remote only" mode
Closes #1010.
-rw-r--r-- | doc/MANUAL.adoc | 35 | ||||
-rw-r--r-- | src/Config.cpp | 10 | ||||
-rw-r--r-- | src/Config.hpp | 8 | ||||
-rw-r--r-- | src/ccache.cpp | 12 | ||||
-rw-r--r-- | src/storage/Storage.cpp | 32 | ||||
-rw-r--r-- | test/CMakeLists.txt | 1 | ||||
-rw-r--r-- | test/suites/remote_only.bash | 33 | ||||
-rw-r--r-- | unittest/test_Config.cpp | 3 |
8 files changed, 111 insertions, 23 deletions
diff --git a/doc/MANUAL.adoc b/doc/MANUAL.adoc index 0e59898f..8577ac1f 100644 --- a/doc/MANUAL.adoc +++ b/doc/MANUAL.adoc @@ -881,12 +881,20 @@ temporary files otherwise. You may also want to set <<config_stats,*stats*>> to If true, ccache will not use any previously stored result. New results will still be cached, possibly overwriting any pre-existing results. +[#config_remote_only] +*remote_only* (*CCACHE_REMOTE_ONLY* or *CCACHE_NOREMOTE_ONLY*, see _<<Boolean values>>_ above):: + + If true, ccache will only use <<config_remote_storage,remote storage>>. The + default is false. Note that cache statistics counters will still be kept in + the local cache directory unless <<config_stats,*stats*>> is false. + [#config_remote_storage] *remote_storage* (*CCACHE_REMOTE_STORAGE*):: This option specifies one or several storage backends (separated by space) - to query after checking the local cache. See _<<Remote storage backends>>_ - for documentation of syntax and available backends. + to query after checking the local cache (unless + <<config_remote_only,*remote_only*>> is true). See _<<Remote storage + backends>>_ for documentation of syntax and available backends. + Examples: + @@ -1034,10 +1042,12 @@ filesystem as the `CCACHE_DIR` path, but this requirement has been relaxed. == Remote storage backends The <<config_remote_storage,*remote_storage*>> option lets you configure ccache -to use one or several other storage backends in addition to the local cache -storage located in <<config_cache_dir,*cache_dir*>>. Note that cache statistics -counters will still be kept in the local cache directory -- remote storage -backends only store compilation results and manifests. +to use one or several remote storage backends. By default, the local cache +directory located in <<config_cache_dir,*cache_dir*>> will be queried first and +remote storage second, but <<config_remote_only,*remote_only*>> can be set to +true to disable local storage. Note that cache statistics counters will still be +kept in the local cache directory -- remote storage backends only store +compilation results and manifests. A remote storage backend is specified with a URL, optionally followed by a pipe (`|`) and a pipe-separated list of attributes. An attribute is _key_=_value_ or @@ -1078,7 +1088,8 @@ Examples: === Storage interaction The table below describes the interaction between local and remote storage on -cache hits and misses: +cache hits and misses if <<config_remote_only,*remote_only*>> is false (which is +the default): [options="header",cols="20%,20%,60%"] |============================================================================== @@ -1094,6 +1105,16 @@ cache hits and misses: ^[2]^ Unless local storage is set to share its cache hits with the <<config_reshare,*reshare*>> option. +If <<config_remote_only,*remote_only*>> is true: + +[options="header",cols="20%,20%,60%"] +|============================================================================== +| *Local storage* | *Remote storage* | *What happens* + +| - | miss | Compile, write to remote, don't write to local +| - | hit | Read from remote, don't write to local + +|============================================================================== === File storage backend diff --git a/src/Config.cpp b/src/Config.cpp index de5b5247..9d2b3ae4 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -91,6 +91,7 @@ enum class ConfigItem { read_only, read_only_direct, recache, + remote_only, remote_storage, reshare, run_second_cpp, @@ -145,6 +146,7 @@ const std::unordered_map<std::string, ConfigKeyTableEntry> k_config_key_table = {"read_only", {ConfigItem::read_only}}, {"read_only_direct", {ConfigItem::read_only_direct}}, {"recache", {ConfigItem::recache}}, + {"remote_only", {ConfigItem::remote_only}}, {"remote_storage", {ConfigItem::remote_storage}}, {"reshare", {ConfigItem::reshare}}, {"run_second_cpp", {ConfigItem::run_second_cpp}}, @@ -193,6 +195,7 @@ const std::unordered_map<std::string, std::string> k_env_variable_table = { {"READONLY", "read_only"}, {"READONLY_DIRECT", "read_only_direct"}, {"RECACHE", "recache"}, + {"REMOTE_ONLY", "remote_only"}, {"REMOTE_STORAGE", "remote_storage"}, {"RESHARE", "reshare"}, {"SECONDARY_STORAGE", "remote_storage"}, // Alias for CCACHE_REMOTE_STORAGE @@ -775,6 +778,9 @@ Config::get_string_value(const std::string& key) const case ConfigItem::recache: return format_bool(m_recache); + case ConfigItem::remote_only: + return format_bool(m_remote_only); + case ConfigItem::remote_storage: return m_remote_storage; @@ -1028,6 +1034,10 @@ Config::set_item(const std::string& key, m_recache = parse_bool(value, env_var_key, negate); break; + case ConfigItem::remote_only: + m_remote_only = parse_bool(value, env_var_key, negate); + break; + case ConfigItem::remote_storage: m_remote_storage = Util::expand_environment_variables(value); break; diff --git a/src/Config.hpp b/src/Config.hpp index 7ee59921..7a01b35e 100644 --- a/src/Config.hpp +++ b/src/Config.hpp @@ -84,6 +84,7 @@ public: bool read_only() const; bool read_only_direct() const; bool recache() const; + bool remote_only() const; const std::string& remote_storage() const; bool reshare() const; bool run_second_cpp() const; @@ -193,6 +194,7 @@ private: bool m_recache = false; bool m_reshare = false; bool m_run_second_cpp = true; + bool m_remote_only = false; std::string m_remote_storage; core::Sloppiness m_sloppiness; bool m_stats = true; @@ -437,6 +439,12 @@ Config::run_second_cpp() const return m_run_second_cpp; } +inline bool +Config::remote_only() const +{ + return m_remote_only; +} + inline const std::string& Config::remote_storage() const { diff --git a/src/ccache.cpp b/src/ccache.cpp index 6cf1c53a..26b4d9fd 100644 --- a/src/ccache.cpp +++ b/src/ccache.cpp @@ -901,10 +901,14 @@ write_result(Context& ctx, core::CacheEntry::Header header(ctx.config, core::CacheEntryType::result); const auto cache_entry_data = core::CacheEntry::serialize(header, serializer); - const auto raw_files = serializer.get_raw_files(); - if (!raw_files.empty()) { - ctx.storage.local.put_raw_files(result_key, raw_files); + + if (!ctx.config.remote_only()) { + const auto& raw_files = serializer.get_raw_files(); + if (!raw_files.empty()) { + ctx.storage.local.put_raw_files(result_key, raw_files); + } } + ctx.storage.put(result_key, core::CacheEntryType::result, cache_entry_data); return true; @@ -1742,7 +1746,7 @@ hash_direct_mode_specific_data(Context& ctx, } }); MTR_END("manifest", "manifest_get"); - if (read_manifests > 1) { + if (read_manifests > 1 && !ctx.config.remote_only()) { MTR_SCOPE("manifest", "merge"); LOG("Storing merged manifest {} locally", manifest_key->to_string()); core::CacheEntry::Header header(ctx.config, core::CacheEntryType::manifest); diff --git a/src/storage/Storage.cpp b/src/storage/Storage.cpp index 11490edc..a607be50 100644 --- a/src/storage/Storage.cpp +++ b/src/storage/Storage.cpp @@ -237,20 +237,24 @@ Storage::get(const Digest& key, { MTR_SCOPE("storage", "get"); - auto value = local.get(key, type); - local.increment_statistic(value ? core::Statistic::local_storage_hit - : core::Statistic::local_storage_miss); - if (value) { - if (m_config.reshare()) { - put_in_remote_storage(key, *value, true); - } - if (entry_receiver(std::move(*value))) { - return; + if (!m_config.remote_only()) { + auto value = local.get(key, type); + local.increment_statistic(value ? core::Statistic::local_storage_hit + : core::Statistic::local_storage_miss); + if (value) { + if (m_config.reshare()) { + put_in_remote_storage(key, *value, true); + } + if (entry_receiver(std::move(*value))) { + return; + } } } get_from_remote_storage(key, [&](util::Bytes&& data) { - local.put(key, type, data, true); + if (!m_config.remote_only()) { + local.put(key, type, data, true); + } return entry_receiver(std::move(data)); }); } @@ -262,7 +266,9 @@ Storage::put(const Digest& key, { MTR_SCOPE("storage", "put"); - local.put(key, type, value); + if (!m_config.remote_only()) { + local.put(key, type, value); + } put_in_remote_storage(key, value, false); } @@ -271,7 +277,9 @@ Storage::remove(const Digest& key, const core::CacheEntryType type) { MTR_SCOPE("storage", "remove"); - local.remove(key, type); + if (!m_config.remote_only()) { + local.remove(key, type); + } remove_from_remote_storage(key); } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 4cd38a38..939a220c 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -54,6 +54,7 @@ addtest(readonly) addtest(readonly_direct) addtest(remote_file) addtest(remote_http) +addtest(remote_only) addtest(remote_redis) addtest(remote_redis_unix) addtest(remote_url) diff --git a/test/suites/remote_only.bash b/test/suites/remote_only.bash new file mode 100644 index 00000000..3dea32ac --- /dev/null +++ b/test/suites/remote_only.bash @@ -0,0 +1,33 @@ +SUITE_remote_only_SETUP() { + unset CCACHE_NODIRECT + export CCACHE_REMOTE_STORAGE="file:$PWD/remote" + export CCACHE_REMOTE_ONLY=1 + + generate_code 1 test.c +} + +SUITE_remote_only() { + # ------------------------------------------------------------------------- + TEST "Base case" + + $CCACHE_COMPILE -c test.c + expect_stat direct_cache_hit 0 + expect_stat cache_miss 1 + expect_stat files_in_cache 0 + expect_stat local_storage_hit 0 + expect_stat local_storage_miss 0 + expect_stat remote_storage_hit 0 + expect_stat remote_storage_miss 2 # result + manifest + expect_file_count 3 '*' remote # CACHEDIR.TAG + result + manifest + + $CCACHE_COMPILE -c test.c + expect_stat direct_cache_hit 1 + expect_stat cache_miss 1 + expect_stat files_in_cache 0 + expect_stat local_storage_hit 0 + expect_stat local_storage_miss 0 + expect_stat remote_storage_hit 2 + expect_stat remote_storage_miss 2 + expect_stat files_in_cache 0 + expect_file_count 3 '*' remote # CACHEDIR.TAG + result + manifest +} diff --git a/unittest/test_Config.cpp b/unittest/test_Config.cpp index 14456811..0226101f 100644 --- a/unittest/test_Config.cpp +++ b/unittest/test_Config.cpp @@ -71,6 +71,7 @@ TEST_CASE("Config: default values") CHECK_FALSE(config.read_only()); CHECK_FALSE(config.read_only_direct()); CHECK_FALSE(config.recache()); + CHECK_FALSE(config.remote_only()); CHECK(config.remote_storage().empty()); CHECK_FALSE(config.reshare()); CHECK(config.run_second_cpp()); @@ -413,6 +414,7 @@ TEST_CASE("Config::visit_items") "read_only = true\n" "read_only_direct = true\n" "recache = true\n" + "remote_only = true\n" "remote_storage = rs\n" "reshare = true\n" "run_second_cpp = false\n" @@ -473,6 +475,7 @@ TEST_CASE("Config::visit_items") "(test.conf) read_only = true", "(test.conf) read_only_direct = true", "(test.conf) recache = true", + "(test.conf) remote_only = true", "(test.conf) remote_storage = rs", "(test.conf) reshare = true", "(test.conf) run_second_cpp = false", |