diff options
25 files changed, 1072 insertions, 931 deletions
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 47e00207..93714230 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -35,18 +35,6 @@ jobs: - os: ubuntu-18.04 compiler: clang - version: "5.0" - - - os: ubuntu-18.04 - compiler: clang - version: "6.0" - - - os: ubuntu-18.04 - compiler: clang - version: "7" - - - os: ubuntu-18.04 - compiler: clang version: "8" - os: ubuntu-20.04 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c3c89fa3..c1bc5e3c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -9,10 +9,6 @@ set( Hash.cpp Logging.cpp ProgressBar.cpp - Result.cpp - ResultExtractor.cpp - ResultInspector.cpp - ResultRetriever.cpp SignalHandler.cpp Stat.cpp TemporaryFile.cpp diff --git a/src/Result.cpp b/src/Result.cpp deleted file mode 100644 index 9f7204f2..00000000 --- a/src/Result.cpp +++ /dev/null @@ -1,439 +0,0 @@ -// Copyright (C) 2019-2022 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "Result.hpp" - -#include "AtomicFile.hpp" -#include "Config.hpp" -#include "Context.hpp" -#include "Fd.hpp" -#include "File.hpp" -#include "Logging.hpp" -#include "Stat.hpp" -#include "Util.hpp" - -#include <ccache.hpp> -#include <core/CacheEntryReader.hpp> -#include <core/CacheEntryWriter.hpp> -#include <core/FileReader.hpp> -#include <core/FileWriter.hpp> -#include <core/Statistic.hpp> -#include <core/exceptions.hpp> -#include <core/wincompat.hpp> -#include <fmtmacros.hpp> -#include <util/path.hpp> - -#include <fcntl.h> -#include <sys/stat.h> -#include <sys/types.h> - -#ifdef HAVE_UNISTD_H -# include <unistd.h> -#endif - -#include <algorithm> - -// Result data format -// ================== -// -// Integers are big-endian. -// -// <payload> ::= <format_ver> <n_entries> <entry>* -// <format_ver> ::= uint8_t -// <n_entries> ::= uint8_t -// <entry> ::= <embedded_file_entry> | <raw_file_entry> -// <embedded_file_entry> ::= <embedded_file_marker> <suffix_len> <suffix> -// <data_len> <data> -// <embedded_file_marker> ::= 0 (uint8_t) -// <embedded_file_type> ::= uint8_t -// <data_len> ::= uint64_t -// <data> ::= data_len bytes -// <raw_file_entry> ::= <raw_file_marker> <suffix_len> <suffix> <file_len> -// <raw_file_marker> ::= 1 (uint8_t) -// <file_len> ::= uint64_t -// <epilogue> ::= <checksum> -// <checksum> ::= uint64_t ; XXH3 of content bytes - -namespace { - -const uint8_t k_result_format_version = 0; - -// File data stored inside the result file. -const uint8_t k_embedded_file_marker = 0; - -// File stored as-is in the file system. -const uint8_t k_raw_file_marker = 1; - -const uint8_t k_max_raw_file_entries = 10; - -std::string -get_raw_file_path(std::string_view result_path, uint8_t entry_number) -{ - if (entry_number >= 10) { - // To support more entries in the future, encode to [0-9a-z]. Note that - // PrimaryStorage::evict currently assumes that the entry number is - // represented as one character. - throw core::Error(FMT("Too high raw file entry number: {}", entry_number)); - } - - const auto prefix = result_path.substr( - 0, result_path.length() - Result::k_file_suffix.length()); - return FMT("{}{}W", prefix, entry_number); -} - -bool -should_store_raw_file(const Config& config, Result::FileType type) -{ - if (!config.file_clone() && !config.hard_link()) { - return false; - } - - // Only store object files as raw files since there are several problems with - // storing other file types: - // - // 1. The compiler unlinks object files before writing to them but it doesn't - // unlink .d files, so just it's possible to corrupt .d files just by - // running the compiler (see ccache issue 599). - // 2. .d files cause trouble for automake if hard-linked (see ccache issue - // 378). - // 3. It's unknown how the compiler treats other file types, so better safe - // than sorry. - // - // It would be possible to store all files in raw form for the file_clone case - // and only hard link object files. However, most likely it's only object - // files that become large enough that it's of interest to clone or hard link - // them, so we keep things simple for now. This will also save i-nodes in the - // cache. - return type == Result::FileType::object; -} - -} // namespace - -namespace Result { - -const std::string k_file_suffix = "R"; -const uint8_t k_magic[4] = {'c', 'C', 'r', 'S'}; -const uint8_t k_version = 1; -const char* const k_unknown_file_type = "<unknown type>"; - -const char* -file_type_to_string(FileType type) -{ - switch (type) { - case FileType::object: - return ".o"; - - case FileType::dependency: - return ".d"; - - case FileType::stderr_output: - return "<stderr>"; - - case FileType::coverage_unmangled: - return ".gcno-unmangled"; - - case FileType::stackusage: - return ".su"; - - case FileType::diagnostic: - return ".dia"; - - case FileType::dwarf_object: - return ".dwo"; - - case FileType::coverage_mangled: - return ".gcno-mangled"; - - case FileType::stdout_output: - return "<stdout>"; - - case FileType::assembler_listing: - return ".al"; - } - - return k_unknown_file_type; -} - -std::string -gcno_file_in_mangled_form(const Context& ctx) -{ - const auto& output_obj = ctx.args_info.output_obj; - const std::string abs_output_obj = - util::is_absolute_path(output_obj) - ? output_obj - : FMT("{}/{}", ctx.apparent_cwd, output_obj); - std::string hashified_obj = abs_output_obj; - std::replace(hashified_obj.begin(), hashified_obj.end(), '/', '#'); - return Util::change_extension(hashified_obj, ".gcno"); -} - -std::string -gcno_file_in_unmangled_form(const Context& ctx) -{ - return Util::change_extension(ctx.args_info.output_obj, ".gcno"); -} - -FileSizeAndCountDiff& -FileSizeAndCountDiff::operator+=(const FileSizeAndCountDiff& other) -{ - size_kibibyte += other.size_kibibyte; - count += other.count; - return *this; -} - -Result::Reader::Reader(core::CacheEntryReader& cache_entry_reader, - const std::string& result_path) - : m_reader(cache_entry_reader), - m_result_path(result_path) -{ -} - -void -Reader::read(Consumer& consumer) -{ - if (m_reader.header().entry_type != core::CacheEntryType::result) { - throw core::Error(FMT("Unexpected cache entry type: {}", - to_string(m_reader.header().entry_type))); - } - - const auto result_format_version = m_reader.read_int<uint8_t>(); - if (result_format_version != k_result_format_version) { - throw core::Error( - FMT("Unknown result format version: {}", result_format_version)); - } - - const auto n_entries = m_reader.read_int<uint8_t>(); - if (n_entries >= k_max_raw_file_entries) { - throw core::Error(FMT( - "Too many raw file entries: {} > {}", n_entries, k_max_raw_file_entries)); - } - - uint8_t i; - for (i = 0; i < n_entries; ++i) { - read_entry(i, consumer); - } - - if (i != n_entries) { - throw core::Error( - FMT("Too few entries (read {}, expected {})", i, n_entries)); - } - - m_reader.finalize(); -} - -void -Reader::read_entry(uint8_t entry_number, Reader::Consumer& consumer) -{ - const auto marker = m_reader.read_int<uint8_t>(); - - switch (marker) { - case k_embedded_file_marker: - case k_raw_file_marker: - break; - - default: - throw core::Error(FMT("Unknown entry type: {}", marker)); - } - - const auto type = m_reader.read_int<UnderlyingFileTypeInt>(); - const auto file_type = FileType(type); - const auto file_len = m_reader.read_int<uint64_t>(); - - if (marker == k_embedded_file_marker) { - consumer.on_entry_start(entry_number, file_type, file_len, std::nullopt); - - uint8_t buf[CCACHE_READ_BUFFER_SIZE]; - size_t remain = file_len; - while (remain > 0) { - size_t n = std::min(remain, sizeof(buf)); - m_reader.read(buf, n); - consumer.on_entry_data(buf, n); - remain -= n; - } - } else { - ASSERT(marker == k_raw_file_marker); - - std::string raw_path; - if (m_result_path != "-") { - raw_path = get_raw_file_path(m_result_path, entry_number); - auto st = Stat::stat(raw_path, Stat::OnError::throw_error); - if (st.size() != file_len) { - throw core::Error( - FMT("Bad file size of {} (actual {} bytes, expected {} bytes)", - raw_path, - st.size(), - file_len)); - } - } - - consumer.on_entry_start(entry_number, file_type, file_len, raw_path); - } - - consumer.on_entry_end(); -} - -Writer::Writer(const Config& config, const std::string& result_path) - : m_config(config), - m_result_path(result_path) -{ -} - -void -Writer::write_data(const FileType file_type, const std::string& data) -{ - m_entries_to_write.push_back(Entry{file_type, ValueType::data, data}); -} - -void -Writer::write_file(const FileType file_type, const std::string& path) -{ - m_entries_to_write.push_back(Entry{file_type, ValueType::path, path}); -} - -nonstd::expected<FileSizeAndCountDiff, std::string> -Writer::finalize() -{ - try { - return do_finalize(); - } catch (const core::Error& e) { - return nonstd::make_unexpected(e.what()); - } -} - -FileSizeAndCountDiff -Writer::do_finalize() -{ - FileSizeAndCountDiff file_size_and_count_diff{0, 0}; - uint64_t payload_size = 0; - payload_size += 1; // format_ver - payload_size += 1; // n_entries - for (const auto& entry : m_entries_to_write) { - payload_size += 1; // embedded_file_marker - payload_size += 1; // embedded_file_type - payload_size += 8; // data_len - payload_size += // data - entry.value_type == ValueType::data - ? entry.value.size() - : Stat::stat(entry.value, Stat::OnError::throw_error).size(); - } - - AtomicFile atomic_result_file(m_result_path, AtomicFile::Mode::binary); - core::CacheEntryHeader header(core::CacheEntryType::result, - compression::type_from_config(m_config), - compression::level_from_config(m_config), - time(nullptr), - CCACHE_VERSION, - m_config.namespace_()); - header.set_entry_size_from_payload_size(payload_size); - - core::FileWriter file_writer(atomic_result_file.stream()); - core::CacheEntryWriter writer(file_writer, header); - - writer.write_int(k_result_format_version); - writer.write_int<uint8_t>(m_entries_to_write.size()); - - uint8_t entry_number = 0; - for (const auto& entry : m_entries_to_write) { - const bool store_raw = entry.value_type == ValueType::path - && should_store_raw_file(m_config, entry.file_type); - const uint64_t entry_size = - entry.value_type == ValueType::data - ? entry.value.size() - : Stat::stat(entry.value, Stat::OnError::throw_error).size(); - - LOG("Storing {} entry #{} {} ({} bytes){}", - store_raw ? "raw" : "embedded", - entry_number, - file_type_to_string(entry.file_type), - entry_size, - entry.value_type == ValueType::data ? "" - : FMT(" from {}", entry.value)); - - writer.write_int<uint8_t>(store_raw ? k_raw_file_marker - : k_embedded_file_marker); - writer.write_int(UnderlyingFileTypeInt(entry.file_type)); - writer.write_int(entry_size); - - if (store_raw) { - file_size_and_count_diff += - write_raw_file_entry(entry.value, entry_number); - } else if (entry.value_type == ValueType::data) { - writer.write(entry.value.data(), entry.value.size()); - } else { - write_embedded_file_entry(writer, entry.value, entry_size); - } - - ++entry_number; - } - - writer.finalize(); - atomic_result_file.commit(); - - return file_size_and_count_diff; -} - -void -Result::Writer::write_embedded_file_entry(core::CacheEntryWriter& writer, - const std::string& path, - const uint64_t file_size) -{ - Fd file(open(path.c_str(), O_RDONLY | O_BINARY)); - if (!file) { - throw core::Error(FMT("Failed to open {} for reading", path)); - } - - uint64_t remain = file_size; - while (remain > 0) { - uint8_t buf[CCACHE_READ_BUFFER_SIZE]; - size_t n = std::min(remain, static_cast<uint64_t>(sizeof(buf))); - auto bytes_read = read(*file, buf, n); - if (bytes_read == -1) { - if (errno == EINTR) { - continue; - } - throw core::Error( - FMT("Error reading from {}: {}", path, strerror(errno))); - } - if (bytes_read == 0) { - throw core::Error(FMT("Error reading from {}: end of file", path)); - } - writer.write(buf, bytes_read); - remain -= bytes_read; - } -} - -FileSizeAndCountDiff -Result::Writer::write_raw_file_entry(const std::string& path, - uint8_t entry_number) -{ - const auto raw_file = get_raw_file_path(m_result_path, entry_number); - const auto old_stat = Stat::stat(raw_file); - try { - Util::clone_hard_link_or_copy_file(m_config, path, raw_file, true); - } catch (core::Error& e) { - throw core::Error( - FMT("Failed to store {} as raw file {}: {}", path, raw_file, e.what())); - } - const auto new_stat = Stat::stat(raw_file); - return { - Util::size_change_kibibyte(old_stat, new_stat), - (new_stat ? 1 : 0) - (old_stat ? 1 : 0), - }; -} - -} // namespace Result diff --git a/src/ResultExtractor.cpp b/src/ResultExtractor.cpp deleted file mode 100644 index ec38c018..00000000 --- a/src/ResultExtractor.cpp +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright (C) 2020-2022 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "ResultExtractor.hpp" - -#include "Util.hpp" -#include "fmtmacros.hpp" - -#include <core/exceptions.hpp> -#include <core/wincompat.hpp> -#include <fmtmacros.hpp> -#include <util/file.hpp> - -#include <fcntl.h> -#include <sys/stat.h> -#include <sys/types.h> - -ResultExtractor::ResultExtractor(const std::string& directory) - : m_directory(directory) -{ -} - -void -ResultExtractor::on_entry_start(uint8_t /*entry_number*/, - Result::FileType file_type, - uint64_t /*file_len*/, - std::optional<std::string> raw_file) -{ - std::string suffix = Result::file_type_to_string(file_type); - if (suffix == Result::k_unknown_file_type) { - suffix = - FMT(".type_{}", static_cast<Result::UnderlyingFileTypeInt>(file_type)); - } else if (suffix[0] == '<') { - suffix[0] = '.'; - suffix.resize(suffix.length() - 1); - } - - m_dest_path = FMT("{}/ccache-result{}", m_directory, suffix); - - if (!raw_file) { - m_dest_fd = Fd( - open(m_dest_path.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666)); - if (!m_dest_fd) { - throw core::Error( - FMT("Failed to open {} for writing: {}", m_dest_path, strerror(errno))); - } - } else if (raw_file->empty()) { - PRINT_RAW(stderr, - "Note: Can't extract raw file since reading from stdin\n"); - } else { - try { - Util::copy_file(*raw_file, m_dest_path, false); - } catch (core::Error& e) { - throw core::Error( - FMT("Failed to copy {} to {}: {}", *raw_file, m_dest_path, e.what())); - } - } -} - -void -ResultExtractor::on_entry_data(const uint8_t* data, size_t size) -{ - ASSERT(m_dest_fd); - - const auto result = util::write_fd(*m_dest_fd, data, size); - if (!result) { - throw core::Error( - FMT("Failed to write to {}: {}", m_dest_path, result.error())); - } -} - -void -ResultExtractor::on_entry_end() -{ - if (m_dest_fd) { - m_dest_fd.close(); - } -} diff --git a/src/ResultRetriever.cpp b/src/ResultRetriever.cpp deleted file mode 100644 index 5e2ec0f6..00000000 --- a/src/ResultRetriever.cpp +++ /dev/null @@ -1,208 +0,0 @@ -// Copyright (C) 2020-2022 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "ResultRetriever.hpp" - -#include "Context.hpp" -#include "Depfile.hpp" -#include "Logging.hpp" - -#include <core/exceptions.hpp> -#include <core/wincompat.hpp> -#include <fmtmacros.hpp> -#include <util/file.hpp> - -#include <fcntl.h> -#include <sys/stat.h> -#include <sys/types.h> - -#ifdef HAVE_UNISTD_H -# include <unistd.h> -#endif - -using Result::FileType; - -ResultRetriever::ResultRetriever(Context& ctx) : m_ctx(ctx) -{ -} - -void -ResultRetriever::on_entry_start(uint8_t entry_number, - FileType file_type, - uint64_t file_len, - std::optional<std::string> raw_file) -{ - LOG("Reading {} entry #{} {} ({} bytes)", - raw_file ? "raw" : "embedded", - entry_number, - Result::file_type_to_string(file_type), - file_len); - - std::string dest_path; - m_dest_file_type = file_type; - - switch (file_type) { - case FileType::object: - dest_path = m_ctx.args_info.output_obj; - break; - - case FileType::dependency: - // Dependency file: Open destination file but accumulate data in m_dest_data - // and write it in on_entry_end. - if (m_ctx.args_info.generating_dependencies) { - dest_path = m_ctx.args_info.output_dep; - m_dest_data.reserve(file_len); - } - break; - - case FileType::stdout_output: - case FileType::stderr_output: - // Stdout/stderr data: Don't open a destination file. Instead accumulate it - // in m_dest_data and write it in on_entry_end. - m_dest_data.reserve(file_len); - break; - - case FileType::coverage_unmangled: - if (m_ctx.args_info.generating_coverage) { - dest_path = Util::change_extension(m_ctx.args_info.output_obj, ".gcno"); - } - break; - - case FileType::stackusage: - if (m_ctx.args_info.generating_stackusage) { - dest_path = m_ctx.args_info.output_su; - } - break; - - case FileType::diagnostic: - if (m_ctx.args_info.generating_diagnostics) { - dest_path = m_ctx.args_info.output_dia; - } - break; - - case FileType::dwarf_object: - if (m_ctx.args_info.seen_split_dwarf - && m_ctx.args_info.output_obj != "/dev/null") { - dest_path = m_ctx.args_info.output_dwo; - } - break; - - case FileType::coverage_mangled: - if (m_ctx.args_info.generating_coverage) { - dest_path = Result::gcno_file_in_mangled_form(m_ctx); - } - break; - - case FileType::assembler_listing: - dest_path = m_ctx.args_info.output_al; - break; - } - - if (file_type == FileType::stdout_output - || file_type == FileType::stderr_output) { - // Written in on_entry_end. - } else if (dest_path.empty()) { - LOG_RAW("Not writing"); - } else if (dest_path == "/dev/null") { - LOG_RAW("Not writing to /dev/null"); - } else if (raw_file) { - Util::clone_hard_link_or_copy_file( - m_ctx.config, *raw_file, dest_path, false); - - // Update modification timestamp to save the file from LRU cleanup (and, if - // hard-linked, to make the object file newer than the source file). - util::set_timestamps(*raw_file); - } else { - LOG("Writing to {}", dest_path); - m_dest_fd = Fd( - open(dest_path.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666)); - if (!m_dest_fd) { - throw WriteError( - FMT("Failed to open {} for writing: {}", dest_path, strerror(errno))); - } - m_dest_path = dest_path; - } -} - -void -ResultRetriever::on_entry_data(const uint8_t* data, size_t size) -{ - ASSERT(!((m_dest_file_type == FileType::stdout_output - || m_dest_file_type == FileType::stderr_output) - && m_dest_fd)); - - if (m_dest_file_type == FileType::stdout_output - || m_dest_file_type == FileType::stderr_output - || (m_dest_file_type == FileType::dependency && !m_dest_path.empty())) { - m_dest_data.append(reinterpret_cast<const char*>(data), size); - } else if (m_dest_fd) { - const auto result = util::write_fd(*m_dest_fd, data, size); - if (!result) { - throw WriteError( - FMT("Failed to write to {}: {}", m_dest_path, result.error())); - } - } -} - -void -ResultRetriever::on_entry_end() -{ - if (m_dest_file_type == FileType::stdout_output) { - LOG("Writing to file descriptor {}", STDOUT_FILENO); - Util::send_to_fd(m_ctx, m_dest_data, STDOUT_FILENO); - } else if (m_dest_file_type == FileType::stderr_output) { - LOG("Writing to file descriptor {}", STDERR_FILENO); - Util::send_to_fd(m_ctx, m_dest_data, STDERR_FILENO); - } else if (m_dest_file_type == FileType::dependency && !m_dest_path.empty()) { - write_dependency_file(); - } - - if (m_dest_fd) { - m_dest_fd.close(); - } - m_dest_path.clear(); - m_dest_data.clear(); -} - -void -ResultRetriever::write_dependency_file() -{ - ASSERT(m_ctx.args_info.dependency_target); - - auto write_data = [&](auto data, auto size) { - const auto result = util::write_fd(*m_dest_fd, data, size); - if (!result) { - throw WriteError( - FMT("Failed to write to {}: {}", m_dest_path, result.error())); - } - }; - - size_t start_pos = 0; - const size_t colon_pos = m_dest_data.find(": "); - if (colon_pos != std::string::npos) { - const auto obj_in_dep_file = - std::string_view(m_dest_data).substr(0, colon_pos); - const auto& dep_target = *m_ctx.args_info.dependency_target; - if (obj_in_dep_file != dep_target) { - write_data(dep_target.data(), dep_target.length()); - start_pos = colon_pos; - } - } - - write_data(m_dest_data.data() + start_pos, m_dest_data.length() - start_pos); -} diff --git a/src/ccache.cpp b/src/ccache.cpp index afbce17b..6b12739f 100644 --- a/src/ccache.cpp +++ b/src/ccache.cpp @@ -29,8 +29,6 @@ #include "Hash.hpp" #include "Logging.hpp" #include "MiniTrace.hpp" -#include "Result.hpp" -#include "ResultRetriever.hpp" #include "SignalHandler.hpp" #include "TemporaryFile.hpp" #include "UmaskScope.hpp" @@ -50,6 +48,8 @@ #include <core/FileReader.hpp> #include <core/FileWriter.hpp> #include <core/Manifest.hpp> +#include <core/Result.hpp> +#include <core/ResultRetriever.hpp> #include <core/Statistics.hpp> #include <core/StatsLog.hpp> #include <core/exceptions.hpp> @@ -849,8 +849,8 @@ find_coverage_file(const Context& ctx) // (in CWD) if -fprofile-dir=DIR is present (regardless of DIR) instead of the // traditional /dir/to/example.gcno. - std::string mangled_form = Result::gcno_file_in_mangled_form(ctx); - std::string unmangled_form = Result::gcno_file_in_unmangled_form(ctx); + std::string mangled_form = core::Result::gcno_file_in_mangled_form(ctx); + std::string unmangled_form = core::Result::gcno_file_in_unmangled_form(ctx); std::string found_file; if (Stat::stat(mangled_form)) { LOG("Found coverage file {}", mangled_form); @@ -880,64 +880,94 @@ write_result(Context& ctx, const std::string& stdout_data, const std::string& stderr_data) { - Result::Writer result_writer(ctx.config, result_path); + core::Result::Serializer serializer(ctx.config); if (!stderr_data.empty()) { - result_writer.write_data(Result::FileType::stderr_output, stderr_data); + serializer.add_data(core::Result::FileType::stderr_output, stderr_data); } // Write stdout only after stderr (better with MSVC), as ResultRetriever // will later print process them in the order they are read. if (!stdout_data.empty()) { - result_writer.write_data(Result::FileType::stdout_output, stdout_data); + serializer.add_data(core::Result::FileType::stdout_output, stdout_data); } if (obj_stat) { - result_writer.write_file(Result::FileType::object, - ctx.args_info.output_obj); + serializer.add_file(core::Result::FileType::object, + ctx.args_info.output_obj); } if (ctx.args_info.generating_dependencies) { - result_writer.write_file(Result::FileType::dependency, - ctx.args_info.output_dep); + serializer.add_file(core::Result::FileType::dependency, + ctx.args_info.output_dep); } if (ctx.args_info.generating_coverage) { const auto coverage_file = find_coverage_file(ctx); if (!coverage_file.found) { return false; } - result_writer.write_file(coverage_file.mangled - ? Result::FileType::coverage_mangled - : Result::FileType::coverage_unmangled, - coverage_file.path); + serializer.add_file(coverage_file.mangled + ? core::Result::FileType::coverage_mangled + : core::Result::FileType::coverage_unmangled, + coverage_file.path); } if (ctx.args_info.generating_stackusage) { - result_writer.write_file(Result::FileType::stackusage, - ctx.args_info.output_su); + serializer.add_file(core::Result::FileType::stackusage, + ctx.args_info.output_su); } if (ctx.args_info.generating_diagnostics) { - result_writer.write_file(Result::FileType::diagnostic, - ctx.args_info.output_dia); + serializer.add_file(core::Result::FileType::diagnostic, + ctx.args_info.output_dia); } if (ctx.args_info.seen_split_dwarf && Stat::stat(ctx.args_info.output_dwo)) { // Only store .dwo file if it was created by the compiler (GCC and Clang // behave differently e.g. for "-gsplit-dwarf -g1"). - result_writer.write_file(Result::FileType::dwarf_object, - ctx.args_info.output_dwo); + serializer.add_file(core::Result::FileType::dwarf_object, + ctx.args_info.output_dwo); } if (!ctx.args_info.output_al.empty()) { - result_writer.write_file(Result::FileType::assembler_listing, - ctx.args_info.output_al); + serializer.add_file(core::Result::FileType::assembler_listing, + ctx.args_info.output_al); } - const auto file_size_and_count_diff = result_writer.finalize(); - if (file_size_and_count_diff) { + AtomicFile atomic_result_file(result_path, AtomicFile::Mode::binary); + core::CacheEntryHeader header(core::CacheEntryType::result, + compression::type_from_config(ctx.config), + compression::level_from_config(ctx.config), + time(nullptr), + CCACHE_VERSION, + ctx.config.namespace_()); + header.set_entry_size_from_payload_size(serializer.serialized_size()); + + core::FileWriter file_writer(atomic_result_file.stream()); + core::CacheEntryWriter writer(file_writer, header); + + std::vector<uint8_t> payload; + payload.reserve(serializer.serialized_size()); + const auto serialize_result = serializer.serialize(payload); + for (auto [file_number, source_path] : serialize_result.raw_files) { + const auto dest_path = storage::primary::PrimaryStorage::get_raw_file_path( + result_path, file_number); + const auto old_stat = Stat::stat(dest_path); + try { + Util::clone_hard_link_or_copy_file( + ctx.config, source_path, dest_path, true); + } catch (core::Error& e) { + LOG("Failed to store {} as raw file {}: {}", + source_path, + dest_path, + e.what()); + throw; + } + const auto new_stat = Stat::stat(dest_path); ctx.storage.primary.increment_statistic( - Statistic::cache_size_kibibyte, file_size_and_count_diff->size_kibibyte); - ctx.storage.primary.increment_statistic(Statistic::files_in_cache, - file_size_and_count_diff->count); - } else { - LOG("Error: {}", file_size_and_count_diff.error()); - return false; + Statistic::cache_size_kibibyte, + Util::size_change_kibibyte(old_stat, new_stat)); + ctx.storage.primary.increment_statistic( + Statistic::files_in_cache, (new_stat ? 1 : 0) - (old_stat ? 1 : 0)); } + writer.write(payload.data(), payload.size()); + writer.finalize(); + atomic_result_file.commit(); + return true; } @@ -1110,8 +1140,13 @@ to_cache(Context& ctx, MTR_BEGIN("result", "result_put"); const bool added = ctx.storage.put( *result_key, core::CacheEntryType::result, [&](const auto& path) { - return write_result( - ctx, path, obj_stat, result->stdout_data, result->stderr_data); + try { + return write_result( + ctx, path, obj_stat, result->stdout_data, result->stderr_data); + } catch (core::Error& e) { + LOG("Error writing to {}: {}", path, e.what()); + return false; + } }); MTR_END("result", "result_put"); if (!added) { @@ -1897,7 +1932,7 @@ calculate_result_and_manifest_key(Context& ctx, bool found_ccbin = false; hash.hash_delimiter("result version"); - hash.hash(Result::k_version); + hash.hash(core::Result::k_version); if (direct_mode) { hash.hash_delimiter("manifest version"); @@ -2012,11 +2047,14 @@ from_cache(Context& ctx, FromCacheCallMode mode, const Digest& result_key) } core::FileReader file_reader(file.get()); core::CacheEntryReader cache_entry_reader(file_reader); - Result::Reader result_reader(cache_entry_reader, *result_path); - ResultRetriever result_retriever(ctx); - - result_reader.read(result_retriever); - } catch (ResultRetriever::WriteError& e) { + std::vector<uint8_t> payload; + payload.resize(cache_entry_reader.header().payload_size()); + cache_entry_reader.read(payload.data(), payload.size()); + core::Result::Deserializer deserializer(payload); + core::ResultRetriever result_retriever(ctx, result_path); + deserializer.visit(result_retriever); + cache_entry_reader.finalize(); + } catch (core::ResultRetriever::WriteError& e) { LOG( "Write error when retrieving result from {}: {}", *result_path, e.what()); return nonstd::make_unexpected(Statistic::bad_output_file); diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 54549c8e..83a6d4b0 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -4,6 +4,10 @@ set( CacheEntryReader.cpp CacheEntryWriter.cpp Manifest.cpp + Result.cpp + ResultExtractor.cpp + ResultInspector.cpp + ResultRetriever.cpp Statistics.cpp StatisticsCounters.cpp StatsLog.cpp diff --git a/src/core/CacheEntryDataReader.hpp b/src/core/CacheEntryDataReader.hpp new file mode 100644 index 00000000..12466f79 --- /dev/null +++ b/src/core/CacheEntryDataReader.hpp @@ -0,0 +1,95 @@ +// Copyright (C) 2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include <Util.hpp> +#include <core/exceptions.hpp> +#include <fmtmacros.hpp> +#include <util/string.hpp> + +#include <third_party/nonstd/span.hpp> + +#include <cstddef> +#include <string_view> + +namespace core { + +class CacheEntryDataReader +{ +public: + CacheEntryDataReader(nonstd::span<const uint8_t> data); + + // Read `size` bytes. Throws `core::Error` on failure. + nonstd::span<const uint8_t> read_bytes(size_t size); + + // Read a string of length `length`. Throws `core::Error` on failure. + std::string_view read_str(size_t length); + + // Read an integer. Throws `core::Error` on failure. + template<typename T> T read_int(); + + // Read an integer into `value`. Throws `core::Error` on failure. + template<typename T> void read_int(T& value); + +private: + nonstd::span<const uint8_t> m_data; +}; + +inline CacheEntryDataReader::CacheEntryDataReader( + nonstd::span<const uint8_t> data) + : m_data(data) +{ +} + +inline nonstd::span<const uint8_t> +CacheEntryDataReader::read_bytes(size_t size) +{ + if (size > m_data.size()) { + throw core::Error(FMT("CacheEntryDataReader: data underflow of {} bytes", + size - m_data.size())); + } + const auto bytes = m_data.first(size); + m_data = m_data.subspan(size); + return bytes; +} + +inline std::string_view +CacheEntryDataReader::read_str(const size_t length) +{ + return util::to_string_view(read_bytes(length)); +} + +template<typename T> +inline T +CacheEntryDataReader::read_int() +{ + const auto buffer = read_bytes(sizeof(T)); + T value; + Util::big_endian_to_int(buffer.data(), value); + return value; +} + +template<typename T> +inline void +CacheEntryDataReader::read_int(T& value) +{ + value = read_int<T>(); +} + +} // namespace core diff --git a/src/core/CacheEntryDataWriter.hpp b/src/core/CacheEntryDataWriter.hpp new file mode 100644 index 00000000..122bee76 --- /dev/null +++ b/src/core/CacheEntryDataWriter.hpp @@ -0,0 +1,79 @@ +// Copyright (C) 2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include <Util.hpp> +#include <core/exceptions.hpp> +#include <fmtmacros.hpp> +#include <util/string.hpp> + +#include <third_party/nonstd/span.hpp> + +#include <cstddef> +#include <cstring> +#include <string_view> +#include <vector> + +namespace core { + +class CacheEntryDataWriter +{ +public: + CacheEntryDataWriter(std::vector<uint8_t>& output); + + // Write `data`. Throws `core::Error` on failure. + void write_bytes(nonstd::span<const uint8_t> data); + + // Write `data`. Throws `core::Error` on failure. + void write_str(std::string_view data); + + // Write integer `value`. Throws `core::Error` on failure. + template<typename T> void write_int(T value); + +private: + std::vector<uint8_t>& m_output; +}; + +inline CacheEntryDataWriter::CacheEntryDataWriter(std::vector<uint8_t>& output) + : m_output(output) +{ +} + +inline void +CacheEntryDataWriter::write_bytes(nonstd::span<const uint8_t> data) +{ + m_output.insert(m_output.end(), data.begin(), data.end()); +} + +template<typename T> +inline void +CacheEntryDataWriter::write_int(const T value) +{ + uint8_t buffer[sizeof(T)]; + Util::int_to_big_endian(value, buffer); + write_bytes(buffer); +} + +inline void +CacheEntryDataWriter::write_str(std::string_view value) +{ + write_bytes(util::to_span(value)); +} + +} // namespace core diff --git a/src/core/CacheEntryHeader.cpp b/src/core/CacheEntryHeader.cpp index e9cf93b8..b1bbe9a2 100644 --- a/src/core/CacheEntryHeader.cpp +++ b/src/core/CacheEntryHeader.cpp @@ -18,6 +18,7 @@ #include "CacheEntryHeader.hpp" +#include <core/exceptions.hpp> #include <fmtmacros.hpp> const size_t k_static_header_fields_size = @@ -57,10 +58,19 @@ CacheEntryHeader::CacheEntryHeader(const core::CacheEntryType entry_type_, { } -uint64_t +uint32_t CacheEntryHeader::payload_size() const { - return entry_size - non_payload_size(); + const auto payload_size = entry_size - non_payload_size(); + // In order to support 32-bit ccache builds, restrict size to uint32_t for + // now. This restriction can be lifted when we drop 32-bit support. + const auto max = std::numeric_limits<uint32_t>::max(); + if (payload_size > max) { + throw core::Error( + FMT("Serialized result too large ({} > {})", payload_size, max)); + } + + return payload_size; } void diff --git a/src/core/CacheEntryHeader.hpp b/src/core/CacheEntryHeader.hpp index dcc32e1c..a0f68918 100644 --- a/src/core/CacheEntryHeader.hpp +++ b/src/core/CacheEntryHeader.hpp @@ -76,7 +76,7 @@ struct CacheEntryHeader std::string namespace_; uint64_t entry_size; - uint64_t payload_size() const; + uint32_t payload_size() const; void set_entry_size_from_payload_size(uint64_t payload_size); void inspect(FILE* stream) const; diff --git a/src/core/Result.cpp b/src/core/Result.cpp new file mode 100644 index 00000000..43526f83 --- /dev/null +++ b/src/core/Result.cpp @@ -0,0 +1,318 @@ +// Copyright (C) 2019-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "Result.hpp" + +#include "Config.hpp" +#include "Context.hpp" +#include "Fd.hpp" +#include "File.hpp" +#include "Logging.hpp" +#include "Stat.hpp" +#include "Util.hpp" + +#include <ccache.hpp> +#include <core/CacheEntryDataReader.hpp> +#include <core/CacheEntryDataWriter.hpp> +#include <core/Statistic.hpp> +#include <core/exceptions.hpp> +#include <core/wincompat.hpp> +#include <fmtmacros.hpp> +#include <util/file.hpp> +#include <util/path.hpp> +#include <util/string.hpp> + +#include <fcntl.h> +#include <sys/stat.h> +#include <sys/types.h> + +#ifdef HAVE_UNISTD_H +# include <unistd.h> +#endif + +#include <algorithm> + +// Result data format +// ================== +// +// Integers are big-endian. +// +// <payload> ::= <format_ver> <n_files> <file_entry>* +// <format_ver> ::= uint8_t +// <n_files> ::= uint8_t +// <file_entry> ::= <embedded_file_entry> | <raw_file_entry> +// <embedded_file_entry> ::= <embedded_file_marker> <file_type> <file_size> +// <file_data> +// <embedded_file_marker> ::= 0 (uint8_t) +// <file_type> ::= uint8_t (see Result::FileType) +// <file_size> ::= uint64_t +// <file_data> ::= file_size bytes +// <raw_file_entry> ::= <raw_file_marker> <file_type> <file_size> +// <raw_file_marker> ::= 1 (uint8_t) +// <file_size> ::= uint64_t + +namespace { + +const uint8_t k_result_format_version = 0; + +// File data stored inside the result file. +const uint8_t k_embedded_file_marker = 0; + +// File stored as-is in the file system. +const uint8_t k_raw_file_marker = 1; + +const uint8_t k_max_raw_file_entries = 10; + +bool +should_store_raw_file(const Config& config, core::Result::FileType type) +{ + if (!config.file_clone() && !config.hard_link()) { + return false; + } + + // Only store object files as raw files since there are several problems with + // storing other file types: + // + // 1. The compiler unlinks object files before writing to them but it doesn't + // unlink .d files, so just it's possible to corrupt .d files just by + // running the compiler (see ccache issue 599). + // 2. .d files cause trouble for automake if hard-linked (see ccache issue + // 378). + // 3. It's unknown how the compiler treats other file types, so better safe + // than sorry. + // + // It would be possible to store all files in raw form for the file_clone case + // and only hard link object files. However, most likely it's only object + // files that become large enough that it's of interest to clone or hard link + // them, so we keep things simple for now. This will also save i-nodes in the + // cache. + return type == core::Result::FileType::object; +} + +} // namespace + +namespace core { + +namespace Result { + +const uint8_t k_version = 1; + +const char* const k_unknown_file_type = "<unknown type>"; + +const char* +file_type_to_string(FileType type) +{ + switch (type) { + case FileType::object: + return ".o"; + + case FileType::dependency: + return ".d"; + + case FileType::stderr_output: + return "<stderr>"; + + case FileType::coverage_unmangled: + return ".gcno-unmangled"; + + case FileType::stackusage: + return ".su"; + + case FileType::diagnostic: + return ".dia"; + + case FileType::dwarf_object: + return ".dwo"; + + case FileType::coverage_mangled: + return ".gcno-mangled"; + + case FileType::stdout_output: + return "<stdout>"; + + case FileType::assembler_listing: + return ".al"; + } + + return k_unknown_file_type; +} + +std::string +gcno_file_in_mangled_form(const Context& ctx) +{ + const auto& output_obj = ctx.args_info.output_obj; + const std::string abs_output_obj = + util::is_absolute_path(output_obj) + ? output_obj + : FMT("{}/{}", ctx.apparent_cwd, output_obj); + std::string hashified_obj = abs_output_obj; + std::replace(hashified_obj.begin(), hashified_obj.end(), '/', '#'); + return Util::change_extension(hashified_obj, ".gcno"); +} + +std::string +gcno_file_in_unmangled_form(const Context& ctx) +{ + return Util::change_extension(ctx.args_info.output_obj, ".gcno"); +} + +Deserializer::Deserializer(nonstd::span<const uint8_t> data) : m_data(data) +{ +} + +void +Deserializer::visit(Deserializer::Visitor& visitor) const +{ + CacheEntryDataReader reader(m_data); + const auto result_format_version = reader.read_int<uint8_t>(); + if (result_format_version != k_result_format_version) { + throw Error(FMT("Unknown result format version: {} != {}", + result_format_version, + k_result_format_version)); + } + + const auto n_files = reader.read_int<uint8_t>(); + if (n_files >= k_max_raw_file_entries) { + throw Error(FMT( + "Too many raw file entries: {} > {}", n_files, k_max_raw_file_entries)); + } + + uint8_t file_number; + for (file_number = 0; file_number < n_files; ++file_number) { + const auto marker = reader.read_int<uint8_t>(); + switch (marker) { + case k_embedded_file_marker: + case k_raw_file_marker: + break; + + default: + throw Error(FMT("Unknown entry type: {}", marker)); + } + + const auto type = reader.read_int<UnderlyingFileTypeInt>(); + const auto file_type = FileType(type); + const auto file_size = reader.read_int<uint64_t>(); + + if (marker == k_embedded_file_marker) { + visitor.on_embedded_file( + file_number, file_type, reader.read_bytes(file_size)); + } else { + ASSERT(marker == k_raw_file_marker); + visitor.on_raw_file(file_number, file_type, file_size); + } + } + + if (file_number != n_files) { + throw Error( + FMT("Too few entries (read {}, expected {})", file_number, n_files)); + } +} + +Serializer::Serializer(const Config& config) + : m_config(config), + m_serialized_size(1 + 1) // format_ver + n_files +{ +} + +void +Serializer::add_data(const FileType file_type, std::string_view data) +{ + m_serialized_size += 1 + 1 + 8; // marker + file_type + file_size + m_serialized_size += data.size(); + m_file_entries.push_back(FileEntry{file_type, util::to_span(data)}); +} + +void +Serializer::add_file(const FileType file_type, const std::string& path) +{ + m_serialized_size += 1 + 1 + 8; // marker + file_type + file_size + if (!should_store_raw_file(m_config, file_type)) { + m_serialized_size += Stat::stat(path, Stat::OnError::throw_error).size(); + } + m_file_entries.push_back(FileEntry{file_type, path}); +} + +uint32_t +Serializer::serialized_size() const +{ + // In order to support 32-bit ccache builds, restrict size to uint32_t for + // now. This restriction can be lifted when we drop 32-bit support. + const auto max = std::numeric_limits<uint32_t>::max(); + if (m_serialized_size > max) { + throw Error( + FMT("Serialized result too large ({} > {})", m_serialized_size, max)); + } + return m_serialized_size; +} + +Serializer::SerializeResult +Serializer::serialize(std::vector<uint8_t>& output) +{ + SerializeResult serialize_result; + CacheEntryDataWriter writer(output); + + writer.write_int(k_result_format_version); + writer.write_int<uint8_t>(m_file_entries.size()); + + uint8_t file_number = 0; + for (const auto& entry : m_file_entries) { + const bool is_file_entry = std::holds_alternative<std::string>(entry.data); + const bool store_raw = + is_file_entry && should_store_raw_file(m_config, entry.file_type); + const uint64_t file_size = + is_file_entry ? Stat::stat(std::get<std::string>(entry.data), + Stat::OnError::throw_error) + .size() + : std::get<nonstd::span<const uint8_t>>(entry.data).size(); + + LOG("Storing {} entry #{} {} ({} bytes){}", + store_raw ? "raw" : "embedded", + file_number, + file_type_to_string(entry.file_type), + file_size, + is_file_entry ? FMT(" from {}", std::get<std::string>(entry.data)) + : ""); + + writer.write_int<uint8_t>(store_raw ? k_raw_file_marker + : k_embedded_file_marker); + writer.write_int(UnderlyingFileTypeInt(entry.file_type)); + writer.write_int(file_size); + + if (store_raw) { + serialize_result.raw_files.emplace(file_number, + std::get<std::string>(entry.data)); + } else if (is_file_entry) { + const auto& path = std::get<std::string>(entry.data); + const auto data = util::read_file<std::vector<uint8_t>>(path); + if (!data) { + throw Error(FMT("Failed to read {}: {}", path, data.error())); + } + writer.write_bytes(*data); + } else { + writer.write_bytes(std::get<nonstd::span<const uint8_t>>(entry.data)); + } + + ++file_number; + } + + return serialize_result; +} + +} // namespace Result + +} // namespace core diff --git a/src/Result.hpp b/src/core/Result.hpp index a67d7106..935fe29a 100644 --- a/src/Result.hpp +++ b/src/core/Result.hpp @@ -18,29 +18,25 @@ #pragma once -#include <core/Reader.hpp> +#include <util/types.hpp> -#include "third_party/nonstd/expected.hpp" +#include <third_party/nonstd/span.hpp> #include <cstdint> -#include <map> -#include <optional> #include <string> +#include <unordered_map> +#include <variant> #include <vector> -namespace core { - -class CacheEntryReader; -class CacheEntryWriter; +class Config; +class Context; -} // namespace core +namespace core { -class Context; +class CacheEntryDataParser; namespace Result { -extern const std::string k_file_suffix; -extern const uint8_t k_magic[4]; extern const uint8_t k_version; extern const char* const k_unknown_file_type; @@ -92,79 +88,71 @@ const char* file_type_to_string(FileType type); std::string gcno_file_in_mangled_form(const Context& ctx); std::string gcno_file_in_unmangled_form(const Context& ctx); -struct FileSizeAndCountDiff -{ - int64_t size_kibibyte; - int64_t count; - - FileSizeAndCountDiff& operator+=(const FileSizeAndCountDiff& other); -}; - -// This class knows how to read a result cache entry. -class Reader +// This class knows how to deserializer a result cache entry. +class Deserializer { public: - Reader(core::CacheEntryReader& cache_entry_reader, - const std::string& result_path); + // Read a result from `data`. + Deserializer(nonstd::span<const uint8_t> data); - class Consumer + class Visitor { public: - virtual ~Consumer() = default; - - virtual void on_entry_start(uint8_t entry_number, - FileType file_type, - uint64_t file_len, - std::optional<std::string> raw_file) = 0; - virtual void on_entry_data(const uint8_t* data, size_t size) = 0; - virtual void on_entry_end() = 0; + virtual ~Visitor() = default; + + virtual void on_embedded_file(uint8_t file_number, + FileType file_type, + nonstd::span<const uint8_t> data) = 0; + virtual void on_raw_file(uint8_t file_number, + FileType file_type, + uint64_t file_size) = 0; }; // Throws core::Error on error. - void read(Consumer& consumer); + void visit(Visitor& visitor) const; private: - core::CacheEntryReader& m_reader; - const std::string m_result_path; + nonstd::span<const uint8_t> m_data; - void read_entry(uint8_t entry_number, Reader::Consumer& consumer); + void parse_file_entry(CacheEntryDataParser& parser, + uint8_t file_number) const; }; -// This class knows how to write a result cache entry. -class Writer +// This class knows how to serialize a result cache entry. +class Serializer { public: - Writer(const Config& config, const std::string& result_path); + Serializer(const Config& config); - // Register content to include in the result. Does not throw. - void write_data(FileType file_type, const std::string& data); + // Register data to include in the result. The data must live until + // serialize() has been called. + void add_data(FileType file_type, std::string_view data); - // Register a file path whose content should be included in the result. Does - // not throw. - void write_file(FileType file_type, const std::string& path); + // Register a file path whose content should be included in the result. + void add_file(FileType file_type, const std::string& path); - // Write registered entries to the result. Returns an error message on error. - nonstd::expected<FileSizeAndCountDiff, std::string> finalize(); + uint32_t serialized_size() const; -private: - enum class ValueType { data, path }; - struct Entry + struct SerializeResult { - FileType file_type; - ValueType value_type; - std::string value; + // Raw files to store in primary storage. + std::unordered_map<uint8_t /*index*/, std::string /*path*/> raw_files; }; + SerializeResult serialize(std::vector<uint8_t>& output); + +private: const Config& m_config; - const std::string m_result_path; - std::vector<Entry> m_entries_to_write; - - FileSizeAndCountDiff do_finalize(); - static void write_embedded_file_entry(core::CacheEntryWriter& writer, - const std::string& path, - uint64_t file_size); - FileSizeAndCountDiff write_raw_file_entry(const std::string& path, - uint8_t entry_number); + uint64_t m_serialized_size; + + struct FileEntry + { + FileType file_type; + std::variant<nonstd::span<const uint8_t>, std::string> data; + }; + std::vector<FileEntry> m_file_entries; }; } // namespace Result + +} // namespace core diff --git a/src/core/ResultExtractor.cpp b/src/core/ResultExtractor.cpp new file mode 100644 index 00000000..d88146f0 --- /dev/null +++ b/src/core/ResultExtractor.cpp @@ -0,0 +1,91 @@ +// Copyright (C) 2020-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "ResultExtractor.hpp" + +#include "Util.hpp" +#include "fmtmacros.hpp" + +#include <core/exceptions.hpp> +#include <core/wincompat.hpp> +#include <fmtmacros.hpp> +#include <util/file.hpp> + +#include <fcntl.h> +#include <sys/stat.h> +#include <sys/types.h> + +#include <vector> + +namespace core { + +ResultExtractor::ResultExtractor( + const std::string& output_directory, + std::optional<GetRawFilePathFunction> get_raw_file_path) + : m_output_directory(output_directory), + m_get_raw_file_path(get_raw_file_path) +{ +} + +void +ResultExtractor::on_embedded_file(uint8_t /*file_number*/, + Result::FileType file_type, + nonstd::span<const uint8_t> data) +{ + std::string suffix = Result::file_type_to_string(file_type); + if (suffix == Result::k_unknown_file_type) { + suffix = + FMT(".type_{}", static_cast<Result::UnderlyingFileTypeInt>(file_type)); + } else if (suffix[0] == '<') { + suffix[0] = '.'; + suffix.resize(suffix.length() - 1); + } + + const auto dest_path = FMT("{}/ccache-result{}", m_output_directory, suffix); + const auto result = util::write_file(dest_path, data); + if (!result) { + throw Error(FMT("Failed to write to {}: {}", dest_path, result.error())); + } +} + +void +ResultExtractor::on_raw_file(uint8_t file_number, + Result::FileType file_type, + uint64_t file_size) +{ + if (!m_get_raw_file_path) { + throw Error("Raw entry for non-local result"); + } + const auto raw_file_path = (*m_get_raw_file_path)(file_number); + const auto st = Stat::stat(raw_file_path, Stat::OnError::throw_error); + if (st.size() != file_size) { + throw Error(FMT("Bad file size of {} (actual {} bytes, expected {} bytes)", + raw_file_path, + st.size(), + file_size)); + } + + const auto data = + util::read_file<std::vector<uint8_t>>(raw_file_path, file_size); + if (!data) { + throw Error(FMT("Failed to read {}: {}", raw_file_path, data.error())); + } + on_embedded_file(file_number, file_type, *data); +} + +} // namespace core diff --git a/src/ResultExtractor.hpp b/src/core/ResultExtractor.hpp index b6bbc1e1..204835bb 100644 --- a/src/ResultExtractor.hpp +++ b/src/core/ResultExtractor.hpp @@ -19,25 +19,37 @@ #pragma once #include "Fd.hpp" -#include "Result.hpp" -class Context; +#include <core/Result.hpp> + +#include <functional> +#include <optional> +#include <string> + +namespace core { // This class extracts the parts of a result entry to a directory. -class ResultExtractor : public Result::Reader::Consumer +class ResultExtractor : public Result::Deserializer::Visitor { public: - ResultExtractor(const std::string& directory); + using GetRawFilePathFunction = std::function<std::string(uint8_t)>; - void on_entry_start(uint8_t entry_number, - Result::FileType file_type, - uint64_t file_len, - std::optional<std::string> raw_file) override; - void on_entry_data(const uint8_t* data, size_t size) override; - void on_entry_end() override; + //`result_path` should be the path to the local result entry file if the + // result comes from primary storage. + ResultExtractor( + const std::string& output_directory, + std::optional<GetRawFilePathFunction> get_raw_file_path = std::nullopt); + + void on_embedded_file(uint8_t file_number, + Result::FileType file_type, + nonstd::span<const uint8_t> data) override; + void on_raw_file(uint8_t file_number, + Result::FileType file_type, + uint64_t file_size) override; private: - const std::string m_directory; - Fd m_dest_fd; - std::string m_dest_path; + std::string m_output_directory; + std::optional<GetRawFilePathFunction> m_get_raw_file_path; }; + +} // namespace core diff --git a/src/ResultInspector.cpp b/src/core/ResultInspector.cpp index 0c4753ce..3ef88c6f 100644 --- a/src/ResultInspector.cpp +++ b/src/core/ResultInspector.cpp @@ -22,30 +22,34 @@ #include "Logging.hpp" #include "fmtmacros.hpp" +namespace core { + ResultInspector::ResultInspector(FILE* stream) : m_stream(stream) { } void -ResultInspector::on_entry_start(uint8_t entry_number, - Result::FileType file_type, - uint64_t file_len, - std::optional<std::string> raw_file) +ResultInspector::on_embedded_file(uint8_t file_number, + Result::FileType file_type, + nonstd::span<const uint8_t> data) { PRINT(m_stream, - "{} file #{}: {} ({} bytes)\n", - raw_file ? "Raw" : "Embedded", - entry_number, + "Embedded file #{}: {} ({} bytes)\n", + file_number, Result::file_type_to_string(file_type), - file_len); + data.size()); } void -ResultInspector::on_entry_data(const uint8_t* /*data*/, size_t /*size*/) +ResultInspector::on_raw_file(uint8_t file_number, + Result::FileType file_type, + uint64_t file_size) { + PRINT(m_stream, + "Raw file #{}: {} ({} bytes)\n", + file_number, + Result::file_type_to_string(file_type), + file_size); } -void -ResultInspector::on_entry_end() -{ -} +} // namespace core diff --git a/src/ResultInspector.hpp b/src/core/ResultInspector.hpp index d2df8ae2..94a9b8c6 100644 --- a/src/ResultInspector.hpp +++ b/src/core/ResultInspector.hpp @@ -18,24 +18,28 @@ #pragma once -#include "Result.hpp" +#include <core/Result.hpp> #include <cstdint> #include <cstdio> +namespace core { + // This class writes information about the result entry to `stream`. -class ResultInspector : public Result::Reader::Consumer +class ResultInspector : public Result::Deserializer::Visitor { public: ResultInspector(FILE* stream); - void on_entry_start(uint8_t entry_number, - Result::FileType file_type, - uint64_t file_len, - std::optional<std::string> raw_file) override; - void on_entry_data(const uint8_t* data, size_t size) override; - void on_entry_end() override; + void on_embedded_file(uint8_t file_number, + Result::FileType file_type, + nonstd::span<const uint8_t> data) override; + void on_raw_file(uint8_t file_number, + Result::FileType file_type, + uint64_t file_size) override; private: FILE* m_stream; }; + +} // namespace core diff --git a/src/core/ResultRetriever.cpp b/src/core/ResultRetriever.cpp new file mode 100644 index 00000000..6bea0adb --- /dev/null +++ b/src/core/ResultRetriever.cpp @@ -0,0 +1,216 @@ +// Copyright (C) 2020-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "ResultRetriever.hpp" + +#include "Context.hpp" +#include "Depfile.hpp" +#include "Logging.hpp" + +#include <Context.hpp> +#include <Stat.hpp> +#include <core/exceptions.hpp> +#include <core/wincompat.hpp> +#include <fmtmacros.hpp> +#include <util/file.hpp> +#include <util/string.hpp> + +#include <fcntl.h> +#include <sys/stat.h> +#include <sys/types.h> + +#ifdef HAVE_UNISTD_H +# include <unistd.h> +#endif + +namespace core { + +using Result::FileType; + +ResultRetriever::ResultRetriever(const Context& ctx, + std::optional<std::string> path) + : m_ctx(ctx), + m_path(path) +{ +} + +void +ResultRetriever::on_embedded_file(uint8_t file_number, + FileType file_type, + nonstd::span<const uint8_t> data) +{ + LOG("Reading embedded entry #{} {} ({} bytes)", + file_number, + Result::file_type_to_string(file_type), + data.size()); + + if (file_type == FileType::stdout_output) { + Util::send_to_fd(m_ctx, util::to_string_view(data), STDOUT_FILENO); + } else if (file_type == FileType::stderr_output) { + Util::send_to_fd(m_ctx, util::to_string_view(data), STDERR_FILENO); + } else { + const auto dest_path = get_dest_path(file_type); + if (dest_path.empty()) { + LOG_RAW("Not writing"); + } else if (dest_path == "/dev/null") { + LOG_RAW("Not writing to /dev/null"); + } else { + LOG("Writing to {}", dest_path); + if (file_type == FileType::dependency) { + write_dependency_file(dest_path, data); + } else { + const auto result = util::write_file(dest_path, data); + if (!result) { + throw WriteError( + FMT("Failed to write to {}: {}", dest_path, result.error())); + } + } + } + } +} + +void +ResultRetriever::on_raw_file(uint8_t file_number, + FileType file_type, + uint64_t file_size) +{ + LOG("Reading raw entry #{} {} ({} bytes)", + file_number, + Result::file_type_to_string(file_type), + file_size); + + if (!m_path) { + throw core::Error("Raw entry for non-local result"); + } + const auto raw_file_path = + storage::primary::PrimaryStorage::get_raw_file_path(*m_path, file_number); + const auto st = Stat::stat(raw_file_path, Stat::OnError::throw_error); + if (st.size() != file_size) { + throw core::Error( + FMT("Bad file size of {} (actual {} bytes, expected {} bytes)", + raw_file_path, + st.size(), + file_size)); + } + + const auto dest_path = get_dest_path(file_type); + if (!dest_path.empty()) { + Util::clone_hard_link_or_copy_file( + m_ctx.config, raw_file_path, dest_path, false); + + // Update modification timestamp to save the file from LRU cleanup (and, if + // hard-linked, to make the object file newer than the source file). + util::set_timestamps(raw_file_path); + } else { + // Should never happen. + LOG("Did not copy {} since destination path is unknown for type {}", + raw_file_path, + static_cast<Result::UnderlyingFileTypeInt>(file_type)); + } +} + +std::string +ResultRetriever::get_dest_path(FileType file_type) const +{ + switch (file_type) { + case FileType::object: + return m_ctx.args_info.output_obj; + + case FileType::dependency: + if (m_ctx.args_info.generating_dependencies) { + return m_ctx.args_info.output_dep; + } + break; + + case FileType::stdout_output: + case FileType::stderr_output: + // Should never get here. + break; + + case FileType::coverage_unmangled: + if (m_ctx.args_info.generating_coverage) { + return Util::change_extension(m_ctx.args_info.output_obj, ".gcno"); + } + break; + + case FileType::stackusage: + if (m_ctx.args_info.generating_stackusage) { + return m_ctx.args_info.output_su; + } + break; + + case FileType::diagnostic: + if (m_ctx.args_info.generating_diagnostics) { + return m_ctx.args_info.output_dia; + } + break; + + case FileType::dwarf_object: + if (m_ctx.args_info.seen_split_dwarf + && m_ctx.args_info.output_obj != "/dev/null") { + return m_ctx.args_info.output_dwo; + } + break; + + case FileType::coverage_mangled: + if (m_ctx.args_info.generating_coverage) { + return Result::gcno_file_in_mangled_form(m_ctx); + } + break; + + case FileType::assembler_listing: + return m_ctx.args_info.output_al; + } + + return {}; +} + +void +ResultRetriever::write_dependency_file(const std::string& path, + nonstd::span<const uint8_t> data) +{ + ASSERT(m_ctx.args_info.dependency_target); + + Fd fd(open(path.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666)); + if (!fd) { + throw WriteError(FMT("Failed to open {} for writing", path)); + } + + auto write_data = [&](auto data, auto size) { + const auto result = util::write_fd(*fd, data, size); + if (!result) { + throw WriteError(FMT("Failed to write to {}: {}", path, result.error())); + } + }; + + std::string_view str_data = util::to_string_view(data); + size_t start_pos = 0; + const size_t colon_pos = str_data.find(": "); + if (colon_pos != std::string::npos) { + const auto obj_in_dep_file = str_data.substr(0, colon_pos); + const auto& dep_target = *m_ctx.args_info.dependency_target; + if (obj_in_dep_file != dep_target) { + write_data(dep_target.data(), dep_target.length()); + start_pos = colon_pos; + } + } + + write_data(str_data.data() + start_pos, str_data.length() - start_pos); +} + +} // namespace core diff --git a/src/ResultRetriever.hpp b/src/core/ResultRetriever.hpp index a114698e..d2e814ba 100644 --- a/src/ResultRetriever.hpp +++ b/src/core/ResultRetriever.hpp @@ -19,41 +19,45 @@ #pragma once #include "Fd.hpp" -#include "Result.hpp" +#include <core/Result.hpp> #include <core/exceptions.hpp> +#include <optional> + class Context; +namespace core { + // This class retrieves a result entry to the local file system. -class ResultRetriever : public Result::Reader::Consumer +class ResultRetriever : public Result::Deserializer::Visitor { public: - class WriteError : public core::Error + class WriteError : public Error { - using core::Error::Error; + using Error::Error; }; - ResultRetriever(Context& ctx); + //`path` should be the path to the local result entry file if the result comes + // from primary storage. + ResultRetriever(const Context& ctx, + std::optional<std::string> path = std::nullopt); - void on_entry_start(uint8_t entry_number, - Result::FileType file_type, - uint64_t file_len, - std::optional<std::string> raw_file) override; - void on_entry_data(const uint8_t* data, size_t size) override; - void on_entry_end() override; + void on_embedded_file(uint8_t file_number, + Result::FileType file_type, + nonstd::span<const uint8_t> data) override; + void on_raw_file(uint8_t file_number, + Result::FileType file_type, + uint64_t file_size) override; private: - Context& m_ctx; - Result::FileType m_dest_file_type{}; - Fd m_dest_fd; - std::string m_dest_path; - - // Collects the full data of stderr output (since we want to potentially strip - // color codes which could span chunk boundaries) or dependency data (since we - // potentially want to rewrite the dependency target which in theory can span - // a chunk boundary). - std::string m_dest_data; - - void write_dependency_file(); + const Context& m_ctx; + std::optional<std::string> m_path; + + std::string get_dest_path(Result::FileType file_type) const; + + void write_dependency_file(const std::string& path, + nonstd::span<const uint8_t> data); }; + +} // namespace core diff --git a/src/core/mainoptions.cpp b/src/core/mainoptions.cpp index 87c318c4..baa842b2 100644 --- a/src/core/mainoptions.cpp +++ b/src/core/mainoptions.cpp @@ -24,13 +24,13 @@ #include <Hash.hpp> #include <InodeCache.hpp> #include <ProgressBar.hpp> -#include <Result.hpp> -#include <ResultExtractor.hpp> -#include <ResultInspector.hpp> #include <ccache.hpp> #include <core/CacheEntryReader.hpp> #include <core/FileReader.hpp> #include <core/Manifest.hpp> +#include <core/Result.hpp> +#include <core/ResultExtractor.hpp> +#include <core/ResultInspector.hpp> #include <core/Statistics.hpp> #include <core/StatsLog.hpp> #include <core/exceptions.hpp> @@ -175,17 +175,21 @@ inspect_path(const std::string& path) case core::CacheEntryType::manifest: { core::Manifest manifest; manifest.read(cache_entry_reader); - cache_entry_reader.finalize(); manifest.dump(stdout); break; } case core::CacheEntryType::result: - Result::Reader result_reader(cache_entry_reader, path); + std::vector<uint8_t> data; + data.resize(cache_entry_reader.header().payload_size()); + cache_entry_reader.read(data.data(), data.size()); + Result::Deserializer result_deserializer(data); ResultInspector result_inspector(stdout); - result_reader.read(result_inspector); + result_deserializer.visit(result_inspector); break; } + cache_entry_reader.finalize(); + return EXIT_SUCCESS; } @@ -441,16 +445,27 @@ process_main_options(int argc, const char* const* argv) } case EXTRACT_RESULT: { - ResultExtractor result_extractor("."); File file = arg == "-" ? File(stdin) : File(arg, "rb"); if (!file) { PRINT(stderr, "Error: Failed to open \"{}\"", arg); return EXIT_FAILURE; } + std::optional<ResultExtractor::GetRawFilePathFunction> get_raw_file_path; + if (arg == "-") { + get_raw_file_path = [&](uint8_t file_number) { + return storage::primary::PrimaryStorage::get_raw_file_path( + arg, file_number); + }; + } + ResultExtractor result_extractor(".", get_raw_file_path); core::FileReader file_reader(file.get()); core::CacheEntryReader cache_entry_reader(file_reader); - Result::Reader result_reader(cache_entry_reader, arg); - result_reader.read(result_extractor); + std::vector<uint8_t> data; + data.resize(cache_entry_reader.header().payload_size()); + cache_entry_reader.read(data.data(), data.size()); + Result::Deserializer result_deserializer(data); + result_deserializer.visit(result_extractor); + cache_entry_reader.finalize(); return EXIT_SUCCESS; } diff --git a/src/storage/primary/CacheFile.cpp b/src/storage/primary/CacheFile.cpp index 1a65e583..a2716e8e 100644 --- a/src/storage/primary/CacheFile.cpp +++ b/src/storage/primary/CacheFile.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2021 Joel Rosdahl and other contributors +// Copyright (C) 2019-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,8 +18,8 @@ #include "CacheFile.hpp" -#include <Result.hpp> #include <core/Manifest.hpp> +#include <core/Result.hpp> #include <util/string.hpp> const Stat& @@ -37,7 +37,7 @@ CacheFile::type() const { if (util::ends_with(m_path, "M")) { return Type::manifest; - } else if (util::ends_with(m_path, Result::k_file_suffix)) { + } else if (util::ends_with(m_path, "R")) { return Type::result; } else if (util::ends_with(m_path, "W")) { return Type::raw; diff --git a/src/storage/primary/PrimaryStorage.cpp b/src/storage/primary/PrimaryStorage.cpp index b5cf31cb..c8e19167 100644 --- a/src/storage/primary/PrimaryStorage.cpp +++ b/src/storage/primary/PrimaryStorage.cpp @@ -259,6 +259,21 @@ PrimaryStorage::remove(const Digest& key, const core::CacheEntryType type) } } +std::string +PrimaryStorage::get_raw_file_path(std::string_view result_path, + uint8_t file_number) +{ + if (file_number >= 10) { + // To support more entries in the future, encode to [0-9a-z]. Note that + // PrimaryStorage::evict currently assumes that the entry number is + // represented as one character. + throw core::Error(FMT("Too high raw file entry number: {}", file_number)); + } + + const auto prefix = result_path.substr(0, result_path.length() - 1); + return FMT("{}{}W", prefix, file_number); +} + void PrimaryStorage::increment_statistic(const Statistic statistic, const int64_t value) diff --git a/src/storage/primary/PrimaryStorage.hpp b/src/storage/primary/PrimaryStorage.hpp index d9f558c5..7a33198a 100644 --- a/src/storage/primary/PrimaryStorage.hpp +++ b/src/storage/primary/PrimaryStorage.hpp @@ -59,6 +59,9 @@ public: void remove(const Digest& key, core::CacheEntryType type); + static std::string get_raw_file_path(std::string_view result_path, + uint8_t file_number); + // --- Statistics --- void increment_statistic(core::Statistic statistic, int64_t value = 1); diff --git a/src/storage/primary/PrimaryStorage_compress.cpp b/src/storage/primary/PrimaryStorage_compress.cpp index 1f54c487..333fc1f4 100644 --- a/src/storage/primary/PrimaryStorage_compress.cpp +++ b/src/storage/primary/PrimaryStorage_compress.cpp @@ -22,7 +22,6 @@ #include <Context.hpp> #include <File.hpp> #include <Logging.hpp> -#include <Result.hpp> #include <ThreadPool.hpp> #include <assertions.hpp> #include <compression/ZstdCompressor.hpp> @@ -31,6 +30,7 @@ #include <core/FileReader.hpp> #include <core/FileWriter.hpp> #include <core/Manifest.hpp> +#include <core/Result.hpp> #include <core/exceptions.hpp> #include <core/wincompat.hpp> #include <fmtmacros.hpp> diff --git a/test/suites/no_compression.bash b/test/suites/no_compression.bash index 820a13d1..ec7d7608 100644 --- a/test/suites/no_compression.bash +++ b/test/suites/no_compression.bash @@ -66,7 +66,8 @@ SUITE_no_compression() { expect_stat files_in_cache 2 result_file=$(find $CCACHE_DIR -name '*R') - printf foo | dd of=$result_file bs=3 count=1 seek=20 conv=notrunc >&/dev/null + # Write BAD at byte 300. + printf BAD | dd of=$result_file bs=3 count=1 seek=100 conv=notrunc >&/dev/null $CCACHE_COMPILE -c test.c expect_stat direct_cache_hit 1 |