diff options
46 files changed, 2030 insertions, 2109 deletions
diff --git a/Makefile.in b/Makefile.in index 117a0483..360a894a 100644 --- a/Makefile.in +++ b/Makefile.in @@ -32,22 +32,25 @@ Q=$(if $(quiet),@) non_third_party_sources = \ src/AtomicFile.cpp \ + src/CacheEntryReader.cpp \ + src/CacheEntryWriter.cpp \ src/CacheFile.cpp \ + src/Compression.cpp \ + src/Compressor.cpp \ src/Config.cpp \ + src/Decompressor.cpp \ + src/NullCompressor.cpp \ + src/NullDecompressor.cpp \ src/ProgressBar.cpp \ src/Util.cpp \ + src/ZstdCompressor.cpp \ + src/ZstdDecompressor.cpp \ src/args.cpp \ src/ccache.cpp \ src/cleanup.cpp \ - src/common_header.cpp \ src/compopt.cpp \ - src/compr_none.cpp \ - src/compr_zstd.cpp \ src/compress.cpp \ - src/compression.cpp \ src/counters.cpp \ - src/decompr_none.cpp \ - src/decompr_zstd.cpp \ src/execute.cpp \ src/exitfn.cpp \ src/hash.cpp \ @@ -76,13 +79,14 @@ ccache_objs = $(patsubst %.c, %.o, $(patsubst %.cpp, %.o, $(ccache_sources))) test_suites += unittest/test_AtomicFile.cpp test_suites += unittest/test_Checksum.cpp +test_suites += unittest/test_Compression.cpp test_suites += unittest/test_Config.cpp +test_suites += unittest/test_NullCompression.cpp test_suites += unittest/test_Util.cpp +test_suites += unittest/test_ZstdCompression.cpp test_suites += unittest/test_args.cpp test_suites += unittest/test_argument_processing.cpp test_suites += unittest/test_compopt.cpp -test_suites += unittest/test_compr_none.cpp -test_suites += unittest/test_compr_zstd.cpp test_suites += unittest/test_counters.cpp test_suites += unittest/test_hash.cpp test_suites += unittest/test_hashutil.cpp @@ -36,22 +36,30 @@ built_dist_files = $(generated_sources) $(generated_docs) non_third_party_headers = \ src/AtomicFile.hpp \ + src/CacheEntryReader.hpp \ + src/CacheEntryWriter.hpp \ src/CacheFile.hpp \ src/Checksum.hpp \ + src/Compression.hpp \ + src/Compressor.hpp \ + src/Decompressor.hpp \ src/Config.hpp \ src/Error.hpp \ + src/File.hpp \ + src/NonCopyable.hpp \ + src/NullCompressor.hpp \ + src/NullDecompressor.hpp \ src/ProgressBar.hpp \ src/Util.hpp \ + src/ZstdCompressor.hpp \ + src/ZstdDecompressor.hpp \ src/ccache.hpp \ src/cleanup.hpp \ - src/common_header.hpp \ src/compopt.hpp \ src/compress.hpp \ - src/compression.hpp \ src/counters.hpp \ src/hash.hpp \ src/hashutil.hpp \ - src/int_bytes_conversion.hpp \ src/language.hpp \ src/macroskip.hpp \ src/manifest.hpp \ diff --git a/doc/MANUAL.adoc b/doc/MANUAL.adoc index 9d27efb7..5304dd15 100644 --- a/doc/MANUAL.adoc +++ b/doc/MANUAL.adoc @@ -381,8 +381,7 @@ Compression will be disabled if file cloning (the files using the real-time compression algorithm Zstandard. The setting only has effect if <<config_compression,*compression*>> is enabled (which it is by default). Zstandard is extremely fast for decompression and very fast - for compression for lower - compression levels. The default is 0. + for compression for lower compression levels. The default is 0. + Semantics of *compression_level*: + diff --git a/src/CacheEntryReader.cpp b/src/CacheEntryReader.cpp new file mode 100644 index 00000000..82dae950 --- /dev/null +++ b/src/CacheEntryReader.cpp @@ -0,0 +1,102 @@ +// Copyright (C) 2019 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "CacheEntryReader.hpp" + +#include "Checksum.hpp" +#include "Compressor.hpp" +#include "Error.hpp" +#include "ccache.hpp" + +#include <fmt/core.h> + +CacheEntryReader::CacheEntryReader(FILE* stream, + const uint8_t expected_magic[4], + uint8_t expected_version, + Checksum* checksum) + : m_checksum(checksum) +{ + uint8_t header_bytes[15]; + if (fread(header_bytes, sizeof(header_bytes), 1, stream) != 1) { + throw Error("Error reading header"); + } + + memcpy(m_magic, header_bytes, sizeof(m_magic)); + m_version = header_bytes[4]; + m_compression_type = Compression::type_from_int(header_bytes[5]); + m_compression_level = header_bytes[6]; + Util::big_endian_to_int(header_bytes + 7, m_content_size); + + if (memcmp(m_magic, expected_magic, sizeof(m_magic)) != 0) { + throw Error(fmt::format("Bad magic value 0x{:02x}{:02x}{:02x}{:02x}", + m_magic[0], + m_magic[1], + m_magic[2], + m_magic[3])); + } + if (m_version != expected_version) { + throw Error(fmt::format( + "Unknown version (actual {}, expected {})", m_version, expected_version)); + } + if (m_compression_type == Compression::Type::none) { + // Since we have the size available, let's use it as a super primitive + // consistency check for the non-compressed case. (A real checksum is used + // for compressed data.) + struct stat st; + if (x_fstat(fileno(stream), &st) != 0) { + throw Error(fmt::format("Failed to fstat: {}", strerror(errno))); + } + if (static_cast<uint64_t>(st.st_size) != m_content_size) { + throw Error(fmt::format( + "Bad uncompressed file size (actual {} bytes, expected {} bytes)", + st.st_size, + m_content_size)); + } + } + + if (m_checksum) { + m_checksum->update(header_bytes, sizeof(header_bytes)); + } + + m_decompressor = Decompressor::create_from_type(m_compression_type, stream); +} + +void +CacheEntryReader::dump_header(FILE* dump_stream) +{ + fmt::print(dump_stream, "Magic: {:.4}\n", m_magic); + fmt::print(dump_stream, "Version: {}\n", m_version); + fmt::print(dump_stream, + "Compression type: {}\n", + Compression::type_to_string(m_compression_type)); + fmt::print(dump_stream, "Compression level: {}\n", m_compression_level); + fmt::print(dump_stream, "Content size: {}\n", m_content_size); +} + +void +CacheEntryReader::read(void* data, size_t count) +{ + m_decompressor->read(data, count); + m_checksum->update(data, count); +} + +void +CacheEntryReader::finalize() +{ + m_decompressor->finalize(); +} diff --git a/src/CacheEntryReader.hpp b/src/CacheEntryReader.hpp new file mode 100644 index 00000000..17f4682e --- /dev/null +++ b/src/CacheEntryReader.hpp @@ -0,0 +1,103 @@ +// Copyright (C) 2019 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include "Decompressor.hpp" +#include "Util.hpp" + +#include <cstdio> +#include <memory> + +class Checksum; + +// This class knows how to read a cache entry with a common header and a +// payload part that is different depending on the cache entry type (result or +// manifest). +class CacheEntryReader +{ +public: + // Constructor. + // + // Parameters: + // - stream: Stream to read header and payload from. + // - expected_magic: Expected magic bytes (first four bytes of the file). + // - expected_version: Expected file format version. + // - checksum: Checksum state that will be updated with the read bytes, or + // nullptr for no checksumming. + CacheEntryReader(FILE* stream, + const uint8_t expected_magic[4], + uint8_t expected_version, + Checksum* checksum = nullptr); + + // Dump header information in text format. + // + // Parameters: + // - dump_stream: Stream to write to. + void dump_header(FILE* dump_stream); + + // Read data into a buffer from the payload. + // + // Parameters: + // - data: Buffer to write data to. + // - count: How many bytes to write. + // + // Throws Error on failure. + void read(void* data, size_t count); + + // Read an unsigned integer from the payload. + // + // Parameters: + // - value: Variable to write to. + // + // Throws Error on failure. + template<typename T> void read(T& value); + + // Close for reading. + // + // This method potentially verifies the end state after reading the cache + // entry and throws Error if any integrity issues are found. + void finalize(); + + // Get size of the content (header + payload). + uint64_t content_size() const; + +private: + std::unique_ptr<Decompressor> m_decompressor; + Checksum* m_checksum; + char m_magic[4]; + uint8_t m_version; + Compression::Type m_compression_type; + int8_t m_compression_level; + uint64_t m_content_size; +}; + +template<typename T> +inline void +CacheEntryReader::read(T& value) +{ + uint8_t buffer[sizeof(T)]; + read(buffer, sizeof(T)); + Util::big_endian_to_int(buffer, value); +} + +inline uint64_t +CacheEntryReader::content_size() const +{ + return m_content_size; +} diff --git a/src/CacheEntryWriter.cpp b/src/CacheEntryWriter.cpp new file mode 100644 index 00000000..15696390 --- /dev/null +++ b/src/CacheEntryWriter.cpp @@ -0,0 +1,57 @@ +// Copyright (C) 2019 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "CacheEntryWriter.hpp" + +#include "Checksum.hpp" + +CacheEntryWriter::CacheEntryWriter(FILE* stream, + const uint8_t magic[4], + uint8_t version, + Compression::Type compression_type, + int8_t compression_level, + uint64_t content_size, + Checksum& checksum) + : m_compressor( + Compressor::create_from_type(compression_type, stream, compression_level)), + m_checksum(checksum) +{ + uint8_t header_bytes[15]; + memcpy(header_bytes, magic, 4); + header_bytes[4] = version; + header_bytes[5] = static_cast<uint8_t>(compression_type); + header_bytes[6] = m_compressor->actual_compression_level(); + Util::int_to_big_endian(content_size, header_bytes + 7); + if (fwrite(header_bytes, sizeof(header_bytes), 1, stream) != 1) { + throw Error("Failed to write cache entry header"); + } + checksum.update(header_bytes, sizeof(header_bytes)); +} + +void +CacheEntryWriter::write(const void* data, size_t count) +{ + m_compressor->write(data, count); + m_checksum.update(data, count); +} + +void +CacheEntryWriter::finalize() +{ + m_compressor->finalize(); +} diff --git a/src/CacheEntryWriter.hpp b/src/CacheEntryWriter.hpp new file mode 100644 index 00000000..aa02a01b --- /dev/null +++ b/src/CacheEntryWriter.hpp @@ -0,0 +1,88 @@ +// Copyright (C) 2019 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include "Compressor.hpp" +#include "Util.hpp" + +#include <cstdio> +#include <memory> + +class Checksum; + +// This class knows how to write a cache entry with a common header and a +// payload part that is different depending on the cache entry type (result or +// manifest). +class CacheEntryWriter +{ +public: + // Constructor. + // + // Parameters: + // - stream: Stream to write header + payload to. + // - magic: File format magic bytes. + // - version: File format version. + // - compression_type: Compression type to use. + // - compression_level: Compression level to use. + // - content_size: Content size. + // - checksum: Checksum state that will be updated with the written bytes. + CacheEntryWriter(FILE* stream, + const uint8_t magic[4], + uint8_t version, + Compression::Type compression_type, + int8_t compression_level, + uint64_t content_size, + Checksum& checksum); + + // Write data to the payload from a buffer. + // + // Parameters: + // - data: Data to write. + // - count: Size of data to write. + // + // Throws Error on failure. + void write(const void* data, size_t count); + + // Write an unsigned integer to the payload. + // + // Parameters: + // - value: Value to write. + // + // Throws Error on failure. + template<typename T> void write(T value); + + // Close for writing. + // + // This method potentially verifies the end state after writing the cache + // entry and throws Error if any integrity issues are found. + void finalize(); + +private: + std::unique_ptr<Compressor> m_compressor; + Checksum& m_checksum; +}; + +template<typename T> +inline void +CacheEntryWriter::write(T value) +{ + uint8_t buffer[sizeof(T)]; + Util::int_to_big_endian(value, buffer); + write(buffer, sizeof(T)); +} diff --git a/src/compression.cpp b/src/Compression.cpp index 1fb682a2..08d50e53 100644 --- a/src/compression.cpp +++ b/src/Compression.cpp @@ -16,60 +16,52 @@ // this program; if not, write to the Free Software Foundation, Inc., 51 // Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -#include "compression.hpp" +#include "Compression.hpp" #include "Config.hpp" +#include "Error.hpp" + +namespace Compression { int8_t -compression_level_from_config(void) +level_from_config() { return g_config.compression() ? g_config.compression_level() : 0; } -enum compression_type -compression_type_from_config(void) +Type +type_from_config() { - return g_config.compression() ? COMPR_TYPE_ZSTD : COMPR_TYPE_NONE; + return g_config.compression() ? Type::zstd : Type::none; } -const char* -compression_type_to_string(uint8_t type) +Type +type_from_int(uint8_t type) { switch (type) { - case COMPR_TYPE_NONE: - return "none"; + case static_cast<uint8_t>(Type::none): + return Type::none; - case COMPR_TYPE_ZSTD: - return "zstd"; + case static_cast<uint8_t>(Type::zstd): + return Type::zstd; } - return "unknown"; + throw Error(fmt::format("Unknown type: {}", type)); } -struct compressor* -compressor_from_type(uint8_t type) +std::string +type_to_string(Type type) { switch (type) { - case COMPR_TYPE_NONE: - return &compressor_none_impl; + case Type::none: + return "none"; - case COMPR_TYPE_ZSTD: - return &compressor_zstd_impl; + case Type::zstd: + return "zstd"; } - return NULL; + assert(false); + return {}; } -struct decompressor* -decompressor_from_type(uint8_t type) -{ - switch (type) { - case COMPR_TYPE_NONE: - return &decompressor_none_impl; - - case COMPR_TYPE_ZSTD: - return &decompressor_zstd_impl; - } - - return NULL; -} +} // namespace Compression diff --git a/src/Compression.hpp b/src/Compression.hpp new file mode 100644 index 00000000..ac973d99 --- /dev/null +++ b/src/Compression.hpp @@ -0,0 +1,40 @@ +// Copyright (C) 2019 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include "system.hpp" + +#include <string> + +namespace Compression { + +enum class Type : uint8_t { + none = 0, + zstd = 1, +}; + +int8_t level_from_config(); + +Type type_from_config(); + +Type type_from_int(uint8_t type); + +std::string type_to_string(Compression::Type type); + +} // namespace Compression diff --git a/src/Compressor.cpp b/src/Compressor.cpp new file mode 100644 index 00000000..8ccc418c --- /dev/null +++ b/src/Compressor.cpp @@ -0,0 +1,40 @@ +// Copyright (C) 2019 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "Compressor.hpp" + +#include "NullCompressor.hpp" +#include "StdMakeUnique.hpp" +#include "ZstdCompressor.hpp" + +std::unique_ptr<Compressor> +Compressor::create_from_type(Compression::Type type, + FILE* stream, + int8_t compression_level) +{ + switch (type) { + case Compression::Type::none: + return std::make_unique<NullCompressor>(stream); + + case Compression::Type::zstd: + return std::make_unique<ZstdCompressor>(stream, compression_level); + } + + assert(false); + return {}; +} diff --git a/src/Compressor.hpp b/src/Compressor.hpp new file mode 100644 index 00000000..2cc1fc0e --- /dev/null +++ b/src/Compressor.hpp @@ -0,0 +1,65 @@ +// Copyright (C) 2019 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include "Compression.hpp" + +#include <memory> + +class Compressor +{ +public: + virtual ~Compressor() = default; + + // Create a compressor for the specified type. + // + // Parameters: + // - type: The type. + // - stream: The stream to write to. + // - compression_level: Desired compression level. + static std::unique_ptr<Compressor> create_from_type(Compression::Type type, + FILE* stream, + int8_t compression_level); + + // Get the actual compression level used for the compressed stream. + virtual int8_t actual_compression_level() const = 0; + + // Write data from a buffer to the compressed stream. + // + // Parameters: + // - data: Data to write. + // - count: Size of data to write. + // + // Throws Error on failure. + virtual void write(const void* data, size_t count) = 0; + + // Write an unsigned integer to the compressed stream. + // + // Parameters: + // - value: Value to write. + // + // Throws Error on failure. + template<typename T> void write(T value); + + // Finalize compression. + // + // This method checks that the end state of the compressed stream is correct + // and throws Error if not. + virtual void finalize() = 0; +}; diff --git a/src/Decompressor.cpp b/src/Decompressor.cpp new file mode 100644 index 00000000..f41caf64 --- /dev/null +++ b/src/Decompressor.cpp @@ -0,0 +1,38 @@ +// Copyright (C) 2019 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "Decompressor.hpp" + +#include "NullDecompressor.hpp" +#include "StdMakeUnique.hpp" +#include "ZstdDecompressor.hpp" + +std::unique_ptr<Decompressor> +Decompressor::create_from_type(Compression::Type type, FILE* stream) +{ + switch (type) { + case Compression::Type::none: + return std::make_unique<NullDecompressor>(stream); + + case Compression::Type::zstd: + return std::make_unique<ZstdDecompressor>(stream); + } + + assert(false); + return {}; +} diff --git a/src/Decompressor.hpp b/src/Decompressor.hpp new file mode 100644 index 00000000..dd2a4e99 --- /dev/null +++ b/src/Decompressor.hpp @@ -0,0 +1,53 @@ +// Copyright (C) 2019 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include "Compression.hpp" + +#include <cstdio> +#include <memory> + +class Decompressor +{ +public: + virtual ~Decompressor() = default; + + // Create a decompressor for the specified type. + // + // Parameters: + // - type: The type. + // - stream: The stream to read from. + static std::unique_ptr<Decompressor> create_from_type(Compression::Type type, + FILE* stream); + + // Read data into a buffer from the compressed stream. + // + // Parameters: + // - data: Buffer to write decompressed data to. + // - count: How many bytes to write. + // + // Throws Error on failure. + virtual void read(void* data, size_t count) = 0; + + // Finalize decompression. + // + // This method checks that the end state of the compressed stream is correct + // and throws Error if not. + virtual void finalize() = 0; +}; diff --git a/src/File.hpp b/src/File.hpp new file mode 100644 index 00000000..0e639db4 --- /dev/null +++ b/src/File.hpp @@ -0,0 +1,64 @@ +// Copyright (C) 2019 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include <cstdio> +#include <string> + +class File +{ +public: + File(const std::string& path, const char* mode) + { + open(path, mode); + } + + void + open(const std::string& path, const char* mode) + { + close(); + m_file = fopen(path.c_str(), mode); + } + + void + close() + { + if (m_file) { + fclose(m_file); + m_file = nullptr; + } + } + + ~File() + { + close(); + } + + operator bool() const + { + return m_file; + } + + FILE* + get() + { + return m_file; + } + +private: + FILE* m_file = nullptr; +}; diff --git a/src/NonCopyable.hpp b/src/NonCopyable.hpp new file mode 100644 index 00000000..86004a93 --- /dev/null +++ b/src/NonCopyable.hpp @@ -0,0 +1,29 @@ +// Copyright (C) 2019 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +class NonCopyable +{ +protected: + NonCopyable() = default; + +private: + NonCopyable(const NonCopyable&) = delete; + NonCopyable& operator=(const NonCopyable&) = delete; +}; diff --git a/src/NullCompressor.cpp b/src/NullCompressor.cpp new file mode 100644 index 00000000..f54438df --- /dev/null +++ b/src/NullCompressor.cpp @@ -0,0 +1,47 @@ +// Copyright (C) 2019 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "NullCompressor.hpp" + +#include "Error.hpp" + +NullCompressor::NullCompressor(FILE* stream) : m_stream(stream) +{ +} + +int8_t +NullCompressor::actual_compression_level() const +{ + return 0; +} + +void +NullCompressor::write(const void* data, size_t count) +{ + if (fwrite(data, 1, count, m_stream) != count) { + throw Error("failed to write to uncompressed stream"); + } +} + +void +NullCompressor::finalize() +{ + if (fflush(m_stream) != 0) { + throw Error("failed to finalize uncompressed stream"); + } +} diff --git a/src/NullCompressor.hpp b/src/NullCompressor.hpp new file mode 100644 index 00000000..1dfdf55c --- /dev/null +++ b/src/NullCompressor.hpp @@ -0,0 +1,38 @@ +// Copyright (C) 2019 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include "Compressor.hpp" +#include "NonCopyable.hpp" + +// A compressor of an uncompressed stream. +class NullCompressor : public Compressor, NonCopyable +{ +public: + // Parameters: + // - stream: The file to write data to. + explicit NullCompressor(FILE* stream); + + int8_t actual_compression_level() const override; + void write(const void* data, size_t count) override; + void finalize() override; + +private: + FILE* m_stream; +}; diff --git a/src/NullDecompressor.cpp b/src/NullDecompressor.cpp new file mode 100644 index 00000000..789de4fa --- /dev/null +++ b/src/NullDecompressor.cpp @@ -0,0 +1,41 @@ +// Copyright (C) 2019 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "NullDecompressor.hpp" + +#include "Error.hpp" + +NullDecompressor::NullDecompressor(FILE* stream) : m_stream(stream) +{ +} + +void +NullDecompressor::read(void* data, size_t count) +{ + if (fread(data, count, 1, m_stream) != 1) { + throw Error("failed to read from uncompressed stream"); + } +} + +void +NullDecompressor::finalize() +{ + if (fgetc(m_stream) != EOF) { + throw Error("garbage data at end of uncompressed stream"); + } +} diff --git a/src/NullDecompressor.hpp b/src/NullDecompressor.hpp new file mode 100644 index 00000000..cef5b445 --- /dev/null +++ b/src/NullDecompressor.hpp @@ -0,0 +1,37 @@ +// Copyright (C) 2019 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include "Decompressor.hpp" +#include "NonCopyable.hpp" + +// A decompressor of an uncompressed stream. +class NullDecompressor : public Decompressor, NonCopyable +{ +public: + // Parameters: + // - stream: The file to read data from. + explicit NullDecompressor(FILE* stream); + + void read(void* data, size_t count) override; + void finalize() override; + +private: + FILE* m_stream; +}; diff --git a/src/Util.cpp b/src/Util.cpp index 26dfac1f..20aff655 100644 --- a/src/Util.cpp +++ b/src/Util.cpp @@ -19,7 +19,6 @@ #include "Util.hpp" #include "ccache.hpp" -#include "util.hpp" #include <algorithm> #include <fstream> diff --git a/src/ZstdCompressor.cpp b/src/ZstdCompressor.cpp new file mode 100644 index 00000000..e3342b4f --- /dev/null +++ b/src/ZstdCompressor.cpp @@ -0,0 +1,112 @@ +// Copyright (C) 2019 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "ZstdCompressor.hpp" + +#include "Error.hpp" +#include "ccache.hpp" + +const uint8_t k_default_zstd_compression_level = -1; + +ZstdCompressor::ZstdCompressor(FILE* stream, int8_t compression_level) + : m_stream(stream), m_zstd_stream(ZSTD_createCStream()) +{ + if (compression_level == 0) { + compression_level = k_default_zstd_compression_level; + cc_log("Using default compression level %d", compression_level); + } + + // libzstd 1.3.4 and newer support negative levels. However, the query + // function ZSTD_minCLevel did not appear until 1.3.6, so perform detection + // based on version instead. + if (ZSTD_versionNumber() < 10304 && compression_level < 1) { + cc_log( + "Using compression level 1 (minimum level supported by libzstd) instead" + " of %d", + compression_level); + compression_level = 1; + } + + m_compression_level = std::min<int>(compression_level, ZSTD_maxCLevel()); + if (m_compression_level != compression_level) { + cc_log("Using compression level %d (max libzstd level) instead of %d", + m_compression_level, + compression_level); + } + + size_t ret = ZSTD_initCStream(m_zstd_stream, m_compression_level); + if (ZSTD_isError(ret)) { + ZSTD_freeCStream(m_zstd_stream); + throw Error("error initializing zstd compression stream"); + } +} + +ZstdCompressor::~ZstdCompressor() +{ + ZSTD_freeCStream(m_zstd_stream); +} + +int8_t +ZstdCompressor::actual_compression_level() const +{ + return m_compression_level; +} + +void +ZstdCompressor::write(const void* data, size_t count) +{ + m_zstd_in.src = data; + m_zstd_in.size = count; + m_zstd_in.pos = 0; + + int flush = data ? 0 : 1; + + size_t ret; + while (m_zstd_in.pos < m_zstd_in.size) { + uint8_t buffer[READ_BUFFER_SIZE]; + m_zstd_out.dst = buffer; + m_zstd_out.size = sizeof(buffer); + m_zstd_out.pos = 0; + ret = ZSTD_compressStream(m_zstd_stream, &m_zstd_out, &m_zstd_in); + assert(!(ZSTD_isError(ret))); + size_t compressed_bytes = m_zstd_out.pos; + if (fwrite(buffer, 1, compressed_bytes, m_stream) != compressed_bytes + || ferror(m_stream)) { + throw Error("failed to write to zstd output stream "); + } + } + ret = flush; + while (ret > 0) { + uint8_t buffer[READ_BUFFER_SIZE]; + m_zstd_out.dst = buffer; + m_zstd_out.size = sizeof(buffer); + m_zstd_out.pos = 0; + ret = ZSTD_endStream(m_zstd_stream, &m_zstd_out); + size_t compressed_bytes = m_zstd_out.pos; + if (fwrite(buffer, 1, compressed_bytes, m_stream) != compressed_bytes + || ferror(m_stream)) { + throw Error("failed to write to zstd output stream"); + } + } +} + +void +ZstdCompressor::finalize() +{ + write(nullptr, 0); +} diff --git a/src/ZstdCompressor.hpp b/src/ZstdCompressor.hpp new file mode 100644 index 00000000..641c2b76 --- /dev/null +++ b/src/ZstdCompressor.hpp @@ -0,0 +1,47 @@ +// Copyright (C) 2019 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include "Compressor.hpp" +#include "NonCopyable.hpp" + +#include <zstd.h> + +// A compressor of a Zstandard stream. +class ZstdCompressor : public Compressor, NonCopyable +{ +public: + // Parameters: + // - stream: The file to write data to. + // - compression_level: Desired compression level. + ZstdCompressor(FILE* stream, int8_t compression_level); + + ~ZstdCompressor() override; + + int8_t actual_compression_level() const override; + void write(const void* data, size_t count) override; + void finalize() override; + +private: + FILE* m_stream; + ZSTD_CStream* m_zstd_stream; + ZSTD_inBuffer m_zstd_in; + ZSTD_outBuffer m_zstd_out; + int8_t m_compression_level; +}; diff --git a/src/ZstdDecompressor.cpp b/src/ZstdDecompressor.cpp new file mode 100644 index 00000000..9b865860 --- /dev/null +++ b/src/ZstdDecompressor.cpp @@ -0,0 +1,82 @@ +// Copyright (C) 2019 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "ZstdDecompressor.hpp" + +#include "Error.hpp" + +ZstdDecompressor::ZstdDecompressor(FILE* stream) + : m_stream(stream), + m_input_size(0), + m_input_consumed(0), + m_zstd_stream(ZSTD_createDStream()), + m_reached_stream_end(false) +{ + size_t ret = ZSTD_initDStream(m_zstd_stream); + if (ZSTD_isError(ret)) { + ZSTD_freeDStream(m_zstd_stream); + throw Error("failed to initialize zstd decompression stream"); + } +} + +ZstdDecompressor::~ZstdDecompressor() +{ + ZSTD_freeDStream(m_zstd_stream); +} + +void +ZstdDecompressor::read(void* data, size_t count) +{ + size_t bytes_read = 0; + while (bytes_read < count) { + assert(m_input_size >= m_input_consumed); + if (m_input_size == m_input_consumed) { + m_input_size = fread(m_input_buffer, 1, sizeof(m_input_buffer), m_stream); + if (m_input_size == 0) { + throw Error("failed to read from zstd input stream"); + } + m_input_consumed = 0; + } + + m_zstd_in.src = (m_input_buffer + m_input_consumed); + m_zstd_in.size = m_input_size - m_input_consumed; + m_zstd_in.pos = 0; + + m_zstd_out.dst = static_cast<uint8_t*>(data) + bytes_read; + m_zstd_out.size = count - bytes_read; + m_zstd_out.pos = 0; + size_t ret = ZSTD_decompressStream(m_zstd_stream, &m_zstd_out, &m_zstd_in); + if (ZSTD_isError(ret)) { + throw Error("failed to read from zstd input stream"); + } + if (ret == 0) { + m_reached_stream_end = true; + break; + } + bytes_read += m_zstd_out.pos; + m_input_consumed += m_zstd_in.pos; + } +} + +void +ZstdDecompressor::finalize() +{ + if (!m_reached_stream_end) { + throw Error("garbage data at end of zstd input stream"); + } +} diff --git a/src/ZstdDecompressor.hpp b/src/ZstdDecompressor.hpp new file mode 100644 index 00000000..45616280 --- /dev/null +++ b/src/ZstdDecompressor.hpp @@ -0,0 +1,49 @@ +// Copyright (C) 2019 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include "Decompressor.hpp" +#include "ccache.hpp" + +#include <fstream> +#include <zstd.h> + +// A decompressor of a Zstandard stream. +class ZstdDecompressor : public Decompressor +{ +public: + // Parameters: + // - stream: The file to read data from. + explicit ZstdDecompressor(FILE* stream); + + ~ZstdDecompressor() override; + + void read(void* data, size_t count) override; + void finalize() override; + +private: + FILE* m_stream; + char m_input_buffer[READ_BUFFER_SIZE]; + size_t m_input_size; + size_t m_input_consumed; + ZSTD_DStream* m_zstd_stream; + ZSTD_inBuffer m_zstd_in; + ZSTD_outBuffer m_zstd_out; + bool m_reached_stream_end; +}; diff --git a/src/ccache.cpp b/src/ccache.cpp index 737cfd80..80e13676 100644 --- a/src/ccache.cpp +++ b/src/ccache.cpp @@ -1196,13 +1196,11 @@ update_manifest_file(void) MTR_BEGIN("manifest", "manifest_put"); cc_log("Adding result name to %s", manifest_path); - if (manifest_put(manifest_path, cached_result_name, g_included_files)) { - if (x_stat(manifest_path, &st) == 0) { - stats_update_size( - manifest_stats_file, file_size(&st) - old_size, old_size == 0 ? 1 : 0); - } - } else { + if (!manifest_put(manifest_path, *cached_result_name, g_included_files)) { cc_log("Failed to add result name to %s", manifest_path); + } else if (x_stat(manifest_path, &st) == 0) { + stats_update_size( + manifest_stats_file, file_size(&st) - old_size, old_size == 0 ? 1 : 0); } MTR_END("manifest", "manifest_put"); } @@ -1437,32 +1435,31 @@ to_cache(struct args* args, struct hash* depend_mode_hash) stats_update(STATS_ERROR); failed(); } - struct result_files* result_files = result_files_init(); + ResultFileMap result_file_map; if (st.st_size > 0) { - result_files_add(result_files, tmp_stderr, RESULT_STDERR_NAME); + result_file_map.emplace(k_result_stderr_name, tmp_stderr); } - result_files_add(result_files, output_obj, ".o"); + result_file_map.emplace(".o", output_obj); if (generating_dependencies) { - result_files_add(result_files, output_dep, ".d"); + result_file_map.emplace(".d", output_dep); } if (generating_coverage) { - result_files_add(result_files, output_cov, ".gcno"); + result_file_map.emplace(".gcno", output_cov); } if (generating_stackusage) { - result_files_add(result_files, output_su, ".su"); + result_file_map.emplace(".su", output_su); } if (generating_diagnostics) { - result_files_add(result_files, output_dia, ".dia"); + result_file_map.emplace(".dia", output_dia); } if (seen_split_dwarf && stat(output_dwo, &st) == 0) { // Only copy .dwo file if it was created by the compiler (GCC and Clang // behave differently e.g. for "-gsplit-dwarf -g1"). - result_files_add(result_files, output_dwo, ".dwo"); + result_file_map.emplace(".dwo", output_dwo); } struct stat orig_dest_st; bool orig_dest_existed = stat(cached_result_path, &orig_dest_st) == 0; - result_put(cached_result_path, result_files); - result_files_free(result_files); + result_put(cached_result_path, result_file_map); cc_log("Stored in cache: %s", cached_result_path); @@ -1862,11 +1859,11 @@ calculate_result_name(struct args* args, struct hash* hash, int direct_mode) bool found_ccbin = false; hash_delimiter(hash, "result version"); - hash_int(hash, RESULT_VERSION); + hash_int(hash, k_result_version); if (direct_mode) { hash_delimiter(hash, "manifest version"); - hash_int(hash, MANIFEST_VERSION); + hash_int(hash, k_manifest_version); } // clang will emit warnings for unused linker flags, so we shouldn't skip @@ -2176,28 +2173,27 @@ from_cache(enum fromcache_call_mode mode, bool put_result_in_manifest) int tmp_stderr_fd = create_tmp_fd(&tmp_stderr); close(tmp_stderr_fd); - struct result_files* result_files = result_files_init(); + ResultFileMap result_file_map; if (!str_eq(output_obj, "/dev/null")) { - result_files_add(result_files, output_obj, ".o"); + result_file_map.emplace(".o", output_obj); if (seen_split_dwarf) { - result_files_add(result_files, output_dwo, ".dwo"); + result_file_map.emplace(".dwo", output_dwo); } } - result_files_add(result_files, tmp_stderr, RESULT_STDERR_NAME); + result_file_map.emplace(k_result_stderr_name, tmp_stderr); if (produce_dep_file) { - result_files_add(result_files, output_dep, ".d"); + result_file_map.emplace(".d", output_dep); } if (generating_coverage) { - result_files_add(result_files, output_cov, ".gcno"); + result_file_map.emplace(".gcno", output_cov); } if (generating_stackusage) { - result_files_add(result_files, output_su, ".su"); + result_file_map.emplace(".su", output_su); } if (generating_diagnostics) { - result_files_add(result_files, output_dia, ".dia"); + result_file_map.emplace(".dia", output_dia); } - bool ok = result_get(cached_result_path, result_files); - result_files_free(result_files); + bool ok = result_get(cached_result_path, result_file_map); if (!ok) { cc_log("Failed to get result from cache"); tmp_unlink(tmp_stderr); @@ -3957,15 +3953,11 @@ ccache_main_options(int argc, char* argv[]) switch (c) { case DUMP_MANIFEST: initialize(); - manifest_dump(optarg, stdout); - break; + return manifest_dump(optarg, stdout) ? 0 : 1; case DUMP_RESULT: initialize(); - if (!result_dump(optarg, stdout)) { - return 1; - } - break; + return result_dump(optarg, stdout) ? 0 : 1; case HASH_FILE: { initialize(); diff --git a/src/ccache.hpp b/src/ccache.hpp index 4bf7d7f7..f41e0fe7 100644 --- a/src/ccache.hpp +++ b/src/ccache.hpp @@ -22,7 +22,6 @@ #include "system.hpp" #include "counters.hpp" -#include "util.hpp" #include "third_party/minitrace.h" diff --git a/src/common_header.cpp b/src/common_header.cpp deleted file mode 100644 index c8baba6f..00000000 --- a/src/common_header.cpp +++ /dev/null @@ -1,160 +0,0 @@ -// Copyright (C) 2019 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "common_header.hpp" - -#include "ccache.hpp" -#include "int_bytes_conversion.hpp" - -bool -common_header_initialize_for_writing(struct common_header* header, - FILE* output, - const char magic[4], - uint8_t version, - uint8_t compression_type, - int8_t compression_level, - uint64_t content_size, - Checksum& checksum, - struct compressor** compressor, - struct compr_state** compr_state) -{ - memcpy(header->magic, magic, 4); - header->version = version; - header->compression_type = compression_type; - header->compression_level = compression_level; - header->content_size = content_size; - - *compressor = compressor_from_type(header->compression_type); - assert(*compressor); - *compr_state = - (*compressor)->init(output, header->compression_level, &checksum); - if (!*compr_state) { - cc_log("Failed to initialize compressor"); - return false; - } - header->compression_level = - (*compressor)->get_actual_compression_level(*compr_state); - - uint8_t header_bytes[COMMON_HEADER_SIZE]; - memcpy(header_bytes, header->magic, 4); - header_bytes[4] = header->version; - header_bytes[5] = header->compression_type; - header_bytes[6] = header->compression_level; - BYTES_FROM_UINT64(header_bytes + 7, header->content_size); - if (fwrite(header_bytes, sizeof(header_bytes), 1, output) != 1) { - cc_log("Failed to write common file header"); - return false; - } - checksum.update(header_bytes, sizeof(header_bytes)); - return true; -} - -bool -common_header_initialize_for_reading(struct common_header* header, - FILE* input, - const char expected_magic[4], - uint8_t expected_version, - struct decompressor** decompressor, - struct decompr_state** decompr_state, - Checksum* checksum, - char** errmsg) -{ - uint8_t header_bytes[COMMON_HEADER_SIZE]; - if (fread(header_bytes, sizeof(header_bytes), 1, input) != 1) { - *errmsg = format("Failed to read common header"); - return false; - } - - memcpy(header->magic, header_bytes, 4); - header->version = header_bytes[4]; - header->compression_type = header_bytes[5]; - header->compression_level = header_bytes[6]; - header->content_size = UINT64_FROM_BYTES(header_bytes + 7); - - if (memcmp(header->magic, expected_magic, sizeof(header->magic)) != 0) { - *errmsg = format("Bad magic value 0x%x%x%x%x", - header->magic[0], - header->magic[1], - header->magic[2], - header->magic[3]); - return false; - } - - if (header->version != expected_version) { - *errmsg = format("Unknown version (actual %u, expected %u)", - header->version, - expected_version); - return false; - } - - if (header->compression_type == COMPR_TYPE_NONE) { - // Since we have the size available, let's use it as a super primitive - // consistency check for the non-compressed case. (A real checksum is used - // for compressed data.) - struct stat st; - if (x_fstat(fileno(input), &st) != 0) { - return false; - } - if ((uint64_t)st.st_size != header->content_size) { - *errmsg = format( - "Bad uncompressed file size (actual %llu bytes, expected %llu bytes)", - (unsigned long long)st.st_size, - (unsigned long long)header->content_size); - return false; - } - } - - if (!decompressor) { - return true; - } - - *decompressor = decompressor_from_type(header->compression_type); - if (!*decompressor) { - *errmsg = format("Unknown compression type: %u", header->compression_type); - return false; - } - - if (checksum) { - checksum->update(header_bytes, sizeof(header_bytes)); - } - - *decompr_state = (*decompressor)->init(input, checksum); - if (!*decompr_state) { - *errmsg = x_strdup("Failed to initialize decompressor"); - return false; - } - - return true; -} - -void -common_header_dump(const struct common_header* header, FILE* f) -{ - fprintf(f, - "Magic: %c%c%c%c\n", - header->magic[0], - header->magic[1], - header->magic[2], - header->magic[3]); - fprintf(f, "Version: %u\n", header->version); - fprintf(f, - "Compression type: %s\n", - compression_type_to_string(header->compression_type)); - fprintf(f, "Compression level: %d\n", header->compression_level); - fprintf(f, "Content size: %" PRIu64 "\n", header->content_size); -} diff --git a/src/common_header.hpp b/src/common_header.hpp deleted file mode 100644 index 5c32fb62..00000000 --- a/src/common_header.hpp +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright (C) 2019 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include "Checksum.hpp" -#include "compression.hpp" - -#define COMMON_HEADER_SIZE 15 - -struct common_header -{ - char magic[4]; - uint8_t version; - uint8_t compression_type; - int8_t compression_level; - uint64_t content_size; -}; - -// Initialize a common_header and write the header data to a file. -// -// header: Header to initialize. -// output: Open file to write to. -// magic: File format magic bytes. -// version: File format version. -// compression_type: Compression type to use. -// compression_level: Compression level to use. -// content_size: Content size. -// compressor: Compressor created from compression_type. -// compressor_state: State for the compressor. -// checksum: Checksum state that will be updated with the written -// bytes. -bool common_header_initialize_for_writing(struct common_header* header, - FILE* output, - const char magic[4], - uint8_t version, - uint8_t compression_type, - int8_t compression_level, - uint64_t content_size, - Checksum& checksum, - struct compressor** compressor, - struct compr_state** compr_state); - -// Initialize a common_header by reading header data from a file. -// -// header: Header to initialize. -// input: Open file to read from. -// expected_magic: Expected file format magic bytes. -// expected_version: Expected file format version. -// decompressor: Decompressor created from the compression type field in -// the header. Pass NULL to not create a decompressor. -// decompressor_state: State for the decompressor. Should be NULL if -// decompressor is NULL. -// checksum: Checksum state that will be updated with the read bytes. -// May be NULL for no checksumming. -bool common_header_initialize_for_reading(struct common_header* header, - FILE* input, - const char expected_magic[4], - uint8_t expected_version, - struct decompressor** decompressor, - struct decompr_state** decompr_state, - Checksum* checksum, - char** errmsg); - -void common_header_dump(const struct common_header* header, FILE* f); diff --git a/src/compr_none.cpp b/src/compr_none.cpp deleted file mode 100644 index e9dc628d..00000000 --- a/src/compr_none.cpp +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (C) 2019 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "compression.hpp" - -struct state -{ - FILE* output; - Checksum* checksum; -}; - -static struct compr_state* -compr_none_init(FILE* output, int8_t level, Checksum* checksum) -{ - auto state = static_cast<struct state*>(malloc(sizeof(struct state))); - state->output = output; - state->checksum = checksum; - (void)level; - return (struct compr_state*)state; -} - -static int8_t -compr_none_get_actual_compression_level(struct compr_state* handle) -{ - (void)handle; - return 0; -} - -static bool -compr_none_write(struct compr_state* handle, const void* data, size_t size) -{ - struct state* state = (struct state*)handle; - size_t ret = fwrite(data, size, 1, state->output); - if (state->checksum) { - state->checksum->update(data, size); - } - return ret == 1; -} - -static bool -compr_none_free(struct compr_state* handle) -{ - struct state* state = (struct state*)handle; - bool result = ferror(state->output) == 0; - free(state); - return result; -} - -struct compressor compressor_none_impl = { - compr_none_init, - compr_none_get_actual_compression_level, - compr_none_write, - compr_none_free}; diff --git a/src/compr_zstd.cpp b/src/compr_zstd.cpp deleted file mode 100644 index dd9b41ad..00000000 --- a/src/compr_zstd.cpp +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright (C) 2019 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "ccache.hpp" -#include "compression.hpp" - -#include <algorithm> -#include <zstd.h> - -#define DEFAULT_ZSTD_COMPRESSION_LEVEL -1 - -struct state -{ - FILE* output; - Checksum* checksum; - ZSTD_CStream* stream; - ZSTD_inBuffer in; - ZSTD_outBuffer out; - bool failed; - int8_t compression_level; -}; - -static struct compr_state* -compr_zstd_init(FILE* output, int8_t level, Checksum* checksum) -{ - auto state = static_cast<struct state*>(malloc(sizeof(struct state))); - state->output = output; - state->checksum = checksum; - state->stream = ZSTD_createCStream(); - state->failed = false; - - if (level == 0) { - level = DEFAULT_ZSTD_COMPRESSION_LEVEL; - cc_log("Using default compression level %d", level); - } - - // libzstd 1.3.4 and newer support negative levels. However, the query - // function ZSTD_minCLevel did not appear until 1.3.6, so perform detection - // based on version instead. - if (ZSTD_versionNumber() < 10304 && level < 1) { - cc_log( - "Using compression level 1 (minimum level supported by libzstd) instead" - " of %d", - level); - level = 1; - } - - state->compression_level = std::min<int>(level, ZSTD_maxCLevel()); - if (state->compression_level != level) { - cc_log("Using compression level %d (max libzstd level) instead of %d", - state->compression_level, - level); - } - - size_t ret = ZSTD_initCStream(state->stream, state->compression_level); - if (ZSTD_isError(ret)) { - ZSTD_freeCStream(state->stream); - free(state); - return NULL; - } - return (struct compr_state*)state; -} - -static int8_t -compr_zstd_get_actual_compression_level(struct compr_state* handle) -{ - struct state* state = (struct state*)handle; - return state->compression_level; -} - -static bool -compr_zstd_write(struct compr_state* handle, const void* data, size_t size) -{ - if (!handle) { - return false; - } - struct state* state = (struct state*)handle; - - if (state->checksum) { - state->checksum->update(data, size); - } - - state->in.src = data; - state->in.size = size; - state->in.pos = 0; - - int flush = data ? 0 : 1; - - size_t ret; - while (state->in.pos < state->in.size) { - unsigned char buffer[READ_BUFFER_SIZE]; - state->out.dst = buffer; - state->out.size = sizeof(buffer); - state->out.pos = 0; - ret = ZSTD_compressStream(state->stream, &state->out, &state->in); - assert(!(ZSTD_isError(ret))); - size_t compressed_bytes = state->out.pos; - if (fwrite(buffer, 1, compressed_bytes, state->output) != compressed_bytes - || ferror(state->output)) { - state->failed = true; - return false; - } - } - ret = flush; - while (ret > 0) { - unsigned char buffer[READ_BUFFER_SIZE]; - state->out.dst = buffer; - state->out.size = sizeof(buffer); - state->out.pos = 0; - ret = ZSTD_endStream(state->stream, &state->out); - size_t compressed_bytes = state->out.pos; - if (fwrite(buffer, 1, compressed_bytes, state->output) != compressed_bytes - || ferror(state->output)) { - state->failed = true; - return false; - } - } - - return true; -} - -static bool -compr_zstd_free(struct compr_state* handle) -{ - if (!handle) { - return false; - } - - struct state* state = (struct state*)handle; - - compr_zstd_write(handle, NULL, 0); - ZSTD_freeCStream(state->stream); - bool success = !state->failed; - free(state); - return success; -} - -struct compressor compressor_zstd_impl = { - compr_zstd_init, - compr_zstd_get_actual_compression_level, - compr_zstd_write, - compr_zstd_free}; diff --git a/src/compress.cpp b/src/compress.cpp index 21e9d1d6..6cbd37c0 100644 --- a/src/compress.cpp +++ b/src/compress.cpp @@ -18,8 +18,9 @@ #include "compress.hpp" +#include "CacheEntryReader.hpp" +#include "File.hpp" #include "ccache.hpp" -#include "common_header.hpp" #include "manifest.hpp" #include "result.hpp" @@ -27,25 +28,22 @@ static bool get_content_size(const std::string& path, - const char* magic, + const uint8_t magic[4], uint8_t version, - size_t* size) + uint64_t& size) { - char* errmsg; - FILE* f = fopen(path.c_str(), "rb"); + File f(path, "rb"); if (!f) { cc_log("Failed to open %s for reading: %s", path.c_str(), strerror(errno)); return false; } - struct common_header header; - bool success = common_header_initialize_for_reading( - &header, f, magic, version, NULL, NULL, NULL, &errmsg); - fclose(f); - if (success) { - *size = header.content_size; - } - return success; + try { + size = CacheEntryReader(f.get(), magic, version).content_size(); + return true; + } catch (const Error&) { + return false; + } } void @@ -72,14 +70,14 @@ compress_stats(const Config& config, on_disk_size += file_size(&file->stat()); - size_t content_size = 0; + uint64_t content_size = 0; bool is_compressible; if (file->type() == CacheFile::Type::manifest) { is_compressible = get_content_size( - file->path(), MANIFEST_MAGIC, MANIFEST_VERSION, &content_size); + file->path(), k_manifest_magic, k_manifest_version, content_size); } else if (file->type() == CacheFile::Type::result) { is_compressible = get_content_size( - file->path(), RESULT_MAGIC, RESULT_VERSION, &content_size); + file->path(), k_result_magic, k_result_version, content_size); } else { is_compressible = false; } diff --git a/src/compression.hpp b/src/compression.hpp deleted file mode 100644 index 4bd60d03..00000000 --- a/src/compression.hpp +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright (C) 2019 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include "Checksum.hpp" - -struct compr_state; - -struct compressor -{ - // Create and initialize a compressor. - // - // output: The file to write compressed data to. - // compression_level: Desired compression level. - // checksum: Checksum state to update (NULL for no checksum). - struct compr_state* (*init)(FILE* output, - int8_t compression_level, - Checksum* checksum); - - // Get the actual compression level that will be used. - int8_t (*get_actual_compression_level)(struct compr_state* state); - - // Compress data. - // - // data: Buffer to read decompressed data from. - // size: How many bytes to read. - // - // Returns false on failure, otherwise true. - bool (*write)(struct compr_state* state, const void* data, size_t size); - - // Finalize compressor and free its state. - // - // Returns false if finalization failed or if any previous operation failed, - // otherwise true. - bool (*free)(struct compr_state* state); -}; - -struct decompr_state; - -struct decompressor -{ - // Create and initialize a decompressor. - // - // input: The file to read compressed data from. - // checksum: Checksum state to update (NULL for no checksum). - struct decompr_state* (*init)(FILE* input, Checksum* checksum); - - // Decompress data. - // - // data: Buffer to write decompressed data to. - // size: How many bytes to write. - // - // Returns false on failure, otherwise true. - bool (*read)(struct decompr_state* state, void* data, size_t size); - - // Finalize the decompressor and free its state. - // - // Returns false if finalization failed or if any previous operation failed, - // otherwise true. - bool (*free)(struct decompr_state* state); -}; - -enum compression_type { COMPR_TYPE_NONE = 0, COMPR_TYPE_ZSTD = 1 }; - -extern struct compressor compressor_none_impl; -extern struct decompressor decompressor_none_impl; - -extern struct compressor compressor_zstd_impl; -extern struct decompressor decompressor_zstd_impl; - -int8_t compression_level_from_config(void); -enum compression_type compression_type_from_config(void); -const char* compression_type_to_string(uint8_t type); -struct compressor* compressor_from_type(uint8_t type); -struct decompressor* decompressor_from_type(uint8_t type); diff --git a/src/decompr_none.cpp b/src/decompr_none.cpp deleted file mode 100644 index 29018d22..00000000 --- a/src/decompr_none.cpp +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright (C) 2019 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "compression.hpp" - -struct state -{ - FILE* input; - Checksum* checksum; - bool failed; -}; - -static struct decompr_state* -decompr_none_init(FILE* input, Checksum* checksum) -{ - auto state = static_cast<struct state*>(malloc(sizeof(struct state))); - state->input = input; - state->checksum = checksum; - state->failed = false; - return (struct decompr_state*)state; -} - -static bool -decompr_none_read(struct decompr_state* handle, void* data, size_t size) -{ - struct state* state = (struct state*)handle; - - bool result = fread(data, 1, size, state->input) == size; - if (result && state->checksum) { - state->checksum->update(data, size); - } - if (!result) { - state->failed = true; - } - return result; -} - -static bool -decompr_none_free(struct decompr_state* handle) -{ - struct state* state = (struct state*)handle; - bool result = !state->failed; - free(state); - return result; -} - -struct decompressor decompressor_none_impl = { - decompr_none_init, decompr_none_read, decompr_none_free}; diff --git a/src/decompr_zstd.cpp b/src/decompr_zstd.cpp deleted file mode 100644 index e187abc3..00000000 --- a/src/decompr_zstd.cpp +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright (C) 2019 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "ccache.hpp" -#include "compression.hpp" - -#include <zstd.h> - -enum stream_state { - STREAM_STATE_READING, - STREAM_STATE_FAILED, - STREAM_STATE_END -}; - -struct state -{ - FILE* input; - Checksum* checksum; - char input_buffer[READ_BUFFER_SIZE]; - size_t input_size; - size_t input_consumed; - ZSTD_DStream* stream; - ZSTD_inBuffer in; - ZSTD_outBuffer out; - enum stream_state stream_state; -}; - -static struct decompr_state* -decompr_zstd_init(FILE* input, Checksum* checksum) -{ - auto state = static_cast<struct state*>(malloc(sizeof(struct state))); - - state->input = input; - state->checksum = checksum; - state->input_size = 0; - state->input_consumed = 0; - state->stream = ZSTD_createDStream(); - state->stream_state = STREAM_STATE_READING; - - size_t ret = ZSTD_initDStream(state->stream); - if (ZSTD_isError(ret)) { - ZSTD_freeDStream(state->stream); - free(state); - return NULL; - } - return (struct decompr_state*)state; -} - -static bool -decompr_zstd_read(struct decompr_state* handle, void* data, size_t size) -{ - if (!handle) { - return false; - } - struct state* state = (struct state*)handle; - - size_t bytes_read = 0; - while (bytes_read < size) { - assert(state->input_size >= state->input_consumed); - if (state->input_size == state->input_consumed) { - state->input_size = fread( - state->input_buffer, 1, sizeof(state->input_buffer), state->input); - if (state->input_size == 0) { - state->stream_state = STREAM_STATE_FAILED; - return false; - } - state->input_consumed = 0; - } - - state->in.src = (state->input_buffer + state->input_consumed); - state->in.size = state->input_size - state->input_consumed; - state->in.pos = 0; - - state->out.dst = ((char*)data + bytes_read); - state->out.size = size - bytes_read; - state->out.pos = 0; - size_t ret = ZSTD_decompressStream(state->stream, &state->out, &state->in); - if (ZSTD_isError(ret)) { - state->stream_state = STREAM_STATE_FAILED; - return false; - } - if (state->checksum) { - state->checksum->update(state->out.dst, state->out.pos); - } - if (ret == 0) { - state->stream_state = STREAM_STATE_END; - break; - } - bytes_read += state->out.pos; - state->input_consumed += state->in.pos; - } - - return true; -} - -static bool -decompr_zstd_free(struct decompr_state* handle) -{ - if (!handle) { - return false; - } - struct state* state = (struct state*)handle; - ZSTD_freeDStream(state->stream); - bool success = state->stream_state == STREAM_STATE_END; - free(handle); - return success; -} - -struct decompressor decompressor_zstd_impl = { - decompr_zstd_init, decompr_zstd_read, decompr_zstd_free}; diff --git a/src/int_bytes_conversion.hpp b/src/int_bytes_conversion.hpp deleted file mode 100644 index 47540dae..00000000 --- a/src/int_bytes_conversion.hpp +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright (C) 2019 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#define BYTES_FROM_UINT16(bytes, uint16) \ - do { \ - (bytes)[0] = (uint16) >> 8 & 0xFF; \ - (bytes)[1] = (uint16) >> 0 & 0xFF; \ - } while (false) - -#define UINT16_FROM_BYTES(bytes) \ - ((uint16_t)(uint8_t)(bytes)[0] << 8 | (uint16_t)(uint8_t)(bytes)[1] << 0) - -#define BYTES_FROM_UINT32(bytes, uint32) \ - do { \ - (bytes)[0] = (uint32) >> 24 & 0xFF; \ - (bytes)[1] = (uint32) >> 16 & 0xFF; \ - (bytes)[2] = (uint32) >> 8 & 0xFF; \ - (bytes)[3] = (uint32) >> 0 & 0xFF; \ - } while (false) - -#define UINT32_FROM_BYTES(bytes) \ - ((uint32_t)(uint8_t)(bytes)[0] << 24 | (uint32_t)(uint8_t)(bytes)[1] << 16 \ - | (uint32_t)(uint8_t)(bytes)[2] << 8 | (uint32_t)(uint8_t)(bytes)[3] << 0) - -#define BYTES_FROM_INT64(bytes, int64) \ - do { \ - (bytes)[0] = (int64) >> 56 & 0xFF; \ - (bytes)[1] = (int64) >> 48 & 0xFF; \ - (bytes)[2] = (int64) >> 40 & 0xFF; \ - (bytes)[3] = (int64) >> 32 & 0xFF; \ - (bytes)[4] = (int64) >> 24 & 0xFF; \ - (bytes)[5] = (int64) >> 16 & 0xFF; \ - (bytes)[6] = (int64) >> 8 & 0xFF; \ - (bytes)[7] = (int64) >> 0 & 0xFF; \ - } while (false) - -#define INT64_FROM_BYTES(bytes) \ - ((int64_t)(uint8_t)(bytes)[0] << 56 | (int64_t)(uint8_t)(bytes)[1] << 48 \ - | (int64_t)(uint8_t)(bytes)[2] << 40 | (int64_t)(uint8_t)(bytes)[3] << 32 \ - | (int64_t)(uint8_t)(bytes)[4] << 24 | (int64_t)(uint8_t)(bytes)[5] << 16 \ - | (int64_t)(uint8_t)(bytes)[6] << 8 | (int64_t)(uint8_t)(bytes)[7] << 0) - -#define BYTES_FROM_UINT64(bytes, uint64) \ - do { \ - (bytes)[0] = (uint64) >> 56 & 0xFF; \ - (bytes)[1] = (uint64) >> 48 & 0xFF; \ - (bytes)[2] = (uint64) >> 40 & 0xFF; \ - (bytes)[3] = (uint64) >> 32 & 0xFF; \ - (bytes)[4] = (uint64) >> 24 & 0xFF; \ - (bytes)[5] = (uint64) >> 16 & 0xFF; \ - (bytes)[6] = (uint64) >> 8 & 0xFF; \ - (bytes)[7] = (uint64) >> 0 & 0xFF; \ - } while (false) - -#define UINT64_FROM_BYTES(bytes) \ - ((uint64_t)(uint8_t)(bytes)[0] << 56 | (uint64_t)(uint8_t)(bytes)[1] << 48 \ - | (uint64_t)(uint8_t)(bytes)[2] << 40 | (uint64_t)(uint8_t)(bytes)[3] << 32 \ - | (uint64_t)(uint8_t)(bytes)[4] << 24 | (uint64_t)(uint8_t)(bytes)[5] << 16 \ - | (uint64_t)(uint8_t)(bytes)[6] << 8 | (uint64_t)(uint8_t)(bytes)[7] << 0) diff --git a/src/legacy_util.cpp b/src/legacy_util.cpp index 90e10c83..651ccb19 100644 --- a/src/legacy_util.cpp +++ b/src/legacy_util.cpp @@ -17,8 +17,6 @@ // this program; if not, write to the Free Software Foundation, Inc., 51 // Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -#include "util.hpp" - #include "Config.hpp" #include "Error.hpp" #include "Util.hpp" diff --git a/src/manifest.cpp b/src/manifest.cpp index 527f999e..36d81163 100644 --- a/src/manifest.cpp +++ b/src/manifest.cpp @@ -18,12 +18,16 @@ #include "manifest.hpp" +#include "AtomicFile.hpp" +#include "CacheEntryReader.hpp" +#include "CacheEntryWriter.hpp" #include "Checksum.hpp" +#include "Config.hpp" +#include "File.hpp" +#include "StdMakeUnique.hpp" #include "ccache.hpp" -#include "common_header.hpp" -#include "compression.hpp" +#include "hash.hpp" #include "hashutil.hpp" -#include "int_bytes_conversion.hpp" // Manifest data format // ==================== @@ -100,18 +104,13 @@ // 1: Introduced in ccache 3.0. (Files are always compressed with gzip.) // 2: Introduced in ccache 4.0. -const char MANIFEST_MAGIC[4] = {'c', 'C', 'm', 'F'}; -static const uint32_t MAX_MANIFEST_ENTRIES = 100; -static const uint32_t MAX_MANIFEST_FILE_INFO_ENTRIES = 10000; +const uint8_t k_manifest_magic[4] = {'c', 'C', 'm', 'F'}; +const uint8_t k_manifest_version = 2; +const uint32_t k_max_manifest_entries = 100; +const uint32_t k_max_manifest_file_info_entries = 10000; namespace { -struct file -{ - uint16_t path_len; // strlen(path) - char* path; // NUL-terminated -}; - struct FileInfo { // Index to n_files. @@ -152,329 +151,265 @@ template<> struct hash<FileInfo> } // namespace std -struct result +struct ResultEntry { - // Number of entries in file_info_indexes. - uint32_t n_file_info_indexes; // Indexes to file_infos. - uint32_t* file_info_indexes; + std::vector<uint32_t> file_info_indexes; + // Name of the result. struct digest name; }; -struct manifest +struct ManifestData { - struct common_header header; - // Referenced include files. - uint32_t n_files; - struct file* files; + std::vector<std::string> files; // Information about referenced include files. - uint32_t n_file_infos; - struct FileInfo* file_infos; + std::vector<FileInfo> file_infos; // Result names plus references to include file infos. - uint32_t n_results; - struct result* results; + std::vector<ResultEntry> results; + + void + add_result_entry( + const struct digest& result_digest, + const std::unordered_map<std::string, digest>& included_files) + { + std::unordered_map<std::string, uint32_t /*index*/> mf_files; + for (uint32_t i = 0; i < files.size(); ++i) { + mf_files.emplace(files[i], i); + } + + std::unordered_map<FileInfo, uint32_t /*index*/> mf_file_infos; + for (uint32_t i = 0; i < file_infos.size(); ++i) { + mf_file_infos.emplace(file_infos[i], i); + } + + std::vector<uint32_t> file_info_indexes; + for (const auto& item : included_files) { + file_info_indexes.push_back( + get_file_info_index(item.first, item.second, mf_files, mf_file_infos)); + } + + results.push_back(ResultEntry{std::move(file_info_indexes), result_digest}); + } + +private: + uint32_t + get_file_info_index( + const std::string& path, + const digest& digest, + const std::unordered_map<std::string, uint32_t>& mf_files, + const std::unordered_map<FileInfo, uint32_t>& mf_file_infos) + { + struct FileInfo fi; + + auto f_it = mf_files.find(path); + if (f_it != mf_files.end()) { + fi.index = f_it->second; + } else { + files.push_back(path); + fi.index = files.size() - 1; + } + + fi.digest = digest; + + // file_stat.st_{m,c}time have a resolution of 1 second, so we can cache + // the file's mtime and ctime only if they're at least one second older + // than time_of_compilation. + // + // st->ctime may be 0, so we have to check time_of_compilation against + // MAX(mtime, ctime). + + struct stat file_stat; + if (stat(path.c_str(), &file_stat) != -1) { + if (time_of_compilation + > std::max(file_stat.st_mtime, file_stat.st_ctime)) { + fi.mtime = file_stat.st_mtime; + fi.ctime = file_stat.st_ctime; + } else { + fi.mtime = -1; + fi.ctime = -1; + } + fi.fsize = file_stat.st_size; + } else { + fi.mtime = -1; + fi.ctime = -1; + fi.fsize = 0; + } + + auto fi_it = mf_file_infos.find(fi); + if (fi_it != mf_file_infos.end()) { + return fi_it->second; + } else { + file_infos.push_back(fi); + return file_infos.size() - 1; + } + } }; -struct file_stats +struct FileStats { uint64_t size; int64_t mtime; int64_t ctime; }; -static void -free_manifest(struct manifest* mf) +static std::unique_ptr<ManifestData> +read_manifest(const std::string& path, FILE* dump_stream = nullptr) { - for (uint32_t i = 0; i < mf->n_files; i++) { - free(mf->files[i].path); - } - free(mf->files); - free(mf->file_infos); - for (uint32_t i = 0; i < mf->n_results; i++) { - free(mf->results[i].file_info_indexes); + File file(path, "rb"); + if (!file) { + return {}; } - free(mf->results); - free(mf); -} - -#define READ_BYTES(buf, length) \ - do { \ - if (!decompressor->read(decompr_state, buf, length)) { \ - goto out; \ - } \ - } while (false) - -#define READ_UINT16(var) \ - do { \ - char buf_[2]; \ - READ_BYTES(buf_, sizeof(buf_)); \ - (var) = UINT16_FROM_BYTES(buf_); \ - } while (false) - -#define READ_UINT32(var) \ - do { \ - char buf_[4]; \ - READ_BYTES(buf_, sizeof(buf_)); \ - (var) = UINT32_FROM_BYTES(buf_); \ - } while (false) - -#define READ_INT64(var) \ - do { \ - char buf_[8]; \ - READ_BYTES(buf_, sizeof(buf_)); \ - (var) = INT64_FROM_BYTES(buf_); \ - } while (false) - -#define READ_UINT64(var) \ - do { \ - char buf_[8]; \ - READ_BYTES(buf_, sizeof(buf_)); \ - (var) = UINT64_FROM_BYTES(buf_); \ - } while (false) - -#define READ_STR(str_var, len_var) \ - do { \ - READ_UINT16(len_var); \ - (str_var) = static_cast<char*>(x_malloc(len_var + 1)); \ - READ_BYTES(str_var, len_var); \ - str_var[len_var] = '\0'; \ - } while (false) - -static struct manifest* -create_empty_manifest(void) -{ - auto mf = static_cast<manifest*>(x_malloc(sizeof(manifest))); - mf->n_files = 0; - mf->files = NULL; - mf->n_file_infos = 0; - mf->file_infos = NULL; - mf->n_results = 0; - mf->results = NULL; - - return mf; -} -static struct manifest* -read_manifest(const char* path, char** errmsg) -{ - bool success = false; - struct manifest* mf = create_empty_manifest(); - struct decompressor* decompressor = NULL; - struct decompr_state* decompr_state = NULL; - *errmsg = NULL; Checksum checksum; - uint64_t actual_checksum; - uint64_t expected_checksum; + CacheEntryReader reader( + file.get(), k_manifest_magic, k_manifest_version, &checksum); - FILE* f = fopen(path, "rb"); - if (!f) { - *errmsg = x_strdup("No such manifest file"); - goto out; + if (dump_stream) { + reader.dump_header(dump_stream); } - if (!common_header_initialize_for_reading(&mf->header, - f, - MANIFEST_MAGIC, - MANIFEST_VERSION, - &decompressor, - &decompr_state, - &checksum, - errmsg)) { - goto out; - } + auto mf = std::make_unique<ManifestData>(); - READ_UINT32(mf->n_files); - mf->files = static_cast<file*>(x_calloc(mf->n_files, sizeof(file))); - for (uint32_t i = 0; i < mf->n_files; i++) { - READ_STR(mf->files[i].path, mf->files[i].path_len); + uint32_t entry_count; + reader.read(entry_count); + for (uint32_t i = 0; i < entry_count; ++i) { + mf->files.emplace_back(); + auto& entry = mf->files.back(); + + uint16_t length; + reader.read(length); + entry.assign(length, 0); + reader.read(&entry[0], length); } - READ_UINT32(mf->n_file_infos); - mf->file_infos = - static_cast<FileInfo*>(x_calloc(mf->n_file_infos, sizeof(FileInfo))); - for (uint32_t i = 0; i < mf->n_file_infos; i++) { - READ_UINT32(mf->file_infos[i].index); - READ_BYTES(mf->file_infos[i].digest.bytes, DIGEST_SIZE); - READ_UINT64(mf->file_infos[i].fsize); - READ_INT64(mf->file_infos[i].mtime); - READ_INT64(mf->file_infos[i].ctime); + reader.read(entry_count); + for (uint32_t i = 0; i < entry_count; ++i) { + mf->file_infos.emplace_back(); + auto& entry = mf->file_infos.back(); + + reader.read(entry.index); + reader.read(entry.digest.bytes, DIGEST_SIZE); + reader.read(entry.fsize); + reader.read(entry.mtime); + reader.read(entry.ctime); } - READ_UINT32(mf->n_results); - mf->results = static_cast<result*>(x_calloc(mf->n_results, sizeof(result))); - for (uint32_t i = 0; i < mf->n_results; i++) { - READ_UINT32(mf->results[i].n_file_info_indexes); - mf->results[i].file_info_indexes = static_cast<uint32_t*>( - x_calloc(mf->results[i].n_file_info_indexes, sizeof(uint32_t))); - for (uint32_t j = 0; j < mf->results[i].n_file_info_indexes; j++) { - READ_UINT32(mf->results[i].file_info_indexes[j]); + reader.read(entry_count); + for (uint32_t i = 0; i < entry_count; ++i) { + mf->results.emplace_back(); + auto& entry = mf->results.back(); + + uint32_t file_info_count; + reader.read(file_info_count); + for (uint32_t j = 0; j < file_info_count; ++j) { + uint32_t file_info_index; + reader.read(file_info_index); + entry.file_info_indexes.push_back(file_info_index); } - READ_BYTES(mf->results[i].name.bytes, DIGEST_SIZE); + reader.read(entry.name.bytes, DIGEST_SIZE); } - actual_checksum = checksum.digest(); - READ_UINT64(expected_checksum); - if (actual_checksum == expected_checksum) { - success = true; - } else { - *errmsg = format("Incorrect checksum (actual %016llx, expected %016llx)", - (unsigned long long)actual_checksum, - (unsigned long long)expected_checksum); + uint64_t actual_checksum = checksum.digest(); + uint64_t expected_checksum; + reader.read(expected_checksum); + if (actual_checksum != expected_checksum) { + throw Error( + fmt::format("Incorrect checksum (actual 0x{:016x}, expected 0x{:016x})", + actual_checksum, + expected_checksum)); } -out: - if (decompressor && !decompressor->free(decompr_state)) { - success = false; - } - if (f) { - fclose(f); - } - if (!success) { - if (!*errmsg) { - *errmsg = x_strdup("Corrupt manifest file"); - } - free_manifest(mf); - mf = NULL; - } + reader.finalize(); return mf; } -#define WRITE_BYTES(buf, length) \ - do { \ - if (!compressor->write(compr_state, buf, length)) { \ - goto out; \ - } \ - } while (false) - -#define WRITE_UINT16(var) \ - do { \ - char buf_[2]; \ - BYTES_FROM_UINT16(buf_, (var)); \ - WRITE_BYTES(buf_, sizeof(buf_)); \ - } while (false) - -#define WRITE_UINT32(var) \ - do { \ - char buf_[4]; \ - BYTES_FROM_UINT32(buf_, (var)); \ - WRITE_BYTES(buf_, sizeof(buf_)); \ - } while (false) - -#define WRITE_INT64(var) \ - do { \ - char buf_[8]; \ - BYTES_FROM_INT64(buf_, (var)); \ - WRITE_BYTES(buf_, sizeof(buf_)); \ - } while (false) - -#define WRITE_UINT64(var) \ - do { \ - char buf_[8]; \ - BYTES_FROM_UINT64(buf_, (var)); \ - WRITE_BYTES(buf_, sizeof(buf_)); \ - } while (false) - static bool -write_manifest(FILE* f, const struct manifest* mf) +write_manifest(const std::string& path, const ManifestData& mf) { - int ret = false; - - uint64_t content_size = COMMON_HEADER_SIZE; + uint64_t content_size = 15; content_size += 4; // n_files - for (size_t i = 0; i < mf->n_files; i++) { - content_size += 2 + mf->files[i].path_len; + for (size_t i = 0; i < mf.files.size(); ++i) { + content_size += 2 + mf.files[i].length(); } content_size += 4; // n_file_infos - content_size += mf->n_file_infos * (4 + DIGEST_SIZE + 8 + 8 + 8); + content_size += mf.file_infos.size() * (4 + DIGEST_SIZE + 8 + 8 + 8); content_size += 4; // n_results - for (size_t i = 0; i < mf->n_results; i++) { + for (size_t i = 0; i < mf.results.size(); ++i) { content_size += 4; // n_file_info_indexes - content_size += mf->results[i].n_file_info_indexes * 4; + content_size += mf.results[i].file_info_indexes.size() * 4; content_size += DIGEST_SIZE; } content_size += 8; // checksum Checksum checksum; - struct common_header header; - struct compressor* compressor; - struct compr_state* compr_state; - if (!common_header_initialize_for_writing(&header, - f, - MANIFEST_MAGIC, - MANIFEST_VERSION, - compression_type_from_config(), - compression_level_from_config(), - content_size, - checksum, - &compressor, - &compr_state)) { - goto out; - } - - WRITE_UINT32(mf->n_files); - for (uint32_t i = 0; i < mf->n_files; i++) { - WRITE_UINT16(mf->files[i].path_len); - WRITE_BYTES(mf->files[i].path, mf->files[i].path_len); - } - - WRITE_UINT32(mf->n_file_infos); - for (uint32_t i = 0; i < mf->n_file_infos; i++) { - WRITE_UINT32(mf->file_infos[i].index); - WRITE_BYTES(mf->file_infos[i].digest.bytes, DIGEST_SIZE); - WRITE_UINT64(mf->file_infos[i].fsize); - WRITE_INT64(mf->file_infos[i].mtime); - WRITE_INT64(mf->file_infos[i].ctime); - } - - WRITE_UINT32(mf->n_results); - for (uint32_t i = 0; i < mf->n_results; i++) { - WRITE_UINT32(mf->results[i].n_file_info_indexes); - for (uint32_t j = 0; j < mf->results[i].n_file_info_indexes; j++) { - WRITE_UINT32(mf->results[i].file_info_indexes[j]); + AtomicFile atomic_manifest_file(path, AtomicFile::Mode::binary); + CacheEntryWriter writer(atomic_manifest_file.stream(), + k_manifest_magic, + k_manifest_version, + Compression::type_from_config(), + Compression::level_from_config(), + content_size, + checksum); + writer.write<uint32_t>(mf.files.size()); + for (uint32_t i = 0; i < mf.files.size(); ++i) { + writer.write<uint16_t>(mf.files[i].length()); + writer.write(mf.files[i].data(), mf.files[i].length()); + } + + writer.write<uint32_t>(mf.file_infos.size()); + for (uint32_t i = 0; i < mf.file_infos.size(); ++i) { + writer.write<uint32_t>(mf.file_infos[i].index); + writer.write(mf.file_infos[i].digest.bytes, DIGEST_SIZE); + writer.write(mf.file_infos[i].fsize); + writer.write(mf.file_infos[i].mtime); + writer.write(mf.file_infos[i].ctime); + } + + writer.write<uint32_t>(mf.results.size()); + for (uint32_t i = 0; i < mf.results.size(); ++i) { + writer.write<uint32_t>(mf.results[i].file_info_indexes.size()); + for (uint32_t j = 0; j < mf.results[i].file_info_indexes.size(); ++j) { + writer.write(mf.results[i].file_info_indexes[j]); } - WRITE_BYTES(mf->results[i].name.bytes, DIGEST_SIZE); + writer.write(mf.results[i].name.bytes, DIGEST_SIZE); } - WRITE_UINT64(checksum.digest()); - - ret = compressor->free(compr_state); - -out: - if (!ret) { - cc_log("Error writing to manifest file"); - } - return ret; + writer.write(checksum.digest()); + writer.finalize(); + atomic_manifest_file.commit(); + return true; } static bool verify_result(const Config& config, - struct manifest* mf, - struct result* result, - std::unordered_map<std::string, file_stats>* stated_files, - std::unordered_map<std::string, digest>* hashed_files) + const ManifestData& mf, + const ResultEntry& result, + std::unordered_map<std::string, FileStats>& stated_files, + std::unordered_map<std::string, digest>& hashed_files) { - for (uint32_t i = 0; i < result->n_file_info_indexes; i++) { - struct FileInfo* fi = &mf->file_infos[result->file_info_indexes[i]]; - char* path = mf->files[fi->index].path; - auto stated_files_iter = stated_files->find(path); - if (stated_files_iter == stated_files->end()) { + for (uint32_t i = 0; i < result.file_info_indexes.size(); ++i) { + const auto& fi = mf.file_infos[result.file_info_indexes[i]]; + const auto& path = mf.files[fi.index]; + + auto stated_files_iter = stated_files.find(path); + if (stated_files_iter == stated_files.end()) { struct stat file_stat; - if (x_stat(path, &file_stat) != 0) { + if (x_stat(path.c_str(), &file_stat) != 0) { return false; } - file_stats st; + FileStats st; st.size = file_stat.st_size; st.mtime = file_stat.st_mtime; st.ctime = file_stat.st_ctime; - stated_files_iter = stated_files->emplace(path, st).first; + stated_files_iter = stated_files.emplace(path, st).first; } - const file_stats& fs = stated_files_iter->second; + const FileStats& fs = stated_files_iter->second; - if (fi->fsize != fs.size) { + if (fi.fsize != fs.size) { return false; } @@ -482,35 +417,36 @@ verify_result(const Config& config, // and will error out if that header is later used without rebuilding. if ((guessed_compiler == GUESSED_CLANG || guessed_compiler == GUESSED_UNKNOWN) - && output_is_precompiled_header && fi->mtime != fs.mtime) { - cc_log("Precompiled header includes %s, which has a new mtime", path); + && output_is_precompiled_header && fi.mtime != fs.mtime) { + cc_log("Precompiled header includes %s, which has a new mtime", + path.c_str()); return false; } if (config.sloppiness() & SLOPPY_FILE_STAT_MATCHES) { if (!(config.sloppiness() & SLOPPY_FILE_STAT_MATCHES_CTIME)) { - if (fi->mtime == fs.mtime && fi->ctime == fs.ctime) { - cc_log("mtime/ctime hit for %s", path); + if (fi.mtime == fs.mtime && fi.ctime == fs.ctime) { + cc_log("mtime/ctime hit for %s", path.c_str()); continue; } else { - cc_log("mtime/ctime miss for %s", path); + cc_log("mtime/ctime miss for %s", path.c_str()); } } else { - if (fi->mtime == fs.mtime) { - cc_log("mtime hit for %s", path); + if (fi.mtime == fs.mtime) { + cc_log("mtime hit for %s", path.c_str()); continue; } else { - cc_log("mtime miss for %s", path); + cc_log("mtime miss for %s", path.c_str()); } } } - auto hashed_files_iter = hashed_files->find(path); - if (hashed_files_iter == hashed_files->end()) { + auto hashed_files_iter = hashed_files.find(path); + if (hashed_files_iter == hashed_files.end()) { struct hash* hash = hash_init(); - int ret = hash_source_code_file(config, hash, path); + int ret = hash_source_code_file(config, hash, path.c_str()); if (ret & HASH_SOURCE_CODE_ERROR) { - cc_log("Failed hashing %s", path); + cc_log("Failed hashing %s", path.c_str()); hash_free(hash); return false; } @@ -522,10 +458,10 @@ verify_result(const Config& config, digest actual; hash_result_as_bytes(hash, &actual); hash_free(hash); - hashed_files_iter = hashed_files->emplace(path, actual).first; + hashed_files_iter = hashed_files.emplace(path, actual).first; } - if (!digests_equal(&fi->digest, &hashed_files_iter->second)) { + if (!digests_equal(&fi.digest, &hashed_files_iter->second)) { return false; } } @@ -533,196 +469,68 @@ verify_result(const Config& config, return true; } -static void -create_file_index_map( - struct file* files, - uint32_t len, - std::unordered_map<std::string, uint32_t /*index*/>* mf_files) -{ - for (uint32_t i = 0; i < len; i++) { - mf_files->emplace(files[i].path, i); - } -} - -static void -create_file_info_index_map( - struct FileInfo* infos, - uint32_t len, - std::unordered_map<FileInfo, uint32_t /*index*/>* mf_file_infos) -{ - for (uint32_t i = 0; i < len; i++) { - mf_file_infos->emplace(infos[i], i); - } -} - -static uint32_t -get_include_file_index( - struct manifest* mf, - const std::string& path, - const std::unordered_map<std::string, uint32_t>& mf_files) -{ - auto it = mf_files.find(path); - if (it != mf_files.end()) { - return it->second; - } - - uint32_t n = mf->n_files; - mf->files = static_cast<file*>(x_realloc(mf->files, (n + 1) * sizeof(file))); - mf->n_files++; - mf->files[n].path_len = path.size(); - mf->files[n].path = x_strdup(path.c_str()); - return n; -} - -static uint32_t -get_file_info_index(struct manifest* mf, - const std::string& path, - const digest& digest, - const std::unordered_map<std::string, uint32_t>& mf_files, - const std::unordered_map<FileInfo, uint32_t>& mf_file_infos) -{ - struct FileInfo fi; - fi.index = get_include_file_index(mf, path, mf_files); - fi.digest = digest; - - // file_stat.st_{m,c}time has a resolution of 1 second, so we can cache the - // file's mtime and ctime only if they're at least one second older than - // time_of_compilation. - // - // st->ctime may be 0, so we have to check time_of_compilation against - // MAX(mtime, ctime). - - struct stat file_stat; - if (stat(path.c_str(), &file_stat) != -1) { - if (time_of_compilation - > std::max(file_stat.st_mtime, file_stat.st_ctime)) { - fi.mtime = file_stat.st_mtime; - fi.ctime = file_stat.st_ctime; - } else { - fi.mtime = -1; - fi.ctime = -1; - } - fi.fsize = file_stat.st_size; - } else { - fi.mtime = -1; - fi.ctime = -1; - fi.fsize = 0; - } - - auto it = mf_file_infos.find(fi); - if (it != mf_file_infos.end()) { - return it->second; - } - - uint32_t n = mf->n_file_infos; - mf->file_infos = static_cast<FileInfo*>( - x_realloc(mf->file_infos, (n + 1) * sizeof(FileInfo))); - mf->n_file_infos++; - mf->file_infos[n] = fi; - return n; -} - -static void -add_file_info_indexes( - uint32_t* indexes, - uint32_t size, - struct manifest* mf, - const std::unordered_map<std::string, digest>& included_files) -{ - if (size == 0) { - return; - } - - std::unordered_map<std::string, uint32_t /*index*/> mf_files; - create_file_index_map(mf->files, mf->n_files, &mf_files); - - std::unordered_map<FileInfo, uint32_t /*index*/> mf_file_infos; - create_file_info_index_map(mf->file_infos, mf->n_file_infos, &mf_file_infos); - - uint32_t i = 0; - for (const auto& item : included_files) { - const auto& path = item.first; - const auto& digest = item.second; - indexes[i] = get_file_info_index(mf, path, digest, mf_files, mf_file_infos); - i++; - } - assert(i == size); -} - -static void -add_result_entry(struct manifest* mf, - struct digest* result_digest, - const std::unordered_map<std::string, digest>& included_files) -{ - uint32_t n_results = mf->n_results; - mf->results = static_cast<result*>( - x_realloc(mf->results, (n_results + 1) * sizeof(result))); - mf->n_results++; - struct result* result = &mf->results[n_results]; - - uint32_t n_fii = included_files.size(); - result->n_file_info_indexes = n_fii; - result->file_info_indexes = - static_cast<uint32_t*>(x_malloc(n_fii * sizeof(uint32_t))); - add_file_info_indexes(result->file_info_indexes, n_fii, mf, included_files); - result->name = *result_digest; -} - // Try to get the result name from a manifest file. Caller frees. Returns NULL // on failure. struct digest* -manifest_get(const Config& config, const char* manifest_path) +manifest_get(const Config& config, const std::string& path) { - char* errmsg; - struct manifest* mf = read_manifest(manifest_path, &errmsg); - if (!mf) { - cc_log("%s", errmsg); - return NULL; + std::unique_ptr<ManifestData> mf; + try { + mf = read_manifest(path); + if (mf) { + // Update modification timestamp to save files from LRU cleanup. + update_mtime(path.c_str()); + } else { + cc_log("No such result file"); + return nullptr; + } + } catch (const Error& e) { + cc_log("Error: %s", e.what()); + return nullptr; } - std::unordered_map<std::string, file_stats> stated_files; + std::unordered_map<std::string, FileStats> stated_files; std::unordered_map<std::string, digest> hashed_files; // Check newest result first since it's a bit more likely to match. struct digest* name = NULL; - for (uint32_t i = mf->n_results; i > 0; i--) { + for (uint32_t i = mf->results.size(); i > 0; i--) { if (verify_result( - config, mf, &mf->results[i - 1], &stated_files, &hashed_files)) { + config, *mf, mf->results[i - 1], stated_files, hashed_files)) { name = static_cast<digest*>(x_malloc(sizeof(digest))); *name = mf->results[i - 1].name; - goto out; + break; } } -out: - free_manifest(mf); - if (name) { - // Update modification timestamp to save files from LRU cleanup. - update_mtime(manifest_path); - } return name; } // Put the result name into a manifest file given a set of included files. // Returns true on success, otherwise false. bool -manifest_put(const char* manifest_path, - struct digest* result_name, +manifest_put(const std::string& path, + const struct digest& result_name, const std::unordered_map<std::string, digest>& included_files) { // We don't bother to acquire a lock when writing the manifest to disk. A // race between two processes will only result in one lost entry, which is // not a big deal, and it's also very unlikely. - char* errmsg; - struct manifest* mf = read_manifest(manifest_path, &errmsg); - if (!mf) { - // New or corrupt file. - mf = create_empty_manifest(); - free(errmsg); // Ignore. + std::unique_ptr<ManifestData> mf; + try { + mf = read_manifest(path); + if (!mf) { + // Manifest file didn't exist. + mf = std::make_unique<ManifestData>(); + } + } catch (const Error& e) { + cc_log("Error: %s", e.what()); + // Manifest file was corrupt, ignore. + mf = std::make_unique<ManifestData>(); } - if (mf->n_results > MAX_MANIFEST_ENTRIES) { + if (mf->results.size() > k_max_manifest_entries) { // Normally, there shouldn't be many result entries in the manifest since // new entries are added only if an include file has changed but not the // source file, and you typically change source files more often than @@ -734,96 +542,71 @@ manifest_put(const char* manifest_path, // LRU order and discarding the old ones. An easy way is to throw away all // entries when there are too many. Let's do that for now. cc_log("More than %u entries in manifest file; discarding", - MAX_MANIFEST_ENTRIES); - free_manifest(mf); - mf = create_empty_manifest(); - } else if (mf->n_file_infos > MAX_MANIFEST_FILE_INFO_ENTRIES) { + k_max_manifest_entries); + mf = std::make_unique<ManifestData>(); + } else if (mf->file_infos.size() > k_max_manifest_file_info_entries) { // Rarely, FileInfo entries can grow large in pathological cases where // many included files change, but the main file does not. This also puts // an upper bound on the number of FileInfo entries. cc_log("More than %u FileInfo entries in manifest file; discarding", - MAX_MANIFEST_FILE_INFO_ENTRIES); - free_manifest(mf); - mf = create_empty_manifest(); + k_max_manifest_file_info_entries); + mf = std::make_unique<ManifestData>(); } - add_result_entry(mf, result_name, included_files); - - int ret = false; - char* tmp_file = format("%s.tmp", manifest_path); - int fd = create_tmp_fd(&tmp_file); - FILE* f = fdopen(fd, "wb"); - if (!f) { - cc_log("Failed to fdopen %s", tmp_file); - goto out; - } - if (write_manifest(f, mf)) { - fclose(f); - f = NULL; - if (x_rename(tmp_file, manifest_path) == 0) { - ret = true; - } else { - cc_log("Failed to rename %s to %s", tmp_file, manifest_path); - } - } else { - cc_log("Failed to write manifest file %s", tmp_file); - } + mf->add_result_entry(result_name, included_files); -out: - if (f) { - fclose(f); - } - if (mf) { - free_manifest(mf); - } - if (tmp_file) { - free(tmp_file); + try { + write_manifest(path, *mf); + return true; + } catch (const Error& e) { + cc_log("Error: %s", e.what()); + return false; } - return ret; } bool -manifest_dump(const char* manifest_path, FILE* stream) +manifest_dump(const std::string& path, FILE* stream) { - char* errmsg; - struct manifest* mf = read_manifest(manifest_path, &errmsg); - if (!mf) { - assert(errmsg); - fprintf(stream, "Error: %s\n", errmsg); - free(errmsg); + std::unique_ptr<ManifestData> mf; + try { + mf = read_manifest(path, stream); + } catch (const Error& e) { + fmt::print(stream, "Error: {}\n", e.what()); return false; } - common_header_dump(&mf->header, stream); + if (!mf) { + fmt::print(stream, "Error: No such file: {}\n", path); + return false; + } - fprintf(stream, "File paths (%u):\n", (unsigned)mf->n_files); - for (unsigned i = 0; i < mf->n_files; ++i) { - fprintf(stream, " %u: %s\n", i, mf->files[i].path); + fmt::print(stream, "File paths ({}):\n", mf->files.size()); + for (unsigned i = 0; i < mf->files.size(); ++i) { + fmt::print(stream, " {}: {}\n", i, mf->files[i]); } - fprintf(stream, "File infos (%u):\n", (unsigned)mf->n_file_infos); - for (unsigned i = 0; i < mf->n_file_infos; ++i) { + fmt::print(stream, "File infos ({}):\n", mf->file_infos.size()); + for (unsigned i = 0; i < mf->file_infos.size(); ++i) { char digest[DIGEST_STRING_BUFFER_SIZE]; - fprintf(stream, " %u:\n", i); - fprintf(stream, " Path index: %u\n", mf->file_infos[i].index); + fmt::print(stream, " {}:\n", i); + fmt::print(stream, " Path index: {}\n", mf->file_infos[i].index); digest_as_string(&mf->file_infos[i].digest, digest); - fprintf(stream, " Hash: %s\n", digest); - fprintf(stream, " File size: %" PRIu64 "\n", mf->file_infos[i].fsize); - fprintf(stream, " Mtime: %lld\n", (long long)mf->file_infos[i].mtime); - fprintf(stream, " Ctime: %lld\n", (long long)mf->file_infos[i].ctime); + fmt::print(stream, " Hash: {}\n", digest); + fmt::print(stream, " File size: {}\n", mf->file_infos[i].fsize); + fmt::print(stream, " Mtime: {}\n", mf->file_infos[i].mtime); + fmt::print(stream, " Ctime: {}\n", mf->file_infos[i].ctime); } - fprintf(stream, "Results (%u):\n", (unsigned)mf->n_results); - for (unsigned i = 0; i < mf->n_results; ++i) { + fmt::print(stream, "Results ({}):\n", mf->results.size()); + for (unsigned i = 0; i < mf->results.size(); ++i) { char name[DIGEST_STRING_BUFFER_SIZE]; - fprintf(stream, " %u:\n", i); - fprintf(stream, " File info indexes:"); - for (unsigned j = 0; j < mf->results[i].n_file_info_indexes; ++j) { - fprintf(stream, " %u", mf->results[i].file_info_indexes[j]); + fmt::print(stream, " {}:\n", i); + fmt::print(stream, " File info indexes:"); + for (unsigned j = 0; j < mf->results[i].file_info_indexes.size(); ++j) { + fmt::print(stream, " {}", mf->results[i].file_info_indexes[j]); } - fprintf(stream, "\n"); + fmt::print(stream, "\n"); digest_as_string(&mf->results[i].name, name); - fprintf(stream, " Name: %s\n", name); + fmt::print(stream, " Name: {}\n", name); } - free_manifest(mf); return true; } diff --git a/src/manifest.hpp b/src/manifest.hpp index 71f8e194..9822c45e 100644 --- a/src/manifest.hpp +++ b/src/manifest.hpp @@ -20,15 +20,18 @@ #include "system.hpp" -#include "Config.hpp" -#include "hashutil.hpp" +#include <string> +#include <unordered_map> -extern const char MANIFEST_MAGIC[4]; -#define MANIFEST_VERSION 2 +class Config; +struct digest; -struct digest* manifest_get(const Config& config, const char* manifest_path); +extern const uint8_t k_manifest_magic[4]; +extern const uint8_t k_manifest_version; + +struct digest* manifest_get(const Config& config, const std::string& path); bool -manifest_put(const char* manifest_path, - struct digest* result_digest, +manifest_put(const std::string& path, + const struct digest& result_name, const std::unordered_map<std::string, digest>& included_files); -bool manifest_dump(const char* manifest_path, FILE* stream); +bool manifest_dump(const std::string& path, FILE* stream); diff --git a/src/result.cpp b/src/result.cpp index b3f3d2e9..ca17a788 100644 --- a/src/result.cpp +++ b/src/result.cpp @@ -18,12 +18,15 @@ #include "result.hpp" +#include "AtomicFile.hpp" +#include "CacheEntryReader.hpp" +#include "CacheEntryWriter.hpp" +#include "Checksum.hpp" #include "Config.hpp" +#include "Error.hpp" +#include "File.hpp" +#include "Util.hpp" #include "ccache.hpp" -#include "common_header.hpp" -#include "compression.hpp" -#include "hash.hpp" -#include "int_bytes_conversion.hpp" // Result data format // ================== @@ -85,501 +88,342 @@ extern char* stats_file; -const char RESULT_MAGIC[4] = {'c', 'C', 'r', 'S'}; +const uint8_t k_result_magic[4] = {'c', 'C', 'r', 'S'}; +const uint8_t k_result_version = 1; +const std::string k_result_stderr_name = "<stderr>"; -enum { - // File data stored inside the result file. - EMBEDDED_FILE_MARKER = 0, +// File data stored inside the result file. +const uint8_t k_embedded_file_marker = 0; - // File stored as-is in the file system. - RAW_FILE_MARKER = 1 -}; +// File stored as-is in the file system. +const uint8_t k_raw_file_marker = 1; -struct result_file -{ - char* suffix; - char* path; - uint64_t size; -}; - -struct result_files -{ - uint32_t n_files; - struct result_file* files; - uint64_t* sizes; -}; - -typedef bool (*read_entry_fn)(struct decompressor* decompressor, - struct decompr_state* decompr_state, - const char* result_path_in_cache, - uint32_t entry_number, - const struct result_files* list, - FILE* dump_stream); - -typedef bool (*write_entry_fn)(struct compressor* compressor, - struct compr_state* compr_state, - const char* result_path_in_cache, - uint32_t entry_number, - const struct result_file* file); - -struct result_files* -result_files_init(void) -{ - auto list = static_cast<result_files*>(x_malloc(sizeof(result_files))); - list->n_files = 0; - list->files = NULL; - list->sizes = NULL; - - return list; -} - -void -result_files_add(struct result_files* list, - const char* path, - const char* suffix) -{ - uint32_t n = list->n_files; - list->files = static_cast<result_file*>( - x_realloc(list->files, (n + 1) * sizeof(result_file))); - list->sizes = - static_cast<uint64_t*>(x_realloc(list->sizes, (n + 1) * sizeof(uint64_t))); - struct result_file* f = &list->files[list->n_files]; - list->n_files++; - - struct stat st; - x_stat(path, &st); - - f->suffix = x_strdup(suffix); - f->path = x_strdup(path); - f->size = st.st_size; -} - -void -result_files_free(struct result_files* list) -{ - for (uint32_t i = 0; i < list->n_files; i++) { - free(list->files[i].suffix); - free(list->files[i].path); - } - free(list->files); - list->files = NULL; - free(list->sizes); - list->sizes = NULL; +using ReadEntryFunction = void (*)(CacheEntryReader& reader, + const std::string& result_path_in_cache, + uint32_t entry_number, + const ResultFileMap* result_file_map, + FILE* dump_stream); - free(list); -} - -#define READ_BYTES(buf, length) \ - do { \ - if (!decompressor->read(decompr_state, buf, length)) { \ - goto out; \ - } \ - } while (false) - -#define READ_BYTE(var) READ_BYTES(&var, 1) +using WriteEntryFunction = + void (*)(CacheEntryWriter& writer, + const std::string& result_path_in_cache, + uint32_t entry_number, + const ResultFileMap::value_type& suffix_and_path); -#define READ_UINT64(var) \ - do { \ - char buf_[8]; \ - READ_BYTES(buf_, sizeof(buf_)); \ - (var) = UINT64_FROM_BYTES(buf_); \ - } while (false) - -static bool -read_embedded_file_entry(struct decompressor* decompressor, - struct decompr_state* decompr_state, - const char* result_path_in_cache, +static void +read_embedded_file_entry(CacheEntryReader& reader, + const std::string& /*result_path_in_cache*/, uint32_t entry_number, - const struct result_files* list, + const ResultFileMap* result_file_map, FILE* dump_stream) { - (void)result_path_in_cache; - bool found = false; - bool success = false; - FILE* subfile = NULL; - uint8_t suffix_len; - READ_BYTE(suffix_len); + reader.read(suffix_len); - char suffix[256 + 1]; - READ_BYTES(suffix, suffix_len); - suffix[suffix_len] = '\0'; + char suffix[256]; + reader.read(suffix, suffix_len); uint64_t file_len; - READ_UINT64(file_len); + reader.read(file_len); + bool content_read = false; if (dump_stream) { - fprintf(dump_stream, - "Embedded file #%u: %s (%" PRIu64 " bytes)\n", - entry_number, - suffix, - file_len); + fmt::print(dump_stream, + "Embedded file #{}: {} ({} bytes)\n", + entry_number, + suffix, + file_len); } else { cc_log("Retrieving embedded file #%u %s (%llu bytes)", entry_number, suffix, (unsigned long long)file_len); - for (uint32_t i = 0; i < list->n_files; i++) { - if (str_eq(suffix, list->files[i].suffix)) { - found = true; + std::string suffix_str(suffix, suffix_len); + const auto it = result_file_map->find(suffix_str); + if (it != result_file_map->end()) { + content_read = true; - cc_log("Copying to %s", list->files[i].path); + const auto& path = it->second; + cc_log("Copying to %s", path.c_str()); - subfile = fopen(list->files[i].path, "wb"); - if (!subfile) { - cc_log("Failed to open %s for writing", list->files[i].path); - goto out; - } - char buf[READ_BUFFER_SIZE]; - size_t remain = file_len; - while (remain > 0) { - size_t n = std::min(remain, sizeof(buf)); - READ_BYTES(buf, n); - if (fwrite(buf, 1, n, subfile) != n) { - goto out; - } - remain -= n; + File subfile(path, "wb"); + if (!subfile) { + throw Error(fmt::format( + "Failed to open {} for writing: {}", path, strerror(errno))); + } + uint8_t buf[READ_BUFFER_SIZE]; + size_t remain = file_len; + while (remain > 0) { + size_t n = std::min(remain, sizeof(buf)); + reader.read(buf, n); + if (fwrite(buf, n, 1, subfile.get()) != 1) { + throw Error(fmt::format("Failed to write to {}", path)); } - fclose(subfile); - subfile = NULL; - - break; + remain -= n; } } } - if (!found) { + if (!content_read) { // Discard the file data. - char buf[READ_BUFFER_SIZE]; + uint8_t buf[READ_BUFFER_SIZE]; size_t remain = file_len; while (remain > 0) { size_t n = std::min(remain, sizeof(buf)); - READ_BYTES(buf, n); + reader.read(buf, n); remain -= n; } } - - success = true; - -out: - if (subfile) { - fclose(subfile); - } - - return success; } -static char* -get_raw_file_path(const char* result_path_in_cache, uint32_t entry_number) +static std::string +get_raw_file_path(const std::string& result_path_in_cache, + uint32_t entry_number) { - return format("%.*s_%u.raw", - (int)strlen(result_path_in_cache) - 7, // .result - result_path_in_cache, - entry_number); + return fmt::format("{:{}}_{}.raw", + result_path_in_cache.c_str(), + result_path_in_cache.length() - 7, // .result + entry_number); } static bool -copy_raw_file(const char* source, const char* dest, bool to_cache) +copy_raw_file(const std::string& source, const std::string& dest, bool to_cache) { if (g_config.file_clone()) { - cc_log("Cloning %s to %s", source, dest); - if (clone_file(source, dest, to_cache)) { + cc_log("Cloning %s to %s", source.c_str(), dest.c_str()); + if (clone_file(source.c_str(), dest.c_str(), to_cache)) { return true; } cc_log("Failed to clone: %s", strerror(errno)); } if (g_config.hard_link()) { - x_try_unlink(dest); - cc_log("Hard linking %s to %s", source, dest); - int ret = link(source, dest); + x_try_unlink(dest.c_str()); + cc_log("Hard linking %s to %s", source.c_str(), dest.c_str()); + int ret = link(source.c_str(), dest.c_str()); if (ret == 0) { return true; } cc_log("Failed to hard link: %s", strerror(errno)); } - cc_log("Copying %s to %s", source, dest); - return copy_file(source, dest, to_cache); + cc_log("Copying %s to %s", source.c_str(), dest.c_str()); + return copy_file(source.c_str(), dest.c_str(), to_cache); } -static bool -read_raw_file_entry(struct decompressor* decompressor, - struct decompr_state* decompr_state, - const char* result_path_in_cache, +static void +read_raw_file_entry(CacheEntryReader& reader, + const std::string& result_path_in_cache, uint32_t entry_number, - const struct result_files* list, - FILE* dump_stream) + const ResultFileMap* result_file_map, + std::FILE* dump_stream) { - bool success = false; - char* raw_path = get_raw_file_path(result_path_in_cache, entry_number); - uint8_t suffix_len; - READ_BYTE(suffix_len); + reader.read(suffix_len); - char suffix[256 + 1]; - READ_BYTES(suffix, suffix_len); - suffix[suffix_len] = '\0'; + char suffix[256]; + reader.read(suffix, suffix_len); uint64_t file_len; - READ_UINT64(file_len); + reader.read(file_len); if (dump_stream) { - fprintf(dump_stream, - "Raw file #%u: %s (%" PRIu64 " bytes)\n", - entry_number, - suffix, - file_len); + fmt::print(dump_stream, + "Raw file #{}: {} ({} bytes)\n", + entry_number, + suffix, + file_len); } else { cc_log("Retrieving raw file #%u %s (%llu bytes)", entry_number, suffix, (unsigned long long)file_len); + auto raw_path = get_raw_file_path(result_path_in_cache, entry_number); struct stat st; - if (x_stat(raw_path, &st) != 0) { - goto out; + if (x_stat(raw_path.c_str(), &st) != 0) { + throw Error( + fmt::format("Failed to stat {}: {}", raw_path, strerror(errno))); } if ((uint64_t)st.st_size != file_len) { - cc_log("Bad file size of %s (actual %llu bytes, expected %llu bytes)", - raw_path, - (unsigned long long)st.st_size, - (unsigned long long)file_len); - goto out; + throw Error( + fmt::format("Bad file size of {} (actual {} bytes, expected {} bytes)", + raw_path, + st.st_size, + file_len)); } - for (uint32_t i = 0; i < list->n_files; i++) { - if (str_eq(suffix, list->files[i].suffix)) { - if (!copy_raw_file(raw_path, list->files[i].path, false)) { - goto out; - } - // Update modification timestamp to save the file from LRU cleanup - // (and, if hard-linked, to make the object file newer than the source - // file). - update_mtime(raw_path); - break; + std::string suffix_str(suffix, suffix_len); + const auto it = result_file_map->find(suffix_str); + if (it != result_file_map->end()) { + const auto& dest_path = it->second; + if (!copy_raw_file(raw_path, dest_path, false)) { + throw Error( + fmt::format("Failed to copy raw file {} to {}", raw_path, dest_path)); } + // Update modification timestamp to save the file from LRU cleanup + // (and, if hard-linked, to make the object file newer than the source + // file). + update_mtime(raw_path.c_str()); } } - - success = true; - -out: - free(raw_path); - return success; } static bool -read_result(const char* path, - struct result_files* list, - FILE* dump_stream, - char** errmsg) +read_result(const std::string& path, + const ResultFileMap* result_file_map, + FILE* dump_stream) { - *errmsg = NULL; - bool cache_miss = false; - bool success = false; - struct decompressor* decompressor = NULL; - struct decompr_state* decompr_state = NULL; - Checksum checksum; - - FILE* f = fopen(path, "rb"); - if (!f) { - cache_miss = true; - goto out; + File file(path, "rb"); + if (!file) { + // Cache miss. + return false; } - struct common_header header; - if (!common_header_initialize_for_reading(&header, - f, - RESULT_MAGIC, - RESULT_VERSION, - &decompressor, - &decompr_state, - &checksum, - errmsg)) { - goto out; - } + Checksum checksum; + CacheEntryReader reader( + file.get(), k_result_magic, k_result_version, &checksum); if (dump_stream) { - common_header_dump(&header, dump_stream); + reader.dump_header(dump_stream); } uint8_t n_entries; - READ_BYTE(n_entries); + reader.read(n_entries); uint32_t i; - for (i = 0; i < n_entries; i++) { + for (i = 0; i < n_entries; ++i) { uint8_t marker; - READ_BYTE(marker); + reader.read(marker); - read_entry_fn read_entry; + ReadEntryFunction read_entry; switch (marker) { - case EMBEDDED_FILE_MARKER: + case k_embedded_file_marker: read_entry = read_embedded_file_entry; break; - case RAW_FILE_MARKER: + case k_raw_file_marker: read_entry = read_raw_file_entry; break; default: - *errmsg = format("Unknown entry type: %u", marker); - goto out; + throw Error(fmt::format("Unknown entry type: {}", marker)); } - if (!read_entry(decompressor, decompr_state, path, i, list, dump_stream)) { - goto out; - } + read_entry(reader, path, i, result_file_map, dump_stream); } if (i != n_entries) { - *errmsg = format("Too few entries (read %u, expected %u)", i, n_entries); - goto out; + throw Error( + fmt::format("Too few entries (read {}, expected {})", i, n_entries)); } - { - uint64_t actual_checksum = checksum.digest(); - uint64_t expected_checksum; - READ_UINT64(expected_checksum); - - if (actual_checksum == expected_checksum) { - success = true; - } else { - *errmsg = format("Incorrect checksum (actual %016llx, expected %016llx)", - (unsigned long long)actual_checksum, - (unsigned long long)expected_checksum); - } + uint64_t actual_checksum = checksum.digest(); + uint64_t expected_checksum; + reader.read(expected_checksum); + if (actual_checksum != expected_checksum) { + throw Error( + fmt::format("Incorrect checksum (actual 0x{:016x}, expected 0x{:016x})", + actual_checksum, + expected_checksum)); } -out: - if (decompressor && !decompressor->free(decompr_state)) { - success = false; - } - if (f) { - fclose(f); - } - if (!success && !cache_miss && !*errmsg) { - *errmsg = x_strdup("Corrupt result"); - } - return success; + reader.finalize(); + return true; } -#define WRITE_BYTES(buf, length) \ - do { \ - if (!compressor->write(compr_state, buf, length)) { \ - goto error; \ - } \ - } while (false) - -#define WRITE_BYTE(var) \ - do { \ - char ch_ = var; \ - WRITE_BYTES(&ch_, 1); \ - } while (false) - -#define WRITE_UINT64(var) \ - do { \ - char buf_[8]; \ - BYTES_FROM_UINT64(buf_, (var)); \ - WRITE_BYTES(buf_, sizeof(buf_)); \ - } while (false) - -static bool -write_embedded_file_entry(struct compressor* compressor, - struct compr_state* compr_state, - const char* result_path_in_cache, +static void +write_embedded_file_entry(CacheEntryWriter& writer, + const std::string& /*result_path_in_cache*/, uint32_t entry_number, - const struct result_file* file) + const ResultFileMap::value_type& suffix_and_path) { - (void)result_path_in_cache; - bool success = false; - size_t suffix_len; - FILE* f; - char buf[READ_BUFFER_SIZE]; - size_t remain; + const auto& suffix = suffix_and_path.first; + const auto& source_path = suffix_and_path.second; + + uint64_t source_file_size; + if (!Util::get_file_size(source_path, source_file_size)) { + throw Error( + fmt::format("Failed to stat {}: {}", source_path, strerror(errno))); + } cc_log("Storing embedded file #%u %s (%llu bytes) from %s", entry_number, - file->suffix, - (unsigned long long)file->size, - file->path); - - WRITE_BYTE(EMBEDDED_FILE_MARKER); - suffix_len = strlen(file->suffix); - WRITE_BYTE(suffix_len); - WRITE_BYTES(file->suffix, suffix_len); - WRITE_UINT64(file->size); - - f = fopen(file->path, "rb"); - if (!f) { - cc_log("Failed to open %s for reading", file->path); - goto error; + suffix.c_str(), + (unsigned long long)source_file_size, + source_path.c_str()); + + writer.write<uint8_t>(k_embedded_file_marker); + writer.write<uint8_t>(suffix.length()); + writer.write(suffix.data(), suffix.length()); + writer.write(source_file_size); + + File file(source_path, "rb"); + if (!file) { + throw Error(fmt::format("Failed to open {} for reading", source_path)); } - remain = file->size; + + size_t remain = source_file_size; while (remain > 0) { + uint8_t buf[READ_BUFFER_SIZE]; size_t n = std::min(remain, sizeof(buf)); - if (fread(buf, 1, n, f) != n) { - goto error; + if (fread(buf, n, 1, file.get()) != 1) { + throw Error(fmt::format("Error reading from {}", source_path)); } - WRITE_BYTES(buf, n); + writer.write(buf, n); remain -= n; } - fclose(f); - - success = true; - -error: - return success; } -static bool -write_raw_file_entry(struct compressor* compressor, - struct compr_state* compr_state, - const char* result_path_in_cache, +static void +write_raw_file_entry(CacheEntryWriter& writer, + const std::string& result_path_in_cache, uint32_t entry_number, - const struct result_file* file) + const ResultFileMap::value_type& suffix_and_path) { - bool success = false; - size_t suffix_len; - char* raw_file; - struct stat old_stat; - bool old_existed; - struct stat new_stat; - bool new_exists; + const auto& suffix = suffix_and_path.first; + const auto& source_path = suffix_and_path.second; + + uint64_t source_file_size; + if (!Util::get_file_size(source_path, source_file_size)) { + throw Error( + fmt::format("Failed to stat {}: {}", source_path, strerror(errno))); + } + size_t old_size; size_t new_size; cc_log("Storing raw file #%u %s (%llu bytes) from %s", entry_number, - file->suffix, - (unsigned long long)file->size, - file->path); - - WRITE_BYTE(RAW_FILE_MARKER); - suffix_len = strlen(file->suffix); - WRITE_BYTE(suffix_len); - WRITE_BYTES(file->suffix, suffix_len); - WRITE_UINT64(file->size); - - raw_file = get_raw_file_path(result_path_in_cache, entry_number); - old_existed = stat(raw_file, &old_stat) == 0; - success = copy_raw_file(file->path, raw_file, true); - new_exists = stat(raw_file, &new_stat) == 0; - free(raw_file); + suffix.c_str(), + (unsigned long long)source_file_size, + source_path.c_str()); + + writer.write<uint8_t>(k_raw_file_marker); + writer.write<uint8_t>(suffix.length()); + writer.write(suffix.data(), suffix.length()); + writer.write(source_file_size); + + auto raw_file = get_raw_file_path(result_path_in_cache, entry_number); + struct stat old_stat; + bool old_existed = stat(raw_file.c_str(), &old_stat) == 0; + if (!copy_raw_file(source_path, raw_file, true)) { + throw Error( + fmt::format("Failed to store {} as raw file {}", source_path, raw_file)); + } + struct stat new_stat; + bool new_exists = stat(raw_file.c_str(), &new_stat) == 0; old_size = old_existed ? file_size(&old_stat) : 0; new_size = new_exists ? file_size(&new_stat) : 0; stats_update_size(stats_file, new_size - old_size, (new_exists ? 1 : 0) - (old_existed ? 1 : 0)); - -error: - return success; } static bool -should_store_raw_file(const char* suffix) +should_store_raw_file(const std::string& suffix) { if (!g_config.file_clone() && !g_config.hard_link()) { return false; @@ -599,132 +443,105 @@ should_store_raw_file(const char* suffix) // could be fixed by letting read_raw_file_entry refuse to hard link .d // files, but it's easier to simply always store them embedded. This will // also save i-nodes in the cache. - return !str_eq(suffix, RESULT_STDERR_NAME) && !str_eq(suffix, ".d"); + return suffix != k_result_stderr_name && suffix != ".d"; } -static bool -write_result(const struct result_files* list, - struct compressor* compressor, - struct compr_state* compr_state, - Checksum& checksum, - const char* result_path_in_cache) +static void +write_result(const std::string& path, const ResultFileMap& result_file_map) { - WRITE_BYTE(list->n_files); - - for (uint32_t i = 0; i < list->n_files; i++) { - write_entry_fn write_entry = should_store_raw_file(list->files[i].suffix) - ? write_raw_file_entry - : write_embedded_file_entry; - if (!write_entry( - compressor, compr_state, result_path_in_cache, i, &list->files[i])) { - goto error; + uint64_t content_size = 15; + content_size += 1; // n_entries + for (const auto& pair : result_file_map) { + const auto& suffix = pair.first; + const auto& result_file = pair.second; + uint64_t source_file_size; + if (!Util::get_file_size(result_file, source_file_size)) { + throw Error( + fmt::format("Failed to stat {}: {}", result_file, strerror(errno))); } + content_size += 1; // embedded_file_marker + content_size += 1; // suffix_len + content_size += suffix.length(); // suffix + content_size += 8; // data_len + content_size += source_file_size; // data } + content_size += 8; // checksum - WRITE_UINT64(checksum.digest()); - - return true; + Checksum checksum; + AtomicFile atomic_result_file(path, AtomicFile::Mode::binary); + CacheEntryWriter writer(atomic_result_file.stream(), + k_result_magic, + k_result_version, + Compression::type_from_config(), + Compression::level_from_config(), + content_size, + checksum); + + writer.write<uint8_t>(result_file_map.size()); + + size_t entry_number = 0; + for (const auto& pair : result_file_map) { + const auto& suffix = pair.first; + WriteEntryFunction write_entry = should_store_raw_file(suffix) + ? write_raw_file_entry + : write_embedded_file_entry; + write_entry(writer, path, entry_number, pair); + ++entry_number; + } -error: - cc_log("Error writing to result file"); - return false; + writer.write(checksum.digest()); + writer.finalize(); + atomic_result_file.commit(); } bool -result_get(const char* path, struct result_files* list) +result_get(const std::string& path, const ResultFileMap& result_file_map) { - cc_log("Getting result %s", path); - - char* errmsg; - bool success = read_result(path, list, NULL, &errmsg); - if (success) { - // Update modification timestamp to save files from LRU cleanup. - update_mtime(path); - } else if (errmsg) { - cc_log("Error: %s", errmsg); - free(errmsg); - } else { - cc_log("No such result file"); + cc_log("Getting result %s", path.c_str()); + + try { + bool cache_hit = read_result(path, &result_file_map, nullptr); + if (cache_hit) { + // Update modification timestamp to save files from LRU cleanup. + update_mtime(path.c_str()); + } else { + cc_log("No such result file"); + } + return cache_hit; + } catch (const Error& e) { + cc_log("Error: %s", e.what()); + return false; } - return success; } bool -result_put(const char* path, struct result_files* list) +result_put(const std::string& path, const ResultFileMap& result_file_map) { - bool ret = false; - Checksum checksum; - bool ok; - uint64_t content_size; - - char* tmp_file = format("%s.tmp", path); - int fd = create_tmp_fd(&tmp_file); - FILE* f = fdopen(fd, "wb"); - if (!f) { - cc_log("Failed to fdopen %s", tmp_file); - goto out; - } - - content_size = COMMON_HEADER_SIZE; - content_size += 1; // n_entries - for (uint32_t i = 0; i < list->n_files; i++) { - content_size += 1; // embedded_file_marker - content_size += 1; // suffix_len - content_size += strlen(list->files[i].suffix); // suffix - content_size += 8; // data_len - content_size += list->files[i].size; // data - } - content_size += 8; // checksum - - struct common_header header; - struct compressor* compressor; - struct compr_state* compr_state; - if (!common_header_initialize_for_writing(&header, - f, - RESULT_MAGIC, - RESULT_VERSION, - compression_type_from_config(), - compression_level_from_config(), - content_size, - checksum, - &compressor, - &compr_state)) { - goto out; - } + cc_log("Storing result %s", path.c_str()); - ok = write_result(list, compressor, compr_state, checksum, path) - && compressor->free(compr_state); - if (!ok) { - cc_log("Failed to write result file"); - goto out; - } - - fclose(f); - f = NULL; - if (x_rename(tmp_file, path) == 0) { - ret = true; - } else { - cc_log("Failed to rename %s to %s", tmp_file, path); - } - -out: - free(tmp_file); - if (f) { - fclose(f); + try { + write_result(path, result_file_map); + return true; + } catch (const Error& e) { + cc_log("Error: %s", e.what()); + return false; } - return ret; } bool -result_dump(const char* path, FILE* stream) +result_dump(const std::string& path, FILE* stream) { assert(stream); - char* errmsg; - bool success = read_result(path, NULL, stream, &errmsg); - if (errmsg) { - fprintf(stream, "Error: %s\n", errmsg); - free(errmsg); + try { + if (read_result(path, nullptr, stream)) { + return true; + } else { + fmt::print(stream, "Error: No such file: {}\n", path); + } + } catch (const Error& e) { + fmt::print(stream, "Error: {}\n", e.what()); } - return success; + + return false; } diff --git a/src/result.hpp b/src/result.hpp index e49d407e..79a27c9d 100644 --- a/src/result.hpp +++ b/src/result.hpp @@ -20,19 +20,15 @@ #include "system.hpp" -#include <cstdio> +#include <map> +#include <string> -extern const char RESULT_MAGIC[4]; -#define RESULT_VERSION 1 -#define RESULT_STDERR_NAME "<stderr>" +extern const uint8_t k_result_magic[4]; +extern const uint8_t k_result_version; +extern const std::string k_result_stderr_name; -struct result_files; +typedef std::map<std::string /*suffix*/, std::string /*path*/> ResultFileMap; -struct result_files* result_files_init(void); -void -result_files_add(struct result_files* c, const char* path, const char* suffix); -void result_files_free(struct result_files* c); - -bool result_get(const char* path, struct result_files* list); -bool result_put(const char* path, struct result_files* list); -bool result_dump(const char* path, FILE* stream); +bool result_get(const std::string& path, const ResultFileMap& result_file_map); +bool result_put(const std::string& path, const ResultFileMap& result_file_map); +bool result_dump(const std::string& path, FILE* stream); diff --git a/unittest/main.cpp b/unittest/main.cpp index 9636b461..f5b3b01d 100644 --- a/unittest/main.cpp +++ b/unittest/main.cpp @@ -22,8 +22,6 @@ unsigned suite_args(unsigned); unsigned suite_argument_processing(unsigned); unsigned suite_compopt(unsigned); -unsigned suite_compr_type_none(unsigned); -unsigned suite_compr_type_zstd(unsigned); unsigned suite_conf(unsigned); unsigned suite_counters(unsigned); unsigned suite_hash(unsigned); @@ -36,8 +34,6 @@ const suite_fn k_legacy_suites[] = { &suite_args, &suite_argument_processing, &suite_compopt, - &suite_compr_type_none, - &suite_compr_type_zstd, &suite_counters, &suite_hash, &suite_hashutil, diff --git a/unittest/test_Compression.cpp b/unittest/test_Compression.cpp new file mode 100644 index 00000000..97c2b051 --- /dev/null +++ b/unittest/test_Compression.cpp @@ -0,0 +1,46 @@ +// Copyright (C) 2019 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "../src/Compression.hpp" + +#include <catch.hpp> + +using Catch::Equals; + +TEST_CASE("Compression::level_from_config") +{ + CHECK(Compression::level_from_config() == 0); +} + +TEST_CASE("Compression::type_from_config") +{ + CHECK(Compression::type_from_config() == Compression::Type::zstd); +} + +TEST_CASE("Compression::type_from_int") +{ + CHECK(Compression::type_from_int(0) == Compression::Type::none); + CHECK(Compression::type_from_int(1) == Compression::Type::zstd); + CHECK_THROWS_WITH(Compression::type_from_int(2), Equals("Unknown type: 2")); +} + +TEST_CASE("Compression::type_to_string") +{ + CHECK(Compression::type_to_string(Compression::Type::none) == "none"); + CHECK(Compression::type_to_string(Compression::Type::zstd) == "zstd"); +} diff --git a/unittest/test_NullCompression.cpp b/unittest/test_NullCompression.cpp new file mode 100644 index 00000000..43e0c198 --- /dev/null +++ b/unittest/test_NullCompression.cpp @@ -0,0 +1,64 @@ +// Copyright (C) 2019 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "../src/Compression.hpp" +#include "../src/Compressor.hpp" +#include "../src/Decompressor.hpp" +#include "../src/File.hpp" + +#include <catch.hpp> + +using Catch::Equals; + +TEST_CASE("Compression::Type::none roundtrip") +{ + File f("data.uncompressed", "w"); + auto compressor = + Compressor::create_from_type(Compression::Type::none, f.get(), 1); + CHECK(compressor->actual_compression_level() == 0); + compressor->write("foobar", 6); + compressor->finalize(); + + f.open("data.uncompressed", "r"); + auto decompressor = + Decompressor::create_from_type(Compression::Type::none, f.get()); + + char buffer[4]; + decompressor->read(buffer, 4); + CHECK(memcmp(buffer, "foob", 4) == 0); + + SECTION("Garbage data") + { + // Not reached the end. + CHECK_THROWS_WITH(decompressor->finalize(), + Equals("garbage data at end of uncompressed stream")); + } + + SECTION("Read to end") + { + decompressor->read(buffer, 2); + CHECK(memcmp(buffer, "ar", 2) == 0); + + // Reached the end. + decompressor->finalize(); + + // Nothing left to read. + CHECK_THROWS_WITH(decompressor->read(buffer, 1), + Equals("failed to read from uncompressed stream")); + } +} diff --git a/unittest/test_ZstdCompression.cpp b/unittest/test_ZstdCompression.cpp new file mode 100644 index 00000000..d9973d92 --- /dev/null +++ b/unittest/test_ZstdCompression.cpp @@ -0,0 +1,112 @@ +// Copyright (C) 2019 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "../src/Compression.hpp" +#include "../src/Compressor.hpp" +#include "../src/Decompressor.hpp" +#include "../src/File.hpp" + +#include <catch.hpp> + +using Catch::Equals; + +TEST_CASE("Small Compression::Type::zstd roundtrip") +{ + File f("data.zstd", "w"); + auto compressor = + Compressor::create_from_type(Compression::Type::zstd, f.get(), 1); + CHECK(compressor->actual_compression_level() == 1); + compressor->write("foobar", 6); + compressor->finalize(); + + f.open("data.zstd", "r"); + auto decompressor = + Decompressor::create_from_type(Compression::Type::zstd, f.get()); + + char buffer[4]; + decompressor->read(buffer, 4); + CHECK(memcmp(buffer, "foob", 4) == 0); + + // Not reached the end. + CHECK_THROWS_WITH(decompressor->finalize(), + Equals("garbage data at end of zstd input stream")); + + decompressor->read(buffer, 2); + CHECK(memcmp(buffer, "ar", 2) == 0); + + // Reached the end. + decompressor->finalize(); + + // Nothing left to read. + CHECK_THROWS_WITH(decompressor->read(buffer, 1), + Equals("failed to read from zstd input stream")); +} + +TEST_CASE("Large compressible Compression::Type::zstd roundtrip") +{ + char data[] = "The quick brown fox jumps over the lazy dog"; + + File f("data.zstd", "w"); + auto compressor = + Compressor::create_from_type(Compression::Type::zstd, f.get(), 1); + for (size_t i = 0; i < 1000; i++) { + compressor->write(data, sizeof(data)); + } + compressor->finalize(); + + f.open("data.zstd", "r"); + auto decompressor = + Decompressor::create_from_type(Compression::Type::zstd, f.get()); + + char buffer[sizeof(data)]; + for (size_t i = 0; i < 1000; i++) { + decompressor->read(buffer, sizeof(buffer)); + CHECK(memcmp(buffer, data, sizeof(data)) == 0); + } + + // Reached the end. + decompressor->finalize(); + + // Nothing left to read. + CHECK_THROWS_WITH(decompressor->read(buffer, 1), + Equals("failed to read from zstd input stream")); +} + +TEST_CASE("Large uncompressible Compression::Type::zstd roundtrip") +{ + char data[100000]; + for (size_t i = 0; i < sizeof(data); i++) { + data[i] = rand() % 256; + } + + File f("data.zstd", "w"); + auto compressor = + Compressor::create_from_type(Compression::Type::zstd, f.get(), 1); + compressor->write(data, sizeof(data)); + compressor->finalize(); + + f.open("data.zstd", "r"); + auto decompressor = + Decompressor::create_from_type(Compression::Type::zstd, f.get()); + + char buffer[sizeof(data)]; + decompressor->read(buffer, sizeof(buffer)); + CHECK(memcmp(buffer, data, sizeof(data)) == 0); + + decompressor->finalize(); +} diff --git a/unittest/test_compr_none.cpp b/unittest/test_compr_none.cpp deleted file mode 100644 index 69563c95..00000000 --- a/unittest/test_compr_none.cpp +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (C) 2019 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "../src/compression.hpp" -#include "framework.hpp" -#include "util.hpp" - -TEST_SUITE(compr_type_none) - -TEST(small_roundtrip) -{ - const uint64_t expected_foobar_checksum = 0xa2aa05ed9085aaf9ULL; - - Checksum checksum; - - FILE* f = fopen("data.uncompressed", "w"); - struct compressor* compr_none = compressor_from_type(COMPR_TYPE_NONE); - struct compr_state* c_state = compr_none->init(f, -1, &checksum); - CHECK(c_state); - - CHECK(compr_none->write(c_state, "foobar", 6)); - - CHECK(compr_none->free(c_state)); - fclose(f); - - CHECK_INT_EQ(checksum.digest(), expected_foobar_checksum); - - checksum.reset(); - f = fopen("data.uncompressed", "r"); - struct decompressor* decompr_none = decompressor_from_type(COMPR_TYPE_NONE); - struct decompr_state* d_state = decompr_none->init(f, &checksum); - CHECK(d_state); - - char buffer[4]; - CHECK(decompr_none->read(d_state, buffer, 4)); - CHECK(memcmp(buffer, "foob", 4) == 0); - CHECK(decompr_none->read(d_state, buffer, 2)); - CHECK(memcmp(buffer, "ar", 2) == 0); - - // Nothing left to read. - CHECK(!decompr_none->read(d_state, buffer, 1)); - - // Error state is remembered. - CHECK(!decompr_none->free(d_state)); - fclose(f); - - CHECK_INT_EQ(checksum.digest(), expected_foobar_checksum); -} - -TEST_SUITE_END diff --git a/unittest/test_compr_zstd.cpp b/unittest/test_compr_zstd.cpp deleted file mode 100644 index 3a4859a9..00000000 --- a/unittest/test_compr_zstd.cpp +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright (C) 2019 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "../src/compression.hpp" -#include "framework.hpp" -#include "util.hpp" - -TEST_SUITE(compr_type_zstd) - -TEST(small_roundtrip) -{ - const uint64_t expected_foobar_checksum = 0xa2aa05ed9085aaf9ULL; - - Checksum checksum; - - FILE* f = fopen("data.zstd", "w"); - struct compressor* compr_zstd = compressor_from_type(COMPR_TYPE_ZSTD); - struct compr_state* c_state = compr_zstd->init(f, -1, &checksum); - CHECK(c_state); - - CHECK(compr_zstd->write(c_state, "foobar", 6)); - - CHECK(compr_zstd->free(c_state)); - fclose(f); - - CHECK_INT_EQ(checksum.digest(), expected_foobar_checksum); - - checksum.reset(); - f = fopen("data.zstd", "r"); - struct decompressor* decompr_zstd = decompressor_from_type(COMPR_TYPE_ZSTD); - struct decompr_state* d_state = decompr_zstd->init(f, &checksum); - CHECK(d_state); - - char buffer[4]; - CHECK(decompr_zstd->read(d_state, buffer, 4)); - CHECK(memcmp(buffer, "foob", 4) == 0); - CHECK(decompr_zstd->read(d_state, buffer, 2)); - CHECK(memcmp(buffer, "ar", 2) == 0); - - // Nothing left to read. - CHECK(!decompr_zstd->read(d_state, buffer, 1)); - - // Error state is remembered. - CHECK(!decompr_zstd->free(d_state)); - fclose(f); - - CHECK_INT_EQ(checksum.digest(), expected_foobar_checksum); -} - -TEST(large_compressible_roundtrip) -{ - char data[] = "The quick brown fox jumps over the lazy dog"; - - FILE* f = fopen("data.zstd", "w"); - struct compressor* compr_zstd = compressor_from_type(COMPR_TYPE_ZSTD); - struct compr_state* c_state = compr_zstd->init(f, 1, NULL); - CHECK(c_state); - - for (size_t i = 0; i < 1000; i++) { - CHECK(compr_zstd->write(c_state, data, sizeof(data))); - } - - CHECK(compr_zstd->free(c_state)); - fclose(f); - - f = fopen("data.zstd", "r"); - struct decompressor* decompr_zstd = decompressor_from_type(COMPR_TYPE_ZSTD); - struct decompr_state* d_state = decompr_zstd->init(f, NULL); - CHECK(d_state); - - char buffer[sizeof(data)]; - for (size_t i = 0; i < 1000; i++) { - CHECK(decompr_zstd->read(d_state, buffer, sizeof(buffer))); - CHECK(memcmp(buffer, data, sizeof(data)) == 0); - } - - // Nothing left to read. - CHECK(!decompr_zstd->read(d_state, buffer, 1)); - - // Error state is remembered. - CHECK(!decompr_zstd->free(d_state)); - fclose(f); -} - -TEST(large_uncompressible_roundtrip) -{ - char data[100000]; - for (size_t i = 0; i < sizeof(data); i++) { - data[i] = rand() % 256; - } - - FILE* f = fopen("data.zstd", "w"); - struct compressor* compr_zstd = compressor_from_type(COMPR_TYPE_ZSTD); - struct compr_state* c_state = compr_zstd->init(f, 1, NULL); - CHECK(c_state); - - CHECK(compr_zstd->write(c_state, data, sizeof(data))); - - CHECK(compr_zstd->free(c_state)); - fclose(f); - - f = fopen("data.zstd", "r"); - struct decompressor* decompr_zstd = decompressor_from_type(COMPR_TYPE_ZSTD); - struct decompr_state* d_state = decompr_zstd->init(f, NULL); - CHECK(d_state); - - char buffer[sizeof(data)]; - CHECK(decompr_zstd->read(d_state, buffer, sizeof(buffer))); - CHECK(memcmp(buffer, data, sizeof(data)) == 0); - - CHECK(decompr_zstd->free(d_state)); - fclose(f); -} - -TEST_SUITE_END |