summaryrefslogtreecommitdiff
path: root/pack-check.c
diff options
context:
space:
mode:
authorNguyễn Thái Ngọc Duy <pclouds@gmail.com>2016-07-13 17:44:04 +0200
committerJunio C Hamano <gitster@pobox.com>2016-07-13 09:15:29 -0700
commitec9d224903053e045d99c36149703501098b021c (patch)
tree7fcec73417f173951a14454a1c2a2c403b334fa4 /pack-check.c
parentaf92a645d30b9ac775cdfe5dd56ea1d66fb6e492 (diff)
downloadgit-ec9d224903053e045d99c36149703501098b021c.tar.gz
fsck: use streaming interface for large blobs in packnd/pack-ofs-4gb-limit
For blobs, we want to make sure the on-disk data is not corrupted (i.e. can be inflated and produce the expected SHA-1). Blob content is opaque, there's nothing else inside to check for. For really large blobs, we may want to avoid unpacking the entire blob in memory, just to check whether it produces the same SHA-1. On 32-bit systems, we may not have enough virtual address space for such memory allocation. And even on 64-bit where it's not a problem, allocating a lot more memory could result in kicking other parts of systems to swap file, generating lots of I/O and slowing everything down. For this particular operation, not unpacking the blob and letting check_sha1_signature, which supports streaming interface, do the job is sufficient. check_sha1_signature() is not shown in the diff, unfortunately. But if will be called when "data_valid && !data" is false. We will call the callback function "fn" with NULL as "data". The only callback of this function is fsck_obj_buffer(), which does not touch "data" at all if it's a blob. Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Diffstat (limited to 'pack-check.c')
-rw-r--r--pack-check.c23
1 files changed, 21 insertions, 2 deletions
diff --git a/pack-check.c b/pack-check.c
index 1da89a41ce..d123846ea2 100644
--- a/pack-check.c
+++ b/pack-check.c
@@ -105,6 +105,8 @@ static int verify_packfile(struct packed_git *p,
void *data;
enum object_type type;
unsigned long size;
+ off_t curpos;
+ int data_valid;
if (p->index_version > 1) {
off_t offset = entries[i].offset;
@@ -116,8 +118,25 @@ static int verify_packfile(struct packed_git *p,
sha1_to_hex(entries[i].sha1),
p->pack_name, (uintmax_t)offset);
}
- data = unpack_entry(p, entries[i].offset, &type, &size);
- if (!data)
+
+ curpos = entries[i].offset;
+ type = unpack_object_header(p, w_curs, &curpos, &size);
+ unuse_pack(w_curs);
+
+ if (type == OBJ_BLOB && big_file_threshold <= size) {
+ /*
+ * Let check_sha1_signature() check it with
+ * the streaming interface; no point slurping
+ * the data in-core only to discard.
+ */
+ data = NULL;
+ data_valid = 0;
+ } else {
+ data = unpack_entry(p, entries[i].offset, &type, &size);
+ data_valid = 1;
+ }
+
+ if (data_valid && !data)
err = error("cannot unpack %s from %s at offset %"PRIuMAX"",
sha1_to_hex(entries[i].sha1), p->pack_name,
(uintmax_t)entries[i].offset);