path: root/src/odb.c

/*
 * Copyright (C) 2009-2011 the libgit2 contributors
 *
 * This file is part of libgit2, distributed under the GNU GPL v2 with
 * a Linking Exception. For full terms see the included COPYING file.
 */

#include "common.h"
#include "git2/zlib.h"
#include "git2/object.h"
#include "fileops.h"
#include "hash.h"
#include "odb.h"
#include "delta-apply.h"

#include "git2/odb_backend.h"

#define GIT_ALTERNATES_FILE "info/alternates"

/* TODO: is this correct? */
#define GIT_LOOSE_PRIORITY 2
#define GIT_PACKED_PRIORITY 1

typedef struct
{
	git_odb_backend *backend;
	int priority;
	int is_alternate;
} backend_internal;

static int format_object_header(char *hdr, size_t n, size_t obj_len, git_otype obj_type)
{
	const char *type_str = git_object_type2string(obj_type);
	int len = p_snprintf(hdr, n, "%s %"PRIuZ, type_str, obj_len);

	if (len < 0 || len >= (int)n)
		return git__throw(GIT_ERROR, "Cannot format object header. Length is out of bounds");

	return len+1;
}

int git_odb__hashobj(git_oid *id, git_rawobj *obj)
{
	git_buf_vec vec[2];
	char header[64];
	int hdrlen;

	assert(id && obj);

	if (!git_object_typeisloose(obj->type))
		return git__throw(GIT_ERROR, "Failed to hash object. Wrong object type");

	if (!obj->data && obj->len != 0)
		return git__throw(GIT_ERROR, "Failed to hash object. No data given");

	if ((hdrlen = format_object_header(header, sizeof(header), obj->len, obj->type)) < 0)
		return git__rethrow(hdrlen, "Failed to hash object");

	vec[0].data = header;
	vec[0].len = hdrlen;
	vec[1].data = obj->data;
	vec[1].len = obj->len;

	git_hash_vec(id, vec, 2);

	return GIT_SUCCESS;
}


static git_odb_object *new_odb_object(const git_oid *oid, git_rawobj *source)
{
	git_odb_object *object = git__malloc(sizeof(git_odb_object));
	memset(object, 0x0, sizeof(git_odb_object));

	git_oid_cpy(&object->cached.oid, oid);
	memcpy(&object->raw, source, sizeof(git_rawobj));

	return object;
}

static void free_odb_object(void *o)
{
	git_odb_object *object = (git_odb_object *)o;

	if (object != NULL) {
		git__free(object->raw.data);
		git__free(object);
	}
}

const git_oid *git_odb_object_id(git_odb_object *object)
{
	return &object->cached.oid;
}

const void *git_odb_object_data(git_odb_object *object)
{
	return object->raw.data;
}

size_t git_odb_object_size(git_odb_object *object)
{
	return object->raw.len;
}

git_otype git_odb_object_type(git_odb_object *object)
{
	return object->raw.type;
}

void git_odb_object_free(git_odb_object *object)
{
	git_cached_obj_decref((git_cached_obj *)object, &free_odb_object);
}

int git_odb__hashfd(git_oid *out, git_file fd, size_t size, git_otype type)
{
	int hdr_len;
	char hdr[64], buffer[2048];
	git_hash_ctx *ctx;

	hdr_len = format_object_header(hdr, sizeof(hdr), size, type);
	if (hdr_len < 0)
		return git__throw(GIT_ERROR, "Failed to format blob header. Length is out of bounds");

	ctx = git_hash_new_ctx();

	git_hash_update(ctx, hdr, hdr_len);

	while (size > 0) {
		ssize_t read_len = read(fd, buffer, sizeof(buffer));

		if (read_len < 0) {
			git_hash_free_ctx(ctx);
			return git__throw(GIT_EOSERR, "Error when reading file: %s", strerror(errno));
		}

		git_hash_update(ctx, buffer, read_len);
		size -= read_len;
	}

	git_hash_final(out, ctx);
	git_hash_free_ctx(ctx);

	return GIT_SUCCESS;
}

int git_odb__hashlink(git_oid *out, const char *path)
{
	struct stat st;
	int error;
	git_off_t size;

	error = p_lstat(path, &st);
	if (error < 0)
		return git__throw(GIT_EOSERR, "Failed to stat blob. %s", strerror(errno));

	size = st.st_size;

	if (!git__is_sizet(size))
		return git__throw(GIT_EOSERR, "File size overflow for 32-bit systems");

	if (S_ISLNK(st.st_mode)) {
		char *link_data;
		ssize_t read_len;

		link_data = git__malloc(size);
		if (link_data == NULL)
			return GIT_ENOMEM;

		read_len = p_readlink(path, link_data, size + 1);
		if (read_len != (ssize_t)size)
			return git__throw(GIT_EOSERR, "Failed to read symlink data");

		error = git_odb_hash(out, link_data, (size_t)size, GIT_OBJ_BLOB);
		free(link_data);
	} else { 
		int fd;

		if ((fd = p_open(path, O_RDONLY)) < 0)
			return git__throw(GIT_ENOTFOUND, "Could not open '%s'", path);

		error = git_odb__hashfd(out, fd, (size_t)size, GIT_OBJ_BLOB);
		p_close(fd);
	}

	return error;
}

int git_odb_hashfile(git_oid *out, const char *path, git_otype type)
{
	int fd, error;
	git_off_t size;

	if ((fd = p_open(path, O_RDONLY)) < 0)
		return git__throw(GIT_ENOTFOUND, "Could not open '%s'", path);

	if ((size = git_futils_filesize(fd)) < 0 || !git__is_sizet(size)) {
		p_close(fd);
		return git__throw(GIT_EOSERR,
			"File size overflow. The object is too big to fit in 32-bit mode");
	}

	error = git_odb__hashfd(out, fd, (size_t)size, type);

	p_close(fd);
	return error;
}

int git_odb_hash(git_oid *id, const void *data, size_t len, git_otype type)
{
	git_rawobj raw;

	assert(id);

	raw.data = (void *)data;
	raw.len = len;
	raw.type = type;

	return git_odb__hashobj(id, &raw);
}

/**
 * FAKE WSTREAM
 */

typedef struct {
	git_odb_stream stream;
	char *buffer;
	size_t size, written;
	git_otype type;
} fake_wstream;

static int fake_wstream__fwrite(git_oid *oid, git_odb_stream *_stream)
{
	fake_wstream *stream = (fake_wstream *)_stream;
	return _stream->backend->write(oid, _stream->backend, stream->buffer, stream->size, stream->type);
}

static int fake_wstream__write(git_odb_stream *_stream, const char *data, size_t len)
{
	fake_wstream *stream = (fake_wstream *)_stream;

	if (stream->written + len > stream->size)
		return GIT_ENOMEM;

	memcpy(stream->buffer + stream->written, data, len);
	stream->written += len;
	return GIT_SUCCESS;
}

static void fake_wstream__free(git_odb_stream *_stream)
{
	fake_wstream *stream = (fake_wstream *)_stream;

	git__free(stream->buffer);
	git__free(stream);
}

static int init_fake_wstream(git_odb_stream **stream_p, git_odb_backend *backend, size_t size, git_otype type)
{
	fake_wstream *stream;

	stream = git__calloc(1, sizeof(fake_wstream));
	if (stream == NULL)
		return GIT_ENOMEM;

	stream->size = size;
	stream->type = type;
	stream->buffer = git__malloc(size);
	if (stream->buffer == NULL) {
		git__free(stream);
		return GIT_ENOMEM;
	}

	stream->stream.backend = backend;
	stream->stream.read = NULL; /* read only */
	stream->stream.write = &fake_wstream__write;
	stream->stream.finalize_write = &fake_wstream__fwrite;
	stream->stream.free = &fake_wstream__free;
	stream->stream.mode = GIT_STREAM_WRONLY;

	*stream_p = (git_odb_stream *)stream;
	return GIT_SUCCESS;
}

/***********************************************************
 *
 * OBJECT DATABASE PUBLIC API
 *
 * Public calls for the ODB functionality
 *
 ***********************************************************/

static int backend_sort_cmp(const void *a, const void *b)
{
	const backend_internal *backend_a = (const backend_internal *)(a);
	const backend_internal *backend_b = (const backend_internal *)(b);

	if (backend_a->is_alternate == backend_b->is_alternate)
		return (backend_b->priority - backend_a->priority);

	return backend_a->is_alternate ? 1 : -1;
}

int git_odb_new(git_odb **out)
{
	int error;

	git_odb *db = git__calloc(1, sizeof(*db));
	if (!db)
		return GIT_ENOMEM;

	error = git_cache_init(&db->cache, GIT_DEFAULT_CACHE_SIZE, &free_odb_object);
	if (error < GIT_SUCCESS) {
		git__free(db);
		return git__rethrow(error, "Failed to create object database");
	}

	if ((error = git_vector_init(&db->backends, 4, backend_sort_cmp)) < GIT_SUCCESS) {
		git__free(db);
		return git__rethrow(error, "Failed to create object database");
	}

	*out = db;
	GIT_REFCOUNT_INC(db);
	return GIT_SUCCESS;
}

static int add_backend_internal(git_odb *odb, git_odb_backend *backend, int priority, int is_alternate)
{
	backend_internal *internal;

	assert(odb && backend);

	if (backend->odb != NULL && backend->odb != odb)
		return git__throw(GIT_EBUSY, "The backend is already owned by another ODB");

	internal = git__malloc(sizeof(backend_internal));
	if (internal == NULL)
		return GIT_ENOMEM;

	internal->backend = backend;
	internal->priority = priority;
	internal->is_alternate = is_alternate;

	if (git_vector_insert(&odb->backends, internal) < 0) {
		git__free(internal);
		return GIT_ENOMEM;
	}

	git_vector_sort(&odb->backends);
	internal->backend->odb = odb;
	return GIT_SUCCESS;
}

int git_odb_add_backend(git_odb *odb, git_odb_backend *backend, int priority)
{
	return add_backend_internal(odb, backend, priority, 0);
}

int git_odb_add_alternate(git_odb *odb, git_odb_backend *backend, int priority)
{
	return add_backend_internal(odb, backend, priority, 1);
}

static int add_default_backends(git_odb *db, const char *objects_dir, int as_alternates)
{
	git_odb_backend *loose, *packed;
	int error;

	/* add the loose object backend */
	error = git_odb_backend_loose(&loose, objects_dir, -1, 0);
	if (error < GIT_SUCCESS)
		return error;

	error = add_backend_internal(db, loose, GIT_LOOSE_PRIORITY, as_alternates);
	if (error < GIT_SUCCESS)
		return git__rethrow(error, "Failed to add backend");

	/* add the packed file backend */
	error = git_odb_backend_pack(&packed, objects_dir);
	if (error < GIT_SUCCESS)
		return error;

	error = add_backend_internal(db, packed, GIT_PACKED_PRIORITY, as_alternates);
	if (error < GIT_SUCCESS)
		return git__rethrow(error, "Failed to add backend");

	return GIT_SUCCESS;
}

static int load_alternates(git_odb *odb, const char *objects_dir)
{
	git_buf alternates_path = GIT_BUF_INIT;
	char *buffer;
	git_fbuffer alternates_buf = GIT_FBUFFER_INIT;
	const char *alternate;
	int error;

	error = git_buf_joinpath(&alternates_path, objects_dir, GIT_ALTERNATES_FILE);
	if (error < GIT_SUCCESS)
		return error;

	if (git_path_exists(alternates_path.ptr) < GIT_SUCCESS) {
		git_buf_free(&alternates_path);
		return GIT_SUCCESS;
	}

	if (git_futils_readbuffer(&alternates_buf, alternates_path.ptr) < GIT_SUCCESS) {
		git_buf_free(&alternates_path);
		return git__throw(GIT_EOSERR, "Failed to add backend. Can't read alternates");
	}

	buffer = (char *)alternates_buf.data;
	error = GIT_SUCCESS;

	/* add each alternate as a new backend; one alternate per line */
	while ((alternate = git__strtok(&buffer, "\r\n")) != NULL) {
		if (*alternate == '\0' || *alternate == '#')
			continue;

		/* relative path: build based on the current `objects` folder */
		if (*alternate == '.') {
			error = git_buf_joinpath(&alternates_path, objects_dir, alternate);
			if (error < GIT_SUCCESS)
				break;
			alternate = git_buf_cstr(&alternates_path);
		}

		if ((error = add_default_backends(odb, alternate, 1)) < GIT_SUCCESS)
			break;
	}

	git_buf_free(&alternates_path);
	git_futils_freebuffer(&alternates_buf);
	if (error < GIT_SUCCESS)
		return git__rethrow(error, "Failed to load alternates");
	return error;
}

int git_odb_open(git_odb **out, const char *objects_dir)
{
	git_odb *db;
	int error;

	assert(out && objects_dir);

	*out = NULL;

	if ((error = git_odb_new(&db)) < 0)
		return git__rethrow(error, "Failed to open ODB");

	if ((error = add_default_backends(db, objects_dir, 0)) < GIT_SUCCESS)
		goto cleanup;

	if ((error = load_alternates(db, objects_dir)) < GIT_SUCCESS)
		goto cleanup;

	*out = db;
	return GIT_SUCCESS;

cleanup:
	git_odb_free(db);
	return error; /* error already set - pass through */
}

static void odb_free(git_odb *db)
{
	unsigned int i;

	for (i = 0; i < db->backends.length; ++i) {
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *backend = internal->backend;

		if (backend->free) backend->free(backend);
		else git__free(backend);

		git__free(internal);
	}

	git_vector_free(&db->backends);
	git_cache_free(&db->cache);
	git__free(db);
}

void git_odb_free(git_odb *db)
{
	if (db == NULL)
		return;

	GIT_REFCOUNT_DEC(db, odb_free);
}

int git_odb_exists(git_odb *db, const git_oid *id)
{
	git_odb_object *object;
	unsigned int i;
	int found = 0;

	assert(db && id);

	if ((object = git_cache_get(&db->cache, id)) != NULL) {
		git_odb_object_free(object);
		return 1;
	}

	for (i = 0; i < db->backends.length && !found; ++i) {
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;

		if (b->exists != NULL)
			found = b->exists(b, id);
	}

	return found;
}

int git_odb_read_header(size_t *len_p, git_otype *type_p, git_odb *db, const git_oid *id)
{
	unsigned int i;
	int error = GIT_ENOTFOUND;
	git_odb_object *object;

	assert(db && id);

	if ((object = git_cache_get(&db->cache, id)) != NULL) {
		*len_p = object->raw.len;
		*type_p = object->raw.type;
		git_odb_object_free(object);
		return GIT_SUCCESS;
	}

	for (i = 0; i < db->backends.length && error < 0; ++i) {
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;

		if (b->read_header != NULL)
			error = b->read_header(len_p, type_p, b, id);
	}

	if (error == GIT_EPASSTHROUGH)
		return GIT_SUCCESS;

	/*
	 * no backend could read only the header.
	 * try reading the whole object and freeing the contents
	 */
	if (error < 0) {
		if ((error = git_odb_read(&object, db, id)) < GIT_SUCCESS)
			return error; /* error already set - pass through */

		*len_p = object->raw.len;
		*type_p = object->raw.type;
		git_odb_object_free(object);
	}

	return GIT_SUCCESS;
}

int git_odb_read(git_odb_object **out, git_odb *db, const git_oid *id)
{
	unsigned int i;
	int error = GIT_ENOTFOUND;
	git_rawobj raw;

	assert(out && db && id);

	*out = git_cache_get(&db->cache, id);
	if (*out != NULL)
		return GIT_SUCCESS;

	for (i = 0; i < db->backends.length && error < 0; ++i) {
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;

		if (b->read != NULL)
			error = b->read(&raw.data, &raw.len, &raw.type, b, id);
	}

	if (error == GIT_EPASSTHROUGH || error == GIT_SUCCESS) {
		*out = git_cache_try_store(&db->cache, new_odb_object(id, &raw));
		return GIT_SUCCESS;
	}

	return git__rethrow(error, "Failed to read object");
}

int git_odb_read_prefix(git_odb_object **out, git_odb *db, const git_oid *short_id, unsigned int len)
{
	unsigned int i;
	int error = GIT_ENOTFOUND;
	git_oid full_oid;
	git_rawobj raw;
	int found = 0;

	assert(out && db);

	if (len < GIT_OID_MINPREFIXLEN)
		return git__throw(GIT_EAMBIGUOUSOIDPREFIX, "Failed to lookup object. Prefix length is lower than %d.", GIT_OID_MINPREFIXLEN);

	if (len > GIT_OID_HEXSZ)
		len = GIT_OID_HEXSZ;

	if (len == GIT_OID_HEXSZ) {
		*out = git_cache_get(&db->cache, short_id);
		if (*out != NULL)
			return GIT_SUCCESS;
	}

	for (i = 0; i < db->backends.length && found < 2; ++i) {
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;

		if (b->read != NULL) {
			error = b->read_prefix(&full_oid, &raw.data, &raw.len, &raw.type, b, short_id, len);
			switch (error) {
			case GIT_SUCCESS:
				found++;
				break;
			case GIT_ENOTFOUND:
			case GIT_EPASSTHROUGH:
				break;
			case GIT_EAMBIGUOUSOIDPREFIX:
				return git__rethrow(error, "Failed to read object. Ambiguous sha1 prefix");
			default:
				return git__rethrow(error, "Failed to read object");
			}
		}
	}

	if (found == 1) {
		*out = git_cache_try_store(&db->cache, new_odb_object(&full_oid, &raw));
	} else if (found > 1) {
		return git__throw(GIT_EAMBIGUOUSOIDPREFIX, "Failed to read object. Ambiguous sha1 prefix");
	} else {
		return git__throw(GIT_ENOTFOUND, "Failed to read object. Object not found");
	}

	return GIT_SUCCESS;
}

int git_odb_write(git_oid *oid, git_odb *db, const void *data, size_t len, git_otype type)
{
	unsigned int i;
	int error = GIT_ERROR;
	git_odb_stream *stream;

	assert(oid && db);

	for (i = 0; i < db->backends.length && error < 0; ++i) {
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;

		/* we don't write in alternates! */
		if (internal->is_alternate)
			continue;

		if (b->write != NULL)
			error = b->write(oid, b, data, len, type);
	}

	if (error == GIT_EPASSTHROUGH || error == GIT_SUCCESS)
		return GIT_SUCCESS;

	/* if no backends were able to write the object directly, we try a streaming
	 * write to the backends; just write the whole object into the stream in one
	 * push */

	if ((error = git_odb_open_wstream(&stream, db, len, type)) == GIT_SUCCESS) {
		stream->write(stream, data, len);
		error = stream->finalize_write(oid, stream);
		stream->free(stream);
		return GIT_SUCCESS;
	}

	return git__rethrow(error, "Failed to write object");
}

int git_odb_open_wstream(git_odb_stream **stream, git_odb *db, size_t size, git_otype type)
{
	unsigned int i;
	int error = GIT_ERROR;

	assert(stream && db);

	for (i = 0; i < db->backends.length && error < 0; ++i) {
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;

		/* we don't write in alternates! */
		if (internal->is_alternate)
			continue;

		if (b->writestream != NULL)
			error = b->writestream(stream, b, size, type);
		else if (b->write != NULL)
			error = init_fake_wstream(stream, b, size, type);
	}

	if (error == GIT_EPASSTHROUGH || error == GIT_SUCCESS)
		return GIT_SUCCESS;

	return git__rethrow(error, "Failed to open write stream");
}

int git_odb_open_rstream(git_odb_stream **stream, git_odb *db, const git_oid *oid)
{
	unsigned int i;
	int error = GIT_ERROR;

	assert(stream && db);

	for (i = 0; i < db->backends.length && error < 0; ++i) {
		backend_internal *internal = git_vector_get(&db->backends, i);
		git_odb_backend *b = internal->backend;

		if (b->readstream != NULL)
			error = b->readstream(stream, b, oid);
	}

	if (error == GIT_EPASSTHROUGH || error == GIT_SUCCESS)
		return GIT_SUCCESS;

	return git__rethrow(error, "Failed to open read stream");
}