Initial Implementation of progress reports during push

This adds the basics of progress reporting during push. While progress
for all aspects of a push operation are not reported with this change,
it lays the foundation to add these later. Push progress reporting
can be improved in the future - and consumers of the API should
just get more accurate information at that point.

The main areas where this is lacking are:

1) packbuilding progress: does not report progress during deltafication,
   as this involves coordinating progress from multiple threads.

2) network progress: reports progress as objects and bytes are going
   to be written to the subtransport (instead of as client gets
   confirmation that they have been received by the server) and leaves
   out some of the bytes that are transfered as part of the push protocol.
   Basically, this reports the pack bytes that are written to the
   subtransport. It does not report the bytes sent on the wire that
   are received by the server. This should be a good estimate of
   progress (and an improvement over no progress).

6 files changed, 169 insertions, 6 deletions

diff --git a/src/pack-objects.c b/src/pack-objects.c
index 7f427e3bd..2a2f36223 100644
--- a/src/pack-objects.c
+++ b/src/pack-objects.c
@@ -14,6 +14,7 @@
 #include "pack.h"
 #include "thread-utils.h"
 #include "tree.h"
+#include "util.h"
 
 #include "git2/pack.h"
 #include "git2/commit.h"
@@ -57,6 +58,9 @@ struct pack_write_context {
 #define git_packbuilder__progress_lock(pb) GIT_PACKBUILDER__MUTEX_OP(pb, progress_mutex, lock)
 #define git_packbuilder__progress_unlock(pb) GIT_PACKBUILDER__MUTEX_OP(pb, progress_mutex, unlock)
 
+/* The minimal interval between progress updates (in seconds). */
+#define MIN_PROGRESS_UPDATE_INTERVAL 0.5
+
 static unsigned name_hash(const char *name)
 {
 	unsigned c, hash = 0;
@@ -212,6 +216,14 @@ int git_packbuilder_insert(git_packbuilder *pb, const git_oid *oid,
 	assert(ret != 0);
 	kh_value(pb->object_ix, pos) = po;
 
+	if (pb->progress_cb) {
+		double current_time = git__timer();
+		if ((current_time - pb->last_progress_report_time) >= MIN_PROGRESS_UPDATE_INTERVAL) {
+			pb->last_progress_report_time = current_time;
+			pb->progress_cb(GIT_PACKBUILDER_ADDING_OBJECTS, pb->nr_objects, 0, pb->progress_cb_payload);
+		}
+	}
+
 	pb->done = false;
 	return 0;
 }
@@ -1207,6 +1219,13 @@ static int prepare_pack(git_packbuilder *pb)
 	if (pb->nr_objects == 0 || pb->done)
 		return 0; /* nothing to do */
 
+	/*
+	 * Although we do not report progress during deltafication, we
+	 * at least report that we are in the deltafication stage
+	 */
+	if (pb->progress_cb)
+			pb->progress_cb(GIT_PACKBUILDER_DELTAFICATION, 0, pb->nr_objects, pb->progress_cb_payload);
+
 	delta_list = git__malloc(pb->nr_objects * sizeof(*delta_list));
 	GITERR_CHECK_ALLOC(delta_list);
 
@@ -1348,6 +1367,17 @@ uint32_t git_packbuilder_written(git_packbuilder *pb)
 	return pb->nr_written;
 }
 
+int git_packbuilder_set_callbacks(git_packbuilder *pb, git_packbuilder_progress progress_cb, void *progress_cb_payload)
+{
+	if (!pb)
+		return -1;
+
+	pb->progress_cb = progress_cb;
+	pb->progress_cb_payload = progress_cb_payload;
+
+	return 0;
+}
+
 void git_packbuilder_free(git_packbuilder *pb)
 {
 	if (pb == NULL)
diff --git a/src/pack-objects.h b/src/pack-objects.h
index 8e7ba7f78..0c94a5a7a 100644
--- a/src/pack-objects.h
+++ b/src/pack-objects.h
@@ -16,6 +16,7 @@
 #include "netops.h"
 
 #include "git2/oid.h"
+#include "git2/pack.h"
 
 #define GIT_PACK_WINDOW 10 /* number of objects to possibly delta against */
 #define GIT_PACK_DEPTH 50 /* max delta depth */
@@ -79,6 +80,10 @@ struct git_packbuilder {
 
 	int nr_threads; /* nr of threads to use */
 
+	git_packbuilder_progress progress_cb;
+	void *progress_cb_payload;
+	double last_progress_report_time; /* the time progress was last reported */
+
 	bool done;
 };
 
diff --git a/src/push.c b/src/push.c
index eaaa46248..698079253 100644
--- a/src/push.c
+++ b/src/push.c
@@ -70,6 +70,25 @@ int git_push_set_options(git_push *push, const git_push_options *opts)
 	return 0;
 }
 
+int git_push_set_callbacks(
+	git_push *push,
+	git_packbuilder_progress pack_progress_cb,
+	void *pack_progress_cb_payload,
+	git_push_transfer_progress transfer_progress_cb,
+	void *transfer_progress_cb_payload)
+{
+	if (!push)
+		return -1;
+
+	push->pack_progress_cb = pack_progress_cb;
+	push->pack_progress_cb_payload = pack_progress_cb_payload;
+
+	push->transfer_progress_cb = transfer_progress_cb;
+	push->transfer_progress_cb_payload = transfer_progress_cb_payload;
+
+	return 0;
+}
+
 static void free_refspec(push_spec *spec)
 {
 	if (spec == NULL)
@@ -583,6 +602,10 @@ static int do_push(git_push *push)
 
 	git_packbuilder_set_threads(push->pb, push->pb_parallelism);
 
+	if (push->pack_progress_cb)
+		if ((error = git_packbuilder_set_callbacks(push->pb, push->pack_progress_cb, push->pack_progress_cb_payload)) < 0)
+			goto on_error;
+
 	if ((error = calculate_work(push)) < 0 ||
 		(error = queue_objects(push)) < 0 ||
 		(error = transport->push(transport, push)) < 0)
diff --git a/src/push.h b/src/push.h
index e982b8385..6c8bf7229 100644
--- a/src/push.h
+++ b/src/push.h
@@ -39,6 +39,11 @@ struct git_push {
 
 	/* options */
 	unsigned pb_parallelism;
+
+	git_packbuilder_progress pack_progress_cb;
+	void *pack_progress_cb_payload;
+	git_push_transfer_progress transfer_progress_cb;
+	void *transfer_progress_cb_payload;
 };
 
 /**
diff --git a/src/transports/smart_protocol.c b/src/transports/smart_protocol.c
index 0cd5e831d..156b69e1f 100644
--- a/src/transports/smart_protocol.c
+++ b/src/transports/smart_protocol.c
@@ -13,8 +13,11 @@
 #include "push.h"
 #include "pack-objects.h"
 #include "remote.h"
+#include "util.h"
 
 #define NETWORK_XFER_THRESHOLD (100*1024)
+/* The minimal interval between progress updates (in seconds). */
+#define MIN_PROGRESS_UPDATE_INTERVAL 0.5
 
 int git_smart__store_refs(transport_smart *t, int flushes)
 {
@@ -801,22 +804,53 @@ static int update_refs_from_report(
 	return 0;
 }
 
+struct push_packbuilder_payload
+{
+	git_smart_subtransport_stream *stream;
+	git_packbuilder *pb;
+	git_push_transfer_progress cb;
+	void *cb_payload;
+	size_t last_bytes;
+	double last_progress_report_time;
+};
+
 static int stream_thunk(void *buf, size_t size, void *data)
 {
-	git_smart_subtransport_stream *s = (git_smart_subtransport_stream *)data;
+	int error = 0;
+	struct push_packbuilder_payload *payload = data;
+
+	if ((error = payload->stream->write(payload->stream, (const char *)buf, size)) < 0)
+		return error;
+
+	if (payload->cb) {
+		double current_time = git__timer();
+		payload->last_bytes += size;
 
-	return s->write(s, (const char *)buf, size);
+		if ((current_time - payload->last_progress_report_time) >= MIN_PROGRESS_UPDATE_INTERVAL) {
+			payload->last_progress_report_time = current_time;
+			payload->cb(payload->pb->nr_written, payload->pb->nr_objects, payload->last_bytes, payload->cb_payload);
+		}
+	}
+
+	return error;
 }
 
 int git_smart__push(git_transport *transport, git_push *push)
 {
 	transport_smart *t = (transport_smart *)transport;
-	git_smart_subtransport_stream *s;
+	struct push_packbuilder_payload packbuilder_payload = {0};
 	git_buf pktline = GIT_BUF_INIT;
 	int error = -1, need_pack = 0;
 	push_spec *spec;
 	unsigned int i;
 
+	packbuilder_payload.pb = push->pb;
+
+	if (push->transfer_progress_cb) {
+		packbuilder_payload.cb = push->transfer_progress_cb;
+		packbuilder_payload.cb_payload = push->transfer_progress_cb_payload;
+	}
+
 #ifdef PUSH_DEBUG
 {
 	git_remote_head *head;
@@ -848,12 +882,12 @@ int git_smart__push(git_transport *transport, git_push *push)
 		}
 	}
 
-	if (git_smart__get_push_stream(t, &s) < 0 ||
+	if (git_smart__get_push_stream(t, &packbuilder_payload.stream) < 0 ||
 		gen_pktline(&pktline, push) < 0 ||
-		s->write(s, git_buf_cstr(&pktline), git_buf_len(&pktline)) < 0)
+		packbuilder_payload.stream->write(packbuilder_payload.stream, git_buf_cstr(&pktline), git_buf_len(&pktline)) < 0)
 		goto on_error;
 
-	if (need_pack && git_packbuilder_foreach(push->pb, &stream_thunk, s) < 0)
+	if (need_pack && git_packbuilder_foreach(push->pb, &stream_thunk, &packbuilder_payload) < 0)
 		goto on_error;
 
 	/* If we sent nothing or the server doesn't support report-status, then
@@ -863,6 +897,11 @@ int git_smart__push(git_transport *transport, git_push *push)
 	else if (parse_report(&t->buffer, push) < 0)
 		goto on_error;
 
+	/* If progress is being reported write the final report */
+	if (push->transfer_progress_cb) {
+		push->transfer_progress_cb(push->pb->nr_written, push->pb->nr_objects, packbuilder_payload.last_bytes, push->transfer_progress_cb_payload);
+	}
+
 	if (push->status.length &&
 		update_refs_from_report(&t->refs, &push->specs, &push->status) < 0)
 		goto on_error;
diff --git a/src/util.h b/src/util.h
index bd93b46b5..78e2f2e79 100644
--- a/src/util.h
+++ b/src/util.h
@@ -353,4 +353,65 @@ GIT_INLINE(void) git__memzero(void *data, size_t size)
 #endif
 }
 
+#ifdef GIT_WIN32
+
+GIT_INLINE(double) git__timer(void)
+{
+	/* We need the initial tick count to detect if the tick
+	 * count has rolled over. */
+	static DWORD initial_tick_count = 0;
+
+	/* GetTickCount returns the number of milliseconds that have
+	 * elapsed since the system was started. */
+	DWORD count = GetTickCount();
+
+	if(initial_tick_count == 0) {
+		initial_tick_count = count;
+	} else if (count < initial_tick_count) {
+		/* The tick count has rolled over - adjust for it. */
+		count = (0xFFFFFFFF - initial_tick_count) + count;
+	}
+
+	return (double) count / (double) 1000;
+}
+
+#elif __APPLE__
+
+#include <mach/mach_time.h>
+
+double git__timer(void)
+{
+   uint64_t time = mach_absolute_time();
+   static double scaling_factor = 0;
+
+   if (scaling_factor == 0) {
+       mach_timebase_info_data_t info;
+       (void)mach_timebase_info(&info);
+       scaling_factor = (double)info.numer / (double)info.denom;
+   }
+
+   return (double)time * scaling_factor / 1.0E-9;
+}
+
+#else
+
+#include <sys/time.h>
+
+GIT_INLINE(double) git__timer(void)
+{
+	struct timespec tp;
+
+	if (clock_gettime(CLOCK_MONOTONIC, &tp) == 0) {
+		return (double) tp.tv_sec + (double) tp.tv_nsec / 1E-9;
+	} else {
+		/* Fall back to using gettimeofday */
+		struct timeval tv;
+		struct timezone tz;
+		gettimeofday(&tv, &tz);
+		return (double)tv.tv_sec + (double)tv.tv_usec / 1E-6;
+	}
+}
+
+#endif
+
 #endif /* INCLUDE_util_h__ */