oid: git_oid_shorten_add() sets GITERR_INVALID when OID set is full

2 files changed, 14 insertions, 5 deletions

diff --git a/include/git2/oid.h b/include/git2/oid.h
index 662338d93..8bf43a279 100644
--- a/include/git2/oid.h
+++ b/include/git2/oid.h
@@ -241,13 +241,13 @@ GIT_EXTERN(git_oid_shorten *) git_oid_shorten_new(size_t min_length);
  * or freed.
  *
  * For performance reasons, there is a hard-limit of how many
- * OIDs can be added to a single set (around ~22000, assuming
+ * OIDs can be added to a single set (around ~32000, assuming
  * a mostly randomized distribution), which should be enough
  * for any kind of program, and keeps the algorithm fast and
  * memory-efficient.
  *
  * Attempting to add more than those OIDs will result in a
- * GIT_ENOMEM error
+ * GITERR_INVALID error
  *
  * @param os a `git_oid_shorten` instance
  * @param text_id an OID in text form
diff --git a/src/oid.c b/src/oid.c
index 8300e46c1..a70b7e099 100644
--- a/src/oid.c
+++ b/src/oid.c
@@ -369,8 +369,10 @@ int git_oid_shorten_add(git_oid_shorten *os, const char *text_oid)
 	bool is_leaf;
 	node_index idx;
 
-	if (os->full)
+	if (os->full) {
+		giterr_set(GITERR_INVALID, "Unable to shorten OID - OID set full");
 		return -1;
+	}
 
 	if (text_oid == NULL)
 		return os->min_length;
@@ -396,12 +398,19 @@ int git_oid_shorten_add(git_oid_shorten *os, const char *text_oid)
 			node->tail = NULL;
 
 			node = push_leaf(os, idx, git__fromhex(tail[0]), &tail[1]);
-			GITERR_CHECK_ALLOC(node);
+			if (node == NULL) {
+				if (os->full)
+					giterr_set(GITERR_INVALID, "Unable to shorten OID - OID set full");
+				return -1;
+			}
 		}
 
 		if (node->children[c] == 0) {
-			if (push_leaf(os, idx, c, &text_oid[i + 1]) == NULL)
+			if (push_leaf(os, idx, c, &text_oid[i + 1]) == NULL) {
+				if (os->full)
+					giterr_set(GITERR_INVALID, "Unable to shorten OID - OID set full");
 				return -1;
+			}
 			break;
 		}