nettle: use the correct type for hash and MAC functionshash3.3

In addition allow for hashing of more than UINT_MAX data.

9 files changed, 100 insertions, 66 deletions

diff --git a/lib/accelerated/x86/hmac-padlock.c b/lib/accelerated/x86/hmac-padlock.c
index e516c385d9..5e4a5eb026 100644
--- a/lib/accelerated/x86/hmac-padlock.c
+++ b/lib/accelerated/x86/hmac-padlock.c
@@ -33,6 +33,7 @@
 #include <aes-padlock.h>
 #include <sha-padlock.h>
 #include <algorithms.h>
+#include <x86-common.h>
 
 #ifdef HAVE_LIBNETTLE
 
@@ -40,9 +41,9 @@
 #define OPAD 0x5c
 #define MAX_SHA_DIGEST_SIZE (512/8)
 
-typedef void (*update_func) (void *, unsigned, const uint8_t *);
-typedef void (*digest_func) (void *, unsigned, uint8_t *);
-typedef void (*set_key_func) (void *, unsigned, const uint8_t *);
+typedef void (*update_func) (void *, size_t, const uint8_t *);
+typedef void (*digest_func) (void *, size_t, uint8_t *);
+typedef void (*set_key_func) (void *, size_t, const uint8_t *);
 
 struct padlock_hmac_ctx {
 	union {
@@ -63,91 +64,91 @@ struct padlock_hmac_ctx {
 
 static void
 padlock_hmac_sha1_set_key(struct hmac_sha1_ctx *ctx,
-			  unsigned key_length, const uint8_t * key)
+			  size_t key_length, const uint8_t * key)
 {
 	HMAC_SET_KEY(ctx, &padlock_sha1, key_length, key);
 }
 
 static void
 padlock_hmac_sha1_update(struct hmac_sha1_ctx *ctx,
-			 unsigned length, const uint8_t * data)
+			 size_t length, const uint8_t * data)
 {
 	padlock_sha1_update(&ctx->state, length, data);
 }
 
 static void
 padlock_hmac_sha1_digest(struct hmac_sha1_ctx *ctx,
-			 unsigned length, uint8_t * digest)
+			 size_t length, uint8_t * digest)
 {
 	HMAC_DIGEST(ctx, &padlock_sha1, length, digest);
 }
 
 static void
 padlock_hmac_sha256_set_key(struct hmac_sha256_ctx *ctx,
-			    unsigned key_length, const uint8_t * key)
+			    size_t key_length, const uint8_t * key)
 {
 	HMAC_SET_KEY(ctx, &padlock_sha256, key_length, key);
 }
 
 static void
 padlock_hmac_sha256_update(struct hmac_sha256_ctx *ctx,
-			   unsigned length, const uint8_t * data)
+			   size_t length, const uint8_t * data)
 {
 	padlock_sha256_update(&ctx->state, length, data);
 }
 
 static void
 padlock_hmac_sha256_digest(struct hmac_sha256_ctx *ctx,
-			   unsigned length, uint8_t * digest)
+			   size_t length, uint8_t * digest)
 {
 	HMAC_DIGEST(ctx, &padlock_sha256, length, digest);
 }
 
 static void
 padlock_hmac_sha224_set_key(struct hmac_sha224_ctx *ctx,
-			    unsigned key_length, const uint8_t * key)
+			    size_t key_length, const uint8_t * key)
 {
 	HMAC_SET_KEY(ctx, &padlock_sha224, key_length, key);
 }
 
 static void
 padlock_hmac_sha224_digest(struct hmac_sha224_ctx *ctx,
-			   unsigned length, uint8_t * digest)
+			   size_t length, uint8_t * digest)
 {
 	HMAC_DIGEST(ctx, &padlock_sha224, length, digest);
 }
 
 static void
 padlock_hmac_sha384_set_key(struct hmac_sha384_ctx *ctx,
-			    unsigned key_length, const uint8_t * key)
+			    size_t key_length, const uint8_t * key)
 {
 	HMAC_SET_KEY(ctx, &padlock_sha384, key_length, key);
 }
 
 static void
 padlock_hmac_sha384_digest(struct hmac_sha384_ctx *ctx,
-			   unsigned length, uint8_t * digest)
+			   size_t length, uint8_t * digest)
 {
 	HMAC_DIGEST(ctx, &padlock_sha384, length, digest);
 }
 
 static void
 padlock_hmac_sha512_set_key(struct hmac_sha512_ctx *ctx,
-			    unsigned key_length, const uint8_t * key)
+			    size_t key_length, const uint8_t * key)
 {
 	HMAC_SET_KEY(ctx, &padlock_sha512, key_length, key);
 }
 
 static void
 padlock_hmac_sha512_update(struct hmac_sha512_ctx *ctx,
-			   unsigned length, const uint8_t * data)
+			   size_t length, const uint8_t * data)
 {
 	padlock_sha512_update(&ctx->state, length, data);
 }
 
 static void
 padlock_hmac_sha512_digest(struct hmac_sha512_ctx *ctx,
-			   unsigned length, uint8_t * digest)
+			   size_t length, uint8_t * digest)
 {
 	HMAC_DIGEST(ctx, &padlock_sha512, length, digest);
 }
diff --git a/lib/accelerated/x86/hmac-x86-ssse3.c b/lib/accelerated/x86/hmac-x86-ssse3.c
index 0d5004d30c..e733be60f8 100644
--- a/lib/accelerated/x86/hmac-x86-ssse3.c
+++ b/lib/accelerated/x86/hmac-x86-ssse3.c
@@ -33,12 +33,13 @@
 #include <aes-x86.h>
 #include <sha-x86.h>
 #include <algorithms.h>
+#include <x86-common.h>
 
 #ifdef HAVE_LIBNETTLE
 
-typedef void (*update_func) (void *, unsigned, const uint8_t *);
-typedef void (*digest_func) (void *, unsigned, uint8_t *);
-typedef void (*set_key_func) (void *, unsigned, const uint8_t *);
+typedef void (*update_func) (void *, size_t, const uint8_t *);
+typedef void (*digest_func) (void *, size_t, uint8_t *);
+typedef void (*set_key_func) (void *, size_t, const uint8_t *);
 
 struct x86_hmac_ctx {
 	union {
@@ -61,56 +62,56 @@ struct x86_hmac_ctx {
 
 static void
 x86_hmac_sha1_set_key(struct hmac_sha1_ctx *ctx,
-			  unsigned key_length, const uint8_t * key)
+			  size_t key_length, const uint8_t * key)
 {
 	HMAC_SET_KEY(ctx, &x86_sha1, key_length, key);
 }
 
 static void
 x86_hmac_sha1_update(struct hmac_sha1_ctx *ctx,
-			 unsigned length, const uint8_t * data)
+			 size_t length, const uint8_t * data)
 {
 	x86_sha1_update(&ctx->state, length, data);
 }
 
 static void
 x86_hmac_sha1_digest(struct hmac_sha1_ctx *ctx,
-			 unsigned length, uint8_t * digest)
+			 size_t length, uint8_t * digest)
 {
 	HMAC_DIGEST(ctx, &x86_sha1, length, digest);
 }
 
 static void
 x86_hmac_sha256_set_key(struct hmac_sha256_ctx *ctx,
-			    unsigned key_length, const uint8_t * key)
+			    size_t key_length, const uint8_t * key)
 {
 	HMAC_SET_KEY(ctx, &x86_sha256, key_length, key);
 }
 
 static void
 x86_hmac_sha256_update(struct hmac_sha256_ctx *ctx,
-			   unsigned length, const uint8_t * data)
+			   size_t length, const uint8_t * data)
 {
 	x86_sha256_update(&ctx->state, length, data);
 }
 
 static void
 x86_hmac_sha256_digest(struct hmac_sha256_ctx *ctx,
-			   unsigned length, uint8_t * digest)
+			   size_t length, uint8_t * digest)
 {
 	HMAC_DIGEST(ctx, &x86_sha256, length, digest);
 }
 
 static void
 x86_hmac_sha224_set_key(struct hmac_sha224_ctx *ctx,
-			    unsigned key_length, const uint8_t * key)
+			    size_t key_length, const uint8_t * key)
 {
 	HMAC_SET_KEY(ctx, &x86_sha224, key_length, key);
 }
 
 static void
 x86_hmac_sha224_digest(struct hmac_sha224_ctx *ctx,
-			   unsigned length, uint8_t * digest)
+			   size_t length, uint8_t * digest)
 {
 	HMAC_DIGEST(ctx, &x86_sha224, length, digest);
 }
@@ -118,35 +119,35 @@ x86_hmac_sha224_digest(struct hmac_sha224_ctx *ctx,
 #ifdef ENABLE_SHA512
 static void
 x86_hmac_sha384_set_key(struct hmac_sha384_ctx *ctx,
-			    unsigned key_length, const uint8_t * key)
+			    size_t key_length, const uint8_t * key)
 {
 	HMAC_SET_KEY(ctx, &x86_sha384, key_length, key);
 }
 
 static void
 x86_hmac_sha384_digest(struct hmac_sha384_ctx *ctx,
-			   unsigned length, uint8_t * digest)
+			   size_t length, uint8_t * digest)
 {
 	HMAC_DIGEST(ctx, &x86_sha384, length, digest);
 }
 
 static void
 x86_hmac_sha512_set_key(struct hmac_sha512_ctx *ctx,
-			    unsigned key_length, const uint8_t * key)
+			    size_t key_length, const uint8_t * key)
 {
 	HMAC_SET_KEY(ctx, &x86_sha512, key_length, key);
 }
 
 static void
 x86_hmac_sha512_update(struct hmac_sha512_ctx *ctx,
-			   unsigned length, const uint8_t * data)
+			   size_t length, const uint8_t * data)
 {
 	x86_sha512_update(&ctx->state, length, data);
 }
 
 static void
 x86_hmac_sha512_digest(struct hmac_sha512_ctx *ctx,
-			   unsigned length, uint8_t * digest)
+			   size_t length, uint8_t * digest)
 {
 	HMAC_DIGEST(ctx, &x86_sha512, length, digest);
 }
diff --git a/lib/accelerated/x86/sha-padlock.c b/lib/accelerated/x86/sha-padlock.c
index 70defe4704..76a862a7b3 100644
--- a/lib/accelerated/x86/sha-padlock.c
+++ b/lib/accelerated/x86/sha-padlock.c
@@ -34,9 +34,9 @@
 
 #ifdef HAVE_LIBNETTLE
 
-typedef void (*update_func) (void *, unsigned, const uint8_t *);
-typedef void (*digest_func) (void *, unsigned, uint8_t *);
-typedef void (*set_key_func) (void *, unsigned, const uint8_t *);
+typedef void (*update_func) (void *, size_t, const uint8_t *);
+typedef void (*digest_func) (void *, size_t, uint8_t *);
+typedef void (*set_key_func) (void *, size_t, const uint8_t *);
 typedef void (*init_func) (void *);
 
 struct padlock_hash_ctx {
@@ -60,7 +60,7 @@ wrap_padlock_hash_update(void *_ctx, const void *text, size_t textsize)
 {
 	struct padlock_hash_ctx *ctx = _ctx;
 
-	ctx->update(ctx->ctx_ptr, textsize, text);
+	_NETTLE_UPDATE(ctx->update, ctx->ctx_ptr, textsize, text);
 
 	return GNUTLS_E_SUCCESS;
 }
@@ -81,21 +81,21 @@ static void wrap_padlock_hash_deinit(void *hd)
 
 void
 padlock_sha1_update(struct sha1_ctx *ctx,
-		    unsigned length, const uint8_t * data)
+		    _NETTLE_SIZE_T length, const uint8_t * data)
 {
 	MD_UPDATE(ctx, length, data, SHA1_COMPRESS, MD1_INCR(ctx));
 }
 
 void
 padlock_sha256_update(struct sha256_ctx *ctx,
-		      unsigned length, const uint8_t * data)
+		      _NETTLE_SIZE_T length, const uint8_t * data)
 {
 	MD_UPDATE(ctx, length, data, SHA256_COMPRESS, MD1_INCR(ctx));
 }
 
 void
 padlock_sha512_update(struct sha512_ctx *ctx,
-		      unsigned length, const uint8_t * data)
+		      _NETTLE_SIZE_T length, const uint8_t * data)
 {
 	MD_UPDATE(ctx, length, data, SHA512_COMPRESS, MD_INCR(ctx));
 }
@@ -136,7 +136,7 @@ _nettle_write_be32(unsigned length, uint8_t * dst, uint32_t * src)
 
 static void
 padlock_sha1_digest(struct sha1_ctx *ctx,
-		    unsigned length, uint8_t * digest)
+		    size_t length, uint8_t * digest)
 {
 #ifdef USE_NETTLE3
 	uint64_t bit_count;
@@ -167,7 +167,7 @@ padlock_sha1_digest(struct sha1_ctx *ctx,
 
 static void
 padlock_sha256_digest(struct sha256_ctx *ctx,
-		      unsigned length, uint8_t * digest)
+		      size_t length, uint8_t * digest)
 {
 #ifdef USE_NETTLE3
 	uint64_t bit_count;
@@ -200,7 +200,7 @@ padlock_sha256_digest(struct sha256_ctx *ctx,
 
 static void
 padlock_sha512_digest(struct sha512_ctx *ctx,
-		      unsigned length, uint8_t * digest)
+		      size_t length, uint8_t * digest)
 {
 	uint64_t high, low;
 
diff --git a/lib/accelerated/x86/sha-padlock.h b/lib/accelerated/x86/sha-padlock.h
index 5cbe1d13b0..ce18ef8e33 100644
--- a/lib/accelerated/x86/sha-padlock.h
+++ b/lib/accelerated/x86/sha-padlock.h
@@ -2,6 +2,7 @@
 #define SHA_PADLOCK_H
 
 #include <nettle/sha.h>
+#include "nettle/gnettle.h"
 
 void padlock_sha1_oneshot(void *ctx, const void *inp, size_t len);
 void padlock_sha256_oneshot(void *ctx, const void *inp, size_t len);
@@ -18,11 +19,11 @@ int wrap_padlock_hash_fast(gnutls_digest_algorithm_t algo,
 			   void *digest);
 
 void padlock_sha1_update(struct sha1_ctx *ctx,
-			 unsigned length, const uint8_t * data);
+			 _NETTLE_SIZE_T length, const uint8_t * data);
 void padlock_sha256_update(struct sha256_ctx *ctx,
-			   unsigned length, const uint8_t * data);
+			   _NETTLE_SIZE_T length, const uint8_t * data);
 void padlock_sha512_update(struct sha512_ctx *ctx,
-			   unsigned length, const uint8_t * data);
+			   _NETTLE_SIZE_T length, const uint8_t * data);
 
 extern const struct nettle_hash padlock_sha1;
 extern const struct nettle_hash padlock_sha224;
diff --git a/lib/accelerated/x86/sha-x86-ssse3.c b/lib/accelerated/x86/sha-x86-ssse3.c
index 4fc106c87d..efcb5abf04 100644
--- a/lib/accelerated/x86/sha-x86-ssse3.c
+++ b/lib/accelerated/x86/sha-x86-ssse3.c
@@ -35,9 +35,9 @@ void sha1_block_data_order(void *c, const void *p, size_t len);
 void sha256_block_data_order(void *c, const void *p, size_t len);
 void sha512_block_data_order(void *c, const void *p, size_t len);
 
-typedef void (*update_func) (void *, unsigned, const uint8_t *);
-typedef void (*digest_func) (void *, unsigned, uint8_t *);
-typedef void (*set_key_func) (void *, unsigned, const uint8_t *);
+typedef void (*update_func) (void *, size_t, const uint8_t *);
+typedef void (*digest_func) (void *, size_t, uint8_t *);
+typedef void (*set_key_func) (void *, size_t, const uint8_t *);
 typedef void (*init_func) (void *);
 
 struct x86_hash_ctx {
@@ -63,7 +63,7 @@ wrap_x86_hash_update(void *_ctx, const void *text, size_t textsize)
 {
 	struct x86_hash_ctx *ctx = _ctx;
 
-	ctx->update(ctx->ctx_ptr, textsize, text);
+	_NETTLE_UPDATE(ctx->update, ctx->ctx_ptr, textsize, text);
 
 	return GNUTLS_E_SUCCESS;
 }
@@ -73,7 +73,7 @@ static void wrap_x86_hash_deinit(void *hd)
 	gnutls_free(hd);
 }
 
-void x86_sha1_update(struct sha1_ctx *ctx, size_t length,
+void x86_sha1_update(struct sha1_ctx *ctx, _NETTLE_SIZE_T length,
 		     const uint8_t * data)
 {
 	struct {
@@ -136,7 +136,7 @@ void x86_sha1_update(struct sha1_ctx *ctx, size_t length,
 
 }
 
-void x86_sha256_update(struct sha256_ctx *ctx, size_t length,
+void x86_sha256_update(struct sha256_ctx *ctx, _NETTLE_SIZE_T length,
 		     const uint8_t * data)
 {
 	struct {
@@ -189,7 +189,7 @@ void x86_sha256_update(struct sha256_ctx *ctx, size_t length,
 }
 
 #ifdef ENABLE_SHA512
-void x86_sha512_update(struct sha512_ctx *ctx, size_t length,
+void x86_sha512_update(struct sha512_ctx *ctx, _NETTLE_SIZE_T length,
 		     const uint8_t * data)
 {
 	struct {
@@ -344,7 +344,7 @@ static int wrap_x86_hash_fast(gnutls_digest_algorithm_t algo,
 	if (ret < 0)
 		return gnutls_assert_val(ret);
 
-	ctx.update(&ctx, text_size, text);
+	_NETTLE_UPDATE(ctx.update, &ctx, text_size, text);
 	ctx.digest(&ctx, ctx.length, digest);
 
 	return 0;
diff --git a/lib/accelerated/x86/sha-x86.h b/lib/accelerated/x86/sha-x86.h
index 3304212538..245fe32464 100644
--- a/lib/accelerated/x86/sha-x86.h
+++ b/lib/accelerated/x86/sha-x86.h
@@ -2,6 +2,7 @@
 #define SHA_X86_H
 
 #include <nettle/sha.h>
+#include "nettle/gnettle.h"
 
 /* nettle's SHA512 is faster than openssl's */
 #undef ENABLE_SHA512
@@ -12,9 +13,9 @@ extern const struct nettle_hash x86_sha256;
 extern const struct nettle_hash x86_sha384;
 extern const struct nettle_hash x86_sha512;
 
-void x86_sha1_update(struct sha1_ctx *ctx, size_t length, const uint8_t * data);
-void x86_sha256_update(struct sha256_ctx *ctx, size_t length, const uint8_t * data);
-void x86_sha512_update(struct sha512_ctx *ctx, size_t length, const uint8_t * data);
+void x86_sha1_update(struct sha1_ctx *ctx, _NETTLE_SIZE_T length, const uint8_t * data);
+void x86_sha256_update(struct sha256_ctx *ctx, _NETTLE_SIZE_T length, const uint8_t * data);
+void x86_sha512_update(struct sha512_ctx *ctx, _NETTLE_SIZE_T length, const uint8_t * data);
 
 extern const gnutls_crypto_digest_st _gnutls_sha_x86_ssse3;
 extern const gnutls_crypto_mac_st _gnutls_hmac_sha_x86_ssse3;
diff --git a/lib/accelerated/x86/x86-common.h b/lib/accelerated/x86/x86-common.h
index 3c3f6e808f..6261bce534 100644
--- a/lib/accelerated/x86/x86-common.h
+++ b/lib/accelerated/x86/x86-common.h
@@ -22,6 +22,8 @@
 
 #include <config.h>
 
+#include "nettle/gnettle.h"
+
 extern unsigned int _gnutls_x86_cpuid_s[3];
 
 #if defined(ASM_X86)
diff --git a/lib/nettle/gnettle.h b/lib/nettle/gnettle.h
index e27149c3ea..7616dea2b1 100644
--- a/lib/nettle/gnettle.h
+++ b/lib/nettle/gnettle.h
@@ -1,3 +1,30 @@
+#ifndef GNETTLE_H
+# define GNETTLE_H
+
+#include <config.h>
+#include <minmax.h>
+#include <limits.h>
+
 #define PRIME_CHECK_PARAM 8
 #define TOMPZ(x) ((__mpz_struct*)(x))
 #define SIZEOF_MPZT sizeof(__mpz_struct)
+
+#ifdef USE_NETTLE3
+# define _NETTLE_SIZE_T size_t
+# define _NETTLE_UPDATE(func, ctx, size, data) func(ctx, size, data)
+#else
+# define _NETTLE_SIZE_T unsigned
+#define _NETTLE_UPDATE(func, ptr, size, data) { \
+	size_t _rsize = size; \
+	const uint8_t *_t = text; \
+	while(size > 0) { \
+		_rsize = MIN(size, UINT_MAX); \
+		func(ptr, _rsize, _t); \
+		size -= _rsize; \
+		_t += _rsize; \
+	} \
+}
+
+#endif
+
+#endif
diff --git a/lib/nettle/mac.c b/lib/nettle/mac.c
index cfa37e3479..cf4fd70c2b 100644
--- a/lib/nettle/mac.c
+++ b/lib/nettle/mac.c
@@ -31,12 +31,13 @@
 #include <nettle/sha.h>
 #include <nettle/hmac.h>
 #include <nettle/umac.h>
+#include "gnettle.h"
 #include <fips.h>
 
-typedef void (*update_func) (void *, unsigned, const uint8_t *);
-typedef void (*digest_func) (void *, unsigned, uint8_t *);
-typedef void (*set_key_func) (void *, unsigned, const uint8_t *);
-typedef void (*set_nonce_func) (void *, unsigned, const uint8_t *);
+typedef void (*update_func) (void *, _NETTLE_SIZE_T, const uint8_t *);
+typedef void (*digest_func) (void *, _NETTLE_SIZE_T, uint8_t *);
+typedef void (*set_key_func) (void *, _NETTLE_SIZE_T, const uint8_t *);
+typedef void (*set_nonce_func) (void *, _NETTLE_SIZE_T, const uint8_t *);
 
 static int wrap_nettle_hash_init(gnutls_digest_algorithm_t algo,
 				 void **_ctx);
@@ -80,7 +81,7 @@ struct nettle_mac_ctx {
 };
 
 static void
-_wrap_umac96_set_key(void *ctx, unsigned len, const uint8_t * key)
+_wrap_umac96_set_key(void *ctx, _NETTLE_SIZE_T len, const uint8_t * key)
 {
 	if (unlikely(len != 16))
 		abort();
@@ -88,7 +89,7 @@ _wrap_umac96_set_key(void *ctx, unsigned len, const uint8_t * key)
 }
 
 static void
-_wrap_umac128_set_key(void *ctx, unsigned len, const uint8_t * key)
+_wrap_umac128_set_key(void *ctx, _NETTLE_SIZE_T len, const uint8_t * key)
 {
 	if (unlikely(len != 16))
 		abort();
@@ -188,7 +189,7 @@ static int wrap_nettle_mac_fast(gnutls_mac_algorithm_t algo,
 	if (ctx.set_nonce)
 		ctx.set_nonce(&ctx, nonce_size, nonce);
 	ctx.set_key(&ctx, key_size, key);
-	ctx.update(&ctx, text_size, text);
+	_NETTLE_UPDATE(ctx.update, &ctx, text_size, text);
 	ctx.digest(&ctx, ctx.length, digest);
 	
 	zeroize_temp_key(&ctx, sizeof(ctx));
@@ -269,7 +270,7 @@ wrap_nettle_mac_update(void *_ctx, const void *text, size_t textsize)
 {
 	struct nettle_mac_ctx *ctx = _ctx;
 
-	ctx->update(ctx->ctx_ptr, textsize, text);
+	_NETTLE_UPDATE(ctx->update, ctx->ctx_ptr, textsize, text);
 
 	return GNUTLS_E_SUCCESS;
 }
@@ -305,7 +306,7 @@ wrap_nettle_hash_update(void *_ctx, const void *text, size_t textsize)
 {
 	struct nettle_hash_ctx *ctx = _ctx;
 
-	ctx->update(ctx->ctx_ptr, textsize, text);
+	_NETTLE_UPDATE(ctx->update, ctx->ctx_ptr, textsize, text);
 
 	return GNUTLS_E_SUCCESS;
 }
@@ -411,7 +412,7 @@ static int wrap_nettle_hash_fast(gnutls_digest_algorithm_t algo,
 	if (ret < 0)
 		return gnutls_assert_val(ret);
 
-	ctx.update(&ctx, text_size, text);
+	_NETTLE_UPDATE(ctx.update, &ctx, text_size, text);
 	ctx.digest(&ctx, ctx.length, digest);
 
 	return 0;