Bug 698135: Import security fix for OpenJPEG

https://pdfium.googlesource.com/pdfium/+/463b77b4f1e4257cd89f3460b5a6fdb102f44265%5E%21/#F0

Patch doesn't apply cleanly, but requires some manual conflict
resolution.

1 files changed, 41 insertions, 34 deletions

diff --git a/openjpeg/src/lib/openjp2/dwt.c b/openjpeg/src/lib/openjp2/dwt.c
index 9e2a36158..d8f69a943 100644
--- a/openjpeg/src/lib/openjp2/dwt.c
+++ b/openjpeg/src/lib/openjp2/dwt.c
@@ -46,14 +46,12 @@
 /** @defgroup DWT DWT - Implementation of a discrete wavelet transform */
 /*@{*/
 
-#define OPJ_WS(i) v->mem[(i)*2]
-#define OPJ_WD(i) v->mem[(1+(i)*2)]
-
 /** @name Local data structures */
 /*@{*/
 
 typedef struct dwt_local {
 	OPJ_INT32* mem;
+	OPJ_SIZE_T mem_count;
 	OPJ_INT32 dn;
 	OPJ_INT32 sn;
 	OPJ_INT32 cas;
@@ -107,16 +105,16 @@ static void opj_dwt_interleave_v(opj_dwt_t* v, OPJ_INT32 *a, OPJ_INT32 x);
 /**
 Forward 5-3 wavelet transform in 1-D
 */
-static void opj_dwt_encode_1(OPJ_INT32 *a, OPJ_INT32 dn, OPJ_INT32 sn, OPJ_INT32 cas);
+static void opj_dwt_encode_1(OPJ_INT32 *a, OPJ_SIZE_T a_count, OPJ_INT32 dn, OPJ_INT32 sn, OPJ_INT32 cas);
 /**
 Inverse 5-3 wavelet transform in 1-D
 */
 static void opj_dwt_decode_1(opj_dwt_t *v);
-static void opj_dwt_decode_1_(OPJ_INT32 *a, OPJ_INT32 dn, OPJ_INT32 sn, OPJ_INT32 cas);
+static void opj_dwt_decode_1_(OPJ_INT32 *a, OPJ_SIZE_T a_count, OPJ_INT32 dn, OPJ_INT32 sn, OPJ_INT32 cas);
 /**
 Forward 9-7 wavelet transform in 1-D
 */
-static void opj_dwt_encode_1_real(OPJ_INT32 *a, OPJ_INT32 dn, OPJ_INT32 sn, OPJ_INT32 cas);
+static void opj_dwt_encode_1_real(OPJ_INT32 *a, OPJ_SIZE_T a_count, OPJ_INT32 dn, OPJ_INT32 sn, OPJ_INT32 cas);
 /**
 Explicit calculation of the Quantization Stepsizes 
 */
@@ -124,10 +122,10 @@ static void opj_dwt_encode_stepsize(OPJ_INT32 stepsize, OPJ_INT32 numbps, opj_st
 /**
 Inverse wavelet transform in 2-D.
 */
-static OPJ_BOOL opj_dwt_decode_tile(opj_thread_pool_t* tp, opj_tcd_tilecomp_t* tilec, OPJ_UINT32 i, DWT1DFN fn);
+static OPJ_BOOL opj_dwt_decode_tile(opj_thread_pool_t* tp, const opj_tcd_tilecomp_t* tilec, OPJ_UINT32 i, DWT1DFN fn);
 
-static OPJ_BOOL opj_dwt_encode_procedure(	opj_tcd_tilecomp_t * tilec,
-										    void (*p_function)(OPJ_INT32 *, OPJ_INT32,OPJ_INT32,OPJ_INT32) );
+static OPJ_BOOL opj_dwt_encode_procedure(const opj_tcd_tilecomp_t * tilec,
+										    void (*p_function)(OPJ_INT32 *, OPJ_SIZE_T,OPJ_INT32,OPJ_INT32,OPJ_INT32) );
 
 static OPJ_UINT32 opj_dwt_max_resolution(opj_tcd_resolution_t* OPJ_RESTRICT r, OPJ_UINT32 i);
 
@@ -156,13 +154,20 @@ static void opj_v4dwt_decode_step2(opj_v4_t* l, opj_v4_t* w, OPJ_INT32 k, OPJ_IN
 
 /*@}*/
 
-#define OPJ_S(i) a[(i)*2]
-#define OPJ_D(i) a[(1+(i)*2)]
-#define OPJ_S_(i) ((i)<0?OPJ_S(0):((i)>=sn?OPJ_S(sn-1):OPJ_S(i)))
-#define OPJ_D_(i) ((i)<0?OPJ_D(0):((i)>=dn?OPJ_D(dn-1):OPJ_D(i)))
+#define IDX_S(i) (i)*2
+#define IDX_D(i) 1+(i)*2
+#define UNDERFLOW_SN(i) ((i)>=sn&&sn>0)
+#define UNDERFLOW_DN(i) ((i)>=dn&&dn>0)
+#define OVERFLOW_S(i) (IDX_S(i)>=a_count)
+#define OVERFLOW_D(i) (IDX_D(i)>=a_count)
+
+#define OPJ_S(i) a[IDX_S(i)]
+#define OPJ_D(i) a[IDX_D(i)]
+#define OPJ_S_(i) ((i)<0?OPJ_S(0):(UNDERFLOW_SN(i)?OPJ_S(sn-1):OVERFLOW_S(i)?OPJ_S(i-1):OPJ_S(i)))
+#define OPJ_D_(i) ((i)<0?OPJ_D(0):(UNDERFLOW_DN(i)?OPJ_D(dn-1):OVERFLOW_D(i)?OPJ_D(i-1):OPJ_D(i)))
 /* new */
-#define OPJ_SS_(i) ((i)<0?OPJ_S(0):((i)>=dn?OPJ_S(dn-1):OPJ_S(i)))
-#define OPJ_DD_(i) ((i)<0?OPJ_D(0):((i)>=sn?OPJ_D(sn-1):OPJ_D(i)))
+#define OPJ_SS_(i) ((i)<0?OPJ_S(0):(UNDERFLOW_DN(i)?OPJ_S(dn-1):OVERFLOW_S(i)?OPJ_S(i-1):OPJ_S(i)))
+#define OPJ_DD_(i) ((i)<0?OPJ_D(0):(UNDERFLOW_SN(i)?OPJ_D(sn-1):OVERFLOW_D(i)?OPJ_D(i-1):OPJ_D(i)))
 
 /* <summary>                                                              */
 /* This table contains the norms of the 5-3 wavelets for different bands. */
@@ -283,7 +288,7 @@ static void opj_dwt_interleave_v(opj_dwt_t* v, OPJ_INT32 *a, OPJ_INT32 x) {
 /* <summary>                            */
 /* Forward 5-3 wavelet transform in 1-D. */
 /* </summary>                           */
-static void opj_dwt_encode_1(OPJ_INT32 *a, OPJ_INT32 dn, OPJ_INT32 sn, OPJ_INT32 cas) {
+static void opj_dwt_encode_1(OPJ_INT32 *a, OPJ_SIZE_T a_count, OPJ_INT32 dn, OPJ_INT32 sn, OPJ_INT32 cas) {
 	OPJ_INT32 i;
 	
 	if (!cas) {
@@ -304,7 +309,7 @@ static void opj_dwt_encode_1(OPJ_INT32 *a, OPJ_INT32 dn, OPJ_INT32 sn, OPJ_INT32
 /* <summary>                            */
 /* Inverse 5-3 wavelet transform in 1-D. */
 /* </summary>                           */ 
-static void opj_dwt_decode_1_(OPJ_INT32 *a, OPJ_INT32 dn, OPJ_INT32 sn, OPJ_INT32 cas) {
+static void opj_dwt_decode_1_(OPJ_INT32 *a, OPJ_SIZE_T a_count, OPJ_INT32 dn, OPJ_INT32 sn, OPJ_INT32 cas) {
 	OPJ_INT32 i;
 	
 	if (!cas) {
@@ -326,13 +331,13 @@ static void opj_dwt_decode_1_(OPJ_INT32 *a, OPJ_INT32 dn, OPJ_INT32 sn, OPJ_INT3
 /* Inverse 5-3 wavelet transform in 1-D. */
 /* </summary>                           */ 
 static void opj_dwt_decode_1(opj_dwt_t *v) {
-	opj_dwt_decode_1_(v->mem, v->dn, v->sn, v->cas);
+	opj_dwt_decode_1_(v->mem, v->mem_count, v->dn, v->sn, v->cas);
 }
 
 /* <summary>                             */
 /* Forward 9-7 wavelet transform in 1-D. */
 /* </summary>                            */
-static void opj_dwt_encode_1_real(OPJ_INT32 *a, OPJ_INT32 dn, OPJ_INT32 sn, OPJ_INT32 cas) {
+static void opj_dwt_encode_1_real(OPJ_INT32 *a, OPJ_SIZE_T a_count, OPJ_INT32 dn, OPJ_INT32 sn, OPJ_INT32 cas) {
 	OPJ_INT32 i;
 	if (!cas) {
 		if ((dn > 0) || (sn > 1)) {	/* NEW :  CASE ONE ELEMENT */
@@ -385,7 +390,7 @@ static void opj_dwt_encode_stepsize(OPJ_INT32 stepsize, OPJ_INT32 numbps, opj_st
 /* <summary>                            */
 /* Forward 5-3 wavelet transform in 2-D. */
 /* </summary>                           */
-static INLINE OPJ_BOOL opj_dwt_encode_procedure(opj_tcd_tilecomp_t * tilec,void (*p_function)(OPJ_INT32 *, OPJ_INT32,OPJ_INT32,OPJ_INT32) )
+static INLINE OPJ_BOOL opj_dwt_encode_procedure(const opj_tcd_tilecomp_t * tilec,void (*p_function)(OPJ_INT32 *, OPJ_SIZE_T,OPJ_INT32,OPJ_INT32,OPJ_INT32) )
 {
 	OPJ_INT32 i, j, k;
 	OPJ_INT32 *a = 00;
@@ -395,7 +400,8 @@ static INLINE OPJ_BOOL opj_dwt_encode_procedure(opj_tcd_tilecomp_t * tilec,void
 
 	OPJ_INT32 rw;			/* width of the resolution level computed   */
 	OPJ_INT32 rh;			/* height of the resolution level computed  */
-	size_t l_data_size;
+	OPJ_SIZE_T l_data_count;
+	OPJ_SIZE_T l_data_size;
 
 	opj_tcd_resolution_t * l_cur_res = 0;
 	opj_tcd_resolution_t * l_last_res = 0;
@@ -407,13 +413,13 @@ static INLINE OPJ_BOOL opj_dwt_encode_procedure(opj_tcd_tilecomp_t * tilec,void
 	l_cur_res = tilec->resolutions + l;
 	l_last_res = l_cur_res - 1;
 
-	l_data_size = opj_dwt_max_resolution( tilec->resolutions,tilec->numresolutions);
+	l_data_count = opj_dwt_max_resolution( tilec->resolutions,tilec->numresolutions);
 	/* overflow check */
-	if (l_data_size > (SIZE_MAX / sizeof(OPJ_INT32))) {
+	if (l_data_count > (SIZE_MAX / sizeof(OPJ_INT32))) {
 		/* FIXME event manager error callback */
 		return OPJ_FALSE;
 	}
-	l_data_size *= sizeof(OPJ_INT32);
+	l_data_size = l_data_count * (OPJ_UINT32)sizeof(OPJ_INT32);
 	bj = (OPJ_INT32*)opj_malloc(l_data_size);
 	/* l_data_size is equal to 0 when numresolutions == 1 but bj is not used */
 	/* in that case, so do not error out */
@@ -445,7 +451,7 @@ static INLINE OPJ_BOOL opj_dwt_encode_procedure(opj_tcd_tilecomp_t * tilec,void
 				bj[k] = aj[k*w];
 			}
 
-			(*p_function) (bj, dn, sn, cas_col);
+			(*p_function) (bj, l_data_count, dn, sn, cas_col);
 
 			opj_dwt_deinterleave_v(bj, aj, dn, sn, w, cas_col);
 		}
@@ -456,7 +462,7 @@ static INLINE OPJ_BOOL opj_dwt_encode_procedure(opj_tcd_tilecomp_t * tilec,void
 		for (j = 0; j < rh; j++) {
 			aj = a + j * w;
 			for (k = 0; k < rw; k++)  bj[k] = aj[k];
-			(*p_function) (bj, dn, sn, cas_row);
+			(*p_function) (bj, l_data_count, dn, sn, cas_row);
 			opj_dwt_deinterleave_h(bj, aj, dn, sn, cas_row);
 		}
 
@@ -627,7 +633,7 @@ static void opj_dwt_decode_v_func(void* user_data, opj_tls_t* tls)
 /* <summary>                            */
 /* Inverse wavelet transform in 2-D.    */
 /* </summary>                           */
-static OPJ_BOOL opj_dwt_decode_tile(opj_thread_pool_t* tp, opj_tcd_tilecomp_t* tilec, OPJ_UINT32 numres, DWT1DFN dwt_1D)
+static OPJ_BOOL opj_dwt_decode_tile(opj_thread_pool_t* tp, const opj_tcd_tilecomp_t* tilec, OPJ_UINT32 numres, DWT1DFN dwt_1D)
 {
 	opj_dwt_t h;
 	opj_dwt_t v;
@@ -638,26 +644,25 @@ static OPJ_BOOL opj_dwt_decode_tile(opj_thread_pool_t* tp, opj_tcd_tilecomp_t* t
 	OPJ_UINT32 rh = (OPJ_UINT32)(tr->y1 - tr->y0);	/* height of the resolution level computed */
 
 	OPJ_UINT32 w = (OPJ_UINT32)(tilec->x1 - tilec->x0);
-	size_t h_mem_size;
 	int num_threads;
 	
 	if (numres == 1U) {
 		return OPJ_TRUE;
 	}
 	num_threads = opj_thread_pool_get_thread_count(tp);
-	h_mem_size = opj_dwt_max_resolution(tr, numres);
+	h.mem_count = opj_dwt_max_resolution(tr, numres);
 	/* overflow check */
-	if (h_mem_size > (SIZE_MAX / sizeof(OPJ_INT32))) {
+	if (h.mem_count > (SIZE_MAX / sizeof(OPJ_INT32))) {
 		/* FIXME event manager error callback */
 		return OPJ_FALSE;
 	}
-	h_mem_size *= sizeof(OPJ_INT32);
-	h.mem = (OPJ_INT32*)opj_aligned_malloc(h_mem_size);
+	h.mem = (OPJ_INT32*)opj_aligned_malloc(h.mem_count * sizeof(OPJ_INT32));
 	if (! h.mem){
 		/* FIXME event manager error callback */
 		return OPJ_FALSE;
 	}
 
+	v.mem_count = h.mem_count;
 	v.mem = h.mem;
 
 	while( --numres) {
@@ -717,7 +722,8 @@ static OPJ_BOOL opj_dwt_decode_tile(opj_thread_pool_t* tp, opj_tcd_tilecomp_t* t
 				if( j == (num_jobs - 1U) ) { /* this will take care of the overflow */
 					job->max_j = rh;
 				}
-				job->h.mem = (OPJ_INT32*)opj_aligned_malloc(h_mem_size);
+				job->h.mem = (OPJ_INT32*)opj_aligned_malloc(h.mem_count * sizeof(OPJ_INT32));
+                                job->h.mem_count = h.mem_count;
 				if (!job->h.mem)
 				{
 					/* FIXME event manager error callback */
@@ -781,7 +787,8 @@ static OPJ_BOOL opj_dwt_decode_tile(opj_thread_pool_t* tp, opj_tcd_tilecomp_t* t
 				if( j == (num_jobs - 1U) ) { /* this will take care of the overflow */
 					job->max_j = rw;
 				}
-				job->v.mem = (OPJ_INT32*)opj_aligned_malloc(h_mem_size);
+				job->v.mem = (OPJ_INT32*)opj_aligned_malloc(h.mem_count * sizeof(OPJ_INT32));
+                                job->v.mem_count = h.mem_count;
 				if (!job->v.mem)
 				{
 					/* FIXME event manager error callback */