openjpegdec: Properly offset and shift for all formats when converting

And while at it, also clean up some code.

1 files changed, 114 insertions, 133 deletions

diff --git a/ext/openjpeg/gstopenjpegdec.c b/ext/openjpeg/gstopenjpegdec.c
index 5c6a2287a..f8929e09b 100644
--- a/ext/openjpeg/gstopenjpegdec.c
+++ b/ext/openjpeg/gstopenjpegdec.c
@@ -199,29 +199,30 @@ gst_openjpeg_dec_set_format (GstVideoDecoder * decoder,
 static void
 fill_frame_packed8_4 (GstVideoFrame * frame, opj_image_t * image)
 {
-  gint x, y, w, h;
+  gint x, y, w, h, c;
   guint8 *data_out, *tmp;
   const gint *data_in[4];
   gint dstride;
+  gint off[4];
 
   w = GST_VIDEO_FRAME_WIDTH (frame);
   h = GST_VIDEO_FRAME_HEIGHT (frame);
   data_out = GST_VIDEO_FRAME_PLANE_DATA (frame, 0);
   dstride = GST_VIDEO_FRAME_PLANE_STRIDE (frame, 0);
 
-  data_in[0] = image->comps[0].data;
-  data_in[1] = image->comps[1].data;
-  data_in[2] = image->comps[2].data;
-  data_in[3] = image->comps[3].data;
+  for (c = 0; c < 4; c++) {
+    data_in[c] = image->comps[c].data;
+    off[c] = 0x80 * image->comps[c].sgnd;
+  }
 
   for (y = 0; y < h; y++) {
     tmp = data_out;
 
     for (x = 0; x < w; x++) {
-      tmp[0] = *data_in[3];
-      tmp[1] = *data_in[0];
-      tmp[2] = *data_in[1];
-      tmp[3] = *data_in[2];
+      tmp[0] = off[3] + *data_in[3];
+      tmp[1] = off[0] + *data_in[0];
+      tmp[2] = off[1] + *data_in[1];
+      tmp[3] = off[2] + *data_in[2];
 
       tmp += 4;
       data_in[0]++;
@@ -236,35 +237,33 @@ fill_frame_packed8_4 (GstVideoFrame * frame, opj_image_t * image)
 static void
 fill_frame_packed16_4 (GstVideoFrame * frame, opj_image_t * image)
 {
-  gint x, y, w, h;
+  gint x, y, w, h, c;
   guint16 *data_out, *tmp;
   const gint *data_in[4];
   gint dstride;
-  gint shift[4];
+  gint shift[4], off[4];
 
   w = GST_VIDEO_FRAME_WIDTH (frame);
   h = GST_VIDEO_FRAME_HEIGHT (frame);
   data_out = GST_VIDEO_FRAME_PLANE_DATA (frame, 0);
   dstride = GST_VIDEO_FRAME_PLANE_STRIDE (frame, 0) / 2;
 
-  data_in[0] = image->comps[0].data;
-  data_in[1] = image->comps[1].data;
-  data_in[2] = image->comps[2].data;
-  data_in[3] = image->comps[3].data;
-
-  shift[0] = 16 - image->comps[0].prec;
-  shift[1] = 16 - image->comps[1].prec;
-  shift[2] = 16 - image->comps[2].prec;
-  shift[3] = 16 - image->comps[3].prec;
+  for (c = 0; c < 4; c++) {
+    data_in[c] = image->comps[c].data;
+    off[c] = (1 << (image->comps[c].prec - 1)) * image->comps[c].sgnd;
+    shift[c] =
+        MAX (MIN (GST_VIDEO_FRAME_COMP_DEPTH (frame, c) - image->comps[c].prec,
+            8), 0);
+  }
 
   for (y = 0; y < h; y++) {
     tmp = data_out;
 
     for (x = 0; x < w; x++) {
-      tmp[0] = *data_in[3] << shift[3];
-      tmp[1] = *data_in[0] << shift[0];
-      tmp[2] = *data_in[1] << shift[1];
-      tmp[3] = *data_in[2] << shift[2];
+      tmp[0] = off[3] + (*data_in[3] << shift[3]);
+      tmp[1] = off[0] + (*data_in[0] << shift[0]);
+      tmp[2] = off[1] + (*data_in[1] << shift[1]);
+      tmp[3] = off[2] + (*data_in[2] << shift[2]);
 
       tmp += 4;
       data_in[0]++;
@@ -279,27 +278,29 @@ fill_frame_packed16_4 (GstVideoFrame * frame, opj_image_t * image)
 static void
 fill_frame_packed8_3 (GstVideoFrame * frame, opj_image_t * image)
 {
-  gint x, y, w, h;
+  gint x, y, w, h, c;
   guint8 *data_out, *tmp;
   const gint *data_in[3];
   gint dstride;
+  gint off[3];
 
   w = GST_VIDEO_FRAME_WIDTH (frame);
   h = GST_VIDEO_FRAME_HEIGHT (frame);
   data_out = GST_VIDEO_FRAME_PLANE_DATA (frame, 0);
   dstride = GST_VIDEO_FRAME_PLANE_STRIDE (frame, 0);
 
-  data_in[0] = image->comps[0].data;
-  data_in[1] = image->comps[1].data;
-  data_in[2] = image->comps[2].data;
+  for (c = 0; c < 3; c++) {
+    data_in[c] = image->comps[c].data;
+    off[c] = 0x80 * image->comps[c].sgnd;
+  };
 
   for (y = 0; y < h; y++) {
     tmp = data_out;
 
     for (x = 0; x < w; x++) {
-      tmp[1] = *data_in[0];
-      tmp[2] = *data_in[1];
-      tmp[3] = *data_in[2];
+      tmp[1] = off[0] + *data_in[0];
+      tmp[2] = off[1] + *data_in[1];
+      tmp[3] = off[2] + *data_in[2];
 
       tmp += 4;
       data_in[0]++;
@@ -313,32 +314,32 @@ fill_frame_packed8_3 (GstVideoFrame * frame, opj_image_t * image)
 static void
 fill_frame_packed16_3 (GstVideoFrame * frame, opj_image_t * image)
 {
-  gint x, y, w, h;
+  gint x, y, w, h, c;
   guint16 *data_out, *tmp;
   const gint *data_in[3];
   gint dstride;
-  gint shift[3];
+  gint shift[3], off[3];
 
   w = GST_VIDEO_FRAME_WIDTH (frame);
   h = GST_VIDEO_FRAME_HEIGHT (frame);
   data_out = GST_VIDEO_FRAME_PLANE_DATA (frame, 0);
   dstride = GST_VIDEO_FRAME_PLANE_STRIDE (frame, 0) / 2;
 
-  data_in[0] = image->comps[0].data;
-  data_in[1] = image->comps[1].data;
-  data_in[2] = image->comps[2].data;
-
-  shift[0] = 16 - image->comps[0].prec;
-  shift[1] = 16 - image->comps[1].prec;
-  shift[2] = 16 - image->comps[2].prec;
+  for (c = 0; c < 3; c++) {
+    data_in[c] = image->comps[c].data;
+    off[c] = (1 << (image->comps[c].prec - 1)) * image->comps[c].sgnd;
+    shift[c] =
+        MAX (MIN (GST_VIDEO_FRAME_COMP_DEPTH (frame, c) - image->comps[c].prec,
+            8), 0);
+  }
 
   for (y = 0; y < h; y++) {
     tmp = data_out;
 
     for (x = 0; x < w; x++) {
-      tmp[1] = *data_in[0] << shift[0];
-      tmp[2] = *data_in[1] << shift[1];
-      tmp[3] = *data_in[2] << shift[2];
+      tmp[1] = off[0] + (*data_in[0] << shift[0]);
+      tmp[2] = off[1] + (*data_in[1] << shift[1]);
+      tmp[3] = off[2] + (*data_in[2] << shift[2]);
 
       tmp += 4;
       data_in[0]++;
@@ -356,6 +357,7 @@ fill_frame_planar8_1 (GstVideoFrame * frame, opj_image_t * image)
   guint8 *data_out, *tmp;
   const gint *data_in;
   gint dstride;
+  gint off;
 
   w = GST_VIDEO_FRAME_WIDTH (frame);
   h = GST_VIDEO_FRAME_HEIGHT (frame);
@@ -363,12 +365,13 @@ fill_frame_planar8_1 (GstVideoFrame * frame, opj_image_t * image)
   dstride = GST_VIDEO_FRAME_PLANE_STRIDE (frame, 0);
 
   data_in = image->comps[0].data;
+  off = 0x80 * image->comps[0].sgnd;
 
   for (y = 0; y < h; y++) {
     tmp = data_out;
 
     for (x = 0; x < w; x++) {
-      *tmp = *data_in;
+      *tmp = off + *data_in;
 
       tmp++;
       data_in++;
@@ -384,7 +387,7 @@ fill_frame_planar16_1 (GstVideoFrame * frame, opj_image_t * image)
   guint16 *data_out, *tmp;
   const gint *data_in;
   gint dstride;
-  gint shift;
+  gint shift, off;
 
   w = GST_VIDEO_FRAME_WIDTH (frame);
   h = GST_VIDEO_FRAME_HEIGHT (frame);
@@ -393,13 +396,16 @@ fill_frame_planar16_1 (GstVideoFrame * frame, opj_image_t * image)
 
   data_in = image->comps[0].data;
 
-  shift = 16 - image->comps[0].prec;
+  off = (1 << (image->comps[0].prec - 1)) * image->comps[0].sgnd;
+  shift =
+      MAX (MIN (GST_VIDEO_FRAME_COMP_DEPTH (frame, 0) - image->comps[0].prec,
+          8), 0);
 
   for (y = 0; y < h; y++) {
     tmp = data_out;
 
     for (x = 0; x < w; x++) {
-      *tmp = *data_in << shift;
+      *tmp = off + (*data_in << shift);
 
       tmp++;
       data_in++;
@@ -414,7 +420,7 @@ fill_frame_planar8_3 (GstVideoFrame * frame, opj_image_t * image)
   gint c, x, y, w, h;
   guint8 *data_out, *tmp;
   const gint *data_in;
-  gint dstride;
+  gint dstride, off;
 
   for (c = 0; c < 3; c++) {
     w = GST_VIDEO_FRAME_COMP_WIDTH (frame, c);
@@ -422,12 +428,13 @@ fill_frame_planar8_3 (GstVideoFrame * frame, opj_image_t * image)
     dstride = GST_VIDEO_FRAME_COMP_STRIDE (frame, c);
     data_out = GST_VIDEO_FRAME_COMP_DATA (frame, c);
     data_in = image->comps[c].data;
+    off = 0x80 * image->comps[c].sgnd;
 
     for (y = 0; y < h; y++) {
       tmp = data_out;
 
       for (x = 0; x < w; x++) {
-        *tmp = *data_in;
+        *tmp = off + *data_in;
         tmp++;
         data_in++;
       }
@@ -443,7 +450,7 @@ fill_frame_planar16_3 (GstVideoFrame * frame, opj_image_t * image)
   guint16 *data_out, *tmp;
   const gint *data_in;
   gint dstride;
-  gint shift;
+  gint shift, off;
 
   for (c = 0; c < 3; c++) {
     w = GST_VIDEO_FRAME_COMP_WIDTH (frame, c);
@@ -451,13 +458,16 @@ fill_frame_planar16_3 (GstVideoFrame * frame, opj_image_t * image)
     dstride = GST_VIDEO_FRAME_COMP_STRIDE (frame, c) / 2;
     data_out = (guint16 *) GST_VIDEO_FRAME_COMP_DATA (frame, c);
     data_in = image->comps[c].data;
-    shift = 16 - image->comps[c].prec;
+    off = (1 << (image->comps[c].prec - 1)) * image->comps[c].sgnd;
+    shift =
+        MAX (MIN (GST_VIDEO_FRAME_COMP_DEPTH (frame, c) - image->comps[c].prec,
+            8), 0);
 
     for (y = 0; y < h; y++) {
       tmp = data_out;
 
       for (x = 0; x < w; x++) {
-        *tmp = *data_in << shift;
+        *tmp = off + (*data_in << shift);
         tmp++;
         data_in++;
       }
@@ -469,37 +479,32 @@ fill_frame_planar16_3 (GstVideoFrame * frame, opj_image_t * image)
 static void
 fill_frame_planar8_3_generic (GstVideoFrame * frame, opj_image_t * image)
 {
-  gint x, y, w, h;
+  gint x, y, w, h, c;
   guint8 *data_out, *tmp;
   const gint *data_in[3];
   gint dstride;
-  gint dx[3], dy[3];
+  gint dx[3], dy[3], off[3];
 
   w = GST_VIDEO_FRAME_WIDTH (frame);
   h = GST_VIDEO_FRAME_HEIGHT (frame);
   data_out = GST_VIDEO_FRAME_PLANE_DATA (frame, 0);
   dstride = GST_VIDEO_FRAME_PLANE_STRIDE (frame, 0);
 
-  data_in[0] = image->comps[0].data;
-  data_in[1] = image->comps[1].data;
-  data_in[2] = image->comps[2].data;
-
-  dx[0] = image->comps[0].dx;
-  dx[1] = image->comps[1].dx;
-  dx[2] = image->comps[2].dx;
-
-  dy[0] = image->comps[0].dy;
-  dy[1] = image->comps[1].dy;
-  dy[2] = image->comps[2].dy;
+  for (c = 0; c < 3; c++) {
+    data_in[c] = image->comps[c].data;
+    dx[c] = image->comps[c].dx;
+    dy[c] = image->comps[c].dy;
+    off[c] = 0x80 * image->comps[c].sgnd;
+  }
 
   for (y = 0; y < h; y++) {
     tmp = data_out;
 
     for (x = 0; x < w; x++) {
       tmp[0] = 0xff;
-      tmp[1] = data_in[0][((y / dy[0]) * w + x) / dx[0]];
-      tmp[2] = data_in[1][((y / dy[1]) * w + x) / dx[1]];
-      tmp[3] = data_in[2][((y / dy[2]) * w + x) / dx[2]];
+      tmp[1] = off[0] + data_in[0][((y / dy[0]) * w + x) / dx[0]];
+      tmp[2] = off[1] + data_in[1][((y / dy[1]) * w + x) / dx[1]];
+      tmp[3] = off[2] + data_in[2][((y / dy[2]) * w + x) / dx[2]];
       tmp += 4;
     }
     data_out += dstride;
@@ -509,40 +514,32 @@ fill_frame_planar8_3_generic (GstVideoFrame * frame, opj_image_t * image)
 static void
 fill_frame_planar8_4_generic (GstVideoFrame * frame, opj_image_t * image)
 {
-  gint x, y, w, h;
+  gint x, y, w, h, c;
   guint8 *data_out, *tmp;
   const gint *data_in[4];
   gint dstride;
-  gint dx[4], dy[4];
+  gint dx[4], dy[4], off[4];
 
   w = GST_VIDEO_FRAME_WIDTH (frame);
   h = GST_VIDEO_FRAME_HEIGHT (frame);
   data_out = GST_VIDEO_FRAME_PLANE_DATA (frame, 0);
   dstride = GST_VIDEO_FRAME_PLANE_STRIDE (frame, 0);
 
-  data_in[0] = image->comps[0].data;
-  data_in[1] = image->comps[1].data;
-  data_in[2] = image->comps[2].data;
-  data_in[3] = image->comps[3].data;
-
-  dx[0] = image->comps[0].dx;
-  dx[1] = image->comps[1].dx;
-  dx[2] = image->comps[2].dx;
-  dx[3] = image->comps[3].dx;
-
-  dy[0] = image->comps[0].dy;
-  dy[1] = image->comps[1].dy;
-  dy[2] = image->comps[2].dy;
-  dy[3] = image->comps[3].dy;
+  for (c = 0; c < 4; c++) {
+    data_in[c] = image->comps[c].data;
+    dx[c] = image->comps[c].dx;
+    dy[c] = image->comps[c].dy;
+    off[c] = 0x80 * image->comps[c].sgnd;
+  }
 
   for (y = 0; y < h; y++) {
     tmp = data_out;
 
     for (x = 0; x < w; x++) {
-      tmp[0] = data_in[3][((y / dy[3]) * w + x) / dx[3]];
-      tmp[1] = data_in[0][((y / dy[0]) * w + x) / dx[0]];
-      tmp[2] = data_in[1][((y / dy[1]) * w + x) / dx[1]];
-      tmp[3] = data_in[2][((y / dy[2]) * w + x) / dx[2]];
+      tmp[0] = off[3] + data_in[3][((y / dy[3]) * w + x) / dx[3]];
+      tmp[1] = off[0] + data_in[0][((y / dy[0]) * w + x) / dx[0]];
+      tmp[2] = off[1] + data_in[1][((y / dy[1]) * w + x) / dx[1]];
+      tmp[3] = off[2] + data_in[2][((y / dy[2]) * w + x) / dx[2]];
       tmp += 4;
     }
     data_out += dstride;
@@ -552,41 +549,35 @@ fill_frame_planar8_4_generic (GstVideoFrame * frame, opj_image_t * image)
 static void
 fill_frame_planar16_3_generic (GstVideoFrame * frame, opj_image_t * image)
 {
-  gint x, y, w, h;
+  gint x, y, w, h, c;
   guint16 *data_out, *tmp;
   const gint *data_in[3];
   gint dstride;
-  gint dx[3], dy[3], shift[3];
+  gint dx[3], dy[3], shift[3], off[3];
 
   w = GST_VIDEO_FRAME_WIDTH (frame);
   h = GST_VIDEO_FRAME_HEIGHT (frame);
   data_out = (guint16 *) GST_VIDEO_FRAME_PLANE_DATA (frame, 0);
   dstride = GST_VIDEO_FRAME_PLANE_STRIDE (frame, 0) / 2;
 
-  data_in[0] = image->comps[0].data;
-  data_in[1] = image->comps[1].data;
-  data_in[2] = image->comps[2].data;
-
-  dx[0] = image->comps[0].dx;
-  dx[1] = image->comps[1].dx;
-  dx[2] = image->comps[2].dx;
-
-  dy[0] = image->comps[0].dy;
-  dy[1] = image->comps[1].dy;
-  dy[2] = image->comps[2].dy;
-
-  shift[0] = 16 - image->comps[0].prec;
-  shift[1] = 16 - image->comps[1].prec;
-  shift[2] = 16 - image->comps[2].prec;
+  for (c = 0; c < 3; c++) {
+    dx[c] = image->comps[c].dx;
+    dy[c] = image->comps[c].dy;
+    data_in[c] = image->comps[c].data;
+    off[c] = (1 << (image->comps[c].prec - 1)) * image->comps[c].sgnd;
+    shift[c] =
+        MAX (MIN (GST_VIDEO_FRAME_COMP_DEPTH (frame, c) - image->comps[c].prec,
+            8), 0);
+  }
 
   for (y = 0; y < h; y++) {
     tmp = data_out;
 
     for (x = 0; x < w; x++) {
       tmp[0] = 0xff;
-      tmp[1] = data_in[0][((y / dy[0]) * w + x) / dx[0]] << shift[0];
-      tmp[2] = data_in[1][((y / dy[1]) * w + x) / dx[1]] << shift[1];
-      tmp[3] = data_in[2][((y / dy[2]) * w + x) / dx[2]] << shift[2];
+      tmp[1] = off[0] + (data_in[0][((y / dy[0]) * w + x) / dx[0]] << shift[0]);
+      tmp[2] = off[1] + (data_in[1][((y / dy[1]) * w + x) / dx[1]] << shift[1]);
+      tmp[3] = off[2] + (data_in[2][((y / dy[2]) * w + x) / dx[2]] << shift[2]);
       tmp += 4;
     }
     data_out += dstride;
@@ -596,45 +587,35 @@ fill_frame_planar16_3_generic (GstVideoFrame * frame, opj_image_t * image)
 static void
 fill_frame_planar16_4_generic (GstVideoFrame * frame, opj_image_t * image)
 {
-  gint x, y, w, h;
+  gint x, y, w, h, c;
   guint16 *data_out, *tmp;
   const gint *data_in[4];
   gint dstride;
-  gint dx[4], dy[4], shift[4];
+  gint dx[4], dy[4], shift[4], off[4];
 
   w = GST_VIDEO_FRAME_WIDTH (frame);
   h = GST_VIDEO_FRAME_HEIGHT (frame);
   data_out = (guint16 *) GST_VIDEO_FRAME_PLANE_DATA (frame, 0);
   dstride = GST_VIDEO_FRAME_PLANE_STRIDE (frame, 0) / 2;
 
-  data_in[0] = image->comps[0].data;
-  data_in[1] = image->comps[1].data;
-  data_in[2] = image->comps[2].data;
-  data_in[3] = image->comps[3].data;
-
-  dx[0] = image->comps[0].dx;
-  dx[1] = image->comps[1].dx;
-  dx[2] = image->comps[2].dx;
-  dx[3] = image->comps[3].dx;
-
-  dy[0] = image->comps[0].dy;
-  dy[1] = image->comps[1].dy;
-  dy[2] = image->comps[2].dy;
-  dy[3] = image->comps[3].dy;
-
-  shift[0] = 16 - image->comps[0].prec;
-  shift[1] = 16 - image->comps[1].prec;
-  shift[2] = 16 - image->comps[2].prec;
-  shift[3] = 16 - image->comps[3].prec;
+  for (c = 0; c < 4; c++) {
+    dx[c] = image->comps[c].dx;
+    dy[c] = image->comps[c].dy;
+    data_in[c] = image->comps[c].data;
+    off[c] = (1 << (image->comps[c].prec - 1)) * image->comps[c].sgnd;
+    shift[c] =
+        MAX (MIN (GST_VIDEO_FRAME_COMP_DEPTH (frame, c) - image->comps[c].prec,
+            8), 0);
+  }
 
   for (y = 0; y < h; y++) {
     tmp = data_out;
 
     for (x = 0; x < w; x++) {
-      tmp[0] = data_in[3][((y / dy[3]) * w + x) / dx[3]] << shift[3];
-      tmp[1] = data_in[0][((y / dy[0]) * w + x) / dx[0]] << shift[0];
-      tmp[2] = data_in[1][((y / dy[1]) * w + x) / dx[1]] << shift[1];
-      tmp[3] = data_in[2][((y / dy[2]) * w + x) / dx[2]] << shift[2];
+      tmp[0] = off[3] + (data_in[3][((y / dy[3]) * w + x) / dx[3]] << shift[3]);
+      tmp[1] = off[0] + (data_in[0][((y / dy[0]) * w + x) / dx[0]] << shift[0]);
+      tmp[2] = off[1] + (data_in[1][((y / dy[1]) * w + x) / dx[1]] << shift[1]);
+      tmp[3] = off[2] + (data_in[2][((y / dy[2]) * w + x) / dx[2]] << shift[2]);
       tmp += 4;
     }
     data_out += dstride;