applemedia: avoid implicit color conversions

Rework the GL texture code a little to avoid implicit color conversion inside
AVF/VT on both iOS and OSX.

4 files changed, 94 insertions, 49 deletions

diff --git a/sys/applemedia/avfvideosrc.m b/sys/applemedia/avfvideosrc.m
index 8d456a726..695f4749a 100644
--- a/sys/applemedia/avfvideosrc.m
+++ b/sys/applemedia/avfvideosrc.m
@@ -93,6 +93,7 @@ G_DEFINE_TYPE (GstAVFVideoSrc, gst_avf_video_src, GST_TYPE_PUSH_SRC);
   BOOL stopRequest;
 
   GstCaps *caps;
+  GstVideoFormat internalFormat;
   GstVideoFormat format;
   gint width, height;
   GstClockTime latency;
@@ -625,6 +626,7 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
   width = info.width;
   height = info.height;
   format = info.finfo->format;
+  internalFormat = GST_VIDEO_FORMAT_UNKNOWN;
   latency = gst_util_uint64_scale (GST_SECOND, info.fps_d, info.fps_n);
 
   dispatch_sync (mainQueue, ^{
@@ -664,22 +666,28 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
       }
     }
 
+    internalFormat = format;
     switch (format) {
       case GST_VIDEO_FORMAT_NV12:
         newformat = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
         break;
       case GST_VIDEO_FORMAT_UYVY:
-         newformat = kCVPixelFormatType_422YpCbCr8;
+        newformat = kCVPixelFormatType_422YpCbCr8;
         break;
       case GST_VIDEO_FORMAT_YUY2:
-         newformat = kCVPixelFormatType_422YpCbCr8_yuvs;
+        newformat = kCVPixelFormatType_422YpCbCr8_yuvs;
         break;
       case GST_VIDEO_FORMAT_RGBA:
-        /* In order to do RGBA, we negotiate BGRA (since RGBA is not supported
-         * if not in textures) and then we get RGBA textures via
-         * CVOpenGL*TextureCacheCreateTextureFromImage. Computers. */
+#if !HAVE_IOS
+        newformat = kCVPixelFormatType_422YpCbCr8;
+        internalFormat = GST_VIDEO_FORMAT_UYVY;
+#else
+        newformat = kCVPixelFormatType_32BGRA;
+        internalFormat = GST_VIDEO_FORMAT_BGRA;
+#endif
+        break;
       case GST_VIDEO_FORMAT_BGRA:
-         newformat = kCVPixelFormatType_32BGRA;
+        newformat = kCVPixelFormatType_32BGRA;
         break;
       default:
         *successPtr = NO;
@@ -688,10 +696,10 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
         return;
     }
 
-    GST_DEBUG_OBJECT(element,
-       "Width: %d Height: %d Format: %" GST_FOURCC_FORMAT,
-       width, height,
-       GST_FOURCC_ARGS (gst_video_format_to_fourcc (format)));
+    GST_INFO_OBJECT(element,
+        "width: %d height: %d format: %s internalFormat: %s", width, height,
+        gst_video_format_to_string (format),
+        gst_video_format_to_string (internalFormat));
 
     output.videoSettings = [NSDictionary
         dictionaryWithObject:[NSNumber numberWithInt:newformat]
@@ -790,10 +798,10 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
     gst_query_unref (query);
 
     if (glContext) {
-      GST_INFO_OBJECT (element, "pushing textures. GL context %p", glContext);
+      GST_INFO_OBJECT (element, "pushing textures. Internal format %s, context %p",
+          gst_video_format_to_string (internalFormat), glContext);
       textureCache = gst_core_video_texture_cache_new (glContext);
-      gst_core_video_texture_cache_set_format (textureCache,
-          "NV12", caps);
+      gst_core_video_texture_cache_set_format (textureCache, internalFormat, caps);
       gst_object_unref (glContext);
     } else {
       GST_WARNING_OBJECT (element, "got memory:GLMemory caps but not GL context from downstream element");
diff --git a/sys/applemedia/corevideotexturecache.h b/sys/applemedia/corevideotexturecache.h
index b90cbf4ab..d53ec41d2 100644
--- a/sys/applemedia/corevideotexturecache.h
+++ b/sys/applemedia/corevideotexturecache.h
@@ -42,7 +42,7 @@ typedef struct _GstCoreVideoTextureCache
 GstCoreVideoTextureCache *gst_core_video_texture_cache_new (GstGLContext * ctx);
 void gst_core_video_texture_cache_free (GstCoreVideoTextureCache * cache);
 void gst_core_video_texture_cache_set_format (GstCoreVideoTextureCache * cache,
-    const gchar * input_format, GstCaps * out_caps);
+    GstVideoFormat in_format, GstCaps * out_caps);
 gboolean gst_core_video_texture_cache_upload (GstVideoGLTextureUploadMeta * meta, guint texture_id[4]);
 GstBuffer * gst_core_video_texture_cache_get_gl_buffer (GstCoreVideoTextureCache * cache,
         GstBuffer * cv_buffer);
diff --git a/sys/applemedia/corevideotexturecache.m b/sys/applemedia/corevideotexturecache.m
index f935d3870..a35dcf65d 100644
--- a/sys/applemedia/corevideotexturecache.m
+++ b/sys/applemedia/corevideotexturecache.m
@@ -84,7 +84,7 @@ gst_core_video_texture_cache_free (GstCoreVideoTextureCache * cache)
 
 void
 gst_core_video_texture_cache_set_format (GstCoreVideoTextureCache * cache,
-    const gchar * input_format, GstCaps * out_caps)
+    GstVideoFormat in_format, GstCaps * out_caps)
 {
   GstCaps *in_caps;
   GstCapsFeatures *features;
@@ -97,7 +97,8 @@ gst_core_video_texture_cache_set_format (GstCoreVideoTextureCache * cache,
   gst_video_info_from_caps (&cache->output_info, out_caps); 
   
   in_caps = gst_caps_copy (out_caps);
-  gst_caps_set_simple (in_caps, "format", G_TYPE_STRING, input_format, NULL);
+  gst_caps_set_simple (in_caps, "format",
+          G_TYPE_STRING, gst_video_format_to_string (in_format), NULL);
   features = gst_caps_get_features (in_caps, 0);
   gst_caps_features_add (features, GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
   gst_video_info_from_caps (&cache->input_info, in_caps);
@@ -127,6 +128,7 @@ static gboolean
 gl_mem_from_buffer (GstCoreVideoTextureCache * cache,
         GstBuffer * buffer, GstMemory **mem1, GstMemory **mem2)
 {
+  gboolean ret = TRUE;
 #if !HAVE_IOS
   CVOpenGLTextureRef texture = NULL;
 #else
@@ -139,49 +141,78 @@ gl_mem_from_buffer (GstCoreVideoTextureCache * cache,
 
 #if !HAVE_IOS
   CVOpenGLTextureCacheFlush (cache->cache, 0);
-
-  if (CVOpenGLTextureCacheCreateTextureFromImage (kCFAllocatorDefault,
-      cache->cache, pixel_buf, NULL, &texture) != kCVReturnSuccess)
-    goto error;
-
-  *mem1 = (GstMemory *) gst_gl_memory_wrapped_texture (cache->ctx,
-      CVOpenGLTextureGetName (texture), CVOpenGLTextureGetTarget (texture),
-      &cache->input_info, 0, NULL, texture, (GDestroyNotify) CFRelease);
 #else
   CVOpenGLESTextureCacheFlush (cache->cache, 0);
+#endif
 
-  if (CVOpenGLESTextureCacheCreateTextureFromImage (kCFAllocatorDefault,
-      cache->cache, pixel_buf, NULL, GL_TEXTURE_2D, GL_LUMINANCE,
-      GST_VIDEO_INFO_WIDTH (&cache->input_info),
-      GST_VIDEO_INFO_HEIGHT (&cache->input_info),
-      GL_LUMINANCE, GL_UNSIGNED_BYTE, 0, &texture) != kCVReturnSuccess)
-    goto error;
-
-  *mem1 = (GstMemory *) gst_gl_memory_wrapped_texture (cache->ctx,
-      CVOpenGLESTextureGetName (texture), CVOpenGLESTextureGetTarget (texture),
-      &cache->input_info, 0, NULL, texture, (GDestroyNotify) CFRelease);
-
-  if (CVOpenGLESTextureCacheCreateTextureFromImage (kCFAllocatorDefault,
-      cache->cache, pixel_buf, NULL, GL_TEXTURE_2D, GL_LUMINANCE_ALPHA,
-      GST_VIDEO_INFO_WIDTH (&cache->input_info) / 2,
-      GST_VIDEO_INFO_HEIGHT (&cache->input_info) / 2,
-      GL_LUMINANCE_ALPHA, GL_UNSIGNED_BYTE, 1, &texture) != kCVReturnSuccess)
-    goto error;
-
-  *mem2 = (GstMemory *) gst_gl_memory_wrapped_texture (cache->ctx,
-      CVOpenGLESTextureGetName (texture), CVOpenGLESTextureGetTarget (texture),
-      &cache->input_info, 0, NULL, texture, (GDestroyNotify) CFRelease);
+  switch (GST_VIDEO_INFO_FORMAT (&cache->input_info)) {
+#if !HAVE_IOS
+      case GST_VIDEO_FORMAT_UYVY:
+        /* both avfvideosrc and vtdec on OSX when doing GLMemory negotiate UYVY
+         * under the hood, which means a single output texture. */
+        if (CVOpenGLTextureCacheCreateTextureFromImage (kCFAllocatorDefault,
+              cache->cache, pixel_buf, NULL, &texture) != kCVReturnSuccess)
+          goto error;
+
+        *mem1 = (GstMemory *) gst_gl_memory_wrapped_texture (cache->ctx,
+            CVOpenGLTextureGetName (texture), CVOpenGLTextureGetTarget (texture),
+            &cache->input_info, 0, NULL, texture, (GDestroyNotify) CFRelease);
+        break;
+#else
+      case GST_VIDEO_FORMAT_BGRA:
+        /* avfvideosrc does BGRA on iOS when doing GLMemory */
+        if (CVOpenGLESTextureCacheCreateTextureFromImage (kCFAllocatorDefault,
+              cache->cache, pixel_buf, NULL, GL_TEXTURE_2D, GL_RGBA,
+              GST_VIDEO_INFO_WIDTH (&cache->input_info),
+              GST_VIDEO_INFO_HEIGHT (&cache->input_info),
+              GL_RGBA, GL_UNSIGNED_BYTE, 0, &texture) != kCVReturnSuccess)
+          goto error;
+
+        *mem1 = (GstMemory *) gst_gl_memory_wrapped_texture (cache->ctx,
+            CVOpenGLESTextureGetName (texture), CVOpenGLESTextureGetTarget (texture),
+            &cache->input_info, 0, NULL, texture, (GDestroyNotify) CFRelease);
+        break;
+      case GST_VIDEO_FORMAT_NV12:
+        /* vtdec does NV12 on iOS when doing GLMemory */
+        if (CVOpenGLESTextureCacheCreateTextureFromImage (kCFAllocatorDefault,
+              cache->cache, pixel_buf, NULL, GL_TEXTURE_2D, GL_LUMINANCE,
+              GST_VIDEO_INFO_WIDTH (&cache->input_info),
+              GST_VIDEO_INFO_HEIGHT (&cache->input_info),
+              GL_LUMINANCE, GL_UNSIGNED_BYTE, 0, &texture) != kCVReturnSuccess)
+          goto error;
+
+        *mem1 = (GstMemory *) gst_gl_memory_wrapped_texture (cache->ctx,
+            CVOpenGLESTextureGetName (texture), CVOpenGLESTextureGetTarget (texture),
+            &cache->input_info, 0, NULL, texture, (GDestroyNotify) CFRelease);
+
+        if (CVOpenGLESTextureCacheCreateTextureFromImage (kCFAllocatorDefault,
+              cache->cache, pixel_buf, NULL, GL_TEXTURE_2D, GL_LUMINANCE_ALPHA,
+              GST_VIDEO_INFO_WIDTH (&cache->input_info) / 2,
+              GST_VIDEO_INFO_HEIGHT (&cache->input_info) / 2,
+              GL_LUMINANCE_ALPHA, GL_UNSIGNED_BYTE, 1, &texture) != kCVReturnSuccess)
+          goto error;
+
+        *mem2 = (GstMemory *) gst_gl_memory_wrapped_texture (cache->ctx,
+            CVOpenGLESTextureGetName (texture), CVOpenGLESTextureGetTarget (texture),
+            &cache->input_info, 0, NULL, texture, (GDestroyNotify) CFRelease);
+        break;
 #endif
+      default:
+        g_warn_if_reached ();
+        ret = FALSE;
+    }
 
-  return TRUE;
+  return ret;
 
 error:
+  ret = FALSE;
+
   if (*mem1)
       gst_memory_unref (*mem1);
   if (*mem2)
       gst_memory_unref (*mem2);
 
-  return FALSE;
+  return ret;
 }
 
 static void
diff --git a/sys/applemedia/vtdec.c b/sys/applemedia/vtdec.c
index 28319999a..a696fcfcc 100644
--- a/sys/applemedia/vtdec.c
+++ b/sys/applemedia/vtdec.c
@@ -313,7 +313,6 @@ gst_vtdec_set_format (GstVideoDecoder * decoder, GstVideoCodecState * state)
   vtdec->format_description = format_description;
 
   output_format = gst_vtdec_negotiate_output_format (vtdec);
-
   if (!gst_vtdec_create_session (vtdec, output_format))
     return FALSE;
 
@@ -710,10 +709,17 @@ gst_vtdec_push_frames_if_needed (GstVtdec * vtdec, gboolean drain,
   if (gst_pad_check_reconfigure (decoder->srcpad)) {
     gst_video_decoder_negotiate (decoder);
     if (vtdec->texture_cache) {
+      GstVideoFormat internal_format;
       GstVideoCodecState *output_state =
           gst_video_decoder_get_output_state (decoder);
+
+#ifdef HAVE_IOS
+      internal_format = GST_VIDEO_FORMAT_NV12;
+#else
+      internal_format = GST_VIDEO_FORMAT_UYVY;
+#endif
       gst_core_video_texture_cache_set_format (vtdec->texture_cache,
-          GST_VTDEC_VIDEO_FORMAT_STR, output_state->caps);
+          internal_format, output_state->caps);
       gst_video_codec_state_unref (output_state);
     }
   }