mfvideosrc: Add support for jpeg on Win32 application

Enable reading jpeg data from webcam if it's supported.
Note that this would be enabled only for Win32.
For UWP, we need to research more about how to support jpeg.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/1342>

5 files changed, 134 insertions, 27 deletions

diff --git a/sys/mediafoundation/gstmfsourceobject.c b/sys/mediafoundation/gstmfsourceobject.c
index 32457a40c..e23f958a2 100644
--- a/sys/mediafoundation/gstmfsourceobject.c
+++ b/sys/mediafoundation/gstmfsourceobject.c
@@ -300,6 +300,20 @@ gst_mf_source_object_fill (GstMFSourceObject * object, GstBuffer * buffer)
   return klass->fill (object, buffer);
 }
 
+GstFlowReturn
+gst_mf_source_object_create (GstMFSourceObject * object, GstBuffer ** buffer)
+{
+  GstMFSourceObjectClass *klass;
+
+  g_return_val_if_fail (GST_IS_MF_SOURCE_OBJECT (object), GST_FLOW_ERROR);
+  g_return_val_if_fail (buffer != NULL, GST_FLOW_ERROR);
+
+  klass = GST_MF_SOURCE_OBJECT_GET_CLASS (object);
+  g_assert (klass->create != NULL);
+
+  return klass->create (object, buffer);
+}
+
 void
 gst_mf_source_object_set_flushing (GstMFSourceObject * object,
     gboolean flushing)
diff --git a/sys/mediafoundation/gstmfsourceobject.h b/sys/mediafoundation/gstmfsourceobject.h
index 9c04575bd..aae86e4ea 100644
--- a/sys/mediafoundation/gstmfsourceobject.h
+++ b/sys/mediafoundation/gstmfsourceobject.h
@@ -82,6 +82,9 @@ struct _GstMFSourceObjectClass
   GstFlowReturn (*fill)        (GstMFSourceObject * object,
                                 GstBuffer * buffer);
 
+  GstFlowReturn (*create)      (GstMFSourceObject * object,
+                                GstBuffer ** buffer);
+
   gboolean      (*unlock)      (GstMFSourceObject * object);
 
   gboolean      (*unlock_stop) (GstMFSourceObject * object);
@@ -98,9 +101,14 @@ gboolean        gst_mf_source_object_start        (GstMFSourceObject * object);
 
 gboolean        gst_mf_source_object_stop         (GstMFSourceObject * object);
 
+/* Used for raw format */
 GstFlowReturn   gst_mf_source_object_fill         (GstMFSourceObject * object,
                                                    GstBuffer * buffer);
 
+/* Used for compressed/raw format */
+GstFlowReturn   gst_mf_source_object_create       (GstMFSourceObject * object,
+                                                   GstBuffer ** buffer);
+
 void            gst_mf_source_object_set_flushing (GstMFSourceObject * object,
                                                    gboolean flushing);
 
diff --git a/sys/mediafoundation/gstmfsourcereader.cpp b/sys/mediafoundation/gstmfsourcereader.cpp
index ff0ee9850..f9dba1b7b 100644
--- a/sys/mediafoundation/gstmfsourcereader.cpp
+++ b/sys/mediafoundation/gstmfsourcereader.cpp
@@ -78,6 +78,8 @@ static gboolean gst_mf_source_reader_stop  (GstMFSourceObject * object);
 static gboolean gst_mf_source_reader_close (GstMFSourceObject * object);
 static GstFlowReturn gst_mf_source_reader_fill (GstMFSourceObject * object,
     GstBuffer * buffer);
+static GstFlowReturn gst_mf_source_reader_create (GstMFSourceObject * object,
+    GstBuffer ** buffer);
 static gboolean gst_mf_source_reader_unlock (GstMFSourceObject * object);
 static gboolean gst_mf_source_reader_unlock_stop (GstMFSourceObject * object);
 static GstCaps * gst_mf_source_reader_get_caps (GstMFSourceObject * object);
@@ -101,6 +103,7 @@ gst_mf_source_reader_class_init (GstMFSourceReaderClass * klass)
   source_class->stop = GST_DEBUG_FUNCPTR (gst_mf_source_reader_stop);
   source_class->close = GST_DEBUG_FUNCPTR (gst_mf_source_reader_close);
   source_class->fill = GST_DEBUG_FUNCPTR (gst_mf_source_reader_fill);
+  source_class->create = GST_DEBUG_FUNCPTR (gst_mf_source_reader_create);
   source_class->unlock = GST_DEBUG_FUNCPTR (gst_mf_source_reader_unlock);
   source_class->unlock_stop =
       GST_DEBUG_FUNCPTR (gst_mf_source_reader_unlock_stop);
@@ -291,10 +294,12 @@ gst_mf_source_reader_start (GstMFSourceObject * object)
 
   type = self->cur_type;
 
-  hr = type->media_type->SetUINT32 (MF_MT_DEFAULT_STRIDE,
-      GST_VIDEO_INFO_PLANE_STRIDE (&self->info, 0));
-  if (!gst_mf_result (hr))
-    return FALSE;
+  if (GST_VIDEO_INFO_FORMAT (&self->info) != GST_VIDEO_FORMAT_ENCODED) {
+    hr = type->media_type->SetUINT32 (MF_MT_DEFAULT_STRIDE,
+        GST_VIDEO_INFO_PLANE_STRIDE (&self->info, 0));
+    if (!gst_mf_result (hr))
+      return FALSE;
+  }
 
   hr = self->reader->SetStreamSelection (type->stream_index, TRUE);
   if (!gst_mf_result (hr))
@@ -360,15 +365,10 @@ gst_mf_source_reader_read_sample (GstMFSourceReader * self)
 }
 
 static GstFlowReturn
-gst_mf_source_reader_fill (GstMFSourceObject * object, GstBuffer * buffer)
+gst_mf_source_reader_get_media_buffer (GstMFSourceReader * self,
+    IMFMediaBuffer ** media_buffer)
 {
-  GstMFSourceReader *self = GST_MF_SOURCE_READER (object);
   GstFlowReturn ret = GST_FLOW_OK;
-  HRESULT hr;
-  GstVideoFrame frame;
-  BYTE *data;
-  gint i, j;
-  ComPtr<IMFMediaBuffer> media_buffer;
 
   while (g_queue_is_empty (self->queue)) {
     ret = gst_mf_source_reader_read_sample (self);
@@ -383,7 +383,25 @@ gst_mf_source_reader_fill (GstMFSourceObject * object, GstBuffer * buffer)
     g_mutex_unlock (&self->lock);
   }
 
-  media_buffer.Attach ((IMFMediaBuffer *) g_queue_pop_head (self->queue));
+  *media_buffer = (IMFMediaBuffer *) g_queue_pop_head (self->queue);
+
+  return GST_FLOW_OK;
+}
+
+static GstFlowReturn
+gst_mf_source_reader_fill (GstMFSourceObject * object, GstBuffer * buffer)
+{
+  GstMFSourceReader *self = GST_MF_SOURCE_READER (object);
+  GstFlowReturn ret = GST_FLOW_OK;
+  ComPtr<IMFMediaBuffer> media_buffer;
+  GstVideoFrame frame;
+  BYTE *data;
+  gint i, j;
+  HRESULT hr;
+
+  ret = gst_mf_source_reader_get_media_buffer (self, &media_buffer);
+  if (ret != GST_FLOW_OK)
+    return ret;
 
   hr = media_buffer->Lock (&data, NULL, NULL);
   if (!gst_mf_result (hr)) {
@@ -420,7 +438,47 @@ gst_mf_source_reader_fill (GstMFSourceObject * object, GstBuffer * buffer)
   gst_video_frame_unmap (&frame);
   media_buffer->Unlock ();
 
-  return ret;
+  return GST_FLOW_OK;
+}
+
+static GstFlowReturn
+gst_mf_source_reader_create (GstMFSourceObject * object, GstBuffer ** buffer)
+{
+  GstMFSourceReader *self = GST_MF_SOURCE_READER (object);
+  GstFlowReturn ret = GST_FLOW_OK;
+  ComPtr<IMFMediaBuffer> media_buffer;
+  HRESULT hr;
+  BYTE *data;
+  DWORD len = 0;
+  GstBuffer *buf;
+  GstMapInfo info;
+
+  ret = gst_mf_source_reader_get_media_buffer (self, &media_buffer);
+  if (ret != GST_FLOW_OK)
+    return ret;
+
+  hr = media_buffer->Lock (&data, NULL, &len);
+  if (!gst_mf_result (hr) || len == 0) {
+    GST_ERROR_OBJECT (self, "Failed to lock media buffer");
+    return GST_FLOW_ERROR;
+  }
+
+  buf = gst_buffer_new_and_alloc (len);
+  if (!buf) {
+    GST_ERROR_OBJECT (self, "Cannot allocate buffer");
+    media_buffer->Unlock ();
+    return GST_FLOW_ERROR;
+  }
+
+  gst_buffer_map (buf, &info, GST_MAP_WRITE);
+  memcpy (info.data, data, len);
+  gst_buffer_unmap (buf, &info);
+
+  media_buffer->Unlock ();
+
+  *buffer = buf;
+
+  return GST_FLOW_OK;
 }
 
 static gboolean
diff --git a/sys/mediafoundation/gstmfutils.cpp b/sys/mediafoundation/gstmfutils.cpp
index 1a00c0f8e..886c4985a 100644
--- a/sys/mediafoundation/gstmfutils.cpp
+++ b/sys/mediafoundation/gstmfutils.cpp
@@ -78,6 +78,7 @@ static struct
   {MFVideoFormat_H265, "video/x-h265"},
   {MFVideoFormat_VP80, "video/x-vp8"},
   {MFVideoFormat_VP90, "video/x-vp9"},
+  {MFVideoFormat_MJPG, "image/jpeg"},
 };
 
 GstVideoFormat
@@ -149,10 +150,10 @@ gst_mf_media_type_to_video_caps (IMFMediaType * media_type)
     return NULL;
   }
 
-  if (raw_format) {
-    hr = MFGetAttributeSize (media_type, MF_MT_FRAME_SIZE, &width, &height);
-    if (FAILED (hr) || !width || !height) {
-      GST_WARNING ("Couldn't get frame size, hr: 0x%x", (guint) hr);
+  hr = MFGetAttributeSize (media_type, MF_MT_FRAME_SIZE, &width, &height);
+  if (FAILED (hr) || !width || !height) {
+    GST_WARNING ("Couldn't get frame size, hr: 0x%x", (guint) hr);
+    if (raw_format) {
       gst_caps_unref (caps);
 
       return NULL;
diff --git a/sys/mediafoundation/gstmfvideosrc.c b/sys/mediafoundation/gstmfvideosrc.c
index 25a793417..23ffa15e7 100644
--- a/sys/mediafoundation/gstmfvideosrc.c
+++ b/sys/mediafoundation/gstmfvideosrc.c
@@ -52,11 +52,22 @@
 GST_DEBUG_CATEGORY (gst_mf_video_src_debug);
 #define GST_CAT_DEFAULT gst_mf_video_src_debug
 
+#if GST_MF_WINAPI_ONLY_APP
+/* FIXME: need support JPEG for UWP */
+#define SRC_TEMPLATE_CAPS \
+    GST_VIDEO_CAPS_MAKE (GST_MF_VIDEO_FORMATS)
+#else
+#define SRC_TEMPLATE_CAPS \
+    GST_VIDEO_CAPS_MAKE (GST_MF_VIDEO_FORMATS) "; " \
+        "image/jpeg, width = " GST_VIDEO_SIZE_RANGE ", " \
+        "height = " GST_VIDEO_SIZE_RANGE ", " \
+        "framerate = " GST_VIDEO_FPS_RANGE
+#endif
+
 static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src",
     GST_PAD_SRC,
     GST_PAD_ALWAYS,
-    GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE (GST_MF_VIDEO_FORMATS))
-    );
+    GST_STATIC_CAPS (SRC_TEMPLATE_CAPS));
 
 struct _GstMFVideoSrc
 {
@@ -101,8 +112,8 @@ static GstCaps *gst_mf_video_src_fixate (GstBaseSrc * src, GstCaps * caps);
 static gboolean gst_mf_video_src_unlock (GstBaseSrc * src);
 static gboolean gst_mf_video_src_unlock_stop (GstBaseSrc * src);
 
-static GstFlowReturn gst_mf_video_src_fill (GstPushSrc * pushsrc,
-    GstBuffer * buffer);
+static GstFlowReturn gst_mf_video_src_create (GstPushSrc * pushsrc,
+    GstBuffer ** buffer);
 
 #define gst_mf_video_src_parent_class parent_class
 G_DEFINE_TYPE (GstMFVideoSrc, gst_mf_video_src, GST_TYPE_PUSH_SRC);
@@ -151,7 +162,7 @@ gst_mf_video_src_class_init (GstMFVideoSrcClass * klass)
   basesrc_class->unlock = GST_DEBUG_FUNCPTR (gst_mf_video_src_unlock);
   basesrc_class->unlock_stop = GST_DEBUG_FUNCPTR (gst_mf_video_src_unlock_stop);
 
-  pushsrc_class->fill = GST_DEBUG_FUNCPTR (gst_mf_video_src_fill);
+  pushsrc_class->create = GST_DEBUG_FUNCPTR (gst_mf_video_src_create);
 
   GST_DEBUG_CATEGORY_INIT (gst_mf_video_src_debug, "mfvideosrc", 0,
       "mfvideosrc");
@@ -281,7 +292,8 @@ gst_mf_video_src_set_caps (GstBaseSrc * src, GstCaps * caps)
   }
 
   gst_video_info_from_caps (&self->info, caps);
-  gst_base_src_set_blocksize (src, GST_VIDEO_INFO_SIZE (&self->info));
+  if (GST_VIDEO_INFO_FORMAT (&self->info) != GST_VIDEO_FORMAT_ENCODED)
+    gst_base_src_set_blocksize (src, GST_VIDEO_INFO_SIZE (&self->info));
 
   return TRUE;
 }
@@ -355,10 +367,11 @@ gst_mf_video_src_unlock_stop (GstBaseSrc * src)
 }
 
 static GstFlowReturn
-gst_mf_video_src_fill (GstPushSrc * pushsrc, GstBuffer * buffer)
+gst_mf_video_src_create (GstPushSrc * pushsrc, GstBuffer ** buffer)
 {
   GstMFVideoSrc *self = GST_MF_VIDEO_SRC (pushsrc);
   GstFlowReturn ret = GST_FLOW_OK;
+  GstBuffer *buf = NULL;
 
   if (!self->started) {
     if (!gst_mf_source_object_start (self->source)) {
@@ -370,13 +383,26 @@ gst_mf_video_src_fill (GstPushSrc * pushsrc, GstBuffer * buffer)
     self->started = TRUE;
   }
 
-  ret = gst_mf_source_object_fill (self->source, buffer);
+  if (GST_VIDEO_INFO_FORMAT (&self->info) != GST_VIDEO_FORMAT_ENCODED) {
+    ret = GST_BASE_SRC_CLASS (parent_class)->alloc (GST_BASE_SRC (self), 0,
+        GST_VIDEO_INFO_SIZE (&self->info), &buf);
+
+    if (ret != GST_FLOW_OK)
+      return ret;
+
+    ret = gst_mf_source_object_fill (self->source, buf);
+  } else {
+    ret = gst_mf_source_object_create (self->source, &buf);
+  }
+
   if (ret != GST_FLOW_OK)
     return ret;
 
-  GST_BUFFER_OFFSET (buffer) = self->n_frames;
-  GST_BUFFER_OFFSET_END (buffer) = GST_BUFFER_OFFSET (buffer) + 1;
+  GST_BUFFER_OFFSET (buf) = self->n_frames;
+  GST_BUFFER_OFFSET_END (buf) = GST_BUFFER_OFFSET (buf) + 1;
   self->n_frames++;
 
+  *buffer = buf;
+
   return GST_FLOW_OK;
 }