amc: MediaCodec::getOutputBuffer() can return NULL without exception

Happens when doing zerocopy rendering, or when passing a wrong index to it.
Handle this properly for zerocopy rendering, fail properly for the other
cases.

https://bugzilla.gnome.org/show_bug.cgi?id=760961

4 files changed, 72 insertions, 17 deletions

diff --git a/sys/androidmedia/gstamc.c b/sys/androidmedia/gstamc.c
index 5a72e0e1d..5b4e93618 100644
--- a/sys/androidmedia/gstamc.c
+++ b/sys/androidmedia/gstamc.c
@@ -369,21 +369,23 @@ gst_amc_codec_get_output_buffer (GstAmcCodec * codec, gint index, GError ** err)
           media_codec.get_output_buffer, &buffer, index))
     goto done;
 
-  ret = g_new0 (GstAmcBuffer, 1);
-  ret->object = gst_amc_jni_object_make_global (env, buffer);
-  if (!ret->object) {
-    gst_amc_jni_set_error (env, err, GST_LIBRARY_ERROR,
-        GST_LIBRARY_ERROR_FAILED, "Failed to create global buffer reference");
-    goto error;
-  }
+  if (buffer != NULL) {
+    ret = g_new0 (GstAmcBuffer, 1);
+    ret->object = gst_amc_jni_object_make_global (env, buffer);
+    if (!ret->object) {
+      gst_amc_jni_set_error (env, err, GST_LIBRARY_ERROR,
+          GST_LIBRARY_ERROR_FAILED, "Failed to create global buffer reference");
+      goto error;
+    }
 
-  ret->data = (*env)->GetDirectBufferAddress (env, ret->object);
-  if (!ret->data) {
-    gst_amc_jni_set_error (env, err, GST_LIBRARY_ERROR,
-        GST_LIBRARY_ERROR_FAILED, "Failed to get buffer address");
-    goto error;
+    ret->data = (*env)->GetDirectBufferAddress (env, ret->object);
+    if (!ret->data) {
+      gst_amc_jni_set_error (env, err, GST_LIBRARY_ERROR,
+          GST_LIBRARY_ERROR_FAILED, "Failed to get buffer address");
+      goto error;
+    }
+    ret->size = (*env)->GetDirectBufferCapacity (env, ret->object);
   }
-  ret->size = (*env)->GetDirectBufferCapacity (env, ret->object);
 
 done:
 
diff --git a/sys/androidmedia/gstamcaudiodec.c b/sys/androidmedia/gstamcaudiodec.c
index ce86a9686..fbd6d0969 100644
--- a/sys/androidmedia/gstamcaudiodec.c
+++ b/sys/androidmedia/gstamcaudiodec.c
@@ -493,8 +493,10 @@ retry:
   is_eos = ! !(buffer_info.flags & BUFFER_FLAG_END_OF_STREAM);
 
   buf = gst_amc_codec_get_output_buffer (self->codec, idx, &err);
-  if (!buf)
+  if (err)
     goto failed_to_get_output_buffer;
+  else if (!buf)
+    goto got_null_output_buffer;
 
   if (buffer_info.size > 0) {
     GstBuffer *outbuf;
@@ -699,6 +701,22 @@ failed_to_get_output_buffer:
     g_mutex_unlock (&self->drain_lock);
     return;
   }
+
+got_null_output_buffer:
+  {
+    GST_ELEMENT_ERROR (self, LIBRARY, SETTINGS, (NULL),
+        ("Got no output buffer"));
+    gst_pad_push_event (GST_AUDIO_DECODER_SRC_PAD (self), gst_event_new_eos ());
+    gst_pad_pause_task (GST_AUDIO_DECODER_SRC_PAD (self));
+    self->downstream_flow_ret = GST_FLOW_ERROR;
+    GST_AUDIO_DECODER_STREAM_UNLOCK (self);
+    g_mutex_lock (&self->drain_lock);
+    self->draining = FALSE;
+    g_cond_broadcast (&self->drain_cond);
+    g_mutex_unlock (&self->drain_lock);
+    return;
+  }
+
 invalid_buffer_size:
   {
     GST_ELEMENT_ERROR (self, LIBRARY, FAILED, (NULL),
diff --git a/sys/androidmedia/gstamcvideodec.c b/sys/androidmedia/gstamcvideodec.c
index 1581d0613..90be00269 100644
--- a/sys/androidmedia/gstamcvideodec.c
+++ b/sys/androidmedia/gstamcvideodec.c
@@ -1271,9 +1271,12 @@ retry:
   is_eos = ! !(buffer_info.flags & BUFFER_FLAG_END_OF_STREAM);
 
   buf = gst_amc_codec_get_output_buffer (self->codec, idx, &err);
-  if (!buf)
+  if (err)
     goto failed_to_get_output_buffer;
 
+  if (self->codec_config != AMC_CODEC_CONFIG_WITH_SURFACE && !buf)
+    goto got_null_output_buffer;
+
   if (frame
       && (deadline =
           gst_video_decoder_get_max_decode_time (GST_VIDEO_DECODER (self),
@@ -1574,6 +1577,21 @@ failed_to_get_output_buffer:
     return;
   }
 
+got_null_output_buffer:
+  {
+    GST_ELEMENT_ERROR (self, LIBRARY, SETTINGS, (NULL),
+        ("Got no output buffer"));
+    gst_pad_push_event (GST_VIDEO_DECODER_SRC_PAD (self), gst_event_new_eos ());
+    gst_pad_pause_task (GST_VIDEO_DECODER_SRC_PAD (self));
+    self->downstream_flow_ret = GST_FLOW_ERROR;
+    GST_VIDEO_DECODER_STREAM_UNLOCK (self);
+    g_mutex_lock (&self->drain_lock);
+    self->draining = FALSE;
+    g_cond_broadcast (&self->drain_cond);
+    g_mutex_unlock (&self->drain_lock);
+    return;
+  }
+
 invalid_buffer:
   {
     GST_ELEMENT_ERROR (self, LIBRARY, SETTINGS, (NULL),
diff --git a/sys/androidmedia/gstamcvideoenc.c b/sys/androidmedia/gstamcvideoenc.c
index 5198fe67b..fc4436c24 100644
--- a/sys/androidmedia/gstamcvideoenc.c
+++ b/sys/androidmedia/gstamcvideoenc.c
@@ -1023,8 +1023,10 @@ process_buffer:
   is_eos = ! !(buffer_info.flags & BUFFER_FLAG_END_OF_STREAM);
 
   buf = gst_amc_codec_get_output_buffer (self->codec, idx, &err);
-  if (!buf)
+  if (err)
     goto failed_to_get_output_buffer;
+  else if (!buf)
+    goto got_null_output_buffer;
 
   flow_ret =
       gst_amc_video_enc_handle_output_frame (self, buf, &buffer_info, frame);
@@ -1147,7 +1149,22 @@ failed_to_get_output_buffer:
     GST_ELEMENT_ERROR_FROM_ERROR (self, err);
     gst_pad_push_event (GST_VIDEO_ENCODER_SRC_PAD (self), gst_event_new_eos ());
     gst_pad_pause_task (GST_VIDEO_ENCODER_SRC_PAD (self));
-    self->downstream_flow_ret = GST_FLOW_NOT_NEGOTIATED;
+    self->downstream_flow_ret = GST_FLOW_ERROR;
+    GST_VIDEO_ENCODER_STREAM_UNLOCK (self);
+    g_mutex_lock (&self->drain_lock);
+    self->draining = FALSE;
+    g_cond_broadcast (&self->drain_cond);
+    g_mutex_unlock (&self->drain_lock);
+    return;
+  }
+
+got_null_output_buffer:
+  {
+    GST_ELEMENT_ERROR (self, LIBRARY, SETTINGS, (NULL),
+        ("Got no output buffer"));
+    gst_pad_push_event (GST_VIDEO_ENCODER_SRC_PAD (self), gst_event_new_eos ());
+    gst_pad_pause_task (GST_VIDEO_ENCODER_SRC_PAD (self));
+    self->downstream_flow_ret = GST_FLOW_ERROR;
     GST_VIDEO_ENCODER_STREAM_UNLOCK (self);
     g_mutex_lock (&self->drain_lock);
     self->draining = FALSE;