androidmedia: when flushing, better handle IllegalStateException received from getOutputBuffer

1. Similar to 880f3d8, don't consider not getting an output buffer as
   an error during flushing. I've seen the following sometimes when
   encoding:

   W GStreamer+amcvideoenc: java.lang.IllegalStateException
   W GStreamer+amcvideoenc:         at android.media.MediaCodec.getBuffer(Native Method)
   W GStreamer+amcvideoenc:         at android.media.MediaCodec.getOutputBuffer(MediaCodec.java:2886)

2. For amcvideodec/enc, call _find_nearest_frame (which grabs a fresh
   reference on a GstVideoCodecFrame) after we have an output buffer,
   so as to not leak the reference, in case getting an output buffer
   fails.

   Otherwise, if we get an error grabbing the output buffer, we leak
   the reference to the frame. This can cause issues with a
   v4l2bufferpool feeding the encoder not being able to clean itself
   up properly due to buffers still being marked as in-use.

https://bugzilla.gnome.org/show_bug.cgi?id=791258

3 files changed, 30 insertions, 15 deletions

diff --git a/sys/androidmedia/gstamcaudiodec.c b/sys/androidmedia/gstamcaudiodec.c
index 29a03e11f..8e0571063 100644
--- a/sys/androidmedia/gstamcaudiodec.c
+++ b/sys/androidmedia/gstamcaudiodec.c
@@ -495,10 +495,15 @@ retry:
   is_eos = ! !(buffer_info.flags & BUFFER_FLAG_END_OF_STREAM);
 
   buf = gst_amc_codec_get_output_buffer (self->codec, idx, &err);
-  if (err)
+  if (err) {
+    if (self->flushing) {
+      g_clear_error (&err);
+      goto flushing;
+    }
     goto failed_to_get_output_buffer;
-  else if (!buf)
+  } else if (!buf) {
     goto got_null_output_buffer;
+  }
 
   if (buffer_info.size > 0) {
     GstBuffer *outbuf;
diff --git a/sys/androidmedia/gstamcvideodec.c b/sys/androidmedia/gstamcvideodec.c
index becfbc52f..69475c9e9 100644
--- a/sys/androidmedia/gstamcvideodec.c
+++ b/sys/androidmedia/gstamcvideodec.c
@@ -1308,19 +1308,24 @@ retry:
       " flags 0x%08x", idx, buffer_info.offset, buffer_info.size,
       buffer_info.presentation_time_us, buffer_info.flags);
 
-  frame =
-      _find_nearest_frame (self,
-      gst_util_uint64_scale (buffer_info.presentation_time_us, GST_USECOND, 1));
-
-  is_eos = ! !(buffer_info.flags & BUFFER_FLAG_END_OF_STREAM);
-
   buf = gst_amc_codec_get_output_buffer (self->codec, idx, &err);
-  if (err)
+  if (err) {
+    if (self->flushing) {
+      g_clear_error (&err);
+      goto flushing;
+    }
     goto failed_to_get_output_buffer;
+  }
 
   if (self->codec_config != AMC_CODEC_CONFIG_WITH_SURFACE && !buf)
     goto got_null_output_buffer;
 
+  frame =
+      _find_nearest_frame (self,
+      gst_util_uint64_scale (buffer_info.presentation_time_us, GST_USECOND, 1));
+
+  is_eos = ! !(buffer_info.flags & BUFFER_FLAG_END_OF_STREAM);
+
   if (frame
       && (deadline =
           gst_video_decoder_get_max_decode_time (GST_VIDEO_DECODER (self),
diff --git a/sys/androidmedia/gstamcvideoenc.c b/sys/androidmedia/gstamcvideoenc.c
index 61c06bca1..fe50a4d34 100644
--- a/sys/androidmedia/gstamcvideoenc.c
+++ b/sys/androidmedia/gstamcvideoenc.c
@@ -1022,18 +1022,23 @@ process_buffer:
       " flags 0x%08x", idx, buffer_info.size, buffer_info.presentation_time_us,
       buffer_info.flags);
 
+  buf = gst_amc_codec_get_output_buffer (self->codec, idx, &err);
+  if (err) {
+    if (self->flushing) {
+      g_clear_error (&err);
+      goto flushing;
+    }
+    goto failed_to_get_output_buffer;
+  } else if (!buf) {
+    goto got_null_output_buffer;
+  }
+
   frame =
       _find_nearest_frame (self,
       gst_util_uint64_scale (buffer_info.presentation_time_us, GST_USECOND, 1));
 
   is_eos = ! !(buffer_info.flags & BUFFER_FLAG_END_OF_STREAM);
 
-  buf = gst_amc_codec_get_output_buffer (self->codec, idx, &err);
-  if (err)
-    goto failed_to_get_output_buffer;
-  else if (!buf)
-    goto got_null_output_buffer;
-
   flow_ret =
       gst_amc_video_enc_handle_output_frame (self, buf, &buffer_info, frame);