fix: append H264 SPS and PPS in front of every IDR NAL units

This is to keep consistency with iOS proxy.

2 files changed, 140 insertions, 2 deletions

diff --git a/sdl_android/src/main/java/com/smartdevicelink/encoder/EncoderUtils.java b/sdl_android/src/main/java/com/smartdevicelink/encoder/EncoderUtils.java
new file mode 100644
index 000000000..362564da8
--- /dev/null
+++ b/sdl_android/src/main/java/com/smartdevicelink/encoder/EncoderUtils.java
@@ -0,0 +1,107 @@
+/*
+ * Copyright (c) 2017, Xevo Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice, this
+ *   list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ *
+ * * Neither the name of the copyright holder nor the names of its contributors
+ *   may be used to endorse or promote products derived from this software
+ *   without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package com.smartdevicelink.encoder;
+
+import android.annotation.TargetApi;
+import android.media.MediaFormat;
+import android.os.Build;
+import android.util.Log;
+
+import java.nio.ByteBuffer;
+
+@TargetApi(Build.VERSION_CODES.JELLY_BEAN)
+public final class EncoderUtils {
+    private final static String TAG = "EncoderUtils";
+
+    /**
+     * Extracts codec-specific data from MediaFormat instance
+     *
+     * Currently, only AVC is supported.
+     *
+     * @param format MediaFormat instance retrieved from MediaCodec
+     * @return byte array containing codec-specific data, or null if an error occurred
+     */
+    public static byte[] getCodecSpecificData(MediaFormat format) {
+        if (format == null) {
+            return null;
+        }
+
+        String name = format.getString(MediaFormat.KEY_MIME);
+        if (name == null) {
+            return null;
+        }
+
+        // same as MediaFormat.MIMETYPE_VIDEO_AVC but it requires API level 21
+        if (name.equals("video/avc")) {
+            return getAVCCodecSpecificData(format);
+        } else {
+            Log.w(TAG, "Retrieving codec-specific data for " + name + " is not supported");
+            return null;
+        }
+    }
+
+    /**
+     * Extracts H.264 codec-specific data (SPS and PPS) from MediaFormat instance
+     *
+     * The codec-specific data is in byte-stream format; 4-byte start codes (0x00 0x00 0x00 0x01)
+     * are added in front of SPS and PPS NAL units.
+     *
+     * @param format MediaFormat instance retrieved from MediaCodec
+     * @return byte array containing codec-specific data, or null if an error occurred
+     */
+    private static byte[] getAVCCodecSpecificData(MediaFormat format) {
+        // For H.264, "csd-0" contains SPS and "csd-1" contains PPS. Refer to the documentation
+        // of MediaCodec.
+        if (!(format.containsKey("csd-0") && format.containsKey("csd-1"))) {
+            Log.w(TAG, "H264 codec specific data not found");
+            return null;
+        }
+
+        ByteBuffer sps = format.getByteBuffer("csd-0");
+        int spsLen = sps.remaining();
+        ByteBuffer pps = format.getByteBuffer("csd-1");
+        int ppsLen = pps.remaining();
+
+        byte[] output = new byte[spsLen + ppsLen];
+        try {
+            sps.get(output, 0, spsLen);
+            pps.get(output, spsLen, ppsLen);
+        } catch (Exception e) {
+            // should not happen
+            Log.w(TAG, "Error while copying H264 codec specific data: " + e);
+            return null;
+        }
+
+        return output;
+    }
+
+    private EncoderUtils() {}
+}
diff --git a/sdl_android/src/main/java/com/smartdevicelink/encoder/SdlEncoder.java b/sdl_android/src/main/java/com/smartdevicelink/encoder/SdlEncoder.java
index 81e344840..29c640f70 100644
--- a/sdl_android/src/main/java/com/smartdevicelink/encoder/SdlEncoder.java
+++ b/sdl_android/src/main/java/com/smartdevicelink/encoder/SdlEncoder.java
@@ -9,11 +9,14 @@ import android.media.MediaCodec;
 import android.media.MediaCodecInfo;
 import android.media.MediaFormat;
 import android.os.Build;
+import android.util.Log;
 import android.view.Surface;
 
 @TargetApi(Build.VERSION_CODES.JELLY_BEAN_MR2)
 public class SdlEncoder {
-	
+
+	private static final String TAG = "SdlEncoder";
+
 	// parameters for the encoder
 	private static final String _MIME_TYPE = "video/avc"; // H.264/AVC video
 	// private static final String MIME_TYPE = "video/mp4v-es"; //MPEG4 video
@@ -29,7 +32,9 @@ public class SdlEncoder {
 	
 	// allocate one of these up front so we don't need to do it every time
 	private MediaCodec.BufferInfo mBufferInfo;
-	
+
+	// Codec-specific data (SPS and PPS)
+	private byte[] mH264CodecSpecificData = null;
 
 	public SdlEncoder () {
 	}
@@ -114,6 +119,7 @@ public class SdlEncoder {
 			}
 			mOutputStream = null;
 		}
+		mH264CodecSpecificData = null;
 	}
 
 	/**
@@ -147,8 +153,33 @@ public class SdlEncoder {
 				// not expected for an encoder
 				encoderOutputBuffers = mEncoder.getOutputBuffers();
 			} else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
+				if (mH264CodecSpecificData == null) {
+					MediaFormat format = mEncoder.getOutputFormat();
+					mH264CodecSpecificData = EncoderUtils.getCodecSpecificData(format);
+				} else {
+					Log.w(TAG, "Output format change notified more than once, ignoring.");
+				}
 			} else if (encoderStatus < 0) {
 			} else {
+				if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
+					// If we already retrieve codec specific data via OUTPUT_FORMAT_CHANGED event,
+					// we do not need this data.
+					if (mH264CodecSpecificData != null) {
+						mBufferInfo.size = 0;
+					} else {
+						Log.i(TAG, "H264 codec specific data not retrieved yet.");
+					}
+				}
+				// append SPS and PPS in front of every IDR NAL unit
+				if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_KEY_FRAME) != 0
+						&& mBufferInfo.size != 0
+						&& mH264CodecSpecificData != null) {
+					try {
+						mOutputStream.write(mH264CodecSpecificData, 0,
+								mH264CodecSpecificData.length);
+					} catch (Exception e) {}
+				}
+
 				if (mBufferInfo.size != 0) {
 					byte[] dataToWrite = new byte[mBufferInfo.size];
 					encoderOutputBuffers[encoderStatus].get(dataToWrite,