Reduced min SDK version on virtual display encoder from 21 to 19

Should cover +90% of all phones at time of commit

2 files changed, 184 insertions, 61 deletions

diff --git a/sdl_android/src/main/java/com/smartdevicelink/encoder/VirtualDisplayEncoder.java b/sdl_android/src/main/java/com/smartdevicelink/encoder/VirtualDisplayEncoder.java
index 22c515957..0fe303c71 100644
--- a/sdl_android/src/main/java/com/smartdevicelink/encoder/VirtualDisplayEncoder.java
+++ b/sdl_android/src/main/java/com/smartdevicelink/encoder/VirtualDisplayEncoder.java
@@ -52,7 +52,7 @@ import com.smartdevicelink.streaming.video.VideoStreamingParameters;
 
 import java.nio.ByteBuffer;
 
-@TargetApi(21)
+@TargetApi(19)
 @SuppressWarnings("NullableProblems")
 public class VirtualDisplayEncoder {
     private static final String TAG = "VirtualDisplayEncoder";
@@ -66,6 +66,11 @@ public class VirtualDisplayEncoder {
     private Boolean initPassed = false;
     private final Object STREAMING_LOCK = new Object();
 
+    // Codec-specific data (SPS and PPS)
+    private byte[] mH264CodecSpecificData = null;
+
+    //For older (<21) OS versions
+    private Thread encoderThread;
 
 
     /**
@@ -74,12 +79,12 @@ public class VirtualDisplayEncoder {
      * @param context to create the virtual display
      * @param outputListener the listener that the video frames will be sent through
      * @param streamingParams parameters to create the virtual display and encoder
-     * @throws Exception if the API level is <21 or supplied parameters were null
+     * @throws Exception if the API level is <19 or supplied parameters were null
      */
     public void init(Context context, IVideoStreamListener outputListener, VideoStreamingParameters streamingParams) throws Exception {
-        if (android.os.Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) {
-            Log.e(TAG, "API level of 21 required for VirtualDisplayEncoder");
-            throw new Exception("API level of 21 required");
+        if (android.os.Build.VERSION.SDK_INT < Build.VERSION_CODES.KITKAT) {
+            Log.e(TAG, "API level of 19 required for VirtualDisplayEncoder");
+            throw new Exception("API level of 19 required");
         }
 
         if (context == null || outputListener == null || streamingParams == null || streamingParams.getResolution() == null || streamingParams.getFormat() == null) {
@@ -87,7 +92,7 @@ public class VirtualDisplayEncoder {
             throw new Exception("init parameters cannot be null");
         }
 
-        this.mDisplayManager = (DisplayManager)context.getSystemService(Context.DISPLAY_SERVICE);
+        this.mDisplayManager = (DisplayManager) context.getSystemService(Context.DISPLAY_SERVICE);
 
         this.streamingParams.update(streamingParams);
 
@@ -102,12 +107,12 @@ public class VirtualDisplayEncoder {
     }
 
     @SuppressWarnings("unused")
-    public void setStreamingParams(int displayDensity, ImageResolution resolution, int frameRate, int bitrate, int interval, VideoStreamingFormat format){
-        this.streamingParams = new VideoStreamingParameters(displayDensity,frameRate,bitrate,interval,resolution,format);
+    public void setStreamingParams(int displayDensity, ImageResolution resolution, int frameRate, int bitrate, int interval, VideoStreamingFormat format) {
+        this.streamingParams = new VideoStreamingParameters(displayDensity, frameRate, bitrate, interval, resolution, format);
     }
 
     @SuppressWarnings("unused")
-    public void setStreamingParams(VideoStreamingParameters streamingParams){
+    public void setStreamingParams(VideoStreamingParameters streamingParams) {
         this.streamingParams = streamingParams;
     }
 
@@ -116,11 +121,11 @@ public class VirtualDisplayEncoder {
      * Prepares the encoder and virtual display.
      */
     public void start() throws Exception {
-        if(!initPassed){
+        if (!initPassed) {
             Log.e(TAG, "VirtualDisplayEncoder was not properly initialized with the init() method.");
             return;
         }
-        if(streamingParams == null || streamingParams.getResolution() == null || streamingParams.getFormat() == null){
+        if (streamingParams == null || streamingParams.getResolution() == null || streamingParams.getFormat() == null) {
             return;
         }
 
@@ -144,11 +149,15 @@ public class VirtualDisplayEncoder {
     }
 
     public void shutDown() {
-        if(!initPassed){
+        if (!initPassed) {
             Log.e(TAG, "VirtualDisplayEncoder was not properly initialized with the init() method.");
             return;
         }
         try {
+            if (encoderThread != null) {
+                encoderThread.interrupt();
+                encoderThread = null;
+            }
 
             if (mVideoEncoder != null) {
                 mVideoEncoder.stop();
@@ -165,15 +174,14 @@ public class VirtualDisplayEncoder {
                 inputSurface.release();
                 inputSurface = null;
             }
-        }
-        catch (Exception ex){
+        } catch (Exception ex) {
             Log.e(TAG, "shutDown() failed");
         }
     }
 
     private Surface prepareVideoEncoder() {
 
-        if(streamingParams == null || streamingParams.getResolution() == null || streamingParams.getFormat() == null){
+        if (streamingParams == null || streamingParams.getResolution() == null || streamingParams.getFormat() == null) {
             return null;
         }
 
@@ -196,48 +204,67 @@ public class VirtualDisplayEncoder {
         try {
             mVideoEncoder = MediaCodec.createEncoderByType(videoMimeType);
             mVideoEncoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
-            Surface surface = mVideoEncoder.createInputSurface();
-            mVideoEncoder.setCallback(new MediaCodec.Callback() {
-                @Override
-                public void onInputBufferAvailable(MediaCodec codec, int index) {
-                    // nothing to do here
-                }
+            Surface surface = mVideoEncoder.createInputSurface(); //prepared
+
+            if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
+                mVideoEncoder.setCallback(new MediaCodec.Callback() {
+                    @Override
+                    public void onInputBufferAvailable(MediaCodec codec, int index) {
+                        // nothing to do here
+                    }
 
-                @Override
-                public void onOutputBufferAvailable(MediaCodec codec, int index, MediaCodec.BufferInfo info) {
-                    try {
-                        ByteBuffer encodedData = codec.getOutputBuffer(index);
-                        if(encodedData!=null) {
-
-                            encodedData.position(info.offset);
-                            encodedData.limit(info.offset + info.size);
-
-                            if (info.size != 0) {
-                                byte[] dataToWrite = new byte[info.size];
-                                encodedData.get(dataToWrite,
-                                        info.offset, info.size);
-                                if (mOutputListener != null) {
-                                    mOutputListener.sendFrame(dataToWrite, 0, dataToWrite.length, info.presentationTimeUs);
+                    @TargetApi(Build.VERSION_CODES.LOLLIPOP)
+                    @Override
+                    public void onOutputBufferAvailable(MediaCodec codec, int index, MediaCodec.BufferInfo info) {
+                        try {
+                            ByteBuffer encodedData = codec.getOutputBuffer(index);
+                            if (encodedData != null) {
+                                if (info.size != 0) {
+                                    byte[] dataToWrite;// = new byte[info.size];
+                                    int dataOffset = 0;
+
+                                    // append SPS and PPS in front of every IDR NAL unit
+                                    if ((info.flags & MediaCodec.BUFFER_FLAG_KEY_FRAME) != 0 && mH264CodecSpecificData != null) {
+                                        dataToWrite = new byte[mH264CodecSpecificData.length + info.size];
+                                        System.arraycopy(mH264CodecSpecificData, 0, dataToWrite, 0, mH264CodecSpecificData.length);
+                                        dataOffset = mH264CodecSpecificData.length;
+                                    } else {
+                                        dataToWrite = new byte[info.size];
+                                    }
+
+                                    encodedData.position(info.offset);
+                                    encodedData.limit(info.offset + info.size);
+
+                                    encodedData.get(dataToWrite, dataOffset, info.size);
+                                    if (mOutputListener != null) {
+                                        mOutputListener.sendFrame(dataToWrite, 0, dataToWrite.length, info.presentationTimeUs);
+                                    }
                                 }
-                            }
 
-                            codec.releaseOutputBuffer(index, false);
+                                codec.releaseOutputBuffer(index, false);
+                            }
+                        } catch (Exception ex) {
+                            ex.printStackTrace();
                         }
-                    } catch (Exception ex) {
-                        ex.printStackTrace();
                     }
-                }
 
-                @Override
-                public void onError(MediaCodec codec, MediaCodec.CodecException e) {
-                    e.printStackTrace();
-                }
+                    @Override
+                    public void onError(MediaCodec codec, MediaCodec.CodecException e) {
+                        e.printStackTrace();
+                    }
 
-                @Override
-                public void onOutputFormatChanged(MediaCodec codec, MediaFormat format) {
-                    // nothing to do here
-                }
-            });
+                    @Override
+                    public void onOutputFormatChanged(MediaCodec codec, MediaFormat format) {
+                        mH264CodecSpecificData = EncoderUtils.getCodecSpecificData(format);
+                    }
+                });
+            } else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT) {
+                //Implied from previous if check that this has to be older than Build.VERSION_CODES.LOLLIPOP
+                encoderThread = new Thread(new EncoderCompat());
+
+            } else {
+                Log.e(TAG, "Unable to start encoder. Android OS version " + Build.VERSION.SDK_INT + "is not supported");
+            }
 
             return surface;
 
@@ -251,28 +278,123 @@ public class VirtualDisplayEncoder {
         if (mVideoEncoder != null) {
             mVideoEncoder.start();
         }
+        if (encoderThread != null) {
+            encoderThread.start();
+        }
     }
 
-    public Display getVirtualDisplay(){
+    public Display getVirtualDisplay() {
         return virtualDisplay.getDisplay();
     }
 
-    private String getMimeForFormat(VideoStreamingFormat format){
-        if(format != null){
+    private String getMimeForFormat(VideoStreamingFormat format) {
+        if (format != null) {
             VideoStreamingCodec codec = format.getCodec();
-            if(codec != null){
-                switch(codec){
+            if (codec != null) {
+                switch (codec) { //MediaFormat constants are only available in Android 21+
                     case VP8:
-                        return MediaFormat.MIMETYPE_VIDEO_VP8;
+                        return "video/x-vnd.on2.vp8"; //MediaFormat.MIMETYPE_VIDEO_VP8
                     case VP9:
-                        return MediaFormat.MIMETYPE_VIDEO_VP9;
+                        return "video/x-vnd.on2.vp9"; //MediaFormat.MIMETYPE_VIDEO_VP9
                     case H264: //Fall through
                     default:
-                        return MediaFormat.MIMETYPE_VIDEO_AVC;
+                        return "video/avc"; // MediaFormat.MIMETYPE_VIDEO_AVC
                 }
             }
         }
         return null;
     }
 
+    private class EncoderCompat implements Runnable {
+
+        @Override
+        public void run() {
+            try {
+                drainEncoder(false);
+            } catch (Exception e) {
+                Log.w(TAG, "Error attempting to drain encoder");
+            } finally {
+                mVideoEncoder.release();
+            }
+        }
+
+        @SuppressWarnings("deprecation")
+        void drainEncoder(boolean endOfStream) {
+            if (mVideoEncoder == null || mOutputListener == null) {
+                return;
+            }
+
+            if (endOfStream) {
+                mVideoEncoder.signalEndOfInputStream();
+            }
+
+            ByteBuffer[] encoderOutputBuffers = mVideoEncoder.getOutputBuffers();
+            Thread currentThread = Thread.currentThread();
+            while (!currentThread.isInterrupted()) {
+                int encoderStatus = mVideoEncoder.dequeueOutputBuffer(mVideoBufferInfo, -1);
+                if(encoderStatus < 0){
+                    if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
+                        // no output available yet
+                        if (!endOfStream) {
+                            break; // out of while
+                        }
+                    } else if (encoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
+                        // not expected for an encoder
+                        encoderOutputBuffers = mVideoEncoder.getOutputBuffers();
+                    } else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
+                        if (mH264CodecSpecificData == null) {
+                            MediaFormat format = mVideoEncoder.getOutputFormat();
+                            mH264CodecSpecificData = EncoderUtils.getCodecSpecificData(format);
+                        } else {
+                            Log.w(TAG, "Output format change notified more than once, ignoring.");
+                        }
+                    }
+                } else {
+                    if ((mVideoBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
+                        // If we already retrieve codec specific data via OUTPUT_FORMAT_CHANGED event,
+                        // we do not need this data.
+                        if (mH264CodecSpecificData != null) {
+                            mVideoBufferInfo.size = 0;
+                        } else {
+                            Log.i(TAG, "H264 codec specific data not retrieved yet.");
+                        }
+                    }
+
+                    if (mVideoBufferInfo.size != 0) {
+                        ByteBuffer encoderOutputBuffer = encoderOutputBuffers[encoderStatus];
+                        byte[] dataToWrite;
+                        int dataOffset = 0;
+
+                        // append SPS and PPS in front of every IDR NAL unit
+                        if ((mVideoBufferInfo.flags & MediaCodec.BUFFER_FLAG_KEY_FRAME) != 0 && mH264CodecSpecificData != null) {
+                            dataToWrite = new byte[mH264CodecSpecificData.length + mVideoBufferInfo.size];
+                            System.arraycopy(mH264CodecSpecificData, 0, dataToWrite, 0, mH264CodecSpecificData.length);
+                            dataOffset = mH264CodecSpecificData.length;
+                        } else {
+                            dataToWrite = new byte[mVideoBufferInfo.size];
+                        }
+
+                        try {
+                            encoderOutputBuffer.position(mVideoBufferInfo.offset);
+                            encoderOutputBuffer.limit(mVideoBufferInfo.offset + mVideoBufferInfo.size);
+
+                            encoderOutputBuffer.get(dataToWrite, dataOffset, mVideoBufferInfo.size);
+
+                            if (mOutputListener != null) {
+                                mOutputListener.sendFrame(dataToWrite, 0, dataToWrite.length, mVideoBufferInfo.presentationTimeUs);
+                            }
+                        } catch (Exception e) {
+                            e.printStackTrace();
+                        }
+                    }
+
+                    mVideoEncoder.releaseOutputBuffer(encoderStatus, false);
+
+                    if ((mVideoBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
+                        break; // out of while
+                    }
+                }
+            }
+        }
+    }
 }
\ No newline at end of file
diff --git a/sdl_android/src/main/java/com/smartdevicelink/proxy/SdlProxyBase.java b/sdl_android/src/main/java/com/smartdevicelink/proxy/SdlProxyBase.java
index d924f88cd..e1e80d5a0 100644
--- a/sdl_android/src/main/java/com/smartdevicelink/proxy/SdlProxyBase.java
+++ b/sdl_android/src/main/java/com/smartdevicelink/proxy/SdlProxyBase.java
@@ -4037,7 +4037,7 @@ public abstract class SdlProxyBase<proxyListenerType extends IProxyListenerBase>
     }

 

 	/**

-	 * Starts streaming a remote display to the module if there is a connected session. This method of streaming requires the device to be on API level 21 or higher

+	 * Starts streaming a remote display to the module if there is a connected session. This method of streaming requires the device to be on API level 19 or higher

 	 * @param context a context that can be used to create the remote display

 	 * @param remoteDisplay class object of the remote display. This class will be used to create an instance of the remote display and will be projected to the module

 	 * @param parameters streaming parameters to be used when streaming. If null is sent in, the default/optimized options will be used.

@@ -4046,7 +4046,7 @@ public abstract class SdlProxyBase<proxyListenerType extends IProxyListenerBase>
 	 *

 	 * @param encrypted a flag of if the stream should be encrypted. Only set if you have a supplied encryption library that the module can understand.

 	 */

-	@TargetApi(21)

+	@TargetApi(19)

 	public void startRemoteDisplayStream(Context context, final Class<? extends SdlRemoteDisplay> remoteDisplay, final VideoStreamingParameters parameters, final boolean encrypted){

 		if(getWiProVersion() >= 5 && !_systemCapabilityManager.isCapabilitySupported(SystemCapabilityType.VIDEO_STREAMING)){

 			Log.e(TAG, "Video streaming not supported on this module");

@@ -6231,7 +6231,7 @@ public abstract class SdlProxyBase<proxyListenerType extends IProxyListenerBase>
 	 * VideoStreamingManager houses all the elements needed to create a scoped, streaming manager for video projection. It is only a private, instance

 	 * dependant class at the moment until it can become public. Once the class is public and API defined, it will be moved into the SdlSession class

 	 */

-	@TargetApi(21)

+	@TargetApi(19)

 	private class VideoStreamingManager implements ISdlServiceListener{

 		Context context;

 		ISdl internalInterface;

@@ -6276,6 +6276,7 @@ public abstract class SdlProxyBase<proxyListenerType extends IProxyListenerBase>
 			} catch (Exception e) {

 				e.printStackTrace();

 			}

+			Log.d(TAG, parameters.toString());

 		}

 

 		public void stopStreaming(){