From 3a403034a7603c72763c3918288cd3230f1f32ea Mon Sep 17 00:00:00 2001 From: Joey Grover Date: Thu, 12 Oct 2017 14:34:53 -0400 Subject: Reduced min SDK version on virtual display encoder from 21 to 19 Should cover +90% of all phones at time of commit --- .../encoder/VirtualDisplayEncoder.java | 238 ++++++++++++++++----- .../com/smartdevicelink/proxy/SdlProxyBase.java | 7 +- 2 files changed, 184 insertions(+), 61 deletions(-) diff --git a/sdl_android/src/main/java/com/smartdevicelink/encoder/VirtualDisplayEncoder.java b/sdl_android/src/main/java/com/smartdevicelink/encoder/VirtualDisplayEncoder.java index 22c515957..0fe303c71 100644 --- a/sdl_android/src/main/java/com/smartdevicelink/encoder/VirtualDisplayEncoder.java +++ b/sdl_android/src/main/java/com/smartdevicelink/encoder/VirtualDisplayEncoder.java @@ -52,7 +52,7 @@ import com.smartdevicelink.streaming.video.VideoStreamingParameters; import java.nio.ByteBuffer; -@TargetApi(21) +@TargetApi(19) @SuppressWarnings("NullableProblems") public class VirtualDisplayEncoder { private static final String TAG = "VirtualDisplayEncoder"; @@ -66,6 +66,11 @@ public class VirtualDisplayEncoder { private Boolean initPassed = false; private final Object STREAMING_LOCK = new Object(); + // Codec-specific data (SPS and PPS) + private byte[] mH264CodecSpecificData = null; + + //For older (<21) OS versions + private Thread encoderThread; /** @@ -74,12 +79,12 @@ public class VirtualDisplayEncoder { * @param context to create the virtual display * @param outputListener the listener that the video frames will be sent through * @param streamingParams parameters to create the virtual display and encoder - * @throws Exception if the API level is <21 or supplied parameters were null + * @throws Exception if the API level is <19 or supplied parameters were null */ public void init(Context context, IVideoStreamListener outputListener, VideoStreamingParameters streamingParams) throws Exception { - if (android.os.Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) { - Log.e(TAG, "API level of 21 required for VirtualDisplayEncoder"); - throw new Exception("API level of 21 required"); + if (android.os.Build.VERSION.SDK_INT < Build.VERSION_CODES.KITKAT) { + Log.e(TAG, "API level of 19 required for VirtualDisplayEncoder"); + throw new Exception("API level of 19 required"); } if (context == null || outputListener == null || streamingParams == null || streamingParams.getResolution() == null || streamingParams.getFormat() == null) { @@ -87,7 +92,7 @@ public class VirtualDisplayEncoder { throw new Exception("init parameters cannot be null"); } - this.mDisplayManager = (DisplayManager)context.getSystemService(Context.DISPLAY_SERVICE); + this.mDisplayManager = (DisplayManager) context.getSystemService(Context.DISPLAY_SERVICE); this.streamingParams.update(streamingParams); @@ -102,12 +107,12 @@ public class VirtualDisplayEncoder { } @SuppressWarnings("unused") - public void setStreamingParams(int displayDensity, ImageResolution resolution, int frameRate, int bitrate, int interval, VideoStreamingFormat format){ - this.streamingParams = new VideoStreamingParameters(displayDensity,frameRate,bitrate,interval,resolution,format); + public void setStreamingParams(int displayDensity, ImageResolution resolution, int frameRate, int bitrate, int interval, VideoStreamingFormat format) { + this.streamingParams = new VideoStreamingParameters(displayDensity, frameRate, bitrate, interval, resolution, format); } @SuppressWarnings("unused") - public void setStreamingParams(VideoStreamingParameters streamingParams){ + public void setStreamingParams(VideoStreamingParameters streamingParams) { this.streamingParams = streamingParams; } @@ -116,11 +121,11 @@ public class VirtualDisplayEncoder { * Prepares the encoder and virtual display. */ public void start() throws Exception { - if(!initPassed){ + if (!initPassed) { Log.e(TAG, "VirtualDisplayEncoder was not properly initialized with the init() method."); return; } - if(streamingParams == null || streamingParams.getResolution() == null || streamingParams.getFormat() == null){ + if (streamingParams == null || streamingParams.getResolution() == null || streamingParams.getFormat() == null) { return; } @@ -144,11 +149,15 @@ public class VirtualDisplayEncoder { } public void shutDown() { - if(!initPassed){ + if (!initPassed) { Log.e(TAG, "VirtualDisplayEncoder was not properly initialized with the init() method."); return; } try { + if (encoderThread != null) { + encoderThread.interrupt(); + encoderThread = null; + } if (mVideoEncoder != null) { mVideoEncoder.stop(); @@ -165,15 +174,14 @@ public class VirtualDisplayEncoder { inputSurface.release(); inputSurface = null; } - } - catch (Exception ex){ + } catch (Exception ex) { Log.e(TAG, "shutDown() failed"); } } private Surface prepareVideoEncoder() { - if(streamingParams == null || streamingParams.getResolution() == null || streamingParams.getFormat() == null){ + if (streamingParams == null || streamingParams.getResolution() == null || streamingParams.getFormat() == null) { return null; } @@ -196,48 +204,67 @@ public class VirtualDisplayEncoder { try { mVideoEncoder = MediaCodec.createEncoderByType(videoMimeType); mVideoEncoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); - Surface surface = mVideoEncoder.createInputSurface(); - mVideoEncoder.setCallback(new MediaCodec.Callback() { - @Override - public void onInputBufferAvailable(MediaCodec codec, int index) { - // nothing to do here - } + Surface surface = mVideoEncoder.createInputSurface(); //prepared + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { + mVideoEncoder.setCallback(new MediaCodec.Callback() { + @Override + public void onInputBufferAvailable(MediaCodec codec, int index) { + // nothing to do here + } - @Override - public void onOutputBufferAvailable(MediaCodec codec, int index, MediaCodec.BufferInfo info) { - try { - ByteBuffer encodedData = codec.getOutputBuffer(index); - if(encodedData!=null) { - - encodedData.position(info.offset); - encodedData.limit(info.offset + info.size); - - if (info.size != 0) { - byte[] dataToWrite = new byte[info.size]; - encodedData.get(dataToWrite, - info.offset, info.size); - if (mOutputListener != null) { - mOutputListener.sendFrame(dataToWrite, 0, dataToWrite.length, info.presentationTimeUs); + @TargetApi(Build.VERSION_CODES.LOLLIPOP) + @Override + public void onOutputBufferAvailable(MediaCodec codec, int index, MediaCodec.BufferInfo info) { + try { + ByteBuffer encodedData = codec.getOutputBuffer(index); + if (encodedData != null) { + if (info.size != 0) { + byte[] dataToWrite;// = new byte[info.size]; + int dataOffset = 0; + + // append SPS and PPS in front of every IDR NAL unit + if ((info.flags & MediaCodec.BUFFER_FLAG_KEY_FRAME) != 0 && mH264CodecSpecificData != null) { + dataToWrite = new byte[mH264CodecSpecificData.length + info.size]; + System.arraycopy(mH264CodecSpecificData, 0, dataToWrite, 0, mH264CodecSpecificData.length); + dataOffset = mH264CodecSpecificData.length; + } else { + dataToWrite = new byte[info.size]; + } + + encodedData.position(info.offset); + encodedData.limit(info.offset + info.size); + + encodedData.get(dataToWrite, dataOffset, info.size); + if (mOutputListener != null) { + mOutputListener.sendFrame(dataToWrite, 0, dataToWrite.length, info.presentationTimeUs); + } } - } - codec.releaseOutputBuffer(index, false); + codec.releaseOutputBuffer(index, false); + } + } catch (Exception ex) { + ex.printStackTrace(); } - } catch (Exception ex) { - ex.printStackTrace(); } - } - @Override - public void onError(MediaCodec codec, MediaCodec.CodecException e) { - e.printStackTrace(); - } + @Override + public void onError(MediaCodec codec, MediaCodec.CodecException e) { + e.printStackTrace(); + } - @Override - public void onOutputFormatChanged(MediaCodec codec, MediaFormat format) { - // nothing to do here - } - }); + @Override + public void onOutputFormatChanged(MediaCodec codec, MediaFormat format) { + mH264CodecSpecificData = EncoderUtils.getCodecSpecificData(format); + } + }); + } else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT) { + //Implied from previous if check that this has to be older than Build.VERSION_CODES.LOLLIPOP + encoderThread = new Thread(new EncoderCompat()); + + } else { + Log.e(TAG, "Unable to start encoder. Android OS version " + Build.VERSION.SDK_INT + "is not supported"); + } return surface; @@ -251,28 +278,123 @@ public class VirtualDisplayEncoder { if (mVideoEncoder != null) { mVideoEncoder.start(); } + if (encoderThread != null) { + encoderThread.start(); + } } - public Display getVirtualDisplay(){ + public Display getVirtualDisplay() { return virtualDisplay.getDisplay(); } - private String getMimeForFormat(VideoStreamingFormat format){ - if(format != null){ + private String getMimeForFormat(VideoStreamingFormat format) { + if (format != null) { VideoStreamingCodec codec = format.getCodec(); - if(codec != null){ - switch(codec){ + if (codec != null) { + switch (codec) { //MediaFormat constants are only available in Android 21+ case VP8: - return MediaFormat.MIMETYPE_VIDEO_VP8; + return "video/x-vnd.on2.vp8"; //MediaFormat.MIMETYPE_VIDEO_VP8 case VP9: - return MediaFormat.MIMETYPE_VIDEO_VP9; + return "video/x-vnd.on2.vp9"; //MediaFormat.MIMETYPE_VIDEO_VP9 case H264: //Fall through default: - return MediaFormat.MIMETYPE_VIDEO_AVC; + return "video/avc"; // MediaFormat.MIMETYPE_VIDEO_AVC } } } return null; } + private class EncoderCompat implements Runnable { + + @Override + public void run() { + try { + drainEncoder(false); + } catch (Exception e) { + Log.w(TAG, "Error attempting to drain encoder"); + } finally { + mVideoEncoder.release(); + } + } + + @SuppressWarnings("deprecation") + void drainEncoder(boolean endOfStream) { + if (mVideoEncoder == null || mOutputListener == null) { + return; + } + + if (endOfStream) { + mVideoEncoder.signalEndOfInputStream(); + } + + ByteBuffer[] encoderOutputBuffers = mVideoEncoder.getOutputBuffers(); + Thread currentThread = Thread.currentThread(); + while (!currentThread.isInterrupted()) { + int encoderStatus = mVideoEncoder.dequeueOutputBuffer(mVideoBufferInfo, -1); + if(encoderStatus < 0){ + if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) { + // no output available yet + if (!endOfStream) { + break; // out of while + } + } else if (encoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { + // not expected for an encoder + encoderOutputBuffers = mVideoEncoder.getOutputBuffers(); + } else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { + if (mH264CodecSpecificData == null) { + MediaFormat format = mVideoEncoder.getOutputFormat(); + mH264CodecSpecificData = EncoderUtils.getCodecSpecificData(format); + } else { + Log.w(TAG, "Output format change notified more than once, ignoring."); + } + } + } else { + if ((mVideoBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) { + // If we already retrieve codec specific data via OUTPUT_FORMAT_CHANGED event, + // we do not need this data. + if (mH264CodecSpecificData != null) { + mVideoBufferInfo.size = 0; + } else { + Log.i(TAG, "H264 codec specific data not retrieved yet."); + } + } + + if (mVideoBufferInfo.size != 0) { + ByteBuffer encoderOutputBuffer = encoderOutputBuffers[encoderStatus]; + byte[] dataToWrite; + int dataOffset = 0; + + // append SPS and PPS in front of every IDR NAL unit + if ((mVideoBufferInfo.flags & MediaCodec.BUFFER_FLAG_KEY_FRAME) != 0 && mH264CodecSpecificData != null) { + dataToWrite = new byte[mH264CodecSpecificData.length + mVideoBufferInfo.size]; + System.arraycopy(mH264CodecSpecificData, 0, dataToWrite, 0, mH264CodecSpecificData.length); + dataOffset = mH264CodecSpecificData.length; + } else { + dataToWrite = new byte[mVideoBufferInfo.size]; + } + + try { + encoderOutputBuffer.position(mVideoBufferInfo.offset); + encoderOutputBuffer.limit(mVideoBufferInfo.offset + mVideoBufferInfo.size); + + encoderOutputBuffer.get(dataToWrite, dataOffset, mVideoBufferInfo.size); + + if (mOutputListener != null) { + mOutputListener.sendFrame(dataToWrite, 0, dataToWrite.length, mVideoBufferInfo.presentationTimeUs); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + mVideoEncoder.releaseOutputBuffer(encoderStatus, false); + + if ((mVideoBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { + break; // out of while + } + } + } + } + } } \ No newline at end of file diff --git a/sdl_android/src/main/java/com/smartdevicelink/proxy/SdlProxyBase.java b/sdl_android/src/main/java/com/smartdevicelink/proxy/SdlProxyBase.java index d924f88cd..e1e80d5a0 100644 --- a/sdl_android/src/main/java/com/smartdevicelink/proxy/SdlProxyBase.java +++ b/sdl_android/src/main/java/com/smartdevicelink/proxy/SdlProxyBase.java @@ -4037,7 +4037,7 @@ public abstract class SdlProxyBase } /** - * Starts streaming a remote display to the module if there is a connected session. This method of streaming requires the device to be on API level 21 or higher + * Starts streaming a remote display to the module if there is a connected session. This method of streaming requires the device to be on API level 19 or higher * @param context a context that can be used to create the remote display * @param remoteDisplay class object of the remote display. This class will be used to create an instance of the remote display and will be projected to the module * @param parameters streaming parameters to be used when streaming. If null is sent in, the default/optimized options will be used. @@ -4046,7 +4046,7 @@ public abstract class SdlProxyBase * * @param encrypted a flag of if the stream should be encrypted. Only set if you have a supplied encryption library that the module can understand. */ - @TargetApi(21) + @TargetApi(19) public void startRemoteDisplayStream(Context context, final Class remoteDisplay, final VideoStreamingParameters parameters, final boolean encrypted){ if(getWiProVersion() >= 5 && !_systemCapabilityManager.isCapabilitySupported(SystemCapabilityType.VIDEO_STREAMING)){ Log.e(TAG, "Video streaming not supported on this module"); @@ -6231,7 +6231,7 @@ public abstract class SdlProxyBase * VideoStreamingManager houses all the elements needed to create a scoped, streaming manager for video projection. It is only a private, instance * dependant class at the moment until it can become public. Once the class is public and API defined, it will be moved into the SdlSession class */ - @TargetApi(21) + @TargetApi(19) private class VideoStreamingManager implements ISdlServiceListener{ Context context; ISdl internalInterface; @@ -6276,6 +6276,7 @@ public abstract class SdlProxyBase } catch (Exception e) { e.printStackTrace(); } + Log.d(TAG, parameters.toString()); } public void stopStreaming(){ -- cgit v1.2.1