Add public method -sendVideoData:pts:

Also, latest PTS value is kept in SDLStreamingMediaLifecycleManager
so that video image with outdated PTS will be rejected.

6 files changed, 66 insertions, 4 deletions

diff --git a/SmartDeviceLink/SDLStreamingMediaLifecycleManager.h b/SmartDeviceLink/SDLStreamingMediaLifecycleManager.h
index d1b0e446a..4ca6ebea9 100644
--- a/SmartDeviceLink/SDLStreamingMediaLifecycleManager.h
+++ b/SmartDeviceLink/SDLStreamingMediaLifecycleManager.h
@@ -146,6 +146,16 @@ extern SDLAudioStreamState *const SDLAudioStreamStateShuttingDown;
 - (BOOL)sendVideoData:(CVImageBufferRef)imageBuffer;
 
 /**
+ *  This method receives raw image data and will run iOS8+'s hardware video encoder to turn the data into a video stream, which will then be passed to the connected head unit.
+ *
+ *  @param imageBuffer  A CVImageBufferRef to be encoded by Video Toolbox
+ *  @param pts A presentation timestamp for the frame, or kCMTimeInvalid if timestamp is unknown. If it's valid, it must be greater than the previous one.
+ *
+ *  @return Whether or not the data was successfully encoded and sent.
+ */
+- (BOOL)sendVideoData:(CVImageBufferRef)imageBuffer pts:(CMTime)pts;
+
+/**
  *  This method receives PCM audio data and will attempt to send that data across to the head unit for immediate playback
  *
  *  @param audioData    The data in PCM audio format, to be played
diff --git a/SmartDeviceLink/SDLStreamingMediaLifecycleManager.m b/SmartDeviceLink/SDLStreamingMediaLifecycleManager.m
index b56313eba..a2265e0e7 100644
--- a/SmartDeviceLink/SDLStreamingMediaLifecycleManager.m
+++ b/SmartDeviceLink/SDLStreamingMediaLifecycleManager.m
@@ -64,6 +64,8 @@ static NSUInteger const SDLFramesToSendOnBackground = 30;
 
 @property (assign, nonatomic) CV_NULLABLE CVPixelBufferRef backgroundingPixelBuffer;
 
+@property (assign, nonatomic) CMTime lastPTS;
+
 @end
 
 
@@ -114,6 +116,8 @@ static NSUInteger const SDLFramesToSendOnBackground = 30;
     [[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(sdl_appStateDidUpdate:) name:UIApplicationDidBecomeActiveNotification object:nil];
     [[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(sdl_appStateDidUpdate:) name:UIApplicationWillResignActiveNotification object:nil];
     
+    _lastPTS = kCMTimeInvalid;
+
     return self;
 }
 
@@ -136,6 +140,10 @@ static NSUInteger const SDLFramesToSendOnBackground = 30;
 }
 
 - (BOOL)sendVideoData:(CVImageBufferRef)imageBuffer {
+    return [self sendVideoData:imageBuffer pts:kCMTimeInvalid];
+}
+
+- (BOOL)sendVideoData:(CVImageBufferRef)imageBuffer pts:(CMTime)pts {
     if (!self.isVideoConnected) {
         SDLLogW(@"Attempted to send video data, but not connected");
         return NO;
@@ -146,8 +154,20 @@ static NSUInteger const SDLFramesToSendOnBackground = 30;
         SDLLogW(@"Attempted to send video data, but the app is not in LIMITED or FULL HMI state");
         return NO;
     }
-    
-    return [self.videoEncoder encodeFrame:imageBuffer];
+
+    /*
+     * reject input image for following cases:
+     * - pts is not increasing
+     * - app tries to send images while background images are shown
+     */
+    if (CMTIME_IS_VALID(self.lastPTS) && CMTIME_IS_VALID(pts)
+        && CMTIME_COMPARE_INLINE(pts, <=, self.lastPTS)) {
+        SDLLogW(@"The video data is out of date");
+        return NO;
+    }
+    self.lastPTS = pts;
+
+    return [self.videoEncoder encodeFrame:imageBuffer pts:pts];
 }
 
 - (BOOL)sendAudioData:(NSData*)audioData {
@@ -297,6 +317,7 @@ static NSUInteger const SDLFramesToSendOnBackground = 30;
             
             self.backgroundingPixelBuffer = backgroundingPixelBuffer;
         }
+        self.lastPTS = kCMTimeInvalid;
     }
     
     [[NSNotificationCenter defaultCenter] postNotificationName:SDLVideoStreamDidStartNotification object:nil];
@@ -517,8 +538,14 @@ static NSUInteger const SDLFramesToSendOnBackground = 30;
         return;
     }
     
+    const CMTime interval = CMTimeMake(1, 30);
     for (int frameCount = 0; frameCount < SDLFramesToSendOnBackground; frameCount++) {
-        [self.videoEncoder encodeFrame:self.backgroundingPixelBuffer];
+        if (CMTIME_IS_VALID(self.lastPTS)) {
+            self.lastPTS = CMTimeAdd(self.lastPTS, interval);
+            [self.videoEncoder encodeFrame:self.backgroundingPixelBuffer pts:self.lastPTS];
+        } else {
+            [self.videoEncoder encodeFrame:self.backgroundingPixelBuffer];
+        }
     }
 }
 
diff --git a/SmartDeviceLink/SDLStreamingMediaManager.h b/SmartDeviceLink/SDLStreamingMediaManager.h
index 232856d40..e47beeecc 100644
--- a/SmartDeviceLink/SDLStreamingMediaManager.h
+++ b/SmartDeviceLink/SDLStreamingMediaManager.h
@@ -114,6 +114,16 @@ NS_ASSUME_NONNULL_BEGIN
 - (BOOL)sendVideoData:(CVImageBufferRef)imageBuffer;
 
 /**
+ *  This method receives raw image data and will run iOS8+'s hardware video encoder to turn the data into a video stream, which will then be passed to the connected head unit.
+ *
+ *  @param imageBuffer A CVImageBufferRef to be encoded by Video Toolbox
+ *  @param pts A presentation timestamp for the frame, or kCMTimeInvalid if timestamp is unknown. If it's valid, it must be greater than the previous one.
+ *
+ *  @return Whether or not the data was successfully encoded and sent.
+ */
+- (BOOL)sendVideoData:(CVImageBufferRef)imageBuffer pts:(CMTime)pts;
+
+/**
  *  This method receives PCM audio data and will attempt to send that data across to the head unit for immediate playback
  *
  *  @param audioData The data in PCM audio format, to be played
diff --git a/SmartDeviceLink/SDLStreamingMediaManager.m b/SmartDeviceLink/SDLStreamingMediaManager.m
index 1c4291931..a3e98262a 100644
--- a/SmartDeviceLink/SDLStreamingMediaManager.m
+++ b/SmartDeviceLink/SDLStreamingMediaManager.m
@@ -53,6 +53,10 @@ NS_ASSUME_NONNULL_BEGIN
     return [self.lifecycleManager sendVideoData:imageBuffer];
 }
 
+- (BOOL)sendVideoData:(CVImageBufferRef)imageBuffer pts:(CMTime)pts {
+    return [self.lifecycleManager sendVideoData:imageBuffer pts:pts];
+}
+
 - (BOOL)sendAudioData:(NSData*)audioData {
     return [self.lifecycleManager sendAudioData:audioData];
 }
diff --git a/SmartDeviceLink/SDLVideoEncoder.h b/SmartDeviceLink/SDLVideoEncoder.h
index e48f8c311..4cdaac87c 100644
--- a/SmartDeviceLink/SDLVideoEncoder.h
+++ b/SmartDeviceLink/SDLVideoEncoder.h
@@ -60,6 +60,8 @@ extern NSString *const SDLErrorDomainVideoEncoder;
 
 - (BOOL)encodeFrame:(CVImageBufferRef)imageBuffer;
 
+- (BOOL)encodeFrame:(CVImageBufferRef)imageBuffer pts:(CMTime)pts;
+
 /**
  *  Creates a new pixel buffer using the pixelBufferPool property.
  */
diff --git a/SmartDeviceLink/SDLVideoEncoder.m b/SmartDeviceLink/SDLVideoEncoder.m
index 426b0075b..57e266a64 100644
--- a/SmartDeviceLink/SDLVideoEncoder.m
+++ b/SmartDeviceLink/SDLVideoEncoder.m
@@ -125,7 +125,16 @@ static NSDictionary<NSString *, id>* _defaultVideoEncoderSettings;
 }
 
 - (BOOL)encodeFrame:(CVImageBufferRef)imageBuffer {
-    OSStatus status = VTCompressionSessionEncodeFrame(_compressionSession, imageBuffer, CMTimeMake(self.currentFrameNumber++, 30), kCMTimeInvalid, NULL, (__bridge void *)self, NULL);
+    return [self encodeFrame:imageBuffer pts:kCMTimeInvalid];
+}
+
+- (BOOL)encodeFrame:(CVImageBufferRef)imageBuffer pts:(CMTime)pts {
+    if (!CMTIME_IS_VALID(pts)) {
+        pts = CMTimeMake(self.currentFrameNumber, 30);
+    }
+    self.currentFrameNumber++;
+
+    OSStatus status = VTCompressionSessionEncodeFrame(_compressionSession, imageBuffer, pts, kCMTimeInvalid, NULL, (__bridge void *)self, NULL);
 
     return (status == noErr);
 }