created new compression session to send background frames on, added new delegate method to SDLVideoEncoderDelegate to pass saved encoded frames to the StreamingVidLifecycleManager

4 files changed, 114 insertions, 3 deletions

diff --git a/SmartDeviceLink/SDLH264VideoEncoder.h b/SmartDeviceLink/SDLH264VideoEncoder.h
index 58bde0508..c519347f5 100644
--- a/SmartDeviceLink/SDLH264VideoEncoder.h
+++ b/SmartDeviceLink/SDLH264VideoEncoder.h
@@ -66,6 +66,10 @@ extern NSString *const SDLErrorDomainVideoEncoder;
 
 - (BOOL)encodeFrame:(CVImageBufferRef)imageBuffer presentationTimestamp:(CMTime)presentationTimestamp;
 
+- (BOOL)encodeSavedFrame:(CVImageBufferRef)imageBuffer;
+
+- (BOOL)encodeSavedFrame:(CVImageBufferRef)imageBuffer presentationTimestamp:(CMTime)presentationTimestamp;
+
 /**
  *  Creates a new pixel buffer using the pixelBufferPool property.
  */
diff --git a/SmartDeviceLink/SDLH264VideoEncoder.m b/SmartDeviceLink/SDLH264VideoEncoder.m
index 7ce981f5e..34ec4d2c8 100644
--- a/SmartDeviceLink/SDLH264VideoEncoder.m
+++ b/SmartDeviceLink/SDLH264VideoEncoder.m
@@ -22,6 +22,7 @@ static NSDictionary<NSString *, id>* _defaultVideoEncoderSettings;
 @interface SDLH264VideoEncoder ()
 
 @property (assign, nonatomic, nullable) VTCompressionSessionRef compressionSession;
+@property (assign, nonatomic, nullable) VTCompressionSessionRef backgroundFrameSession;
 @property (assign, nonatomic, nullable) CFDictionaryRef sdl_pixelBufferOptions;
 @property (assign, nonatomic) NSUInteger currentFrameNumber;
 @property (assign, nonatomic) double timestampOffset;
@@ -57,11 +58,24 @@ static NSDictionary<NSString *, id>* _defaultVideoEncoderSettings;
     }
     
     _compressionSession = NULL;
+    _backgroundFrameSession = NULL;
     _currentFrameNumber = 0;
     _videoEncoderSettings = properties;
     _dimensions = dimensions;
     
     _delegate = delegate;
+
+    OSStatus backgroundStatus;
+
+    backgroundStatus = VTCompressionSessionCreate(NULL, (int32_t)dimensions.width, (int32_t)dimensions.height, kCMVideoCodecType_H264, NULL, self.sdl_pixelBufferOptions, NULL, &sdl_backgroundVideoEncoderOutputCallback, (__bridge void *)self, &_backgroundFrameSession);
+    if (backgroundStatus != noErr) {
+        if (error != NULL) {
+            *error = [NSError errorWithDomain:SDLErrorDomainVideoEncoder code:SDLVideoEncoderErrorConfigurationCompressionSessionCreationFailure userInfo:@{@"OSStatus":@(backgroundStatus), NSLocalizedDescriptionKey:@"Compression session could not be created"}];
+        }
+
+        return nil;
+    }
+
     
     OSStatus status;
     
@@ -89,6 +103,16 @@ static NSDictionary<NSString *, id>* _defaultVideoEncoderSettings;
         
         return nil;
     }
+
+    // background
+    backgroundStatus = VTSessionCopySupportedPropertyDictionary(self.compressionSession, &supportedProperties);
+    if (backgroundStatus != noErr) {
+        if (error != NULL) {
+            *error = [NSError errorWithDomain:SDLErrorDomainVideoEncoder code:SDLVideoEncoderErrorConfigurationCompressionSessionSetPropertyFailure userInfo:@{@"OSStatus":@(backgroundStatus), NSLocalizedDescriptionKey:[NSString stringWithFormat:@"\"%@\" are not supported properties.", supportedProperties]}];
+        }
+
+        return nil;
+    }
     
     NSArray* videoEncoderKeys = self.videoEncoderSettings.allKeys;
     for (NSString *key in videoEncoderKeys) {
@@ -101,6 +125,19 @@ static NSDictionary<NSString *, id>* _defaultVideoEncoderSettings;
         }
     }
     CFRelease(supportedProperties);
+
+    // background
+    for (NSString *key in videoEncoderKeys) {
+        id value = self.videoEncoderSettings[key];
+
+        backgroundStatus = VTSessionSetProperty(self.backgroundFrameSession, (__bridge CFStringRef)key, (__bridge CFTypeRef)value);
+        if (backgroundStatus != noErr) {
+            if (error != NULL) {
+                *error = [NSError errorWithDomain:SDLErrorDomainVideoEncoder code:SDLVideoEncoderErrorConfigurationCompressionSessionSetPropertyFailure userInfo:@{@"OSStatus": @(backgroundStatus), NSLocalizedDescriptionKey:[NSString stringWithFormat:@"Setting key failed \"%@\"", key]}];
+            }
+            return nil;
+        }
+    }
     
     // Populate the video encoder settings from provided dictionary.
     for (NSString *key in videoEncoderKeys) {
@@ -140,6 +177,33 @@ static NSDictionary<NSString *, id>* _defaultVideoEncoderSettings;
         CFRelease(self.compressionSession);
         self.compressionSession = NULL;
     }
+
+    if (self.backgroundFrameSession != NULL) {
+        VTCompressionSessionInvalidate(self.backgroundFrameSession);
+        CFRelease(self.backgroundFrameSession);
+        self.backgroundFrameSession = NULL;
+    }
+}
+
+- (BOOL)encodeSavedFrame:(CVImageBufferRef)imageBuffer {
+    return [self encodeSavedFrame:imageBuffer presentationTimestamp:kCMTimeInvalid];
+
+}
+
+- (BOOL)encodeSavedFrame:(CVImageBufferRef)imageBuffer presentationTimestamp:(CMTime)presentationTimestamp {
+    if (!CMTIME_IS_VALID(presentationTimestamp)) {
+        int32_t timeRate = 30;
+        if (self.videoEncoderSettings[(__bridge NSString *)kVTCompressionPropertyKey_ExpectedFrameRate] != nil) {
+            timeRate = ((NSNumber *)self.videoEncoderSettings[(__bridge NSString *)kVTCompressionPropertyKey_ExpectedFrameRate]).intValue;
+        }
+
+        presentationTimestamp = CMTimeMake((int64_t)self.currentFrameNumber, timeRate);
+    }
+    self.currentFrameNumber++;
+
+    OSStatus status = VTCompressionSessionEncodeFrame(_backgroundFrameSession, imageBuffer, presentationTimestamp, kCMTimeInvalid, NULL, (__bridge void *)self, NULL);
+
+    return (status == noErr);
 }
 
 - (BOOL)encodeFrame:(CVImageBufferRef)imageBuffer {
@@ -198,6 +262,38 @@ static NSDictionary<NSString *, id>* _defaultVideoEncoderSettings;
 #pragma mark - Private
 #pragma mark Callback
 
+void sdl_backgroundVideoEncoderOutputCallback(void * CM_NULLABLE outputCallbackRefCon, void * CM_NULLABLE sourceFrameRefCon, OSStatus status, VTEncodeInfoFlags infoFlags, CM_NULLABLE CMSampleBufferRef sampleBuffer) {
+    // If there was an error in the encoding, drop the frame
+    if (status != noErr) {
+        SDLLogW(@"Error encoding video frame: %d", (int)status);
+        return;
+    }
+
+    if (outputCallbackRefCon == NULL || sourceFrameRefCon == NULL || sampleBuffer == NULL) {
+        return;
+    }
+
+    SDLH264VideoEncoder *encoder = (__bridge SDLH264VideoEncoder *)sourceFrameRefCon;
+    NSArray *nalUnits = [encoder.class sdl_extractNalUnitsFromSampleBuffer:sampleBuffer];
+
+    const CMTime presentationTimestampInCMTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer);
+    double presentationTimestamp = 0.0;
+    if (CMTIME_IS_VALID(presentationTimestampInCMTime)) {
+        presentationTimestamp = CMTimeGetSeconds(presentationTimestampInCMTime);
+    }
+    if (encoder.timestampOffset == 0.0) {
+        // remember this first timestamp as the offset
+        encoder.timestampOffset = presentationTimestamp;
+    }
+
+    NSArray *packets = [encoder.packetizer createPackets:nalUnits
+                                   presentationTimestamp:(presentationTimestamp - encoder.timestampOffset)];
+
+    if ([encoder.delegate respondsToSelector:@selector(videoEncoder:hasEncodedFramesToBeSaved:)]) {
+        [encoder.delegate videoEncoder:encoder hasEncodedFramesToBeSaved:packets];
+    }
+}
+
 /// Callback function that VideoToolbox calls when encoding is complete.
 /// @param outputCallbackRefCon The callback's reference value
 /// @param sourceFrameRefCon The frame's reference value
diff --git a/SmartDeviceLink/SDLStreamingVideoLifecycleManager.m b/SmartDeviceLink/SDLStreamingVideoLifecycleManager.m
index be9135cb8..39c16e338 100644
--- a/SmartDeviceLink/SDLStreamingVideoLifecycleManager.m
+++ b/SmartDeviceLink/SDLStreamingVideoLifecycleManager.m
@@ -81,6 +81,8 @@ typedef void(^SDLVideoCapabilityResponseHandler)(SDLVideoStreamingCapability *_N
 
 @property (assign, nonatomic, readwrite, getter=isVideoEncrypted) BOOL videoEncrypted;
 
+@property (nonatomic, copy) NSArray *savedBackgroundFrames;
+
 /**
  * SSRC of RTP header field.
  *
@@ -301,7 +303,9 @@ typedef void(^SDLVideoCapabilityResponseHandler)(SDLVideoStreamingCapability *_N
     }
 
     if (_showVideoBackgroundDisplay) {
-        [self sdl_sendBackgroundFrames];
+        for (NSData *packet in _savedBackgroundFrames) {
+            [self.videoEncoder.delegate videoEncoder:self.videoEncoder hasEncodedFrame:packet];
+        }
     }
     [self.touchManager cancelPendingTouches];
 
@@ -666,6 +670,10 @@ typedef void(^SDLVideoCapabilityResponseHandler)(SDLVideoStreamingCapability *_N
     }
 }
 
+- (void)videoEncoder:(SDLH264VideoEncoder *)encoder hasEncodedFramesToBeSaved:(NSArray *)encodedVideoFrames {
+    _savedBackgroundFrames = encodedVideoFrames;
+}
+
 #pragma mark - Streaming session helpers
 
 - (void)sdl_startVideoSession {
@@ -737,9 +745,9 @@ typedef void(^SDLVideoCapabilityResponseHandler)(SDLVideoStreamingCapability *_N
     for (int frameCount = 0; frameCount < FramesToSendOnBackground; frameCount++) {
         if (CMTIME_IS_VALID(self.lastPresentationTimestamp)) {
             self.lastPresentationTimestamp = CMTimeAdd(self.lastPresentationTimestamp, interval);
-            [self.videoEncoder encodeFrame:self.backgroundingPixelBuffer presentationTimestamp:self.lastPresentationTimestamp];
+            [self.videoEncoder encodeSavedFrame:self.backgroundingPixelBuffer presentationTimestamp:self.lastPresentationTimestamp];
         } else {
-            [self.videoEncoder encodeFrame:self.backgroundingPixelBuffer];
+            [self.videoEncoder encodeSavedFrame:self.backgroundingPixelBuffer];
         }
     }
 }
diff --git a/SmartDeviceLink/SDLVideoEncoderDelegate.h b/SmartDeviceLink/SDLVideoEncoderDelegate.h
index a8bf27f22..790daa3fb 100644
--- a/SmartDeviceLink/SDLVideoEncoderDelegate.h
+++ b/SmartDeviceLink/SDLVideoEncoderDelegate.h
@@ -14,4 +14,7 @@
 
 - (void)videoEncoder:(SDLH264VideoEncoder *)encoder hasEncodedFrame:(NSData*)encodedVideo;
 
+@optional
+- (void)videoEncoder:(SDLH264VideoEncoder *)encoder hasEncodedFramesToBeSaved:(NSArray *)encodedVideoFrames;
+
 @end