Merge pull request #604 from smartdevicelink/feature/issue_536_input_stream_file_manager

Implement SDL 0025 - Stream file manager uploads from disk

7 files changed, 223 insertions, 80 deletions

diff --git a/SmartDeviceLink/SDLError.h b/SmartDeviceLink/SDLError.h
index 09ec20ee8..717fb7ac5 100644
--- a/SmartDeviceLink/SDLError.h
+++ b/SmartDeviceLink/SDLError.h
@@ -39,6 +39,7 @@ extern SDLErrorDomain *const SDLErrorDomainFileManager;
 + (NSError *)sdl_fileManager_noKnownFileError;
 + (NSError *)sdl_fileManager_unableToStartError;
 + (NSError *)sdl_fileManager_unableToUploadError;
++ (NSError *)sdl_fileManager_fileDoesNotExistError;
 
 @end
 
diff --git a/SmartDeviceLink/SDLError.m b/SmartDeviceLink/SDLError.m
index 511f91deb..69ec729d5 100644
--- a/SmartDeviceLink/SDLError.m
+++ b/SmartDeviceLink/SDLError.m
@@ -140,6 +140,17 @@ SDLErrorDomain *const SDLErrorDomainFileManager = @"com.sdl.filemanager.error";
     return [NSError errorWithDomain:SDLErrorDomainFileManager code:SDLFileManagerErrorUnableToUpload userInfo:userInfo];
 }
 
+#pragma mark SDLUploadFileOperation
+
++ (NSError *)sdl_fileManager_fileDoesNotExistError {
+    NSDictionary<NSString *, NSString *> *userInfo = @{
+                                                       NSLocalizedDescriptionKey: NSLocalizedString(@"The file manager was unable to send the file", nil),
+                                                       NSLocalizedFailureReasonErrorKey: NSLocalizedString(@"This could be because the file does not exist at the specified file path or that passed data is invalid", nil),
+                                                       NSLocalizedRecoverySuggestionErrorKey: NSLocalizedString(@"Make sure that the the correct file path is being set and that the passed data is valid", nil)
+                                                       };
+return [NSError errorWithDomain:SDLErrorDomainFileManager code:SDLFileManagerErrorFileDoesNotExist userInfo:userInfo];
+}
+
 @end
 
 
diff --git a/SmartDeviceLink/SDLErrorConstants.h b/SmartDeviceLink/SDLErrorConstants.h
index c439d3fd6..9f532a6fe 100644
--- a/SmartDeviceLink/SDLErrorConstants.h
+++ b/SmartDeviceLink/SDLErrorConstants.h
@@ -62,4 +62,8 @@ typedef NS_ENUM(NSInteger, SDLFileManagerError) {
      * The file manager was unable to send this file.
      */
     SDLFileManagerErrorUnableToUpload = -4,
+    /**
+     * The file manager could not find the local file
+     */
+    SDLFileManagerErrorFileDoesNotExist = -5,
 };
diff --git a/SmartDeviceLink/SDLFile.h b/SmartDeviceLink/SDLFile.h
index 67a430705..eb21f3f3d 100644
--- a/SmartDeviceLink/SDLFile.h
+++ b/SmartDeviceLink/SDLFile.h
@@ -41,10 +41,19 @@ NS_ASSUME_NONNULL_BEGIN
 @property (copy, nonatomic, readonly) NSData *data;
 
 /**
+ *  The size of the binary data of the SDLFile.
+ */
+@property (nonatomic, readonly) unsigned long long fileSize;
+
+/**
  *  The system will attempt to determine the type of file that you have passed in. It will default to BINARY if it does not recognize the file type or the file type is not supported by SDL.
  */
 @property (strong, nonatomic, readonly) SDLFileType fileType;
 
+/**
+ * A stream to pull binary data from a SDLFile. The stream only pulls required data from the file on disk or in memory. This reduces memory usage while uploading a large file to the remote system as each chunk of data can be released immediately after it is uploaded.
+ */
+@property (nonatomic, readonly) NSInputStream *inputStream;
 
 - (instancetype)init NS_UNAVAILABLE;
 
diff --git a/SmartDeviceLink/SDLFile.m b/SmartDeviceLink/SDLFile.m
index e7bed7f74..9fc4d5423 100644
--- a/SmartDeviceLink/SDLFile.m
+++ b/SmartDeviceLink/SDLFile.m
@@ -23,6 +23,7 @@ NS_ASSUME_NONNULL_BEGIN
 @property (assign, nonatomic, readwrite) BOOL persistent;
 @property (copy, nonatomic, readwrite) NSString *name;
 
+@property (nonatomic, readwrite) NSInputStream *inputStream;
 @end
 
 
@@ -86,17 +87,50 @@ NS_ASSUME_NONNULL_BEGIN
     return [[self alloc] initWithData:data name:name fileExtension:extension persistent:NO];
 }
 
-
 #pragma mark - Getters
 
 - (NSData *)data {
     if (_data.length == 0 && _fileURL != nil) {
-        _data = [NSData dataWithContentsOfURL:_fileURL];
+        return [NSData dataWithContentsOfURL:_fileURL];
     }
 
     return _data;
 }
 
+/**
+ Initalizes a socket from which to read data.
+
+ @return A socket
+ */
+- (NSInputStream *)inputStream {
+    if (!_inputStream) {
+        if (_fileURL) {
+            // Data in file
+            _inputStream = [[NSInputStream alloc] initWithURL:_fileURL];
+        } else if (_data.length != 0) {
+            // Data in memory
+            _inputStream = [[NSInputStream alloc] initWithData:_data];
+        }
+    }
+    return _inputStream;
+}
+
+/**
+ Gets the size of the data. The data may be stored on disk or it may already be in the application's memory.
+
+ @return The size of the data.
+ */
+- (unsigned long long)fileSize {
+    if (_fileURL) {
+        // Data in file
+        NSString *path = [_fileURL path];
+        return [[NSFileManager defaultManager] attributesOfItemAtPath:path error:nil].fileSize;
+    } else if (_data) {
+        // Data in memory
+        return _data.length;
+    }
+    return 0;
+}
 
 #pragma mark - File Type
 
@@ -121,7 +155,6 @@ NS_ASSUME_NONNULL_BEGIN
     }
 }
 
-
 #pragma mark - NSCopying
 
 - (id)copyWithZone:(nullable NSZone *)zone {
diff --git a/SmartDeviceLink/SDLUploadFileOperation.h b/SmartDeviceLink/SDLUploadFileOperation.h
index 5b3d55f24..40c4b36c4 100644
--- a/SmartDeviceLink/SDLUploadFileOperation.h
+++ b/SmartDeviceLink/SDLUploadFileOperation.h
@@ -11,6 +11,7 @@
 #import "SDLAsynchronousOperation.h"
 #import "SDLFileManagerConstants.h"
 
+
 @protocol SDLConnectionManagerType;
 @class SDLFileWrapper;
 
@@ -32,4 +33,4 @@ NS_ASSUME_NONNULL_BEGIN
 
 @end
 
-NS_ASSUME_NONNULL_END
\ No newline at end of file
+NS_ASSUME_NONNULL_END
diff --git a/SmartDeviceLink/SDLUploadFileOperation.m b/SmartDeviceLink/SDLUploadFileOperation.m
index b1648bd4c..acde2d721 100644
--- a/SmartDeviceLink/SDLUploadFileOperation.m
+++ b/SmartDeviceLink/SDLUploadFileOperation.m
@@ -9,6 +9,7 @@
 #import "SDLUploadFileOperation.h"
 
 #import "SDLConnectionManagerType.h"
+#import "SDLError.h"
 #import "SDLFile.h"
 #import "SDLFileWrapper.h"
 #import "SDLGlobals.h"
@@ -21,11 +22,10 @@ NS_ASSUME_NONNULL_BEGIN
 
 #pragma mark - SDLUploadFileOperation
 
-@interface SDLUploadFileOperation ()
+@interface SDLUploadFileOperation () <NSStreamDelegate>
 
 @property (strong, nonatomic) SDLFileWrapper *fileWrapper;
 @property (weak, nonatomic) id<SDLConnectionManagerType> connectionManager;
-
 @end
 
 
@@ -51,112 +51,183 @@ NS_ASSUME_NONNULL_BEGIN
 
 - (void)start {
     [super start];
-
-    [self sdl_sendPutFiles:[self.class sdl_splitFile:self.fileWrapper.file mtuSize:[SDLGlobals sharedGlobals].maxMTUSize] withCompletion:self.fileWrapper.completionHandler];
+    [self sdl_sendPutFiles:self.fileWrapper.file mtuSize:[SDLGlobals sharedGlobals].maxMTUSize withCompletion:self.fileWrapper.completionHandler];
 }
 
-- (void)sdl_sendPutFiles:(NSArray<SDLPutFile *> *)putFiles withCompletion:(SDLFileManagerUploadCompletionHandler)completion {
-    __block BOOL stop = NO;
+/**
+ Sends data asynchronously to the SDL Core by breaking the data into smaller packets, each of which is sent via a putfile. To prevent large files from eating up memory, the data packet is deleted once it is sent via a putfile. If the SDL Core receives all the putfiles successfully, a success response with the amount of free storage space left on the SDL Core is returned. Otherwise the error returned by the SDL Core is passed along.
+
+ @param file The file containing the data to be sent to the SDL Core
+ @param mtuSize The maximum packet size allowed
+ @param completion Closure returning whether or not the upload was a success
+ */
+- (void)sdl_sendPutFiles:(SDLFile *)file mtuSize:(NSUInteger)mtuSize withCompletion:(SDLFileManagerUploadCompletionHandler)completion  {
     __block NSError *streamError = nil;
     __block NSUInteger bytesAvailable = 0;
     __block NSInteger highestCorrelationIDReceived = -1;
 
+    NSInputStream *inputStream = [self sdl_openInputStreamWithFile:file];
+
+    // If the file does not exist or the passed data is nil, return an error
+    if (inputStream == nil) {
+        return completion(NO, bytesAvailable, [NSError sdl_fileManager_fileDoesNotExistError]);
+    }
+
     dispatch_group_t putFileGroup = dispatch_group_create();
     dispatch_group_enter(putFileGroup);
 
-    // When the putfiles all complete, run this block
+    // Waits for all packets be sent before returning whether or not the upload was a success
     __weak typeof(self) weakself = self;
     dispatch_group_notify(putFileGroup, dispatch_get_main_queue(), ^{
-        if (streamError != nil || stop) {
+        typeof(weakself) strongself = weakself;
+        if (streamError != nil || strongself.isCancelled) {
             completion(NO, bytesAvailable, streamError);
         } else {
             completion(YES, bytesAvailable, nil);
         }
-
         [weakself finishOperation];
     });
 
-    for (SDLPutFile *putFile in putFiles) {
+    // Break the data into small pieces, each of which will be sent in a separate putfile
+    NSUInteger currentOffset = 0;
+    for (int i = 0; i < (((file.fileSize - 1) / mtuSize) + 1); i++) {
         dispatch_group_enter(putFileGroup);
+
+        // The putfile's length parameter is based on the current offset
+        SDLPutFile *putFile = [[SDLPutFile alloc] initWithFileName:file.name fileType:file.fileType persistentFile:file.isPersistent];
+        putFile.offset = @(currentOffset);
+        putFile.length = @([[self class] sdl_getPutFileLengthForOffset:currentOffset fileSize:file.fileSize mtuSize:mtuSize]);
+
+        // Get a chunk of data from the input stream
+        NSUInteger dataSize = [[self class] sdl_getDataSizeForOffset:currentOffset fileSize:file.fileSize mtuSize:mtuSize];
+        putFile.bulkData = [[self class] sdl_getDataChunkWithSize:dataSize inputStream:inputStream];
+        currentOffset += dataSize;
+
         __weak typeof(self) weakself = self;
-        [self.connectionManager sendManagerRequest:putFile
-                               withResponseHandler:^(__kindof SDLRPCRequest *_Nullable request, __kindof SDLRPCResponse *_Nullable response, NSError *_Nullable error) {
-                                   typeof(weakself) strongself = weakself;
-                                   // If we've already encountered an error, then just abort
-                                   // TODO: Is this the right way to handle this case? Should we just abort everything in the future? Should we be deleting what we sent? Should we have an automatic retry strategy based on what the error was?
-                                   if (strongself.isCancelled) {
-                                       stop = YES;
-                                   }
-
-                                   if (stop) {
-                                       dispatch_group_leave(putFileGroup);
-                                       BLOCK_RETURN;
-                                   }
-
-                                   // If we encounted an error, abort in the future and call the completion handler
-                                   if (error != nil || response == nil || ![response.success boolValue]) {
-                                       stop = YES;
-                                       streamError = error;
-                                       
-                                       dispatch_group_leave(putFileGroup);
-                                       BLOCK_RETURN;
-                                   }
-
-                                   // If we haven't encounted an error
-                                   SDLPutFileResponse *putFileResponse = (SDLPutFileResponse *)response;
-
-                                   // We need to do this to make sure our bytesAvailable is accurate
-                                   if ([request.correlationID integerValue] > highestCorrelationIDReceived) {
-                                       highestCorrelationIDReceived = [request.correlationID integerValue];
-                                       bytesAvailable = [putFileResponse.spaceAvailable unsignedIntegerValue];
-                                   }
-
-                                   dispatch_group_leave(putFileGroup);
-                               }];
+        [self.connectionManager sendManagerRequest:putFile withResponseHandler:^(__kindof SDLRPCRequest *_Nullable request, __kindof SDLRPCResponse *_Nullable response, NSError *_Nullable error) {
+            typeof(weakself) strongself = weakself;
+
+            // Check if the upload process has been cancelled by another packet. If so, stop the upload process.
+            // TODO: Is this the right way to handle this case? Should we just abort everything in the future? Should we be deleting what we sent? Should we have an automatic retry strategy based on what the error was?
+            if (strongself.isCancelled) {
+                dispatch_group_leave(putFileGroup);
+                BLOCK_RETURN;
+            }
+
+            // If the SDL Core returned an error, cancel the upload the process in the future
+            if (error != nil || response == nil || ![response.success boolValue] || strongself.isCancelled) {
+                [strongself cancel];
+                streamError = error;
+                dispatch_group_leave(putFileGroup);
+                BLOCK_RETURN;
+            }
+
+            // If no errors, watch for a response containing the amount of storage left on the SDL Core
+            if ([[self class] sdl_newHighestCorrelationID:request highestCorrelationIDReceived:highestCorrelationIDReceived]) {
+                highestCorrelationIDReceived = [request.correlationID integerValue];
+                bytesAvailable = [(SDLPutFileResponse *)response spaceAvailable].unsignedIntegerValue;
+            }
+
+            dispatch_group_leave(putFileGroup);
+        }];
     }
-
     dispatch_group_leave(putFileGroup);
 }
 
-+ (NSArray<SDLPutFile *> *)sdl_splitFile:(SDLFile *)file mtuSize:(NSUInteger)mtuSize {
-    NSData *fileData = [file.data copy];
-    NSUInteger currentOffset = 0;
-    NSMutableArray<SDLPutFile *> *putFiles = [NSMutableArray array];
+/**
+ Opens a socket for reading data.
+
+ @param file The file containing the data or the file url of the data
+ */
+- (NSInputStream *)sdl_openInputStreamWithFile:(SDLFile *)file {
+    NSInputStream *inputStream = file.inputStream;
+    [inputStream setDelegate:self];
+    [inputStream scheduleInRunLoop:[NSRunLoop currentRunLoop]
+                           forMode:NSDefaultRunLoopMode];
+    [inputStream open];
+    return inputStream;
+}
 
-    // http://stackoverflow.com/a/503201 Make sure we get the exact number of packets we need
-    for (int i = 0; i < (((fileData.length - 1) / mtuSize) + 1); i++) {
-        SDLPutFile *putFile = [[SDLPutFile alloc] initWithFileName:file.name fileType:file.fileType persistentFile:file.isPersistent];
-        putFile.offset = @(currentOffset);
+/**
+ Returns the length of the data being sent in the putfile. The first putfile's length is unique in that it sends the full size of the data. For the rest of the putfiles, the length parameter is equal to the size of the chunk of data being sent in the putfile.
+
+ @param currentOffset The current position in the file
+ @param fileSize The size of the file
+ @param mtuSize The maximum packet size allowed
+ @return The the length of the data being sent in the putfile
+ */
++ (NSUInteger)sdl_getPutFileLengthForOffset:(NSUInteger)currentOffset fileSize:(unsigned long long)fileSize mtuSize:(NSUInteger)mtuSize {
+    NSInteger putFileLength = 0;
+    if (currentOffset == 0) {
+        // The first putfile sends the full file size
+        putFileLength = (NSInteger)fileSize;
+    } else if ((fileSize - currentOffset) < mtuSize) {
+        // The last putfile sends the size of the remaining data
+        putFileLength = (NSInteger)(fileSize - currentOffset);
+    } else {
+        // All other putfiles send the maximum allowed packet size
+        putFileLength = mtuSize;
+    }
+    return putFileLength;
+}
 
-        // Set the length putfile based on the offset
-        if (currentOffset == 0) {
-            // If the offset is 0, the putfile expects to have the full file length within it
-            putFile.length = @(fileData.length);
-        } else if ((fileData.length - currentOffset) < mtuSize) {
-            // The file length remaining is less than our total MTU size, so use the file length remaining
-            putFile.length = @(fileData.length - currentOffset);
-        } else {
-            // The file length remaining is greater than our total MTU size, and the offset is not zero, we will fill the packet with our max MTU size
-            putFile.length = @(mtuSize);
-        }
+/**
+ Gets the size of the data to be sent in a packet. Packet size can not be greater than the max MTU size allowed by the SDL Core.
+
+ @param currentOffset The position in the file where to start reading data
+ @param fileSize The size of the file
+ @param mtuSize The maximum packet size allowed
+ @return The size of the data to be sent in the packet.
+ */
++ (NSUInteger)sdl_getDataSizeForOffset:(NSUInteger)currentOffset fileSize:(unsigned long long)fileSize mtuSize:(NSUInteger)mtuSize {
+    NSInteger dataSize = 0;
+    NSUInteger fileSizeRemaining = (NSUInteger)(fileSize - currentOffset);
+    if (fileSizeRemaining < mtuSize) {
+        dataSize = fileSizeRemaining;
+    } else {
+        dataSize = mtuSize;
+    }
+    return dataSize;
+}
 
-        // Place the data and set the new offset
-        if (currentOffset == 0 && (mtuSize < [putFile.length unsignedIntegerValue])) {
-            putFile.bulkData = [fileData subdataWithRange:NSMakeRange(currentOffset, mtuSize)];
-            currentOffset = mtuSize;
-        } else {
-            putFile.bulkData = [fileData subdataWithRange:NSMakeRange(currentOffset, [putFile.length unsignedIntegerValue])];
-            currentOffset += [putFile.length unsignedIntegerValue];
-        }
+/**
+ Reads a chunk of data from a socket.
 
-        [putFiles addObject:putFile];
+ @param size The amount of data to read from the input stream
+ @param inputStream The socket from which to read the data
+ @return The data read from the socket
+ */
++ (nullable NSData *)sdl_getDataChunkWithSize:(NSInteger)size inputStream:(NSInputStream *)inputStream {
+    if (size <= 0) {
+        return nil;
+    }
+    
+    Byte buffer[size];
+    NSInteger bytesRead = [inputStream read:buffer maxLength:size];
+    if (bytesRead) {
+        return [[NSData alloc] initWithBytes:(const void*)buffer length:size];
+    } else {
+        // TODO: return a custom error?
+        return nil;
     }
-
-    return putFiles;
 }
 
+/**
+ One of the responses returned by the SDL Core will contain the correct remaining free storage size on the SDL Core. Since communication with the SDL Core is asynchronous, there is no way to predict which response contains the correct bytes available other than to watch for the largest correlation id, since that will be the last response sent by the SDL Core.
+
+ @param request The newest response returned by the SDL Core for a putfile
+ @param highestCorrelationIDReceived The largest currently received correlation id
+ @return Whether or not the newest request contains the highest correlationId
+ */
++ (BOOL)sdl_newHighestCorrelationID:(SDLRPCRequest *)request highestCorrelationIDReceived:(NSInteger)highestCorrelationIDReceived {
+    if ([request.correlationID integerValue] > highestCorrelationIDReceived) {
+        return true;
+    }
+
+    return false;
+}
 
-#pragma mark Property Overrides
+#pragma mark - Property Overrides
 
 - (nullable NSString *)name {
     return self.fileWrapper.file.name;
@@ -166,6 +237,19 @@ NS_ASSUME_NONNULL_BEGIN
     return NSOperationQueuePriorityNormal;
 }
 
+#pragma mark - NSStreamDelegate
+
+- (void)stream:(NSStream *)aStream handleEvent:(NSStreamEvent)eventCode {
+    switch (eventCode) {
+        case NSStreamEventEndEncountered:
+            // Close the input stream once all the data has been read
+            [aStream close];
+            [aStream removeFromRunLoop:[NSRunLoop currentRunLoop] forMode:NSDefaultRunLoopMode];
+        default:
+            break;
+    }
+}
+
 @end
 
 NS_ASSUME_NONNULL_END