modified filetransfer to be more robust in restarting transfers

Signed-off-by: Alexander Wenzel <Alexander.AW.Wenzel@bmw.de>

3 files changed, 615 insertions, 266 deletions

diff --git a/src/system/dlt-system-filetransfer.c b/src/system/dlt-system-filetransfer.c
index 1a6debf..d0ff7c7 100644
--- a/src/system/dlt-system-filetransfer.c
+++ b/src/system/dlt-system-filetransfer.c
@@ -57,8 +57,13 @@
 #include "dlt_filetransfer.h"
 
 #define INOTIFY_SZ (sizeof(struct inotify_event))
-#define INOTIFY_LEN (INOTIFY_SZ + 256)
+#define INOTIFY_LEN (INOTIFY_SZ + NAME_MAX + 1)
 #define Z_CHUNK_SZ 1024*128
+#define COMPRESS_EXTENSION ".gz"
+#define SUBDIR_COMPRESS ".tocompress"
+#define SUBDIR_TOSEND ".tosend"
+
+
 
 extern DltSystemThreads threads;
 // From dlt_filetransfer
@@ -68,292 +73,642 @@ DLT_IMPORT_CONTEXT(dltsystem);
 DLT_DECLARE_CONTEXT(filetransferContext)
 
 typedef struct {
-	int handle;
-	int fd[DLT_SYSTEM_LOG_DIRS_MAX];
+    int handle;
+    int fd[DLT_SYSTEM_LOG_DIRS_MAX];
 } s_ft_inotify;
 
 s_ft_inotify ino;
 
-char *unique_name(const char *src)
+
+char *origin_name(char *src){
+    if (strlen( (char*) basename(src)) > 10 ){
+        return (char*)(basename(src)+10);
+    }
+    else{
+        DLT_LOG(dltsystem, DLT_LOG_ERROR,
+                DLT_STRING("dlt-system-filetransfer, error in recreating origin name!"));
+        return NULL;
+    }
+}
+
+char *unique_name(char *src)
 {
-	DLT_LOG(dltsystem, DLT_LOG_DEBUG,
-			DLT_STRING("dlt-system-filetransfer, creating unique temporary file name."));
-	time_t t = time(NULL);
-	unsigned long l = getFileSerialNumber(src) ^ t;
-	// Length of ULONG_MAX + 1
-	char *ret = malloc(11);
-	MALLOC_ASSERT(ret);
-	snprintf(ret, 11, "%lu", l);
-	return ret;
+    DLT_LOG(dltsystem, DLT_LOG_DEBUG,
+            DLT_STRING("dlt-system-filetransfer, creating unique temporary file name."));
+    time_t t = time(NULL);
+    unsigned long l = getFileSerialNumber(src) ^ t;
+    // Length of ULONG_MAX + 1
+    //char *ret = malloc(11);
+
+    char *basename_f = basename(src);
+    // Length of ULONG_MAX + 1
+    int len = 11+strlen(basename_f);
+    if (len > NAME_MAX){
+        DLT_LOG(dltsystem, DLT_LOG_WARN,
+                DLT_STRING("dlt-system-filetransfer, unique name creation needs to shorten the filename:"),DLT_STRING(basename_f));
+        len = NAME_MAX;
+    }
+
+    char *ret = malloc(len);
+
+    MALLOC_ASSERT(ret);
+    snprintf(ret, len, "%lu%s", l,basename_f);
+    return ret;
 }
 
-char *compress_file(char *src, int level)
+/**
+  * Function which only calls the relevant part to transfer the payload
+  */
+
+void send_dumped_file(FiletransferOptions const *opts,char *dst_tosend){
+
+
+    char *fn = origin_name(dst_tosend);
+    DLT_LOG(dltsystem, DLT_LOG_DEBUG,
+            DLT_STRING("dlt-system-filetransfer, sending dumped file:"),DLT_STRING(fn));
+    if(dlt_user_log_file_header_alias(&filetransferContext, dst_tosend, fn) == 0)
+    {
+        int pkgcount = dlt_user_log_file_packagesCount(&filetransferContext, dst_tosend);
+        int lastpkg = 0;
+        int success = 1;
+        while(lastpkg < pkgcount)
+        {
+            int total = 2;
+            int used = 2;
+            dlt_user_check_buffer(&total, &used);
+            while((total-used) < (total/2))
+            {
+                struct timespec t;
+                t.tv_sec = 0;
+                t.tv_nsec = 1000000ul*opts->TimeoutBetweenLogs;
+                nanosleep(&t, NULL);
+                dlt_user_check_buffer(&total, &used);
+            }
+            lastpkg++;
+            if(dlt_user_log_file_data(&filetransferContext, dst_tosend, lastpkg, opts->TimeoutBetweenLogs) < 0)
+            {
+                success = 0;
+                break;
+            }
+        }
+        if (success)
+            dlt_user_log_file_end(&filetransferContext, dst_tosend, 1);
+    }
+    DLT_LOG(dltsystem, DLT_LOG_DEBUG,
+            DLT_STRING("dlt-system-filetransfer, sent dumped file"));
+}
+
+/**
+ * compress file, delete the source file
+ * modification: compress into subdirectory
+ * File whis is compress will be deleted afterwards
+ *  @param src File to be sent
+ *  @param dst destination where to compress the file
+ * @param level of compression
+**/
+int compress_file_to(char *src, char *dst, int level)
 {
-	DLT_LOG(dltsystem, DLT_LOG_DEBUG,
-			DLT_STRING("dlt-system-filetransfer, compressing file."));
-	char *buf;
-	char *dst = malloc(strlen(src)+4);
-	MALLOC_ASSERT(dst);
-	char dst_mode[8];
-	sprintf(dst, "%s.gz", src);
-	sprintf(dst_mode, "wb%d", level);
-
-	gzFile dst_file;
-	FILE *src_file;
-
-	dst_file = gzopen(dst, dst_mode);
-	if(dst_file == Z_NULL)
-	{
-		free(dst);
-		return NULL;
-	}
-
-	src_file = fopen(src, "r");
-
-	if(src_file == NULL)
-	{
-		gzclose(dst_file);
-		free(dst);
-		return NULL;
-	}
-
-	buf = malloc(Z_CHUNK_SZ);
-	MALLOC_ASSERT(buf);
-
-	while(!feof(src_file))
-	{
-		int read = fread(buf, 1, Z_CHUNK_SZ, src_file);
-		if(ferror(src_file))
-		{
-			free(buf);
-			free(dst);
-			gzclose(dst_file);
-                        fclose(src_file);
-			return NULL;
-		}
-		gzwrite(dst_file, buf, read);
-	}
-
-	if(remove(src) < 0)
-		DLT_LOG(dltsystem, DLT_LOG_WARN, DLT_STRING("Could not remove file"), DLT_STRING(src));
-	free(buf);
-	fclose(src_file);
-	gzclose(dst_file);
-	return dst;
+    DLT_LOG(dltsystem, DLT_LOG_DEBUG,
+            DLT_STRING("dlt-system-filetransfer, compressing file from:"),DLT_STRING(src),DLT_STRING("to:"),DLT_STRING(dst));
+    char *buf;
+
+
+    char dst_mode[8];
+    snprintf(dst_mode,8, "wb%d", level);
+
+    gzFile dst_file;
+    FILE *src_file;
+
+    dst_file = gzopen(dst, dst_mode);
+    if(dst_file == Z_NULL)
+    {
+
+        return -1;
+    }
+
+    src_file = fopen(src, "r");
+
+    if(src_file == NULL)
+    {
+        gzclose(dst_file);
+
+        return -1;
+    }
+
+    buf = malloc(Z_CHUNK_SZ);
+    MALLOC_ASSERT(buf);
+
+    while(!feof(src_file))
+    {
+        int read = fread(buf, 1, Z_CHUNK_SZ, src_file);
+        if(ferror(src_file))
+        {
+            free(buf);
+
+            gzclose(dst_file);
+            fclose(src_file);
+            return -1;
+        }
+        gzwrite(dst_file, buf, read);
+    }
+
+    if(remove(src) < 0)
+        DLT_LOG(dltsystem, DLT_LOG_WARN, DLT_STRING("Could not remove file"), DLT_STRING(src));
+    free(buf);
+    fclose(src_file);
+    gzclose(dst_file);
+
+    return 0;
 }
 
-int send_one(char *src, FiletransferOptions opts, int which)
+//!Sends one file over DLT.
+/**
+ * If configured in opts, compresses it, then sends it.
+ * uses subdirecties for compressing and before sending, to avoid that those files get changed in the meanwhile
+ *
+ */
+int send_one(char *src, FiletransferOptions const *opts, int which)
 {
-	DLT_LOG(dltsystem, DLT_LOG_DEBUG,
-			DLT_STRING("dlt-system-filetransfer, sending a file."));
-	sleep(opts.TimeDelay);
-
-	// Prepare all needed file names
-	char *fn = basename(src);
-	char *rn = unique_name(src);
-	char *dst = malloc(strlen(opts.TempDir)+strlen(rn)+2);
-        char *dst_tmp = NULL;
-	MALLOC_ASSERT(fn);
-	MALLOC_ASSERT(rn);
-	MALLOC_ASSERT(dst);
-
-	sprintf(dst, "%s/%s", opts.TempDir, rn);
-	if(rename(src, dst) < 0)
-	{
-		DLT_LOG(dltsystem, DLT_LOG_ERROR,
-				DLT_STRING("Could not move file"),
-				DLT_STRING(src),
-				DLT_STRING(dst));
-		free(rn);
-		free(dst);
-		return -1;
-	}
-
-	// Compress if needed
-	if(opts.Compression[which] > 0)
-	{
-                dst_tmp = dst;
-                dst = compress_file(dst_tmp, opts.CompressionLevel[which]);
-                free(dst_tmp);//no more used
-		char *old_fn = fn;
-		fn = malloc(strlen(old_fn)+4);
-		MALLOC_ASSERT(fn);
-		sprintf(fn, "%s.gz", old_fn);
-	}
-
-        if(dlt_user_log_file_header_alias(&filetransferContext, dst, fn) == 0)
-	{
-                int pkgcount = dlt_user_log_file_packagesCount(&filetransferContext, dst);
-		int lastpkg = 0;
-		while(lastpkg < pkgcount)
+    DLT_LOG(dltsystem, DLT_LOG_DEBUG,
+            DLT_STRING("dlt-system-filetransfer, sending a file."));
+    sleep(opts->TimeDelay);
+
+    // Prepare all needed file names
+    char *fn = basename(src);
+    char *fdir = malloc(strlen(src)+1);
+    MALLOC_ASSERT(fdir);
+    strncpy(fdir,src,strlen(src));
+    *(fdir+strlen(fdir))='\0';
+    fdir = dirname(fdir);//dirname overwrites its argument anyway
+    char *dst_tosend;//file which is going to be sent
+
+    char *rn = unique_name(src);//new unique filename based on inode
+
+
+    MALLOC_ASSERT(fn);
+    MALLOC_ASSERT(rn);
+
+    // Compress if needed
+    if(opts->Compression[which] > 0)
+    {
+        DLT_LOG(dltsystem, DLT_LOG_DEBUG,
+                DLT_STRING("dlt-system-filetransfer, Moving file to tmp directory for compressing it."));
+
+        char *dst_tocompress;//file which is going to be compressed, the compressed one is named dst_tosend
+
+
+        int len = strlen(fdir)+strlen(SUBDIR_COMPRESS)+strlen(rn)+3;//the filename in .tocompress +2 for 2*"/", +1 for \0
+        dst_tocompress = malloc(len);
+        MALLOC_ASSERT(dst_tocompress);
+
+        snprintf(dst_tocompress,len,"%s/%s/%s",fdir,SUBDIR_COMPRESS,rn);
+
+
+
+
+        //moving in subdir, from where it can be compressed
+        if(rename(src, dst_tocompress) < 0)
+        {
+            DLT_LOG(dltsystem, DLT_LOG_ERROR,
+                    DLT_STRING("Could not move file"),
+                    DLT_STRING(src),
+                    DLT_STRING(dst_tocompress));
+            free(rn);
+            free(dst_tocompress);
+            free(fdir);
+            return -1;
+        }
+        len = strlen(fdir)+strlen(SUBDIR_TOSEND)+strlen(rn)+strlen(COMPRESS_EXTENSION)+3;//the resulting filename in .tosend +2 for 2*"/", +1 for \0
+        dst_tosend = malloc(len);
+        MALLOC_ASSERT(dst_tosend);
+        snprintf(dst_tosend,len,"%s/%s/%s%s",fdir,SUBDIR_TOSEND,rn,COMPRESS_EXTENSION);
+
+
+
+        if (compress_file_to(dst_tocompress,dst_tosend, opts->CompressionLevel[which]) != 0){
+            free(rn);
+            free(dst_tosend);
+            free(dst_tocompress);
+            free(fdir);
+            return -1;
+        }
+        free(dst_tocompress);
+
+    }
+    else{
+        //move it directly into "tosend"
+        DLT_LOG(dltsystem, DLT_LOG_DEBUG,
+                DLT_STRING("dlt-system-filetransfer, Moving file to tmp directory."));
+        int len = strlen(fdir)+strlen(SUBDIR_TOSEND)+strlen(rn)+3;
+        dst_tosend = malloc(len);//the resulting filename in .tosend +2 for 2*"/", +1 for \0
+
+        snprintf(dst_tosend,len,"%s/%s/%s",fdir,SUBDIR_TOSEND,rn);
+
+        DLT_LOG(dltsystem, DLT_LOG_DEBUG,
+                DLT_STRING("dlt-system-filetransfer, Rename:"),DLT_STRING(src),DLT_STRING("to: "),DLT_STRING(dst_tosend));
+        //moving in subdir, from where it can be compressed
+        if(rename(src, dst_tosend) < 0)
+        {
+            DLT_LOG(dltsystem, DLT_LOG_ERROR,
+                    DLT_STRING("Could not move file"),
+                    DLT_STRING(src),
+                    DLT_STRING(dst_tosend));
+            free(rn);
+            free(dst_tosend);
+            free(fdir);
+            return -1;
+        }
+
+    }
+
+    DLT_LOG(dltsystem, DLT_LOG_DEBUG,
+            DLT_STRING("dlt-system-filetransfer, File ready to send"));
+
+    send_dumped_file(opts,dst_tosend);
+
+
+    free(rn);
+    free(dst_tosend);
+    free(fdir);
+
+    return 0;
+}
+
+
+int flush_dir_send(FiletransferOptions const *opts, const char *compress_dir,  const char *send_dir){
+
+    struct dirent *dp;
+    DIR *dir;
+    dir = opendir(send_dir);
+
+    if(dir != NULL)
+    {
+        while((dp = readdir(dir)) != NULL)
+        {
+            if(dp->d_type != DT_REG)
+                continue;
+            char *fn;
+            DLT_LOG(dltsystem, DLT_LOG_DEBUG,
+                    DLT_STRING("dlt-system-filetransfer, old compressed file found in send directory:"),DLT_STRING(dp->d_name));
+            int len = strlen(send_dir)+strlen(dp->d_name)+2;
+            fn = malloc(len);
+            MALLOC_ASSERT(fn);
+            snprintf(fn,len, "%s/%s", send_dir, dp->d_name);
+
+
+            //if we have a file here and in the to_compress dir, we delete the to_send file: we can not be sure, that it has been properly compressed!
+            if (!strncmp( dp->d_name+strlen(dp->d_name)-strlen(COMPRESS_EXTENSION),COMPRESS_EXTENSION,strlen(COMPRESS_EXTENSION)))
+            {
+
+                //ends with ".gz"
+                //old file name (not: path) would have been:
+                char tmp[strlen(dp->d_name)-strlen(COMPRESS_EXTENSION)+1];
+                strncpy(tmp,dp->d_name,strlen(dp->d_name)-strlen(COMPRESS_EXTENSION));
+                tmp[strlen(dp->d_name)-3]='\0';
+
+                int len = strlen(tmp)+strlen(compress_dir)+1+1;//2 sizes + 1*"/" + \0
+                char *path_uncompressed = malloc(len);
+                MALLOC_ASSERT(path_uncompressed);
+                snprintf(path_uncompressed,len,"%s/%s",compress_dir,tmp);
+
+                struct stat sb;
+                if (stat(path_uncompressed,&sb)==-1)
+                {
+                    //uncompressed equivalent does not exist. We can send it out.
+                    DLT_LOG(dltsystem, DLT_LOG_DEBUG,
+                            DLT_STRING("dlt-system-filetransfer, sending file."));
+
+                    send_dumped_file(opts,fn);
+                }
+                else
                 {
-                        int total = 2;
-                        int used = 2;
-                        dlt_user_check_buffer(&total, &used);
-                        while((total-used) < (total/2))
+                    //There is an uncompressed file. Compression seems to have been interrupted -> delete the compressed file instead of sending it!
+                    DLT_LOG(dltsystem, DLT_LOG_DEBUG,
+                            DLT_STRING("dlt-system-filetransfer, uncompressed version exists. Deleting partially compressed version."));
+                    if (sb.st_mode & S_IFREG)
+                    {
+
+
+                        if( remove(fn ) != 0 )
                         {
-                                struct timespec t;
-                                t.tv_sec = 0;
-                                t.tv_nsec = 1000000ul*opts.TimeoutBetweenLogs;
-                                nanosleep(&t, NULL);
-                                dlt_user_check_buffer(&total, &used);
+                            //"Error deleting file". Continue? If we would cancel, maybe the dump is never sent! Deletion would again be tried in next LC.
+                            DLT_LOG(dltsystem, DLT_LOG_ERROR,
+                                    DLT_STRING("Error deleting file:"),DLT_STRING(fn));
                         }
-                        lastpkg++;
-                        if(dlt_user_log_file_data(&filetransferContext, dst, lastpkg, opts.TimeoutBetweenLogs) < 0)
-                                break;
+                    }
+                    else
+                    {
+                        //"Oldfile is a not reg file. Is this possible? Can we compress a directory?: %s\n",path_uncompressed);
+                        DLT_LOG(dltsystem, DLT_LOG_DEBUG,
+                                DLT_STRING("dlt-system-filetransfer, Oldfile is a not regular file! Do we have a problem?"),DLT_STRING(fn));
+                    }
+
                 }
-                dlt_user_log_file_end(&filetransferContext, dst, 1);
-	}
-
-	if(opts.Compression[which] > 0)
-		free(fn);
-	free(rn);
-	free(dst);
-	return 0;
+                free(path_uncompressed);//it is no more used. It would be transferred in next step.
+            }//it is a .gz file
+            else{
+                //uncompressed file. We can just resend it, the action to put it here was a move action.
+                DLT_LOG(dltsystem, DLT_LOG_DEBUG,
+                        DLT_STRING("dlt-system-filetransfer, Sending uncompressed file from previous LC."),DLT_STRING(fn));
+                send_dumped_file(opts,fn);
+            }
+            free(fn);
+        }
+    }
+    else
+    {
+        DLT_LOG(dltsystem, DLT_LOG_ERROR,
+                DLT_STRING("Could not open directory"),
+                DLT_STRING(send_dir));
+        return -1;
+    }
+    closedir(dir);//end: send_dir
+    return 0;
+}
+
+
+int flush_dir_compress(FiletransferOptions const *opts, int which, const char *compress_dir, const char *send_dir){
+
+    //check for files in compress_dir. Assumption: a file which lies here, should have been compressed, but that action was interrupted.
+    //As it can arrive here only by a rename, it is most likely to be a complete file
+    struct dirent *dp;
+    DIR *dir;
+    dir = opendir(compress_dir);
+    if(dir != NULL)
+    {
+        while((dp = readdir(dir)) != NULL)
+        {
+            if(dp->d_type != DT_REG)
+                continue;
+            DLT_LOG(dltsystem, DLT_LOG_DEBUG,
+                    DLT_STRING("dlt-system-filetransfer, old file found in compress-directory."));
+
+
+            //compress file into to_send dir
+            int len = strlen(compress_dir)+strlen(dp->d_name)+2;
+            char *cd_filename = malloc(len);
+            MALLOC_ASSERT(cd_filename);
+            snprintf(cd_filename,len,"%s/%s",compress_dir,dp->d_name);
+
+
+            len = strlen(send_dir)+strlen(dp->d_name)+strlen(COMPRESS_EXTENSION)+2;
+            char *dst_tosend = malloc(len);//the resulting filename in .tosend +2 for 1*"/", +1 for \0 + .gz
+            MALLOC_ASSERT(dst_tosend);
+            snprintf(dst_tosend,len,"%s/%s%s",send_dir,dp->d_name,COMPRESS_EXTENSION);
+
+            if (compress_file_to(cd_filename,dst_tosend, opts->CompressionLevel[which]) != 0){
+                free(dst_tosend);
+                free(cd_filename);
+                closedir(dir);
+                return -1;
+            }
+
+            //send file
+            send_dumped_file(opts,dst_tosend);
+            free(dst_tosend);
+            free(cd_filename);
+        }
+    }
+    else
+    {
+        DLT_LOG(dltsystem, DLT_LOG_ERROR,
+                DLT_STRING("Could not open directory"),
+                DLT_STRING(compress_dir));
+        return -1;
+    }
+    closedir(dir);//end: compress_dir
+
+    return 0;
+}
+
+int flush_dir_original(FiletransferOptions const *opts, int which){
+    struct dirent *dp;
+    DIR *dir;
+    const char *sdir = opts->Directory[which];
+    dir = opendir(sdir);
+    if(dir != NULL)
+    {
+        while((dp = readdir(dir)) != NULL)
+        {
+            if(dp->d_type != DT_REG){
+                //we don't send directories
+                continue;
+            }
+            DLT_LOG(dltsystem, DLT_LOG_DEBUG,
+                    DLT_STRING("dlt-system-filetransfer, old file found in directory."));
+            int len = strlen(sdir)+strlen(dp->d_name)+2;
+            char *fn = malloc(len);
+            MALLOC_ASSERT(fn);
+            snprintf(fn,len, "%s/%s", sdir, dp->d_name);
+            if(send_one(fn, opts, which) < 0)
+            {
+                closedir(dir);
+                free(fn);
+                return -1;
+            }
+            free(fn);
+        }
+    }
+    else
+    {
+        DLT_LOG(dltsystem, DLT_LOG_ERROR,
+                DLT_STRING("Could not open directory"),
+                DLT_STRING(sdir));
+        return -1;
+    }
+    closedir(dir);
+    return 0;
 }
 
-int flush_dir(FiletransferOptions opts, int which)
+//!Cleans the surveyed directories and subdirectories. Sends residing files into trace
+/**
+ * @param opts FiletransferOptions
+ * @param which which directory is affected -> position in list of opts->Directory
+ * @return Returns 0 if everything was okay. If there was a failure a value < 0 will be returned.
+ */
+int flush_dir(FiletransferOptions const *opts, int which)
 {
-	DLT_LOG(dltsystem, DLT_LOG_DEBUG,
-			DLT_STRING("dlt-system-filetransfer, flush directory of old files."));
-	const char *sdir = opts.Directory[which];
-	char *fn;
-	struct dirent *dp;
-	DIR *dir;
-
-	dir = opendir(sdir);
-	if(dir != NULL)
-	{
-		while((dp = readdir(dir)) != NULL)
-		{
-			if(dp->d_type != DT_REG)
-				continue;
-			DLT_LOG(dltsystem, DLT_LOG_DEBUG,
-					DLT_STRING("dlt-system-filetransfer, old file found in directory."));
-			fn = malloc(strlen(sdir)+dp->d_reclen+2);
-			MALLOC_ASSERT(fn);
-			sprintf(fn, "%s/%s", sdir, dp->d_name);
-			if(send_one(fn, opts, which) < 0)
-                        {
-                                closedir(dir);
-				return -1;
-                        }
-		}
-	}
-	else
-	{
-		DLT_LOG(dltsystem, DLT_LOG_ERROR,
-				DLT_STRING("Could not open directory"),
-				DLT_STRING(sdir));
-		return -1;
-	}
-	closedir(dir);
-	return 0;
+
+
+    DLT_LOG(dltsystem, DLT_LOG_DEBUG,
+            DLT_STRING("dlt-system-filetransfer, flush directory of old files."));
+
+    char *compress_dir;
+    char *send_dir;
+    int len = strlen(opts->Directory[which])+strlen(SUBDIR_COMPRESS)+2;
+    compress_dir = malloc (len);
+    MALLOC_ASSERT(compress_dir);
+    snprintf(compress_dir,len,"%s/%s",opts->Directory[which],SUBDIR_COMPRESS);
+
+    len = strlen(opts->Directory[which])+strlen(SUBDIR_TOSEND)+2;
+    send_dir = malloc (len);
+    MALLOC_ASSERT(send_dir);
+    snprintf(send_dir,len,"%s/%s",opts->Directory[which],SUBDIR_TOSEND);
+
+    //1st: scan the tosend directory.
+    if ( 0 != flush_dir_send(opts, compress_dir, send_dir) ){
+        free(send_dir);
+        free(compress_dir);
+        return -1;
+    }
+
+    //1nd: scan the tocompress directory.
+    if (0 != flush_dir_compress(opts, which, compress_dir, send_dir)){
+        free(send_dir);
+        free(compress_dir);
+        return -1;
+    }
+
+    free(send_dir);//no more used
+    free(compress_dir);
+
+    //last step: scan the original directory - we can reuse the send_one function
+    if ( 0 != flush_dir_original(opts,which)){
+        return -1;
+    }
+
+    return 0;
 }
 
-int init_filetransfer_dirs(FiletransferOptions opts)
+//!Initializes the surveyed directories
+/**On startup, the inotifiy handlers are created, and existing files shall be sent into DLT stream
+ * @param opts FiletransferOptions
+ * @return Returns 0 if everything was okay. If there was a failure a value < 0 will be returned.
+ */
+int init_filetransfer_dirs(FiletransferOptions const *opts)
 {
-	DLT_LOG(dltsystem, DLT_LOG_DEBUG,
-			DLT_STRING("dlt-system-filetransfer, initializing inotify on directories."));
-	ino.handle = inotify_init();
-	int i;
-
-	if(ino.handle < 0)
-	{
-		DLT_LOG(filetransferContext, DLT_LOG_FATAL,
-		DLT_STRING("Failed to initialize inotify in dlt-system file transfer."));
-		return -1;
-	}
-	for(i = 0;i < opts.Count;i++)
-	{
-		ino.fd[i] = inotify_add_watch(ino.handle, opts.Directory[i],
-				IN_CLOSE_WRITE|IN_MOVED_TO);
-		if(ino.fd[i] < 0)
-		{
-			char buf[1024];
-			snprintf(buf, 1024, "Failed to add inotify watch to directory %s in dlt-system file transfer.",
-					opts.Directory[i]);
-			DLT_LOG(filetransferContext, DLT_LOG_FATAL,
-			DLT_STRING(buf));
-			return -1;
-		}
-		flush_dir(opts, i);
-	}
-	return 0;
+    DLT_LOG(dltsystem, DLT_LOG_DEBUG,
+            DLT_STRING("dlt-system-filetransfer, initializing inotify on directories."));
+    ino.handle = inotify_init();
+    int i;
+
+    if(ino.handle < 0)
+    {
+        DLT_LOG(filetransferContext, DLT_LOG_FATAL,
+                DLT_STRING("Failed to initialize inotify in dlt-system file transfer."));
+        return -1;
+    }
+
+    for(i = 0;i < opts->Count;i++)
+    {
+        //create subdirectories for processing the files
+
+        char *subdirpath;
+        int len = strlen(opts->Directory[i])+strlen(SUBDIR_COMPRESS)+2;
+        subdirpath= malloc (len);
+        MALLOC_ASSERT(subdirpath);
+        snprintf(subdirpath,len,"%s/%s",opts->Directory[i],SUBDIR_COMPRESS);
+        int ret = mkdir(subdirpath,0777);
+
+        if (0 != ret && EEXIST != errno){
+            DLT_LOG(dltsystem, DLT_LOG_ERROR,
+                    DLT_STRING("dlt-system-filetransfer, error creating subdirectory: "),DLT_STRING(subdirpath),DLT_STRING(" Errorcode: "),DLT_INT(errno));
+            free (subdirpath);
+            return -1;
+        }
+        free(subdirpath);
+
+        len = strlen(opts->Directory[i])+strlen(SUBDIR_TOSEND)+2;
+        subdirpath= malloc (len);
+        MALLOC_ASSERT(subdirpath);
+        snprintf(subdirpath,len,"%s/%s",opts->Directory[i],SUBDIR_TOSEND);
+        mkdir(subdirpath,0777);
+        if (0 != ret && EEXIST != errno){
+            DLT_LOG(dltsystem, DLT_LOG_ERROR,
+                    DLT_STRING("dlt-system-filetransfer, error creating subdirectory: "),DLT_STRING(subdirpath),DLT_STRING(" Errorcode: "),DLT_INT(errno));
+            free (subdirpath);
+            return -1;
+        }
+        free(subdirpath);
+
+
+        ino.fd[i] = inotify_add_watch(ino.handle, opts->Directory[i],
+                                      IN_CLOSE_WRITE|IN_MOVED_TO);
+        if(ino.fd[i] < 0)
+        {
+            char buf[1024];
+            snprintf(buf, 1024, "Failed to add inotify watch to directory %s in dlt-system file transfer.",
+                     opts->Directory[i]);
+            DLT_LOG(filetransferContext, DLT_LOG_FATAL,
+                    DLT_STRING(buf));
+            return -1;
+        }
+
+
+        flush_dir(opts, i);
+
+    }
+    return 0;
 }
 
-int wait_for_files(FiletransferOptions opts)
+int wait_for_files(FiletransferOptions const *opts)
 {
-	DLT_LOG(dltsystem, DLT_LOG_DEBUG, DLT_STRING("dlt-system-filetransfer, waiting for files."));
-	static char buf[INOTIFY_LEN];
-	int len = read(ino.handle, buf, INOTIFY_LEN);
-	if(len < 0)
-	{
-		DLT_LOG(filetransferContext, DLT_LOG_ERROR,
-		DLT_STRING("Error while waiting for files in dlt-system file transfer."));
-		return -1;
-	}
-
-	int i = 0;
-        while(i < (len-INOTIFY_SZ))
-	{
-		struct inotify_event *ie = (struct inotify_event *)&buf[i];
-		if(ie->len > 0)
-		{
-			if(ie->mask & IN_CLOSE_WRITE || ie->mask & IN_MOVED_TO)
-			{
-				int j;
-				for(j = 0;j < opts.Count;j++)
-				{
-					if(ie->wd == ino.fd[j])
-					{
-                                                DLT_LOG(dltsystem, DLT_LOG_DEBUG, DLT_STRING("dlt-system-filetransfer, found new file."));
-                                                if ((i + INOTIFY_SZ + ie->len) < INOTIFY_LEN){
-                                                        char *tosend = malloc(strlen(opts.Directory[j])+ie->len+1);
-                                                        sprintf(tosend, "%s/%s", opts.Directory[j], ie->name);
-                                                        send_one(tosend, opts, j);
-                                                        free(tosend);
-                                                }
-                                                else{
-                                                        DLT_LOG(dltsystem, DLT_LOG_ERROR, DLT_STRING("dlt-system-filetransfer, filename out of bounds!."));
-                                                }
-                                        }
-				}
-			}
-		}
-		i += INOTIFY_SZ + ie->len;
-	}
-	return 0;
+    DLT_LOG(dltsystem, DLT_LOG_DEBUG, DLT_STRING("dlt-system-filetransfer, waiting for files."));
+    static char buf[INOTIFY_LEN];
+    int len = read(ino.handle, buf, INOTIFY_LEN);
+    if(len < 0)
+    {
+        DLT_LOG(filetransferContext, DLT_LOG_ERROR,
+                DLT_STRING("Error while waiting for files in dlt-system file transfer."));
+        return -1;
+    }
+
+    int i = 0;
+    while(i<len)
+    {
+        struct inotify_event *ie = (struct inotify_event *)&buf[i];
+        if(ie->len > 0)
+        {
+            if(ie->mask & IN_CLOSE_WRITE || ie->mask & IN_MOVED_TO)
+            {
+                int j;
+                for(j = 0;j < opts->Count;j++)
+                {
+                    if(ie->wd == ino.fd[j])
+                    {
+                        DLT_LOG(dltsystem, DLT_LOG_DEBUG, DLT_STRING("dlt-system-filetransfer, found new file."));
+                        int length = strlen(opts->Directory[j])+ie->len+1;
+                        char *tosend = malloc(length);
+                        snprintf(tosend,length, "%s/%s", opts->Directory[j], ie->name);
+                        send_one(tosend, opts, j);
+                        free(tosend);
+                    }
+                }
+            }
+        }
+        i += INOTIFY_SZ + ie->len;
+    }
+    return 0;
 }
 
 void filetransfer_thread(void *v_conf)
 {
-	DLT_LOG(dltsystem, DLT_LOG_DEBUG, DLT_STRING("dlt-system-filetransfer, in thread."));
-	DltSystemConfiguration *conf = (DltSystemConfiguration *) v_conf;
-	DLT_REGISTER_CONTEXT(filetransferContext, conf->Filetransfer.ContextId,
-			"File transfer manager.");
-
-	sleep(conf->Filetransfer.TimeStartup);
-
-	if(init_filetransfer_dirs(conf->Filetransfer) < 0)
-		return;
-
-	while(!threads.shutdown)
-	{
-		if(wait_for_files(conf->Filetransfer) < 0)
-		{
-			DLT_LOG(dltsystem, DLT_LOG_ERROR, DLT_STRING("Error while waiting files. File transfer shutdown."));
-			return;
-		}
-		sleep(conf->Filetransfer.TimeDelay);
-	}
+    DLT_LOG(dltsystem, DLT_LOG_DEBUG, DLT_STRING("dlt-system-filetransfer, in thread."));
+    DltSystemConfiguration *conf = (DltSystemConfiguration *) v_conf;
+    DLT_REGISTER_CONTEXT(filetransferContext, conf->Filetransfer.ContextId,
+                         "File transfer manager.");
+
+    sleep(conf->Filetransfer.TimeStartup);
+
+    if(init_filetransfer_dirs(&(conf->Filetransfer)) < 0)
+        return;
+
+    while(!threads.shutdown)
+    {
+        if(wait_for_files(&(conf->Filetransfer)) < 0)
+        {
+            DLT_LOG(dltsystem, DLT_LOG_ERROR, DLT_STRING("Error while waiting files. File transfer shutdown."));
+            return;
+        }
+        sleep(conf->Filetransfer.TimeDelay);
+    }
 }
 
 void start_filetransfer(DltSystemConfiguration *conf)
 {
-	DLT_LOG(dltsystem, DLT_LOG_DEBUG, DLT_STRING("dlt-system-filetransfer, start."));
-	static pthread_attr_t t_attr;
-	static pthread_t pt;
-	pthread_create(&pt, &t_attr, (void *)filetransfer_thread, conf);
-	threads.threads[threads.count++] = pt;
+    DLT_LOG(dltsystem, DLT_LOG_DEBUG, DLT_STRING("dlt-system-filetransfer, start."));
+    static pthread_attr_t t_attr;
+    static pthread_t pt;
+    pthread_create(&pt, &t_attr, (void *)filetransfer_thread, conf);
+    threads.threads[threads.count++] = pt;
 }
diff --git a/src/system/dlt-system.c b/src/system/dlt-system.c
index 93e0873..8575d2f 100644
--- a/src/system/dlt-system.c
+++ b/src/system/dlt-system.c
@@ -58,6 +58,7 @@ int main(int argc, char* argv[])
 {
 	DltSystemCliOptions options;
 	DltSystemConfiguration config;
+
 #if defined(DLT_SYSTEMD_WATCHDOG_ENABLE) || defined(DLT_SYSTEMD_ENABLE)
 	int ret;
 #endif
@@ -113,6 +114,7 @@ int main(int argc, char* argv[])
 
 	DLT_LOG(dltsystem, DLT_LOG_DEBUG, DLT_STRING("Launching threads."));
 
+
 	start_threads(&config);
 	join_threads();
 	return 0;
diff --git a/src/system/dlt-system.conf b/src/system/dlt-system.conf
index eee34c3..1675cb6 100644
--- a/src/system/dlt-system.conf
+++ b/src/system/dlt-system.conf
@@ -41,20 +41,12 @@ FiletransferTimeDelay = 10
 # Time in ms seconds to wait between two file transfer logs of a single file to DLT.  (Default: 10)
 FiletransferTimeoutBetweenLogs = 10
 
-# Temporary directory to use.
-# File transfer will move a file to this directory
-# while it is being compressed and sent
-# Make sure that dlt-system has all the necessary rights to 
-# Read, Write, Remove and Move files between all these directories.
-# It is safest and most efficient to have all the directories 
-# under one file system.  
-FiletransferTempDir = /tmp
-
 # You can define multiple file transfer directories
 # Define the directory to watch, whether to compress
 # the file with zlib and the zlib compression level
 # For parsing purposes, FiletransferCompressionLevel
 # must be the last one of three values.
+# For compressing and sending following subdirectories are used: .tocompress and .tosend
 FiletransferDirectory = /var/dlt/ft1
 FiletransferCompression = 1
 FiletransferCompressionLevel = 5