console: Tool to merge multiple DLT files

- Sort in dlt-sortbytimestamp: first sorted by time,
  second sorted by timestamp.
- Support multiple compressed files in dlt-convert.

These changes is based on use case:
User can expects to join several dlt files which are compressed in
tar.gz file (maybe per component). Moreover, currently the existing
sort is only applicaple for single boot cycle. There should be a
support for sorting multiple cyclic boots in one dlt file.

Signed-off-by: Bui Nguyen Quoc Thanh <Thanh.BuiNguyenQuoc@vn.bosch.com>

2 files changed, 279 insertions, 44 deletions

diff --git a/src/console/dlt-convert.c b/src/console/dlt-convert.c
index ce9ce67..2ce5401 100644
--- a/src/console/dlt-convert.c
+++ b/src/console/dlt-convert.c
@@ -73,11 +73,14 @@
  * Initials    Date         Comment
  * aw          13.01.2010   initial
  */
+
+#include <dirent.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
 #include <string.h>
 #include <ctype.h>
+#include <errno.h>
 
 #include <sys/stat.h>
 #include <fcntl.h>
@@ -87,6 +90,10 @@
 #include "dlt_common.h"
 
 #define DLT_CONVERT_TEXTBUFSIZE  10024   /* Size of buffer for text output */
+#define COMMAND_SIZE        1024    /* Size of command */
+#define FILENAME_SIZE       1024    /* Size of filename */
+#define DLT_EXTENSION       "dlt"
+#define DLT_CONVERT_WS      "/tmp/dlt_convert_workspace/"
 
 /**
  * Print usage information of tool.
@@ -117,6 +124,66 @@ void usage()
     printf("  -b number     First messages to be handled\n");
     printf("  -e number     Last message to be handled\n");
     printf("  -w            Follow dlt file while file is increasing\n");
+    printf("  -t            Handling input compressed files (tar.gz)\n");
+}
+
+char *get_filename_ext(const char *filename)
+{
+    if (filename == NULL)
+            fprintf(stderr, "ERROR: %s: invalid arguments\n", __FUNCTION__);
+
+    char *dot = strrchr(filename, '.');
+    if(!dot || dot == filename)
+        return "";
+    return dot + 1;
+}
+
+void empty_dir(const char *dir)
+{
+    struct dirent **files = { 0 };
+    struct stat st;
+    uint32_t n = 0;
+    char tmp_filename[FILENAME_SIZE] = { 0 };
+    uint32_t i;
+
+    if (dir == NULL)
+        fprintf(stderr, "ERROR: %s: invalid arguments\n", __FUNCTION__);
+
+    if (stat(dir, &st) == 0) {
+        if (S_ISDIR(st.st_mode)) {
+            n = scandir(dir, &files, NULL, alphasort);
+
+            /* Do not include /. and /.. */
+            if (n < 2)
+                fprintf(stderr, "ERROR: Failed to scan %s with error %s\n",
+                        dir, strerror(errno));
+            else if (n == 2)
+                printf("%s is already empty\n", dir);
+            else {
+                for (i = 2; i < n; i++) {
+                    memset(tmp_filename, 0, FILENAME_SIZE);
+                    snprintf(tmp_filename, FILENAME_SIZE, "%s%s", dir, files[i]->d_name);
+
+                    if (remove(tmp_filename) != 0)
+                        fprintf(stderr, "ERROR: Failed to delete %s with error %s\n",
+                                tmp_filename, strerror(errno));
+                }
+                if (files) {
+                    for (i = 0; i < n ; i++)
+                        if (files[i]) {
+                            free(files[i]);
+                            files[i] = NULL;
+                        }
+                    free(files);
+                    files = NULL;
+                }
+            }
+        }
+        else
+            fprintf(stderr, "ERROR: %s is not a directory\n", dir);
+    }
+    else
+        fprintf(stderr, "ERROR: Failed to stat %s with error %s\n", dir, strerror(errno));
 }
 
 /**
@@ -131,6 +198,7 @@ int main(int argc, char *argv[])
     int xflag = 0;
     int mflag = 0;
     int wflag = 0;
+    int tflag = 0;
     char *fvalue = 0;
     char *bvalue = 0;
     char *evalue = 0;
@@ -146,15 +214,25 @@ int main(int argc, char *argv[])
 
     int num, begin, end;
 
-    char text[DLT_CONVERT_TEXTBUFSIZE];
+    char text[DLT_CONVERT_TEXTBUFSIZE] = { 0 };
+
+    /* For handling compressed files */
+    char command[COMMAND_SIZE] = { 0 };
+    char tmp_filename[FILENAME_SIZE] = { 0 };
+    struct stat st = { 0 };
+    struct dirent **files = { 0 };
+    int n = 0;
+    int i = 0;
 
     struct iovec iov[2];
-    int bytes_written;
+    int bytes_written = 0;
+    int syserr = 0;
 
     opterr = 0;
 
-    while ((c = getopt (argc, argv, "vcashxmwf:b:e:o:")) != -1)
-        switch (c) {
+    while ((c = getopt (argc, argv, "vcashxmwtf:b:e:o:")) != -1) {
+        switch (c)
+        {
         case 'v':
         {
             vflag = 1;
@@ -190,6 +268,11 @@ int main(int argc, char *argv[])
             wflag = 1;
             break;
         }
+        case 't':
+        {
+            tflag = 1;
+            break;
+        }
         case 'h':
         {
             usage();
@@ -230,12 +313,12 @@ int main(int argc, char *argv[])
         }
         default:
         {
-            abort();
             return -1;    /*for parasoft */
         }
         }
+    }
 
-    /* initialise structure to use DLT file */
+    /* Initialize structure to use DLT file */
     dlt_file_init(&file, vflag);
 
     /* first parse filter file if filter parameter is used */
@@ -258,10 +341,69 @@ int main(int argc, char *argv[])
         }
     }
 
+    if (tflag) {
+        /* Prepare the temp dir to untar compressed files */
+        if (stat(DLT_CONVERT_WS, &st) == -1) {
+            if (mkdir(DLT_CONVERT_WS, 0700) != 0) {
+                fprintf(stderr,"ERROR: Cannot create temp dir %s!\n", DLT_CONVERT_WS);
+                if (ovalue)
+                    close(ohandle);
+
+                return -1;
+            }
+        }
+        else {
+            if (S_ISDIR(st.st_mode))
+                empty_dir(DLT_CONVERT_WS);
+            else
+                fprintf(stderr, "ERROR: %s is not a directory", DLT_CONVERT_WS);
+        }
+
+        for (index = optind; index < argc; index++) {
+            memset(command, 0, COMMAND_SIZE);
+
+            /* Check extension of input file
+             * If it is a compressed file, uncompress it
+             */
+            if (strcmp(get_filename_ext(argv[index]), DLT_EXTENSION) != 0)
+                snprintf(command, COMMAND_SIZE, "tar xf %s -C %s",
+                        argv[index], DLT_CONVERT_WS);
+            else
+                snprintf(command, COMMAND_SIZE, "cp %s %s",
+                        argv[index], DLT_CONVERT_WS);
+
+            syserr = system(command);
+            if (syserr != 0)
+                fprintf(stderr, "ERROR: Failed to execute command [%s] with error [%d]\n",
+                        command, syserr);
+        }
+
+        n = scandir(DLT_CONVERT_WS, &files, NULL, alphasort);
+        if (n == -1) {
+            fprintf(stderr,"ERROR: Cannot scan temp dir %s!\n", DLT_CONVERT_WS);
+            if (ovalue)
+                close(ohandle);
+
+            return -1;
+        }
+
+        /* do not include ./ and ../ in the files */
+        argc = optind + (n - 2);
+    }
+
     for (index = optind; index < argc; index++) {
-        /* load, analyse data file and create index list */
+        if (tflag) {
+            memset(tmp_filename, 0, FILENAME_SIZE);
+            snprintf(tmp_filename, FILENAME_SIZE, "%s%s",
+                    DLT_CONVERT_WS, files[index - optind + 2]->d_name);
+
+            argv[index] = tmp_filename;
+        }
+
+        /* load, analyze data file and create index list */
         if (dlt_file_open(&file, argv[index], vflag) >= DLT_RETURN_OK) {
-            while (dlt_file_read(&file, vflag) >= DLT_RETURN_OK) {}
+            while (dlt_file_read(&file, vflag) >= DLT_RETURN_OK) {
+            }
         }
 
         if (aflag || sflag || xflag || mflag || ovalue) {
@@ -277,11 +419,17 @@ int main(int argc, char *argv[])
 
             if ((begin < 0) || (begin >= file.counter)) {
                 fprintf(stderr, "ERROR: Selected first message %d is out of range!\n", begin);
+                if (ovalue)
+                    close(ohandle);
+
                 return -1;
             }
 
             if ((end < 0) || (end >= file.counter) || (end < begin)) {
                 fprintf(stderr, "ERROR: Selected end message %d is out of range!\n", end);
+                if (ovalue)
+                    close(ohandle);
+
                 return -1;
             }
 
@@ -292,8 +440,7 @@ int main(int argc, char *argv[])
                     printf("%d ", num);
                     dlt_message_print_hex(&(file.msg), text, DLT_CONVERT_TEXTBUFSIZE, vflag);
                 }
-                else if (aflag)
-                {
+                else if (aflag) {
                     printf("%d ", num);
 
                     dlt_message_header(&(file.msg), text, DLT_CONVERT_TEXTBUFSIZE, vflag);
@@ -304,13 +451,11 @@ int main(int argc, char *argv[])
 
                     printf("[%s]\n", text);
                 }
-                else if (mflag)
-                {
+                else if (mflag) {
                     printf("%d ", num);
                     dlt_message_print_mixed_plain(&(file.msg), text, DLT_CONVERT_TEXTBUFSIZE, vflag);
                 }
-                else if (sflag)
-                {
+                else if (sflag) {
                     printf("%d ", num);
 
                     dlt_message_header(&(file.msg), text, DLT_CONVERT_TEXTBUFSIZE, vflag);
@@ -329,6 +474,10 @@ int main(int argc, char *argv[])
 
                     if (0 > bytes_written) {
                         printf("in main: writev(ohandle, iov, 2); returned an error!");
+
+                        if (ovalue)
+                            close(ohandle);
+
                         dlt_file_free(&file, vflag);
                         return -1;
                     }
@@ -337,7 +486,8 @@ int main(int argc, char *argv[])
                 /* check for new messages if follow flag set */
                 if (wflag && (num == end)) {
                     while (1) {
-                        while (dlt_file_read(&file, 0) >= 0) {}
+                        while (dlt_file_read(&file, 0) >= 0){
+                        }
 
                         if (end == (file.counter - 1)) {
                             /* Sleep if no new message was received */
@@ -367,6 +517,17 @@ int main(int argc, char *argv[])
     if (ovalue)
         close(ohandle);
 
+    if (tflag) {
+        empty_dir(DLT_CONVERT_WS);
+        if (files) {
+            for (i = 0; i < n ; i++)
+                if (files[i])
+                    free(files[i]);
+
+            free(files);
+        }
+        rmdir(DLT_CONVERT_WS);
+    }
     if (index == optind) {
         /* no file selected, show usage and terminate */
         fprintf(stderr, "ERROR: No file selected\n");
diff --git a/src/console/dlt-sortbytimestamp.c b/src/console/dlt-sortbytimestamp.c
index c65c692..6f8498a 100644
--- a/src/console/dlt-sortbytimestamp.c
+++ b/src/console/dlt-sortbytimestamp.c
@@ -73,10 +73,13 @@
 #include "dlt_common.h"
 
 #define DLT_VERBUFSIZE  255
+#define FIFTY_SEC_IN_MSEC 500000
+#define THREE_MIN_IN_SEC  180
 
 typedef struct sTimestampIndex {
     int num;
     uint32_t tmsp;
+    uint32_t systmsp;
 } TimestampIndex;
 
 int verbosity = 0;
@@ -84,8 +87,7 @@ int verbosity = 0;
 /**
  * Print information, conditional upon requested verbosity level
  */
-void verbose(int level, char *msg, ...)
-{
+void verbose(int level, char *msg, ...) {
     if (level <= verbosity) {
         if (verbosity > 1) { /* timestamp */
             time_t tnow = time((time_t *)0);
@@ -103,6 +105,7 @@ void verbose(int level, char *msg, ...)
         va_list args;
         va_start (args, msg);
         vprintf(msg, args);
+        va_end(args);
 
         /* lines without a terminal newline aren't guaranteed to be displayed */
         if (msg[len - 1] != '\n')
@@ -112,22 +115,36 @@ void verbose(int level, char *msg, ...)
 
 /**
  * Comparison function for use with qsort
+ * Used for time stamp
  */
-int compare_index_timestamps(const void *a, const void *b)
-{
+int compare_index_timestamps(const void *a, const void *b) {
+    int ret = -1;
     if (((TimestampIndex *)a)->tmsp > ((TimestampIndex *)b)->tmsp)
-        return 1;
+        ret = 1;
     else if (((TimestampIndex *)a)->tmsp == ((TimestampIndex *)b)->tmsp)
-        return 0;
+        ret = 0;
 
-    return -1;
+    return ret;
+}
+
+/**
+ * Comparison function for use with qsort
+ * Used for system time
+ */
+int compare_index_systime(const void *a, const void *b) {
+    int ret = -1;
+    if(((TimestampIndex *) a)->systmsp > ((TimestampIndex *) b)->systmsp)
+        ret = 1;
+    else if(((TimestampIndex *) a)->systmsp == ((TimestampIndex *) b)->systmsp)
+        ret = 0;
+
+    return ret;
 }
 
 /**
  * Write the messages in the order specified by the given index
  */
-void write_messages(int ohandle, DltFile *file, TimestampIndex *timestamps, int message_count)
-{
+void write_messages(int ohandle, DltFile *file, TimestampIndex *timestamps, int message_count) {
     struct iovec iov[2];
     int bytes_written;
 
@@ -158,8 +175,7 @@ void write_messages(int ohandle, DltFile *file, TimestampIndex *timestamps, int
 /**
  * Print usage information of tool.
  */
-void usage()
-{
+void usage() {
     char version[DLT_VERBUFSIZE];
 
     dlt_get_version(version, DLT_VERBUFSIZE);
@@ -182,8 +198,7 @@ void usage()
 /**
  * Main function of tool.
  */
-int main(int argc, char *argv[])
-{
+int main(int argc, char *argv[]) {
     int vflag = 0;
     int cflag = 0;
     char *fvalue = 0;
@@ -193,8 +208,15 @@ int main(int argc, char *argv[])
     char *ovalue = 0;
 
     TimestampIndex *timestamp_index = 0;
+    TimestampIndex *temp_timestamp_index = 0;
     int32_t message_count = 0;
 
+    uint32_t count = 0;
+    int start = 0;
+    uint32_t delta_tmsp = 0;
+    uint32_t delta_systime = 0;
+    size_t i;
+
     int c;
 
     DltFile file;
@@ -208,7 +230,7 @@ int main(int argc, char *argv[])
 
     verbose(1, "Configuring\n");
 
-    while ((c = getopt (argc, argv, "vchf:b:e:")) != -1)
+    while ((c = getopt (argc, argv, "vchf:b:e:")) != -1) {
         switch (c) {
         case 'v':
         {
@@ -255,10 +277,11 @@ int main(int argc, char *argv[])
         }
         default:
         {
-            abort();
+            usage();
             return -1;    /*for parasoft */
         }
         }
+    }
 
     /* Don't use vflag on quietest levels */
     if (verbosity > 2)
@@ -266,7 +289,7 @@ int main(int argc, char *argv[])
 
     verbose (1, "Initializing\n");
 
-    /* initialise structure to use DLT file */
+    /* Initialize structure to use DLT file */
     dlt_file_init(&file, vflag);
 
     /* first parse filter file if filter parameter is used */
@@ -312,9 +335,10 @@ int main(int argc, char *argv[])
 
     verbose(1, "Loading\n");
 
-    /* load, analyse data file and create index list */
+    /* load, analyze data file and create index list */
     if (dlt_file_open(&file, ivalue, vflag) >= DLT_RETURN_OK) {
-        while (dlt_file_read(&file, vflag) >= DLT_RETURN_OK) {}
+        while (dlt_file_read(&file, vflag) >= DLT_RETURN_OK) {
+        }
     }
 
     if (cflag) {
@@ -334,13 +358,14 @@ int main(int argc, char *argv[])
     else
         end = file.counter - 1;
 
-    if ((begin < 0) || (begin >= file.counter) || (begin > end)) {
-        fprintf(stderr, "ERROR: Selected first message %d is out of range!\n", begin);
-        return -1;
-    }
+    if ((begin < 0) || (end < 0) || (begin > end) ||
+        (begin >= file.counter) || (end >= file.counter)) {
+        fprintf(stderr, "ERROR: Selected message [begin-end]-[%d-%d] is out of range!\n", begin, end);
+
+        dlt_file_free(&file, vflag);
+        if (ovalue)
+            close(ohandle);
 
-    if (end >= file.counter) {
-        fprintf(stderr, "ERROR: Selected end message %d is out of range!\n", end);
         return -1;
     }
 
@@ -350,11 +375,15 @@ int main(int argc, char *argv[])
 
     message_count = 1 + end - begin;
 
-    timestamp_index = (TimestampIndex *)malloc(sizeof(TimestampIndex) * message_count);
+    timestamp_index = (TimestampIndex *) malloc(sizeof(TimestampIndex) * (message_count + 1));
 
-    if (timestamp_index == 0) {
+    if (timestamp_index == NULL) {
         fprintf(stderr, "ERROR: Failed to allocate memory for message index!\n");
+
         dlt_file_free(&file, vflag);
+        if (ovalue)
+            close(ohandle);
+
         return -1;
     }
 
@@ -363,17 +392,62 @@ int main(int argc, char *argv[])
     for (num = begin; num <= end; num++) {
         dlt_file_message(&file, num, vflag);
         timestamp_index[num - begin].num = num;
+        timestamp_index[num - begin].systmsp = file.msg.storageheader->seconds;
         timestamp_index[num - begin].tmsp = file.msg.headerextra.tmsp;
     }
 
+    /* This step is extending the array one more element by copying the first element */
+    timestamp_index[num].num = timestamp_index[0].num;
+    timestamp_index[num].systmsp = timestamp_index[0].systmsp;
+    timestamp_index[num].tmsp = timestamp_index[0].tmsp;
+
     verbose(1, "Sorting\n");
-    qsort((void *)timestamp_index, message_count, sizeof(TimestampIndex), compare_index_timestamps);
+    qsort((void *) timestamp_index, message_count, sizeof(TimestampIndex), compare_index_systime);
 
-    write_messages(ohandle, &file, timestamp_index, message_count);
-    close(ohandle);
+    for (num = begin; num <= end; num++) {
+        delta_tmsp = abs(timestamp_index[num + 1].tmsp - timestamp_index[num].tmsp);
+        delta_systime = abs(timestamp_index[num + 1].systmsp - timestamp_index[num].systmsp);
+
+        /*
+         * Here is a try to detect a new cycle of boot in system.
+         * Relatively, if there are gaps whose systime is larger than 3 mins and
+         * timestamp is larger than 15 secs should be identified as a new boot cycle.
+         */
+        count++;
+        if(delta_tmsp > FIFTY_SEC_IN_MSEC || delta_systime >= THREE_MIN_IN_SEC) {
+            temp_timestamp_index = (TimestampIndex *) malloc(sizeof(TimestampIndex) * count);
+
+            if (temp_timestamp_index == NULL) {
+                fprintf(stderr, "ERROR: Failed to allocate memory for array\n");
+
+                dlt_file_free(&file, vflag);
+                if (ovalue)
+                    close(ohandle);
+
+                return -1;
+            }
+
+            for (i = 0; i < count; i++) {
+                memcpy((void*) &temp_timestamp_index[i],
+                        (void*) &timestamp_index[start + i],
+                        sizeof(TimestampIndex));
+            }
+            qsort((void *) temp_timestamp_index, count, sizeof(TimestampIndex),
+                    compare_index_timestamps);
+
+            write_messages(ohandle, &file, temp_timestamp_index, count);
+            free(temp_timestamp_index);
+            temp_timestamp_index = NULL;
+            start = start + count;
+            count = 0;
+        }
+    }
+    if (ovalue)
+        close(ohandle);
 
     verbose(1, "Tidying up.\n");
     free(timestamp_index);
+    timestamp_index = NULL;
     dlt_file_free(&file, vflag);
     return 0;
 }