console: Tool to merge multiple DLT files

- Sort in dlt-sortbytimestamp: first sorted by time,
  second sorted by timestamp.
- Support multiple compressed files in dlt-convert.

These changes is based on use case:
User can expects to join several dlt files which are compressed in
tar.gz file (maybe per component). Moreover, currently the existing
sort is only applicaple for single boot cycle. There should be a
support for sorting multiple cyclic boots in one dlt file.

Signed-off-by: Bui Nguyen Quoc Thanh <Thanh.BuiNguyenQuoc@vn.bosch.com>

4 files changed, 292 insertions, 52 deletions

diff --git a/doc/dlt-convert.1.md b/doc/dlt-convert.1.md
index c0df353..19f855a 100644
--- a/doc/dlt-convert.1.md
+++ b/doc/dlt-convert.1.md
@@ -6,7 +6,7 @@
 
 # SYNOPSIS
 
-**dlt-convert** \[**-h**\] \[**-a**\] \[**-x**\] \[**-m**\] \[**-s**\] \[**-o** filename\] \[**-v**\] \[**-c**\] \[**-f** filterfile\] \[**-b** number\] \[**-e** number\] \[**-w**\] file1 \[file2\] \[file3\]
+**dlt-convert** \[**-h**\] \[**-a**\] \[**-x**\] \[**-m**\] \[**-s**\] \[**-t**\] \[**-o** filename\] \[**-v**\] \[**-c**\] \[**-f** filterfile\] \[**-b** number\] \[**-e** number\] \[**-w**\] file1 \[file2\] \[file3\]
 
 # DESCRIPTION
 
@@ -63,6 +63,10 @@ Use two files and Output file to join DLT files.
 
 :   Follow dlt file while file is increasing.
 
+-t
+
+:   Handling the compressed input files (tar.gz).
+
 # EXAMPLES
 
 Convert DLT file into ASCII:
@@ -71,9 +75,12 @@ Convert DLT file into ASCII:
 Cut a specific range, e.g. from message 1 to message 3 from a file called log.dlt and store the result to a file called newlog.dlt:
     **dlt-convert -b 1 -e 3 -o newlog.dlt log.dlt**
 
-Paste two dlt files log1.dlt and log2.dlt to a new file called newlog.dlt::
+Paste two dlt files log1.dlt and log2.dlt to a new file called newlog.dlt:
     **dlt-convert -o newlog.dlt log1.dlt log2.dlt**
 
+Handle the compressed input files and join inputs into a new file called newlog.dlt:
+    **dlt-convert -t -o newlog.dlt log1.dlt compressed_log2.tar.gz**
+
 # EXIT STATUS
 
 Non zero is returned in case of failure.
diff --git a/doc/dlt-sortbytimestamp.1.md b/doc/dlt-sortbytimestamp.1.md
index 9f66320..caf0256 100644
--- a/doc/dlt-sortbytimestamp.1.md
+++ b/doc/dlt-sortbytimestamp.1.md
@@ -2,7 +2,7 @@
 
 # NAME
 
-**dlt-sortbytimestamp** - Re-order DLT Logging files according to message creation time
+**dlt-sortbytimestamp** - Re-order DLT Logging files according to message creation time and timestamp.
 
 # SYNOPSIS
 
@@ -12,13 +12,11 @@
 
 By default messages in DLT files are ordered according to the time the logger received them. This can unhelpful when tracing a sequence of events on a busy multi-threaded/multi-core system, because thread pre-emption combined with multiple processes attempting to log messages simultaneously means that the order in which the messages are received may vary significantly from the order in which they were created.
 
-*dlt-sortbytimestamp* re-orders a DLT input file's messages according their creation timestamp, and writes them to an output DLT file.
+*dlt-sortbytimestamp* is able to re-order a DLT input file's messages according both their creation time and timestamp, and writes them to an output DLT file.
 
-# IMPORTANT NOTE
+# NOTE
 
-Message timestamps are recorded relative to boot time. DLT files can contain messages from more than one reboot cycle. Because timestamping is reset to zero at each boot, simply running *dlt-sortbytimestamp* against a multi-boot-cycle DLT input file will produce a tangled mess.
-
-Use the *-b* and/or *-e* options to specify a range of messages within a single reboot cycle and all will be well.
+Use the \*-b\* and/or \*-e\* options to specify a range of messages within a single reboot cycle and all will be well.
 
 Hint: use *dlt-viewer* to ascertain the endpoints of the range in question.
 
diff --git a/src/console/dlt-convert.c b/src/console/dlt-convert.c
index ce9ce67..2ce5401 100644
--- a/src/console/dlt-convert.c
+++ b/src/console/dlt-convert.c
@@ -73,11 +73,14 @@
  * Initials    Date         Comment
  * aw          13.01.2010   initial
  */
+
+#include <dirent.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
 #include <string.h>
 #include <ctype.h>
+#include <errno.h>
 
 #include <sys/stat.h>
 #include <fcntl.h>
@@ -87,6 +90,10 @@
 #include "dlt_common.h"
 
 #define DLT_CONVERT_TEXTBUFSIZE  10024   /* Size of buffer for text output */
+#define COMMAND_SIZE        1024    /* Size of command */
+#define FILENAME_SIZE       1024    /* Size of filename */
+#define DLT_EXTENSION       "dlt"
+#define DLT_CONVERT_WS      "/tmp/dlt_convert_workspace/"
 
 /**
  * Print usage information of tool.
@@ -117,6 +124,66 @@ void usage()
     printf("  -b number     First messages to be handled\n");
     printf("  -e number     Last message to be handled\n");
     printf("  -w            Follow dlt file while file is increasing\n");
+    printf("  -t            Handling input compressed files (tar.gz)\n");
+}
+
+char *get_filename_ext(const char *filename)
+{
+    if (filename == NULL)
+            fprintf(stderr, "ERROR: %s: invalid arguments\n", __FUNCTION__);
+
+    char *dot = strrchr(filename, '.');
+    if(!dot || dot == filename)
+        return "";
+    return dot + 1;
+}
+
+void empty_dir(const char *dir)
+{
+    struct dirent **files = { 0 };
+    struct stat st;
+    uint32_t n = 0;
+    char tmp_filename[FILENAME_SIZE] = { 0 };
+    uint32_t i;
+
+    if (dir == NULL)
+        fprintf(stderr, "ERROR: %s: invalid arguments\n", __FUNCTION__);
+
+    if (stat(dir, &st) == 0) {
+        if (S_ISDIR(st.st_mode)) {
+            n = scandir(dir, &files, NULL, alphasort);
+
+            /* Do not include /. and /.. */
+            if (n < 2)
+                fprintf(stderr, "ERROR: Failed to scan %s with error %s\n",
+                        dir, strerror(errno));
+            else if (n == 2)
+                printf("%s is already empty\n", dir);
+            else {
+                for (i = 2; i < n; i++) {
+                    memset(tmp_filename, 0, FILENAME_SIZE);
+                    snprintf(tmp_filename, FILENAME_SIZE, "%s%s", dir, files[i]->d_name);
+
+                    if (remove(tmp_filename) != 0)
+                        fprintf(stderr, "ERROR: Failed to delete %s with error %s\n",
+                                tmp_filename, strerror(errno));
+                }
+                if (files) {
+                    for (i = 0; i < n ; i++)
+                        if (files[i]) {
+                            free(files[i]);
+                            files[i] = NULL;
+                        }
+                    free(files);
+                    files = NULL;
+                }
+            }
+        }
+        else
+            fprintf(stderr, "ERROR: %s is not a directory\n", dir);
+    }
+    else
+        fprintf(stderr, "ERROR: Failed to stat %s with error %s\n", dir, strerror(errno));
 }
 
 /**
@@ -131,6 +198,7 @@ int main(int argc, char *argv[])
     int xflag = 0;
     int mflag = 0;
     int wflag = 0;
+    int tflag = 0;
     char *fvalue = 0;
     char *bvalue = 0;
     char *evalue = 0;
@@ -146,15 +214,25 @@ int main(int argc, char *argv[])
 
     int num, begin, end;
 
-    char text[DLT_CONVERT_TEXTBUFSIZE];
+    char text[DLT_CONVERT_TEXTBUFSIZE] = { 0 };
+
+    /* For handling compressed files */
+    char command[COMMAND_SIZE] = { 0 };
+    char tmp_filename[FILENAME_SIZE] = { 0 };
+    struct stat st = { 0 };
+    struct dirent **files = { 0 };
+    int n = 0;
+    int i = 0;
 
     struct iovec iov[2];
-    int bytes_written;
+    int bytes_written = 0;
+    int syserr = 0;
 
     opterr = 0;
 
-    while ((c = getopt (argc, argv, "vcashxmwf:b:e:o:")) != -1)
-        switch (c) {
+    while ((c = getopt (argc, argv, "vcashxmwtf:b:e:o:")) != -1) {
+        switch (c)
+        {
         case 'v':
         {
             vflag = 1;
@@ -190,6 +268,11 @@ int main(int argc, char *argv[])
             wflag = 1;
             break;
         }
+        case 't':
+        {
+            tflag = 1;
+            break;
+        }
         case 'h':
         {
             usage();
@@ -230,12 +313,12 @@ int main(int argc, char *argv[])
         }
         default:
         {
-            abort();
             return -1;    /*for parasoft */
         }
         }
+    }
 
-    /* initialise structure to use DLT file */
+    /* Initialize structure to use DLT file */
     dlt_file_init(&file, vflag);
 
     /* first parse filter file if filter parameter is used */
@@ -258,10 +341,69 @@ int main(int argc, char *argv[])
         }
     }
 
+    if (tflag) {
+        /* Prepare the temp dir to untar compressed files */
+        if (stat(DLT_CONVERT_WS, &st) == -1) {
+            if (mkdir(DLT_CONVERT_WS, 0700) != 0) {
+                fprintf(stderr,"ERROR: Cannot create temp dir %s!\n", DLT_CONVERT_WS);
+                if (ovalue)
+                    close(ohandle);
+
+                return -1;
+            }
+        }
+        else {
+            if (S_ISDIR(st.st_mode))
+                empty_dir(DLT_CONVERT_WS);
+            else
+                fprintf(stderr, "ERROR: %s is not a directory", DLT_CONVERT_WS);
+        }
+
+        for (index = optind; index < argc; index++) {
+            memset(command, 0, COMMAND_SIZE);
+
+            /* Check extension of input file
+             * If it is a compressed file, uncompress it
+             */
+            if (strcmp(get_filename_ext(argv[index]), DLT_EXTENSION) != 0)
+                snprintf(command, COMMAND_SIZE, "tar xf %s -C %s",
+                        argv[index], DLT_CONVERT_WS);
+            else
+                snprintf(command, COMMAND_SIZE, "cp %s %s",
+                        argv[index], DLT_CONVERT_WS);
+
+            syserr = system(command);
+            if (syserr != 0)
+                fprintf(stderr, "ERROR: Failed to execute command [%s] with error [%d]\n",
+                        command, syserr);
+        }
+
+        n = scandir(DLT_CONVERT_WS, &files, NULL, alphasort);
+        if (n == -1) {
+            fprintf(stderr,"ERROR: Cannot scan temp dir %s!\n", DLT_CONVERT_WS);
+            if (ovalue)
+                close(ohandle);
+
+            return -1;
+        }
+
+        /* do not include ./ and ../ in the files */
+        argc = optind + (n - 2);
+    }
+
     for (index = optind; index < argc; index++) {
-        /* load, analyse data file and create index list */
+        if (tflag) {
+            memset(tmp_filename, 0, FILENAME_SIZE);
+            snprintf(tmp_filename, FILENAME_SIZE, "%s%s",
+                    DLT_CONVERT_WS, files[index - optind + 2]->d_name);
+
+            argv[index] = tmp_filename;
+        }
+
+        /* load, analyze data file and create index list */
         if (dlt_file_open(&file, argv[index], vflag) >= DLT_RETURN_OK) {
-            while (dlt_file_read(&file, vflag) >= DLT_RETURN_OK) {}
+            while (dlt_file_read(&file, vflag) >= DLT_RETURN_OK) {
+            }
         }
 
         if (aflag || sflag || xflag || mflag || ovalue) {
@@ -277,11 +419,17 @@ int main(int argc, char *argv[])
 
             if ((begin < 0) || (begin >= file.counter)) {
                 fprintf(stderr, "ERROR: Selected first message %d is out of range!\n", begin);
+                if (ovalue)
+                    close(ohandle);
+
                 return -1;
             }
 
             if ((end < 0) || (end >= file.counter) || (end < begin)) {
                 fprintf(stderr, "ERROR: Selected end message %d is out of range!\n", end);
+                if (ovalue)
+                    close(ohandle);
+
                 return -1;
             }
 
@@ -292,8 +440,7 @@ int main(int argc, char *argv[])
                     printf("%d ", num);
                     dlt_message_print_hex(&(file.msg), text, DLT_CONVERT_TEXTBUFSIZE, vflag);
                 }
-                else if (aflag)
-                {
+                else if (aflag) {
                     printf("%d ", num);
 
                     dlt_message_header(&(file.msg), text, DLT_CONVERT_TEXTBUFSIZE, vflag);
@@ -304,13 +451,11 @@ int main(int argc, char *argv[])
 
                     printf("[%s]\n", text);
                 }
-                else if (mflag)
-                {
+                else if (mflag) {
                     printf("%d ", num);
                     dlt_message_print_mixed_plain(&(file.msg), text, DLT_CONVERT_TEXTBUFSIZE, vflag);
                 }
-                else if (sflag)
-                {
+                else if (sflag) {
                     printf("%d ", num);
 
                     dlt_message_header(&(file.msg), text, DLT_CONVERT_TEXTBUFSIZE, vflag);
@@ -329,6 +474,10 @@ int main(int argc, char *argv[])
 
                     if (0 > bytes_written) {
                         printf("in main: writev(ohandle, iov, 2); returned an error!");
+
+                        if (ovalue)
+                            close(ohandle);
+
                         dlt_file_free(&file, vflag);
                         return -1;
                     }
@@ -337,7 +486,8 @@ int main(int argc, char *argv[])
                 /* check for new messages if follow flag set */
                 if (wflag && (num == end)) {
                     while (1) {
-                        while (dlt_file_read(&file, 0) >= 0) {}
+                        while (dlt_file_read(&file, 0) >= 0){
+                        }
 
                         if (end == (file.counter - 1)) {
                             /* Sleep if no new message was received */
@@ -367,6 +517,17 @@ int main(int argc, char *argv[])
     if (ovalue)
         close(ohandle);
 
+    if (tflag) {
+        empty_dir(DLT_CONVERT_WS);
+        if (files) {
+            for (i = 0; i < n ; i++)
+                if (files[i])
+                    free(files[i]);
+
+            free(files);
+        }
+        rmdir(DLT_CONVERT_WS);
+    }
     if (index == optind) {
         /* no file selected, show usage and terminate */
         fprintf(stderr, "ERROR: No file selected\n");
diff --git a/src/console/dlt-sortbytimestamp.c b/src/console/dlt-sortbytimestamp.c
index c65c692..6f8498a 100644
--- a/src/console/dlt-sortbytimestamp.c
+++ b/src/console/dlt-sortbytimestamp.c
@@ -73,10 +73,13 @@
 #include "dlt_common.h"
 
 #define DLT_VERBUFSIZE  255
+#define FIFTY_SEC_IN_MSEC 500000
+#define THREE_MIN_IN_SEC  180
 
 typedef struct sTimestampIndex {
     int num;
     uint32_t tmsp;
+    uint32_t systmsp;
 } TimestampIndex;
 
 int verbosity = 0;
@@ -84,8 +87,7 @@ int verbosity = 0;
 /**
  * Print information, conditional upon requested verbosity level
  */
-void verbose(int level, char *msg, ...)
-{
+void verbose(int level, char *msg, ...) {
     if (level <= verbosity) {
         if (verbosity > 1) { /* timestamp */
             time_t tnow = time((time_t *)0);
@@ -103,6 +105,7 @@ void verbose(int level, char *msg, ...)
         va_list args;
         va_start (args, msg);
         vprintf(msg, args);
+        va_end(args);
 
         /* lines without a terminal newline aren't guaranteed to be displayed */
         if (msg[len - 1] != '\n')
@@ -112,22 +115,36 @@ void verbose(int level, char *msg, ...)
 
 /**
  * Comparison function for use with qsort
+ * Used for time stamp
  */
-int compare_index_timestamps(const void *a, const void *b)
-{
+int compare_index_timestamps(const void *a, const void *b) {
+    int ret = -1;
     if (((TimestampIndex *)a)->tmsp > ((TimestampIndex *)b)->tmsp)
-        return 1;
+        ret = 1;
     else if (((TimestampIndex *)a)->tmsp == ((TimestampIndex *)b)->tmsp)
-        return 0;
+        ret = 0;
 
-    return -1;
+    return ret;
+}
+
+/**
+ * Comparison function for use with qsort
+ * Used for system time
+ */
+int compare_index_systime(const void *a, const void *b) {
+    int ret = -1;
+    if(((TimestampIndex *) a)->systmsp > ((TimestampIndex *) b)->systmsp)
+        ret = 1;
+    else if(((TimestampIndex *) a)->systmsp == ((TimestampIndex *) b)->systmsp)
+        ret = 0;
+
+    return ret;
 }
 
 /**
  * Write the messages in the order specified by the given index
  */
-void write_messages(int ohandle, DltFile *file, TimestampIndex *timestamps, int message_count)
-{
+void write_messages(int ohandle, DltFile *file, TimestampIndex *timestamps, int message_count) {
     struct iovec iov[2];
     int bytes_written;
 
@@ -158,8 +175,7 @@ void write_messages(int ohandle, DltFile *file, TimestampIndex *timestamps, int
 /**
  * Print usage information of tool.
  */
-void usage()
-{
+void usage() {
     char version[DLT_VERBUFSIZE];
 
     dlt_get_version(version, DLT_VERBUFSIZE);
@@ -182,8 +198,7 @@ void usage()
 /**
  * Main function of tool.
  */
-int main(int argc, char *argv[])
-{
+int main(int argc, char *argv[]) {
     int vflag = 0;
     int cflag = 0;
     char *fvalue = 0;
@@ -193,8 +208,15 @@ int main(int argc, char *argv[])
     char *ovalue = 0;
 
     TimestampIndex *timestamp_index = 0;
+    TimestampIndex *temp_timestamp_index = 0;
     int32_t message_count = 0;
 
+    uint32_t count = 0;
+    int start = 0;
+    uint32_t delta_tmsp = 0;
+    uint32_t delta_systime = 0;
+    size_t i;
+
     int c;
 
     DltFile file;
@@ -208,7 +230,7 @@ int main(int argc, char *argv[])
 
     verbose(1, "Configuring\n");
 
-    while ((c = getopt (argc, argv, "vchf:b:e:")) != -1)
+    while ((c = getopt (argc, argv, "vchf:b:e:")) != -1) {
         switch (c) {
         case 'v':
         {
@@ -255,10 +277,11 @@ int main(int argc, char *argv[])
         }
         default:
         {
-            abort();
+            usage();
             return -1;    /*for parasoft */
         }
         }
+    }
 
     /* Don't use vflag on quietest levels */
     if (verbosity > 2)
@@ -266,7 +289,7 @@ int main(int argc, char *argv[])
 
     verbose (1, "Initializing\n");
 
-    /* initialise structure to use DLT file */
+    /* Initialize structure to use DLT file */
     dlt_file_init(&file, vflag);
 
     /* first parse filter file if filter parameter is used */
@@ -312,9 +335,10 @@ int main(int argc, char *argv[])
 
     verbose(1, "Loading\n");
 
-    /* load, analyse data file and create index list */
+    /* load, analyze data file and create index list */
     if (dlt_file_open(&file, ivalue, vflag) >= DLT_RETURN_OK) {
-        while (dlt_file_read(&file, vflag) >= DLT_RETURN_OK) {}
+        while (dlt_file_read(&file, vflag) >= DLT_RETURN_OK) {
+        }
     }
 
     if (cflag) {
@@ -334,13 +358,14 @@ int main(int argc, char *argv[])
     else
         end = file.counter - 1;
 
-    if ((begin < 0) || (begin >= file.counter) || (begin > end)) {
-        fprintf(stderr, "ERROR: Selected first message %d is out of range!\n", begin);
-        return -1;
-    }
+    if ((begin < 0) || (end < 0) || (begin > end) ||
+        (begin >= file.counter) || (end >= file.counter)) {
+        fprintf(stderr, "ERROR: Selected message [begin-end]-[%d-%d] is out of range!\n", begin, end);
+
+        dlt_file_free(&file, vflag);
+        if (ovalue)
+            close(ohandle);
 
-    if (end >= file.counter) {
-        fprintf(stderr, "ERROR: Selected end message %d is out of range!\n", end);
         return -1;
     }
 
@@ -350,11 +375,15 @@ int main(int argc, char *argv[])
 
     message_count = 1 + end - begin;
 
-    timestamp_index = (TimestampIndex *)malloc(sizeof(TimestampIndex) * message_count);
+    timestamp_index = (TimestampIndex *) malloc(sizeof(TimestampIndex) * (message_count + 1));
 
-    if (timestamp_index == 0) {
+    if (timestamp_index == NULL) {
         fprintf(stderr, "ERROR: Failed to allocate memory for message index!\n");
+
         dlt_file_free(&file, vflag);
+        if (ovalue)
+            close(ohandle);
+
         return -1;
     }
 
@@ -363,17 +392,62 @@ int main(int argc, char *argv[])
     for (num = begin; num <= end; num++) {
         dlt_file_message(&file, num, vflag);
         timestamp_index[num - begin].num = num;
+        timestamp_index[num - begin].systmsp = file.msg.storageheader->seconds;
         timestamp_index[num - begin].tmsp = file.msg.headerextra.tmsp;
     }
 
+    /* This step is extending the array one more element by copying the first element */
+    timestamp_index[num].num = timestamp_index[0].num;
+    timestamp_index[num].systmsp = timestamp_index[0].systmsp;
+    timestamp_index[num].tmsp = timestamp_index[0].tmsp;
+
     verbose(1, "Sorting\n");
-    qsort((void *)timestamp_index, message_count, sizeof(TimestampIndex), compare_index_timestamps);
+    qsort((void *) timestamp_index, message_count, sizeof(TimestampIndex), compare_index_systime);
 
-    write_messages(ohandle, &file, timestamp_index, message_count);
-    close(ohandle);
+    for (num = begin; num <= end; num++) {
+        delta_tmsp = abs(timestamp_index[num + 1].tmsp - timestamp_index[num].tmsp);
+        delta_systime = abs(timestamp_index[num + 1].systmsp - timestamp_index[num].systmsp);
+
+        /*
+         * Here is a try to detect a new cycle of boot in system.
+         * Relatively, if there are gaps whose systime is larger than 3 mins and
+         * timestamp is larger than 15 secs should be identified as a new boot cycle.
+         */
+        count++;
+        if(delta_tmsp > FIFTY_SEC_IN_MSEC || delta_systime >= THREE_MIN_IN_SEC) {
+            temp_timestamp_index = (TimestampIndex *) malloc(sizeof(TimestampIndex) * count);
+
+            if (temp_timestamp_index == NULL) {
+                fprintf(stderr, "ERROR: Failed to allocate memory for array\n");
+
+                dlt_file_free(&file, vflag);
+                if (ovalue)
+                    close(ohandle);
+
+                return -1;
+            }
+
+            for (i = 0; i < count; i++) {
+                memcpy((void*) &temp_timestamp_index[i],
+                        (void*) &timestamp_index[start + i],
+                        sizeof(TimestampIndex));
+            }
+            qsort((void *) temp_timestamp_index, count, sizeof(TimestampIndex),
+                    compare_index_timestamps);
+
+            write_messages(ohandle, &file, temp_timestamp_index, count);
+            free(temp_timestamp_index);
+            temp_timestamp_index = NULL;
+            start = start + count;
+            count = 0;
+        }
+    }
+    if (ovalue)
+        close(ohandle);
 
     verbose(1, "Tidying up.\n");
     free(timestamp_index);
+    timestamp_index = NULL;
     dlt_file_free(&file, vflag);
     return 0;
 }