Restrict dlt-receive to write max. n-bytes

 - dlt-receive gets a new parameter to restrict the maximum size of the
   written file. When the limit is exceeded, a new file is opened
 - testscript (bash) in src/tests provided to demonstrate functionality

Signed-off-by: Stefan Vacek <stefan.vacek@intel.com>

4 files changed, 322 insertions, 7 deletions

diff --git a/doc/dlt-receive.1 b/doc/dlt-receive.1
index 9efbbca..5f1665b 100644
--- a/doc/dlt-receive.1
+++ b/doc/dlt-receive.1
@@ -31,7 +31,7 @@
 dlt-receive \- Console based client for DLT Logging
 .SH "SYNOPSIS"
 .sp
-\fBdlt\-receive\fR [\-h] [\-a] [\-x] [\-m] [\-s][\-o filename] [\-v] [\-y] [\-b baudrate] [\-e ecuid]hostname/serial_device_name
+\fBdlt\-receive\fR [\-h] [\-a] [\-x] [\-m] [\-s] [\-o filename] [\-c limit] [\-v] [\-y] [\-b baudrate] [\-e ecuid] hostname/serial_device_name
 .SH "DESCRIPTION"
 .sp
 Receive DLT messages from DLT daemon and print or store the messages\&.
@@ -67,6 +67,11 @@ Print DLT file; only headers\&.
 Output messages in new DLT file\&.
 .RE
 .PP
+\fB\-c\fR
+.RS 4
+Set limit when storing messages in file\&. When limit is reached, a new file is opened\&. Use K,M,G as suffix to specify kilo\-, mega\-, giga\-bytes respectively, e\&.g\&. 1M for one megabyte (Default: unlimited)\&.
+.RE
+.PP
 \fB\-v\fR
 .RS 4
 Verbose mode\&.
@@ -98,13 +103,21 @@ Print received message headers received from a serila interface
 dlt\-receive \-s \-y /dev/ttySO
 .RE
 .PP
-Store received message headers from a dlt\-daemon to a log file called log\&.dlt and filter them for e\&.g\&. Application ID ABCD and Context ID EFGH (Write:ABCD EFGH as single line to a file claled filter\&.txt)
+Store received message headers from a dlt\-daemon to a log file called log\&.dlt and filter them for e\&.g\&. Application ID ABCD and Context ID EFGH (Write:ABCD EFGH as single line to a file called filter\&.txt)
 .RS 4
 dlt\-receive \-s \-o log\&.dlt \-f filter\&.txt localhost
 .RE
+.PP
+Store incoming messages in file(s) and restrict file sizes to 1 megabyte\&. If limit is reached, log\&.dlt will be renamed into log\&.0\&.dlt, log\&.1\&.dlt, \&... No files will be overwritten in this mode
+.RS 4
+dlt\-receive \-o log\&.dlt \-c 1M localhost
+.RE
 .SH "EXIT STATUS"
 .sp
 Non zero is returned in case of failure\&.
+.SH "NOTES"
+.sp
+Be aware that dlt\-receive will never delete any files\&. Instead, it creates a new file\&.
 .SH "AUTHOR"
 .sp
 Alexander Wenzel (alexander\&.aw\&.wenzel (at) bmw\&.de)
diff --git a/doc/dlt-receive.1.txt b/doc/dlt-receive.1.txt
index 4f1d933..25a876d 100644
--- a/doc/dlt-receive.1.txt
+++ b/doc/dlt-receive.1.txt
@@ -8,7 +8,7 @@ dlt-receive - Console based client for DLT Logging
 
 SYNOPSIS
 --------
-*dlt-receive* [-h] [-a] [-x] [-m] [-s][-o filename] [-v] [-y] [-b baudrate] [-e ecuid]hostname/serial_device_name
+*dlt-receive* [-h] [-a] [-x] [-m] [-s] [-o filename] [-c limit] [-v] [-y] [-b baudrate] [-e ecuid] hostname/serial_device_name
 
 DESCRIPTION
 -----------
@@ -34,6 +34,9 @@ Print DLT file; only headers.
 *-o*::
 Output messages in new DLT file.
 
+*-c*::
+Set limit when storing messages in file. When limit is reached, a new file is opened. Use K,M,G as suffix to specify kilo-, mega-, giga-bytes respectively, e.g. 1M for one megabyte (Default: unlimited).
+
 *-v*::
 Verbose mode.
 
@@ -54,13 +57,20 @@ Print received message headers received from a dlt-daemon running on localhost::
 Print received message headers received from a serila interface::
     dlt-receive -s -y /dev/ttySO
 
-Store received message headers from a dlt-daemon to a log file called log.dlt and filter them for e.g. Application ID ABCD and Context ID EFGH (Write:ABCD EFGH as single line to a file claled filter.txt)::
+Store received message headers from a dlt-daemon to a log file called log.dlt and filter them for e.g. Application ID ABCD and Context ID EFGH (Write:ABCD EFGH as single line to a file called filter.txt)::
     dlt-receive -s -o log.dlt -f filter.txt localhost
+    
+Store incoming messages in file(s) and restrict file sizes to 1 megabyte. If limit is reached, log.dlt will be renamed into log.0.dlt, log.1.dlt, ... No files will be overwritten in this mode::
+    dlt-receive -o log.dlt -c 1M localhost
 
 EXIT STATUS
 -----------
 Non zero is returned in case of failure.
 
+NOTES
+-----
+Be aware that dlt-receive will never delete any files. Instead, it creates a new file.
+
 AUTHOR
 ------
 Alexander Wenzel (alexander.aw.wenzel (at) bmw.de)
diff --git a/src/console/dlt-receive.c b/src/console/dlt-receive.c
index 41a0a8c..071c1ba 100644
--- a/src/console/dlt-receive.c
+++ b/src/console/dlt-receive.c
@@ -73,6 +73,11 @@
 #include <sys/stat.h>   /* for S_IRUSR, S_IWUSR, S_IRGRP, S_IROTH */
 #include <fcntl.h>      /* for open() */
 #include <sys/uio.h>    /* for writev() */
+#include <errno.h>
+#include <string.h>
+#include <glob.h>
+#include <syslog.h>
+#include <linux/limits.h> /* for PATH_MAX */
 
 #include "dlt_client.h"
 
@@ -91,11 +96,15 @@ typedef struct {
     int vflag;
     int yflag;
     char *ovalue;
+    char *ovaluebase; /* ovalue without ".dlt" */
     char *fvalue;
     char *evalue;
     int bvalue;
+    int64_t climit;
     char ecuid[4];
     int ohandle;
+    int64_t totalbytes; /* bytes written so far into the output file, used to check the file size limit */
+    int part_num;    /* number of current output file if limit was exceeded */
     DltFile file;
     DltFilter filter;
 } DltReceiveData;
@@ -124,9 +133,168 @@ void usage()
     printf("  -b baudrate   Serial device baudrate (Default: 115200)\n");
     printf("  -e ecuid      Set ECU ID (Default: RECV)\n");
     printf("  -o filename   Output messages in new DLT file\n");
+    printf("  -c limit      Restrict file size to <limit> bytes when output to file\n");
+    printf("                When limit is reached, a new file is opened. Use K,M,G as\n");
+    printf("                suffix to specify kilo-, mega-, giga-bytes respectively\n");
     printf("  -f filename   Enable filtering of messages\n");
 }
 
+
+int64_t convert_arg_to_byte_size(char * arg)
+{
+    size_t i;
+    int64_t factor;
+    int64_t result;
+    /* check if valid input */
+    for (i = 0; i<strlen(arg)-1; ++i)
+    {
+        if (!isdigit(arg[i]))
+        {
+            return -2;
+        }
+    }
+
+    /* last character */
+    factor = 1;
+    if ((arg[strlen(arg)-1] == 'K') || (arg[strlen(arg)-1] == 'k'))
+    {
+        factor = 1024;
+    }
+    else if ((arg[strlen(arg)-1] == 'M') || (arg[strlen(arg)-1] == 'm'))
+    {
+        factor = 1024 * 1024;
+    }
+    else if ((arg[strlen(arg)-1] == 'G') || (arg[strlen(arg)-1] == 'g'))
+    {
+        factor = 1024 * 1024 * 1024;
+    }
+    else
+    {
+        if (!isdigit(arg[strlen(arg)-1]))
+        {
+            return -2;
+        }
+    }
+
+    /* range checking */
+    int64_t const mult = atoll(arg);
+    if (((INT64_MAX)/factor) < mult)
+    {
+      /* Would overflow! */
+      return -2;
+    }
+
+    result = factor * mult;
+
+    /* The result be at least the size of one message
+     * One message consists of its header + user data:
+     */
+    DltMessage msg;
+    int64_t min_size = sizeof(msg.headerbuffer);
+    min_size += 2048 /* DLT_USER_BUF_MAX_SIZE */;
+
+    if (min_size > result)
+    {
+        char tmp[256];
+        snprintf(tmp, 256, "ERROR: Specified limit: %li is smaller than a the size of a single message: %li !\n", result, min_size);
+        dlt_log(LOG_ERR, tmp);
+        result = -2;
+    }
+
+    return result;
+}
+
+
+/*
+ * open output file
+ */
+int dlt_receive_open_output_file(DltReceiveData * dltdata)
+{
+    /* if (file_already_exists) */
+    glob_t outer;
+    if (glob(dltdata->ovalue, GLOB_TILDE_CHECK | GLOB_NOSORT, NULL, &outer) == 0)
+    {
+        if (dltdata->vflag)
+        {
+          char tmp[256];
+          snprintf(tmp, 256, "File %s already exists, need to rename first\n", dltdata->ovalue);
+          dlt_log(LOG_INFO, tmp);
+        }
+
+        if (dltdata->part_num < 0)
+        {
+            char pattern[PATH_MAX+1];
+            pattern[PATH_MAX] = 0;
+            snprintf(pattern, PATH_MAX, "%s.*.dlt", dltdata->ovaluebase);
+            glob_t inner;
+
+            /* sort does not help here because we have to traverse the
+             * full result in any case. Remember, a sorted list would look like:
+             * foo.1.dlt
+             * foo.10.dlt
+             * foo.1000.dlt
+             * foo.11.dlt
+             */
+            if (glob(pattern, GLOB_TILDE_CHECK | GLOB_NOSORT, NULL, &inner) == 0)
+            {
+              /* search for the highest number used */
+              size_t i;
+              for (i= 0; i<inner.gl_pathc; ++i)
+              {
+                /* convert string that follows the period after the initial portion,
+                 * e.g. gt.gl_pathv[i] = foo.1.dlt -> atoi("1.dlt");
+                 */
+                int cur = atoi(&inner.gl_pathv[i][strlen(dltdata->ovaluebase)+1]);
+                if (cur > dltdata->part_num)
+                {
+                  dltdata->part_num = cur;
+                }
+              }
+            }
+            globfree(&inner);
+
+            ++dltdata->part_num;
+
+        }
+
+        char filename[PATH_MAX+1];
+        filename[PATH_MAX] = 0;
+
+        snprintf(filename, PATH_MAX, "%s.%i.dlt", dltdata->ovaluebase, dltdata->part_num++);
+        if (rename(dltdata->ovalue, filename) != 0)
+        {
+          char tmp[256];
+          snprintf(tmp, 256, "ERROR: rename %s to %s failed with error %s\n", dltdata->ovalue, filename, strerror(errno));
+          dlt_log(LOG_ERR, tmp);
+        }
+        else
+        {
+          if (dltdata->vflag)
+          {
+            char tmp[256];
+            snprintf(tmp, 256, "Renaming existing file from %s to %s\n", dltdata->ovalue, filename);
+            dlt_log(LOG_INFO, tmp);
+          }
+        }
+
+    } /* if (file_already_exists) */
+    globfree(&outer);
+
+    dltdata->ohandle = open(dltdata->ovalue, O_WRONLY|O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+    return dltdata->ohandle;
+}
+
+
+void dlt_receive_close_output_file(DltReceiveData * dltdata)
+{
+    if (dltdata->ohandle)
+    {
+        close(dltdata->ohandle);
+        dltdata->ohandle = -1;
+    }
+}
+
+
 /**
  * Main function of tool.
  */
@@ -145,15 +313,19 @@ int main(int argc, char* argv[])
     dltdata.vflag = 0;
     dltdata.yflag = 0;
     dltdata.ovalue = 0;
+    dltdata.ovaluebase = 0;
     dltdata.fvalue = 0;
     dltdata.evalue = 0;
     dltdata.bvalue = 0;
+    dltdata.climit = -1; /* default: -1 = unlimited */
     dltdata.ohandle=-1;
+    dltdata.totalbytes = 0;
+    dltdata.part_num = -1;
 
     /* Fetch command line arguments */
     opterr = 0;
 
-    while ((c = getopt (argc, argv, "vashyxmf:o:e:b:")) != -1)
+    while ((c = getopt (argc, argv, "vashyxmf:o:e:b:c:")) != -1)
         switch (c)
         {
         case 'v':
@@ -199,6 +371,13 @@ int main(int argc, char* argv[])
         case 'o':
 			{
             	dltdata.ovalue = optarg;
+            	size_t to_copy = strlen(dltdata.ovalue);
+            	if (strcmp(&dltdata.ovalue[to_copy-4], ".dlt") == 0)
+            	{
+            	  to_copy = to_copy - 4;
+            	}
+
+            	dltdata.ovaluebase = strndup(dltdata.ovalue, to_copy);
             	break;
 			}
         case 'e':
@@ -211,9 +390,22 @@ int main(int argc, char* argv[])
             	dltdata.bvalue = atoi(optarg);
             	break;
 			}
+
+        case 'c':
+			{
+            	dltdata.climit = convert_arg_to_byte_size(optarg);
+            	if (dltdata.climit < -1)
+            	{
+		            fprintf (stderr, "Invalid argument for option -c.\n");
+			        /* unknown or wrong option used, show usage information and terminate */
+			        usage();
+			        return -1;
+            	}
+            	break;
+			}
         case '?':
 			{
-		        if (optopt == 'o' || optopt == 'f')
+		        if (optopt == 'o' || optopt == 'f' || optopt == 'c')
 				{
 		            fprintf (stderr, "Option -%c requires an argument.\n", optopt);
 		        }
@@ -299,7 +491,17 @@ int main(int argc, char* argv[])
     /* open DLT output file */
     if (dltdata.ovalue)
     {
-        dltdata.ohandle = open(dltdata.ovalue,O_WRONLY|O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); /* mode: wb */
+        if (dltdata.climit > -1)
+        {
+          char tmp[256];
+          snprintf(tmp, 256, "Using file size limit of %li bytes\n", dltdata.climit);
+          dlt_log(LOG_INFO, tmp);
+          dltdata.ohandle = dlt_receive_open_output_file(&dltdata);
+        }
+        else /* in case no limit for the output file is given, we simply overwrite any existing file */
+        {
+          dltdata.ohandle = open(dltdata.ovalue, O_WRONLY|O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+        }
 
         if (dltdata.ohandle == -1)
         {
@@ -335,6 +537,8 @@ int main(int argc, char* argv[])
         close(dltdata.ohandle);
 	}
 
+    free(dltdata.ovaluebase);
+
     dlt_file_free(&(dltdata.file),dltdata.vflag);
 
     dlt_filter_free(&(dltdata.filter),dltdata.vflag);
@@ -405,8 +609,24 @@ int dlt_receive_message_callback(DltMessage *message, void *data)
             iov[1].iov_base = message->databuffer;
             iov[1].iov_len = message->datasize;
 
+            if (dltdata->climit > -1)
+            {
+            	int bytes_to_write = message->headersize + message->datasize;
+            	if ((bytes_to_write + dltdata->totalbytes > dltdata->climit))
+            	{
+                dlt_receive_close_output_file(dltdata);
+            		if (dlt_receive_open_output_file(dltdata) < 0)
+            		{
+                        printf("ERROR: dlt_receive_message_callback: Unable to open log when maximum filesize was reached!\n");
+                        return -1;
+            		}
+            		dltdata->totalbytes = 0;
+            	}
+            }
             bytes_written = writev(dltdata->ohandle, iov, 2);
 
+            dltdata->totalbytes += bytes_written;
+
             if (0 > bytes_written){
                     printf("dlt_receive_message_callback: writev(dltdata->ohandle, iov, 2); returned an error!" );
                     return -1;
diff --git a/src/tests/dlt-test-receiver-multiple-files.sh b/src/tests/dlt-test-receiver-multiple-files.sh
new file mode 100755
index 0000000..013cbe6
--- /dev/null
+++ b/src/tests/dlt-test-receiver-multiple-files.sh
@@ -0,0 +1,72 @@
+#!/bin/bash
+
+# check if dlt-daemon is running
+daemon_running=`/usr/bin/ps -C dlt-daemon | wc -l`
+daemon_pid=0
+
+if [ "$daemon_running" -lt "2" ]; then
+  echo "No daemon running, starting one myself"
+  /usr/bin/dlt-daemon > /tmp/dlt_daemon_dlt_receiver_test.txt & 
+  daemon_pid=$!
+  echo "daemon pid: " ${daemon_pid}
+else
+  echo "dlt-daemon already running"
+fi
+
+# create a directory in /tmp where all logs will be stored
+output_dir=`mktemp -d /tmp/DLT_TESTING_XXXXXX`
+echo "Using directory " ${output_dir}
+
+# start dlt-receive (in background) and store PID
+echo "Starting dlt-receive"
+/usr/bin/dlt-receive -o ${output_dir}/dlt_test.dlt localhost &
+dlt_receive_pid=$!
+disown
+
+# start dlt-example-user to create some logs
+# sleep time: 100ms
+# number of messages: 10
+/usr/bin/dlt-example-user -g -d 100 -n 10 TEST_MESSAGE_ONE
+
+# stop dlt-receive
+kill ${dlt_receive_pid}
+
+# show content of /tmp
+echo "log-file after first run"
+ls -l ${output_dir}
+
+# start dlt-receive (in background) and store PID
+echo "Starting dlt-receive"
+/usr/bin/dlt-receive -o ${output_dir}/dlt_test.dlt localhost &
+dlt_receive_pid=$!
+disown
+
+# start dlt-example-user to create some logs (use different number of messages)
+/usr/bin/dlt-example-user -d 100 -n 20 TEST_MESSAGE_TWO
+
+# show content of /tmp --> original file was overwritten
+kill ${dlt_receive_pid}
+echo "log-file after second run"
+ls -l ${output_dir}
+
+# start dlt-receive with small maximum file size (in background) and store PID
+echo "Starting dlt-receive"
+/usr/bin/dlt-receive -o ${output_dir}/dlt_test.dlt -c 3K localhost &
+dlt_receive_pid=$!
+disown
+
+# start dlt-example-user to create some logs (use even more messages then before) 
+/usr/bin/dlt-example-user -d 20 -n 500 TEST_MESSAGE_THREE
+
+# show content of /tmp --> multiple files were created, the original file was preserved
+echo "log-file after third run (should show multiple files)"
+ls -l ${output_dir}
+
+# directory will not be cleaned up
+echo "Used directory " ${output_dir}
+
+
+if [ "${daemon_pid}" -ne "0" ]; then
+  sleep 1
+  kill ${daemon_pid}
+fi