Add dlt-sortbytimestamp utility plus documentation (#73)

* Add dlt-sortbytimestamp utility plus documentation

    Add a commandline utility to sort a DLT file by timestamp.

    By default a DLT file's messages are in the order they were 
    received by the logger. This is not ideal for tracing problems on
    systems with multi-threaded programmes running on multi-core CPUs
    since message reception order will not always (or even often)
    correspond to message creation order.

    The documentation deals with how to handle the problem case of DLT
    files containing messages from multiple boot cycles.

6 files changed, 222 insertions, 13 deletions

diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt
index d04bf8d..263dbe0 100644
--- a/doc/CMakeLists.txt
+++ b/doc/CMakeLists.txt
@@ -19,12 +19,12 @@ if(WITH_DOC)
     find_package(Doxygen)
 
     configure_file(${CMAKE_SOURCE_DIR}/doc/doxygen.cfg.cmake ${CMAKE_BINARY_DIR}/doc/doxygen.cfg @ONLY)
-	
+
 	add_custom_target (doc ALL
   		COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_BINARY_DIR}/doc/doxygen.cfg
   		WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/doc
 	)
-	
+
 	add_custom_target (doc-manuals ALL
 		COMMAND mkdir -p ${CMAKE_BINARY_DIR}/doc/manuals
 		COMMAND mkdir -p ${CMAKE_BINARY_DIR}/doc/manuals/images
@@ -40,6 +40,7 @@ if(WITH_DOC)
   		COMMAND asciidoc -a TOC1 -o ${CMAKE_BINARY_DIR}/doc/manuals/dlt-system.1.html ${CMAKE_SOURCE_DIR}/doc/dlt-system.1.txt
   		COMMAND asciidoc -a TOC1 -o ${CMAKE_BINARY_DIR}/doc/manuals/dlt-receive.1.html ${CMAKE_SOURCE_DIR}/doc/dlt-receive.1.txt
   		COMMAND asciidoc -a TOC1 -o ${CMAKE_BINARY_DIR}/doc/manuals/dlt-convert.1.html ${CMAKE_SOURCE_DIR}/doc/dlt-convert.1.txt
+  		COMMAND asciidoc -a TOC1 -o ${CMAKE_BINARY_DIR}/doc/manuals/dlt-sortbytimestamp.1.html ${CMAKE_SOURCE_DIR}/doc/dlt-sortbytimestamp.1.txt
   		COMMAND asciidoc -a TOC1 -o ${CMAKE_BINARY_DIR}/doc/manuals/dlt.conf.5.html ${CMAKE_SOURCE_DIR}/doc/dlt.conf.5.txt
   		COMMAND asciidoc -a TOC1 -o ${CMAKE_BINARY_DIR}/doc/manuals/dlt-system.conf.5.html ${CMAKE_SOURCE_DIR}/doc/dlt-system.conf.5.txt
   		COMMAND asciidoc -a TOC1 -o ${CMAKE_BINARY_DIR}/doc/manuals/dlt_book.html ${CMAKE_SOURCE_DIR}/doc/dlt_book.txt
@@ -58,12 +59,13 @@ if(WITH_DOC)
 		COMMAND a2x --doctype manpage --format manpage ${CMAKE_SOURCE_DIR}/doc/dlt-system.1.txt
 		COMMAND a2x --doctype manpage --format manpage ${CMAKE_SOURCE_DIR}/doc/dlt-receive.1.txt
 		COMMAND a2x --doctype manpage --format manpage ${CMAKE_SOURCE_DIR}/doc/dlt-convert.1.txt
+		COMMAND a2x --doctype manpage --format manpage ${CMAKE_SOURCE_DIR}/doc/dlt-sortbytimestamp.1.txt
 		COMMAND a2x --doctype manpage --format manpage ${CMAKE_SOURCE_DIR}/doc/dlt.conf.5.txt
 		COMMAND a2x --doctype manpage --format manpage ${CMAKE_SOURCE_DIR}/doc/dlt-system.conf.5.txt
 		COMMAND a2x --doctype manpage --format manpage ${CMAKE_SOURCE_DIR}/doc/dlt-logstorage-ctrl.1.txt
 		COMMAND a2x --doctype manpage --format manpage ${CMAKE_SOURCE_DIR}/doc/dlt-passive-node-ctrl.1.txt
 	)
-	
+
 endif(WITH_DOC)
 
 if(WITH_MAN)
@@ -73,18 +75,19 @@ if(WITH_MAN)
 	             PATHS /bin
 	                   /usr/bin
 	                   /usr/local/bin)
-	
+
 	if(NOT GZIP_TOOL)
 	  MESSAGE(FATAL_ERROR "Could not find gzip for man page compression.")
 	endif(NOT GZIP_TOOL)
-	
+
 	set(MAN_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 	set(MAN_BUILD_DIR ${CMAKE_BINARY_DIR}/doc)
-	
+
 	set(MAN_SRC
 				${MAN_SRC_DIR}/dlt.conf.5
 				${MAN_SRC_DIR}/dlt-system.conf.5
 				${MAN_SRC_DIR}/dlt-convert.1
+				${MAN_SRC_DIR}/dlt-sortbytimestamp.1
 				${MAN_SRC_DIR}/dlt-daemon.1
 				${MAN_SRC_DIR}/dlt-receive.1
 				${MAN_SRC_DIR}/dlt-system.1
@@ -94,6 +97,7 @@ if(WITH_MAN)
 				${MAN_BUILD_DIR}/dlt.conf.5
 				${MAN_BUILD_DIR}/dlt-system.conf.5
 				${MAN_BUILD_DIR}/dlt-convert.1
+				${MAN_BUILD_DIR}/dlt-sortbytimestamp.1
 				${MAN_BUILD_DIR}/dlt-daemon.1
 				${MAN_BUILD_DIR}/dlt-receive.1
 				${MAN_BUILD_DIR}/dlt-system.1
@@ -103,37 +107,40 @@ if(WITH_MAN)
 				${MAN_BUILD_DIR}/dlt.conf.5.gz
 				${MAN_BUILD_DIR}/dlt-system.conf.5.gz
 				${MAN_BUILD_DIR}/dlt-convert.1.gz
+				${MAN_BUILD_DIR}/dlt-sortbytimestamp.1.gz
 				${MAN_BUILD_DIR}/dlt-daemon.1.gz
 				${MAN_BUILD_DIR}/dlt-receive.1.gz
 				${MAN_BUILD_DIR}/dlt-system.1.gz
 				${MAN_BUILD_DIR}/dlt-logstorage-ctrl.1.gz
 				${MAN_BUILD_DIR}/dlt-passive-node-ctrl.1.gz)
-	
+
 	foreach(MAN ${MAN_SRC})
 		execute_process(COMMAND ${CMAKE_COMMAND} -E copy ${MAN} ${MAN_BUILD_DIR})
 	endforeach(MAN)
-	
+
 	add_custom_target(compress_man ALL
 				COMMAND ${GZIP_TOOL} -c ${MAN_BUILD_DIR}/dlt.conf.5 > ${MAN_BUILD_DIR}/dlt.conf.5.gz
 				COMMAND ${GZIP_TOOL} -c ${MAN_BUILD_DIR}/dlt-system.conf.5 > ${MAN_BUILD_DIR}/dlt-system.conf.5.gz
 				COMMAND ${GZIP_TOOL} -c ${MAN_BUILD_DIR}/dlt-convert.1  > ${MAN_BUILD_DIR}/dlt-convert.1.gz
+				COMMAND ${GZIP_TOOL} -c ${MAN_BUILD_DIR}/dlt-sortbytimestamp.1  > ${MAN_BUILD_DIR}/dlt-sortbytimestamp.1.gz
 				COMMAND ${GZIP_TOOL} -c ${MAN_BUILD_DIR}/dlt-daemon.1 > ${MAN_BUILD_DIR}/dlt-daemon.1.gz
 				COMMAND ${GZIP_TOOL} -c ${MAN_BUILD_DIR}/dlt-receive.1 > ${MAN_BUILD_DIR}/dlt-receive.1.gz
 				COMMAND ${GZIP_TOOL} -c ${MAN_BUILD_DIR}/dlt-system.1 > ${MAN_BUILD_DIR}/dlt-system.1.gz
 				COMMAND ${GZIP_TOOL} -c ${MAN_BUILD_DIR}/dlt-logstorage-ctrl.1 > ${MAN_BUILD_DIR}/dlt-logstorage-ctrl.1.gz
 				COMMAND ${GZIP_TOOL} -c ${MAN_BUILD_DIR}/dlt-passive-node-ctrl.1 > ${MAN_BUILD_DIR}/dlt-passive-node-ctrl.1.gz)
-	
+
 	# If user has not set the base dir for man pages, use a default location
 	set(MAN_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}/share/man)
-	
+
 	install(FILES
 			${MAN_BUILD_DIR}/dlt.conf.5.gz
 			${MAN_BUILD_DIR}/dlt-system.conf.5.gz
 		DESTINATION ${MAN_INSTALL_DIR}/man5
 	)
-	
+
 	install(FILES
 			${MAN_BUILD_DIR}/dlt-convert.1.gz
+			${MAN_BUILD_DIR}/dlt-sortbytimestamp.1.gz
 			${MAN_BUILD_DIR}/dlt-daemon.1.gz
 			${MAN_BUILD_DIR}/dlt-receive.1.gz
 			${MAN_BUILD_DIR}/dlt-system.1.gz
diff --git a/doc/dlt-sortbytimestamp.1 b/doc/dlt-sortbytimestamp.1
new file mode 100644
index 0000000..7f0915f
--- /dev/null
+++ b/doc/dlt-sortbytimestamp.1
@@ -0,0 +1,103 @@
+'\" t
+.\"     Title: dlt-sortbytimestamp
+.\"    Author: [see the "AUTHOR" section]
+.\" Generator: DocBook XSL Stylesheets v1.79.1 <http://docbook.sf.net/>
+.\"      Date: 07/17/2018
+.\"    Manual: \ \&
+.\"    Source: \ \&
+.\"  Language: English
+.\"
+.TH "DLT\-SORTBYTIMESTAMP" "1" "07/17/2018" "\ \&" "\ \&"
+.\" -----------------------------------------------------------------
+.\" * Define some portability stuff
+.\" -----------------------------------------------------------------
+.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.\" http://bugs.debian.org/507673
+.\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html
+.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.ie \n(.g .ds Aq \(aq
+.el       .ds Aq '
+.\" -----------------------------------------------------------------
+.\" * set default formatting
+.\" -----------------------------------------------------------------
+.\" disable hyphenation
+.nh
+.\" disable justification (adjust text to left margin only)
+.ad l
+.\" -----------------------------------------------------------------
+.\" * MAIN CONTENT STARTS HERE *
+.\" -----------------------------------------------------------------
+.SH "NAME"
+dlt-sortbytimestamp \- Re\-order DLT Logging files according to message creation time
+.SH "SYNOPSIS"
+.sp
+\fBdlt\-sortbytimestamp\fR [\-h] [\-v] [\-c] [\-f filterfile] [\-b number] [\-e number] dltfile_in dltfile_out
+.SH "DESCRIPTION"
+.sp
+By default messages in DLT files are ordered according to the time the logger received them\&. This can unhelpful when tracing a sequence of events on a busy multi\-threaded/multi\-core system, because thread pre\-emption combined with multiple processes attempting to log messages simultaneously means that the order in which the messages are received may vary significantly from the order in which they were created\&.
+.sp
+\fBdlt\-sortbytimestamp\fR re\-orders a DLT input file\(cqs messages according their creation timestamp, and writes them to an output DLT file\&.
+.SH "IMPORTANT NOTE"
+.sp
+Message timestamps are recorded relative to boot time\&. DLT files can contain messages from more than one reboot cycle\&. Because timestamping is reset to zero at each boot, simply running \fBdlt\-sortbytimestamp\fR against a multi\-boot\-cycle DLT input file will produce a tangled mess\&.
+.sp
+Use the \fB\-b\fR and/or \fB\-e\fR options to specify a range of messages within a single reboot cycle and all will be well\&.
+.sp
+Hint: use \fBdlt\-viewer\fR to ascertain the endpoints of the range in question\&.
+.SH "OPTIONS"
+.PP
+\fB\-h\fR
+.RS 4
+Display a short help text\&.
+.RE
+.PP
+\fB\-v\fR
+.RS 4
+Verbose mode\&. Repeat to give more information\&.
+.RE
+.PP
+\fB\-c\fR
+.RS 4
+Count number of messages\&.
+.RE
+.PP
+\fB\-f\fR
+.RS 4
+Enable filtering of messages\&. Incompatible with range options\&.
+.RE
+.PP
+\fB\-b\fR
+.RS 4
+First message to be handled\&. Zero based index\&.
+.RE
+.PP
+\fB\-e\fR
+.RS 4
+Last message to be handled\&. Zero based index\&.
+.RE
+.SH "EXAMPLES"
+.PP
+Sort an entire file by message timestamp
+.RS 4
+\fBdlt\-sortbytimestamp input\&.dlt output\&.dlt\fR
+.RE
+.PP
+Sort a specific range, e\&.g\&. from message 1,000,000 to message 1,500,000 from a file called input\&.dlt and store the result in a file called output\&.dlt
+.RS 4
+\fBdlt\-sortbytimestamp \-b 1000000 \-e 1500000 input\&.dlt output\&.dlt\fR
+.RE
+.SH "EXIT STATUS"
+.sp
+Non zero is returned in case of failure\&.
+.SH "AUTHOR"
+.sp
+Jonathan Sambrook (jonathan\&.sambrook (at) codethink\&.co\&.uk) Alexander Wenzel (alexander\&.aw\&.wenzel (at) bmw\&.de)
+.SH "COPYRIGHT"
+.sp
+Copyright \(co 2018 Codethink Ltd\&. Copyright \(co 2015 BMW AG\&. License MPL\-2\&.0: Mozilla Public License version 2\&.0 http://mozilla\&.org/MPL/2\&.0/\&.
+.SH "RESOURCES"
+.sp
+Main web site: http://projects\&.genivi\&.org/diagnostic\-log\-trace Mailinglist: https://lists\&.genivi\&.org/mailman/listinfo/genivi\-diagnostic\-log\-and\-trace
+.SH "SEE ALSO"
+.sp
+dlt\-convert(1) dlt\-daemon(1)
diff --git a/doc/dlt-sortbytimestamp.1.txt b/doc/dlt-sortbytimestamp.1.txt
new file mode 100644
index 0000000..cb92051
--- /dev/null
+++ b/doc/dlt-sortbytimestamp.1.txt
@@ -0,0 +1,81 @@
+DLT-SORTBYTIMESTAMP(1)
+======================
+:doctype: manpage
+
+NAME
+----
+dlt-sortbytimestamp - Re-order DLT Logging files according to message creation time
+
+SYNOPSIS
+--------
+*dlt-sortbytimestamp* [-h] [-v] [-c] [-f filterfile] [-b number] [-e number] dltfile_in dltfile_out
+
+DESCRIPTION
+-----------
+By default messages in DLT files are ordered according to the time the logger received them. This can unhelpful when tracing a sequence of events on a busy multi-threaded/multi-core system, because thread pre-emption combined with multiple processes attempting to log messages simultaneously means that the order in which the messages are received may vary significantly from the order in which they were created.
+
+*dlt-sortbytimestamp* re-orders a DLT input file's messages according their creation timestamp, and writes them to an output DLT file.
+
+IMPORTANT NOTE
+--------------
+Message timestamps are recorded relative to boot time. DLT files can contain messages from more than one reboot cycle. Because timestamping is reset to zero at each boot, simply running *dlt-sortbytimestamp* against a multi-boot-cycle DLT input file will produce a tangled mess.
+
+Use the *-b* and/or *-e* options to specify a range of messages within a single reboot cycle and all will be well.
+
+Hint: use *dlt-viewer* to ascertain the endpoints of the range in question.
+
+OPTIONS
+-------
+*-h*::
+    Display a short help text.
+
+*-v*::
+    Verbose mode. Repeat to give more information.
+
+*-c*::
+    Count number of messages.
+
+*-f*::
+    Enable filtering of messages. Incompatible with range options.
+
+*-b*::
+    First message to be handled. Zero based index.
+
+*-e*::
+    Last message to be handled. Zero based index.
+
+EXAMPLES
+--------
+
+Sort an entire file by message timestamp::
+    *dlt-sortbytimestamp input.dlt output.dlt*
+
+Sort a specific range, e.g. from message 1,000,000 to message 1,500,000 from a file called input.dlt and store the result in a file called output.dlt::
+    *dlt-sortbytimestamp -b 1000000 -e 1500000 input.dlt output.dlt*
+
+
+EXIT STATUS
+-----------
+Non zero is returned in case of failure.
+
+AUTHOR
+------
+Jonathan Sambrook (jonathan.sambrook (at) codethink.co.uk)
+Alexander Wenzel (alexander.aw.wenzel (at) bmw.de)
+
+COPYRIGHT
+---------
+Copyright (C) 2018 Codethink Ltd.
+Copyright (C) 2015 BMW AG.
+License MPL-2.0: Mozilla Public License version 2.0 <http://mozilla.org/MPL/2.0/>.
+
+RESOURCES
+---------
+Main web site: <http://projects.genivi.org/diagnostic-log-trace> +
+Mailinglist: <https://lists.genivi.org/mailman/listinfo/genivi-diagnostic-log-and-trace>
+
+SEE ALSO
+--------
+dlt-convert(1)
+dlt-daemon(1)
+
diff --git a/doc/dlt_book.txt b/doc/dlt_book.txt
index 1026bb4..8a2c91e 100644
--- a/doc/dlt_book.txt
+++ b/doc/dlt_book.txt
@@ -78,6 +78,8 @@ include::dlt-system.conf.5.txt[]
 
 include::dlt-convert.1.txt[]
 
+include::dlt-sortbytimestamp.1.txt[]
+
 include::dlt-receive.1.txt[]
 
 include::dlt-logstorage-ctrl.1.txt[]
diff --git a/doc/dlt_design_specification.txt b/doc/dlt_design_specification.txt
index 4d52a2a..6d1bdf4 100644
--- a/doc/dlt_design_specification.txt
+++ b/doc/dlt_design_specification.txt
@@ -531,7 +531,7 @@ As this document has the focus on the DLT Daemon and the DLT user library, only
 | src | | Source Code
 | src | shared | Shared source code (between DLT daemon and DLT user library)
 | src | adaptor | Adaptors to DLT daemon:dlt-adaptor-stdin (for connection over stdin) dlt-adaptor-udp (for connection over UDP)
-| src | console | Console utilities: dlt-receive and dlt-convert
+| src | console | Console utilities: dlt-receive, dlt-convert, and dlt-sortbytimestamp
 | src | daemon | DLT Daemon
 | src | example | Examples for usage of the DLT user library:dlt-example-user (Macro IF) anddlt-example-user-func (Function IF) andwintestclient (MS Windows based test client)
 | src | lib | DLT library functions
diff --git a/doc/dlt_user_manual.txt b/doc/dlt_user_manual.txt
index 8b937b1..c8f444e 100644
--- a/doc/dlt_user_manual.txt
+++ b/doc/dlt_user_manual.txt
@@ -36,6 +36,7 @@ The SW components of DLT:
 * dlt-adaptor-stdin
 * dlt-adaptor-udp
 * dlt-convert
+* dlt-sortbytimestamp
 * dlt-receive
 * dlt-dbus
 * dlt-kpi
@@ -137,12 +138,13 @@ See Manpage dlt-system(1).
 See Manpage dlt-system.conf(5).
 
 == DLT command line tools
-Three command line tools are provided together with DLT daemon implementation.
+Four command line tools are provided together with DLT daemon implementation.
 
 These tools are:
 
 * Data logger: dlt-receive
 * Converter: dlt-convert
+* Sorter: dlt-sortbytimestamp
 * Configuration: dlt-control (planned)
 
 
@@ -162,6 +164,20 @@ The dlt-convert console utility is used to read DLT files, print DLT messages in
 ==== Command line interface
 See Manpage dlt-convert(1).
 
+=== dlt-sortbytimestamp
+
+==== Overview
+By default messages in DLT files are ordered according to the time the logger received them. This can unhelpful when tracing a sequence of events on a busy multi-threaded/multi-core system, because thread pre-emption combined with multiple processes attempting to log messages simultaneously means that the order in which the messages are received may vary significantly from the order in which they were created.
+
+The dlt-sortbytimestamp console utility is used to re-order the messages in DLT files, sorting them according to when the messages were created.
+
+Filters can be used to filter messages OR a range of messages to be processed can specified.
+
+Ranges are essential when processing a DLT file covering more than a single reboot cycle. This is because message timestamps are recorded relative to boot time and thus timestamping is reset to zero at each boot. Events from different boot cycles will have differing receive times, but may have identical boot relative timestamps. Using *dlt-sortbytimestamp* to sort such a DLT file will result in a tangled mess. Use ranges to avoid this. Use *dlt-viewer* to ascertain the start and end message indices of the desired range.
+
+==== Command line interface
+See Manpage dlt-sortbytimestamp(1).
+
 == DLT adaptors
 The DLT adaptors are used to interface legacy linux applications with the DLT daemon. Therefore, there are two adaptors: