ovsdb-server: Report time since last connect and disconnect for each manager.

Only the time connected (if connected) or disconnected (if disconnected) is
currently reported for each manager. Change to reporting both in seconds since
the last connect and disconnect events respectively. An empty value indicates
no previous connection or disconnection.

This can help diagnose certain connectivity problems, e.g. flapping.

Requested-by: Peter Balland <peter@nicira.com>
Bug #4833.

1 files changed, 39 insertions, 85 deletions

diff --git a/tests/reconnect.at b/tests/reconnect.at
index 6a9ae1e84..bb37170ec 100644
--- a/tests/reconnect.at
+++ b/tests/reconnect.at
@@ -59,8 +59,10 @@ run
   should connect
 connected
   in ACTIVE for 0 ms (0 ms backoff)
+  created 1000, last received 1000, last connected 1000
   1 successful connections out of 1 attempts, seqno 1
-  connected (0 ms), total 0 ms connected
+  connected
+  last connected 0 ms ago, connected 0 ms total
 
 # Send inactivity probe.
 timeout
@@ -68,7 +70,6 @@ timeout
 
 ### t=6000 ###
   in ACTIVE for 5000 ms (0 ms backoff)
-  connected (5000 ms), total 5000 ms connected
 run
   should send probe
   in IDLE for 0 ms (0 ms backoff)
@@ -79,13 +80,12 @@ timeout
 
 ### t=11000 ###
   in IDLE for 5000 ms (0 ms backoff)
-  connected (10000 ms), total 10000 ms connected
 run
   should disconnect
 disconnected
   in BACKOFF for 0 ms (1000 ms backoff)
   1 successful connections out of 1 attempts, seqno 2
-  not connected (0 ms), total 10000 ms connected
+  disconnected
   disconnected at 11000 ms (0 ms ago)
 ])
 
@@ -126,14 +126,13 @@ advance 500
 
 ### t=1500 ###
   in CONNECTING for 500 ms (0 ms backoff)
-  disconnected for 500 ms
 run
 connected
   in ACTIVE for 0 ms (0 ms backoff)
   created 1000, last received 1000, last connected 1500
   1 successful connections out of 1 attempts, seqno 1
-  connected (0 ms), total 0 ms connected
-  disconnected for 0 ms
+  connected
+  last connected 0 ms ago, connected 0 ms total
 
 # Send inactivity probe.
 timeout
@@ -141,7 +140,6 @@ timeout
 
 ### t=6500 ###
   in ACTIVE for 5000 ms (0 ms backoff)
-  connected (5000 ms), total 5000 ms connected
 run
   should send probe
   in IDLE for 0 ms (0 ms backoff)
@@ -152,13 +150,12 @@ timeout
 
 ### t=11500 ###
   in IDLE for 5000 ms (0 ms backoff)
-  connected (10000 ms), total 10000 ms connected
 run
   should disconnect
 disconnected
   in BACKOFF for 0 ms (1000 ms backoff)
   1 successful connections out of 1 attempts, seqno 2
-  not connected (0 ms), total 10000 ms connected
+  disconnected
   disconnected at 11500 ms (0 ms ago)
 ])
 
@@ -239,7 +236,6 @@ timeout
 
 ### t=2000 ###
   in CONNECTING for 1000 ms (0 ms backoff)
-  disconnected for 1000 ms
 run
   should disconnect
 connect-failed
@@ -252,7 +248,6 @@ timeout
 
 ### t=3000 ###
   in BACKOFF for 1000 ms (1000 ms backoff)
-  disconnected for 2000 ms
 run
   should connect
 
@@ -264,7 +259,6 @@ timeout
 
 ### t=4000 ###
   in CONNECTING for 1000 ms (1000 ms backoff)
-  disconnected for 3000 ms
 run
   should disconnect
 connect-failed
@@ -277,7 +271,6 @@ timeout
 
 ### t=6000 ###
   in BACKOFF for 2000 ms (2000 ms backoff)
-  disconnected for 5000 ms
 run
   should connect
 
@@ -289,7 +282,6 @@ timeout
 
 ### t=8000 ###
   in CONNECTING for 2000 ms (2000 ms backoff)
-  disconnected for 7000 ms
 run
   should disconnect
 connect-failed
@@ -302,7 +294,6 @@ timeout
 
 ### t=12000 ###
   in BACKOFF for 4000 ms (4000 ms backoff)
-  disconnected for 11000 ms
 run
   should connect
 
@@ -314,7 +305,6 @@ timeout
 
 ### t=16000 ###
   in CONNECTING for 4000 ms (4000 ms backoff)
-  disconnected for 15000 ms
 run
   should disconnect
 connect-failed
@@ -327,7 +317,6 @@ timeout
 
 ### t=24000 ###
   in BACKOFF for 8000 ms (8000 ms backoff)
-  disconnected for 23000 ms
 run
   should connect
 
@@ -339,7 +328,6 @@ timeout
 
 ### t=32000 ###
   in CONNECTING for 8000 ms (8000 ms backoff)
-  disconnected for 31000 ms
 run
   should disconnect
 connect-failed
@@ -352,7 +340,6 @@ timeout
 
 ### t=40000 ###
   in BACKOFF for 8000 ms (8000 ms backoff)
-  disconnected for 39000 ms
 run
   should connect
 
@@ -364,7 +351,6 @@ timeout
 
 ### t=48000 ###
   in CONNECTING for 8000 ms (8000 ms backoff)
-  disconnected for 47000 ms
 run
   should disconnect
 connect-failed
@@ -422,14 +408,15 @@ run
   should connect
 connected
   in ACTIVE for 0 ms (0 ms backoff)
+  created 1000, last received 1000, last connected 1000
   1 successful connections out of 1 attempts, seqno 1
-  connected (0 ms), total 0 ms connected
+  connected
+  last connected 0 ms ago, connected 0 ms total
 timeout
   advance 5000 ms
 
 ### t=6000 ###
   in ACTIVE for 5000 ms (0 ms backoff)
-  connected (5000 ms), total 5000 ms connected
 run
   should send probe
   in IDLE for 0 ms (0 ms backoff)
@@ -438,13 +425,12 @@ timeout
 
 ### t=11000 ###
   in IDLE for 5000 ms (0 ms backoff)
-  connected (10000 ms), total 10000 ms connected
 run
   should disconnect
 disconnected
   in BACKOFF for 0 ms (1000 ms backoff)
   1 successful connections out of 1 attempts, seqno 2
-  not connected (0 ms), total 10000 ms connected
+  disconnected
   disconnected at 11000 ms (0 ms ago)
 
 # Back off for 1000 ms.
@@ -453,7 +439,7 @@ timeout
 
 ### t=12000 ###
   in BACKOFF for 1000 ms (1000 ms backoff)
-  disconnected for 1000 ms
+  last connected 11000 ms ago, connected 10000 ms total
 run
   should connect
 
@@ -464,14 +450,13 @@ connected
   in ACTIVE for 0 ms (1000 ms backoff)
   created 1000, last received 1000, last connected 12000
   2 successful connections out of 2 attempts, seqno 3
-  connected (0 ms), total 10000 ms connected
-  disconnected for 0 ms
+  connected
+  last connected 0 ms ago, connected 10000 ms total
 timeout
   advance 5000 ms
 
 ### t=17000 ###
   in ACTIVE for 5000 ms (1000 ms backoff)
-  connected (5000 ms), total 15000 ms connected
 run
   should send probe
   in IDLE for 0 ms (1000 ms backoff)
@@ -480,13 +465,12 @@ timeout
 
 ### t=22000 ###
   in IDLE for 5000 ms (1000 ms backoff)
-  connected (10000 ms), total 20000 ms connected
 run
   should disconnect
 disconnected
   in BACKOFF for 0 ms (2000 ms backoff)
   2 successful connections out of 2 attempts, seqno 4
-  not connected (0 ms), total 20000 ms connected
+  disconnected
   disconnected at 22000 ms (0 ms ago)
 
 # Back off for 2000 ms.
@@ -495,7 +479,7 @@ timeout
 
 ### t=24000 ###
   in BACKOFF for 2000 ms (2000 ms backoff)
-  disconnected for 2000 ms
+  last connected 12000 ms ago, connected 20000 ms total
 run
   should connect
 
@@ -506,14 +490,13 @@ connected
   in ACTIVE for 0 ms (2000 ms backoff)
   created 1000, last received 1000, last connected 24000
   3 successful connections out of 3 attempts, seqno 5
-  connected (0 ms), total 20000 ms connected
-  disconnected for 0 ms
+  connected
+  last connected 0 ms ago, connected 20000 ms total
 timeout
   advance 5000 ms
 
 ### t=29000 ###
   in ACTIVE for 5000 ms (2000 ms backoff)
-  connected (5000 ms), total 25000 ms connected
 run
   should send probe
   in IDLE for 0 ms (2000 ms backoff)
@@ -522,13 +505,12 @@ timeout
 
 ### t=34000 ###
   in IDLE for 5000 ms (2000 ms backoff)
-  connected (10000 ms), total 30000 ms connected
 run
   should disconnect
 disconnected
   in BACKOFF for 0 ms (4000 ms backoff)
   3 successful connections out of 3 attempts, seqno 6
-  not connected (0 ms), total 30000 ms connected
+  disconnected
   disconnected at 34000 ms (0 ms ago)
 
 # Back off for 4000 ms.
@@ -537,7 +519,7 @@ timeout
 
 ### t=38000 ###
   in BACKOFF for 4000 ms (4000 ms backoff)
-  disconnected for 4000 ms
+  last connected 14000 ms ago, connected 30000 ms total
 ])
 
 ######################################################################
@@ -595,7 +577,6 @@ timeout
 
 ### t=2000 ###
   in CONNECTING for 1000 ms (0 ms backoff)
-  disconnected for 1000 ms
 run
   should disconnect
 connect-failed
@@ -608,7 +589,6 @@ timeout
 
 ### t=3000 ###
   in BACKOFF for 1000 ms (1000 ms backoff)
-  disconnected for 2000 ms
 run
   should connect
 
@@ -620,7 +600,6 @@ timeout
 
 ### t=4000 ###
   in CONNECTING for 1000 ms (1000 ms backoff)
-  disconnected for 3000 ms
 run
   should disconnect
 connect-failed
@@ -633,7 +612,6 @@ timeout
 
 ### t=6000 ###
   in BACKOFF for 2000 ms (2000 ms backoff)
-  disconnected for 5000 ms
 run
   should connect
 
@@ -644,25 +622,23 @@ advance 500
 
 ### t=6500 ###
   in CONNECTING for 500 ms (2000 ms backoff)
-  disconnected for 5500 ms
 run
 connected
   in ACTIVE for 0 ms (2000 ms backoff)
   created 1000, last received 1000, last connected 6500
   1 successful connections out of 3 attempts, seqno 1
-  connected (0 ms), total 0 ms connected
-  disconnected for 0 ms
+  connected
+  last connected 0 ms ago, connected 0 ms total
 
 # Connection drops after another 250 ms.
 advance 250
 
 ### t=6750 ###
   in ACTIVE for 250 ms (2000 ms backoff)
-  connected (250 ms), total 250 ms connected
 disconnected
   in BACKOFF for 0 ms (4000 ms backoff)
   1 successful connections out of 3 attempts, seqno 2
-  not connected (0 ms), total 250 ms connected
+  disconnected
   disconnected at 6750 ms (0 ms ago)
 run
 
@@ -672,7 +648,7 @@ timeout
 
 ### t=10750 ###
   in BACKOFF for 4000 ms (4000 ms backoff)
-  disconnected for 4000 ms
+  last connected 4250 ms ago, connected 250 ms total
 run
   should connect
 ])
@@ -743,7 +719,6 @@ timeout
 
 ### t=2000 ###
   in CONNECTING for 1000 ms (0 ms backoff)
-  disconnected for 1000 ms
 run
   should disconnect
 connect-failed
@@ -756,7 +731,6 @@ timeout
 
 ### t=3000 ###
   in BACKOFF for 1000 ms (1000 ms backoff)
-  disconnected for 2000 ms
 run
   should connect
 
@@ -768,7 +742,6 @@ timeout
 
 ### t=4000 ###
   in CONNECTING for 1000 ms (1000 ms backoff)
-  disconnected for 3000 ms
 run
   should disconnect
 connect-failed
@@ -781,7 +754,6 @@ timeout
 
 ### t=6000 ###
   in BACKOFF for 2000 ms (2000 ms backoff)
-  disconnected for 5000 ms
 run
   should connect
 
@@ -792,21 +764,19 @@ advance 500
 
 ### t=6500 ###
   in CONNECTING for 500 ms (2000 ms backoff)
-  disconnected for 5500 ms
 run
 connected
   in ACTIVE for 0 ms (2000 ms backoff)
   created 1000, last received 1000, last connected 6500
   1 successful connections out of 3 attempts, seqno 1
-  connected (0 ms), total 0 ms connected
-  disconnected for 0 ms
+  connected
+  last connected 0 ms ago, connected 0 ms total
 
 # Connection receives 3 chunks of data spaced 250 ms apart.
 advance 250
 
 ### t=6750 ###
   in ACTIVE for 250 ms (2000 ms backoff)
-  connected (250 ms), total 250 ms connected
 run
 received
   created 1000, last received 6750, last connected 6500
@@ -814,7 +784,6 @@ advance 250
 
 ### t=7000 ###
   in ACTIVE for 500 ms (2000 ms backoff)
-  connected (500 ms), total 500 ms connected
 run
 received
   created 1000, last received 7000, last connected 6500
@@ -822,7 +791,6 @@ advance 250
 
 ### t=7250 ###
   in ACTIVE for 750 ms (2000 ms backoff)
-  connected (750 ms), total 750 ms connected
 run
 received
   created 1000, last received 7250, last connected 6500
@@ -831,7 +799,7 @@ received
 disconnected
   in BACKOFF for 0 ms (4000 ms backoff)
   1 successful connections out of 3 attempts, seqno 2
-  not connected (0 ms), total 750 ms connected
+  disconnected
   disconnected at 7250 ms (0 ms ago)
 run
 
@@ -841,7 +809,7 @@ timeout
 
 ### t=11250 ###
   in BACKOFF for 4000 ms (4000 ms backoff)
-  disconnected for 4000 ms
+  last connected 4750 ms ago, connected 750 ms total
 run
   should connect
 ])
@@ -912,7 +880,6 @@ timeout
 
 ### t=2000 ###
   in CONNECTING for 1000 ms (0 ms backoff)
-  disconnected for 1000 ms
 run
   should disconnect
 connect-failed
@@ -925,7 +892,6 @@ timeout
 
 ### t=3000 ###
   in BACKOFF for 1000 ms (1000 ms backoff)
-  disconnected for 2000 ms
 run
   should connect
 
@@ -937,7 +903,6 @@ timeout
 
 ### t=4000 ###
   in CONNECTING for 1000 ms (1000 ms backoff)
-  disconnected for 3000 ms
 run
   should disconnect
 connect-failed
@@ -950,7 +915,6 @@ timeout
 
 ### t=6000 ###
   in BACKOFF for 2000 ms (2000 ms backoff)
-  disconnected for 5000 ms
 run
   should connect
 
@@ -961,21 +925,19 @@ advance 500
 
 ### t=6500 ###
   in CONNECTING for 500 ms (2000 ms backoff)
-  disconnected for 5500 ms
 run
 connected
   in ACTIVE for 0 ms (2000 ms backoff)
   created 1000, last received 1000, last connected 6500
   1 successful connections out of 3 attempts, seqno 1
-  connected (0 ms), total 0 ms connected
-  disconnected for 0 ms
+  connected
+  last connected 0 ms ago, connected 0 ms total
 
 # Connection receives 3 chunks of data spaced 2000 ms apart.
 advance 2000
 
 ### t=8500 ###
   in ACTIVE for 2000 ms (2000 ms backoff)
-  connected (2000 ms), total 2000 ms connected
 run
 received
   created 1000, last received 8500, last connected 6500
@@ -983,7 +945,6 @@ advance 2000
 
 ### t=10500 ###
   in ACTIVE for 4000 ms (2000 ms backoff)
-  connected (4000 ms), total 4000 ms connected
 run
 received
   created 1000, last received 10500, last connected 6500
@@ -991,7 +952,6 @@ advance 2000
 
 ### t=12500 ###
   in ACTIVE for 6000 ms (2000 ms backoff)
-  connected (6000 ms), total 6000 ms connected
 run
 received
   created 1000, last received 12500, last connected 6500
@@ -1000,7 +960,7 @@ received
 disconnected
   in BACKOFF for 0 ms (1000 ms backoff)
   1 successful connections out of 3 attempts, seqno 2
-  not connected (0 ms), total 6000 ms connected
+  disconnected
   disconnected at 12500 ms (0 ms ago)
 run
 
@@ -1010,7 +970,7 @@ timeout
 
 ### t=13500 ###
   in BACKOFF for 1000 ms (1000 ms backoff)
-  disconnected for 1000 ms
+  last connected 7000 ms ago, connected 6000 ms total
 run
   should connect
 ])
@@ -1053,7 +1013,6 @@ timeout
 
 ### t=2000 ###
   in BACKOFF for 1000 ms (1000 ms backoff)
-  disconnected for 1000 ms
 
 # Connection fails quickly again.
 run
@@ -1069,7 +1028,6 @@ timeout
 
 ### t=4000 ###
   in BACKOFF for 2000 ms (2000 ms backoff)
-  disconnected for 3000 ms
 ])
 
 ######################################################################
@@ -1102,8 +1060,10 @@ run
   should connect
 connected
   in ACTIVE for 0 ms (0 ms backoff)
+  created 1000, last received 1000, last connected 1000
   1 successful connections out of 1 attempts, seqno 1
-  connected (0 ms), total 0 ms connected
+  connected
+  last connected 0 ms ago, connected 0 ms total
 
 # Send inactivity probe.
 timeout
@@ -1111,7 +1071,6 @@ timeout
 
 ### t=6000 ###
   in ACTIVE for 5000 ms (0 ms backoff)
-  connected (5000 ms), total 5000 ms connected
 run
   should send probe
   in IDLE for 0 ms (0 ms backoff)
@@ -1122,13 +1081,12 @@ timeout
 
 ### t=11000 ###
   in IDLE for 5000 ms (0 ms backoff)
-  connected (10000 ms), total 10000 ms connected
 run
   should disconnect
 disconnected
   in VOID for 0 ms (1000 ms backoff)
   1 successful connections out of 1 attempts, seqno 2
-  not connected (0 ms), total 10000 ms connected
+  disconnected
   disconnected at 11000 ms (0 ms ago)
 ])
 
@@ -1212,7 +1170,6 @@ timeout
 
 ### t=2000 ###
   in BACKOFF for 1000 ms (1000 ms backoff)
-  disconnected for 1000 ms
 run
   should connect
 listening
@@ -1223,15 +1180,14 @@ connected
   in ACTIVE for 0 ms (1000 ms backoff)
   created 1000, last received 1000, last connected 2000
   1 successful connections out of 1 attempts, seqno 1
-  connected (0 ms), total 0 ms connected
-  disconnected for 0 ms
+  connected
+  last connected 0 ms ago, connected 0 ms total
 received
   created 1000, last received 2000, last connected 2000
 advance 1000
 
 ### t=3000 ###
   in ACTIVE for 1000 ms (1000 ms backoff)
-  connected (1000 ms), total 1000 ms connected
 received
   created 1000, last received 3000, last connected 2000
 
@@ -1241,7 +1197,6 @@ timeout
 
 ### t=8000 ###
   in ACTIVE for 6000 ms (1000 ms backoff)
-  connected (6000 ms), total 6000 ms connected
 run
   should send probe
   in IDLE for 0 ms (1000 ms backoff)
@@ -1250,13 +1205,12 @@ timeout
 
 ### t=13000 ###
   in IDLE for 5000 ms (1000 ms backoff)
-  connected (11000 ms), total 11000 ms connected
 run
   should disconnect
 disconnected
   in BACKOFF for 0 ms (0 ms backoff)
   1 successful connections out of 1 attempts, seqno 2
-  not connected (0 ms), total 11000 ms connected
+  disconnected
   disconnected at 13000 ms (0 ms ago)
 
 # Start listening again.