Improve connection cleanup in replication connection pool

Previously when an owner process crashed before it had a chance to release the
worker to the pool, the worker entry was simply deleted. In some cases that
was ok because ibrowse's inactivity timeout would kick in and connection would
stop itself. In other cases, as observed in practice with _changes feed
connection over TLS protocol, inactivity timeout would never fire, so these
deleted connections would slowly accumulate leaking memory and filling the
process table. TLS connection would keep an associated session
open as well making things even worse.

To prevent the connection leak, explicitly unlink and kill the worker.

1 files changed, 5 insertions, 1 deletions

diff --git a/src/couch_replicator/src/couch_replicator_connection.erl b/src/couch_replicator/src/couch_replicator_connection.erl
index 9c6472360..f3e4a864d 100644
--- a/src/couch_replicator/src/couch_replicator_connection.erl
+++ b/src/couch_replicator/src/couch_replicator_connection.erl
@@ -156,7 +156,11 @@ handle_cast({connection_close_interval, V}, State) ->
 handle_info({'DOWN', Ref, process, _Pid, _Reason}, State) ->
     couch_stats:increment_counter([couch_replicator, connection,
         owner_crashes]),
-    ets:match_delete(?MODULE, #connection{mref=Ref, _='_'}),
+    Conns = ets:match_object(?MODULE, #connection{mref = Ref, _='_'}),
+    lists:foreach(fun(Conn) ->
+        couch_stats:increment_counter([couch_replicator, connection, closes]),
+        delete_worker(Conn)
+    end, Conns),
     {noreply, State};
 
 % worker crashed