QPID-4081: [Java Tests] Applied patch supplied by Alex Rudyy

git-svn-id: https://svn.apache.org/repos/asf/qpid/trunk@1352838 13f79535-47bb-0310-9956-ffa450edef68

4 files changed, 112 insertions, 2 deletions

diff --git a/qpid/java/systests/src/main/java/org/apache/qpid/test/utils/BrokerHolder.java b/qpid/java/systests/src/main/java/org/apache/qpid/test/utils/BrokerHolder.java
index 66b3fe0c6a..3af57327d2 100644
--- a/qpid/java/systests/src/main/java/org/apache/qpid/test/utils/BrokerHolder.java
+++ b/qpid/java/systests/src/main/java/org/apache/qpid/test/utils/BrokerHolder.java
@@ -25,4 +25,5 @@ public interface BrokerHolder
     String getWorkingDirectory();
     void shutdown();
     void kill();
+    String dumpThreads();
 }
diff --git a/qpid/java/systests/src/main/java/org/apache/qpid/test/utils/InternalBrokerHolder.java b/qpid/java/systests/src/main/java/org/apache/qpid/test/utils/InternalBrokerHolder.java
index 003fabed20..a71a4ef517 100644
--- a/qpid/java/systests/src/main/java/org/apache/qpid/test/utils/InternalBrokerHolder.java
+++ b/qpid/java/systests/src/main/java/org/apache/qpid/test/utils/InternalBrokerHolder.java
@@ -20,6 +20,9 @@
  */
 package org.apache.qpid.test.utils;
 
+import java.lang.management.ManagementFactory;
+import java.lang.management.ThreadInfo;
+import java.lang.management.ThreadMXBean;
 import java.util.Set;
 
 import org.apache.log4j.Logger;
@@ -69,12 +72,44 @@ public class InternalBrokerHolder implements BrokerHolder
     {
         // Can't kill a internal broker as we would also kill ourselves as we share the same JVM.
         shutdown();
-
-        waitUntilPortsAreFree();
     }
 
     private void waitUntilPortsAreFree()
     {
         new PortHelper().waitUntilPortsAreFree(_portsUsedByBroker);
     }
+
+    @Override
+    public String dumpThreads()
+    {
+        ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
+        ThreadInfo[] threadInfos = threadMXBean.dumpAllThreads(true, true);
+        StringBuilder dump = new StringBuilder();
+        dump.append(String.format("%n"));
+        for (ThreadInfo threadInfo : threadInfos)
+        {
+            dump.append(threadInfo);
+        }
+
+        long[] deadLocks = threadMXBean.findDeadlockedThreads();
+        if (deadLocks != null && deadLocks.length > 0)
+        {
+            ThreadInfo[] deadlockedThreads = threadMXBean.getThreadInfo(deadLocks);
+            dump.append(String.format("%n"));
+            dump.append("Deadlock is detected!");
+            dump.append(String.format("%n"));
+            for (ThreadInfo threadInfo : deadlockedThreads)
+            {
+                dump.append(threadInfo);
+            }
+        }
+        return dump.toString();
+    }
+
+    @Override
+    public String toString()
+    {
+        return "InternalBrokerHolder [_portsUsedByBroker=" + _portsUsedByBroker + "]";
+    }
+
 }
diff --git a/qpid/java/systests/src/main/java/org/apache/qpid/test/utils/QpidBrokerTestCase.java b/qpid/java/systests/src/main/java/org/apache/qpid/test/utils/QpidBrokerTestCase.java
index 3338e8b3b9..c5077ccb64 100644
--- a/qpid/java/systests/src/main/java/org/apache/qpid/test/utils/QpidBrokerTestCase.java
+++ b/qpid/java/systests/src/main/java/org/apache/qpid/test/utils/QpidBrokerTestCase.java
@@ -713,11 +713,55 @@ public class QpidBrokerTestCase extends QpidTestCase
 
     public void stopAllBrokers()
     {
+        boolean exceptionOccured = false;
         Set<Integer> runningBrokerPorts = new HashSet<Integer>(getBrokerPortNumbers());
         for (int brokerPortNumber : runningBrokerPorts)
         {
+            if (!stopBrokerSafely(brokerPortNumber))
+            {
+                exceptionOccured = true;
+            }
+        }
+        if (exceptionOccured)
+        {
+            throw new RuntimeException("Exception occured on stopping of test broker. Please, examine logs for details");
+        }
+    }
+
+    protected boolean stopBrokerSafely(int brokerPortNumber)
+    {
+        boolean success = true;
+        BrokerHolder broker = _brokers.get(brokerPortNumber);
+        try
+        {
             stopBroker(brokerPortNumber);
         }
+        catch(Exception e)
+        {
+            success = false;
+            _logger.error("Failed to stop broker " + broker + " at port " + brokerPortNumber, e);
+            if (broker != null)
+            {
+                // save the thread dump in case of dead locks
+                try
+                {
+                    _logger.error("Broker " + broker + " thread dump:" + broker.dumpThreads());
+                }
+                finally
+                {
+                    // try to kill broker
+                    try
+                    {
+                        broker.kill();
+                    }
+                    catch(Exception killException)
+                    {
+                        // ignore
+                    }
+                }
+            }
+        }
+        return success;
     }
 
     public void stopBroker(int port)
diff --git a/qpid/java/systests/src/main/java/org/apache/qpid/test/utils/SpawnedBrokerHolder.java b/qpid/java/systests/src/main/java/org/apache/qpid/test/utils/SpawnedBrokerHolder.java
index ecf90ba431..bce97a574a 100644
--- a/qpid/java/systests/src/main/java/org/apache/qpid/test/utils/SpawnedBrokerHolder.java
+++ b/qpid/java/systests/src/main/java/org/apache/qpid/test/utils/SpawnedBrokerHolder.java
@@ -20,7 +20,9 @@
  */
 package org.apache.qpid.test.utils;
 
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.util.Set;
 
 import org.apache.log4j.Logger;
@@ -158,4 +160,32 @@ public class SpawnedBrokerHolder implements BrokerHolder
         new PortHelper().waitUntilPortsAreFree(_portsUsedByBroker);
     }
 
+    @Override
+    public String dumpThreads()
+    {
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        try
+        {
+            Process process = Runtime.getRuntime().exec("jstack " + _pid);
+            InputStream is = process.getInputStream();
+            byte[] buffer = new byte[1024];
+            int length = -1;
+            while ((length = is.read(buffer)) != -1)
+            {
+                baos.write(buffer, 0, length);
+            }
+         }
+        catch (Exception e)
+        {
+            LOGGER.error("Error whilst collecting thread dump for " + _pid, e);
+        }
+        return new String(baos.toByteArray());
+    }
+
+    @Override
+    public String toString()
+    {
+        return "SpawnedBrokerHolder [_pid=" + _pid + ", _portsUsedByBroker="
+                + _portsUsedByBroker + "]";
+    }
 }