ChangeLogTag: Mon Oct 13 11:37:03 2003 Steve Totten <totten_s@ociweb.com>

6 files changed, 572 insertions, 39 deletions

diff --git a/TAO/ChangeLog b/TAO/ChangeLog
index 5c283fab095..e36dfeda5ed 100644
--- a/TAO/ChangeLog
+++ b/TAO/ChangeLog
@@ -1,3 +1,20 @@
+Mon Oct 13 11:37:03 2003  Steve Totten  <totten_s@ociweb.com>
+
+        * orbsvcs/orbsvcs/FT_ReplicationManager.idl:
+          Added a TAO-specific shutdown() operation to the Replication
+          Manager's interface.
+
+        * orbsvcs/FT_ReplicationManager/FT_FaultConsumer.cpp:
+        * orbsvcs/FT_ReplicationManager/FT_ReplicationManager.cpp:
+        * orbsvcs/FT_ReplicationManager/FT_ReplicationManager.h:
+          Continued working on integration of FT_FaultConsumer with
+          the Replication Manager.  Made several improvements to
+          initializtion and finalization code.  Implemented
+          TAO-specific shutdown() operation on Replication Manager.
+
+        * orbsvcs/tests/FT_App/run_test_replication_mgr.pl:
+          Added this unit test (still in work).
+
 Fri Oct 10 17:19:03 2003  Steve Totten  <totten_s@ociweb.com>
 
         * orbsvcs/FT_ReplicationManager/FT_FaultConsumer.cpp:
@@ -5,7 +22,7 @@ Fri Oct 10 17:19:03 2003  Steve Totten  <totten_s@ociweb.com>
         * orbsvcs/FT_ReplicationManager/FT_ReplicationManager.h:
         * orbsvcs/FT_ReplicationManager/FT_ReplicationManager.mpc:
           More clean up of Replication Manager code.
-          Integrated FT_FaultConsumer into Replication Manager.
+          Initial integration of FT_FaultConsumer into Replication Manager.
 
 Fri Oct 10 13:58:04 2003  Steve Totten  <totten_s@ociweb.com>
 
diff --git a/TAO/orbsvcs/FT_ReplicationManager/FT_FaultConsumer.cpp b/TAO/orbsvcs/FT_ReplicationManager/FT_FaultConsumer.cpp
index 5094b83220e..826af851984 100755
--- a/TAO/orbsvcs/FT_ReplicationManager/FT_FaultConsumer.cpp
+++ b/TAO/orbsvcs/FT_ReplicationManager/FT_FaultConsumer.cpp
@@ -58,6 +58,17 @@ int TAO::FT_FaultConsumer::init (
   FT::ReplicationManager_ptr replication_manager
   ACE_ENV_ARG_DECL)
 {
+
+#if (TAO_DEBUG_LEVEL_NEEDED == 1)
+  if (TAO_debug_level > 1)
+#endif /* (TAO_DEBUG_LEVEL_NEEDED == 1) */
+  {
+    ACE_DEBUG ((LM_DEBUG,
+      ACE_TEXT (
+        "Enter TAO::FT_FaultConsumer::init.\n")
+    ));
+  }
+
   ACE_ASSERT (!CORBA::is_nil (poa));
   ACE_ASSERT (!CORBA::is_nil (fault_notifier));
   ACE_ASSERT (!CORBA::is_nil (replication_manager));
@@ -95,6 +106,16 @@ int TAO::FT_FaultConsumer::init (
     this->consumer_ref_.in(), filter ACE_ENV_ARG_PARAMETER);
   ACE_CHECK_RETURN (-1);
 
+#if (TAO_DEBUG_LEVEL_NEEDED == 1)
+  if (TAO_debug_level > 1)
+#endif /* (TAO_DEBUG_LEVEL_NEEDED == 1) */
+  {
+    ACE_DEBUG ((LM_DEBUG,
+      ACE_TEXT (
+        "Leave TAO::FT_FaultConsumer::init.\n")
+    ));
+  }
+
   // Success.
   return 0;
 }
@@ -106,21 +127,54 @@ int TAO::FT_FaultConsumer::init (
 */
 int TAO::FT_FaultConsumer::fini (ACE_ENV_SINGLE_ARG_DECL)
 {
+
+#if (TAO_DEBUG_LEVEL_NEEDED == 1)
+  if (TAO_debug_level > 1)
+#endif /* (TAO_DEBUG_LEVEL_NEEDED == 1) */
+  {
+    ACE_DEBUG ((LM_DEBUG,
+      ACE_TEXT ("Enter TAO::FT_FaultConsumer::fini.\n")
+    ));
+  }
+
   // Disconnect from the FaultNotifier.
   // Swallow any exception.
   ACE_TRY_NEW_ENV
   {
     if (!CORBA::is_nil (this->fault_notifier_.in()))
     {
+
+#if (TAO_DEBUG_LEVEL_NEEDED == 1)
+      if (TAO_debug_level > 1)
+#endif /* (TAO_DEBUG_LEVEL_NEEDED == 1) */
+      {
+        ACE_DEBUG ((LM_DEBUG,
+          ACE_TEXT (
+            "TAO::FT_FaultConsumer::fini: "
+            "Disconnecting consumer from FaultNotifier.\n")
+        ));
+      }
+
       this->fault_notifier_->disconnect_consumer (
         this->consumer_id_ ACE_ENV_ARG_PARAMETER);
       ACE_TRY_CHECK;
-    }
 
-    // Deactivate ourself from the POA.
-    this->poa_->deactivate_object (
-      this->object_id_.in() ACE_ENV_ARG_PARAMETER);
-    ACE_TRY_CHECK;
+#if (TAO_DEBUG_LEVEL_NEEDED == 1)
+      if (TAO_debug_level > 1)
+#endif /* (TAO_DEBUG_LEVEL_NEEDED == 1) */
+      {
+        ACE_DEBUG ((LM_DEBUG,
+          ACE_TEXT (
+            "TAO::FT_FaultConsumer::fini: "
+            "Deactivating from POA.\n")
+        ));
+      }
+
+      // Deactivate ourself from the POA.
+      this->poa_->deactivate_object (
+        this->object_id_.in() ACE_ENV_ARG_PARAMETER);
+      ACE_TRY_CHECK;
+    }
   }
   ACE_CATCHANY
   {
@@ -133,8 +187,28 @@ int TAO::FT_FaultConsumer::fini (ACE_ENV_SINGLE_ARG_DECL)
   ACE_ENDTRY;
   ACE_CHECK;
 
+#if (TAO_DEBUG_LEVEL_NEEDED == 1)
+  if (TAO_debug_level > 1)
+#endif /* (TAO_DEBUG_LEVEL_NEEDED == 1) */
+  {
+    ACE_DEBUG ((LM_DEBUG,
+      ACE_TEXT (
+        "TAO::FT_FaultConsumer::fini: "
+        "Setting our object reference to nil.\n")
+    ));
+  }
+
   this->consumer_ref_ = CosNotifyComm::StructuredPushConsumer::_nil ();
 
+#if (TAO_DEBUG_LEVEL_NEEDED == 1)
+  if (TAO_debug_level > 1)
+#endif /* (TAO_DEBUG_LEVEL_NEEDED == 1) */
+  {
+    ACE_DEBUG ((LM_DEBUG,
+      ACE_TEXT ("Leave TAO::FT_FaultConsumer::fini.\n")
+    ));
+  }
+
   // Success.
   return 0;
 }
diff --git a/TAO/orbsvcs/FT_ReplicationManager/FT_ReplicationManager.cpp b/TAO/orbsvcs/FT_ReplicationManager/FT_ReplicationManager.cpp
index 29612504cc9..44090424748 100644
--- a/TAO/orbsvcs/FT_ReplicationManager/FT_ReplicationManager.cpp
+++ b/TAO/orbsvcs/FT_ReplicationManager/FT_ReplicationManager.cpp
@@ -110,6 +110,16 @@ const char * TAO::FT_ReplicationManager::identity () const
 
 int TAO::FT_ReplicationManager::init (CORBA::ORB_ptr orb ACE_ENV_ARG_DECL)
 {
+#if (TAO_DEBUG_LEVEL_NEEDED == 1)
+  if (TAO_debug_level > 1)
+#endif /* (TAO_DEBUG_LEVEL_NEEDED == 1) */
+  {
+    ACE_DEBUG ((LM_DEBUG,
+      ACE_TEXT (
+        "Enter TAO::FT_ReplicationManager::init.\n")
+    ));
+  }
+
   int result = 0;
   this->orb_ = CORBA::ORB::_duplicate (orb);
 
@@ -168,6 +178,40 @@ int TAO::FT_ReplicationManager::init (CORBA::ORB_ptr orb ACE_ENV_ARG_DECL)
     this_obj.in() ACE_ENV_ARG_PARAMETER);
   ACE_CHECK_RETURN (-1);
 
+  // If we were given an initial IOR string for a Fault Notifier on the
+  // command line, convert it to an IOR.
+  if (this->fault_notifier_ior_file_ != 0)
+  {
+    CORBA::String_var notifierIOR;
+    if (this->readIORFile (this->fault_notifier_ior_file_, notifierIOR))
+    {
+      CORBA::Object_var notifier_obj = this->orb_->string_to_object (
+        notifierIOR.in() ACE_ENV_ARG_PARAMETER);
+      ACE_CHECK_RETURN (-1);
+      FT::FaultNotifier_var notifier = FT::FaultNotifier::_narrow (
+        notifier_obj.in() ACE_ENV_ARG_PARAMETER);
+      ACE_CHECK_RETURN (-1);
+      this->register_fault_notifier_i (notifier.in() ACE_ENV_ARG_PARAMETER);
+      ACE_CHECK_RETURN (-1);
+    }
+    else
+    {
+      ACE_ERROR_RETURN ((LM_ERROR,
+        ACE_TEXT (
+          "TAO::FT_ReplicationManager::init: "
+          "Could not read %s.\n"),
+        this->fault_notifier_ior_file_),
+        -1);
+    }
+  }
+
+  // Activate the RootPOA.
+  PortableServer::POAManager_var poa_mgr =
+    this->poa_->the_POAManager (ACE_ENV_ARG_PARAMETER);
+  ACE_CHECK_RETURN (-1);
+  poa_mgr->activate (ACE_ENV_ARG_PARAMETER);
+  ACE_CHECK_RETURN (-1);
+
   // Publish our IOR, either to a file or the Naming Service.
   if (this->ior_output_file_ != 0)
   {
@@ -211,37 +255,14 @@ int TAO::FT_ReplicationManager::init (CORBA::ORB_ptr orb ACE_ENV_ARG_DECL)
     ACE_CHECK_RETURN (-1);
   }
 
-  // If we were given an initial IOR string for a Fault Notifier on the
-  // command line, convert it to an IOR.
-  if (this->fault_notifier_ior_file_ != 0)
+#if (TAO_DEBUG_LEVEL_NEEDED == 1)
+  if (TAO_debug_level > 1)
+#endif /* (TAO_DEBUG_LEVEL_NEEDED == 1) */
   {
-    CORBA::String_var notifierIOR;
-    if (this->readIORFile (this->fault_notifier_ior_file_, notifierIOR))
-    {
-      CORBA::Object_var notifier_obj = this->orb_->string_to_object (
-        notifierIOR.in() ACE_ENV_ARG_PARAMETER);
-      ACE_CHECK_RETURN (-1);
-      this->fault_notifier_ = FT::FaultNotifier::_narrow (
-        notifier_obj.in() ACE_ENV_ARG_PARAMETER);
-      ACE_CHECK_RETURN (-1);
-      if (CORBA::is_nil (this->fault_notifier_.in()))
-      {
-        ACE_ERROR_RETURN ((LM_ERROR,
-          ACE_TEXT (
-            "TAO::FT_ReplicationManager::init: "
-            "Bad Fault Notifier object reference provided on command line.\n")),
-          -1);
-      }
-    }
-    else
-    {
-      ACE_ERROR_RETURN ((LM_ERROR,
-        ACE_TEXT (
-          "TAO::FT_ReplicationManager::init: "
-          "Could not read %s.\n"),
-        this->fault_notifier_ior_file_),
-        -1);
-    }
+    ACE_DEBUG ((LM_DEBUG,
+      ACE_TEXT (
+        "Leave TAO::FT_ReplicationManager::init.\n")
+    ));
   }
 
   return result;
@@ -257,6 +278,11 @@ int TAO::FT_ReplicationManager::idle (int & result)
 
 int TAO::FT_ReplicationManager::fini (ACE_ENV_SINGLE_ARG_DECL)
 {
+  int result = 0;
+
+  //@@ Should we ever return -1 from this function, or should we
+  // catch and swallow all exceptions?
+
   if (this->ior_output_file_ != 0)
   {
     ACE_OS::unlink (this->ior_output_file_);
@@ -265,9 +291,14 @@ int TAO::FT_ReplicationManager::fini (ACE_ENV_SINGLE_ARG_DECL)
   if (this->ns_name_ != 0)
   {
     this->naming_context_->unbind (this->this_name_ ACE_ENV_ARG_PARAMETER);
+    ACE_CHECK_RETURN (-1);
     this->ns_name_ = 0;
   }
-  return 0;
+
+  result = this->fault_consumer_.fini (ACE_ENV_SINGLE_ARG_PARAMETER);
+  ACE_CHECK_RETURN (-1);
+
+  return result;
 }
 
 ////////////////////////////////////////////
@@ -305,6 +336,51 @@ TAO::FT_ReplicationManager::register_fault_notifier (
   ACE_ENV_ARG_DECL)
   ACE_THROW_SPEC ((CORBA::SystemException))
 {
+  this->register_fault_notifier_i (fault_notifier ACE_ENV_ARG_DECL);
+}
+
+void
+TAO::FT_ReplicationManager::register_fault_notifier_i (
+  FT::FaultNotifier_ptr fault_notifier
+  ACE_ENV_ARG_DECL)
+  ACE_THROW_SPEC ((CORBA::SystemException))
+{
+
+#if (TAO_DEBUG_LEVEL_NEEDED == 1)
+  if (TAO_debug_level > 1)
+#endif /* (TAO_DEBUG_LEVEL_NEEDED == 1) */
+  {
+    ACE_DEBUG ((LM_DEBUG,
+      ACE_TEXT (
+        "Enter TAO::FT_ReplicationManager::register_fault_notifier_i.\n")
+    ));
+  }
+
+  if (CORBA::is_nil (fault_notifier))
+  {
+    ACE_ERROR ((LM_ERROR,
+      ACE_TEXT (
+        "TAO::FT_ReplicationManager::register_fault_notifier_i: "
+        "Bad Fault Notifier object reference provided.\n")
+    ));
+    ACE_THROW (CORBA::BAD_PARAM (
+      CORBA::SystemException::_tao_minor_code (
+        TAO_DEFAULT_MINOR_CODE,
+        EINVAL),
+      CORBA::COMPLETED_NO));
+  }
+
+#if (TAO_DEBUG_LEVEL_NEEDED == 1)
+  if (TAO_debug_level > 1)
+#endif /* (TAO_DEBUG_LEVEL_NEEDED == 1) */
+  {
+    ACE_DEBUG ((LM_DEBUG,
+      ACE_TEXT (
+        "TAO::FT_ReplicationManager::register_fault_notifier_i: "
+        "Duplicate FaultNotifier object reference.\n")
+    ));
+  }
+
   // Cache new Fault Notifier object reference.
   this->fault_notifier_ = FT::FaultNotifier::_duplicate (fault_notifier);
 
@@ -313,9 +389,32 @@ TAO::FT_ReplicationManager::register_fault_notifier (
   int result = 0;
   ACE_TRY_NEW_ENV
   {
+
+#if (TAO_DEBUG_LEVEL_NEEDED == 1)
+    if (TAO_debug_level > 1)
+#endif /* (TAO_DEBUG_LEVEL_NEEDED == 1) */
+    {
+      ACE_DEBUG ((LM_DEBUG,
+        ACE_TEXT (
+          "TAO::FT_ReplicationManager::register_fault_notifier_i: "
+          "Calling fault_consumer_.fini().\n")
+      ));
+    }
+
     result = this->fault_consumer_.fini (ACE_ENV_SINGLE_ARG_PARAMETER);
     ACE_TRY_CHECK;
 
+#if (TAO_DEBUG_LEVEL_NEEDED == 1)
+    if (TAO_debug_level > 1)
+#endif /* (TAO_DEBUG_LEVEL_NEEDED == 1) */
+    {
+      ACE_DEBUG ((LM_DEBUG,
+        ACE_TEXT (
+          "TAO::FT_ReplicationManager::register_fault_notifier_i: "
+          "Calling fault_consumer_.init().\n")
+      ));
+    }
+
     result = this->fault_consumer_.init (
       this->poa_.in(),
       this->fault_notifier_.in(),
@@ -327,7 +426,7 @@ TAO::FT_ReplicationManager::register_fault_notifier (
   {
     ACE_PRINT_EXCEPTION (ACE_ANY_EXCEPTION,
       ACE_TEXT (
-        "TAO::FT_ReplicationManager::register_fault_notifier: "
+        "TAO::FT_ReplicationManager::register_fault_notifier_i: "
         "Error reinitializing FT_FaultConsumer.\n")
     );
   }
@@ -337,7 +436,7 @@ TAO::FT_ReplicationManager::register_fault_notifier (
   {
     ACE_ERROR((LM_ERROR,
       ACE_TEXT (
-        "TAO::FT_ReplicationManager::register_fault_notifier: "
+        "TAO::FT_ReplicationManager::register_fault_notifier_i: "
         "Could not re-initialize FT_FaultConsumer.\n")
     ));
 
@@ -347,6 +446,17 @@ TAO::FT_ReplicationManager::register_fault_notifier (
         EINVAL),
       CORBA::COMPLETED_NO));
   }
+
+#if (TAO_DEBUG_LEVEL_NEEDED == 1)
+  if (TAO_debug_level > 1)
+#endif /* (TAO_DEBUG_LEVEL_NEEDED == 1) */
+  {
+    ACE_DEBUG ((LM_DEBUG,
+      ACE_TEXT (
+        "Leave TAO::FT_ReplicationManager::register_fault_notifier_i.\n")
+    ));
+  }
+
 }
 
 
@@ -359,6 +469,25 @@ TAO::FT_ReplicationManager::get_fault_notifier (
   return FT::FaultNotifier::_duplicate (this->fault_notifier_.in());
 }
 
+/// TAO-specific shutdown operation.
+void TAO::FT_ReplicationManager::shutdown (
+  ACE_ENV_SINGLE_ARG_DECL)
+  ACE_THROW_SPEC ((CORBA::SystemException))
+{
+  this->shutdown_i (ACE_ENV_SINGLE_ARG_PARAMETER);
+}
+
+void TAO::FT_ReplicationManager::shutdown_i (
+  ACE_ENV_SINGLE_ARG_DECL)
+  ACE_THROW_SPEC ((CORBA::SystemException))
+{
+  int result = 0;
+  result = this->fini (ACE_ENV_SINGLE_ARG_PARAMETER);
+  ACE_CHECK;
+
+  this->orb_->shutdown (0 ACE_ENV_SINGLE_ARG_PARAMETER);
+}
+
 //////////////////////////////////////////////////////
 // PortableGroup::PropertyManager methods
 
diff --git a/TAO/orbsvcs/FT_ReplicationManager/FT_ReplicationManager.h b/TAO/orbsvcs/FT_ReplicationManager/FT_ReplicationManager.h
index c739145682f..07fb3e51a48 100644
--- a/TAO/orbsvcs/FT_ReplicationManager/FT_ReplicationManager.h
+++ b/TAO/orbsvcs/FT_ReplicationManager/FT_ReplicationManager.h
@@ -117,6 +117,23 @@ namespace TAO
     /// Read and IOR from a file.
     int readIORFile (const char * filename, CORBA::String_var & ior);
 
+    /// Registers the Fault Notifier with the Replication Manager.
+    void register_fault_notifier_i (
+        FT::FaultNotifier_ptr fault_notifier
+        ACE_ENV_ARG_DECL_WITH_DEFAULTS
+      )
+      ACE_THROW_SPEC ((
+        CORBA::SystemException
+      ));
+
+    /// Implementation of TAO-specific shutdown operation.
+    void shutdown_i (
+        ACE_ENV_SINGLE_ARG_DECL_WITH_DEFAULTS
+      )
+      ACE_THROW_SPEC ((
+        CORBA::SystemException
+      ));
+
   public:
 
     /**
@@ -144,6 +161,14 @@ namespace TAO
         , FT::InterfaceNotFound
       ));
 
+    /// TAO-specific shutdown operation.
+    virtual void shutdown (
+        ACE_ENV_SINGLE_ARG_DECL_WITH_DEFAULTS
+      )
+      ACE_THROW_SPEC ((
+        CORBA::SystemException
+      ));
+
     //@}
 
     /**
diff --git a/TAO/orbsvcs/orbsvcs/FT_ReplicationManager.idl b/TAO/orbsvcs/orbsvcs/FT_ReplicationManager.idl
index 14e6a31450d..12fa0d389f7 100644
--- a/TAO/orbsvcs/orbsvcs/FT_ReplicationManager.idl
+++ b/TAO/orbsvcs/orbsvcs/FT_ReplicationManager.idl
@@ -49,6 +49,10 @@ module FT
     */
     ObjectGroup create_test_iogr ();
  
+    /**
+    * TAO-specific shutdown operation.
+    */
+    void oneway shutdown ();
   };
 };
 
diff --git a/TAO/orbsvcs/tests/FT_App/run_test_replication_mgr.pl b/TAO/orbsvcs/tests/FT_App/run_test_replication_mgr.pl
new file mode 100755
index 00000000000..976b89c067d
--- /dev/null
+++ b/TAO/orbsvcs/tests/FT_App/run_test_replication_mgr.pl
@@ -0,0 +1,284 @@
+eval '(exit $?0)' && eval 'exec perl -S $0 ${1+"$@"}'
+     & eval 'exec perl -S $0 $argv:q'
+    if 0;
+
+# $Id$
+# -*- perl -*-
+
+# Purpose:
+#       To test the FT_ReplicationManager
+#
+# Process being tested:
+#       FT_ReplicationManager
+#           implements the FT::ReplicationManager and
+#           CosNotifyComm::StructuredPushConsumer interfaces.
+# Processes used in test:
+#       FT_Replica
+#           implements TestReplica interface.
+#           implements PullMonitorable.
+#       Fault_Detector
+#         implements FaultDetectorFactory interface
+#         implements PullMonitorable interface
+#       FT_Client
+#           client for TestReplica interface.
+#           client for PullMonitorable.
+#       Fault_Notifier
+#           implements the FaultNotifier interface.
+#           propagates fault reports to the FaultConsumer.
+#
+# Test Scenario (***Test: marks behavior being tested):
+#   Phase 1:
+#     Start two FT_Replicas
+#       FT_Replicas write TestReplica IORs (FR#1 and FR#2) to files
+#     Start the Fault_Detector
+#       Fault_Detector writes its IOR (FDF) to a file
+#     Start the Fault_Notifier
+#       Fault_Notifier writes its IOR (FN) to a file.
+#     Start the Fault_Consumer
+#       Subscribes to the Fault_Notifier for fault reports.
+#   Phase 2:
+#     Wait for IORs: FR#1, FR#2, FDF, and FN
+#     Start the StubAnalyzer giving it IORS: FR#1, FR#2 FDF, FN
+#      StubAnalyzer calls FDF to create a FaultDetector
+#       for each Replica.
+#      StubAnalyzer subscribes to Fault_Notifier
+#      StubAnalyzer writes dummy message(READY) to a file.
+#   Phase 3:
+#     Wait for READY
+#     Start FT_Client giving it IORS: FR#1 and FR#2. [1]
+#       FT_Client interacts with FR#1.
+#       FT_Client asks FR#1 to fault.  It does so.
+#       FT_Client notices fault and switches to FR#2. [1]
+#       FD#1 notices fault and notifies Fault_Notifier
+#       FD#1 terminates
+#       ***Test: Fault_Notifier forwards notification to StubAnalyzer
+#       StubAnalyzer prints notification.
+#       FT_Client interacts with FR#2.
+#       FT_Client asks FR#2 to shut down.
+#       FT_Client shuts down.
+#       FD#2 notices FR2 is gone, interprets this
+#         as a fault, and sends notification to Fault_Notifier
+#       FD#2 terminates.
+#       ***Test: Fault_Notifier forwards notification to StubAnalyzer
+#       StubAnalyzer prints notification.
+#     Phase 4: shutting down.
+#       All FaultDetectors have terminated so the FaultDetectorFactory
+#        honors the "quit-on-idle" option on it's command line and exits.
+#       StubAnalyzer compares # fault notifications to # replicas. When
+#        they match, it "knows" that the test is over, so it shuts down.
+#        As it does so, it disconnects its fault consumers from the FaultNotifier.
+#       FaultNotifier notices the last fault consumer disconnecting and exits because
+#        the "quit-on-idle" option was specified on the command line.
+#     Phase 5: housekeeping
+#       Wait for all processes to terminate.
+#       Check termination status.
+#       Delete temp files.
+#
+# [1] Client mediated fault tolerance.  These points will
+#     change when IOGR support is available.
+use lib '../../../../bin';
+#use lib '$ENV{ACE_ROOT}/bin';
+use PerlACE::Run_Test;
+
+########################
+#command line options
+#set defaults:
+my($verbose) = 0;         # 1: report perl actions before executing them
+my($debug_builds) = 0;    # 0: use exes from Release directories
+my($simulated) = 1;       # 1: use "client simulated" fault tolerance
+
+foreach $i (@ARGV) {
+  if ($i eq "--debug_build")
+  {
+    $debug_builds = 1;
+  }
+  elsif ($i eq "--no_simulate")  # reverse this once we have FT ORB support
+  {
+    $simulated = 0;
+  }
+  elsif ($i eq "-v")
+  {
+    $verbose += 1;
+  }
+}
+
+
+my($build_directory) = "/Release";
+if ( $debug_builds ) {
+  $build_directory = "";
+}
+
+if ( $verbose > 1) {
+  print "verbose: $verbose\n";
+  print "debug_builds: $debug_builds -> $build_directory\n";
+  print "simulated: $simulated\n";
+}
+
+
+#define temp files
+my($factory1_ior) = PerlACE::LocalFile ("factory1.ior");
+my($factory2_ior) = PerlACE::LocalFile ("factory2.ior");
+my($replica1_ior) = PerlACE::LocalFile ("replica1.ior");
+my($replica2_ior) = PerlACE::LocalFile ("replica2.ior");
+my($detector_ior) = PerlACE::LocalFile ("detector.ior");
+my($notifier_ior) = PerlACE::LocalFile ("notifier.ior");
+my($replmgr_ior) = PerlACE::LocalFile ("replmgr.ior");
+my($ready_file) = PerlACE::LocalFile ("ready.file");
+my($client_data) = PerlACE::LocalFile ("persistent.dat");
+
+#discard junk from previous tests
+unlink $factory1_ior;
+unlink $factory2_ior;
+unlink $replica1_ior;
+unlink $replica2_ior;
+unlink $detector_ior;
+unlink $notifier_ior;
+unlink $ready_file;
+unlink $client_data;
+
+my($status) = 0;
+
+my($REP1) = new PerlACE::Process (".$build_directory/ft_replica", "-o $factory1_ior -t $replica1_ior -l loc1 -i type1 -q");
+my($REP2) = new PerlACE::Process (".$build_directory/ft_replica", "-o $factory2_ior -t $replica2_ior -l loc2 -i type1 -q");
+my($DET) = new PerlACE::Process ("$ENV{'TAO_ROOT'}/orbsvcs/Fault_Detector$build_directory/Fault_Detector", "-o $detector_ior -q");
+my($NOT) = new PerlACE::Process ("$ENV{'TAO_ROOT'}/orbsvcs/Fault_Notifier$build_directory/Fault_Notifier", "-o $notifier_ior -v -q");
+my($CONS) = new PerlACE::Process (".$build_directory/ft_fault_consumer", "-o $ready_file -n $notifier_ior -q -d $detector_ior -r $replica1_ior,$replica2_ior");
+my($REPLM) = new PerlACE::Process ("$ENV{'TAO_ROOT'}/orbsvcs/FT_ReplicationManager$build_directory/FT_ReplicationManager", "-o $replmgr_ior -f $notifier_ior");
+
+my($CL);
+if (simulated) {
+  $CL = new PerlACE::Process (".$build_directory/ft_client", "-f $replica1_ior,$replica2_ior -c testscript");
+}else{
+  #todo figure out how to get iogr
+  $CL = new PerlACE::Process (".$build_directory/ft_client", "-f $replica1_iogr -c testscript");
+}
+
+print "TEST: starting replica1 " . $REP1->CommandLine . "\n" if ($verbose);
+$REP1->Spawn ();
+
+print "TEST: waiting for replica 1's IOR\n" if ($verbose);
+if (PerlACE::waitforfile_timed ($replica1_ior, 5) == -1) {
+    print STDERR "ERROR: cannot find file <$replica1_ior>\n";
+    $REP1->Kill (); $REP1->TimedWait (1);
+    exit 1;
+}
+
+print "\nTEST: starting replica2 " . $REP2->CommandLine . "\n" if ($verbose);
+$REP2->Spawn ();
+
+print "TEST: waiting for replica 2's IOR\n" if ($verbose);
+if (PerlACE::waitforfile_timed ($replica2_ior, 5) == -1) {
+    print STDERR "ERROR: cannot find file <$replica2_ior>\n";
+    $REP1->Kill (); $REP1->TimedWait (1);
+    $REP2->Kill (); $REP2->TimedWait (1);
+    exit 1;
+}
+
+print "\nTEST: starting detector factory " . $DET->CommandLine . "\n" if ($verbose);
+$DET->Spawn ();
+
+print "TEST: waiting for detector's IOR\n" if ($verbose);
+if (PerlACE::waitforfile_timed ($detector_ior, 5) == -1) {
+    print STDERR "ERROR: cannot find file <$detector_ior>\n";
+    $REP1->Kill (); $REP1->TimedWait (1);
+    $REP2->Kill (); $REP2->TimedWait (1);
+    $DET->Kill (); $DET->TimedWait(1);
+    exit 1;
+}
+
+print "\nTEST: starting notifier " . $NOT->CommandLine . "\n" if ($verbose);
+$NOT->Spawn ();
+
+print "TEST: waiting for notifier's IOR\n" if ($verbose);
+if (PerlACE::waitforfile_timed ($notifier_ior, 5) == -1) {
+    print STDERR "ERROR: cannot find file <$notifier_ior>\n";
+    $REP1->Kill (); $REP1->TimedWait (1);
+    $REP2->Kill (); $REP2->TimedWait (1);
+    $DET->Kill (); $DET->TimedWait(1);
+    $NOT->Kill (); $NOT->TimedWait(1);
+    exit 1;
+}
+
+print "\nTEST: starting replication manager " . $REPLM->CommandLine . "\n" if ($verbose);
+$REPLM->Spawn ();
+
+print "TEST: waiting for Replication Manager's IOR file\n" if ($verbose);
+if (PerlACE::waitforfile_timed ($replmgr_ior, 5) == -1) {
+    print STDERR "ERROR: cannot find file <$replmgr_ior>\n";
+    $REP1->Kill (); $REP1->TimedWait (1);
+    $REP2->Kill (); $REP2->TimedWait (1);
+    $DET->Kill (); $DET->TimedWait(1);
+    $NOT->Kill (); $NOT->TimedWait(1);
+    $REPLM->Kill (); $REPLM->TimedWait(1);
+    exit 1;
+}
+
+print "\nTEST: starting fault consumer " . $CONS->CommandLine . "\n" if ($verbose);
+$CONS->Spawn ();
+
+print "TEST: waiting for READY.FILE from fault consumer\n" if ($verbose);
+if (PerlACE::waitforfile_timed ($ready_file, 5) == -1) {
+    print STDERR "ERROR: cannot find file <$ready_file>\n";
+    $REP1->Kill (); $REP1->TimedWait (1);
+    $REP2->Kill (); $REP2->TimedWait (1);
+    $DET->Kill (); $DET->TimedWait(1);
+    $NOT->Kill (); $NOT->TimedWait(1);
+    $CONS->Kill (); $CONS->TimedWait(1);
+    $REPLM->Kill (); $REPLM->TimedWait(1);
+    exit 1;
+}
+
+print "\nTEST: starting client " . $CL->CommandLine . "\n" if ($verbose);
+$client = $CL->SpawnWaitKill (60);
+
+if ($client != 0) {
+    print STDERR "ERROR: client returned $client\n";
+    $status = 1;
+}
+
+print "\nTEST: wait for replica 1.\n" if ($verbose);
+$replica1 = $REP1->WaitKill (5);
+if ($replica1 != 0) {
+    print STDERR "ERROR: replica returned $replica1\n";
+    $status = 1;
+}
+
+print "\nTEST: wait for replica 2.\n" if ($verbose);
+$replica2 = $REP2->WaitKill (5);
+if ($replica2 != 0) {
+    print STDERR "ERROR: replica returned $replica2\n";
+    $status = 1;
+}
+
+print "\nTEST: wait for detector factory to leave.\n" if ($verbose);
+$detector = $DET->WaitKill (20);
+if ($detector != 0) {
+    print STDERR "ERROR: detector returned $detector\n";
+    $status = 1;
+}
+
+print "\nTEST: wait for notifier to leave.\n" if ($verbose);
+$notifier = $NOT->WaitKill (20);
+if ($notifier != 0) {
+    print STDERR "ERROR: notifier returned $notifier\n";
+    $status = 1;
+}
+
+print "\nTEST: wait for fault consumer to leave.\n" if ($verbose);
+$consumer = $CONS->WaitKill (20);
+if ($consumer != 0) {
+    print STDERR "ERROR: fault consumer returned $consumer\n";
+    $status = 1;
+}
+
+print "\nTEST: releasing scratch files.\n" if ($verbose);
+unlink $replica1_ior;
+unlink $replica2_ior;
+unlink $detector_ior;
+unlink $notifier_ior;
+unlink $ready_file;
+
+#client's work file
+unlink $client_data;
+
+exit $status;