ChangeLogTag: Wed Oct 1 08:12:32 2003 Steve Totten <totten_s@ociweb.com>

6 files changed, 682 insertions, 9 deletions

diff --git a/TAO/ChangeLog b/TAO/ChangeLog
index a4846b6de4f..cd30c27fef6 100644
--- a/TAO/ChangeLog
+++ b/TAO/ChangeLog
@@ -1,8 +1,29 @@
+Wed Oct  1 08:12:32 2003  Steve Totten  <totten_s@ociweb.com>
+
+        * orbsvcs/orbsvcs/PortableGroup/PG_ObjectGroupManager.cpp:
+          Minor cleanup of code that gets the IORManipulation object.
+
+        * orbsvcs/FT_ReplicationManager/FT_FaultConsumer.cpp:
+        * orbsvcs/FT_ReplicationManager/FT_FaultConsumer.h:
+          Added TAO::FT_FaultConsumer class.  The FT_FaultConsumer
+          will be used by the Replication Manager to subscribe to the
+          Fault Notifier and process fault reports.  Just a stub
+          implementation right now.
+
+        * orbsvcs/tests/FT_App/FTApp_FaultConsumer_Main.cpp:
+        * orbsvcs/tests/FT_App/run_test_fault_consumer.pl:
+          Added the beginnings of a unit test for the FT_FaultConsumer.
+
+        * orbsvcs/tests/FT_App/FT_App.mpc:
+          Added project to build FT_FaultConsumer unit test.
+
+        These changes were made in the oci_haft branch.
+
 Tue Sep 30 16:31:12 2003  Curt Hibbs  <hibbs_c@ociweb.com>
 
-        * orbsvcs/orbsvcs/PortableGroup/PG_GenericFactory.cpp
-        * orbsvcs/orbsvcs/PortableGroup/PG_Properties_Decoder.cpp
-        * orbsvcs/tests/FT_App/FT_UnitTests.cpp
+        * orbsvcs/orbsvcs/PortableGroup/PG_GenericFactory.cpp:
+        * orbsvcs/orbsvcs/PortableGroup/PG_Properties_Decoder.cpp:
+        * orbsvcs/tests/FT_App/FT_UnitTests.cpp:
           Minor changes to correct the way properties were being constructed
           and extracted.
 
diff --git a/TAO/orbsvcs/FT_ReplicationManager/FT_FaultConsumer.cpp b/TAO/orbsvcs/FT_ReplicationManager/FT_FaultConsumer.cpp
new file mode 100755
index 00000000000..d7c5c627cbb
--- /dev/null
+++ b/TAO/orbsvcs/FT_ReplicationManager/FT_FaultConsumer.cpp
@@ -0,0 +1,198 @@
+/* -*- C++ -*- */
+//=============================================================================
+/**
+ *  @file    FT_FaultConsumer.cpp
+ *
+ *  $Id$
+ *
+ *  This file is part of TAO's implementation of Fault Tolerant CORBA.
+ *  This file provides the implementation of the TAO::FT_FaultConsumer
+ *  class.  The TAO::FT_FaultConsumer connects to the FaultNotifier to
+ *  receive fault reports.  It interacts with the ReplicationManager
+ *  to process fault reports (e.g., to set a new primary on an object
+ *  group or to create a new member of an object group).
+ *
+ *  @author Steve Totten <totten_s@ociweb.com>
+ */
+//=============================================================================
+
+#include "ace/pre.h"
+
+#include "orbsvcs/FT_ReplicationManager/FT_FaultConsumer.h"
+#include "orbsvcs/FT_ReplicationManagerC.h"
+
+ACE_RCSID (FT_FaultConsumer,
+           FT_FaultConsumer,
+           "$Id$")
+
+/// Default constructor.
+TAO::FT_FaultConsumer::FT_FaultConsumer ()
+  : poa_ (PortableServer::POA::_nil ())
+  , fault_notifier_ (FT::FaultNotifier::_nil ())
+  , replication_manager_ (FT::ReplicationManager::_nil ())
+  , consumer_id_ (0)
+  , notifications_ (0)
+{
+}
+
+/// Destructor.
+TAO::FT_FaultConsumer::~FT_FaultConsumer ()
+{
+}
+
+/**
+* Connect to the FT::FaultNotifier.
+* Note: We make the following assumptions about what the
+* application will do:
+* - Create an instance of this consumer class.
+* - Obtain the object reference of the FaultNotifier to which this
+*   consumer should connect.
+* - Call this init() method, passing it the POA with which we
+*   have been activated, the FaultNotifier, and ReplicationManager
+*   object references.
+*/
+int TAO::FT_FaultConsumer::init (
+  PortableServer::POA_ptr poa,
+  FT::FaultNotifier_ptr fault_notifier,
+  FT::ReplicationManager_ptr replication_manager
+  ACE_ENV_ARG_DECL)
+{
+  ACE_ASSERT (!CORBA::is_nil (poa));
+  ACE_ASSERT (!CORBA::is_nil (fault_notifier));
+  ACE_ASSERT (!CORBA::is_nil (replication_manager));
+
+  // Duplicate the object references passed in.
+  this->poa_ = 
+    PortableServer::POA::_duplicate (poa);
+  this->fault_notifier_ = 
+    FT::FaultNotifier::_duplicate (fault_notifier);
+  this->replication_manager_ = 
+    FT::ReplicationManager::_duplicate (replication_manager);
+
+  //@@ Should this init() method activate the consumer in the POA, or
+  // should the application do that?
+  // I don't think this object can activate itself because it doesn't
+  // know the policies on the POA.  So, we assume the application has
+  // already activated us.
+  //@@ For now, let's try just activating it in the POA.
+
+  // Activate this consumer in the POA.
+  PortableServer::ObjectId_var oid = 
+    this->poa_->activate_object (this ACE_ENV_ARG_PARAMETER);
+  ACE_CHECK_RETURN (-1);
+  CORBA::Object_var obj = 
+    this->poa_->id_to_reference (oid.in() ACE_ENV_ARG_PARAMETER);
+  ACE_CHECK_RETURN (-1);
+
+  // Narrow it to CosNotifyComm::StructuredPushConsumer.
+  CosNotifyComm::StructuredPushConsumer_var consumer = 
+    CosNotifyComm::StructuredPushConsumer::_nil ();
+  consumer = CosNotifyComm::StructuredPushConsumer::_narrow (
+    obj.in() ACE_ENV_ARG_PARAMETER);
+  ACE_CHECK_RETURN (-1);
+
+  // Subscribe to the FaultNotifier.
+  CosNotifyFilter::Filter_var filter = CosNotifyFilter::Filter::_nil ();
+  this->consumer_id_ = fault_notifier_->connect_structured_fault_consumer (
+    consumer.in(), filter ACE_ENV_ARG_PARAMETER);
+  ACE_CHECK_RETURN (-1);
+
+  // Success.
+  return 0;
+}
+
+/**
+* Clean house for process shut down.
+* - Disconnect from FT::FaultNotifier.
+* - Deactivate from the POA.
+*/
+int TAO::FT_FaultConsumer::fini (ACE_ENV_SINGLE_ARG_DECL)
+{
+  // Disconnect from the FaultNotifier.
+  this->fault_notifier_->disconnect_consumer (this->consumer_id_
+                                              ACE_ENV_ARG_PARAMETER);
+  ACE_CHECK_RETURN (-1);
+
+  //@@ Should this fini() method deactivate the consumer from the POA, or
+  // should the application do that?
+  //
+  // Deactivate ourselves from the POA.
+  PortableServer::ObjectId_var oid = 
+    this->poa_->servant_to_id (this ACE_ENV_ARG_PARAMETER);
+  ACE_CHECK_RETURN (-1);
+  this->poa_->deactivate_object (oid.in() ACE_ENV_ARG_PARAMETER);
+  ACE_CHECK_RETURN (-1);
+ 
+  // Success.
+  return 0;
+}
+
+
+////////////////
+// CORBA methods
+
+void TAO::FT_FaultConsumer::push_structured_event (
+  const CosNotification::StructuredEvent &event
+  ACE_ENV_ARG_DECL_WITH_DEFAULTS
+  )
+  ACE_THROW_SPEC ((CORBA::SystemException, CosEventComm::Disconnected))
+{
+  this->notifications_ += 1;
+  ACE_DEBUG ((LM_DEBUG,
+    "TAO::FT_FaultConsumer: Received Fault notification(%d):\n"
+    "TAO::FT_FaultConsumer:   Header EventType domain: %s\n"
+    "TAO::FT_FaultConsumer:   Header EventType type: %s\n"
+    "TAO::FT_FaultConsumer:   Header EventName: %s\n",
+    ACE_static_cast (unsigned int, this->notifications_),
+    ACE_static_cast (const char *, event.header.fixed_header.event_type.domain_name),
+    ACE_static_cast (const char *, event.header.fixed_header.event_type.type_name),
+    ACE_static_cast (const char *, event.header.fixed_header.event_name)
+    ));
+
+  const CosNotification::FilterableEventBody & filterable = 
+    event.filterable_data;
+
+  size_t propertyCount = filterable.length ();
+  for (size_t nProp = 0; nProp < propertyCount; ++nProp)
+  {
+    const CosNotification::Property & property = filterable[nProp];
+    ACE_DEBUG ((LM_DEBUG,
+      "TAO::FT_FaultConsumer:   Property Name: %s\n",
+      ACE_static_cast (const char *, property.name)
+      ));
+  }
+
+  //TODO: Finish the implementation of push_structured_event() to:
+  // - analyze the fault
+  // - if it was a primary fault, set a new primary replica
+  // - if the number of replicas falls below MinimumNumberReplicas
+  //   property and membership style is Infrastructure-Controlled,
+  //   create and add a new member.
+
+}
+
+void TAO::FT_FaultConsumer::offer_change (
+    const CosNotification::EventTypeSeq & added,
+    const CosNotification::EventTypeSeq & removed
+    ACE_ENV_ARG_DECL_WITH_DEFAULTS
+  )
+  ACE_THROW_SPEC ((CORBA::SystemException, CosNotifyComm::InvalidEventType))
+{
+  ACE_DEBUG ((LM_DEBUG, 
+    "TAO::FT_FaultConsumer::offer_change() call ignored.\n"
+    ));
+}
+
+void TAO::FT_FaultConsumer::disconnect_structured_push_consumer (
+    ACE_ENV_SINGLE_ARG_DECL_WITH_DEFAULTS
+  )
+  ACE_THROW_SPEC ((CORBA::SystemException))
+{
+  //TODO: For now, we are just ignoring the disconnect callback.
+  ACE_DEBUG ((LM_DEBUG, 
+    "TAO::FT_FaultConsumer::disconnect_structured_push_consumer() call ignored.\n"
+    ));
+}
+
+#include "ace/post.h"
+
diff --git a/TAO/orbsvcs/FT_ReplicationManager/FT_FaultConsumer.h b/TAO/orbsvcs/FT_ReplicationManager/FT_FaultConsumer.h
new file mode 100755
index 00000000000..4b590c5e744
--- /dev/null
+++ b/TAO/orbsvcs/FT_ReplicationManager/FT_FaultConsumer.h
@@ -0,0 +1,161 @@
+/* -*- C++ -*- */
+//=============================================================================
+/**
+ *  @file    FT_FaultConsumer.h
+ *
+ *  $Id$
+ *
+ *  This file is part of TAO's implementation of Fault Tolerant CORBA.
+ *
+ *  @author Steve Totten <totten_s@ociweb.com>
+ */
+//=============================================================================
+
+
+#ifndef FT_FAULT_CONSUMER_H_
+#define FT_FAULT_CONSUMER_H_
+
+#include /**/ "ace/pre.h"
+
+#if !defined (ACE_LACKS_PRAGMA_ONCE)
+#pragma once
+#endif /* ACE_LACKS_PRAGMA_ONCE */
+
+#include "orbsvcs/CosNotifyCommS.h"
+#include "orbsvcs/FT_NotifierC.h"
+#include "orbsvcs/FT_ReplicationManagerC.h"
+
+namespace TAO
+{
+  /**
+  * Implement the CosNotifyComm::StructuredPushConsumer interface.
+  *
+  */
+  class FT_FaultConsumer
+    : public virtual POA_CosNotifyComm::StructuredPushConsumer
+    , public virtual PortableServer::RefCountServantBase
+  {
+
+    //////////////////////
+    // non-CORBA interface
+
+  public:
+    /**
+    * Default constructor.
+    */
+    FT_FaultConsumer ();
+
+    /**
+    * Destructor.
+    */
+    virtual ~FT_FaultConsumer ();
+
+    /**
+    * Connect to the FT::FaultNotifier.
+    * Note: We make the following assumptions about what the
+    * application will do:
+    * - Create an instance of this consumer class.
+    * - Obtain the object reference of the FaultNotifier to which this
+    *   consumer should connect.
+    * - Call this init() method, passing it the POA with which we
+    *   have been activated, the FaultNotifier, and ReplicationManager
+    *   object references.
+    */
+    int init (
+      PortableServer::POA_ptr poa,
+      FT::FaultNotifier_ptr fault_notifier,
+      FT::ReplicationManager_ptr replication_manager
+      ACE_ENV_ARG_DECL);
+
+    /**
+    * Clean house for process shut down.
+    * - Disconnect from FT::FaultNotifier.
+    * - Deactivate from the POA.
+    */
+    int fini (ACE_ENV_SINGLE_ARG_DECL);
+
+  public:
+
+    /////////////////////////
+    // Implementation methods
+  private:
+
+    /**
+    * Extract the value of the MinimumNumberReplicas property from
+    * the_criteria.
+    */
+    int get_minimum_number_replicas (
+      const char * type_id,
+      const PortableGroup::Criteria & the_criteria,
+      CORBA::UShort & minimum_number_replicas) const;
+
+  public:
+
+    /**
+    * @name POA_CosNotifyComm::StructuredPushConsumer Methods
+    *
+    * Methods required by the POA_CosNotifyComm::StructuredPushConsumer 
+    * interface.
+    */
+    //@{
+
+    ////////////////
+    // CORBA methods
+    virtual void push_structured_event (
+      const CosNotification::StructuredEvent &notification
+      ACE_ENV_ARG_DECL_WITH_DEFAULTS
+      )
+      ACE_THROW_SPEC ((CORBA::SystemException, CosEventComm::Disconnected));
+
+   virtual void offer_change (
+        const CosNotification::EventTypeSeq & added,
+        const CosNotification::EventTypeSeq & removed
+        ACE_ENV_ARG_DECL_WITH_DEFAULTS
+      )
+      ACE_THROW_SPEC ((CORBA::SystemException, CosNotifyComm::InvalidEventType));
+
+   virtual void disconnect_structured_push_consumer (
+        ACE_ENV_SINGLE_ARG_DECL_WITH_DEFAULTS
+      )
+      ACE_THROW_SPEC ((CORBA::SystemException));
+
+    //@}
+
+    ////////////////////
+    // Forbidden methods
+  private:
+    /// Copy constructor.
+    FT_FaultConsumer (
+        const FT_FaultConsumer & rhs);
+    /// Assignment operator.
+    FT_FaultConsumer & operator = (
+        const FT_FaultConsumer & rhs);
+
+    ///////////////
+    // Data Members
+  private:
+
+    /// The POA with which we are activated.
+    PortableServer::POA_var poa_;
+
+    /// The FaultNotifier's object reference.
+    FT::FaultNotifier_var fault_notifier_;
+
+    /// The ReplicationManager's object reference.
+    FT::ReplicationManager_var replication_manager_;
+
+    /// ConsumerId assigned by the notifier.
+    FT::FaultNotifier::ConsumerId consumer_id_;
+
+    ///TODO: Remove this later, it is just for testing.
+    // Keep track of how many notifications we have received.
+    size_t notifications_;
+
+  };
+
+} // namespace TAO
+
+#include "ace/post.h"
+
+#endif  /* FT_FAULT_CONSUMER_H_ */
+
diff --git a/TAO/orbsvcs/orbsvcs/PortableGroup/PG_ObjectGroupManager.cpp b/TAO/orbsvcs/orbsvcs/PortableGroup/PG_ObjectGroupManager.cpp
index 5214353697a..c38546f3c04 100644
--- a/TAO/orbsvcs/orbsvcs/PortableGroup/PG_ObjectGroupManager.cpp
+++ b/TAO/orbsvcs/orbsvcs/PortableGroup/PG_ObjectGroupManager.cpp
@@ -629,14 +629,12 @@ TAO_PG_ObjectGroupManager::init (CORBA::ORB_ptr orb, PortableServer::POA_ptr p)
   int result = 0;
 
   // Get an object reference for the ORBs IORManipulation object!
-  CORBA::Object_var IORM =
-    orb_->resolve_initial_references (TAO_OBJID_IORMANIPULATION,
-                                      0
-                                      ACE_ENV_ARG_PARAMETER);
+  CORBA::Object_var IORM = this->orb_->resolve_initial_references (
+    TAO_OBJID_IORMANIPULATION, 0 ACE_ENV_ARG_PARAMETER);
   ACE_TRY_CHECK;
 
-  iorm_ = TAO_IOP::TAO_IOR_Manipulation::_narrow (IORM.in ()
-                                                  ACE_ENV_ARG_PARAMETER);
+  this->iorm_ = TAO_IOP::TAO_IOR_Manipulation::_narrow (
+    IORM.in () ACE_ENV_ARG_PARAMETER);
   ACE_TRY_CHECK;
 
   return result;
diff --git a/TAO/orbsvcs/tests/FT_App/FT_App.mpc b/TAO/orbsvcs/tests/FT_App/FT_App.mpc
index 121a7ea3a1f..a19cc3b4bed 100644
--- a/TAO/orbsvcs/tests/FT_App/FT_App.mpc
+++ b/TAO/orbsvcs/tests/FT_App/FT_App.mpc
@@ -86,6 +86,33 @@ project(*Analyzer): taoclient, fault_tolerance, orbsvcsexe {
   }
 }
 
+project(*FaultConsumer): taoserver, fault_tolerance, orbsvcsexe {
+  exename = ft_fault_consumer
+
+  Source_Files {
+    FTApp_FaultConsumer_Main.cpp
+    ../../FT_ReplicationManager/FT_FaultConsumer.cpp
+  }
+
+//  // Custom folder: script to run this test
+//  Define_Custom(Script){
+//    inputext = .pl, .py, .rb
+//    generic_outputext = .txt
+//  }
+//
+//  Script_Files {
+//    run_test_detector.pl
+//  }
+
+  // explicitly omit IDL files
+  IDL_Files {
+  }
+  Documentation_Files {
+    // pretend the pl file is documentation for now
+    run_test_fault_consumer.pl
+  }
+}
+
 project(*FactoryRegistry): taoclient, fault_tolerance, orbsvcsexe {
   exename = ft_registry
 
diff --git a/TAO/orbsvcs/tests/FT_App/run_test_fault_consumer.pl b/TAO/orbsvcs/tests/FT_App/run_test_fault_consumer.pl
new file mode 100755
index 00000000000..1229bfed924
--- /dev/null
+++ b/TAO/orbsvcs/tests/FT_App/run_test_fault_consumer.pl
@@ -0,0 +1,268 @@
+eval '(exit $?0)' && eval 'exec perl -S $0 ${1+"$@"}'
+     & eval 'exec perl -S $0 $argv:q'
+    if 0;
+
+# $Id$
+# -*- perl -*-
+
+# Purpose:
+#       To test the FaultConsumer
+#
+# Process being tested:
+#       Fault_Consumer
+#           implements FaultConsumer interface.
+# Processes used in test:
+#       FT_Replica
+#           implements TestReplica interface.
+#           implements PullMonitorable.
+#       Fault_Detector
+#         implements FaultDetectorFactory interface
+#         implements PullMonitorable interface
+#       FT_Client
+#           client for TestReplica interface.
+#           client for PullMonitorable.
+#       Fault_Notifier
+#           implements the FaultNotifier interface.
+#           propagates fault reports to the FaultConsumer.
+#
+# Test Scenario (***Test: marks behavior being tested):
+#   Phase 1:
+#     Start two FT_Replicas
+#       FT_Replicas write TestReplica IORs (FR#1 and FR#2) to files
+#     Start the Fault_Detector
+#       Fault_Detector writes its IOR (FDF) to a file
+#     Start the Fault_Notifier
+#       Fault_Notifier writes its IOR (FN) to a file.
+#     Start the Fault_Consumer
+#       Subscribes to the Fault_Notifier for fault reports.
+#   Phase 2:
+#     Wait for IORs: FR#1, FR#2, FDF, and FN
+#     Start the StubAnalyzer giving it IORS: FR#1, FR#2 FDF, FN
+#      StubAnalyzer calls FDF to create a FaultDetector
+#       for each Replica.
+#      StubAnalyzer subscribes to Fault_Notifier
+#      StubAnalyzer writes dummy message(READY) to a file.
+#   Phase 3:
+#     Wait for READY
+#     Start FT_Client giving it IORS: FR#1 and FR#2. [1]
+#       FT_Client interacts with FR#1.
+#       FT_Client asks FR#1 to fault.  It does so.
+#       FT_Client notices fault and switches to FR#2. [1]
+#       FD#1 notices fault and notifies Fault_Notifier
+#       FD#1 terminates
+#       ***Test: Fault_Notifier forwards notification to StubAnalyzer
+#       StubAnalyzer prints notification.
+#       FT_Client interacts with FR#2.
+#       FT_Client asks FR#2 to shut down.
+#       FT_Client shuts down.
+#       FD#2 notices FR2 is gone, interprets this
+#         as a fault, and sends notification to Fault_Notifier
+#       FD#2 terminates.
+#       ***Test: Fault_Notifier forwards notification to StubAnalyzer
+#       StubAnalyzer prints notification.
+#     Phase 4: shutting down.
+#       All FaultDetectors have terminated so the FaultDetectorFactory
+#        honors the "quit-on-idle" option on it's command line and exits.
+#       StubAnalyzer compares # fault notifications to # replicas. When
+#        they match, it "knows" that the test is over, so it shuts down.
+#        As it does so, it disconnects its fault consumers from the FaultNotifier.
+#       FaultNotifier notices the last fault consumer disconnecting and exits because
+#        the "quit-on-idle" option was specified on the command line.
+#     Phase 5: housekeeping
+#       Wait for all processes to terminate.
+#       Check termination status.
+#       Delete temp files.
+#
+# [1] Client mediated fault tolerance.  These points will
+#     change when IOGR support is available.
+use lib '../../../../bin';
+#use lib '$ENV{ACE_ROOT}/bin';
+use PerlACE::Run_Test;
+
+########################
+#command line options
+#set defaults:
+my($verbose) = 0;         # 1: report perl actions before executing them
+my($debug_builds) = 1;    # 0: use exes from Release directories
+my($simulated) = 1;       # 1: use "client simulated" fault tolerance
+
+foreach $i (@ARGV) {
+  if ($i eq "--debug_build")
+  {
+    $debug_builds = 1;
+  }
+  elsif ($i eq "--no_simulate")  # reverse this once we have FT ORB support
+  {
+    $simulated = 0;
+  }
+  elsif ($i eq "-v")
+  {
+    $verbose += 1;
+  }
+}
+
+
+my($build_directory) = "/Release";
+if ( $debug_builds ) {
+  $build_directory = "";
+}
+
+if ( $verbose > 1) {
+  print "verbose: $verbose\n";
+  print "debug_builds: $debug_builds -> $build_directory\n";
+  print "simulated: $simulated\n";
+}
+
+
+#define temp files
+my($factory1_ior) = PerlACE::LocalFile ("factory1.ior");
+my($factory2_ior) = PerlACE::LocalFile ("factory2.ior");
+my($replica1_ior) = PerlACE::LocalFile ("replica1.ior");
+my($replica2_ior) = PerlACE::LocalFile ("replica2.ior");
+my($detector_ior) = PerlACE::LocalFile ("detector.ior");
+my($notifier_ior) = PerlACE::LocalFile ("notifier.ior");
+# my($ready_file) = PerlACE::LocalFile ("ready.file");
+my($client_data) = PerlACE::LocalFile ("persistent.dat");
+
+#discard junk from previous tests
+unlink $factory1_ior;
+unlink $factory2_ior;
+unlink $replica1_ior;
+unlink $replica2_ior;
+unlink $detector_ior;
+unlink $notifier_ior;
+# unlink $ready_file;
+unlink $client_data;
+
+my($status) = 0;
+
+my($REP1) = new PerlACE::Process (".$build_directory/ft_replica", "-o $factory1_ior -t $replica1_ior -r 1 -q");
+my($REP2) = new PerlACE::Process (".$build_directory/ft_replica", "-o $factory2_ior -t $replica2_ior -r 2 -q");
+my($DET) = new PerlACE::Process ("$ENV{'TAO_ROOT'}/orbsvcs/Fault_Detector$build_directory/Fault_Detector", "-o $detector_ior -q");
+my($NOT) = new PerlACE::Process ("$ENV{'TAO_ROOT'}/orbsvcs/Fault_Notifier$build_directory/Fault_Notifier", "-o $notifier_ior -v -q");
+# my($ANA) = new PerlACE::Process (".$build_directory/ft_analyzer", "-o $ready_file -n $notifier_ior -q -d $detector_ior -r $replica1_ior,$replica2_ior");
+my($ANA) = new PerlACE::Process (".$build_directory/ft_fault_consumer");
+
+my($CL);
+if (simulated) {
+  $CL = new PerlACE::Process (".$build_directory/ft_client", "-f $replica1_ior,$replica2_ior -c testscript");
+}else{
+  #todo figure out how to get iogr
+  $CL = new PerlACE::Process (".$build_directory/ft_client", "-f $replica1_iogr -c testscript");
+}
+
+print "TEST: starting replica1 " . $REP1->CommandLine . "\n" if ($verbose);
+$REP1->Spawn ();
+
+print "TEST: waiting for replica 1's IOR\n" if ($verbose);
+if (PerlACE::waitforfile_timed ($replica1_ior, 5) == -1) {
+    print STDERR "ERROR: cannot find file <$replica1_ior>\n";
+    $REP1->Kill (); $REP1->TimedWait (1);
+    exit 1;
+}
+
+print "\nTEST: starting replica2 " . $REP2->CommandLine . "\n" if ($verbose);
+$REP2->Spawn ();
+
+print "TEST: waiting for replica 2's IOR\n" if ($verbose);
+if (PerlACE::waitforfile_timed ($replica2_ior, 5) == -1) {
+    print STDERR "ERROR: cannot find file <$replica2_ior>\n";
+    $REP1->Kill (); $REP1->TimedWait (1);
+    $REP2->Kill (); $REP2->TimedWait (1);
+    exit 1;
+}
+
+print "\nTEST: starting detector factory " . $DET->CommandLine . "\n" if ($verbose);
+$DET->Spawn ();
+
+print "TEST: waiting for detector's IOR\n" if ($verbose);
+if (PerlACE::waitforfile_timed ($detector_ior, 5) == -1) {
+    print STDERR "ERROR: cannot find file <$detector_ior>\n";
+    $REP1->Kill (); $REP1->TimedWait (1);
+    $REP2->Kill (); $REP2->TimedWait (1);
+    $DET->Kill (); $DET2->TimedWait(1);
+    exit 1;
+}
+
+print "\nTEST: starting notifier " . $NOT->CommandLine . "\n" if ($verbose);
+$NOT->Spawn ();
+
+print "TEST: waiting for notifier's IOR\n" if ($verbose);
+if (PerlACE::waitforfile_timed ($notifier_ior, 5) == -1) {
+    print STDERR "ERROR: cannot find file <$notifier_ior>\n";
+    $REP1->Kill (); $REP1->TimedWait (1);
+    $REP2->Kill (); $REP2->TimedWait (1);
+    $DET->Kill (); $DET2->TimedWait(1);
+    $ANA->Kill (); $ANA->TimedWait(1);
+    exit 1;
+}
+
+print "\nTEST: starting analyzer " . $ANA->CommandLine . "\n" if ($verbose);
+$ANA->Spawn ();
+sleep (3);
+
+# print "TEST: waiting for READY.FILE from analyzer\n" if ($verbose);
+# if (PerlACE::waitforfile_timed ($ready_file, 5) == -1) {
+#     print STDERR "ERROR: cannot find file <$ready_file>\n";
+#     $REP1->Kill (); $REP1->TimedWait (1);
+#     $REP2->Kill (); $REP2->TimedWait (1);
+#     $DET->Kill (); $DET2->TimedWait(1);
+#     $NOT->Kill (); $NOT->TimedWait(1);
+#     $ANA->Kill (); $ANA->TimedWait(1);
+#     exit 1;
+# }
+
+print "\nTEST: starting client " . $CL->CommandLine . "\n" if ($verbose);
+$client = $CL->SpawnWaitKill (60);
+
+if ($client != 0) {
+    print STDERR "ERROR: client returned $client\n";
+    $status = 1;
+}
+
+print "\nTEST: wait for replica 1.\n" if ($verbose);
+$replica1 = $REP1->WaitKill (60);
+if ($replica1 != 0) {
+    print STDERR "ERROR: replica returned $replica1\n";
+    $status = 1;
+}
+
+print "\nTEST: wait for replica 2.\n" if ($verbose);
+$replica2 = $REP2->WaitKill (60);
+if ($replica2 != 0) {
+    print STDERR "ERROR: replica returned $replica2\n";
+    $status = 1;
+}
+
+print "\nTEST: wait for detector factory to leave.\n" if ($verbose);
+$detector = $DET->WaitKill (60);
+if ($detector != 0) {
+    print STDERR "ERROR: detector returned $detector\n";
+    $status = 1;
+}
+
+print "\nTEST: wait for notifier to leave.\n" if ($verbose);
+$notifier = $NOT->WaitKill (60);
+if ($notifier != 0) {
+    print STDERR "ERROR: notifier returned $notifier\n";
+    $status = 1;
+}
+
+print "\nTEST: wait for analyzer to leave.\n" if ($verbose);
+$analyzer = $ANA->WaitKill (60);
+if ($analyzer != 0) {
+    print STDERR "ERROR: analyzer returned $analyzer\n";
+    $status = 1;
+}
+
+print "\nTEST: releasing scratch files.\n" if ($verbose);
+unlink $replica1_ior;
+unlink $replica2_ior;
+unlink $detector_ior;
+unlink $notifier_ior;
+# unlink $ready_file;
+
+#client's work file
+unlink $client_data;
+
+exit $status;