ChangeLogTag: Wed Oct 8 13:59:40 2003 Steve Totten <totten_s@ociweb.com>

9 files changed, 595 insertions, 69 deletions

diff --git a/TAO/ChangeLog b/TAO/ChangeLog
index beb90ed1020..31758038ed8 100644
--- a/TAO/ChangeLog
+++ b/TAO/ChangeLog
@@ -1,3 +1,22 @@
+Wed Oct  8 13:59:40 2003  Steve Totten  <totten_s@ociweb.com>
+
+        * orbsvcs/FT_ReplicationManager/FT_FaultConsumer.cpp:
+        * orbsvcs/FT_ReplicationManager/FT_FaultConsumer.h:
+          Added some initial fault analysis logic.
+
+        * orbsvcs/FT_ReplicationManager/FT_FaultEventDescriptor.cpp:
+        * orbsvcs/FT_ReplicationManager/FT_FaultEventDescriptor.h:
+          Added a new helper class for fault analysis.
+
+        * orbsvcs/tests/FT_App/FTApp_FaultConsumer_Main.cpp:
+        * orbsvcs/tests/FT_App/FT_App.mpc:
+        * orbsvcs/tests/FT_App/ReplicationManagerFaultConsumerAdapter.cpp:
+        * orbsvcs/tests/FT_App/ReplicationManagerFaultConsumerAdapter.h:
+        * orbsvcs/tests/FT_App/run_test_fault_consumer.pl:
+          Updated to work with new TAO::Utils::Server_Main<> framework.
+
+        These changes were made in the oci_haft branch.
+
 Tue Oct  7 16:47:50 2003  Dale Wilson  <wilson_d@ociweb.com>
 
         * orbsvcs/tests/FT_App/FT_App.mpc:
diff --git a/TAO/orbsvcs/FT_ReplicationManager/FT_FaultConsumer.cpp b/TAO/orbsvcs/FT_ReplicationManager/FT_FaultConsumer.cpp
index 40f54dadb21..3662e73ea01 100755
--- a/TAO/orbsvcs/FT_ReplicationManager/FT_FaultConsumer.cpp
+++ b/TAO/orbsvcs/FT_ReplicationManager/FT_FaultConsumer.cpp
@@ -18,6 +18,7 @@
 
 #include "orbsvcs/FT_ReplicationManager/FT_FaultConsumer.h"
 #include "orbsvcs/FT_ReplicationManagerC.h"
+#include <tao/debug.h>
 
 ACE_RCSID (FT_FaultConsumer,
            FT_FaultConsumer,
@@ -76,11 +77,10 @@ int TAO::FT_FaultConsumer::init (
   //@@ For now, let's try just activating it in the POA.
 
   // Activate this consumer in the POA.
-  PortableServer::ObjectId_var oid =
-    this->poa_->activate_object (this ACE_ENV_ARG_PARAMETER);
+  this->object_id_ = this->poa_->activate_object (this ACE_ENV_ARG_PARAMETER);
   ACE_CHECK_RETURN (-1);
   CORBA::Object_var obj =
-    this->poa_->id_to_reference (oid.in() ACE_ENV_ARG_PARAMETER);
+    this->poa_->id_to_reference (this->object_id_.in() ACE_ENV_ARG_PARAMETER);
   ACE_CHECK_RETURN (-1);
 
   // Narrow it to CosNotifyComm::StructuredPushConsumer.
@@ -114,10 +114,7 @@ int TAO::FT_FaultConsumer::fini (ACE_ENV_SINGLE_ARG_DECL)
   // should the application do that?
   //
   // Deactivate ourselves from the POA.
-  PortableServer::ObjectId_var oid =
-    this->poa_->servant_to_id (this ACE_ENV_ARG_PARAMETER);
-  ACE_CHECK_RETURN (-1);
-  this->poa_->deactivate_object (oid.in() ACE_ENV_ARG_PARAMETER);
+  this->poa_->deactivate_object (this->object_id_.in() ACE_ENV_ARG_PARAMETER);
   ACE_CHECK_RETURN (-1);
 
   this->consumer_ref_ = CosNotifyComm::StructuredPushConsumer::_nil ();
@@ -142,46 +139,123 @@ size_t TAO::FT_FaultConsumer::notifications () const
 ////////////////
 // CORBA methods
 
+// Receive and process an incoming fault event from the Fault Notifier.
+// First, we validate the event to make sure it is something we can
+// handle.  Then, we analyze it.
+// If it is not an event we can handle, we simply log the error and
+// drop the event.
 void TAO::FT_FaultConsumer::push_structured_event (
   const CosNotification::StructuredEvent &event
   ACE_ENV_ARG_DECL_WITH_DEFAULTS
   )
   ACE_THROW_SPEC ((CORBA::SystemException, CosEventComm::Disconnected))
 {
+  // Debugging support.
   this->notifications_ += 1;
-  ACE_DEBUG ((LM_DEBUG,
-    "TAO::FT_FaultConsumer: Received Fault notification(%d):\n"
-    "TAO::FT_FaultConsumer:   Header EventType domain: %s\n"
-    "TAO::FT_FaultConsumer:   Header EventType type: %s\n"
-    "TAO::FT_FaultConsumer:   Header EventName: %s\n",
-    ACE_static_cast (unsigned int, this->notifications_),
-    ACE_static_cast (const char *, event.header.fixed_header.event_type.domain_name),
-    ACE_static_cast (const char *, event.header.fixed_header.event_type.type_name),
-    ACE_static_cast (const char *, event.header.fixed_header.event_name)
+#if (TAO_DEBUG_LEVEL_NEEDED == 1)
+  if (TAO_debug_level > 1)
+#endif /* (TAO_DEBUG_LEVEL_NEEDED == 1) */
+  {
+    ACE_DEBUG ((LM_DEBUG,
+      ACE_TEXT (
+        "TAO::FT_FaultConsumer::push_structured_event: "
+        "Received Fault notification(%d):\n"),
+      ACE_static_cast (unsigned int, this->notifications_)
     ));
+  }
 
-  const CosNotification::FilterableEventBody & filterable =
-    event.filterable_data;
+  int result = 0;
 
-  size_t propertyCount = filterable.length ();
-  for (size_t nProp = 0; nProp < propertyCount; ++nProp)
+  // Make sure it is an event type we can handle.
+  if (result == 0)
   {
-    const CosNotification::Property & property = filterable[nProp];
-    ACE_DEBUG ((LM_DEBUG,
-      "TAO::FT_FaultConsumer:   Property Name: %s\n",
-      ACE_static_cast (const char *, property.name)
+    result = this->validate_event_type (event);
+    ACE_CHECK;
+    if (result != 0)
+    {
+      ACE_ERROR ((LM_ERROR,
+        ACE_TEXT (
+          "TAO::FT_FaultConsumer::push_structured_event: "
+          "Received invalid fault event type.\n")
       ));
+    }
   }
 
-  //TODO: Finish the implementation of push_structured_event() to:
-  // - analyze the fault
-  // - if it was a primary fault, set a new primary replica
-  // - if the number of replicas falls below MinimumNumberReplicas
-  //   property and membership style is Infrastructure-Controlled,
-  //   create and add a new member.
+  // Analyze the event.
+  if (result == 0)
+  {
+    TAO::FT_FaultEventDescriptor fault_event_desc;
+    result = this->analyze_fault_event (
+      event,
+      fault_event_desc
+      ACE_ENV_ARG_PARAMETER);
+    ACE_CHECK_RETURN (-1);
+    if (result != 0)
+    {
+      ACE_ERROR ((LM_ERROR,
+        ACE_TEXT (
+          "TAO::FT_FaultConsumer::push_structured_event: "
+          "Could not analyze fault event.\n")
+      ));
+    }
 
+    // Debugging support.
+#if (TAO_DEBUG_LEVEL_NEEDED == 1)
+    if (TAO_debug_level > 6)
+#endif /* (TAO_DEBUG_LEVEL_NEEDED == 1) */
+    {
+      ACE_DEBUG ((LM_DEBUG,
+        ACE_TEXT("Getting the location as a string.\n")
+      ));
+
+      // Get the location as a string.
+      ACE_CString loc_as_string;
+      FT::Location_var & loc = fault_event_desc.location;
+
+      for (CORBA::ULong li = 0; li < loc->length(); ++li)
+      {
+        if (li > 0) loc_as_string += "/";
+        // Assume only the "id" field of the CosNaming::Name is used.
+        loc_as_string += CORBA::string_dup (loc[li].id);
+      }
+
+      if (fault_event_desc.all_at_location_failed == 1)
+      {
+        ACE_DEBUG ((LM_DEBUG,
+          ACE_TEXT (
+            "All objects at location <%s> failed.\n"),
+            loc_as_string.c_str()
+        ));
+      }
+
+      if (fault_event_desc.all_of_type_at_location_failed == 1)
+      {
+        ACE_DEBUG ((LM_DEBUG,
+          ACE_TEXT (
+            "All objects of type <%s> at location <%s> failed.\n"),
+            fault_event_desc.type_id.in(),
+            loc_as_string.c_str()
+        ));
+      }
+
+      if (fault_event_desc.object_at_location_failed == 1)
+      {
+        ACE_DEBUG ((LM_DEBUG,
+          ACE_TEXT (
+            "Replica of type <%s> with ObjectGroupId <%Q> "
+            "at location <%s> failed.\n"),
+            fault_event_desc.type_id.in(),
+            fault_event_desc.object_group_id,
+            loc_as_string.c_str()
+        ));
+      }
+    }
+  }
+
+  return;
 }
 
+
 void TAO::FT_FaultConsumer::offer_change (
     const CosNotification::EventTypeSeq & added,
     const CosNotification::EventTypeSeq & removed
@@ -190,8 +264,8 @@ void TAO::FT_FaultConsumer::offer_change (
   ACE_THROW_SPEC ((CORBA::SystemException, CosNotifyComm::InvalidEventType))
 {
   ACE_DEBUG ((LM_DEBUG,
-    "TAO::FT_FaultConsumer::offer_change() call ignored.\n"
-    ));
+    ACE_TEXT("TAO::FT_FaultConsumer::offer_change() call ignored.\n")
+  ));
 }
 
 void TAO::FT_FaultConsumer::disconnect_structured_push_consumer (
@@ -201,7 +275,274 @@ void TAO::FT_FaultConsumer::disconnect_structured_push_consumer (
 {
   //TODO: For now, we are just ignoring the disconnect callback.
   ACE_DEBUG ((LM_DEBUG,
-    "TAO::FT_FaultConsumer::disconnect_structured_push_consumer() call ignored.\n"
+    ACE_TEXT("TAO::FT_FaultConsumer::disconnect_structured_push_consumer() "
+             "call ignored.\n")
+    ));
+}
+
+//////////////////
+// Private methods
+
+// Validate the event to make sure it is one we can handle.
+// If it is not an event we can handle, this function logs the error
+// and returns -1.
+int TAO::FT_FaultConsumer::validate_event_type (
+  const CosNotification::StructuredEvent & event)
+{
+  int result = 0;
+
+  // CORBA 3.0.2, section 23.4.5.1 states:
+  //
+  //   The fault management specification defines one event type:
+  //   ObjectCrashFault. As the name suggests, this event is
+  //   generated by a Fault Detector when it detects that an object
+  //   has crashed.
+
+  // So, the event header's event_type.domain_name must be "FT_CORBA"
+  // and the event header's event_type.type_name must be "ObjectCrashFault".
+  CORBA::String_var domain_name = CORBA::string_dup (
+    event.header.fixed_header.event_type.domain_name);
+  CORBA::String_var type_name = CORBA::string_dup (
+      event.header.fixed_header.event_type.type_name);
+  CORBA::String_var event_name = CORBA::string_dup (
+      event.header.fixed_header.event_name);
+
+  if (result == 0)
+  {
+    //TODO: Get rid of magic strings (though they come from the spec).
+    if (ACE_OS::strcmp (domain_name.in(), "FT_CORBA") != 0 ||
+        ACE_OS::strcmp (type_name.in(), "ObjectCrashFault") != 0)
+    {
+      ACE_ERROR ((LM_ERROR,
+        ACE_TEXT (
+          "TAO::FT_FaultConsumer::validate_event_type: "
+          "Received invalid event type.\n"
+          "EventType domain: <%s>\n"
+          "EventType type: <%s>\n"
+          "EventName: <%s>\n"),
+        domain_name.in(),
+        type_name.in(),
+        event_name.in()
+      ));
+      result = -1;
+    }
+  }
+
+  // CORBA 3.0.2, section 23.4.5.1 also states:
+  //
+  //   The filterable_data part of the event body contains the
+  //   identity of the crashed object as four name-value pairs: the
+  //   fault tolerance domain identifier, the member’s location
+  //   identifier, the repository identifier and the object group
+  //   identifier. The Fault Notifier filters events based on the
+  //   domain_name, the type_name, and the four identifiers.  All
+  //   other fields of the structured event may be set to null.
+  //
+  //   The Fault Detector always sets the following fault event
+  //   fields: domain_name, type_name, FTDomainId, and Location.
+  //
+  // So, at least "FTDomainId" and "Location" must be present:
+  if (result == 0)
+  {
+    if (event.filterable_data.length () >= 2)
+    {
+      // Check for FTDomainId.
+      if (ACE_OS::strcmp (
+            event.filterable_data[0].name.in(), "FTDomainId") != 0)
+      {
+        ACE_ERROR ((LM_ERROR,
+          ACE_TEXT (
+            "TAO::FT_FaultConsumer::validate_event_type: "
+            "Received invalid structured event.\n"
+            "filterable_data[0] must be \"FTDomainId\", not \"%s\"\n"),
+          event.filterable_data[0].name.in()
+        ));
+        result = -1;
+      }
+      else if (ACE_OS::strcmp (
+            event.filterable_data[1].name.in(), "Location") != 0)
+      {
+        ACE_ERROR ((LM_ERROR,
+          ACE_TEXT (
+            "TAO::FT_FaultConsumer::validate_event_type: "
+            "Received invalid structured event.\n"
+            "filterable_data[1] must be \"Location\", not \"%s\"\n"),
+          event.filterable_data[1].name.in()
+        ));
+        result = -1;
+      }
+    }
+    else
+    {
+      ACE_ERROR ((LM_ERROR,
+        ACE_TEXT (
+          "TAO::FT_FaultConsumer::validate_event_type: "
+          "Received invalid structured event.\n"
+          "There must be at least two name/value pairs in "
+          "the filterable_data field, for \"FTDomainId\" and \"Location\".\n")
+      ));
+      result = -1;
+    }
+  }
+
+  return result;
+}
+
+/// Analyze a fault event.
+int TAO::FT_FaultConsumer::analyze_fault_event (
+  const CosNotification::StructuredEvent & event,
+  TAO::FT_FaultEventDescriptor & fault_event_desc
+  ACE_ENV_ARG_DECL_WITH_DEFAULTS
+  )
+  ACE_THROW_SPEC ((CORBA::SystemException, CosEventComm::Disconnected))
+{
+  int result = 0;
+
+  const CosNotification::FilterableEventBody & filterable =
+    event.filterable_data;
+  CORBA::ULong item_count = filterable.length ();
+#if (TAO_DEBUG_LEVEL_NEEDED == 1)
+  if (TAO_debug_level > 6)
+#endif /* (TAO_DEBUG_LEVEL_NEEDED == 1) */
+  {
+    for (CORBA::ULong n_prop = 0; n_prop < item_count; ++n_prop)
+    {
+      ACE_DEBUG ((LM_DEBUG,
+        ACE_TEXT("TAO::FT_FaultConsumer::analyze_fault_event: "
+                 "Property Name: <%s>\n"),
+        filterable[n_prop].name.in()
+      ));
+    }
+  }
+
+  //
+  // Populate the TAO::FT_FaultEventDescriptor structure from the
+  // properties in the event.
+  //
+
+  // Extract the location.
+  if (result == 0)
+  {
+    FT::Location* temp_loc;
+    if ((filterable[1].value >>= temp_loc) == 0)
+    {
+      ACE_ERROR ((LM_ERROR,
+        ACE_TEXT (
+          "TAO::FT_FaultConsumer::analyze_fault_event: "
+          "Could not extract Location value from fault event.\n")
+      ));
+      result = -1;
+    }
+    else
+    {
+      // Make a deep copy of the Location.
+      FT::Location* copy_loc = 0;
+      ACE_NEW_RETURN (copy_loc, FT::Location (*temp_loc), -1);
+      fault_event_desc.location = copy_loc;
+    }
+  }
+
+  // CORBA 3.0.2, section 23.4.5.1 states:
+  //
+  //   The fault detector may or may not set the TypeId and
+  //   ObjectGroupId fields with the following interpretations:
+  //   - Neither is set if all objects at the given location have failed.
+  //   - TypeId is set and ObjectGroupId is not set if all objects at
+  //     the given location with the given type have failed.
+  //   - Both are set if the member with the given ObjectGroupId at the
+  //     given location has failed.
+  if (result == 0)
+  {
+    // All objects at location failed.
+    if (item_count == 2)
+    {
+      fault_event_desc.all_at_location_failed = 1;
+    }
+
+    // All objects of type at location failed.
+    if (item_count == 3)
+    {
+      fault_event_desc.all_of_type_at_location_failed = 1;
+      fault_event_desc.type_id = this->extract_type_id (filterable[2].value);
+    }
+
+    // An object (replica) at a location failed.
+    if (item_count == 4)
+    {
+      fault_event_desc.object_at_location_failed = 1;
+      fault_event_desc.type_id = this->extract_type_id (filterable[2].value);
+      fault_event_desc.object_group_id =
+        this->extract_object_group_id (filterable[3].value);
+    }
+  }
+
+  //TODO: Finish the implementation of analyze_fault_event() to:
+  // - if the ReplicationStyle of the object group is FT::SEMI_ACTIVE,
+  //   and if the failed replica was the primary replica, set a new
+  //   primary replica
+  // - if the number of replicas falls below MinimumNumberReplicas
+  //   property and MembershipStyle of the object group is FT::MEMB_INF_CTRL,
+  //   create and add a new member.
+
+  //Q: How do we get the ObjectGroup from the ObjectGroupId?
+  //A: ??
+
+  //Q: Then, how do we get the properties of the ObjectGroup to
+  // figure out the ReplicationStyle, current primary,
+  // MinimumNumberReplicas, etc.?
+  //A: FT::PropertyManager::get_properties().
+
+  // Assuming we have the ObjectGroup reference...
+  FT::ObjectGroup_var the_object_group;
+
+  // Get the properties associated with this ObjectGroup.
+#if 0
+  if (result == 0)
+  {
+    FT::Properties_var properties =
+      this->replication_manager_->get_properties (
+        the_object_group.in()
+        ACE_ENV_ARG_PARAMETER);
+    ACE_CHECK_RETURN (-1);
+  }
+#endif
+
+  return result;
+}
+
+// Extract a string type_id from CORBA::Any.
+// Caller owns the returned string.
+char*
+TAO::FT_FaultConsumer::extract_type_id (const CORBA::Any& val)
+{
+  const char* type_id_value;
+  if ((val >>= type_id_value) == 0)
+  {
+    ACE_ERROR ((LM_ERROR,
+      ACE_TEXT (
+        "TAO::FT_FaultConsumer::extract_type_id: "
+        "Could not extract TypeId value from any.\n")
     ));
+    return 0;
+  }
+
+  return CORBA::string_dup (type_id_value);
+}
+
+// Extract the ObjectGroupId from CORBA::Any.
+FT::ObjectGroupId
+TAO::FT_FaultConsumer::extract_object_group_id (const CORBA::Any& val)
+{
+  FT::ObjectGroupId id = (FT::ObjectGroupId)0;
+  if ((val >>= id) == 0)
+  {
+    ACE_ERROR ((LM_ERROR,
+      ACE_TEXT (
+        "TAO::FT_FaultConsumer::extract_object_group_id: "
+        "Could not extract ObjectGroupId value from any.\n")
+    ));
+  }
+
+  return id;
 }
 
diff --git a/TAO/orbsvcs/FT_ReplicationManager/FT_FaultConsumer.h b/TAO/orbsvcs/FT_ReplicationManager/FT_FaultConsumer.h
index 559808a69d0..82f82a8d090 100755
--- a/TAO/orbsvcs/FT_ReplicationManager/FT_FaultConsumer.h
+++ b/TAO/orbsvcs/FT_ReplicationManager/FT_FaultConsumer.h
@@ -24,6 +24,7 @@
 #include "orbsvcs/CosNotifyCommS.h"
 #include "orbsvcs/FT_NotifierC.h"
 #include "orbsvcs/FT_ReplicationManagerC.h"
+#include "orbsvcs/FT_ReplicationManager/FT_FaultEventDescriptor.h"
 
 namespace TAO
 {
@@ -92,13 +93,48 @@ namespace TAO
   private:
 
     /**
+    * Validate event type to make sure it is one we can handle.
+    */
+    int validate_event_type (
+      const CosNotification::StructuredEvent & event);
+
+    /**
+    * Analyze a fault event.
+    */
+    int analyze_fault_event (
+      const CosNotification::StructuredEvent & event,
+      TAO::FT_FaultEventDescriptor & fault_event_desc
+      ACE_ENV_ARG_DECL_WITH_DEFAULTS)
+      ACE_THROW_SPEC ((CORBA::SystemException, CosEventComm::Disconnected));
+
+    /// Helper functions for fault analysis.
+    char* extract_type_id (const CORBA::Any& val);
+    FT::ObjectGroupId extract_object_group_id (const CORBA::Any& val);
+
+    /**
+    * Set a new primary member on an object group.
+    */
+    int set_new_primary_on_object_group (
+      FT::ObjectGroup_ptr object_group,
+      const FT::Location & the_location
+      ACE_ENV_ARG_DECL_WITH_DEFAULTS
+    )
+    ACE_THROW_SPEC ((
+      CORBA::SystemException
+      , PortableGroup::ObjectGroupNotFound
+      , PortableGroup::MemberNotFound
+      , FT::PrimaryNotSet
+      , FT::BadReplicationStyle
+    ));
+
+    /**
     * Extract the value of the MinimumNumberReplicas property from
     * the_criteria.
     */
     int get_minimum_number_replicas (
       const char * type_id,
       const PortableGroup::Criteria & the_criteria,
-      CORBA::UShort & minimum_number_replicas) const;
+      CORBA::UShort & minimum_number_replicas);
 
   public:
 
@@ -147,6 +183,9 @@ namespace TAO
     /// The POA with which we are activated.
     PortableServer::POA_var poa_;
 
+    /// The ObjectId from our activation in the POA.
+    PortableServer::ObjectId_var object_id_;
+
     /// The FaultNotifier's object reference.
     FT::FaultNotifier_var fault_notifier_;
 
diff --git a/TAO/orbsvcs/FT_ReplicationManager/FT_FaultEventDescriptor.cpp b/TAO/orbsvcs/FT_ReplicationManager/FT_FaultEventDescriptor.cpp
new file mode 100755
index 00000000000..cd998c80a97
--- /dev/null
+++ b/TAO/orbsvcs/FT_ReplicationManager/FT_FaultEventDescriptor.cpp
@@ -0,0 +1,33 @@
+/* -*- C++ -*- */
+//=============================================================================
+/**
+ *  @file    FT_FaultEventDescriptor.cpp
+ *
+ *  $Id$
+ *
+ *  This file is part of TAO's implementation of Fault Tolerant CORBA.
+ *  This file provides the implementation of the
+ *  TAO::FT_FaultEventDescriptor structure.  The
+ *  TAO::FT_FaultEventDescriptor is a helper type used during
+ *  analysis of fault events.
+ *
+ *  @author Steve Totten <totten_s@ociweb.com>
+ */
+//=============================================================================
+
+#include "orbsvcs/FT_ReplicationManager/FT_FaultEventDescriptor.h"
+
+ACE_RCSID (FT_FaultEventDescriptor,
+           FT_FaultEventDescriptor,
+           "$Id$")
+
+/// Default constructor.
+TAO::FT_FaultEventDescriptor::FT_FaultEventDescriptor ()
+  : all_at_location_failed (0)
+  , all_of_type_at_location_failed (0)
+  , object_at_location_failed (0)
+  , object_is_primary (0)
+  , object_group_id (PortableGroup::ObjectGroupId (0))
+{
+  type_id = CORBA::string_dup ("");
+}
diff --git a/TAO/orbsvcs/FT_ReplicationManager/FT_FaultEventDescriptor.h b/TAO/orbsvcs/FT_ReplicationManager/FT_FaultEventDescriptor.h
new file mode 100755
index 00000000000..735491b81e7
--- /dev/null
+++ b/TAO/orbsvcs/FT_ReplicationManager/FT_FaultEventDescriptor.h
@@ -0,0 +1,62 @@
+/* -*- C++ -*- */
+//=============================================================================
+/**
+ *  @file    FT_FaultEventDescriptor.h
+ *
+ *  $Id$
+ *
+ *  This file is part of TAO's implementation of Fault Tolerant CORBA.
+ *
+ *  @author Steve Totten <totten_s@ociweb.com>
+ */
+//=============================================================================
+
+
+#ifndef FT_FAULT_EVENT_DESCRIPTOR_H_
+#define FT_FAULT_EVENT_DESCRIPTOR_H_
+
+#include /**/ "ace/pre.h"
+
+#if !defined (ACE_LACKS_PRAGMA_ONCE)
+#pragma once
+#endif /* ACE_LACKS_PRAGMA_ONCE */
+
+#include "orbsvcs/FT_CORBAC.h"
+#include "orbsvcs/PortableGroupC.h"
+
+namespace TAO
+{
+  /// Helper class for describing the properties in a fault event.
+  struct FT_FaultEventDescriptor
+  {
+  public:
+
+    /// Default constructor.
+    FT_FaultEventDescriptor ();
+
+    ////////
+    /// Data members.
+
+    /// Flags indicating the "extent" of the fault.
+    int all_at_location_failed;
+    int all_of_type_at_location_failed;
+    int object_at_location_failed;
+    int object_is_primary;
+
+    /// The location of the fault.
+    FT::Location_var location;
+
+    /// The TypeId of the object that faulted.
+    FT::TypeId_var type_id;
+
+    /// The ObjectGroupId of the faulted object.
+    PortableGroup::ObjectGroupId object_group_id;
+
+  };
+
+} // namespace TAO
+
+#include /**/ "ace/post.h"
+
+#endif  /* FT_FAULT_EVENT_DESCRIPTOR_H_ */
+
diff --git a/TAO/orbsvcs/tests/FT_App/FT_App.mpc b/TAO/orbsvcs/tests/FT_App/FT_App.mpc
index 773a43fc6e5..c360c591d9d 100644
--- a/TAO/orbsvcs/tests/FT_App/FT_App.mpc
+++ b/TAO/orbsvcs/tests/FT_App/FT_App.mpc
@@ -87,6 +87,7 @@ project(*FaultConsumer): taoserver, fault_tolerance, orbsvcsexe {
     FTApp_FaultConsumer_Main.cpp
     ReplicationManagerFaultConsumerAdapter.cpp
     ../../FT_ReplicationManager/FT_FaultConsumer.cpp
+    ../../FT_ReplicationManager/FT_FaultEventDescriptor.cpp
   }
 
   // explicitly omit IDL files
diff --git a/TAO/orbsvcs/tests/FT_App/ReplicationManagerFaultConsumerAdapter.cpp b/TAO/orbsvcs/tests/FT_App/ReplicationManagerFaultConsumerAdapter.cpp
index 36833b2066e..0ad3bb04860 100755
--- a/TAO/orbsvcs/tests/FT_App/ReplicationManagerFaultConsumerAdapter.cpp
+++ b/TAO/orbsvcs/tests/FT_App/ReplicationManagerFaultConsumerAdapter.cpp
@@ -154,7 +154,7 @@ int ReplicationManagerFaultConsumerAdapter::parse_args (int argc, char * argv[])
  * Register this object.
  */
 int ReplicationManagerFaultConsumerAdapter::init (
-  TAO_ORB_Manager & orbManager
+  CORBA::ORB_ptr orb
   ACE_ENV_ARG_DECL_WITH_DEFAULTS)
 {
   ACE_DEBUG ((
@@ -163,7 +163,7 @@ int ReplicationManagerFaultConsumerAdapter::init (
   ));
 
   int result = 0;
-  this->orb_ = orbManager.orb();
+  this->orb_ = CORBA::ORB::_duplicate (orb);
 
   //////////////////////////////////////////
   // resolve reference to detector factory
@@ -176,8 +176,12 @@ int ReplicationManagerFaultConsumerAdapter::init (
 
   if (this->readIORFile(this->iorDetectorFile_, factoryIOR))
   {
-    CORBA::Object_var obj = this->orb_->string_to_object(factoryIOR);
-    this->factory_ = ::FT::FaultDetectorFactory::_narrow(obj);
+    CORBA::Object_var obj = this->orb_->string_to_object (
+      factoryIOR.in() ACE_ENV_ARG_PARAMETER);
+    ACE_CHECK_RETURN (-1);
+    this->factory_ = ::FT::FaultDetectorFactory::_narrow (
+      obj.in() ACE_ENV_ARG_PARAMETER);
+    ACE_CHECK_RETURN (-1);
     if (CORBA::is_nil (this->factory_.in()))
     {
       ACE_ERROR_RETURN ((
@@ -209,8 +213,12 @@ int ReplicationManagerFaultConsumerAdapter::init (
   CORBA::String_var notifierIOR;
   if (this->readIORFile(this->iorNotifierFile_, notifierIOR))
   {
-    CORBA::Object_var obj = this->orb_->string_to_object(notifierIOR);
-    this->notifier_ = ::FT::FaultNotifier::_narrow(obj);
+    CORBA::Object_var obj = this->orb_->string_to_object (
+      notifierIOR.in() ACE_ENV_ARG_PARAMETER);
+    ACE_CHECK_RETURN (-1);
+    this->notifier_ = ::FT::FaultNotifier::_narrow (
+      obj.in() ACE_ENV_ARG_PARAMETER);
+    ACE_CHECK_RETURN (-1);
     if (CORBA::is_nil (this->notifier_.in()))
     {
       ACE_ERROR_RETURN ((
@@ -255,18 +263,23 @@ int ReplicationManagerFaultConsumerAdapter::init (
   // - FT::FaultNotifier IOR.
   // - FT::ReplicationManager IOR (fake it for now).
 
-  // Get the RootPOA from the ORBManager.
-  PortableServer::POA_var poa = orbManager.root_poa();
+  // Get the RootPOA from the ORB.
+  CORBA::Object_var poa_obj = this->orb_->resolve_initial_references (
+    "RootPOA" ACE_ENV_ARG_PARAMETER);
+  ACE_CHECK_RETURN (-1);
+  PortableServer::POA_var poa = PortableServer::POA::_narrow (
+    poa_obj.in() ACE_ENV_ARG_PARAMETER);
+  ACE_CHECK_RETURN (-1);
 
   // Fake out the ReplicationManager IOR.
   CORBA::Object_var obj = this->orb_->string_to_object (
     "corbaloc::localhost:1900/ReplicationManager"
     ACE_ENV_ARG_PARAMETER);
-  ACE_TRY_CHECK;
+  ACE_CHECK_RETURN (-1);
   FT::ReplicationManager_var repl_mgr =
     FT::ReplicationManager::_unchecked_narrow (
       obj.in() ACE_ENV_ARG_PARAMETER);
-  ACE_TRY_CHECK;
+  ACE_CHECK_RETURN (-1);
   if (CORBA::is_nil (repl_mgr.in())) {
     ACE_ERROR_RETURN ((
       LM_ERROR,
@@ -290,7 +303,7 @@ int ReplicationManagerFaultConsumerAdapter::init (
     this->notifier_.in(),
     repl_mgr.in()
     ACE_ENV_ARG_PARAMETER);
-  ACE_TRY_CHECK;
+  ACE_CHECK_RETURN (-1);
   if (result != 0)
   {
     ACE_ERROR_RETURN ((
@@ -302,6 +315,13 @@ int ReplicationManagerFaultConsumerAdapter::init (
 
   this->identity_ = "ReplicationManagerFaultConsumerAdapter";
 
+  // Activate the RootPOA.
+  PortableServer::POAManager_var poa_manager =
+    poa->the_POAManager (ACE_ENV_SINGLE_ARG_PARAMETER);
+  ACE_CHECK_RETURN (-1);
+  poa_manager->activate (ACE_ENV_SINGLE_ARG_PARAMETER);
+  ACE_CHECK_RETURN (-1);
+
   /////////////////////////
   // Set up fault detectors
   if (result == 0)
@@ -310,15 +330,23 @@ int ReplicationManagerFaultConsumerAdapter::init (
     // resolve references to replicas
     // create a fault detector for each replica
     size_t replicaCount = this->iorReplicaFiles_.size();
-    for(size_t nRep = 0; result == 0 && nRep < replicaCount; ++nRep)
+    ACE_DEBUG ((LM_DEBUG,
+      ACE_TEXT ("Number of replicas being monitored: (%u)\n"),
+      ACE_static_cast (unsigned int, replicaCount)
+    ));
+    for (size_t nRep = 0; result == 0 && nRep < replicaCount; ++nRep)
     {
       const char * iorName = this->iorReplicaFiles_[nRep];
       CORBA::String_var ior;
       if (this->readIORFile(iorName, ior))
       {
-        CORBA::Object_var obj = this->orb_->string_to_object(ior);
-        FT::PullMonitorable_var replica = FT::PullMonitorable::_narrow(obj);
-        if (CORBA::is_nil(replica))
+        CORBA::Object_var obj = this->orb_->string_to_object (
+          ior.in() ACE_ENV_ARG_PARAMETER);
+        ACE_CHECK_RETURN (-1);
+        FT::PullMonitorable_var replica = FT::PullMonitorable::_narrow (
+          obj.in() ACE_ENV_ARG_PARAMETER);
+        ACE_CHECK_RETURN (-1);
+        if (CORBA::is_nil(replica.in()))
         {
           ACE_ERROR_RETURN ((
             LM_ERROR,
@@ -347,16 +375,19 @@ int ReplicationManagerFaultConsumerAdapter::init (
           encoder.add(::FT::FT_DOMAIN_ID, value);
 
           FT::Location object_location;
-          object_location.length(1);
-          object_location[0].id = CORBA::string_dup("Test location");
+          object_location.length(2);
+          object_location[0].id = CORBA::string_dup("test");
+          object_location[1].id = CORBA::string_dup("Location_A");
           value <<= object_location;
           encoder.add(::FT::FT_LOCATION, value);
 
-          FT::TypeId object_type = 0;
+          FT::TypeId_var object_type = CORBA::string_dup (
+            "IDL:org.omg/CosNaming/NamingContextExt:1.0");
           value <<= object_type;
           encoder.add(::FT::FT_TYPE_ID, value);
 
-          FT::ObjectGroupId group_id = 0;
+          FT::ObjectGroupId group_id =
+            ACE_static_cast (FT::ObjectGroupId, 6191982);
           value <<= group_id;
           encoder.add(::FT::FT_GROUP_ID, value);
 
@@ -382,7 +413,7 @@ int ReplicationManagerFaultConsumerAdapter::init (
               criteria.in(),
               factory_creation_id
               ACE_ENV_ARG_PARAMETER);
-            ACE_TRY_CHECK;
+            ACE_CHECK_RETURN (-1);
           }
         }
       }
diff --git a/TAO/orbsvcs/tests/FT_App/ReplicationManagerFaultConsumerAdapter.h b/TAO/orbsvcs/tests/FT_App/ReplicationManagerFaultConsumerAdapter.h
index 50438a04b5a..bb3fc17fd0d 100755
--- a/TAO/orbsvcs/tests/FT_App/ReplicationManagerFaultConsumerAdapter.h
+++ b/TAO/orbsvcs/tests/FT_App/ReplicationManagerFaultConsumerAdapter.h
@@ -44,7 +44,7 @@ public:
   /**
    * Initialize this object
    */
-  int init (TAO_ORB_Manager & orbManager ACE_ENV_ARG_DECL_WITH_DEFAULTS);
+  int init (CORBA::ORB_ptr orb ACE_ENV_ARG_DECL_WITH_DEFAULTS);
 
   /**
    * Return a string to identify this object for logging/console message purposes.
diff --git a/TAO/orbsvcs/tests/FT_App/run_test_fault_consumer.pl b/TAO/orbsvcs/tests/FT_App/run_test_fault_consumer.pl
index 9447955cb05..a75760fbbdd 100755
--- a/TAO/orbsvcs/tests/FT_App/run_test_fault_consumer.pl
+++ b/TAO/orbsvcs/tests/FT_App/run_test_fault_consumer.pl
@@ -83,7 +83,7 @@ use PerlACE::Run_Test;
 #command line options
 #set defaults:
 my($verbose) = 0;         # 1: report perl actions before executing them
-my($debug_builds) = 1;    # 0: use exes from Release directories
+my($debug_builds) = 0;    # 0: use exes from Release directories
 my($simulated) = 1;       # 1: use "client simulated" fault tolerance
 
 foreach $i (@ARGV) {
@@ -136,11 +136,11 @@ unlink $client_data;
 
 my($status) = 0;
 
-my($REP1) = new PerlACE::Process (".$build_directory/ft_replica", "-o $factory1_ior -t $replica1_ior -r 1 -q");
-my($REP2) = new PerlACE::Process (".$build_directory/ft_replica", "-o $factory2_ior -t $replica2_ior -r 2 -q");
+my($REP1) = new PerlACE::Process (".$build_directory/ft_replica", "-o $factory1_ior -t $replica1_ior -l loc1 -i type1 -q");
+my($REP2) = new PerlACE::Process (".$build_directory/ft_replica", "-o $factory2_ior -t $replica2_ior -l loc2 -i type1 -q");
 my($DET) = new PerlACE::Process ("$ENV{'TAO_ROOT'}/orbsvcs/Fault_Detector$build_directory/Fault_Detector", "-o $detector_ior -q");
 my($NOT) = new PerlACE::Process ("$ENV{'TAO_ROOT'}/orbsvcs/Fault_Notifier$build_directory/Fault_Notifier", "-o $notifier_ior -v -q");
-my($ANA) = new PerlACE::Process (".$build_directory/ft_fault_consumer", "-o $ready_file -n $notifier_ior -q -d $detector_ior -r $replica1_ior,$replica2_ior");
+my($CONS) = new PerlACE::Process (".$build_directory/ft_fault_consumer", "-o $ready_file -n $notifier_ior -q -d $detector_ior -r $replica1_ior,$replica2_ior");
 
 my($CL);
 if (simulated) {
@@ -196,8 +196,8 @@ if (PerlACE::waitforfile_timed ($notifier_ior, 5) == -1) {
     exit 1;
 }
 
-print "\nTEST: starting fault consumer " . $ANA->CommandLine . "\n" if ($verbose);
-$ANA->Spawn ();
+print "\nTEST: starting fault consumer " . $CONS->CommandLine . "\n" if ($verbose);
+$CONS->Spawn ();
 
 print "TEST: waiting for READY.FILE from fault consumer\n" if ($verbose);
 if (PerlACE::waitforfile_timed ($ready_file, 5) == -1) {
@@ -206,7 +206,7 @@ if (PerlACE::waitforfile_timed ($ready_file, 5) == -1) {
     $REP2->Kill (); $REP2->TimedWait (1);
     $DET->Kill (); $DET->TimedWait(1);
     $NOT->Kill (); $NOT->TimedWait(1);
-    $ANA->Kill (); $ANA->TimedWait(1);
+    $CONS->Kill (); $CONS->TimedWait(1);
     exit 1;
 }
 
@@ -219,37 +219,37 @@ if ($client != 0) {
 }
 
 print "\nTEST: wait for replica 1.\n" if ($verbose);
-$replica1 = $REP1->WaitKill (60);
+$replica1 = $REP1->WaitKill (5);
 if ($replica1 != 0) {
     print STDERR "ERROR: replica returned $replica1\n";
     $status = 1;
 }
 
 print "\nTEST: wait for replica 2.\n" if ($verbose);
-$replica2 = $REP2->WaitKill (60);
+$replica2 = $REP2->WaitKill (5);
 if ($replica2 != 0) {
     print STDERR "ERROR: replica returned $replica2\n";
     $status = 1;
 }
 
 print "\nTEST: wait for detector factory to leave.\n" if ($verbose);
-$detector = $DET->WaitKill (60);
+$detector = $DET->WaitKill (20);
 if ($detector != 0) {
     print STDERR "ERROR: detector returned $detector\n";
     $status = 1;
 }
 
 print "\nTEST: wait for notifier to leave.\n" if ($verbose);
-$notifier = $NOT->WaitKill (60);
+$notifier = $NOT->WaitKill (20);
 if ($notifier != 0) {
     print STDERR "ERROR: notifier returned $notifier\n";
     $status = 1;
 }
 
 print "\nTEST: wait for fault consumer to leave.\n" if ($verbose);
-$analyzer = $ANA->WaitKill (60);
-if ($analyzer != 0) {
-    print STDERR "ERROR: fault consumer returned $analyzer\n";
+$consumer = $CONS->WaitKill (20);
+if ($consumer != 0) {
+    print STDERR "ERROR: fault consumer returned $consumer\n";
     $status = 1;
 }