ChangeLogTag: Tue Oct 14 18:30:45 2003 Steve Totten <totten_s@ociweb.com>

6 files changed, 169 insertions, 73 deletions

diff --git a/TAO/ChangeLog b/TAO/ChangeLog
index 83ce2974ba9..aa94c70e5ef 100644
--- a/TAO/ChangeLog
+++ b/TAO/ChangeLog
@@ -1,3 +1,19 @@
+Tue Oct 14 18:30:45 2003  Steve Totten  <totten_s@ociweb.com>
+
+        * orbsvcs/FT_ReplicationManager/FT_ReplicationManager.cpp:
+          Removed some useless debugging statements.
+
+        * orbsvcs/FT_ReplicationManager/FT_ReplicationManagerFaultAnalyzer.cpp:
+        * orbsvcs/FT_ReplicationManager/FT_ReplicationManagerFaultAnalyzer.h:
+          Added fault analysis logic to determine if the failed
+          object was the primary member of its object group.
+
+        * orbsvcs/tests/FT_App/run_test_fault_consumer.pl:
+        * orbsvcs/tests/FT_App/run_test_replication_mgr.pl:
+          Fixed scripts to use new command line syntax.
+
+        These changes were made in the oci_haft branch.
+
 Tue Oct 14 15:21:19 2003  Dale Wilson  <wilson_d@ociweb.com>
 
         * orbsvcs/tests/FT_App/ReplicationManagerFaultConsumerAdapter.cpp:
@@ -5,7 +21,6 @@ Tue Oct 14 15:21:19 2003  Dale Wilson  <wilson_d@ociweb.com>
           change command line option that specify multiple
           eliminate read_ior_file.  Expect file:// ior on command line instead.
 
-
         These changes were made in the oci_haft branch.
 
 Tue Oct 14 13:52:18 2003  Steve Totten  <totten_s@ociweb.com>
diff --git a/TAO/orbsvcs/FT_ReplicationManager/FT_ReplicationManager.cpp b/TAO/orbsvcs/FT_ReplicationManager/FT_ReplicationManager.cpp
index 8f9a0b5c42e..ec007311810 100644
--- a/TAO/orbsvcs/FT_ReplicationManager/FT_ReplicationManager.cpp
+++ b/TAO/orbsvcs/FT_ReplicationManager/FT_ReplicationManager.cpp
@@ -329,17 +329,6 @@ TAO::FT_ReplicationManager::register_fault_notifier_i (
   ACE_ENV_ARG_DECL)
   ACE_THROW_SPEC ((CORBA::SystemException))
 {
-
-#if (TAO_DEBUG_LEVEL_NEEDED == 1)
-  if (TAO_debug_level > 1)
-#endif /* (TAO_DEBUG_LEVEL_NEEDED == 1) */
-  {
-    ACE_DEBUG ((LM_DEBUG,
-      ACE_TEXT (
-        "Enter TAO::FT_ReplicationManager::register_fault_notifier_i.\n")
-    ));
-  }
-
   if (CORBA::is_nil (fault_notifier))
   {
     ACE_ERROR ((LM_ERROR,
@@ -354,17 +343,6 @@ TAO::FT_ReplicationManager::register_fault_notifier_i (
       CORBA::COMPLETED_NO));
   }
 
-#if (TAO_DEBUG_LEVEL_NEEDED == 1)
-  if (TAO_debug_level > 1)
-#endif /* (TAO_DEBUG_LEVEL_NEEDED == 1) */
-  {
-    ACE_DEBUG ((LM_DEBUG,
-      ACE_TEXT (
-        "TAO::FT_ReplicationManager::register_fault_notifier_i: "
-        "Duplicate FaultNotifier object reference.\n")
-    ));
-  }
-
   // Cache new Fault Notifier object reference.
   this->fault_notifier_ = FT::FaultNotifier::_duplicate (fault_notifier);
 
@@ -373,32 +351,9 @@ TAO::FT_ReplicationManager::register_fault_notifier_i (
   int result = 0;
   ACE_TRY_NEW_ENV
   {
-
-#if (TAO_DEBUG_LEVEL_NEEDED == 1)
-    if (TAO_debug_level > 1)
-#endif /* (TAO_DEBUG_LEVEL_NEEDED == 1) */
-    {
-      ACE_DEBUG ((LM_DEBUG,
-        ACE_TEXT (
-          "TAO::FT_ReplicationManager::register_fault_notifier_i: "
-          "Calling fault_consumer_.fini().\n")
-      ));
-    }
-
     result = this->fault_consumer_.fini (ACE_ENV_SINGLE_ARG_PARAMETER);
     ACE_TRY_CHECK;
 
-#if (TAO_DEBUG_LEVEL_NEEDED == 1)
-    if (TAO_debug_level > 1)
-#endif /* (TAO_DEBUG_LEVEL_NEEDED == 1) */
-    {
-      ACE_DEBUG ((LM_DEBUG,
-        ACE_TEXT (
-          "TAO::FT_ReplicationManager::register_fault_notifier_i: "
-          "Calling fault_consumer_.init().\n")
-      ));
-    }
-
     // Create a fault analyzer.
     TAO::FT_FaultAnalyzer * analyzer = 0;
     ACE_NEW_NORETURN (
@@ -448,17 +403,6 @@ TAO::FT_ReplicationManager::register_fault_notifier_i (
         EINVAL),
       CORBA::COMPLETED_NO));
   }
-
-#if (TAO_DEBUG_LEVEL_NEEDED == 1)
-  if (TAO_debug_level > 1)
-#endif /* (TAO_DEBUG_LEVEL_NEEDED == 1) */
-  {
-    ACE_DEBUG ((LM_DEBUG,
-      ACE_TEXT (
-        "Leave TAO::FT_ReplicationManager::register_fault_notifier_i.\n")
-    ));
-  }
-
 }
 
 
diff --git a/TAO/orbsvcs/FT_ReplicationManager/FT_ReplicationManagerFaultAnalyzer.cpp b/TAO/orbsvcs/FT_ReplicationManager/FT_ReplicationManagerFaultAnalyzer.cpp
index a1cd7f7ea4e..f6370fdb3e2 100755
--- a/TAO/orbsvcs/FT_ReplicationManager/FT_ReplicationManagerFaultAnalyzer.cpp
+++ b/TAO/orbsvcs/FT_ReplicationManager/FT_ReplicationManagerFaultAnalyzer.cpp
@@ -17,6 +17,7 @@
 #include "orbsvcs/FT_ReplicationManager/FT_ReplicationManager.h"
 #include "orbsvcs/FT_ReplicationManager/FT_FaultEventDescriptor.h"
 #include "orbsvcs/PortableGroup/PG_Property_Utils.h"
+#include "orbsvcs/FaultTolerance/FT_IOGR_Property.h"
 #include <tao/debug.h>
 
 #define INTEGRATED_WITH_REPLICATION_MANAGER 1
@@ -143,15 +144,18 @@ int TAO::FT_ReplicationManagerFaultAnalyzer::analyze_fault_event (
   //   property and MembershipStyle of the object group is FT::MEMB_INF_CTRL,
   //   create and add a new member.
 
-  //@@ Q: How do we get the ObjectGroup from the ObjectGroupId?
-  //@@ A: Use TAO-specific extension to PortableGroup::ObjectGroupManager
-  // interface (get_object_group_ref_from_id()).
 #if (INTEGRATED_WITH_REPLICATION_MANAGER == 1)
-  if ((fault_event_desc.object_at_location_failed == 1) && (result == 0))
+  // If a specific object at a location failed, we need to determine
+  // if it was the primary replica.
+  if ((result == 0) &&
+      (fault_event_desc.object_at_location_failed == 1))
   {
     FT::ObjectGroup_var the_object_group = FT::ObjectGroup::_nil();
     ACE_TRY_NEW_ENV
     {
+      //@@ Q: How do we get the ObjectGroup from the ObjectGroupId?
+      //@@ A: Use TAO-specific extension to PortableGroup::ObjectGroupManager
+      // interface (get_object_group_ref_from_id()).
       the_object_group =
         this->replication_manager_->get_object_group_ref_from_id (
           fault_event_desc.object_group_id
@@ -160,8 +164,12 @@ int TAO::FT_ReplicationManagerFaultAnalyzer::analyze_fault_event (
     }
     ACE_CATCH (PortableGroup::ObjectGroupNotFound, ex)
     {
-      ACE_PRINT_EXCEPTION (ex,
-        "TAO::FT_ReplicationManagerFaultAnalyzer::analyze_fault_event: ");
+      ACE_PRINT_EXCEPTION (
+        ex,
+        ACE_TEXT (
+          "TAO::FT_ReplicationManagerFaultAnalyzer::analyze_fault_event: ")
+      );
+      result = -1;
     }
     ACE_ENDTRY;
 
@@ -177,8 +185,7 @@ int TAO::FT_ReplicationManagerFaultAnalyzer::analyze_fault_event (
     }
 
     //@@ Q: How do we get the properties of the ObjectGroup to
-    // figure out the ReplicationStyle, current primary,
-    // MinimumNumberReplicas, etc.?
+    // figure out the ReplicationStyle, MinimumNumberReplicas, etc.?
     //@@ A: FT::PropertyManager::get_properties().
 
     // Get the properties associated with this ObjectGroup.
@@ -214,7 +221,7 @@ int TAO::FT_ReplicationManagerFaultAnalyzer::analyze_fault_event (
             ACE_TEXT (
               "TAO::FT_ReplicationManagerFaultAnalyzer::validate_event_type: "
               "MembershipStyleValue = <%d>"),
-              fault_event_desc.membership_style 
+              fault_event_desc.membership_style
           ));
         }
       }
@@ -243,7 +250,7 @@ int TAO::FT_ReplicationManagerFaultAnalyzer::analyze_fault_event (
             ACE_TEXT (
               "TAO::FT_ReplicationManagerFaultAnalyzer::validate_event_type: "
               "ReplicationStyleValue = <%d>"),
-              fault_event_desc.replication_style 
+              fault_event_desc.replication_style
           ));
         }
       }
@@ -273,7 +280,7 @@ int TAO::FT_ReplicationManagerFaultAnalyzer::analyze_fault_event (
             ACE_TEXT (
               "TAO::FT_ReplicationManagerFaultAnalyzer::validate_event_type: "
               "MinimumNumberReplicas = <%d>"),
-              fault_event_desc.minimum_number_replicas 
+              fault_event_desc.minimum_number_replicas
           ));
         }
       }
@@ -303,11 +310,26 @@ int TAO::FT_ReplicationManagerFaultAnalyzer::analyze_fault_event (
             ACE_TEXT (
               "TAO::FT_ReplicationManagerFaultAnalyzer::validate_event_type: "
               "InitialNumberReplicas = <%d>"),
-              fault_event_desc.initial_number_replicas 
+              fault_event_desc.initial_number_replicas
           ));
         }
       }
     }
+
+    // If the ReplicationStyle is COLD_PASSIVE, WARM_PASSIVE, or
+    // SEMI_ACTIVE, we can see if it was the primary replica that
+    // failed.
+    if ((result == 0) &&
+        (fault_event_desc.replication_style == FT::COLD_PASSIVE ||
+         fault_event_desc.replication_style == FT::WARM_PASSIVE ||
+         fault_event_desc.replication_style == FT::SEMI_ACTIVE))
+    {
+      result = this->is_primary_member (
+        the_object_group.in(),
+        fault_event_desc.location,
+        fault_event_desc.object_is_primary);
+    }
+
   }
 #endif /* (INTEGRATED_WITH_REPLICATION_MANAGER == 1) */
 
@@ -540,3 +562,110 @@ int TAO::FT_ReplicationManagerFaultAnalyzer::get_initial_number_replicas (
   return result;
 }
 
+int TAO::FT_ReplicationManagerFaultAnalyzer::is_primary_member (
+  const FT::ObjectGroup_ptr iogr,
+  const FT::Location_var & location,
+  int & object_is_primary)
+{
+
+  //@@ Q: How do we determine if this was a primary that faulted?
+  //@@ A: Get the TagFTGroupTaggedComponent from the IOGR and search
+  // for the primary, using the TAO_FT_IOGR_Property helper class.
+  // Then, compare the TypeId and Location of the failed object with
+  // those of the primary.  If they match, it was a primary fault.
+
+  int result = 0;
+  object_is_primary = 0;
+
+  ACE_TRY_NEW_ENV
+  {
+    // Create an "empty" TAO_FT_IOGR_Property and use it to get the
+    // tagged component.
+    TAO_FT_IOGR_Property temp_ft_prop;
+    FT::TagFTGroupTaggedComponent ft_group_tagged_component;
+    CORBA::Boolean got_tagged_component =
+      temp_ft_prop.get_tagged_component (
+          iogr, ft_group_tagged_component ACE_ENV_ARG_PARAMETER);
+    ACE_TRY_CHECK;
+    if (got_tagged_component)
+    {
+      // Create a new TAO_FT_IOGR_Property with the tagged
+      // component.
+      TAO_FT_IOGR_Property ft_prop (ft_group_tagged_component);
+
+      // Check to see if a primary is set.
+      CORBA::Boolean primary_is_set = ft_prop.is_primary_set (
+        iogr ACE_ENV_ARG_PARAMETER);
+      ACE_TRY_CHECK;
+      if (primary_is_set)
+      {
+        // Get the primary object.
+        CORBA::Object_var primary_obj = ft_prop.get_primary (
+          iogr ACE_ENV_ARG_PARAMETER);
+        ACE_TRY_CHECK;
+        if (CORBA::is_nil (primary_obj.in()))
+        {
+          ACE_ERROR_RETURN ((LM_ERROR,
+            ACE_TEXT (
+              "TAO::FT_ReplicationManagerFaultAnalyzer::is_primary_member: "
+              "Could not get primary IOR from IOGR.\n")),
+            -1);
+        }
+
+        // Get the object reference of the failed member.
+        CORBA::Object_var failed_obj =
+          this->replication_manager_->get_member_ref (
+            iogr, location.in() ACE_ENV_ARG_PARAMETER);
+        ACE_TRY_CHECK;
+        if (CORBA::is_nil (failed_obj.in()))
+        {
+          ACE_ERROR_RETURN ((LM_ERROR,
+            ACE_TEXT (
+              "TAO::FT_ReplicationManagerFaultAnalyzer::is_primary_member: "
+              "Could not get IOR of failed member from IOGR.\n")),
+            -1);
+        }
+
+        // Are the two object refs (primary and failed) equivalent?
+        CORBA::Boolean equiv = primary_obj->_is_equivalent (
+          failed_obj.in() ACE_ENV_ARG_PARAMETER);
+        ACE_TRY_CHECK;
+        if (equiv)
+        {
+          object_is_primary = 1;
+          result = 0;
+        }
+      }
+      else  // primary is not set
+      {
+        ACE_ERROR ((LM_ERROR,
+          ACE_TEXT (
+            "TAO::FT_ReplicationManagerFaultAnalyzer::is_primary_member: "
+            "Primary is not set on IOGR.\n")
+        ));
+        result = -1;
+      }
+    }
+    else // could not get tagged component
+    {
+      ACE_ERROR ((LM_ERROR,
+        ACE_TEXT (
+          "TAO::FT_ReplicationManagerFaultAnalyzer::is_primary_member: "
+          "Could not get tagged component from IOGR.\n")
+      ));
+      result = -1;
+    }
+  }
+  ACE_CATCHANY
+  {
+    ACE_PRINT_EXCEPTION (
+      ACE_ANY_EXCEPTION,
+      ACE_TEXT (
+        "TAO::FT_ReplicationManagerFaultAnalyzer::is_primary_member: ")
+    );
+    result = -1;
+  }
+  ACE_ENDTRY;
+
+  return result;
+}
diff --git a/TAO/orbsvcs/FT_ReplicationManager/FT_ReplicationManagerFaultAnalyzer.h b/TAO/orbsvcs/FT_ReplicationManager/FT_ReplicationManagerFaultAnalyzer.h
index 5b31d600e38..48d8bc4ef33 100755
--- a/TAO/orbsvcs/FT_ReplicationManager/FT_ReplicationManagerFaultAnalyzer.h
+++ b/TAO/orbsvcs/FT_ReplicationManager/FT_ReplicationManagerFaultAnalyzer.h
@@ -109,6 +109,14 @@ namespace TAO
       const FT::Properties & properties,
       FT::InitialNumberReplicasValue & initial_number_replicas);
 
+    // Is the replica at location the primary member of iogr?
+    // Sets is_primary and returns 0 on success.
+    // Returns -1 on failure.
+    int is_primary_member (
+      const FT::ObjectGroup_ptr iogr,
+      const FT::Location_var & location,
+      int & object_is_primary);
+
     ///////////////
     // Data Members
   private:
diff --git a/TAO/orbsvcs/tests/FT_App/run_test_fault_consumer.pl b/TAO/orbsvcs/tests/FT_App/run_test_fault_consumer.pl
index 5a3dd149d51..61aa981c79a 100755
--- a/TAO/orbsvcs/tests/FT_App/run_test_fault_consumer.pl
+++ b/TAO/orbsvcs/tests/FT_App/run_test_fault_consumer.pl
@@ -140,7 +140,7 @@ my($REP1) = new PerlACE::Process (".$build_directory/ft_replica", "-o $factory1_
 my($REP2) = new PerlACE::Process (".$build_directory/ft_replica", "-o $factory2_ior -t $replica2_ior -l loc2 -i type1 -q");
 my($DET) = new PerlACE::Process ("$ENV{'TAO_ROOT'}/orbsvcs/Fault_Detector$build_directory/Fault_Detector", "-o $detector_ior -q");
 my($NOT) = new PerlACE::Process ("$ENV{'TAO_ROOT'}/orbsvcs/Fault_Notifier$build_directory/Fault_Notifier", "-o $notifier_ior -v -q");
-my($CONS) = new PerlACE::Process (".$build_directory/ft_fault_consumer", "-o $ready_file -n $notifier_ior -q -d $detector_ior -r $replica1_ior,$replica2_ior");
+my($CONS) = new PerlACE::Process (".$build_directory/ft_fault_consumer", "-o $ready_file -n file://$notifier_ior -q -d file://$detector_ior -r file://$replica1_ior -r file://$replica2_ior");
 
 my($CL);
 if (simulated) {
diff --git a/TAO/orbsvcs/tests/FT_App/run_test_replication_mgr.pl b/TAO/orbsvcs/tests/FT_App/run_test_replication_mgr.pl
index e70944c75fd..4325c4e7060 100755
--- a/TAO/orbsvcs/tests/FT_App/run_test_replication_mgr.pl
+++ b/TAO/orbsvcs/tests/FT_App/run_test_replication_mgr.pl
@@ -143,15 +143,15 @@ my($REP1) = new PerlACE::Process (".$build_directory/ft_replica", "-o $factory1_
 my($REP2) = new PerlACE::Process (".$build_directory/ft_replica", "-o $factory2_ior -t $replica2_ior -l loc2 -i type1 -q");
 my($DET) = new PerlACE::Process ("$ENV{'TAO_ROOT'}/orbsvcs/Fault_Detector$build_directory/Fault_Detector", "-o $detector_ior -q");
 my($NOT) = new PerlACE::Process ("$ENV{'TAO_ROOT'}/orbsvcs/Fault_Notifier$build_directory/Fault_Notifier", "-o $notifier_ior -v -q");
-my($CONS) = new PerlACE::Process (".$build_directory/ft_fault_consumer", "-o $ready_file -n $notifier_ior -q -d $detector_ior -r$replica1_ior,$replica2_ior");
+my($CONS) = new PerlACE::Process (".$build_directory/ft_fault_consumer", "-o $ready_file -n file://$notifier_ior -q -d file://$detector_ior -r file://$replica1_ior -r file://$replica2_ior");
 my($REPLM) = new PerlACE::Process ("$ENV{'TAO_ROOT'}/orbsvcs/FT_ReplicationManager$build_directory/FT_ReplicationManager", "-o $replmgr_ior -f file://$notifier_ior");
 
 my($CL);
 if (simulated) {
-  $CL = new PerlACE::Process (".$build_directory/ft_client", "-f $replica1_ior,$replica2_ior -c testscript");
+  $CL = new PerlACE::Process (".$build_directory/ft_client", "-f file://$replica1_ior -f file://$replica2_ior -c testscript");
 }else{
   #todo figure out how to get iogr
-  $CL = new PerlACE::Process (".$build_directory/ft_client", "-f $replica1_iogr -c testscript");
+  $CL = new PerlACE::Process (".$build_directory/ft_client", "-f  -f file://$replica1_iogr -c testscript");
 }
 
 print "TEST: starting replica1 " . $REP1->CommandLine . "\n" if ($verbose);