Fri Jun 18 03:25:00 UTC 2010 William R. Otte <wotte@dre.vanderbilt.edu>

        * DAnCE/LocalityManager/Scheduler/Deployment_Completion.h:
        * DAnCE/LocalityManager/Scheduler/Deployment_Completion.cpp:
        * DAnCE/LocalityManager/Scheduler/Deployment_Event.h:
        * DAnCE/LocalityManager/Scheduler/Deployment_Event.inl:
        * DAnCE/LocalityManager/Scheduler/Deployment_Event.cpp:
        * DAnCE/LocalityManager/Scheduler/Deployment_Events.h:
        * DAnCE/LocalityManager/Scheduler/Deployment_Events.inl:
        * DAnCE/LocalityManager/Scheduler/Deployment_Events.cpp:
        * DAnCE/LocalityManager/Scheduler/Scheduler.mpc:

          Improved mechanism to wait on completion of deployment
          events based on the completion counter.

        * DAnCE/NodeApplication/NodeApplication_Impl.h:
        * DAnCE/NodeApplication/NodeApplication_Impl.cpp:

          Updated to use new completion system.

        * DAnCE/tools/Completion/Completion_Counter_Base.h:
        * DAnCE/tools/Completion/Completion_Counter_Base.inl:

          Added new method so subclasses can increment expected
          events.  Also resolved possible deadlock condition
          by releasing locks before making outbound calls
          on user code.

        * DAnCE/LocalityManager/Scheduler/Deployment_Events.tpp:

          Removed this file.

15 files changed, 340 insertions, 114 deletions

diff --git a/CIAO/ChangeLog b/CIAO/ChangeLog
index b8bbbf17586..31de2cb476f 100644
--- a/CIAO/ChangeLog
+++ b/CIAO/ChangeLog
@@ -1,18 +1,50 @@
+Fri Jun 18 03:25:00 UTC 2010  William R. Otte  <wotte@dre.vanderbilt.edu>
+
+        * DAnCE/LocalityManager/Scheduler/Deployment_Completion.h:
+        * DAnCE/LocalityManager/Scheduler/Deployment_Completion.cpp:
+        * DAnCE/LocalityManager/Scheduler/Deployment_Event.h:
+        * DAnCE/LocalityManager/Scheduler/Deployment_Event.inl:
+        * DAnCE/LocalityManager/Scheduler/Deployment_Event.cpp:
+        * DAnCE/LocalityManager/Scheduler/Deployment_Events.h:
+        * DAnCE/LocalityManager/Scheduler/Deployment_Events.inl:
+        * DAnCE/LocalityManager/Scheduler/Deployment_Events.cpp:
+        * DAnCE/LocalityManager/Scheduler/Scheduler.mpc:
+
+          Improved mechanism to wait on completion of deployment
+          events based on the completion counter.
+
+        * DAnCE/NodeApplication/NodeApplication_Impl.h:
+        * DAnCE/NodeApplication/NodeApplication_Impl.cpp:
+
+          Updated to use new completion system.
+
+        * DAnCE/tools/Completion/Completion_Counter_Base.h:
+        * DAnCE/tools/Completion/Completion_Counter_Base.inl:
+
+          Added new method so subclasses can increment expected
+          events.  Also resolved possible deadlock condition
+          by releasing locks before making outbound calls
+          on user code.
+
+        * DAnCE/LocalityManager/Scheduler/Deployment_Events.tpp:
+
+          Removed this file.
+
 Thu Jun 17 19:22:51 UTC 2010  William R. Otte  <wotte@dre.vanderbilt.edu>
 
         * DAnCE/LocalityManager/Daemon/Locality_Manager_Impl.h:
-        
-          Layout change. 
+
+          Layout change.
 
         * DAnCE/LocalityManager/Handler/LocalityActivator_Impl.h:
         * DAnCE/LocalityManager/Handler/LocalityActivator_Impl.cpp:
-        
+
           Bugfixes in the callback code to accomodate non-orb managed threadpools.
 
         * DAnCE/LocalityManager/Handler/Locality_Manager_Handler_Impl.h:
         * DAnCE/LocalityManager/Handler/Locality_Manager_Handler_Impl.cpp:
 
-          Changed how internal Activator is configured. 
+          Changed how internal Activator is configured.
 
         * DAnCE/LocalityManager/Scheduler/Deployment_Events.h:
         * DAnCE/LocalityManager/Scheduler/Deployment_Events.cpp:
diff --git a/CIAO/DAnCE/LocalityManager/Scheduler/Deployment_Completion.cpp b/CIAO/DAnCE/LocalityManager/Scheduler/Deployment_Completion.cpp
new file mode 100644
index 00000000000..5ec39122a6d
--- /dev/null
+++ b/CIAO/DAnCE/LocalityManager/Scheduler/Deployment_Completion.cpp
@@ -0,0 +1,71 @@
+// $Id$
+#include "Deployment_Completion.h"
+
+#include "Logger/Log_Macros.h"
+
+namespace DAnCE
+{
+  Deployment_Completion::Deployment_Completion (void)
+    : Completion_Counter_Base< TAO_SYNCH_MUTEX > (0, 0),
+      mutex_ (),
+      condition_ (this->mutex_)
+  {
+  }
+    
+  Deployment_Completion::~Deployment_Completion (void)
+  {
+  }
+  
+  void
+  Deployment_Completion::accept (Event_Future &future)
+  {
+    future.attach (this);
+    this->increment_exec_count ();
+  }
+
+  void 
+  Deployment_Completion::update (const Event_Future &future)
+  {
+    this->decrement_exec_count ();
+  }
+    
+  bool
+  Deployment_Completion::wait_on_completion (ACE_Time_Value *tv)
+  {
+    ACE_GUARD_RETURN (TAO_SYNCH_MUTEX,
+                      guard,
+                      this->mutex_, false);
+    
+    while (!this->all_completed ())
+      {
+        int retval = this->condition_.wait (tv);
+        
+        if (retval == -1)
+          {
+            DANCE_ERROR (2, (LM_ERROR, DLINFO
+                             ACE_TEXT ("Deployment_Completion::wait_on_completion - ")
+                             ACE_TEXT ("Timed out waiting on event completion\n")));
+            return false;
+          }
+      }
+    
+    DANCE_DEBUG (8, (LM_DEBUG, DLINFO
+                     ACE_TEXT ("Deployment_Completion::wait_on_completion - ")
+                     ACE_TEXT ("All events completed\n")));
+
+    return true;
+  }
+  
+  void
+  Deployment_Completion::on_all_completed ()
+  {
+    this->condition_.broadcast ();
+  }
+  
+  void 
+  Deployment_Completion::on_all_completed_with_failure ()
+  {
+    this->condition_.broadcast ();
+  }
+}
+
diff --git a/CIAO/DAnCE/LocalityManager/Scheduler/Deployment_Completion.h b/CIAO/DAnCE/LocalityManager/Scheduler/Deployment_Completion.h
new file mode 100644
index 00000000000..6fdf2eff75d
--- /dev/null
+++ b/CIAO/DAnCE/LocalityManager/Scheduler/Deployment_Completion.h
@@ -0,0 +1,55 @@
+/**
+ * @file Deployment_Completion.h
+ * @author William R. Otte <wotte@dre.vanderbilt.edu>
+ *
+ * $Id$
+ *
+ * Future observer that signals when the scheduler has completed.
+ */
+
+#ifndef DEPLOYMENT_COMPLETION_H
+#define DEPLOYMENT_COMPLETION_H
+
+#include /**/ "ace/pre.h"
+
+#include "ace/Condition_T.h"
+#include "ace/Future.h"
+#include "Completion/Completion_Counter_Base.h"
+#include "LocalityManager/Scheduler/Deployment_Event.h"
+
+namespace DAnCE
+{
+  /**
+   * @class Event_Handler
+   */
+  class Deployment_Scheduler_Export Deployment_Completion
+    : public virtual ACE_Future_Observer< Event_Result >,
+      public virtual Completion_Counter_Base< TAO_SYNCH_MUTEX >
+  {
+  public:
+    Deployment_Completion (void);
+
+    virtual ~Deployment_Completion (void);
+
+    virtual void update (const Event_Future &future);
+    
+    void accept (Event_Future &future);
+
+    /// Waits for completion of all events, with a timeout
+    bool wait_on_completion (ACE_Time_Value *tv);
+
+  protected:
+    virtual void on_all_completed ();
+
+    virtual void on_all_completed_with_failure ();
+
+  private:
+    TAO_SYNCH_MUTEX mutex_;
+    ACE_Condition< TAO_SYNCH_MUTEX > condition_;
+  };
+
+}
+
+#include /**/ "ace/post.h"
+
+#endif
diff --git a/CIAO/DAnCE/LocalityManager/Scheduler/Deployment_Event.cpp b/CIAO/DAnCE/LocalityManager/Scheduler/Deployment_Event.cpp
new file mode 100644
index 00000000000..b58189c9c55
--- /dev/null
+++ b/CIAO/DAnCE/LocalityManager/Scheduler/Deployment_Event.cpp
@@ -0,0 +1,5 @@
+
+#if !defined (__ACE_INLINE__)
+#include "LocalityManager/Scheduler/Deployment_Event.inl"
+#endif
+
diff --git a/CIAO/DAnCE/LocalityManager/Scheduler/Deployment_Event.h b/CIAO/DAnCE/LocalityManager/Scheduler/Deployment_Event.h
new file mode 100644
index 00000000000..deb95e27ef7
--- /dev/null
+++ b/CIAO/DAnCE/LocalityManager/Scheduler/Deployment_Event.h
@@ -0,0 +1,43 @@
+/**
+ * @file Deployment_Event.h
+ * @author William R. Otte <wotte@dre.vanderbilt.edu>
+ * $Id$
+ */
+
+#ifndef DEPLOYMENT_EVENT_H
+#define DEPLOYMENT_EVENT_H
+
+#include /**/ "ace/pre.h"
+
+#include "ace/Future.h"
+#include "tao/AnyTypeCode/Any.h"
+#include "LocalityManager/Scheduler/Deployment_Scheduler_export.h"
+
+namespace DAnCE
+{
+  struct Deployment_Scheduler_Export Event_Result
+  {
+    Event_Result (void);
+
+    Event_Result (const std::string &id,
+                  bool exception);
+
+    Event_Result (const std::string &id,
+                  bool exception,
+                  CORBA::Any *any);
+
+    std::string id_;
+    bool exception_;
+    CORBA::Any_var contents_;
+  };
+    
+  typedef ACE_Future< Event_Result > Event_Future;
+}
+
+#if defined (__ACE_INLINE__)
+#include "LocalityManager/Scheduler/Deployment_Event.inl"
+#endif
+
+#include /**/ "ace/post.h"
+
+#endif
diff --git a/CIAO/DAnCE/LocalityManager/Scheduler/Deployment_Event.inl b/CIAO/DAnCE/LocalityManager/Scheduler/Deployment_Event.inl
new file mode 100644
index 00000000000..ba36246657b
--- /dev/null
+++ b/CIAO/DAnCE/LocalityManager/Scheduler/Deployment_Event.inl
@@ -0,0 +1,32 @@
+#ifndef DEPLOYMENT_EVENT_INL
+#define DEPLOYMENT_EVENT_INL
+
+namespace DAnCE
+{
+  ACE_INLINE
+  Event_Result::Event_Result (void)
+    : id_ (""),
+      exception_ (true)
+  {
+  }
+
+  ACE_INLINE
+  Event_Result::Event_Result (const std::string &id,
+                              bool exception)
+    : id_ (id),
+      exception_ (exception)
+  {
+  }
+
+  ACE_INLINE
+  Event_Result::Event_Result (const std::string &id,
+                              bool exception,
+                              ::CORBA::Any *any)
+    : id_ (id),
+      exception_ (exception),
+      contents_ (any)
+  {
+  }
+}
+
+#endif
diff --git a/CIAO/DAnCE/LocalityManager/Scheduler/Deployment_Events.cpp b/CIAO/DAnCE/LocalityManager/Scheduler/Deployment_Events.cpp
index 31c1ae7ab14..08806d275f4 100644
--- a/CIAO/DAnCE/LocalityManager/Scheduler/Deployment_Events.cpp
+++ b/CIAO/DAnCE/LocalityManager/Scheduler/Deployment_Events.cpp
@@ -9,15 +9,12 @@
 #include "DAnCE/DAnCE_Utility.h"
 #include "Plugin_Manager.h"
 
+#if !defined (__ACE_INLINE__)
+#include "LocalityManager/Scheduler/Deployment_Events.inl"
+#endif
+
 namespace DAnCE
 {
-  Deployment_Event::Deployment_Event (Event_Future holder,
-                                      const char *inst_type)
-    : holder_ (holder),
-      instance_type_ (inst_type)
-  {
-  }
-
   Install_Instance::Install_Instance (const ::Deployment::DeploymentPlan & plan,
                                       ::CORBA::ULong instanceRef,
                                       const char *inst_type,
@@ -141,10 +138,7 @@ namespace DAnCE
                                          instance_excep.in ());
           }
         
-        Event_Result result;
-        result.id_ = name;
-        result.exception_ = false;
-        result.contents_ = instance_ref._retn ();
+        Event_Result result (name, false, instance_ref._retn ());
         
         DANCE_DEBUG (10, (LM_TRACE, DLINFO
                           ACE_TEXT ("Install_Instance::call - ")
@@ -161,9 +155,8 @@ namespace DAnCE
                          this->instanceRef_,
                          name));
         
-        Event_Result result;
-        result.id_ = name;
-        result.exception_ = true;
+        Event_Result result (name, true);
+
         CORBA::Any *tmp = 0;
         ACE_NEW_NORETURN (tmp,
                           CORBA::Any);
@@ -290,10 +283,8 @@ namespace DAnCE
                                          instance_excep.in ());
           }
         
-        Event_Result result;
-        result.id_ = name;
-        result.exception_ = false;
-        
+        Event_Result result (name, false);
+
         this->holder_.set (result);
       }
     catch (CORBA::Exception &ex)
@@ -304,9 +295,8 @@ namespace DAnCE
                          name,
                          ex._info ().c_str ()));
         
-        Event_Result result;
-        result.id_ = name;
-        result.exception_ = true;
+        Event_Result result (name, true);
+
         CORBA::Any *tmp = 0;
         ACE_NEW_NORETURN (tmp,
                           CORBA::Any);
@@ -396,9 +386,8 @@ namespace DAnCE
         
         ::Deployment::StopError ex_tmp (name,
                                         ex._info ().c_str ());
-        Event_Result result;
-        result.id_ = name;
-        result.exception_ = true;
+        Event_Result result (name, true);
+
         CORBA::Any *tmp = 0;
         ACE_NEW_NORETURN (tmp,
                           CORBA::Any);
@@ -413,9 +402,8 @@ namespace DAnCE
         return -1;
       }
     
-    Event_Result result;
-    result.id_ = name;
-    result.exception_ = false;
+    Event_Result result (name, false);
+
     this->holder_.set (result);
     
     return 0;
diff --git a/CIAO/DAnCE/LocalityManager/Scheduler/Deployment_Events.h b/CIAO/DAnCE/LocalityManager/Scheduler/Deployment_Events.h
index 14dba5b2028..d365e479744 100644
--- a/CIAO/DAnCE/LocalityManager/Scheduler/Deployment_Events.h
+++ b/CIAO/DAnCE/LocalityManager/Scheduler/Deployment_Events.h
@@ -16,45 +16,10 @@
 #include "ace/Future.h"
 #include "Deployment/Deployment_DeploymentPlanC.h"
 #include "LocalityManager/Scheduler/Deployment_Scheduler_export.h"
+#include "LocalityManager/Scheduler/Deployment_Event.h"
 
 namespace DAnCE
 {
-  struct Deployment_Scheduler_Export Event_Result
-  {
-    std::string id_;
-    bool exception_;
-    CORBA::Any_var contents_;
-  };
-    
-  typedef ACE_Future< Event_Result > Event_Future;
-  
-  /**
-   * @class Event_Handler
-   * @brief Future observer that invokes a parameterized functor on the future
-   */
-  template <typename Functor>
-  class Deployment_Scheduler_Export Event_Handler
-    : ACE_Future_Observer< Event_Result >
-  {
-  public:
-    Event_Handler (Functor &specific_handler);
-    
-    virtual ~Event_Handler (void);
-
-    virtual void update (const Event_Result &future);
-    
-    /// Indicate to the observer that there is an additional future
-    /// it is waiting on
-    void add_outstanding (void);
-    
-    /// Return the number of still outstanding future events.
-    size_t count_outstanding (void);
-  private:
-    ACE_Atomic_Op < TAO_SYNCH_MUTEX, unsigned long > outstanding_;
-    
-    Functor &specific_handler_;
-  };
-
   class Deployment_Scheduler_Export Deployment_Event : 
     public virtual ACE_Method_Request
   {
@@ -129,7 +94,7 @@ namespace DAnCE
 }
 
 #if defined (__ACE_INLINE__)
-#include "LocalityManager/Scheduler/Deployment_Events.h"
+#include "LocalityManager/Scheduler/Deployment_Events.inl"
 #endif
 
 #include /**/ "ace/post.h"
diff --git a/CIAO/DAnCE/LocalityManager/Scheduler/Deployment_Events.inl b/CIAO/DAnCE/LocalityManager/Scheduler/Deployment_Events.inl
index 71939059690..a5be9d429cb 100644
--- a/CIAO/DAnCE/LocalityManager/Scheduler/Deployment_Events.inl
+++ b/CIAO/DAnCE/LocalityManager/Scheduler/Deployment_Events.inl
@@ -1,11 +1,25 @@
 // $Id$
 
+#ifndef DEPLOYMENT_EVENTS_INL
+#define DEPLOYMENT_EVENTS_INL
+
 namespace DAnCE
 {
   ACE_INLINE
+  Deployment_Event::Deployment_Event (Event_Future holder,
+                                      const char *inst_type)
+    : holder_ (holder),
+      instance_type_ (inst_type)
+  {
+  }
+
+  ACE_INLINE
   Event_Future
   Deployment_Event::get_future (void)
   {
     return holder_;
   }
 }
+
+#endif
+
diff --git a/CIAO/DAnCE/LocalityManager/Scheduler/Deployment_Events.tpp b/CIAO/DAnCE/LocalityManager/Scheduler/Deployment_Events.tpp
deleted file mode 100644
index bb2bdb3ed33..00000000000
--- a/CIAO/DAnCE/LocalityManager/Scheduler/Deployment_Events.tpp
+++ /dev/null
@@ -1,38 +0,0 @@
-// $Id$
-#ifndef DEPLOYMENT_EVENTS_TPP
-#define DEPLOYMENT_EVENTS_TPP
-
-namespace DAnCE
-{
-  Event_Handler::Event_Handler (Functor &specific_handler)
-    : outstanding_ (0),
-      specific_handler_ (specific_handler)
-  {
-  }
-    
-  Event_Handler::~Event_Handler (void);
-
-  void 
-  Event_Handler::update (const Event_Result &future)
-  {
-    this->specific_handler_ (future);
-    --this->outstanding_;
-  }
-    
-  /// Indicate to the observer that there is an additional future
-  /// it is waiting on
-  void
-  Event_Handler::add_outstanding (void)
-  {
-    ++this->outstanding_;
-  }
-    
-  /// Return the number of still outstanding future events.
-  size_t
-  Event_Handler::count_outstanding (void)
-  {
-    return this->outstanding_.value ();
-  }
-}
-
-#endif
diff --git a/CIAO/DAnCE/LocalityManager/Scheduler/Scheduler.mpc b/CIAO/DAnCE/LocalityManager/Scheduler/Scheduler.mpc
index abbd7b95850..dd10a4bfc35 100644
--- a/CIAO/DAnCE/LocalityManager/Scheduler/Scheduler.mpc
+++ b/CIAO/DAnCE/LocalityManager/Scheduler/Scheduler.mpc
@@ -1,17 +1,27 @@
 // $Id$
 project(DAnCE_Deployment_Scheduler) : dance_lib, dance_output, dance_skel, dance_logger {
   dynamicflags = DEPLOYMENT_SCHEDULER_BUILD_DLL
+ 
+  includes += $(DANCE_ROOT)/tools
   
   Source_Files {
+    Deployment_Event.cpp
     Deployment_Events.cpp
     Plugin_Manager.cpp
     Deployment_Scheduler.cpp
+    Deployment_Completion.cpp
   }
  
+  Inline_Files {
+    Deployment_Event.inl
+  }
+
   Header_Files {
     Plugin_Manager.h
+    Deployment_Event.h
     Deployment_Events.h
     Deployment_Scheduler.h
     Deployment_Scheduler_export.h
+    Deployment_Completion.h
   }
 }
\ No newline at end of file
diff --git a/CIAO/DAnCE/NodeApplication/NodeApplication_Impl.cpp b/CIAO/DAnCE/NodeApplication/NodeApplication_Impl.cpp
index cc591f6bf3f..66d35b5b60d 100644
--- a/CIAO/DAnCE/NodeApplication/NodeApplication_Impl.cpp
+++ b/CIAO/DAnCE/NodeApplication/NodeApplication_Impl.cpp
@@ -17,6 +17,7 @@
 #include "DAnCE/DAnCE_PropertiesC.h"
 #include "DAnCE/DAnCE_LocalityManagerC.h"
 #include "DAnCE/LocalityManager/Scheduler/Plugin_Manager.h"
+#include "DAnCE/LocalityManager/Scheduler/Deployment_Completion.h"
 #include "Split_Plan/Locality_Splitter.h"
 #include "Split_Plan/Split_Plan.h"
 
@@ -39,10 +40,14 @@ NodeApplication_Impl::NodeApplication_Impl (CORBA::ORB_ptr orb,
     poa_ (PortableServer::POA::_duplicate (poa)),
     installer_ (DAnCE::ArtifactInstallation::_duplicate (installer)),
     node_name_ (node_name),
-    scheduler_ ()
+    scheduler_ (),
+    spawn_delay_ (30)
 {
   DANCE_TRACE ("NodeApplication_Impl::NodeApplication_Impl");
 
+  Utility::get_property_value (DAnCE::LOCALITY_TIMEOUT,
+                               properties, this->spawn_delay_);
+
   ::Deployment::Properties prop;
   ::DAnCE::Utility::build_property_sequence (prop, properties);
   PLUGIN_MANAGER::instance ()->set_configuration (prop);
@@ -73,6 +78,7 @@ NodeApplication_Impl::prepare_instances (const LocalitySplitter::TSubPlans& plan
 
   CORBA::ULong plan (0);
   std::list < Event_Future > prepared_instances;
+  Deployment_Completion completion;
 
   // for each sub plan
   LocalitySplitter::TSubPlanConstIterator plans_end (plans, 1);
@@ -116,7 +122,8 @@ NodeApplication_Impl::prepare_instances (const LocalitySplitter::TSubPlans& plan
 
       Install_Instance *event (0);
       Event_Future result;
-      
+      completion.accept (result);
+
       ACE_NEW_THROW_EX (event,
                         Install_Instance (sub_plan,
                                           loc_manager_instance,
@@ -125,19 +132,30 @@ NodeApplication_Impl::prepare_instances (const LocalitySplitter::TSubPlans& plan
                                           ),
                         CORBA::NO_MEMORY ());
       
+      
       prepared_instances.push_back (result);
       this->scheduler_.schedule_event (event);
       ++plan;
     }
   
+  ACE_Time_Value tv (ACE_OS::gettimeofday () + ACE_Time_Value (this->spawn_delay_));
+  
+  if (completion.wait_on_completion (&tv))
+    {
+      DANCE_ERROR (1, (LM_ERROR, DLINFO
+                       ACE_TEXT("NodeApplication_Impl::prepare_instances - ")
+                       ACE_TEXT("Timed out while waiting on completion of scheduler\n")));
+    }
+  
+  tv = ACE_Time_Value::zero;
+
   plan = 0;
   for (std::list < Event_Future >::iterator i = prepared_instances.begin ();
        i != prepared_instances.end ();
        ++i)
     {
       Event_Result event;
-      if (i->get (event,
-                  0 /*need to wait based on spawn delay*/) != 0)        
+      if (i->get (event, &tv) != 0)        
         {
           DANCE_ERROR (1, (LM_ERROR, DLINFO
                            ACE_TEXT("NodeApplication_Impl::prepare_instances - ")
@@ -344,6 +362,8 @@ NodeApplication_Impl::remove_instances (void)
 
   std::list < Event_Future > removed_instances;
 
+  Deployment_Completion completion;
+
   for (LOCALITY_MAP::iterator i = this->localities_.begin ();
        i != this->localities_.end (); ++i)
     {
@@ -351,7 +371,7 @@ NodeApplication_Impl::remove_instances (void)
                        ACE_TEXT ("NodeApplication_Impl::remove_instances - ")
                        ACE_TEXT ("Removing locality <%C>\n"),
                        i->first.c_str ()));
-
+      
       try
         {
           CORBA::Any ref;
@@ -375,6 +395,8 @@ NodeApplication_Impl::remove_instances (void)
                                         result));
               
               removed_instances.push_back (result);
+              completion.accept (result);
+
               this->scheduler_.schedule_event (event);
             }
           else
@@ -411,6 +433,17 @@ NodeApplication_Impl::remove_instances (void)
         }      
     }
   
+  ACE_Time_Value tv (ACE_OS::gettimeofday () + ACE_Time_Value (this->spawn_delay_));
+  
+  if (completion.wait_on_completion (&tv))
+    {
+      DANCE_ERROR (1, (LM_ERROR, DLINFO
+                       ACE_TEXT("NodeApplication_Impl::remove_instances - ")
+                       ACE_TEXT("Timed out while waiting on completion of scheduler\n")));
+    }
+  
+  tv = ACE_Time_Value::zero;
+
   for (std::list < Event_Future >::iterator i = removed_instances.begin ();
        i != removed_instances.end ();
        ++i)
@@ -420,7 +453,7 @@ NodeApplication_Impl::remove_instances (void)
           Event_Result event;
           
           if (i->get (event,
-                      0 /* need to wait based on spawn delay */) != 0)
+                      &tv) != 0)
             {
               DANCE_ERROR (1, (LM_ERROR, DLINFO
                                ACE_TEXT ("NodeApplication_Impl::remove_instances - ")
diff --git a/CIAO/DAnCE/NodeApplication/NodeApplication_Impl.h b/CIAO/DAnCE/NodeApplication/NodeApplication_Impl.h
index 8f77c9b2cfd..c9b0a8e7142 100644
--- a/CIAO/DAnCE/NodeApplication/NodeApplication_Impl.h
+++ b/CIAO/DAnCE/NodeApplication/NodeApplication_Impl.h
@@ -95,6 +95,8 @@ namespace DAnCE
     DAnCE::Deployment_Scheduler scheduler_;
 
     PLAN_MAP sub_plans_;
+    
+    CORBA::ULong spawn_delay_;
   };
 }
 #endif /*NODEAPPLICATION_IMPL_H_*/
diff --git a/CIAO/DAnCE/tools/Completion/Completion_Counter_Base.h b/CIAO/DAnCE/tools/Completion/Completion_Counter_Base.h
index 871755e96ee..3b3d914e12b 100644
--- a/CIAO/DAnCE/tools/Completion/Completion_Counter_Base.h
+++ b/CIAO/DAnCE/tools/Completion/Completion_Counter_Base.h
@@ -37,7 +37,7 @@ namespace DAnCE
       Completion_Counter_Base (unsigned int exec_count,
                                unsigned int fail_count);
       virtual ~Completion_Counter_Base ();
-
+      
       void  decrement_exec_count ();
       void  increment_fail_count (const char* error);
 
@@ -45,10 +45,12 @@ namespace DAnCE
       unsigned int fail_count ();
 
       bool all_completed ();
-
+      
       const errors_type& errors () const;
 
     protected:
+      void  increment_exec_count ();
+
       unsigned int exec_count_i ();
       unsigned int fail_count_i ();
 
@@ -59,6 +61,7 @@ namespace DAnCE
       mutable ACE_LOCK lock_;
       unsigned int exec_count_;
       unsigned int fail_count_;
+
       errors_type errors_;
     };
 
diff --git a/CIAO/DAnCE/tools/Completion/Completion_Counter_Base.inl b/CIAO/DAnCE/tools/Completion/Completion_Counter_Base.inl
index 2ad8282780b..7d567a2eb2e 100644
--- a/CIAO/DAnCE/tools/Completion/Completion_Counter_Base.inl
+++ b/CIAO/DAnCE/tools/Completion/Completion_Counter_Base.inl
@@ -20,6 +20,15 @@ namespace DAnCE
 
     template <class ACE_LOCK>
     ACE_INLINE void
+    Completion_Counter_Base<ACE_LOCK>::increment_exec_count ()
+      {
+        ACE_GUARD (ACE_LOCK, ace_mon, this->lock_);
+
+        ++this->exec_count_;
+      }
+
+    template <class ACE_LOCK>
+    ACE_INLINE void
     Completion_Counter_Base<ACE_LOCK>::decrement_exec_count ()
       {
         ACE_GUARD (ACE_LOCK, ace_mon, this->lock_);
@@ -30,10 +39,12 @@ namespace DAnCE
           {
             if (this->fail_count_ > 0)
             {
+              ace_mon.release ();
               this->on_all_completed_with_failure ();
             }
           else
             {
+              ace_mon.release ();
               this->on_all_completed ();
             }
           }