Sat Jan 16 19:57:13 UTC 2010 John S. Kinnebrew <john.s.kinnebrew@vanderbilt.edu>

4 files changed, 504 insertions, 157 deletions

diff --git a/SA_POP/ChangeLog b/SA_POP/ChangeLog
index eb720c79ccf..f8b2a9d40e1 100644
--- a/SA_POP/ChangeLog
+++ b/SA_POP/ChangeLog
@@ -1,3 +1,12 @@
+Sat Jan 16 19:57:13 UTC 2010  John S. Kinnebrew  <john.s.kinnebrew@vanderbilt.edu>
+
+        * experiments/EU_Performance/Exp_Core/Exp_EU_Planner.h:
+        * experiments/EU_Performance/Exp_Core/Exp_EU_Planner.cpp:
+        * experiments/EU_Performance/SAPOP_Exp_EU.cpp:
+
+          Added experimental runs on a series of networks and cumulative statistics gathering/output.
+
+
 Sat Jan 16 03:39:12 UTC 2010  John S. Kinnebrew  <john.s.kinnebrew@vanderbilt.edu>
 
         * experiments/EU_Performance/Exp_Core/Exp_EU_Planner.cpp:
diff --git a/SA_POP/experiments/EU_Performance/Exp_Core/Exp_EU_Planner.cpp b/SA_POP/experiments/EU_Performance/Exp_Core/Exp_EU_Planner.cpp
index a84ee5ad830..398ceb58770 100644
--- a/SA_POP/experiments/EU_Performance/Exp_Core/Exp_EU_Planner.cpp
+++ b/SA_POP/experiments/EU_Performance/Exp_Core/Exp_EU_Planner.cpp
@@ -20,8 +20,9 @@
 #include <stdlib.h>
 #include <fstream>
 #include <sstream>
+#include <algorithm>
 
-#include "boost/random.hpp"
+#include <boost/random.hpp>
 #include <boost/accumulators/accumulators.hpp>
 #include <boost/accumulators/statistics/stats.hpp>
 #include <boost/accumulators/statistics/mean.hpp>
@@ -46,18 +47,13 @@ Exp_EU_Planner::Exp_EU_Planner (void)
 :do_pause_ (false),
  input_ (0),
  ques_ (0),
- do_stats_out_ (false),
- trial_num_plans_ (0),
- run_num_trial_attempts_ (0),
- run_num_init_plans_ (0),
- run_num_pref_plans_ (0),
- run_num_alt_plans_ (0),
- pref_plan_eu_ (0.0),
- max_plan_eu_ (0.0)
+ do_stats_out_ (false)
 {
+  this->reset_trial_stats ();
+  this->reset_run_stats ();
+
 //    std::ofstream log_trials_out_;
 //    std::ofstream log_runs_out_;
-
   // All other initialization handled by Planner base class constructor.
 };
 
@@ -82,18 +78,31 @@ Exp_EU_Planner::~Exp_EU_Planner (void)
 // Reset all planning statistics for an individual trial.
 void Exp_EU_Planner::reset_trial_stats (void)
 {
-  this->trial_num_plans_ = 0;
-  this->pref_plan_eu_ = 0.0;
-  this->max_plan_eu_ = 0.0;
+  this->trial_results_.num_plans = 0;
+  this->trial_results_.pref_plan_eu = 0.0;
+  this->trial_results_.max_plan_eu = 0.0;
+
+  // Create empty goal.
+  SA_POP::Goal goal;
+  goal.goal_id = "UserSpecifiedGoal ID";
+  goal.name = "User specified goal";
+  goal.abs_times.clear ();
+  goal.rel_times.clear ();
+  goal.goal_conds.clear ();
+  goal.start_window = std::make_pair (0, 0);
+
+  this->trial_results_.goal = goal;
 };
 
 // Reset all planning statistics for an individual experimental run.
 void Exp_EU_Planner::reset_run_stats (void)
 {
-  this->run_num_trial_attempts_ = 0;
-  this->run_num_init_plans_ = 0;
-  this->run_num_pref_plans_ = 0;
-  this->run_num_alt_plans_ = 0;
+  this->run_results_.num_trial_attempts = 0;
+  this->run_results_.num_init_plans = 0;
+  this->run_results_.num_pref_plans = 0;
+  this->run_results_.num_alt_plans = 0;
+  this->run_results_.net_name = "UnknownNetwork";
+  this->run_results_.trials.clear ();
 };
 
 // Set to pause and ask user whether to continue after each plan is generated.
@@ -111,8 +120,14 @@ void Exp_EU_Planner::unset_pause (void)
 };
 
 // Run experiment planning.
-void Exp_EU_Planner::exp_run (size_t sa_max_steps, std::string log_trials_filename, std::string log_runs_filename, size_t num_goal_conds, double percent_init_true, size_t util_min, size_t util_max, size_t max_trial_attempts, size_t num_trials, bool do_log_headers)
+SA_POP::Exp_EU_Run_Results Exp_EU_Planner::exp_run (std::string log_trials_filename, std::string log_runs_filename, std::string net_name, SA_POP::Exp_EU_Trial_Params trial_params, size_t max_trial_attempts, size_t num_trials, bool do_log_headers)
 {
+  // Maximum spreading activation steps to update network is twice
+  // the number of tasks (for worst case of linear network).
+  size_t sa_max_steps = 2 * this->sanet_->get_num_tasks ();
+
+  this->run_results_.net_name = net_name;
+
   // Set flag to output statistics to file.
   this->do_stats_out_ = true;
 
@@ -152,25 +167,29 @@ void Exp_EU_Planner::exp_run (size_t sa_max_steps, std::string log_trials_filena
   for (size_t trial_num = 0; trial_num < num_trials; trial_num++) {
     // If maximum number of trial attempts have been reached,
     // stop running trials.
-    if (this->run_num_trial_attempts_ >= max_trial_attempts)
+    if (this->run_results_.num_trial_attempts >= max_trial_attempts)
       break;
 
     // Initialize trial statistics.
     this->reset_trial_stats ();
 
     // Initialize conditions and goal.
-    SA_POP::Goal goal = this->exp_init (num_goal_conds, percent_init_true, util_min, util_max);
+    SA_POP::Goal goal = this->exp_init (trial_params);
+    this->trial_results_.goal = goal;
 
     // Perform experimental run.
     this->plan (sa_max_steps, goal);
 
     // Increment trial attempts counter.
-    this->run_num_trial_attempts_++;
+    this->run_results_.num_trial_attempts++;
 
     // Trial only valid if at least one alternate plan was generated.
-    if (this->trial_num_plans_ >= 3) {
+    if (this->trial_results_.num_plans >= 3) {
       // Log trial statistics to output file.
       this->log_trial_stats ();
+      
+      // Add to run results.
+      this->run_results_.trials.push_back (this->trial_results_);
     } else {
       // For invalid trial, decrement trial counter.
       trial_num--;
@@ -179,10 +198,13 @@ void Exp_EU_Planner::exp_run (size_t sa_max_steps, std::string log_trials_filena
   }
 
   // Log run statistics to output file.
-  this->log_run_stats ();
+  this->log_run_stats (trial_params);
 
   // Close output files.
   this->log_trials_out_.close ();
+  this->log_runs_out_.close ();
+
+  return this->run_results_;
 };
 
 
@@ -218,15 +240,15 @@ bool Exp_EU_Planner::full_sched ()
 void Exp_EU_Planner::track_stats (SA_POP::Plan plan)
 {
   // Update number of plans generated.
-  this->trial_num_plans_++;
+  this->trial_results_.num_plans++;
 
   // Ignore first plan.
-  if (this->trial_num_plans_ <= 1) {
+  if (this->trial_results_.num_plans <= 1) {
     // First plan.
-    this->trial_num_plans_ = 1;
+    this->trial_results_.num_plans = 1;
 
     // Increment counter of trials with initial plan.
-    this->run_num_init_plans_++;
+    this->run_results_.num_init_plans++;
 
     // Nothing else to do for first plan.
     return;
@@ -236,12 +258,12 @@ void Exp_EU_Planner::track_stats (SA_POP::Plan plan)
   SA_POP::Utility plan_eu = this->calc_plan_eu (plan);
 
   // Second plan is SA-POP preferred plan.
-  if (this->trial_num_plans_ == 2) {
+  if (this->trial_results_.num_plans == 2) {
     // Current maximum plan EU is this (preferred) plan's EU.
-    this->pref_plan_eu_ = this->max_plan_eu_ = plan_eu;
+    this->trial_results_.pref_plan_eu = this->trial_results_.max_plan_eu = plan_eu;
 
     // Increment counter of trials with preferred plan.
-    this->run_num_pref_plans_++;
+    this->run_results_.num_pref_plans++;
 
     // Nothing else to do for first plan.
     return;
@@ -251,27 +273,27 @@ void Exp_EU_Planner::track_stats (SA_POP::Plan plan)
 
   // If this is the first alternate plan (third plan generated),
   // increment counter of trials with at least one alternate plan.
-  if (this->trial_num_plans_ == 3) {
-    this->run_num_alt_plans_++;
+  if (this->trial_results_.num_plans == 3) {
+    this->run_results_.num_alt_plans++;
   }
 
   // If this plan has a higher EU than current max, update max.
-  if (plan_eu > this->max_plan_eu_)
-    this->max_plan_eu_ = plan_eu;
+  if (plan_eu > this->trial_results_.max_plan_eu)
+    this->trial_results_.max_plan_eu = plan_eu;
 };
 
 // Initialize experiment (set values for initial conditions and create goal).
-SA_POP::Goal Exp_EU_Planner::exp_init (size_t num_goal_conds, double percent_init_true, size_t util_min, size_t util_max)
+SA_POP::Goal Exp_EU_Planner::exp_init (SA_POP::Exp_EU_Trial_Params params)
 {
   // Check goal utility min & max for consistency.
-  if (util_min > util_max) {
+  if (params.util_min > params.util_max) {
     std::cerr << "SA-POP ERROR in Exp_EU_Planner::exp_init (): goal utility min is >= goal utility max";
     throw "SA-POP ERROR in Exp_EU_Planner::exp_init (): goal utility min is >= goal utility max";
   }
 
 
   SA_POP::Goal goal;
-  goal.goal_id = "ExperimentGoalID";
+  goal.goal_id = "ExperimentGeneratedGoalID";
   goal.name = "Experiment generated goal";
   goal.abs_times.clear ();
   goal.rel_times.clear ();
@@ -282,7 +304,7 @@ SA_POP::Goal Exp_EU_Planner::exp_init (size_t num_goal_conds, double percent_ini
   SA_POP::CondSet conds = this->sanet_->get_all_conds ();
 
   // Check number of goal conditions for consistency.
-  if (num_goal_conds > conds.size ()) {
+  if (params.num_goal_conds > conds.size ()) {
     std::cerr << "SA-POP ERROR in Exp_EU_Planner::exp_init (): more goal conditions than number of goals";
     throw "SA-POP ERROR in Exp_EU_Planner::exp_init (): more goal conditions than number of goals";
   }
@@ -290,7 +312,7 @@ SA_POP::Goal Exp_EU_Planner::exp_init (size_t num_goal_conds, double percent_ini
   // Initialize random number generators.
   boost::uniform_int<> goal_choice_dist(0,  conds.size () - 1);
   boost::variate_generator<boost::mt19937&, boost::uniform_int<> > gen_goal_choice(this->rand_gen_, goal_choice_dist);
-  boost::uniform_int<> goal_util_dist(util_min, util_max);
+  boost::uniform_int<> goal_util_dist(params.util_min, params.util_max);
   boost::variate_generator<boost::mt19937&, boost::uniform_int<> > gen_goal_util(this->rand_gen_, goal_util_dist);
   boost::uniform_real<> cond_init_dist(0.0, 1.0);
   boost::variate_generator<boost::mt19937&, boost::uniform_real<> > gen_cond_init(this->rand_gen_, cond_init_dist);
@@ -298,7 +320,7 @@ SA_POP::Goal Exp_EU_Planner::exp_init (size_t num_goal_conds, double percent_ini
   // Set goals.
   std::set<SA_POP::CondID> goal_ids;
   goal_ids.clear ();
-  for (size_t goal_ctr = 0; goal_ctr < num_goal_conds; goal_ctr++) {
+  for (size_t goal_ctr = 0; goal_ctr < params.num_goal_conds; goal_ctr++) {
     // Choose goal condition number.
     size_t goal_choice = gen_goal_choice ();
 
@@ -344,7 +366,7 @@ SA_POP::Goal Exp_EU_Planner::exp_init (size_t num_goal_conds, double percent_ini
 
     // Randomly determine whether to set condition to true.
     double init_check = gen_cond_init ();
-    if (init_check <= percent_init_true) {
+    if (init_check <= params.percent_init_true) {
       this->sanet_->update_cond_val ((*cond_iter).id, true);
     }
   }
@@ -390,9 +412,9 @@ void Exp_EU_Planner::log_trial_stats (void)
   }
 
   // Output trial statistics to file.
-  this->log_trials_out_ << this->pref_plan_eu_ << "\t";
-  this->log_trials_out_ << this->max_plan_eu_ << "\t";
-  this->log_trials_out_ << (this->trial_num_plans_ - 2);
+  this->log_trials_out_ << this->trial_results_.pref_plan_eu << "\t";
+  this->log_trials_out_ << this->trial_results_.max_plan_eu << "\t";
+  this->log_trials_out_ << (this->trial_results_.num_plans - 2);
   this->log_trials_out_ << std::endl;
 
 };
@@ -410,8 +432,17 @@ void Exp_EU_Planner::log_run_header (void)
     std::cerr << msg;
     throw msg;
   }
-  
+
   // Output run header to file.
+  this->log_runs_out_ << "SANet name" << "\t";
+  this->log_runs_out_ << "Number of goal conditions" << "\t";
+  this->log_runs_out_ << "Percent initial conditions true" << "\t";
+  this->log_runs_out_ << "Goal utility min" << "\t";
+  this->log_runs_out_ << "Goal utility max" << "\t";
+  this->log_runs_out_ << "Plan EU ratio (preferred : best) Mean" << "\t";
+  this->log_runs_out_ << "Plan EU ratio (preferred : best) Std Dev" << "\t";
+  this->log_runs_out_ << "Trial plans generated Mean" << "\t";
+  this->log_runs_out_ << "Trial plans generated Std Dev" << "\t";
   this->log_runs_out_ << "Trial attempts" << "\t";
   this->log_runs_out_ << "Trials w/ initial plan" << "\t";
   this->log_runs_out_ << "Trials w/ preferred plan" << "\t";
@@ -420,7 +451,7 @@ void Exp_EU_Planner::log_run_header (void)
 };
 
 // Output run statistics to log.
-void Exp_EU_Planner::log_run_stats (void)
+void Exp_EU_Planner::log_run_stats (SA_POP::Exp_EU_Trial_Params trial_params)
 {
   // Only log statistics if flag is set.
   if (!this->do_stats_out_)
@@ -433,11 +464,30 @@ void Exp_EU_Planner::log_run_stats (void)
     throw msg;
   }
 
+  // Calculate cumulative statistics for trials.
+  boost::accumulators::accumulator_set<double, boost::accumulators::stats<boost::accumulators::tag::mean, boost::accumulators::tag::variance> > acc_pref_best_ratio;
+  boost::accumulators::accumulator_set<double, boost::accumulators::stats<boost::accumulators::tag::mean, boost::accumulators::tag::variance> > acc_num_plans;
+  for (std::list<Exp_EU_Trial_Results>::iterator trial_iter = this->run_results_.trials.begin (); trial_iter != this->run_results_.trials.end (); trial_iter++) {
+    double pref_best_ratio = (*trial_iter).pref_plan_eu / (*trial_iter).max_plan_eu;
+
+    acc_pref_best_ratio (pref_best_ratio);
+    acc_num_plans ((*trial_iter).num_plans);
+  }
+
   // Output run statistics to file.
-  this->log_runs_out_ << this->run_num_trial_attempts_ << "\t";
-  this->log_runs_out_ << this->run_num_init_plans_ << "\t";
-  this->log_runs_out_ << this->run_num_pref_plans_ << "\t";
-  this->log_runs_out_ << this->run_num_alt_plans_;
+  this->log_runs_out_ << this->run_results_.net_name << "\t";
+  this->log_runs_out_ << trial_params.num_goal_conds << "\t";
+  this->log_runs_out_ << trial_params.percent_init_true << "\t";
+  this->log_runs_out_ << trial_params.util_min << "\t";
+  this->log_runs_out_ << trial_params.util_max << "\t";
+  this->log_runs_out_ << boost::accumulators::mean(acc_pref_best_ratio) << "\t";
+  this->log_runs_out_ << std::sqrt(boost::accumulators::variance(acc_pref_best_ratio)) << "\t";
+  this->log_runs_out_ << boost::accumulators::mean(acc_num_plans) << "\t";
+  this->log_runs_out_ << std::sqrt(boost::accumulators::variance(acc_num_plans)) << "\t";
+  this->log_runs_out_ << this->run_results_.num_trial_attempts << "\t";
+  this->log_runs_out_ << this->run_results_.num_init_plans << "\t";
+  this->log_runs_out_ << this->run_results_.num_pref_plans << "\t";
+  this->log_runs_out_ << this->run_results_.num_alt_plans;
   this->log_runs_out_ << std::endl;
 };
 
diff --git a/SA_POP/experiments/EU_Performance/Exp_Core/Exp_EU_Planner.h b/SA_POP/experiments/EU_Performance/Exp_Core/Exp_EU_Planner.h
index b935983b95d..0694ab0bd1d 100644
--- a/SA_POP/experiments/EU_Performance/Exp_Core/Exp_EU_Planner.h
+++ b/SA_POP/experiments/EU_Performance/Exp_Core/Exp_EU_Planner.h
@@ -21,6 +21,7 @@
 #include <string>
 #include <set>
 #include <map>
+#include <list>
 #include <fstream>
 
 #include <boost/random.hpp>
@@ -39,6 +40,113 @@
 
 namespace SA_POP {
 
+  /// Container for experimental trial parameters.
+  struct Exp_EU_Trial_Params {
+    /// Number of goal conditions to generate.
+    size_t num_goal_conds;
+    
+    /// Percentage of conditions to set initial value to true.
+    double percent_init_true;
+
+    /// Minimum goal utility (uniform random choice in range [util_min, util_max]).
+    int util_min;
+
+    /// Maximum goal utility (uniform random choice in range [util_min, util_max]).
+    int util_max;
+
+    bool operator== (const Exp_EU_Trial_Params &s) const { return (this->num_goal_conds == s.num_goal_conds && this->percent_init_true == s.percent_init_true && this->util_min == s.util_min && this->util_max == s.util_max); };
+    bool operator!= (const Exp_EU_Trial_Params &s) const { return !(*this == s); };
+    bool operator< (const Exp_EU_Trial_Params &s) const
+    {
+      if (this->num_goal_conds == s.num_goal_conds) {
+        if (this->percent_init_true == s.percent_init_true) {
+          if (this->util_min == s.util_min) {
+            return this->util_max < s.util_max;
+          }
+          return this->util_min < s.util_min;
+        }
+        return this->percent_init_true < s.percent_init_true;
+      }
+      return this->num_goal_conds < s.num_goal_conds;
+    };
+  };
+
+  /// Container for experimental results of a trial.
+  struct Exp_EU_Trial_Results {
+    /// Expected utility of SA-POP preferred plan.
+    SA_POP::Utility pref_plan_eu;
+
+    /// Highest expected utility of any (schedulable/valid) plan (including preferred plan).
+    SA_POP::Utility max_plan_eu;
+
+    /// Number of plans generated.
+    size_t num_plans;
+
+    /// Trial goal.
+    SA_POP::Goal goal;
+
+    bool operator== (const Exp_EU_Trial_Results &s) const { return (this->pref_plan_eu == s.pref_plan_eu && this->max_plan_eu == s.max_plan_eu && this->num_plans == s.num_plans && this->goal == s.goal); };
+    bool operator!= (const Exp_EU_Trial_Results &s) const { return !(*this == s); };
+    bool operator< (const Exp_EU_Trial_Results &s) const
+    {
+      if (this->pref_plan_eu == s.pref_plan_eu) {
+        if (this->max_plan_eu == s.max_plan_eu) {
+          if (this->num_plans == s.num_plans) {
+            return this->goal < s.goal;
+          }
+          return this->num_plans < s.num_plans;
+        }
+        return this->max_plan_eu < s.max_plan_eu;
+      }
+      return this->pref_plan_eu < s.pref_plan_eu;
+    };
+  };
+
+  /// Container for a series of experimental results.
+  struct Exp_EU_Run_Results {
+    /// Run counter for number of trial attempts.
+    size_t num_trial_attempts;
+
+    /// Run counter for number of trials with initial plan generated.
+    size_t num_init_plans;
+
+    /// Run counter for number of trials with preferred plan generated.
+    size_t num_pref_plans;
+
+    /// Run counter for number of trials with at least one alternate plan generated.
+    size_t num_alt_plans;
+
+    /// Name of network.
+    std::string net_name;
+
+    /// List of trial results.
+    std::list<Exp_EU_Trial_Results> trials;
+
+    bool operator== (const Exp_EU_Run_Results &s) const { return (this->num_trial_attempts == s.num_trial_attempts && this->num_init_plans == s.num_init_plans && this->num_pref_plans == s.num_pref_plans && this->num_alt_plans == s.num_alt_plans && this->net_name == s.net_name && this->trials == s.trials); };
+    bool operator!= (const Exp_EU_Run_Results &s) const { return !(*this == s); };
+    bool operator< (const Exp_EU_Run_Results &s) const
+    {
+      if (this->num_trial_attempts == s.num_trial_attempts) {
+        if (this->num_init_plans == s.num_init_plans) {
+          if (this->num_pref_plans == s.num_pref_plans) {
+            if (this->num_alt_plans == s.num_alt_plans) {
+              if (this->net_name == s.net_name) {
+                return this->trials < s.trials;
+              }
+              return this->net_name < s.net_name;
+            }
+            return this->num_alt_plans < s.num_alt_plans;
+          }
+          return this->num_pref_plans < s.num_pref_plans;
+        }
+        return this->num_init_plans < s.num_init_plans;
+      }
+      return this->num_trial_attempts < s.num_trial_attempts;
+    };
+  };
+
+
+
   /**
    * @class Exp_EU_Planner
    *
@@ -73,19 +181,13 @@ namespace SA_POP {
 
     /// Perform experimental trial(s).
     /**
-     * @param sa_max_steps  Maximum steps to run spreading activation.
-     *
      * @param log_trials_filename  Name of file to output trial statistics to.
      *
      * @param log_runs_filename  Name of file to output cumulative statistics from run to.
      *
-     * @param num_goal_conds  Number of goal conditions to generate.
-     *
-     * @param percent_init_true  Percentage of conditions to set initial value to true.
-     *
-     * @param util_min  Minimum goal utility (uniform random choice in range [util_min, util_max]).
+     * @param log_runs_filename  Name of current spreading activation network for logging.
      *
-     * @param util_max  Maximum goal utility (uniform random choice in range [util_min, util_max]).
+     * @param trial_params  Parameters for random initialization of each trial.
      *
      * @param max_trial_attempts  Maximum number of trial attempts to execute in order to reach assigned number of valid trials.
      *
@@ -93,9 +195,8 @@ namespace SA_POP {
      *
      * @param do_log_headers  Flag to output header lines to log files (if true).
      */
-    virtual void exp_run (size_t sa_max_steps, std::string log_trials_filename,
-      std::string log_runs_filename, size_t num_goal_conds, double percent_init_true,
-      size_t util_min, size_t util_max, size_t max_trial_attempts,
+    virtual SA_POP::Exp_EU_Run_Results exp_run (std::string log_trials_filename, std::string log_runs_filename,
+      std::string net_name, SA_POP::Exp_EU_Trial_Params trial_params, size_t max_trial_attempts,
       size_t num_trials = 1, bool do_log_headers = false);
     
 
@@ -132,26 +233,32 @@ namespace SA_POP {
     /// Output file stream for logging trial statistics.
     std::ofstream log_runs_out_;
 
+    /// Results of current experimental trial.
+    SA_POP::Exp_EU_Trial_Results trial_results_;
+
+    /// Results of current experimental run (multiple trials).
+    SA_POP::Exp_EU_Run_Results run_results_;
+
     /// Trial counter for number of plans generated.
-    size_t trial_num_plans_;
+//    size_t trial_num_plans_;
 
     /// Run counter for number of trial attempts.
-    size_t run_num_trial_attempts_;
+//    size_t run_num_trial_attempts_;
 
     /// Run counter for number of trials with initial plan generated.
-    size_t run_num_init_plans_;
+//    size_t run_num_init_plans_;
 
     /// Run counter for number of trials with preferred plan generated.
-    size_t run_num_pref_plans_;
+//    size_t run_num_pref_plans_;
 
     /// Run counter for number of trials with at least one alternate plan generated.
-    size_t run_num_alt_plans_;
+//    size_t run_num_alt_plans_;
 
     /// Expected utility of SA-POP preferred plan.
-    SA_POP::Utility pref_plan_eu_;
+//    SA_POP::Utility pref_plan_eu_;
 
     /// Highest expected utility of any (schedulable/valid) plan (including preferred plan).
-    SA_POP::Utility max_plan_eu_;
+//    SA_POP::Utility max_plan_eu_;
 
 
     // ************************************************************************
@@ -174,7 +281,10 @@ namespace SA_POP {
     virtual void log_trial_stats (void);
 
     /// Output run statistics to log.
-    virtual void log_run_stats (void);
+    /**
+     * @param trial_params  Parameters used for random initialization of trials in this experimental run.
+     */
+    virtual void log_run_stats (SA_POP::Exp_EU_Trial_Params trial_params);
 
     /// Output run header to log.
     virtual void log_run_header (void);
@@ -182,17 +292,11 @@ namespace SA_POP {
     /// Initialize experiment (set values for initial conditions and
     /// create goal).
     /**
-     * @param num_goal_conds  Number of goal conditions to generate.
-     *
-     * @param percent_init_true  Percentage of conditions to set initial value to true.
-     *
-     * @param util_min  Minimum goal utility (uniform random choice in range [util_min, util_max]).
-     *
-     * @param util_max  Maximum goal utility (uniform random choice in range [util_min, util_max]).
+     * @param params  Parameters for random initialization of trial.
      *
      * @return  Goal to use in current experimental run.
      */
-    virtual SA_POP::Goal exp_init (size_t num_goal_conds, double percent_init_true, size_t util_min, size_t util_max);
+    virtual SA_POP::Goal exp_init (SA_POP::Exp_EU_Trial_Params params);
   };
 
 };  /* SA_POP namespace */
diff --git a/SA_POP/experiments/EU_Performance/SAPOP_Exp_EU.cpp b/SA_POP/experiments/EU_Performance/SAPOP_Exp_EU.cpp
index f18f9c908a0..1ad1ee92cfc 100644
--- a/SA_POP/experiments/EU_Performance/SAPOP_Exp_EU.cpp
+++ b/SA_POP/experiments/EU_Performance/SAPOP_Exp_EU.cpp
@@ -21,12 +21,19 @@
 #include <fstream>
 #include <vector>
 #include <map>
+#include <list>
+
+#include <boost/accumulators/accumulators.hpp>
+#include <boost/accumulators/statistics/stats.hpp>
+#include <boost/accumulators/statistics/mean.hpp>
+#include <boost/accumulators/statistics/variance.hpp>
 
 #include "SA_POP_Types.h"
 #include "Exp_Core/Exp_EU_Builder.h"
 #include "Exp_Core/Exp_EU_Planner.h"
 #include "TaskMapFileIn.h"
 #include "SANet/SANetFileIn.h"
+#include "LogFileOut.h"
 #include "LogScreenOut.h"
 #include "LogGraphOut.h"
 //#include "SA_POP_Utils.h"
@@ -138,9 +145,6 @@ void displayConds(SA_POP::Planner *plans, std::vector<SANet::CondID> checks, std
 
 int main (int argc, char* argv[])
 {
-  SA_POP::Exp_EU_Builder builder;
-  SANet::SANetFileIn sanet_in;
-  SA_POP::TaskMapFileIn tm_in;
   SA_POP::Exp_EU_Planner *planner = 0;
   SA_POP::LogGraphOut graph_out	(std::cout, false);
   SA_POP::LogScreenOut screen_out (std::cout);
@@ -152,131 +156,312 @@ int main (int argc, char* argv[])
 
   std::vector<SA_POP::CondID> kconds;
 
-  // Get filenames from user.
-  UserInterface::Question sanet_file_ques ("Task Network file:");
-  if (user_input.ask (sanet_file_ques))
-    sanet_filename = sanet_file_ques.get_answer ();
-  else {
-    std::cerr << "No Task Network filename provided." << std::endl;
-    throw "No Task Network filename provided.";
-  }
-  UserInterface::Question tm_file_ques ("Task Map file:");
-  if (user_input.ask (tm_file_ques))
-    tm_filename = tm_file_ques.get_answer ();
-  else {
-    std::cerr << "No Task Map filename provided." << std::endl;
-    throw "No Task Map filename provided.";
-  }
-
-  
-  // Build task network and task map.
-  try {
-    sanet_in.build_net (sanet_filename, &builder);
-    tm_in.build_task_map (tm_filename, &builder);
-  } catch (std::string e) {
-    std::cerr << "ERROR while building task network and task map from files:";
-    std::cerr << std::endl;
-    std::cerr << e;
-  } catch (...) {
-    std::cerr << "UNKNOWN ERROR while building task network and task map from files." << std::endl;
-  }
-
-  // Get SA-POP Planner.
-  planner = builder.get_exp_eu_planner ();
 
   // Run manual or automatic trial based on user input.
   UserInterface::QuestionBool auto_trials_ques ("Automatic goal & initial condition generation? [(Y)es or (N)o]:", false);
   user_input.ask (auto_trials_ques);
   if (auto_trials_ques.get_answer_bool ()) {
+    std::string log_trials_filename = "log_trials.txt";
+    std::string log_runs_filename = "log_runs.txt";
+    std::string log_exp_filename = "log_exp.txt";
+    size_t num_trials = 1;
+    size_t num_goals = 1;
+    size_t num_percent_conds_init = SA_POP::Default::NumPercentCondsInit;
+    size_t min_util = 100;
+    size_t max_util = min_util;
+    bool do_log_headers = false;
+    std::string plan_out_filename = "log_default_plan_out.txt";
+    bool do_plan_file = false;
+    SA_POP::Exp_EU_Run_Results run_results;
+    bool do_mult_runs = false;
+
+    // Get whether to run a series of 
+    UserInterface::QuestionBool mult_runs_ques ("Run on a series of networks? [(Y)es or (N)o]:", false);
+    user_input.ask (mult_runs_ques);
+    do_mult_runs = mult_runs_ques.get_answer_bool ();
+
     // Get trial log filename.
-    std::string log_trials_filename = "default_trials_filename.log";
     UserInterface::Question log_trials_filename_ques ("Filename for trial log:");
     if (user_input.ask (log_trials_filename_ques))
       log_trials_filename = log_trials_filename_ques.get_answer ();
 
     // Get run log filename.
-    std::string log_runs_filename = "default_runs_filename.log";
     UserInterface::Question log_runs_filename_ques ("Filename for run log:");
     if (user_input.ask (log_runs_filename_ques))
       log_runs_filename = log_runs_filename_ques.get_answer ();
 
+    // If multiple runs, get experiment log filename.
+    if (do_mult_runs) {
+      UserInterface::Question log_exp_filename_ques ("Filename for experiment log:");
+      if (user_input.ask (log_exp_filename_ques))
+        log_exp_filename = log_exp_filename_ques.get_answer ();
+    }
+
     // Get number of experimental trials to perform.
-    size_t num_trials = 1;
-    UserInterface::QuestionInt num_trials_ques ("Number of trials to run:", SA_POP::Default::NumTrialsMin, SA_POP::Default::NumTrialsMax);
+    UserInterface::QuestionInt num_trials_ques ("Number of trials per network:", SA_POP::Default::NumTrialsMin, SA_POP::Default::NumTrialsMax);
     if (user_input.ask (num_trials_ques))
       num_trials = num_trials_ques.get_answer_int ();
 
     // Get number of goal conditions.
-    size_t num_goals = 1;
     UserInterface::QuestionInt num_goals_ques ("Number of goal conditions:", SA_POP::Default::NumGoalsMin, SA_POP::Default::NumGoalsMax);
     if (user_input.ask (num_goals_ques))
       num_goals = num_goals_ques.get_answer_int ();
 
     // Get percent of initial conditions true.
-    size_t num_percent_conds_init = SA_POP::Default::NumPercentCondsInit;
     UserInterface::QuestionInt conds_init_ques ("Percentage of initial conditions true [0 to 100]:", 0, 100);
     if (user_input.ask (conds_init_ques))
       num_percent_conds_init = conds_init_ques.get_answer_int ();
     double percent_conds_init = (double) num_percent_conds_init / 100.0;
 
     // Get min goal utility.
-    size_t min_util = 100;
-    UserInterface::QuestionInt min_util_ques ("Minimum goal utility:", SA_POP::Default::GoalUtilMin, SA_POP::Default::GoalUtilMax);
+    UserInterface::QuestionInt min_util_ques ("Minimum goal condition utility:", SA_POP::Default::GoalUtilMin, SA_POP::Default::GoalUtilMax);
     if (user_input.ask (min_util_ques))
       min_util = min_util_ques.get_answer_int ();
 
     // Get max goal utility.
-    size_t max_util = min_util;
-    UserInterface::QuestionInt max_util_ques ("Maximum goal utility:", min_util, SA_POP::Default::GoalUtilMax);
+    UserInterface::QuestionInt max_util_ques ("Maximum goal condition utility:", min_util, SA_POP::Default::GoalUtilMax);
     if (user_input.ask (max_util_ques))
       max_util = max_util_ques.get_answer_int ();
 
-    // Get maximum steps of spreading activation from user.
-    UserInterface::QuestionInt steps_ques("Max steps to run spreading activation (10000 step limit): ", 0, 10000);
-    if (user_input.ask (steps_ques))
-      max_steps = steps_ques.get_answer_int ();
-
     // Get whether to output log headers.
-    bool do_log_headers = false;
     UserInterface::QuestionBool log_headers_ques ("Output headers in log files? [(Y)es or (N)o]:", false);
     user_input.ask (log_headers_ques);
     do_log_headers = log_headers_ques.get_answer_bool ();
 
-    // Output plans to screen if user wishes.
-    UserInterface::QuestionBool screenout_ques ("Output plans to screen? [(Y)es or (N)o]:", false);
-    user_input.ask (screenout_ques);
-    if (screenout_ques.get_answer_bool ())
-      planner->add_out_adapter (&screen_out);
-
-    // Pause between plans if user wishes.
-    UserInterface::QuestionBool ques_cont_plan ("Continue planning to find next plan? [(Y)es or (N)o]:", false);
-    UserInterface::QuestionBool pause_ques ("Pause between plans? [(Y)es or (N)o]:", false);
-    user_input.ask (pause_ques);
-    if (pause_ques.get_answer_bool ()) {
-      // Set planner to ask whether continue after each plan is generated.
-      planner->set_pause (&user_input, &ques_cont_plan);
-
-      // Output graphical plans if user wishes.
-      UserInterface::QuestionBool graphout_ques ("Output graphical plans? [(Y)es or (N)o]:", false);
-      user_input.ask (graphout_ques);
-      if (graphout_ques.get_answer_bool ())
-        planner->add_out_adapter (&graph_out);
+    // Get whether to output plans to file.
+    UserInterface::QuestionBool fileout_ques ("Output plans to file? [(Y)es or (N)o]:", false);
+    user_input.ask (fileout_ques);
+    do_plan_file = fileout_ques.get_answer_bool ();
+    if (do_plan_file) {
+      // Get plan out filename.
+      UserInterface::Question plan_out_filename_ques ("Filename for plan log:");
+      if (user_input.ask (plan_out_filename_ques))
+        plan_out_filename = plan_out_filename_ques.get_answer ();
     }
 
+    // Create trial parameters structure.
+    SA_POP::Exp_EU_Trial_Params trial_params;
+    trial_params.num_goal_conds = num_goals;
+    trial_params.percent_init_true = percent_conds_init;
+    trial_params.util_min = min_util;
+    trial_params.util_max = max_util;
+
+
+
+    // If multiple runs, get base filename and run experiment.
+    if (do_mult_runs) {
+      std::string base_filename = "network";
+      size_t num_runs = 1;
+      boost::accumulators::accumulator_set<double, boost::accumulators::stats<boost::accumulators::tag::mean, boost::accumulators::tag::variance> > acc_pref_best_ratio;
+      boost::accumulators::accumulator_set<int, boost::accumulators::stats<boost::accumulators::tag::mean, boost::accumulators::tag::variance> > acc_num_plans;
+      boost::accumulators::accumulator_set<int, boost::accumulators::stats<boost::accumulators::tag::mean, boost::accumulators::tag::variance> > acc_trial_attempts;
+      boost::accumulators::accumulator_set<int, boost::accumulators::stats<boost::accumulators::tag::mean, boost::accumulators::tag::variance> > acc_init_plans;
+      boost::accumulators::accumulator_set<int, boost::accumulators::stats<boost::accumulators::tag::mean, boost::accumulators::tag::variance> > acc_pref_plans;
+      boost::accumulators::accumulator_set<int, boost::accumulators::stats<boost::accumulators::tag::mean, boost::accumulators::tag::variance> > acc_alt_plans;
+
+      // Get base filename.
+      UserInterface::Question base_file_ques ("Base filename:");
+      if (user_input.ask (base_file_ques))
+        base_filename = base_file_ques.get_answer ();
+      else {
+        std::cerr << "No base filename provided." << std::endl;
+        throw "No base filename provided.";
+      }
+
+      // Get number of networks to run.
+      UserInterface::QuestionInt num_runs_ques ("Number of networks:", SA_POP::Default::NumTrialsMin, SA_POP::Default::NumTrialsMax);
+      if (user_input.ask (num_runs_ques))
+        num_runs = num_runs_ques.get_answer_int ();
+
+      // Run on requested number of networks.
+      for (size_t run_num = 0; run_num < num_runs; run_num++) {
+
+        // Construct SANet and TM filenames.
+        std::ostringstream sanet_ss;
+        sanet_ss << base_filename << run_num << ".xml";
+        sanet_filename = sanet_ss.str ();
+        std::ostringstream tm_ss;
+        tm_ss << base_filename << run_num << ".tm.xml";
+        tm_filename = tm_ss.str ();
+
+        SA_POP::Exp_EU_Builder builder;
+        SANet::SANetFileIn sanet_in;
+        SA_POP::TaskMapFileIn tm_in;
+
+        // Build task network and task map.
+        try {
+          sanet_in.build_net (sanet_filename, &builder);
+          tm_in.build_task_map (tm_filename, &builder);
+        } catch (std::string e) {
+          std::cerr << "ERROR while building task network and task map from files:";
+          std::cerr << std::endl;
+          std::cerr << e;
+        } catch (...) {
+          std::cerr << "UNKNOWN ERROR while building task network and task map from files." << std::endl;
+        }
+
+        // Get SA-POP Planner.
+        planner = builder.get_exp_eu_planner ();
+
+        // Add file output adapter, if user requested it.
+        SA_POP::LogFileOut file_out (plan_out_filename.c_str ());
+        if (do_plan_file)
+          planner->add_out_adapter (&file_out);
+
+        // Run trials.
+        std::cout << "Running trials on network " << run_num << std::endl;
+        run_results = planner->exp_run (log_trials_filename, log_runs_filename, sanet_filename, trial_params, num_trials * SA_POP::Default::TrialAttemptsMult, num_trials, do_log_headers);
+
+        // Add trials to cumulative statistics.
+        for (std::list<SA_POP::Exp_EU_Trial_Results>::iterator trial_iter = run_results.trials.begin (); trial_iter != run_results.trials.end (); trial_iter++) {
+          double pref_best_ratio = (*trial_iter).pref_plan_eu / (*trial_iter).max_plan_eu;
+
+          acc_pref_best_ratio (pref_best_ratio);
+          acc_num_plans ((*trial_iter).num_plans);
+        }
+
+        // Add run to cumulative statistics.
+        acc_trial_attempts (run_results.num_trial_attempts);
+        acc_init_plans (run_results.num_init_plans);
+        acc_pref_plans (run_results.num_pref_plans);
+        acc_alt_plans (run_results.num_alt_plans);
+
+        // Delete planner.
+        delete planner;
+      }
+
+      // Open/create experiment output file.
+      std::ofstream log_exp_out;
+      log_exp_out.open (log_exp_filename.c_str (), std::ios_base::out | std::ios_base::app);
+      if (log_exp_out == 0){
+        std::string msg = "SA-POP EU EXPERIMENT ERROR: in main():  Unable to open experiment log file (";
+        msg += log_exp_filename;
+        msg += ") for writing (append).";
+        std::cerr << msg;
+        throw msg;
+      }
+
+      // Output experiment results.
+      if (do_log_headers) {
+        // Output experiment header to file.
+        log_exp_out << "Network base name" << "\t";
+        log_exp_out << "Number of goal conditions" << "\t";
+        log_exp_out << "Percent initial conditions true" << "\t";
+        log_exp_out << "Goal utility min" << "\t";
+        log_exp_out << "Goal utility max" << "\t";
+        log_exp_out << "Plan EU ratio (preferred : best) Mean" << "\t";
+        log_exp_out << "Plan EU ratio (preferred : best) Std Dev" << "\t";
+        log_exp_out << "Trial plans generated Mean" << "\t";
+        log_exp_out << "Trial plans generated Std Dev" << "\t";
+        log_exp_out << "Trial attempts Mean" << "\t";
+        log_exp_out << "Trial attempts Std Dev" << "\t";
+        log_exp_out << "Trials w/ initial plan Mean" << "\t";
+        log_exp_out << "Trials w/ initial plan Std Dev" << "\t";
+        log_exp_out << "Trials w/ preferred plan Mean" << "\t";
+        log_exp_out << "Trials w/ preferred plan Std Dev" << "\t";
+        log_exp_out << "Trials w/ alternate plan(s) Mean";
+        log_exp_out << "Trials w/ alternate plan(s) Std Dev";
+        log_exp_out << std::endl;
+      }
+
+      // Output run statistics to file.
+      log_exp_out << base_filename << "\t";
+      log_exp_out << trial_params.num_goal_conds << "\t";
+      log_exp_out << trial_params.percent_init_true << "\t";
+      log_exp_out << trial_params.util_min << "\t";
+      log_exp_out << trial_params.util_max << "\t";
+      log_exp_out << boost::accumulators::mean(acc_pref_best_ratio) << "\t";
+      log_exp_out << std::sqrt(boost::accumulators::variance(acc_pref_best_ratio)) << "\t";
+      log_exp_out << boost::accumulators::mean(acc_num_plans) << "\t";
+      log_exp_out << std::sqrt(boost::accumulators::variance(acc_num_plans)) << "\t";
+      log_exp_out << boost::accumulators::mean(acc_trial_attempts) << "\t";
+      log_exp_out << std::sqrt(boost::accumulators::variance(acc_trial_attempts)) << "\t";
+      log_exp_out << boost::accumulators::mean(acc_init_plans) << "\t";
+      log_exp_out << std::sqrt(boost::accumulators::variance(acc_init_plans)) << "\t";
+      log_exp_out << boost::accumulators::mean(acc_pref_plans) << "\t";
+      log_exp_out << std::sqrt(boost::accumulators::variance(acc_pref_plans)) << "\t";
+      log_exp_out << boost::accumulators::mean(acc_alt_plans) << "\t";
+      log_exp_out << std::sqrt(boost::accumulators::variance(acc_alt_plans)) << "\t";
+      log_exp_out << std::endl;
+      
+    } else {  // Run on single network.
+      // Get filenames from user.
+      UserInterface::Question sanet_file_ques ("Task Network file:");
+      if (user_input.ask (sanet_file_ques))
+        sanet_filename = sanet_file_ques.get_answer ();
+      else {
+        std::cerr << "No Task Network filename provided." << std::endl;
+        throw "No Task Network filename provided.";
+      }
+      UserInterface::Question tm_file_ques ("Task Map file:");
+      if (user_input.ask (tm_file_ques))
+        tm_filename = tm_file_ques.get_answer ();
+      else {
+        std::cerr << "No Task Map filename provided." << std::endl;
+        throw "No Task Map filename provided.";
+      }
+
+    
+      SA_POP::Exp_EU_Builder builder;
+      SANet::SANetFileIn sanet_in;
+      SA_POP::TaskMapFileIn tm_in;
+
+      // Build task network and task map.
+      try {
+        sanet_in.build_net (sanet_filename, &builder);
+        tm_in.build_task_map (tm_filename, &builder);
+      } catch (std::string e) {
+        std::cerr << "ERROR while building task network and task map from files:";
+        std::cerr << std::endl;
+        std::cerr << e;
+      } catch (...) {
+        std::cerr << "UNKNOWN ERROR while building task network and task map from files." << std::endl;
+      }
+
+      // Get SA-POP Planner.
+      planner = builder.get_exp_eu_planner ();
+
+
+      // Output plans to screen if user wishes.
+      UserInterface::QuestionBool screenout_ques ("Output plans to screen? [(Y)es or (N)o]:", false);
+      user_input.ask (screenout_ques);
+      if (screenout_ques.get_answer_bool ())
+        planner->add_out_adapter (&screen_out);
+
+      // Pause between plans if user wishes.
+      UserInterface::QuestionBool pause_ques ("Pause between plans? [(Y)es or (N)o]:", false);
+      UserInterface::QuestionBool ques_cont_plan ("Continue planning to find next plan? [(Y)es or (N)o]:", false);
+      user_input.ask (pause_ques);
+      if (pause_ques.get_answer_bool ()) {
+        // Set planner to ask whether continue after each plan is generated.
+        planner->set_pause (&user_input, &ques_cont_plan);
+
+        // Output graphical plans if user wishes.
+        UserInterface::QuestionBool graphout_ques ("Output graphical plans? [(Y)es or (N)o]:", false);
+        user_input.ask (graphout_ques);
+        if (graphout_ques.get_answer_bool ())
+          planner->add_out_adapter (&graph_out);
+      }
 
 //****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP
-    std::cout << "Experimental parameters:" << std::endl;
-    std::cout << "   Max SA steps: " << max_steps << std::endl;
-    std::cout << "   Num goals: " << num_goals << std::endl;
-    std::cout << "   Percent init conds true: " << percent_conds_init << std::endl;
-    std::cout << "   Goal min utility: " << min_util << std::endl;
-    std::cout << "   Goal max utility: " << max_util << std::endl;
+      std::cout << "Experimental parameters:" << std::endl;
+      std::cout << "   Number of goal conditions: " << trial_params.num_goal_conds << std::endl;
+      std::cout << "   Percent of initial conditions true: " << trial_params.percent_init_true << std::endl;
+      std::cout << "   Goal condition min utility: " << trial_params.util_min << std::endl;
+      std::cout << "   Goal condition max utility: " << trial_params.util_max << std::endl;
 //****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP
 
-    // Run trials.
-    planner->exp_run (max_steps, log_trials_filename, log_runs_filename, num_goals, percent_conds_init, min_util, max_util, num_trials * SA_POP::Default::TrialAttemptsMult, num_trials, do_log_headers);
+      // Run trials.
+      run_results = planner->exp_run (log_trials_filename, log_runs_filename, sanet_filename, trial_params, num_trials * SA_POP::Default::TrialAttemptsMult, num_trials, do_log_headers);
+
+      delete planner;
+    }
   } else {
     
+    SA_POP::Exp_EU_Builder builder;
+    SANet::SANetFileIn sanet_in;
+    SA_POP::TaskMapFileIn tm_in;
+
     // Create empty goal.
     SA_POP::Goal goal;
     goal.goal_id = "UserSpecifiedGoal ID";
@@ -376,10 +561,9 @@ int main (int argc, char* argv[])
 //    SA_POP::Plan plan = planner->get_plan ();
 //    SA_POP::Utility plan_eu = planner->calc_plan_eu (plan);
 //    std::cout << "Expected utility of generated plan:  " << plan_eu << std::endl;
-  }
-
 
-  delete planner;
+    delete planner;
+  }
 
   // Wait for user to end program.
 //  UserInterface::Question end_ques ("Enter any character to end program:");