diff options
| -rw-r--r-- | SA_POP/ChangeLog | 9 | ||||
| -rw-r--r-- | SA_POP/experiments/EU_Performance/Exp_Core/Exp_EU_Planner.cpp | 150 | ||||
| -rw-r--r-- | SA_POP/experiments/EU_Performance/Exp_Core/Exp_EU_Planner.h | 158 | ||||
| -rw-r--r-- | SA_POP/experiments/EU_Performance/SAPOP_Exp_EU.cpp | 344 |
4 files changed, 504 insertions, 157 deletions
diff --git a/SA_POP/ChangeLog b/SA_POP/ChangeLog index eb720c79ccf..f8b2a9d40e1 100644 --- a/SA_POP/ChangeLog +++ b/SA_POP/ChangeLog @@ -1,3 +1,12 @@ +Sat Jan 16 19:57:13 UTC 2010 John S. Kinnebrew <john.s.kinnebrew@vanderbilt.edu> + + * experiments/EU_Performance/Exp_Core/Exp_EU_Planner.h: + * experiments/EU_Performance/Exp_Core/Exp_EU_Planner.cpp: + * experiments/EU_Performance/SAPOP_Exp_EU.cpp: + + Added experimental runs on a series of networks and cumulative statistics gathering/output. + + Sat Jan 16 03:39:12 UTC 2010 John S. Kinnebrew <john.s.kinnebrew@vanderbilt.edu> * experiments/EU_Performance/Exp_Core/Exp_EU_Planner.cpp: diff --git a/SA_POP/experiments/EU_Performance/Exp_Core/Exp_EU_Planner.cpp b/SA_POP/experiments/EU_Performance/Exp_Core/Exp_EU_Planner.cpp index a84ee5ad830..398ceb58770 100644 --- a/SA_POP/experiments/EU_Performance/Exp_Core/Exp_EU_Planner.cpp +++ b/SA_POP/experiments/EU_Performance/Exp_Core/Exp_EU_Planner.cpp @@ -20,8 +20,9 @@ #include <stdlib.h> #include <fstream> #include <sstream> +#include <algorithm> -#include "boost/random.hpp" +#include <boost/random.hpp> #include <boost/accumulators/accumulators.hpp> #include <boost/accumulators/statistics/stats.hpp> #include <boost/accumulators/statistics/mean.hpp> @@ -46,18 +47,13 @@ Exp_EU_Planner::Exp_EU_Planner (void) :do_pause_ (false), input_ (0), ques_ (0), - do_stats_out_ (false), - trial_num_plans_ (0), - run_num_trial_attempts_ (0), - run_num_init_plans_ (0), - run_num_pref_plans_ (0), - run_num_alt_plans_ (0), - pref_plan_eu_ (0.0), - max_plan_eu_ (0.0) + do_stats_out_ (false) { + this->reset_trial_stats (); + this->reset_run_stats (); + // std::ofstream log_trials_out_; // std::ofstream log_runs_out_; - // All other initialization handled by Planner base class constructor. }; @@ -82,18 +78,31 @@ Exp_EU_Planner::~Exp_EU_Planner (void) // Reset all planning statistics for an individual trial. void Exp_EU_Planner::reset_trial_stats (void) { - this->trial_num_plans_ = 0; - this->pref_plan_eu_ = 0.0; - this->max_plan_eu_ = 0.0; + this->trial_results_.num_plans = 0; + this->trial_results_.pref_plan_eu = 0.0; + this->trial_results_.max_plan_eu = 0.0; + + // Create empty goal. + SA_POP::Goal goal; + goal.goal_id = "UserSpecifiedGoal ID"; + goal.name = "User specified goal"; + goal.abs_times.clear (); + goal.rel_times.clear (); + goal.goal_conds.clear (); + goal.start_window = std::make_pair (0, 0); + + this->trial_results_.goal = goal; }; // Reset all planning statistics for an individual experimental run. void Exp_EU_Planner::reset_run_stats (void) { - this->run_num_trial_attempts_ = 0; - this->run_num_init_plans_ = 0; - this->run_num_pref_plans_ = 0; - this->run_num_alt_plans_ = 0; + this->run_results_.num_trial_attempts = 0; + this->run_results_.num_init_plans = 0; + this->run_results_.num_pref_plans = 0; + this->run_results_.num_alt_plans = 0; + this->run_results_.net_name = "UnknownNetwork"; + this->run_results_.trials.clear (); }; // Set to pause and ask user whether to continue after each plan is generated. @@ -111,8 +120,14 @@ void Exp_EU_Planner::unset_pause (void) }; // Run experiment planning. -void Exp_EU_Planner::exp_run (size_t sa_max_steps, std::string log_trials_filename, std::string log_runs_filename, size_t num_goal_conds, double percent_init_true, size_t util_min, size_t util_max, size_t max_trial_attempts, size_t num_trials, bool do_log_headers) +SA_POP::Exp_EU_Run_Results Exp_EU_Planner::exp_run (std::string log_trials_filename, std::string log_runs_filename, std::string net_name, SA_POP::Exp_EU_Trial_Params trial_params, size_t max_trial_attempts, size_t num_trials, bool do_log_headers) { + // Maximum spreading activation steps to update network is twice + // the number of tasks (for worst case of linear network). + size_t sa_max_steps = 2 * this->sanet_->get_num_tasks (); + + this->run_results_.net_name = net_name; + // Set flag to output statistics to file. this->do_stats_out_ = true; @@ -152,25 +167,29 @@ void Exp_EU_Planner::exp_run (size_t sa_max_steps, std::string log_trials_filena for (size_t trial_num = 0; trial_num < num_trials; trial_num++) { // If maximum number of trial attempts have been reached, // stop running trials. - if (this->run_num_trial_attempts_ >= max_trial_attempts) + if (this->run_results_.num_trial_attempts >= max_trial_attempts) break; // Initialize trial statistics. this->reset_trial_stats (); // Initialize conditions and goal. - SA_POP::Goal goal = this->exp_init (num_goal_conds, percent_init_true, util_min, util_max); + SA_POP::Goal goal = this->exp_init (trial_params); + this->trial_results_.goal = goal; // Perform experimental run. this->plan (sa_max_steps, goal); // Increment trial attempts counter. - this->run_num_trial_attempts_++; + this->run_results_.num_trial_attempts++; // Trial only valid if at least one alternate plan was generated. - if (this->trial_num_plans_ >= 3) { + if (this->trial_results_.num_plans >= 3) { // Log trial statistics to output file. this->log_trial_stats (); + + // Add to run results. + this->run_results_.trials.push_back (this->trial_results_); } else { // For invalid trial, decrement trial counter. trial_num--; @@ -179,10 +198,13 @@ void Exp_EU_Planner::exp_run (size_t sa_max_steps, std::string log_trials_filena } // Log run statistics to output file. - this->log_run_stats (); + this->log_run_stats (trial_params); // Close output files. this->log_trials_out_.close (); + this->log_runs_out_.close (); + + return this->run_results_; }; @@ -218,15 +240,15 @@ bool Exp_EU_Planner::full_sched () void Exp_EU_Planner::track_stats (SA_POP::Plan plan) { // Update number of plans generated. - this->trial_num_plans_++; + this->trial_results_.num_plans++; // Ignore first plan. - if (this->trial_num_plans_ <= 1) { + if (this->trial_results_.num_plans <= 1) { // First plan. - this->trial_num_plans_ = 1; + this->trial_results_.num_plans = 1; // Increment counter of trials with initial plan. - this->run_num_init_plans_++; + this->run_results_.num_init_plans++; // Nothing else to do for first plan. return; @@ -236,12 +258,12 @@ void Exp_EU_Planner::track_stats (SA_POP::Plan plan) SA_POP::Utility plan_eu = this->calc_plan_eu (plan); // Second plan is SA-POP preferred plan. - if (this->trial_num_plans_ == 2) { + if (this->trial_results_.num_plans == 2) { // Current maximum plan EU is this (preferred) plan's EU. - this->pref_plan_eu_ = this->max_plan_eu_ = plan_eu; + this->trial_results_.pref_plan_eu = this->trial_results_.max_plan_eu = plan_eu; // Increment counter of trials with preferred plan. - this->run_num_pref_plans_++; + this->run_results_.num_pref_plans++; // Nothing else to do for first plan. return; @@ -251,27 +273,27 @@ void Exp_EU_Planner::track_stats (SA_POP::Plan plan) // If this is the first alternate plan (third plan generated), // increment counter of trials with at least one alternate plan. - if (this->trial_num_plans_ == 3) { - this->run_num_alt_plans_++; + if (this->trial_results_.num_plans == 3) { + this->run_results_.num_alt_plans++; } // If this plan has a higher EU than current max, update max. - if (plan_eu > this->max_plan_eu_) - this->max_plan_eu_ = plan_eu; + if (plan_eu > this->trial_results_.max_plan_eu) + this->trial_results_.max_plan_eu = plan_eu; }; // Initialize experiment (set values for initial conditions and create goal). -SA_POP::Goal Exp_EU_Planner::exp_init (size_t num_goal_conds, double percent_init_true, size_t util_min, size_t util_max) +SA_POP::Goal Exp_EU_Planner::exp_init (SA_POP::Exp_EU_Trial_Params params) { // Check goal utility min & max for consistency. - if (util_min > util_max) { + if (params.util_min > params.util_max) { std::cerr << "SA-POP ERROR in Exp_EU_Planner::exp_init (): goal utility min is >= goal utility max"; throw "SA-POP ERROR in Exp_EU_Planner::exp_init (): goal utility min is >= goal utility max"; } SA_POP::Goal goal; - goal.goal_id = "ExperimentGoalID"; + goal.goal_id = "ExperimentGeneratedGoalID"; goal.name = "Experiment generated goal"; goal.abs_times.clear (); goal.rel_times.clear (); @@ -282,7 +304,7 @@ SA_POP::Goal Exp_EU_Planner::exp_init (size_t num_goal_conds, double percent_ini SA_POP::CondSet conds = this->sanet_->get_all_conds (); // Check number of goal conditions for consistency. - if (num_goal_conds > conds.size ()) { + if (params.num_goal_conds > conds.size ()) { std::cerr << "SA-POP ERROR in Exp_EU_Planner::exp_init (): more goal conditions than number of goals"; throw "SA-POP ERROR in Exp_EU_Planner::exp_init (): more goal conditions than number of goals"; } @@ -290,7 +312,7 @@ SA_POP::Goal Exp_EU_Planner::exp_init (size_t num_goal_conds, double percent_ini // Initialize random number generators. boost::uniform_int<> goal_choice_dist(0, conds.size () - 1); boost::variate_generator<boost::mt19937&, boost::uniform_int<> > gen_goal_choice(this->rand_gen_, goal_choice_dist); - boost::uniform_int<> goal_util_dist(util_min, util_max); + boost::uniform_int<> goal_util_dist(params.util_min, params.util_max); boost::variate_generator<boost::mt19937&, boost::uniform_int<> > gen_goal_util(this->rand_gen_, goal_util_dist); boost::uniform_real<> cond_init_dist(0.0, 1.0); boost::variate_generator<boost::mt19937&, boost::uniform_real<> > gen_cond_init(this->rand_gen_, cond_init_dist); @@ -298,7 +320,7 @@ SA_POP::Goal Exp_EU_Planner::exp_init (size_t num_goal_conds, double percent_ini // Set goals. std::set<SA_POP::CondID> goal_ids; goal_ids.clear (); - for (size_t goal_ctr = 0; goal_ctr < num_goal_conds; goal_ctr++) { + for (size_t goal_ctr = 0; goal_ctr < params.num_goal_conds; goal_ctr++) { // Choose goal condition number. size_t goal_choice = gen_goal_choice (); @@ -344,7 +366,7 @@ SA_POP::Goal Exp_EU_Planner::exp_init (size_t num_goal_conds, double percent_ini // Randomly determine whether to set condition to true. double init_check = gen_cond_init (); - if (init_check <= percent_init_true) { + if (init_check <= params.percent_init_true) { this->sanet_->update_cond_val ((*cond_iter).id, true); } } @@ -390,9 +412,9 @@ void Exp_EU_Planner::log_trial_stats (void) } // Output trial statistics to file. - this->log_trials_out_ << this->pref_plan_eu_ << "\t"; - this->log_trials_out_ << this->max_plan_eu_ << "\t"; - this->log_trials_out_ << (this->trial_num_plans_ - 2); + this->log_trials_out_ << this->trial_results_.pref_plan_eu << "\t"; + this->log_trials_out_ << this->trial_results_.max_plan_eu << "\t"; + this->log_trials_out_ << (this->trial_results_.num_plans - 2); this->log_trials_out_ << std::endl; }; @@ -410,8 +432,17 @@ void Exp_EU_Planner::log_run_header (void) std::cerr << msg; throw msg; } - + // Output run header to file. + this->log_runs_out_ << "SANet name" << "\t"; + this->log_runs_out_ << "Number of goal conditions" << "\t"; + this->log_runs_out_ << "Percent initial conditions true" << "\t"; + this->log_runs_out_ << "Goal utility min" << "\t"; + this->log_runs_out_ << "Goal utility max" << "\t"; + this->log_runs_out_ << "Plan EU ratio (preferred : best) Mean" << "\t"; + this->log_runs_out_ << "Plan EU ratio (preferred : best) Std Dev" << "\t"; + this->log_runs_out_ << "Trial plans generated Mean" << "\t"; + this->log_runs_out_ << "Trial plans generated Std Dev" << "\t"; this->log_runs_out_ << "Trial attempts" << "\t"; this->log_runs_out_ << "Trials w/ initial plan" << "\t"; this->log_runs_out_ << "Trials w/ preferred plan" << "\t"; @@ -420,7 +451,7 @@ void Exp_EU_Planner::log_run_header (void) }; // Output run statistics to log. -void Exp_EU_Planner::log_run_stats (void) +void Exp_EU_Planner::log_run_stats (SA_POP::Exp_EU_Trial_Params trial_params) { // Only log statistics if flag is set. if (!this->do_stats_out_) @@ -433,11 +464,30 @@ void Exp_EU_Planner::log_run_stats (void) throw msg; } + // Calculate cumulative statistics for trials. + boost::accumulators::accumulator_set<double, boost::accumulators::stats<boost::accumulators::tag::mean, boost::accumulators::tag::variance> > acc_pref_best_ratio; + boost::accumulators::accumulator_set<double, boost::accumulators::stats<boost::accumulators::tag::mean, boost::accumulators::tag::variance> > acc_num_plans; + for (std::list<Exp_EU_Trial_Results>::iterator trial_iter = this->run_results_.trials.begin (); trial_iter != this->run_results_.trials.end (); trial_iter++) { + double pref_best_ratio = (*trial_iter).pref_plan_eu / (*trial_iter).max_plan_eu; + + acc_pref_best_ratio (pref_best_ratio); + acc_num_plans ((*trial_iter).num_plans); + } + // Output run statistics to file. - this->log_runs_out_ << this->run_num_trial_attempts_ << "\t"; - this->log_runs_out_ << this->run_num_init_plans_ << "\t"; - this->log_runs_out_ << this->run_num_pref_plans_ << "\t"; - this->log_runs_out_ << this->run_num_alt_plans_; + this->log_runs_out_ << this->run_results_.net_name << "\t"; + this->log_runs_out_ << trial_params.num_goal_conds << "\t"; + this->log_runs_out_ << trial_params.percent_init_true << "\t"; + this->log_runs_out_ << trial_params.util_min << "\t"; + this->log_runs_out_ << trial_params.util_max << "\t"; + this->log_runs_out_ << boost::accumulators::mean(acc_pref_best_ratio) << "\t"; + this->log_runs_out_ << std::sqrt(boost::accumulators::variance(acc_pref_best_ratio)) << "\t"; + this->log_runs_out_ << boost::accumulators::mean(acc_num_plans) << "\t"; + this->log_runs_out_ << std::sqrt(boost::accumulators::variance(acc_num_plans)) << "\t"; + this->log_runs_out_ << this->run_results_.num_trial_attempts << "\t"; + this->log_runs_out_ << this->run_results_.num_init_plans << "\t"; + this->log_runs_out_ << this->run_results_.num_pref_plans << "\t"; + this->log_runs_out_ << this->run_results_.num_alt_plans; this->log_runs_out_ << std::endl; }; diff --git a/SA_POP/experiments/EU_Performance/Exp_Core/Exp_EU_Planner.h b/SA_POP/experiments/EU_Performance/Exp_Core/Exp_EU_Planner.h index b935983b95d..0694ab0bd1d 100644 --- a/SA_POP/experiments/EU_Performance/Exp_Core/Exp_EU_Planner.h +++ b/SA_POP/experiments/EU_Performance/Exp_Core/Exp_EU_Planner.h @@ -21,6 +21,7 @@ #include <string> #include <set> #include <map> +#include <list> #include <fstream> #include <boost/random.hpp> @@ -39,6 +40,113 @@ namespace SA_POP { + /// Container for experimental trial parameters. + struct Exp_EU_Trial_Params { + /// Number of goal conditions to generate. + size_t num_goal_conds; + + /// Percentage of conditions to set initial value to true. + double percent_init_true; + + /// Minimum goal utility (uniform random choice in range [util_min, util_max]). + int util_min; + + /// Maximum goal utility (uniform random choice in range [util_min, util_max]). + int util_max; + + bool operator== (const Exp_EU_Trial_Params &s) const { return (this->num_goal_conds == s.num_goal_conds && this->percent_init_true == s.percent_init_true && this->util_min == s.util_min && this->util_max == s.util_max); }; + bool operator!= (const Exp_EU_Trial_Params &s) const { return !(*this == s); }; + bool operator< (const Exp_EU_Trial_Params &s) const + { + if (this->num_goal_conds == s.num_goal_conds) { + if (this->percent_init_true == s.percent_init_true) { + if (this->util_min == s.util_min) { + return this->util_max < s.util_max; + } + return this->util_min < s.util_min; + } + return this->percent_init_true < s.percent_init_true; + } + return this->num_goal_conds < s.num_goal_conds; + }; + }; + + /// Container for experimental results of a trial. + struct Exp_EU_Trial_Results { + /// Expected utility of SA-POP preferred plan. + SA_POP::Utility pref_plan_eu; + + /// Highest expected utility of any (schedulable/valid) plan (including preferred plan). + SA_POP::Utility max_plan_eu; + + /// Number of plans generated. + size_t num_plans; + + /// Trial goal. + SA_POP::Goal goal; + + bool operator== (const Exp_EU_Trial_Results &s) const { return (this->pref_plan_eu == s.pref_plan_eu && this->max_plan_eu == s.max_plan_eu && this->num_plans == s.num_plans && this->goal == s.goal); }; + bool operator!= (const Exp_EU_Trial_Results &s) const { return !(*this == s); }; + bool operator< (const Exp_EU_Trial_Results &s) const + { + if (this->pref_plan_eu == s.pref_plan_eu) { + if (this->max_plan_eu == s.max_plan_eu) { + if (this->num_plans == s.num_plans) { + return this->goal < s.goal; + } + return this->num_plans < s.num_plans; + } + return this->max_plan_eu < s.max_plan_eu; + } + return this->pref_plan_eu < s.pref_plan_eu; + }; + }; + + /// Container for a series of experimental results. + struct Exp_EU_Run_Results { + /// Run counter for number of trial attempts. + size_t num_trial_attempts; + + /// Run counter for number of trials with initial plan generated. + size_t num_init_plans; + + /// Run counter for number of trials with preferred plan generated. + size_t num_pref_plans; + + /// Run counter for number of trials with at least one alternate plan generated. + size_t num_alt_plans; + + /// Name of network. + std::string net_name; + + /// List of trial results. + std::list<Exp_EU_Trial_Results> trials; + + bool operator== (const Exp_EU_Run_Results &s) const { return (this->num_trial_attempts == s.num_trial_attempts && this->num_init_plans == s.num_init_plans && this->num_pref_plans == s.num_pref_plans && this->num_alt_plans == s.num_alt_plans && this->net_name == s.net_name && this->trials == s.trials); }; + bool operator!= (const Exp_EU_Run_Results &s) const { return !(*this == s); }; + bool operator< (const Exp_EU_Run_Results &s) const + { + if (this->num_trial_attempts == s.num_trial_attempts) { + if (this->num_init_plans == s.num_init_plans) { + if (this->num_pref_plans == s.num_pref_plans) { + if (this->num_alt_plans == s.num_alt_plans) { + if (this->net_name == s.net_name) { + return this->trials < s.trials; + } + return this->net_name < s.net_name; + } + return this->num_alt_plans < s.num_alt_plans; + } + return this->num_pref_plans < s.num_pref_plans; + } + return this->num_init_plans < s.num_init_plans; + } + return this->num_trial_attempts < s.num_trial_attempts; + }; + }; + + + /** * @class Exp_EU_Planner * @@ -73,19 +181,13 @@ namespace SA_POP { /// Perform experimental trial(s). /** - * @param sa_max_steps Maximum steps to run spreading activation. - * * @param log_trials_filename Name of file to output trial statistics to. * * @param log_runs_filename Name of file to output cumulative statistics from run to. * - * @param num_goal_conds Number of goal conditions to generate. - * - * @param percent_init_true Percentage of conditions to set initial value to true. - * - * @param util_min Minimum goal utility (uniform random choice in range [util_min, util_max]). + * @param log_runs_filename Name of current spreading activation network for logging. * - * @param util_max Maximum goal utility (uniform random choice in range [util_min, util_max]). + * @param trial_params Parameters for random initialization of each trial. * * @param max_trial_attempts Maximum number of trial attempts to execute in order to reach assigned number of valid trials. * @@ -93,9 +195,8 @@ namespace SA_POP { * * @param do_log_headers Flag to output header lines to log files (if true). */ - virtual void exp_run (size_t sa_max_steps, std::string log_trials_filename, - std::string log_runs_filename, size_t num_goal_conds, double percent_init_true, - size_t util_min, size_t util_max, size_t max_trial_attempts, + virtual SA_POP::Exp_EU_Run_Results exp_run (std::string log_trials_filename, std::string log_runs_filename, + std::string net_name, SA_POP::Exp_EU_Trial_Params trial_params, size_t max_trial_attempts, size_t num_trials = 1, bool do_log_headers = false); @@ -132,26 +233,32 @@ namespace SA_POP { /// Output file stream for logging trial statistics. std::ofstream log_runs_out_; + /// Results of current experimental trial. + SA_POP::Exp_EU_Trial_Results trial_results_; + + /// Results of current experimental run (multiple trials). + SA_POP::Exp_EU_Run_Results run_results_; + /// Trial counter for number of plans generated. - size_t trial_num_plans_; +// size_t trial_num_plans_; /// Run counter for number of trial attempts. - size_t run_num_trial_attempts_; +// size_t run_num_trial_attempts_; /// Run counter for number of trials with initial plan generated. - size_t run_num_init_plans_; +// size_t run_num_init_plans_; /// Run counter for number of trials with preferred plan generated. - size_t run_num_pref_plans_; +// size_t run_num_pref_plans_; /// Run counter for number of trials with at least one alternate plan generated. - size_t run_num_alt_plans_; +// size_t run_num_alt_plans_; /// Expected utility of SA-POP preferred plan. - SA_POP::Utility pref_plan_eu_; +// SA_POP::Utility pref_plan_eu_; /// Highest expected utility of any (schedulable/valid) plan (including preferred plan). - SA_POP::Utility max_plan_eu_; +// SA_POP::Utility max_plan_eu_; // ************************************************************************ @@ -174,7 +281,10 @@ namespace SA_POP { virtual void log_trial_stats (void); /// Output run statistics to log. - virtual void log_run_stats (void); + /** + * @param trial_params Parameters used for random initialization of trials in this experimental run. + */ + virtual void log_run_stats (SA_POP::Exp_EU_Trial_Params trial_params); /// Output run header to log. virtual void log_run_header (void); @@ -182,17 +292,11 @@ namespace SA_POP { /// Initialize experiment (set values for initial conditions and /// create goal). /** - * @param num_goal_conds Number of goal conditions to generate. - * - * @param percent_init_true Percentage of conditions to set initial value to true. - * - * @param util_min Minimum goal utility (uniform random choice in range [util_min, util_max]). - * - * @param util_max Maximum goal utility (uniform random choice in range [util_min, util_max]). + * @param params Parameters for random initialization of trial. * * @return Goal to use in current experimental run. */ - virtual SA_POP::Goal exp_init (size_t num_goal_conds, double percent_init_true, size_t util_min, size_t util_max); + virtual SA_POP::Goal exp_init (SA_POP::Exp_EU_Trial_Params params); }; }; /* SA_POP namespace */ diff --git a/SA_POP/experiments/EU_Performance/SAPOP_Exp_EU.cpp b/SA_POP/experiments/EU_Performance/SAPOP_Exp_EU.cpp index f18f9c908a0..1ad1ee92cfc 100644 --- a/SA_POP/experiments/EU_Performance/SAPOP_Exp_EU.cpp +++ b/SA_POP/experiments/EU_Performance/SAPOP_Exp_EU.cpp @@ -21,12 +21,19 @@ #include <fstream> #include <vector> #include <map> +#include <list> + +#include <boost/accumulators/accumulators.hpp> +#include <boost/accumulators/statistics/stats.hpp> +#include <boost/accumulators/statistics/mean.hpp> +#include <boost/accumulators/statistics/variance.hpp> #include "SA_POP_Types.h" #include "Exp_Core/Exp_EU_Builder.h" #include "Exp_Core/Exp_EU_Planner.h" #include "TaskMapFileIn.h" #include "SANet/SANetFileIn.h" +#include "LogFileOut.h" #include "LogScreenOut.h" #include "LogGraphOut.h" //#include "SA_POP_Utils.h" @@ -138,9 +145,6 @@ void displayConds(SA_POP::Planner *plans, std::vector<SANet::CondID> checks, std int main (int argc, char* argv[]) { - SA_POP::Exp_EU_Builder builder; - SANet::SANetFileIn sanet_in; - SA_POP::TaskMapFileIn tm_in; SA_POP::Exp_EU_Planner *planner = 0; SA_POP::LogGraphOut graph_out (std::cout, false); SA_POP::LogScreenOut screen_out (std::cout); @@ -152,131 +156,312 @@ int main (int argc, char* argv[]) std::vector<SA_POP::CondID> kconds; - // Get filenames from user. - UserInterface::Question sanet_file_ques ("Task Network file:"); - if (user_input.ask (sanet_file_ques)) - sanet_filename = sanet_file_ques.get_answer (); - else { - std::cerr << "No Task Network filename provided." << std::endl; - throw "No Task Network filename provided."; - } - UserInterface::Question tm_file_ques ("Task Map file:"); - if (user_input.ask (tm_file_ques)) - tm_filename = tm_file_ques.get_answer (); - else { - std::cerr << "No Task Map filename provided." << std::endl; - throw "No Task Map filename provided."; - } - - - // Build task network and task map. - try { - sanet_in.build_net (sanet_filename, &builder); - tm_in.build_task_map (tm_filename, &builder); - } catch (std::string e) { - std::cerr << "ERROR while building task network and task map from files:"; - std::cerr << std::endl; - std::cerr << e; - } catch (...) { - std::cerr << "UNKNOWN ERROR while building task network and task map from files." << std::endl; - } - - // Get SA-POP Planner. - planner = builder.get_exp_eu_planner (); // Run manual or automatic trial based on user input. UserInterface::QuestionBool auto_trials_ques ("Automatic goal & initial condition generation? [(Y)es or (N)o]:", false); user_input.ask (auto_trials_ques); if (auto_trials_ques.get_answer_bool ()) { + std::string log_trials_filename = "log_trials.txt"; + std::string log_runs_filename = "log_runs.txt"; + std::string log_exp_filename = "log_exp.txt"; + size_t num_trials = 1; + size_t num_goals = 1; + size_t num_percent_conds_init = SA_POP::Default::NumPercentCondsInit; + size_t min_util = 100; + size_t max_util = min_util; + bool do_log_headers = false; + std::string plan_out_filename = "log_default_plan_out.txt"; + bool do_plan_file = false; + SA_POP::Exp_EU_Run_Results run_results; + bool do_mult_runs = false; + + // Get whether to run a series of + UserInterface::QuestionBool mult_runs_ques ("Run on a series of networks? [(Y)es or (N)o]:", false); + user_input.ask (mult_runs_ques); + do_mult_runs = mult_runs_ques.get_answer_bool (); + // Get trial log filename. - std::string log_trials_filename = "default_trials_filename.log"; UserInterface::Question log_trials_filename_ques ("Filename for trial log:"); if (user_input.ask (log_trials_filename_ques)) log_trials_filename = log_trials_filename_ques.get_answer (); // Get run log filename. - std::string log_runs_filename = "default_runs_filename.log"; UserInterface::Question log_runs_filename_ques ("Filename for run log:"); if (user_input.ask (log_runs_filename_ques)) log_runs_filename = log_runs_filename_ques.get_answer (); + // If multiple runs, get experiment log filename. + if (do_mult_runs) { + UserInterface::Question log_exp_filename_ques ("Filename for experiment log:"); + if (user_input.ask (log_exp_filename_ques)) + log_exp_filename = log_exp_filename_ques.get_answer (); + } + // Get number of experimental trials to perform. - size_t num_trials = 1; - UserInterface::QuestionInt num_trials_ques ("Number of trials to run:", SA_POP::Default::NumTrialsMin, SA_POP::Default::NumTrialsMax); + UserInterface::QuestionInt num_trials_ques ("Number of trials per network:", SA_POP::Default::NumTrialsMin, SA_POP::Default::NumTrialsMax); if (user_input.ask (num_trials_ques)) num_trials = num_trials_ques.get_answer_int (); // Get number of goal conditions. - size_t num_goals = 1; UserInterface::QuestionInt num_goals_ques ("Number of goal conditions:", SA_POP::Default::NumGoalsMin, SA_POP::Default::NumGoalsMax); if (user_input.ask (num_goals_ques)) num_goals = num_goals_ques.get_answer_int (); // Get percent of initial conditions true. - size_t num_percent_conds_init = SA_POP::Default::NumPercentCondsInit; UserInterface::QuestionInt conds_init_ques ("Percentage of initial conditions true [0 to 100]:", 0, 100); if (user_input.ask (conds_init_ques)) num_percent_conds_init = conds_init_ques.get_answer_int (); double percent_conds_init = (double) num_percent_conds_init / 100.0; // Get min goal utility. - size_t min_util = 100; - UserInterface::QuestionInt min_util_ques ("Minimum goal utility:", SA_POP::Default::GoalUtilMin, SA_POP::Default::GoalUtilMax); + UserInterface::QuestionInt min_util_ques ("Minimum goal condition utility:", SA_POP::Default::GoalUtilMin, SA_POP::Default::GoalUtilMax); if (user_input.ask (min_util_ques)) min_util = min_util_ques.get_answer_int (); // Get max goal utility. - size_t max_util = min_util; - UserInterface::QuestionInt max_util_ques ("Maximum goal utility:", min_util, SA_POP::Default::GoalUtilMax); + UserInterface::QuestionInt max_util_ques ("Maximum goal condition utility:", min_util, SA_POP::Default::GoalUtilMax); if (user_input.ask (max_util_ques)) max_util = max_util_ques.get_answer_int (); - // Get maximum steps of spreading activation from user. - UserInterface::QuestionInt steps_ques("Max steps to run spreading activation (10000 step limit): ", 0, 10000); - if (user_input.ask (steps_ques)) - max_steps = steps_ques.get_answer_int (); - // Get whether to output log headers. - bool do_log_headers = false; UserInterface::QuestionBool log_headers_ques ("Output headers in log files? [(Y)es or (N)o]:", false); user_input.ask (log_headers_ques); do_log_headers = log_headers_ques.get_answer_bool (); - // Output plans to screen if user wishes. - UserInterface::QuestionBool screenout_ques ("Output plans to screen? [(Y)es or (N)o]:", false); - user_input.ask (screenout_ques); - if (screenout_ques.get_answer_bool ()) - planner->add_out_adapter (&screen_out); - - // Pause between plans if user wishes. - UserInterface::QuestionBool ques_cont_plan ("Continue planning to find next plan? [(Y)es or (N)o]:", false); - UserInterface::QuestionBool pause_ques ("Pause between plans? [(Y)es or (N)o]:", false); - user_input.ask (pause_ques); - if (pause_ques.get_answer_bool ()) { - // Set planner to ask whether continue after each plan is generated. - planner->set_pause (&user_input, &ques_cont_plan); - - // Output graphical plans if user wishes. - UserInterface::QuestionBool graphout_ques ("Output graphical plans? [(Y)es or (N)o]:", false); - user_input.ask (graphout_ques); - if (graphout_ques.get_answer_bool ()) - planner->add_out_adapter (&graph_out); + // Get whether to output plans to file. + UserInterface::QuestionBool fileout_ques ("Output plans to file? [(Y)es or (N)o]:", false); + user_input.ask (fileout_ques); + do_plan_file = fileout_ques.get_answer_bool (); + if (do_plan_file) { + // Get plan out filename. + UserInterface::Question plan_out_filename_ques ("Filename for plan log:"); + if (user_input.ask (plan_out_filename_ques)) + plan_out_filename = plan_out_filename_ques.get_answer (); } + // Create trial parameters structure. + SA_POP::Exp_EU_Trial_Params trial_params; + trial_params.num_goal_conds = num_goals; + trial_params.percent_init_true = percent_conds_init; + trial_params.util_min = min_util; + trial_params.util_max = max_util; + + + + // If multiple runs, get base filename and run experiment. + if (do_mult_runs) { + std::string base_filename = "network"; + size_t num_runs = 1; + boost::accumulators::accumulator_set<double, boost::accumulators::stats<boost::accumulators::tag::mean, boost::accumulators::tag::variance> > acc_pref_best_ratio; + boost::accumulators::accumulator_set<int, boost::accumulators::stats<boost::accumulators::tag::mean, boost::accumulators::tag::variance> > acc_num_plans; + boost::accumulators::accumulator_set<int, boost::accumulators::stats<boost::accumulators::tag::mean, boost::accumulators::tag::variance> > acc_trial_attempts; + boost::accumulators::accumulator_set<int, boost::accumulators::stats<boost::accumulators::tag::mean, boost::accumulators::tag::variance> > acc_init_plans; + boost::accumulators::accumulator_set<int, boost::accumulators::stats<boost::accumulators::tag::mean, boost::accumulators::tag::variance> > acc_pref_plans; + boost::accumulators::accumulator_set<int, boost::accumulators::stats<boost::accumulators::tag::mean, boost::accumulators::tag::variance> > acc_alt_plans; + + // Get base filename. + UserInterface::Question base_file_ques ("Base filename:"); + if (user_input.ask (base_file_ques)) + base_filename = base_file_ques.get_answer (); + else { + std::cerr << "No base filename provided." << std::endl; + throw "No base filename provided."; + } + + // Get number of networks to run. + UserInterface::QuestionInt num_runs_ques ("Number of networks:", SA_POP::Default::NumTrialsMin, SA_POP::Default::NumTrialsMax); + if (user_input.ask (num_runs_ques)) + num_runs = num_runs_ques.get_answer_int (); + + // Run on requested number of networks. + for (size_t run_num = 0; run_num < num_runs; run_num++) { + + // Construct SANet and TM filenames. + std::ostringstream sanet_ss; + sanet_ss << base_filename << run_num << ".xml"; + sanet_filename = sanet_ss.str (); + std::ostringstream tm_ss; + tm_ss << base_filename << run_num << ".tm.xml"; + tm_filename = tm_ss.str (); + + SA_POP::Exp_EU_Builder builder; + SANet::SANetFileIn sanet_in; + SA_POP::TaskMapFileIn tm_in; + + // Build task network and task map. + try { + sanet_in.build_net (sanet_filename, &builder); + tm_in.build_task_map (tm_filename, &builder); + } catch (std::string e) { + std::cerr << "ERROR while building task network and task map from files:"; + std::cerr << std::endl; + std::cerr << e; + } catch (...) { + std::cerr << "UNKNOWN ERROR while building task network and task map from files." << std::endl; + } + + // Get SA-POP Planner. + planner = builder.get_exp_eu_planner (); + + // Add file output adapter, if user requested it. + SA_POP::LogFileOut file_out (plan_out_filename.c_str ()); + if (do_plan_file) + planner->add_out_adapter (&file_out); + + // Run trials. + std::cout << "Running trials on network " << run_num << std::endl; + run_results = planner->exp_run (log_trials_filename, log_runs_filename, sanet_filename, trial_params, num_trials * SA_POP::Default::TrialAttemptsMult, num_trials, do_log_headers); + + // Add trials to cumulative statistics. + for (std::list<SA_POP::Exp_EU_Trial_Results>::iterator trial_iter = run_results.trials.begin (); trial_iter != run_results.trials.end (); trial_iter++) { + double pref_best_ratio = (*trial_iter).pref_plan_eu / (*trial_iter).max_plan_eu; + + acc_pref_best_ratio (pref_best_ratio); + acc_num_plans ((*trial_iter).num_plans); + } + + // Add run to cumulative statistics. + acc_trial_attempts (run_results.num_trial_attempts); + acc_init_plans (run_results.num_init_plans); + acc_pref_plans (run_results.num_pref_plans); + acc_alt_plans (run_results.num_alt_plans); + + // Delete planner. + delete planner; + } + + // Open/create experiment output file. + std::ofstream log_exp_out; + log_exp_out.open (log_exp_filename.c_str (), std::ios_base::out | std::ios_base::app); + if (log_exp_out == 0){ + std::string msg = "SA-POP EU EXPERIMENT ERROR: in main(): Unable to open experiment log file ("; + msg += log_exp_filename; + msg += ") for writing (append)."; + std::cerr << msg; + throw msg; + } + + // Output experiment results. + if (do_log_headers) { + // Output experiment header to file. + log_exp_out << "Network base name" << "\t"; + log_exp_out << "Number of goal conditions" << "\t"; + log_exp_out << "Percent initial conditions true" << "\t"; + log_exp_out << "Goal utility min" << "\t"; + log_exp_out << "Goal utility max" << "\t"; + log_exp_out << "Plan EU ratio (preferred : best) Mean" << "\t"; + log_exp_out << "Plan EU ratio (preferred : best) Std Dev" << "\t"; + log_exp_out << "Trial plans generated Mean" << "\t"; + log_exp_out << "Trial plans generated Std Dev" << "\t"; + log_exp_out << "Trial attempts Mean" << "\t"; + log_exp_out << "Trial attempts Std Dev" << "\t"; + log_exp_out << "Trials w/ initial plan Mean" << "\t"; + log_exp_out << "Trials w/ initial plan Std Dev" << "\t"; + log_exp_out << "Trials w/ preferred plan Mean" << "\t"; + log_exp_out << "Trials w/ preferred plan Std Dev" << "\t"; + log_exp_out << "Trials w/ alternate plan(s) Mean"; + log_exp_out << "Trials w/ alternate plan(s) Std Dev"; + log_exp_out << std::endl; + } + + // Output run statistics to file. + log_exp_out << base_filename << "\t"; + log_exp_out << trial_params.num_goal_conds << "\t"; + log_exp_out << trial_params.percent_init_true << "\t"; + log_exp_out << trial_params.util_min << "\t"; + log_exp_out << trial_params.util_max << "\t"; + log_exp_out << boost::accumulators::mean(acc_pref_best_ratio) << "\t"; + log_exp_out << std::sqrt(boost::accumulators::variance(acc_pref_best_ratio)) << "\t"; + log_exp_out << boost::accumulators::mean(acc_num_plans) << "\t"; + log_exp_out << std::sqrt(boost::accumulators::variance(acc_num_plans)) << "\t"; + log_exp_out << boost::accumulators::mean(acc_trial_attempts) << "\t"; + log_exp_out << std::sqrt(boost::accumulators::variance(acc_trial_attempts)) << "\t"; + log_exp_out << boost::accumulators::mean(acc_init_plans) << "\t"; + log_exp_out << std::sqrt(boost::accumulators::variance(acc_init_plans)) << "\t"; + log_exp_out << boost::accumulators::mean(acc_pref_plans) << "\t"; + log_exp_out << std::sqrt(boost::accumulators::variance(acc_pref_plans)) << "\t"; + log_exp_out << boost::accumulators::mean(acc_alt_plans) << "\t"; + log_exp_out << std::sqrt(boost::accumulators::variance(acc_alt_plans)) << "\t"; + log_exp_out << std::endl; + + } else { // Run on single network. + // Get filenames from user. + UserInterface::Question sanet_file_ques ("Task Network file:"); + if (user_input.ask (sanet_file_ques)) + sanet_filename = sanet_file_ques.get_answer (); + else { + std::cerr << "No Task Network filename provided." << std::endl; + throw "No Task Network filename provided."; + } + UserInterface::Question tm_file_ques ("Task Map file:"); + if (user_input.ask (tm_file_ques)) + tm_filename = tm_file_ques.get_answer (); + else { + std::cerr << "No Task Map filename provided." << std::endl; + throw "No Task Map filename provided."; + } + + + SA_POP::Exp_EU_Builder builder; + SANet::SANetFileIn sanet_in; + SA_POP::TaskMapFileIn tm_in; + + // Build task network and task map. + try { + sanet_in.build_net (sanet_filename, &builder); + tm_in.build_task_map (tm_filename, &builder); + } catch (std::string e) { + std::cerr << "ERROR while building task network and task map from files:"; + std::cerr << std::endl; + std::cerr << e; + } catch (...) { + std::cerr << "UNKNOWN ERROR while building task network and task map from files." << std::endl; + } + + // Get SA-POP Planner. + planner = builder.get_exp_eu_planner (); + + + // Output plans to screen if user wishes. + UserInterface::QuestionBool screenout_ques ("Output plans to screen? [(Y)es or (N)o]:", false); + user_input.ask (screenout_ques); + if (screenout_ques.get_answer_bool ()) + planner->add_out_adapter (&screen_out); + + // Pause between plans if user wishes. + UserInterface::QuestionBool pause_ques ("Pause between plans? [(Y)es or (N)o]:", false); + UserInterface::QuestionBool ques_cont_plan ("Continue planning to find next plan? [(Y)es or (N)o]:", false); + user_input.ask (pause_ques); + if (pause_ques.get_answer_bool ()) { + // Set planner to ask whether continue after each plan is generated. + planner->set_pause (&user_input, &ques_cont_plan); + + // Output graphical plans if user wishes. + UserInterface::QuestionBool graphout_ques ("Output graphical plans? [(Y)es or (N)o]:", false); + user_input.ask (graphout_ques); + if (graphout_ques.get_answer_bool ()) + planner->add_out_adapter (&graph_out); + } //****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP - std::cout << "Experimental parameters:" << std::endl; - std::cout << " Max SA steps: " << max_steps << std::endl; - std::cout << " Num goals: " << num_goals << std::endl; - std::cout << " Percent init conds true: " << percent_conds_init << std::endl; - std::cout << " Goal min utility: " << min_util << std::endl; - std::cout << " Goal max utility: " << max_util << std::endl; + std::cout << "Experimental parameters:" << std::endl; + std::cout << " Number of goal conditions: " << trial_params.num_goal_conds << std::endl; + std::cout << " Percent of initial conditions true: " << trial_params.percent_init_true << std::endl; + std::cout << " Goal condition min utility: " << trial_params.util_min << std::endl; + std::cout << " Goal condition max utility: " << trial_params.util_max << std::endl; //****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP****TEMP - // Run trials. - planner->exp_run (max_steps, log_trials_filename, log_runs_filename, num_goals, percent_conds_init, min_util, max_util, num_trials * SA_POP::Default::TrialAttemptsMult, num_trials, do_log_headers); + // Run trials. + run_results = planner->exp_run (log_trials_filename, log_runs_filename, sanet_filename, trial_params, num_trials * SA_POP::Default::TrialAttemptsMult, num_trials, do_log_headers); + + delete planner; + } } else { + SA_POP::Exp_EU_Builder builder; + SANet::SANetFileIn sanet_in; + SA_POP::TaskMapFileIn tm_in; + // Create empty goal. SA_POP::Goal goal; goal.goal_id = "UserSpecifiedGoal ID"; @@ -376,10 +561,9 @@ int main (int argc, char* argv[]) // SA_POP::Plan plan = planner->get_plan (); // SA_POP::Utility plan_eu = planner->calc_plan_eu (plan); // std::cout << "Expected utility of generated plan: " << plan_eu << std::endl; - } - - delete planner; + delete planner; + } // Wait for user to end program. // UserInterface::Question end_ques ("Enter any character to end program:"); |