From fe6fced6f3de7e1d45024849e8d7f9d1986d7cf3 Mon Sep 17 00:00:00 2001 From: "fergus.henderson" Date: Tue, 4 May 2010 20:09:58 +0000 Subject: Apply patch from Ryan Burns : add 1, 3, 5 minute averages of the number of children used on a server to the statistics collected by the stats server. git-svn-id: http://distcc.googlecode.com/svn/trunk@719 01de4be4-8c4a-0410-9132-4925637da917 --- src/serve.c | 2 +- src/stats.c | 149 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++---- src/stats.h | 3 +- src/util.c | 18 ++++++++ src/util.h | 2 + 5 files changed, 162 insertions(+), 12 deletions(-) diff --git a/src/serve.c b/src/serve.c index c31f59f..56c1486 100644 --- a/src/serve.c +++ b/src/serve.c @@ -769,7 +769,7 @@ out_cleanup: if (job_result == STATS_COMPILE_OK) { /* special case, also log compiler, file and time */ - dcc_stats_compile_ok(argv[0], orig_input, time_ms); + dcc_stats_compile_ok(argv[0], orig_input, start, end, time_ms); } else { dcc_stats_event(job_result); } diff --git a/src/stats.c b/src/stats.c index 14944a9..c2ef762 100644 --- a/src/stats.c +++ b/src/stats.c @@ -56,18 +56,27 @@ void dcc_manage_kids(int listen_fd); struct stats_s { int counters[STATS_ENUM_MAX]; + int kids_avg[3]; /* 1, 5, 15m */ int longest_job_time; char longest_job_name[MAX_FILENAME_LEN]; char longest_job_compiler[MAX_FILENAME_LEN]; int io_rate; /* read/write sectors per second */ - int compile_timeseries[300]; /* 300 3-sec time intervals */ + + /* + * linked list of statsdata. This is to keep the last + * couple of minutes of compile stats for analysis. + */ + struct statsdata *sd_root; } dcc_stats; struct statsdata { enum stats_e type; + struct statsdata *next; /* used only for STATS_COMPILE_OK */ + struct timeval start; + struct timeval stop; int time; char filename[MAX_FILENAME_LEN]; char compiler[MAX_FILENAME_LEN]; @@ -113,13 +122,16 @@ void dcc_stats_event(enum stats_e e) { /** * Logs a completed job to stats server **/ -void dcc_stats_compile_ok(char *compiler, char *filename, int time_usec) { +void dcc_stats_compile_ok(char *compiler, char *filename, struct timeval start, + struct timeval stop, int time_usec) { if (arg_stats) { struct statsdata sd; memset(&sd, 0, sizeof(sd)); sd.type = STATS_COMPILE_OK; /* also send compiler, filename & runtime */ + memcpy(&(sd.start), &start, sizeof(struct timeval)); + memcpy(&(sd.stop), &stop, sizeof(struct timeval)); sd.time = time_usec; strncpy(sd.filename, filename, MAX_FILENAME_LEN); strncpy(sd.compiler, compiler, MAX_FILENAME_LEN); @@ -128,6 +140,123 @@ void dcc_stats_compile_ok(char *compiler, char *filename, int time_usec) { } +/* + * tracks the compile times + */ +static void dcc_stats_update_compile_times(struct statsdata *sd) { + struct statsdata *prev_sd = NULL; + struct statsdata *curr_sd = NULL; + struct statsdata *new_sd; + time_t two_min_ago = time(NULL) - 120; + + /* Record file with longest runtime */ + if (dcc_stats.longest_job_time < sd->time) { + dcc_stats.longest_job_time = sd->time; + strncpy(dcc_stats.longest_job_name, sd->filename, + MAX_FILENAME_LEN); + strncpy(dcc_stats.longest_job_compiler, sd->compiler, + MAX_FILENAME_LEN); + } + + /* store stats for compile time calcs */ + new_sd = malloc(sizeof(struct statsdata)); + memcpy(new_sd, sd, sizeof(struct statsdata)); + new_sd->next = dcc_stats.sd_root; + dcc_stats.sd_root = new_sd; + + + /* drop elements older than 2min */ + curr_sd = dcc_stats.sd_root; + while (curr_sd != NULL) { + if (curr_sd->stop.tv_sec < two_min_ago) { + /* delete the stat */ + if (prev_sd == NULL) { + dcc_stats.sd_root = curr_sd->next; + } else { + prev_sd->next = curr_sd->next; + } + free(curr_sd); + curr_sd = prev_sd->next; + } else { + /* we didn't delete anyting. move forward by one */ + prev_sd = curr_sd; + curr_sd = curr_sd->next; + } + } +} + +/* caclulate the avg kids used */ +static void dcc_stats_calc_kid_avg(void) { + static int total_5m[5] = {0}; + static int total_15m[15] = {0}; + static int pos_5m = 0; + static int pos_15m = 0; + static time_t last = 0; + struct timeval now; + struct timeval time_p; + struct statsdata *curr_sd; + int total_running = 0; + int running = 0; + int t = 0; + int x = 0; + int total = 0; + + gettimeofday(&now, NULL); + + /* calculate average kids used over the last minute */ + if ((now.tv_sec - 60) >= last) { + /* we look at 1min ago back to 2 min ago because we only register + * compiles when they complete. If we look right now, we miss all + * the current compiles that haven't completed. + */ + for (t=60; t<120; t++) { + time_p.tv_usec = now.tv_usec; + time_p.tv_sec = now.tv_sec - t; + running = 0; + curr_sd = dcc_stats.sd_root; + + while (curr_sd != NULL) { + if ((dcc_timecmp(curr_sd->start, time_p) <= 0) && + (dcc_timecmp(curr_sd->stop, time_p) >= 0)) { + running++; + } + curr_sd = curr_sd->next; + } + total_running += running; + } + dcc_stats.kids_avg[0] = total_running / 60; + + + /* populate 5m kid avgs */ + total_5m[pos_5m] = dcc_stats.kids_avg[0]; + pos_5m++; + if (pos_5m >= 5) + pos_5m = 0; + + /* calc 5m kid avg */ + total = 0; + for (x=0; x<5; x++) + total += total_5m[x]; + dcc_stats.kids_avg[1] = total/5; + + + /* populate 15m kid avgs */ + total_15m[pos_15m] = dcc_stats.kids_avg[0]; + pos_15m++; + if (pos_15m >= 15) + pos_15m = 0; + + /* calc 15m kid avg */ + total = 0; + for (x=0; x<15; x++) + total += total_15m[x]; + dcc_stats.kids_avg[2] = total/15; + + last = now.tv_sec; + } +} + + /* * Updates a running total of compiles in the last 1, 5, 15 minutes * @@ -253,6 +382,9 @@ dcc_longest_job %s\n\ dcc_longest_job_compiler %s\n\ dcc_longest_job_time_msecs %d\n\ dcc_max_kids %d\n\ +dcc_avg_kids1 %d\n\ +dcc_avg_kids2 %d\n\ +dcc_avg_kids3 %d\n\ dcc_current_load %d\n\ dcc_load1 %1.2lf\n\ dcc_load2 %1.2lf\n\ @@ -296,6 +428,9 @@ dcc_free_space %d MB\n\ dcc_stats.longest_job_compiler, dcc_stats.longest_job_time, dcc_max_kids, + dcc_stats.kids_avg[0], + dcc_stats.kids_avg[1], + dcc_stats.kids_avg[2], dcc_getcurrentload(), loadavg[0], loadavg[1], loadavg[2], ct[0], ct[1], ct[2], @@ -331,14 +466,7 @@ static void dcc_stats_process(struct statsdata *sd) { case STATS_REJ_OVERLOAD: break; case STATS_COMPILE_OK: - /* Record file with longest runtime */ - if (dcc_stats.longest_job_time < sd->time) { - dcc_stats.longest_job_time = sd->time; - strncpy(dcc_stats.longest_job_name, sd->filename, - MAX_FILENAME_LEN); - strncpy(dcc_stats.longest_job_compiler, sd->compiler, - MAX_FILENAME_LEN); - } + dcc_stats_update_compile_times(sd); case STATS_COMPILE_ERROR: case STATS_COMPILE_TIMEOUT: case STATS_CLI_DISCONN: @@ -389,6 +517,7 @@ int dcc_stats_server(int listen_fd) while (1) { dcc_stats_minutely_update(); + dcc_stats_calc_kid_avg(); timeout.tv_sec = 60; timeout.tv_usec = 0; diff --git a/src/stats.h b/src/stats.h index e711fd9..9bde285 100644 --- a/src/stats.h +++ b/src/stats.h @@ -39,7 +39,8 @@ int dcc_stats_init(void); void dcc_stats_init_kid(void); int dcc_stats_server(int listen_fd); void dcc_stats_event(enum stats_e e); -void dcc_stats_compile_ok(char *compiler, char *filename, int time_usec); +void dcc_stats_compile_ok(char *compiler, char *filename, struct timeval start, + struct timeval stop, int time_usec); #ifdef __cplusplus } diff --git a/src/util.c b/src/util.c index 560797f..0762867 100644 --- a/src/util.c +++ b/src/util.c @@ -456,6 +456,24 @@ char *dcc_abspath(const char *path, int path_len) return buf; } +/* Return -1 if a < b, 0 if a == b, and 1 if a > b */ +int dcc_timecmp(struct timeval a, struct timeval b) { + if (a.tv_sec < b.tv_sec) { + return -1; + } else if (a.tv_sec > b.tv_sec) { + return 1; + } else if (a.tv_usec < b.tv_usec) { + /* at this point their tv_sec must be the same */ + return -1; + } else if (a.tv_usec > b.tv_usec) { + return 1; + } else { + /* they must be equal */ + return 0; + } +} + + /* Return the current number of running processes. */ int dcc_getcurrentload(void) { #if defined(linux) diff --git a/src/util.h b/src/util.h index d32b37b..463a383 100644 --- a/src/util.h +++ b/src/util.h @@ -23,8 +23,10 @@ #include #include +#include /* util.c */ +int dcc_timecmp(struct timeval a, struct timeval b); int dcc_getcurrentload(void); void dcc_getloadavg(double loadavg[3]); int argv_contains(char **argv, const char *s); -- cgit v1.2.1