diff options
| -rw-r--r-- | AUTHORS | 1 | ||||
| -rw-r--r-- | lib/automake.mk | 8 | ||||
| -rw-r--r-- | lib/netlink.c | 17 | ||||
| -rw-r--r-- | lib/stream-fd.c | 25 | ||||
| -rw-r--r-- | lib/stress-unixctl.man | 72 | ||||
| -rw-r--r-- | lib/stress.c | 222 | ||||
| -rw-r--r-- | lib/stress.h | 92 | ||||
| -rw-r--r-- | ofproto/ofproto.c | 12 | ||||
| -rw-r--r-- | ovsdb/ovsdb-server.1.in | 1 | ||||
| -rw-r--r-- | ovsdb/ovsdb-server.c | 2 | ||||
| -rw-r--r-- | vswitchd/ovs-vswitchd.8.in | 1 | ||||
| -rw-r--r-- | vswitchd/ovs-vswitchd.c | 2 |
12 files changed, 444 insertions, 11 deletions
@@ -1,6 +1,7 @@ The following people, in alphabetical order, have either authored or signed off on commits in the Open vSwitch version control repository. +Andrew Lambeth wal@nicira.com Andy Southgate andy.southgate@citrix.com Ben Pfaff blp@nicira.com Bryan Phillippe bp@toroki.com diff --git a/lib/automake.mk b/lib/automake.mk index 91d8fff24..6d84a9212 100644 --- a/lib/automake.mk +++ b/lib/automake.mk @@ -133,6 +133,8 @@ lib_libopenvswitch_a_SOURCES = \ lib/stream-unix.c \ lib/stream.c \ lib/stream.h \ + lib/stress.c \ + lib/stress.h \ lib/string.h \ lib/svec.c \ lib/svec.h \ @@ -220,6 +222,7 @@ EXTRA_DIST += \ lib/ssl-peer-ca-cert.man \ lib/ssl.man \ lib/ssl-syn.man \ + lib/stress-unixctl.man \ lib/unixctl.man \ lib/unixctl-syn.man \ lib/vconn-active.man \ @@ -261,6 +264,11 @@ lib/coverage.def: $(DIST_SOURCES) sed -n 's|^COVERAGE_DEFINE(\([_a-zA-Z0-9]\{1,\}\)).*$$|COVERAGE_COUNTER(\1)|p' $(all_sources) | LC_ALL=C sort -u > $@ CLEANFILES += lib/coverage.def +lib/stress.$(OBJEXT): lib/stress.def +lib/stress.def: $(DIST_SOURCES) + sed -n '/^STRESS_OPTION(/,/);$$/{s/);$$/)/;p}' $(all_sources) > $@ +CLEANFILES += lib/stress.def + lib/vlog.$(OBJEXT): lib/vlog-modules.def lib/vlog-modules.def: $(DIST_SOURCES) sed -n 's|^VLOG_DEFINE_\(THIS_\)\{0,1\}MODULE(\([_a-zA-Z0-9]\{1,\}\)).*$$|VLOG_MODULE(\2)|p' $(all_sources) | LC_ALL=C sort -u > $@ diff --git a/lib/netlink.c b/lib/netlink.c index 8806f91d9..ba32ca3a0 100644 --- a/lib/netlink.c +++ b/lib/netlink.c @@ -29,6 +29,7 @@ #include "netlink-protocol.h" #include "ofpbuf.h" #include "poll-loop.h" +#include "stress.h" #include "timeval.h" #include "util.h" #include "vlog.h" @@ -259,6 +260,15 @@ nl_sock_sendv(struct nl_sock *sock, const struct iovec iov[], size_t n_iov, return error; } +/* This stress option is useful for testing that OVS properly tolerates + * -ENOBUFS on NetLink sockets. Such errors are unavoidable because they can + * occur if the kernel cannot temporarily allocate enough GFP_ATOMIC memory to + * reply to a request. They can also occur if messages arrive on a multicast + * channel faster than OVS can process them. */ +STRESS_OPTION( + netlink_overflow, "simulate netlink socket receive buffer overflow", + 5, 1, -1, 100); + /* Tries to receive a netlink message from the kernel on 'sock'. If * successful, stores the received message into '*bufp' and returns 0. The * caller is responsible for destroying the message with ofpbuf_delete(). On @@ -340,9 +350,16 @@ try_again: ofpbuf_delete(buf); return EPROTO; } + + if (STRESS(netlink_overflow)) { + ofpbuf_delete(buf); + return ENOBUFS; + } + *bufp = buf; log_nlmsg(__func__, 0, buf->data, buf->size); COVERAGE_INC(netlink_received); + return 0; } diff --git a/lib/stream-fd.c b/lib/stream-fd.c index ad15dca7e..2026db6af 100644 --- a/lib/stream-fd.c +++ b/lib/stream-fd.c @@ -28,6 +28,7 @@ #include "leak-checker.h" #include "poll-loop.h" #include "socket-util.h" +#include "stress.h" #include "util.h" #include "stream-provider.h" #include "stream.h" @@ -96,19 +97,39 @@ fd_connect(struct stream *stream) return check_connection_completion(s->fd); } +STRESS_OPTION( + stream_flaky_recv, "simulate failure of fd stream recvs", + 100, 0, -1, 0); + static ssize_t fd_recv(struct stream *stream, void *buffer, size_t n) { struct stream_fd *s = stream_fd_cast(stream); - ssize_t retval = read(s->fd, buffer, n); + ssize_t retval; + + if (STRESS(stream_flaky_recv)) { + return -EIO; + } + + retval = read(s->fd, buffer, n); return retval >= 0 ? retval : -errno; } +STRESS_OPTION( + stream_flaky_send, "simulate failure of fd stream sends", + 100, 0, -1, 0); + static ssize_t fd_send(struct stream *stream, const void *buffer, size_t n) { struct stream_fd *s = stream_fd_cast(stream); - ssize_t retval = write(s->fd, buffer, n); + ssize_t retval; + + if (STRESS(stream_flaky_send)) { + return -EIO; + } + + retval = write(s->fd, buffer, n); return (retval > 0 ? retval : retval == 0 ? -EAGAIN : -errno); diff --git a/lib/stress-unixctl.man b/lib/stress-unixctl.man new file mode 100644 index 000000000..ad265eefe --- /dev/null +++ b/lib/stress-unixctl.man @@ -0,0 +1,72 @@ +.SS "STRESS OPTION COMMANDS" +These command manage stress options, which allow developers testing +Open vSwitch to trigger behavior that otherwise would occur only in +corner cases. Developers and testers can thereby more easily discover +bugs that would otherwise manifest only rarely or +nondeterministically. Stress options may cause surprising behavior +even when they do not actually reveal bugs, so they should only be +enabled as part of testing Open vSwitch. +. +.IP "\fBstress/enable\fR" +.IQ "\fBstress/disable\fR" +All stress options are disabled by default. Use \fBstress/enable\fR +to enable stress options and \fBstress/disable\fR to disable them. +. +.IP "\fBstress/list\fR" +Lists and describes the available stress options and their settings in +tabular form. The columns in the table are: +.RS +.IP "NAME" +A single-word identifier for the option, used to identify stress +options to \fBstress/set\fR. +. +.IP "DESCRIPTION" +A description for a person unfamiliar with the detailed internals of +the code what behavior the option affects. +. +.IP "PERIOD" +Currently configured trigger period. If the stress option is +disabled, this is \fBdisabled\fR. Otherwise this is a number giving +the number of occurrences of the event between activations of the +stress option triggers. +. +.IP "MODE" +If the stress option is disabled, this is \fBn/a\fR. Otherwise it is +\fBperiodic\fR if the stress option triggers after exactly the period, +or \fBrandom\fR if it triggers randomly but on average after the +number of occurrences specified by the period. +. +.IP "COUNTER" +If the stress option is disabled, this is \fBn/a\fR. Otherwise it is +the number of occurrences of the event before the next time the stress +option triggers. +. +.IP "HITS" +The number of times that this stress option has triggered since this +program started. +. +.IP "RECOMMENDED" +A suggested period for a person unfamiliar with the internals. It +should put reasonable stress on the system without crippling it. +. +.IP "MINIMUM" +.IQ "MAXIMUM" +Minimum and maximum values allowed for the period. +. +.IP "DEFAULT" +The default period, used when stress options have been enabled (with +\fBstress/enable\fR) but this particular stress option has not been +specifically configured (with \fBstress/set\fR). It is \fBdisabled\fR +if the option is disabled by default. It is nonzero for options that +can be left on at low levels without noticable impact to the end user. +.RE +. +.IP "\fBstress/set \fIoption\fR \fIperiod\fR [\fBrandom\fR|\fBperiodic\fR]" +Sets the period at which stress \fIoption\fR triggers to +\fIperiod\fR. A \fIperiod\fR of 0 disables \fIoption\fR. Specify +\fBrandom\fR to make the option trigger randomly with an average +period of \fIperiod\fR, or \fBperiodic\fR to trigger exactly every +\fIperiod\fR events; the latter is the default. +.IP +If stress options have not been enabled with \fBstress/enable\fR, this +command has no effect. diff --git a/lib/stress.c b/lib/stress.c new file mode 100644 index 000000000..53f6fc02c --- /dev/null +++ b/lib/stress.c @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2010 Nicira Networks. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <config.h> +#include "stress.h" +#include <stdlib.h> +#include <string.h> +#include "unixctl.h" +#include "dynamic-string.h" +#include "random.h" +#include "util.h" +#include "vlog.h" + +VLOG_DEFINE_THIS_MODULE(stress); + +/* The stress options. */ +#if USE_LINKER_SECTIONS +extern struct stress_option *__start_stress_options[]; +extern struct stress_option *__stop_stress_options[]; +#define stress_options __start_stress_options +#define n_stress_options (__stop_stress_options - __start_stress_options) +#else /* !USE_LINKER_SECTIONS */ +#undef STRESS_OPTION +#define STRESS_OPTION(NAME, DESCRIPTION, RECOMMENDED, MIN, MAX, DEFAULT) \ + STRESS_OPTION__(NAME, DESCRIPTION, RECOMMENDED, MIN, MAX, DEFAULT); +#include "stress.def" +#undef STRESS_OPTION + +struct stress_option *stress_options[] = { +#define STRESS_OPTION(NAME, DESCRIPTION, RECOMMENDED, MIN, MAX, DEFAULT) \ + &stress_##NAME, +#include "stress.def" +#undef STRESS_OPTION +}; +#define n_stress_options ARRAY_SIZE(stress_options) +#endif /* !USE_LINKER_SECTIONS */ + +/* Enable stress options? */ +static bool stress_enabled; + +static void +stress_reset(struct stress_option *option) +{ + if (!option->period || !stress_enabled) { + option->counter = UINT_MAX; + } else if (!option->random) { + option->counter = option->period; + } else if (option->period < UINT32_MAX / 2) { + /* Random distribution with mean of option->period. */ + option->counter = random_uint32() % ((2 * option->period) - 1) + 1; + } else { + option->counter = random_uint32(); + } +} + +static void +stress_enable(bool enable) +{ + if (stress_enabled != enable) { + int i; + + stress_enabled = enable; + for (i = 0; i < n_stress_options; i++) { + stress_reset(stress_options[i]); + } + } +} + +bool +stress_sample_slowpath__(struct stress_option *option) +{ + stress_reset(option); + if (option->period && stress_enabled) { + static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 1); + + option->hits++; + VLOG_DBG_RL(&rl, "%s hit (%llu total)", option->name, option->hits); + + return true; + } else { + return false; + } +} + +static void +stress_set(struct stress_option *option, unsigned int period, bool random) +{ + if (period > option->max) { + period = option->max; + } + if (period < option->min) { + period = option->min; + } + if (period != option->period || random != option->random) { + option->random = random; + option->period = period; + stress_reset(option); + } +} + +static void +stress_unixctl_list(struct unixctl_conn *conn, const char *args, + void *aux OVS_UNUSED) +{ + int i, found = 0; + struct ds results; + + ds_init(&results); + ds_put_cstr(&results, "NAME (DESCRIPTION)\n"); + ds_put_format(&results, "%11s %10s %10s %10s\n", + "PERIOD", "MODE", "COUNTER", "HITS"); + ds_put_format(&results, "%11s %10s %10s %10s\n", + "RECOMMENDED", "MINIMUM", "MAXIMUM", "DEFAULT"); + for (i = 0; i < n_stress_options; i++) { + struct stress_option *option = stress_options[i]; + if (!*args || strstr(option->name, args)) { + ds_put_format(&results, "\n%s (%s)\n", + option->name, option->description); + if (option->period) { + ds_put_format(&results, "%11u %10s ", option->period, + option->random ? "random" : "periodic"); + if (stress_enabled) { + ds_put_format(&results, "%10u", option->counter); + } else { + ds_put_cstr(&results, " n/a"); + } + } else { + ds_put_format(&results, "%11s %10s %10s", + "disabled", "n/a", "n/a"); + } + ds_put_format(&results, " %10llu\n", option->hits); + ds_put_format(&results, "%11u %10u %10u ", + option->recommended, option->min, option->max); + if (!option->def) { + ds_put_format(&results, "%10s", "disabled"); + } else { + ds_put_format(&results, "%10u", option->def); + } + ds_put_char(&results, '\n'); + found++; + } + } + if (found) { + unixctl_command_reply(conn, 200, ds_cstr(&results)); + } else { + unixctl_command_reply(conn, 404, ""); + } + ds_destroy(&results); +} + +static void +stress_unixctl_enable(struct unixctl_conn *conn, const char *args OVS_UNUSED, + void *aux OVS_UNUSED) +{ + stress_enable(true); + unixctl_command_reply(conn, 200, ""); +} + +static void +stress_unixctl_disable(struct unixctl_conn *conn, const char *args OVS_UNUSED, + void *aux OVS_UNUSED) +{ + stress_enable(false); + unixctl_command_reply(conn, 200, ""); +} + +static void +stress_unixctl_set(struct unixctl_conn *conn, const char *args_, + void *aux OVS_UNUSED) +{ + int code = 404; + char *args = xstrdup(args_); + char *save_ptr = NULL; + char *option_name; + char *option_val; + + option_name = strtok_r(args, " ", &save_ptr); + option_val = strtok_r(NULL, " ", &save_ptr); + if (option_val) { + int i; + for (i = 0; i < n_stress_options; i++) { + struct stress_option *option = stress_options[i]; + if (!strcmp(option_name, option->name)) { + unsigned int period = strtoul(option_val, NULL, 0); + bool random = strstr(args_, "random"); + + stress_set(option, period, random); + code = 200; + break; + } + } + } + unixctl_command_reply(conn, code, ""); + free(args); +} + +/* Exposes ovs-appctl access to the stress options. + * + * This function is not required to simply reference stress options and have + * them fire at their default periods. + */ +void +stress_init_command(void) +{ + unixctl_command_register("stress/list", stress_unixctl_list, NULL); + unixctl_command_register("stress/set", stress_unixctl_set, NULL); + unixctl_command_register("stress/enable", stress_unixctl_enable, NULL); + unixctl_command_register("stress/disable", stress_unixctl_disable, NULL); +} diff --git a/lib/stress.h b/lib/stress.h new file mode 100644 index 000000000..4244fca9b --- /dev/null +++ b/lib/stress.h @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2010 Nicira Networks. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef STRESS_H +#define STRESS_H 1 + +#include <stdbool.h> + +struct stress_option { + /* Properties. */ + char *name; /* Short identifier string */ + char *description; /* Description of what the option stresses. */ + unsigned int recommended; /* Recommended period. */ + unsigned int min; /* Minimum period that can be set. */ + unsigned int max; /* Maximum period that can be set. */ + unsigned int def; /* Default value. */ + + /* Configuration. */ + unsigned int period; /* Desired period for firing, 0 to disable. */ + bool random; /* Fire randomly or exactly at period? */ + + /* State. */ + unsigned int counter; /* Number of hits before next firing. */ + unsigned long long int hits; /* Hits since last reset. */ +}; + +/* Creates and initializes a global instance of a stress option. + * + * NAME is a single word descriptive identifier for the option. This is the + * token to pass in to the STRESS() macro at the sites where exectution is to + * be controlled by the option. + * + * DESCRIPTION is a quoted string that should describe to a person unfamiliar + * with the detailed internals of the code what behavior the option affects. + * + * RECOMMENDED is a suggested value for a person unfamiliar with the internals. + * It should put reasonable stress on the system without crippling it. + * + * MIN and MAX are the minimum and maximum values allowed for the option. + * + * DEFAULT is the default value for the option. Specify 0 to disable the + * option by default, which should be the usual choice. But some options can + * be left on at low levels without noticable impact to the end user. An + * example would be failing to allocate a buffer for every 100000th packet + * processed by the system. + */ +#if USE_LINKER_SECTIONS +#define STRESS_OPTION(NAME, DESCRIPTION, RECOMMENDED, MIN, MAX, DEFAULT) \ + STRESS_OPTION__(NAME, DESCRIPTION, RECOMMENDED, MIN, MAX, DEFAULT); \ + struct stress_option *stress_option_ptr_##NAME \ + __attribute__((section("stress_options"))) = &stress_##NAME +#else +#define STRESS_OPTION(NAME, DESCRIPTION, RECOMMENDED, MIN, MAX, DEFAULT) \ + extern struct stress_option stress_##NAME +#endif + +/* Yields true if stress option NAME should be triggered, + * false otherwise. */ +#define STRESS(NAME) stress_sample__(&stress_##NAME) + +void stress_init_command(void); + +/* Implementation details. */ + +#define STRESS_OPTION__(NAME, DESCRIPTION, RECOMMENDED, MIN, MAX, DEFAULT) \ + struct stress_option stress_##NAME = \ + { #NAME, DESCRIPTION, RECOMMENDED, MIN, MAX, DEFAULT, \ + DEFAULT ? DEFAULT : 0, /* period */ \ + false, /* random */ \ + UINT_MAX, /* counter */ \ + 0 } /* hits */ + +bool stress_sample_slowpath__(struct stress_option *); +static inline bool stress_sample__(struct stress_option *option) +{ + return --option->counter == 0 && stress_sample_slowpath__(option); +} + +#endif /* STRESS_H */ diff --git a/ofproto/ofproto.c b/ofproto/ofproto.c index c3214c596..bf557cfc3 100644 --- a/ofproto/ofproto.c +++ b/ofproto/ofproto.c @@ -63,22 +63,20 @@ VLOG_DEFINE_THIS_MODULE(ofproto); +COVERAGE_DEFINE(facet_changed_rule); +COVERAGE_DEFINE(facet_revalidate); COVERAGE_DEFINE(odp_overflow); -COVERAGE_DEFINE(ofproto_add_wc_flow); COVERAGE_DEFINE(ofproto_agg_request); COVERAGE_DEFINE(ofproto_costly_flags); COVERAGE_DEFINE(ofproto_ctlr_action); -COVERAGE_DEFINE(ofproto_del_wc_flow); -COVERAGE_DEFINE(ofproto_dp_missed); +COVERAGE_DEFINE(ofproto_del_rule); COVERAGE_DEFINE(ofproto_error); COVERAGE_DEFINE(ofproto_expiration); COVERAGE_DEFINE(ofproto_expired); COVERAGE_DEFINE(ofproto_flows_req); COVERAGE_DEFINE(ofproto_flush); COVERAGE_DEFINE(ofproto_invalidated); -COVERAGE_DEFINE(ofproto_mod_wc_flow); COVERAGE_DEFINE(ofproto_no_packet_in); -COVERAGE_DEFINE(ofproto_odp_unchanged); COVERAGE_DEFINE(ofproto_ofconn_stuck); COVERAGE_DEFINE(ofproto_ofp2odp); COVERAGE_DEFINE(ofproto_packet_in); @@ -86,10 +84,6 @@ COVERAGE_DEFINE(ofproto_packet_out); COVERAGE_DEFINE(ofproto_queue_req); COVERAGE_DEFINE(ofproto_recv_openflow); COVERAGE_DEFINE(ofproto_reinit_ports); -COVERAGE_DEFINE(ofproto_revalidate); -COVERAGE_DEFINE(ofproto_revalidate_moved); -COVERAGE_DEFINE(ofproto_revalidate_rule); -COVERAGE_DEFINE(ofproto_subrule_create); COVERAGE_DEFINE(ofproto_unexpected_rule); COVERAGE_DEFINE(ofproto_uninstallable); COVERAGE_DEFINE(ofproto_update_port); diff --git a/ovsdb/ovsdb-server.1.in b/ovsdb/ovsdb-server.1.in index 79a39c663..a89f09195 100644 --- a/ovsdb/ovsdb-server.1.in +++ b/ovsdb/ovsdb-server.1.in @@ -119,6 +119,7 @@ This command might be useful for debugging issues with database clients. . .so lib/vlog-unixctl.man +.so lib/stress-unixctl.man .SH "SEE ALSO" . .BR ovsdb\-tool (1). diff --git a/ovsdb/ovsdb-server.c b/ovsdb/ovsdb-server.c index c7a93815b..9f63f3a8b 100644 --- a/ovsdb/ovsdb-server.c +++ b/ovsdb/ovsdb-server.c @@ -39,6 +39,7 @@ #include "row.h" #include "stream-ssl.h" #include "stream.h" +#include "stress.h" #include "svec.h" #include "table.h" #include "timeval.h" @@ -87,6 +88,7 @@ main(int argc, char *argv[]) proctitle_init(argc, argv); set_program_name(argv[0]); + stress_init_command(); signal(SIGPIPE, SIG_IGN); process_init(); diff --git a/vswitchd/ovs-vswitchd.8.in b/vswitchd/ovs-vswitchd.8.in index b5c744732..416be246e 100644 --- a/vswitchd/ovs-vswitchd.8.in +++ b/vswitchd/ovs-vswitchd.8.in @@ -183,6 +183,7 @@ status of \fIslave\fR changes. Returns the hash value which would be used for \fImac\fR. . .so lib/vlog-unixctl.man +.so lib/stress-unixctl.man .SH "SEE ALSO" .BR ovs\-appctl (8), .BR ovs\-brcompatd (8), diff --git a/vswitchd/ovs-vswitchd.c b/vswitchd/ovs-vswitchd.c index 01f2ac29c..93f0371c6 100644 --- a/vswitchd/ovs-vswitchd.c +++ b/vswitchd/ovs-vswitchd.c @@ -41,6 +41,7 @@ #include "signals.h" #include "stream-ssl.h" #include "stream.h" +#include "stress.h" #include "svec.h" #include "timeval.h" #include "unixctl.h" @@ -67,6 +68,7 @@ main(int argc, char *argv[]) proctitle_init(argc, argv); set_program_name(argv[0]); + stress_init_command(); remote = parse_options(argc, argv); signal(SIGPIPE, SIG_IGN); sighup = signal_register(SIGHUP); |