Add support for multiple OpenFlow controllers on a single bridge.

With this commit, Open vSwitch permits a bridge to have any number of
OpenFlow controllers.  When multiple controllers are configured, Open
vSwitch connects to all of them simultaneously.  Details of configuration
are in the vswitch schema documentation.

OpenFlow 1.0 does not specify how multiple controllers coordinate in
interacting with a single switch, so more than one controller should be
specified only if the controllers are themselves designed to coordinate
with each other.

An upcoming commit will provide a simple means for coordination between
multiple controllers.

Feature #2495.

4 files changed, 244 insertions, 127 deletions

diff --git a/utilities/ovs-openflowd.8.in b/utilities/ovs-openflowd.8.in
index 4d2f211a0..7a78bf292 100644
--- a/utilities/ovs-openflowd.8.in
+++ b/utilities/ovs-openflowd.8.in
@@ -6,12 +6,12 @@ ovs\-openflowd \- OpenFlow switch implementation
 .
 .SH SYNOPSIS
 .B ovs\-openflowd
-[\fIoptions\fR] \fIdatapath\fR [\fIcontroller\fR]
+[\fIoptions\fR] \fIdatapath\fR [\fIcontroller\fR\&...]
 .
 .SH DESCRIPTION
 The \fBovs\-openflowd\fR program implements an OpenFlow switch using a
-flow-based datapath.  \fBovs\-openflowd\fR connects to an OpenFlow controller
-over TCP or SSL.
+flow-based datapath.  \fBovs\-openflowd\fR connects to one or more
+OpenFlow controllers over TCP or SSL.
 .PP
 The mandatory \fIdatapath\fR argument argument specifies the local datapath
 to relay.  It takes one of the following forms:
@@ -19,17 +19,17 @@ to relay.  It takes one of the following forms:
 .so lib/dpif.man
 .
 .PP
-The optional \fIcontroller\fR argument specifies how to connect to
+The optional \fIcontroller\fR arguments specify how to connect to
 the OpenFlow controller.  It takes one of the following forms:
 .
 .so lib/vconn-active.man
 .
 .PP
-If \fIcontroller\fR is omitted, \fBovs\-openflowd\fR attempts to discover the
-location of the controller automatically (see below).
+If no \fIcontroller\fR is specified, \fBovs\-openflowd\fR attempts to
+discover the location of a controller automatically (see below).
 .
-.SS "Contacting the Controller"
-The OpenFlow switch must be able to contact the OpenFlow controller
+.SS "Contacting Controllers"
+The OpenFlow switch must be able to contact the OpenFlow controllers
 over the network.  It can do so in one of two ways:
 .
 .IP out-of-band
@@ -60,8 +60,8 @@ manually or discovered automatically:
 .RS
 .IP "controller discovery"
 To make \fBovs\-openflowd\fR discover the location of the controller
-automatically, do not specify the location of the controller on the
-\fBovs\-openflowd\fR command line.
+automatically, do not specify a controller on the \fBovs\-openflowd\fR
+command line.
 .IP
 In this mode, \fBovs\-openflowd\fR will broadcast a DHCP request with vendor
 class identifier \fBOpenFlow\fR across the network devices added to
@@ -159,8 +159,8 @@ the local port network device, and start the DHCP client afterward.
 .TP
 \fB--datapath-id=\fIdpid\fR
 Sets \fIdpid\fR, which must consist of exactly 16 hexadecimal digits,
-as the datapath ID that the switch will use to identify itself to the
-OpenFlow controller.
+as the datapath ID that the switch will use to identify itself to
+OpenFlow controllers.
 .IP
 If this option is omitted, the default datapath ID is taken from the
 Ethernet address of the datapath's local port (which is typically
diff --git a/utilities/ovs-openflowd.c b/utilities/ovs-openflowd.c
index 70430c035..1460423d6 100644
--- a/utilities/ovs-openflowd.c
+++ b/utilities/ovs-openflowd.c
@@ -51,7 +51,8 @@
 /* Settings that may be configured by the user. */
 struct ofsettings {
     /* Controller configuration. */
-    struct ofproto_controller controller;
+    struct ofproto_controller *controllers;
+    size_t n_controllers;
 
     /* Datapath. */
     uint64_t datapath_id;       /* Datapath ID. */
@@ -171,7 +172,7 @@ main(int argc, char *argv[])
     if (error) {
         ovs_fatal(error, "failed to configure STP");
     }
-    ofproto_set_controller(ofproto, &s.controller);
+    ofproto_set_controllers(ofproto, s.controllers, s.n_controllers);
 
     daemonize_complete();
 
@@ -266,16 +267,18 @@ parse_options(int argc, char *argv[], struct ofsettings *s)
         {0, 0, 0, 0},
     };
     char *short_options = long_options_to_short_options(long_options);
+    struct ofproto_controller controller_opts;
 
     /* Set defaults that we can figure out before parsing options. */
-    s->controller.max_backoff = 8;
-    s->controller.probe_interval = 5;
-    s->controller.fail = OFPROTO_FAIL_STANDALONE;
-    s->controller.band = OFPROTO_IN_BAND;
-    s->controller.accept_re = NULL;
-    s->controller.update_resolv_conf = true;
-    s->controller.rate_limit = 0;
-    s->controller.burst_limit = 0;
+    controller_opts.target = NULL;
+    controller_opts.max_backoff = 8;
+    controller_opts.probe_interval = 5;
+    controller_opts.fail = OFPROTO_FAIL_STANDALONE;
+    controller_opts.band = OFPROTO_IN_BAND;
+    controller_opts.accept_re = NULL;
+    controller_opts.update_resolv_conf = true;
+    controller_opts.rate_limit = 0;
+    controller_opts.burst_limit = 0;
     s->datapath_id = 0;
     s->mfr_desc = NULL;
     s->hw_desc = NULL;
@@ -325,26 +328,26 @@ parse_options(int argc, char *argv[], struct ofsettings *s)
             break;
 
         case OPT_ACCEPT_VCONN:
-            s->controller.accept_re = optarg;
+            controller_opts.accept_re = optarg;
             break;
 
         case OPT_NO_RESOLV_CONF:
-            s->controller.update_resolv_conf = false;
+            controller_opts.update_resolv_conf = false;
             break;
 
         case OPT_FAIL_MODE:
             if (!strcmp(optarg, "open")) {
-                s->controller.fail = OFPROTO_FAIL_STANDALONE;
+                controller_opts.fail = OFPROTO_FAIL_STANDALONE;
             } else if (!strcmp(optarg, "closed")) {
-                s->controller.fail = OFPROTO_FAIL_SECURE;
+                controller_opts.fail = OFPROTO_FAIL_SECURE;
             } else {
                 ovs_fatal(0, "--fail argument must be \"open\" or \"closed\"");
             }
             break;
 
         case OPT_INACTIVITY_PROBE:
-            s->controller.probe_interval = atoi(optarg);
-            if (s->controller.probe_interval < 5) {
+            controller_opts.probe_interval = atoi(optarg);
+            if (controller_opts.probe_interval < 5) {
                 ovs_fatal(0, "--inactivity-probe argument must be at least 5");
             }
             break;
@@ -362,28 +365,28 @@ parse_options(int argc, char *argv[], struct ofsettings *s)
             break;
 
         case OPT_MAX_BACKOFF:
-            s->controller.max_backoff = atoi(optarg);
-            if (s->controller.max_backoff < 1) {
+            controller_opts.max_backoff = atoi(optarg);
+            if (controller_opts.max_backoff < 1) {
                 ovs_fatal(0, "--max-backoff argument must be at least 1");
-            } else if (s->controller.max_backoff > 3600) {
-                s->controller.max_backoff = 3600;
+            } else if (controller_opts.max_backoff > 3600) {
+                controller_opts.max_backoff = 3600;
             }
             break;
 
         case OPT_RATE_LIMIT:
             if (optarg) {
-                s->controller.rate_limit = atoi(optarg);
-                if (s->controller.rate_limit < 1) {
+                controller_opts.rate_limit = atoi(optarg);
+                if (controller_opts.rate_limit < 1) {
                     ovs_fatal(0, "--rate-limit argument must be at least 1");
                 }
             } else {
-                s->controller.rate_limit = 1000;
+                controller_opts.rate_limit = 1000;
             }
             break;
 
         case OPT_BURST_LIMIT:
-            s->controller.burst_limit = atoi(optarg);
-            if (s->controller.burst_limit < 1) {
+            controller_opts.burst_limit = atoi(optarg);
+            if (controller_opts.burst_limit < 1) {
                 ovs_fatal(0, "--burst-limit argument must be at least 1");
             }
             break;
@@ -397,11 +400,11 @@ parse_options(int argc, char *argv[], struct ofsettings *s)
             break;
 
         case OPT_OUT_OF_BAND:
-            s->controller.band = OFPROTO_OUT_OF_BAND;
+            controller_opts.band = OFPROTO_OUT_OF_BAND;
             break;
 
         case OPT_IN_BAND:
-            s->controller.band = OFPROTO_IN_BAND;
+            controller_opts.band = OFPROTO_IN_BAND;
             break;
 
         case OPT_NETFLOW:
@@ -457,25 +460,46 @@ parse_options(int argc, char *argv[], struct ofsettings *s)
                   "use --help for usage");
     }
 
-    /* Local and remote vconns. */
-    dp_parse_name(argv[0], &s->dp_name, &s->dp_type);
-
-    s->controller.target = argc > 1 ? argv[1] : "discover";
-    if (!strcmp(s->controller.target, "discover")
-        && s->controller.band == OFPROTO_OUT_OF_BAND) {
-        ovs_fatal(0, "Cannot perform discovery with out-of-band control");
-    }
-
     /* Set accept_controller_regex. */
-    if (!s->controller.accept_re) {
-        s->controller.accept_re
+    if (!controller_opts.accept_re) {
+        controller_opts.accept_re
             = stream_ssl_is_configured() ? "^ssl:.*" : "^tcp:.*";
     }
 
     /* Rate limiting. */
-    if (s->controller.rate_limit && s->controller.rate_limit < 100) {
+    if (controller_opts.rate_limit && controller_opts.rate_limit < 100) {
         VLOG_WARN("Rate limit set to unusually low value %d",
-                  s->controller.rate_limit);
+                  controller_opts.rate_limit);
+    }
+
+    /* Local vconns. */
+    dp_parse_name(argv[0], &s->dp_name, &s->dp_type);
+
+    /* Controllers. */
+    s->n_controllers = argc > 1 ? argc - 1 : 1;
+    s->controllers = xmalloc(s->n_controllers * sizeof *s->controllers);
+    if (argc > 1) {
+        size_t i;
+
+        for (i = 0; i < s->n_controllers; i++) {
+            s->controllers[i] = controller_opts;
+            s->controllers[i].target = argv[i + 1];
+        }
+    } else {
+        s->controllers[0] = controller_opts;
+        s->controllers[0].target = "discover";
+    }
+
+    /* Sanity check. */
+    if (controller_opts.band == OFPROTO_OUT_OF_BAND) {
+        size_t i;
+
+        for (i = 0; i < s->n_controllers; i++) {
+            if (!strcmp(s->controllers[i].target, "discover")) {
+                ovs_fatal(0, "Cannot perform discovery with out-of-band "
+                          "control");
+            }
+        }
     }
 }
 
@@ -483,10 +507,10 @@ static void
 usage(void)
 {
     printf("%s: an OpenFlow switch implementation.\n"
-           "usage: %s [OPTIONS] DATAPATH [CONTROLLER]\n"
+           "usage: %s [OPTIONS] DATAPATH [CONTROLLER...]\n"
            "DATAPATH is a local datapath (e.g. \"dp0\").\n"
-           "CONTROLLER is an active OpenFlow connection method; if it is\n"
-           "omitted, then ovs-openflowd performs controller discovery.\n",
+           "Each CONTROLLER is an active OpenFlow connection method.  If\n"
+           "none is given, ovs-openflowd performs controller discovery.\n",
            program_name, program_name);
     vconn_usage(true, true, true);
     printf("\nOpenFlow options:\n"
diff --git a/utilities/ovs-vsctl.8.in b/utilities/ovs-vsctl.8.in
index 84193e875..cd83d0a29 100644
--- a/utilities/ovs-vsctl.8.in
+++ b/utilities/ovs-vsctl.8.in
@@ -299,13 +299,14 @@ output.
 .SS "OpenFlow Controller Connectivity"
 .
 \fBovs\-vswitchd\fR can perform all configured bridging and switching
-locally, or it can be configured to connect a given bridge to an
-external OpenFlow controller, such as NOX.  
+locally, or it can be configured to connect a given bridge to one or
+more external OpenFlow controllers, such as NOX.
 .
-If a \fIbridge\fR argument is given, the settings apply only to the
-specified bridge.  Otherwise, they apply to the Open vSwitch instance,
-and its configuration applies to any bridge that has not been explicitly
-configured through a \fIbridge\fR argument.
+For each of these commands, a \fIbridge\fR of \fBdefault\fR applies
+the configuration as the default for any bridge that has not been
+explicitly configured.  Otherwise, \fIbridge\fR must name a bridge,
+and the settings apply only to that bridge.  (Omitting \fIbridge\fR
+entirely usually has the same effect as specifying \fBdefault\fR.)
 .
 .IP "\fBget\-controller\fR [\fIbridge\fR]"
 Prints the configured controller target.
@@ -313,9 +314,10 @@ Prints the configured controller target.
 .IP "\fBdel\-controller\fR [\fIbridge\fR]"
 Deletes the configured controller target.
 .
-.IP "\fBset\-controller\fR [\fIbridge\fR] \fItarget\fR"
-Sets the configured controller target.  The \fItarget\fR may use any of
-the following forms:
+.IP "\fBset\-controller\fR [\fIbridge\fR] \fItarget\fR\&..."
+Sets the configured controller target or targets.  If more than one
+\fItarget\fR is specified, then \fIbridge\fR may not be omitted.  Each
+\fItarget\fR may use any of the following forms:
 .
 .RS
 .so lib/vconn-active.man
diff --git a/utilities/ovs-vsctl.c b/utilities/ovs-vsctl.c
index fe7838767..091e6bc7b 100644
--- a/utilities/ovs-vsctl.c
+++ b/utilities/ovs-vsctl.c
@@ -489,7 +489,8 @@ struct vsctl_context {
 struct vsctl_bridge {
     struct ovsrec_bridge *br_cfg;
     char *name;
-    struct ovsrec_controller *ctrl;
+    struct ovsrec_controller **ctrl;
+    size_t n_ctrl;
     struct vsctl_bridge *parent;
     int vlan;
 };
@@ -508,7 +509,8 @@ struct vsctl_info {
     struct shash bridges;
     struct shash ports;
     struct shash ifaces;
-    struct ovsrec_controller *ctrl;
+    struct ovsrec_controller **ctrl;
+    size_t n_ctrl;
 };
 
 static char *
@@ -545,7 +547,13 @@ add_bridge(struct vsctl_info *b,
     br->name = xstrdup(name);
     br->parent = parent;
     br->vlan = vlan;
-    br->ctrl = parent ? parent->br_cfg->controller : br_cfg->controller;
+    if (parent) {
+        br->ctrl = parent->br_cfg->controller;
+        br->n_ctrl = parent->br_cfg->n_controller;
+    } else {
+        br->ctrl = br_cfg->controller;
+        br->n_ctrl = br_cfg->n_controller;
+    }
     shash_add(&b->bridges, br->name, br);
     return br;
 }
@@ -610,6 +618,7 @@ get_info(const struct ovsrec_open_vswitch *ovs, struct vsctl_info *info)
     shash_init(&info->ifaces);
 
     info->ctrl = ovs->controller;
+    info->n_ctrl = ovs->n_controller;
 
     shash_init(&bridges);
     shash_init(&ports);
@@ -1440,6 +1449,29 @@ cmd_iface_to_br(struct vsctl_context *ctx)
     free_info(&info);
 }
 
+/* Print targets of the 'n_controllers' in 'controllers' on the output for
+ * 'ctx'. */
+static void
+print_controllers(struct vsctl_context *ctx,
+                  struct ovsrec_controller **controllers,
+                  size_t n_controllers)
+{
+    /* Print the targets in sorted order for reproducibility. */
+    struct svec targets;
+    size_t i;
+
+    svec_init(&targets);
+    for (i = 0; i < n_controllers; i++) {
+        svec_add(&targets, controllers[i]->target);
+    }
+
+    svec_sort(&targets);
+    for (i = 0; i < targets.n; i++) {
+        ds_put_format(&ctx->output, "%s\n", targets.names[i]);
+    }
+    svec_destroy(&targets);
+}
+
 static void
 cmd_get_controller(struct vsctl_context *ctx)
 {
@@ -1447,21 +1479,14 @@ cmd_get_controller(struct vsctl_context *ctx)
 
     get_info(ctx->ovs, &info);
 
-    if (ctx->argc == 1) {
-        /* Return the controller from the "Open_vSwitch" table */
-        if (info.ctrl) {
-            ds_put_format(&ctx->output, "%s\n", info.ctrl->target);
-        }
+    if (ctx->argc == 1 || !strcmp(ctx->argv[1], "default")) {
+        print_controllers(ctx, info.ctrl, info.n_ctrl);
     } else {
-        /* Return the controller for a particular bridge. */
         struct vsctl_bridge *br = find_bridge(&info, ctx->argv[1], true);
-
-        /* If no controller is explicitly defined for the requested
-         * bridge, fallback to the "Open_vSwitch" table's controller. */
-        if (br->ctrl) {
-            ds_put_format(&ctx->output, "%s\n", br->ctrl->target);
-        } else if (info.ctrl) {
-            ds_put_format(&ctx->output, "%s\n", info.ctrl->target);
+        if (br->n_ctrl) {
+            print_controllers(ctx, br->ctrl, br->n_ctrl);
+        } else {
+            print_controllers(ctx, info.ctrl, info.n_ctrl);
         }
     }
 
@@ -1469,60 +1494,117 @@ cmd_get_controller(struct vsctl_context *ctx)
 }
 
 static void
+delete_controllers(struct ovsrec_controller **controllers,
+                   size_t n_controllers)
+{
+    size_t i;
+
+    for (i = 0; i < n_controllers; i++) {
+        ovsrec_controller_delete(controllers[i]);
+    }
+}
+
+static void
 cmd_del_controller(struct vsctl_context *ctx)
 {
     struct vsctl_info info;
 
     get_info(ctx->ovs, &info);
 
-    if (ctx->argc == 1) {
-        if (info.ctrl) {
-            ovsrec_controller_delete(info.ctrl);
-            ovsrec_open_vswitch_set_controller(ctx->ovs, NULL);
+    if (ctx->argc == 1 || !strcmp(ctx->argv[1], "default")) {
+        if (info.n_ctrl) {
+            delete_controllers(info.ctrl, info.n_ctrl);
+            ovsrec_open_vswitch_set_controller(ctx->ovs, NULL, 0);
         }
     } else {
         struct vsctl_bridge *br = find_real_bridge(&info, ctx->argv[1], true);
-
         if (br->ctrl) {
-            ovsrec_controller_delete(br->ctrl);
-            ovsrec_bridge_set_controller(br->br_cfg, NULL);
+            delete_controllers(br->ctrl, br->n_ctrl);
+            ovsrec_bridge_set_controller(br->br_cfg, NULL, 0);
         }
     }
 
     free_info(&info);
 }
 
+static struct ovsrec_controller **
+insert_controllers(struct ovsdb_idl_txn *txn, char *targets[], size_t n)
+{
+    struct ovsrec_controller **controllers;
+    size_t i;
+
+    controllers = xmalloc(n * sizeof *controllers);
+    for (i = 0; i < n; i++) {
+        controllers[i] = ovsrec_controller_insert(txn);
+        ovsrec_controller_set_target(controllers[i], targets[i]);
+    }
+
+    return controllers;
+}
+
+static void
+set_default_controllers(struct vsctl_context *ctx, char *targets[], size_t n)
+{
+    struct ovsrec_controller **controllers;
+
+    delete_controllers(ctx->ovs->controller, ctx->ovs->n_controller);
+
+    controllers = insert_controllers(ctx->txn, targets, n);
+    ovsrec_open_vswitch_set_controller(ctx->ovs, controllers, n);
+    free(controllers);
+}
+
 static void
 cmd_set_controller(struct vsctl_context *ctx)
 {
     struct vsctl_info info;
-    struct ovsrec_controller *ctrl;
 
     get_info(ctx->ovs, &info);
 
     if (ctx->argc == 2) {
-        /* Set the controller in the "Open_vSwitch" table. */
-        if (info.ctrl) {
-            ovsrec_controller_delete(info.ctrl);
-        }
-        ctrl = ovsrec_controller_insert(ctx->txn);
-        ovsrec_controller_set_target(ctrl, ctx->argv[1]);
-        ovsrec_open_vswitch_set_controller(ctx->ovs, ctrl);
+        /* Set one controller in the "Open_vSwitch" table. */
+        set_default_controllers(ctx, &ctx->argv[1], 1);
+    } else if (!strcmp(ctx->argv[1], "default")) {
+        /* Set one or more controllers in the "Open_vSwitch" table. */
+        set_default_controllers(ctx, &ctx->argv[2], ctx->argc - 2);
     } else {
-        /* Set the controller for a particular bridge. */
+        /* Set one or more controllers for a particular bridge. */
         struct vsctl_bridge *br = find_real_bridge(&info, ctx->argv[1], true);
+        struct ovsrec_controller **controllers;
+        size_t n;
 
-        if (br->ctrl) {
-            ovsrec_controller_delete(br->ctrl);
-        }
-        ctrl = ovsrec_controller_insert(ctx->txn);
-        ovsrec_controller_set_target(ctrl, ctx->argv[2]);
-        ovsrec_bridge_set_controller(br->br_cfg, ctrl);
+        delete_controllers(br->ctrl, br->n_ctrl);
+
+        n = ctx->argc - 2;
+        controllers = insert_controllers(ctx->txn, &ctx->argv[2], n);
+        ovsrec_bridge_set_controller(br->br_cfg, controllers, n);
+        free(controllers);
     }
 
     free_info(&info);
 }
 
+static const char *
+get_fail_mode(struct ovsrec_controller **controllers, size_t n_controllers)
+{
+    const char *fail_mode;
+    size_t i;
+
+    fail_mode = NULL;
+    for (i = 0; i < n_controllers; i++) {
+        const char *s = controllers[i]->fail_mode;
+        if (s) {
+            if (!strcmp(s, "secure")) {
+                return s;
+            } else {
+                fail_mode = s;
+            }
+        }
+    }
+
+    return fail_mode;
+}
+
 static void
 cmd_get_fail_mode(struct vsctl_context *ctx)
 {
@@ -1531,23 +1613,18 @@ cmd_get_fail_mode(struct vsctl_context *ctx)
 
     get_info(ctx->ovs, &info);
 
-    if (ctx->argc == 1) {
+    if (ctx->argc == 1 || !strcmp(ctx->argv[1], "default")) {
         /* Return the fail-mode from the "Open_vSwitch" table */
-        if (info.ctrl && info.ctrl->fail_mode) {
-            fail_mode = info.ctrl->fail_mode;
-        }
+        fail_mode = get_fail_mode(info.ctrl, info.n_ctrl);
     } else {
         /* Return the fail-mode for a particular bridge. */
         struct vsctl_bridge *br = find_bridge(&info, ctx->argv[1], true);
 
-        /* If no controller or fail-mode is explicitly defined for the 
-         * requested bridge, fallback to the "Open_vSwitch" table's 
-         * setting. */
-        if (br->ctrl && br->ctrl->fail_mode) {
-            fail_mode = br->ctrl->fail_mode;
-        } else if (info.ctrl && info.ctrl->fail_mode) {
-            fail_mode = info.ctrl->fail_mode;
-        }
+        /* If no controller is defined for the requested bridge, fallback to
+         * the "Open_vSwitch" table's controller. */
+        fail_mode = (br->n_ctrl
+                     ? get_fail_mode(br->ctrl, br->n_ctrl)
+                     : get_fail_mode(info.ctrl, info.n_ctrl));
     }
 
     if (fail_mode && strlen(fail_mode)) {
@@ -1558,22 +1635,29 @@ cmd_get_fail_mode(struct vsctl_context *ctx)
 }
 
 static void
+set_fail_mode(struct ovsrec_controller **controllers, size_t n_controllers,
+              const char *fail_mode)
+{
+    size_t i;
+
+    for (i = 0; i < n_controllers; i++) {
+        ovsrec_controller_set_fail_mode(controllers[i], fail_mode);
+    }
+}
+
+static void
 cmd_del_fail_mode(struct vsctl_context *ctx)
 {
     struct vsctl_info info;
 
     get_info(ctx->ovs, &info);
 
-    if (ctx->argc == 1) {
-        if (info.ctrl && info.ctrl->fail_mode) {
-            ovsrec_controller_set_fail_mode(info.ctrl, NULL);
-        }
+    if (ctx->argc == 1 || !strcmp(ctx->argv[1], "default")) {
+        set_fail_mode(info.ctrl, info.n_ctrl, NULL);
     } else {
         struct vsctl_bridge *br = find_real_bridge(&info, ctx->argv[1], true);
 
-        if (br->ctrl && br->ctrl->fail_mode) {
-            ovsrec_controller_set_fail_mode(br->ctrl, NULL);
-        }
+        set_fail_mode(br->ctrl, br->n_ctrl, NULL);
     }
 
     free_info(&info);
@@ -1583,29 +1667,36 @@ static void
 cmd_set_fail_mode(struct vsctl_context *ctx)
 {
     struct vsctl_info info;
+    const char *bridge;
     const char *fail_mode;
 
     get_info(ctx->ovs, &info);
 
-    fail_mode = (ctx->argc == 2) ? ctx->argv[1] : ctx->argv[2];
+    if (ctx->argc == 2) {
+        bridge = "default";
+        fail_mode = ctx->argv[1];
+    } else {
+        bridge = ctx->argv[1];
+        fail_mode = ctx->argv[2];
+    }
 
     if (strcmp(fail_mode, "standalone") && strcmp(fail_mode, "secure")) {
         vsctl_fatal("fail-mode must be \"standalone\" or \"secure\"");
     }
 
-    if (ctx->argc == 2) {
+    if (!strcmp(bridge, "default")) {
         /* Set the fail-mode in the "Open_vSwitch" table. */
         if (!info.ctrl) {
             vsctl_fatal("no controller declared");
         }
-        ovsrec_controller_set_fail_mode(info.ctrl, fail_mode);
+        set_fail_mode(info.ctrl, info.n_ctrl, fail_mode);
     } else {
-        struct vsctl_bridge *br = find_real_bridge(&info, ctx->argv[1], true);
+        struct vsctl_bridge *br = find_real_bridge(&info, bridge, true);
 
         if (!br->ctrl) {
             vsctl_fatal("no controller declared for %s", br->name);
         }
-        ovsrec_controller_set_fail_mode(br->ctrl, fail_mode);
+        set_fail_mode(br->ctrl, br->n_ctrl, fail_mode);
     }
 
     free_info(&info);
@@ -2544,7 +2635,7 @@ static const struct vsctl_command_syntax all_commands[] = {
     /* Controller commands. */
     {"get-controller", 0, 1, cmd_get_controller, NULL, ""},
     {"del-controller", 0, 1, cmd_del_controller, NULL, ""},
-    {"set-controller", 1, 2, cmd_set_controller, NULL, ""},
+    {"set-controller", 1, INT_MAX, cmd_set_controller, NULL, ""},
     {"get-fail-mode", 0, 1, cmd_get_fail_mode, NULL, ""},
     {"del-fail-mode", 0, 1, cmd_del_fail_mode, NULL, ""},
     {"set-fail-mode", 1, 2, cmd_set_fail_mode, NULL, ""},