Implement serializing the state of packet traversal in "continuations".

One purpose of OpenFlow packet-in messages is to allow a controller to
interpose on the path of a packet through the flow tables.  If, for
example, the controller needs to modify a packet in some way that the
switch doesn't directly support, the controller should be able to
program the switch to send it the packet, then modify the packet and
send it back to the switch to continue through the flow table.

That's the theory.  In practice, this doesn't work with any but the
simplest flow tables.  Packet-in messages simply don't include enough
context to allow the flow table traversal to continue.  For example:

    * Via "resubmit" actions, an Open vSwitch packet can have an
      effective "call stack", but a packet-in can't describe it, and
      so it would be lost.

    * A packet-in can't preserve the stack used by NXAST_PUSH and
      NXAST_POP actions.

    * A packet-in can't preserve the OpenFlow 1.1+ action set.

    * A packet-in can't preserve the state of Open vSwitch mirroring
      or connection tracking.

This commit introduces a solution called "continuations".  A continuation
is the state of a packet's traversal through OpenFlow flow tables.  A
"controller" action with the "pause" flag, which is newly implemented in
this commit, generates a continuation and sends it to the OpenFlow
controller in a packet-in asynchronous message (only NXT_PACKET_IN2
supports continuations, so the controller must configure them with
NXT_SET_PACKET_IN_FORMAT).  The controller processes the packet-in,
possibly modifying some of its data, and sends it back to the switch with
an NXT_RESUME request, which causes flow table traversal to continue.  In
principle, a single packet can be paused and resumed multiple times.

Another way to look at it is:

    - "pause" is an extension of the existing OFPAT_CONTROLLER
      action.  It sends the packet to the controller, with full
      pipeline context (some of which is switch implementation
      dependent, and may thus vary from switch to switch).

    - A continuation is an extension of OFPT_PACKET_IN, allowing for
      implementation dependent metadata.

    - NXT_RESUME is an extension of OFPT_PACKET_OUT, with the
      semantics that the pipeline processing is continued with the
      original translation context from where it was left at the time
      it was paused.

Signed-off-by: Ben Pfaff <blp@ovn.org>
Acked-by: Jarno Rajahalme <jarno@ovn.org>

2 files changed, 93 insertions, 27 deletions

diff --git a/utilities/ovs-ofctl.8.in b/utilities/ovs-ofctl.8.in
index 49e95a71c..981e60ace 100644
--- a/utilities/ovs-ofctl.8.in
+++ b/utilities/ovs-ofctl.8.in
@@ -1505,7 +1505,7 @@ network device that has the same name as the bridge.
 Outputs the packet on the port from which it was received.
 .
 .IP \fBcontroller(\fIkey\fB=\fIvalue\fR...\fB)
-Sends the packet to the OpenFlow controller as a ``packet in''
+Sends the packet and its metadata to the OpenFlow controller as a ``packet in''
 message.  The supported key-value pairs are:
 .RS
 .IP "\fBmax_len=\fInbytes\fR"
@@ -1527,6 +1527,15 @@ OpenFlow.
 Supplies the bytes represented as hex digits \fIhh\fR as additional
 data to the controller in the packet-in message.  Pairs of hex digits
 may be separated by periods for readability.
+.IP "\fBpause\fR"
+Causes the switch to freeze the packet's trip through Open vSwitch
+flow tables and serializes that state into the packet-in message as a
+``continuation,'' an additional property in the \fBNXT_PACKET_IN2\fR
+message.  The controller can later send the continuation back to the
+switch in an \fBNXT_RESUME\fR message, which will restart the packet's
+traversal from the point where it was interrupted.  This permits an
+OpenFlow controller to interpose on a packet midway through processing
+in Open vSwitch.
 .
 .RE
 .IP
diff --git a/utilities/ovs-ofctl.c b/utilities/ovs-ofctl.c
index 0d990aa73..c1876fd3e 100644
--- a/utilities/ovs-ofctl.c
+++ b/utilities/ovs-ofctl.c
@@ -1633,9 +1633,13 @@ ofctl_unblock(struct unixctl_conn *conn, int argc OVS_UNUSED,
 /* Prints to stdout all of the messages received on 'vconn'.
  *
  * Iff 'reply_to_echo_requests' is true, sends a reply to any echo request
- * received on 'vconn'. */
+ * received on 'vconn'.
+ *
+ * If 'resume_continuations' is true, sends an NXT_RESUME in reply to any
+ * NXT_PACKET_IN2 that includes a continuation. */
 static void
-monitor_vconn(struct vconn *vconn, bool reply_to_echo_requests)
+monitor_vconn(struct vconn *vconn, bool reply_to_echo_requests,
+              bool resume_continuations)
 {
     struct barrier_aux barrier_aux = { vconn, NULL };
     struct unixctl_server *server;
@@ -1663,6 +1667,10 @@ monitor_vconn(struct vconn *vconn, bool reply_to_echo_requests)
 
     daemonize_complete();
 
+    enum ofp_version version = vconn_get_version(vconn);
+    enum ofputil_protocol protocol
+        = ofputil_protocol_from_ofp_version(version);
+
     for (;;) {
         struct ofpbuf *b;
         int retval;
@@ -1708,6 +1716,36 @@ monitor_vconn(struct vconn *vconn, bool reply_to_echo_requests)
                     }
                 }
                 break;
+
+            case OFPTYPE_PACKET_IN:
+                if (resume_continuations) {
+                    struct ofputil_packet_in pin;
+                    struct ofpbuf continuation;
+
+                    error = ofputil_decode_packet_in(b->data, true, &pin,
+                                                     NULL, NULL,
+                                                     &continuation);
+                    if (error) {
+                        fprintf(stderr, "decoding packet-in failed: %s",
+                                ofperr_to_string(error));
+                    } else if (continuation.size) {
+                        struct ofpbuf *reply;
+
+                        reply = ofputil_encode_resume(&pin, &continuation,
+                                                      protocol);
+
+                        fprintf(stderr, "send: ");
+                        ofp_print(stderr, reply->data, reply->size,
+                                  verbosity + 2);
+                        fflush(stderr);
+
+                        retval = vconn_send_block(vconn, reply);
+                        if (retval) {
+                            ovs_fatal(retval, "failed to send NXT_RESUME");
+                        }
+                    }
+                }
+                break;
             }
             ofpbuf_delete(b);
         }
@@ -1758,6 +1796,7 @@ ofctl_monitor(struct ovs_cmdl_context *ctx)
     }
 
     open_vconn(ctx->argv[1], &vconn);
+    bool resume_continuations = false;
     for (i = 2; i < ctx->argc; i++) {
         const char *arg = ctx->argv[i];
 
@@ -1784,6 +1823,16 @@ ofctl_monitor(struct ovs_cmdl_context *ctx)
             ofputil_append_flow_monitor_request(&fmr, msg);
             dump_transaction(vconn, msg);
             fflush(stdout);
+        } else if (!strcmp(arg, "resume")) {
+            /* This option is intentionally undocumented because it is meant
+             * only for testing. */
+            resume_continuations = true;
+
+            /* Set miss_send_len to ensure that we get packet-ins. */
+            struct ofputil_switch_config config;
+            fetch_switch_config(vconn, &config);
+            config.miss_send_len = UINT16_MAX;
+            set_switch_config(vconn, &config);
         } else {
             ovs_fatal(0, "%s: unsupported \"monitor\" argument", arg);
         }
@@ -1805,7 +1854,7 @@ ofctl_monitor(struct ovs_cmdl_context *ctx)
         }
     }
 
-    monitor_vconn(vconn, true);
+    monitor_vconn(vconn, true, resume_continuations);
 }
 
 static void
@@ -1814,7 +1863,7 @@ ofctl_snoop(struct ovs_cmdl_context *ctx)
     struct vconn *vconn;
 
     open_vconn__(ctx->argv[1], SNOOP, &vconn);
-    monitor_vconn(vconn, false);
+    monitor_vconn(vconn, false, false);
 }
 
 static void
@@ -3610,35 +3659,43 @@ ofctl_parse_ofp11_match(struct ovs_cmdl_context *ctx OVS_UNUSED)
     ds_destroy(&in);
 }
 
-/* "parse-pcap PCAP": read packets from PCAP and print their flows. */
+/* "parse-pcap PCAP...": read packets from each PCAP file and print their
+ * flows. */
 static void
 ofctl_parse_pcap(struct ovs_cmdl_context *ctx)
 {
-    FILE *pcap;
+    int error = 0;
+    for (int i = 1; i < ctx->argc; i++) {
+        const char *filename = ctx->argv[i];
+        FILE *pcap = ovs_pcap_open(filename, "rb");
+        if (!pcap) {
+            error = errno;
+            ovs_error(error, "%s: open failed", filename);
+            continue;
+        }
 
-    pcap = ovs_pcap_open(ctx->argv[1], "rb");
-    if (!pcap) {
-        ovs_fatal(errno, "%s: open failed", ctx->argv[1]);
-    }
+        for (;;) {
+            struct dp_packet *packet;
+            struct flow flow;
+            int retval;
 
-    for (;;) {
-        struct dp_packet *packet;
-        struct flow flow;
-        int error;
+            retval = ovs_pcap_read(pcap, &packet, NULL);
+            if (retval == EOF) {
+                break;
+            } else if (retval) {
+                error = retval;
+                ovs_error(error, "%s: read failed", filename);
+            }
 
-        error = ovs_pcap_read(pcap, &packet, NULL);
-        if (error == EOF) {
-            break;
-        } else if (error) {
-            ovs_fatal(error, "%s: read failed", ctx->argv[1]);
+            pkt_metadata_init(&packet->md, ODPP_NONE);
+            flow_extract(packet, &flow);
+            flow_print(stdout, &flow);
+            putchar('\n');
+            dp_packet_delete(packet);
         }
-
-        pkt_metadata_init(&packet->md, ODPP_NONE);
-        flow_extract(packet, &flow);
-        flow_print(stdout, &flow);
-        putchar('\n');
-        dp_packet_delete(packet);
+        fclose(pcap);
     }
+    exit(error);
 }
 
 /* "check-vlan VLAN_TCI VLAN_TCI_MASK": converts the specified vlan_tci and
@@ -3976,7 +4033,7 @@ static const struct ovs_cmdl_command all_commands[] = {
     { "parse-instructions", NULL, 1, 1, ofctl_parse_instructions },
     { "parse-ofp10-match", NULL, 0, 0, ofctl_parse_ofp10_match },
     { "parse-ofp11-match", NULL, 0, 0, ofctl_parse_ofp11_match },
-    { "parse-pcap", NULL, 1, 1, ofctl_parse_pcap },
+    { "parse-pcap", NULL, 1, INT_MAX, ofctl_parse_pcap },
     { "check-vlan", NULL, 2, 2, ofctl_check_vlan },
     { "print-error", NULL, 1, 1, ofctl_print_error },
     { "encode-error-reply", NULL, 2, 2, ofctl_encode_error_reply },