datapath-windows: Added recirculation support.

Recirculation support for the OVS extension.

Tested using PING and iperf with Driver Verifier enabled.

Signed-off-by: Sorin Vinturis <svinturis@cloudbasesolutions.com>
Co-authored-by: Alin Gabriel Serdean <aserdean@cloudbasesolutions.com>
Reported-by: Sorin Vinturis <svinturis@cloudbasesolutions.com>
Reported-at: https://github.com/openvswitch/ovs-issues/issues/104
Acked-by: Nithin Raju <nithin@vmware.com>
Signed-off-by: Ben Pfaff <blp@ovn.org>

17 files changed, 852 insertions, 64 deletions

diff --git a/datapath-windows/automake.mk b/datapath-windows/automake.mk
index f29f548a2..04fc97f07 100644
--- a/datapath-windows/automake.mk
+++ b/datapath-windows/automake.mk
@@ -9,6 +9,7 @@ EXTRA_DIST += \
 	datapath-windows/misc/uninstall.cmd \
 	datapath-windows/ovsext.sln \
 	datapath-windows/ovsext/Actions.c \
+	datapath-windows/ovsext/Actions.h \
 	datapath-windows/ovsext/Atomic.h \
 	datapath-windows/ovsext/BufferMgmt.c \
 	datapath-windows/ovsext/BufferMgmt.h \
@@ -45,6 +46,8 @@ EXTRA_DIST += \
 	datapath-windows/ovsext/PacketIO.h \
 	datapath-windows/ovsext/PacketParser.c \
 	datapath-windows/ovsext/PacketParser.h \
+	datapath-windows/ovsext/Recirc.c \
+	datapath-windows/ovsext/Recirc.h \
 	datapath-windows/ovsext/Stt.c \
 	datapath-windows/ovsext/Stt.h \
 	datapath-windows/ovsext/Switch.c \
diff --git a/datapath-windows/ovsext/Actions.c b/datapath-windows/ovsext/Actions.c
index 5a045419d..199f6804e 100644
--- a/datapath-windows/ovsext/Actions.c
+++ b/datapath-windows/ovsext/Actions.c
@@ -16,6 +16,7 @@
 
 #include "precomp.h"
 
+#include "Actions.h"
 #include "Debug.h"
 #include "Event.h"
 #include "Flow.h"
@@ -24,6 +25,7 @@
 #include "NetProto.h"
 #include "Offload.h"
 #include "PacketIO.h"
+#include "Recirc.h"
 #include "Stt.h"
 #include "Switch.h"
 #include "User.h"
@@ -35,6 +37,8 @@
 #endif
 #define OVS_DBG_MOD OVS_DBG_ACTION
 
+#define OVS_DEST_PORTS_ARRAY_MIN_SIZE 2
+
 typedef struct _OVS_ACTION_STATS {
     UINT64 rxGre;
     UINT64 txGre;
@@ -55,6 +59,8 @@ typedef struct _OVS_ACTION_STATS {
     UINT32 cannotGrowDest;
     UINT32 zeroActionLen;
     UINT32 failedChecksum;
+    UINT32 deferredActionsQueueFull;
+    UINT32 deferredActionsExecLimit;
 } OVS_ACTION_STATS, *POVS_ACTION_STATS;
 
 OVS_ACTION_STATS ovsActionStats;
@@ -66,7 +72,6 @@ OVS_ACTION_STATS ovsActionStats;
  * exercised while adding new members to the structure - only add ones that get
  * used across multiple stages in the pipeline/get used in multiple functions.
  */
-#define OVS_DEST_PORTS_ARRAY_MIN_SIZE 2
 typedef struct OvsForwardingContext {
     POVS_SWITCH_CONTEXT switchContext;
     /* The NBL currently used in the pipeline. */
@@ -99,7 +104,7 @@ typedef struct OvsForwardingContext {
      */
     OvsIPv4TunnelKey tunKey;
 
-     /*
+    /*
      * Tunneling - Tx:
      * To store the output port, when it is a tunneled port. We don't foresee
      * multiple tunneled ports as outport for any given NBL.
@@ -117,7 +122,6 @@ typedef struct OvsForwardingContext {
     OVS_PACKET_HDR_INFO layers;
 } OvsForwardingContext;
 
-
 /*
  * --------------------------------------------------------------------------
  * OvsInitForwardingCtx --
@@ -564,10 +568,10 @@ OvsCompleteNBLForwardingCtx(OvsForwardingContext *ovsFwdCtx,
 static __inline NDIS_STATUS
 OvsDoFlowLookupOutput(OvsForwardingContext *ovsFwdCtx)
 {
-    OvsFlowKey key;
-    OvsFlow *flow;
-    UINT64 hash;
-    NDIS_STATUS status;
+    OvsFlowKey key = { 0 };
+    OvsFlow *flow = NULL;
+    UINT64 hash = 0;
+    NDIS_STATUS status = NDIS_STATUS_SUCCESS;
     POVS_VPORT_ENTRY vport =
         OvsFindVportByPortNo(ovsFwdCtx->switchContext, ovsFwdCtx->srcVportNo);
     if (vport == NULL || vport->ovsState != OVS_STATE_CONNECTED) {
@@ -595,11 +599,13 @@ OvsDoFlowLookupOutput(OvsForwardingContext *ovsFwdCtx)
     if (flow) {
         OvsFlowUsed(flow, ovsFwdCtx->curNbl, &ovsFwdCtx->layers);
         ovsFwdCtx->switchContext->datapath.hits++;
-        status = OvsActionsExecute(ovsFwdCtx->switchContext,
-                                   ovsFwdCtx->completionList, ovsFwdCtx->curNbl,
-                                   ovsFwdCtx->srcVportNo, ovsFwdCtx->sendFlags,
-                                   &key, &hash, &ovsFwdCtx->layers,
-                                   flow->actions, flow->actionsLen);
+        status = OvsDoExecuteActions(ovsFwdCtx->switchContext,
+                                     ovsFwdCtx->completionList,
+                                     ovsFwdCtx->curNbl,
+                                     ovsFwdCtx->srcVportNo,
+                                     ovsFwdCtx->sendFlags,
+                                     &key, &hash, &ovsFwdCtx->layers,
+                                     flow->actions, flow->actionsLen);
         ovsFwdCtx->curNbl = NULL;
     } else {
         LIST_ENTRY missedPackets;
@@ -1520,8 +1526,55 @@ OvsExecuteSetAction(OvsForwardingContext *ovsFwdCtx,
 
 /*
  * --------------------------------------------------------------------------
- * OvsActionsExecute --
- *     Interpret and execute the specified 'actions' on the specifed packet
+ * OvsExecuteRecirc --
+ *     The function adds a deferred action to allow the current packet, nbl,
+ *     to re-enter datapath packet processing.
+ * --------------------------------------------------------------------------
+ */
+NDIS_STATUS
+OvsExecuteRecirc(OvsForwardingContext *ovsFwdCtx,
+                 OvsFlowKey *key,
+                 const PNL_ATTR actions,
+                 int rem)
+{
+    POVS_DEFERRED_ACTION deferredAction = NULL;
+    PNET_BUFFER_LIST newNbl = NULL;
+
+    if (!NlAttrIsLast(actions, rem)) {
+        /*
+         * Recirc action is the not the last action of the action list, so we
+         * need to clone the packet.
+         */
+        newNbl = OvsPartialCopyNBL(ovsFwdCtx->switchContext, ovsFwdCtx->curNbl,
+                                   0, 0, TRUE /*copy NBL info*/);
+        /*
+         * Skip the recirc action when out of memory, but continue on with the
+         * rest of the action list.
+         */
+        if (newNbl == NULL) {
+            ovsActionStats.noCopiedNbl++;
+            return NDIS_STATUS_SUCCESS;
+        }
+        ovsFwdCtx->curNbl = newNbl;
+    }
+
+    deferredAction = OvsAddDeferredActions(ovsFwdCtx->curNbl, key, NULL);
+    if (deferredAction) {
+        deferredAction->key.recircId = NlAttrGetU32(actions);
+    } else {
+        if (newNbl) {
+            ovsActionStats.deferredActionsQueueFull++;
+            OvsCompleteNBL(ovsFwdCtx->switchContext, newNbl, TRUE);
+        }
+    }
+
+    return NDIS_STATUS_SUCCESS;
+}
+
+/*
+ * --------------------------------------------------------------------------
+ * OvsDoExecuteActions --
+ *     Interpret and execute the specified 'actions' on the specified packet
  *     'curNbl'. The expectation is that if the packet needs to be dropped
  *     (completed) for some reason, it is added to 'completionList' so that the
  *     caller can complete the packet. If 'completionList' is NULL, the NBL is
@@ -1537,16 +1590,16 @@ OvsExecuteSetAction(OvsForwardingContext *ovsFwdCtx,
  * --------------------------------------------------------------------------
  */
 NDIS_STATUS
-OvsActionsExecute(POVS_SWITCH_CONTEXT switchContext,
-                  OvsCompletionList *completionList,
-                  PNET_BUFFER_LIST curNbl,
-                  UINT32 portNo,
-                  ULONG sendFlags,
-                  OvsFlowKey *key,
-                  UINT64 *hash,
-                  OVS_PACKET_HDR_INFO *layers,
-                  const PNL_ATTR actions,
-                  INT actionsLen)
+OvsDoExecuteActions(POVS_SWITCH_CONTEXT switchContext,
+                    OvsCompletionList *completionList,
+                    PNET_BUFFER_LIST curNbl,
+                    UINT32 portNo,
+                    ULONG sendFlags,
+                    OvsFlowKey *key,
+                    UINT64 *hash,
+                    OVS_PACKET_HDR_INFO *layers,
+                    const PNL_ATTR actions,
+                    INT actionsLen)
 {
     PNL_ATTR a;
     INT rem;
@@ -1695,6 +1748,29 @@ OvsActionsExecute(POVS_SWITCH_CONTEXT switchContext,
             break;
         }
 
+        case OVS_ACTION_ATTR_RECIRC:
+        {
+            if (ovsFwdCtx.destPortsSizeOut > 0 || ovsFwdCtx.tunnelTxNic != NULL
+                || ovsFwdCtx.tunnelRxNic != NULL) {
+                status = OvsOutputBeforeSetAction(&ovsFwdCtx);
+                if (status != NDIS_STATUS_SUCCESS) {
+                    dropReason = L"OVS-adding destination failed";
+                    goto dropit;
+                }
+            }
+
+            status = OvsExecuteRecirc(&ovsFwdCtx, key, (const PNL_ATTR)a, rem);
+            if (status != NDIS_STATUS_SUCCESS) {
+                dropReason = L"OVS-recirculation action failed";
+                goto dropit;
+            }
+
+            if (NlAttrIsLast(a, rem)) {
+                goto exit;
+            }
+            break;
+        }
+
         case OVS_ACTION_ATTR_USERSPACE:
         {
             PNL_ATTR userdataAttr;
@@ -1781,5 +1857,146 @@ dropit:
         OvsCompleteNBLForwardingCtx(&ovsFwdCtx, dropReason);
     }
 
+exit:
+    return status;
+}
+
+/*
+ * --------------------------------------------------------------------------
+ * OvsActionsExecute --
+ *     The function interprets and executes the specified 'actions' on the
+ *     specified packet 'curNbl'. See 'OvsDoExecuteActions' description for
+ *     more details.
+ *
+ *     Also executes deferred actions added by recirculation or sample
+ *     actions.
+ * --------------------------------------------------------------------------
+ */
+NDIS_STATUS
+OvsActionsExecute(POVS_SWITCH_CONTEXT switchContext,
+                  OvsCompletionList *completionList,
+                  PNET_BUFFER_LIST curNbl,
+                  UINT32 portNo,
+                  ULONG sendFlags,
+                  OvsFlowKey *key,
+                  UINT64 *hash,
+                  OVS_PACKET_HDR_INFO *layers,
+                  const PNL_ATTR actions,
+                  INT actionsLen)
+{
+    NDIS_STATUS status = STATUS_SUCCESS;
+
+    status = OvsDoExecuteActions(switchContext, completionList, curNbl,
+                                 portNo, sendFlags, key, hash, layers,
+                                 actions, actionsLen);
+
+    if (status == STATUS_SUCCESS) {
+        status = OvsProcessDeferredActions(switchContext, completionList,
+                                           portNo, sendFlags, layers);
+    }
+
+    return status;
+}
+
+/*
+ * --------------------------------------------------------------------------
+ * OvsDoRecirc --
+ *     The function processes the packet 'curNbl' that re-entered datapath
+ *     packet processing after a recirculation action.
+ * --------------------------------------------------------------------------
+ */
+NDIS_STATUS
+OvsDoRecirc(POVS_SWITCH_CONTEXT switchContext,
+            OvsCompletionList *completionList,
+            PNET_BUFFER_LIST curNbl,
+            OvsFlowKey *key,
+            UINT32 srcPortNo,
+            OVS_PACKET_HDR_INFO *layers)
+{
+    NDIS_STATUS status = NDIS_STATUS_SUCCESS;
+    OvsFlow *flow = NULL;
+    OvsForwardingContext ovsFwdCtx = { 0 };
+    UINT64 hash = 0;
+    ASSERT(layers);
+
+    OvsInitForwardingCtx(&ovsFwdCtx, switchContext, curNbl,
+                         srcPortNo, 0,
+                         NET_BUFFER_LIST_SWITCH_FORWARDING_DETAIL(curNbl),
+                         completionList, layers, TRUE);
+
+    status = OvsExtractFlow(ovsFwdCtx.curNbl, ovsFwdCtx.srcVportNo, key,
+                            &ovsFwdCtx.layers, NULL);
+    if (status != NDIS_STATUS_SUCCESS) {
+        OvsCompleteNBLForwardingCtx(&ovsFwdCtx,
+            L"OVS-Dropped due to extract flow failure");
+        ovsActionStats.failedFlowMiss++;
+        return NDIS_STATUS_FAILURE;
+    }
+
+    flow = OvsLookupFlow(&ovsFwdCtx.switchContext->datapath, key, &hash, FALSE);
+    if (flow) {
+        UINT32 level = OvsDeferredActionsLevelGet();
+
+        if (level > DEFERRED_ACTION_EXEC_LEVEL) {
+            OvsCompleteNBLForwardingCtx(&ovsFwdCtx,
+                L"OVS-Dropped due to deferred actions execution level limit \
+                  reached");
+            ovsActionStats.deferredActionsExecLimit++;
+            ovsFwdCtx.curNbl = NULL;
+            return NDIS_STATUS_FAILURE;
+        }
+
+        OvsFlowUsed(flow, ovsFwdCtx.curNbl, &ovsFwdCtx.layers);
+        ovsFwdCtx.switchContext->datapath.hits++;
+
+        OvsDeferredActionsLevelInc();
+
+        status = OvsDoExecuteActions(ovsFwdCtx.switchContext,
+                                     ovsFwdCtx.completionList,
+                                     ovsFwdCtx.curNbl,
+                                     ovsFwdCtx.srcVportNo,
+                                     ovsFwdCtx.sendFlags,
+                                     key, &hash, &ovsFwdCtx.layers,
+                                     flow->actions, flow->actionsLen);
+        ovsFwdCtx.curNbl = NULL;
+
+        OvsDeferredActionsLevelDec();
+    } else {
+        POVS_VPORT_ENTRY vport = NULL;
+        LIST_ENTRY missedPackets;
+        UINT32 num = 0;
+
+        ovsFwdCtx.switchContext->datapath.misses++;
+        InitializeListHead(&missedPackets);
+        vport = OvsFindVportByPortNo(switchContext, srcPortNo);
+        if (vport == NULL || vport->ovsState != OVS_STATE_CONNECTED) {
+            OvsCompleteNBLForwardingCtx(&ovsFwdCtx,
+                L"OVS-Dropped due to port removal");
+            ovsActionStats.noVport++;
+            return NDIS_STATUS_SUCCESS;
+        }
+        status = OvsCreateAndAddPackets(NULL, 0, OVS_PACKET_CMD_MISS,
+                                        vport, key, ovsFwdCtx.curNbl,
+                                        srcPortNo ==
+                                        switchContext->virtualExternalPortId,
+                                        &ovsFwdCtx.layers,
+                                        ovsFwdCtx.switchContext,
+                                        &missedPackets, &num);
+        if (num) {
+            OvsQueuePackets(&missedPackets, num);
+        }
+        if (status == NDIS_STATUS_SUCCESS) {
+            /* Complete the packet since it was copied to user buffer. */
+            OvsCompleteNBLForwardingCtx(&ovsFwdCtx,
+                L"OVS-Dropped since packet was copied to userspace");
+            ovsActionStats.flowMiss++;
+        } else {
+            OvsCompleteNBLForwardingCtx(&ovsFwdCtx,
+                L"OVS-Dropped due to failure to queue to userspace");
+            ovsActionStats.failedFlowMiss++;
+            status = NDIS_STATUS_FAILURE;
+        }
+    }
+
     return status;
 }
diff --git a/datapath-windows/ovsext/Actions.h b/datapath-windows/ovsext/Actions.h
new file mode 100644
index 000000000..c56c260c3
--- /dev/null
+++ b/datapath-windows/ovsext/Actions.h
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2016 Cloudbase Solutions Srl
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __ACTIONS_H_
+#define __ACTIONS_H_ 1
+
+#include "Switch.h"
+#include "PacketIO.h"
+
+NDIS_STATUS
+OvsActionsExecute(POVS_SWITCH_CONTEXT switchContext,
+                  OvsCompletionList *completionList,
+                  PNET_BUFFER_LIST curNbl,
+                  UINT32 srcVportNo,
+                  ULONG sendFlags,
+                  OvsFlowKey *key,
+                  UINT64 *hash,
+                  OVS_PACKET_HDR_INFO *layers,
+                  const PNL_ATTR actions,
+                  int actionsLen);
+
+NDIS_STATUS
+OvsDoExecuteActions(POVS_SWITCH_CONTEXT switchContext,
+                    OvsCompletionList *completionList,
+                    PNET_BUFFER_LIST curNbl,
+                    UINT32 srcVportNo,
+                    ULONG sendFlags,
+                    OvsFlowKey *key,
+                    UINT64 *hash,
+                    OVS_PACKET_HDR_INFO *layers,
+                    const PNL_ATTR actions,
+                    int actionsLen);
+
+NDIS_STATUS
+OvsDoRecirc(POVS_SWITCH_CONTEXT switchContext,
+            OvsCompletionList *completionList,
+            PNET_BUFFER_LIST curNbl,
+            OvsFlowKey *key,
+            UINT32 srcPortNo,
+            OVS_PACKET_HDR_INFO *layers);
+
+#endif /* __ACTIONS_H_ */
diff --git a/datapath-windows/ovsext/Datapath.c b/datapath-windows/ovsext/Datapath.c
index a9a218d20..464fa9784 100644
--- a/datapath-windows/ovsext/Datapath.c
+++ b/datapath-windows/ovsext/Datapath.c
@@ -30,6 +30,7 @@
 #include "Event.h"
 #include "User.h"
 #include "PacketIO.h"
+#include "Recirc.h"
 #include "NetProto.h"
 #include "Flow.h"
 #include "User.h"
@@ -384,6 +385,8 @@ OvsInit()
     gOvsCtrlLock = &ovsCtrlLockObj;
     NdisAllocateSpinLock(gOvsCtrlLock);
     OvsInitEventQueue();
+    OvsDeferredActionsQueueAlloc();
+    OvsDeferredActionsLevelAlloc();
 }
 
 VOID
@@ -394,6 +397,8 @@ OvsCleanup()
         NdisFreeSpinLock(gOvsCtrlLock);
         gOvsCtrlLock = NULL;
     }
+    OvsDeferredActionsQueueFree();
+    OvsDeferredActionsLevelFree();
 }
 
 VOID
diff --git a/datapath-windows/ovsext/DpInternal.h b/datapath-windows/ovsext/DpInternal.h
index 10ea5e86f..be8ac2547 100644
--- a/datapath-windows/ovsext/DpInternal.h
+++ b/datapath-windows/ovsext/DpInternal.h
@@ -20,7 +20,6 @@
 #include <netioapi.h>
 #define IFNAMSIZ IF_NAMESIZE
 #include "../ovsext/Netlink/Netlink.h"
-#include "Mpls.h"
 
 #define OVS_DP_NUMBER   ((uint32_t) 0)
 
@@ -166,6 +165,7 @@ typedef __declspec(align(8)) struct OvsFlowKey {
         Icmp6Key icmp6Key;       /* size 72 */
         MplsKey mplsKey;         /* size 8 */
     };
+    UINT32 recircId;             /* Recirculation ID.  */
 } OvsFlowKey;
 
 #define OVS_WIN_TUNNEL_KEY_SIZE (sizeof (OvsIPv4TunnelKey))
diff --git a/datapath-windows/ovsext/Flow.c b/datapath-windows/ovsext/Flow.c
index be2d5caf4..0bcab62f6 100644
--- a/datapath-windows/ovsext/Flow.c
+++ b/datapath-windows/ovsext/Flow.c
@@ -110,9 +110,7 @@ const NL_POLICY nlFlowPolicy[] = {
     [OVS_FLOW_ATTR_PROBE] = {.type = NL_A_FLAG, .optional = TRUE}
 };
 
-/* For Parsing nested OVS_FLOW_ATTR_KEY attributes.
- * Some of the attributes like OVS_KEY_ATTR_RECIRC_ID
- * are not supported yet. */
+/* For Parsing nested OVS_FLOW_ATTR_KEY attributes. */
 
 const NL_POLICY nlFlowKeyPolicy[] = {
     [OVS_KEY_ATTR_ENCAP] = {.type = NL_A_VAR_LEN, .optional = TRUE},
@@ -252,7 +250,7 @@ OvsFlowNlCmdHandler(POVS_USER_PARAMS_CONTEXT usrParamsCtx,
                     UINT32 *replyLen)
 {
     NTSTATUS rc = STATUS_SUCCESS;
-    BOOLEAN ok;
+    BOOLEAN ok = FALSE;
     POVS_MESSAGE msgIn = (POVS_MESSAGE)usrParamsCtx->inputBuffer;
     POVS_MESSAGE msgOut = (POVS_MESSAGE)usrParamsCtx->outputBuffer;
     PNL_MSG_HDR nlMsgHdr = &(msgIn->nlMsg);
@@ -496,7 +494,7 @@ _FlowNlGetCmdHandler(POVS_USER_PARAMS_CONTEXT usrParamsCtx,
         /* Get tunnel keys attributes */
         if ((NlAttrParseNested(nlMsgHdr, tunnelKeyAttrOffset,
                                NlAttrLen(keyAttrs[OVS_KEY_ATTR_TUNNEL]),
-                               nlFlowTunnelKeyPolicy, 
+                               nlFlowTunnelKeyPolicy,
                                ARRAY_SIZE(nlFlowTunnelKeyPolicy),
                                tunnelAttrs, ARRAY_SIZE(tunnelAttrs)))
                                != TRUE) {
@@ -846,6 +844,12 @@ MapFlowKeyToNlKey(PNL_BUFFER nlBuf,
         goto error_nested_start;
     }
 
+    if (!NlMsgPutTailU32(nlBuf, OVS_KEY_ATTR_RECIRC_ID,
+                         flowKey->recircId)) {
+        rc = STATUS_UNSUCCESSFUL;
+        goto done;
+    }
+
     /* Ethernet header */
     RtlCopyMemory(&(ethKey.eth_src), flowKey->l2.dlSrc, ETH_ADDR_LEN);
     RtlCopyMemory(&(ethKey.eth_dst), flowKey->l2.dlDst, ETH_ADDR_LEN);
@@ -1368,6 +1372,11 @@ _MapKeyAttrToFlowPut(PNL_ATTR *keyAttrs,
 {
     _MapTunAttrToFlowPut(keyAttrs, tunnelAttrs, destKey);
 
+    if (keyAttrs[OVS_KEY_ATTR_RECIRC_ID]) {
+        destKey->recircId = NlAttrGetU32(keyAttrs[OVS_KEY_ATTR_RECIRC_ID]);
+        destKey->l2.keyLen += sizeof(destKey->recircId);
+    }
+
     /* ===== L2 headers ===== */
     destKey->l2.inPort = NlAttrGetU32(keyAttrs[OVS_KEY_ATTR_IN_PORT]);
 
@@ -1546,7 +1555,7 @@ _MapKeyAttrToFlowPut(PNL_ATTR *keyAttrs,
             mplsFlowPutKey->pad[1] = 0;
             mplsFlowPutKey->pad[2] = 0;
             mplsFlowPutKey->pad[3] = 0;
-            destKey->l2.keyLen += sizeof(MplsKey);
+            destKey->l2.keyLen += OVS_MPLS_KEY_SIZE;
         }
         break;
     }
@@ -2259,6 +2268,8 @@ ReportFlowInfo(OvsFlow *flow,
         }
     }
 
+    info->key.recircId = flow->key.recircId;
+
     return status;
 }
 
@@ -2547,7 +2558,7 @@ OvsTunKeyAttrSize(void)
  *----------------------------------------------------------------------------
  *  OvsProbeSupportedFeature --
  *    Verifies if the probed feature is supported.
- * 
+ *
  * Results:
  *   STATUS_SUCCESS if the probed feature is supported.
  *----------------------------------------------------------------------------
diff --git a/datapath-windows/ovsext/Flow.h b/datapath-windows/ovsext/Flow.h
index 74b9dfb74..78bf7ccb7 100644
--- a/datapath-windows/ovsext/Flow.h
+++ b/datapath-windows/ovsext/Flow.h
@@ -54,8 +54,11 @@ NDIS_STATUS OvsAllocateFlowTable(OVS_DATAPATH *datapath,
 NDIS_STATUS OvsExtractFlow(const NET_BUFFER_LIST *pkt, UINT32 inPort,
                            OvsFlowKey *flow, POVS_PACKET_HDR_INFO layers,
                            OvsIPv4TunnelKey *tunKey);
-OvsFlow *OvsLookupFlow(OVS_DATAPATH *datapath, const OvsFlowKey *key,
+OvsFlow* OvsLookupFlow(OVS_DATAPATH *datapath, const OvsFlowKey *key,
                        UINT64 *hash, BOOLEAN hashValid);
+OvsFlow* OvsLookupFlowRecirc(OVS_DATAPATH *datapath,
+                             const OvsFlowKey *key,
+                             UINT64 *hash);
 UINT64 OvsHashFlow(const OvsFlowKey *key);
 VOID OvsFlowUsed(OvsFlow *flow, const NET_BUFFER_LIST *pkt,
                  const POVS_PACKET_HDR_INFO layers);
diff --git a/datapath-windows/ovsext/Netlink/Netlink.h b/datapath-windows/ovsext/Netlink/Netlink.h
index 99665fb92..8f6a5bea5 100644
--- a/datapath-windows/ovsext/Netlink/Netlink.h
+++ b/datapath-windows/ovsext/Netlink/Netlink.h
@@ -173,6 +173,17 @@ static __inline NlAttrTotalSize(UINT32 payloadSize)
     return NLA_ALIGN(NlAttrSize(payloadSize));
 }
 
+/*
+ * ---------------------------------------------------------------------------
+ * Returns true if the last attribute is reached.
+ * ---------------------------------------------------------------------------
+ */
+BOOLEAN
+static __inline NlAttrIsLast(const PNL_ATTR nla, int rem)
+{
+    return nla->nlaLen == rem;
+}
+
 /* Netlink attribute validation */
 BOOLEAN NlAttrValidate(const PNL_ATTR, const PNL_POLICY);
 
diff --git a/datapath-windows/ovsext/PacketIO.c b/datapath-windows/ovsext/PacketIO.c
index cfbae3406..a0ddc3de7 100644
--- a/datapath-windows/ovsext/PacketIO.c
+++ b/datapath-windows/ovsext/PacketIO.c
@@ -20,6 +20,8 @@
  */
 
 #include "precomp.h"
+
+#include "Actions.h"
 #include "Switch.h"
 #include "Vport.h"
 #include "NetProto.h"
@@ -234,14 +236,14 @@ OvsStartNBLIngress(POVS_SWITCH_CONTEXT switchContext,
     OvsInitCompletionList(&completionList, switchContext, sendCompleteFlags);
 
     for (curNbl = netBufferLists; curNbl != NULL; curNbl = nextNbl) {
-        POVS_VPORT_ENTRY vport;
-        UINT32 portNo;
+        POVS_VPORT_ENTRY vport = NULL;
+        UINT32 portNo = 0;
         OVS_DATAPATH *datapath = &switchContext->datapath;
-        OVS_PACKET_HDR_INFO layers;
-        OvsFlowKey key;
-        UINT64 hash;
-        PNET_BUFFER curNb;
-        POVS_BUFFER_CONTEXT ctx;
+        OVS_PACKET_HDR_INFO layers = { 0 };
+        OvsFlowKey key = { 0 };
+        UINT64 hash = 0;
+        PNET_BUFFER curNb = NULL;
+        POVS_BUFFER_CONTEXT ctx = NULL;
 
         nextNbl = curNbl->Next;
         curNbl->Next = NULL;
diff --git a/datapath-windows/ovsext/PacketIO.h b/datapath-windows/ovsext/PacketIO.h
index 7247869d9..a7c1f7668 100644
--- a/datapath-windows/ovsext/PacketIO.h
+++ b/datapath-windows/ovsext/PacketIO.h
@@ -48,14 +48,4 @@ VOID OvsSendNBLIngress(POVS_SWITCH_CONTEXT switchContext,
                        PNET_BUFFER_LIST netBufferLists,
                        ULONG sendFlags);
 
-NDIS_STATUS OvsActionsExecute(POVS_SWITCH_CONTEXT switchContext,
-                              OvsCompletionList *completionList,
-                              PNET_BUFFER_LIST curNbl, UINT32 srcVportNo,
-                              ULONG sendFlags, OvsFlowKey *key, UINT64 *hash,
-                              OVS_PACKET_HDR_INFO *layers,
-                              const PNL_ATTR actions, int actionsLen);
-
-VOID OvsLookupFlowOutput(POVS_SWITCH_CONTEXT switchContext,
-                         VOID *compList, PNET_BUFFER_LIST curNbl);
-
 #endif /* __PACKETIO_H_ */
diff --git a/datapath-windows/ovsext/Recirc.c b/datapath-windows/ovsext/Recirc.c
new file mode 100644
index 000000000..86e6f51df
--- /dev/null
+++ b/datapath-windows/ovsext/Recirc.c
@@ -0,0 +1,349 @@
+/*
+ * Copyright (c) 2016 Cloudbase Solutions Srl
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Recirc.h"
+#include "Flow.h"
+#include "Jhash.h"
+
+static POVS_DEFERRED_ACTION_QUEUE ovsDeferredActionQueue = NULL;
+static UINT32* ovsDeferredActionLevel = NULL;
+
+/*
+ * --------------------------------------------------------------------------
+ * OvsDeferredActionsQueueAlloc --
+ *     The function allocates per-cpu deferred actions queue.
+ * --------------------------------------------------------------------------
+ */
+BOOLEAN
+OvsDeferredActionsQueueAlloc()
+{
+    ovsDeferredActionQueue =
+        OvsAllocateMemoryPerCpu(sizeof(*ovsDeferredActionQueue),
+                                OVS_RECIRC_POOL_TAG);
+    if (!ovsDeferredActionQueue) {
+        return FALSE;
+    }
+    return TRUE;
+}
+
+/*
+ * --------------------------------------------------------------------------
+ * OvsDeferredActionsQueueFree --
+ *     The function frees per-cpu deferred actions queue.
+ * --------------------------------------------------------------------------
+ */
+VOID
+OvsDeferredActionsQueueFree()
+{
+    OvsFreeMemoryWithTag(ovsDeferredActionQueue,
+                         OVS_RECIRC_POOL_TAG);
+    ovsDeferredActionQueue = NULL;
+}
+
+/*
+ * --------------------------------------------------------------------------
+ * OvsDeferredActionsLevelAlloc --
+ *     The function allocates per-cpu deferred actions execution level.
+ * --------------------------------------------------------------------------
+ */
+BOOLEAN
+OvsDeferredActionsLevelAlloc()
+{
+    ovsDeferredActionLevel =
+        OvsAllocateMemoryPerCpu(sizeof(*ovsDeferredActionLevel),
+                                OVS_RECIRC_POOL_TAG);
+    if (!ovsDeferredActionLevel) {
+        return FALSE;
+    }
+    return TRUE;
+}
+
+/*
+ * --------------------------------------------------------------------------
+ * OvsDeferredActionsLevelFree --
+ *     The function frees per-cpu deferred actions execution level.
+ * --------------------------------------------------------------------------
+ */
+VOID
+OvsDeferredActionsLevelFree()
+{
+    OvsFreeMemoryWithTag(ovsDeferredActionLevel,
+                         OVS_RECIRC_POOL_TAG);
+    ovsDeferredActionLevel = NULL;
+}
+
+/*
+ * --------------------------------------------------------------------------
+ * OvsDeferredActionsQueueGet --
+ *     The function returns the deferred action queue corresponding to the
+ *     current processor.
+ * --------------------------------------------------------------------------
+ */
+POVS_DEFERRED_ACTION_QUEUE
+OvsDeferredActionsQueueGet()
+{
+    POVS_DEFERRED_ACTION_QUEUE queue = NULL;
+    ULONG index = 0;
+    KIRQL oldIrql = KeGetCurrentIrql();
+
+    if (oldIrql < DISPATCH_LEVEL) {
+        KeRaiseIrqlToDpcLevel();
+    }
+
+    index = KeGetCurrentProcessorNumberEx(NULL);
+    queue = &ovsDeferredActionQueue[index];
+
+    if (oldIrql < DISPATCH_LEVEL) {
+        KeLowerIrql(oldIrql);
+    }
+
+    return queue;
+}
+
+/*
+ * --------------------------------------------------------------------------
+ * OvsDeferredActionsLevelGet --
+ *     The function returns the deferred action execution level corresponding
+ *     to the current processor.
+ * --------------------------------------------------------------------------
+ */
+UINT32
+OvsDeferredActionsLevelGet()
+{
+    UINT32 *level = NULL;
+    ULONG index = 0;
+    KIRQL oldIrql = KeGetCurrentIrql();
+
+    if (oldIrql < DISPATCH_LEVEL) {
+        KeRaiseIrqlToDpcLevel();
+    }
+
+    index = KeGetCurrentProcessorNumberEx(NULL);
+    level = &ovsDeferredActionLevel[index];
+
+    if (oldIrql < DISPATCH_LEVEL) {
+        KeLowerIrql(oldIrql);
+    }
+
+    return *level;
+}
+
+/*
+ * --------------------------------------------------------------------------
+ * OvsDeferredActionsLevelInc --
+ *     The function increments the deferred action execution level
+ *     corresponding to the current processor.
+ * --------------------------------------------------------------------------
+ */
+VOID
+OvsDeferredActionsLevelInc()
+{
+    UINT32 *level = NULL;
+    ULONG index = 0;
+    KIRQL oldIrql = KeGetCurrentIrql();
+
+    if (oldIrql < DISPATCH_LEVEL) {
+        KeRaiseIrqlToDpcLevel();
+    }
+
+    index = KeGetCurrentProcessorNumberEx(NULL);
+    level = &ovsDeferredActionLevel[index];
+    (*level)++;
+
+    if (oldIrql < DISPATCH_LEVEL) {
+        KeLowerIrql(oldIrql);
+    }
+}
+
+/*
+ * --------------------------------------------------------------------------
+ * OvsDeferredActionsLevelDec --
+ *     The function decrements the deferred action execution level
+ *     corresponding to the current processor.
+ * --------------------------------------------------------------------------
+ */
+VOID
+OvsDeferredActionsLevelDec()
+{
+    UINT32 *level = NULL;
+    ULONG index = 0;
+    KIRQL oldIrql = KeGetCurrentIrql();
+
+    if (oldIrql < DISPATCH_LEVEL) {
+        KeRaiseIrqlToDpcLevel();
+    }
+
+    index = KeGetCurrentProcessorNumberEx(NULL);
+    level = &ovsDeferredActionLevel[index];
+    (*level)--;
+
+    if (oldIrql < DISPATCH_LEVEL) {
+        KeLowerIrql(oldIrql);
+    }
+}
+
+/*
+ * --------------------------------------------------------------------------
+ * OvsDeferredActionsQueueInit --
+ *     The function resets the queue to be ready for the next packet.
+ * --------------------------------------------------------------------------
+ */
+static
+VOID
+OvsDeferredActionsQueueInit(POVS_DEFERRED_ACTION_QUEUE queue)
+{
+    queue->head = 0;
+    queue->tail = 0;
+}
+
+/*
+ * --------------------------------------------------------------------------
+ * OvsDeferredActionsQueueIsEmpty --
+ *     The function verifies if the queue is empty.
+ * --------------------------------------------------------------------------
+ */
+static
+BOOLEAN
+OvsDeferredActionsQueueIsEmpty(const POVS_DEFERRED_ACTION_QUEUE queue)
+{
+    return (queue->head == queue->tail);
+}
+
+/*
+ * --------------------------------------------------------------------------
+ * OvsDeferredActionsQueuePop --
+ *     The function pops the next queue element.
+ * --------------------------------------------------------------------------
+ */
+static
+POVS_DEFERRED_ACTION
+OvsDeferredActionsQueuePop(POVS_DEFERRED_ACTION_QUEUE queue)
+{
+    POVS_DEFERRED_ACTION deferredAction = NULL;
+    KIRQL oldIrql = KeGetCurrentIrql();
+
+    if (oldIrql < DISPATCH_LEVEL) {
+        KeRaiseIrqlToDpcLevel();
+    }
+
+    if (OvsDeferredActionsQueueIsEmpty(queue)) {
+        /* Reset the queue for the next packet. */
+        OvsDeferredActionsQueueInit(queue);
+    } else {
+        deferredAction = &queue->queue[queue->tail++];
+    }
+
+    if (oldIrql < DISPATCH_LEVEL) {
+        KeLowerIrql(oldIrql);
+    }
+
+    return deferredAction;
+}
+
+/*
+ * --------------------------------------------------------------------------
+ * OvsDeferredActionsQueuePush --
+ *     The function pushes the current element in the deferred actions queue.
+ * --------------------------------------------------------------------------
+ */
+static
+POVS_DEFERRED_ACTION
+OvsDeferredActionsQueuePush(POVS_DEFERRED_ACTION_QUEUE queue)
+{
+    POVS_DEFERRED_ACTION deferredAction = NULL;
+    KIRQL oldIrql = KeGetCurrentIrql();
+
+    if (oldIrql < DISPATCH_LEVEL) {
+        KeRaiseIrqlToDpcLevel();
+    }
+
+    if (queue->head < DEFERRED_ACTION_QUEUE_SIZE) {
+        deferredAction = &queue->queue[queue->head++];
+    }
+
+    if (oldIrql < DISPATCH_LEVEL) {
+        KeLowerIrql(oldIrql);
+    }
+
+    return deferredAction;
+}
+
+/*
+ * --------------------------------------------------------------------------
+ * OvsAddDeferredActions --
+ *     This function adds the deferred action to the current CPU queue and
+ *     returns the new queue entry if the queue is not already full.
+ * --------------------------------------------------------------------------
+ */
+POVS_DEFERRED_ACTION
+OvsAddDeferredActions(PNET_BUFFER_LIST nbl,
+                      OvsFlowKey *key,
+                      const PNL_ATTR actions)
+{
+    POVS_DEFERRED_ACTION_QUEUE queue = OvsDeferredActionsQueueGet();
+    POVS_DEFERRED_ACTION deferredAction = NULL;
+
+    deferredAction = OvsDeferredActionsQueuePush(queue);
+    if (deferredAction) {
+        deferredAction->nbl = nbl;
+        deferredAction->actions = actions;
+        deferredAction->key = *key;
+    }
+
+    return deferredAction;
+}
+
+/*
+ * --------------------------------------------------------------------------
+ * OvsProcessDeferredActions --
+ *     This function processes all deferred actions contained in the queue
+ *     corresponding to the current CPU.
+ * --------------------------------------------------------------------------
+ */
+NDIS_STATUS
+OvsProcessDeferredActions(POVS_SWITCH_CONTEXT switchContext,
+                          OvsCompletionList *completionList,
+                          UINT32 portNo,
+                          ULONG sendFlags,
+                          OVS_PACKET_HDR_INFO *layers)
+{
+    NDIS_STATUS status = NDIS_STATUS_SUCCESS;
+    POVS_DEFERRED_ACTION_QUEUE queue = OvsDeferredActionsQueueGet();
+    POVS_DEFERRED_ACTION deferredAction = NULL;
+
+    /* Process all deferred actions. */
+    while ((deferredAction = OvsDeferredActionsQueuePop(queue)) != NULL) {
+        if (deferredAction->actions) {
+            status = OvsDoExecuteActions(switchContext,
+                                         completionList,
+                                         deferredAction->nbl,
+                                         portNo,
+                                         sendFlags,
+                                         &deferredAction->key, NULL,
+                                         layers, deferredAction->actions,
+                                         NlAttrGetSize(deferredAction->actions));
+        } else {
+            status = OvsDoRecirc(switchContext,
+                                 completionList,
+                                 deferredAction->nbl,
+                                 &deferredAction->key,
+                                 portNo,
+                                 layers);
+        }
+    }
+
+    return status;
+}
diff --git a/datapath-windows/ovsext/Recirc.h b/datapath-windows/ovsext/Recirc.h
new file mode 100644
index 000000000..ee0576347
--- /dev/null
+++ b/datapath-windows/ovsext/Recirc.h
@@ -0,0 +1,136 @@
+/*
+ * Copyright (c) 2016 Cloudbase Solutions Srl
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __RECIRC_H_
+#define __RECIRC_H_ 1
+
+#include "Actions.h"
+
+#define DEFERRED_ACTION_QUEUE_SIZE          10
+#define DEFERRED_ACTION_EXEC_LEVEL           4
+
+typedef struct _OVS_DEFERRED_ACTION {
+    PNET_BUFFER_LIST    nbl;
+    PNL_ATTR            actions;
+    OvsFlowKey          key;
+} OVS_DEFERRED_ACTION, *POVS_DEFERRED_ACTION;
+
+/*
+ * --------------------------------------------------------------------------
+ * '_OVS_DEFERRED_ACTION_QUEUE' structure is responsible for keeping track of
+ * all deferred actions. The maximum number of deferred actions should not
+ * exceed 'DEFERRED_ACTION_QUEUE_SIZE'.
+ * --------------------------------------------------------------------------
+ */
+typedef struct _OVS_DEFERRED_ACTION_QUEUE {
+    UINT32  head;
+    UINT32  tail;
+    OVS_DEFERRED_ACTION queue[DEFERRED_ACTION_QUEUE_SIZE];
+} OVS_DEFERRED_ACTION_QUEUE, *POVS_DEFERRED_ACTION_QUEUE;
+
+/*
+ * --------------------------------------------------------------------------
+ * OvsProcessDeferredActions --
+ *     This function processes all deferred actions contained in the queue
+ *     corresponding to the current CPU.
+ * --------------------------------------------------------------------------
+ */
+NDIS_STATUS
+OvsProcessDeferredActions(POVS_SWITCH_CONTEXT switchContext,
+                          OvsCompletionList *completionList,
+                          UINT32 portNo,
+                          ULONG sendFlags,
+                          OVS_PACKET_HDR_INFO *layers);
+
+/*
+ * --------------------------------------------------------------------------
+ * OvsAddDeferredActions --
+ *     This function adds the deferred action to the current CPU queue and
+ *     returns the new queue entry if the queue is not already full.
+ * --------------------------------------------------------------------------
+ */
+POVS_DEFERRED_ACTION
+OvsAddDeferredActions(PNET_BUFFER_LIST packet,
+                      OvsFlowKey *key,
+                      const PNL_ATTR actions);
+
+/*
+ * --------------------------------------------------------------------------
+ * OvsDeferredActionsQueueAlloc --
+ *     The function allocates per-cpu deferred actions queue.
+ * --------------------------------------------------------------------------
+ */
+BOOLEAN
+OvsDeferredActionsQueueAlloc();
+
+/*
+ * --------------------------------------------------------------------------
+ * OvsDeferredActionsQueueFree --
+ *     The function frees per-cpu deferred actions queue.
+ * --------------------------------------------------------------------------
+ */
+VOID
+OvsDeferredActionsQueueFree();
+
+/*
+ * --------------------------------------------------------------------------
+ * OvsDeferredActionsLevelAlloc --
+ *     The function allocates per-cpu deferred actions execution level.
+ * --------------------------------------------------------------------------
+ */
+BOOLEAN
+OvsDeferredActionsLevelAlloc();
+
+/*
+ * --------------------------------------------------------------------------
+ * OvsDeferredActionsLevelFree --
+ *     The function frees per-cpu deferred actions execution level.
+ * --------------------------------------------------------------------------
+ */
+VOID
+OvsDeferredActionsLevelFree();
+
+/*
+ * --------------------------------------------------------------------------
+ * OvsDeferredActionsLevelGet --
+ *     The function returns the deferred action execution level corresponding
+ *     to the current processor.
+ * --------------------------------------------------------------------------
+ */
+UINT32
+OvsDeferredActionsLevelGet();
+
+/*
+ * --------------------------------------------------------------------------
+ * OvsDeferredActionsLevelInc --
+ *     The function increments the deferred action execution level
+ *     corresponding to the current processor.
+ * --------------------------------------------------------------------------
+ */
+VOID
+OvsDeferredActionsLevelInc();
+
+/*
+ * --------------------------------------------------------------------------
+ * OvsDeferredActionsLevelDec --
+ *     The function decrements the deferred action execution level
+ *     corresponding to the current processor.
+ * --------------------------------------------------------------------------
+*/
+VOID
+OvsDeferredActionsLevelDec();
+
+#endif /* __RECIRC_H_ */
diff --git a/datapath-windows/ovsext/Tunnel.c b/datapath-windows/ovsext/Tunnel.c
index eea4a8495..e957aaf4f 100644
--- a/datapath-windows/ovsext/Tunnel.c
+++ b/datapath-windows/ovsext/Tunnel.c
@@ -39,6 +39,7 @@
 #include "PacketIO.h"
 #include "NetProto.h"
 #include "Flow.h"
+#include "Actions.h"
 
 extern POVS_SWITCH_CONTEXT gOvsSwitchContext;
 
@@ -258,13 +259,13 @@ OvsInjectPacketThroughActions(PNET_BUFFER_LIST pNbl,
                           sendCompleteFlags);
 
     {
-        POVS_VPORT_ENTRY vport;
-        UINT32 portNo;
-        OVS_PACKET_HDR_INFO layers;
-        OvsFlowKey key;
-        UINT64 hash;
-        PNET_BUFFER curNb;
-        OvsFlow *flow;
+        POVS_VPORT_ENTRY vport = NULL;
+        UINT32 portNo = 0;
+        OVS_PACKET_HDR_INFO layers = { 0 };
+        OvsFlowKey key = { 0 };
+        UINT64 hash = 0;
+        PNET_BUFFER curNb = NULL;
+        OvsFlow *flow = NULL;
 
         fwdDetail = NET_BUFFER_LIST_SWITCH_FORWARDING_DETAIL(pNbl);
 
diff --git a/datapath-windows/ovsext/User.c b/datapath-windows/ovsext/User.c
index e97f2b2b6..cadffdab8 100644
--- a/datapath-windows/ovsext/User.c
+++ b/datapath-windows/ovsext/User.c
@@ -22,6 +22,7 @@
 
 #include "precomp.h"
 
+#include "Actions.h"
 #include "Datapath.h"
 #include "Debug.h"
 #include "Event.h"
@@ -388,14 +389,14 @@ NTSTATUS
 OvsExecuteDpIoctl(OvsPacketExecute *execute)
 {
     NTSTATUS                    status = STATUS_SUCCESS;
-    NTSTATUS                    ndisStatus;
+    NTSTATUS                    ndisStatus = STATUS_SUCCESS;
     LOCK_STATE_EX               lockState;
-    PNET_BUFFER_LIST pNbl;
-    PNL_ATTR actions;
+    PNET_BUFFER_LIST            pNbl = NULL;
+    PNL_ATTR                    actions = NULL;
     PNDIS_SWITCH_FORWARDING_DETAIL_NET_BUFFER_LIST_INFO fwdDetail;
-    OvsFlowKey key;
-    OVS_PACKET_HDR_INFO layers;
-    POVS_VPORT_ENTRY vport;
+    OvsFlowKey                  key = { 0 };
+    OVS_PACKET_HDR_INFO         layers = { 0 };
+    POVS_VPORT_ENTRY            vport = NULL;
 
     if (execute->packetLen == 0) {
         status = STATUS_INVALID_PARAMETER;
diff --git a/datapath-windows/ovsext/Util.h b/datapath-windows/ovsext/Util.h
index b2ec798e0..038754dd0 100644
--- a/datapath-windows/ovsext/Util.h
+++ b/datapath-windows/ovsext/Util.h
@@ -36,6 +36,7 @@
 #define OVS_STT_POOL_TAG                'RSVO'
 #define OVS_GRE_POOL_TAG                'GSVO'
 #define OVS_TUNFLT_POOL_TAG             'WSVO'
+#define OVS_RECIRC_POOL_TAG             'CSVO'
 
 VOID *OvsAllocateMemory(size_t size);
 VOID *OvsAllocateMemoryWithTag(size_t size, ULONG tag);
diff --git a/datapath-windows/ovsext/ovsext.vcxproj b/datapath-windows/ovsext/ovsext.vcxproj
index e3aea9717..af718f775 100644
--- a/datapath-windows/ovsext/ovsext.vcxproj
+++ b/datapath-windows/ovsext/ovsext.vcxproj
@@ -71,6 +71,7 @@
   </ImportGroup>
   <ItemGroup Label="WrappedTaskItems">
     <ClInclude Include="..\include\OvsDpInterfaceExt.h" />
+    <ClInclude Include="Actions.h" />
     <ClInclude Include="Atomic.h" />
     <ClInclude Include="BufferMgmt.h" />
     <ClInclude Include="Datapath.h" />
@@ -93,6 +94,7 @@
     <ClInclude Include="PacketIO.h" />
     <ClInclude Include="PacketParser.h" />
     <ClInclude Include="precomp.h" />
+    <ClInclude Include="Recirc.h" />
     <ClInclude Include="resource.h" />
     <ClInclude Include="Stt.h" />
     <ClInclude Include="Switch.h" />
@@ -193,6 +195,7 @@
       <PreCompiledHeader>Create</PreCompiledHeader>
       <PreCompiledHeaderOutputFile>$(IntDir)\precomp.h.pch</PreCompiledHeaderOutputFile>
     </ClCompile>
+    <ClCompile Include="Recirc.c" />
     <ClCompile Include="Stt.c" />
     <ClCompile Include="Switch.c" />
     <ClCompile Include="Tunnel.c" />
diff --git a/lib/ovs-atomic.h b/lib/ovs-atomic.h
index b38c9ef4d..2046ea947 100644
--- a/lib/ovs-atomic.h
+++ b/lib/ovs-atomic.h
@@ -333,7 +333,7 @@
         #include "ovs-atomic-i586.h"
     #elif HAVE_GCC4_ATOMICS
         #include "ovs-atomic-gcc4+.h"
-    #elif _MSC_VER && _M_IX86 >= 500
+    #elif _MSC_VER && _WIN32
         #include "ovs-atomic-msvc.h"
     #else
         /* ovs-atomic-pthreads implementation is provided for portability.