/* * Copyright (c) 2014 VMware, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * This file contains the implementation of the datapath/forwarding * functionality of the OVS. */ #include "precomp.h" #include "Actions.h" #include "Switch.h" #include "Vport.h" #include "NetProto.h" #include "User.h" #include "PacketIO.h" #include "Flow.h" #include "Event.h" #include "User.h" /* Due to an imported header file */ #pragma warning( disable:4505 ) #ifdef OVS_DBG_MOD #undef OVS_DBG_MOD #endif #define OVS_DBG_MOD OVS_DBG_DISPATCH #include "Debug.h" extern NDIS_STRING ovsExtGuidUC; extern NDIS_STRING ovsExtFriendlyNameUC; static VOID OvsFinalizeCompletionList(OvsCompletionList *completionList); static VOID OvsCompleteNBLIngress(POVS_SWITCH_CONTEXT switchContext, PNET_BUFFER_LIST netBufferLists, ULONG sendCompleteFlags, BOOLEAN isSendComplete); VOID OvsInitCompletionList(OvsCompletionList *completionList, POVS_SWITCH_CONTEXT switchContext, ULONG sendCompleteFlags) { ASSERT(completionList); completionList->dropNbl = NULL; completionList->dropNblNext = &completionList->dropNbl; completionList->switchContext = switchContext; completionList->sendCompleteFlags = sendCompleteFlags; } /* Utility function used to complete an NBL. */ VOID OvsAddPktCompletionList(OvsCompletionList *completionList, BOOLEAN incoming, NDIS_SWITCH_PORT_ID sourcePort, PNET_BUFFER_LIST netBufferList, UINT32 netBufferListCount, PNDIS_STRING filterReason) { POVS_BUFFER_CONTEXT ctx; /* XXX: We handle one NBL at a time. */ ASSERT(netBufferList->Next == NULL); /* Make sure it has a context. */ ctx = (POVS_BUFFER_CONTEXT)NET_BUFFER_LIST_CONTEXT_DATA_START(netBufferList); ASSERT(ctx && ctx->magic == OVS_CTX_MAGIC); completionList->switchContext->NdisSwitchHandlers.ReportFilteredNetBufferLists( completionList->switchContext->NdisSwitchContext, &ovsExtGuidUC, &ovsExtFriendlyNameUC, sourcePort, incoming ? NDIS_SWITCH_REPORT_FILTERED_NBL_FLAGS_IS_INCOMING : 0, netBufferListCount, netBufferList, filterReason); *completionList->dropNblNext = netBufferList; completionList->dropNblNext = &netBufferList->Next; ASSERT(completionList->dropNbl); } static __inline VOID OvsReportNBLIngressError(POVS_SWITCH_CONTEXT switchContext, PNET_BUFFER_LIST nblList, PNDIS_STRING filterReason, NDIS_STATUS error) { PNET_BUFFER_LIST nbl = nblList; while (nbl) { PNDIS_SWITCH_FORWARDING_DETAIL_NET_BUFFER_LIST_INFO fwdDetail; fwdDetail = NET_BUFFER_LIST_SWITCH_FORWARDING_DETAIL(nbl); nbl->Status = error; /* This can be optimized by batching NBL's from the same * SourcePortId. */ switchContext->NdisSwitchHandlers.ReportFilteredNetBufferLists( switchContext->NdisSwitchContext, &ovsExtGuidUC, &ovsExtFriendlyNameUC, fwdDetail->SourcePortId, NDIS_SWITCH_REPORT_FILTERED_NBL_FLAGS_IS_INCOMING, 1 /*Nbl count.*/, nbl, filterReason); nbl = NET_BUFFER_LIST_NEXT_NBL(nbl); } } static __inline ULONG OvsGetSendCompleteFlags(ULONG sendFlags) { BOOLEAN dispatch, sameSource; ULONG sendCompleteFlags; dispatch = NDIS_TEST_SEND_AT_DISPATCH_LEVEL(sendFlags); sendCompleteFlags = (dispatch ? NDIS_SEND_COMPLETE_FLAGS_DISPATCH_LEVEL : 0); sameSource = NDIS_TEST_SEND_FLAG(sendFlags, NDIS_SEND_FLAGS_SWITCH_SINGLE_SOURCE); sendCompleteFlags |= (sameSource ? NDIS_SEND_COMPLETE_FLAGS_SWITCH_SINGLE_SOURCE : 0); return sendCompleteFlags; } VOID OvsSendNBLIngress(POVS_SWITCH_CONTEXT switchContext, PNET_BUFFER_LIST netBufferLists, ULONG sendFlags) { if (switchContext->dataFlowState == OvsSwitchPaused) { /* If a filter module is in the Paused state, the filter driver must not * originate any send requests for that filter module. If NDIS calls * FilterSendNetBufferLists, the driver must not call * NdisFSendNetBufferLists to pass on the data until the driver is * restarted. The driver should call NdisFSendNetBufferListsComplete * immediately to complete the send operation. It should set the * complete status in each NET_BUFFER_LIST structure to * NDIS_STATUS_PAUSED. * * http://msdn.microsoft.com/en-us/library/windows/hardware/ * ff549966(v=vs.85).aspx */ NDIS_STRING filterReason; ULONG sendCompleteFlags = OvsGetSendCompleteFlags(sendFlags); RtlInitUnicodeString(&filterReason, L"Switch state PAUSED, drop before FSendNBL."); OvsReportNBLIngressError(switchContext, netBufferLists, &filterReason, NDIS_STATUS_PAUSED); OvsCompleteNBLIngress(switchContext, netBufferLists, sendCompleteFlags, FALSE); return; } ASSERT(switchContext->dataFlowState == OvsSwitchRunning); POVS_BUFFER_CONTEXT ctx = (POVS_BUFFER_CONTEXT)NET_BUFFER_LIST_CONTEXT_DATA_START(netBufferLists); LONG refCount = 1, exchange = 0; InterlockedCompareExchange((LONG volatile *)&refCount, exchange, (LONG)ctx->refCount); if (refCount != exchange) { InterlockedExchange((LONG volatile *)&ctx->sendFlags, sendFlags); InterlockedExchange16((SHORT volatile *)&ctx->pendingSend, 1); return; } InterlockedExchange16((SHORT volatile *)&ctx->pendingSend, 0); NdisFSendNetBufferLists(switchContext->NdisFilterHandle, netBufferLists, NDIS_DEFAULT_PORT_NUMBER, sendFlags); } static __inline BOOLEAN OvsCheckNBLSingleSource(PNET_BUFFER_LIST netBufferLists) { UINT32 sourcePortId = 0; BOOLEAN singleSource = TRUE; PNET_BUFFER_LIST curNbl = netBufferLists; PNDIS_SWITCH_FORWARDING_DETAIL_NET_BUFFER_LIST_INFO info; while (curNbl != NULL) { info = NET_BUFFER_LIST_SWITCH_FORWARDING_DETAIL(curNbl); if (info == NULL) { /* We are not able to determine the source port ID */ singleSource = FALSE; OVS_LOG_INFO("nbl %p has no source port", curNbl); break; } if (curNbl == netBufferLists) { sourcePortId = info->SourcePortId; } else if (info->SourcePortId != sourcePortId) { singleSource = FALSE; OVS_LOG_INFO("Source port in nbl %p is %u, not from %u", curNbl, info->SourcePortId, sourcePortId); break; } curNbl = NET_BUFFER_LIST_NEXT_NBL(curNbl); } return singleSource; } /* * SendNetBufferListsCompleteHandler releases the NetBufferLists with flag * NDIS_SEND_COMPLETE_FLAGS_SWITCH_SINGLE_SOURCE if all the NBLs have same * source port, for cloned NBLs, source port might be changed, although the * cloned NBLs have same source port, there parent NBLs may have different * source ports, so we should have a check before passing the flag to * NdisFSendNetBufferListsComplete. */ static __inline VOID OvsCompleteUpperLayerNBL(NDIS_HANDLE ndisHandle, PNET_BUFFER_LIST netBufferLists, ULONG sendCompleteFlags, BOOLEAN isSendComplete) { BOOLEAN singleSource = TRUE; PNET_BUFFER_LIST curNbl, nextNbl; /* To check whether the NBLs are from the same source port */ if (isSendComplete && (sendCompleteFlags & NDIS_SEND_COMPLETE_FLAGS_SWITCH_SINGLE_SOURCE)) { singleSource = OvsCheckNBLSingleSource(netBufferLists); } if (singleSource) { NdisFSendNetBufferListsComplete(ndisHandle, netBufferLists, sendCompleteFlags); } else { /* * Not from a single source port, releasing the NBls without flag * NDIS_SEND_COMPLETE_FLAGS_SWITCH_SINGLE_SOURCE doesn't help, so * let's release them one by one. */ for (curNbl = netBufferLists; curNbl != NULL; curNbl = nextNbl) { nextNbl = NET_BUFFER_LIST_NEXT_NBL(curNbl); NET_BUFFER_LIST_NEXT_NBL(curNbl) = NULL; NdisFSendNetBufferListsComplete(ndisHandle, curNbl, sendCompleteFlags); } } } static __inline VOID OvsStartNBLIngressError(POVS_SWITCH_CONTEXT switchContext, PNET_BUFFER_LIST nblList, ULONG sendCompleteFlags, PNDIS_STRING filterReason, NDIS_STATUS error) { ASSERT(error); OvsReportNBLIngressError(switchContext, nblList, filterReason, error); OvsCompleteUpperLayerNBL(switchContext->NdisFilterHandle, nblList, sendCompleteFlags, FALSE); } static VOID OvsAppendNativeForwardedPacket(POVS_SWITCH_CONTEXT switchContext, PNET_BUFFER_LIST curNbl, PNET_BUFFER_LIST *nativeNbls, ULONG flags, BOOLEAN isRecv) { POVS_BUFFER_CONTEXT ctx = { 0 }; NDIS_STRING filterReason; *nativeNbls = curNbl; ctx = OvsInitExternalNBLContext(switchContext, curNbl, isRecv); if (ctx == NULL) { RtlInitUnicodeString(&filterReason, L"Cannot allocate native NBL context."); OvsStartNBLIngressError(switchContext, curNbl, flags, &filterReason, NDIS_STATUS_RESOURCES); } } static VOID OvsStartNBLIngress(POVS_SWITCH_CONTEXT switchContext, PNET_BUFFER_LIST netBufferLists, ULONG SendFlags) { NDIS_SWITCH_PORT_ID sourcePort = 0; NDIS_SWITCH_NIC_INDEX sourceIndex = 0; PNDIS_SWITCH_FORWARDING_DETAIL_NET_BUFFER_LIST_INFO fwdDetail; PNET_BUFFER_LIST curNbl = NULL, nextNbl = NULL, lastNbl = NULL; ULONG sendCompleteFlags; UCHAR dispatch; LOCK_STATE_EX lockState, dpLockState; NDIS_STATUS status; NDIS_STRING filterReason; LIST_ENTRY missedPackets; UINT32 num = 0; OvsCompletionList completionList; #if (NDIS_SUPPORT_NDIS640) PNET_BUFFER_LIST nativeForwardedNbls = NULL; PNET_BUFFER_LIST *nextNativeForwardedNbl = &nativeForwardedNbls; #endif dispatch = NDIS_TEST_SEND_AT_DISPATCH_LEVEL(SendFlags)? NDIS_RWL_AT_DISPATCH_LEVEL : 0; sendCompleteFlags = OvsGetSendCompleteFlags(SendFlags); SendFlags |= NDIS_SEND_FLAGS_SWITCH_DESTINATION_GROUP; InitializeListHead(&missedPackets); OvsInitCompletionList(&completionList, switchContext, sendCompleteFlags); for (curNbl = netBufferLists; curNbl != NULL; curNbl = nextNbl) { POVS_VPORT_ENTRY vport = NULL; UINT32 portNo = 0; OVS_DATAPATH *datapath = &switchContext->datapath; OVS_PACKET_HDR_INFO layers = { 0 }; OvsFlowKey key = { 0 }; UINT64 hash = 0; PNET_BUFFER curNb = NULL; POVS_BUFFER_CONTEXT ctx = NULL; nextNbl = curNbl->Next; curNbl->Next = NULL; fwdDetail = NET_BUFFER_LIST_SWITCH_FORWARDING_DETAIL(curNbl); sourcePort = fwdDetail->SourcePortId; sourceIndex = (NDIS_SWITCH_NIC_INDEX)fwdDetail->SourceNicIndex; #if (NDIS_SUPPORT_NDIS640) if (fwdDetail->NativeForwardingRequired) { /* Add current NBL to those that require native forwarding. */ OvsAppendNativeForwardedPacket( switchContext, curNbl, nextNativeForwardedNbl, sendCompleteFlags, OvsIsExternalVportByPortId(switchContext, sourcePort)); continue; } #endif /* NDIS_SUPPORT_NDIS640 */ ctx = OvsInitExternalNBLContext(switchContext, curNbl, OvsIsExternalVportByPortId(switchContext, sourcePort)); if (ctx == NULL) { RtlInitUnicodeString(&filterReason, L"Cannot allocate external NBL context."); OvsStartNBLIngressError(switchContext, curNbl, sendCompleteFlags, &filterReason, NDIS_STATUS_RESOURCES); continue; } /* Ethernet Header is a guaranteed safe access. */ curNb = NET_BUFFER_LIST_FIRST_NB(curNbl); if (curNb->Next != NULL) { /* Create a NET_BUFFER_LIST for each NET_BUFFER. */ status = OvsCreateNewNBLsFromMultipleNBs(switchContext, &curNbl, &lastNbl); if (!NT_SUCCESS(status)) { RtlInitUnicodeString(&filterReason, L"Cannot allocate NBLs with single NB."); OvsAddPktCompletionList(&completionList, TRUE, sourcePort, curNbl, 0, &filterReason); continue; } lastNbl->Next = nextNbl; nextNbl = curNbl->Next; curNbl->Next = NULL; } { OvsFlow *flow; /* Take the DispatchLock so none of the VPORTs disconnect while * we are setting destination ports. * * XXX: acquire/release the dispatch lock for a "batch" of packets * rather than for each packet. */ NdisAcquireRWLockRead(switchContext->dispatchLock, &lockState, dispatch); vport = OvsFindVportByPortIdAndNicIndex(switchContext, sourcePort, sourceIndex); if (vport == NULL || vport->ovsState != OVS_STATE_CONNECTED) { RtlInitUnicodeString(&filterReason, L"OVS-Cannot forward packet from unknown source port"); goto dropit; } else { portNo = vport->portNo; } vport->stats.rxPackets++; vport->stats.rxBytes += NET_BUFFER_DATA_LENGTH(curNb); status = OvsExtractFlow(curNbl, vport->portNo, &key, &layers, NULL); if (status != NDIS_STATUS_SUCCESS) { RtlInitUnicodeString(&filterReason, L"OVS-Flow extract failed"); goto dropit; } ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); OvsAcquireDatapathRead(datapath, &dpLockState, TRUE); flow = OvsLookupFlow(datapath, &key, &hash, FALSE); if (flow) { OvsFlowUsed(flow, curNbl, &layers); datapath->hits++; /* If successful, OvsActionsExecute() consumes the NBL. * Otherwise, it adds it to the completionList. No need to * check the return value. */ OvsActionsExecute(switchContext, &completionList, curNbl, portNo, SendFlags, &key, &hash, &layers, flow->actions, flow->actionsLen); OvsReleaseDatapath(datapath, &dpLockState); NdisReleaseRWLock(switchContext->dispatchLock, &lockState); continue; } else { OvsReleaseDatapath(datapath, &dpLockState); datapath->misses++; status = OvsCreateAndAddPackets(NULL, 0, OVS_PACKET_CMD_MISS, vport, &key, curNbl, OvsIsExternalVportByPortId(switchContext, sourcePort), &layers, switchContext, &missedPackets, &num); if (status == NDIS_STATUS_SUCCESS) { /* Complete the packet since it was copied to user * buffer. */ RtlInitUnicodeString(&filterReason, L"OVS-Dropped since packet was copied to userspace"); } else { RtlInitUnicodeString(&filterReason, L"OVS-Dropped due to failure to queue to userspace"); } goto dropit; } dropit: OvsAddPktCompletionList(&completionList, TRUE, sourcePort, curNbl, 0, &filterReason); NdisReleaseRWLock(switchContext->dispatchLock, &lockState); } } #if (NDIS_SUPPORT_NDIS640) if (nativeForwardedNbls) { /* This is NVGRE encapsulated traffic and is forwarded to NDIS * in order to be handled by the HNV module. */ OvsSendNBLIngress(switchContext, nativeForwardedNbls, SendFlags); } #endif /* NDIS_SUPPORT_NDIS640 */ /* Queue the missed packets. */ OvsQueuePackets(&missedPackets, num); OvsFinalizeCompletionList(&completionList); } /* * -------------------------------------------------------------------------- * Implements filter driver's FilterSendNetBufferLists Function. * -------------------------------------------------------------------------- */ VOID OvsExtSendNBL(NDIS_HANDLE filterModuleContext, PNET_BUFFER_LIST netBufferLists, NDIS_PORT_NUMBER portNumber, ULONG sendFlags) { UNREFERENCED_PARAMETER(portNumber); /* 'filterModuleContext' is the switch context that gets created in the * AttachHandler. */ POVS_SWITCH_CONTEXT switchContext; switchContext = (POVS_SWITCH_CONTEXT) filterModuleContext; if (switchContext->dataFlowState == OvsSwitchPaused) { NDIS_STRING filterReason; ULONG sendCompleteFlags = OvsGetSendCompleteFlags(sendFlags); RtlInitUnicodeString(&filterReason, L"Switch state PAUSED, drop on ingress."); OvsStartNBLIngressError(switchContext, netBufferLists, sendCompleteFlags, &filterReason, NDIS_STATUS_PAUSED); return; } ASSERT(switchContext->dataFlowState == OvsSwitchRunning); OvsStartNBLIngress(switchContext, netBufferLists, sendFlags); } static VOID OvsCompleteNBLIngress(POVS_SWITCH_CONTEXT switchContext, PNET_BUFFER_LIST netBufferLists, ULONG sendCompleteFlags, BOOLEAN isSendComplete) { PNET_BUFFER_LIST curNbl = NULL, nextNbl = NULL; OvsCompletionList newList; newList.dropNbl = NULL; newList.dropNblNext = &newList.dropNbl; for (curNbl = netBufferLists; curNbl != NULL; curNbl = nextNbl) { nextNbl = curNbl->Next; curNbl->Next = NULL; curNbl = OvsCompleteNBL(switchContext, curNbl, TRUE); if (curNbl != NULL) { /* NBL originated from the upper layer. */ *newList.dropNblNext = curNbl; newList.dropNblNext = &curNbl->Next; } } /* Complete the NBL's that were sent by the upper layer. */ if (newList.dropNbl != NULL) { OvsCompleteUpperLayerNBL(switchContext->NdisFilterHandle, newList.dropNbl, sendCompleteFlags, isSendComplete); } } /* * -------------------------------------------------------------------------- * Implements filter driver's FilterSendNetBufferListsComplete function. * -------------------------------------------------------------------------- */ VOID OvsExtSendNBLComplete(NDIS_HANDLE filterModuleContext, PNET_BUFFER_LIST netBufferLists, ULONG sendCompleteFlags) { OvsCompleteNBLIngress((POVS_SWITCH_CONTEXT)filterModuleContext, netBufferLists, sendCompleteFlags, TRUE); } VOID OvsFinalizeCompletionList(OvsCompletionList *completionList) { if (completionList->dropNbl != NULL) { OvsCompleteNBLIngress(completionList->switchContext, completionList->dropNbl, completionList->sendCompleteFlags, FALSE); completionList->dropNbl = NULL; completionList->dropNblNext = &completionList->dropNbl; } } /* * -------------------------------------------------------------------------- * Implements filter driver's FilterCancelSendNetBufferLists function. * * "If a filter driver specifies a FilterSendNetBufferLists function and it * queues send requests, it must also specify a * FilterCancelSendNetBufferLists function." * * http://msdn.microsoft.com/en-us/library/windows/hardware/ * ff549966(v=vs.85).aspx * -------------------------------------------------------------------------- */ VOID OvsExtCancelSendNBL(NDIS_HANDLE filterModuleContext, PVOID CancelId) { UNREFERENCED_PARAMETER(filterModuleContext); UNREFERENCED_PARAMETER(CancelId); /* All send requests get completed synchronously, so there is no need to * implement this callback. */ }