VMM: NEM kick off.

git-svn-id: https://www.virtualbox.org/svn/vbox/trunk@70918 cfe28804-0f27-0410-a406-dd0f0b0b656f

21 files changed, 568 insertions, 39 deletions

diff --git a/Makefile.kmk b/Makefile.kmk
index 24c7cb9bd74..a515cce8b99 100644
--- a/Makefile.kmk
+++ b/Makefile.kmk
@@ -766,6 +766,7 @@ VBOX_CORE_DOXYFILE_INPUT_FIRST =\
 	$(PATH_ROOT)/include/VBox/vmm/hm_svm.h \
 	$(PATH_ROOT)/include/VBox/vmm/hm_vmx.h \
 	$(PATH_ROOT)/include/VBox/vmm/iem.h \
+	$(PATH_ROOT)/include/VBox/vmm/nem.h \
 	$(PATH_ROOT)/include/VBox/vmm/pdm.h \
 	$(PATH_ROOT)/include/VBox/vmm/rem.h \
 	$(PATH_ROOT)/include/VBox/vmm/iom.h \
@@ -783,6 +784,7 @@ VBOX_CORE_DOXYFILE_INPUT_FIRST =\
 	$(PATH_ROOT)/src/VBox/VMM/include/IEMInternal.h \
 	$(PATH_ROOT)/src/VBox/VMM/include/IOMInternal.h \
 	$(PATH_ROOT)/src/VBox/VMM/include/MMInternal.h \
+	$(PATH_ROOT)/src/VBox/VMM/include/NEMInternal.h \
 	$(PATH_ROOT)/src/VBox/VMM/include/PDMInternal.h \
 	$(PATH_ROOT)/src/VBox/VMM/include/PGMInternal.h \
 	$(PATH_ROOT)/src/VBox/VMM/include/GIMInternal.h \
diff --git a/include/VBox/log.h b/include/VBox/log.h
index 34a9fb90ae9..51a76aac8f0 100644
--- a/include/VBox/log.h
+++ b/include/VBox/log.h
@@ -679,6 +679,8 @@ typedef enum LOGGROUP
     LOG_GROUP_MM_POOL,
     /** The NAT service group */
     LOG_GROUP_NAT_SERVICE,
+    /** NEM group. */
+    LOG_GROUP_NEM,
     /** The network adaptor driver group. */
     LOG_GROUP_NET_ADP_DRV,
     /** The network filter driver group. */
@@ -1140,6 +1142,7 @@ typedef enum LOGGROUP
     "MM_PHYS",      \
     "MM_POOL",      \
     "NAT_SERVICE",  \
+    "NEM",          \
     "NET_ADP_DRV",  \
     "NET_FLT_DRV",  \
     "NET_SERVICE",  \
diff --git a/include/VBox/sup.h b/include/VBox/sup.h
index c3104424a35..b23ec3c42a6 100644
--- a/include/VBox/sup.h
+++ b/include/VBox/sup.h
@@ -1642,8 +1642,9 @@ SUPR3DECL(int) SUPR3HardenedLdrLoadPlugIn(const char *pszFilename, PRTLDRMOD phL
  * Check if the host kernel can run in VMX root mode.
  *
  * @returns VINF_SUCCESS if supported, error code indicating why if not.
+ * @param   ppszWhy         Where to return an explanatory message on failure.
  */
-SUPR3DECL(int) SUPR3QueryVTxSupported(void);
+SUPR3DECL(int) SUPR3QueryVTxSupported(const char **ppszWhy);
 
 /**
  * Return VT-x/AMD-V capabilities.
diff --git a/include/VBox/vmm/nem.h b/include/VBox/vmm/nem.h
new file mode 100644
index 00000000000..5fecf1ff842
--- /dev/null
+++ b/include/VBox/vmm/nem.h
@@ -0,0 +1,58 @@
+/** @file
+ * NEM - The Native Execution Manager.
+ */
+
+/*
+ * Copyright (C) 2018 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ *
+ * The contents of this file may alternatively be used under the terms
+ * of the Common Development and Distribution License Version 1.0
+ * (CDDL) only, as it comes in the "COPYING.CDDL" file of the
+ * VirtualBox OSE distribution, in which case the provisions of the
+ * CDDL are applicable instead of those of the GPL.
+ *
+ * You may elect to license modified versions of this file under the
+ * terms and conditions of either the GPL or the CDDL or both.
+ */
+
+#ifndef ___VBox_vmm_nem_h
+#define ___VBox_vmm_nem_h
+
+#include <VBox/types.h>
+#include <VBox/vmm/vmapi.h>
+
+
+RT_C_DECLS_BEGIN
+
+/** @defgroup grp_nem      The Native Execution Manager API
+ * @ingroup grp_vmm
+ * @{
+ */
+
+/** @defgroup grp_hm_r3    The HM ring-3 Context API
+ * @{
+ */
+VMMR3_INT_DECL(int)  NEMR3InitConfig(PVM pVM);
+VMMR3_INT_DECL(int)  NEMR3Init(PVM pVM, bool fFallback, bool fHMForced);
+#ifdef IN_RING3
+VMMR3_INT_DECL(int)  NEMR3InitCompleted(PVM pVM, VMINITCOMPLETED enmWhat);
+#endif
+VMMR3_INT_DECL(int)  NEMR3Term(PVM pVM);
+VMMR3_INT_DECL(void) NEMR3Reset(PVM pVM);
+VMMR3_INT_DECL(void) NEMR3ResetCpu(PVMCPU pVCpu);
+/** @} */
+
+/** @} */
+RT_C_DECLS_END
+
+
+#endif
+
diff --git a/include/VBox/vmm/vm.h b/include/VBox/vmm/vm.h
index c9846b57ad2..43cc6e82eab 100644
--- a/include/VBox/vmm/vm.h
+++ b/include/VBox/vmm/vm.h
@@ -167,6 +167,15 @@ typedef struct VMCPU
         uint8_t             padding[1408];      /* multiple of 64 */
     } em;
 
+    /** NEM part. */
+    union VMCPUUNIONNEM
+    {
+#ifdef ___NEMInternal_h
+        struct NEMCPU       s;
+#endif
+        uint8_t             padding[128];       /* multiple of 64 */
+    } nem;
+
     /** TRPM part. */
     union VMCPUUNIONTRPM
     {
@@ -253,7 +262,7 @@ typedef struct VMCPU
     STAMPROFILEADV          aStatAdHoc[8];                          /* size: 40*8 = 320 */
 
     /** Align the following members on page boundary. */
-    uint8_t                 abAlignment2[2104];
+    uint8_t                 abAlignment2[1976];
 
     /** PGM part. */
     union VMCPUUNIONPGM
@@ -1185,6 +1194,15 @@ typedef struct VM
         uint8_t     padding[256];       /* multiple of 64 */
     } em;
 
+    /** NEM part. */
+    union
+    {
+#ifdef ___NEMInternal_h
+        struct NEM  s;
+#endif
+        uint8_t     padding[128];       /* multiple of 64 */
+    } nem;
+
     /** TM part. */
     union
     {
@@ -1317,13 +1335,13 @@ typedef struct VM
 
     /** Padding for aligning the cpu array on a page boundary. */
 #if defined(VBOX_WITH_REM) && defined(VBOX_WITH_RAW_MODE)
-    uint8_t         abAlignment2[3870];
+    uint8_t         abAlignment2[3742];
 #elif defined(VBOX_WITH_REM) && !defined(VBOX_WITH_RAW_MODE)
-    uint8_t         abAlignment2[1630];
+    uint8_t         abAlignment2[1502];
 #elif !defined(VBOX_WITH_REM) && defined(VBOX_WITH_RAW_MODE)
-    uint8_t         abAlignment2[30];
+    uint8_t         abAlignment2[3998];
 #else
-    uint8_t         abAlignment2[1886];
+    uint8_t         abAlignment2[1758];
 #endif
 
     /* ---- end small stuff ---- */
diff --git a/include/VBox/vmm/vm.mac b/include/VBox/vmm/vm.mac
index 939dedc4c56..e492f41044a 100644
--- a/include/VBox/vmm/vm.mac
+++ b/include/VBox/vmm/vm.mac
@@ -63,6 +63,7 @@ struc VMCPU
     .iem                    resb 18496
     .hm                     resb 5824
     .em                     resb 1408
+    .nem                    resb 128
     .trpm                   resb 128
     .tm                     resb 384
     .vmm                    resb 704
@@ -160,6 +161,7 @@ struc VM
     .pdm                    resb 1920
     .iom                    resb 896
     .em                     resb 256
+    .nem                    resb 128
     .tm                     resb 2496
     .dbgf                   resb 2368
     .ssm                    resb 128
diff --git a/src/VBox/HostDrivers/Support/SUPLib.cpp b/src/VBox/HostDrivers/Support/SUPLib.cpp
index 9efb3372f9a..334c30e63a4 100644
--- a/src/VBox/HostDrivers/Support/SUPLib.cpp
+++ b/src/VBox/HostDrivers/Support/SUPLib.cpp
@@ -1658,10 +1658,11 @@ SUPR3DECL(int) SUPR3GipGetPhys(PRTHCPHYS pHCPhys)
 }
 
 
-SUPR3DECL(int) SUPR3QueryVTxSupported(void)
+SUPR3DECL(int) SUPR3QueryVTxSupported(const char **ppszWhy)
 {
+    *ppszWhy = NULL;
 #ifdef RT_OS_LINUX
-    return suplibOsQueryVTxSupported();
+    return suplibOsQueryVTxSupported(ppszWhy);
 #else
     return VINF_SUCCESS;
 #endif
diff --git a/src/VBox/HostDrivers/Support/SUPLibInternal.h b/src/VBox/HostDrivers/Support/SUPLibInternal.h
index cad1b9b41d7..9ca66a4994c 100644
--- a/src/VBox/HostDrivers/Support/SUPLibInternal.h
+++ b/src/VBox/HostDrivers/Support/SUPLibInternal.h
@@ -360,7 +360,7 @@ int     suplibOsIOCtl(PSUPLIBDATA pThis, uintptr_t uFunction, void *pvReq, size_
 int     suplibOsIOCtlFast(PSUPLIBDATA pThis, uintptr_t uFunction, uintptr_t idCpu);
 int     suplibOsPageAlloc(PSUPLIBDATA pThis, size_t cPages, void **ppvPages);
 int     suplibOsPageFree(PSUPLIBDATA pThis, void *pvPages, size_t cPages);
-int     suplibOsQueryVTxSupported(void);
+int     suplibOsQueryVTxSupported(const char **ppszWhy);
 
 
 /**
diff --git a/src/VBox/HostDrivers/Support/linux/SUPLib-linux.cpp b/src/VBox/HostDrivers/Support/linux/SUPLib-linux.cpp
index 208d3db5601..e0e352f9e34 100644
--- a/src/VBox/HostDrivers/Support/linux/SUPLib-linux.cpp
+++ b/src/VBox/HostDrivers/Support/linux/SUPLib-linux.cpp
@@ -259,16 +259,19 @@ int suplibOsPageFree(PSUPLIBDATA pThis, void *pvPages, size_t cPages)
 }
 
 
-/** Check if the host kernel supports VT-x or not.
+/**
+ * Check if the host kernel supports VT-x or not.
  *
  * Older Linux kernels clear the VMXE bit in the CR4 register (function
  * tlb_flush_all()) leading to a host kernel panic.
+ *
+ * @returns VBox status code (no info).
+ * @param   ppszWhy         Where to return explanatory message.
  */
-int suplibOsQueryVTxSupported(void)
+int suplibOsQueryVTxSupported(const char **ppszWhy)
 {
     char szBuf[256];
     int rc = RTSystemQueryOSInfo(RTSYSOSINFO_RELEASE, szBuf, sizeof(szBuf));
-
     if (RT_SUCCESS(rc))
     {
         char *pszNext;
@@ -298,6 +301,7 @@ int suplibOsQueryVTxSupported(void)
         }
     }
 
+    *ppszWhy = "Linux 2.6.13 or newer required!";
     return VERR_SUPDRV_KERNEL_TOO_OLD_FOR_VTX;
 }
 
diff --git a/src/VBox/Main/src-server/HostImpl.cpp b/src/VBox/Main/src-server/HostImpl.cpp
index ebd1e391b33..d081eb13a2c 100644
--- a/src/VBox/Main/src-server/HostImpl.cpp
+++ b/src/VBox/Main/src-server/HostImpl.cpp
@@ -334,7 +334,8 @@ HRESULT Host::init(VirtualBox *aParent)
                      && (fFeaturesEdx & X86_CPUID_FEATURE_EDX_FXSR)
                    )
                 {
-                    int rc = SUPR3QueryVTxSupported();
+                    const char *pszIgn;
+                    int rc = SUPR3QueryVTxSupported(&pszIgn);
                     if (RT_SUCCESS(rc))
                         m->fVTSupported = true;
                 }
diff --git a/src/VBox/VMM/Makefile.kmk b/src/VBox/VMM/Makefile.kmk
index 5948a3f1bd4..d4fd4f1279d 100644
--- a/src/VBox/VMM/Makefile.kmk
+++ b/src/VBox/VMM/Makefile.kmk
@@ -185,6 +185,7 @@ VBoxVMM_SOURCES  = \
 	VMMR3/MMHyper.cpp \
 	VMMR3/MMPagePool.cpp \
 	VMMR3/MMUkHeap.cpp \
+	VMMR3/NEMR3.cpp \
 	VMMR3/PDM.cpp \
 	VMMR3/PDMBlkCache.cpp \
 	VMMR3/PDMDevice.cpp \
@@ -306,6 +307,9 @@ VBoxVMM_SOURCES.x86 += \
 VBoxVMM_SOURCES.amd64 += \
 	VMMSwitcher/AMD64Stub.asm
 
+VBoxVMM_SOURCES.win += VMMR3/NEMR3Native-win.cpp
+VBoxVMM_DEFS.win    += VBOX_WITH_NATIVE_NEM
+
 VBoxVMM_LIBS = \
 	$(PATH_STAGE_LIB)/DisasmR3$(VBOX_SUFF_LIB)
 ifdef VBOX_WITH_DEBUGGER
diff --git a/src/VBox/VMM/VMMR3/HM.cpp b/src/VBox/VMM/VMMR3/HM.cpp
index 81336af4db0..d482f1c26ec 100644
--- a/src/VBox/VMM/VMMR3/HM.cpp
+++ b/src/VBox/VMM/VMMR3/HM.cpp
@@ -50,6 +50,7 @@
 #include <VBox/vmm/patm.h>
 #include <VBox/vmm/csam.h>
 #include <VBox/vmm/selm.h>
+#include <VBox/vmm/nem.h>
 #ifdef VBOX_WITH_REM
 # include <VBox/vmm/rem.h>
 #endif
@@ -393,10 +394,16 @@ static int                hmR3TermCPU(PVM pVM);
 /**
  * Initializes the HM.
  *
- * This reads the config and check whether VT-x or AMD-V hardware is available
- * if configured to use it.  This is one of the very first components to be
- * initialized after CFGM, so that we can fall back to raw-mode early in the
- * initialization process.
+ * This is the very first component to really do init after CFGM so that we can
+ * establish the predominat execution engine for the VM prior to initializing
+ * other modules.  It takes care of NEM initialization if needed (HM disabled or
+ * not available in HW).
+ *
+ * If VT-x or AMD-V hardware isn't available, HM will try fall back on a native
+ * hypervisor API via NEM, and then back on raw-mode if that isn't available
+ * either.  The fallback to raw-mode will not happen if /HM/HMForced is set
+ * (like for guest using SMP or 64-bit as well as for complicated guest like OS
+ * X, OS/2 and others).
  *
  * Note that a lot of the set up work is done in ring-0 and thus postponed till
  * the ring-3 and ring-0 callback to HMR3InitCompleted.
@@ -448,6 +455,8 @@ VMMR3_INT_DECL(int) HMR3Init(PVM pVM)
      */
     rc = CFGMR3ValidateConfig(pCfgHm, "/HM/",
                               "HMForced"
+                              "|UseNEMInstead"
+                              "|FallbackToNEM"
                               "|EnableNestedPaging"
                               "|EnableUX"
                               "|EnableLargePages"
@@ -491,6 +500,23 @@ VMMR3_INT_DECL(int) HMR3Init(PVM pVM)
     fHMForced = true;
 #endif /* !VBOX_WITH_RAW_MODE */
 
+    /** @cfgm{/HM/UseNEMInstead, bool, true}
+     * Don't use HM, use NEM instead. */
+    bool fUseNEMInstead = false;
+    rc = CFGMR3QueryBoolDef(pCfgHm, "UseNEMInstead", &fUseNEMInstead, false);
+    AssertRCReturn(rc, rc);
+    if (fUseNEMInstead && pVM->fHMEnabled)
+    {
+        LogRel(("HM: Setting fHMEnabled to false because fUseNEMInstead is set.\n"));
+        pVM->fHMEnabled = false;
+    }
+
+    /** @cfgm{/HM/FallbackToNEM, bool, true}
+     * Enables fallback on NEM. */
+    bool fFallbackToNEM = true;
+    rc = CFGMR3QueryBoolDef(pCfgHm, "FallbackToNEM", &fFallbackToNEM, true);
+    AssertRCReturn(rc, rc);
+
     /** @cfgm{/HM/EnableNestedPaging, bool, false}
      * Enables nested paging (aka extended page tables). */
     rc = CFGMR3QueryBoolDef(pCfgHm, "EnableNestedPaging", &pVM->hm.s.fAllowNestedPaging, false);
@@ -642,7 +668,8 @@ VMMR3_INT_DECL(int) HMR3Init(PVM pVM)
             }
             else if (fCaps & SUPVTCAPS_VT_X)
             {
-                rc = SUPR3QueryVTxSupported();
+                const char *pszWhy;
+                rc = SUPR3QueryVTxSupported(&pszWhy);
                 if (RT_SUCCESS(rc))
                 {
                     LogRel(("HM: HMR3Init: VT-x%s%s%s\n",
@@ -653,17 +680,26 @@ VMMR3_INT_DECL(int) HMR3Init(PVM pVM)
                 }
                 else
                 {
-#ifdef RT_OS_LINUX
-                    const char *pszMinReq = " Linux 2.6.13 or newer required!";
-#else
-                    const char *pszMinReq = "";
-#endif
-                    if (fHMForced)
-                        return VMSetError(pVM, rc, RT_SRC_POS, "The host kernel does not support VT-x.%s\n", pszMinReq);
-
-                    /* Fall back to raw-mode. */
-                    LogRel(("HM: HMR3Init: Falling back to raw-mode: The host kernel does not support VT-x.%s\n", pszMinReq));
+                    /*
+                     * Before failing, try fallback to NEM if we're allowed to do that.
+                     */
                     pVM->fHMEnabled = false;
+                    if (fFallbackToNEM)
+                    {
+                        LogRel(("HM: HMR3Init: Attempting fall back to NEM: The host kernel does not support VT-x - %s\n", pszWhy));
+                        int rc2 = NEMR3Init(pVM, true /*fFallback*/, fHMForced);
+                        if (   RT_SUCCESS(rc2)
+                            && pVM->fNEMActive)
+                            rc = VINF_SUCCESS;
+                    }
+                    if (RT_FAILURE(rc))
+                    {
+                        if (fHMForced)
+                            return VMSetError(pVM, rc, RT_SRC_POS, "The host kernel does not support VT-x: %s\n", pszWhy);
+
+                        /* Fall back to raw-mode. */
+                        LogRel(("HM: HMR3Init: Falling back to raw-mode: The host kernel does not support VT-x - %s\n", pszWhy));
+                    }
                 }
             }
             else
@@ -737,19 +773,39 @@ VMMR3_INT_DECL(int) HMR3Init(PVM pVM)
                     break;
 
                 default:
-                    pszMsg = NULL;
-                    break;
+                    return VMSetError(pVM, rc, RT_SRC_POS, "SUPR3QueryVTCaps failed with %Rrc", rc);
             }
-            if (fHMForced && pszMsg)
-                return VM_SET_ERROR(pVM, rc, pszMsg);
-            if (!pszMsg)
-                return VMSetError(pVM, rc, RT_SRC_POS, "SUPR3QueryVTCaps failed with %Rrc", rc);
 
-            /* Fall back to raw-mode. */
-            LogRel(("HM: HMR3Init: Falling back to raw-mode: %s\n", pszMsg));
+            /*
+             * Before failing, try fallback to NEM if we're allowed to do that.
+             */
             pVM->fHMEnabled = false;
+            if (fFallbackToNEM)
+            {
+                LogRel(("HM: HMR3Init: Attempting fall back to NEM: %s\n", pszMsg));
+                int rc2 = NEMR3Init(pVM, true /*fFallback*/, fHMForced);
+                if (   RT_SUCCESS(rc2)
+                    && pVM->fNEMActive)
+                    rc = VINF_SUCCESS;
+            }
+            if (RT_FAILURE(rc))
+            {
+                if (fHMForced)
+                    return VM_SET_ERROR(pVM, rc, pszMsg);
+
+                LogRel(("HM: HMR3Init: Falling back to raw-mode: %s\n", pszMsg));
+            }
         }
     }
+    /*
+     * If NEM is supposed to be used instead, initialize it instead.
+     */
+    else if (fUseNEMInstead)
+    {
+        rc = NEMR3Init(pVM, false /*fFallback*/, fHMForced);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
 
     /* It's now OK to use the predicate function. */
     pVM->fHMEnabledFixed = true;
diff --git a/src/VBox/VMM/VMMR3/NEMR3.cpp b/src/VBox/VMM/VMMR3/NEMR3.cpp
new file mode 100644
index 00000000000..201973a3a2c
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/NEMR3.cpp
@@ -0,0 +1,211 @@
+/* $Id$ */
+/** @file
+ * NEM - Native execution manager.
+ */
+
+/*
+ * Copyright (C) 2018 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/** @page pg_nem NEM - Native Execution Manager.
+ *
+ * Later.
+ *
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_NEM
+#include <VBox/vmm/nem.h>
+#include "NEMInternal.h"
+#include <VBox/vmm/vm.h>
+
+
+
+/**
+ * Basic init and configuration reading.
+ *
+ * Always call NEMR3Term after calling this.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ */
+VMMR3_INT_DECL(int) NEMR3InitConfig(PVM pVM)
+{
+    LogFlow(("NEMR3Init\n"));
+
+    /*
+     * Assert alignment and sizes.
+     */
+    AssertCompileMemberAlignment(VM, nem.s, 64);
+    AssertCompile(sizeof(pVM->nem.s) <= sizeof(pVM->nem.padding));
+
+    /*
+     * Initialize state info so NEMR3Term will always be happy.
+     * No returning prior to setting magics!
+     */
+    pVM->nem.s.u32Magic = NEM_MAGIC;
+    for (VMCPUID iCpu = 0; iCpu < pVM->cCpus; iCpu++)
+        pVM->aCpus[iCpu].nem.s.u32Magic = NEMCPU_MAGIC;
+
+    /*
+     * Read configuration.
+     */
+    PCFGMNODE pCfgNem = CFGMR3GetChild(CFGMR3GetRoot(pVM), "NEM/");
+
+    /*
+     * Validate the NEM settings.
+     */
+    int rc = CFGMR3ValidateConfig(pCfgNem,
+                                  "/NEM/",
+                                  "Enabled",
+                                  "" /* pszValidNodes */, "NEM" /* pszWho */, 0 /* uInstance */);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /** @cfgm{/NEM/NEMEnabled, bool, true}
+     * Whether NEM is enabled. */
+    rc = CFGMR3QueryBoolDef(pCfgNem, "Enabled", &pVM->nem.s.fEnabled, true);
+    AssertLogRelRCReturn(rc, rc);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * This is called by HMR3Init() when HM cannot be used.
+ *
+ * Sets VM::fNEMActive if we can use a native hypervisor API to execute the VM.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   fFallback   Whether this is a fallback call.  Cleared if the VM is
+ *                      configured to use NEM instead of HM.
+ * @param   fForced     Whether /HM/HMForced was set.  If set and we fail to
+ *                      enable NEM, we'll return a failure status code.
+ *                      Otherwise we'll assume HMR3Init falls back on raw-mode.
+ */
+VMMR3_INT_DECL(int) NEMR3Init(PVM pVM, bool fFallback, bool fForced)
+{
+    Assert(!pVM->fNEMActive);
+    int rc;
+    if (pVM->nem.s.fEnabled)
+    {
+#ifdef VBOX_WITH_NATIVE_NEM
+        rc = nemR3NativeInit(pVM, fFallback, fForced);
+#else
+        RT_NOREF(fFallback);
+        rc = VINF_SUCCESS;
+#endif
+        if (RT_SUCCESS(rc))
+        {
+            if (pVM->fNEMActive)
+                LogRel(("NEM: NEMR3Init: Active.\n"));
+            else
+            {
+                LogRel(("NEM: NEMR3Init: Not available.\n"));
+                if (fForced)
+                    rc = VERR_NEM_NOT_AVAILABLE;
+            }
+        }
+        else
+            LogRel(("NEM: NEMR3Init: Native init failed: %Rrc.\n", rc));
+    }
+    else
+    {
+        LogRel(("NEM: NEMR3Init: Disabled.\n"));
+        rc = fForced ? VERR_NEM_NOT_ENABLED : VINF_SUCCESS;
+    }
+    return rc;
+}
+
+
+/**
+ * Called when a init phase has completed.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   enmWhat             The phase that completed.
+ */
+VMMR3_INT_DECL(int) NEMR3InitCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
+{
+    int rc = VINF_SUCCESS;
+#ifdef VBOX_WITH_NATIVE_NEM
+    if (pVM->fNEMActive)
+        rc = nemR3NativeInitCompleted(pVM, enmWhat);
+#else
+    RT_NOREF(pVM, enmWhat);
+#endif
+    return rc;
+}
+
+
+/**
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ */
+VMMR3_INT_DECL(int) NEMR3Term(PVM pVM)
+{
+    AssertReturn(pVM->nem.s.u32Magic == NEM_MAGIC, VERR_WRONG_ORDER);
+    for (VMCPUID iCpu = 0; iCpu < pVM->cCpus; iCpu++)
+        AssertReturn(pVM->aCpus[iCpu].nem.s.u32Magic == NEMCPU_MAGIC, VERR_WRONG_ORDER);
+
+    /* Do native termination. */
+    int rc = VINF_SUCCESS;
+#ifdef VBOX_WITH_NATIVE_NEM
+    if (pVM->fNEMActive)
+        rc = nemR3NativeTerm(pVM);
+#endif
+
+    /* Mark it as terminated. */
+    for (VMCPUID iCpu = 0; iCpu < pVM->cCpus; iCpu++)
+        pVM->aCpus[iCpu].nem.s.u32Magic = NEMCPU_MAGIC_DEAD;
+    pVM->nem.s.u32Magic = NEM_MAGIC_DEAD;
+    return rc;
+}
+
+
+/**
+ * The VM is being reset.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+VMMR3_INT_DECL(void) NEMR3Reset(PVM pVM)
+{
+#ifdef VBOX_WITH_NATIVE_NEM
+    if (pVM->fNEMActive)
+        nemR3NativeReset(pVM);
+#else
+    RT_NOREF(pVM);
+#endif
+}
+
+
+/**
+ * Resets a virtual CPU.
+ *
+ * Used to bring up secondary CPUs on SMP as well as CPU hot plugging.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure to reset.
+ */
+VMMR3_INT_DECL(void) NEMR3ResetCpu(PVMCPU pVCpu)
+{
+#ifdef VBOX_WITH_NATIVE_NEM
+    if (pVCpu->pVMR3->fNEMActive)
+        nemR3NativeResetCpu(pVCpu);
+#else
+    RT_NOREF(pVCpu);
+#endif
+}
+
diff --git a/src/VBox/VMM/VMMR3/NEMR3Native-win.cpp b/src/VBox/VMM/VMMR3/NEMR3Native-win.cpp
new file mode 100644
index 00000000000..da6bfe27fda
--- /dev/null
+++ b/src/VBox/VMM/VMMR3/NEMR3Native-win.cpp
@@ -0,0 +1,60 @@
+/* $Id$ */
+/** @file
+ * NEM - Native execution manager, native ring-3 Windows backend.
+ */
+
+/*
+ * Copyright (C) 2018 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_NEM
+#include <VBox/vmm/nem.h>
+#include "NEMInternal.h"
+#include <VBox/vmm/vm.h>
+
+
+
+int nemR3NativeInit(PVM pVM, bool fFallback, bool fForced)
+{
+    NOREF(pVM); NOREF(fFallback); NOREF(fForced);
+    return VINF_SUCCESS;
+}
+
+
+int nemR3NativeInitCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
+{
+    NOREF(pVM); NOREF(enmWhat);
+    return VINF_SUCCESS;
+}
+
+
+int nemR3NativeTerm(PVM pVM)
+{
+    NOREF(pVM);
+    return VINF_SUCCESS;
+}
+
+
+void nemR3NativeReset(PVM pVM)
+{
+    NOREF(pVM);
+}
+
+
+void nemR3NativeResetCpu(PVMCPU pVCpu)
+{
+    NOREF(pVCpu);
+}
+
diff --git a/src/VBox/VMM/VMMR3/VM.cpp b/src/VBox/VMM/VMMR3/VM.cpp
index fed03a7a623..b1af3384c38 100644
--- a/src/VBox/VMM/VMMR3/VM.cpp
+++ b/src/VBox/VMM/VMMR3/VM.cpp
@@ -60,6 +60,7 @@
 #ifdef VBOX_WITH_REM
 # include <VBox/vmm/rem.h>
 #endif
+#include <VBox/vmm/nem.h>
 #include <VBox/vmm/apic.h>
 #include <VBox/vmm/tm.h>
 #include <VBox/vmm/stam.h>
@@ -924,9 +925,11 @@ static int vmR3InitRing3(PVM pVM, PUVM pUVM)
 
     /*
      * Init all R3 components, the order here might be important.
-     * HM shall be initialized first!
+     * NEM and HM shall be initialized first!
      */
-    rc = HMR3Init(pVM);
+    rc = NEMR3InitConfig(pVM);
+    if (RT_SUCCESS(rc))
+        rc = HMR3Init(pVM);
     if (RT_SUCCESS(rc))
     {
         rc = MMR3Init(pVM);
@@ -1075,7 +1078,7 @@ static int vmR3InitRing3(PVM pVM, PUVM pUVM)
         int rc2 = HMR3Term(pVM);
         AssertRC(rc2);
     }
-
+    NEMR3Term(pVM);
 
     LogFlow(("vmR3InitRing3: returns %Rrc\n", rc));
     return rc;
@@ -1168,6 +1171,8 @@ static int vmR3InitDoCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
     if (RT_SUCCESS(rc))
         rc = HMR3InitCompleted(pVM, enmWhat);
     if (RT_SUCCESS(rc))
+        rc = NEMR3InitCompleted(pVM, enmWhat);
+    if (RT_SUCCESS(rc))
         rc = PGMR3InitCompleted(pVM, enmWhat);
     if (RT_SUCCESS(rc))
         rc = CPUMR3InitCompleted(pVM, enmWhat);
@@ -2515,6 +2520,8 @@ DECLCALLBACK(int) vmR3Destroy(PVM pVM)
 #endif
         rc = HMR3Term(pVM);
         AssertRC(rc);
+        rc = NEMR3Term(pVM);
+        AssertRC(rc);
         rc = PGMR3Term(pVM);
         AssertRC(rc);
         rc = VMMR3Term(pVM); /* Terminates the ring-0 code! */
@@ -2803,6 +2810,7 @@ static DECLCALLBACK(VBOXSTRICTRC) vmR3SoftReset(PVM pVM, PVMCPU pVCpu, void *pvU
         CPUMR3Reset(pVM);               /* This must come *after* PDM (due to APIC base MSR caching). */
         EMR3Reset(pVM);
         HMR3Reset(pVM);                 /* This must come *after* PATM, CSAM, CPUM, SELM and TRPM. */
+        NEMR3Reset(pVM);
 
         /*
          * Since EMT(0) is the last to go thru here, it will advance the state.
@@ -2906,6 +2914,7 @@ static DECLCALLBACK(VBOXSTRICTRC) vmR3HardReset(PVM pVM, PVMCPU pVCpu, void *pvU
         TMR3Reset(pVM);
         EMR3Reset(pVM);
         HMR3Reset(pVM);                 /* This must come *after* PATM, CSAM, CPUM, SELM and TRPM. */
+        NEMR3Reset(pVM);
 
         /*
          * Do memory setup.
@@ -4631,6 +4640,7 @@ static DECLCALLBACK(int) vmR3HotUnplugCpu(PVM pVM, VMCPUID idCpu)
     CPUMR3ResetCpu(pVM, pVCpu);
     EMR3ResetCpu(pVCpu);
     HMR3ResetCpu(pVCpu);
+    NEMR3ResetCpu(pVCpu);
     return VINF_EM_WAIT_SIPI;
 }
 
diff --git a/src/VBox/VMM/VMMR3/VMM.cpp b/src/VBox/VMM/VMMR3/VMM.cpp
index 639f24d92e7..c8ff640c5f9 100644
--- a/src/VBox/VMM/VMMR3/VMM.cpp
+++ b/src/VBox/VMM/VMMR3/VMM.cpp
@@ -111,6 +111,7 @@
 #include <VBox/vmm/cpum.h>
 #include <VBox/vmm/gim.h>
 #include <VBox/vmm/mm.h>
+#include <VBox/vmm/nem.h>
 #include <VBox/vmm/iom.h>
 #include <VBox/vmm/trpm.h>
 #include <VBox/vmm/selm.h>
@@ -1493,6 +1494,7 @@ static DECLCALLBACK(int) vmmR3SendInitIpi(PVM pVM, VMCPUID idCpu)
     CPUMR3ResetCpu(pVM, pVCpu);
     EMR3ResetCpu(pVCpu);
     HMR3ResetCpu(pVCpu);
+    NEMR3ResetCpu(pVCpu);
 
     /* This will trickle up on the target EMT. */
     return VINF_EM_WAIT_SIPI;
diff --git a/src/VBox/VMM/include/NEMInternal.h b/src/VBox/VMM/include/NEMInternal.h
new file mode 100644
index 00000000000..fd1c7f32a4e
--- /dev/null
+++ b/src/VBox/VMM/include/NEMInternal.h
@@ -0,0 +1,88 @@
+/* $Id$ */
+/** @file
+ * NEM - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2018 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef ___NEMInternal_h
+#define ___NEMInternal_h
+
+#include <VBox/cdefs.h>
+#include <VBox/types.h>
+#include <VBox/vmm/nem.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/vmapi.h>
+
+RT_C_DECLS_BEGIN
+
+
+/** @defgroup grp_nem_int      Internal
+ * @ingroup grp_nem
+ * @internal
+ * @{
+ */
+
+/**
+ * NEM VM Instance data.
+ */
+typedef struct NEM
+{
+    /** NEM_MAGIC. */
+    uint32_t                    u32Magic;
+
+    /** Set if enabled. */
+    bool                        fEnabled;
+
+} NEM;
+/** Pointer to NEM VM instance data. */
+typedef NEM *PNEM;
+
+/** NEM::u32Magic value. */
+#define NEM_MAGIC               UINT32_C(0x004d454e)
+/** NEM::u32Magic value after termination. */
+#define NEM_MAGIC_DEAD          UINT32_C(0xdead1111)
+
+
+/**
+ * NEM VMCPU Instance data.
+ */
+typedef struct NEMCPU
+{
+    /** NEMCPU_MAGIC. */
+    uint32_t                    u32Magic;
+
+} NEMCPU;
+/** Pointer to NEM VMCPU instance data. */
+typedef NEMCPU *PNEMCPU;
+
+/** NEMCPU::u32Magic value. */
+#define NEMCPU_MAGIC            UINT32_C(0x4d454e20)
+/** NEMCPU::u32Magic value after termination. */
+#define NEMCPU_MAGIC_DEAD       UINT32_C(0xdead2222)
+
+#ifdef IN_RING3
+int     nemR3NativeInit(PVM pVM, bool fFallback, bool fForced);
+int     nemR3NativeInitCompleted(PVM pVM, VMINITCOMPLETED enmWhat);
+int     nemR3NativeTerm(PVM pVM);
+void    nemR3NativeReset(PVM pVM);
+void    nemR3NativeResetCpu(PVMCPU pVCpu);
+#endif
+
+
+/** @} */
+
+RT_C_DECLS_END
+
+#endif
+
diff --git a/src/VBox/VMM/testcase/tstVMStruct.h b/src/VBox/VMM/testcase/tstVMStruct.h
index 031833ebf87..542ebd14a91 100644
--- a/src/VBox/VMM/testcase/tstVMStruct.h
+++ b/src/VBox/VMM/testcase/tstVMStruct.h
@@ -432,6 +432,9 @@
     GEN_CHECK_OFF_DOT(MMLOOKUPHYPER, u.MMIO2.off);
     GEN_CHECK_OFF(MMLOOKUPHYPER, pszDesc);
 
+    GEN_CHECK_SIZE(NEM);
+    GEN_CHECK_SIZE(NEMCPU);
+
     GEN_CHECK_SIZE(PDM);
     GEN_CHECK_OFF(PDM, CritSect);
     GEN_CHECK_OFF(PDM, NopCritSect);
diff --git a/src/VBox/VMM/testcase/tstVMStructDTrace.cpp b/src/VBox/VMM/testcase/tstVMStructDTrace.cpp
index bf9dbbcd08d..8c7fe65e3dc 100644
--- a/src/VBox/VMM/testcase/tstVMStructDTrace.cpp
+++ b/src/VBox/VMM/testcase/tstVMStructDTrace.cpp
@@ -51,6 +51,7 @@
 #include "EMInternal.h"
 #include "IEMInternal.h"
 #include "REMInternal.h"
+#include "NEMInternal.h"
 #ifdef VBOX_WITH_RAW_MODE
 # include "CSAMInternal.h"
 # include "PATMInternal.h"
diff --git a/src/VBox/VMM/testcase/tstVMStructRC.cpp b/src/VBox/VMM/testcase/tstVMStructRC.cpp
index 02c62073bc5..0c23f5f7cac 100644
--- a/src/VBox/VMM/testcase/tstVMStructRC.cpp
+++ b/src/VBox/VMM/testcase/tstVMStructRC.cpp
@@ -81,6 +81,7 @@ AssertCompileSize(RTHCPHYS, 8);
 #include "EMInternal.h"
 #include "IEMInternal.h"
 #include "REMInternal.h"
+#include "NEMInternal.h"
 #include <VBox/vmm/vm.h>
 #include <VBox/vmm/hm_vmx.h>
 #include <VBox/param.h>
diff --git a/src/VBox/VMM/testcase/tstVMStructSize.cpp b/src/VBox/VMM/testcase/tstVMStructSize.cpp
index 29cade6c1ac..d8d28646842 100644
--- a/src/VBox/VMM/testcase/tstVMStructSize.cpp
+++ b/src/VBox/VMM/testcase/tstVMStructSize.cpp
@@ -52,6 +52,7 @@
 #include "EMInternal.h"
 #include "IEMInternal.h"
 #include "REMInternal.h"
+#include "NEMInternal.h"
 #include "../VMMR0/GMMR0Internal.h"
 #include "../VMMR0/GVMMR0Internal.h"
 #ifdef VBOX_WITH_RAW_MODE
@@ -226,6 +227,7 @@ int main()
 #endif
     CHECK_PADDING_VM(64, em);
     /*CHECK_PADDING_VM(64, iem);*/
+    CHECK_PADDING_VM(64, nem);
     CHECK_PADDING_VM(64, tm);
     PRINT_OFFSET(VM, tm.s.VirtualSyncLock);
     CHECK_PADDING_VM(64, dbgf);
@@ -242,6 +244,7 @@ int main()
     CHECK_PADDING_VMCPU(64, iem);
     CHECK_PADDING_VMCPU(64, hm);
     CHECK_PADDING_VMCPU(64, em);
+    CHECK_PADDING_VMCPU(64, nem);
     CHECK_PADDING_VMCPU(64, trpm);
     CHECK_PADDING_VMCPU(64, tm);
     CHECK_PADDING_VMCPU(64, vmm);