VMM.cpp@ 90992

Last change on this file since 90992 was 90992, checked in by vboxsync, 4 years ago
VMM: Eliminated VMMCALLRING3_PGM_LOCK. bugref:6695
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 111.1 KB

Line
1	/* $Id: VMM.cpp 90992 2021-08-30 09:52:14Z vboxsync $ */
2	/** @file
3	* VMM - The Virtual Machine Monitor Core.
4	*/
5
6	/*
7	* Copyright (C) 2006-2020 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.215389.xyz. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	//#define NO_SUPCALLR0VMM
19
20	/** @page pg_vmm VMM - The Virtual Machine Monitor
21	*
22	* The VMM component is two things at the moment, it's a component doing a few
23	* management and routing tasks, and it's the whole virtual machine monitor
24	* thing. For hysterical reasons, it is not doing all the management that one
25	* would expect, this is instead done by @ref pg_vm. We'll address this
26	* misdesign eventually, maybe.
27	*
28	* VMM is made up of these components:
29	* - @subpage pg_cfgm
30	* - @subpage pg_cpum
31	* - @subpage pg_dbgf
32	* - @subpage pg_em
33	* - @subpage pg_gim
34	* - @subpage pg_gmm
35	* - @subpage pg_gvmm
36	* - @subpage pg_hm
37	* - @subpage pg_iem
38	* - @subpage pg_iom
39	* - @subpage pg_mm
40	* - @subpage pg_nem
41	* - @subpage pg_pdm
42	* - @subpage pg_pgm
43	* - @subpage pg_selm
44	* - @subpage pg_ssm
45	* - @subpage pg_stam
46	* - @subpage pg_tm
47	* - @subpage pg_trpm
48	* - @subpage pg_vm
49	*
50	*
51	* @see @ref grp_vmm @ref grp_vm @subpage pg_vmm_guideline @subpage pg_raw
52	*
53	*
54	* @section sec_vmmstate VMM State
55	*
56	* @image html VM_Statechart_Diagram.gif
57	*
58	* To be written.
59	*
60	*
61	* @subsection subsec_vmm_init VMM Initialization
62	*
63	* To be written.
64	*
65	*
66	* @subsection subsec_vmm_term VMM Termination
67	*
68	* To be written.
69	*
70	*
71	* @section sec_vmm_limits VMM Limits
72	*
73	* There are various resource limits imposed by the VMM and it's
74	* sub-components. We'll list some of them here.
75	*
76	* On 64-bit hosts:
77	* - Max 8191 VMs. Imposed by GVMM's handle allocation (GVMM_MAX_HANDLES),
78	* can be increased up to 64K - 1.
79	* - Max 16TB - 64KB of the host memory can be used for backing VM RAM and
80	* ROM pages. The limit is imposed by the 32-bit page ID used by GMM.
81	* - A VM can be assigned all the memory we can use (16TB), however, the
82	* Main API will restrict this to 2TB (MM_RAM_MAX_IN_MB).
83	* - Max 32 virtual CPUs (VMM_MAX_CPU_COUNT).
84	*
85	* On 32-bit hosts:
86	* - Max 127 VMs. Imposed by GMM's per page structure.
87	* - Max 64GB - 64KB of the host memory can be used for backing VM RAM and
88	* ROM pages. The limit is imposed by the 28-bit page ID used
89	* internally in GMM. It is also limited by PAE.
90	* - A VM can be assigned all the memory GMM can allocate, however, the
91	* Main API will restrict this to 3584MB (MM_RAM_MAX_IN_MB).
92	* - Max 32 virtual CPUs (VMM_MAX_CPU_COUNT).
93	*
94	*/
95
96
97	/*********************************************************************************************************************************
98	* Header Files *
99	*********************************************************************************************************************************/
100	#define LOG_GROUP LOG_GROUP_VMM
101	#include <VBox/vmm/vmm.h>
102	#include <VBox/vmm/vmapi.h>
103	#include <VBox/vmm/pgm.h>
104	#include <VBox/vmm/cfgm.h>
105	#include <VBox/vmm/pdmqueue.h>
106	#include <VBox/vmm/pdmcritsect.h>
107	#include <VBox/vmm/pdmcritsectrw.h>
108	#include <VBox/vmm/pdmapi.h>
109	#include <VBox/vmm/cpum.h>
110	#include <VBox/vmm/gim.h>
111	#include <VBox/vmm/mm.h>
112	#include <VBox/vmm/nem.h>
113	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
114	# include <VBox/vmm/iem.h>
115	#endif
116	#include <VBox/vmm/iom.h>
117	#include <VBox/vmm/trpm.h>
118	#include <VBox/vmm/selm.h>
119	#include <VBox/vmm/em.h>
120	#include <VBox/sup.h>
121	#include <VBox/vmm/dbgf.h>
122	#include <VBox/vmm/apic.h>
123	#include <VBox/vmm/ssm.h>
124	#include <VBox/vmm/tm.h>
125	#include "VMMInternal.h"
126	#include <VBox/vmm/vmcc.h>
127
128	#include <VBox/err.h>
129	#include <VBox/param.h>
130	#include <VBox/version.h>
131	#include <VBox/vmm/hm.h>
132	#include <iprt/assert.h>
133	#include <iprt/alloc.h>
134	#include <iprt/asm.h>
135	#include <iprt/time.h>
136	#include <iprt/semaphore.h>
137	#include <iprt/stream.h>
138	#include <iprt/string.h>
139	#include <iprt/stdarg.h>
140	#include <iprt/ctype.h>
141	#include <iprt/x86.h>
142
143
144	/*********************************************************************************************************************************
145	* Defined Constants And Macros *
146	*********************************************************************************************************************************/
147	/** The saved state version. */
148	#define VMM_SAVED_STATE_VERSION 4
149	/** The saved state version used by v3.0 and earlier. (Teleportation) */
150	#define VMM_SAVED_STATE_VERSION_3_0 3
151
152	/** Macro for flushing the ring-0 logging. */
153	#define VMM_FLUSH_R0_LOG(a_pVM, a_pVCpu, a_pLogger, a_pR3Logger) \
154	do { \
155	size_t const idxBuf = (a_pLogger)->idxBuf % VMMLOGGER_BUFFER_COUNT; \
156	if ( (a_pLogger)->aBufs[idxBuf].AuxDesc.offBuf == 0 \
157	\|\| (a_pLogger)->aBufs[idxBuf].AuxDesc.fFlushedIndicator) \
158	{ /* likely? */ } \
159	else \
160	vmmR3LogReturnFlush(a_pVM, a_pVCpu, a_pLogger, idxBuf, a_pR3Logger); \
161	} while (0)
162
163
164	/*********************************************************************************************************************************
165	* Internal Functions *
166	*********************************************************************************************************************************/
167	static int vmmR3InitStacks(PVM pVM);
168	static void vmmR3InitRegisterStats(PVM pVM);
169	static DECLCALLBACK(int) vmmR3Save(PVM pVM, PSSMHANDLE pSSM);
170	static DECLCALLBACK(int) vmmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
171	#if 0 /* pointless when timers doesn't run on EMT */
172	static DECLCALLBACK(void) vmmR3YieldEMT(PVM pVM, TMTIMERHANDLE hTimer, void *pvUser);
173	#endif
174	static VBOXSTRICTRC vmmR3EmtRendezvousCommon(PVM pVM, PVMCPU pVCpu, bool fIsCaller,
175	uint32_t fFlags, PFNVMMEMTRENDEZVOUS pfnRendezvous, void *pvUser);
176	static int vmmR3ServiceCallRing3Request(PVM pVM, PVMCPU pVCpu);
177	static FNRTTHREAD vmmR3LogFlusher;
178	static void vmmR3LogReturnFlush(PVM pVM, PVMCPU pVCpu, PVMMR3CPULOGGER pShared, size_t idxBuf,
179	PRTLOGGER pDstLogger);
180	static DECLCALLBACK(void) vmmR3InfoFF(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
181
182
183
184	/**
185	* Initializes the VMM.
186	*
187	* @returns VBox status code.
188	* @param pVM The cross context VM structure.
189	*/
190	VMMR3_INT_DECL(int) VMMR3Init(PVM pVM)
191	{
192	LogFlow(("VMMR3Init\n"));
193
194	/*
195	* Assert alignment, sizes and order.
196	*/
197	AssertCompile(sizeof(pVM->vmm.s) <= sizeof(pVM->vmm.padding));
198	AssertCompile(RT_SIZEOFMEMB(VMCPU, vmm.s) <= RT_SIZEOFMEMB(VMCPU, vmm.padding));
199
200	/*
201	* Init basic VM VMM members.
202	*/
203	pVM->vmm.s.pahEvtRendezvousEnterOrdered = NULL;
204	pVM->vmm.s.hEvtRendezvousEnterOneByOne = NIL_RTSEMEVENT;
205	pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce = NIL_RTSEMEVENTMULTI;
206	pVM->vmm.s.hEvtMulRendezvousDone = NIL_RTSEMEVENTMULTI;
207	pVM->vmm.s.hEvtRendezvousDoneCaller = NIL_RTSEMEVENT;
208	pVM->vmm.s.hEvtMulRendezvousRecursionPush = NIL_RTSEMEVENTMULTI;
209	pVM->vmm.s.hEvtMulRendezvousRecursionPop = NIL_RTSEMEVENTMULTI;
210	pVM->vmm.s.hEvtRendezvousRecursionPushCaller = NIL_RTSEMEVENT;
211	pVM->vmm.s.hEvtRendezvousRecursionPopCaller = NIL_RTSEMEVENT;
212	pVM->vmm.s.nsProgramStart = RTTimeProgramStartNanoTS();
213
214	#if 0 /* pointless when timers doesn't run on EMT */
215	/** @cfgm{/YieldEMTInterval, uint32_t, 1, UINT32_MAX, 23, ms}
216	* The EMT yield interval. The EMT yielding is a hack we employ to play a
217	* bit nicer with the rest of the system (like for instance the GUI).
218	*/
219	int rc = CFGMR3QueryU32Def(CFGMR3GetRoot(pVM), "YieldEMTInterval", &pVM->vmm.s.cYieldEveryMillies,
220	23 /* Value arrived at after experimenting with the grub boot prompt. */);
221	AssertMsgRCReturn(rc, ("Configuration error. Failed to query \"YieldEMTInterval\", rc=%Rrc\n", rc), rc);
222	#endif
223
224	/** @cfgm{/VMM/UsePeriodicPreemptionTimers, boolean, true}
225	* Controls whether we employ per-cpu preemption timers to limit the time
226	* spent executing guest code. This option is not available on all
227	* platforms and we will silently ignore this setting then. If we are
228	* running in VT-x mode, we will use the VMX-preemption timer instead of
229	* this one when possible.
230	*/
231	PCFGMNODE pCfgVMM = CFGMR3GetChild(CFGMR3GetRoot(pVM), "VMM");
232	int rc = CFGMR3QueryBoolDef(pCfgVMM, "UsePeriodicPreemptionTimers", &pVM->vmm.s.fUsePeriodicPreemptionTimers, true);
233	AssertMsgRCReturn(rc, ("Configuration error. Failed to query \"VMM/UsePeriodicPreemptionTimers\", rc=%Rrc\n", rc), rc);
234
235	/*
236	* Initialize the VMM rendezvous semaphores.
237	*/
238	pVM->vmm.s.pahEvtRendezvousEnterOrdered = (PRTSEMEVENT)MMR3HeapAlloc(pVM, MM_TAG_VMM, sizeof(RTSEMEVENT) * pVM->cCpus);
239	if (!pVM->vmm.s.pahEvtRendezvousEnterOrdered)
240	return VERR_NO_MEMORY;
241	for (VMCPUID i = 0; i < pVM->cCpus; i++)
242	pVM->vmm.s.pahEvtRendezvousEnterOrdered[i] = NIL_RTSEMEVENT;
243	for (VMCPUID i = 0; i < pVM->cCpus; i++)
244	{
245	rc = RTSemEventCreate(&pVM->vmm.s.pahEvtRendezvousEnterOrdered[i]);
246	AssertRCReturn(rc, rc);
247	}
248	rc = RTSemEventCreate(&pVM->vmm.s.hEvtRendezvousEnterOneByOne);
249	AssertRCReturn(rc, rc);
250	rc = RTSemEventMultiCreate(&pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce);
251	AssertRCReturn(rc, rc);
252	rc = RTSemEventMultiCreate(&pVM->vmm.s.hEvtMulRendezvousDone);
253	AssertRCReturn(rc, rc);
254	rc = RTSemEventCreate(&pVM->vmm.s.hEvtRendezvousDoneCaller);
255	AssertRCReturn(rc, rc);
256	rc = RTSemEventMultiCreate(&pVM->vmm.s.hEvtMulRendezvousRecursionPush);
257	AssertRCReturn(rc, rc);
258	rc = RTSemEventMultiCreate(&pVM->vmm.s.hEvtMulRendezvousRecursionPop);
259	AssertRCReturn(rc, rc);
260	rc = RTSemEventCreate(&pVM->vmm.s.hEvtRendezvousRecursionPushCaller);
261	AssertRCReturn(rc, rc);
262	rc = RTSemEventCreate(&pVM->vmm.s.hEvtRendezvousRecursionPopCaller);
263	AssertRCReturn(rc, rc);
264
265	/*
266	* Register the saved state data unit.
267	*/
268	rc = SSMR3RegisterInternal(pVM, "vmm", 1, VMM_SAVED_STATE_VERSION, VMM_STACK_SIZE + sizeof(RTGCPTR),
269	NULL, NULL, NULL,
270	NULL, vmmR3Save, NULL,
271	NULL, vmmR3Load, NULL);
272	if (RT_FAILURE(rc))
273	return rc;
274
275	/*
276	* Register the Ring-0 VM handle with the session for fast ioctl calls.
277	*/
278	rc = SUPR3SetVMForFastIOCtl(VMCC_GET_VMR0_FOR_CALL(pVM));
279	if (RT_FAILURE(rc))
280	return rc;
281
282	/*
283	* Init various sub-components.
284	*/
285	rc = vmmR3InitStacks(pVM);
286	if (RT_SUCCESS(rc))
287	{
288	#ifdef VBOX_WITH_NMI
289	/*
290	* Allocate mapping for the host APIC.
291	*/
292	rc = MMR3HyperReserve(pVM, PAGE_SIZE, "Host APIC", &pVM->vmm.s.GCPtrApicBase);
293	AssertRC(rc);
294	#endif
295	if (RT_SUCCESS(rc))
296	{
297	/*
298	* Start the log flusher thread.
299	*/
300	rc = RTThreadCreate(&pVM->vmm.s.hLogFlusherThread, vmmR3LogFlusher, pVM, 0 /cbStack/,
301	RTTHREADTYPE_IO, RTTHREADFLAGS_WAITABLE, "R0LogWrk");
302	if (RT_SUCCESS(rc))
303	{
304
305	/*
306	* Debug info and statistics.
307	*/
308	DBGFR3InfoRegisterInternal(pVM, "fflags", "Displays the current Forced actions Flags.", vmmR3InfoFF);
309	vmmR3InitRegisterStats(pVM);
310	vmmInitFormatTypes();
311
312	return VINF_SUCCESS;
313	}
314	}
315	}
316	/** @todo Need failure cleanup? */
317
318	return rc;
319	}
320
321
322	/**
323	* Allocate & setup the VMM RC stack(s) (for EMTs).
324	*
325	* The stacks are also used for long jumps in Ring-0.
326	*
327	* @returns VBox status code.
328	* @param pVM The cross context VM structure.
329	*
330	* @remarks The optional guard page gets it protection setup up during R3 init
331	* completion because of init order issues.
332	*/
333	static int vmmR3InitStacks(PVM pVM)
334	{
335	int rc = VINF_SUCCESS;
336	#ifdef VMM_R0_SWITCH_STACK
337	uint32_t fFlags = MMHYPER_AONR_FLAGS_KERNEL_MAPPING;
338	#else
339	uint32_t fFlags = 0;
340	#endif
341
342	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
343	{
344	PVMCPU pVCpu = pVM->apCpusR3[idCpu];
345
346	#ifdef VBOX_STRICT_VMM_STACK
347	rc = MMR3HyperAllocOnceNoRelEx(pVM, PAGE_SIZE + VMM_STACK_SIZE + PAGE_SIZE,
348	#else
349	rc = MMR3HyperAllocOnceNoRelEx(pVM, VMM_STACK_SIZE,
350	#endif
351	PAGE_SIZE, MM_TAG_VMM, fFlags, (void **)&pVCpu->vmm.s.pbEMTStackR3);
352	if (RT_SUCCESS(rc))
353	{
354	#ifdef VBOX_STRICT_VMM_STACK
355	pVCpu->vmm.s.pbEMTStackR3 += PAGE_SIZE;
356	#endif
357	pVCpu->vmm.s.CallRing3JmpBufR0.pvSavedStack = MMHyperR3ToR0(pVM, pVCpu->vmm.s.pbEMTStackR3);
358
359	}
360	}
361
362	return rc;
363	}
364
365
366	/**
367	* VMMR3Init worker that register the statistics with STAM.
368	*
369	* @param pVM The cross context VM structure.
370	*/
371	static void vmmR3InitRegisterStats(PVM pVM)
372	{
373	RT_NOREF_PV(pVM);
374
375	/*
376	* Statistics.
377	*/
378	STAM_REG(pVM, &pVM->vmm.s.StatRunGC, STAMTYPE_COUNTER, "/VMM/RunGC", STAMUNIT_OCCURENCES, "Number of context switches.");
379	STAM_REG(pVM, &pVM->vmm.s.StatRZRetNormal, STAMTYPE_COUNTER, "/VMM/RZRet/Normal", STAMUNIT_OCCURENCES, "Number of VINF_SUCCESS returns.");
380	STAM_REG(pVM, &pVM->vmm.s.StatRZRetInterrupt, STAMTYPE_COUNTER, "/VMM/RZRet/Interrupt", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_INTERRUPT returns.");
381	STAM_REG(pVM, &pVM->vmm.s.StatRZRetInterruptHyper, STAMTYPE_COUNTER, "/VMM/RZRet/InterruptHyper", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_INTERRUPT_HYPER returns.");
382	STAM_REG(pVM, &pVM->vmm.s.StatRZRetGuestTrap, STAMTYPE_COUNTER, "/VMM/RZRet/GuestTrap", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_GUEST_TRAP returns.");
383	STAM_REG(pVM, &pVM->vmm.s.StatRZRetRingSwitch, STAMTYPE_COUNTER, "/VMM/RZRet/RingSwitch", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_RING_SWITCH returns.");
384	STAM_REG(pVM, &pVM->vmm.s.StatRZRetRingSwitchInt, STAMTYPE_COUNTER, "/VMM/RZRet/RingSwitchInt", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_RING_SWITCH_INT returns.");
385	STAM_REG(pVM, &pVM->vmm.s.StatRZRetStaleSelector, STAMTYPE_COUNTER, "/VMM/RZRet/StaleSelector", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_STALE_SELECTOR returns.");
386	STAM_REG(pVM, &pVM->vmm.s.StatRZRetIRETTrap, STAMTYPE_COUNTER, "/VMM/RZRet/IRETTrap", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_IRET_TRAP returns.");
387	STAM_REG(pVM, &pVM->vmm.s.StatRZRetEmulate, STAMTYPE_COUNTER, "/VMM/RZRet/Emulate", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION returns.");
388	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchEmulate, STAMTYPE_COUNTER, "/VMM/RZRet/PatchEmulate", STAMUNIT_OCCURENCES, "Number of VINF_PATCH_EMULATE_INSTR returns.");
389	STAM_REG(pVM, &pVM->vmm.s.StatRZRetIORead, STAMTYPE_COUNTER, "/VMM/RZRet/IORead", STAMUNIT_OCCURENCES, "Number of VINF_IOM_R3_IOPORT_READ returns.");
390	STAM_REG(pVM, &pVM->vmm.s.StatRZRetIOWrite, STAMTYPE_COUNTER, "/VMM/RZRet/IOWrite", STAMUNIT_OCCURENCES, "Number of VINF_IOM_R3_IOPORT_WRITE returns.");
391	STAM_REG(pVM, &pVM->vmm.s.StatRZRetIOCommitWrite, STAMTYPE_COUNTER, "/VMM/RZRet/IOCommitWrite", STAMUNIT_OCCURENCES, "Number of VINF_IOM_R3_IOPORT_COMMIT_WRITE returns.");
392	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIORead, STAMTYPE_COUNTER, "/VMM/RZRet/MMIORead", STAMUNIT_OCCURENCES, "Number of VINF_IOM_R3_MMIO_READ returns.");
393	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOWrite, STAMTYPE_COUNTER, "/VMM/RZRet/MMIOWrite", STAMUNIT_OCCURENCES, "Number of VINF_IOM_R3_MMIO_WRITE returns.");
394	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOCommitWrite, STAMTYPE_COUNTER, "/VMM/RZRet/MMIOCommitWrite", STAMUNIT_OCCURENCES, "Number of VINF_IOM_R3_MMIO_COMMIT_WRITE returns.");
395	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOReadWrite, STAMTYPE_COUNTER, "/VMM/RZRet/MMIOReadWrite", STAMUNIT_OCCURENCES, "Number of VINF_IOM_R3_MMIO_READ_WRITE returns.");
396	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOPatchRead, STAMTYPE_COUNTER, "/VMM/RZRet/MMIOPatchRead", STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_MMIO_PATCH_READ returns.");
397	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOPatchWrite, STAMTYPE_COUNTER, "/VMM/RZRet/MMIOPatchWrite", STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_MMIO_PATCH_WRITE returns.");
398	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMSRRead, STAMTYPE_COUNTER, "/VMM/RZRet/MSRRead", STAMUNIT_OCCURENCES, "Number of VINF_CPUM_R3_MSR_READ returns.");
399	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMSRWrite, STAMTYPE_COUNTER, "/VMM/RZRet/MSRWrite", STAMUNIT_OCCURENCES, "Number of VINF_CPUM_R3_MSR_WRITE returns.");
400	STAM_REG(pVM, &pVM->vmm.s.StatRZRetLDTFault, STAMTYPE_COUNTER, "/VMM/RZRet/LDTFault", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_GDT_FAULT returns.");
401	STAM_REG(pVM, &pVM->vmm.s.StatRZRetGDTFault, STAMTYPE_COUNTER, "/VMM/RZRet/GDTFault", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_LDT_FAULT returns.");
402	STAM_REG(pVM, &pVM->vmm.s.StatRZRetIDTFault, STAMTYPE_COUNTER, "/VMM/RZRet/IDTFault", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_IDT_FAULT returns.");
403	STAM_REG(pVM, &pVM->vmm.s.StatRZRetTSSFault, STAMTYPE_COUNTER, "/VMM/RZRet/TSSFault", STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_TSS_FAULT returns.");
404	STAM_REG(pVM, &pVM->vmm.s.StatRZRetCSAMTask, STAMTYPE_COUNTER, "/VMM/RZRet/CSAMTask", STAMUNIT_OCCURENCES, "Number of VINF_CSAM_PENDING_ACTION returns.");
405	STAM_REG(pVM, &pVM->vmm.s.StatRZRetSyncCR3, STAMTYPE_COUNTER, "/VMM/RZRet/SyncCR", STAMUNIT_OCCURENCES, "Number of VINF_PGM_SYNC_CR3 returns.");
406	STAM_REG(pVM, &pVM->vmm.s.StatRZRetMisc, STAMTYPE_COUNTER, "/VMM/RZRet/Misc", STAMUNIT_OCCURENCES, "Number of misc returns.");
407	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchInt3, STAMTYPE_COUNTER, "/VMM/RZRet/PatchInt3", STAMUNIT_OCCURENCES, "Number of VINF_PATM_PATCH_INT3 returns.");
408	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchPF, STAMTYPE_COUNTER, "/VMM/RZRet/PatchPF", STAMUNIT_OCCURENCES, "Number of VINF_PATM_PATCH_TRAP_PF returns.");
409	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchGP, STAMTYPE_COUNTER, "/VMM/RZRet/PatchGP", STAMUNIT_OCCURENCES, "Number of VINF_PATM_PATCH_TRAP_GP returns.");
410	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchIretIRQ, STAMTYPE_COUNTER, "/VMM/RZRet/PatchIret", STAMUNIT_OCCURENCES, "Number of VINF_PATM_PENDING_IRQ_AFTER_IRET returns.");
411	STAM_REG(pVM, &pVM->vmm.s.StatRZRetRescheduleREM, STAMTYPE_COUNTER, "/VMM/RZRet/ScheduleREM", STAMUNIT_OCCURENCES, "Number of VINF_EM_RESCHEDULE_REM returns.");
412	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3Total, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
413	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3Unknown, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/Unknown", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns without responsible force flag.");
414	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3FF, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/ToR3", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns with VMCPU_FF_TO_R3.");
415	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3TMVirt, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/TMVirt", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns with VM_FF_TM_VIRTUAL_SYNC.");
416	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3HandyPages, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/Handy", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns with VM_FF_PGM_NEED_HANDY_PAGES.");
417	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3PDMQueues, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/PDMQueue", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns with VM_FF_PDM_QUEUES.");
418	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3Rendezvous, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/Rendezvous", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns with VM_FF_EMT_RENDEZVOUS.");
419	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3Timer, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/Timer", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns with VMCPU_FF_TIMER.");
420	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3DMA, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/DMA", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns with VM_FF_PDM_DMA.");
421	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3CritSect, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/CritSect", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns with VMCPU_FF_PDM_CRITSECT.");
422	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3Iem, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/IEM", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns with VMCPU_FF_IEM.");
423	STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3Iom, STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/IOM", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns with VMCPU_FF_IOM.");
424	STAM_REG(pVM, &pVM->vmm.s.StatRZRetTimerPending, STAMTYPE_COUNTER, "/VMM/RZRet/TimerPending", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TIMER_PENDING returns.");
425	STAM_REG(pVM, &pVM->vmm.s.StatRZRetInterruptPending, STAMTYPE_COUNTER, "/VMM/RZRet/InterruptPending", STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_INTERRUPT_PENDING returns.");
426	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPATMDuplicateFn, STAMTYPE_COUNTER, "/VMM/RZRet/PATMDuplicateFn", STAMUNIT_OCCURENCES, "Number of VINF_PATM_DUPLICATE_FUNCTION returns.");
427	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPGMChangeMode, STAMTYPE_COUNTER, "/VMM/RZRet/PGMChangeMode", STAMUNIT_OCCURENCES, "Number of VINF_PGM_CHANGE_MODE returns.");
428	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPGMFlushPending, STAMTYPE_COUNTER, "/VMM/RZRet/PGMFlushPending", STAMUNIT_OCCURENCES, "Number of VINF_PGM_POOL_FLUSH_PENDING returns.");
429	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPendingRequest, STAMTYPE_COUNTER, "/VMM/RZRet/PendingRequest", STAMUNIT_OCCURENCES, "Number of VINF_EM_PENDING_REQUEST returns.");
430	STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchTPR, STAMTYPE_COUNTER, "/VMM/RZRet/PatchTPR", STAMUNIT_OCCURENCES, "Number of VINF_EM_HM_PATCH_TPR_INSTR returns.");
431	STAM_REG(pVM, &pVM->vmm.s.StatRZRetCallRing3, STAMTYPE_COUNTER, "/VMM/RZCallR3/Misc", STAMUNIT_OCCURENCES, "Number of Other ring-3 calls.");
432	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMLock, STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMLock", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_LOCK calls.");
433	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMPoolGrow, STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMPoolGrow", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_POOL_GROW calls.");
434	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMMapChunk, STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMMapChunk", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_MAP_CHUNK calls.");
435	STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMAllocHandy, STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMAllocHandy", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES calls.");
436	STAM_REG(pVM, &pVM->vmm.s.StatRZCallVMSetError, STAMTYPE_COUNTER, "/VMM/RZCallR3/VMSetError", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_VM_SET_ERROR calls.");
437	STAM_REG(pVM, &pVM->vmm.s.StatRZCallVMSetRuntimeError, STAMTYPE_COUNTER, "/VMM/RZCallR3/VMRuntimeError", STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_VM_SET_RUNTIME_ERROR calls.");
438
439	STAMR3Register(pVM, &pVM->vmm.s.StatLogFlusherFlushes, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, "/VMM/LogFlush/00-Flushes", STAMUNIT_OCCURENCES, "Total number of buffer flushes");
440	STAMR3Register(pVM, &pVM->vmm.s.StatLogFlusherNoWakeUp, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, "/VMM/LogFlush/00-NoWakups", STAMUNIT_OCCURENCES, "Times the flusher thread didn't need waking up.");
441
442	#ifdef VBOX_WITH_STATISTICS
443	for (VMCPUID i = 0; i < pVM->cCpus; i++)
444	{
445	PVMCPU pVCpu = pVM->apCpusR3[i];
446	STAMR3RegisterF(pVM, &pVCpu->vmm.s.CallRing3JmpBufR0.cbUsedMax, STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Max amount of stack used.", "/VMM/Stack/CPU%u/Max", i);
447	STAMR3RegisterF(pVM, &pVCpu->vmm.s.CallRing3JmpBufR0.cbUsedAvg, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, "Average stack usage.", "/VMM/Stack/CPU%u/Avg", i);
448	STAMR3RegisterF(pVM, &pVCpu->vmm.s.CallRing3JmpBufR0.cUsedTotal, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of stack usages.", "/VMM/Stack/CPU%u/Uses", i);
449	}
450	#endif
451	for (VMCPUID i = 0; i < pVM->cCpus; i++)
452	{
453	PVMCPU pVCpu = pVM->apCpusR3[i];
454	STAMR3RegisterF(pVM, &pVCpu->vmm.s.StatR0HaltBlock, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_CALL, "", "/PROF/CPU%u/VM/Halt/R0HaltBlock", i);
455	STAMR3RegisterF(pVM, &pVCpu->vmm.s.StatR0HaltBlockOnTime, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_CALL, "", "/PROF/CPU%u/VM/Halt/R0HaltBlockOnTime", i);
456	STAMR3RegisterF(pVM, &pVCpu->vmm.s.StatR0HaltBlockOverslept, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_CALL, "", "/PROF/CPU%u/VM/Halt/R0HaltBlockOverslept", i);
457	STAMR3RegisterF(pVM, &pVCpu->vmm.s.StatR0HaltBlockInsomnia, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_CALL, "", "/PROF/CPU%u/VM/Halt/R0HaltBlockInsomnia", i);
458	STAMR3RegisterF(pVM, &pVCpu->vmm.s.StatR0HaltExec, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "", "/PROF/CPU%u/VM/Halt/R0HaltExec", i);
459	STAMR3RegisterF(pVM, &pVCpu->vmm.s.StatR0HaltExecFromSpin, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "", "/PROF/CPU%u/VM/Halt/R0HaltExec/FromSpin", i);
460	STAMR3RegisterF(pVM, &pVCpu->vmm.s.StatR0HaltExecFromBlock, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "", "/PROF/CPU%u/VM/Halt/R0HaltExec/FromBlock", i);
461	STAMR3RegisterF(pVM, &pVCpu->vmm.s.StatR0HaltToR3, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "", "/PROF/CPU%u/VM/Halt/R0HaltToR3", i);
462	STAMR3RegisterF(pVM, &pVCpu->vmm.s.StatR0HaltToR3FromSpin, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "", "/PROF/CPU%u/VM/Halt/R0HaltToR3/FromSpin", i);
463	STAMR3RegisterF(pVM, &pVCpu->vmm.s.StatR0HaltToR3Other, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "", "/PROF/CPU%u/VM/Halt/R0HaltToR3/Other", i);
464	STAMR3RegisterF(pVM, &pVCpu->vmm.s.StatR0HaltToR3PendingFF, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "", "/PROF/CPU%u/VM/Halt/R0HaltToR3/PendingFF", i);
465	STAMR3RegisterF(pVM, &pVCpu->vmm.s.StatR0HaltToR3SmallDelta, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "", "/PROF/CPU%u/VM/Halt/R0HaltToR3/SmallDelta", i);
466	STAMR3RegisterF(pVM, &pVCpu->vmm.s.StatR0HaltToR3PostNoInt, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "", "/PROF/CPU%u/VM/Halt/R0HaltToR3/PostWaitNoInt", i);
467	STAMR3RegisterF(pVM, &pVCpu->vmm.s.StatR0HaltToR3PostPendingFF,STAMTYPE_COUNTER,STAMVISIBILITY_ALWAYS,STAMUNIT_OCCURENCES, "", "/PROF/CPU%u/VM/Halt/R0HaltToR3/PostWaitPendingFF", i);
468	STAMR3RegisterF(pVM, &pVCpu->vmm.s.cR0Halts, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "", "/PROF/CPU%u/VM/Halt/R0HaltHistoryCounter", i);
469	STAMR3RegisterF(pVM, &pVCpu->vmm.s.cR0HaltsSucceeded, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "", "/PROF/CPU%u/VM/Halt/R0HaltHistorySucceeded", i);
470	STAMR3RegisterF(pVM, &pVCpu->vmm.s.cR0HaltsToRing3, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "", "/PROF/CPU%u/VM/Halt/R0HaltHistoryToRing3", i);
471
472	STAMR3RegisterF(pVM, &pVCpu->cEmtHashCollisions, STAMTYPE_U8, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "", "/VMM/EmtHashCollisions/Emt%02u", i);
473
474	PVMMR3CPULOGGER pShared = &pVCpu->vmm.s.u.s.Logger;
475	STAMR3RegisterF(pVM, &pShared->StatFlushes, STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES, "", "/VMM/LogFlush/CPU%u/Reg", i);
476	STAMR3RegisterF(pVM, &pShared->StatCannotBlock, STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES, "", "/VMM/LogFlush/CPU%u/Reg/CannotBlock", i);
477	STAMR3RegisterF(pVM, &pShared->StatWait, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, "", "/VMM/LogFlush/CPU%u/Reg/Wait", i);
478	STAMR3RegisterF(pVM, &pShared->StatRaces, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, "", "/VMM/LogFlush/CPU%u/Reg/Races", i);
479	STAMR3RegisterF(pVM, &pShared->StatRacesToR0, STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES, "", "/VMM/LogFlush/CPU%u/Reg/RacesToR0", i);
480	STAMR3RegisterF(pVM, &pShared->cbDropped, STAMTYPE_U32, STAMVISIBILITY_USED, STAMUNIT_BYTES, "", "/VMM/LogFlush/CPU%u/Reg/cbDropped", i);
481	STAMR3RegisterF(pVM, &pShared->cbBuf, STAMTYPE_U32, STAMVISIBILITY_USED, STAMUNIT_BYTES, "", "/VMM/LogFlush/CPU%u/Reg/cbBuf", i);
482	STAMR3RegisterF(pVM, &pShared->idxBuf, STAMTYPE_U32, STAMVISIBILITY_USED, STAMUNIT_BYTES, "", "/VMM/LogFlush/CPU%u/Reg/idxBuf", i);
483
484	pShared = &pVCpu->vmm.s.u.s.RelLogger;
485	STAMR3RegisterF(pVM, &pShared->StatFlushes, STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES, "", "/VMM/LogFlush/CPU%u/Rel", i);
486	STAMR3RegisterF(pVM, &pShared->StatCannotBlock, STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES, "", "/VMM/LogFlush/CPU%u/Rel/CannotBlock", i);
487	STAMR3RegisterF(pVM, &pShared->StatWait, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, "", "/VMM/LogFlush/CPU%u/Rel/Wait", i);
488	STAMR3RegisterF(pVM, &pShared->StatRaces, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, "", "/VMM/LogFlush/CPU%u/Rel/Races", i);
489	STAMR3RegisterF(pVM, &pShared->StatRacesToR0, STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES, "", "/VMM/LogFlush/CPU%u/Rel/RacesToR0", i);
490	STAMR3RegisterF(pVM, &pShared->cbDropped, STAMTYPE_U32, STAMVISIBILITY_USED, STAMUNIT_BYTES, "", "/VMM/LogFlush/CPU%u/Rel/cbDropped", i);
491	STAMR3RegisterF(pVM, &pShared->cbBuf, STAMTYPE_U32, STAMVISIBILITY_USED, STAMUNIT_BYTES, "", "/VMM/LogFlush/CPU%u/Rel/cbBuf", i);
492	STAMR3RegisterF(pVM, &pShared->idxBuf, STAMTYPE_U32, STAMVISIBILITY_USED, STAMUNIT_BYTES, "", "/VMM/LogFlush/CPU%u/Rel/idxBuf", i);
493	}
494	}
495
496
497	/**
498	* Worker for VMMR3InitR0 that calls ring-0 to do EMT specific initialization.
499	*
500	* @returns VBox status code.
501	* @param pVM The cross context VM structure.
502	* @param pVCpu The cross context per CPU structure.
503	* @thread EMT(pVCpu)
504	*/
505	static DECLCALLBACK(int) vmmR3InitR0Emt(PVM pVM, PVMCPU pVCpu)
506	{
507	return VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_VMMR0_INIT_EMT, 0, NULL);
508	}
509
510
511	/**
512	* Initializes the R0 VMM.
513	*
514	* @returns VBox status code.
515	* @param pVM The cross context VM structure.
516	*/
517	VMMR3_INT_DECL(int) VMMR3InitR0(PVM pVM)
518	{
519	int rc;
520	PVMCPU pVCpu = VMMGetCpu(pVM);
521	Assert(pVCpu && pVCpu->idCpu == 0);
522
523	/*
524	* Make sure the ring-0 loggers are up to date.
525	*/
526	rc = VMMR3UpdateLoggers(pVM);
527	if (RT_FAILURE(rc))
528	return rc;
529
530	/*
531	* Call Ring-0 entry with init code.
532	*/
533	for (;;)
534	{
535	#ifdef NO_SUPCALLR0VMM
536	//rc = VERR_GENERAL_FAILURE;
537	rc = VINF_SUCCESS;
538	#else
539	rc = SUPR3CallVMMR0Ex(VMCC_GET_VMR0_FOR_CALL(pVM), 0 /idCpu/, VMMR0_DO_VMMR0_INIT, RT_MAKE_U64(VMMGetSvnRev(), vmmGetBuildType()), NULL);
540	#endif
541	/*
542	* Flush the logs.
543	*/
544	#ifdef LOG_ENABLED
545	VMM_FLUSH_R0_LOG(pVM, pVCpu, &pVCpu->vmm.s.u.s.Logger, NULL);
546	#endif
547	VMM_FLUSH_R0_LOG(pVM, pVCpu, &pVCpu->vmm.s.u.s.RelLogger, RTLogRelGetDefaultInstance());
548	if (rc != VINF_VMM_CALL_HOST)
549	break;
550	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
551	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
552	break;
553	/* Resume R0 */
554	}
555
556	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
557	{
558	LogRel(("VMM: R0 init failed, rc=%Rra\n", rc));
559	if (RT_SUCCESS(rc))
560	rc = VERR_IPE_UNEXPECTED_INFO_STATUS;
561	}
562
563	/* Log whether thread-context hooks are used (on Linux this can depend on how the kernel is configured). */
564	if (pVM->vmm.s.fIsUsingContextHooks)
565	LogRel(("VMM: Enabled thread-context hooks\n"));
566	else
567	LogRel(("VMM: Thread-context hooks unavailable\n"));
568
569	/* Log RTThreadPreemptIsPendingTrusty() and RTThreadPreemptIsPossible() results. */
570	if (pVM->vmm.s.fIsPreemptPendingApiTrusty)
571	LogRel(("VMM: RTThreadPreemptIsPending() can be trusted\n"));
572	else
573	LogRel(("VMM: Warning! RTThreadPreemptIsPending() cannot be trusted! Need to update kernel info?\n"));
574	if (pVM->vmm.s.fIsPreemptPossible)
575	LogRel(("VMM: Kernel preemption is possible\n"));
576	else
577	LogRel(("VMM: Kernel preemption is not possible it seems\n"));
578
579	/*
580	* Send all EMTs to ring-0 to get their logger initialized.
581	*/
582	for (VMCPUID idCpu = 0; RT_SUCCESS(rc) && idCpu < pVM->cCpus; idCpu++)
583	rc = VMR3ReqCallWait(pVM, idCpu, (PFNRT)vmmR3InitR0Emt, 2, pVM, pVM->apCpusR3[idCpu]);
584
585	return rc;
586	}
587
588
589	/**
590	* Called when an init phase completes.
591	*
592	* @returns VBox status code.
593	* @param pVM The cross context VM structure.
594	* @param enmWhat Which init phase.
595	*/
596	VMMR3_INT_DECL(int) VMMR3InitCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
597	{
598	int rc = VINF_SUCCESS;
599
600	switch (enmWhat)
601	{
602	case VMINITCOMPLETED_RING3:
603	{
604	#if 0 /* pointless when timers doesn't run on EMT */
605	/*
606	* Create the EMT yield timer.
607	*/
608	rc = TMR3TimerCreate(pVM, TMCLOCK_REAL, vmmR3YieldEMT, NULL, TMTIMER_FLAGS_NO_RING0,
609	"EMT Yielder", &pVM->vmm.s.hYieldTimer);
610	AssertRCReturn(rc, rc);
611
612	rc = TMTimerSetMillies(pVM, pVM->vmm.s.hYieldTimer, pVM->vmm.s.cYieldEveryMillies);
613	AssertRCReturn(rc, rc);
614	#endif
615	break;
616	}
617
618	case VMINITCOMPLETED_HM:
619	{
620	/*
621	* Disable the periodic preemption timers if we can use the
622	* VMX-preemption timer instead.
623	*/
624	if ( pVM->vmm.s.fUsePeriodicPreemptionTimers
625	&& HMR3IsVmxPreemptionTimerUsed(pVM))
626	pVM->vmm.s.fUsePeriodicPreemptionTimers = false;
627	LogRel(("VMM: fUsePeriodicPreemptionTimers=%RTbool\n", pVM->vmm.s.fUsePeriodicPreemptionTimers));
628
629	/*
630	* Last chance for GIM to update its CPUID leaves if it requires
631	* knowledge/information from HM initialization.
632	*/
633	rc = GIMR3InitCompleted(pVM);
634	AssertRCReturn(rc, rc);
635
636	/*
637	* CPUM's post-initialization (print CPUIDs).
638	*/
639	CPUMR3LogCpuIdAndMsrFeatures(pVM);
640	break;
641	}
642
643	default: /* shuts up gcc */
644	break;
645	}
646
647	return rc;
648	}
649
650
651	/**
652	* Terminate the VMM bits.
653	*
654	* @returns VBox status code.
655	* @param pVM The cross context VM structure.
656	*/
657	VMMR3_INT_DECL(int) VMMR3Term(PVM pVM)
658	{
659	PVMCPU pVCpu = VMMGetCpu(pVM);
660	Assert(pVCpu && pVCpu->idCpu == 0);
661
662	/*
663	* Call Ring-0 entry with termination code.
664	*/
665	int rc;
666	for (;;)
667	{
668	#ifdef NO_SUPCALLR0VMM
669	//rc = VERR_GENERAL_FAILURE;
670	rc = VINF_SUCCESS;
671	#else
672	rc = SUPR3CallVMMR0Ex(VMCC_GET_VMR0_FOR_CALL(pVM), 0 /idCpu/, VMMR0_DO_VMMR0_TERM, 0, NULL);
673	#endif
674	/*
675	* Flush the logs.
676	*/
677	#ifdef LOG_ENABLED
678	VMM_FLUSH_R0_LOG(pVM, pVCpu, &pVCpu->vmm.s.u.s.Logger, NULL);
679	#endif
680	VMM_FLUSH_R0_LOG(pVM, pVCpu, &pVCpu->vmm.s.u.s.RelLogger, RTLogRelGetDefaultInstance());
681	if (rc != VINF_VMM_CALL_HOST)
682	break;
683	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
684	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
685	break;
686	/* Resume R0 */
687	}
688	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
689	{
690	LogRel(("VMM: VMMR3Term: R0 term failed, rc=%Rra. (warning)\n", rc));
691	if (RT_SUCCESS(rc))
692	rc = VERR_IPE_UNEXPECTED_INFO_STATUS;
693	}
694
695	for (VMCPUID i = 0; i < pVM->cCpus; i++)
696	{
697	RTSemEventDestroy(pVM->vmm.s.pahEvtRendezvousEnterOrdered[i]);
698	pVM->vmm.s.pahEvtRendezvousEnterOrdered[i] = NIL_RTSEMEVENT;
699	}
700	RTSemEventDestroy(pVM->vmm.s.hEvtRendezvousEnterOneByOne);
701	pVM->vmm.s.hEvtRendezvousEnterOneByOne = NIL_RTSEMEVENT;
702	RTSemEventMultiDestroy(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce);
703	pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce = NIL_RTSEMEVENTMULTI;
704	RTSemEventMultiDestroy(pVM->vmm.s.hEvtMulRendezvousDone);
705	pVM->vmm.s.hEvtMulRendezvousDone = NIL_RTSEMEVENTMULTI;
706	RTSemEventDestroy(pVM->vmm.s.hEvtRendezvousDoneCaller);
707	pVM->vmm.s.hEvtRendezvousDoneCaller = NIL_RTSEMEVENT;
708	RTSemEventMultiDestroy(pVM->vmm.s.hEvtMulRendezvousRecursionPush);
709	pVM->vmm.s.hEvtMulRendezvousRecursionPush = NIL_RTSEMEVENTMULTI;
710	RTSemEventMultiDestroy(pVM->vmm.s.hEvtMulRendezvousRecursionPop);
711	pVM->vmm.s.hEvtMulRendezvousRecursionPop = NIL_RTSEMEVENTMULTI;
712	RTSemEventDestroy(pVM->vmm.s.hEvtRendezvousRecursionPushCaller);
713	pVM->vmm.s.hEvtRendezvousRecursionPushCaller = NIL_RTSEMEVENT;
714	RTSemEventDestroy(pVM->vmm.s.hEvtRendezvousRecursionPopCaller);
715	pVM->vmm.s.hEvtRendezvousRecursionPopCaller = NIL_RTSEMEVENT;
716
717	vmmTermFormatTypes();
718
719	/*
720	* Wait for the log flusher thread to complete.
721	*/
722	if (pVM->vmm.s.hLogFlusherThread != NIL_RTTHREAD)
723	{
724	int rc2 = RTThreadWait(pVM->vmm.s.hLogFlusherThread, RT_MS_30SEC, NULL);
725	AssertLogRelRC(rc2);
726	if (RT_SUCCESS(rc2))
727	pVM->vmm.s.hLogFlusherThread = NIL_RTTHREAD;
728	}
729
730	return rc;
731	}
732
733
734	/**
735	* Applies relocations to data and code managed by this
736	* component. This function will be called at init and
737	* whenever the VMM need to relocate it self inside the GC.
738	*
739	* The VMM will need to apply relocations to the core code.
740	*
741	* @param pVM The cross context VM structure.
742	* @param offDelta The relocation delta.
743	*/
744	VMMR3_INT_DECL(void) VMMR3Relocate(PVM pVM, RTGCINTPTR offDelta)
745	{
746	LogFlow(("VMMR3Relocate: offDelta=%RGv\n", offDelta));
747	RT_NOREF(offDelta);
748
749	/*
750	* Update the logger.
751	*/
752	VMMR3UpdateLoggers(pVM);
753	}
754
755
756	/**
757	* Worker for VMMR3UpdateLoggers.
758	*/
759	static int vmmR3UpdateLoggersWorker(PVM pVM, PVMCPU pVCpu, PRTLOGGER pSrcLogger, bool fReleaseLogger)
760	{
761	/*
762	* Get the group count.
763	*/
764	uint32_t uGroupsCrc32 = 0;
765	uint32_t cGroups = 0;
766	uint64_t fFlags = 0;
767	int rc = RTLogQueryBulk(pSrcLogger, &fFlags, &uGroupsCrc32, &cGroups, NULL);
768	Assert(rc == VERR_BUFFER_OVERFLOW);
769
770	/*
771	* Allocate the request of the right size.
772	*/
773	uint32_t const cbReq = RT_UOFFSETOF_DYN(VMMR0UPDATELOGGERSREQ, afGroups[cGroups]);
774	PVMMR0UPDATELOGGERSREQ pReq = (PVMMR0UPDATELOGGERSREQ)RTMemAllocZVar(cbReq);
775	if (pReq)
776	{
777	pReq->Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
778	pReq->Hdr.cbReq = cbReq;
779	pReq->cGroups = cGroups;
780	rc = RTLogQueryBulk(pSrcLogger, &pReq->fFlags, &pReq->uGroupCrc32, &pReq->cGroups, pReq->afGroups);
781	AssertRC(rc);
782	if (RT_SUCCESS(rc))
783	rc = VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_VMMR0_UPDATE_LOGGERS, fReleaseLogger, &pReq->Hdr);
784
785	RTMemFree(pReq);
786	}
787	else
788	rc = VERR_NO_MEMORY;
789	return rc;
790	}
791
792
793	/**
794	* Updates the settings for the RC and R0 loggers.
795	*
796	* @returns VBox status code.
797	* @param pVM The cross context VM structure.
798	* @thread EMT
799	*/
800	VMMR3_INT_DECL(int) VMMR3UpdateLoggers(PVM pVM)
801	{
802	VM_ASSERT_EMT(pVM);
803	PVMCPU pVCpu = VMMGetCpu(pVM);
804	AssertReturn(pVCpu, VERR_VM_THREAD_NOT_EMT);
805
806	/*
807	* Each EMT has each own logger instance.
808	*/
809	/* Debug logging.*/
810	int rcDebug = VINF_SUCCESS;
811	#ifdef LOG_ENABLED
812	PRTLOGGER const pDefault = RTLogDefaultInstance();
813	if (pDefault)
814	rcDebug = vmmR3UpdateLoggersWorker(pVM, pVCpu, pDefault, false /fReleaseLogger/);
815	#else
816	RT_NOREF(pVM);
817	#endif
818
819	/* Release logging. */
820	int rcRelease = VINF_SUCCESS;
821	PRTLOGGER const pRelease = RTLogRelGetDefaultInstance();
822	if (pRelease)
823	rcRelease = vmmR3UpdateLoggersWorker(pVM, pVCpu, pRelease, true /fReleaseLogger/);
824
825	return RT_SUCCESS(rcDebug) ? rcRelease : rcDebug;
826	}
827
828
829	/**
830	* @callback_method_impl{FNRTTHREAD, Ring-0 log flusher thread.}
831	*/
832	static DECLCALLBACK(int) vmmR3LogFlusher(RTTHREAD hThreadSelf, void *pvUser)
833	{
834	PVM const pVM = (PVM)pvUser;
835	RT_NOREF(hThreadSelf);
836
837	/* Reset the flusher state before we start: */
838	pVM->vmm.s.LogFlusherItem.u32 = UINT32_MAX;
839
840	/*
841	* The work loop.
842	*/
843	for (;;)
844	{
845	/*
846	* Wait for work.
847	*/
848	int rc = SUPR3CallVMMR0Ex(VMCC_GET_VMR0_FOR_CALL(pVM), NIL_VMCPUID, VMMR0_DO_VMMR0_LOG_FLUSHER, 0, NULL);
849	if (RT_SUCCESS(rc))
850	{
851	/* Paranoia: Make another copy of the request, to make sure the validated data can't be changed. */
852	VMMLOGFLUSHERENTRY Item;
853	Item.u32 = pVM->vmm.s.LogFlusherItem.u32;
854	if ( Item.s.idCpu < pVM->cCpus
855	&& Item.s.idxLogger < VMMLOGGER_IDX_MAX
856	&& Item.s.idxBuffer < VMMLOGGER_BUFFER_COUNT)
857	{
858	/*
859	* Verify the request.
860	*/
861	PVMCPU const pVCpu = pVM->apCpusR3[Item.s.idCpu];
862	PVMMR3CPULOGGER const pShared = &pVCpu->vmm.s.u.aLoggers[Item.s.idxLogger];
863	uint32_t const cbToFlush = pShared->aBufs[Item.s.idxBuffer].AuxDesc.offBuf;
864	if (cbToFlush > 0)
865	{
866	if (cbToFlush <= pShared->cbBuf)
867	{
868	char * const pchBufR3 = pShared->aBufs[Item.s.idxBuffer].pchBufR3;
869	if (pchBufR3)
870	{
871	/*
872	* Do the flushing.
873	*/
874	PRTLOGGER const pLogger = Item.s.idxLogger == VMMLOGGER_IDX_REGULAR
875	? RTLogGetDefaultInstance() : RTLogRelGetDefaultInstance();
876	if (pLogger)
877	{
878	char szBefore[128];
879	RTStrPrintf(szBefore, sizeof(szBefore),
880	"FLUSH idCpu=%u idxLogger=%u idxBuffer=%u cbToFlush=%#x fFlushed=%RTbool cbDropped=%#x\n",
881	Item.s.idCpu, Item.s.idxLogger, Item.s.idxBuffer, cbToFlush,
882	pShared->aBufs[Item.s.idxBuffer].AuxDesc.fFlushedIndicator, pShared->cbDropped);
883	RTLogBulkWrite(pLogger, szBefore, pchBufR3, cbToFlush, "FLUSH DONE\n");
884	}
885	}
886	else
887	Log(("vmmR3LogFlusher: idCpu=%u idxLogger=%u idxBuffer=%u cbToFlush=%#x: Warning! No ring-3 buffer pointer!\n",
888	Item.s.idCpu, Item.s.idxLogger, Item.s.idxBuffer, cbToFlush));
889	}
890	else
891	Log(("vmmR3LogFlusher: idCpu=%u idxLogger=%u idxBuffer=%u cbToFlush=%#x: Warning! Exceeds %#x bytes buffer size!\n",
892	Item.s.idCpu, Item.s.idxLogger, Item.s.idxBuffer, cbToFlush, pShared->cbBuf));
893	}
894	else
895	Log(("vmmR3LogFlusher: idCpu=%u idxLogger=%u idxBuffer=%u cbToFlush=%#x: Warning! Zero bytes to flush!\n",
896	Item.s.idCpu, Item.s.idxLogger, Item.s.idxBuffer, cbToFlush));
897
898	/*
899	* Mark the descriptor as flushed and set the request flag for same.
900	*/
901	pShared->aBufs[Item.s.idxBuffer].AuxDesc.fFlushedIndicator = true;
902	}
903	else
904	{
905	Assert(Item.s.idCpu == UINT16_MAX);
906	Assert(Item.s.idxLogger == UINT8_MAX);
907	Assert(Item.s.idxBuffer == UINT8_MAX);
908	}
909	}
910	/*
911	* Interrupted can happen, just ignore it.
912	*/
913	else if (rc == VERR_INTERRUPTED)
914	{ /* ignore*/ }
915	/*
916	* The ring-0 termination code will set the shutdown flag and wake us
917	* up, and we should return with object destroyed. In case there is
918	* some kind of race, we might also get sempahore destroyed.
919	*/
920	else if ( rc == VERR_OBJECT_DESTROYED
921	\|\| rc == VERR_SEM_DESTROYED
922	\|\| rc == VERR_INVALID_HANDLE)
923	{
924	LogRel(("vmmR3LogFlusher: Terminating (%Rrc)\n", rc));
925	return VINF_SUCCESS;
926	}
927	/*
928	* There shouldn't be any other errors...
929	*/
930	else
931	{
932	LogRelMax(64, ("vmmR3LogFlusher: VMMR0_DO_VMMR0_LOG_FLUSHER -> %Rrc\n", rc));
933	AssertRC(rc);
934	RTThreadSleep(1);
935	}
936	}
937	}
938
939
940	/**
941	* Helper for VMM_FLUSH_R0_LOG that does the flushing.
942	*
943	* @param pVM The cross context VM structure.
944	* @param pVCpu The cross context virtual CPU structure of the calling
945	* EMT.
946	* @param pShared The shared logger data.
947	* @param idxBuf The buffer to flush.
948	* @param pDstLogger The destination IPRT logger.
949	*/
950	static void vmmR3LogReturnFlush(PVM pVM, PVMCPU pVCpu, PVMMR3CPULOGGER pShared, size_t idxBuf, PRTLOGGER pDstLogger)
951	{
952	uint32_t const cbToFlush = pShared->aBufs[idxBuf].AuxDesc.offBuf;
953	const char pszBefore = cbToFlush < 256 ? NULL : "FLUSH*\n";
954	const char pszAfter = cbToFlush < 256 ? NULL : "END*\n";
955
956	#if VMMLOGGER_BUFFER_COUNT > 1
957	/*
958	* When we have more than one log buffer, the flusher thread may still be
959	* working on the previous buffer when we get here.
960	*/
961	char szBefore[64];
962	if (pShared->cFlushing > 0)
963	{
964	STAM_REL_PROFILE_START(&pShared->StatRaces, a);
965	uint64_t const nsStart = RTTimeNanoTS();
966
967	/* A no-op, but it takes the lock and the hope is that we end up waiting
968	on the flusher to finish up. */
969	RTLogBulkWrite(pDstLogger, NULL, "", 0, NULL);
970	if (pShared->cFlushing != 0)
971	{
972	RTLogBulkWrite(pDstLogger, NULL, "", 0, NULL);
973
974	/* If no luck, go to ring-0 and to proper waiting. */
975	if (pShared->cFlushing != 0)
976	{
977	STAM_REL_COUNTER_INC(&pShared->StatRacesToR0);
978	SUPR3CallVMMR0Ex(VMCC_GET_VMR0_FOR_CALL(pVM), pVCpu->idCpu, VMMR0_DO_VMMR0_LOG_WAIT_FLUSHED, 0, NULL);
979	}
980	}
981
982	RTStrPrintf(szBefore, sizeof(szBefore), "%sFLUSH waited %'RU64 ns\n",
983	pShared->cFlushing == 0 ? "" : " MISORDERED", RTTimeNanoTS() - nsStart);
984	pszBefore = szBefore;
985	STAM_REL_PROFILE_STOP(&pShared->StatRaces, a);
986	}
987	#else
988	RT_NOREF(pVM, pVCpu);
989	#endif
990
991	RTLogBulkWrite(pDstLogger, pszBefore, pShared->aBufs[idxBuf].pchBufR3, cbToFlush, pszAfter);
992	pShared->aBufs[idxBuf].AuxDesc.fFlushedIndicator = true;
993	}
994
995
996	/**
997	* Gets the pointer to a buffer containing the R0/RC RTAssertMsg1Weak output.
998	*
999	* @returns Pointer to the buffer.
1000	* @param pVM The cross context VM structure.
1001	*/
1002	VMMR3DECL(const char *) VMMR3GetRZAssertMsg1(PVM pVM)
1003	{
1004	return pVM->vmm.s.szRing0AssertMsg1;
1005	}
1006
1007
1008	/**
1009	* Returns the VMCPU of the specified virtual CPU.
1010	*
1011	* @returns The VMCPU pointer. NULL if @a idCpu or @a pUVM is invalid.
1012	*
1013	* @param pUVM The user mode VM handle.
1014	* @param idCpu The ID of the virtual CPU.
1015	*/
1016	VMMR3DECL(PVMCPU) VMMR3GetCpuByIdU(PUVM pUVM, RTCPUID idCpu)
1017	{
1018	UVM_ASSERT_VALID_EXT_RETURN(pUVM, NULL);
1019	AssertReturn(idCpu < pUVM->cCpus, NULL);
1020	VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, NULL);
1021	return pUVM->pVM->apCpusR3[idCpu];
1022	}
1023
1024
1025	/**
1026	* Gets the pointer to a buffer containing the R0/RC RTAssertMsg2Weak output.
1027	*
1028	* @returns Pointer to the buffer.
1029	* @param pVM The cross context VM structure.
1030	*/
1031	VMMR3DECL(const char *) VMMR3GetRZAssertMsg2(PVM pVM)
1032	{
1033	return pVM->vmm.s.szRing0AssertMsg2;
1034	}
1035
1036
1037	/**
1038	* Execute state save operation.
1039	*
1040	* @returns VBox status code.
1041	* @param pVM The cross context VM structure.
1042	* @param pSSM SSM operation handle.
1043	*/
1044	static DECLCALLBACK(int) vmmR3Save(PVM pVM, PSSMHANDLE pSSM)
1045	{
1046	LogFlow(("vmmR3Save:\n"));
1047
1048	/*
1049	* Save the started/stopped state of all CPUs except 0 as it will always
1050	* be running. This avoids breaking the saved state version. :-)
1051	*/
1052	for (VMCPUID i = 1; i < pVM->cCpus; i++)
1053	SSMR3PutBool(pSSM, VMCPUSTATE_IS_STARTED(VMCPU_GET_STATE(pVM->apCpusR3[i])));
1054
1055	return SSMR3PutU32(pSSM, UINT32_MAX); /* terminator */
1056	}
1057
1058
1059	/**
1060	* Execute state load operation.
1061	*
1062	* @returns VBox status code.
1063	* @param pVM The cross context VM structure.
1064	* @param pSSM SSM operation handle.
1065	* @param uVersion Data layout version.
1066	* @param uPass The data pass.
1067	*/
1068	static DECLCALLBACK(int) vmmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
1069	{
1070	LogFlow(("vmmR3Load:\n"));
1071	Assert(uPass == SSM_PASS_FINAL); NOREF(uPass);
1072
1073	/*
1074	* Validate version.
1075	*/
1076	if ( uVersion != VMM_SAVED_STATE_VERSION
1077	&& uVersion != VMM_SAVED_STATE_VERSION_3_0)
1078	{
1079	AssertMsgFailed(("vmmR3Load: Invalid version uVersion=%u!\n", uVersion));
1080	return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
1081	}
1082
1083	if (uVersion <= VMM_SAVED_STATE_VERSION_3_0)
1084	{
1085	/* Ignore the stack bottom, stack pointer and stack bits. */
1086	RTRCPTR RCPtrIgnored;
1087	SSMR3GetRCPtr(pSSM, &RCPtrIgnored);
1088	SSMR3GetRCPtr(pSSM, &RCPtrIgnored);
1089	#ifdef RT_OS_DARWIN
1090	if ( SSMR3HandleVersion(pSSM) >= VBOX_FULL_VERSION_MAKE(3,0,0)
1091	&& SSMR3HandleVersion(pSSM) < VBOX_FULL_VERSION_MAKE(3,1,0)
1092	&& SSMR3HandleRevision(pSSM) >= 48858
1093	&& ( !strcmp(SSMR3HandleHostOSAndArch(pSSM), "darwin.x86")
1094	\|\| !strcmp(SSMR3HandleHostOSAndArch(pSSM), "") )
1095	)
1096	SSMR3Skip(pSSM, 16384);
1097	else
1098	SSMR3Skip(pSSM, 8192);
1099	#else
1100	SSMR3Skip(pSSM, 8192);
1101	#endif
1102	}
1103
1104	/*
1105	* Restore the VMCPU states. VCPU 0 is always started.
1106	*/
1107	VMCPU_SET_STATE(pVM->apCpusR3[0], VMCPUSTATE_STARTED);
1108	for (VMCPUID i = 1; i < pVM->cCpus; i++)
1109	{
1110	bool fStarted;
1111	int rc = SSMR3GetBool(pSSM, &fStarted);
1112	if (RT_FAILURE(rc))
1113	return rc;
1114	VMCPU_SET_STATE(pVM->apCpusR3[i], fStarted ? VMCPUSTATE_STARTED : VMCPUSTATE_STOPPED);
1115	}
1116
1117	/* terminator */
1118	uint32_t u32;
1119	int rc = SSMR3GetU32(pSSM, &u32);
1120	if (RT_FAILURE(rc))
1121	return rc;
1122	if (u32 != UINT32_MAX)
1123	{
1124	AssertMsgFailed(("u32=%#x\n", u32));
1125	return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
1126	}
1127	return VINF_SUCCESS;
1128	}
1129
1130
1131	/**
1132	* Suspends the CPU yielder.
1133	*
1134	* @param pVM The cross context VM structure.
1135	*/
1136	VMMR3_INT_DECL(void) VMMR3YieldSuspend(PVM pVM)
1137	{
1138	#if 0 /* pointless when timers doesn't run on EMT */
1139	VMCPU_ASSERT_EMT(pVM->apCpusR3[0]);
1140	if (!pVM->vmm.s.cYieldResumeMillies)
1141	{
1142	uint64_t u64Now = TMTimerGet(pVM, pVM->vmm.s.hYieldTimer);
1143	uint64_t u64Expire = TMTimerGetExpire(pVM, pVM->vmm.s.hYieldTimer);
1144	if (u64Now >= u64Expire \|\| u64Expire == ~(uint64_t)0)
1145	pVM->vmm.s.cYieldResumeMillies = pVM->vmm.s.cYieldEveryMillies;
1146	else
1147	pVM->vmm.s.cYieldResumeMillies = TMTimerToMilli(pVM, pVM->vmm.s.hYieldTimer, u64Expire - u64Now);
1148	TMTimerStop(pVM, pVM->vmm.s.hYieldTimer);
1149	}
1150	pVM->vmm.s.u64LastYield = RTTimeNanoTS();
1151	#else
1152	RT_NOREF(pVM);
1153	#endif
1154	}
1155
1156
1157	/**
1158	* Stops the CPU yielder.
1159	*
1160	* @param pVM The cross context VM structure.
1161	*/
1162	VMMR3_INT_DECL(void) VMMR3YieldStop(PVM pVM)
1163	{
1164	#if 0 /* pointless when timers doesn't run on EMT */
1165	if (!pVM->vmm.s.cYieldResumeMillies)
1166	TMTimerStop(pVM, pVM->vmm.s.hYieldTimer);
1167	pVM->vmm.s.cYieldResumeMillies = pVM->vmm.s.cYieldEveryMillies;
1168	pVM->vmm.s.u64LastYield = RTTimeNanoTS();
1169	#else
1170	RT_NOREF(pVM);
1171	#endif
1172	}
1173
1174
1175	/**
1176	* Resumes the CPU yielder when it has been a suspended or stopped.
1177	*
1178	* @param pVM The cross context VM structure.
1179	*/
1180	VMMR3_INT_DECL(void) VMMR3YieldResume(PVM pVM)
1181	{
1182	#if 0 /* pointless when timers doesn't run on EMT */
1183	if (pVM->vmm.s.cYieldResumeMillies)
1184	{
1185	TMTimerSetMillies(pVM, pVM->vmm.s.hYieldTimer, pVM->vmm.s.cYieldResumeMillies);
1186	pVM->vmm.s.cYieldResumeMillies = 0;
1187	}
1188	#else
1189	RT_NOREF(pVM);
1190	#endif
1191	}
1192
1193
1194	#if 0 /* pointless when timers doesn't run on EMT */
1195	/**
1196	* @callback_method_impl{FNTMTIMERINT, EMT yielder}
1197	*
1198	* @todo This is a UNI core/thread thing, really... Should be reconsidered.
1199	*/
1200	static DECLCALLBACK(void) vmmR3YieldEMT(PVM pVM, TMTIMERHANDLE hTimer, void *pvUser)
1201	{
1202	NOREF(pvUser);
1203
1204	/*
1205	* This really needs some careful tuning. While we shouldn't be too greedy since
1206	* that'll cause the rest of the system to stop up, we shouldn't be too nice either
1207	* because that'll cause us to stop up.
1208	*
1209	* The current logic is to use the default interval when there is no lag worth
1210	* mentioning, but when we start accumulating lag we don't bother yielding at all.
1211	*
1212	* (This depends on the TMCLOCK_VIRTUAL_SYNC to be scheduled before TMCLOCK_REAL
1213	* so the lag is up to date.)
1214	*/
1215	const uint64_t u64Lag = TMVirtualSyncGetLag(pVM);
1216	if ( u64Lag < 50000000 /* 50ms */
1217	\|\| ( u64Lag < 1000000000 /* 1s */
1218	&& RTTimeNanoTS() - pVM->vmm.s.u64LastYield < 500000000 /* 500 ms */)
1219	)
1220	{
1221	uint64_t u64Elapsed = RTTimeNanoTS();
1222	pVM->vmm.s.u64LastYield = u64Elapsed;
1223
1224	RTThreadYield();
1225
1226	#ifdef LOG_ENABLED
1227	u64Elapsed = RTTimeNanoTS() - u64Elapsed;
1228	Log(("vmmR3YieldEMT: %RI64 ns\n", u64Elapsed));
1229	#endif
1230	}
1231	TMTimerSetMillies(pVM, hTimer, pVM->vmm.s.cYieldEveryMillies);
1232	}
1233	#endif
1234
1235
1236	/**
1237	* Executes guest code (Intel VT-x and AMD-V).
1238	*
1239	* @param pVM The cross context VM structure.
1240	* @param pVCpu The cross context virtual CPU structure.
1241	*/
1242	VMMR3_INT_DECL(int) VMMR3HmRunGC(PVM pVM, PVMCPU pVCpu)
1243	{
1244	Log2(("VMMR3HmRunGC: (cs:rip=%04x:%RX64)\n", CPUMGetGuestCS(pVCpu), CPUMGetGuestRIP(pVCpu)));
1245
1246	for (;;)
1247	{
1248	int rc;
1249	do
1250	{
1251	#ifdef NO_SUPCALLR0VMM
1252	rc = VERR_GENERAL_FAILURE;
1253	#else
1254	rc = SUPR3CallVMMR0Fast(VMCC_GET_VMR0_FOR_CALL(pVM), VMMR0_DO_HM_RUN, pVCpu->idCpu);
1255	if (RT_LIKELY(rc == VINF_SUCCESS))
1256	rc = pVCpu->vmm.s.iLastGZRc;
1257	#endif
1258	} while (rc == VINF_EM_RAW_INTERRUPT_HYPER);
1259
1260	#if 0 /** @todo triggers too often */
1261	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TO_R3));
1262	#endif
1263
1264	/*
1265	* Flush the logs
1266	*/
1267	#ifdef LOG_ENABLED
1268	VMM_FLUSH_R0_LOG(pVM, pVCpu, &pVCpu->vmm.s.u.s.Logger, NULL);
1269	#endif
1270	VMM_FLUSH_R0_LOG(pVM, pVCpu, &pVCpu->vmm.s.u.s.RelLogger, RTLogRelGetDefaultInstance());
1271	if (rc != VINF_VMM_CALL_HOST)
1272	{
1273	Log2(("VMMR3HmRunGC: returns %Rrc (cs:rip=%04x:%RX64)\n", rc, CPUMGetGuestCS(pVCpu), CPUMGetGuestRIP(pVCpu)));
1274	return rc;
1275	}
1276	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
1277	if (RT_FAILURE(rc))
1278	return rc;
1279	/* Resume R0 */
1280	}
1281	}
1282
1283
1284	/**
1285	* Perform one of the fast I/O control VMMR0 operation.
1286	*
1287	* @returns VBox strict status code.
1288	* @param pVM The cross context VM structure.
1289	* @param pVCpu The cross context virtual CPU structure.
1290	* @param enmOperation The operation to perform.
1291	*/
1292	VMMR3_INT_DECL(VBOXSTRICTRC) VMMR3CallR0EmtFast(PVM pVM, PVMCPU pVCpu, VMMR0OPERATION enmOperation)
1293	{
1294	for (;;)
1295	{
1296	VBOXSTRICTRC rcStrict;
1297	do
1298	{
1299	#ifdef NO_SUPCALLR0VMM
1300	rcStrict = VERR_GENERAL_FAILURE;
1301	#else
1302	rcStrict = SUPR3CallVMMR0Fast(VMCC_GET_VMR0_FOR_CALL(pVM), enmOperation, pVCpu->idCpu);
1303	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
1304	rcStrict = pVCpu->vmm.s.iLastGZRc;
1305	#endif
1306	} while (rcStrict == VINF_EM_RAW_INTERRUPT_HYPER);
1307
1308	/*
1309	* Flush the logs
1310	*/
1311	#ifdef LOG_ENABLED
1312	VMM_FLUSH_R0_LOG(pVM, pVCpu, &pVCpu->vmm.s.u.s.Logger, NULL);
1313	#endif
1314	VMM_FLUSH_R0_LOG(pVM, pVCpu, &pVCpu->vmm.s.u.s.RelLogger, RTLogRelGetDefaultInstance());
1315	if (rcStrict != VINF_VMM_CALL_HOST)
1316	return rcStrict;
1317	int rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
1318	if (RT_FAILURE(rc))
1319	return rc;
1320	/* Resume R0 */
1321	}
1322	}
1323
1324
1325	/**
1326	* VCPU worker for VMMR3SendStartupIpi.
1327	*
1328	* @param pVM The cross context VM structure.
1329	* @param idCpu Virtual CPU to perform SIPI on.
1330	* @param uVector The SIPI vector.
1331	*/
1332	static DECLCALLBACK(int) vmmR3SendStarupIpi(PVM pVM, VMCPUID idCpu, uint32_t uVector)
1333	{
1334	PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
1335	VMCPU_ASSERT_EMT(pVCpu);
1336
1337	/*
1338	* In the INIT state, the target CPU is only responsive to an SIPI.
1339	* This is also true for when when the CPU is in VMX non-root mode.
1340	*
1341	* See AMD spec. 16.5 "Interprocessor Interrupts (IPI)".
1342	* See Intel spec. 26.6.2 "Activity State".
1343	*/
1344	if (EMGetState(pVCpu) != EMSTATE_WAIT_SIPI)
1345	return VINF_SUCCESS;
1346
1347	PCPUMCTX pCtx = CPUMQueryGuestCtxPtr(pVCpu);
1348	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1349	if (CPUMIsGuestInVmxRootMode(pCtx))
1350	{
1351	/* If the CPU is in VMX non-root mode we must cause a VM-exit. */
1352	if (CPUMIsGuestInVmxNonRootMode(pCtx))
1353	return VBOXSTRICTRC_TODO(IEMExecVmxVmexitStartupIpi(pVCpu, uVector));
1354
1355	/* If the CPU is in VMX root mode (and not in VMX non-root mode) SIPIs are blocked. */
1356	return VINF_SUCCESS;
1357	}
1358	#endif
1359
1360	pCtx->cs.Sel = uVector << 8;
1361	pCtx->cs.ValidSel = uVector << 8;
1362	pCtx->cs.fFlags = CPUMSELREG_FLAGS_VALID;
1363	pCtx->cs.u64Base = uVector << 12;
1364	pCtx->cs.u32Limit = UINT32_C(0x0000ffff);
1365	pCtx->rip = 0;
1366
1367	Log(("vmmR3SendSipi for VCPU %d with vector %x\n", idCpu, uVector));
1368
1369	# if 1 /* If we keep the EMSTATE_WAIT_SIPI method, then move this to EM.cpp. */
1370	EMSetState(pVCpu, EMSTATE_HALTED);
1371	return VINF_EM_RESCHEDULE;
1372	# else /* And if we go the VMCPU::enmState way it can stay here. */
1373	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STOPPED);
1374	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
1375	return VINF_SUCCESS;
1376	# endif
1377	}
1378
1379
1380	/**
1381	* VCPU worker for VMMR3SendInitIpi.
1382	*
1383	* @returns VBox status code.
1384	* @param pVM The cross context VM structure.
1385	* @param idCpu Virtual CPU to perform SIPI on.
1386	*/
1387	static DECLCALLBACK(int) vmmR3SendInitIpi(PVM pVM, VMCPUID idCpu)
1388	{
1389	PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
1390	VMCPU_ASSERT_EMT(pVCpu);
1391
1392	Log(("vmmR3SendInitIpi for VCPU %d\n", idCpu));
1393
1394	/** @todo r=ramshankar: We should probably block INIT signal when the CPU is in
1395	* wait-for-SIPI state. Verify. */
1396
1397	/* If the CPU is in VMX non-root mode, INIT signals cause VM-exits. */
1398	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1399	PCCPUMCTX pCtx = CPUMQueryGuestCtxPtr(pVCpu);
1400	if (CPUMIsGuestInVmxNonRootMode(pCtx))
1401	return VBOXSTRICTRC_TODO(IEMExecVmxVmexit(pVCpu, VMX_EXIT_INIT_SIGNAL, 0 /* uExitQual */));
1402	#endif
1403
1404	/** @todo Figure out how to handle a SVM nested-guest intercepts here for INIT
1405	* IPI (e.g. SVM_EXIT_INIT). */
1406
1407	PGMR3ResetCpu(pVM, pVCpu);
1408	PDMR3ResetCpu(pVCpu); /* Only clears pending interrupts force flags */
1409	APICR3InitIpi(pVCpu);
1410	TRPMR3ResetCpu(pVCpu);
1411	CPUMR3ResetCpu(pVM, pVCpu);
1412	EMR3ResetCpu(pVCpu);
1413	HMR3ResetCpu(pVCpu);
1414	NEMR3ResetCpu(pVCpu, true /fInitIpi/);
1415
1416	/* This will trickle up on the target EMT. */
1417	return VINF_EM_WAIT_SIPI;
1418	}
1419
1420
1421	/**
1422	* Sends a Startup IPI to the virtual CPU by setting CS:EIP into
1423	* vector-dependent state and unhalting processor.
1424	*
1425	* @param pVM The cross context VM structure.
1426	* @param idCpu Virtual CPU to perform SIPI on.
1427	* @param uVector SIPI vector.
1428	*/
1429	VMMR3_INT_DECL(void) VMMR3SendStartupIpi(PVM pVM, VMCPUID idCpu, uint32_t uVector)
1430	{
1431	AssertReturnVoid(idCpu < pVM->cCpus);
1432
1433	int rc = VMR3ReqCallNoWait(pVM, idCpu, (PFNRT)vmmR3SendStarupIpi, 3, pVM, idCpu, uVector);
1434	AssertRC(rc);
1435	}
1436
1437
1438	/**
1439	* Sends init IPI to the virtual CPU.
1440	*
1441	* @param pVM The cross context VM structure.
1442	* @param idCpu Virtual CPU to perform int IPI on.
1443	*/
1444	VMMR3_INT_DECL(void) VMMR3SendInitIpi(PVM pVM, VMCPUID idCpu)
1445	{
1446	AssertReturnVoid(idCpu < pVM->cCpus);
1447
1448	int rc = VMR3ReqCallNoWait(pVM, idCpu, (PFNRT)vmmR3SendInitIpi, 2, pVM, idCpu);
1449	AssertRC(rc);
1450	}
1451
1452
1453	/**
1454	* Registers the guest memory range that can be used for patching.
1455	*
1456	* @returns VBox status code.
1457	* @param pVM The cross context VM structure.
1458	* @param pPatchMem Patch memory range.
1459	* @param cbPatchMem Size of the memory range.
1460	*/
1461	VMMR3DECL(int) VMMR3RegisterPatchMemory(PVM pVM, RTGCPTR pPatchMem, unsigned cbPatchMem)
1462	{
1463	VM_ASSERT_EMT(pVM);
1464	if (HMIsEnabled(pVM))
1465	return HMR3EnablePatching(pVM, pPatchMem, cbPatchMem);
1466
1467	return VERR_NOT_SUPPORTED;
1468	}
1469
1470
1471	/**
1472	* Deregisters the guest memory range that can be used for patching.
1473	*
1474	* @returns VBox status code.
1475	* @param pVM The cross context VM structure.
1476	* @param pPatchMem Patch memory range.
1477	* @param cbPatchMem Size of the memory range.
1478	*/
1479	VMMR3DECL(int) VMMR3DeregisterPatchMemory(PVM pVM, RTGCPTR pPatchMem, unsigned cbPatchMem)
1480	{
1481	if (HMIsEnabled(pVM))
1482	return HMR3DisablePatching(pVM, pPatchMem, cbPatchMem);
1483
1484	return VINF_SUCCESS;
1485	}
1486
1487
1488	/**
1489	* Common recursion handler for the other EMTs.
1490	*
1491	* @returns Strict VBox status code.
1492	* @param pVM The cross context VM structure.
1493	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1494	* @param rcStrict Current status code to be combined with the one
1495	* from this recursion and returned.
1496	*/
1497	static VBOXSTRICTRC vmmR3EmtRendezvousCommonRecursion(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rcStrict)
1498	{
1499	int rc2;
1500
1501	/*
1502	* We wait here while the initiator of this recursion reconfigures
1503	* everything. The last EMT to get in signals the initiator.
1504	*/
1505	if (ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsRecursingPush) == pVM->cCpus)
1506	{
1507	rc2 = RTSemEventSignal(pVM->vmm.s.hEvtRendezvousRecursionPushCaller);
1508	AssertLogRelRC(rc2);
1509	}
1510
1511	rc2 = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousRecursionPush, RT_INDEFINITE_WAIT);
1512	AssertLogRelRC(rc2);
1513
1514	/*
1515	* Do the normal rendezvous processing.
1516	*/
1517	VBOXSTRICTRC rcStrict2 = vmmR3EmtRendezvousCommon(pVM, pVCpu, false /* fIsCaller */, pVM->vmm.s.fRendezvousFlags,
1518	pVM->vmm.s.pfnRendezvous, pVM->vmm.s.pvRendezvousUser);
1519
1520	/*
1521	* Wait for the initiator to restore everything.
1522	*/
1523	rc2 = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousRecursionPop, RT_INDEFINITE_WAIT);
1524	AssertLogRelRC(rc2);
1525
1526	/*
1527	* Last thread out of here signals the initiator.
1528	*/
1529	if (ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsRecursingPop) == pVM->cCpus)
1530	{
1531	rc2 = RTSemEventSignal(pVM->vmm.s.hEvtRendezvousRecursionPopCaller);
1532	AssertLogRelRC(rc2);
1533	}
1534
1535	/*
1536	* Merge status codes and return.
1537	*/
1538	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
1539	if ( rcStrict2 != VINF_SUCCESS
1540	&& ( rcStrict == VINF_SUCCESS
1541	\|\| rcStrict > rcStrict2))
1542	rcStrict = rcStrict2;
1543	return rcStrict;
1544	}
1545
1546
1547	/**
1548	* Count returns and have the last non-caller EMT wake up the caller.
1549	*
1550	* @returns VBox strict informational status code for EM scheduling. No failures
1551	* will be returned here, those are for the caller only.
1552	*
1553	* @param pVM The cross context VM structure.
1554	* @param rcStrict The current accumulated recursive status code,
1555	* to be merged with i32RendezvousStatus and
1556	* returned.
1557	*/
1558	DECL_FORCE_INLINE(VBOXSTRICTRC) vmmR3EmtRendezvousNonCallerReturn(PVM pVM, VBOXSTRICTRC rcStrict)
1559	{
1560	VBOXSTRICTRC rcStrict2 = ASMAtomicReadS32(&pVM->vmm.s.i32RendezvousStatus);
1561
1562	uint32_t cReturned = ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsReturned);
1563	if (cReturned == pVM->cCpus - 1U)
1564	{
1565	int rc = RTSemEventSignal(pVM->vmm.s.hEvtRendezvousDoneCaller);
1566	AssertLogRelRC(rc);
1567	}
1568
1569	/*
1570	* Merge the status codes, ignoring error statuses in this code path.
1571	*/
1572	AssertLogRelMsgReturn( rcStrict2 <= VINF_SUCCESS
1573	\|\| (rcStrict2 >= VINF_EM_FIRST && rcStrict2 <= VINF_EM_LAST),
1574	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)),
1575	VERR_IPE_UNEXPECTED_INFO_STATUS);
1576
1577	if (RT_SUCCESS(rcStrict2))
1578	{
1579	if ( rcStrict2 != VINF_SUCCESS
1580	&& ( rcStrict == VINF_SUCCESS
1581	\|\| rcStrict > rcStrict2))
1582	rcStrict = rcStrict2;
1583	}
1584	return rcStrict;
1585	}
1586
1587
1588	/**
1589	* Common worker for VMMR3EmtRendezvous and VMMR3EmtRendezvousFF.
1590	*
1591	* @returns VBox strict informational status code for EM scheduling. No failures
1592	* will be returned here, those are for the caller only. When
1593	* fIsCaller is set, VINF_SUCCESS is always returned.
1594	*
1595	* @param pVM The cross context VM structure.
1596	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1597	* @param fIsCaller Whether we're the VMMR3EmtRendezvous caller or
1598	* not.
1599	* @param fFlags The flags.
1600	* @param pfnRendezvous The callback.
1601	* @param pvUser The user argument for the callback.
1602	*/
1603	static VBOXSTRICTRC vmmR3EmtRendezvousCommon(PVM pVM, PVMCPU pVCpu, bool fIsCaller,
1604	uint32_t fFlags, PFNVMMEMTRENDEZVOUS pfnRendezvous, void *pvUser)
1605	{
1606	int rc;
1607	VBOXSTRICTRC rcStrictRecursion = VINF_SUCCESS;
1608
1609	/*
1610	* Enter, the last EMT triggers the next callback phase.
1611	*/
1612	uint32_t cEntered = ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsEntered);
1613	if (cEntered != pVM->cCpus)
1614	{
1615	if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE)
1616	{
1617	/* Wait for our turn. */
1618	for (;;)
1619	{
1620	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousEnterOneByOne, RT_INDEFINITE_WAIT);
1621	AssertLogRelRC(rc);
1622	if (!pVM->vmm.s.fRendezvousRecursion)
1623	break;
1624	rcStrictRecursion = vmmR3EmtRendezvousCommonRecursion(pVM, pVCpu, rcStrictRecursion);
1625	}
1626	}
1627	else if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE)
1628	{
1629	/* Wait for the last EMT to arrive and wake everyone up. */
1630	rc = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce, RT_INDEFINITE_WAIT);
1631	AssertLogRelRC(rc);
1632	Assert(!pVM->vmm.s.fRendezvousRecursion);
1633	}
1634	else if ( (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
1635	\|\| (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING)
1636	{
1637	/* Wait for our turn. */
1638	for (;;)
1639	{
1640	rc = RTSemEventWait(pVM->vmm.s.pahEvtRendezvousEnterOrdered[pVCpu->idCpu], RT_INDEFINITE_WAIT);
1641	AssertLogRelRC(rc);
1642	if (!pVM->vmm.s.fRendezvousRecursion)
1643	break;
1644	rcStrictRecursion = vmmR3EmtRendezvousCommonRecursion(pVM, pVCpu, rcStrictRecursion);
1645	}
1646	}
1647	else
1648	{
1649	Assert((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE);
1650
1651	/*
1652	* The execute once is handled specially to optimize the code flow.
1653	*
1654	* The last EMT to arrive will perform the callback and the other
1655	* EMTs will wait on the Done/DoneCaller semaphores (instead of
1656	* the EnterOneByOne/AllAtOnce) in the meanwhile. When the callback
1657	* returns, that EMT will initiate the normal return sequence.
1658	*/
1659	if (!fIsCaller)
1660	{
1661	for (;;)
1662	{
1663	rc = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousDone, RT_INDEFINITE_WAIT);
1664	AssertLogRelRC(rc);
1665	if (!pVM->vmm.s.fRendezvousRecursion)
1666	break;
1667	rcStrictRecursion = vmmR3EmtRendezvousCommonRecursion(pVM, pVCpu, rcStrictRecursion);
1668	}
1669
1670	return vmmR3EmtRendezvousNonCallerReturn(pVM, rcStrictRecursion);
1671	}
1672	return VINF_SUCCESS;
1673	}
1674	}
1675	else
1676	{
1677	/*
1678	* All EMTs are waiting, clear the FF and take action according to the
1679	* execution method.
1680	*/
1681	VM_FF_CLEAR(pVM, VM_FF_EMT_RENDEZVOUS);
1682
1683	if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE)
1684	{
1685	/* Wake up everyone. */
1686	rc = RTSemEventMultiSignal(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce);
1687	AssertLogRelRC(rc);
1688	}
1689	else if ( (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
1690	\|\| (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING)
1691	{
1692	/* Figure out who to wake up and wake it up. If it's ourself, then
1693	it's easy otherwise wait for our turn. */
1694	VMCPUID iFirst = (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
1695	? 0
1696	: pVM->cCpus - 1U;
1697	if (pVCpu->idCpu != iFirst)
1698	{
1699	rc = RTSemEventSignal(pVM->vmm.s.pahEvtRendezvousEnterOrdered[iFirst]);
1700	AssertLogRelRC(rc);
1701	for (;;)
1702	{
1703	rc = RTSemEventWait(pVM->vmm.s.pahEvtRendezvousEnterOrdered[pVCpu->idCpu], RT_INDEFINITE_WAIT);
1704	AssertLogRelRC(rc);
1705	if (!pVM->vmm.s.fRendezvousRecursion)
1706	break;
1707	rcStrictRecursion = vmmR3EmtRendezvousCommonRecursion(pVM, pVCpu, rcStrictRecursion);
1708	}
1709	}
1710	}
1711	/* else: execute the handler on the current EMT and wake up one or more threads afterwards. */
1712	}
1713
1714
1715	/*
1716	* Do the callback and update the status if necessary.
1717	*/
1718	if ( !(fFlags & VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR)
1719	\|\| RT_SUCCESS(ASMAtomicUoReadS32(&pVM->vmm.s.i32RendezvousStatus)) )
1720	{
1721	VBOXSTRICTRC rcStrict2 = pfnRendezvous(pVM, pVCpu, pvUser);
1722	if (rcStrict2 != VINF_SUCCESS)
1723	{
1724	AssertLogRelMsg( rcStrict2 <= VINF_SUCCESS
1725	\|\| (rcStrict2 >= VINF_EM_FIRST && rcStrict2 <= VINF_EM_LAST),
1726	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
1727	int32_t i32RendezvousStatus;
1728	do
1729	{
1730	i32RendezvousStatus = ASMAtomicUoReadS32(&pVM->vmm.s.i32RendezvousStatus);
1731	if ( rcStrict2 == i32RendezvousStatus
1732	\|\| RT_FAILURE(i32RendezvousStatus)
1733	\|\| ( i32RendezvousStatus != VINF_SUCCESS
1734	&& rcStrict2 > i32RendezvousStatus))
1735	break;
1736	} while (!ASMAtomicCmpXchgS32(&pVM->vmm.s.i32RendezvousStatus, VBOXSTRICTRC_VAL(rcStrict2), i32RendezvousStatus));
1737	}
1738	}
1739
1740	/*
1741	* Increment the done counter and take action depending on whether we're
1742	* the last to finish callback execution.
1743	*/
1744	uint32_t cDone = ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsDone);
1745	if ( cDone != pVM->cCpus
1746	&& (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE)
1747	{
1748	/* Signal the next EMT? */
1749	if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE)
1750	{
1751	rc = RTSemEventSignal(pVM->vmm.s.hEvtRendezvousEnterOneByOne);
1752	AssertLogRelRC(rc);
1753	}
1754	else if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING)
1755	{
1756	Assert(cDone == pVCpu->idCpu + 1U);
1757	rc = RTSemEventSignal(pVM->vmm.s.pahEvtRendezvousEnterOrdered[pVCpu->idCpu + 1U]);
1758	AssertLogRelRC(rc);
1759	}
1760	else if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING)
1761	{
1762	Assert(pVM->cCpus - cDone == pVCpu->idCpu);
1763	rc = RTSemEventSignal(pVM->vmm.s.pahEvtRendezvousEnterOrdered[pVM->cCpus - cDone - 1U]);
1764	AssertLogRelRC(rc);
1765	}
1766
1767	/* Wait for the rest to finish (the caller waits on hEvtRendezvousDoneCaller). */
1768	if (!fIsCaller)
1769	{
1770	for (;;)
1771	{
1772	rc = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousDone, RT_INDEFINITE_WAIT);
1773	AssertLogRelRC(rc);
1774	if (!pVM->vmm.s.fRendezvousRecursion)
1775	break;
1776	rcStrictRecursion = vmmR3EmtRendezvousCommonRecursion(pVM, pVCpu, rcStrictRecursion);
1777	}
1778	}
1779	}
1780	else
1781	{
1782	/* Callback execution is all done, tell the rest to return. */
1783	rc = RTSemEventMultiSignal(pVM->vmm.s.hEvtMulRendezvousDone);
1784	AssertLogRelRC(rc);
1785	}
1786
1787	if (!fIsCaller)
1788	return vmmR3EmtRendezvousNonCallerReturn(pVM, rcStrictRecursion);
1789	return rcStrictRecursion;
1790	}
1791
1792
1793	/**
1794	* Called in response to VM_FF_EMT_RENDEZVOUS.
1795	*
1796	* @returns VBox strict status code - EM scheduling. No errors will be returned
1797	* here, nor will any non-EM scheduling status codes be returned.
1798	*
1799	* @param pVM The cross context VM structure.
1800	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1801	*
1802	* @thread EMT
1803	*/
1804	VMMR3_INT_DECL(int) VMMR3EmtRendezvousFF(PVM pVM, PVMCPU pVCpu)
1805	{
1806	Assert(!pVCpu->vmm.s.fInRendezvous);
1807	Log(("VMMR3EmtRendezvousFF: EMT%#u\n", pVCpu->idCpu));
1808	pVCpu->vmm.s.fInRendezvous = true;
1809	VBOXSTRICTRC rcStrict = vmmR3EmtRendezvousCommon(pVM, pVCpu, false /* fIsCaller */, pVM->vmm.s.fRendezvousFlags,
1810	pVM->vmm.s.pfnRendezvous, pVM->vmm.s.pvRendezvousUser);
1811	pVCpu->vmm.s.fInRendezvous = false;
1812	Log(("VMMR3EmtRendezvousFF: EMT%#u returns %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict)));
1813	return VBOXSTRICTRC_TODO(rcStrict);
1814	}
1815
1816
1817	/**
1818	* Helper for resetting an single wakeup event sempahore.
1819	*
1820	* @returns VERR_TIMEOUT on success, RTSemEventWait status otherwise.
1821	* @param hEvt The event semaphore to reset.
1822	*/
1823	static int vmmR3HlpResetEvent(RTSEMEVENT hEvt)
1824	{
1825	for (uint32_t cLoops = 0; ; cLoops++)
1826	{
1827	int rc = RTSemEventWait(hEvt, 0 /cMsTimeout/);
1828	if (rc != VINF_SUCCESS \|\| cLoops > _4K)
1829	return rc;
1830	}
1831	}
1832
1833
1834	/**
1835	* Worker for VMMR3EmtRendezvous that handles recursion.
1836	*
1837	* @returns VBox strict status code. This will be the first error,
1838	* VINF_SUCCESS, or an EM scheduling status code.
1839	*
1840	* @param pVM The cross context VM structure.
1841	* @param pVCpu The cross context virtual CPU structure of the
1842	* calling EMT.
1843	* @param fFlags Flags indicating execution methods. See
1844	* grp_VMMR3EmtRendezvous_fFlags.
1845	* @param pfnRendezvous The callback.
1846	* @param pvUser User argument for the callback.
1847	*
1848	* @thread EMT(pVCpu)
1849	*/
1850	static VBOXSTRICTRC vmmR3EmtRendezvousRecursive(PVM pVM, PVMCPU pVCpu, uint32_t fFlags,
1851	PFNVMMEMTRENDEZVOUS pfnRendezvous, void *pvUser)
1852	{
1853	Log(("vmmR3EmtRendezvousRecursive: %#x EMT#%u depth=%d\n", fFlags, pVCpu->idCpu, pVM->vmm.s.cRendezvousRecursions));
1854	AssertLogRelReturn(pVM->vmm.s.cRendezvousRecursions < 3, VERR_DEADLOCK);
1855	Assert(pVCpu->vmm.s.fInRendezvous);
1856
1857	/*
1858	* Save the current state.
1859	*/
1860	uint32_t const fParentFlags = pVM->vmm.s.fRendezvousFlags;
1861	uint32_t const cParentDone = pVM->vmm.s.cRendezvousEmtsDone;
1862	int32_t const iParentStatus = pVM->vmm.s.i32RendezvousStatus;
1863	PFNVMMEMTRENDEZVOUS const pfnParent = pVM->vmm.s.pfnRendezvous;
1864	void * const pvParentUser = pVM->vmm.s.pvRendezvousUser;
1865
1866	/*
1867	* Check preconditions and save the current state.
1868	*/
1869	AssertReturn( (fParentFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
1870	\|\| (fParentFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING
1871	\|\| (fParentFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE
1872	\|\| (fParentFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE,
1873	VERR_INTERNAL_ERROR);
1874	AssertReturn(pVM->vmm.s.cRendezvousEmtsEntered == pVM->cCpus, VERR_INTERNAL_ERROR_2);
1875	AssertReturn(pVM->vmm.s.cRendezvousEmtsReturned == 0, VERR_INTERNAL_ERROR_3);
1876
1877	/*
1878	* Reset the recursion prep and pop semaphores.
1879	*/
1880	int rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousRecursionPush);
1881	AssertLogRelRCReturn(rc, rc);
1882	rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousRecursionPop);
1883	AssertLogRelRCReturn(rc, rc);
1884	rc = vmmR3HlpResetEvent(pVM->vmm.s.hEvtRendezvousRecursionPushCaller);
1885	AssertLogRelMsgReturn(rc == VERR_TIMEOUT, ("%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
1886	rc = vmmR3HlpResetEvent(pVM->vmm.s.hEvtRendezvousRecursionPopCaller);
1887	AssertLogRelMsgReturn(rc == VERR_TIMEOUT, ("%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
1888
1889	/*
1890	* Usher the other thread into the recursion routine.
1891	*/
1892	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsRecursingPush, 0);
1893	ASMAtomicWriteBool(&pVM->vmm.s.fRendezvousRecursion, true);
1894
1895	uint32_t cLeft = pVM->cCpus - (cParentDone + 1U);
1896	if ((fParentFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE)
1897	while (cLeft-- > 0)
1898	{
1899	rc = RTSemEventSignal(pVM->vmm.s.hEvtRendezvousEnterOneByOne);
1900	AssertLogRelRC(rc);
1901	}
1902	else if ((fParentFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING)
1903	{
1904	Assert(cLeft == pVM->cCpus - (pVCpu->idCpu + 1U));
1905	for (VMCPUID iCpu = pVCpu->idCpu + 1U; iCpu < pVM->cCpus; iCpu++)
1906	{
1907	rc = RTSemEventSignal(pVM->vmm.s.pahEvtRendezvousEnterOrdered[iCpu]);
1908	AssertLogRelRC(rc);
1909	}
1910	}
1911	else if ((fParentFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING)
1912	{
1913	Assert(cLeft == pVCpu->idCpu);
1914	for (VMCPUID iCpu = pVCpu->idCpu; iCpu > 0; iCpu--)
1915	{
1916	rc = RTSemEventSignal(pVM->vmm.s.pahEvtRendezvousEnterOrdered[iCpu - 1U]);
1917	AssertLogRelRC(rc);
1918	}
1919	}
1920	else
1921	AssertLogRelReturn((fParentFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE,
1922	VERR_INTERNAL_ERROR_4);
1923
1924	rc = RTSemEventMultiSignal(pVM->vmm.s.hEvtMulRendezvousDone);
1925	AssertLogRelRC(rc);
1926	rc = RTSemEventSignal(pVM->vmm.s.hEvtRendezvousDoneCaller);
1927	AssertLogRelRC(rc);
1928
1929
1930	/*
1931	* Wait for the EMTs to wake up and get out of the parent rendezvous code.
1932	*/
1933	if (ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsRecursingPush) != pVM->cCpus)
1934	{
1935	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousRecursionPushCaller, RT_INDEFINITE_WAIT);
1936	AssertLogRelRC(rc);
1937	}
1938
1939	ASMAtomicWriteBool(&pVM->vmm.s.fRendezvousRecursion, false);
1940
1941	/*
1942	* Clear the slate and setup the new rendezvous.
1943	*/
1944	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1945	{
1946	rc = vmmR3HlpResetEvent(pVM->vmm.s.pahEvtRendezvousEnterOrdered[i]);
1947	AssertLogRelMsg(rc == VERR_TIMEOUT, ("%Rrc\n", rc));
1948	}
1949	rc = vmmR3HlpResetEvent(pVM->vmm.s.hEvtRendezvousEnterOneByOne); AssertLogRelMsg(rc == VERR_TIMEOUT, ("%Rrc\n", rc));
1950	rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce); AssertLogRelRC(rc);
1951	rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousDone); AssertLogRelRC(rc);
1952	rc = vmmR3HlpResetEvent(pVM->vmm.s.hEvtRendezvousDoneCaller); AssertLogRelMsg(rc == VERR_TIMEOUT, ("%Rrc\n", rc));
1953
1954	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsEntered, 0);
1955	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsDone, 0);
1956	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsReturned, 0);
1957	ASMAtomicWriteS32(&pVM->vmm.s.i32RendezvousStatus, VINF_SUCCESS);
1958	ASMAtomicWritePtr((void * volatile )&pVM->vmm.s.pfnRendezvous, (void )(uintptr_t)pfnRendezvous);
1959	ASMAtomicWritePtr(&pVM->vmm.s.pvRendezvousUser, pvUser);
1960	ASMAtomicWriteU32(&pVM->vmm.s.fRendezvousFlags, fFlags);
1961	ASMAtomicIncU32(&pVM->vmm.s.cRendezvousRecursions);
1962
1963	/*
1964	* We're ready to go now, do normal rendezvous processing.
1965	*/
1966	rc = RTSemEventMultiSignal(pVM->vmm.s.hEvtMulRendezvousRecursionPush);
1967	AssertLogRelRC(rc);
1968
1969	VBOXSTRICTRC rcStrict = vmmR3EmtRendezvousCommon(pVM, pVCpu, true /fIsCaller/, fFlags, pfnRendezvous, pvUser);
1970
1971	/*
1972	* The caller waits for the other EMTs to be done, return and waiting on the
1973	* pop semaphore.
1974	*/
1975	for (;;)
1976	{
1977	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousDoneCaller, RT_INDEFINITE_WAIT);
1978	AssertLogRelRC(rc);
1979	if (!pVM->vmm.s.fRendezvousRecursion)
1980	break;
1981	rcStrict = vmmR3EmtRendezvousCommonRecursion(pVM, pVCpu, rcStrict);
1982	}
1983
1984	/*
1985	* Get the return code and merge it with the above recursion status.
1986	*/
1987	VBOXSTRICTRC rcStrict2 = pVM->vmm.s.i32RendezvousStatus;
1988	if ( rcStrict2 != VINF_SUCCESS
1989	&& ( rcStrict == VINF_SUCCESS
1990	\|\| rcStrict > rcStrict2))
1991	rcStrict = rcStrict2;
1992
1993	/*
1994	* Restore the parent rendezvous state.
1995	*/
1996	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1997	{
1998	rc = vmmR3HlpResetEvent(pVM->vmm.s.pahEvtRendezvousEnterOrdered[i]);
1999	AssertLogRelMsg(rc == VERR_TIMEOUT, ("%Rrc\n", rc));
2000	}
2001	rc = vmmR3HlpResetEvent(pVM->vmm.s.hEvtRendezvousEnterOneByOne); AssertLogRelMsg(rc == VERR_TIMEOUT, ("%Rrc\n", rc));
2002	rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce); AssertLogRelRC(rc);
2003	rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousDone); AssertLogRelRC(rc);
2004	rc = vmmR3HlpResetEvent(pVM->vmm.s.hEvtRendezvousDoneCaller); AssertLogRelMsg(rc == VERR_TIMEOUT, ("%Rrc\n", rc));
2005
2006	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsEntered, pVM->cCpus);
2007	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsReturned, 0);
2008	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsDone, cParentDone);
2009	ASMAtomicWriteS32(&pVM->vmm.s.i32RendezvousStatus, iParentStatus);
2010	ASMAtomicWriteU32(&pVM->vmm.s.fRendezvousFlags, fParentFlags);
2011	ASMAtomicWritePtr(&pVM->vmm.s.pvRendezvousUser, pvParentUser);
2012	ASMAtomicWritePtr((void * volatile )&pVM->vmm.s.pfnRendezvous, (void )(uintptr_t)pfnParent);
2013
2014	/*
2015	* Usher the other EMTs back to their parent recursion routine, waiting
2016	* for them to all get there before we return (makes sure they've been
2017	* scheduled and are past the pop event sem, see below).
2018	*/
2019	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsRecursingPop, 0);
2020	rc = RTSemEventMultiSignal(pVM->vmm.s.hEvtMulRendezvousRecursionPop);
2021	AssertLogRelRC(rc);
2022
2023	if (ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsRecursingPop) != pVM->cCpus)
2024	{
2025	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousRecursionPopCaller, RT_INDEFINITE_WAIT);
2026	AssertLogRelRC(rc);
2027	}
2028
2029	/*
2030	* We must reset the pop semaphore on the way out (doing the pop caller too,
2031	* just in case). The parent may be another recursion.
2032	*/
2033	rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousRecursionPop); AssertLogRelRC(rc);
2034	rc = vmmR3HlpResetEvent(pVM->vmm.s.hEvtRendezvousRecursionPopCaller); AssertLogRelMsg(rc == VERR_TIMEOUT, ("%Rrc\n", rc));
2035
2036	ASMAtomicDecU32(&pVM->vmm.s.cRendezvousRecursions);
2037
2038	Log(("vmmR3EmtRendezvousRecursive: %#x EMT#%u depth=%d returns %Rrc\n",
2039	fFlags, pVCpu->idCpu, pVM->vmm.s.cRendezvousRecursions, VBOXSTRICTRC_VAL(rcStrict)));
2040	return rcStrict;
2041	}
2042
2043
2044	/**
2045	* EMT rendezvous.
2046	*
2047	* Gathers all the EMTs and execute some code on each of them, either in a one
2048	* by one fashion or all at once.
2049	*
2050	* @returns VBox strict status code. This will be the first error,
2051	* VINF_SUCCESS, or an EM scheduling status code.
2052	*
2053	* @retval VERR_DEADLOCK if recursion is attempted using a rendezvous type that
2054	* doesn't support it or if the recursion is too deep.
2055	*
2056	* @param pVM The cross context VM structure.
2057	* @param fFlags Flags indicating execution methods. See
2058	* grp_VMMR3EmtRendezvous_fFlags. The one-by-one,
2059	* descending and ascending rendezvous types support
2060	* recursion from inside @a pfnRendezvous.
2061	* @param pfnRendezvous The callback.
2062	* @param pvUser User argument for the callback.
2063	*
2064	* @thread Any.
2065	*/
2066	VMMR3DECL(int) VMMR3EmtRendezvous(PVM pVM, uint32_t fFlags, PFNVMMEMTRENDEZVOUS pfnRendezvous, void *pvUser)
2067	{
2068	/*
2069	* Validate input.
2070	*/
2071	AssertReturn(pVM, VERR_INVALID_VM_HANDLE);
2072	AssertMsg( (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_INVALID
2073	&& (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) <= VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING
2074	&& !(fFlags & ~VMMEMTRENDEZVOUS_FLAGS_VALID_MASK), ("%#x\n", fFlags));
2075	AssertMsg( !(fFlags & VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR)
2076	\|\| ( (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE
2077	&& (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE),
2078	("type %u\n", fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK));
2079
2080	VBOXSTRICTRC rcStrict;
2081	PVMCPU pVCpu = VMMGetCpu(pVM);
2082	if (!pVCpu)
2083	{
2084	/*
2085	* Forward the request to an EMT thread.
2086	*/
2087	Log(("VMMR3EmtRendezvous: %#x non-EMT\n", fFlags));
2088	if (!(fFlags & VMMEMTRENDEZVOUS_FLAGS_PRIORITY))
2089	rcStrict = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)VMMR3EmtRendezvous, 4, pVM, fFlags, pfnRendezvous, pvUser);
2090	else
2091	rcStrict = VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)VMMR3EmtRendezvous, 4, pVM, fFlags, pfnRendezvous, pvUser);
2092	Log(("VMMR3EmtRendezvous: %#x non-EMT returns %Rrc\n", fFlags, VBOXSTRICTRC_VAL(rcStrict)));
2093	}
2094	else if ( pVM->cCpus == 1
2095	\|\| ( pVM->enmVMState == VMSTATE_DESTROYING
2096	&& VMR3GetActiveEmts(pVM->pUVM) < pVM->cCpus ) )
2097	{
2098	/*
2099	* Shortcut for the single EMT case.
2100	*
2101	* We also ends up here if EMT(0) (or others) tries to issue a rendezvous
2102	* during vmR3Destroy after other emulation threads have started terminating.
2103	*/
2104	if (!pVCpu->vmm.s.fInRendezvous)
2105	{
2106	Log(("VMMR3EmtRendezvous: %#x EMT (uni)\n", fFlags));
2107	pVCpu->vmm.s.fInRendezvous = true;
2108	pVM->vmm.s.fRendezvousFlags = fFlags;
2109	rcStrict = pfnRendezvous(pVM, pVCpu, pvUser);
2110	pVCpu->vmm.s.fInRendezvous = false;
2111	}
2112	else
2113	{
2114	/* Recursion. Do the same checks as in the SMP case. */
2115	Log(("VMMR3EmtRendezvous: %#x EMT (uni), recursion depth=%d\n", fFlags, pVM->vmm.s.cRendezvousRecursions));
2116	uint32_t fType = pVM->vmm.s.fRendezvousFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK;
2117	AssertLogRelReturn( !pVCpu->vmm.s.fInRendezvous
2118	\|\| fType == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
2119	\|\| fType == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING
2120	\|\| fType == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE
2121	\|\| fType == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE
2122	, VERR_DEADLOCK);
2123
2124	AssertLogRelReturn(pVM->vmm.s.cRendezvousRecursions < 3, VERR_DEADLOCK);
2125	pVM->vmm.s.cRendezvousRecursions++;
2126	uint32_t const fParentFlags = pVM->vmm.s.fRendezvousFlags;
2127	pVM->vmm.s.fRendezvousFlags = fFlags;
2128
2129	rcStrict = pfnRendezvous(pVM, pVCpu, pvUser);
2130
2131	pVM->vmm.s.fRendezvousFlags = fParentFlags;
2132	pVM->vmm.s.cRendezvousRecursions--;
2133	}
2134	Log(("VMMR3EmtRendezvous: %#x EMT (uni) returns %Rrc\n", fFlags, VBOXSTRICTRC_VAL(rcStrict)));
2135	}
2136	else
2137	{
2138	/*
2139	* Spin lock. If busy, check for recursion, if not recursing wait for
2140	* the other EMT to finish while keeping a lookout for the RENDEZVOUS FF.
2141	*/
2142	int rc;
2143	rcStrict = VINF_SUCCESS;
2144	if (RT_UNLIKELY(!ASMAtomicCmpXchgU32(&pVM->vmm.s.u32RendezvousLock, 0x77778888, 0)))
2145	{
2146	/* Allow recursion in some cases. */
2147	if ( pVCpu->vmm.s.fInRendezvous
2148	&& ( (pVM->vmm.s.fRendezvousFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
2149	\|\| (pVM->vmm.s.fRendezvousFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING
2150	\|\| (pVM->vmm.s.fRendezvousFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE
2151	\|\| (pVM->vmm.s.fRendezvousFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE
2152	))
2153	return VBOXSTRICTRC_TODO(vmmR3EmtRendezvousRecursive(pVM, pVCpu, fFlags, pfnRendezvous, pvUser));
2154
2155	AssertLogRelMsgReturn(!pVCpu->vmm.s.fInRendezvous, ("fRendezvousFlags=%#x\n", pVM->vmm.s.fRendezvousFlags),
2156	VERR_DEADLOCK);
2157
2158	Log(("VMMR3EmtRendezvous: %#x EMT#%u, waiting for lock...\n", fFlags, pVCpu->idCpu));
2159	while (!ASMAtomicCmpXchgU32(&pVM->vmm.s.u32RendezvousLock, 0x77778888, 0))
2160	{
2161	if (VM_FF_IS_SET(pVM, VM_FF_EMT_RENDEZVOUS))
2162	{
2163	rc = VMMR3EmtRendezvousFF(pVM, pVCpu);
2164	if ( rc != VINF_SUCCESS
2165	&& ( rcStrict == VINF_SUCCESS
2166	\|\| rcStrict > rc))
2167	rcStrict = rc;
2168	/** @todo Perhaps deal with termination here? */
2169	}
2170	ASMNopPause();
2171	}
2172	}
2173
2174	Log(("VMMR3EmtRendezvous: %#x EMT#%u\n", fFlags, pVCpu->idCpu));
2175	Assert(!VM_FF_IS_SET(pVM, VM_FF_EMT_RENDEZVOUS));
2176	Assert(!pVCpu->vmm.s.fInRendezvous);
2177	pVCpu->vmm.s.fInRendezvous = true;
2178
2179	/*
2180	* Clear the slate and setup the rendezvous. This is a semaphore ping-pong orgy. :-)
2181	*/
2182	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2183	{
2184	rc = RTSemEventWait(pVM->vmm.s.pahEvtRendezvousEnterOrdered[i], 0);
2185	AssertLogRelMsg(rc == VERR_TIMEOUT \|\| rc == VINF_SUCCESS, ("%Rrc\n", rc));
2186	}
2187	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousEnterOneByOne, 0); AssertLogRelMsg(rc == VERR_TIMEOUT \|\| rc == VINF_SUCCESS, ("%Rrc\n", rc));
2188	rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce); AssertLogRelRC(rc);
2189	rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousDone); AssertLogRelRC(rc);
2190	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousDoneCaller, 0); AssertLogRelMsg(rc == VERR_TIMEOUT \|\| rc == VINF_SUCCESS, ("%Rrc\n", rc));
2191	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsEntered, 0);
2192	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsDone, 0);
2193	ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsReturned, 0);
2194	ASMAtomicWriteS32(&pVM->vmm.s.i32RendezvousStatus, VINF_SUCCESS);
2195	ASMAtomicWritePtr((void * volatile )&pVM->vmm.s.pfnRendezvous, (void )(uintptr_t)pfnRendezvous);
2196	ASMAtomicWritePtr(&pVM->vmm.s.pvRendezvousUser, pvUser);
2197	ASMAtomicWriteU32(&pVM->vmm.s.fRendezvousFlags, fFlags);
2198
2199	/*
2200	* Set the FF and poke the other EMTs.
2201	*/
2202	VM_FF_SET(pVM, VM_FF_EMT_RENDEZVOUS);
2203	VMR3NotifyGlobalFFU(pVM->pUVM, VMNOTIFYFF_FLAGS_POKE);
2204
2205	/*
2206	* Do the same ourselves.
2207	*/
2208	VBOXSTRICTRC rcStrict2 = vmmR3EmtRendezvousCommon(pVM, pVCpu, true /* fIsCaller */, fFlags, pfnRendezvous, pvUser);
2209
2210	/*
2211	* The caller waits for the other EMTs to be done and return before doing
2212	* the cleanup. This makes away with wakeup / reset races we would otherwise
2213	* risk in the multiple release event semaphore code (hEvtRendezvousDoneCaller).
2214	*/
2215	for (;;)
2216	{
2217	rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousDoneCaller, RT_INDEFINITE_WAIT);
2218	AssertLogRelRC(rc);
2219	if (!pVM->vmm.s.fRendezvousRecursion)
2220	break;
2221	rcStrict2 = vmmR3EmtRendezvousCommonRecursion(pVM, pVCpu, rcStrict2);
2222	}
2223
2224	/*
2225	* Get the return code and clean up a little bit.
2226	*/
2227	VBOXSTRICTRC rcStrict3 = pVM->vmm.s.i32RendezvousStatus;
2228	ASMAtomicWriteNullPtr((void * volatile *)&pVM->vmm.s.pfnRendezvous);
2229
2230	ASMAtomicWriteU32(&pVM->vmm.s.u32RendezvousLock, 0);
2231	pVCpu->vmm.s.fInRendezvous = false;
2232
2233	/*
2234	* Merge rcStrict, rcStrict2 and rcStrict3.
2235	*/
2236	AssertRC(VBOXSTRICTRC_VAL(rcStrict));
2237	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
2238	if ( rcStrict2 != VINF_SUCCESS
2239	&& ( rcStrict == VINF_SUCCESS
2240	\|\| rcStrict > rcStrict2))
2241	rcStrict = rcStrict2;
2242	if ( rcStrict3 != VINF_SUCCESS
2243	&& ( rcStrict == VINF_SUCCESS
2244	\|\| rcStrict > rcStrict3))
2245	rcStrict = rcStrict3;
2246	Log(("VMMR3EmtRendezvous: %#x EMT#%u returns %Rrc\n", fFlags, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict)));
2247	}
2248
2249	AssertLogRelMsgReturn( rcStrict <= VINF_SUCCESS
2250	\|\| (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST),
2251	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)),
2252	VERR_IPE_UNEXPECTED_INFO_STATUS);
2253	return VBOXSTRICTRC_VAL(rcStrict);
2254	}
2255
2256
2257	/**
2258	* Interface for vmR3SetHaltMethodU.
2259	*
2260	* @param pVCpu The cross context virtual CPU structure of the
2261	* calling EMT.
2262	* @param fMayHaltInRing0 The new state.
2263	* @param cNsSpinBlockThreshold The spin-vs-blocking threashold.
2264	* @thread EMT(pVCpu)
2265	*
2266	* @todo Move the EMT handling to VMM (or EM). I soooooo regret that VM
2267	* component.
2268	*/
2269	VMMR3_INT_DECL(void) VMMR3SetMayHaltInRing0(PVMCPU pVCpu, bool fMayHaltInRing0, uint32_t cNsSpinBlockThreshold)
2270	{
2271	LogFlow(("VMMR3SetMayHaltInRing0(#%u, %d, %u)\n", pVCpu->idCpu, fMayHaltInRing0, cNsSpinBlockThreshold));
2272	pVCpu->vmm.s.fMayHaltInRing0 = fMayHaltInRing0;
2273	pVCpu->vmm.s.cNsSpinBlockThreshold = cNsSpinBlockThreshold;
2274	}
2275
2276
2277	/**
2278	* Read from the ring 0 jump buffer stack.
2279	*
2280	* @returns VBox status code.
2281	*
2282	* @param pVM The cross context VM structure.
2283	* @param idCpu The ID of the source CPU context (for the address).
2284	* @param R0Addr Where to start reading.
2285	* @param pvBuf Where to store the data we've read.
2286	* @param cbRead The number of bytes to read.
2287	*/
2288	VMMR3_INT_DECL(int) VMMR3ReadR0Stack(PVM pVM, VMCPUID idCpu, RTHCUINTPTR R0Addr, void *pvBuf, size_t cbRead)
2289	{
2290	PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
2291	AssertReturn(pVCpu, VERR_INVALID_PARAMETER);
2292	AssertReturn(cbRead < ~(size_t)0 / 2, VERR_INVALID_PARAMETER);
2293
2294	int rc;
2295	#ifdef VMM_R0_SWITCH_STACK
2296	RTHCUINTPTR off = R0Addr - MMHyperCCToR0(pVM, pVCpu->vmm.s.pbEMTStackR3);
2297	#else
2298	RTHCUINTPTR off = pVCpu->vmm.s.CallRing3JmpBufR0.cbSavedStack - (pVCpu->vmm.s.CallRing3JmpBufR0.SpCheck - R0Addr);
2299	#endif
2300	if ( off < VMM_STACK_SIZE
2301	&& off + cbRead <= VMM_STACK_SIZE)
2302	{
2303	memcpy(pvBuf, &pVCpu->vmm.s.pbEMTStackR3[off], cbRead);
2304	rc = VINF_SUCCESS;
2305	}
2306	else
2307	rc = VERR_INVALID_POINTER;
2308
2309	/* Supply the setjmp return RIP/EIP. */
2310	if ( pVCpu->vmm.s.CallRing3JmpBufR0.UnwindRetPcLocation + sizeof(RTR0UINTPTR) > R0Addr
2311	&& pVCpu->vmm.s.CallRing3JmpBufR0.UnwindRetPcLocation < R0Addr + cbRead)
2312	{
2313	uint8_t const pbSrc = (uint8_t const )&pVCpu->vmm.s.CallRing3JmpBufR0.UnwindRetPcValue;
2314	size_t cbSrc = sizeof(pVCpu->vmm.s.CallRing3JmpBufR0.UnwindRetPcValue);
2315	size_t offDst = 0;
2316	if (R0Addr < pVCpu->vmm.s.CallRing3JmpBufR0.UnwindRetPcLocation)
2317	offDst = pVCpu->vmm.s.CallRing3JmpBufR0.UnwindRetPcLocation - R0Addr;
2318	else if (R0Addr > pVCpu->vmm.s.CallRing3JmpBufR0.UnwindRetPcLocation)
2319	{
2320	size_t offSrc = R0Addr - pVCpu->vmm.s.CallRing3JmpBufR0.UnwindRetPcLocation;
2321	Assert(offSrc < cbSrc);
2322	pbSrc -= offSrc;
2323	cbSrc -= offSrc;
2324	}
2325	if (cbSrc > cbRead - offDst)
2326	cbSrc = cbRead - offDst;
2327	memcpy((uint8_t *)pvBuf + offDst, pbSrc, cbSrc);
2328
2329	if (cbSrc == cbRead)
2330	rc = VINF_SUCCESS;
2331	}
2332
2333	return rc;
2334	}
2335
2336
2337	/**
2338	* Used by the DBGF stack unwinder to initialize the register state.
2339	*
2340	* @param pUVM The user mode VM handle.
2341	* @param idCpu The ID of the CPU being unwound.
2342	* @param pState The unwind state to initialize.
2343	*/
2344	VMMR3_INT_DECL(void) VMMR3InitR0StackUnwindState(PUVM pUVM, VMCPUID idCpu, struct RTDBGUNWINDSTATE *pState)
2345	{
2346	PVMCPU pVCpu = VMMR3GetCpuByIdU(pUVM, idCpu);
2347	AssertReturnVoid(pVCpu);
2348
2349	/*
2350	* Locate the resume point on the stack.
2351	*/
2352	#ifdef VMM_R0_SWITCH_STACK
2353	uintptr_t off = pVCpu->vmm.s.CallRing3JmpBufR0.SpResume - MMHyperCCToR0(pVCpu->pVMR3, pVCpu->vmm.s.pbEMTStackR3);
2354	AssertReturnVoid(off < VMM_STACK_SIZE);
2355	#else
2356	uintptr_t off = 0;
2357	#endif
2358
2359	#ifdef RT_ARCH_AMD64
2360	/*
2361	* This code must match the .resume stuff in VMMR0JmpA-amd64.asm exactly.
2362	*/
2363	# ifdef VBOX_STRICT
2364	Assert((uint64_t const )&pVCpu->vmm.s.pbEMTStackR3[off] == UINT32_C(0x7eadf00d));
2365	off += 8; /* RESUME_MAGIC */
2366	# endif
2367	# ifdef RT_OS_WINDOWS
2368	off += 0xa0; /* XMM6 thru XMM15 */
2369	# endif
2370	pState->u.x86.uRFlags = (uint64_t const )&pVCpu->vmm.s.pbEMTStackR3[off];
2371	off += 8;
2372	pState->u.x86.auRegs[X86_GREG_xBX] = (uint64_t const )&pVCpu->vmm.s.pbEMTStackR3[off];
2373	off += 8;
2374	# ifdef RT_OS_WINDOWS
2375	pState->u.x86.auRegs[X86_GREG_xSI] = (uint64_t const )&pVCpu->vmm.s.pbEMTStackR3[off];
2376	off += 8;
2377	pState->u.x86.auRegs[X86_GREG_xDI] = (uint64_t const )&pVCpu->vmm.s.pbEMTStackR3[off];
2378	off += 8;
2379	# endif
2380	pState->u.x86.auRegs[X86_GREG_x12] = (uint64_t const )&pVCpu->vmm.s.pbEMTStackR3[off];
2381	off += 8;
2382	pState->u.x86.auRegs[X86_GREG_x13] = (uint64_t const )&pVCpu->vmm.s.pbEMTStackR3[off];
2383	off += 8;
2384	pState->u.x86.auRegs[X86_GREG_x14] = (uint64_t const )&pVCpu->vmm.s.pbEMTStackR3[off];
2385	off += 8;
2386	pState->u.x86.auRegs[X86_GREG_x15] = (uint64_t const )&pVCpu->vmm.s.pbEMTStackR3[off];
2387	off += 8;
2388	pState->u.x86.auRegs[X86_GREG_xBP] = (uint64_t const )&pVCpu->vmm.s.pbEMTStackR3[off];
2389	off += 8;
2390	pState->uPc = (uint64_t const )&pVCpu->vmm.s.pbEMTStackR3[off];
2391	off += 8;
2392
2393	#elif defined(RT_ARCH_X86)
2394	/*
2395	* This code must match the .resume stuff in VMMR0JmpA-x86.asm exactly.
2396	*/
2397	# ifdef VBOX_STRICT
2398	Assert((uint32_t const )&pVCpu->vmm.s.pbEMTStackR3[off] == UINT32_C(0x7eadf00d));
2399	off += 4; /* RESUME_MAGIC */
2400	# endif
2401	pState->u.x86.uRFlags = (uint32_t const )&pVCpu->vmm.s.pbEMTStackR3[off];
2402	off += 4;
2403	pState->u.x86.auRegs[X86_GREG_xBX] = (uint32_t const )&pVCpu->vmm.s.pbEMTStackR3[off];
2404	off += 4;
2405	pState->u.x86.auRegs[X86_GREG_xSI] = (uint32_t const )&pVCpu->vmm.s.pbEMTStackR3[off];
2406	off += 4;
2407	pState->u.x86.auRegs[X86_GREG_xDI] = (uint32_t const )&pVCpu->vmm.s.pbEMTStackR3[off];
2408	off += 4;
2409	pState->u.x86.auRegs[X86_GREG_xBP] = (uint32_t const )&pVCpu->vmm.s.pbEMTStackR3[off];
2410	off += 4;
2411	pState->uPc = (uint32_t const )&pVCpu->vmm.s.pbEMTStackR3[off];
2412	off += 4;
2413	#else
2414	# error "Port me"
2415	#endif
2416
2417	/*
2418	* This is all we really need here, though the above helps if the assembly
2419	* doesn't contain unwind info (currently only on win/64, so that is useful).
2420	*/
2421	pState->u.x86.auRegs[X86_GREG_xBP] = pVCpu->vmm.s.CallRing3JmpBufR0.SavedEbp;
2422	pState->u.x86.auRegs[X86_GREG_xSP] = pVCpu->vmm.s.CallRing3JmpBufR0.SpResume;
2423	}
2424
2425
2426	/**
2427	* Wrapper for SUPR3CallVMMR0Ex which will deal with VINF_VMM_CALL_HOST returns.
2428	*
2429	* @returns VBox status code.
2430	* @param pVM The cross context VM structure.
2431	* @param uOperation Operation to execute.
2432	* @param u64Arg Constant argument.
2433	* @param pReqHdr Pointer to a request header. See SUPR3CallVMMR0Ex for
2434	* details.
2435	*/
2436	VMMR3DECL(int) VMMR3CallR0(PVM pVM, uint32_t uOperation, uint64_t u64Arg, PSUPVMMR0REQHDR pReqHdr)
2437	{
2438	PVMCPU pVCpu = VMMGetCpu(pVM);
2439	AssertReturn(pVCpu, VERR_VM_THREAD_NOT_EMT);
2440	return VMMR3CallR0Emt(pVM, pVCpu, (VMMR0OPERATION)uOperation, u64Arg, pReqHdr);
2441	}
2442
2443
2444	/**
2445	* Wrapper for SUPR3CallVMMR0Ex which will deal with VINF_VMM_CALL_HOST returns.
2446	*
2447	* @returns VBox status code.
2448	* @param pVM The cross context VM structure.
2449	* @param pVCpu The cross context VM structure.
2450	* @param enmOperation Operation to execute.
2451	* @param u64Arg Constant argument.
2452	* @param pReqHdr Pointer to a request header. See SUPR3CallVMMR0Ex for
2453	* details.
2454	*/
2455	VMMR3_INT_DECL(int) VMMR3CallR0Emt(PVM pVM, PVMCPU pVCpu, VMMR0OPERATION enmOperation, uint64_t u64Arg, PSUPVMMR0REQHDR pReqHdr)
2456	{
2457	int rc;
2458	for (;;)
2459	{
2460	#ifdef NO_SUPCALLR0VMM
2461	rc = VERR_GENERAL_FAILURE;
2462	#else
2463	rc = SUPR3CallVMMR0Ex(VMCC_GET_VMR0_FOR_CALL(pVM), pVCpu->idCpu, enmOperation, u64Arg, pReqHdr);
2464	#endif
2465	/*
2466	* Flush the logs.
2467	*/
2468	#ifdef LOG_ENABLED
2469	VMM_FLUSH_R0_LOG(pVM, pVCpu, &pVCpu->vmm.s.u.s.Logger, NULL);
2470	#endif
2471	VMM_FLUSH_R0_LOG(pVM, pVCpu, &pVCpu->vmm.s.u.s.RelLogger, RTLogRelGetDefaultInstance());
2472	if (rc != VINF_VMM_CALL_HOST)
2473	break;
2474	rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
2475	if (RT_FAILURE(rc) \|\| (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
2476	break;
2477	/* Resume R0 */
2478	}
2479
2480	AssertLogRelMsgReturn(rc == VINF_SUCCESS \|\| RT_FAILURE(rc),
2481	("enmOperation=%u rc=%Rrc\n", enmOperation, rc),
2482	VERR_IPE_UNEXPECTED_INFO_STATUS);
2483	return rc;
2484	}
2485
2486
2487	/**
2488	* Service a call to the ring-3 host code.
2489	*
2490	* @returns VBox status code.
2491	* @param pVM The cross context VM structure.
2492	* @param pVCpu The cross context virtual CPU structure.
2493	* @remarks Careful with critsects.
2494	*/
2495	static int vmmR3ServiceCallRing3Request(PVM pVM, PVMCPU pVCpu)
2496	{
2497	/*
2498	* We must also check for pending critsect exits or else we can deadlock
2499	* when entering other critsects here.
2500	*/
2501	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PDM_CRITSECT))
2502	PDMCritSectBothFF(pVM, pVCpu);
2503
2504	switch (pVCpu->vmm.s.enmCallRing3Operation)
2505	{
2506	/*
2507	* Grow the PGM pool.
2508	*/
2509	case VMMCALLRING3_PGM_POOL_GROW:
2510	{
2511	pVCpu->vmm.s.rcCallRing3 = PGMR3PoolGrow(pVM, pVCpu);
2512	break;
2513	}
2514
2515	/*
2516	* Maps an page allocation chunk into ring-3 so ring-0 can use it.
2517	*/
2518	case VMMCALLRING3_PGM_MAP_CHUNK:
2519	{
2520	pVCpu->vmm.s.rcCallRing3 = PGMR3PhysChunkMap(pVM, pVCpu->vmm.s.u64CallRing3Arg);
2521	break;
2522	}
2523
2524	/*
2525	* Allocates more handy pages.
2526	*/
2527	case VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES:
2528	{
2529	pVCpu->vmm.s.rcCallRing3 = PGMR3PhysAllocateHandyPages(pVM);
2530	break;
2531	}
2532
2533	/*
2534	* Allocates a large page.
2535	*/
2536	case VMMCALLRING3_PGM_ALLOCATE_LARGE_HANDY_PAGE:
2537	{
2538	pVCpu->vmm.s.rcCallRing3 = PGMR3PhysAllocateLargeHandyPage(pVM, pVCpu->vmm.s.u64CallRing3Arg);
2539	break;
2540	}
2541
2542	/*
2543	* Set the VM error message.
2544	*/
2545	case VMMCALLRING3_VM_SET_ERROR:
2546	VMR3SetErrorWorker(pVM);
2547	pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
2548	break;
2549
2550	/*
2551	* Set the VM runtime error message.
2552	*/
2553	case VMMCALLRING3_VM_SET_RUNTIME_ERROR:
2554	pVCpu->vmm.s.rcCallRing3 = VMR3SetRuntimeErrorWorker(pVM);
2555	break;
2556
2557	/*
2558	* Signal a ring 0 hypervisor assertion.
2559	* Cancel the longjmp operation that's in progress.
2560	*/
2561	case VMMCALLRING3_VM_R0_ASSERTION:
2562	pVCpu->vmm.s.enmCallRing3Operation = VMMCALLRING3_INVALID;
2563	pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call = false;
2564	#ifdef RT_ARCH_X86
2565	pVCpu->vmm.s.CallRing3JmpBufR0.eip = 0;
2566	#else
2567	pVCpu->vmm.s.CallRing3JmpBufR0.rip = 0;
2568	#endif
2569	#ifdef VMM_R0_SWITCH_STACK
2570	(uint64_t )pVCpu->vmm.s.pbEMTStackR3 = 0; /* clear marker */
2571	#endif
2572	LogRel(("%s", pVM->vmm.s.szRing0AssertMsg1));
2573	LogRel(("%s", pVM->vmm.s.szRing0AssertMsg2));
2574	return VERR_VMM_RING0_ASSERTION;
2575
2576	default:
2577	AssertMsgFailed(("enmCallRing3Operation=%d\n", pVCpu->vmm.s.enmCallRing3Operation));
2578	return VERR_VMM_UNKNOWN_RING3_CALL;
2579	}
2580
2581	pVCpu->vmm.s.enmCallRing3Operation = VMMCALLRING3_INVALID;
2582	return VINF_SUCCESS;
2583	}
2584
2585
2586	/**
2587	* Displays the Force action Flags.
2588	*
2589	* @param pVM The cross context VM structure.
2590	* @param pHlp The output helpers.
2591	* @param pszArgs The additional arguments (ignored).
2592	*/
2593	static DECLCALLBACK(void) vmmR3InfoFF(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
2594	{
2595	int c;
2596	uint32_t f;
2597	NOREF(pszArgs);
2598
2599	#define PRINT_FLAG(prf,flag) do { \
2600	if (f & (prf##flag)) \
2601	{ \
2602	static const char *s_psz = #flag; \
2603	if (!(c % 6)) \
2604	pHlp->pfnPrintf(pHlp, "%s\n %s", c ? "," : "", s_psz); \
2605	else \
2606	pHlp->pfnPrintf(pHlp, ", %s", s_psz); \
2607	c++; \
2608	f &= ~(prf##flag); \
2609	} \
2610	} while (0)
2611
2612	#define PRINT_GROUP(prf,grp,sfx) do { \
2613	if (f & (prf##grp##sfx)) \
2614	{ \
2615	static const char *s_psz = #grp; \
2616	if (!(c % 5)) \
2617	pHlp->pfnPrintf(pHlp, "%s %s", c ? ",\n" : " Groups:\n", s_psz); \
2618	else \
2619	pHlp->pfnPrintf(pHlp, ", %s", s_psz); \
2620	c++; \
2621	} \
2622	} while (0)
2623
2624	/*
2625	* The global flags.
2626	*/
2627	const uint32_t fGlobalForcedActions = pVM->fGlobalForcedActions;
2628	pHlp->pfnPrintf(pHlp, "Global FFs: %#RX32", fGlobalForcedActions);
2629
2630	/* show the flag mnemonics */
2631	c = 0;
2632	f = fGlobalForcedActions;
2633	PRINT_FLAG(VM_FF_,TM_VIRTUAL_SYNC);
2634	PRINT_FLAG(VM_FF_,PDM_QUEUES);
2635	PRINT_FLAG(VM_FF_,PDM_DMA);
2636	PRINT_FLAG(VM_FF_,DBGF);
2637	PRINT_FLAG(VM_FF_,REQUEST);
2638	PRINT_FLAG(VM_FF_,CHECK_VM_STATE);
2639	PRINT_FLAG(VM_FF_,RESET);
2640	PRINT_FLAG(VM_FF_,EMT_RENDEZVOUS);
2641	PRINT_FLAG(VM_FF_,PGM_NEED_HANDY_PAGES);
2642	PRINT_FLAG(VM_FF_,PGM_NO_MEMORY);
2643	PRINT_FLAG(VM_FF_,PGM_POOL_FLUSH_PENDING);
2644	PRINT_FLAG(VM_FF_,DEBUG_SUSPEND);
2645	if (f)
2646	pHlp->pfnPrintf(pHlp, "%s\n Unknown bits: %#RX32\n", c ? "," : "", f);
2647	else
2648	pHlp->pfnPrintf(pHlp, "\n");
2649
2650	/* the groups */
2651	c = 0;
2652	f = fGlobalForcedActions;
2653	PRINT_GROUP(VM_FF_,EXTERNAL_SUSPENDED,_MASK);
2654	PRINT_GROUP(VM_FF_,EXTERNAL_HALTED,_MASK);
2655	PRINT_GROUP(VM_FF_,HIGH_PRIORITY_PRE,_MASK);
2656	PRINT_GROUP(VM_FF_,HIGH_PRIORITY_PRE_RAW,_MASK);
2657	PRINT_GROUP(VM_FF_,HIGH_PRIORITY_POST,_MASK);
2658	PRINT_GROUP(VM_FF_,NORMAL_PRIORITY_POST,_MASK);
2659	PRINT_GROUP(VM_FF_,NORMAL_PRIORITY,_MASK);
2660	PRINT_GROUP(VM_FF_,ALL_REM,_MASK);
2661	if (c)
2662	pHlp->pfnPrintf(pHlp, "\n");
2663
2664	/*
2665	* Per CPU flags.
2666	*/
2667	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2668	{
2669	PVMCPU pVCpu = pVM->apCpusR3[i];
2670	const uint64_t fLocalForcedActions = pVCpu->fLocalForcedActions;
2671	pHlp->pfnPrintf(pHlp, "CPU %u FFs: %#RX64", i, fLocalForcedActions);
2672
2673	/* show the flag mnemonics */
2674	c = 0;
2675	f = fLocalForcedActions;
2676	PRINT_FLAG(VMCPU_FF_,INTERRUPT_APIC);
2677	PRINT_FLAG(VMCPU_FF_,INTERRUPT_PIC);
2678	PRINT_FLAG(VMCPU_FF_,TIMER);
2679	PRINT_FLAG(VMCPU_FF_,INTERRUPT_NMI);
2680	PRINT_FLAG(VMCPU_FF_,INTERRUPT_SMI);
2681	PRINT_FLAG(VMCPU_FF_,PDM_CRITSECT);
2682	PRINT_FLAG(VMCPU_FF_,UNHALT);
2683	PRINT_FLAG(VMCPU_FF_,IEM);
2684	PRINT_FLAG(VMCPU_FF_,UPDATE_APIC);
2685	PRINT_FLAG(VMCPU_FF_,DBGF);
2686	PRINT_FLAG(VMCPU_FF_,REQUEST);
2687	PRINT_FLAG(VMCPU_FF_,HM_UPDATE_CR3);
2688	PRINT_FLAG(VMCPU_FF_,HM_UPDATE_PAE_PDPES);
2689	PRINT_FLAG(VMCPU_FF_,PGM_SYNC_CR3);
2690	PRINT_FLAG(VMCPU_FF_,PGM_SYNC_CR3_NON_GLOBAL);
2691	PRINT_FLAG(VMCPU_FF_,TLB_FLUSH);
2692	PRINT_FLAG(VMCPU_FF_,INHIBIT_INTERRUPTS);
2693	PRINT_FLAG(VMCPU_FF_,BLOCK_NMIS);
2694	PRINT_FLAG(VMCPU_FF_,TO_R3);
2695	PRINT_FLAG(VMCPU_FF_,IOM);
2696	if (f)
2697	pHlp->pfnPrintf(pHlp, "%s\n Unknown bits: %#RX64\n", c ? "," : "", f);
2698	else
2699	pHlp->pfnPrintf(pHlp, "\n");
2700
2701	if (fLocalForcedActions & VMCPU_FF_INHIBIT_INTERRUPTS)
2702	pHlp->pfnPrintf(pHlp, " intr inhibit RIP: %RGp\n", EMGetInhibitInterruptsPC(pVCpu));
2703
2704	/* the groups */
2705	c = 0;
2706	f = fLocalForcedActions;
2707	PRINT_GROUP(VMCPU_FF_,EXTERNAL_SUSPENDED,_MASK);
2708	PRINT_GROUP(VMCPU_FF_,EXTERNAL_HALTED,_MASK);
2709	PRINT_GROUP(VMCPU_FF_,HIGH_PRIORITY_PRE,_MASK);
2710	PRINT_GROUP(VMCPU_FF_,HIGH_PRIORITY_PRE_RAW,_MASK);
2711	PRINT_GROUP(VMCPU_FF_,HIGH_PRIORITY_POST,_MASK);
2712	PRINT_GROUP(VMCPU_FF_,NORMAL_PRIORITY_POST,_MASK);
2713	PRINT_GROUP(VMCPU_FF_,NORMAL_PRIORITY,_MASK);
2714	PRINT_GROUP(VMCPU_FF_,RESUME_GUEST,_MASK);
2715	PRINT_GROUP(VMCPU_FF_,HM_TO_R3,_MASK);
2716	PRINT_GROUP(VMCPU_FF_,ALL_REM,_MASK);
2717	if (c)
2718	pHlp->pfnPrintf(pHlp, "\n");
2719	}
2720
2721	#undef PRINT_FLAG
2722	#undef PRINT_GROUP
2723	}
2724

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR3/VMM.cpp@ 90992

Download in other formats: