VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllPool.cpp@ 20110

Last change on this file since 20110 was 20058, checked in by vboxsync, 16 years ago

Deal with pool pages being modified while we wait for the pgm lock in access handlers.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 161.3 KB
Line 
1/* $Id: PGMAllPool.cpp 20058 2009-05-27 08:29:57Z vboxsync $ */
2/** @file
3 * PGM Shadow Page Pool.
4 */
5
6/*
7 * Copyright (C) 2006-2007 Sun Microsystems, Inc.
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.215389.xyz. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 *
17 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18 * Clara, CA 95054 USA or visit http://www.sun.com if you need
19 * additional information or have any questions.
20 */
21
22
23/*******************************************************************************
24* Header Files *
25*******************************************************************************/
26#define LOG_GROUP LOG_GROUP_PGM_POOL
27#include <VBox/pgm.h>
28#include <VBox/mm.h>
29#include <VBox/em.h>
30#include <VBox/cpum.h>
31#ifdef IN_RC
32# include <VBox/patm.h>
33#endif
34#include "PGMInternal.h"
35#include <VBox/vm.h>
36#include <VBox/disopcode.h>
37#include <VBox/hwacc_vmx.h>
38
39#include <VBox/log.h>
40#include <VBox/err.h>
41#include <iprt/asm.h>
42#include <iprt/string.h>
43
44
45/*******************************************************************************
46* Internal Functions *
47*******************************************************************************/
48__BEGIN_DECLS
49static void pgmPoolFlushAllInt(PPGMPOOL pPool);
50#ifdef PGMPOOL_WITH_USER_TRACKING
51DECLINLINE(unsigned) pgmPoolTrackGetShadowEntrySize(PGMPOOLKIND enmKind);
52DECLINLINE(unsigned) pgmPoolTrackGetGuestEntrySize(PGMPOOLKIND enmKind);
53static void pgmPoolTrackDeref(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
54#endif
55#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
56static void pgmPoolTracDerefGCPhysHint(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhysHint);
57#endif
58#ifdef PGMPOOL_WITH_CACHE
59static int pgmPoolTrackAddUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable);
60#endif
61#ifdef PGMPOOL_WITH_MONITORING
62static void pgmPoolMonitorModifiedRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
63#endif
64#ifndef IN_RING3
65DECLEXPORT(int) pgmPoolAccessHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser);
66#endif
67#ifdef LOG_ENABLED
68static const char *pgmPoolPoolKindToStr(uint8_t enmKind);
69#endif
70
71void pgmPoolTrackFlushGCPhysPT(PVM pVM, PPGMPAGE pPhysPage, uint16_t iShw, uint16_t cRefs);
72void pgmPoolTrackFlushGCPhysPTs(PVM pVM, PPGMPAGE pPhysPage, uint16_t iPhysExt);
73int pgmPoolTrackFlushGCPhysPTsSlow(PVM pVM, PPGMPAGE pPhysPage);
74PPGMPOOLPHYSEXT pgmPoolTrackPhysExtAlloc(PVM pVM, uint16_t *piPhysExt);
75void pgmPoolTrackPhysExtFree(PVM pVM, uint16_t iPhysExt);
76void pgmPoolTrackPhysExtFreeList(PVM pVM, uint16_t iPhysExt);
77
78__END_DECLS
79
80
81/**
82 * Checks if the specified page pool kind is for a 4MB or 2MB guest page.
83 *
84 * @returns true if it's the shadow of a 4MB or 2MB guest page, otherwise false.
85 * @param enmKind The page kind.
86 */
87DECLINLINE(bool) pgmPoolIsBigPage(PGMPOOLKIND enmKind)
88{
89 switch (enmKind)
90 {
91 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
92 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
93 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
94 return true;
95 default:
96 return false;
97 }
98}
99
100/** @def PGMPOOL_PAGE_2_LOCKED_PTR
101 * Maps a pool page pool into the current context and lock it (RC only).
102 *
103 * @returns VBox status code.
104 * @param pVM The VM handle.
105 * @param pPage The pool page.
106 *
107 * @remark In RC this uses PGMGCDynMapHCPage(), so it will consume of the
108 * small page window employeed by that function. Be careful.
109 * @remark There is no need to assert on the result.
110 */
111#if defined(IN_RC)
112DECLINLINE(void *) PGMPOOL_PAGE_2_LOCKED_PTR(PVM pVM, PPGMPOOLPAGE pPage)
113{
114 void *pv = pgmPoolMapPageInlined(&pVM->pgm.s, pPage);
115
116 /* Make sure the dynamic mapping will not be reused. */
117 if (pv)
118 PGMDynLockHCPage(pVM, (uint8_t *)pv);
119
120 return pv;
121}
122#else
123# define PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage) PGMPOOL_PAGE_2_PTR(pVM, pPage)
124#endif
125
126/** @def PGMPOOL_UNLOCK_PTR
127 * Unlock a previously locked dynamic caching (RC only).
128 *
129 * @returns VBox status code.
130 * @param pVM The VM handle.
131 * @param pPage The pool page.
132 *
133 * @remark In RC this uses PGMGCDynMapHCPage(), so it will consume of the
134 * small page window employeed by that function. Be careful.
135 * @remark There is no need to assert on the result.
136 */
137#if defined(IN_RC)
138DECLINLINE(void) PGMPOOL_UNLOCK_PTR(PVM pVM, void *pvPage)
139{
140 if (pvPage)
141 PGMDynUnlockHCPage(pVM, (uint8_t *)pvPage);
142}
143#else
144# define PGMPOOL_UNLOCK_PTR(pVM, pPage) do {} while (0)
145#endif
146
147
148#ifdef PGMPOOL_WITH_MONITORING
149/**
150 * Determin the size of a write instruction.
151 * @returns number of bytes written.
152 * @param pDis The disassembler state.
153 */
154static unsigned pgmPoolDisasWriteSize(PDISCPUSTATE pDis)
155{
156 /*
157 * This is very crude and possibly wrong for some opcodes,
158 * but since it's not really supposed to be called we can
159 * probably live with that.
160 */
161 return DISGetParamSize(pDis, &pDis->param1);
162}
163
164
165/**
166 * Flushes a chain of pages sharing the same access monitor.
167 *
168 * @returns VBox status code suitable for scheduling.
169 * @param pPool The pool.
170 * @param pPage A page in the chain.
171 */
172int pgmPoolMonitorChainFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
173{
174 LogFlow(("pgmPoolMonitorChainFlush: Flush page %RGp type=%d\n", pPage->GCPhys, pPage->enmKind));
175
176 /*
177 * Find the list head.
178 */
179 uint16_t idx = pPage->idx;
180 if (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
181 {
182 while (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
183 {
184 idx = pPage->iMonitoredPrev;
185 Assert(idx != pPage->idx);
186 pPage = &pPool->aPages[idx];
187 }
188 }
189
190 /*
191 * Iterate the list flushing each shadow page.
192 */
193 int rc = VINF_SUCCESS;
194 for (;;)
195 {
196 idx = pPage->iMonitoredNext;
197 Assert(idx != pPage->idx);
198 if (pPage->idx >= PGMPOOL_IDX_FIRST)
199 {
200 int rc2 = pgmPoolFlushPage(pPool, pPage);
201 AssertRC(rc2);
202 }
203 /* next */
204 if (idx == NIL_PGMPOOL_IDX)
205 break;
206 pPage = &pPool->aPages[idx];
207 }
208 return rc;
209}
210
211
212/**
213 * Wrapper for getting the current context pointer to the entry being modified.
214 *
215 * @returns VBox status code suitable for scheduling.
216 * @param pVM VM Handle.
217 * @param pvDst Destination address
218 * @param pvSrc Source guest virtual address.
219 * @param GCPhysSrc The source guest physical address.
220 * @param cb Size of data to read
221 */
222DECLINLINE(int) pgmPoolPhysSimpleReadGCPhys(PVM pVM, void *pvDst, CTXTYPE(RTGCPTR, RTHCPTR, RTGCPTR) pvSrc, RTGCPHYS GCPhysSrc, size_t cb)
223{
224#if defined(IN_RING3)
225 memcpy(pvDst, (RTHCPTR)((uintptr_t)pvSrc & ~(RTHCUINTPTR)(cb - 1)), cb);
226 return VINF_SUCCESS;
227#else
228 /* @todo in RC we could attempt to use the virtual address, although this can cause many faults (PAE Windows XP guest). */
229 return PGMPhysSimpleReadGCPhys(pVM, pvDst, GCPhysSrc & ~(RTGCPHYS)(cb - 1), cb);
230#endif
231}
232
233/**
234 * Process shadow entries before they are changed by the guest.
235 *
236 * For PT entries we will clear them. For PD entries, we'll simply check
237 * for mapping conflicts and set the SyncCR3 FF if found.
238 *
239 * @param pVCpu VMCPU handle
240 * @param pPool The pool.
241 * @param pPage The head page.
242 * @param GCPhysFault The guest physical fault address.
243 * @param uAddress In R0 and GC this is the guest context fault address (flat).
244 * In R3 this is the host context 'fault' address.
245 * @param pCpu The disassembler state for figuring out the write size.
246 * This need not be specified if the caller knows we won't do cross entry accesses.
247 */
248void pgmPoolMonitorChainChanging(PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhysFault, CTXTYPE(RTGCPTR, RTHCPTR, RTGCPTR) pvAddress, PDISCPUSTATE pCpu)
249{
250 Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
251 const unsigned off = GCPhysFault & PAGE_OFFSET_MASK;
252 const unsigned cbWrite = (pCpu) ? pgmPoolDisasWriteSize(pCpu) : 0;
253 PVM pVM = pPool->CTX_SUFF(pVM);
254
255 LogFlow(("pgmPoolMonitorChainChanging: %RGv phys=%RGp kind=%s cbWrite=%d\n", (RTGCPTR)pvAddress, GCPhysFault, pgmPoolPoolKindToStr(pPage->enmKind), cbWrite));
256 for (;;)
257 {
258 union
259 {
260 void *pv;
261 PX86PT pPT;
262 PX86PTPAE pPTPae;
263 PX86PD pPD;
264 PX86PDPAE pPDPae;
265 PX86PDPT pPDPT;
266 PX86PML4 pPML4;
267 } uShw;
268
269 uShw.pv = NULL;
270 switch (pPage->enmKind)
271 {
272 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
273 {
274 uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
275 const unsigned iShw = off / sizeof(X86PTE);
276 LogFlow(("PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT iShw=%x\n", iShw));
277 if (uShw.pPT->a[iShw].n.u1Present)
278 {
279# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
280 X86PTE GstPte;
281
282 int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
283 AssertRC(rc);
284 Log4(("pgmPoolMonitorChainChanging 32_32: deref %016RX64 GCPhys %08RX32\n", uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK, GstPte.u & X86_PTE_PG_MASK));
285 pgmPoolTracDerefGCPhysHint(pPool, pPage,
286 uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK,
287 GstPte.u & X86_PTE_PG_MASK);
288# endif
289 ASMAtomicWriteSize(&uShw.pPT->a[iShw], 0);
290 }
291 break;
292 }
293
294 /* page/2 sized */
295 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
296 {
297 uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
298 if (!((off ^ pPage->GCPhys) & (PAGE_SIZE / 2)))
299 {
300 const unsigned iShw = (off / sizeof(X86PTE)) & (X86_PG_PAE_ENTRIES - 1);
301 LogFlow(("PGMPOOLKIND_PAE_PT_FOR_32BIT_PT iShw=%x\n", iShw));
302 if (uShw.pPTPae->a[iShw].n.u1Present)
303 {
304# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
305 X86PTE GstPte;
306 int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
307 AssertRC(rc);
308
309 Log4(("pgmPoolMonitorChainChanging pae_32: deref %016RX64 GCPhys %08RX32\n", uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK, GstPte.u & X86_PTE_PG_MASK));
310 pgmPoolTracDerefGCPhysHint(pPool, pPage,
311 uShw.pPTPae->a[iShw].u & X86_PTE_PAE_PG_MASK,
312 GstPte.u & X86_PTE_PG_MASK);
313# endif
314 ASMAtomicWriteSize(&uShw.pPTPae->a[iShw], 0);
315 }
316 }
317 break;
318 }
319
320 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
321 case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
322 case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
323 case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
324 {
325 unsigned iGst = off / sizeof(X86PDE);
326 unsigned iShwPdpt = iGst / 256;
327 unsigned iShw = (iGst % 256) * 2;
328 uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
329
330 LogFlow(("pgmPoolMonitorChainChanging PAE for 32 bits: iGst=%x iShw=%x idx = %d page idx=%d\n", iGst, iShw, iShwPdpt, pPage->enmKind - PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD));
331 if (iShwPdpt == pPage->enmKind - (unsigned)PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD)
332 {
333 for (unsigned i = 0; i < 2; i++)
334 {
335# ifndef IN_RING0
336 if ((uShw.pPDPae->a[iShw + i].u & (PGM_PDFLAGS_MAPPING | X86_PDE_P)) == (PGM_PDFLAGS_MAPPING | X86_PDE_P))
337 {
338 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
339 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
340 LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShwPdpt=%#x iShw=%#x!\n", iShwPdpt, iShw+i));
341 break;
342 }
343 else
344# endif /* !IN_RING0 */
345 if (uShw.pPDPae->a[iShw+i].n.u1Present)
346 {
347 LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw+i, uShw.pPDPae->a[iShw+i].u));
348 pgmPoolFree(pVM,
349 uShw.pPDPae->a[iShw+i].u & X86_PDE_PAE_PG_MASK,
350 pPage->idx,
351 iShw + i);
352 ASMAtomicWriteSize(&uShw.pPDPae->a[iShw+i], 0);
353 }
354
355 /* paranoia / a bit assumptive. */
356 if ( pCpu
357 && (off & 3)
358 && (off & 3) + cbWrite > 4)
359 {
360 const unsigned iShw2 = iShw + 2 + i;
361 if (iShw2 < RT_ELEMENTS(uShw.pPDPae->a))
362 {
363# ifndef IN_RING0
364 if ((uShw.pPDPae->a[iShw2].u & (PGM_PDFLAGS_MAPPING | X86_PDE_P)) == (PGM_PDFLAGS_MAPPING | X86_PDE_P))
365 {
366 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
367 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
368 LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShwPdpt=%#x iShw2=%#x!\n", iShwPdpt, iShw2));
369 break;
370 }
371 else
372# endif /* !IN_RING0 */
373 if (uShw.pPDPae->a[iShw2].n.u1Present)
374 {
375 LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPae->a[iShw2].u));
376 pgmPoolFree(pVM,
377 uShw.pPDPae->a[iShw2].u & X86_PDE_PAE_PG_MASK,
378 pPage->idx,
379 iShw2);
380 ASMAtomicWriteSize(&uShw.pPDPae->a[iShw2].u, 0);
381 }
382 }
383 }
384 }
385 }
386 break;
387 }
388
389 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
390 {
391 uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
392 const unsigned iShw = off / sizeof(X86PTEPAE);
393 if (uShw.pPTPae->a[iShw].n.u1Present)
394 {
395# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
396 X86PTEPAE GstPte;
397 int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
398 AssertRC(rc);
399
400 Log4(("pgmPoolMonitorChainChanging pae: deref %016RX64 GCPhys %016RX64\n", uShw.pPTPae->a[iShw].u & X86_PTE_PAE_PG_MASK, GstPte.u & X86_PTE_PAE_PG_MASK));
401 pgmPoolTracDerefGCPhysHint(pPool, pPage,
402 uShw.pPTPae->a[iShw].u & X86_PTE_PAE_PG_MASK,
403 GstPte.u & X86_PTE_PAE_PG_MASK);
404# endif
405 ASMAtomicWriteSize(&uShw.pPTPae->a[iShw].u, 0);
406 }
407
408 /* paranoia / a bit assumptive. */
409 if ( pCpu
410 && (off & 7)
411 && (off & 7) + cbWrite > sizeof(X86PTEPAE))
412 {
413 const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PTEPAE);
414 AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPTPae->a));
415
416 if (uShw.pPTPae->a[iShw2].n.u1Present)
417 {
418# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
419 X86PTEPAE GstPte;
420# ifdef IN_RING3
421 int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, (RTHCPTR)((RTHCUINTPTR)pvAddress + sizeof(GstPte)), GCPhysFault + sizeof(GstPte), sizeof(GstPte));
422# else
423 int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress + sizeof(GstPte), GCPhysFault + sizeof(GstPte), sizeof(GstPte));
424# endif
425 AssertRC(rc);
426 Log4(("pgmPoolMonitorChainChanging pae: deref %016RX64 GCPhys %016RX64\n", uShw.pPTPae->a[iShw2].u & X86_PTE_PAE_PG_MASK, GstPte.u & X86_PTE_PAE_PG_MASK));
427 pgmPoolTracDerefGCPhysHint(pPool, pPage,
428 uShw.pPTPae->a[iShw2].u & X86_PTE_PAE_PG_MASK,
429 GstPte.u & X86_PTE_PAE_PG_MASK);
430# endif
431 ASMAtomicWriteSize(&uShw.pPTPae->a[iShw2].u ,0);
432 }
433 }
434 break;
435 }
436
437 case PGMPOOLKIND_32BIT_PD:
438 {
439 uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
440 const unsigned iShw = off / sizeof(X86PTE); // ASSUMING 32-bit guest paging!
441
442 LogFlow(("pgmPoolMonitorChainChanging: PGMPOOLKIND_32BIT_PD %x\n", iShw));
443# ifndef IN_RING0
444 if (uShw.pPD->a[iShw].u & PGM_PDFLAGS_MAPPING)
445 {
446 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
447 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
448 STAM_COUNTER_INC(&(pVCpu->pgm.s.StatRZGuestCR3WriteConflict));
449 LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw=%#x!\n", iShw));
450 break;
451 }
452# endif /* !IN_RING0 */
453# ifndef IN_RING0
454 else
455# endif /* !IN_RING0 */
456 {
457 if (uShw.pPD->a[iShw].n.u1Present)
458 {
459 LogFlow(("pgmPoolMonitorChainChanging: 32 bit pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPD->a[iShw].u));
460 pgmPoolFree(pVM,
461 uShw.pPD->a[iShw].u & X86_PDE_PAE_PG_MASK,
462 pPage->idx,
463 iShw);
464 ASMAtomicWriteSize(&uShw.pPD->a[iShw].u, 0);
465 }
466 }
467 /* paranoia / a bit assumptive. */
468 if ( pCpu
469 && (off & 3)
470 && (off & 3) + cbWrite > sizeof(X86PTE))
471 {
472 const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PTE);
473 if ( iShw2 != iShw
474 && iShw2 < RT_ELEMENTS(uShw.pPD->a))
475 {
476# ifndef IN_RING0
477 if (uShw.pPD->a[iShw2].u & PGM_PDFLAGS_MAPPING)
478 {
479 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
480 STAM_COUNTER_INC(&(pVCpu->pgm.s.StatRZGuestCR3WriteConflict));
481 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
482 LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
483 break;
484 }
485# endif /* !IN_RING0 */
486# ifndef IN_RING0
487 else
488# endif /* !IN_RING0 */
489 {
490 if (uShw.pPD->a[iShw2].n.u1Present)
491 {
492 LogFlow(("pgmPoolMonitorChainChanging: 32 bit pd iShw=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPD->a[iShw2].u));
493 pgmPoolFree(pVM,
494 uShw.pPD->a[iShw2].u & X86_PDE_PAE_PG_MASK,
495 pPage->idx,
496 iShw2);
497 ASMAtomicWriteSize(&uShw.pPD->a[iShw2].u, 0);
498 }
499 }
500 }
501 }
502#if 0 /* useful when running PGMAssertCR3(), a bit too troublesome for general use (TLBs). */
503 if ( uShw.pPD->a[iShw].n.u1Present
504 && !VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
505 {
506 LogFlow(("pgmPoolMonitorChainChanging: iShw=%#x: %RX32 -> freeing it!\n", iShw, uShw.pPD->a[iShw].u));
507# ifdef IN_RC /* TLB load - we're pushing things a bit... */
508 ASMProbeReadByte(pvAddress);
509# endif
510 pgmPoolFree(pVM, uShw.pPD->a[iShw].u & X86_PDE_PG_MASK, pPage->idx, iShw);
511 ASMAtomicWriteSize(&uShw.pPD->a[iShw].u, 0);
512 }
513#endif
514 break;
515 }
516
517 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
518 {
519 uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
520 const unsigned iShw = off / sizeof(X86PDEPAE);
521#ifndef IN_RING0
522 if (uShw.pPDPae->a[iShw].u & PGM_PDFLAGS_MAPPING)
523 {
524 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
525 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
526 STAM_COUNTER_INC(&(pVCpu->pgm.s.StatRZGuestCR3WriteConflict));
527 LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw=%#x!\n", iShw));
528 break;
529 }
530#endif /* !IN_RING0 */
531 /*
532 * Causes trouble when the guest uses a PDE to refer to the whole page table level
533 * structure. (Invalidate here; faults later on when it tries to change the page
534 * table entries -> recheck; probably only applies to the RC case.)
535 */
536# ifndef IN_RING0
537 else
538# endif /* !IN_RING0 */
539 {
540 if (uShw.pPDPae->a[iShw].n.u1Present)
541 {
542 LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPae->a[iShw].u));
543 pgmPoolFree(pVM,
544 uShw.pPDPae->a[iShw].u & X86_PDE_PAE_PG_MASK,
545 pPage->idx,
546 iShw);
547 ASMAtomicWriteSize(&uShw.pPDPae->a[iShw].u, 0);
548 }
549 }
550 /* paranoia / a bit assumptive. */
551 if ( pCpu
552 && (off & 7)
553 && (off & 7) + cbWrite > sizeof(X86PDEPAE))
554 {
555 const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDEPAE);
556 AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPDPae->a));
557
558#ifndef IN_RING0
559 if ( iShw2 != iShw
560 && uShw.pPDPae->a[iShw2].u & PGM_PDFLAGS_MAPPING)
561 {
562 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
563 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
564 STAM_COUNTER_INC(&(pVCpu->pgm.s.StatRZGuestCR3WriteConflict));
565 LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
566 break;
567 }
568#endif /* !IN_RING0 */
569# ifndef IN_RING0
570 else
571# endif /* !IN_RING0 */
572 if (uShw.pPDPae->a[iShw2].n.u1Present)
573 {
574 LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPae->a[iShw2].u));
575 pgmPoolFree(pVM,
576 uShw.pPDPae->a[iShw2].u & X86_PDE_PAE_PG_MASK,
577 pPage->idx,
578 iShw2);
579 ASMAtomicWriteSize(&uShw.pPDPae->a[iShw2].u, 0);
580 }
581 }
582 break;
583 }
584
585 case PGMPOOLKIND_PAE_PDPT:
586 {
587 /*
588 * Hopefully this doesn't happen very often:
589 * - touching unused parts of the page
590 * - messing with the bits of pd pointers without changing the physical address
591 */
592 /* PDPT roots are not page aligned; 32 byte only! */
593 const unsigned offPdpt = GCPhysFault - pPage->GCPhys;
594
595 uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
596 const unsigned iShw = offPdpt / sizeof(X86PDPE);
597 if (iShw < X86_PG_PAE_PDPE_ENTRIES) /* don't use RT_ELEMENTS(uShw.pPDPT->a), because that's for long mode only */
598 {
599# ifndef IN_RING0
600 if (uShw.pPDPT->a[iShw].u & PGM_PLXFLAGS_MAPPING)
601 {
602 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
603 STAM_COUNTER_INC(&(pVCpu->pgm.s.StatRZGuestCR3WriteConflict));
604 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
605 LogFlow(("pgmPoolMonitorChainChanging: Detected pdpt conflict at iShw=%#x!\n", iShw));
606 break;
607 }
608# endif /* !IN_RING0 */
609# ifndef IN_RING0
610 else
611# endif /* !IN_RING0 */
612 if (uShw.pPDPT->a[iShw].n.u1Present)
613 {
614 LogFlow(("pgmPoolMonitorChainChanging: pae pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPT->a[iShw].u));
615 pgmPoolFree(pVM,
616 uShw.pPDPT->a[iShw].u & X86_PDPE_PG_MASK,
617 pPage->idx,
618 iShw);
619 ASMAtomicWriteSize(&uShw.pPDPT->a[iShw].u, 0);
620 }
621
622 /* paranoia / a bit assumptive. */
623 if ( pCpu
624 && (offPdpt & 7)
625 && (offPdpt & 7) + cbWrite > sizeof(X86PDPE))
626 {
627 const unsigned iShw2 = (offPdpt + cbWrite - 1) / sizeof(X86PDPE);
628 if ( iShw2 != iShw
629 && iShw2 < X86_PG_PAE_PDPE_ENTRIES)
630 {
631# ifndef IN_RING0
632 if (uShw.pPDPT->a[iShw2].u & PGM_PLXFLAGS_MAPPING)
633 {
634 Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
635 STAM_COUNTER_INC(&(pVCpu->pgm.s.StatRZGuestCR3WriteConflict));
636 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
637 LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
638 break;
639 }
640# endif /* !IN_RING0 */
641# ifndef IN_RING0
642 else
643# endif /* !IN_RING0 */
644 if (uShw.pPDPT->a[iShw2].n.u1Present)
645 {
646 LogFlow(("pgmPoolMonitorChainChanging: pae pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPT->a[iShw2].u));
647 pgmPoolFree(pVM,
648 uShw.pPDPT->a[iShw2].u & X86_PDPE_PG_MASK,
649 pPage->idx,
650 iShw2);
651 ASMAtomicWriteSize(&uShw.pPDPT->a[iShw2].u, 0);
652 }
653 }
654 }
655 }
656 break;
657 }
658
659#ifndef IN_RC
660 case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
661 {
662 uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
663 const unsigned iShw = off / sizeof(X86PDEPAE);
664 Assert(!(uShw.pPDPae->a[iShw].u & PGM_PDFLAGS_MAPPING));
665 if (uShw.pPDPae->a[iShw].n.u1Present)
666 {
667 LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPae->a[iShw].u));
668 pgmPoolFree(pVM,
669 uShw.pPDPae->a[iShw].u & X86_PDE_PAE_PG_MASK,
670 pPage->idx,
671 iShw);
672 ASMAtomicWriteSize(&uShw.pPDPae->a[iShw].u, 0);
673 }
674 /* paranoia / a bit assumptive. */
675 if ( pCpu
676 && (off & 7)
677 && (off & 7) + cbWrite > sizeof(X86PDEPAE))
678 {
679 const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDEPAE);
680 AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPDPae->a));
681
682 Assert(!(uShw.pPDPae->a[iShw2].u & PGM_PDFLAGS_MAPPING));
683 if (uShw.pPDPae->a[iShw2].n.u1Present)
684 {
685 LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPae->a[iShw2].u));
686 pgmPoolFree(pVM,
687 uShw.pPDPae->a[iShw2].u & X86_PDE_PAE_PG_MASK,
688 pPage->idx,
689 iShw2);
690 ASMAtomicWriteSize(&uShw.pPDPae->a[iShw2].u, 0);
691 }
692 }
693 break;
694 }
695
696 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
697 {
698 /*
699 * Hopefully this doesn't happen very often:
700 * - messing with the bits of pd pointers without changing the physical address
701 */
702 if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
703 {
704 uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
705 const unsigned iShw = off / sizeof(X86PDPE);
706 if (uShw.pPDPT->a[iShw].n.u1Present)
707 {
708 LogFlow(("pgmPoolMonitorChainChanging: pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPT->a[iShw].u));
709 pgmPoolFree(pVM, uShw.pPDPT->a[iShw].u & X86_PDPE_PG_MASK, pPage->idx, iShw);
710 ASMAtomicWriteSize(&uShw.pPDPT->a[iShw].u, 0);
711 }
712 /* paranoia / a bit assumptive. */
713 if ( pCpu
714 && (off & 7)
715 && (off & 7) + cbWrite > sizeof(X86PDPE))
716 {
717 const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDPE);
718 if (uShw.pPDPT->a[iShw2].n.u1Present)
719 {
720 LogFlow(("pgmPoolMonitorChainChanging: pdpt iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPT->a[iShw2].u));
721 pgmPoolFree(pVM, uShw.pPDPT->a[iShw2].u & X86_PDPE_PG_MASK, pPage->idx, iShw2);
722 ASMAtomicWriteSize(&uShw.pPDPT->a[iShw2].u, 0);
723 }
724 }
725 }
726 break;
727 }
728
729 case PGMPOOLKIND_64BIT_PML4:
730 {
731 /*
732 * Hopefully this doesn't happen very often:
733 * - messing with the bits of pd pointers without changing the physical address
734 */
735 if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
736 {
737 uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
738 const unsigned iShw = off / sizeof(X86PDPE);
739 if (uShw.pPML4->a[iShw].n.u1Present)
740 {
741 LogFlow(("pgmPoolMonitorChainChanging: pml4 iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPML4->a[iShw].u));
742 pgmPoolFree(pVM, uShw.pPML4->a[iShw].u & X86_PML4E_PG_MASK, pPage->idx, iShw);
743 ASMAtomicWriteSize(&uShw.pPML4->a[iShw].u, 0);
744 }
745 /* paranoia / a bit assumptive. */
746 if ( pCpu
747 && (off & 7)
748 && (off & 7) + cbWrite > sizeof(X86PDPE))
749 {
750 const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PML4E);
751 if (uShw.pPML4->a[iShw2].n.u1Present)
752 {
753 LogFlow(("pgmPoolMonitorChainChanging: pml4 iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPML4->a[iShw2].u));
754 pgmPoolFree(pVM, uShw.pPML4->a[iShw2].u & X86_PML4E_PG_MASK, pPage->idx, iShw2);
755 ASMAtomicWriteSize(&uShw.pPML4->a[iShw2].u, 0);
756 }
757 }
758 }
759 break;
760 }
761#endif /* IN_RING0 */
762
763 default:
764 AssertFatalMsgFailed(("enmKind=%d\n", pPage->enmKind));
765 }
766 PGMPOOL_UNLOCK_PTR(pVM, uShw.pv);
767
768 /* next */
769 if (pPage->iMonitoredNext == NIL_PGMPOOL_IDX)
770 return;
771 pPage = &pPool->aPages[pPage->iMonitoredNext];
772 }
773}
774
775# ifndef IN_RING3
776/**
777 * Checks if a access could be a fork operation in progress.
778 *
779 * Meaning, that the guest is setuping up the parent process for Copy-On-Write.
780 *
781 * @returns true if it's likly that we're forking, otherwise false.
782 * @param pPool The pool.
783 * @param pCpu The disassembled instruction.
784 * @param offFault The access offset.
785 */
786DECLINLINE(bool) pgmPoolMonitorIsForking(PPGMPOOL pPool, PDISCPUSTATE pCpu, unsigned offFault)
787{
788 /*
789 * i386 linux is using btr to clear X86_PTE_RW.
790 * The functions involved are (2.6.16 source inspection):
791 * clear_bit
792 * ptep_set_wrprotect
793 * copy_one_pte
794 * copy_pte_range
795 * copy_pmd_range
796 * copy_pud_range
797 * copy_page_range
798 * dup_mmap
799 * dup_mm
800 * copy_mm
801 * copy_process
802 * do_fork
803 */
804 if ( pCpu->pCurInstr->opcode == OP_BTR
805 && !(offFault & 4)
806 /** @todo Validate that the bit index is X86_PTE_RW. */
807 )
808 {
809 STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,Fork));
810 return true;
811 }
812 return false;
813}
814
815
816/**
817 * Determine whether the page is likely to have been reused.
818 *
819 * @returns true if we consider the page as being reused for a different purpose.
820 * @returns false if we consider it to still be a paging page.
821 * @param pVM VM Handle.
822 * @param pRegFrame Trap register frame.
823 * @param pCpu The disassembly info for the faulting instruction.
824 * @param pvFault The fault address.
825 *
826 * @remark The REP prefix check is left to the caller because of STOSD/W.
827 */
828DECLINLINE(bool) pgmPoolMonitorIsReused(PVM pVM, PCPUMCTXCORE pRegFrame, PDISCPUSTATE pCpu, RTGCPTR pvFault)
829{
830#ifndef IN_RC
831 /** @todo could make this general, faulting close to rsp should be safe reuse heuristic. */
832 if ( HWACCMHasPendingIrq(pVM)
833 && (pRegFrame->rsp - pvFault) < 32)
834 {
835 /* Fault caused by stack writes while trying to inject an interrupt event. */
836 Log(("pgmPoolMonitorIsReused: reused %RGv for interrupt stack (rsp=%RGv).\n", pvFault, pRegFrame->rsp));
837 return true;
838 }
839#else
840 NOREF(pVM); NOREF(pvFault);
841#endif
842
843 switch (pCpu->pCurInstr->opcode)
844 {
845 /* call implies the actual push of the return address faulted */
846 case OP_CALL:
847 Log4(("pgmPoolMonitorIsReused: CALL\n"));
848 return true;
849 case OP_PUSH:
850 Log4(("pgmPoolMonitorIsReused: PUSH\n"));
851 return true;
852 case OP_PUSHF:
853 Log4(("pgmPoolMonitorIsReused: PUSHF\n"));
854 return true;
855 case OP_PUSHA:
856 Log4(("pgmPoolMonitorIsReused: PUSHA\n"));
857 return true;
858 case OP_FXSAVE:
859 Log4(("pgmPoolMonitorIsReused: FXSAVE\n"));
860 return true;
861 case OP_MOVNTI: /* solaris - block_zero_no_xmm */
862 Log4(("pgmPoolMonitorIsReused: MOVNTI\n"));
863 return true;
864 case OP_MOVNTDQ: /* solaris - hwblkclr & hwblkpagecopy */
865 Log4(("pgmPoolMonitorIsReused: MOVNTDQ\n"));
866 return true;
867 case OP_MOVSWD:
868 case OP_STOSWD:
869 if ( pCpu->prefix == (PREFIX_REP|PREFIX_REX)
870 && pRegFrame->rcx >= 0x40
871 )
872 {
873 Assert(pCpu->mode == CPUMODE_64BIT);
874
875 Log(("pgmPoolMonitorIsReused: OP_STOSQ\n"));
876 return true;
877 }
878 return false;
879 }
880 if ( (pCpu->param1.flags & USE_REG_GEN32)
881 && (pCpu->param1.base.reg_gen == USE_REG_ESP))
882 {
883 Log4(("pgmPoolMonitorIsReused: ESP\n"));
884 return true;
885 }
886
887 return false;
888}
889
890
891/**
892 * Flushes the page being accessed.
893 *
894 * @returns VBox status code suitable for scheduling.
895 * @param pVM The VM handle.
896 * @param pVCpu The VMCPU handle.
897 * @param pPool The pool.
898 * @param pPage The pool page (head).
899 * @param pCpu The disassembly of the write instruction.
900 * @param pRegFrame The trap register frame.
901 * @param GCPhysFault The fault address as guest physical address.
902 * @param pvFault The fault address.
903 */
904static int pgmPoolAccessHandlerFlush(PVM pVM, PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pCpu,
905 PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
906{
907 /*
908 * First, do the flushing.
909 */
910 int rc = pgmPoolMonitorChainFlush(pPool, pPage);
911
912 /*
913 * Emulate the instruction (xp/w2k problem, requires pc/cr2/sp detection).
914 */
915 uint32_t cbWritten;
916 int rc2 = EMInterpretInstructionCPU(pVM, pVCpu, pCpu, pRegFrame, pvFault, &cbWritten);
917 if (RT_SUCCESS(rc2))
918 pRegFrame->rip += pCpu->opsize;
919 else if (rc2 == VERR_EM_INTERPRETER)
920 {
921#ifdef IN_RC
922 if (PATMIsPatchGCAddr(pVM, (RTRCPTR)pRegFrame->eip))
923 {
924 LogFlow(("pgmPoolAccessHandlerPTWorker: Interpretation failed for patch code %04x:%RGv, ignoring.\n",
925 pRegFrame->cs, (RTGCPTR)pRegFrame->eip));
926 rc = VINF_SUCCESS;
927 STAM_COUNTER_INC(&pPool->StatMonitorRZIntrFailPatch2);
928 }
929 else
930#endif
931 {
932 rc = VINF_EM_RAW_EMULATE_INSTR;
933 STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,EmulateInstr));
934 }
935 }
936 else
937 rc = rc2;
938
939 /* See use in pgmPoolAccessHandlerSimple(). */
940 PGM_INVL_VCPU_TLBS(pVCpu);
941
942 LogFlow(("pgmPoolAccessHandlerPT: returns %Rrc (flushed)\n", rc));
943 return rc;
944
945}
946
947
948/**
949 * Handles the STOSD write accesses.
950 *
951 * @returns VBox status code suitable for scheduling.
952 * @param pVM The VM handle.
953 * @param pPool The pool.
954 * @param pPage The pool page (head).
955 * @param pCpu The disassembly of the write instruction.
956 * @param pRegFrame The trap register frame.
957 * @param GCPhysFault The fault address as guest physical address.
958 * @param pvFault The fault address.
959 */
960DECLINLINE(int) pgmPoolAccessHandlerSTOSD(PVM pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pCpu,
961 PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
962{
963 Assert(pCpu->mode == CPUMODE_32BIT);
964
965 Log3(("pgmPoolAccessHandlerSTOSD\n"));
966
967 /*
968 * Increment the modification counter and insert it into the list
969 * of modified pages the first time.
970 */
971 if (!pPage->cModifications++)
972 pgmPoolMonitorModifiedInsert(pPool, pPage);
973
974 /*
975 * Execute REP STOSD.
976 *
977 * This ASSUMES that we're not invoked by Trap0e on in a out-of-sync
978 * write situation, meaning that it's safe to write here.
979 */
980 PVMCPU pVCpu = VMMGetCpu(pPool->CTX_SUFF(pVM));
981 RTGCUINTPTR pu32 = (RTGCUINTPTR)pvFault;
982 while (pRegFrame->ecx)
983 {
984#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
985 uint32_t iPrevSubset = PGMDynMapPushAutoSubset(pVCpu);
986 pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, (RTGCPTR)pu32, NULL);
987 PGMDynMapPopAutoSubset(pVCpu, iPrevSubset);
988#else
989 pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, (RTGCPTR)pu32, NULL);
990#endif
991#ifdef IN_RC
992 *(uint32_t *)pu32 = pRegFrame->eax;
993#else
994 PGMPhysSimpleWriteGCPhys(pVM, GCPhysFault, &pRegFrame->eax, 4);
995#endif
996 pu32 += 4;
997 GCPhysFault += 4;
998 pRegFrame->edi += 4;
999 pRegFrame->ecx--;
1000 }
1001 pRegFrame->rip += pCpu->opsize;
1002
1003#ifdef IN_RC
1004 /* See use in pgmPoolAccessHandlerSimple(). */
1005 PGM_INVL_VCPU_TLBS(pVCpu);
1006#endif
1007
1008 LogFlow(("pgmPoolAccessHandlerSTOSD: returns\n"));
1009 return VINF_SUCCESS;
1010}
1011
1012
1013/**
1014 * Handles the simple write accesses.
1015 *
1016 * @returns VBox status code suitable for scheduling.
1017 * @param pVM The VM handle.
1018 * @param pVCpu The VMCPU handle.
1019 * @param pPool The pool.
1020 * @param pPage The pool page (head).
1021 * @param pCpu The disassembly of the write instruction.
1022 * @param pRegFrame The trap register frame.
1023 * @param GCPhysFault The fault address as guest physical address.
1024 * @param pvFault The fault address.
1025 */
1026DECLINLINE(int) pgmPoolAccessHandlerSimple(PVM pVM, PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pCpu,
1027 PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
1028{
1029 Log3(("pgmPoolAccessHandlerSimple\n"));
1030 /*
1031 * Increment the modification counter and insert it into the list
1032 * of modified pages the first time.
1033 */
1034 if (!pPage->cModifications++)
1035 pgmPoolMonitorModifiedInsert(pPool, pPage);
1036
1037 /*
1038 * Clear all the pages. ASSUMES that pvFault is readable.
1039 */
1040#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
1041 uint32_t iPrevSubset = PGMDynMapPushAutoSubset(pVCpu);
1042 pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, pvFault, pCpu);
1043 PGMDynMapPopAutoSubset(pVCpu, iPrevSubset);
1044#else
1045 pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, pvFault, pCpu);
1046#endif
1047
1048 /*
1049 * Interpret the instruction.
1050 */
1051 uint32_t cb;
1052 int rc = EMInterpretInstructionCPU(pVM, pVCpu, pCpu, pRegFrame, pvFault, &cb);
1053 if (RT_SUCCESS(rc))
1054 pRegFrame->rip += pCpu->opsize;
1055 else if (rc == VERR_EM_INTERPRETER)
1056 {
1057 LogFlow(("pgmPoolAccessHandlerPTWorker: Interpretation failed for %04x:%RGv - opcode=%d\n",
1058 pRegFrame->cs, (RTGCPTR)pRegFrame->rip, pCpu->pCurInstr->opcode));
1059 rc = VINF_EM_RAW_EMULATE_INSTR;
1060 STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,EmulateInstr));
1061 }
1062
1063#ifdef IN_RC
1064 /*
1065 * Quick hack, with logging enabled we're getting stale
1066 * code TLBs but no data TLB for EIP and crash in EMInterpretDisasOne.
1067 * Flushing here is BAD and expensive, I think EMInterpretDisasOne will
1068 * have to be fixed to support this. But that'll have to wait till next week.
1069 *
1070 * An alternative is to keep track of the changed PTEs together with the
1071 * GCPhys from the guest PT. This may proove expensive though.
1072 *
1073 * At the moment, it's VITAL that it's done AFTER the instruction interpreting
1074 * because we need the stale TLBs in some cases (XP boot). This MUST be fixed properly!
1075 */
1076 PGM_INVL_VCPU_TLBS(pVCpu);
1077#endif
1078
1079 LogFlow(("pgmPoolAccessHandlerSimple: returns %Rrc cb=%d\n", rc, cb));
1080 return rc;
1081}
1082
1083/**
1084 * \#PF Handler callback for PT write accesses.
1085 *
1086 * @returns VBox status code (appropriate for GC return).
1087 * @param pVM VM Handle.
1088 * @param uErrorCode CPU Error code.
1089 * @param pRegFrame Trap register frame.
1090 * NULL on DMA and other non CPU access.
1091 * @param pvFault The fault address (cr2).
1092 * @param GCPhysFault The GC physical address corresponding to pvFault.
1093 * @param pvUser User argument.
1094 */
1095DECLEXPORT(int) pgmPoolAccessHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
1096{
1097 STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), a);
1098 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1099 PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)pvUser;
1100 PVMCPU pVCpu = VMMGetCpu(pVM);
1101
1102 LogFlow(("pgmPoolAccessHandler: pvFault=%RGv pPage=%p:{.idx=%d} GCPhysFault=%RGp\n", pvFault, pPage, pPage->idx, GCPhysFault));
1103
1104 /*
1105 * We should ALWAYS have the list head as user parameter. This
1106 * is because we use that page to record the changes.
1107 */
1108 Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
1109
1110 /*
1111 * Disassemble the faulting instruction.
1112 */
1113 DISCPUSTATE Cpu;
1114 int rc = EMInterpretDisasOne(pVM, pVCpu, pRegFrame, &Cpu, NULL);
1115 AssertRCReturn(rc, rc);
1116
1117 pgmLock(pVM);
1118 if (PHYS_PAGE_ADDRESS(GCPhysFault) != PHYS_PAGE_ADDRESS(pPage->GCPhys))
1119 {
1120 /* Pool page changed while we were waiting for the lock; ignore. */
1121 Log(("CPU%d: pgmPoolAccessHandler pgm pool page for %RGp changed (to %RGp) while waiting!\n", pVCpu->idCpu, PHYS_PAGE_ADDRESS(GCPhysFault), PHYS_PAGE_ADDRESS(pPage->GCPhys)));
1122 STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), &pPool->CTX_MID_Z(StatMonitor,Handled), a);
1123 pgmUnlock(pVM);
1124 return VINF_SUCCESS;
1125 }
1126
1127 /*
1128 * Check if it's worth dealing with.
1129 */
1130 bool fReused = false;
1131 if ( ( pPage->cModifications < 48 /** @todo #define */ /** @todo need to check that it's not mapping EIP. */ /** @todo adjust this! */
1132 || pgmPoolIsPageLocked(&pVM->pgm.s, pPage)
1133 )
1134 && !(fReused = pgmPoolMonitorIsReused(pVM, pRegFrame, &Cpu, pvFault))
1135 && !pgmPoolMonitorIsForking(pPool, &Cpu, GCPhysFault & PAGE_OFFSET_MASK))
1136 {
1137 /*
1138 * Simple instructions, no REP prefix.
1139 */
1140 if (!(Cpu.prefix & (PREFIX_REP | PREFIX_REPNE)))
1141 {
1142 rc = pgmPoolAccessHandlerSimple(pVM, pVCpu, pPool, pPage, &Cpu, pRegFrame, GCPhysFault, pvFault);
1143 STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), &pPool->CTX_MID_Z(StatMonitor,Handled), a);
1144 pgmUnlock(pVM);
1145 return rc;
1146 }
1147
1148 /*
1149 * Windows is frequently doing small memset() operations (netio test 4k+).
1150 * We have to deal with these or we'll kill the cache and performance.
1151 */
1152 if ( Cpu.pCurInstr->opcode == OP_STOSWD
1153 && CPUMGetGuestCPL(pVCpu, pRegFrame) == 0
1154 && pRegFrame->ecx <= 0x20
1155 && pRegFrame->ecx * 4 <= PAGE_SIZE - ((uintptr_t)pvFault & PAGE_OFFSET_MASK)
1156 && !((uintptr_t)pvFault & 3)
1157 && (pRegFrame->eax == 0 || pRegFrame->eax == 0x80) /* the two values observed. */
1158 && Cpu.mode == CPUMODE_32BIT
1159 && Cpu.opmode == CPUMODE_32BIT
1160 && Cpu.addrmode == CPUMODE_32BIT
1161 && Cpu.prefix == PREFIX_REP
1162 && !pRegFrame->eflags.Bits.u1DF
1163 )
1164 {
1165 rc = pgmPoolAccessHandlerSTOSD(pVM, pPool, pPage, &Cpu, pRegFrame, GCPhysFault, pvFault);
1166 STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), &pPool->CTX_MID_Z(StatMonitor,RepStosd), a);
1167 pgmUnlock(pVM);
1168 return rc;
1169 }
1170
1171 /* REP prefix, don't bother. */
1172 STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,RepPrefix));
1173 Log4(("pgmPoolAccessHandler: eax=%#x ecx=%#x edi=%#x esi=%#x rip=%RGv opcode=%d prefix=%#x\n",
1174 pRegFrame->eax, pRegFrame->ecx, pRegFrame->edi, pRegFrame->esi, (RTGCPTR)pRegFrame->rip, Cpu.pCurInstr->opcode, Cpu.prefix));
1175 }
1176
1177 /*
1178 * Not worth it, so flush it.
1179 *
1180 * If we considered it to be reused, don't go back to ring-3
1181 * to emulate failed instructions since we usually cannot
1182 * interpret then. This may be a bit risky, in which case
1183 * the reuse detection must be fixed.
1184 */
1185 rc = pgmPoolAccessHandlerFlush(pVM, pVCpu, pPool, pPage, &Cpu, pRegFrame, GCPhysFault, pvFault);
1186 if (rc == VINF_EM_RAW_EMULATE_INSTR && fReused)
1187 rc = VINF_SUCCESS;
1188 STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), &pPool->CTX_MID_Z(StatMonitor,FlushPage), a);
1189 pgmUnlock(pVM);
1190 return rc;
1191}
1192
1193# endif /* !IN_RING3 */
1194#endif /* PGMPOOL_WITH_MONITORING */
1195
1196#ifdef PGMPOOL_WITH_CACHE
1197
1198/**
1199 * Inserts a page into the GCPhys hash table.
1200 *
1201 * @param pPool The pool.
1202 * @param pPage The page.
1203 */
1204DECLINLINE(void) pgmPoolHashInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1205{
1206 Log3(("pgmPoolHashInsert: %RGp\n", pPage->GCPhys));
1207 Assert(pPage->GCPhys != NIL_RTGCPHYS); Assert(pPage->iNext == NIL_PGMPOOL_IDX);
1208 uint16_t iHash = PGMPOOL_HASH(pPage->GCPhys);
1209 pPage->iNext = pPool->aiHash[iHash];
1210 pPool->aiHash[iHash] = pPage->idx;
1211}
1212
1213
1214/**
1215 * Removes a page from the GCPhys hash table.
1216 *
1217 * @param pPool The pool.
1218 * @param pPage The page.
1219 */
1220DECLINLINE(void) pgmPoolHashRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1221{
1222 Log3(("pgmPoolHashRemove: %RGp\n", pPage->GCPhys));
1223 uint16_t iHash = PGMPOOL_HASH(pPage->GCPhys);
1224 if (pPool->aiHash[iHash] == pPage->idx)
1225 pPool->aiHash[iHash] = pPage->iNext;
1226 else
1227 {
1228 uint16_t iPrev = pPool->aiHash[iHash];
1229 for (;;)
1230 {
1231 const int16_t i = pPool->aPages[iPrev].iNext;
1232 if (i == pPage->idx)
1233 {
1234 pPool->aPages[iPrev].iNext = pPage->iNext;
1235 break;
1236 }
1237 if (i == NIL_PGMPOOL_IDX)
1238 {
1239 AssertReleaseMsgFailed(("GCPhys=%RGp idx=%#x\n", pPage->GCPhys, pPage->idx));
1240 break;
1241 }
1242 iPrev = i;
1243 }
1244 }
1245 pPage->iNext = NIL_PGMPOOL_IDX;
1246}
1247
1248
1249/**
1250 * Frees up one cache page.
1251 *
1252 * @returns VBox status code.
1253 * @retval VINF_SUCCESS on success.
1254 * @param pPool The pool.
1255 * @param iUser The user index.
1256 */
1257static int pgmPoolCacheFreeOne(PPGMPOOL pPool, uint16_t iUser)
1258{
1259#ifndef IN_RC
1260 const PVM pVM = pPool->CTX_SUFF(pVM);
1261#endif
1262 Assert(pPool->iAgeHead != pPool->iAgeTail); /* We shouldn't be here if there < 2 cached entries! */
1263 STAM_COUNTER_INC(&pPool->StatCacheFreeUpOne);
1264
1265 /*
1266 * Select one page from the tail of the age list.
1267 */
1268 PPGMPOOLPAGE pPage;
1269 for (unsigned iLoop = 0; ; iLoop++)
1270 {
1271 uint16_t iToFree = pPool->iAgeTail;
1272 if (iToFree == iUser)
1273 iToFree = pPool->aPages[iToFree].iAgePrev;
1274/* This is the alternative to the SyncCR3 pgmPoolCacheUsed calls.
1275 if (pPool->aPages[iToFree].iUserHead != NIL_PGMPOOL_USER_INDEX)
1276 {
1277 uint16_t i = pPool->aPages[iToFree].iAgePrev;
1278 for (unsigned j = 0; j < 10 && i != NIL_PGMPOOL_USER_INDEX; j++, i = pPool->aPages[i].iAgePrev)
1279 {
1280 if (pPool->aPages[iToFree].iUserHead == NIL_PGMPOOL_USER_INDEX)
1281 continue;
1282 iToFree = i;
1283 break;
1284 }
1285 }
1286*/
1287 Assert(iToFree != iUser);
1288 AssertRelease(iToFree != NIL_PGMPOOL_IDX);
1289 pPage = &pPool->aPages[iToFree];
1290
1291 /*
1292 * Reject any attempts at flushing the currently active shadow CR3 mapping.
1293 * Call pgmPoolCacheUsed to move the page to the head of the age list.
1294 */
1295 if (!pgmPoolIsPageLocked(&pPool->CTX_SUFF(pVM)->pgm.s, pPage))
1296 break;
1297 LogFlow(("pgmPoolCacheFreeOne: refuse CR3 mapping\n"));
1298 pgmPoolCacheUsed(pPool, pPage);
1299 AssertLogRelReturn(iLoop < 8192, VERR_INTERNAL_ERROR);
1300 }
1301
1302 /*
1303 * Found a usable page, flush it and return.
1304 */
1305 int rc = pgmPoolFlushPage(pPool, pPage);
1306 /* This flush was initiated by us and not the guest, so explicitly flush the TLB. */
1307 if (rc == VINF_SUCCESS)
1308 PGM_INVL_ALL_VCPU_TLBS(pVM);
1309 return rc;
1310}
1311
1312
1313/**
1314 * Checks if a kind mismatch is really a page being reused
1315 * or if it's just normal remappings.
1316 *
1317 * @returns true if reused and the cached page (enmKind1) should be flushed
1318 * @returns false if not reused.
1319 * @param enmKind1 The kind of the cached page.
1320 * @param enmKind2 The kind of the requested page.
1321 */
1322static bool pgmPoolCacheReusedByKind(PGMPOOLKIND enmKind1, PGMPOOLKIND enmKind2)
1323{
1324 switch (enmKind1)
1325 {
1326 /*
1327 * Never reuse them. There is no remapping in non-paging mode.
1328 */
1329 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1330 case PGMPOOLKIND_32BIT_PD_PHYS:
1331 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1332 case PGMPOOLKIND_PAE_PD_PHYS:
1333 case PGMPOOLKIND_PAE_PDPT_PHYS:
1334 case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
1335 case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
1336 case PGMPOOLKIND_EPT_PT_FOR_PHYS:
1337 case PGMPOOLKIND_EPT_PD_FOR_PHYS:
1338 case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
1339 case PGMPOOLKIND_PAE_PDPT_FOR_32BIT: /* never reuse them for other types */
1340 return false;
1341
1342 /*
1343 * It's perfectly fine to reuse these, except for PAE and non-paging stuff.
1344 */
1345 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
1346 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1347 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1348 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1349 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
1350 case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
1351 case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
1352 case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
1353 case PGMPOOLKIND_32BIT_PD:
1354 case PGMPOOLKIND_PAE_PDPT:
1355 switch (enmKind2)
1356 {
1357 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
1358 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1359 case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
1360 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
1361 case PGMPOOLKIND_64BIT_PML4:
1362 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
1363 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1364 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1365 case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
1366 case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
1367 case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
1368 case PGMPOOLKIND_EPT_PD_FOR_PHYS:
1369 case PGMPOOLKIND_EPT_PT_FOR_PHYS:
1370 return true;
1371 default:
1372 return false;
1373 }
1374
1375 /*
1376 * It's perfectly fine to reuse these, except for PAE and non-paging stuff.
1377 */
1378 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
1379 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1380 case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
1381 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
1382 case PGMPOOLKIND_64BIT_PML4:
1383 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
1384 switch (enmKind2)
1385 {
1386 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
1387 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1388 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1389 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1390 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
1391 case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
1392 case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
1393 case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
1394 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1395 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1396 case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
1397 case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
1398 case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
1399 case PGMPOOLKIND_EPT_PD_FOR_PHYS:
1400 case PGMPOOLKIND_EPT_PT_FOR_PHYS:
1401 return true;
1402 default:
1403 return false;
1404 }
1405
1406 /*
1407 * These cannot be flushed, and it's common to reuse the PDs as PTs.
1408 */
1409 case PGMPOOLKIND_ROOT_NESTED:
1410 return false;
1411
1412 default:
1413 AssertFatalMsgFailed(("enmKind1=%d\n", enmKind1));
1414 }
1415}
1416
1417
1418/**
1419 * Attempts to satisfy a pgmPoolAlloc request from the cache.
1420 *
1421 * @returns VBox status code.
1422 * @retval VINF_PGM_CACHED_PAGE on success.
1423 * @retval VERR_FILE_NOT_FOUND if not found.
1424 * @param pPool The pool.
1425 * @param GCPhys The GC physical address of the page we're gonna shadow.
1426 * @param enmKind The kind of mapping.
1427 * @param iUser The shadow page pool index of the user table.
1428 * @param iUserTable The index into the user table (shadowed).
1429 * @param ppPage Where to store the pointer to the page.
1430 */
1431static int pgmPoolCacheAlloc(PPGMPOOL pPool, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, uint16_t iUser, uint32_t iUserTable, PPPGMPOOLPAGE ppPage)
1432{
1433#ifndef IN_RC
1434 const PVM pVM = pPool->CTX_SUFF(pVM);
1435#endif
1436 /*
1437 * Look up the GCPhys in the hash.
1438 */
1439 unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
1440 Log3(("pgmPoolCacheAlloc: %RGp kind %s iUser=%x iUserTable=%x SLOT=%d\n", GCPhys, pgmPoolPoolKindToStr(enmKind), iUser, iUserTable, i));
1441 if (i != NIL_PGMPOOL_IDX)
1442 {
1443 do
1444 {
1445 PPGMPOOLPAGE pPage = &pPool->aPages[i];
1446 Log4(("pgmPoolCacheAlloc: slot %d found page %RGp\n", i, pPage->GCPhys));
1447 if (pPage->GCPhys == GCPhys)
1448 {
1449 if ((PGMPOOLKIND)pPage->enmKind == enmKind)
1450 {
1451 /* Put it at the start of the use list to make sure pgmPoolTrackAddUser
1452 * doesn't flush it in case there are no more free use records.
1453 */
1454 pgmPoolCacheUsed(pPool, pPage);
1455
1456 int rc = pgmPoolTrackAddUser(pPool, pPage, iUser, iUserTable);
1457 if (RT_SUCCESS(rc))
1458 {
1459 Assert((PGMPOOLKIND)pPage->enmKind == enmKind);
1460 *ppPage = pPage;
1461 STAM_COUNTER_INC(&pPool->StatCacheHits);
1462 return VINF_PGM_CACHED_PAGE;
1463 }
1464 return rc;
1465 }
1466
1467 /*
1468 * The kind is different. In some cases we should now flush the page
1469 * as it has been reused, but in most cases this is normal remapping
1470 * of PDs as PT or big pages using the GCPhys field in a slightly
1471 * different way than the other kinds.
1472 */
1473 if (pgmPoolCacheReusedByKind((PGMPOOLKIND)pPage->enmKind, enmKind))
1474 {
1475 STAM_COUNTER_INC(&pPool->StatCacheKindMismatches);
1476 pgmPoolFlushPage(pPool, pPage);
1477 PGM_INVL_VCPU_TLBS(VMMGetCpu(pVM)); /* see PT handler. */
1478 break;
1479 }
1480 }
1481
1482 /* next */
1483 i = pPage->iNext;
1484 } while (i != NIL_PGMPOOL_IDX);
1485 }
1486
1487 Log3(("pgmPoolCacheAlloc: Missed GCPhys=%RGp enmKind=%s\n", GCPhys, pgmPoolPoolKindToStr(enmKind)));
1488 STAM_COUNTER_INC(&pPool->StatCacheMisses);
1489 return VERR_FILE_NOT_FOUND;
1490}
1491
1492
1493/**
1494 * Inserts a page into the cache.
1495 *
1496 * @param pPool The pool.
1497 * @param pPage The cached page.
1498 * @param fCanBeCached Set if the page is fit for caching from the caller's point of view.
1499 */
1500static void pgmPoolCacheInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fCanBeCached)
1501{
1502 /*
1503 * Insert into the GCPhys hash if the page is fit for that.
1504 */
1505 Assert(!pPage->fCached);
1506 if (fCanBeCached)
1507 {
1508 pPage->fCached = true;
1509 pgmPoolHashInsert(pPool, pPage);
1510 Log3(("pgmPoolCacheInsert: Caching %p:{.Core=%RHp, .idx=%d, .enmKind=%s, GCPhys=%RGp}\n",
1511 pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
1512 STAM_COUNTER_INC(&pPool->StatCacheCacheable);
1513 }
1514 else
1515 {
1516 Log3(("pgmPoolCacheInsert: Not caching %p:{.Core=%RHp, .idx=%d, .enmKind=%s, GCPhys=%RGp}\n",
1517 pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
1518 STAM_COUNTER_INC(&pPool->StatCacheUncacheable);
1519 }
1520
1521 /*
1522 * Insert at the head of the age list.
1523 */
1524 pPage->iAgePrev = NIL_PGMPOOL_IDX;
1525 pPage->iAgeNext = pPool->iAgeHead;
1526 if (pPool->iAgeHead != NIL_PGMPOOL_IDX)
1527 pPool->aPages[pPool->iAgeHead].iAgePrev = pPage->idx;
1528 else
1529 pPool->iAgeTail = pPage->idx;
1530 pPool->iAgeHead = pPage->idx;
1531}
1532
1533
1534/**
1535 * Flushes a cached page.
1536 *
1537 * @param pPool The pool.
1538 * @param pPage The cached page.
1539 */
1540static void pgmPoolCacheFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1541{
1542 Log3(("pgmPoolCacheFlushPage: %RGp\n", pPage->GCPhys));
1543
1544 /*
1545 * Remove the page from the hash.
1546 */
1547 if (pPage->fCached)
1548 {
1549 pPage->fCached = false;
1550 pgmPoolHashRemove(pPool, pPage);
1551 }
1552 else
1553 Assert(pPage->iNext == NIL_PGMPOOL_IDX);
1554
1555 /*
1556 * Remove it from the age list.
1557 */
1558 if (pPage->iAgeNext != NIL_PGMPOOL_IDX)
1559 pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->iAgePrev;
1560 else
1561 pPool->iAgeTail = pPage->iAgePrev;
1562 if (pPage->iAgePrev != NIL_PGMPOOL_IDX)
1563 pPool->aPages[pPage->iAgePrev].iAgeNext = pPage->iAgeNext;
1564 else
1565 pPool->iAgeHead = pPage->iAgeNext;
1566 pPage->iAgeNext = NIL_PGMPOOL_IDX;
1567 pPage->iAgePrev = NIL_PGMPOOL_IDX;
1568}
1569
1570#endif /* PGMPOOL_WITH_CACHE */
1571#ifdef PGMPOOL_WITH_MONITORING
1572
1573/**
1574 * Looks for pages sharing the monitor.
1575 *
1576 * @returns Pointer to the head page.
1577 * @returns NULL if not found.
1578 * @param pPool The Pool
1579 * @param pNewPage The page which is going to be monitored.
1580 */
1581static PPGMPOOLPAGE pgmPoolMonitorGetPageByGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pNewPage)
1582{
1583#ifdef PGMPOOL_WITH_CACHE
1584 /*
1585 * Look up the GCPhys in the hash.
1586 */
1587 RTGCPHYS GCPhys = pNewPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1);
1588 unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
1589 if (i == NIL_PGMPOOL_IDX)
1590 return NULL;
1591 do
1592 {
1593 PPGMPOOLPAGE pPage = &pPool->aPages[i];
1594 if ( pPage->GCPhys - GCPhys < PAGE_SIZE
1595 && pPage != pNewPage)
1596 {
1597 switch (pPage->enmKind)
1598 {
1599 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1600 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1601 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1602 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
1603 case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
1604 case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
1605 case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
1606 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
1607 case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
1608 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
1609 case PGMPOOLKIND_64BIT_PML4:
1610 case PGMPOOLKIND_32BIT_PD:
1611 case PGMPOOLKIND_PAE_PDPT:
1612 {
1613 /* find the head */
1614 while (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
1615 {
1616 Assert(pPage->iMonitoredPrev != pPage->idx);
1617 pPage = &pPool->aPages[pPage->iMonitoredPrev];
1618 }
1619 return pPage;
1620 }
1621
1622 /* ignore, no monitoring. */
1623 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1624 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
1625 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
1626 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1627 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1628 case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
1629 case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
1630 case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
1631 case PGMPOOLKIND_EPT_PD_FOR_PHYS:
1632 case PGMPOOLKIND_EPT_PT_FOR_PHYS:
1633 case PGMPOOLKIND_ROOT_NESTED:
1634 case PGMPOOLKIND_PAE_PD_PHYS:
1635 case PGMPOOLKIND_PAE_PDPT_PHYS:
1636 case PGMPOOLKIND_32BIT_PD_PHYS:
1637 case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
1638 break;
1639 default:
1640 AssertFatalMsgFailed(("enmKind=%d idx=%d\n", pPage->enmKind, pPage->idx));
1641 }
1642 }
1643
1644 /* next */
1645 i = pPage->iNext;
1646 } while (i != NIL_PGMPOOL_IDX);
1647#endif
1648 return NULL;
1649}
1650
1651
1652/**
1653 * Enabled write monitoring of a guest page.
1654 *
1655 * @returns VBox status code.
1656 * @retval VINF_SUCCESS on success.
1657 * @param pPool The pool.
1658 * @param pPage The cached page.
1659 */
1660static int pgmPoolMonitorInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1661{
1662 LogFlow(("pgmPoolMonitorInsert %RGp\n", pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1)));
1663
1664 /*
1665 * Filter out the relevant kinds.
1666 */
1667 switch (pPage->enmKind)
1668 {
1669 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1670 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1671 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
1672 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1673 case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
1674 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
1675 case PGMPOOLKIND_64BIT_PML4:
1676 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
1677 case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
1678 case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
1679 case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
1680 case PGMPOOLKIND_32BIT_PD:
1681 case PGMPOOLKIND_PAE_PDPT:
1682 break;
1683
1684 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1685 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
1686 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
1687 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1688 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1689 case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
1690 case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
1691 case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
1692 case PGMPOOLKIND_EPT_PD_FOR_PHYS:
1693 case PGMPOOLKIND_EPT_PT_FOR_PHYS:
1694 case PGMPOOLKIND_ROOT_NESTED:
1695 /* Nothing to monitor here. */
1696 return VINF_SUCCESS;
1697
1698 case PGMPOOLKIND_32BIT_PD_PHYS:
1699 case PGMPOOLKIND_PAE_PDPT_PHYS:
1700 case PGMPOOLKIND_PAE_PD_PHYS:
1701 case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
1702 /* Nothing to monitor here. */
1703 return VINF_SUCCESS;
1704#ifdef PGMPOOL_WITH_MIXED_PT_CR3
1705 break;
1706#else
1707 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
1708#endif
1709 default:
1710 AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind));
1711 }
1712
1713 /*
1714 * Install handler.
1715 */
1716 int rc;
1717 PPGMPOOLPAGE pPageHead = pgmPoolMonitorGetPageByGCPhys(pPool, pPage);
1718 if (pPageHead)
1719 {
1720 Assert(pPageHead != pPage); Assert(pPageHead->iMonitoredNext != pPage->idx);
1721 Assert(pPageHead->iMonitoredPrev != pPage->idx);
1722 pPage->iMonitoredPrev = pPageHead->idx;
1723 pPage->iMonitoredNext = pPageHead->iMonitoredNext;
1724 if (pPageHead->iMonitoredNext != NIL_PGMPOOL_IDX)
1725 pPool->aPages[pPageHead->iMonitoredNext].iMonitoredPrev = pPage->idx;
1726 pPageHead->iMonitoredNext = pPage->idx;
1727 rc = VINF_SUCCESS;
1728 }
1729 else
1730 {
1731 Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX); Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
1732 PVM pVM = pPool->CTX_SUFF(pVM);
1733 const RTGCPHYS GCPhysPage = pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1);
1734 rc = PGMHandlerPhysicalRegisterEx(pVM, PGMPHYSHANDLERTYPE_PHYSICAL_WRITE,
1735 GCPhysPage, GCPhysPage + (PAGE_SIZE - 1),
1736 pPool->pfnAccessHandlerR3, MMHyperCCToR3(pVM, pPage),
1737 pPool->pfnAccessHandlerR0, MMHyperCCToR0(pVM, pPage),
1738 pPool->pfnAccessHandlerRC, MMHyperCCToRC(pVM, pPage),
1739 pPool->pszAccessHandler);
1740 /** @todo we should probably deal with out-of-memory conditions here, but for now increasing
1741 * the heap size should suffice. */
1742 AssertFatalMsgRC(rc, ("PGMHandlerPhysicalRegisterEx %RGp failed with %Rrc\n", GCPhysPage, rc));
1743 Assert(!(pVM->pgm.s.fGlobalSyncFlags & PGM_GLOBAL_SYNC_CLEAR_PGM_POOL) || VMCPU_FF_ISSET(VMMGetCpu(pVM), VMCPU_FF_PGM_SYNC_CR3));
1744 }
1745 pPage->fMonitored = true;
1746 return rc;
1747}
1748
1749
1750/**
1751 * Disables write monitoring of a guest page.
1752 *
1753 * @returns VBox status code.
1754 * @retval VINF_SUCCESS on success.
1755 * @param pPool The pool.
1756 * @param pPage The cached page.
1757 */
1758static int pgmPoolMonitorFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1759{
1760 /*
1761 * Filter out the relevant kinds.
1762 */
1763 switch (pPage->enmKind)
1764 {
1765 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1766 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1767 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
1768 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1769 case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
1770 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
1771 case PGMPOOLKIND_64BIT_PML4:
1772 case PGMPOOLKIND_32BIT_PD:
1773 case PGMPOOLKIND_PAE_PDPT:
1774 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
1775 case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
1776 case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
1777 case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
1778 break;
1779
1780 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1781 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
1782 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
1783 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1784 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1785 case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
1786 case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
1787 case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
1788 case PGMPOOLKIND_EPT_PD_FOR_PHYS:
1789 case PGMPOOLKIND_EPT_PT_FOR_PHYS:
1790 case PGMPOOLKIND_ROOT_NESTED:
1791 case PGMPOOLKIND_PAE_PD_PHYS:
1792 case PGMPOOLKIND_PAE_PDPT_PHYS:
1793 case PGMPOOLKIND_32BIT_PD_PHYS:
1794 /* Nothing to monitor here. */
1795 return VINF_SUCCESS;
1796
1797#ifdef PGMPOOL_WITH_MIXED_PT_CR3
1798 break;
1799#endif
1800 default:
1801 AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind));
1802 }
1803
1804 /*
1805 * Remove the page from the monitored list or uninstall it if last.
1806 */
1807 const PVM pVM = pPool->CTX_SUFF(pVM);
1808 int rc;
1809 if ( pPage->iMonitoredNext != NIL_PGMPOOL_IDX
1810 || pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
1811 {
1812 if (pPage->iMonitoredPrev == NIL_PGMPOOL_IDX)
1813 {
1814 PPGMPOOLPAGE pNewHead = &pPool->aPages[pPage->iMonitoredNext];
1815 pNewHead->iMonitoredPrev = NIL_PGMPOOL_IDX;
1816 rc = PGMHandlerPhysicalChangeCallbacks(pVM, pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1),
1817 pPool->pfnAccessHandlerR3, MMHyperCCToR3(pVM, pNewHead),
1818 pPool->pfnAccessHandlerR0, MMHyperCCToR0(pVM, pNewHead),
1819 pPool->pfnAccessHandlerRC, MMHyperCCToRC(pVM, pNewHead),
1820 pPool->pszAccessHandler);
1821 AssertFatalRCSuccess(rc);
1822 pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
1823 }
1824 else
1825 {
1826 pPool->aPages[pPage->iMonitoredPrev].iMonitoredNext = pPage->iMonitoredNext;
1827 if (pPage->iMonitoredNext != NIL_PGMPOOL_IDX)
1828 {
1829 pPool->aPages[pPage->iMonitoredNext].iMonitoredPrev = pPage->iMonitoredPrev;
1830 pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
1831 }
1832 pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
1833 rc = VINF_SUCCESS;
1834 }
1835 }
1836 else
1837 {
1838 rc = PGMHandlerPhysicalDeregister(pVM, pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1));
1839 AssertFatalRC(rc);
1840 AssertMsg(!(pVM->pgm.s.fGlobalSyncFlags & PGM_GLOBAL_SYNC_CLEAR_PGM_POOL) || VMCPU_FF_ISSET(VMMGetCpu(pVM), VMCPU_FF_PGM_SYNC_CR3),
1841 ("%#x %#x\n", pVM->pgm.s.fGlobalSyncFlags, pVM->fGlobalForcedActions));
1842 }
1843 pPage->fMonitored = false;
1844
1845 /*
1846 * Remove it from the list of modified pages (if in it).
1847 */
1848 pgmPoolMonitorModifiedRemove(pPool, pPage);
1849
1850 return rc;
1851}
1852
1853
1854/**
1855 * Inserts the page into the list of modified pages.
1856 *
1857 * @param pPool The pool.
1858 * @param pPage The page.
1859 */
1860void pgmPoolMonitorModifiedInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1861{
1862 Log3(("pgmPoolMonitorModifiedInsert: idx=%d\n", pPage->idx));
1863 AssertMsg( pPage->iModifiedNext == NIL_PGMPOOL_IDX
1864 && pPage->iModifiedPrev == NIL_PGMPOOL_IDX
1865 && pPool->iModifiedHead != pPage->idx,
1866 ("Next=%d Prev=%d idx=%d cModifications=%d Head=%d cModifiedPages=%d\n",
1867 pPage->iModifiedNext, pPage->iModifiedPrev, pPage->idx, pPage->cModifications,
1868 pPool->iModifiedHead, pPool->cModifiedPages));
1869
1870 pPage->iModifiedNext = pPool->iModifiedHead;
1871 if (pPool->iModifiedHead != NIL_PGMPOOL_IDX)
1872 pPool->aPages[pPool->iModifiedHead].iModifiedPrev = pPage->idx;
1873 pPool->iModifiedHead = pPage->idx;
1874 pPool->cModifiedPages++;
1875#ifdef VBOX_WITH_STATISTICS
1876 if (pPool->cModifiedPages > pPool->cModifiedPagesHigh)
1877 pPool->cModifiedPagesHigh = pPool->cModifiedPages;
1878#endif
1879}
1880
1881
1882/**
1883 * Removes the page from the list of modified pages and resets the
1884 * moficiation counter.
1885 *
1886 * @param pPool The pool.
1887 * @param pPage The page which is believed to be in the list of modified pages.
1888 */
1889static void pgmPoolMonitorModifiedRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1890{
1891 Log3(("pgmPoolMonitorModifiedRemove: idx=%d cModifications=%d\n", pPage->idx, pPage->cModifications));
1892 if (pPool->iModifiedHead == pPage->idx)
1893 {
1894 Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX);
1895 pPool->iModifiedHead = pPage->iModifiedNext;
1896 if (pPage->iModifiedNext != NIL_PGMPOOL_IDX)
1897 {
1898 pPool->aPages[pPage->iModifiedNext].iModifiedPrev = NIL_PGMPOOL_IDX;
1899 pPage->iModifiedNext = NIL_PGMPOOL_IDX;
1900 }
1901 pPool->cModifiedPages--;
1902 }
1903 else if (pPage->iModifiedPrev != NIL_PGMPOOL_IDX)
1904 {
1905 pPool->aPages[pPage->iModifiedPrev].iModifiedNext = pPage->iModifiedNext;
1906 if (pPage->iModifiedNext != NIL_PGMPOOL_IDX)
1907 {
1908 pPool->aPages[pPage->iModifiedNext].iModifiedPrev = pPage->iModifiedPrev;
1909 pPage->iModifiedNext = NIL_PGMPOOL_IDX;
1910 }
1911 pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
1912 pPool->cModifiedPages--;
1913 }
1914 else
1915 Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX);
1916 pPage->cModifications = 0;
1917}
1918
1919
1920/**
1921 * Zaps the list of modified pages, resetting their modification counters in the process.
1922 *
1923 * @param pVM The VM handle.
1924 */
1925void pgmPoolMonitorModifiedClearAll(PVM pVM)
1926{
1927 pgmLock(pVM);
1928 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1929 LogFlow(("pgmPoolMonitorModifiedClearAll: cModifiedPages=%d\n", pPool->cModifiedPages));
1930
1931 unsigned cPages = 0; NOREF(cPages);
1932 uint16_t idx = pPool->iModifiedHead;
1933 pPool->iModifiedHead = NIL_PGMPOOL_IDX;
1934 while (idx != NIL_PGMPOOL_IDX)
1935 {
1936 PPGMPOOLPAGE pPage = &pPool->aPages[idx];
1937 idx = pPage->iModifiedNext;
1938 pPage->iModifiedNext = NIL_PGMPOOL_IDX;
1939 pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
1940 pPage->cModifications = 0;
1941 Assert(++cPages);
1942 }
1943 AssertMsg(cPages == pPool->cModifiedPages, ("%d != %d\n", cPages, pPool->cModifiedPages));
1944 pPool->cModifiedPages = 0;
1945 pgmUnlock(pVM);
1946}
1947
1948
1949#ifdef IN_RING3
1950/**
1951 * Callback to clear all shadow pages and clear all modification counters.
1952 *
1953 * @returns VBox status code.
1954 * @param pVM The VM handle.
1955 * @param pvUser Unused parameter
1956 * @remark Should only be used when monitoring is available, thus placed in
1957 * the PGMPOOL_WITH_MONITORING #ifdef.
1958 */
1959DECLCALLBACK(int) pgmPoolClearAll(PVM pVM, void *pvUser)
1960{
1961 NOREF(pvUser);
1962 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1963 STAM_PROFILE_START(&pPool->StatClearAll, c);
1964 LogFlow(("pgmPoolClearAll: cUsedPages=%d\n", pPool->cUsedPages));
1965
1966 pgmLock(pVM);
1967
1968 /*
1969 * Iterate all the pages until we've encountered all that in use.
1970 * This is simple but not quite optimal solution.
1971 */
1972 unsigned cModifiedPages = 0; NOREF(cModifiedPages);
1973 unsigned cLeft = pPool->cUsedPages;
1974 unsigned iPage = pPool->cCurPages;
1975 while (--iPage >= PGMPOOL_IDX_FIRST)
1976 {
1977 PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
1978 if (pPage->GCPhys != NIL_RTGCPHYS)
1979 {
1980 switch (pPage->enmKind)
1981 {
1982 /*
1983 * We only care about shadow page tables.
1984 */
1985 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1986 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1987 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1988 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
1989 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1990 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
1991 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1992 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1993 {
1994#ifdef PGMPOOL_WITH_USER_TRACKING
1995 if (pPage->cPresent)
1996#endif
1997 {
1998 void *pvShw = PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pPage);
1999 STAM_PROFILE_START(&pPool->StatZeroPage, z);
2000 ASMMemZeroPage(pvShw);
2001 STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
2002#ifdef PGMPOOL_WITH_USER_TRACKING
2003 pPage->cPresent = 0;
2004 pPage->iFirstPresent = ~0;
2005#endif
2006 }
2007 }
2008 /* fall thru */
2009
2010 default:
2011 Assert(!pPage->cModifications || ++cModifiedPages);
2012 Assert(pPage->iModifiedNext == NIL_PGMPOOL_IDX || pPage->cModifications);
2013 Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX || pPage->cModifications);
2014 pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2015 pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
2016 pPage->cModifications = 0;
2017 break;
2018
2019 }
2020 if (!--cLeft)
2021 break;
2022 }
2023 }
2024
2025 /* swipe the special pages too. */
2026 for (iPage = PGMPOOL_IDX_FIRST_SPECIAL; iPage < PGMPOOL_IDX_FIRST; iPage++)
2027 {
2028 PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
2029 if (pPage->GCPhys != NIL_RTGCPHYS)
2030 {
2031 Assert(!pPage->cModifications || ++cModifiedPages);
2032 Assert(pPage->iModifiedNext == NIL_PGMPOOL_IDX || pPage->cModifications);
2033 Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX || pPage->cModifications);
2034 pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2035 pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
2036 pPage->cModifications = 0;
2037 }
2038 }
2039
2040#ifndef DEBUG_michael
2041 AssertMsg(cModifiedPages == pPool->cModifiedPages, ("%d != %d\n", cModifiedPages, pPool->cModifiedPages));
2042#endif
2043 pPool->iModifiedHead = NIL_PGMPOOL_IDX;
2044 pPool->cModifiedPages = 0;
2045
2046#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
2047 /*
2048 * Clear all the GCPhys links and rebuild the phys ext free list.
2049 */
2050 for (PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRanges);
2051 pRam;
2052 pRam = pRam->CTX_SUFF(pNext))
2053 {
2054 unsigned iPage = pRam->cb >> PAGE_SHIFT;
2055 while (iPage-- > 0)
2056 PGM_PAGE_SET_TRACKING(&pRam->aPages[iPage], 0);
2057 }
2058
2059 pPool->iPhysExtFreeHead = 0;
2060 PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
2061 const unsigned cMaxPhysExts = pPool->cMaxPhysExts;
2062 for (unsigned i = 0; i < cMaxPhysExts; i++)
2063 {
2064 paPhysExts[i].iNext = i + 1;
2065 paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX;
2066 paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX;
2067 paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX;
2068 }
2069 paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
2070#endif
2071
2072 pPool->cPresent = 0;
2073 pgmUnlock(pVM);
2074 PGM_INVL_ALL_VCPU_TLBS(pVM);
2075 STAM_PROFILE_STOP(&pPool->StatClearAll, c);
2076 return VINF_SUCCESS;
2077}
2078#endif /* IN_RING3 */
2079
2080
2081/**
2082 * Handle SyncCR3 pool tasks
2083 *
2084 * @returns VBox status code.
2085 * @retval VINF_SUCCESS if successfully added.
2086 * @retval VINF_PGM_SYNC_CR3 is it needs to be deferred to ring 3 (GC only)
2087 * @param pVM The VM handle.
2088 * @remark Should only be used when monitoring is available, thus placed in
2089 * the PGMPOOL_WITH_MONITORING #ifdef.
2090 */
2091int pgmPoolSyncCR3(PVM pVM)
2092{
2093 LogFlow(("pgmPoolSyncCR3\n"));
2094 /*
2095 * When monitoring shadowed pages, we reset the modification counters on CR3 sync.
2096 * Occasionally we will have to clear all the shadow page tables because we wanted
2097 * to monitor a page which was mapped by too many shadowed page tables. This operation
2098 * sometimes refered to as a 'lightweight flush'.
2099 */
2100# ifdef IN_RING3 /* Don't flush in ring-0 or raw mode, it's taking too long. */
2101 if (ASMBitTestAndClear(&pVM->pgm.s.fGlobalSyncFlags, PGM_GLOBAL_SYNC_CLEAR_PGM_POOL_BIT))
2102 {
2103 VMMR3AtomicExecuteHandler(pVM, pgmPoolClearAll, NULL);
2104# else /* !IN_RING3 */
2105 if (pVM->pgm.s.fGlobalSyncFlags & PGM_GLOBAL_SYNC_CLEAR_PGM_POOL)
2106 {
2107 LogFlow(("SyncCR3: PGM_GLOBAL_SYNC_CLEAR_PGM_POOL is set -> VINF_PGM_SYNC_CR3\n"));
2108 VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
2109 return VINF_PGM_SYNC_CR3;
2110# endif /* !IN_RING3 */
2111 }
2112 else
2113 pgmPoolMonitorModifiedClearAll(pVM);
2114
2115 return VINF_SUCCESS;
2116}
2117
2118#endif /* PGMPOOL_WITH_MONITORING */
2119#ifdef PGMPOOL_WITH_USER_TRACKING
2120
2121/**
2122 * Frees up at least one user entry.
2123 *
2124 * @returns VBox status code.
2125 * @retval VINF_SUCCESS if successfully added.
2126 * @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
2127 * @param pPool The pool.
2128 * @param iUser The user index.
2129 */
2130static int pgmPoolTrackFreeOneUser(PPGMPOOL pPool, uint16_t iUser)
2131{
2132 STAM_COUNTER_INC(&pPool->StatTrackFreeUpOneUser);
2133#ifdef PGMPOOL_WITH_CACHE
2134 /*
2135 * Just free cached pages in a braindead fashion.
2136 */
2137 /** @todo walk the age list backwards and free the first with usage. */
2138 int rc = VINF_SUCCESS;
2139 do
2140 {
2141 int rc2 = pgmPoolCacheFreeOne(pPool, iUser);
2142 if (RT_FAILURE(rc2) && rc == VINF_SUCCESS)
2143 rc = rc2;
2144 } while (pPool->iUserFreeHead == NIL_PGMPOOL_USER_INDEX);
2145 return rc;
2146#else
2147 /*
2148 * Lazy approach.
2149 */
2150 /* @todo This path no longer works (CR3 root pages will be flushed)!! */
2151 AssertCompileFailed();
2152 Assert(!CPUMIsGuestInLongMode(pVM));
2153 pgmPoolFlushAllInt(pPool);
2154 return VERR_PGM_POOL_FLUSHED;
2155#endif
2156}
2157
2158
2159/**
2160 * Inserts a page into the cache.
2161 *
2162 * This will create user node for the page, insert it into the GCPhys
2163 * hash, and insert it into the age list.
2164 *
2165 * @returns VBox status code.
2166 * @retval VINF_SUCCESS if successfully added.
2167 * @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
2168 * @param pPool The pool.
2169 * @param pPage The cached page.
2170 * @param GCPhys The GC physical address of the page we're gonna shadow.
2171 * @param iUser The user index.
2172 * @param iUserTable The user table index.
2173 */
2174DECLINLINE(int) pgmPoolTrackInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhys, uint16_t iUser, uint32_t iUserTable)
2175{
2176 int rc = VINF_SUCCESS;
2177 PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
2178
2179 LogFlow(("pgmPoolTrackInsert GCPhys=%RGp iUser %x iUserTable %x\n", GCPhys, iUser, iUserTable));
2180
2181#ifdef VBOX_STRICT
2182 /*
2183 * Check that the entry doesn't already exists.
2184 */
2185 if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
2186 {
2187 uint16_t i = pPage->iUserHead;
2188 do
2189 {
2190 Assert(i < pPool->cMaxUsers);
2191 AssertMsg(paUsers[i].iUser != iUser || paUsers[i].iUserTable != iUserTable, ("%x %x vs new %x %x\n", paUsers[i].iUser, paUsers[i].iUserTable, iUser, iUserTable));
2192 i = paUsers[i].iNext;
2193 } while (i != NIL_PGMPOOL_USER_INDEX);
2194 }
2195#endif
2196
2197 /*
2198 * Find free a user node.
2199 */
2200 uint16_t i = pPool->iUserFreeHead;
2201 if (i == NIL_PGMPOOL_USER_INDEX)
2202 {
2203 int rc = pgmPoolTrackFreeOneUser(pPool, iUser);
2204 if (RT_FAILURE(rc))
2205 return rc;
2206 i = pPool->iUserFreeHead;
2207 }
2208
2209 /*
2210 * Unlink the user node from the free list,
2211 * initialize and insert it into the user list.
2212 */
2213 pPool->iUserFreeHead = paUsers[i].iNext;
2214 paUsers[i].iNext = NIL_PGMPOOL_USER_INDEX;
2215 paUsers[i].iUser = iUser;
2216 paUsers[i].iUserTable = iUserTable;
2217 pPage->iUserHead = i;
2218
2219 /*
2220 * Insert into cache and enable monitoring of the guest page if enabled.
2221 *
2222 * Until we implement caching of all levels, including the CR3 one, we'll
2223 * have to make sure we don't try monitor & cache any recursive reuse of
2224 * a monitored CR3 page. Because all windows versions are doing this we'll
2225 * have to be able to do combined access monitoring, CR3 + PT and
2226 * PD + PT (guest PAE).
2227 *
2228 * Update:
2229 * We're now cooperating with the CR3 monitor if an uncachable page is found.
2230 */
2231#if defined(PGMPOOL_WITH_MONITORING) || defined(PGMPOOL_WITH_CACHE)
2232# ifdef PGMPOOL_WITH_MIXED_PT_CR3
2233 const bool fCanBeMonitored = true;
2234# else
2235 bool fCanBeMonitored = pPool->CTX_SUFF(pVM)->pgm.s.GCPhysGstCR3Monitored == NIL_RTGCPHYS
2236 || (GCPhys & X86_PTE_PAE_PG_MASK) != (pPool->CTX_SUFF(pVM)->pgm.s.GCPhysGstCR3Monitored & X86_PTE_PAE_PG_MASK)
2237 || pgmPoolIsBigPage((PGMPOOLKIND)pPage->enmKind);
2238# endif
2239# ifdef PGMPOOL_WITH_CACHE
2240 pgmPoolCacheInsert(pPool, pPage, fCanBeMonitored); /* This can be expanded. */
2241# endif
2242 if (fCanBeMonitored)
2243 {
2244# ifdef PGMPOOL_WITH_MONITORING
2245 rc = pgmPoolMonitorInsert(pPool, pPage);
2246 AssertRC(rc);
2247 }
2248# endif
2249#endif /* PGMPOOL_WITH_MONITORING */
2250 return rc;
2251}
2252
2253
2254# ifdef PGMPOOL_WITH_CACHE /* (only used when the cache is enabled.) */
2255/**
2256 * Adds a user reference to a page.
2257 *
2258 * This will move the page to the head of the
2259 *
2260 * @returns VBox status code.
2261 * @retval VINF_SUCCESS if successfully added.
2262 * @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
2263 * @param pPool The pool.
2264 * @param pPage The cached page.
2265 * @param iUser The user index.
2266 * @param iUserTable The user table.
2267 */
2268static int pgmPoolTrackAddUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
2269{
2270 PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
2271
2272 Log3(("pgmPoolTrackAddUser GCPhys = %RGp iUser %x iUserTable %x\n", pPage->GCPhys, iUser, iUserTable));
2273
2274# ifdef VBOX_STRICT
2275 /*
2276 * Check that the entry doesn't already exists. We only allow multiple users of top-level paging structures (SHW_POOL_ROOT_IDX).
2277 */
2278 if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
2279 {
2280 uint16_t i = pPage->iUserHead;
2281 do
2282 {
2283 Assert(i < pPool->cMaxUsers);
2284 AssertMsg(iUser != PGMPOOL_IDX_PD || iUser != PGMPOOL_IDX_PDPT || iUser != PGMPOOL_IDX_NESTED_ROOT || iUser != PGMPOOL_IDX_AMD64_CR3 ||
2285 paUsers[i].iUser != iUser || paUsers[i].iUserTable != iUserTable, ("%x %x vs new %x %x\n", paUsers[i].iUser, paUsers[i].iUserTable, iUser, iUserTable));
2286 i = paUsers[i].iNext;
2287 } while (i != NIL_PGMPOOL_USER_INDEX);
2288 }
2289# endif
2290
2291 /*
2292 * Allocate a user node.
2293 */
2294 uint16_t i = pPool->iUserFreeHead;
2295 if (i == NIL_PGMPOOL_USER_INDEX)
2296 {
2297 int rc = pgmPoolTrackFreeOneUser(pPool, iUser);
2298 if (RT_FAILURE(rc))
2299 return rc;
2300 i = pPool->iUserFreeHead;
2301 }
2302 pPool->iUserFreeHead = paUsers[i].iNext;
2303
2304 /*
2305 * Initialize the user node and insert it.
2306 */
2307 paUsers[i].iNext = pPage->iUserHead;
2308 paUsers[i].iUser = iUser;
2309 paUsers[i].iUserTable = iUserTable;
2310 pPage->iUserHead = i;
2311
2312# ifdef PGMPOOL_WITH_CACHE
2313 /*
2314 * Tell the cache to update its replacement stats for this page.
2315 */
2316 pgmPoolCacheUsed(pPool, pPage);
2317# endif
2318 return VINF_SUCCESS;
2319}
2320# endif /* PGMPOOL_WITH_CACHE */
2321
2322
2323/**
2324 * Frees a user record associated with a page.
2325 *
2326 * This does not clear the entry in the user table, it simply replaces the
2327 * user record to the chain of free records.
2328 *
2329 * @param pPool The pool.
2330 * @param HCPhys The HC physical address of the shadow page.
2331 * @param iUser The shadow page pool index of the user table.
2332 * @param iUserTable The index into the user table (shadowed).
2333 */
2334static void pgmPoolTrackFreeUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
2335{
2336 /*
2337 * Unlink and free the specified user entry.
2338 */
2339 PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
2340
2341 Log3(("pgmPoolTrackFreeUser %RGp %x %x\n", pPage->GCPhys, iUser, iUserTable));
2342 /* Special: For PAE and 32-bit paging, there is usually no more than one user. */
2343 uint16_t i = pPage->iUserHead;
2344 if ( i != NIL_PGMPOOL_USER_INDEX
2345 && paUsers[i].iUser == iUser
2346 && paUsers[i].iUserTable == iUserTable)
2347 {
2348 pPage->iUserHead = paUsers[i].iNext;
2349
2350 paUsers[i].iUser = NIL_PGMPOOL_IDX;
2351 paUsers[i].iNext = pPool->iUserFreeHead;
2352 pPool->iUserFreeHead = i;
2353 return;
2354 }
2355
2356 /* General: Linear search. */
2357 uint16_t iPrev = NIL_PGMPOOL_USER_INDEX;
2358 while (i != NIL_PGMPOOL_USER_INDEX)
2359 {
2360 if ( paUsers[i].iUser == iUser
2361 && paUsers[i].iUserTable == iUserTable)
2362 {
2363 if (iPrev != NIL_PGMPOOL_USER_INDEX)
2364 paUsers[iPrev].iNext = paUsers[i].iNext;
2365 else
2366 pPage->iUserHead = paUsers[i].iNext;
2367
2368 paUsers[i].iUser = NIL_PGMPOOL_IDX;
2369 paUsers[i].iNext = pPool->iUserFreeHead;
2370 pPool->iUserFreeHead = i;
2371 return;
2372 }
2373 iPrev = i;
2374 i = paUsers[i].iNext;
2375 }
2376
2377 /* Fatal: didn't find it */
2378 AssertFatalMsgFailed(("Didn't find the user entry! iUser=%#x iUserTable=%#x GCPhys=%RGp\n",
2379 iUser, iUserTable, pPage->GCPhys));
2380}
2381
2382
2383/**
2384 * Gets the entry size of a shadow table.
2385 *
2386 * @param enmKind The kind of page.
2387 *
2388 * @returns The size of the entry in bytes. That is, 4 or 8.
2389 * @returns If the kind is not for a table, an assertion is raised and 0 is
2390 * returned.
2391 */
2392DECLINLINE(unsigned) pgmPoolTrackGetShadowEntrySize(PGMPOOLKIND enmKind)
2393{
2394 switch (enmKind)
2395 {
2396 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2397 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2398 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2399 case PGMPOOLKIND_32BIT_PD:
2400 case PGMPOOLKIND_32BIT_PD_PHYS:
2401 return 4;
2402
2403 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2404 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2405 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2406 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2407 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2408 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2409 case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2410 case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2411 case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2412 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2413 case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2414 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2415 case PGMPOOLKIND_64BIT_PML4:
2416 case PGMPOOLKIND_PAE_PDPT:
2417 case PGMPOOLKIND_ROOT_NESTED:
2418 case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2419 case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2420 case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2421 case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2422 case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2423 case PGMPOOLKIND_PAE_PD_PHYS:
2424 case PGMPOOLKIND_PAE_PDPT_PHYS:
2425 return 8;
2426
2427 default:
2428 AssertFatalMsgFailed(("enmKind=%d\n", enmKind));
2429 }
2430}
2431
2432
2433/**
2434 * Gets the entry size of a guest table.
2435 *
2436 * @param enmKind The kind of page.
2437 *
2438 * @returns The size of the entry in bytes. That is, 0, 4 or 8.
2439 * @returns If the kind is not for a table, an assertion is raised and 0 is
2440 * returned.
2441 */
2442DECLINLINE(unsigned) pgmPoolTrackGetGuestEntrySize(PGMPOOLKIND enmKind)
2443{
2444 switch (enmKind)
2445 {
2446 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2447 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2448 case PGMPOOLKIND_32BIT_PD:
2449 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2450 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2451 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2452 case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2453 case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2454 case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2455 return 4;
2456
2457 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2458 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2459 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2460 case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2461 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2462 case PGMPOOLKIND_64BIT_PML4:
2463 case PGMPOOLKIND_PAE_PDPT:
2464 return 8;
2465
2466 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2467 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2468 case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2469 case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2470 case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2471 case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2472 case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2473 case PGMPOOLKIND_ROOT_NESTED:
2474 case PGMPOOLKIND_PAE_PD_PHYS:
2475 case PGMPOOLKIND_PAE_PDPT_PHYS:
2476 case PGMPOOLKIND_32BIT_PD_PHYS:
2477 /** @todo can we return 0? (nobody is calling this...) */
2478 AssertFailed();
2479 return 0;
2480
2481 default:
2482 AssertFatalMsgFailed(("enmKind=%d\n", enmKind));
2483 }
2484}
2485
2486#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
2487
2488/**
2489 * Scans one shadow page table for mappings of a physical page.
2490 *
2491 * @param pVM The VM handle.
2492 * @param pPhysPage The guest page in question.
2493 * @param iShw The shadow page table.
2494 * @param cRefs The number of references made in that PT.
2495 */
2496static void pgmPoolTrackFlushGCPhysPTInt(PVM pVM, PCPGMPAGE pPhysPage, uint16_t iShw, uint16_t cRefs)
2497{
2498 LogFlow(("pgmPoolTrackFlushGCPhysPT: pPhysPage=%R[pgmpage] iShw=%d cRefs=%d\n", pPhysPage, iShw, cRefs));
2499 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2500
2501 /*
2502 * Assert sanity.
2503 */
2504 Assert(cRefs == 1);
2505 AssertFatalMsg(iShw < pPool->cCurPages && iShw != NIL_PGMPOOL_IDX, ("iShw=%d\n", iShw));
2506 PPGMPOOLPAGE pPage = &pPool->aPages[iShw];
2507
2508 /*
2509 * Then, clear the actual mappings to the page in the shadow PT.
2510 */
2511 switch (pPage->enmKind)
2512 {
2513 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2514 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2515 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2516 {
2517 const uint32_t u32 = PGM_PAGE_GET_HCPHYS(pPhysPage) | X86_PTE_P;
2518 PX86PT pPT = (PX86PT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
2519 for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
2520 if ((pPT->a[i].u & (X86_PTE_PG_MASK | X86_PTE_P)) == u32)
2521 {
2522 Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX32 cRefs=%#x\n", i, pPT->a[i], cRefs));
2523 pPT->a[i].u = 0;
2524 cRefs--;
2525 if (!cRefs)
2526 return;
2527 }
2528#ifdef LOG_ENABLED
2529 RTLogPrintf("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent);
2530 for (unsigned i = 0; i < RT_ELEMENTS(pPT->a); i++)
2531 if ((pPT->a[i].u & (X86_PTE_PG_MASK | X86_PTE_P)) == u32)
2532 {
2533 RTLogPrintf("i=%d cRefs=%d\n", i, cRefs--);
2534 pPT->a[i].u = 0;
2535 }
2536#endif
2537 AssertFatalMsgFailed(("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent));
2538 break;
2539 }
2540
2541 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2542 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2543 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2544 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2545 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2546 {
2547 const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) | X86_PTE_P;
2548 PX86PTPAE pPT = (PX86PTPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
2549 for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
2550 if ((pPT->a[i].u & (X86_PTE_PAE_PG_MASK | X86_PTE_P)) == u64)
2551 {
2552 Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX64 cRefs=%#x\n", i, pPT->a[i], cRefs));
2553 pPT->a[i].u = 0;
2554 cRefs--;
2555 if (!cRefs)
2556 return;
2557 }
2558#ifdef LOG_ENABLED
2559 RTLogPrintf("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent);
2560 for (unsigned i = 0; i < RT_ELEMENTS(pPT->a); i++)
2561 if ((pPT->a[i].u & (X86_PTE_PAE_PG_MASK | X86_PTE_P)) == u64)
2562 {
2563 RTLogPrintf("i=%d cRefs=%d\n", i, cRefs--);
2564 pPT->a[i].u = 0;
2565 }
2566#endif
2567 AssertFatalMsgFailed(("cRefs=%d iFirstPresent=%d cPresent=%d u64=%RX64\n", cRefs, pPage->iFirstPresent, pPage->cPresent, u64));
2568 break;
2569 }
2570
2571 case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2572 {
2573 const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) | X86_PTE_P;
2574 PEPTPT pPT = (PEPTPT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
2575 for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
2576 if ((pPT->a[i].u & (EPT_PTE_PG_MASK | X86_PTE_P)) == u64)
2577 {
2578 Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX64 cRefs=%#x\n", i, pPT->a[i], cRefs));
2579 pPT->a[i].u = 0;
2580 cRefs--;
2581 if (!cRefs)
2582 return;
2583 }
2584#ifdef LOG_ENABLED
2585 RTLogPrintf("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent);
2586 for (unsigned i = 0; i < RT_ELEMENTS(pPT->a); i++)
2587 if ((pPT->a[i].u & (EPT_PTE_PG_MASK | X86_PTE_P)) == u64)
2588 {
2589 RTLogPrintf("i=%d cRefs=%d\n", i, cRefs--);
2590 pPT->a[i].u = 0;
2591 }
2592#endif
2593 AssertFatalMsgFailed(("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent));
2594 break;
2595 }
2596
2597 default:
2598 AssertFatalMsgFailed(("enmKind=%d iShw=%d\n", pPage->enmKind, iShw));
2599 }
2600}
2601
2602
2603/**
2604 * Scans one shadow page table for mappings of a physical page.
2605 *
2606 * @param pVM The VM handle.
2607 * @param pPhysPage The guest page in question.
2608 * @param iShw The shadow page table.
2609 * @param cRefs The number of references made in that PT.
2610 */
2611void pgmPoolTrackFlushGCPhysPT(PVM pVM, PPGMPAGE pPhysPage, uint16_t iShw, uint16_t cRefs)
2612{
2613 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
2614 LogFlow(("pgmPoolTrackFlushGCPhysPT: pPhysPage=%R[pgmpage] iShw=%d cRefs=%d\n", pPhysPage, iShw, cRefs));
2615 STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPT, f);
2616 pgmPoolTrackFlushGCPhysPTInt(pVM, pPhysPage, iShw, cRefs);
2617 PGM_PAGE_SET_TRACKING(pPhysPage, 0);
2618 STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPT, f);
2619}
2620
2621
2622/**
2623 * Flushes a list of shadow page tables mapping the same physical page.
2624 *
2625 * @param pVM The VM handle.
2626 * @param pPhysPage The guest page in question.
2627 * @param iPhysExt The physical cross reference extent list to flush.
2628 */
2629void pgmPoolTrackFlushGCPhysPTs(PVM pVM, PPGMPAGE pPhysPage, uint16_t iPhysExt)
2630{
2631 Assert(PGMIsLockOwner(pVM));
2632 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2633 STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPTs, f);
2634 LogFlow(("pgmPoolTrackFlushGCPhysPTs: pPhysPage=%R[pgmpage] iPhysExt\n", pPhysPage, iPhysExt));
2635
2636 const uint16_t iPhysExtStart = iPhysExt;
2637 PPGMPOOLPHYSEXT pPhysExt;
2638 do
2639 {
2640 Assert(iPhysExt < pPool->cMaxPhysExts);
2641 pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
2642 for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
2643 if (pPhysExt->aidx[i] != NIL_PGMPOOL_IDX)
2644 {
2645 pgmPoolTrackFlushGCPhysPTInt(pVM, pPhysPage, pPhysExt->aidx[i], 1);
2646 pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
2647 }
2648
2649 /* next */
2650 iPhysExt = pPhysExt->iNext;
2651 } while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
2652
2653 /* insert the list into the free list and clear the ram range entry. */
2654 pPhysExt->iNext = pPool->iPhysExtFreeHead;
2655 pPool->iPhysExtFreeHead = iPhysExtStart;
2656 PGM_PAGE_SET_TRACKING(pPhysPage, 0);
2657
2658 STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTs, f);
2659}
2660
2661#endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
2662
2663/**
2664 * Flushes all shadow page table mappings of the given guest page.
2665 *
2666 * This is typically called when the host page backing the guest one has been
2667 * replaced or when the page protection was changed due to an access handler.
2668 *
2669 * @returns VBox status code.
2670 * @retval VINF_SUCCESS if all references has been successfully cleared.
2671 * @retval VINF_PGM_SYNC_CR3 if we're better off with a CR3 sync and a page
2672 * pool cleaning. FF and sync flags are set.
2673 *
2674 * @param pVM The VM handle.
2675 * @param pPhysPage The guest page in question.
2676 * @param pfFlushTLBs This is set to @a true if the shadow TLBs should be
2677 * flushed, it is NOT touched if this isn't necessary.
2678 * The caller MUST initialized this to @a false.
2679 */
2680int pgmPoolTrackFlushGCPhys(PVM pVM, PPGMPAGE pPhysPage, bool *pfFlushTLBs)
2681{
2682 pgmLock(pVM);
2683 int rc = VINF_SUCCESS;
2684#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
2685 const uint16_t u16 = PGM_PAGE_GET_TRACKING(pPhysPage);
2686 if (u16)
2687 {
2688 /*
2689 * The zero page is currently screwing up the tracking and we'll
2690 * have to flush the whole shebang. Unless VBOX_WITH_NEW_LAZY_PAGE_ALLOC
2691 * is defined, zero pages won't normally be mapped. Some kind of solution
2692 * will be needed for this problem of course, but it will have to wait...
2693 */
2694 if (PGM_PAGE_IS_ZERO(pPhysPage))
2695 rc = VINF_PGM_GCPHYS_ALIASED;
2696 else
2697 {
2698# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2699 /* Start a subset here because pgmPoolTrackFlushGCPhysPTsSlow and
2700 pgmPoolTrackFlushGCPhysPTs will/may kill the pool otherwise. */
2701 PVMCPU pVCpu = VMMGetCpu(pVM);
2702 uint32_t iPrevSubset = PGMDynMapPushAutoSubset(pVCpu);
2703# endif
2704
2705 if (PGMPOOL_TD_GET_CREFS(u16) != PGMPOOL_TD_CREFS_PHYSEXT)
2706 pgmPoolTrackFlushGCPhysPT(pVM,
2707 pPhysPage,
2708 PGMPOOL_TD_GET_IDX(u16),
2709 PGMPOOL_TD_GET_CREFS(u16));
2710 else if (u16 != PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED))
2711 pgmPoolTrackFlushGCPhysPTs(pVM, pPhysPage, PGMPOOL_TD_GET_IDX(u16));
2712 else
2713 rc = pgmPoolTrackFlushGCPhysPTsSlow(pVM, pPhysPage);
2714 *pfFlushTLBs = true;
2715
2716# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2717 PGMDynMapPopAutoSubset(pVCpu, iPrevSubset);
2718# endif
2719 }
2720 }
2721
2722#elif defined(PGMPOOL_WITH_CACHE)
2723 if (PGM_PAGE_IS_ZERO(pPhysPage))
2724 rc = VINF_PGM_GCPHYS_ALIASED;
2725 else
2726 {
2727# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2728 /* Start a subset here because pgmPoolTrackFlushGCPhysPTsSlow kill the pool otherwise. */
2729 PVMCPU pVCpu = VMMGetCpu(pVM);
2730 uint32_t iPrevSubset = PGMDynMapPushAutoSubset(pVCpu);
2731# endif
2732 rc = pgmPoolTrackFlushGCPhysPTsSlow(pVM, pPhysPage);
2733 if (rc == VINF_SUCCESS)
2734 *pfFlushTLBs = true;
2735 }
2736
2737# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2738 PGMDynMapPopAutoSubset(pVCpu, iPrevSubset);
2739# endif
2740
2741#else
2742 rc = VINF_PGM_GCPHYS_ALIASED;
2743#endif
2744
2745 if (rc == VINF_PGM_GCPHYS_ALIASED)
2746 {
2747 pVM->pgm.s.fGlobalSyncFlags |= PGM_GLOBAL_SYNC_CLEAR_PGM_POOL;
2748 for (unsigned i=0;i<pVM->cCPUs;i++)
2749 {
2750 PVMCPU pVCpu = &pVM->aCpus[i];
2751 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2752 }
2753 rc = VINF_PGM_SYNC_CR3;
2754 }
2755 pgmUnlock(pVM);
2756 return rc;
2757}
2758
2759
2760/**
2761 * Scans all shadow page tables for mappings of a physical page.
2762 *
2763 * This may be slow, but it's most likely more efficient than cleaning
2764 * out the entire page pool / cache.
2765 *
2766 * @returns VBox status code.
2767 * @retval VINF_SUCCESS if all references has been successfully cleared.
2768 * @retval VINF_PGM_GCPHYS_ALIASED if we're better off with a CR3 sync and
2769 * a page pool cleaning.
2770 *
2771 * @param pVM The VM handle.
2772 * @param pPhysPage The guest page in question.
2773 */
2774int pgmPoolTrackFlushGCPhysPTsSlow(PVM pVM, PPGMPAGE pPhysPage)
2775{
2776 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2777 STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPTsSlow, s);
2778 LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: cUsedPages=%d cPresent=%d pPhysPage=%R[pgmpage]\n",
2779 pPool->cUsedPages, pPool->cPresent, pPhysPage));
2780
2781#if 1
2782 /*
2783 * There is a limit to what makes sense.
2784 */
2785 if (pPool->cPresent > 1024)
2786 {
2787 LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: giving up... (cPresent=%d)\n", pPool->cPresent));
2788 STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTsSlow, s);
2789 return VINF_PGM_GCPHYS_ALIASED;
2790 }
2791#endif
2792
2793 /*
2794 * Iterate all the pages until we've encountered all that in use.
2795 * This is simple but not quite optimal solution.
2796 */
2797 const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) | X86_PTE_P;
2798 const uint32_t u32 = u64;
2799 unsigned cLeft = pPool->cUsedPages;
2800 unsigned iPage = pPool->cCurPages;
2801 while (--iPage >= PGMPOOL_IDX_FIRST)
2802 {
2803 PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
2804 if (pPage->GCPhys != NIL_RTGCPHYS)
2805 {
2806 switch (pPage->enmKind)
2807 {
2808 /*
2809 * We only care about shadow page tables.
2810 */
2811 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2812 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2813 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2814 {
2815 unsigned cPresent = pPage->cPresent;
2816 PX86PT pPT = (PX86PT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
2817 for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
2818 if (pPT->a[i].n.u1Present)
2819 {
2820 if ((pPT->a[i].u & (X86_PTE_PG_MASK | X86_PTE_P)) == u32)
2821 {
2822 //Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX32\n", iPage, i, pPT->a[i]));
2823 pPT->a[i].u = 0;
2824 }
2825 if (!--cPresent)
2826 break;
2827 }
2828 break;
2829 }
2830
2831 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2832 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2833 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2834 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2835 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2836 {
2837 unsigned cPresent = pPage->cPresent;
2838 PX86PTPAE pPT = (PX86PTPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
2839 for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
2840 if (pPT->a[i].n.u1Present)
2841 {
2842 if ((pPT->a[i].u & (X86_PTE_PAE_PG_MASK | X86_PTE_P)) == u64)
2843 {
2844 //Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX64\n", iPage, i, pPT->a[i]));
2845 pPT->a[i].u = 0;
2846 }
2847 if (!--cPresent)
2848 break;
2849 }
2850 break;
2851 }
2852 }
2853 if (!--cLeft)
2854 break;
2855 }
2856 }
2857
2858 PGM_PAGE_SET_TRACKING(pPhysPage, 0);
2859 STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTsSlow, s);
2860 return VINF_SUCCESS;
2861}
2862
2863
2864/**
2865 * Clears the user entry in a user table.
2866 *
2867 * This is used to remove all references to a page when flushing it.
2868 */
2869static void pgmPoolTrackClearPageUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PCPGMPOOLUSER pUser)
2870{
2871 Assert(pUser->iUser != NIL_PGMPOOL_IDX);
2872 Assert(pUser->iUser < pPool->cCurPages);
2873 uint32_t iUserTable = pUser->iUserTable;
2874
2875 /*
2876 * Map the user page.
2877 */
2878 PPGMPOOLPAGE pUserPage = &pPool->aPages[pUser->iUser];
2879 union
2880 {
2881 uint64_t *pau64;
2882 uint32_t *pau32;
2883 } u;
2884 u.pau64 = (uint64_t *)PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pUserPage);
2885
2886 LogFlow(("pgmPoolTrackClearPageUser: clear %x in %s (%RGp) (flushing %s)\n", iUserTable, pgmPoolPoolKindToStr(pUserPage->enmKind), pUserPage->Core.Key, pgmPoolPoolKindToStr(pPage->enmKind)));
2887
2888 /* Safety precaution in case we change the paging for other modes too in the future. */
2889 Assert(!pgmPoolIsPageLocked(&pPool->CTX_SUFF(pVM)->pgm.s, pPage));
2890
2891#ifdef VBOX_STRICT
2892 /*
2893 * Some sanity checks.
2894 */
2895 switch (pUserPage->enmKind)
2896 {
2897 case PGMPOOLKIND_32BIT_PD:
2898 case PGMPOOLKIND_32BIT_PD_PHYS:
2899 Assert(iUserTable < X86_PG_ENTRIES);
2900 break;
2901 case PGMPOOLKIND_PAE_PDPT:
2902 case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
2903 case PGMPOOLKIND_PAE_PDPT_PHYS:
2904 Assert(iUserTable < 4);
2905 Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
2906 break;
2907 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2908 case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2909 case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2910 case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2911 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2912 case PGMPOOLKIND_PAE_PD_PHYS:
2913 Assert(iUserTable < X86_PG_PAE_ENTRIES);
2914 break;
2915 case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2916 Assert(iUserTable < X86_PG_PAE_ENTRIES);
2917 Assert(!(u.pau64[iUserTable] & PGM_PDFLAGS_MAPPING));
2918 break;
2919 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2920 Assert(iUserTable < X86_PG_PAE_ENTRIES);
2921 Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
2922 break;
2923 case PGMPOOLKIND_64BIT_PML4:
2924 Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
2925 /* GCPhys >> PAGE_SHIFT is the index here */
2926 break;
2927 case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2928 case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2929 Assert(iUserTable < X86_PG_PAE_ENTRIES);
2930 break;
2931
2932 case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2933 case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2934 Assert(iUserTable < X86_PG_PAE_ENTRIES);
2935 break;
2936
2937 case PGMPOOLKIND_ROOT_NESTED:
2938 Assert(iUserTable < X86_PG_PAE_ENTRIES);
2939 break;
2940
2941 default:
2942 AssertMsgFailed(("enmKind=%d\n", pUserPage->enmKind));
2943 break;
2944 }
2945#endif /* VBOX_STRICT */
2946
2947 /*
2948 * Clear the entry in the user page.
2949 */
2950 switch (pUserPage->enmKind)
2951 {
2952 /* 32-bit entries */
2953 case PGMPOOLKIND_32BIT_PD:
2954 case PGMPOOLKIND_32BIT_PD_PHYS:
2955 u.pau32[iUserTable] = 0;
2956 break;
2957
2958 /* 64-bit entries */
2959 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2960 case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2961 case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2962 case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2963 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2964#if defined(IN_RC)
2965 /* In 32 bits PAE mode we *must* invalidate the TLB when changing a PDPT entry; the CPU fetches them only during cr3 load, so any
2966 * non-present PDPT will continue to cause page faults.
2967 */
2968 ASMReloadCR3();
2969#endif
2970 /* no break */
2971 case PGMPOOLKIND_PAE_PD_PHYS:
2972 case PGMPOOLKIND_PAE_PDPT_PHYS:
2973 case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2974 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2975 case PGMPOOLKIND_64BIT_PML4:
2976 case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2977 case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2978 case PGMPOOLKIND_PAE_PDPT:
2979 case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
2980 case PGMPOOLKIND_ROOT_NESTED:
2981 case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2982 case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2983 u.pau64[iUserTable] = 0;
2984 break;
2985
2986 default:
2987 AssertFatalMsgFailed(("enmKind=%d iUser=%#x iUserTable=%#x\n", pUserPage->enmKind, pUser->iUser, pUser->iUserTable));
2988 }
2989}
2990
2991
2992/**
2993 * Clears all users of a page.
2994 */
2995static void pgmPoolTrackClearPageUsers(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2996{
2997 /*
2998 * Free all the user records.
2999 */
3000 LogFlow(("pgmPoolTrackClearPageUsers %RGp\n", pPage->GCPhys));
3001
3002 PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
3003 uint16_t i = pPage->iUserHead;
3004 while (i != NIL_PGMPOOL_USER_INDEX)
3005 {
3006 /* Clear enter in user table. */
3007 pgmPoolTrackClearPageUser(pPool, pPage, &paUsers[i]);
3008
3009 /* Free it. */
3010 const uint16_t iNext = paUsers[i].iNext;
3011 paUsers[i].iUser = NIL_PGMPOOL_IDX;
3012 paUsers[i].iNext = pPool->iUserFreeHead;
3013 pPool->iUserFreeHead = i;
3014
3015 /* Next. */
3016 i = iNext;
3017 }
3018 pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
3019}
3020
3021#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
3022
3023/**
3024 * Allocates a new physical cross reference extent.
3025 *
3026 * @returns Pointer to the allocated extent on success. NULL if we're out of them.
3027 * @param pVM The VM handle.
3028 * @param piPhysExt Where to store the phys ext index.
3029 */
3030PPGMPOOLPHYSEXT pgmPoolTrackPhysExtAlloc(PVM pVM, uint16_t *piPhysExt)
3031{
3032 Assert(PGMIsLockOwner(pVM));
3033 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3034 uint16_t iPhysExt = pPool->iPhysExtFreeHead;
3035 if (iPhysExt == NIL_PGMPOOL_PHYSEXT_INDEX)
3036 {
3037 STAM_COUNTER_INC(&pPool->StamTrackPhysExtAllocFailures);
3038 return NULL;
3039 }
3040 PPGMPOOLPHYSEXT pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
3041 pPool->iPhysExtFreeHead = pPhysExt->iNext;
3042 pPhysExt->iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
3043 *piPhysExt = iPhysExt;
3044 return pPhysExt;
3045}
3046
3047
3048/**
3049 * Frees a physical cross reference extent.
3050 *
3051 * @param pVM The VM handle.
3052 * @param iPhysExt The extent to free.
3053 */
3054void pgmPoolTrackPhysExtFree(PVM pVM, uint16_t iPhysExt)
3055{
3056 Assert(PGMIsLockOwner(pVM));
3057 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3058 Assert(iPhysExt < pPool->cMaxPhysExts);
3059 PPGMPOOLPHYSEXT pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
3060 for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
3061 pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
3062 pPhysExt->iNext = pPool->iPhysExtFreeHead;
3063 pPool->iPhysExtFreeHead = iPhysExt;
3064}
3065
3066
3067/**
3068 * Frees a physical cross reference extent.
3069 *
3070 * @param pVM The VM handle.
3071 * @param iPhysExt The extent to free.
3072 */
3073void pgmPoolTrackPhysExtFreeList(PVM pVM, uint16_t iPhysExt)
3074{
3075 Assert(PGMIsLockOwner(pVM));
3076 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3077
3078 const uint16_t iPhysExtStart = iPhysExt;
3079 PPGMPOOLPHYSEXT pPhysExt;
3080 do
3081 {
3082 Assert(iPhysExt < pPool->cMaxPhysExts);
3083 pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
3084 for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
3085 pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
3086
3087 /* next */
3088 iPhysExt = pPhysExt->iNext;
3089 } while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
3090
3091 pPhysExt->iNext = pPool->iPhysExtFreeHead;
3092 pPool->iPhysExtFreeHead = iPhysExtStart;
3093}
3094
3095
3096/**
3097 * Insert a reference into a list of physical cross reference extents.
3098 *
3099 * @returns The new tracking data for PGMPAGE.
3100 *
3101 * @param pVM The VM handle.
3102 * @param iPhysExt The physical extent index of the list head.
3103 * @param iShwPT The shadow page table index.
3104 *
3105 */
3106static uint16_t pgmPoolTrackPhysExtInsert(PVM pVM, uint16_t iPhysExt, uint16_t iShwPT)
3107{
3108 Assert(PGMIsLockOwner(pVM));
3109 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3110 PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
3111
3112 /* special common case. */
3113 if (paPhysExts[iPhysExt].aidx[2] == NIL_PGMPOOL_IDX)
3114 {
3115 paPhysExts[iPhysExt].aidx[2] = iShwPT;
3116 STAM_COUNTER_INC(&pVM->pgm.s.StatTrackAliasedMany);
3117 LogFlow(("pgmPoolTrackPhysExtAddref: %d:{,,%d}\n", iPhysExt, iShwPT));
3118 return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
3119 }
3120
3121 /* general treatment. */
3122 const uint16_t iPhysExtStart = iPhysExt;
3123 unsigned cMax = 15;
3124 for (;;)
3125 {
3126 Assert(iPhysExt < pPool->cMaxPhysExts);
3127 for (unsigned i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
3128 if (paPhysExts[iPhysExt].aidx[i] == NIL_PGMPOOL_IDX)
3129 {
3130 paPhysExts[iPhysExt].aidx[i] = iShwPT;
3131 STAM_COUNTER_INC(&pVM->pgm.s.StatTrackAliasedMany);
3132 LogFlow(("pgmPoolTrackPhysExtAddref: %d:{%d} i=%d cMax=%d\n", iPhysExt, iShwPT, i, cMax));
3133 return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExtStart);
3134 }
3135 if (!--cMax)
3136 {
3137 STAM_COUNTER_INC(&pVM->pgm.s.StatTrackOverflows);
3138 pgmPoolTrackPhysExtFreeList(pVM, iPhysExtStart);
3139 LogFlow(("pgmPoolTrackPhysExtAddref: overflow (1) iShwPT=%d\n", iShwPT));
3140 return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
3141 }
3142 }
3143
3144 /* add another extent to the list. */
3145 PPGMPOOLPHYSEXT pNew = pgmPoolTrackPhysExtAlloc(pVM, &iPhysExt);
3146 if (!pNew)
3147 {
3148 STAM_COUNTER_INC(&pVM->pgm.s.StatTrackOverflows);
3149 pgmPoolTrackPhysExtFreeList(pVM, iPhysExtStart);
3150 return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
3151 }
3152 pNew->iNext = iPhysExtStart;
3153 pNew->aidx[0] = iShwPT;
3154 LogFlow(("pgmPoolTrackPhysExtAddref: added new extent %d:{%d}->%d\n", iPhysExt, iShwPT, iPhysExtStart));
3155 return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
3156}
3157
3158
3159/**
3160 * Add a reference to guest physical page where extents are in use.
3161 *
3162 * @returns The new tracking data for PGMPAGE.
3163 *
3164 * @param pVM The VM handle.
3165 * @param u16 The ram range flags (top 16-bits).
3166 * @param iShwPT The shadow page table index.
3167 */
3168uint16_t pgmPoolTrackPhysExtAddref(PVM pVM, uint16_t u16, uint16_t iShwPT)
3169{
3170 pgmLock(pVM);
3171 if (PGMPOOL_TD_GET_CREFS(u16) != PGMPOOL_TD_CREFS_PHYSEXT)
3172 {
3173 /*
3174 * Convert to extent list.
3175 */
3176 Assert(PGMPOOL_TD_GET_CREFS(u16) == 1);
3177 uint16_t iPhysExt;
3178 PPGMPOOLPHYSEXT pPhysExt = pgmPoolTrackPhysExtAlloc(pVM, &iPhysExt);
3179 if (pPhysExt)
3180 {
3181 LogFlow(("pgmPoolTrackPhysExtAddref: new extent: %d:{%d, %d}\n", iPhysExt, PGMPOOL_TD_GET_IDX(u16), iShwPT));
3182 STAM_COUNTER_INC(&pVM->pgm.s.StatTrackAliased);
3183 pPhysExt->aidx[0] = PGMPOOL_TD_GET_IDX(u16);
3184 pPhysExt->aidx[1] = iShwPT;
3185 u16 = PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
3186 }
3187 else
3188 u16 = PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
3189 }
3190 else if (u16 != PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED))
3191 {
3192 /*
3193 * Insert into the extent list.
3194 */
3195 u16 = pgmPoolTrackPhysExtInsert(pVM, PGMPOOL_TD_GET_IDX(u16), iShwPT);
3196 }
3197 else
3198 STAM_COUNTER_INC(&pVM->pgm.s.StatTrackAliasedLots);
3199 pgmUnlock(pVM);
3200 return u16;
3201}
3202
3203
3204/**
3205 * Clear references to guest physical memory.
3206 *
3207 * @param pPool The pool.
3208 * @param pPage The page.
3209 * @param pPhysPage Pointer to the aPages entry in the ram range.
3210 */
3211void pgmPoolTrackPhysExtDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMPAGE pPhysPage)
3212{
3213 const unsigned cRefs = PGM_PAGE_GET_TD_CREFS(pPhysPage);
3214 AssertFatalMsg(cRefs == PGMPOOL_TD_CREFS_PHYSEXT, ("cRefs=%d pPhysPage=%R[pgmpage] pPage=%p:{.idx=%d}\n", cRefs, pPhysPage, pPage, pPage->idx));
3215
3216 uint16_t iPhysExt = PGM_PAGE_GET_TD_IDX(pPhysPage);
3217 if (iPhysExt != PGMPOOL_TD_IDX_OVERFLOWED)
3218 {
3219 PVM pVM = pPool->CTX_SUFF(pVM);
3220 pgmLock(pVM);
3221
3222 uint16_t iPhysExtPrev = NIL_PGMPOOL_PHYSEXT_INDEX;
3223 PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
3224 do
3225 {
3226 Assert(iPhysExt < pPool->cMaxPhysExts);
3227
3228 /*
3229 * Look for the shadow page and check if it's all freed.
3230 */
3231 for (unsigned i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
3232 {
3233 if (paPhysExts[iPhysExt].aidx[i] == pPage->idx)
3234 {
3235 paPhysExts[iPhysExt].aidx[i] = NIL_PGMPOOL_IDX;
3236
3237 for (i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
3238 if (paPhysExts[iPhysExt].aidx[i] != NIL_PGMPOOL_IDX)
3239 {
3240 Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d\n", pPhysPage, pPage->idx));
3241 pgmUnlock(pVM);
3242 return;
3243 }
3244
3245 /* we can free the node. */
3246 const uint16_t iPhysExtNext = paPhysExts[iPhysExt].iNext;
3247 if ( iPhysExtPrev == NIL_PGMPOOL_PHYSEXT_INDEX
3248 && iPhysExtNext == NIL_PGMPOOL_PHYSEXT_INDEX)
3249 {
3250 /* lonely node */
3251 pgmPoolTrackPhysExtFree(pVM, iPhysExt);
3252 Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d lonely\n", pPhysPage, pPage->idx));
3253 PGM_PAGE_SET_TRACKING(pPhysPage, 0);
3254 }
3255 else if (iPhysExtPrev == NIL_PGMPOOL_PHYSEXT_INDEX)
3256 {
3257 /* head */
3258 Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d head\n", pPhysPage, pPage->idx));
3259 PGM_PAGE_SET_TRACKING(pPhysPage, PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExtNext));
3260 pgmPoolTrackPhysExtFree(pVM, iPhysExt);
3261 }
3262 else
3263 {
3264 /* in list */
3265 Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d\n", pPhysPage, pPage->idx));
3266 paPhysExts[iPhysExtPrev].iNext = iPhysExtNext;
3267 pgmPoolTrackPhysExtFree(pVM, iPhysExt);
3268 }
3269 iPhysExt = iPhysExtNext;
3270 pgmUnlock(pVM);
3271 return;
3272 }
3273 }
3274
3275 /* next */
3276 iPhysExtPrev = iPhysExt;
3277 iPhysExt = paPhysExts[iPhysExt].iNext;
3278 } while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
3279
3280 pgmUnlock(pVM);
3281 AssertFatalMsgFailed(("not-found! cRefs=%d pPhysPage=%R[pgmpage] pPage=%p:{.idx=%d}\n", cRefs, pPhysPage, pPage, pPage->idx));
3282 }
3283 else /* nothing to do */
3284 Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage]\n", pPhysPage));
3285}
3286
3287
3288/**
3289 * Clear references to guest physical memory.
3290 *
3291 * This is the same as pgmPoolTracDerefGCPhys except that the guest physical address
3292 * is assumed to be correct, so the linear search can be skipped and we can assert
3293 * at an earlier point.
3294 *
3295 * @param pPool The pool.
3296 * @param pPage The page.
3297 * @param HCPhys The host physical address corresponding to the guest page.
3298 * @param GCPhys The guest physical address corresponding to HCPhys.
3299 */
3300static void pgmPoolTracDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhys)
3301{
3302 /*
3303 * Walk range list.
3304 */
3305 PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRanges);
3306 while (pRam)
3307 {
3308 RTGCPHYS off = GCPhys - pRam->GCPhys;
3309 if (off < pRam->cb)
3310 {
3311 /* does it match? */
3312 const unsigned iPage = off >> PAGE_SHIFT;
3313 Assert(PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]));
3314#ifdef LOG_ENABLED
3315RTHCPHYS HCPhysPage = PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]);
3316Log2(("pgmPoolTracDerefGCPhys %RHp vs %RHp\n", HCPhysPage, HCPhys));
3317#endif
3318 if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
3319 {
3320 pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage]);
3321 return;
3322 }
3323 break;
3324 }
3325 pRam = pRam->CTX_SUFF(pNext);
3326 }
3327 AssertFatalMsgFailed(("HCPhys=%RHp GCPhys=%RGp\n", HCPhys, GCPhys));
3328}
3329
3330
3331/**
3332 * Clear references to guest physical memory.
3333 *
3334 * @param pPool The pool.
3335 * @param pPage The page.
3336 * @param HCPhys The host physical address corresponding to the guest page.
3337 * @param GCPhysHint The guest physical address which may corresponding to HCPhys.
3338 */
3339static void pgmPoolTracDerefGCPhysHint(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhysHint)
3340{
3341 Log4(("pgmPoolTracDerefGCPhysHint %RHp %RGp\n", HCPhys, GCPhysHint));
3342
3343 /*
3344 * Walk range list.
3345 */
3346 PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRanges);
3347 while (pRam)
3348 {
3349 RTGCPHYS off = GCPhysHint - pRam->GCPhys;
3350 if (off < pRam->cb)
3351 {
3352 /* does it match? */
3353 const unsigned iPage = off >> PAGE_SHIFT;
3354 Assert(PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]));
3355 if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
3356 {
3357 pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage]);
3358 return;
3359 }
3360 break;
3361 }
3362 pRam = pRam->CTX_SUFF(pNext);
3363 }
3364
3365 /*
3366 * Damn, the hint didn't work. We'll have to do an expensive linear search.
3367 */
3368 STAM_COUNTER_INC(&pPool->StatTrackLinearRamSearches);
3369 pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRanges);
3370 while (pRam)
3371 {
3372 unsigned iPage = pRam->cb >> PAGE_SHIFT;
3373 while (iPage-- > 0)
3374 {
3375 if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
3376 {
3377 Log4(("pgmPoolTracDerefGCPhysHint: Linear HCPhys=%RHp GCPhysHint=%RGp GCPhysReal=%RGp\n",
3378 HCPhys, GCPhysHint, pRam->GCPhys + (iPage << PAGE_SHIFT)));
3379 pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage]);
3380 return;
3381 }
3382 }
3383 pRam = pRam->CTX_SUFF(pNext);
3384 }
3385
3386 AssertFatalMsgFailed(("HCPhys=%RHp GCPhysHint=%RGp\n", HCPhys, GCPhysHint));
3387}
3388
3389
3390/**
3391 * Clear references to guest physical memory in a 32-bit / 32-bit page table.
3392 *
3393 * @param pPool The pool.
3394 * @param pPage The page.
3395 * @param pShwPT The shadow page table (mapping of the page).
3396 * @param pGstPT The guest page table.
3397 */
3398DECLINLINE(void) pgmPoolTrackDerefPT32Bit32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PT pShwPT, PCX86PT pGstPT)
3399{
3400 for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
3401 if (pShwPT->a[i].n.u1Present)
3402 {
3403 Log4(("pgmPoolTrackDerefPT32Bit32Bit: i=%d pte=%RX32 hint=%RX32\n",
3404 i, pShwPT->a[i].u & X86_PTE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK));
3405 pgmPoolTracDerefGCPhysHint(pPool, pPage, pShwPT->a[i].u & X86_PTE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK);
3406 if (!--pPage->cPresent)
3407 break;
3408 }
3409}
3410
3411
3412/**
3413 * Clear references to guest physical memory in a PAE / 32-bit page table.
3414 *
3415 * @param pPool The pool.
3416 * @param pPage The page.
3417 * @param pShwPT The shadow page table (mapping of the page).
3418 * @param pGstPT The guest page table (just a half one).
3419 */
3420DECLINLINE(void) pgmPoolTrackDerefPTPae32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PTPAE pShwPT, PCX86PT pGstPT)
3421{
3422 for (unsigned i = 0; i < RT_ELEMENTS(pShwPT->a); i++)
3423 if (pShwPT->a[i].n.u1Present)
3424 {
3425 Log4(("pgmPoolTrackDerefPTPae32Bit: i=%d pte=%RX64 hint=%RX32\n",
3426 i, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK));
3427 pgmPoolTracDerefGCPhysHint(pPool, pPage, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK);
3428 }
3429}
3430
3431
3432/**
3433 * Clear references to guest physical memory in a PAE / PAE page table.
3434 *
3435 * @param pPool The pool.
3436 * @param pPage The page.
3437 * @param pShwPT The shadow page table (mapping of the page).
3438 * @param pGstPT The guest page table.
3439 */
3440DECLINLINE(void) pgmPoolTrackDerefPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PTPAE pShwPT, PCX86PTPAE pGstPT)
3441{
3442 for (unsigned i = 0; i < RT_ELEMENTS(pShwPT->a); i++)
3443 if (pShwPT->a[i].n.u1Present)
3444 {
3445 Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX32 hint=%RX32\n",
3446 i, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PAE_PG_MASK));
3447 pgmPoolTracDerefGCPhysHint(pPool, pPage, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PAE_PG_MASK);
3448 }
3449}
3450
3451
3452/**
3453 * Clear references to guest physical memory in a 32-bit / 4MB page table.
3454 *
3455 * @param pPool The pool.
3456 * @param pPage The page.
3457 * @param pShwPT The shadow page table (mapping of the page).
3458 */
3459DECLINLINE(void) pgmPoolTrackDerefPT32Bit4MB(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PT pShwPT)
3460{
3461 RTGCPHYS GCPhys = pPage->GCPhys;
3462 for (unsigned i = 0; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
3463 if (pShwPT->a[i].n.u1Present)
3464 {
3465 Log4(("pgmPoolTrackDerefPT32Bit4MB: i=%d pte=%RX32 GCPhys=%RGp\n",
3466 i, pShwPT->a[i].u & X86_PTE_PG_MASK, GCPhys));
3467 pgmPoolTracDerefGCPhys(pPool, pPage, pShwPT->a[i].u & X86_PTE_PG_MASK, GCPhys);
3468 }
3469}
3470
3471
3472/**
3473 * Clear references to guest physical memory in a PAE / 2/4MB page table.
3474 *
3475 * @param pPool The pool.
3476 * @param pPage The page.
3477 * @param pShwPT The shadow page table (mapping of the page).
3478 */
3479DECLINLINE(void) pgmPoolTrackDerefPTPaeBig(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PTPAE pShwPT)
3480{
3481 RTGCPHYS GCPhys = pPage->GCPhys;
3482 for (unsigned i = 0; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
3483 if (pShwPT->a[i].n.u1Present)
3484 {
3485 Log4(("pgmPoolTrackDerefPTPaeBig: i=%d pte=%RX64 hint=%RGp\n",
3486 i, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, GCPhys));
3487 pgmPoolTracDerefGCPhys(pPool, pPage, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, GCPhys);
3488 }
3489}
3490
3491#endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
3492
3493
3494/**
3495 * Clear references to shadowed pages in a 32 bits page directory.
3496 *
3497 * @param pPool The pool.
3498 * @param pPage The page.
3499 * @param pShwPD The shadow page directory (mapping of the page).
3500 */
3501DECLINLINE(void) pgmPoolTrackDerefPD(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PD pShwPD)
3502{
3503 for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
3504 {
3505 if ( pShwPD->a[i].n.u1Present
3506 && !(pShwPD->a[i].u & PGM_PDFLAGS_MAPPING)
3507 )
3508 {
3509 PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & X86_PDE_PG_MASK);
3510 if (pSubPage)
3511 pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
3512 else
3513 AssertFatalMsgFailed(("%x\n", pShwPD->a[i].u & X86_PDE_PG_MASK));
3514 }
3515 }
3516}
3517
3518/**
3519 * Clear references to shadowed pages in a PAE (legacy or 64 bits) page directory.
3520 *
3521 * @param pPool The pool.
3522 * @param pPage The page.
3523 * @param pShwPD The shadow page directory (mapping of the page).
3524 */
3525DECLINLINE(void) pgmPoolTrackDerefPDPae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPAE pShwPD)
3526{
3527 for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
3528 {
3529 if ( pShwPD->a[i].n.u1Present
3530 && !(pShwPD->a[i].u & PGM_PDFLAGS_MAPPING)
3531 )
3532 {
3533 PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & X86_PDE_PAE_PG_MASK);
3534 if (pSubPage)
3535 pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
3536 else
3537 AssertFatalMsgFailed(("%RX64\n", pShwPD->a[i].u & X86_PDE_PAE_PG_MASK));
3538 /** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
3539 }
3540 }
3541}
3542
3543/**
3544 * Clear references to shadowed pages in a PAE page directory pointer table.
3545 *
3546 * @param pPool The pool.
3547 * @param pPage The page.
3548 * @param pShwPDPT The shadow page directory pointer table (mapping of the page).
3549 */
3550DECLINLINE(void) pgmPoolTrackDerefPDPTPae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPT pShwPDPT)
3551{
3552 for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
3553 {
3554 if ( pShwPDPT->a[i].n.u1Present
3555 && !(pShwPDPT->a[i].u & PGM_PLXFLAGS_MAPPING)
3556 )
3557 {
3558 PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPDPT->a[i].u & X86_PDPE_PG_MASK);
3559 if (pSubPage)
3560 pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
3561 else
3562 AssertFatalMsgFailed(("%RX64\n", pShwPDPT->a[i].u & X86_PDPE_PG_MASK));
3563 }
3564 }
3565}
3566
3567
3568/**
3569 * Clear references to shadowed pages in a 64-bit page directory pointer table.
3570 *
3571 * @param pPool The pool.
3572 * @param pPage The page.
3573 * @param pShwPDPT The shadow page directory pointer table (mapping of the page).
3574 */
3575DECLINLINE(void) pgmPoolTrackDerefPDPT64Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPT pShwPDPT)
3576{
3577 for (unsigned i = 0; i < RT_ELEMENTS(pShwPDPT->a); i++)
3578 {
3579 Assert(!(pShwPDPT->a[i].u & PGM_PLXFLAGS_MAPPING));
3580 if (pShwPDPT->a[i].n.u1Present)
3581 {
3582 PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPDPT->a[i].u & X86_PDPE_PG_MASK);
3583 if (pSubPage)
3584 pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
3585 else
3586 AssertFatalMsgFailed(("%RX64\n", pShwPDPT->a[i].u & X86_PDPE_PG_MASK));
3587 /** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
3588 }
3589 }
3590}
3591
3592
3593/**
3594 * Clear references to shadowed pages in a 64-bit level 4 page table.
3595 *
3596 * @param pPool The pool.
3597 * @param pPage The page.
3598 * @param pShwPML4 The shadow page directory pointer table (mapping of the page).
3599 */
3600DECLINLINE(void) pgmPoolTrackDerefPML464Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PML4 pShwPML4)
3601{
3602 for (unsigned i = 0; i < RT_ELEMENTS(pShwPML4->a); i++)
3603 {
3604 if (pShwPML4->a[i].n.u1Present)
3605 {
3606 PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPML4->a[i].u & X86_PDPE_PG_MASK);
3607 if (pSubPage)
3608 pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
3609 else
3610 AssertFatalMsgFailed(("%RX64\n", pShwPML4->a[i].u & X86_PML4E_PG_MASK));
3611 /** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
3612 }
3613 }
3614}
3615
3616
3617/**
3618 * Clear references to shadowed pages in an EPT page table.
3619 *
3620 * @param pPool The pool.
3621 * @param pPage The page.
3622 * @param pShwPML4 The shadow page directory pointer table (mapping of the page).
3623 */
3624DECLINLINE(void) pgmPoolTrackDerefPTEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPT pShwPT)
3625{
3626 RTGCPHYS GCPhys = pPage->GCPhys;
3627 for (unsigned i = 0; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
3628 if (pShwPT->a[i].n.u1Present)
3629 {
3630 Log4(("pgmPoolTrackDerefPTEPT: i=%d pte=%RX64 GCPhys=%RX64\n",
3631 i, pShwPT->a[i].u & EPT_PTE_PG_MASK, pPage->GCPhys));
3632 pgmPoolTracDerefGCPhys(pPool, pPage, pShwPT->a[i].u & EPT_PTE_PG_MASK, GCPhys);
3633 }
3634}
3635
3636
3637/**
3638 * Clear references to shadowed pages in an EPT page directory.
3639 *
3640 * @param pPool The pool.
3641 * @param pPage The page.
3642 * @param pShwPD The shadow page directory (mapping of the page).
3643 */
3644DECLINLINE(void) pgmPoolTrackDerefPDEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPD pShwPD)
3645{
3646 for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
3647 {
3648 if (pShwPD->a[i].n.u1Present)
3649 {
3650 PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & EPT_PDE_PG_MASK);
3651 if (pSubPage)
3652 pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
3653 else
3654 AssertFatalMsgFailed(("%RX64\n", pShwPD->a[i].u & EPT_PDE_PG_MASK));
3655 /** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
3656 }
3657 }
3658}
3659
3660
3661/**
3662 * Clear references to shadowed pages in an EPT page directory pointer table.
3663 *
3664 * @param pPool The pool.
3665 * @param pPage The page.
3666 * @param pShwPDPT The shadow page directory pointer table (mapping of the page).
3667 */
3668DECLINLINE(void) pgmPoolTrackDerefPDPTEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPDPT pShwPDPT)
3669{
3670 for (unsigned i = 0; i < RT_ELEMENTS(pShwPDPT->a); i++)
3671 {
3672 if (pShwPDPT->a[i].n.u1Present)
3673 {
3674 PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPDPT->a[i].u & EPT_PDPTE_PG_MASK);
3675 if (pSubPage)
3676 pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
3677 else
3678 AssertFatalMsgFailed(("%RX64\n", pShwPDPT->a[i].u & EPT_PDPTE_PG_MASK));
3679 /** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
3680 }
3681 }
3682}
3683
3684
3685/**
3686 * Clears all references made by this page.
3687 *
3688 * This includes other shadow pages and GC physical addresses.
3689 *
3690 * @param pPool The pool.
3691 * @param pPage The page.
3692 */
3693static void pgmPoolTrackDeref(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
3694{
3695 /*
3696 * Map the shadow page and take action according to the page kind.
3697 */
3698 void *pvShw = PGMPOOL_PAGE_2_LOCKED_PTR(pPool->CTX_SUFF(pVM), pPage);
3699 switch (pPage->enmKind)
3700 {
3701#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
3702 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3703 {
3704 STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
3705 void *pvGst;
3706 int rc = PGM_GCPHYS_2_PTR(pPool->CTX_SUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
3707 pgmPoolTrackDerefPT32Bit32Bit(pPool, pPage, (PX86PT)pvShw, (PCX86PT)pvGst);
3708 STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
3709 break;
3710 }
3711
3712 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3713 {
3714 STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
3715 void *pvGst;
3716 int rc = PGM_GCPHYS_2_PTR_EX(pPool->CTX_SUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
3717 pgmPoolTrackDerefPTPae32Bit(pPool, pPage, (PX86PTPAE)pvShw, (PCX86PT)pvGst);
3718 STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
3719 break;
3720 }
3721
3722 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3723 {
3724 STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
3725 void *pvGst;
3726 int rc = PGM_GCPHYS_2_PTR(pPool->CTX_SUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
3727 pgmPoolTrackDerefPTPaePae(pPool, pPage, (PX86PTPAE)pvShw, (PCX86PTPAE)pvGst);
3728 STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
3729 break;
3730 }
3731
3732 case PGMPOOLKIND_32BIT_PT_FOR_PHYS: /* treat it like a 4 MB page */
3733 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3734 {
3735 STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
3736 pgmPoolTrackDerefPT32Bit4MB(pPool, pPage, (PX86PT)pvShw);
3737 STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
3738 break;
3739 }
3740
3741 case PGMPOOLKIND_PAE_PT_FOR_PHYS: /* treat it like a 2 MB page */
3742 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3743 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3744 {
3745 STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
3746 pgmPoolTrackDerefPTPaeBig(pPool, pPage, (PX86PTPAE)pvShw);
3747 STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
3748 break;
3749 }
3750
3751#else /* !PGMPOOL_WITH_GCPHYS_TRACKING */
3752 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3753 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3754 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3755 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3756 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3757 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3758 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
3759 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
3760 break;
3761#endif /* !PGMPOOL_WITH_GCPHYS_TRACKING */
3762
3763 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
3764 case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
3765 case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
3766 case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
3767 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3768 case PGMPOOLKIND_PAE_PD_PHYS:
3769 case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
3770 case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
3771 pgmPoolTrackDerefPDPae(pPool, pPage, (PX86PDPAE)pvShw);
3772 break;
3773
3774 case PGMPOOLKIND_32BIT_PD_PHYS:
3775 case PGMPOOLKIND_32BIT_PD:
3776 pgmPoolTrackDerefPD(pPool, pPage, (PX86PD)pvShw);
3777 break;
3778
3779 case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
3780 case PGMPOOLKIND_PAE_PDPT:
3781 case PGMPOOLKIND_PAE_PDPT_PHYS:
3782 pgmPoolTrackDerefPDPTPae(pPool, pPage, (PX86PDPT)pvShw);
3783 break;
3784
3785 case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
3786 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
3787 pgmPoolTrackDerefPDPT64Bit(pPool, pPage, (PX86PDPT)pvShw);
3788 break;
3789
3790 case PGMPOOLKIND_64BIT_PML4:
3791 pgmPoolTrackDerefPML464Bit(pPool, pPage, (PX86PML4)pvShw);
3792 break;
3793
3794 case PGMPOOLKIND_EPT_PT_FOR_PHYS:
3795 pgmPoolTrackDerefPTEPT(pPool, pPage, (PEPTPT)pvShw);
3796 break;
3797
3798 case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3799 pgmPoolTrackDerefPDEPT(pPool, pPage, (PEPTPD)pvShw);
3800 break;
3801
3802 case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
3803 pgmPoolTrackDerefPDPTEPT(pPool, pPage, (PEPTPDPT)pvShw);
3804 break;
3805
3806 default:
3807 AssertFatalMsgFailed(("enmKind=%d\n", pPage->enmKind));
3808 }
3809
3810 /* paranoia, clear the shadow page. Remove this laser (i.e. let Alloc and ClearAll do it). */
3811 STAM_PROFILE_START(&pPool->StatZeroPage, z);
3812 ASMMemZeroPage(pvShw);
3813 STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
3814 pPage->fZeroed = true;
3815 PGMPOOL_UNLOCK_PTR(pPool->CTX_SUFF(pVM), pvShw);
3816}
3817#endif /* PGMPOOL_WITH_USER_TRACKING */
3818
3819/**
3820 * Flushes a pool page.
3821 *
3822 * This moves the page to the free list after removing all user references to it.
3823 *
3824 * @returns VBox status code.
3825 * @retval VINF_SUCCESS on success.
3826 * @param pPool The pool.
3827 * @param HCPhys The HC physical address of the shadow page.
3828 */
3829int pgmPoolFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
3830{
3831 PVM pVM = pPool->CTX_SUFF(pVM);
3832
3833 int rc = VINF_SUCCESS;
3834 STAM_PROFILE_START(&pPool->StatFlushPage, f);
3835 LogFlow(("pgmPoolFlushPage: pPage=%p:{.Key=%RHp, .idx=%d, .enmKind=%s, .GCPhys=%RGp}\n",
3836 pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
3837
3838 /*
3839 * Quietly reject any attempts at flushing any of the special root pages.
3840 */
3841 if (pPage->idx < PGMPOOL_IDX_FIRST)
3842 {
3843 AssertFailed(); /* can no longer happen */
3844 Log(("pgmPoolFlushPage: special root page, rejected. enmKind=%s idx=%d\n", pgmPoolPoolKindToStr(pPage->enmKind), pPage->idx));
3845 return VINF_SUCCESS;
3846 }
3847
3848 pgmLock(pVM);
3849
3850 /*
3851 * Quietly reject any attempts at flushing the currently active shadow CR3 mapping
3852 */
3853 if (pgmPoolIsPageLocked(&pVM->pgm.s, pPage))
3854 {
3855 AssertMsg( pPage->enmKind == PGMPOOLKIND_64BIT_PML4
3856 || pPage->enmKind == PGMPOOLKIND_PAE_PDPT
3857 || pPage->enmKind == PGMPOOLKIND_PAE_PDPT_FOR_32BIT
3858 || pPage->enmKind == PGMPOOLKIND_32BIT_PD
3859 || pPage->enmKind == PGMPOOLKIND_PAE_PD_FOR_PAE_PD
3860 || pPage->enmKind == PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD
3861 || pPage->enmKind == PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD
3862 || pPage->enmKind == PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD
3863 || pPage->enmKind == PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD,
3864 ("Can't free the shadow CR3! (%RHp vs %RHp kind=%d\n", PGMGetHyperCR3(VMMGetCpu(pVM)), pPage->Core.Key, pPage->enmKind));
3865 Log(("pgmPoolFlushPage: current active shadow CR3, rejected. enmKind=%s idx=%d\n", pgmPoolPoolKindToStr(pPage->enmKind), pPage->idx));
3866 pgmUnlock(pVM);
3867 return VINF_SUCCESS;
3868 }
3869
3870#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3871 /* Start a subset so we won't run out of mapping space. */
3872 PVMCPU pVCpu = VMMGetCpu(pVM);
3873 uint32_t iPrevSubset = PGMDynMapPushAutoSubset(pVCpu);
3874#endif
3875
3876 /*
3877 * Mark the page as being in need of a ASMMemZeroPage().
3878 */
3879 pPage->fZeroed = false;
3880
3881#ifdef PGMPOOL_WITH_USER_TRACKING
3882 /*
3883 * Clear the page.
3884 */
3885 pgmPoolTrackClearPageUsers(pPool, pPage);
3886 STAM_PROFILE_START(&pPool->StatTrackDeref,a);
3887 pgmPoolTrackDeref(pPool, pPage);
3888 STAM_PROFILE_STOP(&pPool->StatTrackDeref,a);
3889#endif
3890
3891#ifdef PGMPOOL_WITH_CACHE
3892 /*
3893 * Flush it from the cache.
3894 */
3895 pgmPoolCacheFlushPage(pPool, pPage);
3896#endif /* PGMPOOL_WITH_CACHE */
3897
3898#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3899 /* Heavy stuff done. */
3900 PGMDynMapPopAutoSubset(pVCpu, iPrevSubset);
3901#endif
3902
3903#ifdef PGMPOOL_WITH_MONITORING
3904 /*
3905 * Deregistering the monitoring.
3906 */
3907 if (pPage->fMonitored)
3908 rc = pgmPoolMonitorFlush(pPool, pPage);
3909#endif
3910
3911 /*
3912 * Free the page.
3913 */
3914 Assert(pPage->iNext == NIL_PGMPOOL_IDX);
3915 pPage->iNext = pPool->iFreeHead;
3916 pPool->iFreeHead = pPage->idx;
3917 pPage->enmKind = PGMPOOLKIND_FREE;
3918 pPage->GCPhys = NIL_RTGCPHYS;
3919 pPage->fReusedFlushPending = false;
3920
3921 pPool->cUsedPages--;
3922 pgmUnlock(pVM);
3923 STAM_PROFILE_STOP(&pPool->StatFlushPage, f);
3924 return rc;
3925}
3926
3927
3928/**
3929 * Frees a usage of a pool page.
3930 *
3931 * The caller is responsible to updating the user table so that it no longer
3932 * references the shadow page.
3933 *
3934 * @param pPool The pool.
3935 * @param HCPhys The HC physical address of the shadow page.
3936 * @param iUser The shadow page pool index of the user table.
3937 * @param iUserTable The index into the user table (shadowed).
3938 */
3939void pgmPoolFreeByPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
3940{
3941 PVM pVM = pPool->CTX_SUFF(pVM);
3942
3943 STAM_PROFILE_START(&pPool->StatFree, a);
3944 LogFlow(("pgmPoolFreeByPage: pPage=%p:{.Key=%RHp, .idx=%d, enmKind=%s} iUser=%#x iUserTable=%#x\n",
3945 pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), iUser, iUserTable));
3946 Assert(pPage->idx >= PGMPOOL_IDX_FIRST);
3947 pgmLock(pVM);
3948#ifdef PGMPOOL_WITH_USER_TRACKING
3949 pgmPoolTrackFreeUser(pPool, pPage, iUser, iUserTable);
3950#endif
3951#ifdef PGMPOOL_WITH_CACHE
3952 if (!pPage->fCached)
3953#endif
3954 pgmPoolFlushPage(pPool, pPage);
3955 pgmUnlock(pVM);
3956 STAM_PROFILE_STOP(&pPool->StatFree, a);
3957}
3958
3959
3960/**
3961 * Makes one or more free page free.
3962 *
3963 * @returns VBox status code.
3964 * @retval VINF_SUCCESS on success.
3965 * @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
3966 *
3967 * @param pPool The pool.
3968 * @param enmKind Page table kind
3969 * @param iUser The user of the page.
3970 */
3971static int pgmPoolMakeMoreFreePages(PPGMPOOL pPool, PGMPOOLKIND enmKind, uint16_t iUser)
3972{
3973 PVM pVM = pPool->CTX_SUFF(pVM);
3974
3975 LogFlow(("pgmPoolMakeMoreFreePages: iUser=%#x\n", iUser));
3976
3977 /*
3978 * If the pool isn't full grown yet, expand it.
3979 */
3980 if ( pPool->cCurPages < pPool->cMaxPages
3981#if defined(IN_RC)
3982 /* Hack alert: we can't deal with jumps to ring 3 when called from MapCR3 and allocating pages for PAE PDs. */
3983 && enmKind != PGMPOOLKIND_PAE_PD_FOR_PAE_PD
3984 && (enmKind < PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD || enmKind > PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD)
3985#endif
3986 )
3987 {
3988 STAM_PROFILE_ADV_SUSPEND(&pPool->StatAlloc, a);
3989#ifdef IN_RING3
3990 int rc = PGMR3PoolGrow(pVM);
3991#else
3992 int rc = CTXALLMID(VMM, CallHost)(pVM, VMMCALLHOST_PGM_POOL_GROW, 0);
3993#endif
3994 if (RT_FAILURE(rc))
3995 return rc;
3996 STAM_PROFILE_ADV_RESUME(&pPool->StatAlloc, a);
3997 if (pPool->iFreeHead != NIL_PGMPOOL_IDX)
3998 return VINF_SUCCESS;
3999 }
4000
4001#ifdef PGMPOOL_WITH_CACHE
4002 /*
4003 * Free one cached page.
4004 */
4005 return pgmPoolCacheFreeOne(pPool, iUser);
4006#else
4007 /*
4008 * Flush the pool.
4009 *
4010 * If we have tracking enabled, it should be possible to come up with
4011 * a cheap replacement strategy...
4012 */
4013 /* @todo This path no longer works (CR3 root pages will be flushed)!! */
4014 AssertCompileFailed();
4015 Assert(!CPUMIsGuestInLongMode(pVM));
4016 pgmPoolFlushAllInt(pPool);
4017 return VERR_PGM_POOL_FLUSHED;
4018#endif
4019}
4020
4021
4022/**
4023 * Allocates a page from the pool.
4024 *
4025 * This page may actually be a cached page and not in need of any processing
4026 * on the callers part.
4027 *
4028 * @returns VBox status code.
4029 * @retval VINF_SUCCESS if a NEW page was allocated.
4030 * @retval VINF_PGM_CACHED_PAGE if a CACHED page was returned.
4031 * @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
4032 * @param pVM The VM handle.
4033 * @param GCPhys The GC physical address of the page we're gonna shadow.
4034 * For 4MB and 2MB PD entries, it's the first address the
4035 * shadow PT is covering.
4036 * @param enmKind The kind of mapping.
4037 * @param iUser The shadow page pool index of the user table.
4038 * @param iUserTable The index into the user table (shadowed).
4039 * @param ppPage Where to store the pointer to the page. NULL is stored here on failure.
4040 */
4041int pgmPoolAlloc(PVM pVM, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, uint16_t iUser, uint32_t iUserTable, PPPGMPOOLPAGE ppPage)
4042{
4043 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4044 STAM_PROFILE_ADV_START(&pPool->StatAlloc, a);
4045 LogFlow(("pgmPoolAlloc: GCPhys=%RGp enmKind=%s iUser=%#x iUserTable=%#x\n", GCPhys, pgmPoolPoolKindToStr(enmKind), iUser, iUserTable));
4046 *ppPage = NULL;
4047 /** @todo CSAM/PGMPrefetchPage messes up here during CSAMR3CheckGates
4048 * (TRPMR3SyncIDT) because of FF priority. Try fix that?
4049 * Assert(!(pVM->pgm.s.fGlobalSyncFlags & PGM_GLOBAL_SYNC_CLEAR_PGM_POOL)); */
4050
4051 pgmLock(pVM);
4052
4053#ifdef PGMPOOL_WITH_CACHE
4054 if (pPool->fCacheEnabled)
4055 {
4056 int rc2 = pgmPoolCacheAlloc(pPool, GCPhys, enmKind, iUser, iUserTable, ppPage);
4057 if (RT_SUCCESS(rc2))
4058 {
4059 pgmUnlock(pVM);
4060 STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
4061 LogFlow(("pgmPoolAlloc: cached returns %Rrc *ppPage=%p:{.Key=%RHp, .idx=%d}\n", rc2, *ppPage, (*ppPage)->Core.Key, (*ppPage)->idx));
4062 return rc2;
4063 }
4064 }
4065#endif
4066
4067 /*
4068 * Allocate a new one.
4069 */
4070 int rc = VINF_SUCCESS;
4071 uint16_t iNew = pPool->iFreeHead;
4072 if (iNew == NIL_PGMPOOL_IDX)
4073 {
4074 rc = pgmPoolMakeMoreFreePages(pPool, enmKind, iUser);
4075 if (RT_FAILURE(rc))
4076 {
4077 pgmUnlock(pVM);
4078 Log(("pgmPoolAlloc: returns %Rrc (Free)\n", rc));
4079 STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
4080 return rc;
4081 }
4082 iNew = pPool->iFreeHead;
4083 AssertReleaseReturn(iNew != NIL_PGMPOOL_IDX, VERR_INTERNAL_ERROR);
4084 }
4085
4086 /* unlink the free head */
4087 PPGMPOOLPAGE pPage = &pPool->aPages[iNew];
4088 pPool->iFreeHead = pPage->iNext;
4089 pPage->iNext = NIL_PGMPOOL_IDX;
4090
4091 /*
4092 * Initialize it.
4093 */
4094 pPool->cUsedPages++; /* physical handler registration / pgmPoolTrackFlushGCPhysPTsSlow requirement. */
4095 pPage->enmKind = enmKind;
4096 pPage->GCPhys = GCPhys;
4097 pPage->fSeenNonGlobal = false; /* Set this to 'true' to disable this feature. */
4098 pPage->fMonitored = false;
4099 pPage->fCached = false;
4100 pPage->fReusedFlushPending = false;
4101#ifdef PGMPOOL_WITH_MONITORING
4102 pPage->cModifications = 0;
4103 pPage->iModifiedNext = NIL_PGMPOOL_IDX;
4104 pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
4105#else
4106 pPage->fCR3Mix = false;
4107#endif
4108#ifdef PGMPOOL_WITH_USER_TRACKING
4109 pPage->cPresent = 0;
4110 pPage->iFirstPresent = ~0;
4111
4112 /*
4113 * Insert into the tracking and cache. If this fails, free the page.
4114 */
4115 int rc3 = pgmPoolTrackInsert(pPool, pPage, GCPhys, iUser, iUserTable);
4116 if (RT_FAILURE(rc3))
4117 {
4118 pPool->cUsedPages--;
4119 pPage->enmKind = PGMPOOLKIND_FREE;
4120 pPage->GCPhys = NIL_RTGCPHYS;
4121 pPage->iNext = pPool->iFreeHead;
4122 pPool->iFreeHead = pPage->idx;
4123 pgmUnlock(pVM);
4124 STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
4125 Log(("pgmPoolAlloc: returns %Rrc (Insert)\n", rc3));
4126 return rc3;
4127 }
4128#endif /* PGMPOOL_WITH_USER_TRACKING */
4129
4130 /*
4131 * Commit the allocation, clear the page and return.
4132 */
4133#ifdef VBOX_WITH_STATISTICS
4134 if (pPool->cUsedPages > pPool->cUsedPagesHigh)
4135 pPool->cUsedPagesHigh = pPool->cUsedPages;
4136#endif
4137
4138 if (!pPage->fZeroed)
4139 {
4140 STAM_PROFILE_START(&pPool->StatZeroPage, z);
4141 void *pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
4142 ASMMemZeroPage(pv);
4143 STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
4144 }
4145
4146 *ppPage = pPage;
4147 pgmUnlock(pVM);
4148 LogFlow(("pgmPoolAlloc: returns %Rrc *ppPage=%p:{.Key=%RHp, .idx=%d, .fCached=%RTbool, .fMonitored=%RTbool}\n",
4149 rc, pPage, pPage->Core.Key, pPage->idx, pPage->fCached, pPage->fMonitored));
4150 STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
4151 return rc;
4152}
4153
4154
4155/**
4156 * Frees a usage of a pool page.
4157 *
4158 * @param pVM The VM handle.
4159 * @param HCPhys The HC physical address of the shadow page.
4160 * @param iUser The shadow page pool index of the user table.
4161 * @param iUserTable The index into the user table (shadowed).
4162 */
4163void pgmPoolFree(PVM pVM, RTHCPHYS HCPhys, uint16_t iUser, uint32_t iUserTable)
4164{
4165 LogFlow(("pgmPoolFree: HCPhys=%RHp iUser=%#x iUserTable=%#x\n", HCPhys, iUser, iUserTable));
4166 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4167 pgmPoolFreeByPage(pPool, pgmPoolGetPage(pPool, HCPhys), iUser, iUserTable);
4168}
4169
4170/**
4171 * Internal worker for finding a 'in-use' shadow page give by it's physical address.
4172 *
4173 * @returns Pointer to the shadow page structure.
4174 * @param pPool The pool.
4175 * @param HCPhys The HC physical address of the shadow page.
4176 */
4177PPGMPOOLPAGE pgmPoolGetPage(PPGMPOOL pPool, RTHCPHYS HCPhys)
4178{
4179 PVM pVM = pPool->CTX_SUFF(pVM);
4180
4181 /*
4182 * Look up the page.
4183 */
4184 pgmLock(pVM);
4185 PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, HCPhys & X86_PTE_PAE_PG_MASK);
4186 pgmUnlock(pVM);
4187
4188 AssertFatalMsg(pPage && pPage->enmKind != PGMPOOLKIND_FREE, ("HCPhys=%RHp pPage=%p idx=%d\n", HCPhys, pPage, (pPage) ? pPage->idx : 0));
4189 return pPage;
4190}
4191
4192
4193#ifdef IN_RING3
4194/**
4195 * Flushes the entire cache.
4196 *
4197 * It will assert a global CR3 flush (FF) and assumes the caller is aware of this
4198 * and execute this CR3 flush.
4199 *
4200 * @param pPool The pool.
4201 */
4202void pgmR3PoolReset(PVM pVM)
4203{
4204 PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4205
4206 Assert(PGMIsLockOwner(pVM));
4207 STAM_PROFILE_START(&pPool->StatFlushAllInt, a);
4208 LogFlow(("pgmPoolFlushAllInt:\n"));
4209
4210 /*
4211 * If there are no pages in the pool, there is nothing to do.
4212 */
4213 if (pPool->cCurPages <= PGMPOOL_IDX_FIRST)
4214 {
4215 STAM_PROFILE_STOP(&pPool->StatFlushAllInt, a);
4216 return;
4217 }
4218
4219 /*
4220 * Exit the shadow mode since we're going to clear everything,
4221 * including the root page.
4222 */
4223 for (unsigned i=0;i<pVM->cCPUs;i++)
4224 {
4225 PVMCPU pVCpu = &pVM->aCpus[i];
4226 pgmR3ExitShadowModeBeforePoolFlush(pVM, pVCpu);
4227 }
4228
4229 /*
4230 * Nuke the free list and reinsert all pages into it.
4231 */
4232 for (unsigned i = pPool->cCurPages - 1; i >= PGMPOOL_IDX_FIRST; i--)
4233 {
4234 PPGMPOOLPAGE pPage = &pPool->aPages[i];
4235
4236 Assert(pPage->Core.Key == MMPage2Phys(pVM, pPage->pvPageR3));
4237#ifdef PGMPOOL_WITH_MONITORING
4238 if (pPage->fMonitored)
4239 pgmPoolMonitorFlush(pPool, pPage);
4240 pPage->iModifiedNext = NIL_PGMPOOL_IDX;
4241 pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
4242 pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
4243 pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
4244 pPage->cModifications = 0;
4245#endif
4246 pPage->GCPhys = NIL_RTGCPHYS;
4247 pPage->enmKind = PGMPOOLKIND_FREE;
4248 Assert(pPage->idx == i);
4249 pPage->iNext = i + 1;
4250 pPage->fZeroed = false; /* This could probably be optimized, but better safe than sorry. */
4251 pPage->fSeenNonGlobal = false;
4252 pPage->fMonitored= false;
4253 pPage->fCached = false;
4254 pPage->fReusedFlushPending = false;
4255#ifdef PGMPOOL_WITH_USER_TRACKING
4256 pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
4257#else
4258 pPage->fCR3Mix = false;
4259#endif
4260#ifdef PGMPOOL_WITH_CACHE
4261 pPage->iAgeNext = NIL_PGMPOOL_IDX;
4262 pPage->iAgePrev = NIL_PGMPOOL_IDX;
4263#endif
4264 pPage->cLocked = 0;
4265 }
4266 pPool->aPages[pPool->cCurPages - 1].iNext = NIL_PGMPOOL_IDX;
4267 pPool->iFreeHead = PGMPOOL_IDX_FIRST;
4268 pPool->cUsedPages = 0;
4269
4270#ifdef PGMPOOL_WITH_USER_TRACKING
4271 /*
4272 * Zap and reinitialize the user records.
4273 */
4274 pPool->cPresent = 0;
4275 pPool->iUserFreeHead = 0;
4276 PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
4277 const unsigned cMaxUsers = pPool->cMaxUsers;
4278 for (unsigned i = 0; i < cMaxUsers; i++)
4279 {
4280 paUsers[i].iNext = i + 1;
4281 paUsers[i].iUser = NIL_PGMPOOL_IDX;
4282 paUsers[i].iUserTable = 0xfffffffe;
4283 }
4284 paUsers[cMaxUsers - 1].iNext = NIL_PGMPOOL_USER_INDEX;
4285#endif
4286
4287#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
4288 /*
4289 * Clear all the GCPhys links and rebuild the phys ext free list.
4290 */
4291 for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
4292 pRam;
4293 pRam = pRam->CTX_SUFF(pNext))
4294 {
4295 unsigned iPage = pRam->cb >> PAGE_SHIFT;
4296 while (iPage-- > 0)
4297 PGM_PAGE_SET_TRACKING(&pRam->aPages[iPage], 0);
4298 }
4299
4300 pPool->iPhysExtFreeHead = 0;
4301 PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
4302 const unsigned cMaxPhysExts = pPool->cMaxPhysExts;
4303 for (unsigned i = 0; i < cMaxPhysExts; i++)
4304 {
4305 paPhysExts[i].iNext = i + 1;
4306 paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX;
4307 paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX;
4308 paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX;
4309 }
4310 paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
4311#endif
4312
4313#ifdef PGMPOOL_WITH_MONITORING
4314 /*
4315 * Just zap the modified list.
4316 */
4317 pPool->cModifiedPages = 0;
4318 pPool->iModifiedHead = NIL_PGMPOOL_IDX;
4319#endif
4320
4321#ifdef PGMPOOL_WITH_CACHE
4322 /*
4323 * Clear the GCPhys hash and the age list.
4324 */
4325 for (unsigned i = 0; i < RT_ELEMENTS(pPool->aiHash); i++)
4326 pPool->aiHash[i] = NIL_PGMPOOL_IDX;
4327 pPool->iAgeHead = NIL_PGMPOOL_IDX;
4328 pPool->iAgeTail = NIL_PGMPOOL_IDX;
4329#endif
4330
4331 /*
4332 * Reinsert active pages into the hash and ensure monitoring chains are correct.
4333 */
4334 for (unsigned i = PGMPOOL_IDX_FIRST_SPECIAL; i < PGMPOOL_IDX_FIRST; i++)
4335 {
4336 PPGMPOOLPAGE pPage = &pPool->aPages[i];
4337 pPage->iNext = NIL_PGMPOOL_IDX;
4338#ifdef PGMPOOL_WITH_MONITORING
4339 pPage->iModifiedNext = NIL_PGMPOOL_IDX;
4340 pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
4341 pPage->cModifications = 0;
4342 /* ASSUMES that we're not sharing with any of the other special pages (safe for now). */
4343 pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
4344 pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
4345 if (pPage->fMonitored)
4346 {
4347 int rc = PGMHandlerPhysicalChangeCallbacks(pVM, pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1),
4348 pPool->pfnAccessHandlerR3, MMHyperCCToR3(pVM, pPage),
4349 pPool->pfnAccessHandlerR0, MMHyperCCToR0(pVM, pPage),
4350 pPool->pfnAccessHandlerRC, MMHyperCCToRC(pVM, pPage),
4351 pPool->pszAccessHandler);
4352 AssertFatalRCSuccess(rc);
4353# ifdef PGMPOOL_WITH_CACHE
4354 pgmPoolHashInsert(pPool, pPage);
4355# endif
4356 }
4357#endif
4358#ifdef PGMPOOL_WITH_USER_TRACKING
4359 Assert(pPage->iUserHead == NIL_PGMPOOL_USER_INDEX); /* for now */
4360#endif
4361#ifdef PGMPOOL_WITH_CACHE
4362 Assert(pPage->iAgeNext == NIL_PGMPOOL_IDX);
4363 Assert(pPage->iAgePrev == NIL_PGMPOOL_IDX);
4364#endif
4365 }
4366
4367 for (unsigned i=0;i<pVM->cCPUs;i++)
4368 {
4369 PVMCPU pVCpu = &pVM->aCpus[i];
4370 /*
4371 * Re-enter the shadowing mode and assert Sync CR3 FF.
4372 */
4373 pgmR3ReEnterShadowModeAfterPoolFlush(pVM, pVCpu);
4374 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
4375 }
4376
4377 STAM_PROFILE_STOP(&pPool->StatFlushAllInt, a);
4378}
4379#endif /* IN_RING3 */
4380
4381#ifdef LOG_ENABLED
4382static const char *pgmPoolPoolKindToStr(uint8_t enmKind)
4383{
4384 switch(enmKind)
4385 {
4386 case PGMPOOLKIND_INVALID:
4387 return "PGMPOOLKIND_INVALID";
4388 case PGMPOOLKIND_FREE:
4389 return "PGMPOOLKIND_FREE";
4390 case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
4391 return "PGMPOOLKIND_32BIT_PT_FOR_PHYS";
4392 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
4393 return "PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT";
4394 case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
4395 return "PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB";
4396 case PGMPOOLKIND_PAE_PT_FOR_PHYS:
4397 return "PGMPOOLKIND_PAE_PT_FOR_PHYS";
4398 case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
4399 return "PGMPOOLKIND_PAE_PT_FOR_32BIT_PT";
4400 case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
4401 return "PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB";
4402 case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
4403 return "PGMPOOLKIND_PAE_PT_FOR_PAE_PT";
4404 case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
4405 return "PGMPOOLKIND_PAE_PT_FOR_PAE_2MB";
4406 case PGMPOOLKIND_32BIT_PD:
4407 return "PGMPOOLKIND_32BIT_PD";
4408 case PGMPOOLKIND_32BIT_PD_PHYS:
4409 return "PGMPOOLKIND_32BIT_PD_PHYS";
4410 case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
4411 return "PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD";
4412 case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
4413 return "PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD";
4414 case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
4415 return "PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD";
4416 case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
4417 return "PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD";
4418 case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
4419 return "PGMPOOLKIND_PAE_PD_FOR_PAE_PD";
4420 case PGMPOOLKIND_PAE_PD_PHYS:
4421 return "PGMPOOLKIND_PAE_PD_PHYS";
4422 case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
4423 return "PGMPOOLKIND_PAE_PDPT_FOR_32BIT";
4424 case PGMPOOLKIND_PAE_PDPT:
4425 return "PGMPOOLKIND_PAE_PDPT";
4426 case PGMPOOLKIND_PAE_PDPT_PHYS:
4427 return "PGMPOOLKIND_PAE_PDPT_PHYS";
4428 case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
4429 return "PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT";
4430 case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
4431 return "PGMPOOLKIND_64BIT_PDPT_FOR_PHYS";
4432 case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
4433 return "PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD";
4434 case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
4435 return "PGMPOOLKIND_64BIT_PD_FOR_PHYS";
4436 case PGMPOOLKIND_64BIT_PML4:
4437 return "PGMPOOLKIND_64BIT_PML4";
4438 case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
4439 return "PGMPOOLKIND_EPT_PDPT_FOR_PHYS";
4440 case PGMPOOLKIND_EPT_PD_FOR_PHYS:
4441 return "PGMPOOLKIND_EPT_PD_FOR_PHYS";
4442 case PGMPOOLKIND_EPT_PT_FOR_PHYS:
4443 return "PGMPOOLKIND_EPT_PT_FOR_PHYS";
4444 case PGMPOOLKIND_ROOT_NESTED:
4445 return "PGMPOOLKIND_ROOT_NESTED";
4446 }
4447 return "Unknown kind!";
4448}
4449#endif /* LOG_ENABLED*/
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette