alloc-ef-r0drv.cpp@ 83546

Last change on this file since 83546 was 83546, checked in by vboxsync, 5 years ago
IPRT: Added RTMemFreeZ, RTMemTmpFreeZ, and RTMemEf* variants. bugref:9698
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 30.9 KB

Line
1	/* $Id: alloc-ef-r0drv.cpp 83546 2020-04-04 10:46:18Z vboxsync $ */
2	/** @file
3	* IPRT - Memory Allocation, electric fence for ring-0 drivers.
4	*/
5
6	/*
7	* Copyright (C) 2006-2020 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.215389.xyz. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* The contents of this file may alternatively be used under the terms
18	* of the Common Development and Distribution License Version 1.0
19	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	* VirtualBox OSE distribution, in which case the provisions of the
21	* CDDL are applicable instead of those of the GPL.
22	*
23	* You may elect to license modified versions of this file under the
24	* terms and conditions of either the GPL or the CDDL or both.
25	*/
26
27
28	/*********************************************************************************************************************************
29	* Header Files *
30	*********************************************************************************************************************************/
31	#define RTMEM_NO_WRAP_TO_EF_APIS
32	#include "internal/iprt.h"
33	#include <iprt/mem.h>
34
35	#include <iprt/alloc.h>
36	#include <iprt/asm.h>
37	#include <iprt/asm-amd64-x86.h>
38	#include <iprt/assert.h>
39	#include <iprt/errcore.h>
40	#include <iprt/log.h>
41	#include <iprt/memobj.h>
42	#include <iprt/param.h>
43	#include <iprt/string.h>
44	#include <iprt/thread.h>
45
46	#include "internal/mem.h"
47
48
49	/*********************************************************************************************************************************
50	* Defined Constants And Macros *
51	*********************************************************************************************************************************/
52	#if defined(DOXYGEN_RUNNING)
53	# define RTR0MEM_EF_IN_FRONT
54	#endif
55
56	/** @def RTR0MEM_EF_SIZE
57	* The size of the fence. This must be page aligned.
58	*/
59	#define RTR0MEM_EF_SIZE PAGE_SIZE
60
61	/** @def RTR0MEM_EF_ALIGNMENT
62	* The allocation alignment, power of two of course.
63	*
64	* Use this for working around misaligned sizes, usually stemming from
65	* allocating a string or something after the main structure. When you
66	* encounter this, please fix the allocation to RTMemAllocVar or RTMemAllocZVar.
67	*/
68	#if 0
69	# define RTR0MEM_EF_ALIGNMENT (ARCH_BITS / 8)
70	#else
71	# define RTR0MEM_EF_ALIGNMENT 1
72	#endif
73
74	/** @def RTR0MEM_EF_IN_FRONT
75	* Define this to put the fence up in front of the block.
76	* The default (when this isn't defined) is to up it up after the block.
77	*/
78	//# define RTR0MEM_EF_IN_FRONT
79
80	/** @def RTR0MEM_EF_FREE_DELAYED
81	* This define will enable free() delay and protection of the freed data
82	* while it's being delayed. The value of RTR0MEM_EF_FREE_DELAYED defines
83	* the threshold of the delayed blocks.
84	* Delayed blocks does not consume any physical memory, only virtual address space.
85	*/
86	#define RTR0MEM_EF_FREE_DELAYED (20 * _1M)
87
88	/** @def RTR0MEM_EF_FREE_FILL
89	* This define will enable memset(,RTR0MEM_EF_FREE_FILL,)'ing the user memory
90	* in the block before freeing/decommitting it. This is useful in GDB since GDB
91	* appears to be able to read the content of the page even after it's been
92	* decommitted.
93	*/
94	#define RTR0MEM_EF_FREE_FILL 'f'
95
96	/** @def RTR0MEM_EF_FILLER
97	* This define will enable memset(,RTR0MEM_EF_FILLER,)'ing the allocated
98	* memory when the API doesn't require it to be zero'd.
99	*/
100	#define RTR0MEM_EF_FILLER 0xef
101
102	/** @def RTR0MEM_EF_NOMAN_FILLER
103	* This define will enable memset(,RTR0MEM_EF_NOMAN_FILLER,)'ing the
104	* unprotected but not allocated area of memory, the so called no man's land.
105	*/
106	#define RTR0MEM_EF_NOMAN_FILLER 0xaa
107
108	/** @def RTR0MEM_EF_FENCE_FILLER
109	* This define will enable memset(,RTR0MEM_EF_FENCE_FILLER,)'ing the
110	* fence itself, as debuggers can usually read them.
111	*/
112	#define RTR0MEM_EF_FENCE_FILLER 0xcc
113
114
115	/*********************************************************************************************************************************
116	* Header Files *
117	*********************************************************************************************************************************/
118	#ifdef RT_OS_WINDOWS
119	# include <iprt/win/windows.h>
120	#elif !defined(RT_OS_FREEBSD)
121	# include <sys/mman.h>
122	#endif
123	#include <iprt/avl.h>
124	#include <iprt/thread.h>
125
126
127	/*********************************************************************************************************************************
128	* Structures and Typedefs *
129	*********************************************************************************************************************************/
130	/**
131	* Allocation types.
132	*/
133	typedef enum RTMEMTYPE
134	{
135	RTMEMTYPE_RTMEMALLOC,
136	RTMEMTYPE_RTMEMALLOCZ,
137	RTMEMTYPE_RTMEMREALLOC,
138	RTMEMTYPE_RTMEMFREE,
139	RTMEMTYPE_RTMEMFREEZ,
140
141	RTMEMTYPE_NEW,
142	RTMEMTYPE_NEW_ARRAY,
143	RTMEMTYPE_DELETE,
144	RTMEMTYPE_DELETE_ARRAY
145	} RTMEMTYPE;
146
147	/**
148	* Node tracking a memory allocation.
149	*/
150	typedef struct RTR0MEMEFBLOCK
151	{
152	/** Avl node code, key is the user block pointer. */
153	AVLPVNODECORE Core;
154	/** Allocation type. */
155	RTMEMTYPE enmType;
156	/** The memory object. */
157	RTR0MEMOBJ hMemObj;
158	/** The unaligned size of the block. */
159	size_t cbUnaligned;
160	/** The aligned size of the block. */
161	size_t cbAligned;
162	/** The allocation tag (read-only string). */
163	const char *pszTag;
164	/** The return address of the allocator function. */
165	void *pvCaller;
166	/** Line number of the alloc call. */
167	unsigned iLine;
168	/** File from within the allocation was made. */
169	const char *pszFile;
170	/** Function from within the allocation was made. */
171	const char *pszFunction;
172	} RTR0MEMEFBLOCK, *PRTR0MEMEFBLOCK;
173
174
175
176	/*********************************************************************************************************************************
177	* Global Variables *
178	*********************************************************************************************************************************/
179	/** Spinlock protecting the all the block's globals. */
180	static volatile uint32_t g_BlocksLock;
181	/** Tree tracking the allocations. */
182	static AVLPVTREE g_BlocksTree;
183
184	#ifdef RTR0MEM_EF_FREE_DELAYED
185	/** Tail of the delayed blocks. */
186	static volatile PRTR0MEMEFBLOCK g_pBlocksDelayHead;
187	/** Tail of the delayed blocks. */
188	static volatile PRTR0MEMEFBLOCK g_pBlocksDelayTail;
189	/** Number of bytes in the delay list (includes fences). */
190	static volatile size_t g_cbBlocksDelay;
191	#endif /* RTR0MEM_EF_FREE_DELAYED */
192
193	/** Array of pointers free watches for. */
194	void *gapvRTMemFreeWatch[4] = {NULL, NULL, NULL, NULL};
195	/** Enable logging of all freed memory. */
196	bool gfRTMemFreeLog = false;
197
198
199	/*********************************************************************************************************************************
200	* Internal Functions *
201	*********************************************************************************************************************************/
202
203
204	/**
205	* @callback_method_impl{FNRTSTROUTPUT}
206	*/
207	static DECLCALLBACK(size_t) rtR0MemEfWrite(void pvArg, const char pachChars, size_t cbChars)
208	{
209	RT_NOREF1(pvArg);
210	if (cbChars)
211	{
212	RTLogWriteDebugger(pachChars, cbChars);
213	RTLogWriteStdOut(pachChars, cbChars);
214	RTLogWriteUser(pachChars, cbChars);
215	}
216	return cbChars;
217	}
218
219
220	/**
221	* Complains about something.
222	*/
223	static void rtR0MemComplain(const char pszOp, const char pszFormat, ...)
224	{
225	va_list args;
226	RTStrFormat(rtR0MemEfWrite, NULL, NULL, NULL, "RTMem error: %s: ", pszOp);
227	va_start(args, pszFormat);
228	RTStrFormatV(rtR0MemEfWrite, NULL, NULL, NULL, pszFormat, args);
229	va_end(args);
230	RTAssertDoPanic();
231	}
232
233	/**
234	* Log an event.
235	*/
236	DECLINLINE(void) rtR0MemLog(const char pszOp, const char pszFormat, ...)
237	{
238	#if 0
239	va_list args;
240	RTStrFormat(rtR0MemEfWrite, NULL, NULL, NULL, "RTMem info: %s: ", pszOp);
241	va_start(args, pszFormat);
242	RTStrFormatV(rtR0MemEfWrite, NULL, NULL, NULL, pszFormat, args);
243	va_end(args);
244	#else
245	NOREF(pszOp); NOREF(pszFormat);
246	#endif
247	}
248
249
250
251	/**
252	* Acquires the lock.
253	*/
254	DECLINLINE(RTCCUINTREG) rtR0MemBlockLock(void)
255	{
256	RTCCUINTREG uRet;
257	unsigned c = 0;
258	if (RTThreadPreemptIsEnabled(NIL_RTTHREAD))
259	{
260	for (;;)
261	{
262	uRet = ASMIntDisableFlags();
263	if (ASMAtomicCmpXchgU32(&g_BlocksLock, 1, 0))
264	break;
265	ASMSetFlags(uRet);
266	RTThreadSleepNoLog(((++c) >> 2) & 31);
267	}
268	}
269	else
270	{
271	for (;;)
272	{
273	uRet = ASMIntDisableFlags();
274	if (ASMAtomicCmpXchgU32(&g_BlocksLock, 1, 0))
275	break;
276	ASMSetFlags(uRet);
277	ASMNopPause();
278	if (++c & 3)
279	ASMNopPause();
280	}
281	}
282	return uRet;
283	}
284
285
286	/**
287	* Releases the lock.
288	*/
289	DECLINLINE(void) rtR0MemBlockUnlock(RTCCUINTREG fSavedIntFlags)
290	{
291	Assert(g_BlocksLock == 1);
292	ASMAtomicXchgU32(&g_BlocksLock, 0);
293	ASMSetFlags(fSavedIntFlags);
294	}
295
296
297	/**
298	* Creates a block.
299	*/
300	DECLINLINE(PRTR0MEMEFBLOCK) rtR0MemBlockCreate(RTMEMTYPE enmType, size_t cbUnaligned, size_t cbAligned,
301	const char pszTag, void pvCaller, RT_SRC_POS_DECL)
302	{
303	PRTR0MEMEFBLOCK pBlock = (PRTR0MEMEFBLOCK)RTMemAlloc(sizeof(*pBlock));
304	if (pBlock)
305	{
306	pBlock->enmType = enmType;
307	pBlock->cbUnaligned = cbUnaligned;
308	pBlock->cbAligned = cbAligned;
309	pBlock->pszTag = pszTag;
310	pBlock->pvCaller = pvCaller;
311	pBlock->iLine = iLine;
312	pBlock->pszFile = pszFile;
313	pBlock->pszFunction = pszFunction;
314	}
315	return pBlock;
316	}
317
318
319	/**
320	* Frees a block.
321	*/
322	DECLINLINE(void) rtR0MemBlockFree(PRTR0MEMEFBLOCK pBlock)
323	{
324	RTMemFree(pBlock);
325	}
326
327
328	/**
329	* Insert a block from the tree.
330	*/
331	DECLINLINE(void) rtR0MemBlockInsert(PRTR0MEMEFBLOCK pBlock, void *pv, RTR0MEMOBJ hMemObj)
332	{
333	pBlock->Core.Key = pv;
334	pBlock->hMemObj = hMemObj;
335	RTCCUINTREG fSavedIntFlags = rtR0MemBlockLock();
336	bool fRc = RTAvlPVInsert(&g_BlocksTree, &pBlock->Core);
337	rtR0MemBlockUnlock(fSavedIntFlags);
338	AssertRelease(fRc);
339	}
340
341
342	/**
343	* Remove a block from the tree and returns it to the caller.
344	*/
345	DECLINLINE(PRTR0MEMEFBLOCK) rtR0MemBlockRemove(void *pv)
346	{
347	RTCCUINTREG fSavedIntFlags = rtR0MemBlockLock();
348	PRTR0MEMEFBLOCK pBlock = (PRTR0MEMEFBLOCK)RTAvlPVRemove(&g_BlocksTree, pv);
349	rtR0MemBlockUnlock(fSavedIntFlags);
350	return pBlock;
351	}
352
353
354	/**
355	* Gets a block.
356	*/
357	DECLINLINE(PRTR0MEMEFBLOCK) rtR0MemBlockGet(void *pv)
358	{
359	RTCCUINTREG fSavedIntFlags = rtR0MemBlockLock();
360	PRTR0MEMEFBLOCK pBlock = (PRTR0MEMEFBLOCK)RTAvlPVGet(&g_BlocksTree, pv);
361	rtR0MemBlockUnlock(fSavedIntFlags);
362	return pBlock;
363	}
364
365
366	/**
367	* Dumps one allocation.
368	*/
369	static DECLCALLBACK(int) RTMemDumpOne(PAVLPVNODECORE pNode, void *pvUser)
370	{
371	PRTR0MEMEFBLOCK pBlock = (PRTR0MEMEFBLOCK)pNode;
372	RTStrFormat(rtR0MemEfWrite, NULL, NULL, NULL, "%p %08lx(+%02lx) %p\n",
373	pBlock->Core.Key,
374	(unsigned long)pBlock->cbUnaligned,
375	(unsigned long)(pBlock->cbAligned - pBlock->cbUnaligned),
376	pBlock->pvCaller);
377	NOREF(pvUser);
378	return 0;
379	}
380
381
382	/**
383	* Dumps the allocated blocks.
384	* This is something which you should call from gdb.
385	*/
386	RT_C_DECLS_BEGIN
387	void RTMemDump(void);
388	RT_C_DECLS_END
389
390	void RTMemDump(void)
391	{
392	RTStrFormat(rtR0MemEfWrite, NULL, NULL, NULL, "address size(alg) caller\n");
393	RTAvlPVDoWithAll(&g_BlocksTree, true, RTMemDumpOne, NULL);
394	}
395
396	#ifdef RTR0MEM_EF_FREE_DELAYED
397
398	/**
399	* Insert a delayed block.
400	*/
401	DECLINLINE(void) rtR0MemBlockDelayInsert(PRTR0MEMEFBLOCK pBlock)
402	{
403	size_t cbBlock = RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) + RTR0MEM_EF_SIZE;
404	pBlock->Core.pRight = NULL;
405	pBlock->Core.pLeft = NULL;
406	RTCCUINTREG fSavedIntFlags = rtR0MemBlockLock();
407	if (g_pBlocksDelayHead)
408	{
409	g_pBlocksDelayHead->Core.pLeft = (PAVLPVNODECORE)pBlock;
410	pBlock->Core.pRight = (PAVLPVNODECORE)g_pBlocksDelayHead;
411	g_pBlocksDelayHead = pBlock;
412	}
413	else
414	{
415	g_pBlocksDelayTail = pBlock;
416	g_pBlocksDelayHead = pBlock;
417	}
418	g_cbBlocksDelay += cbBlock;
419	rtR0MemBlockUnlock(fSavedIntFlags);
420	}
421
422	/**
423	* Removes a delayed block.
424	*/
425	DECLINLINE(PRTR0MEMEFBLOCK) rtR0MemBlockDelayRemove(void)
426	{
427	PRTR0MEMEFBLOCK pBlock = NULL;
428	RTCCUINTREG fSavedIntFlags = rtR0MemBlockLock();
429	if (g_cbBlocksDelay > RTR0MEM_EF_FREE_DELAYED)
430	{
431	pBlock = g_pBlocksDelayTail;
432	if (pBlock)
433	{
434	g_pBlocksDelayTail = (PRTR0MEMEFBLOCK)pBlock->Core.pLeft;
435	if (pBlock->Core.pLeft)
436	pBlock->Core.pLeft->pRight = NULL;
437	else
438	g_pBlocksDelayHead = NULL;
439	g_cbBlocksDelay -= RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) + RTR0MEM_EF_SIZE;
440	}
441	}
442	rtR0MemBlockUnlock(fSavedIntFlags);
443	return pBlock;
444	}
445
446	#endif /* RTR0MEM_EF_FREE_DELAYED */
447
448
449	static void rtR0MemFreeBlock(PRTR0MEMEFBLOCK pBlock, const char *pszOp)
450	{
451	void *pv = pBlock->Core.Key;
452	# ifdef RTR0MEM_EF_IN_FRONT
453	void pvBlock = (char )pv - RTR0MEM_EF_SIZE;
454	# else
455	void pvBlock = (void )((uintptr_t)pv & ~(uintptr_t)PAGE_OFFSET_MASK);
456	# endif
457	size_t cbBlock = RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) + RTR0MEM_EF_SIZE;
458
459	int rc = RTR0MemObjProtect(pBlock->hMemObj, 0 /offSub/, RT_ALIGN_Z(cbBlock, PAGE_SIZE), RTMEM_PROT_READ \| RTMEM_PROT_WRITE);
460	if (RT_FAILURE(rc))
461	rtR0MemComplain(pszOp, "RTR0MemObjProtect([%p], 0, %#x, RTMEM_PROT_READ \| RTMEM_PROT_WRITE) -> %Rrc\n",
462	pvBlock, cbBlock, rc);
463
464	rc = RTR0MemObjFree(pBlock->hMemObj, true /fFreeMappings/);
465	if (RT_FAILURE(rc))
466	rtR0MemComplain(pszOp, "RTR0MemObjFree([%p LB %#x]) -> %Rrc\n", pvBlock, cbBlock, rc);
467	pBlock->hMemObj = NIL_RTR0MEMOBJ;
468
469	rtR0MemBlockFree(pBlock);
470	}
471
472
473	/**
474	* Initialize call, we shouldn't fail here.
475	*/
476	void rtR0MemEfInit(void)
477	{
478
479	}
480
481	/**
482	* @callback_method_impl{AVLPVCALLBACK}
483	*/
484	static DECLCALLBACK(int) rtR0MemEfDestroyBlock(PAVLPVNODECORE pNode, void *pvUser)
485	{
486	PRTR0MEMEFBLOCK pBlock = (PRTR0MEMEFBLOCK)pNode;
487
488	/* Note! pszFile and pszFunction may be invalid at this point. */
489	rtR0MemComplain("rtR0MemEfDestroyBlock", "Leaking %zu bytes at %p (iLine=%u pvCaller=%p)\n",
490	pBlock->cbAligned, pBlock->Core.Key, pBlock->iLine, pBlock->pvCaller);
491
492	rtR0MemFreeBlock(pBlock, "rtR0MemEfDestroyBlock");
493
494	NOREF(pvUser);
495	return VINF_SUCCESS;
496	}
497
498
499	/**
500	* Termination call.
501	*
502	* Will check and free memory.
503	*/
504	void rtR0MemEfTerm(void)
505	{
506	#ifdef RTR0MEM_EF_FREE_DELAYED
507	/*
508	* Release delayed frees.
509	*/
510	RTCCUINTREG fSavedIntFlags = rtR0MemBlockLock();
511	for (;;)
512	{
513	PRTR0MEMEFBLOCK pBlock = g_pBlocksDelayTail;
514	if (pBlock)
515	{
516	g_pBlocksDelayTail = (PRTR0MEMEFBLOCK)pBlock->Core.pLeft;
517	if (pBlock->Core.pLeft)
518	pBlock->Core.pLeft->pRight = NULL;
519	else
520	g_pBlocksDelayHead = NULL;
521	rtR0MemBlockUnlock(fSavedIntFlags);
522
523	rtR0MemFreeBlock(pBlock, "rtR0MemEfTerm");
524
525	rtR0MemBlockLock();
526	}
527	else
528	break;
529	}
530	g_cbBlocksDelay = 0;
531	rtR0MemBlockUnlock(fSavedIntFlags);
532	#endif
533
534	/*
535	* Complain about leaks. Then release them.
536	*/
537	RTAvlPVDestroy(&g_BlocksTree, rtR0MemEfDestroyBlock, NULL);
538	}
539
540
541	/**
542	* Internal allocator.
543	*/
544	static void * rtR0MemAlloc(const char *pszOp, RTMEMTYPE enmType, size_t cbUnaligned, size_t cbAligned,
545	const char pszTag, void pvCaller, RT_SRC_POS_DECL)
546	{
547	/*
548	* Sanity.
549	*/
550	if ( RT_ALIGN_Z(RTR0MEM_EF_SIZE, PAGE_SIZE) != RTR0MEM_EF_SIZE
551	&& RTR0MEM_EF_SIZE <= 0)
552	{
553	rtR0MemComplain(pszOp, "Invalid E-fence size! %#x\n", RTR0MEM_EF_SIZE);
554	return NULL;
555	}
556	if (!cbUnaligned)
557	{
558	#if 1
559	rtR0MemComplain(pszOp, "Request of ZERO bytes allocation!\n");
560	return NULL;
561	#else
562	cbAligned = cbUnaligned = 1;
563	#endif
564	}
565
566	#ifndef RTR0MEM_EF_IN_FRONT
567	/* Alignment decreases fence accuracy, but this is at least partially
568	* counteracted by filling and checking the alignment padding. When the
569	* fence is in front then then no extra alignment is needed. */
570	cbAligned = RT_ALIGN_Z(cbAligned, RTR0MEM_EF_ALIGNMENT);
571	#endif
572
573	/*
574	* Allocate the trace block.
575	*/
576	PRTR0MEMEFBLOCK pBlock = rtR0MemBlockCreate(enmType, cbUnaligned, cbAligned, pszTag, pvCaller, RT_SRC_POS_ARGS);
577	if (!pBlock)
578	{
579	rtR0MemComplain(pszOp, "Failed to allocate trace block!\n");
580	return NULL;
581	}
582
583	/*
584	* Allocate a block with page alignment space + the size of the E-fence.
585	*/
586	void *pvBlock = NULL;
587	RTR0MEMOBJ hMemObj;
588	size_t cbBlock = RT_ALIGN_Z(cbAligned, PAGE_SIZE) + RTR0MEM_EF_SIZE;
589	int rc = RTR0MemObjAllocPage(&hMemObj, cbBlock, false /fExecutable/);
590	if (RT_SUCCESS(rc))
591	pvBlock = RTR0MemObjAddress(hMemObj);
592	if (pvBlock)
593	{
594	/*
595	* Calc the start of the fence and the user block
596	* and then change the page protection of the fence.
597	*/
598	#ifdef RTR0MEM_EF_IN_FRONT
599	void *pvEFence = pvBlock;
600	void pv = (char )pvEFence + RTR0MEM_EF_SIZE;
601	# ifdef RTR0MEM_EF_NOMAN_FILLER
602	memset((char *)pv + cbUnaligned, RTR0MEM_EF_NOMAN_FILLER, cbBlock - RTR0MEM_EF_SIZE - cbUnaligned);
603	# endif
604	#else
605	void pvEFence = (char )pvBlock + (cbBlock - RTR0MEM_EF_SIZE);
606	void pv = (char )pvEFence - cbAligned;
607	# ifdef RTR0MEM_EF_NOMAN_FILLER
608	memset(pvBlock, RTR0MEM_EF_NOMAN_FILLER, cbBlock - RTR0MEM_EF_SIZE - cbAligned);
609	memset((char *)pv + cbUnaligned, RTR0MEM_EF_NOMAN_FILLER, cbAligned - cbUnaligned);
610	# endif
611	#endif
612
613	#ifdef RTR0MEM_EF_FENCE_FILLER
614	memset(pvEFence, RTR0MEM_EF_FENCE_FILLER, RTR0MEM_EF_SIZE);
615	#endif
616	rc = RTR0MemObjProtect(hMemObj, (uint8_t )pvEFence - (uint8_t )pvBlock, RTR0MEM_EF_SIZE, RTMEM_PROT_NONE);
617	if (!rc)
618	{
619	rtR0MemBlockInsert(pBlock, pv, hMemObj);
620	if (enmType == RTMEMTYPE_RTMEMALLOCZ)
621	memset(pv, 0, cbUnaligned);
622	#ifdef RTR0MEM_EF_FILLER
623	else
624	memset(pv, RTR0MEM_EF_FILLER, cbUnaligned);
625	#endif
626
627	rtR0MemLog(pszOp, "returns %p (pvBlock=%p cbBlock=%#x pvEFence=%p cbUnaligned=%#x)\n", pv, pvBlock, cbBlock, pvEFence, cbUnaligned);
628	return pv;
629	}
630	rtR0MemComplain(pszOp, "RTMemProtect failed, pvEFence=%p size %d, rc=%d\n", pvEFence, RTR0MEM_EF_SIZE, rc);
631	RTR0MemObjFree(hMemObj, true /fFreeMappings/);
632	}
633	else
634	{
635	rtR0MemComplain(pszOp, "Failed to allocated %zu (%zu) bytes (rc=%Rrc).\n", cbBlock, cbUnaligned, rc);
636	if (RT_SUCCESS(rc))
637	RTR0MemObjFree(hMemObj, true /fFreeMappings/);
638	}
639
640	rtR0MemBlockFree(pBlock);
641	return NULL;
642	}
643
644
645	/**
646	* Internal free.
647	*/
648	static void rtR0MemFree(const char pszOp, RTMEMTYPE enmType, void pv, size_t cbUser, void *pvCaller, RT_SRC_POS_DECL)
649	{
650	NOREF(enmType); RT_SRC_POS_NOREF();
651
652	/*
653	* Simple case.
654	*/
655	if (!pv)
656	return;
657
658	/*
659	* Check watch points.
660	*/
661	for (unsigned i = 0; i < RT_ELEMENTS(gapvRTMemFreeWatch); i++)
662	if (gapvRTMemFreeWatch[i] == pv)
663	RTAssertDoPanic();
664
665	/*
666	* Find the block.
667	*/
668	PRTR0MEMEFBLOCK pBlock = rtR0MemBlockRemove(pv);
669	if (pBlock)
670	{
671	if (gfRTMemFreeLog)
672	RTLogPrintf("RTMem %s: pv=%p pvCaller=%p cbUnaligned=%#x\n", pszOp, pv, pvCaller, pBlock->cbUnaligned);
673
674	#ifdef RTR0MEM_EF_NOMAN_FILLER
675	/*
676	* Check whether the no man's land is untouched.
677	*/
678	# ifdef RTR0MEM_EF_IN_FRONT
679	void pvWrong = ASMMemFirstMismatchingU8((char )pv + pBlock->cbUnaligned,
680	RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) - pBlock->cbUnaligned,
681	RTR0MEM_EF_NOMAN_FILLER);
682	# else
683	/* Alignment must match allocation alignment in rtMemAlloc(). */
684	void pvWrong = ASMMemFirstMismatchingU8((char )pv + pBlock->cbUnaligned,
685	pBlock->cbAligned - pBlock->cbUnaligned,
686	RTR0MEM_EF_NOMAN_FILLER);
687	if (pvWrong)
688	RTAssertDoPanic();
689	pvWrong = ASMMemFirstMismatchingU8((void *)((uintptr_t)pv & ~(uintptr_t)PAGE_OFFSET_MASK),
690	RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE) - pBlock->cbAligned,
691	RTR0MEM_EF_NOMAN_FILLER);
692	# endif
693	if (pvWrong)
694	RTAssertDoPanic();
695	#endif
696
697	/*
698	* Fill the user part of the block.
699	*/
700	AssertMsg(enmType != RTMEMTYPE_RTMEMFREEZ \|\| cbUser == pBlock->cbUnaligned,
701	("cbUser=%#zx cbUnaligned=%#zx\n", cbUser, pBlock->cbUnaligned));
702	if (enmType == RTMEMTYPE_RTMEMFREEZ)
703	RT_BZERO(pv, pBlock->cbUnaligned);
704	#ifdef RTR0MEM_EF_FREE_FILL
705	else
706	memset(pv, RTR0MEM_EF_FREE_FILL, pBlock->cbUnaligned);
707	#endif
708
709	#if defined(RTR0MEM_EF_FREE_DELAYED) && RTR0MEM_EF_FREE_DELAYED > 0
710	/*
711	* We're doing delayed freeing.
712	* That means we'll expand the E-fence to cover the entire block.
713	*/
714	int rc = RTR0MemObjProtect(pBlock->hMemObj,
715	# ifdef RTR0MEM_EF_IN_FRONT
716	RTR0MEM_EF_SIZE,
717	# else
718	0 /offSub/,
719	# endif
720	RT_ALIGN_Z(pBlock->cbAligned, PAGE_SIZE),
721	RTMEM_PROT_NONE);
722	if (RT_SUCCESS(rc))
723	{
724	/*
725	* Insert it into the free list and process pending frees.
726	*/
727	rtR0MemBlockDelayInsert(pBlock);
728	while ((pBlock = rtR0MemBlockDelayRemove()) != NULL)
729	rtR0MemFreeBlock(pBlock, pszOp);
730	}
731	else
732	rtR0MemComplain(pszOp, "Failed to expand the efence of pv=%p cb=%d, rc=%d.\n", pv, pBlock, rc);
733
734	#else /* !RTR0MEM_EF_FREE_DELAYED */
735	rtR0MemFreeBlock(pBlock, pszOp);
736	#endif /* !RTR0MEM_EF_FREE_DELAYED */
737	}
738	else
739	rtR0MemComplain(pszOp, "pv=%p not found! Incorrect free!\n", pv);
740	}
741
742
743	/**
744	* Internal realloc.
745	*/
746	static void rtR0MemRealloc(const char pszOp, RTMEMTYPE enmType, void *pvOld, size_t cbNew,
747	const char pszTag, void pvCaller, RT_SRC_POS_DECL)
748	{
749	/*
750	* Allocate new and copy.
751	*/
752	if (!pvOld)
753	return rtR0MemAlloc(pszOp, enmType, cbNew, cbNew, pszTag, pvCaller, RT_SRC_POS_ARGS);
754	if (!cbNew)
755	{
756	rtR0MemFree(pszOp, RTMEMTYPE_RTMEMREALLOC, pvOld, 0, pvCaller, RT_SRC_POS_ARGS);
757	return NULL;
758	}
759
760	/*
761	* Get the block, allocate the new, copy the data, free the old one.
762	*/
763	PRTR0MEMEFBLOCK pBlock = rtR0MemBlockGet(pvOld);
764	if (pBlock)
765	{
766	void *pvRet = rtR0MemAlloc(pszOp, enmType, cbNew, cbNew, pszTag, pvCaller, RT_SRC_POS_ARGS);
767	if (pvRet)
768	{
769	memcpy(pvRet, pvOld, RT_MIN(cbNew, pBlock->cbUnaligned));
770	rtR0MemFree(pszOp, RTMEMTYPE_RTMEMREALLOC, pvOld, 0, pvCaller, RT_SRC_POS_ARGS);
771	}
772	return pvRet;
773	}
774	rtR0MemComplain(pszOp, "pvOld=%p was not found!\n", pvOld);
775	return NULL;
776	}
777
778
779
780
781	RTDECL(void ) RTMemEfTmpAlloc(size_t cb, const char pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
782	{
783	return rtR0MemAlloc("TmpAlloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
784	}
785
786
787	RTDECL(void ) RTMemEfTmpAllocZ(size_t cb, const char pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
788	{
789	return rtR0MemAlloc("TmpAlloc", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
790	}
791
792
793	RTDECL(void) RTMemEfTmpFree(void *pv, RT_SRC_POS_DECL) RT_NO_THROW_DEF
794	{
795	if (pv)
796	rtR0MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, 0, ASMReturnAddress(), RT_SRC_POS_ARGS);
797	}
798
799
800	RTDECL(void) RTMemEfTmpFreeZ(void *pv, size_t cb, RT_SRC_POS_DECL) RT_NO_THROW_DEF
801	{
802	if (pv)
803	rtR0MemFree("FreeZ", RTMEMTYPE_RTMEMFREEZ, pv, cb, ASMReturnAddress(), RT_SRC_POS_ARGS);
804	}
805
806
807	RTDECL(void ) RTMemEfAlloc(size_t cb, const char pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
808	{
809	return rtR0MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
810	}
811
812
813	RTDECL(void ) RTMemEfAllocZ(size_t cb, const char pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
814	{
815	return rtR0MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
816	}
817
818
819	RTDECL(void ) RTMemEfAllocVar(size_t cbUnaligned, const char pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
820	{
821	size_t cbAligned;
822	if (cbUnaligned >= 16)
823	cbAligned = RT_ALIGN_Z(cbUnaligned, 16);
824	else
825	cbAligned = RT_ALIGN_Z(cbUnaligned, sizeof(void *));
826	return rtR0MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
827	}
828
829
830	RTDECL(void ) RTMemEfAllocZVar(size_t cbUnaligned, const char pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
831	{
832	size_t cbAligned;
833	if (cbUnaligned >= 16)
834	cbAligned = RT_ALIGN_Z(cbUnaligned, 16);
835	else
836	cbAligned = RT_ALIGN_Z(cbUnaligned, sizeof(void *));
837	return rtR0MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
838	}
839
840
841	RTDECL(void ) RTMemEfRealloc(void pvOld, size_t cbNew, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
842	{
843	return rtR0MemRealloc("Realloc", RTMEMTYPE_RTMEMREALLOC, pvOld, cbNew, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
844	}
845
846	RTDECL(void ) RTMemEfReallocZ(void pvOld, size_t cbOld, size_t cbNew, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
847	{
848	void *pvDst = rtR0MemRealloc("Realloc", RTMEMTYPE_RTMEMREALLOC, pvOld, cbNew, pszTag, ASMReturnAddress(), RT_SRC_POS_ARGS);
849	if (pvDst && cbNew > cbOld)
850	memset((uint8_t *)pvDst + cbOld, 0, cbNew - cbOld);
851	return pvDst;
852	}
853
854
855	RTDECL(void) RTMemEfFree(void *pv, RT_SRC_POS_DECL) RT_NO_THROW_DEF
856	{
857	if (pv)
858	rtR0MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, 0, ASMReturnAddress(), RT_SRC_POS_ARGS);
859	}
860
861
862	RTDECL(void) RTMemEfFreeZ(void *pv, size_t cb, RT_SRC_POS_DECL) RT_NO_THROW_DEF
863	{
864	if (pv)
865	rtR0MemFree("Free", RTMEMTYPE_RTMEMFREEZ, pv, cb, ASMReturnAddress(), RT_SRC_POS_ARGS);
866	}
867
868
869	RTDECL(void ) RTMemEfDup(const void pvSrc, size_t cb, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
870	{
871	void *pvDst = RTMemEfAlloc(cb, pszTag, RT_SRC_POS_ARGS);
872	if (pvDst)
873	memcpy(pvDst, pvSrc, cb);
874	return pvDst;
875	}
876
877
878	RTDECL(void ) RTMemEfDupEx(const void pvSrc, size_t cbSrc, size_t cbExtra, const char *pszTag, RT_SRC_POS_DECL) RT_NO_THROW_DEF
879	{
880	void *pvDst = RTMemEfAlloc(cbSrc + cbExtra, pszTag, RT_SRC_POS_ARGS);
881	if (pvDst)
882	{
883	memcpy(pvDst, pvSrc, cbSrc);
884	memset((uint8_t *)pvDst + cbSrc, 0, cbExtra);
885	}
886	return pvDst;
887	}
888
889
890
891
892	/*
893	*
894	* The NP (no position) versions.
895	*
896	*/
897
898
899
900	RTDECL(void ) RTMemEfTmpAllocNP(size_t cb, const char pszTag) RT_NO_THROW_DEF
901	{
902	return rtR0MemAlloc("TmpAlloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL);
903	}
904
905
906	RTDECL(void ) RTMemEfTmpAllocZNP(size_t cb, const char pszTag) RT_NO_THROW_DEF
907	{
908	return rtR0MemAlloc("TmpAllocZ", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL);
909	}
910
911
912	RTDECL(void) RTMemEfTmpFreeNP(void *pv) RT_NO_THROW_DEF
913	{
914	if (pv)
915	rtR0MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, 0, ASMReturnAddress(), NULL, 0, NULL);
916	}
917
918
919	RTDECL(void) RTMemEfTmpFreeZNP(void *pv, size_t cb) RT_NO_THROW_DEF
920	{
921	if (pv)
922	rtR0MemFree("FreeZ", RTMEMTYPE_RTMEMFREEZ, pv, cb, ASMReturnAddress(), NULL, 0, NULL);
923	}
924
925
926	RTDECL(void ) RTMemEfAllocNP(size_t cb, const char pszTag) RT_NO_THROW_DEF
927	{
928	return rtR0MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL);
929	}
930
931
932	RTDECL(void ) RTMemEfAllocZNP(size_t cb, const char pszTag) RT_NO_THROW_DEF
933	{
934	return rtR0MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cb, cb, pszTag, ASMReturnAddress(), NULL, 0, NULL);
935	}
936
937
938	RTDECL(void ) RTMemEfAllocVarNP(size_t cbUnaligned, const char pszTag) RT_NO_THROW_DEF
939	{
940	size_t cbAligned;
941	if (cbUnaligned >= 16)
942	cbAligned = RT_ALIGN_Z(cbUnaligned, 16);
943	else
944	cbAligned = RT_ALIGN_Z(cbUnaligned, sizeof(void *));
945	return rtR0MemAlloc("Alloc", RTMEMTYPE_RTMEMALLOC, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), NULL, 0, NULL);
946	}
947
948
949	RTDECL(void ) RTMemEfAllocZVarNP(size_t cbUnaligned, const char pszTag) RT_NO_THROW_DEF
950	{
951	size_t cbAligned;
952	if (cbUnaligned >= 16)
953	cbAligned = RT_ALIGN_Z(cbUnaligned, 16);
954	else
955	cbAligned = RT_ALIGN_Z(cbUnaligned, sizeof(void *));
956	return rtR0MemAlloc("AllocZ", RTMEMTYPE_RTMEMALLOCZ, cbUnaligned, cbAligned, pszTag, ASMReturnAddress(), NULL, 0, NULL);
957	}
958
959
960	RTDECL(void ) RTMemEfReallocNP(void pvOld, size_t cbNew, const char *pszTag) RT_NO_THROW_DEF
961	{
962	return rtR0MemRealloc("Realloc", RTMEMTYPE_RTMEMREALLOC, pvOld, cbNew, pszTag, ASMReturnAddress(), NULL, 0, NULL);
963	}
964
965
966	RTDECL(void ) RTMemEfReallocZNP(void pvOld, size_t cbOld, size_t cbNew, const char *pszTag) RT_NO_THROW_DEF
967	{
968	void *pvDst = rtR0MemRealloc("ReallocZ", RTMEMTYPE_RTMEMREALLOC, pvOld, cbNew, pszTag, ASMReturnAddress(), NULL, 0, NULL);
969	if (pvDst && cbNew > cbOld)
970	memset((uint8_t *)pvDst + cbOld, 0, cbNew - cbOld);
971	return pvDst;
972	}
973
974
975	RTDECL(void) RTMemEfFreeNP(void *pv) RT_NO_THROW_DEF
976	{
977	if (pv)
978	rtR0MemFree("Free", RTMEMTYPE_RTMEMFREE, pv, 0, ASMReturnAddress(), NULL, 0, NULL);
979	}
980
981
982	RTDECL(void) RTMemEfFreeZNP(void *pv, size_t cb) RT_NO_THROW_DEF
983	{
984	if (pv)
985	rtR0MemFree("FreeZ", RTMEMTYPE_RTMEMFREEZ, pv, cb, ASMReturnAddress(), NULL, 0, NULL);
986	}
987
988
989	RTDECL(void ) RTMemEfDupNP(const void pvSrc, size_t cb, const char *pszTag) RT_NO_THROW_DEF
990	{
991	void *pvDst = RTMemEfAlloc(cb, pszTag, NULL, 0, NULL);
992	if (pvDst)
993	memcpy(pvDst, pvSrc, cb);
994	return pvDst;
995	}
996
997
998	RTDECL(void ) RTMemEfDupExNP(const void pvSrc, size_t cbSrc, size_t cbExtra, const char *pszTag) RT_NO_THROW_DEF
999	{
1000	void *pvDst = RTMemEfAlloc(cbSrc + cbExtra, pszTag, NULL, 0, NULL);
1001	if (pvDst)
1002	{
1003	memcpy(pvDst, pvSrc, cbSrc);
1004	memset((uint8_t *)pvDst + cbSrc, 0, cbExtra);
1005	}
1006	return pvDst;
1007	}
1008

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/r0drv/alloc-ef-r0drv.cpp@ 83546

Download in other formats: