memcache.cpp@ 55253

Last change on this file since 55253 was 55253, checked in by vboxsync, 10 years ago
memcache.cpp: Enabled the free list (unused).
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 18.9 KB

Line
1	/* $Id: memcache.cpp 55253 2015-04-14 14:56:38Z vboxsync $ */
2	/** @file
3	* IPRT - Memory Object Allocation Cache.
4	*/
5
6	/*
7	* Copyright (C) 2006-2012 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.215389.xyz. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* The contents of this file may alternatively be used under the terms
18	* of the Common Development and Distribution License Version 1.0
19	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	* VirtualBox OSE distribution, in which case the provisions of the
21	* CDDL are applicable instead of those of the GPL.
22	*
23	* You may elect to license modified versions of this file under the
24	* terms and conditions of either the GPL or the CDDL or both.
25	*/
26
27
28	/*******************************************************************************
29	* Header Files *
30	*******************************************************************************/
31	#include <iprt/memcache.h>
32	#include "internal/iprt.h"
33
34	#include <iprt/assert.h>
35	#include <iprt/asm.h>
36	#include <iprt/critsect.h>
37	#include <iprt/err.h>
38	#include <iprt/mem.h>
39	#include <iprt/param.h>
40
41	#include "internal/magics.h"
42
43
44	/*******************************************************************************
45	* Structures and Typedefs *
46	*******************************************************************************/
47	/** Pointer to a cache instance. */
48	typedef struct RTMEMCACHEINT *PRTMEMCACHEINT;
49	/** Pointer to a cache page. */
50	typedef struct RTMEMCACHEPAGE *PRTMEMCACHEPAGE;
51
52
53
54	/**
55	* A free object.
56	*
57	* @remarks This only works if the objects don't have a constructor or
58	* destructor and are big enough.
59	*/
60	typedef struct RTMEMCACHEFREEOBJ
61	{
62	/** Pointer to the next free object */
63	struct RTMEMCACHEFREEOBJ * volatile pNext;
64	} RTMEMCACHEFREEOBJ;
65	/** Pointer to a free object. */
66	typedef RTMEMCACHEFREEOBJ *PRTMEMCACHEFREEOBJ;
67
68
69	/**
70	* A cache page.
71	*
72	* This is a page of memory that we split up in to a bunch object sized chunks
73	* and hand out to the cache users. The bitmap is updated in an atomic fashion
74	* so that we don't have to take any locks when freeing or allocating memory.
75	*/
76	typedef struct RTMEMCACHEPAGE
77	{
78	/** Pointer to the cache owning this page.
79	* This is used for validation purposes only. */
80	PRTMEMCACHEINT pCache;
81	/** Pointer to the next page.
82	* This is marked as volatile since we'll be adding new entries to the list
83	* without taking any locks. */
84	PRTMEMCACHEPAGE volatile pNext;
85	/** Bitmap tracking allocated blocks. */
86	void volatile *pbmAlloc;
87	/** Bitmap tracking which blocks that has been thru the constructor. */
88	void volatile *pbmCtor;
89	/** Pointer to the object array. */
90	uint8_t *pbObjects;
91	/** The number of objects on this page. */
92	uint32_t cObjects;
93
94	/** Padding to force cFree into the next cache line. (ASSUMES CL = 64) */
95	uint8_t abPadding[ARCH_BITS == 32 ? 64 - 64 : 64 - 58 - 4];
96	/** The number of free objects. */
97	int32_t volatile cFree;
98	} RTMEMCACHEPAGE;
99	AssertCompileMemberOffset(RTMEMCACHEPAGE, cFree, 64);
100
101
102	/**
103	* Memory object cache instance.
104	*/
105	typedef struct RTMEMCACHEINT
106	{
107	/** Magic value (RTMEMCACHE_MAGIC). */
108	uint32_t u32Magic;
109	/** The object size. */
110	uint32_t cbObject;
111	/** Object alignment. */
112	uint32_t cbAlignment;
113	/** The per page object count. */
114	uint32_t cPerPage;
115	/** Number of bits in the bitmap.
116	* @remarks This is higher or equal to cPerPage and it is aligned such that
117	* the search operation will be most efficient on x86/AMD64. */
118	uint32_t cBits;
119	/** The maximum number of objects. */
120	uint32_t cMax;
121	/** Whether to the use the free list or not. */
122	bool fUseFreeList;
123	/** Head of the page list. */
124	PRTMEMCACHEPAGE pPageHead;
125	/** Poiner to the insertion point in the page list. */
126	PRTMEMCACHEPAGE volatile *ppPageNext;
127	/** Constructor callback. */
128	PFNMEMCACHECTOR pfnCtor;
129	/** Destructor callback. */
130	PFNMEMCACHEDTOR pfnDtor;
131	/** Callback argument. */
132	void *pvUser;
133	/** Critical section serializing page allocation and similar. */
134	RTCRITSECT CritSect;
135
136	/** The total object count. */
137	uint32_t volatile cTotal;
138	/** The number of free objects. */
139	int32_t volatile cFree;
140	/** This may point to a page with free entries. */
141	PRTMEMCACHEPAGE volatile pPageHint;
142	/** Stack of free items.
143	* These are marked as used in the allocation bitmaps.
144	*
145	* @todo This doesn't scale well when several threads are beating on the
146	* cache. Also, it totally doesn't work when the objects are too
147	* small. */
148	PRTMEMCACHEFREEOBJ volatile pFreeTop;
149	} RTMEMCACHEINT;
150
151
152	/*******************************************************************************
153	* Internal Functions *
154	*******************************************************************************/
155	static void rtMemCacheFreeList(RTMEMCACHEINT *pThis, PRTMEMCACHEFREEOBJ pHead);
156
157
158	RTDECL(int) RTMemCacheCreate(PRTMEMCACHE phMemCache, size_t cbObject, size_t cbAlignment, uint32_t cMaxObjects,
159	PFNMEMCACHECTOR pfnCtor, PFNMEMCACHEDTOR pfnDtor, void *pvUser, uint32_t fFlags)
160
161	{
162	AssertPtr(phMemCache);
163	AssertPtrNull(pfnCtor);
164	AssertPtrNull(pfnDtor);
165	AssertReturn(!pfnDtor \|\| pfnCtor, VERR_INVALID_PARAMETER);
166	AssertReturn(cbObject > 0, VERR_INVALID_PARAMETER);
167	AssertReturn(cbObject <= PAGE_SIZE / 8, VERR_INVALID_PARAMETER);
168	AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
169
170	if (cbAlignment == 0)
171	{
172	if (cbObject <= 2)
173	cbAlignment = cbObject;
174	else if (cbObject <= 4)
175	cbAlignment = 4;
176	else if (cbObject <= 8)
177	cbAlignment = 8;
178	else if (cbObject <= 16)
179	cbAlignment = 16;
180	else if (cbObject <= 32)
181	cbAlignment = 32;
182	else
183	cbAlignment = 64;
184	}
185	else
186	{
187	AssertReturn(!((cbAlignment - 1) & cbAlignment), VERR_NOT_POWER_OF_TWO);
188	AssertReturn(cbAlignment <= 64, VERR_OUT_OF_RANGE);
189	}
190
191	/*
192	* Allocate and initialize the instance memory.
193	*/
194	RTMEMCACHEINT pThis = (RTMEMCACHEINT )RTMemAlloc(sizeof(*pThis));
195	if (!pThis)
196	return VERR_NO_MEMORY;
197	int rc = RTCritSectInit(&pThis->CritSect);
198	if (RT_FAILURE(rc))
199	{
200	RTMemFree(pThis);
201	return rc;
202	}
203
204	pThis->u32Magic = RTMEMCACHE_MAGIC;
205	pThis->cbObject = (uint32_t)RT_ALIGN_Z(cbObject, cbAlignment);
206	pThis->cbAlignment = (uint32_t)cbAlignment;
207	pThis->cPerPage = (uint32_t)((PAGE_SIZE - RT_ALIGN_Z(sizeof(RTMEMCACHEPAGE), cbAlignment)) / pThis->cbObject);
208	while ( RT_ALIGN_Z(sizeof(RTMEMCACHEPAGE), 8)
209	+ pThis->cPerPage * pThis->cbObject
210	+ RT_ALIGN(pThis->cPerPage, 64) / 8 * 2
211	> PAGE_SIZE)
212	pThis->cPerPage--;
213	pThis->cBits = RT_ALIGN(pThis->cPerPage, 64);
214	pThis->cMax = cMaxObjects;
215	pThis->fUseFreeList = cbObject >= sizeof(RTMEMCACHEFREEOBJ)
216	&& !pfnCtor
217	&& !pfnDtor;
218	pThis->pPageHead = NULL;
219	pThis->ppPageNext = &pThis->pPageHead;
220	pThis->pfnCtor = pfnCtor;
221	pThis->pfnDtor = pfnDtor;
222	pThis->pvUser = pvUser;
223	pThis->cTotal = 0;
224	pThis->cFree = 0;
225	pThis->pPageHint = NULL;
226	pThis->pFreeTop = NULL;
227
228	*phMemCache = pThis;
229	return VINF_SUCCESS;
230	}
231
232
233	RTDECL(int) RTMemCacheDestroy(RTMEMCACHE hMemCache)
234	{
235	RTMEMCACHEINT *pThis = hMemCache;
236	if (!pThis)
237	return VINF_SUCCESS;
238	AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
239	AssertReturn(pThis->u32Magic == RTMEMCACHE_MAGIC, VERR_INVALID_HANDLE);
240
241	#if 0 /def RT_STRICT - don't require eveything to be freed. Caches are very convenient for lazy cleanup. /
242	uint32_t cFree = pThis->cFree;
243	for (PRTMEMCACHEFREEOBJ pFree = pThis->pFreeTop; pFree && cFree < pThis->cTotal + 5; pFree = pFree->pNext)
244	cFree++;
245	AssertMsg(cFree == pThis->cTotal, ("cFree=%u cTotal=%u\n", cFree, pThis->cTotal));
246	#endif
247
248	/*
249	* Destroy it.
250	*/
251	AssertReturn(ASMAtomicCmpXchgU32(&pThis->u32Magic, RTMEMCACHE_MAGIC_DEAD, RTMEMCACHE_MAGIC), VERR_INVALID_HANDLE);
252	RTCritSectDelete(&pThis->CritSect);
253
254	while (pThis->pPageHead)
255	{
256	PRTMEMCACHEPAGE pPage = pThis->pPageHead;
257	pThis->pPageHead = pPage->pNext;
258	pPage->cFree = 0;
259
260	if (pThis->pfnDtor)
261	{
262	uint32_t iObj = pPage->cObjects;
263	while (iObj-- > 0)
264	if (ASMBitTestAndClear(pPage->pbmCtor, iObj))
265	pThis->pfnDtor(hMemCache, pPage->pbObjects + iObj * pThis->cbObject, pThis->pvUser);
266	}
267
268	RTMemPageFree(pPage, PAGE_SIZE);
269	}
270
271	RTMemFree(pThis);
272	return VINF_SUCCESS;
273	}
274
275
276	/**
277	* Grows the cache.
278	*
279	* @returns IPRT status code.
280	* @param pThis The memory cache instance.
281	*/
282	static int rtMemCacheGrow(RTMEMCACHEINT *pThis)
283	{
284	/*
285	* Enter the critical section here to avoid allocation races leading to
286	* wasted memory (++) and make it easier to link in the new page.
287	*/
288	RTCritSectEnter(&pThis->CritSect);
289	int rc = VINF_SUCCESS;
290	if (pThis->cFree < 0)
291	{
292	/*
293	* Allocate and initialize the new page.
294	*
295	* We put the constructor bitmap at the lower end right after cFree.
296	* We then push the object array to the end of the page and place the
297	* allocation bitmap below it. The hope is to increase the chance that
298	* the allocation bitmap is in a different cache line than cFree since
299	* this increases performance markably when lots of threads are beating
300	* on the cache.
301	*/
302	PRTMEMCACHEPAGE pPage = (PRTMEMCACHEPAGE)RTMemPageAlloc(PAGE_SIZE);
303	if (pPage)
304	{
305	uint32_t const cObjects = RT_MIN(pThis->cPerPage, pThis->cMax - pThis->cTotal);
306
307	ASMMemZeroPage(pPage);
308	pPage->pCache = pThis;
309	pPage->pNext = NULL;
310	pPage->cFree = cObjects;
311	pPage->cObjects = cObjects;
312	uint8_t pb = (uint8_t )(pPage + 1);
313	pb = RT_ALIGN_PT(pb, 8, uint8_t *);
314	pPage->pbmCtor = pb;
315	pb = (uint8_t )pPage + PAGE_SIZE - pThis->cbObject cObjects;
316	pPage->pbObjects = pb; Assert(RT_ALIGN_P(pb, pThis->cbAlignment) == pb);
317	pb -= pThis->cBits / 8;
318	pb = (uint8_t *)((uintptr_t)pb & ~(uintptr_t)7);
319	pPage->pbmAlloc = pb;
320	Assert((uintptr_t)pPage->pbmCtor + pThis->cBits / 8 <= (uintptr_t)pPage->pbmAlloc);
321
322	/* Mark the bitmap padding and any unused objects as allocated. */
323	for (uint32_t iBit = cObjects; iBit < pThis->cBits; iBit++)
324	ASMBitSet(pPage->pbmAlloc, iBit);
325
326	/* Make it the hint. */
327	ASMAtomicWritePtr(&pThis->pPageHint, pPage);
328
329	/* Link the page in at the end of the list. */
330	ASMAtomicWritePtr(pThis->ppPageNext, pPage);
331	pThis->ppPageNext = &pPage->pNext;
332
333	/* Add it to the page counts. */
334	ASMAtomicAddS32(&pThis->cFree, cObjects);
335	ASMAtomicAddU32(&pThis->cTotal, cObjects);
336	}
337	else
338	rc = VERR_NO_MEMORY;
339	}
340	RTCritSectLeave(&pThis->CritSect);
341	return rc;
342	}
343
344
345	/**
346	* Grabs a an object in a page.
347	* @returns New cFree value on success (0 or higher), -1 on failure.
348	* @param pPage Pointer to the page.
349	*/
350	DECL_FORCE_INLINE(int32_t) rtMemCacheGrabObj(PRTMEMCACHEPAGE pPage)
351	{
352	int32_t cFreeNew = ASMAtomicDecS32(&pPage->cFree);
353	if (cFreeNew < 0)
354	{
355	ASMAtomicIncS32(&pPage->cFree);
356	return -1;
357	}
358	return cFreeNew;
359	}
360
361
362	RTDECL(int) RTMemCacheAllocEx(RTMEMCACHE hMemCache, void **ppvObj)
363	{
364	RTMEMCACHEINT *pThis = hMemCache;
365	AssertPtrReturn(pThis, VERR_INVALID_PARAMETER);
366	AssertReturn(pThis->u32Magic == RTMEMCACHE_MAGIC, VERR_INVALID_PARAMETER);
367
368	/*
369	* Try grab a free object from the stack.
370	*/
371	PRTMEMCACHEFREEOBJ pObj = ASMAtomicUoReadPtrT(&pThis->pFreeTop, PRTMEMCACHEFREEOBJ);
372	if (pObj)
373	{
374	pObj = ASMAtomicXchgPtrT(&pThis->pFreeTop, NULL, PRTMEMCACHEFREEOBJ);
375	if (pObj)
376	{
377	if (pObj->pNext)
378	{
379	Assert(pObj->pNext != pObj);
380	PRTMEMCACHEFREEOBJ pAllocRace = ASMAtomicXchgPtrT(&pThis->pFreeTop, pObj->pNext, PRTMEMCACHEFREEOBJ);
381	if (pAllocRace)
382	rtMemCacheFreeList(pThis, pAllocRace);
383	}
384
385	pObj->pNext = NULL;
386	*ppvObj = pObj;
387	return VINF_SUCCESS;
388	}
389	}
390
391	/*
392	* Try grab a free object at the cache level.
393	*/
394	int32_t cNewFree = ASMAtomicDecS32(&pThis->cFree);
395	if (RT_LIKELY(cNewFree < 0))
396	{
397	uint32_t cTotal = ASMAtomicUoReadU32(&pThis->cTotal);
398	if ( (uint32_t)(cTotal + -cNewFree) > pThis->cMax
399	\|\| (uint32_t)(cTotal + -cNewFree) <= cTotal)
400	{
401	ASMAtomicIncS32(&pThis->cFree);
402	return VERR_MEM_CACHE_MAX_SIZE;
403	}
404
405	int rc = rtMemCacheGrow(pThis);
406	if (RT_FAILURE(rc))
407	{
408	ASMAtomicIncS32(&pThis->cFree);
409	return rc;
410	}
411	}
412
413	/*
414	* Grab a free object at the page level.
415	*/
416	PRTMEMCACHEPAGE pPage = ASMAtomicReadPtrT(&pThis->pPageHint, PRTMEMCACHEPAGE);
417	int32_t iObj = pPage ? rtMemCacheGrabObj(pPage) : -1;
418	if (iObj < 0)
419	{
420	for (unsigned cLoops = 0; ; cLoops++)
421	{
422	for (pPage = pThis->pPageHead; pPage; pPage = pPage->pNext)
423	{
424	iObj = rtMemCacheGrabObj(pPage);
425	if (iObj >= 0)
426	{
427	if (iObj > 0)
428	ASMAtomicWritePtr(&pThis->pPageHint, pPage);
429	break;
430	}
431	}
432	if (iObj >= 0)
433	break;
434	Assert(cLoops != 2);
435	Assert(cLoops < 10);
436	}
437	}
438	Assert(iObj >= 0);
439	Assert((uint32_t)iObj < pThis->cMax);
440
441	/*
442	* Find a free object in the allocation bitmap. Use the new cFree count
443	* as a hint.
444	*/
445	if (ASMAtomicBitTestAndSet(pPage->pbmAlloc, iObj))
446	{
447	for (unsigned cLoops2 = 0;; cLoops2++)
448	{
449	iObj = ASMBitFirstClear(pPage->pbmAlloc, pThis->cBits);
450	if (RT_LIKELY(iObj >= 0))
451	{
452	if (!ASMAtomicBitTestAndSet(pPage->pbmAlloc, iObj))
453	break;
454	}
455	else
456	ASMMemoryFence();
457	Assert(cLoops2 != 40);
458	}
459	Assert(iObj >= 0);
460	}
461	void pvObj = &pPage->pbObjects[iObj pThis->cbObject];
462	Assert((uintptr_t)pvObj - (uintptr_t)pPage < PAGE_SIZE);
463
464	/*
465	* Call the constructor?
466	*/
467	if ( pThis->pfnCtor
468	&& !ASMAtomicBitTestAndSet(pPage->pbmCtor, iObj))
469	{
470	int rc = pThis->pfnCtor(hMemCache, pvObj, pThis->pvUser);
471	if (RT_FAILURE(rc))
472	{
473	ASMAtomicBitClear(pPage->pbmCtor, iObj);
474	RTMemCacheFree(pThis, pvObj);
475	return rc;
476	}
477	}
478
479	*ppvObj = pvObj;
480	return VINF_SUCCESS;
481	}
482
483
484	RTDECL(void *) RTMemCacheAlloc(RTMEMCACHE hMemCache)
485	{
486	void *pvObj;
487	int rc = RTMemCacheAllocEx(hMemCache, &pvObj);
488	if (RT_SUCCESS(rc))
489	return pvObj;
490	return NULL;
491	}
492
493
494
495	/**
496	* Really frees one object.
497	*
498	* @param pThis The memory cache.
499	* @param pvObj The memory object to free.
500	*/
501	static void rtMemCacheFreeOne(RTMEMCACHEINT pThis, void pvObj)
502	{
503	/* Note: Do NOT attempt to poison the object! */
504
505	/*
506	* Find the cache page. The page structure is at the start of the page.
507	*/
508	PRTMEMCACHEPAGE pPage = (PRTMEMCACHEPAGE)(((uintptr_t)pvObj) & ~(uintptr_t)PAGE_OFFSET_MASK);
509	Assert(pPage->pCache == pThis);
510	Assert(ASMAtomicUoReadS32(&pPage->cFree) < (int32_t)pThis->cPerPage);
511
512	/*
513	* Clear the bitmap bit and update the two object counter. Order matters!
514	*/
515	uintptr_t offObj = (uintptr_t)pvObj - (uintptr_t)pPage->pbObjects;
516	uintptr_t iObj = offObj / pThis->cbObject;
517	Assert(iObj * pThis->cbObject == offObj);
518	Assert(iObj < pThis->cPerPage);
519	AssertReturnVoid(ASMAtomicBitTestAndClear(pPage->pbmAlloc, iObj));
520
521	ASMAtomicIncS32(&pPage->cFree);
522	ASMAtomicIncS32(&pThis->cFree);
523	}
524
525
526	/**
527	* Really frees a list of 'freed' object.
528	*
529	* @param pThis The memory cache.
530	* @param pHead The head of the list.
531	*/
532	static void rtMemCacheFreeList(RTMEMCACHEINT *pThis, PRTMEMCACHEFREEOBJ pHead)
533	{
534	while (pHead)
535	{
536	PRTMEMCACHEFREEOBJ pFreeMe = pHead;
537	pHead = pHead->pNext;
538	pFreeMe->pNext = NULL;
539	ASMCompilerBarrier();
540	rtMemCacheFreeOne(pThis, pFreeMe);
541	}
542	}
543
544
545
546	RTDECL(void) RTMemCacheFree(RTMEMCACHE hMemCache, void *pvObj)
547	{
548	if (!pvObj)
549	return;
550
551	RTMEMCACHEINT *pThis = hMemCache;
552	AssertPtrReturnVoid(pThis);
553	AssertReturnVoid(pThis->u32Magic == RTMEMCACHE_MAGIC);
554
555	AssertPtr(pvObj);
556	Assert(RT_ALIGN_P(pvObj, pThis->cbAlignment) == pvObj);
557
558	if (!pThis->fUseFreeList)
559	rtMemCacheFreeOne(pThis, pvObj);
560	else
561	{
562	# ifdef RT_STRICT
563	/* This is the same as the other branch, except it's not actually freed. */
564	PRTMEMCACHEPAGE pPage = (PRTMEMCACHEPAGE)(((uintptr_t)pvObj) & ~(uintptr_t)PAGE_OFFSET_MASK);
565	Assert(pPage->pCache == pThis);
566	Assert(ASMAtomicUoReadS32(&pPage->cFree) < (int32_t)pThis->cPerPage);
567	uintptr_t offObj = (uintptr_t)pvObj - (uintptr_t)pPage->pbObjects;
568	uintptr_t iObj = offObj / pThis->cbObject;
569	Assert(iObj * pThis->cbObject == offObj);
570	Assert(iObj < pThis->cPerPage);
571	AssertReturnVoid(ASMBitTest(pPage->pbmAlloc, (int32_t)iObj));
572	# endif
573
574	/*
575	* Push it onto the free stack.
576	*/
577	PRTMEMCACHEFREEOBJ pObj = (PRTMEMCACHEFREEOBJ)pvObj;
578	pObj->pNext = ASMAtomicXchgPtrT(&pThis->pFreeTop, NULL, PRTMEMCACHEFREEOBJ);
579	PRTMEMCACHEFREEOBJ pFreeRace = ASMAtomicXchgPtrT(&pThis->pFreeTop, pObj, PRTMEMCACHEFREEOBJ);
580	if (pFreeRace)
581	rtMemCacheFreeList(pThis, pFreeRace);
582	}
583	}
584

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/common/alloc/memcache.cpp@ 55253

Download in other formats: