VirtualBox

source: vbox/trunk/src/VBox/Runtime/common/fuzz/fuzz-target-recorder.cpp@ 77544

Last change on this file since 77544 was 77544, checked in by vboxsync, 6 years ago

Runtime/fuzz: Updates, add a target state recording mechanism to record changes in target behavior caused by mutated inputs. This allows to decide which mutated input gets added to the input corpus and which one gets discarded. Currently this is only able to record the stdout/stderr channels of the fuzzed process but other sources to detect changed behvior will get added in the future
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 14.9 KB
Line 
1/* $Id: fuzz-target-recorder.cpp 77544 2019-03-03 20:07:01Z vboxsync $ */
2/** @file
3 * IPRT - Fuzzing framework API, target state recorder.
4 */
5
6/*
7 * Copyright (C) 2019 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.215389.xyz. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 *
17 * The contents of this file may alternatively be used under the terms
18 * of the Common Development and Distribution License Version 1.0
19 * (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20 * VirtualBox OSE distribution, in which case the provisions of the
21 * CDDL are applicable instead of those of the GPL.
22 *
23 * You may elect to license modified versions of this file under the
24 * terms and conditions of either the GPL or the CDDL or both.
25 */
26
27
28/*********************************************************************************************************************************
29* Header Files *
30*********************************************************************************************************************************/
31#include <iprt/fuzz.h>
32#include "internal/iprt.h"
33
34#include <iprt/asm.h>
35#include <iprt/assert.h>
36#include <iprt/avl.h>
37#include <iprt/ctype.h>
38#include <iprt/err.h>
39#include <iprt/file.h>
40#include <iprt/list.h>
41#include <iprt/mem.h>
42#include <iprt/pipe.h>
43#include <iprt/semaphore.h>
44#include <iprt/string.h>
45
46
47
48/*********************************************************************************************************************************
49* Structures and Typedefs *
50*********************************************************************************************************************************/
51/** Pointer to the internal fuzzed target recorder state. */
52typedef struct RTFUZZTGTRECINT *PRTFUZZTGTRECINT;
53
54
55/**
56 * Stdout/Stderr buffer.
57 */
58typedef struct RTFUZZTGTSTDOUTERRBUF
59{
60 /** Current amount buffered. */
61 size_t cbBuf;
62 /** Maxmium amount to buffer. */
63 size_t cbBufMax;
64 /** Base pointer to the data buffer. */
65 uint8_t *pbBase;
66} RTFUZZTGTSTDOUTERRBUF;
67/** Pointer to a stdout/stderr buffer. */
68typedef RTFUZZTGTSTDOUTERRBUF *PRTFUZZTGTSTDOUTERRBUF;
69
70
71/**
72 * Internal fuzzed target state.
73 */
74typedef struct RTFUZZTGTSTATEINT
75{
76 /** Node for the list of states. */
77 RTLISTNODE NdStates;
78 /** Magic identifying the structure. */
79 uint32_t u32Magic;
80 /** Reference counter. */
81 volatile uint32_t cRefs;
82 /** The owning recorder instance. */
83 PRTFUZZTGTRECINT pTgtRec;
84 /** Flag whether the state is finalized. */
85 bool fFinalized;
86 /** Flag whether the state is contained in the recorded set. */
87 bool fInRecSet;
88 /** The stdout data buffer. */
89 RTFUZZTGTSTDOUTERRBUF StdOutBuf;
90 /** The stderr data buffer. */
91 RTFUZZTGTSTDOUTERRBUF StdErrBuf;
92} RTFUZZTGTSTATEINT;
93/** Pointer to an internal fuzzed target state. */
94typedef RTFUZZTGTSTATEINT *PRTFUZZTGTSTATEINT;
95
96
97/**
98 * Recorder states node in the AVL tree.
99 */
100typedef struct RTFUZZTGTRECNODE
101{
102 /** The AVL tree core (keyed by stdout/stderr buffer sizes). */
103 AVLU64NODECORE Core;
104 /** The list anchor for the individual states. */
105 RTLISTANCHOR LstStates;
106} RTFUZZTGTRECNODE;
107/** Pointer to a recorder states node. */
108typedef RTFUZZTGTRECNODE *PRTFUZZTGTRECNODE;
109
110
111/**
112 * Internal fuzzed target recorder state.
113 */
114typedef struct RTFUZZTGTRECINT
115{
116 /** Magic value for identification. */
117 uint32_t u32Magic;
118 /** Reference counter. */
119 volatile uint32_t cRefs;
120 /** Semaphore protecting the states tree. */
121 RTSEMRW hSemRwStates;
122 /** The AVL tree for indexing the recorded state (keyed by stdout/stderr buffer size). */
123 AVLU64TREE TreeStates;
124} RTFUZZTGTRECINT;
125
126
127/*********************************************************************************************************************************
128* Internal Functions *
129*********************************************************************************************************************************/
130
131/**
132 * Initializes the given stdout/stderr buffer.
133 *
134 * @returns nothing.
135 * @param pBuf The buffer to initialize.
136 */
137static void rtFuzzTgtStdOutErrBufInit(PRTFUZZTGTSTDOUTERRBUF pBuf)
138{
139 pBuf->cbBuf = 0;
140 pBuf->cbBufMax = 0;
141 pBuf->pbBase = NULL;
142}
143
144
145/**
146 * Frees all allocated resources in the given stdout/stderr buffer.
147 *
148 * @returns nothing.
149 * @param pBuf The buffer to free.
150 */
151static void rtFuzzTgtStdOutErrBufFree(PRTFUZZTGTSTDOUTERRBUF pBuf)
152{
153 if (pBuf->pbBase)
154 RTMemFree(pBuf->pbBase);
155}
156
157
158/**
159 * Fills the given stdout/stderr buffer from the given pipe.
160 *
161 * @returns IPRT status code.
162 * @param pBuf The buffer to fill.
163 * @param hPipeRead The pipe to read from.
164 */
165static int rtFuzzTgtStdOutErrBufFillFromPipe(PRTFUZZTGTSTDOUTERRBUF pBuf, RTPIPE hPipeRead)
166{
167 int rc = VINF_SUCCESS;
168
169 size_t cbRead = 0;
170 size_t cbThisRead = 0;
171 do
172 {
173 cbThisRead = pBuf->cbBufMax - pBuf->cbBuf;
174 if (!cbThisRead)
175 {
176 /* Try to increase the buffer. */
177 uint8_t *pbNew = (uint8_t *)RTMemRealloc(pBuf->pbBase, pBuf->cbBufMax + _4K);
178 if (RT_LIKELY(pbNew))
179 {
180 pBuf->cbBufMax += _4K;
181 pBuf->pbBase = pbNew;
182 }
183 cbThisRead = pBuf->cbBufMax - pBuf->cbBuf;
184 }
185
186 if (cbThisRead)
187 {
188 rc = RTPipeRead(hPipeRead, pBuf->pbBase + pBuf->cbBuf, cbThisRead, &cbRead);
189 if (RT_SUCCESS(rc))
190 pBuf->cbBuf += cbRead;
191 }
192 else
193 rc = VERR_NO_MEMORY;
194 } while ( RT_SUCCESS(rc)
195 && cbRead == cbThisRead);
196
197 return rc;
198}
199
200
201/**
202 * Destorys the given fuzzer target recorder freeing all allocated resources.
203 *
204 * @returns nothing.
205 * @param pThis The fuzzer target recorder instance.
206 */
207static void rtFuzzTgtRecDestroy(PRTFUZZTGTRECINT pThis)
208{
209 RT_NOREF(pThis);
210}
211
212
213/**
214 * Destorys the given fuzzer target recorder freeing all allocated resources.
215 *
216 * @returns nothing.
217 * @param pThis The fuzzed target state instance.
218 */
219static void rtFuzzTgtStateDestroy(PRTFUZZTGTSTATEINT pThis)
220{
221 pThis->u32Magic = ~0; /** @todo Dead magic */
222 rtFuzzTgtStdOutErrBufFree(&pThis->StdOutBuf);
223 rtFuzzTgtStdOutErrBufFree(&pThis->StdErrBuf);
224 RTMemFree(pThis);
225}
226
227
228RTDECL(int) RTFuzzTgtRecorderCreate(PRTFUZZTGTREC phFuzzTgtRec)
229{
230 int rc;
231 PRTFUZZTGTRECINT pThis = (PRTFUZZTGTRECINT)RTMemAllocZ(sizeof(*pThis));
232 if (RT_LIKELY(pThis))
233 {
234 pThis->u32Magic = 0; /** @todo */
235 pThis->cRefs = 1;
236 pThis->TreeStates = NULL;
237
238 rc = RTSemRWCreate(&pThis->hSemRwStates);
239 if (RT_SUCCESS(rc))
240 {
241 *phFuzzTgtRec = pThis;
242 return VINF_SUCCESS;
243 }
244
245 RTMemFree(pThis);
246 }
247 else
248 rc = VERR_NO_MEMORY;
249
250 return rc;
251}
252
253
254RTDECL(uint32_t) RTFuzzTgtRecorderRetain(RTFUZZTGTREC hFuzzTgtRec)
255{
256 PRTFUZZTGTRECINT pThis = hFuzzTgtRec;
257
258 AssertPtrReturn(pThis, UINT32_MAX);
259
260 uint32_t cRefs = ASMAtomicIncU32(&pThis->cRefs);
261 AssertMsg(cRefs > 1 && cRefs < _1M, ("%#x %p\n", cRefs, pThis));
262 return cRefs;
263}
264
265
266RTDECL(uint32_t) RTFuzzTgtRecorderRelease(RTFUZZTGTREC hFuzzTgtRec)
267{
268 PRTFUZZTGTRECINT pThis = hFuzzTgtRec;
269 if (pThis == NIL_RTFUZZTGTREC)
270 return 0;
271 AssertPtrReturn(pThis, UINT32_MAX);
272
273 uint32_t cRefs = ASMAtomicDecU32(&pThis->cRefs);
274 AssertMsg(cRefs < _1M, ("%#x %p\n", cRefs, pThis));
275 if (cRefs == 0)
276 rtFuzzTgtRecDestroy(pThis);
277 return cRefs;
278}
279
280
281RTDECL(int) RTFuzzTgtRecorderCreateNewState(RTFUZZTGTREC hFuzzTgtRec, PRTFUZZTGTSTATE phFuzzTgtState)
282{
283 PRTFUZZTGTRECINT pThis = hFuzzTgtRec;
284 AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
285 AssertPtrReturn(phFuzzTgtState, VERR_INVALID_POINTER);
286
287 int rc = VINF_SUCCESS;
288 PRTFUZZTGTSTATEINT pState = (PRTFUZZTGTSTATEINT)RTMemAllocZ(sizeof(*pState));
289 if (RT_LIKELY(pState))
290 {
291 pState->u32Magic = 0; /** @todo */
292 pState->cRefs = 1;
293 pState->pTgtRec = pThis;
294 pState->fFinalized = false;
295 rtFuzzTgtStdOutErrBufInit(&pState->StdOutBuf);
296 rtFuzzTgtStdOutErrBufInit(&pState->StdErrBuf);
297 *phFuzzTgtState = pState;
298 }
299 else
300 rc = VERR_NO_MEMORY;
301
302 return rc;
303}
304
305
306RTDECL(uint32_t) RTFuzzTgtStateRetain(RTFUZZTGTSTATE hFuzzTgtState)
307{
308 PRTFUZZTGTSTATEINT pThis = hFuzzTgtState;
309
310 AssertPtrReturn(pThis, UINT32_MAX);
311
312 uint32_t cRefs = ASMAtomicIncU32(&pThis->cRefs);
313 AssertMsg(cRefs > 1 && cRefs < _1M, ("%#x %p\n", cRefs, pThis));
314 return cRefs;
315}
316
317
318RTDECL(uint32_t) RTFuzzTgtStateRelease(RTFUZZTGTSTATE hFuzzTgtState)
319{
320 PRTFUZZTGTSTATEINT pThis = hFuzzTgtState;
321 if (pThis == NIL_RTFUZZTGTSTATE)
322 return 0;
323 AssertPtrReturn(pThis, UINT32_MAX);
324
325 uint32_t cRefs = ASMAtomicDecU32(&pThis->cRefs);
326 AssertMsg(cRefs < _1M, ("%#x %p\n", cRefs, pThis));
327 if (cRefs == 0 && !pThis->fInRecSet)
328 rtFuzzTgtStateDestroy(pThis);
329 return cRefs;
330}
331
332
333RTDECL(int) RTFuzzTgtStateReset(RTFUZZTGTSTATE hFuzzTgtState)
334{
335 PRTFUZZTGTSTATEINT pThis = hFuzzTgtState;
336 AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
337
338 /* Clear the buffers. */
339 pThis->StdOutBuf.cbBuf = 0;
340 pThis->StdErrBuf.cbBuf = 0;
341 pThis->fFinalized = false;
342 return VINF_SUCCESS;
343}
344
345
346RTDECL(int) RTFuzzTgtStateFinalize(RTFUZZTGTSTATE hFuzzTgtState)
347{
348 PRTFUZZTGTSTATEINT pThis = hFuzzTgtState;
349 AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
350
351 /*
352 * As we key both the stdout and stderr sizes in one 64bit
353 * AVL tree core we only have 32bit for each and refuse buffers
354 * exceeding this size (very unlikely for now).
355 */
356 if (RT_UNLIKELY( pThis->StdOutBuf.cbBuf > UINT32_MAX
357 || pThis->StdErrBuf.cbBuf > UINT32_MAX))
358 return VERR_BUFFER_OVERFLOW;
359
360 pThis->fFinalized = true;
361 return VINF_SUCCESS;
362}
363
364
365RTDECL(int) RTFuzzTgtStateAddToRecorder(RTFUZZTGTSTATE hFuzzTgtState)
366{
367 PRTFUZZTGTSTATEINT pThis = hFuzzTgtState;
368 AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
369
370 if (!pThis->fFinalized)
371 {
372 int rc = RTFuzzTgtStateFinalize(pThis);
373 if (RT_FAILURE(rc))
374 return rc;
375 }
376
377 PRTFUZZTGTRECINT pTgtRec = pThis->pTgtRec;
378 uint64_t uKey = (pThis->StdOutBuf.cbBuf << 32) | pThis->StdErrBuf.cbBuf;
379
380 /* Try to find a node matching the stdout and sterr sizes first. */
381 int rc = RTSemRWRequestRead(pTgtRec->hSemRwStates, RT_INDEFINITE_WAIT); AssertRC(rc);
382 PRTFUZZTGTRECNODE pNode = (PRTFUZZTGTRECNODE)RTAvlU64Get(&pTgtRec->TreeStates, uKey);
383 if (pNode)
384 {
385 /* Traverse the states and check if any matches the stdout and stderr buffers exactly. */
386 PRTFUZZTGTSTATEINT pIt;
387 bool fMatchFound = false;
388 RTListForEach(&pNode->LstStates, pIt, RTFUZZTGTSTATEINT, NdStates)
389 {
390 Assert( pThis->StdOutBuf.cbBuf == pIt->StdOutBuf.cbBuf
391 && pThis->StdErrBuf.cbBuf == pIt->StdErrBuf.cbBuf);
392 if ( ( !pThis->StdOutBuf.cbBuf
393 || !memcmp(pThis->StdOutBuf.pbBase, pIt->StdOutBuf.pbBase, pThis->StdOutBuf.cbBuf))
394 && ( !pThis->StdErrBuf.cbBuf
395 || !memcmp(pThis->StdErrBuf.pbBase, pIt->StdErrBuf.pbBase, pThis->StdErrBuf.cbBuf)))
396 {
397 fMatchFound = true;
398 break;
399 }
400 }
401
402 rc = RTSemRWReleaseRead(pTgtRec->hSemRwStates); AssertRC(rc);
403 if (!fMatchFound)
404 {
405 rc = RTSemRWRequestWrite(pTgtRec->hSemRwStates, RT_INDEFINITE_WAIT); AssertRC(rc);
406 RTListAppend(&pNode->LstStates, &pThis->NdStates);
407 rc = RTSemRWReleaseWrite(pTgtRec->hSemRwStates); AssertRC(rc);
408 pThis->fInRecSet = true;
409 }
410 else
411 rc = VERR_ALREADY_EXISTS;
412 }
413 else
414 {
415 rc = RTSemRWReleaseRead(pTgtRec->hSemRwStates); AssertRC(rc);
416
417 /* No node found, create new one and insert in to the tree right away. */
418 pNode = (PRTFUZZTGTRECNODE)RTMemAllocZ(sizeof(*pNode));
419 if (RT_LIKELY(pNode))
420 {
421 pNode->Core.Key = uKey;
422 RTListInit(&pNode->LstStates);
423 RTListAppend(&pNode->LstStates, &pThis->NdStates);
424 rc = RTSemRWRequestWrite(pTgtRec->hSemRwStates, RT_INDEFINITE_WAIT); AssertRC(rc);
425 bool fIns = RTAvlU64Insert(&pTgtRec->TreeStates, &pNode->Core);
426 if (!fIns)
427 {
428 /* Someone raced us, get the new node and append there. */
429 RTMemFree(pNode);
430 pNode = (PRTFUZZTGTRECNODE)RTAvlU64Get(&pTgtRec->TreeStates, uKey);
431 AssertPtr(pNode);
432 RTListAppend(&pNode->LstStates, &pThis->NdStates);
433 }
434 rc = RTSemRWReleaseWrite(pTgtRec->hSemRwStates); AssertRC(rc);
435 pThis->fInRecSet = true;
436 }
437 else
438 rc = VERR_NO_MEMORY;
439 }
440
441 return rc;
442}
443
444
445RTDECL(int) RTFuzzTgtStateAppendStdoutFromBuf(RTFUZZTGTSTATE hFuzzTgtState, const void *pvStdOut, size_t cbStdOut)
446{
447 PRTFUZZTGTSTATEINT pThis = hFuzzTgtState;
448 AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
449 AssertReturn(!pThis->fFinalized, VERR_WRONG_ORDER);
450
451 RT_NOREF(pvStdOut, cbStdOut);
452 return VERR_NOT_IMPLEMENTED;
453}
454
455
456RTDECL(int) RTFuzzTgtStateAppendStderrFromBuf(RTFUZZTGTSTATE hFuzzTgtState, const void *pvStdErr, size_t cbStdErr)
457{
458 PRTFUZZTGTSTATEINT pThis = hFuzzTgtState;
459 AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
460 AssertReturn(!pThis->fFinalized, VERR_WRONG_ORDER);
461
462 RT_NOREF(pvStdErr, cbStdErr);
463 return VERR_NOT_IMPLEMENTED;
464}
465
466
467RTDECL(int) RTFuzzTgtStateAppendStdoutFromPipe(RTFUZZTGTSTATE hFuzzTgtState, RTPIPE hPipe)
468{
469 PRTFUZZTGTSTATEINT pThis = hFuzzTgtState;
470 AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
471 AssertReturn(!pThis->fFinalized, VERR_WRONG_ORDER);
472
473 return rtFuzzTgtStdOutErrBufFillFromPipe(&pThis->StdOutBuf, hPipe);
474}
475
476
477RTDECL(int) RTFuzzTgtStateAppendStderrFromPipe(RTFUZZTGTSTATE hFuzzTgtState, RTPIPE hPipe)
478{
479 PRTFUZZTGTSTATEINT pThis = hFuzzTgtState;
480 AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
481 AssertReturn(!pThis->fFinalized, VERR_WRONG_ORDER);
482
483 return rtFuzzTgtStdOutErrBufFillFromPipe(&pThis->StdErrBuf, hPipe);
484}
485
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette