timer-r0drv-linux.c@ 9368

Last change on this file since 9368 was 9368, checked in by vboxsync, 17 years ago
RTCpuSetIsEqual
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 28.2 KB

Line
1	/* $Id: timer-r0drv-linux.c 9368 2008-06-03 22:30:13Z vboxsync $ */
2	/** @file
3	* IPRT - Timers, Ring-0 Driver, Linux.
4	*/
5
6	/*
7	* Copyright (C) 2006-2008 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.215389.xyz. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* The contents of this file may alternatively be used under the terms
18	* of the Common Development and Distribution License Version 1.0
19	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	* VirtualBox OSE distribution, in which case the provisions of the
21	* CDDL are applicable instead of those of the GPL.
22	*
23	* You may elect to license modified versions of this file under the
24	* terms and conditions of either the GPL or the CDDL or both.
25	*
26	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
27	* Clara, CA 95054 USA or visit http://www.sun.com if you need
28	* additional information or have any questions.
29	*/
30
31	/*******************************************************************************
32	* Header Files *
33	*******************************************************************************/
34	#include "the-linux-kernel.h"
35
36	#include <iprt/timer.h>
37	#include <iprt/time.h>
38	#include <iprt/mp.h>
39	#include <iprt/cpuset.h>
40	#include <iprt/spinlock.h>
41	#include <iprt/err.h>
42	#include <iprt/asm.h>
43	#include <iprt/assert.h>
44	#include <iprt/alloc.h>
45
46	#include "internal/magics.h"
47
48	#if !defined(RT_USE_LINUX_HRTIMER) && LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 23) /* ?? */
49	# define RT_USE_LINUX_HRTIMER
50	#endif
51
52	#if defined(RT_USE_LINUX_HRTIMER) && LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 16)
53	# error "RT_USE_LINUX_HRTIMER requires 2.6.16 or later, sorry."
54	#endif
55
56
57	/*******************************************************************************
58	* Structures and Typedefs *
59	*******************************************************************************/
60	/**
61	* Timer state machine.
62	*
63	* This is used to try handle the issues with MP events and
64	* timers that runs on all CPUs. It's relatively nasty :-/
65	*/
66	typedef enum RTTIMERLNXSTATE
67	{
68	/** Stopped. */
69	RTTIMERLNXSTATE_STOPPED = 0,
70	/** Transient state; next ACTIVE. */
71	RTTIMERLNXSTATE_STARTING,
72	/** Transient state; next ACTIVE. (not really necessary) */
73	RTTIMERLNXSTATE_MP_STARTING,
74	/** Active. */
75	RTTIMERLNXSTATE_ACTIVE,
76	/** Transient state; next STOPPED. */
77	RTTIMERLNXSTATE_STOPPING,
78	/** Transient state; next STOPPED. */
79	RTTIMERLNXSTATE_MP_STOPPING,
80	/** The usual 32-bit hack. */
81	RTTIMERLNXSTATE_32BIT_HACK = 0x7fffffff
82	} RTTIMERLNXSTATE;
83
84
85	/**
86	* A Linux sub-timer.
87	*/
88	typedef struct RTTIMERLNXSUBTIMER
89	{
90	/** The linux timer structure. */
91	#ifdef RT_USE_LINUX_HRTIMER
92	struct hrtimer LnxTimer;
93	#else
94	struct timer_list LnxTimer;
95	#endif
96	/** The start of the current run (ns).
97	* This is used to calculate when the timer ought to fire the next time. */
98	uint64_t u64StartTS;
99	/** The start of the current run (ns).
100	* This is used to calculate when the timer ought to fire the next time. */
101	uint64_t u64NextTS;
102	/** The current tick number (since u64StartTS). */
103	uint64_t iTick;
104	/** Pointer to the parent timer. */
105	PRTTIMER pParent;
106	/** The current sub-timer state. */
107	RTTIMERLNXSTATE volatile enmState;
108	} RTTIMERLNXSUBTIMER;
109	/** Pointer to a linux sub-timer. */
110	typedef RTTIMERLNXSUBTIMER *PRTTIMERLNXSUBTIMER;
111	AssertCompileMemberOffset(RTTIMERLNXSUBTIMER, LnxTimer, 0);
112
113
114	/**
115	* The internal representation of an Linux timer handle.
116	*/
117	typedef struct RTTIMER
118	{
119	/** Magic.
120	* This is RTTIMER_MAGIC, but changes to something else before the timer
121	* is destroyed to indicate clearly that thread should exit. */
122	uint32_t volatile u32Magic;
123	/** Spinlock synchronizing the fSuspended and MP event handling.
124	* This is NIL_RTSPINLOCK if cCpus == 1. */
125	RTSPINLOCK hSpinlock;
126	/** Flag indicating the the timer is suspended. */
127	bool volatile fSuspended;
128	/** Whether the timer must run on one specific CPU or not. */
129	bool fSpecificCpu;
130	#ifdef CONFIG_SMP
131	/** Whether the timer must run on all CPUs or not. */
132	bool fAllCpus;
133	#endif /* else: All -> specific on non-SMP kernels */
134	/** The CPU it must run on if fSpecificCpu is set. */
135	RTCPUID idCpu;
136	/** The number of CPUs this timer should run on. */
137	RTCPUID cCpus;
138	/** Callback. */
139	PFNRTTIMER pfnTimer;
140	/** User argument. */
141	void *pvUser;
142	/** The timer interval. 0 if one-shot. */
143	uint64_t u64NanoInterval;
144	/** Sub-timers.
145	* Normally there is just one, but for RTTIMER_FLAGS_CPU_ALL this will contain
146	* an entry for all possible cpus. In that case the index will be the same as
147	* for the RTCpuSet. */
148	RTTIMERLNXSUBTIMER aSubTimers[1];
149	} RTTIMER;
150
151
152	/**
153	* A rtTimerLinuxStartOnCpu and rtTimerLinuxStartOnCpu argument package.
154	*/
155	typedef struct RTTIMERLINUXSTARTONCPUARGS
156	{
157	/** The current time (RTTimeNanoTS). */
158	uint64_t u64Now;
159	/** When to start firing (delta). */
160	uint64_t u64First;
161	} RTTIMERLINUXSTARTONCPUARGS;
162	/** Pointer to a rtTimerLinuxStartOnCpu argument package. */
163	typedef RTTIMERLINUXSTARTONCPUARGS *PRTTIMERLINUXSTARTONCPUARGS;
164
165
166	/*******************************************************************************
167	* Internal Functions *
168	*******************************************************************************/
169	#ifdef RT_USE_LINUX_HRTIMER
170	static enum hrtimer_restart rtTimerLinuxCallback(struct hrtimer *pHrTimer);
171	#else
172	static void rtTimerLinuxCallback(unsigned long ulUser);
173	#endif
174	#ifdef CONFIG_SMP
175	static int rtTimerLnxStartAll(PRTTIMER pTimer, PRTTIMERLINUXSTARTONCPUARGS pArgs);
176	static int rtTimerLnxStopAll(PRTTIMER pTimer);
177	static DECLCALLBACK(void) rtTimerLinuxMpEvent(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvUser);
178	#endif
179
180
181	RTDECL(uint32_t) RTTimerGetSystemGranularity(void)
182	{
183	#ifdef RT_USE_LINUX_HRTIMER
184	/** @todo later... */
185	return 1000000000 / HZ; /* ns */
186	#else
187	return 1000000000 / HZ; /* ns */
188	#endif
189	}
190
191
192	RTDECL(int) RTTimerRequestSystemGranularity(uint32_t u32Request, uint32_t *pu32Granted)
193	{
194	return VERR_NOT_SUPPORTED;
195	}
196
197
198	RTDECL(int) RTTimerReleaseSystemGranularity(uint32_t u32Granted)
199	{
200	return VERR_NOT_SUPPORTED;
201	}
202
203
204	RTDECL(int) RTTimerCreateEx(PRTTIMER ppTimer, uint64_t u64NanoInterval, unsigned fFlags, PFNRTTIMER pfnTimer, void pvUser)
205	{
206	PRTTIMER pTimer;
207	RTCPUID iCpu;
208	unsigned cCpus;
209
210	*ppTimer = NULL;
211
212	/*
213	* Validate flags.
214	*/
215	if (!RTTIMER_FLAGS_IS_VALID(fFlags))
216	return VERR_INVALID_PARAMETER;
217	if ( (fFlags & RTTIMER_FLAGS_CPU_SPECIFIC)
218	&& (fFlags & RTTIMER_FLAGS_CPU_ALL) != RTTIMER_FLAGS_CPU_ALL
219	&& !RTMpIsCpuOnline(fFlags & RTTIMER_FLAGS_CPU_MASK))
220	return (fFlags & RTTIMER_FLAGS_CPU_MASK) > RTMpGetMaxCpuId()
221	? VERR_CPU_NOT_FOUND
222	: VERR_CPU_OFFLINE;
223
224	/*
225	* Allocate the timer handler.
226	*/
227	cCpus = 1;
228	#ifdef CONFIG_SMP
229	if ((fFlags & RTTIMER_FLAGS_CPU_ALL) == RTTIMER_FLAGS_CPU_ALL)
230	{
231	cCpus = RTMpGetMaxCpuId() + 1;
232	Assert(cCpus <= RTCPUSET_MAX_CPUS); /* On linux we have a 1:1 relationship between cpuid and set index. */
233	AssertReturn(u64NanoInterval, VERR_NOT_IMPLEMENTED); /* We don't implement single shot on all cpus, sorry. */
234	}
235	#endif
236
237	pTimer = (PRTTIMER)RTMemAllocZ(RT_OFFSETOF(RTTIMER, aSubTimers[cCpus]));
238	if (!pTimer)
239	return VERR_NO_MEMORY;
240
241	/*
242	* Initialize it.
243	*/
244	pTimer->u32Magic = RTTIMER_MAGIC;
245	pTimer->hSpinlock = NIL_RTSPINLOCK;
246	pTimer->fSuspended = true;
247	#ifdef CONFIG_SMP
248	pTimer->fSpecificCpu = (fFlags & RTTIMER_FLAGS_CPU_SPECIFIC) && (fFlags & RTTIMER_FLAGS_CPU_ALL) != RTTIMER_FLAGS_CPU_ALL;
249	pTimer->fAllCpus = (fFlags & RTTIMER_FLAGS_CPU_ALL) == RTTIMER_FLAGS_CPU_ALL;
250	pTimer->idCpu = fFlags & RTTIMER_FLAGS_CPU_MASK;
251	#else
252	pTimer->fSpecificCpu = !!(fFlags & RTTIMER_FLAGS_CPU_SPECIFIC);
253	pTimer->idCpu = RTMpCpuId();
254	#endif
255	pTimer->cCpus = cCpus;
256	pTimer->pfnTimer = pfnTimer;
257	pTimer->pvUser = pvUser;
258	pTimer->u64NanoInterval = u64NanoInterval;
259
260	for (iCpu = 0; iCpu < cCpus; iCpu++)
261	{
262	#ifdef RT_USE_LINUX_HRTIMER
263	hrtimer_init(&pTimer->aSubTimers[iCpu].LnxTimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
264	pTimer->aSubTimers[iCpu].LnxTimer.function = rtTimerLinuxCallback;
265	#else
266	init_timer(&pTimer->aSubTimers[iCpu].LnxTimer);
267	pTimer->aSubTimers[iCpu].LnxTimer.data = (unsigned long)pTimer;
268	pTimer->aSubTimers[iCpu].LnxTimer.function = rtTimerLinuxCallback;
269	pTimer->aSubTimers[iCpu].LnxTimer.expires = jiffies;
270	#endif
271	pTimer->aSubTimers[iCpu].u64StartTS = 0;
272	pTimer->aSubTimers[iCpu].u64NextTS = 0;
273	pTimer->aSubTimers[iCpu].iTick = 0;
274	pTimer->aSubTimers[iCpu].pParent = pTimer;
275	pTimer->aSubTimers[iCpu].enmState = RTTIMERLNXSTATE_STOPPED;
276	}
277
278	#ifdef CONFIG_SMP
279	/*
280	* If this is running on ALL cpus, we'll have to register a callback
281	* for MP events (so timers can be started/stopped on cpus going
282	* online/offline). We also create the spinlock for syncrhonizing
283	* stop/start/mp-event.
284	*/
285	if (cCpus > 1)
286	{
287	int rc = RTSpinlockCreate(&pTimer->hSpinlock);
288	if (RT_SUCCESS(rc))
289	rc = RTMpNotificationRegister(rtTimerLinuxMpEvent, pTimer);
290	else
291	pTimer->hSpinlock = NIL_RTSPINLOCK;
292	if (RT_FAILURE(rc))
293	{
294	RTTimerDestroy(pTimer);
295	return rc;
296	}
297	}
298	#endif /* CONFIG_SMP */
299
300	*ppTimer = pTimer;
301	return VINF_SUCCESS;
302	}
303
304
305	RTDECL(int) RTTimerDestroy(PRTTIMER pTimer)
306	{
307	RTSPINLOCK hSpinlock;
308
309	/* It's ok to pass NULL pointer. */
310	if (pTimer == /NIL_RTTIMER/ NULL)
311	return VINF_SUCCESS;
312	AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
313	AssertPtrReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
314
315	/*
316	* Remove the MP notifications first because it'll reduce the risk of
317	* us overtaking any MP event that might theoretically be racing us here.
318	*/
319	hSpinlock = pTimer->hSpinlock;
320	#ifdef CONFIG_SMP
321	if ( pTimer->cCpus > 1
322	&& hSpinlock != NIL_RTSPINLOCK)
323	{
324	int rc = RTMpNotificationDeregister(rtTimerLinuxMpEvent, pTimer);
325	AssertRC(rc);
326	}
327	#endif /* CONFIG_SMP */
328
329	/*
330	* Stop the timer if it's running.
331	*/
332	if (!ASMAtomicUoReadBool(&pTimer->fSuspended)) /* serious paranoia */
333	RTTimerStop(pTimer);
334
335	/*
336	* Uninitialize the structure and free the associated resources.
337	* The spinlock goes last.
338	*/
339	ASMAtomicWriteU32(&pTimer->u32Magic, ~RTTIMER_MAGIC);
340	RTMemFree(pTimer);
341	if (hSpinlock != NIL_RTSPINLOCK)
342	RTSpinlockDestroy(hSpinlock);
343
344	return VINF_SUCCESS;
345	}
346
347
348	/**
349	* Sets the state.
350	*/
351	DECLINLINE(void) rtTimerLnxSetState(RTTIMERLNXSTATE volatile *penmState, RTTIMERLNXSTATE enmNewState)
352	{
353	ASMAtomicWriteU32((uint32_t volatile *)penmState, enmNewState);
354	}
355
356
357	/**
358	* Sets the state if it has a certain value.
359	*/
360	DECLINLINE(bool) rtTimerLnxCmpXchgState(RTTIMERLNXSTATE volatile *penmState, RTTIMERLNXSTATE enmNewState, RTTIMERLNXSTATE enmCurState)
361	{
362	return ASMAtomicCmpXchgU32((uint32_t volatile *)penmState, enmNewState, enmCurState);
363	}
364
365
366	/**
367	* Gets the state.
368	*/
369	DECLINLINE(RTTIMERLNXSTATE) rtTimerLnxGetState(RTTIMERLNXSTATE volatile *penmState)
370	{
371	return (RTTIMERLNXSTATE)ASMAtomicUoReadU32((uint32_t volatile *)penmState);
372	}
373
374
375	/**
376	* Starts a sub-timer (RTTimerStart).
377	*
378	* @param pSubTimer The sub-timer to start.
379	* @param u64Now The current timestamp (RTTimeNanoTS()).
380	* @param u64First The interval from u64Now to the first time the timer should fire.
381	*/
382	static void rtTimerLnxStartSubTimer(PRTTIMERLNXSUBTIMER pSubTimer, uint64_t u64Now, uint64_t u64First)
383	{
384	/*
385	* Calc when it should start firing.
386	*/
387	uint64_t u64NextTS = u64Now + u64First;
388	pSubTimer->u64StartTS = u64Now;
389	pSubTimer->u64NextTS = u64NextTS;
390
391	#ifdef RT_USE_LINUX_HRTIMER
392	{
393	/* ASSUMES RTTimeNanoTS() is implemented using ktime_get_ts(). */
394	struct timespec Ts;
395	ktime_t Kt;
396	Ts.tv_sec = u64NextTS / 1000000000;
397	Ts.tv_nsec = u64NextTS % 1000000000;
398	Kt = timespec_to_ktime(Ts);
399	hrtimer_start(&pSubTimer->LnxTimer, Kt, HRTIMER_MODE_ABS);
400	}
401	#else
402	{
403	uint64_t cJiffies = !u64First ? 0 : u64First / TICK_NSEC;
404	if (cJiffies > MAX_JIFFY_OFFSET)
405	cJiffies = MAX_JIFFY_OFFSET;
406	mod_timer(&pSubTimer->LnxTimer, jiffies + cJiffies);
407	}
408	#endif
409
410	rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE);
411	}
412
413
414	/**
415	* Stops a sub-timer (RTTimerStart and rtTimerLinuxMpEvent()).
416	*
417	* @param pSubTimer The sub-timer.
418	*/
419	static void rtTimerLnxStopSubTimer(PRTTIMERLNXSUBTIMER pSubTimer)
420	{
421	#ifdef RT_USE_LINUX_HRTIMER
422	hrtimer_cancel(&pSubTimer->LnxTimer);
423	#else
424	if (timer_pending(&pSubTimer->LnxTimer))
425	del_timer_sync(&pSubTimer->LnxTimer);
426	#endif
427
428	rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED);
429	}
430
431
432	/**
433	* Callback function use by RTTimerStart via RTMpOnSpecific to start
434	* a timer running on a specific CPU.
435	*
436	* @param idCpu The current CPU.
437	* @param pvUser1 Pointer to the timer.
438	* @param pvUser2 Pointer to the argument structure.
439	*/
440	static DECLCALLBACK(void) rtTimerLnxStartOnSpecificCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
441	{
442	PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
443	PRTTIMER pTimer = (PRTTIMER)pvUser1;
444	rtTimerLnxStartSubTimer(&pTimer->aSubTimers[0], pArgs->u64Now, pArgs->u64First);
445	}
446
447
448	RTDECL(int) RTTimerStart(PRTTIMER pTimer, uint64_t u64First)
449	{
450	RTTIMERLINUXSTARTONCPUARGS Args;
451	int rc2;
452
453	/*
454	* Validate.
455	*/
456	AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
457	AssertPtrReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
458
459	if (!pTimer->fSuspended)
460	return VERR_TIMER_ACTIVE;
461
462	Args.u64First = u64First;
463	#ifdef CONFIG_SMP
464	if (pTimer->fAllCpus)
465	return rtTimerLnxStartAll(pTimer, &Args);
466	#endif
467
468	/*
469	* This is pretty straight forwards.
470	*/
471	Args.u64Now = RTTimeNanoTS();
472	rtTimerLnxSetState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STARTING);
473	ASMAtomicWriteBool(&pTimer->fSuspended, false);
474	if (!pTimer->fSpecificCpu)
475	rtTimerLnxStartSubTimer(&pTimer->aSubTimers[0], Args.u64Now, Args.u64First);
476	else
477	{
478	rc2 = RTMpOnSpecific(pTimer->idCpu, rtTimerLnxStartOnSpecificCpu, pTimer, &Args);
479	if (RT_FAILURE(rc2))
480	{
481	/* Suspend it, the cpu id is probably invalid or offline. */
482	ASMAtomicWriteBool(&pTimer->fSuspended, true);
483	rtTimerLnxSetState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STOPPED);
484	return rc2;
485	}
486	}
487
488	return VINF_SUCCESS;
489	}
490
491
492	RTDECL(int) RTTimerStop(PRTTIMER pTimer)
493	{
494
495	/*
496	* Validate.
497	*/
498	AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
499	AssertPtrReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
500
501	if (pTimer->fSuspended)
502	return VERR_TIMER_SUSPENDED;
503
504	#ifdef CONFIG_SMP
505	if (pTimer->fAllCpus)
506	return rtTimerLnxStopAll(pTimer);
507	#endif
508
509	/*
510	* Cancel the timer.
511	*/
512	ASMAtomicWriteBool(&pTimer->fSuspended, true); /* just to be on the safe side. */
513	rtTimerLnxSetState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STOPPING);
514	rtTimerLnxStopSubTimer(&pTimer->aSubTimers[0]);
515
516	return VINF_SUCCESS;
517	}
518
519
520
521	#ifdef RT_USE_LINUX_HRTIMER
522	/**
523	* Timer callback function.
524	* @returns HRTIMER_NORESTART or HRTIMER_RESTART depending on whether it's a one-shot or interval timer.
525	* @param pHrTimer Pointer to the sub-timer structure.
526	*/
527	static enum hrtimer_restart rtTimerLinuxCallback(struct hrtimer *pHrTimer)
528	#else
529	/**
530	* Timer callback function.
531	* @param ulUser Address of the sub-timer structure.
532	*/
533	static void rtTimerLinuxCallback(unsigned long ulUser)
534	#endif
535	{
536	#ifdef RT_USE_LINUX_HRTIMER
537	enum hrtimer_restart rc;
538	PRTTIMERLNXSUBTIMER pSubTimer = (PRTTIMERLNXSUBTIMER)pHrTimer;
539	#else
540	PRTTIMERLNXSUBTIMER pSubTimer = (PRTTIMERLNXSUBTIMER)ulUser;
541	#endif
542	PRTTIMER pTimer = pSubTimer->pParent;
543
544	/*
545	* Don't call the handler if the timer has been suspended.
546	* Also, when running on all CPUS, make sure we don't call out twice
547	* on a CPU because of timer migration.
548	*
549	* For the specific cpu case, we're just ignoring timer migration for now... (bad)
550	*/
551	if ( pTimer->fSuspended
552	#ifdef CONFIG_SMP
553	\|\| ( pTimer->fAllCpus
554	&& (pSubTimer - &pTimer->aSubTimers[0]) != RTMpCpuId())
555	#endif
556	)
557	{
558	rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_ACTIVE);
559	# ifdef RT_USE_LINUX_HRTIMER
560	rc = HRTIMER_NORESTART;
561	# endif
562	}
563	else if (!pTimer->u64NanoInterval)
564	{
565	/*
566	* One shot timer, stop it before dispatching it.
567	*/
568	if (pTimer->cCpus == 1)
569	ASMAtomicWriteBool(&pTimer->fSuspended, true);
570	rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_ACTIVE);
571	#ifdef RT_USE_LINUX_HRTIMER
572	rc = HRTIMER_NORESTART;
573	#else
574	/* detached before we're called, nothing to do for this case. */
575	#endif
576
577	pTimer->pfnTimer(pTimer, pTimer->pvUser);
578	}
579	else
580	{
581	/*
582	* Interval timer, calculate the next timeout and re-arm it.
583	*
584	* The first time around, we'll re-adjust the u64StartTS to
585	* try prevent some jittering if we were started at a bad time.
586	* This may of course backfire with highres timers...
587	*/
588	const uint64_t u64NanoTS = RTTimeNanoTS();
589	#if 0 /* this needs to be tested before it's enabled. */
590	if ( pSubTimer->iTick == 0
591	&& pTimer->u64NanoInterval < u64NanoTS)
592	pSubTimer->u64StartTS = pSubTimer->u64FirstTS = u64NanoTS - 1000; /* ugly */
593	#endif
594	pSubTimer->iTick++;
595	pSubTimer->u64NextTS = pSubTimer->u64StartTS
596	+ pSubTimer->iTick * pTimer->u64NanoInterval;
597	if (pSubTimer->u64NextTS < u64NanoTS)
598	pSubTimer->u64NextTS = u64NanoTS + RTTimerGetSystemGranularity() / 2;
599
600	#ifdef RT_USE_LINUX_HRTIMER
601	{
602	/* ASSUMES RTTimeNanoTS() is implemented using ktime_get_ts(). */
603	struct timespec Ts;
604	Ts.tv_sec = pSubTimer->u64NextTS / 1000000000;
605	Ts.tv_nsec = pSubTimer->u64NextTS % 1000000000;
606	pSubTimer->LnxTimer.expires = timespec_to_ktime(Ts);
607	rc = HRTIMER_RESTART;
608	}
609	#else
610	{
611	uint64_t offDelta = pSubTimer->u64NextTS - u64NanoTS;
612	uint64_t cJiffies = offDelta / TICK_NSEC; /* (We end up doing 64-bit div here which ever way we go.) */
613	if (cJiffies > MAX_JIFFY_OFFSET)
614	cJiffies = MAX_JIFFY_OFFSET;
615	else if (cJiffies == 0)
616	cJiffies = 1;
617	mod_timer(&pSubTimer->LnxTimer, jiffies + cJiffies);
618	}
619	#endif
620
621	/*
622	* Run the timer.
623	*/
624	pTimer->pfnTimer(pTimer, pTimer->pvUser);
625	}
626
627	#ifdef RT_USE_LINUX_HRTIMER
628	return rc;
629	#endif
630	}
631
632
633
634	#ifdef CONFIG_SMP
635
636	/* */
637	/* */
638	/* The ALL CPUs stuff */
639	/* The ALL CPUs stuff */
640	/* The ALL CPUs stuff */
641	/* */
642	/* */
643
644
645
646
647	/**
648	* Per-cpu callback function (RTMpOnAll/RTMpOnSpecific).
649	*
650	* @param idCpu The current CPU.
651	* @param pvUser1 Pointer to the timer.
652	* @param pvUser2 Pointer to the argument structure.
653	*/
654	static DECLCALLBACK(void) rtTimerLnxStartAllOnCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
655	{
656	PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
657	PRTTIMER pTimer = (PRTTIMER)pvUser1;
658	Assert(idCpu < pTimer->cCpus);
659	rtTimerLnxStartSubTimer(&pTimer->aSubTimers[idCpu], pArgs->u64Now, pArgs->u64First);
660	}
661
662
663	/**
664	* Worker for RTTimerStart() that takes care of the ugly bit.s
665	*
666	* @returns RTTimerStart() return value.
667	* @param pTimer The timer.
668	* @param pArgs The argument structure.
669	*/
670	static int rtTimerLnxStartAll(PRTTIMER pTimer, PRTTIMERLINUXSTARTONCPUARGS pArgs)
671	{
672	RTSPINLOCKTMP Tmp;
673	RTCPUID iCpu;
674	RTCPUSET OnlineSet;
675	RTCPUSET OnlineSet2;
676	int rc2;
677
678	/*
679	* Prepare all the sub-timers for the startup and then flag the timer
680	* as a whole as non-suspended, make sure we get them all before
681	* clearing fSuspended as the MP handler will be waiting on this
682	* should something happen while we're looping.
683	*/
684	RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
685
686	do
687	{
688	RTMpGetOnlineSet(&OnlineSet);
689	for (iCpu = 0; iCpu <= pTimer->cCpus; iCpu++)
690	{
691	Assert(pTimer->aSubTimers[iCpu].enmState != RTTIMERLNXSTATE_MP_STOPPING);
692	rtTimerLnxSetState(&pTimer->aSubTimers[iCpu].enmState,
693	RTCpuSetIsMember(&OnlineSet, iCpu)
694	? RTTIMERLNXSTATE_STARTING
695	: RTTIMERLNXSTATE_STOPPED);
696	}
697	} while (!RTCpuSetIsEqual(&OnlineSet, RTMpGetOnlineSet(&OnlineSet2)));
698
699	ASMAtomicWriteBool(&pTimer->fSuspended, false);
700
701	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
702
703	/*
704	* Start them (can't find any exported function that allows me to
705	* do this without the cross calls).
706	*/
707	pArgs->u64Now = RTTimeNanoTS();
708	rc2 = RTMpOnAll(rtTimerLnxStartAllOnCpu, pTimer, pArgs);
709	AssertRC(rc2); /* screw this if it fails. */
710
711	/*
712	* Reset the sub-timers who didn't start up (ALL CPUs case).
713	* CPUs that comes online between the
714	*/
715	RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
716
717	for (iCpu = 0; iCpu <= pTimer->cCpus; iCpu++)
718	if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_STARTING))
719	{
720	/** @todo very odd case for a rainy day. Cpus that temporarily went offline while
721	* we were between calls needs to nudged as the MP handler will ignore events for
722	* them because of the STARTING state. This is an extremely unlikely case - not that
723	* that means anything in my experience... ;-) */
724	}
725
726	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
727
728	return VINF_SUCCESS;
729	}
730
731
732	/**
733	* Worker for RTTimerStop() that takes care of the ugly SMP bits.
734	*
735	* @returns RTTimerStop() return value.
736	* @param pTimer The timer (valid).
737	*/
738	static int rtTimerLnxStopAll(PRTTIMER pTimer)
739	{
740	RTCPUID iCpu;
741	RTSPINLOCKTMP Tmp;
742
743
744	/*
745	* Mark the timer as suspended and flag all timers as stopping, except
746	* for those being stopped by an MP event.
747	*/
748	RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
749
750	ASMAtomicWriteBool(&pTimer->fSuspended, true);
751	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
752	{
753	RTTIMERLNXSTATE enmState;
754	do
755	{
756	enmState = rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState);
757	if ( enmState == RTTIMERLNXSTATE_STOPPED
758	\|\| enmState == RTTIMERLNXSTATE_MP_STOPPING)
759	break;
760	Assert(enmState == RTTIMERLNXSTATE_ACTIVE);
761	} while (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState, RTTIMERLNXSTATE_STOPPING, enmState));
762	}
763
764	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
765
766	/*
767	* Do the actual stopping. Fortunately, this doesn't require any IPIs.
768	* Unfortunately it cannot be done synchronously from within the spinlock,
769	* because we might end up in an active waiting for a handler to complete.
770	*/
771	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
772	if (rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState) == RTTIMERLNXSTATE_STOPPING)
773	rtTimerLnxStopSubTimer(&pTimer->aSubTimers[iCpu]);
774
775	return VINF_SUCCESS;
776	}
777
778
779	/**
780	* Per-cpu callback function (RTMpOnSpecific) used by rtTimerLinuxMpEvent()
781	* to start a sub-timer on a cpu that just have come online.
782	*
783	* @param idCpu The current CPU.
784	* @param pvUser1 Pointer to the timer.
785	* @param pvUser2 Pointer to the argument structure.
786	*/
787	static DECLCALLBACK(void) rtTimerLinuxMpStartOnCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
788	{
789	PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
790	PRTTIMER pTimer = (PRTTIMER)pvUser1;
791	RTSPINLOCK hSpinlock;
792	Assert(idCpu < pTimer->cCpus);
793
794	/*
795	* We have to be kind of careful here as we might RTTimerStop (and RTTimerDestroy),
796	* thus the paranoia.
797	*/
798	hSpinlock = pTimer->hSpinlock;
799	if ( hSpinlock != NIL_RTSPINLOCK
800	&& pTimer->u32Magic == RTTIMER_MAGIC)
801	{
802	RTSPINLOCKTMP Tmp;
803	RTSpinlockAcquire(hSpinlock, &Tmp);
804
805	if ( !pTimer->fSuspended
806	&& pTimer->u32Magic == RTTIMER_MAGIC)
807	{
808	/* We're sane and the timer is not suspended yet. */
809	PRTTIMERLNXSUBTIMER pSubTimer = &pTimer->aSubTimers[idCpu];
810	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STARTING, RTTIMERLNXSTATE_STOPPED))
811	rtTimerLnxStartSubTimer(pSubTimer, pArgs->u64Now, pArgs->u64First);
812	}
813
814	RTSpinlockRelease(hSpinlock, &Tmp);
815	}
816	}
817
818
819	/**
820	* MP event notification callback.
821	*
822	* @param enmEvent The event.
823	* @param idCpu The cpu it applies to.
824	* @param pvUser The timer.
825	*/
826	static DECLCALLBACK(void) rtTimerLinuxMpEvent(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvUser)
827	{
828	PRTTIMER pTimer = (PRTTIMER)pvUser;
829	PRTTIMERLNXSUBTIMER pSubTimer = &pTimer->aSubTimers[idCpu];
830	RTSPINLOCK hSpinlock;
831	RTSPINLOCKTMP Tmp;
832
833	Assert(idCpu < pTimer->cCpus);
834
835	/*
836	* Some initial paranoia.
837	*/
838	if (pTimer->u32Magic != RTTIMER_MAGIC)
839	return;
840	hSpinlock = pTimer->hSpinlock;
841	if (hSpinlock == NIL_RTSPINLOCK)
842	return;
843
844	RTSpinlockAcquireNoInts(hSpinlock, &Tmp);
845
846	/* Is it active? */
847	if ( !pTimer->fSuspended
848	&& !pTimer->u32Magic == RTTIMER_MAGIC)
849	{
850	switch (enmEvent)
851	{
852	/*
853	* Try do it without leaving the spin lock, but if we have to, retake it
854	* when we're on the right cpu.
855	*/
856	case RTMPEVENT_ONLINE:
857	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STARTING, RTTIMERLNXSTATE_STOPPED))
858	{
859	RTTIMERLINUXSTARTONCPUARGS Args;
860	Args.u64Now = RTTimeNanoTS();
861	Args.u64First = pTimer->u64NanoInterval;
862
863	if (RTMpCpuId() == idCpu)
864	rtTimerLnxStartSubTimer(pSubTimer, Args.u64Now, Args.u64First);
865	else
866	{
867	rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED); /* we'll recheck it. */
868	RTSpinlockReleaseNoInts(hSpinlock, &Tmp);
869
870	RTMpOnSpecific(idCpu, rtTimerLinuxMpStartOnCpu, pTimer, &Args);
871	return; /* we've left the spinlock */
872	}
873	}
874	break;
875
876	/*
877	* The CPU is (going) offline, make sure the sub-timer is stopped.
878	*
879	* Linux will migrate it to a different CPU, but we don't want this. The
880	* timer function is checking for this.
881	*/
882	case RTMPEVENT_OFFLINE:
883	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STOPPING, RTTIMERLNXSTATE_ACTIVE))
884	{
885	RTSpinlockAcquireNoInts(hSpinlock, &Tmp);
886
887	rtTimerLnxStopSubTimer(pSubTimer);
888	return; /* we've left the spinlock */
889	}
890	break;
891	}
892	}
893
894	RTSpinlockAcquireNoInts(hSpinlock, &Tmp);
895	}
896
897	#endif /* CONFIG_SMP */

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/r0drv/linux/timer-r0drv-linux.c@ 9368

Download in other formats: