timer-r0drv-linux.c@ 9370

Last change on this file since 9370 was 9370, checked in by vboxsync, 17 years ago
comment.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 28.2 KB

Line
1	/* $Id: timer-r0drv-linux.c 9370 2008-06-03 22:31:33Z vboxsync $ */
2	/** @file
3	* IPRT - Timers, Ring-0 Driver, Linux.
4	*/
5
6	/*
7	* Copyright (C) 2006-2008 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.215389.xyz. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* The contents of this file may alternatively be used under the terms
18	* of the Common Development and Distribution License Version 1.0
19	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	* VirtualBox OSE distribution, in which case the provisions of the
21	* CDDL are applicable instead of those of the GPL.
22	*
23	* You may elect to license modified versions of this file under the
24	* terms and conditions of either the GPL or the CDDL or both.
25	*
26	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
27	* Clara, CA 95054 USA or visit http://www.sun.com if you need
28	* additional information or have any questions.
29	*/
30
31	/*******************************************************************************
32	* Header Files *
33	*******************************************************************************/
34	#include "the-linux-kernel.h"
35
36	#include <iprt/timer.h>
37	#include <iprt/time.h>
38	#include <iprt/mp.h>
39	#include <iprt/cpuset.h>
40	#include <iprt/spinlock.h>
41	#include <iprt/err.h>
42	#include <iprt/asm.h>
43	#include <iprt/assert.h>
44	#include <iprt/alloc.h>
45
46	#include "internal/magics.h"
47
48	#if !defined(RT_USE_LINUX_HRTIMER) && LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 23) /* ?? */
49	# define RT_USE_LINUX_HRTIMER
50	#endif
51
52	/* This check must match the ktime usage in rtTimeGetSystemNanoTS() / time-r0drv-linux.c. */
53	#if defined(RT_USE_LINUX_HRTIMER) \
54	&& LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 16)
55	# error "RT_USE_LINUX_HRTIMER requires 2.6.16 or later, sorry."
56	#endif
57
58
59	/*******************************************************************************
60	* Structures and Typedefs *
61	*******************************************************************************/
62	/**
63	* Timer state machine.
64	*
65	* This is used to try handle the issues with MP events and
66	* timers that runs on all CPUs. It's relatively nasty :-/
67	*/
68	typedef enum RTTIMERLNXSTATE
69	{
70	/** Stopped. */
71	RTTIMERLNXSTATE_STOPPED = 0,
72	/** Transient state; next ACTIVE. */
73	RTTIMERLNXSTATE_STARTING,
74	/** Transient state; next ACTIVE. (not really necessary) */
75	RTTIMERLNXSTATE_MP_STARTING,
76	/** Active. */
77	RTTIMERLNXSTATE_ACTIVE,
78	/** Transient state; next STOPPED. */
79	RTTIMERLNXSTATE_STOPPING,
80	/** Transient state; next STOPPED. */
81	RTTIMERLNXSTATE_MP_STOPPING,
82	/** The usual 32-bit hack. */
83	RTTIMERLNXSTATE_32BIT_HACK = 0x7fffffff
84	} RTTIMERLNXSTATE;
85
86
87	/**
88	* A Linux sub-timer.
89	*/
90	typedef struct RTTIMERLNXSUBTIMER
91	{
92	/** The linux timer structure. */
93	#ifdef RT_USE_LINUX_HRTIMER
94	struct hrtimer LnxTimer;
95	#else
96	struct timer_list LnxTimer;
97	#endif
98	/** The start of the current run (ns).
99	* This is used to calculate when the timer ought to fire the next time. */
100	uint64_t u64StartTS;
101	/** The start of the current run (ns).
102	* This is used to calculate when the timer ought to fire the next time. */
103	uint64_t u64NextTS;
104	/** The current tick number (since u64StartTS). */
105	uint64_t iTick;
106	/** Pointer to the parent timer. */
107	PRTTIMER pParent;
108	/** The current sub-timer state. */
109	RTTIMERLNXSTATE volatile enmState;
110	} RTTIMERLNXSUBTIMER;
111	/** Pointer to a linux sub-timer. */
112	typedef RTTIMERLNXSUBTIMER *PRTTIMERLNXSUBTIMER;
113	AssertCompileMemberOffset(RTTIMERLNXSUBTIMER, LnxTimer, 0);
114
115
116	/**
117	* The internal representation of an Linux timer handle.
118	*/
119	typedef struct RTTIMER
120	{
121	/** Magic.
122	* This is RTTIMER_MAGIC, but changes to something else before the timer
123	* is destroyed to indicate clearly that thread should exit. */
124	uint32_t volatile u32Magic;
125	/** Spinlock synchronizing the fSuspended and MP event handling.
126	* This is NIL_RTSPINLOCK if cCpus == 1. */
127	RTSPINLOCK hSpinlock;
128	/** Flag indicating the the timer is suspended. */
129	bool volatile fSuspended;
130	/** Whether the timer must run on one specific CPU or not. */
131	bool fSpecificCpu;
132	#ifdef CONFIG_SMP
133	/** Whether the timer must run on all CPUs or not. */
134	bool fAllCpus;
135	#endif /* else: All -> specific on non-SMP kernels */
136	/** The CPU it must run on if fSpecificCpu is set. */
137	RTCPUID idCpu;
138	/** The number of CPUs this timer should run on. */
139	RTCPUID cCpus;
140	/** Callback. */
141	PFNRTTIMER pfnTimer;
142	/** User argument. */
143	void *pvUser;
144	/** The timer interval. 0 if one-shot. */
145	uint64_t u64NanoInterval;
146	/** Sub-timers.
147	* Normally there is just one, but for RTTIMER_FLAGS_CPU_ALL this will contain
148	* an entry for all possible cpus. In that case the index will be the same as
149	* for the RTCpuSet. */
150	RTTIMERLNXSUBTIMER aSubTimers[1];
151	} RTTIMER;
152
153
154	/**
155	* A rtTimerLinuxStartOnCpu and rtTimerLinuxStartOnCpu argument package.
156	*/
157	typedef struct RTTIMERLINUXSTARTONCPUARGS
158	{
159	/** The current time (RTTimeNanoTS). */
160	uint64_t u64Now;
161	/** When to start firing (delta). */
162	uint64_t u64First;
163	} RTTIMERLINUXSTARTONCPUARGS;
164	/** Pointer to a rtTimerLinuxStartOnCpu argument package. */
165	typedef RTTIMERLINUXSTARTONCPUARGS *PRTTIMERLINUXSTARTONCPUARGS;
166
167
168	/*******************************************************************************
169	* Internal Functions *
170	*******************************************************************************/
171	#ifdef RT_USE_LINUX_HRTIMER
172	static enum hrtimer_restart rtTimerLinuxCallback(struct hrtimer *pHrTimer);
173	#else
174	static void rtTimerLinuxCallback(unsigned long ulUser);
175	#endif
176	#ifdef CONFIG_SMP
177	static int rtTimerLnxStartAll(PRTTIMER pTimer, PRTTIMERLINUXSTARTONCPUARGS pArgs);
178	static int rtTimerLnxStopAll(PRTTIMER pTimer);
179	static DECLCALLBACK(void) rtTimerLinuxMpEvent(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvUser);
180	#endif
181
182
183	RTDECL(uint32_t) RTTimerGetSystemGranularity(void)
184	{
185	#ifdef RT_USE_LINUX_HRTIMER
186	/** @todo later... */
187	return 1000000000 / HZ; /* ns */
188	#else
189	return 1000000000 / HZ; /* ns */
190	#endif
191	}
192
193
194	RTDECL(int) RTTimerRequestSystemGranularity(uint32_t u32Request, uint32_t *pu32Granted)
195	{
196	return VERR_NOT_SUPPORTED;
197	}
198
199
200	RTDECL(int) RTTimerReleaseSystemGranularity(uint32_t u32Granted)
201	{
202	return VERR_NOT_SUPPORTED;
203	}
204
205
206	RTDECL(int) RTTimerCreateEx(PRTTIMER ppTimer, uint64_t u64NanoInterval, unsigned fFlags, PFNRTTIMER pfnTimer, void pvUser)
207	{
208	PRTTIMER pTimer;
209	RTCPUID iCpu;
210	unsigned cCpus;
211
212	*ppTimer = NULL;
213
214	/*
215	* Validate flags.
216	*/
217	if (!RTTIMER_FLAGS_IS_VALID(fFlags))
218	return VERR_INVALID_PARAMETER;
219	if ( (fFlags & RTTIMER_FLAGS_CPU_SPECIFIC)
220	&& (fFlags & RTTIMER_FLAGS_CPU_ALL) != RTTIMER_FLAGS_CPU_ALL
221	&& !RTMpIsCpuOnline(fFlags & RTTIMER_FLAGS_CPU_MASK))
222	return (fFlags & RTTIMER_FLAGS_CPU_MASK) > RTMpGetMaxCpuId()
223	? VERR_CPU_NOT_FOUND
224	: VERR_CPU_OFFLINE;
225
226	/*
227	* Allocate the timer handler.
228	*/
229	cCpus = 1;
230	#ifdef CONFIG_SMP
231	if ((fFlags & RTTIMER_FLAGS_CPU_ALL) == RTTIMER_FLAGS_CPU_ALL)
232	{
233	cCpus = RTMpGetMaxCpuId() + 1;
234	Assert(cCpus <= RTCPUSET_MAX_CPUS); /* On linux we have a 1:1 relationship between cpuid and set index. */
235	AssertReturn(u64NanoInterval, VERR_NOT_IMPLEMENTED); /* We don't implement single shot on all cpus, sorry. */
236	}
237	#endif
238
239	pTimer = (PRTTIMER)RTMemAllocZ(RT_OFFSETOF(RTTIMER, aSubTimers[cCpus]));
240	if (!pTimer)
241	return VERR_NO_MEMORY;
242
243	/*
244	* Initialize it.
245	*/
246	pTimer->u32Magic = RTTIMER_MAGIC;
247	pTimer->hSpinlock = NIL_RTSPINLOCK;
248	pTimer->fSuspended = true;
249	#ifdef CONFIG_SMP
250	pTimer->fSpecificCpu = (fFlags & RTTIMER_FLAGS_CPU_SPECIFIC) && (fFlags & RTTIMER_FLAGS_CPU_ALL) != RTTIMER_FLAGS_CPU_ALL;
251	pTimer->fAllCpus = (fFlags & RTTIMER_FLAGS_CPU_ALL) == RTTIMER_FLAGS_CPU_ALL;
252	pTimer->idCpu = fFlags & RTTIMER_FLAGS_CPU_MASK;
253	#else
254	pTimer->fSpecificCpu = !!(fFlags & RTTIMER_FLAGS_CPU_SPECIFIC);
255	pTimer->idCpu = RTMpCpuId();
256	#endif
257	pTimer->cCpus = cCpus;
258	pTimer->pfnTimer = pfnTimer;
259	pTimer->pvUser = pvUser;
260	pTimer->u64NanoInterval = u64NanoInterval;
261
262	for (iCpu = 0; iCpu < cCpus; iCpu++)
263	{
264	#ifdef RT_USE_LINUX_HRTIMER
265	hrtimer_init(&pTimer->aSubTimers[iCpu].LnxTimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
266	pTimer->aSubTimers[iCpu].LnxTimer.function = rtTimerLinuxCallback;
267	#else
268	init_timer(&pTimer->aSubTimers[iCpu].LnxTimer);
269	pTimer->aSubTimers[iCpu].LnxTimer.data = (unsigned long)pTimer;
270	pTimer->aSubTimers[iCpu].LnxTimer.function = rtTimerLinuxCallback;
271	pTimer->aSubTimers[iCpu].LnxTimer.expires = jiffies;
272	#endif
273	pTimer->aSubTimers[iCpu].u64StartTS = 0;
274	pTimer->aSubTimers[iCpu].u64NextTS = 0;
275	pTimer->aSubTimers[iCpu].iTick = 0;
276	pTimer->aSubTimers[iCpu].pParent = pTimer;
277	pTimer->aSubTimers[iCpu].enmState = RTTIMERLNXSTATE_STOPPED;
278	}
279
280	#ifdef CONFIG_SMP
281	/*
282	* If this is running on ALL cpus, we'll have to register a callback
283	* for MP events (so timers can be started/stopped on cpus going
284	* online/offline). We also create the spinlock for syncrhonizing
285	* stop/start/mp-event.
286	*/
287	if (cCpus > 1)
288	{
289	int rc = RTSpinlockCreate(&pTimer->hSpinlock);
290	if (RT_SUCCESS(rc))
291	rc = RTMpNotificationRegister(rtTimerLinuxMpEvent, pTimer);
292	else
293	pTimer->hSpinlock = NIL_RTSPINLOCK;
294	if (RT_FAILURE(rc))
295	{
296	RTTimerDestroy(pTimer);
297	return rc;
298	}
299	}
300	#endif /* CONFIG_SMP */
301
302	*ppTimer = pTimer;
303	return VINF_SUCCESS;
304	}
305
306
307	RTDECL(int) RTTimerDestroy(PRTTIMER pTimer)
308	{
309	RTSPINLOCK hSpinlock;
310
311	/* It's ok to pass NULL pointer. */
312	if (pTimer == /NIL_RTTIMER/ NULL)
313	return VINF_SUCCESS;
314	AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
315	AssertPtrReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
316
317	/*
318	* Remove the MP notifications first because it'll reduce the risk of
319	* us overtaking any MP event that might theoretically be racing us here.
320	*/
321	hSpinlock = pTimer->hSpinlock;
322	#ifdef CONFIG_SMP
323	if ( pTimer->cCpus > 1
324	&& hSpinlock != NIL_RTSPINLOCK)
325	{
326	int rc = RTMpNotificationDeregister(rtTimerLinuxMpEvent, pTimer);
327	AssertRC(rc);
328	}
329	#endif /* CONFIG_SMP */
330
331	/*
332	* Stop the timer if it's running.
333	*/
334	if (!ASMAtomicUoReadBool(&pTimer->fSuspended)) /* serious paranoia */
335	RTTimerStop(pTimer);
336
337	/*
338	* Uninitialize the structure and free the associated resources.
339	* The spinlock goes last.
340	*/
341	ASMAtomicWriteU32(&pTimer->u32Magic, ~RTTIMER_MAGIC);
342	RTMemFree(pTimer);
343	if (hSpinlock != NIL_RTSPINLOCK)
344	RTSpinlockDestroy(hSpinlock);
345
346	return VINF_SUCCESS;
347	}
348
349
350	/**
351	* Sets the state.
352	*/
353	DECLINLINE(void) rtTimerLnxSetState(RTTIMERLNXSTATE volatile *penmState, RTTIMERLNXSTATE enmNewState)
354	{
355	ASMAtomicWriteU32((uint32_t volatile *)penmState, enmNewState);
356	}
357
358
359	/**
360	* Sets the state if it has a certain value.
361	*/
362	DECLINLINE(bool) rtTimerLnxCmpXchgState(RTTIMERLNXSTATE volatile *penmState, RTTIMERLNXSTATE enmNewState, RTTIMERLNXSTATE enmCurState)
363	{
364	return ASMAtomicCmpXchgU32((uint32_t volatile *)penmState, enmNewState, enmCurState);
365	}
366
367
368	/**
369	* Gets the state.
370	*/
371	DECLINLINE(RTTIMERLNXSTATE) rtTimerLnxGetState(RTTIMERLNXSTATE volatile *penmState)
372	{
373	return (RTTIMERLNXSTATE)ASMAtomicUoReadU32((uint32_t volatile *)penmState);
374	}
375
376
377	/**
378	* Starts a sub-timer (RTTimerStart).
379	*
380	* @param pSubTimer The sub-timer to start.
381	* @param u64Now The current timestamp (RTTimeNanoTS()).
382	* @param u64First The interval from u64Now to the first time the timer should fire.
383	*/
384	static void rtTimerLnxStartSubTimer(PRTTIMERLNXSUBTIMER pSubTimer, uint64_t u64Now, uint64_t u64First)
385	{
386	/*
387	* Calc when it should start firing.
388	*/
389	uint64_t u64NextTS = u64Now + u64First;
390	pSubTimer->u64StartTS = u64Now;
391	pSubTimer->u64NextTS = u64NextTS;
392
393	#ifdef RT_USE_LINUX_HRTIMER
394	{
395	/* ASSUMES RTTimeNanoTS() is implemented using ktime_get_ts(). */
396	struct timespec Ts;
397	ktime_t Kt;
398	Ts.tv_sec = u64NextTS / 1000000000;
399	Ts.tv_nsec = u64NextTS % 1000000000;
400	Kt = timespec_to_ktime(Ts);
401	hrtimer_start(&pSubTimer->LnxTimer, Kt, HRTIMER_MODE_ABS);
402	}
403	#else
404	{
405	uint64_t cJiffies = !u64First ? 0 : u64First / TICK_NSEC;
406	if (cJiffies > MAX_JIFFY_OFFSET)
407	cJiffies = MAX_JIFFY_OFFSET;
408	mod_timer(&pSubTimer->LnxTimer, jiffies + cJiffies);
409	}
410	#endif
411
412	rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE);
413	}
414
415
416	/**
417	* Stops a sub-timer (RTTimerStart and rtTimerLinuxMpEvent()).
418	*
419	* @param pSubTimer The sub-timer.
420	*/
421	static void rtTimerLnxStopSubTimer(PRTTIMERLNXSUBTIMER pSubTimer)
422	{
423	#ifdef RT_USE_LINUX_HRTIMER
424	hrtimer_cancel(&pSubTimer->LnxTimer);
425	#else
426	if (timer_pending(&pSubTimer->LnxTimer))
427	del_timer_sync(&pSubTimer->LnxTimer);
428	#endif
429
430	rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED);
431	}
432
433
434	/**
435	* Callback function use by RTTimerStart via RTMpOnSpecific to start
436	* a timer running on a specific CPU.
437	*
438	* @param idCpu The current CPU.
439	* @param pvUser1 Pointer to the timer.
440	* @param pvUser2 Pointer to the argument structure.
441	*/
442	static DECLCALLBACK(void) rtTimerLnxStartOnSpecificCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
443	{
444	PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
445	PRTTIMER pTimer = (PRTTIMER)pvUser1;
446	rtTimerLnxStartSubTimer(&pTimer->aSubTimers[0], pArgs->u64Now, pArgs->u64First);
447	}
448
449
450	RTDECL(int) RTTimerStart(PRTTIMER pTimer, uint64_t u64First)
451	{
452	RTTIMERLINUXSTARTONCPUARGS Args;
453	int rc2;
454
455	/*
456	* Validate.
457	*/
458	AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
459	AssertPtrReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
460
461	if (!pTimer->fSuspended)
462	return VERR_TIMER_ACTIVE;
463
464	Args.u64First = u64First;
465	#ifdef CONFIG_SMP
466	if (pTimer->fAllCpus)
467	return rtTimerLnxStartAll(pTimer, &Args);
468	#endif
469
470	/*
471	* This is pretty straight forwards.
472	*/
473	Args.u64Now = RTTimeNanoTS();
474	rtTimerLnxSetState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STARTING);
475	ASMAtomicWriteBool(&pTimer->fSuspended, false);
476	if (!pTimer->fSpecificCpu)
477	rtTimerLnxStartSubTimer(&pTimer->aSubTimers[0], Args.u64Now, Args.u64First);
478	else
479	{
480	rc2 = RTMpOnSpecific(pTimer->idCpu, rtTimerLnxStartOnSpecificCpu, pTimer, &Args);
481	if (RT_FAILURE(rc2))
482	{
483	/* Suspend it, the cpu id is probably invalid or offline. */
484	ASMAtomicWriteBool(&pTimer->fSuspended, true);
485	rtTimerLnxSetState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STOPPED);
486	return rc2;
487	}
488	}
489
490	return VINF_SUCCESS;
491	}
492
493
494	RTDECL(int) RTTimerStop(PRTTIMER pTimer)
495	{
496
497	/*
498	* Validate.
499	*/
500	AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
501	AssertPtrReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
502
503	if (pTimer->fSuspended)
504	return VERR_TIMER_SUSPENDED;
505
506	#ifdef CONFIG_SMP
507	if (pTimer->fAllCpus)
508	return rtTimerLnxStopAll(pTimer);
509	#endif
510
511	/*
512	* Cancel the timer.
513	*/
514	ASMAtomicWriteBool(&pTimer->fSuspended, true); /* just to be on the safe side. */
515	rtTimerLnxSetState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STOPPING);
516	rtTimerLnxStopSubTimer(&pTimer->aSubTimers[0]);
517
518	return VINF_SUCCESS;
519	}
520
521
522
523	#ifdef RT_USE_LINUX_HRTIMER
524	/**
525	* Timer callback function.
526	* @returns HRTIMER_NORESTART or HRTIMER_RESTART depending on whether it's a one-shot or interval timer.
527	* @param pHrTimer Pointer to the sub-timer structure.
528	*/
529	static enum hrtimer_restart rtTimerLinuxCallback(struct hrtimer *pHrTimer)
530	#else
531	/**
532	* Timer callback function.
533	* @param ulUser Address of the sub-timer structure.
534	*/
535	static void rtTimerLinuxCallback(unsigned long ulUser)
536	#endif
537	{
538	#ifdef RT_USE_LINUX_HRTIMER
539	enum hrtimer_restart rc;
540	PRTTIMERLNXSUBTIMER pSubTimer = (PRTTIMERLNXSUBTIMER)pHrTimer;
541	#else
542	PRTTIMERLNXSUBTIMER pSubTimer = (PRTTIMERLNXSUBTIMER)ulUser;
543	#endif
544	PRTTIMER pTimer = pSubTimer->pParent;
545
546	/*
547	* Don't call the handler if the timer has been suspended.
548	* Also, when running on all CPUS, make sure we don't call out twice
549	* on a CPU because of timer migration.
550	*
551	* For the specific cpu case, we're just ignoring timer migration for now... (bad)
552	*/
553	if ( pTimer->fSuspended
554	#ifdef CONFIG_SMP
555	\|\| ( pTimer->fAllCpus
556	&& (pSubTimer - &pTimer->aSubTimers[0]) != RTMpCpuId())
557	#endif
558	)
559	{
560	rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_ACTIVE);
561	# ifdef RT_USE_LINUX_HRTIMER
562	rc = HRTIMER_NORESTART;
563	# endif
564	}
565	else if (!pTimer->u64NanoInterval)
566	{
567	/*
568	* One shot timer, stop it before dispatching it.
569	*/
570	if (pTimer->cCpus == 1)
571	ASMAtomicWriteBool(&pTimer->fSuspended, true);
572	rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_ACTIVE);
573	#ifdef RT_USE_LINUX_HRTIMER
574	rc = HRTIMER_NORESTART;
575	#else
576	/* detached before we're called, nothing to do for this case. */
577	#endif
578
579	pTimer->pfnTimer(pTimer, pTimer->pvUser);
580	}
581	else
582	{
583	/*
584	* Interval timer, calculate the next timeout and re-arm it.
585	*
586	* The first time around, we'll re-adjust the u64StartTS to
587	* try prevent some jittering if we were started at a bad time.
588	* This may of course backfire with highres timers...
589	*/
590	const uint64_t u64NanoTS = RTTimeNanoTS();
591	#if 0 /* this needs to be tested before it's enabled. */
592	if ( pSubTimer->iTick == 0
593	&& pTimer->u64NanoInterval < u64NanoTS)
594	pSubTimer->u64StartTS = pSubTimer->u64FirstTS = u64NanoTS - 1000; /* ugly */
595	#endif
596	pSubTimer->iTick++;
597	pSubTimer->u64NextTS = pSubTimer->u64StartTS
598	+ pSubTimer->iTick * pTimer->u64NanoInterval;
599	if (pSubTimer->u64NextTS < u64NanoTS)
600	pSubTimer->u64NextTS = u64NanoTS + RTTimerGetSystemGranularity() / 2;
601
602	#ifdef RT_USE_LINUX_HRTIMER
603	{
604	/* ASSUMES RTTimeNanoTS() is implemented using ktime_get_ts(). */
605	struct timespec Ts;
606	Ts.tv_sec = pSubTimer->u64NextTS / 1000000000;
607	Ts.tv_nsec = pSubTimer->u64NextTS % 1000000000;
608	pSubTimer->LnxTimer.expires = timespec_to_ktime(Ts);
609	rc = HRTIMER_RESTART;
610	}
611	#else
612	{
613	uint64_t offDelta = pSubTimer->u64NextTS - u64NanoTS;
614	uint64_t cJiffies = offDelta / TICK_NSEC; /* (We end up doing 64-bit div here which ever way we go.) */
615	if (cJiffies > MAX_JIFFY_OFFSET)
616	cJiffies = MAX_JIFFY_OFFSET;
617	else if (cJiffies == 0)
618	cJiffies = 1;
619	mod_timer(&pSubTimer->LnxTimer, jiffies + cJiffies);
620	}
621	#endif
622
623	/*
624	* Run the timer.
625	*/
626	pTimer->pfnTimer(pTimer, pTimer->pvUser);
627	}
628
629	#ifdef RT_USE_LINUX_HRTIMER
630	return rc;
631	#endif
632	}
633
634
635
636	#ifdef CONFIG_SMP
637
638	/* */
639	/* */
640	/* The ALL CPUs stuff */
641	/* The ALL CPUs stuff */
642	/* The ALL CPUs stuff */
643	/* */
644	/* */
645
646
647
648
649	/**
650	* Per-cpu callback function (RTMpOnAll/RTMpOnSpecific).
651	*
652	* @param idCpu The current CPU.
653	* @param pvUser1 Pointer to the timer.
654	* @param pvUser2 Pointer to the argument structure.
655	*/
656	static DECLCALLBACK(void) rtTimerLnxStartAllOnCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
657	{
658	PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
659	PRTTIMER pTimer = (PRTTIMER)pvUser1;
660	Assert(idCpu < pTimer->cCpus);
661	rtTimerLnxStartSubTimer(&pTimer->aSubTimers[idCpu], pArgs->u64Now, pArgs->u64First);
662	}
663
664
665	/**
666	* Worker for RTTimerStart() that takes care of the ugly bit.s
667	*
668	* @returns RTTimerStart() return value.
669	* @param pTimer The timer.
670	* @param pArgs The argument structure.
671	*/
672	static int rtTimerLnxStartAll(PRTTIMER pTimer, PRTTIMERLINUXSTARTONCPUARGS pArgs)
673	{
674	RTSPINLOCKTMP Tmp;
675	RTCPUID iCpu;
676	RTCPUSET OnlineSet;
677	RTCPUSET OnlineSet2;
678	int rc2;
679
680	/*
681	* Prepare all the sub-timers for the startup and then flag the timer
682	* as a whole as non-suspended, make sure we get them all before
683	* clearing fSuspended as the MP handler will be waiting on this
684	* should something happen while we're looping.
685	*/
686	RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
687
688	do
689	{
690	RTMpGetOnlineSet(&OnlineSet);
691	for (iCpu = 0; iCpu <= pTimer->cCpus; iCpu++)
692	{
693	Assert(pTimer->aSubTimers[iCpu].enmState != RTTIMERLNXSTATE_MP_STOPPING);
694	rtTimerLnxSetState(&pTimer->aSubTimers[iCpu].enmState,
695	RTCpuSetIsMember(&OnlineSet, iCpu)
696	? RTTIMERLNXSTATE_STARTING
697	: RTTIMERLNXSTATE_STOPPED);
698	}
699	} while (!RTCpuSetIsEqual(&OnlineSet, RTMpGetOnlineSet(&OnlineSet2)));
700
701	ASMAtomicWriteBool(&pTimer->fSuspended, false);
702
703	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
704
705	/*
706	* Start them (can't find any exported function that allows me to
707	* do this without the cross calls).
708	*/
709	pArgs->u64Now = RTTimeNanoTS();
710	rc2 = RTMpOnAll(rtTimerLnxStartAllOnCpu, pTimer, pArgs);
711	AssertRC(rc2); /* screw this if it fails. */
712
713	/*
714	* Reset the sub-timers who didn't start up (ALL CPUs case).
715	* CPUs that comes online between the
716	*/
717	RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
718
719	for (iCpu = 0; iCpu <= pTimer->cCpus; iCpu++)
720	if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_STARTING))
721	{
722	/** @todo very odd case for a rainy day. Cpus that temporarily went offline while
723	* we were between calls needs to nudged as the MP handler will ignore events for
724	* them because of the STARTING state. This is an extremely unlikely case - not that
725	* that means anything in my experience... ;-) */
726	}
727
728	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
729
730	return VINF_SUCCESS;
731	}
732
733
734	/**
735	* Worker for RTTimerStop() that takes care of the ugly SMP bits.
736	*
737	* @returns RTTimerStop() return value.
738	* @param pTimer The timer (valid).
739	*/
740	static int rtTimerLnxStopAll(PRTTIMER pTimer)
741	{
742	RTCPUID iCpu;
743	RTSPINLOCKTMP Tmp;
744
745
746	/*
747	* Mark the timer as suspended and flag all timers as stopping, except
748	* for those being stopped by an MP event.
749	*/
750	RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
751
752	ASMAtomicWriteBool(&pTimer->fSuspended, true);
753	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
754	{
755	RTTIMERLNXSTATE enmState;
756	do
757	{
758	enmState = rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState);
759	if ( enmState == RTTIMERLNXSTATE_STOPPED
760	\|\| enmState == RTTIMERLNXSTATE_MP_STOPPING)
761	break;
762	Assert(enmState == RTTIMERLNXSTATE_ACTIVE);
763	} while (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState, RTTIMERLNXSTATE_STOPPING, enmState));
764	}
765
766	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
767
768	/*
769	* Do the actual stopping. Fortunately, this doesn't require any IPIs.
770	* Unfortunately it cannot be done synchronously from within the spinlock,
771	* because we might end up in an active waiting for a handler to complete.
772	*/
773	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
774	if (rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState) == RTTIMERLNXSTATE_STOPPING)
775	rtTimerLnxStopSubTimer(&pTimer->aSubTimers[iCpu]);
776
777	return VINF_SUCCESS;
778	}
779
780
781	/**
782	* Per-cpu callback function (RTMpOnSpecific) used by rtTimerLinuxMpEvent()
783	* to start a sub-timer on a cpu that just have come online.
784	*
785	* @param idCpu The current CPU.
786	* @param pvUser1 Pointer to the timer.
787	* @param pvUser2 Pointer to the argument structure.
788	*/
789	static DECLCALLBACK(void) rtTimerLinuxMpStartOnCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
790	{
791	PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
792	PRTTIMER pTimer = (PRTTIMER)pvUser1;
793	RTSPINLOCK hSpinlock;
794	Assert(idCpu < pTimer->cCpus);
795
796	/*
797	* We have to be kind of careful here as we might RTTimerStop (and RTTimerDestroy),
798	* thus the paranoia.
799	*/
800	hSpinlock = pTimer->hSpinlock;
801	if ( hSpinlock != NIL_RTSPINLOCK
802	&& pTimer->u32Magic == RTTIMER_MAGIC)
803	{
804	RTSPINLOCKTMP Tmp;
805	RTSpinlockAcquire(hSpinlock, &Tmp);
806
807	if ( !pTimer->fSuspended
808	&& pTimer->u32Magic == RTTIMER_MAGIC)
809	{
810	/* We're sane and the timer is not suspended yet. */
811	PRTTIMERLNXSUBTIMER pSubTimer = &pTimer->aSubTimers[idCpu];
812	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STARTING, RTTIMERLNXSTATE_STOPPED))
813	rtTimerLnxStartSubTimer(pSubTimer, pArgs->u64Now, pArgs->u64First);
814	}
815
816	RTSpinlockRelease(hSpinlock, &Tmp);
817	}
818	}
819
820
821	/**
822	* MP event notification callback.
823	*
824	* @param enmEvent The event.
825	* @param idCpu The cpu it applies to.
826	* @param pvUser The timer.
827	*/
828	static DECLCALLBACK(void) rtTimerLinuxMpEvent(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvUser)
829	{
830	PRTTIMER pTimer = (PRTTIMER)pvUser;
831	PRTTIMERLNXSUBTIMER pSubTimer = &pTimer->aSubTimers[idCpu];
832	RTSPINLOCK hSpinlock;
833	RTSPINLOCKTMP Tmp;
834
835	Assert(idCpu < pTimer->cCpus);
836
837	/*
838	* Some initial paranoia.
839	*/
840	if (pTimer->u32Magic != RTTIMER_MAGIC)
841	return;
842	hSpinlock = pTimer->hSpinlock;
843	if (hSpinlock == NIL_RTSPINLOCK)
844	return;
845
846	RTSpinlockAcquireNoInts(hSpinlock, &Tmp);
847
848	/* Is it active? */
849	if ( !pTimer->fSuspended
850	&& !pTimer->u32Magic == RTTIMER_MAGIC)
851	{
852	switch (enmEvent)
853	{
854	/*
855	* Try do it without leaving the spin lock, but if we have to, retake it
856	* when we're on the right cpu.
857	*/
858	case RTMPEVENT_ONLINE:
859	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STARTING, RTTIMERLNXSTATE_STOPPED))
860	{
861	RTTIMERLINUXSTARTONCPUARGS Args;
862	Args.u64Now = RTTimeNanoTS();
863	Args.u64First = pTimer->u64NanoInterval;
864
865	if (RTMpCpuId() == idCpu)
866	rtTimerLnxStartSubTimer(pSubTimer, Args.u64Now, Args.u64First);
867	else
868	{
869	rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED); /* we'll recheck it. */
870	RTSpinlockReleaseNoInts(hSpinlock, &Tmp);
871
872	RTMpOnSpecific(idCpu, rtTimerLinuxMpStartOnCpu, pTimer, &Args);
873	return; /* we've left the spinlock */
874	}
875	}
876	break;
877
878	/*
879	* The CPU is (going) offline, make sure the sub-timer is stopped.
880	*
881	* Linux will migrate it to a different CPU, but we don't want this. The
882	* timer function is checking for this.
883	*/
884	case RTMPEVENT_OFFLINE:
885	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STOPPING, RTTIMERLNXSTATE_ACTIVE))
886	{
887	RTSpinlockAcquireNoInts(hSpinlock, &Tmp);
888
889	rtTimerLnxStopSubTimer(pSubTimer);
890	return; /* we've left the spinlock */
891	}
892	break;
893	}
894	}
895
896	RTSpinlockAcquireNoInts(hSpinlock, &Tmp);
897	}
898
899	#endif /* CONFIG_SMP */

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/r0drv/linux/timer-r0drv-linux.c@ 9370

Download in other formats: