VirtualBox

source: vbox/trunk/src/VBox/Additions/common/VBoxService/VBoxServiceCpuHotPlug.cpp@ 33464

Last change on this file since 33464 was 33464, checked in by vboxsync, 15 years ago

*: Fixes for incorrect RTStrAPrintf usage (it does NOT return an IPRT status code) and the odd bugs nearby.

  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 20.8 KB
Line 
1/* $Id: VBoxServiceCpuHotPlug.cpp 33464 2010-10-26 12:27:50Z vboxsync $ */
2/** @file
3 * VBoxService - Guest Additions CPU Hot Plugging Service.
4 */
5
6/*
7 * Copyright (C) 2010 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.215389.xyz. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#include <iprt/assert.h>
22#include <iprt/dir.h>
23#include <iprt/file.h>
24#include <iprt/mem.h>
25#include <iprt/string.h>
26#include <iprt/thread.h>
27#include <VBox/VBoxGuestLib.h>
28#include "VBoxServiceInternal.h"
29
30#ifdef RT_OS_LINUX
31# include <iprt/linux/sysfs.h>
32# include <errno.h> /* For the sysfs API */
33#endif
34
35
36/*******************************************************************************
37* Defined Constants And Macros *
38*******************************************************************************/
39#ifdef RT_OS_LINUX
40/** @name Paths to access the CPU device
41 * @{
42 */
43# define SYSFS_ACPI_CPU_PATH "/sys/devices"
44# define SYSFS_CPU_PATH "/sys/devices/system/cpu"
45/** @} */
46
47/** Path component for the ACPI CPU path. */
48typedef struct SYSFSCPUPATHCOMP
49{
50 /** Flag whether the name is suffixed with a number */
51 bool fNumberedSuffix;
52 /** Name of the component */
53 const char *pcszName;
54} SYSFSCPUPATHCOMP, *PSYSFSCPUPATHCOMP;
55/** Pointer to a const component. */
56typedef const SYSFSCPUPATHCOMP *PCSYSFSCPUPATHCOMP;
57
58/**
59 * Structure which defines how the entries are assembled.
60 */
61typedef struct SYSFSCPUPATH
62{
63 /** Id when probing for the correct path. */
64 uint32_t uId;
65 /** Array holding the possible components. */
66 PCSYSFSCPUPATHCOMP aComponentsPossible;
67 /** Number of entries in the array, excluding the terminator. */
68 unsigned cComponents;
69 /** Directory handle */
70 PRTDIR pDir;
71 /** Current directory to try. */
72 char *pszPath;
73} SYSFSCPUPATH, *PSYSFSCPUPATH;
74
75/** Content of uId if the path wasn't probed yet. */
76#define ACPI_CPU_PATH_NOT_PROBED UINT32_MAX
77
78/** Possible combinations of all path components for level 1. */
79const SYSFSCPUPATHCOMP g_aAcpiCpuPathLvl1[] =
80{
81 /** LNXSYSTEM:<id> */
82 {true, "LNXSYSTM:"}
83};
84
85/** Possible combinations of all path components for level 2. */
86const SYSFSCPUPATHCOMP g_aAcpiCpuPathLvl2[] =
87{
88 /** device:<id> */
89 {true, "device:"},
90 /** LNXSYBUS:<id> */
91 {true, "LNXSYBUS:"}
92};
93
94/** Possible combinations of all path components for level 3 */
95const SYSFSCPUPATHCOMP g_aAcpiCpuPathLvl3[] =
96{
97 /** ACPI0004:<id> */
98 {true, "ACPI0004:"}
99};
100
101/** Possible combinations of all path components for level 4 */
102const SYSFSCPUPATHCOMP g_aAcpiCpuPathLvl4[] =
103{
104 /** LNXCPU:<id> */
105 {true, "LNXCPU:"},
106 /** ACPI_CPU:<id> */
107 {true, "ACPI_CPU:"}
108};
109
110/** All possible combinations. */
111SYSFSCPUPATH g_aAcpiCpuPath[] =
112{
113 /** Level 1 */
114 {ACPI_CPU_PATH_NOT_PROBED, g_aAcpiCpuPathLvl1, RT_ELEMENTS(g_aAcpiCpuPathLvl1), NULL, NULL},
115 /** Level 2 */
116 {ACPI_CPU_PATH_NOT_PROBED, g_aAcpiCpuPathLvl2, RT_ELEMENTS(g_aAcpiCpuPathLvl2), NULL, NULL},
117 /** Level 3 */
118 {ACPI_CPU_PATH_NOT_PROBED, g_aAcpiCpuPathLvl3, RT_ELEMENTS(g_aAcpiCpuPathLvl3), NULL, NULL},
119 /** Level 4 */
120 {ACPI_CPU_PATH_NOT_PROBED, g_aAcpiCpuPathLvl4, RT_ELEMENTS(g_aAcpiCpuPathLvl4), NULL, NULL},
121};
122#endif
123
124#ifdef RT_OS_LINUX
125/**
126 * Probes for the correct path to the ACPI CPU object in sysfs for the
127 * various different kernel versions and distro's.
128 *
129 * @returns VBox status code.
130 */
131static int VBoxServiceCpuHotPlugProbePath(void)
132{
133 int rc = VINF_SUCCESS;
134
135 /* Probe for the correct path if we didn't already. */
136 if (RT_UNLIKELY(g_aAcpiCpuPath[0].uId == ACPI_CPU_PATH_NOT_PROBED))
137 {
138 char *pszPath = NULL; /** < Current path, increasing while we dig deeper. */
139
140 pszPath = RTStrDup(SYSFS_ACPI_CPU_PATH);
141 if (!pszPath)
142 return VERR_NO_MEMORY;
143
144 /*
145 * Simple algorithm to find the path.
146 * Performance is not a real problem because it is
147 * only executed once.
148 */
149 for (unsigned iLvlCurr = 0; iLvlCurr < RT_ELEMENTS(g_aAcpiCpuPath); iLvlCurr++)
150 {
151 PSYSFSCPUPATH pAcpiCpuPathLvl = &g_aAcpiCpuPath[iLvlCurr];
152
153 for (unsigned iCompCurr = 0; iCompCurr < pAcpiCpuPathLvl->cComponents; iCompCurr++)
154 {
155 PCSYSFSCPUPATHCOMP pPathComponent = &pAcpiCpuPathLvl->aComponentsPossible[iCompCurr];
156
157 /* Open the directory */
158 PRTDIR pDirCurr = NULL;
159 char *pszPathTmp = RTPathJoinA(pszPath, pPathComponent->pcszName);
160 if (pszPathTmp)
161 {
162 rc = RTDirOpenFiltered(&pDirCurr, pszPathTmp, RTDIRFILTER_WINNT);
163 RTStrFree(pszPathTmp);
164 }
165 else
166 rc = VERR_NO_STR_MEMORY;
167 if (RT_FAILURE(rc))
168 break;
169
170 /* Search if the current directory contains one of the possible parts. */
171 size_t cchName = strlen(pPathComponent->pcszName);
172 RTDIRENTRY DirFolderContent;
173 bool fFound = false;
174 while (RT_SUCCESS(RTDirRead(pDirCurr, &DirFolderContent, NULL))) /* Assumption that szName has always enough space */
175 {
176 if ( DirFolderContent.cbName >= cchName
177 && !strncmp(DirFolderContent.szName, pPathComponent->pcszName, cchName))
178 {
179 /* Found, use the complete name to dig deeper. */
180 fFound = true;
181 pAcpiCpuPathLvl->uId = iCompCurr;
182 char *pszPathLvl = RTPathJoinA(pszPath, DirFolderContent.szName);
183 if (pszPathLvl)
184 {
185 RTStrFree(pszPath);
186 pszPath = pszPathLvl;
187 }
188 else
189 rc = VERR_NO_STR_MEMORY;
190 break;
191 }
192 }
193 RTDirClose(pDirCurr);
194
195 if (fFound)
196 break;
197 } /* For every possible component. */
198
199 /* No matching component for this part, no need to continue */
200 if (RT_FAILURE(rc))
201 break;
202 } /* For every level */
203
204 VBoxServiceVerbose(1, "Final path after probing %s rc=%Rrc\n", pszPath, rc);
205 RTStrFree(pszPath);
206 }
207
208 return rc;
209}
210
211/**
212 * Returns the path of the ACPI CPU device with the given core and package ID.
213 *
214 * @returns VBox status code.
215 * @param ppszPath Where to store the path.
216 * @param idCpuCore The core ID of the CPU.
217 * @param idCpuPackage The package ID of the CPU.
218 */
219static int VBoxServiceCpuHotPlugGetACPIDevicePath(char **ppszPath, uint32_t idCpuCore, uint32_t idCpuPackage)
220{
221 int rc = VINF_SUCCESS;
222
223 AssertPtrReturn(ppszPath, VERR_INVALID_PARAMETER);
224
225 rc = VBoxServiceCpuHotPlugProbePath();
226 if (RT_SUCCESS(rc))
227 {
228 /* Build the path from all components. */
229 bool fFound = false;
230 unsigned iLvlCurr = 0;
231 char *pszPath = NULL;
232 char *pszPathDir = NULL;
233 PSYSFSCPUPATH pAcpiCpuPathLvl = &g_aAcpiCpuPath[iLvlCurr];
234
235 /* Init everything. */
236 Assert(pAcpiCpuPathLvl->uId != ACPI_CPU_PATH_NOT_PROBED);
237 pszPath = RTPathJoinA(SYSFS_ACPI_CPU_PATH, pAcpiCpuPathLvl->aComponentsPossible[pAcpiCpuPathLvl->uId].pcszName);
238 if (!pszPath)
239 return VERR_NO_STR_MEMORY;
240
241 pAcpiCpuPathLvl->pszPath = RTStrDup(SYSFS_ACPI_CPU_PATH);
242 if (!pAcpiCpuPathLvl->pszPath)
243 {
244 RTStrFree(pszPath);
245 return VERR_NO_STR_MEMORY;
246 }
247
248 /* Open the directory */
249 rc = RTDirOpenFiltered(&pAcpiCpuPathLvl->pDir, pszPath, RTDIRFILTER_WINNT);
250 if (RT_SUCCESS(rc))
251 {
252 RTStrFree(pszPath);
253
254 /* Search for CPU */
255 while (!fFound)
256 {
257 /* Get the next directory. */
258 RTDIRENTRY DirFolderContent;
259 rc = RTDirRead(pAcpiCpuPathLvl->pDir, &DirFolderContent, NULL);
260 if (RT_SUCCESS(rc))
261 {
262 /* Create the new path. */
263 char *pszPathCurr = RTPathJoinA(pAcpiCpuPathLvl->pszPath, DirFolderContent.szName);
264 if (!pszPathCurr)
265 {
266 rc = VERR_NO_STR_MEMORY;
267 break;
268 }
269
270 /* If this is the last level check for the given core and package id. */
271 if (iLvlCurr == RT_ELEMENTS(g_aAcpiCpuPath) - 1)
272 {
273 /* Get the sysdev */
274 uint32_t idCore = RTLinuxSysFsReadIntFile(10, "%s/sysdev/topology/core_id",
275 pszPathCurr);
276 uint32_t idPackage = RTLinuxSysFsReadIntFile(10, "%s/sysdev/topology/physical_package_id",
277 pszPathCurr);
278 if ( idCore == idCpuCore
279 && idPackage == idCpuPackage)
280 {
281 /* Return the path */
282 pszPath = pszPathCurr;
283 fFound = true;
284 VBoxServiceVerbose(3, "CPU found\n");
285 break;
286 }
287 else
288 {
289 /* Get the next directory. */
290 RTStrFree(pszPathCurr);
291 VBoxServiceVerbose(3, "CPU doesn't match, next directory\n");
292 }
293 }
294 else
295 {
296 /* Go deeper */
297 iLvlCurr++;
298
299 VBoxServiceVerbose(3, "Going deeper (iLvlCurr=%u)\n", iLvlCurr);
300
301 pAcpiCpuPathLvl = &g_aAcpiCpuPath[iLvlCurr];
302
303 Assert(!pAcpiCpuPathLvl->pDir);
304 Assert(!pAcpiCpuPathLvl->pszPath);
305 pAcpiCpuPathLvl->pszPath = pszPathCurr;
306 PCSYSFSCPUPATHCOMP pPathComponent = &pAcpiCpuPathLvl->aComponentsPossible[pAcpiCpuPathLvl->uId];
307
308 Assert(pAcpiCpuPathLvl->uId != ACPI_CPU_PATH_NOT_PROBED);
309
310 pszPathDir = RTPathJoinA(pszPathCurr, pPathComponent->pcszName);
311 if (!pszPathDir)
312 {
313 rc = VERR_NO_STR_MEMORY;
314 break;
315 }
316
317 VBoxServiceVerbose(3, "New path %s\n", pszPathDir);
318
319 /* Open the directory */
320 rc = RTDirOpenFiltered(&pAcpiCpuPathLvl->pDir, pszPathDir, RTDIRFILTER_WINNT);
321 if (RT_FAILURE(rc))
322 break;
323 }
324 }
325 else
326 {
327 /* Go back one level and try to get the next entry. */
328 Assert(iLvlCurr > 0);
329
330 RTDirClose(pAcpiCpuPathLvl->pDir);
331 RTStrFree(pAcpiCpuPathLvl->pszPath);
332 pAcpiCpuPathLvl->pDir = NULL;
333 pAcpiCpuPathLvl->pszPath = NULL;
334
335 iLvlCurr--;
336 pAcpiCpuPathLvl = &g_aAcpiCpuPath[iLvlCurr];
337 VBoxServiceVerbose(3, "Directory not found, going back (iLvlCurr=%u)\n", iLvlCurr);
338 }
339 } /* while not found */
340 } /* Successful init */
341
342 /* Cleanup */
343 for (unsigned i = 0; i < RT_ELEMENTS(g_aAcpiCpuPath); i++)
344 {
345 if (g_aAcpiCpuPath[i].pDir)
346 RTDirClose(g_aAcpiCpuPath[i].pDir);
347 if (g_aAcpiCpuPath[i].pszPath)
348 RTStrFree(g_aAcpiCpuPath[i].pszPath);
349 g_aAcpiCpuPath[i].pDir = NULL;
350 g_aAcpiCpuPath[i].pszPath = NULL;
351 }
352 if (pszPathDir)
353 RTStrFree(pszPathDir);
354 if (RT_FAILURE(rc) && pszPath)
355 RTStrFree(pszPath);
356
357 if (RT_SUCCESS(rc))
358 *ppszPath = pszPath;
359 }
360
361 return rc;
362}
363#endif /* RT_OS_LINUX */
364
365
366/** @copydoc VBOXSERVICE::pfnPreInit */
367static DECLCALLBACK(int) VBoxServiceCpuHotPlugPreInit(void)
368{
369 return VINF_SUCCESS;
370}
371
372
373/** @copydoc VBOXSERVICE::pfnOption */
374static DECLCALLBACK(int) VBoxServiceCpuHotPlugOption(const char **ppszShort, int argc, char **argv, int *pi)
375{
376 NOREF(ppszShort);
377 NOREF(argc);
378 NOREF(argv);
379 NOREF(pi);
380 return VINF_SUCCESS;
381}
382
383
384/** @copydoc VBOXSERVICE::pfnInit */
385static DECLCALLBACK(int) VBoxServiceCpuHotPlugInit(void)
386{
387 return VINF_SUCCESS;
388}
389
390
391/**
392 * Handles VMMDevCpuEventType_Plug.
393 *
394 * @param idCpuCore The CPU core ID.
395 * @param idCpuPackage The CPU package ID.
396 */
397static void VBoxServiceCpuHotPlugHandlePlugEvent(uint32_t idCpuCore, uint32_t idCpuPackage)
398{
399#ifdef RT_OS_LINUX
400 /*
401 * The topology directory (containing the physical and core id properties)
402 * is not available until the CPU is online. So we just iterate over all directories
403 * and enable every CPU which is not online already.
404 * Because the directory might not be available immediately we try a few times.
405 *
406 * @todo: Maybe use udev to monitor hot-add events from the kernel
407 */
408 bool fCpuOnline = false;
409 unsigned cTries = 5;
410
411 do
412 {
413 PRTDIR pDirDevices = NULL;
414 int rc = RTDirOpen(&pDirDevices, SYSFS_CPU_PATH);
415 if (RT_SUCCESS(rc))
416 {
417 RTDIRENTRY DirFolderContent;
418 while (RT_SUCCESS(RTDirRead(pDirDevices, &DirFolderContent, NULL))) /* Assumption that szName has always enough space */
419 {
420 /** @todo r-bird: This code is bringing all CPUs online; the idCpuCore and
421 * idCpuPackage parameters are unused!
422 * aeichner: These files are not available at this point unfortunately. (see comment above)
423 * bird: Yes, but isn't that easily dealt with by doing:
424 * if (matching_topology() || !have_topology_directory())
425 * bring_cpu_online()
426 * That could save you the cpu0 and cpuidle checks to.
427 */
428 /*
429 * Check if this is a CPU object.
430 * cpu0 is excluded because it is not possible to change the state
431 * of the first CPU on Linux (it doesn't even have an online file)
432 * and cpuidle is no CPU device. Prevents error messages later.
433 */
434 if( !strncmp(DirFolderContent.szName, "cpu", 3)
435 && strncmp(DirFolderContent.szName, "cpu0", 4)
436 && strncmp(DirFolderContent.szName, "cpuidle", 7))
437 {
438 /* Get the sysdev */
439 RTFILE hFileCpuOnline = NIL_RTFILE;
440
441 rc = RTFileOpenF(&hFileCpuOnline, RTFILE_O_WRITE | RTFILE_O_OPEN | RTFILE_O_DENY_NONE,
442 "%s/%s/online", SYSFS_CPU_PATH, DirFolderContent.szName);
443 if (RT_SUCCESS(rc))
444 {
445 /* Write a 1 to online the CPU */
446 rc = RTFileWrite(hFileCpuOnline, "1", 1, NULL);
447 RTFileClose(hFileCpuOnline);
448 if (RT_SUCCESS(rc))
449 {
450 VBoxServiceVerbose(1, "CpuHotPlug: CPU %u/%u was brought online\n", idCpuPackage, idCpuCore);
451 fCpuOnline = true;
452 break;
453 }
454 /* Error means CPU not present or online already */
455 }
456 else
457 VBoxServiceError("CpuHotPlug: Failed to open \"%s/%s/online\" rc=%Rrc\n",
458 SYSFS_CPU_PATH, DirFolderContent.szName, rc);
459 }
460 }
461 }
462 else
463 VBoxServiceError("CpuHotPlug: Failed to open path %s rc=%Rrc\n", SYSFS_CPU_PATH, rc);
464
465 /* Sleep a bit */
466 if (!fCpuOnline)
467 RTThreadSleep(10);
468
469 } while ( !fCpuOnline
470 && cTries-- > 0);
471#else
472# error "Port me"
473#endif
474}
475
476
477/**
478 * Handles VMMDevCpuEventType_Unplug.
479 *
480 * @param idCpuCore The CPU core ID.
481 * @param idCpuPackage The CPU package ID.
482 */
483static void VBoxServiceCpuHotPlugHandleUnplugEvent(uint32_t idCpuCore, uint32_t idCpuPackage)
484{
485#ifdef RT_OS_LINUX
486 char *pszCpuDevicePath = NULL;
487 int rc = VBoxServiceCpuHotPlugGetACPIDevicePath(&pszCpuDevicePath, idCpuCore, idCpuPackage);
488 if (RT_SUCCESS(rc))
489 {
490 RTFILE hFileCpuEject;
491 rc = RTFileOpenF(&hFileCpuEject, RTFILE_O_WRITE | RTFILE_O_OPEN | RTFILE_O_DENY_NONE,
492 "%s/eject", pszCpuDevicePath);
493 if (RT_SUCCESS(rc))
494 {
495 /* Write a 1 to eject the CPU */
496 rc = RTFileWrite(hFileCpuEject, "1", 1, NULL);
497 if (RT_SUCCESS(rc))
498 VBoxServiceVerbose(1, "CpuHotPlug: CPU %u/%u was ejected\n", idCpuPackage, idCpuCore);
499 else
500 VBoxServiceError("CpuHotPlug: Failed to eject CPU %u/%u rc=%Rrc\n", idCpuPackage, idCpuCore, rc);
501
502 RTFileClose(hFileCpuEject);
503 }
504 else
505 VBoxServiceError("CpuHotPlug: Failed to open \"%s/eject\" rc=%Rrc\n", pszCpuDevicePath, rc);
506 RTStrFree(pszCpuDevicePath);
507 }
508 else
509 VBoxServiceError("CpuHotPlug: Failed to get CPU device path rc=%Rrc\n", rc);
510#else
511# error "Port me"
512#endif
513}
514
515
516/** @copydoc VBOXSERVICE::pfnWorker */
517DECLCALLBACK(int) VBoxServiceCpuHotPlugWorker(bool volatile *pfShutdown)
518{
519 /*
520 * Tell the control thread that it can continue spawning services.
521 */
522 RTThreadUserSignal(RTThreadSelf());
523
524 /*
525 * Enable the CPU hotplug notifier.
526 */
527 int rc = VbglR3CpuHotPlugInit();
528 if (RT_FAILURE(rc))
529 return rc;
530
531 /*
532 * The Work Loop.
533 */
534 for (;;)
535 {
536 /* Wait for CPU hot plugging event. */
537 uint32_t idCpuCore;
538 uint32_t idCpuPackage;
539 VMMDevCpuEventType enmEventType;
540 rc = VbglR3CpuHotPlugWaitForEvent(&enmEventType, &idCpuCore, &idCpuPackage);
541 if (RT_SUCCESS(rc))
542 {
543 VBoxServiceVerbose(3, "CpuHotPlug: Event happened idCpuCore=%u idCpuPackage=%u enmEventType=%d\n",
544 idCpuCore, idCpuPackage, enmEventType);
545 switch (enmEventType)
546 {
547 case VMMDevCpuEventType_Plug:
548 VBoxServiceCpuHotPlugHandlePlugEvent(idCpuCore, idCpuPackage);
549 break;
550
551 case VMMDevCpuEventType_Unplug:
552 VBoxServiceCpuHotPlugHandleUnplugEvent(idCpuCore, idCpuPackage);
553 break;
554
555 default:
556 {
557 static uint32_t s_iErrors = 0;
558 if (s_iErrors++ < 10)
559 VBoxServiceError("CpuHotPlug: Unknown event: idCpuCore=%u idCpuPackage=%u enmEventType=%d\n",
560 idCpuCore, idCpuPackage, enmEventType);
561 break;
562 }
563 }
564 }
565 else if (rc != VERR_INTERRUPTED && rc != VERR_TRY_AGAIN)
566 {
567 VBoxServiceError("CpuHotPlug: VbglR3CpuHotPlugWaitForEvent returned %Rrc\n", rc);
568 break;
569 }
570
571 if (*pfShutdown)
572 break;
573 }
574
575 VbglR3CpuHotPlugTerm();
576 return rc;
577}
578
579
580/** @copydoc VBOXSERVICE::pfnStop */
581static DECLCALLBACK(void) VBoxServiceCpuHotPlugStop(void)
582{
583 VbglR3InterruptEventWaits();
584 return;
585}
586
587
588/** @copydoc VBOXSERVICE::pfnTerm */
589static DECLCALLBACK(void) VBoxServiceCpuHotPlugTerm(void)
590{
591 return;
592}
593
594
595/**
596 * The 'timesync' service description.
597 */
598VBOXSERVICE g_CpuHotPlug =
599{
600 /* pszName. */
601 "cpuhotplug",
602 /* pszDescription. */
603 "CPU hot plugging monitor",
604 /* pszUsage. */
605 NULL,
606 /* pszOptions. */
607 NULL,
608 /* methods */
609 VBoxServiceCpuHotPlugPreInit,
610 VBoxServiceCpuHotPlugOption,
611 VBoxServiceCpuHotPlugInit,
612 VBoxServiceCpuHotPlugWorker,
613 VBoxServiceCpuHotPlugStop,
614 VBoxServiceCpuHotPlugTerm
615};
616
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette