utf-8.cpp@ 30859

Last change on this file since 30859 was 30859, checked in by vboxsync, 15 years ago
iprt/string.h: added RTStrPurgeEncoding.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 43.6 KB

Line
1	/* $Id: utf-8.cpp 30859 2010-07-15 16:20:17Z vboxsync $ */
2	/** @file
3	* IPRT - UTF-8 Decoding.
4	*/
5
6	/*
7	* Copyright (C) 2006-2009 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.215389.xyz. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* The contents of this file may alternatively be used under the terms
18	* of the Common Development and Distribution License Version 1.0
19	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	* VirtualBox OSE distribution, in which case the provisions of the
21	* CDDL are applicable instead of those of the GPL.
22	*
23	* You may elect to license modified versions of this file under the
24	* terms and conditions of either the GPL or the CDDL or both.
25	*/
26
27
28	/*******************************************************************************
29	* Header Files *
30	*******************************************************************************/
31	#include <iprt/string.h>
32	#include "internal/iprt.h"
33
34	#include <iprt/uni.h>
35	#include <iprt/alloc.h>
36	#include <iprt/assert.h>
37	#include <iprt/err.h>
38	#include "internal/string.h"
39
40
41
42	/**
43	* Get get length in code points of a UTF-8 encoded string.
44	* The string is validated while doing this.
45	*
46	* @returns IPRT status code.
47	* @param psz Pointer to the UTF-8 string.
48	* @param cch The max length of the string. (btw cch = cb)
49	* Use RTSTR_MAX if all of the string is to be examined.
50	* @param pcuc Where to store the length in unicode code points.
51	* @param pcchActual Where to store the actual size of the UTF-8 string
52	* on success (cch = cb again). Optional.
53	*/
54	int rtUtf8Length(const char psz, size_t cch, size_t pcuc, size_t *pcchActual)
55	{
56	const unsigned char puch = (const unsigned char )psz;
57	size_t cCodePoints = 0;
58	while (cch > 0)
59	{
60	const unsigned char uch = *puch;
61	if (!uch)
62	break;
63	if (uch & RT_BIT(7))
64	{
65	/* figure sequence length and validate the first byte */
66	unsigned cb;
67	if ((uch & (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5))) == (RT_BIT(7) \| RT_BIT(6)))
68	cb = 2;
69	else if ((uch & (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5) \| RT_BIT(4))) == (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5)))
70	cb = 3;
71	else if ((uch & (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5) \| RT_BIT(4) \| RT_BIT(3))) == (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5) \| RT_BIT(4)))
72	cb = 4;
73	else if ((uch & (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5) \| RT_BIT(4) \| RT_BIT(3) \| RT_BIT(2))) == (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5) \| RT_BIT(4) \| RT_BIT(3)))
74	cb = 5;
75	else if ((uch & (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5) \| RT_BIT(4) \| RT_BIT(3) \| RT_BIT(2) \| RT_BIT(1))) == (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5) \| RT_BIT(4) \| RT_BIT(3) \| RT_BIT(2)))
76	cb = 6;
77	else
78	{
79	RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.*Rhxs\n", RT_MIN(cch, 10), puch));
80	return VERR_INVALID_UTF8_ENCODING;
81	}
82
83	/* check length */
84	if (cb > cch)
85	{
86	RTStrAssertMsgFailed(("Invalid UTF-8 length: cb=%d cch=%d (%.*Rhxs)\n", cb, cch, RT_MIN(cch, 10), puch));
87	return VERR_INVALID_UTF8_ENCODING;
88	}
89
90	/* validate the rest */
91	switch (cb)
92	{
93	case 6:
94	RTStrAssertMsgReturn((puch[5] & (RT_BIT(7) \| RT_BIT(6))) == RT_BIT(7), ("6/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
95	case 5:
96	RTStrAssertMsgReturn((puch[4] & (RT_BIT(7) \| RT_BIT(6))) == RT_BIT(7), ("5/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
97	case 4:
98	RTStrAssertMsgReturn((puch[3] & (RT_BIT(7) \| RT_BIT(6))) == RT_BIT(7), ("4/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
99	case 3:
100	RTStrAssertMsgReturn((puch[2] & (RT_BIT(7) \| RT_BIT(6))) == RT_BIT(7), ("3/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
101	case 2:
102	RTStrAssertMsgReturn((puch[1] & (RT_BIT(7) \| RT_BIT(6))) == RT_BIT(7), ("2/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
103	break;
104	}
105
106	/* validate the code point. */
107	RTUNICP uc;
108	switch (cb)
109	{
110	case 6:
111	uc = (puch[5] & 0x3f)
112	\| ((RTUNICP)(puch[4] & 0x3f) << 6)
113	\| ((RTUNICP)(puch[3] & 0x3f) << 12)
114	\| ((RTUNICP)(puch[2] & 0x3f) << 18)
115	\| ((RTUNICP)(puch[1] & 0x3f) << 24)
116	\| ((RTUNICP)(uch & 0x01) << 30);
117	RTStrAssertMsgReturn(uc >= 0x04000000 && uc <= 0x7fffffff,
118	("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
119	break;
120	case 5:
121	uc = (puch[4] & 0x3f)
122	\| ((RTUNICP)(puch[3] & 0x3f) << 6)
123	\| ((RTUNICP)(puch[2] & 0x3f) << 12)
124	\| ((RTUNICP)(puch[1] & 0x3f) << 18)
125	\| ((RTUNICP)(uch & 0x03) << 24);
126	RTStrAssertMsgReturn(uc >= 0x00200000 && uc <= 0x03ffffff,
127	("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
128	break;
129	case 4:
130	uc = (puch[3] & 0x3f)
131	\| ((RTUNICP)(puch[2] & 0x3f) << 6)
132	\| ((RTUNICP)(puch[1] & 0x3f) << 12)
133	\| ((RTUNICP)(uch & 0x07) << 18);
134	RTStrAssertMsgReturn(uc >= 0x00010000 && uc <= 0x001fffff,
135	("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
136	break;
137	case 3:
138	uc = (puch[2] & 0x3f)
139	\| ((RTUNICP)(puch[1] & 0x3f) << 6)
140	\| ((RTUNICP)(uch & 0x0f) << 12);
141	RTStrAssertMsgReturn(uc >= 0x00000800 && uc <= 0x0000fffd,
142	("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch),
143	uc == 0xffff \|\| uc == 0xfffe ? VERR_CODE_POINT_ENDIAN_INDICATOR : VERR_INVALID_UTF8_ENCODING);
144	RTStrAssertMsgReturn(uc < 0xd800 \|\| uc > 0xdfff,
145	("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_CODE_POINT_SURROGATE);
146	break;
147	case 2:
148	uc = (puch[1] & 0x3f)
149	\| ((RTUNICP)(uch & 0x1f) << 6);
150	RTStrAssertMsgReturn(uc >= 0x00000080 && uc <= 0x000007ff,
151	("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
152	break;
153	}
154
155	/* advance */
156	cch -= cb;
157	puch += cb;
158	}
159	else
160	{
161	/* one ASCII byte */
162	puch++;
163	cch--;
164	}
165	cCodePoints++;
166	}
167
168	/* done */
169	*pcuc = cCodePoints;
170	if (pcchActual)
171	pcchActual = puch - (unsigned char const )psz;
172	return VINF_SUCCESS;
173	}
174
175
176	/**
177	* Decodes and UTF-8 string into an array of unicode code point.
178	*
179	* Since we know the input is valid, we do not perform encoding or length checks.
180	*
181	* @returns iprt status code.
182	* @param psz The UTF-8 string to recode. This is a valid encoding.
183	* @param cch The number of chars (the type char, so bytes if you like) to process of the UTF-8 string.
184	* The recoding will stop when cch or '\\0' is reached. Pass RTSTR_MAX to process up to '\\0'.
185	* @param paCps Where to store the code points array.
186	* @param cCps The number of RTUNICP items the paCps buffer can hold, excluding the terminator ('\\0').
187	*/
188	static int rtUtf8Decode(const char *psz, size_t cch, PRTUNICP paCps, size_t cCps)
189	{
190	int rc = VINF_SUCCESS;
191	const unsigned char puch = (const unsigned char )psz;
192	PRTUNICP pCp = paCps;
193	while (cch > 0)
194	{
195	/* read the next char and check for terminator. */
196	const unsigned char uch = *puch;
197	if (!uch)
198	break;
199
200	/* check for output overflow */
201	if (RT_UNLIKELY(cCps < 1))
202	{
203	rc = VERR_BUFFER_OVERFLOW;
204	break;
205	}
206	cCps--;
207
208	/* decode and recode the code point */
209	if (!(uch & RT_BIT(7)))
210	{
211	*pCp++ = uch;
212	puch++;
213	cch--;
214	}
215	#ifdef RT_STRICT
216	else if (!(uch & RT_BIT(6)))
217	AssertMsgFailed(("Internal error!\n"));
218	#endif
219	else if (!(uch & RT_BIT(5)))
220	{
221	*pCp++ = (puch[1] & 0x3f)
222	\| ((uint16_t)(uch & 0x1f) << 6);
223	puch += 2;
224	cch -= 2;
225	}
226	else if (!(uch & RT_BIT(4)))
227	{
228	*pCp++ = (puch[2] & 0x3f)
229	\| ((uint16_t)(puch[1] & 0x3f) << 6)
230	\| ((uint16_t)(uch & 0x0f) << 12);
231	puch += 3;
232	cch -= 3;
233	}
234	else if (!(uch & RT_BIT(3)))
235	{
236	*pCp++ = (puch[3] & 0x3f)
237	\| ((RTUNICP)(puch[2] & 0x3f) << 6)
238	\| ((RTUNICP)(puch[1] & 0x3f) << 12)
239	\| ((RTUNICP)(uch & 0x07) << 18);
240	puch += 4;
241	cch -= 4;
242	}
243	else if (!(uch & RT_BIT(2)))
244	{
245	*pCp++ = (puch[4] & 0x3f)
246	\| ((RTUNICP)(puch[3] & 0x3f) << 6)
247	\| ((RTUNICP)(puch[2] & 0x3f) << 12)
248	\| ((RTUNICP)(puch[1] & 0x3f) << 18)
249	\| ((RTUNICP)(uch & 0x03) << 24);
250	puch += 5;
251	cch -= 6;
252	}
253	else
254	{
255	Assert(!(uch & RT_BIT(1)));
256	*pCp++ = (puch[5] & 0x3f)
257	\| ((RTUNICP)(puch[4] & 0x3f) << 6)
258	\| ((RTUNICP)(puch[3] & 0x3f) << 12)
259	\| ((RTUNICP)(puch[2] & 0x3f) << 18)
260	\| ((RTUNICP)(puch[1] & 0x3f) << 24)
261	\| ((RTUNICP)(uch & 0x01) << 30);
262	puch += 6;
263	cch -= 6;
264	}
265	}
266
267	/* done */
268	*pCp = 0;
269	return rc;
270	}
271
272
273	RTDECL(size_t) RTStrUniLen(const char *psz)
274	{
275	size_t cCodePoints;
276	int rc = rtUtf8Length(psz, RTSTR_MAX, &cCodePoints, NULL);
277	return RT_SUCCESS(rc) ? cCodePoints : 0;
278	}
279	RT_EXPORT_SYMBOL(RTStrUniLen);
280
281
282	RTDECL(int) RTStrUniLenEx(const char psz, size_t cch, size_t pcCps)
283	{
284	size_t cCodePoints;
285	int rc = rtUtf8Length(psz, cch, &cCodePoints, NULL);
286	if (pcCps)
287	*pcCps = RT_SUCCESS(rc) ? cCodePoints : 0;
288	return rc;
289	}
290	RT_EXPORT_SYMBOL(RTStrUniLenEx);
291
292
293	RTDECL(int) RTStrValidateEncoding(const char *psz)
294	{
295	return RTStrValidateEncodingEx(psz, RTSTR_MAX, 0);
296	}
297	RT_EXPORT_SYMBOL(RTStrValidateEncoding);
298
299
300	RTDECL(int) RTStrValidateEncodingEx(const char *psz, size_t cch, uint32_t fFlags)
301	{
302	AssertReturn(!(fFlags & ~(RTSTR_VALIDATE_ENCODING_ZERO_TERMINATED)), VERR_INVALID_PARAMETER);
303	AssertPtr(psz);
304
305	/*
306	* Use rtUtf8Length for the job.
307	*/
308	size_t cchActual;
309	size_t cCpsIgnored;
310	int rc = rtUtf8Length(psz, cch, &cCpsIgnored, &cchActual);
311	if (RT_SUCCESS(rc))
312	{
313	if ( (fFlags & RTSTR_VALIDATE_ENCODING_ZERO_TERMINATED)
314	&& cchActual >= cch)
315	rc = VERR_BUFFER_OVERFLOW;
316	}
317	return rc;
318	}
319	RT_EXPORT_SYMBOL(RTStrValidateEncodingEx);
320
321
322	RTDECL(bool) RTStrIsValidEncoding(const char *psz)
323	{
324	int rc = RTStrValidateEncodingEx(psz, RTSTR_MAX, 0);
325	return RT_SUCCESS(rc);
326	}
327	RT_EXPORT_SYMBOL(RTStrIsValidEncoding);
328
329
330	RTDECL(size_t) RTStrPurgeEncoding(char *psz)
331	{
332	size_t cErrors = 0;
333	for (;;)
334	{
335	RTUNICP Cp;
336	int rc = RTStrGetCpEx((const char **)&psz, &Cp);
337	if (RT_SUCCESS(rc))
338	{
339	if (!Cp)
340	break;
341	}
342	else
343	{
344	psz[-1] = '?';
345	cErrors++;
346	}
347	}
348	return cErrors;
349	}
350	RT_EXPORT_SYMBOL(RTStrPurgeEncoding);
351
352
353	RTDECL(int) RTStrToUni(const char pszString, PRTUNICP ppaCps)
354	{
355	/*
356	* Validate input.
357	*/
358	Assert(VALID_PTR(pszString));
359	Assert(VALID_PTR(ppaCps));
360	*ppaCps = NULL;
361
362	/*
363	* Validate the UTF-8 input and count its code points.
364	*/
365	size_t cCps;
366	int rc = rtUtf8Length(pszString, RTSTR_MAX, &cCps, NULL);
367	if (RT_SUCCESS(rc))
368	{
369	/*
370	* Allocate buffer.
371	*/
372	PRTUNICP paCps = (PRTUNICP)RTMemAlloc((cCps + 1) * sizeof(RTUNICP));
373	if (paCps)
374	{
375	/*
376	* Decode the string.
377	*/
378	rc = rtUtf8Decode(pszString, RTSTR_MAX, paCps, cCps);
379	if (RT_SUCCESS(rc))
380	{
381	*ppaCps = paCps;
382	return rc;
383	}
384	RTMemFree(paCps);
385	}
386	else
387	rc = VERR_NO_CODE_POINT_MEMORY;
388	}
389	return rc;
390	}
391	RT_EXPORT_SYMBOL(RTStrToUni);
392
393
394	RTDECL(int) RTStrToUniEx(const char pszString, size_t cchString, PRTUNICP ppaCps, size_t cCps, size_t *pcCps)
395	{
396	/*
397	* Validate input.
398	*/
399	Assert(VALID_PTR(pszString));
400	Assert(VALID_PTR(ppaCps));
401	Assert(!pcCps \|\| VALID_PTR(pcCps));
402
403	/*
404	* Validate the UTF-8 input and count the code points.
405	*/
406	size_t cCpsResult;
407	int rc = rtUtf8Length(pszString, cchString, &cCpsResult, NULL);
408	if (RT_SUCCESS(rc))
409	{
410	if (pcCps)
411	*pcCps = cCpsResult;
412
413	/*
414	* Check buffer size / Allocate buffer.
415	*/
416	bool fShouldFree;
417	PRTUNICP paCpsResult;
418	if (cCps > 0 && *ppaCps)
419	{
420	fShouldFree = false;
421	if (cCps <= cCpsResult)
422	return VERR_BUFFER_OVERFLOW;
423	paCpsResult = *ppaCps;
424	}
425	else
426	{
427	*ppaCps = NULL;
428	fShouldFree = true;
429	cCps = RT_MAX(cCpsResult + 1, cCps);
430	paCpsResult = (PRTUNICP)RTMemAlloc(cCps * sizeof(RTUNICP));
431	}
432	if (paCpsResult)
433	{
434	/*
435	* Encode the UTF-16 string.
436	*/
437	rc = rtUtf8Decode(pszString, cchString, paCpsResult, cCps - 1);
438	if (RT_SUCCESS(rc))
439	{
440	*ppaCps = paCpsResult;
441	return rc;
442	}
443	if (fShouldFree)
444	RTMemFree(paCpsResult);
445	}
446	else
447	rc = VERR_NO_CODE_POINT_MEMORY;
448	}
449	return rc;
450	}
451	RT_EXPORT_SYMBOL(RTStrToUniEx);
452
453
454	/**
455	* Calculates the UTF-16 length of a string, validating the encoding while doing so.
456	*
457	* @returns IPRT status code.
458	* @param psz Pointer to the UTF-8 string.
459	* @param cch The max length of the string. (btw cch = cb)
460	* Use RTSTR_MAX if all of the string is to be examined.
461	* @param pcwc Where to store the length of the UTF-16 string as a number of RTUTF16 characters.
462	*/
463	static int rtUtf8CalcUtf16Length(const char psz, size_t cch, size_t pcwc)
464	{
465	const unsigned char puch = (const unsigned char )psz;
466	size_t cwc = 0;
467	while (cch > 0)
468	{
469	const unsigned char uch = *puch;
470	if (!uch)
471	break;
472	if (!(uch & RT_BIT(7)))
473	{
474	/* one ASCII byte */
475	cwc++;
476	puch++;
477	cch--;
478	}
479	else
480	{
481	/* figure sequence length and validate the first byte */
482	unsigned cb;
483	if ((uch & (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5))) == (RT_BIT(7) \| RT_BIT(6)))
484	cb = 2;
485	else if ((uch & (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5) \| RT_BIT(4))) == (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5)))
486	cb = 3;
487	else if ((uch & (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5) \| RT_BIT(4) \| RT_BIT(3))) == (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5) \| RT_BIT(4)))
488	cb = 4;
489	else if ((uch & (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5) \| RT_BIT(4) \| RT_BIT(3) \| RT_BIT(2))) == (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5) \| RT_BIT(4) \| RT_BIT(3)))
490	cb = 5;
491	else if ((uch & (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5) \| RT_BIT(4) \| RT_BIT(3) \| RT_BIT(2) \| RT_BIT(1))) == (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5) \| RT_BIT(4) \| RT_BIT(3) \| RT_BIT(2)))
492	cb = 6;
493	else
494	{
495	RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.*Rhxs\n", RT_MIN(cch, 10), puch));
496	return VERR_INVALID_UTF8_ENCODING;
497	}
498
499	/* check length */
500	if (cb > cch)
501	{
502	RTStrAssertMsgFailed(("Invalid UTF-8 length: cb=%d cch=%d (%.*Rhxs)\n", cb, cch, RT_MIN(cch, 10), puch));
503	return VERR_INVALID_UTF8_ENCODING;
504	}
505
506	/* validate the rest */
507	switch (cb)
508	{
509	case 6:
510	RTStrAssertMsgReturn((puch[5] & (RT_BIT(7) \| RT_BIT(6))) == RT_BIT(7), ("6/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
511	case 5:
512	RTStrAssertMsgReturn((puch[4] & (RT_BIT(7) \| RT_BIT(6))) == RT_BIT(7), ("5/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
513	case 4:
514	RTStrAssertMsgReturn((puch[3] & (RT_BIT(7) \| RT_BIT(6))) == RT_BIT(7), ("4/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
515	case 3:
516	RTStrAssertMsgReturn((puch[2] & (RT_BIT(7) \| RT_BIT(6))) == RT_BIT(7), ("3/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
517	case 2:
518	RTStrAssertMsgReturn((puch[1] & (RT_BIT(7) \| RT_BIT(6))) == RT_BIT(7), ("2/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
519	break;
520	}
521
522	/* validate the code point. */
523	RTUNICP uc;
524	switch (cb)
525	{
526	case 6:
527	uc = (puch[5] & 0x3f)
528	\| ((RTUNICP)(puch[4] & 0x3f) << 6)
529	\| ((RTUNICP)(puch[3] & 0x3f) << 12)
530	\| ((RTUNICP)(puch[2] & 0x3f) << 18)
531	\| ((RTUNICP)(puch[1] & 0x3f) << 24)
532	\| ((RTUNICP)(uch & 0x01) << 30);
533	RTStrAssertMsgReturn(uc >= 0x04000000 && uc <= 0x7fffffff,
534	("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
535	RTStrAssertMsgFailed(("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch));
536	return VERR_CANT_RECODE_AS_UTF16;
537	case 5:
538	uc = (puch[4] & 0x3f)
539	\| ((RTUNICP)(puch[3] & 0x3f) << 6)
540	\| ((RTUNICP)(puch[2] & 0x3f) << 12)
541	\| ((RTUNICP)(puch[1] & 0x3f) << 18)
542	\| ((RTUNICP)(uch & 0x03) << 24);
543	RTStrAssertMsgReturn(uc >= 0x00200000 && uc <= 0x03ffffff,
544	("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
545	RTStrAssertMsgFailed(("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch));
546	return VERR_CANT_RECODE_AS_UTF16;
547	case 4:
548	uc = (puch[3] & 0x3f)
549	\| ((RTUNICP)(puch[2] & 0x3f) << 6)
550	\| ((RTUNICP)(puch[1] & 0x3f) << 12)
551	\| ((RTUNICP)(uch & 0x07) << 18);
552	RTStrAssertMsgReturn(uc >= 0x00010000 && uc <= 0x001fffff,
553	("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
554	RTStrAssertMsgReturn(uc <= 0x0010ffff,
555	("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_CANT_RECODE_AS_UTF16);
556	cwc++;
557	break;
558	case 3:
559	uc = (puch[2] & 0x3f)
560	\| ((RTUNICP)(puch[1] & 0x3f) << 6)
561	\| ((RTUNICP)(uch & 0x0f) << 12);
562	RTStrAssertMsgReturn(uc >= 0x00000800 && uc <= 0x0000fffd,
563	("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch),
564	uc == 0xffff \|\| uc == 0xfffe ? VERR_CODE_POINT_ENDIAN_INDICATOR : VERR_INVALID_UTF8_ENCODING);
565	RTStrAssertMsgReturn(uc < 0xd800 \|\| uc > 0xdfff,
566	("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_CODE_POINT_SURROGATE);
567	break;
568	case 2:
569	uc = (puch[1] & 0x3f)
570	\| ((RTUNICP)(uch & 0x1f) << 6);
571	RTStrAssertMsgReturn(uc >= 0x00000080 && uc <= 0x000007ff,
572	("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
573	break;
574	}
575
576	/* advance */
577	cch -= cb;
578	puch += cb;
579	cwc++;
580	}
581	}
582
583	/* done */
584	*pcwc = cwc;
585	return VINF_SUCCESS;
586	}
587
588
589	/**
590	* Recodes a valid UTF-8 string as UTF-16.
591	*
592	* Since we know the input is valid, we do not perform encoding or length checks.
593	*
594	* @returns iprt status code.
595	* @param psz The UTF-8 string to recode. This is a valid encoding.
596	* @param cch The number of chars (the type char, so bytes if you like) to process of the UTF-8 string.
597	* The recoding will stop when cch or '\\0' is reached. Pass RTSTR_MAX to process up to '\\0'.
598	* @param pwsz Where to store the UTF-16 string.
599	* @param cwc The number of RTUTF16 items the pwsz buffer can hold, excluding the terminator ('\\0').
600	*/
601	static int rtUtf8RecodeAsUtf16(const char *psz, size_t cch, PRTUTF16 pwsz, size_t cwc)
602	{
603	int rc = VINF_SUCCESS;
604	const unsigned char puch = (const unsigned char )psz;
605	PRTUTF16 pwc = pwsz;
606	while (cch > 0)
607	{
608	/* read the next char and check for terminator. */
609	const unsigned char uch = *puch;
610	if (!uch)
611	break;
612
613	/* check for output overflow */
614	if (RT_UNLIKELY(cwc < 1))
615	{
616	rc = VERR_BUFFER_OVERFLOW;
617	break;
618	}
619	cwc--;
620
621	/* decode and recode the code point */
622	if (!(uch & RT_BIT(7)))
623	{
624	*pwc++ = uch;
625	puch++;
626	cch--;
627	}
628	else if ((uch & (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5))) == (RT_BIT(7) \| RT_BIT(6)))
629	{
630	uint16_t uc = (puch[1] & 0x3f)
631	\| ((uint16_t)(uch & 0x1f) << 6);
632	*pwc++ = uc;
633	puch += 2;
634	cch -= 2;
635	}
636	else if ((uch & (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5) \| RT_BIT(4))) == (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5)))
637	{
638	uint16_t uc = (puch[2] & 0x3f)
639	\| ((uint16_t)(puch[1] & 0x3f) << 6)
640	\| ((uint16_t)(uch & 0x0f) << 12);
641	*pwc++ = uc;
642	puch += 3;
643	cch -= 3;
644	}
645	else
646	{
647	/* generate surrugate pair */
648	Assert((uch & (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5) \| RT_BIT(4) \| RT_BIT(3))) == (RT_BIT(7) \| RT_BIT(6) \| RT_BIT(5) \| RT_BIT(4)));
649	RTUNICP uc = (puch[3] & 0x3f)
650	\| ((RTUNICP)(puch[2] & 0x3f) << 6)
651	\| ((RTUNICP)(puch[1] & 0x3f) << 12)
652	\| ((RTUNICP)(uch & 0x07) << 18);
653	if (RT_UNLIKELY(cwc < 1))
654	{
655	rc = VERR_BUFFER_OVERFLOW;
656	break;
657	}
658	cwc--;
659
660	uc -= 0x10000;
661	*pwc++ = 0xd800 \| (uc >> 10);
662	*pwc++ = 0xdc00 \| (uc & 0x3ff);
663	puch += 4;
664	cch -= 4;
665	}
666	}
667
668	/* done */
669	*pwc = '\0';
670	return rc;
671	}
672
673
674	RTDECL(int) RTStrToUtf16(const char pszString, PRTUTF16 ppwszString)
675	{
676	/*
677	* Validate input.
678	*/
679	Assert(VALID_PTR(ppwszString));
680	Assert(VALID_PTR(pszString));
681	*ppwszString = NULL;
682
683	/*
684	* Validate the UTF-8 input and calculate the length of the UTF-16 string.
685	*/
686	size_t cwc;
687	int rc = rtUtf8CalcUtf16Length(pszString, RTSTR_MAX, &cwc);
688	if (RT_SUCCESS(rc))
689	{
690	/*
691	* Allocate buffer.
692	*/
693	PRTUTF16 pwsz = (PRTUTF16)RTMemAlloc((cwc + 1) * sizeof(RTUTF16));
694	if (pwsz)
695	{
696	/*
697	* Encode the UTF-16 string.
698	*/
699	rc = rtUtf8RecodeAsUtf16(pszString, RTSTR_MAX, pwsz, cwc);
700	if (RT_SUCCESS(rc))
701	{
702	*ppwszString = pwsz;
703	return rc;
704	}
705	RTMemFree(pwsz);
706	}
707	else
708	rc = VERR_NO_UTF16_MEMORY;
709	}
710	return rc;
711	}
712	RT_EXPORT_SYMBOL(RTStrToUtf16);
713
714
715	RTDECL(int) RTStrToUtf16Ex(const char pszString, size_t cchString, PRTUTF16 ppwsz, size_t cwc, size_t *pcwc)
716	{
717	/*
718	* Validate input.
719	*/
720	Assert(VALID_PTR(pszString));
721	Assert(VALID_PTR(ppwsz));
722	Assert(!pcwc \|\| VALID_PTR(pcwc));
723
724	/*
725	* Validate the UTF-8 input and calculate the length of the UTF-16 string.
726	*/
727	size_t cwcResult;
728	int rc = rtUtf8CalcUtf16Length(pszString, cchString, &cwcResult);
729	if (RT_SUCCESS(rc))
730	{
731	if (pcwc)
732	*pcwc = cwcResult;
733
734	/*
735	* Check buffer size / Allocate buffer.
736	*/
737	bool fShouldFree;
738	PRTUTF16 pwszResult;
739	if (cwc > 0 && *ppwsz)
740	{
741	fShouldFree = false;
742	if (cwc <= cwcResult)
743	return VERR_BUFFER_OVERFLOW;
744	pwszResult = *ppwsz;
745	}
746	else
747	{
748	*ppwsz = NULL;
749	fShouldFree = true;
750	cwc = RT_MAX(cwcResult + 1, cwc);
751	pwszResult = (PRTUTF16)RTMemAlloc(cwc * sizeof(RTUTF16));
752	}
753	if (pwszResult)
754	{
755	/*
756	* Encode the UTF-16 string.
757	*/
758	rc = rtUtf8RecodeAsUtf16(pszString, cchString, pwszResult, cwc - 1);
759	if (RT_SUCCESS(rc))
760	{
761	*ppwsz = pwszResult;
762	return rc;
763	}
764	if (fShouldFree)
765	RTMemFree(pwszResult);
766	}
767	else
768	rc = VERR_NO_UTF16_MEMORY;
769	}
770	return rc;
771	}
772	RT_EXPORT_SYMBOL(RTStrToUtf16Ex);
773
774
775	RTDECL(size_t) RTStrCalcUtf16Len(const char *psz)
776	{
777	size_t cwc;
778	int rc = rtUtf8CalcUtf16Length(psz, RTSTR_MAX, &cwc);
779	return RT_SUCCESS(rc) ? cwc : 0;
780	}
781	RT_EXPORT_SYMBOL(RTStrCalcUtf16Len);
782
783
784	RTDECL(int) RTStrCalcUtf16LenEx(const char psz, size_t cch, size_t pcwc)
785	{
786	size_t cwc;
787	int rc = rtUtf8CalcUtf16Length(psz, cch, &cwc);
788	if (pcwc)
789	*pcwc = RT_SUCCESS(rc) ? cwc : ~(size_t)0;
790	return rc;
791	}
792	RT_EXPORT_SYMBOL(RTStrCalcUtf16LenEx);
793
794
795	/**
796	* Handle invalid encodings passed to RTStrGetCp() and RTStrGetCpEx().
797	* @returns rc
798	* @param ppsz The pointer to the string position point.
799	* @param pCp Where to store RTUNICP_INVALID.
800	* @param rc The iprt error code.
801	*/
802	static int rtStrGetCpExFailure(const char **ppsz, PRTUNICP pCp, int rc)
803	{
804	/*
805	* Try find a valid encoding.
806	*/
807	(ppsz)++; /* @todo code this! */
808	*pCp = RTUNICP_INVALID;
809	return rc;
810	}
811
812
813	RTDECL(RTUNICP) RTStrGetCpInternal(const char *psz)
814	{
815	RTUNICP Cp;
816	RTStrGetCpExInternal(&psz, &Cp);
817	return Cp;
818	}
819	RT_EXPORT_SYMBOL(RTStrGetCpInternal);
820
821
822	RTDECL(int) RTStrGetCpExInternal(const char **ppsz, PRTUNICP pCp)
823	{
824	const unsigned char puch = (const unsigned char )*ppsz;
825	const unsigned char uch = *puch;
826	RTUNICP uc;
827
828	/* ASCII ? */
829	if (!(uch & RT_BIT(7)))
830	{
831	uc = uch;
832	puch++;
833	}
834	else if (uch & RT_BIT(6))
835	{
836	/* figure the length and validate the first octet. */
837	unsigned cb;
838	if (!(uch & RT_BIT(5)))
839	cb = 2;
840	else if (!(uch & RT_BIT(4)))
841	cb = 3;
842	else if (!(uch & RT_BIT(3)))
843	cb = 4;
844	else if (!(uch & RT_BIT(2)))
845	cb = 5;
846	else if (!(uch & RT_BIT(1)))
847	cb = 6;
848	else
849	{
850	RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.Rhxs\n", RT_MIN(strlen((char )puch), 10), puch));
851	return rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING);
852	}
853
854	/* validate the rest */
855	switch (cb)
856	{
857	case 6:
858	RTStrAssertMsgReturn((puch[5] & 0xc0) == 0x80, ("6/%u: %.Rhxs\n", cb, RT_MIN(cb + 10, strlen((char )puch)), puch),
859	rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
860	case 5:
861	RTStrAssertMsgReturn((puch[4] & 0xc0) == 0x80, ("5/%u: %.Rhxs\n", cb, RT_MIN(cb + 10, strlen((char )puch)), puch),
862	rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
863	case 4:
864	RTStrAssertMsgReturn((puch[3] & 0xc0) == 0x80, ("4/%u: %.Rhxs\n", cb, RT_MIN(cb + 10, strlen((char )puch)), puch),
865	rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
866	case 3:
867	RTStrAssertMsgReturn((puch[2] & 0xc0) == 0x80, ("3/%u: %.Rhxs\n", cb, RT_MIN(cb + 10, strlen((char )puch)), puch),
868	rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
869	case 2:
870	RTStrAssertMsgReturn((puch[1] & 0xc0) == 0x80, ("2/%u: %.Rhxs\n", cb, RT_MIN(cb + 10, strlen((char )puch)), puch),
871	rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
872	break;
873	}
874
875	/* get and validate the code point. */
876	switch (cb)
877	{
878	case 6:
879	uc = (puch[5] & 0x3f)
880	\| ((RTUNICP)(puch[4] & 0x3f) << 6)
881	\| ((RTUNICP)(puch[3] & 0x3f) << 12)
882	\| ((RTUNICP)(puch[2] & 0x3f) << 18)
883	\| ((RTUNICP)(puch[1] & 0x3f) << 24)
884	\| ((RTUNICP)(uch & 0x01) << 30);
885	RTStrAssertMsgReturn(uc >= 0x04000000 && uc <= 0x7fffffff,
886	("%u: cp=%#010RX32: %.Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char )puch)), puch),
887	rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
888	break;
889	case 5:
890	uc = (puch[4] & 0x3f)
891	\| ((RTUNICP)(puch[3] & 0x3f) << 6)
892	\| ((RTUNICP)(puch[2] & 0x3f) << 12)
893	\| ((RTUNICP)(puch[1] & 0x3f) << 18)
894	\| ((RTUNICP)(uch & 0x03) << 24);
895	RTStrAssertMsgReturn(uc >= 0x00200000 && uc <= 0x03ffffff,
896	("%u: cp=%#010RX32: %.Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char )puch)), puch),
897	rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
898	break;
899	case 4:
900	uc = (puch[3] & 0x3f)
901	\| ((RTUNICP)(puch[2] & 0x3f) << 6)
902	\| ((RTUNICP)(puch[1] & 0x3f) << 12)
903	\| ((RTUNICP)(uch & 0x07) << 18);
904	RTStrAssertMsgReturn(uc >= 0x00010000 && uc <= 0x001fffff,
905	("%u: cp=%#010RX32: %.Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char )puch)), puch),
906	rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
907	break;
908	case 3:
909	uc = (puch[2] & 0x3f)
910	\| ((RTUNICP)(puch[1] & 0x3f) << 6)
911	\| ((RTUNICP)(uch & 0x0f) << 12);
912	RTStrAssertMsgReturn(uc >= 0x00000800 && uc <= 0x0000fffd,
913	("%u: cp=%#010RX32: %.Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char )puch)), puch),
914	rtStrGetCpExFailure(ppsz, pCp, uc == 0xffff \|\| uc == 0xfffe ? VERR_CODE_POINT_ENDIAN_INDICATOR : VERR_INVALID_UTF8_ENCODING));
915	RTStrAssertMsgReturn(uc < 0xd800 \|\| uc > 0xdfff,
916	("%u: cp=%#010RX32: %.Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char )puch)), puch),
917	rtStrGetCpExFailure(ppsz, pCp, VERR_CODE_POINT_SURROGATE));
918	break;
919	case 2:
920	uc = (puch[1] & 0x3f)
921	\| ((RTUNICP)(uch & 0x1f) << 6);
922	RTStrAssertMsgReturn(uc >= 0x00000080 && uc <= 0x000007ff,
923	("%u: cp=%#010RX32: %.Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char )puch)), puch),
924	rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
925	break;
926	default: /* impossible, but GCC is bitching. */
927	uc = RTUNICP_INVALID;
928	break;
929	}
930	puch += cb;
931	}
932	else
933	{
934	/* 6th bit is always set. */
935	RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.Rhxs\n", RT_MIN(strlen((char )puch), 10), puch));
936	return rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING);
937	}
938	*pCp = uc;
939	ppsz = (const char )puch;
940	return VINF_SUCCESS;
941	}
942	RT_EXPORT_SYMBOL(RTStrGetCpExInternal);
943
944
945	/**
946	* Handle invalid encodings passed to RTStrGetCpNEx().
947	* @returns rc
948	* @param ppsz The pointer to the string position point.
949	* @param pcch Pointer to the string length.
950	* @param pCp Where to store RTUNICP_INVALID.
951	* @param rc The iprt error code.
952	*/
953	static int rtStrGetCpNExFailure(const char *ppsz, size_t pcch, PRTUNICP pCp, int rc)
954	{
955	/*
956	* Try find a valid encoding.
957	*/
958	(ppsz)++; /* @todo code this! */
959	(*pcch)--;
960	*pCp = RTUNICP_INVALID;
961	return rc;
962	}
963
964
965	RTDECL(int) RTStrGetCpNExInternal(const char *ppsz, size_t pcch, PRTUNICP pCp)
966	{
967	const unsigned char puch = (const unsigned char )*ppsz;
968	const unsigned char uch = *puch;
969	size_t cch = *pcch;
970	RTUNICP uc;
971
972	if (cch == 0)
973	{
974	*pCp = RTUNICP_INVALID;
975	return VERR_END_OF_STRING;
976	}
977
978	/* ASCII ? */
979	if (!(uch & RT_BIT(7)))
980	{
981	uc = uch;
982	puch++;
983	cch--;
984	}
985	else if (uch & RT_BIT(6))
986	{
987	/* figure the length and validate the first octet. */
988	unsigned cb;
989	if (!(uch & RT_BIT(5)))
990	cb = 2;
991	else if (!(uch & RT_BIT(4)))
992	cb = 3;
993	else if (!(uch & RT_BIT(3)))
994	cb = 4;
995	else if (!(uch & RT_BIT(2)))
996	cb = 5;
997	else if (!(uch & RT_BIT(1)))
998	cb = 6;
999	else
1000	{
1001	RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.Rhxs\n", RT_MIN(strlen((char )puch), 10), puch));
1002	return rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING);
1003	}
1004
1005	if (cb > cch)
1006	return rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING);
1007
1008	/* validate the rest */
1009	switch (cb)
1010	{
1011	case 6:
1012	RTStrAssertMsgReturn((puch[5] & 0xc0) == 0x80, ("6/%u: %.Rhxs\n", cb, RT_MIN(cb + 10, strlen((char )puch)), puch),
1013	rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1014	case 5:
1015	RTStrAssertMsgReturn((puch[4] & 0xc0) == 0x80, ("5/%u: %.Rhxs\n", cb, RT_MIN(cb + 10, strlen((char )puch)), puch),
1016	rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1017	case 4:
1018	RTStrAssertMsgReturn((puch[3] & 0xc0) == 0x80, ("4/%u: %.Rhxs\n", cb, RT_MIN(cb + 10, strlen((char )puch)), puch),
1019	rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1020	case 3:
1021	RTStrAssertMsgReturn((puch[2] & 0xc0) == 0x80, ("3/%u: %.Rhxs\n", cb, RT_MIN(cb + 10, strlen((char )puch)), puch),
1022	rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1023	case 2:
1024	RTStrAssertMsgReturn((puch[1] & 0xc0) == 0x80, ("2/%u: %.Rhxs\n", cb, RT_MIN(cb + 10, strlen((char )puch)), puch),
1025	rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1026	break;
1027	}
1028
1029	/* get and validate the code point. */
1030	switch (cb)
1031	{
1032	case 6:
1033	uc = (puch[5] & 0x3f)
1034	\| ((RTUNICP)(puch[4] & 0x3f) << 6)
1035	\| ((RTUNICP)(puch[3] & 0x3f) << 12)
1036	\| ((RTUNICP)(puch[2] & 0x3f) << 18)
1037	\| ((RTUNICP)(puch[1] & 0x3f) << 24)
1038	\| ((RTUNICP)(uch & 0x01) << 30);
1039	RTStrAssertMsgReturn(uc >= 0x04000000 && uc <= 0x7fffffff,
1040	("%u: cp=%#010RX32: %.Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char )puch)), puch),
1041	rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1042	break;
1043	case 5:
1044	uc = (puch[4] & 0x3f)
1045	\| ((RTUNICP)(puch[3] & 0x3f) << 6)
1046	\| ((RTUNICP)(puch[2] & 0x3f) << 12)
1047	\| ((RTUNICP)(puch[1] & 0x3f) << 18)
1048	\| ((RTUNICP)(uch & 0x03) << 24);
1049	RTStrAssertMsgReturn(uc >= 0x00200000 && uc <= 0x03ffffff,
1050	("%u: cp=%#010RX32: %.Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char )puch)), puch),
1051	rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1052	break;
1053	case 4:
1054	uc = (puch[3] & 0x3f)
1055	\| ((RTUNICP)(puch[2] & 0x3f) << 6)
1056	\| ((RTUNICP)(puch[1] & 0x3f) << 12)
1057	\| ((RTUNICP)(uch & 0x07) << 18);
1058	RTStrAssertMsgReturn(uc >= 0x00010000 && uc <= 0x001fffff,
1059	("%u: cp=%#010RX32: %.Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char )puch)), puch),
1060	rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1061	break;
1062	case 3:
1063	uc = (puch[2] & 0x3f)
1064	\| ((RTUNICP)(puch[1] & 0x3f) << 6)
1065	\| ((RTUNICP)(uch & 0x0f) << 12);
1066	RTStrAssertMsgReturn(uc >= 0x00000800 && uc <= 0x0000fffd,
1067	("%u: cp=%#010RX32: %.Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char )puch)), puch),
1068	rtStrGetCpNExFailure(ppsz, pcch, pCp, uc == 0xffff \|\| uc == 0xfffe ? VERR_CODE_POINT_ENDIAN_INDICATOR : VERR_INVALID_UTF8_ENCODING));
1069	RTStrAssertMsgReturn(uc < 0xd800 \|\| uc > 0xdfff,
1070	("%u: cp=%#010RX32: %.Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char )puch)), puch),
1071	rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_CODE_POINT_SURROGATE));
1072	break;
1073	case 2:
1074	uc = (puch[1] & 0x3f)
1075	\| ((RTUNICP)(uch & 0x1f) << 6);
1076	RTStrAssertMsgReturn(uc >= 0x00000080 && uc <= 0x000007ff,
1077	("%u: cp=%#010RX32: %.Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char )puch)), puch),
1078	rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1079	break;
1080	default: /* impossible, but GCC is bitching. */
1081	uc = RTUNICP_INVALID;
1082	break;
1083	}
1084	puch += cb;
1085	cch -= cb;
1086	}
1087	else
1088	{
1089	/* 6th bit is always set. */
1090	RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.Rhxs\n", RT_MIN(strlen((char )puch), 10), puch));
1091	return rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING);
1092	}
1093	*pCp = uc;
1094	ppsz = (const char )puch;
1095	(*pcch) = cch;
1096	return VINF_SUCCESS;
1097	}
1098	RT_EXPORT_SYMBOL(RTStrGetCpNExInternal);
1099
1100
1101	RTDECL(char ) RTStrPutCpInternal(char psz, RTUNICP uc)
1102	{
1103	unsigned char puch = (unsigned char )psz;
1104	if (uc < 0x80)
1105	*puch++ = (unsigned char )uc;
1106	else if (uc < 0x00000800)
1107	{
1108	*puch++ = 0xc0 \| (uc >> 6);
1109	*puch++ = 0x80 \| (uc & 0x3f);
1110	}
1111	else if (uc < 0x00010000)
1112	{
1113	if ( uc < 0x0000d8000
1114	\|\| ( uc > 0x0000dfff
1115	&& uc < 0x0000fffe))
1116	{
1117	*puch++ = 0xe0 \| (uc >> 12);
1118	*puch++ = 0x80 \| ((uc >> 6) & 0x3f);
1119	*puch++ = 0x80 \| (uc & 0x3f);
1120	}
1121	else
1122	{
1123	AssertMsgFailed(("Invalid code point U+%05x!\n", uc));
1124	*puch++ = 0x7f;
1125	}
1126	}
1127	else if (uc < 0x00200000)
1128	{
1129	*puch++ = 0xf0 \| (uc >> 18);
1130	*puch++ = 0x80 \| ((uc >> 12) & 0x3f);
1131	*puch++ = 0x80 \| ((uc >> 6) & 0x3f);
1132	*puch++ = 0x80 \| (uc & 0x3f);
1133	}
1134	else if (uc < 0x04000000)
1135	{
1136	*puch++ = 0xf8 \| (uc >> 24);
1137	*puch++ = 0x80 \| ((uc >> 18) & 0x3f);
1138	*puch++ = 0x80 \| ((uc >> 12) & 0x3f);
1139	*puch++ = 0x80 \| ((uc >> 6) & 0x3f);
1140	*puch++ = 0x80 \| (uc & 0x3f);
1141	}
1142	else if (uc <= 0x7fffffff)
1143	{
1144	*puch++ = 0xfc \| (uc >> 30);
1145	*puch++ = 0x80 \| ((uc >> 24) & 0x3f);
1146	*puch++ = 0x80 \| ((uc >> 18) & 0x3f);
1147	*puch++ = 0x80 \| ((uc >> 12) & 0x3f);
1148	*puch++ = 0x80 \| ((uc >> 6) & 0x3f);
1149	*puch++ = 0x80 \| (uc & 0x3f);
1150	}
1151	else
1152	{
1153	AssertMsgFailed(("Invalid code point U+%08x!\n", uc));
1154	*puch++ = 0x7f;
1155	}
1156
1157	return (char *)puch;
1158	}
1159	RT_EXPORT_SYMBOL(RTStrPutCpInternal);
1160
1161
1162	RTDECL(char ) RTStrPrevCp(const char pszStart, const char *psz)
1163	{
1164	if (pszStart < psz)
1165	{
1166	/* simple char? */
1167	const unsigned char puch = (const unsigned char )psz;
1168	unsigned uch = *--puch;
1169	if (!(uch & RT_BIT(7)))
1170	return (char *)puch;
1171	RTStrAssertMsgReturn(!(uch & RT_BIT(6)), ("uch=%#x\n", uch), (char *)pszStart);
1172
1173	/* two or more. */
1174	uint32_t uMask = 0xffffffc0;
1175	while ( (const unsigned char *)pszStart < puch
1176	&& !(uMask & 1))
1177	{
1178	uch = *--puch;
1179	if ((uch & 0xc0) != 0x80)
1180	{
1181	RTStrAssertMsgReturn((uch & (uMask >> 1)) == (uMask & 0xff),
1182	("Invalid UTF-8 encoding: %.Rhxs puch=%p psz=%p\n", psz - (char )puch, puch, psz),
1183	(char *)pszStart);
1184	return (char *)puch;
1185	}
1186	uMask >>= 1;
1187	}
1188	RTStrAssertMsgFailed(("Invalid UTF-8 encoding: %.Rhxs puch=%p psz=%p\n", psz - (char )puch, puch, psz));
1189	}
1190	return (char *)pszStart;
1191	}
1192	RT_EXPORT_SYMBOL(RTStrPrevCp);
1193

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/common/string/utf-8.cpp@ 30859

Download in other formats: