VirtualBox

source: vbox/trunk/src/VBox/Runtime/common/string/utf-8.cpp@ 31199

Last change on this file since 31199 was 31199, checked in by vboxsync, 15 years ago

Runtime/string: add Latin-1 <-> Utf-8 conversion APIs - tests and probably fixes to follow
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 53.9 KB
Line 
1/* $Id: utf-8.cpp 31199 2010-07-29 10:54:16Z vboxsync $ */
2/** @file
3 * IPRT - UTF-8 Decoding.
4 */
5
6/*
7 * Copyright (C) 2006-2010 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.215389.xyz. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 *
17 * The contents of this file may alternatively be used under the terms
18 * of the Common Development and Distribution License Version 1.0
19 * (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20 * VirtualBox OSE distribution, in which case the provisions of the
21 * CDDL are applicable instead of those of the GPL.
22 *
23 * You may elect to license modified versions of this file under the
24 * terms and conditions of either the GPL or the CDDL or both.
25 */
26
27
28/*******************************************************************************
29* Header Files *
30*******************************************************************************/
31#include <iprt/string.h>
32#include "internal/iprt.h"
33
34#include <iprt/uni.h>
35#include <iprt/alloc.h>
36#include <iprt/assert.h>
37#include <iprt/err.h>
38#include "internal/string.h"
39
40
41
42/**
43 * Get get length in code points of a UTF-8 encoded string.
44 * The string is validated while doing this.
45 *
46 * @returns IPRT status code.
47 * @param psz Pointer to the UTF-8 string.
48 * @param cch The max length of the string. (btw cch = cb)
49 * Use RTSTR_MAX if all of the string is to be examined.
50 * @param pcuc Where to store the length in unicode code points.
51 * @param pcchActual Where to store the actual size of the UTF-8 string
52 * on success (cch = cb again). Optional.
53 */
54int rtUtf8Length(const char *psz, size_t cch, size_t *pcuc, size_t *pcchActual)
55{
56 const unsigned char *puch = (const unsigned char *)psz;
57 size_t cCodePoints = 0;
58 while (cch > 0)
59 {
60 const unsigned char uch = *puch;
61 if (!uch)
62 break;
63 if (uch & RT_BIT(7))
64 {
65 /* figure sequence length and validate the first byte */
66 unsigned cb;
67 if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5))) == (RT_BIT(7) | RT_BIT(6)))
68 cb = 2;
69 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5)))
70 cb = 3;
71 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4)))
72 cb = 4;
73 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3) | RT_BIT(2))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3)))
74 cb = 5;
75 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3) | RT_BIT(2) | RT_BIT(1))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3) | RT_BIT(2)))
76 cb = 6;
77 else
78 {
79 RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.*Rhxs\n", RT_MIN(cch, 10), puch));
80 return VERR_INVALID_UTF8_ENCODING;
81 }
82
83 /* check length */
84 if (cb > cch)
85 {
86 RTStrAssertMsgFailed(("Invalid UTF-8 length: cb=%d cch=%d (%.*Rhxs)\n", cb, cch, RT_MIN(cch, 10), puch));
87 return VERR_INVALID_UTF8_ENCODING;
88 }
89
90 /* validate the rest */
91 switch (cb)
92 {
93 case 6:
94 RTStrAssertMsgReturn((puch[5] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("6/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
95 case 5:
96 RTStrAssertMsgReturn((puch[4] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("5/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
97 case 4:
98 RTStrAssertMsgReturn((puch[3] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("4/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
99 case 3:
100 RTStrAssertMsgReturn((puch[2] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("3/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
101 case 2:
102 RTStrAssertMsgReturn((puch[1] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("2/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
103 break;
104 }
105
106 /* validate the code point. */
107 RTUNICP uc;
108 switch (cb)
109 {
110 case 6:
111 uc = (puch[5] & 0x3f)
112 | ((RTUNICP)(puch[4] & 0x3f) << 6)
113 | ((RTUNICP)(puch[3] & 0x3f) << 12)
114 | ((RTUNICP)(puch[2] & 0x3f) << 18)
115 | ((RTUNICP)(puch[1] & 0x3f) << 24)
116 | ((RTUNICP)(uch & 0x01) << 30);
117 RTStrAssertMsgReturn(uc >= 0x04000000 && uc <= 0x7fffffff,
118 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
119 break;
120 case 5:
121 uc = (puch[4] & 0x3f)
122 | ((RTUNICP)(puch[3] & 0x3f) << 6)
123 | ((RTUNICP)(puch[2] & 0x3f) << 12)
124 | ((RTUNICP)(puch[1] & 0x3f) << 18)
125 | ((RTUNICP)(uch & 0x03) << 24);
126 RTStrAssertMsgReturn(uc >= 0x00200000 && uc <= 0x03ffffff,
127 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
128 break;
129 case 4:
130 uc = (puch[3] & 0x3f)
131 | ((RTUNICP)(puch[2] & 0x3f) << 6)
132 | ((RTUNICP)(puch[1] & 0x3f) << 12)
133 | ((RTUNICP)(uch & 0x07) << 18);
134 RTStrAssertMsgReturn(uc >= 0x00010000 && uc <= 0x001fffff,
135 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
136 break;
137 case 3:
138 uc = (puch[2] & 0x3f)
139 | ((RTUNICP)(puch[1] & 0x3f) << 6)
140 | ((RTUNICP)(uch & 0x0f) << 12);
141 RTStrAssertMsgReturn(uc >= 0x00000800 && uc <= 0x0000fffd,
142 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch),
143 uc == 0xffff || uc == 0xfffe ? VERR_CODE_POINT_ENDIAN_INDICATOR : VERR_INVALID_UTF8_ENCODING);
144 RTStrAssertMsgReturn(uc < 0xd800 || uc > 0xdfff,
145 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_CODE_POINT_SURROGATE);
146 break;
147 case 2:
148 uc = (puch[1] & 0x3f)
149 | ((RTUNICP)(uch & 0x1f) << 6);
150 RTStrAssertMsgReturn(uc >= 0x00000080 && uc <= 0x000007ff,
151 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
152 break;
153 }
154
155 /* advance */
156 cch -= cb;
157 puch += cb;
158 }
159 else
160 {
161 /* one ASCII byte */
162 puch++;
163 cch--;
164 }
165 cCodePoints++;
166 }
167
168 /* done */
169 *pcuc = cCodePoints;
170 if (pcchActual)
171 *pcchActual = puch - (unsigned char const *)psz;
172 return VINF_SUCCESS;
173}
174
175
176/**
177 * Decodes and UTF-8 string into an array of unicode code point.
178 *
179 * Since we know the input is valid, we do *not* perform encoding or length checks.
180 *
181 * @returns iprt status code.
182 * @param psz The UTF-8 string to recode. This is a valid encoding.
183 * @param cch The number of chars (the type char, so bytes if you like) to process of the UTF-8 string.
184 * The recoding will stop when cch or '\\0' is reached. Pass RTSTR_MAX to process up to '\\0'.
185 * @param paCps Where to store the code points array.
186 * @param cCps The number of RTUNICP items the paCps buffer can hold, excluding the terminator ('\\0').
187 */
188static int rtUtf8Decode(const char *psz, size_t cch, PRTUNICP paCps, size_t cCps)
189{
190 int rc = VINF_SUCCESS;
191 const unsigned char *puch = (const unsigned char *)psz;
192 PRTUNICP pCp = paCps;
193 while (cch > 0)
194 {
195 /* read the next char and check for terminator. */
196 const unsigned char uch = *puch;
197 if (!uch)
198 break;
199
200 /* check for output overflow */
201 if (RT_UNLIKELY(cCps < 1))
202 {
203 rc = VERR_BUFFER_OVERFLOW;
204 break;
205 }
206 cCps--;
207
208 /* decode and recode the code point */
209 if (!(uch & RT_BIT(7)))
210 {
211 *pCp++ = uch;
212 puch++;
213 cch--;
214 }
215#ifdef RT_STRICT
216 else if (!(uch & RT_BIT(6)))
217 AssertMsgFailed(("Internal error!\n"));
218#endif
219 else if (!(uch & RT_BIT(5)))
220 {
221 *pCp++ = (puch[1] & 0x3f)
222 | ((uint16_t)(uch & 0x1f) << 6);
223 puch += 2;
224 cch -= 2;
225 }
226 else if (!(uch & RT_BIT(4)))
227 {
228 *pCp++ = (puch[2] & 0x3f)
229 | ((uint16_t)(puch[1] & 0x3f) << 6)
230 | ((uint16_t)(uch & 0x0f) << 12);
231 puch += 3;
232 cch -= 3;
233 }
234 else if (!(uch & RT_BIT(3)))
235 {
236 *pCp++ = (puch[3] & 0x3f)
237 | ((RTUNICP)(puch[2] & 0x3f) << 6)
238 | ((RTUNICP)(puch[1] & 0x3f) << 12)
239 | ((RTUNICP)(uch & 0x07) << 18);
240 puch += 4;
241 cch -= 4;
242 }
243 else if (!(uch & RT_BIT(2)))
244 {
245 *pCp++ = (puch[4] & 0x3f)
246 | ((RTUNICP)(puch[3] & 0x3f) << 6)
247 | ((RTUNICP)(puch[2] & 0x3f) << 12)
248 | ((RTUNICP)(puch[1] & 0x3f) << 18)
249 | ((RTUNICP)(uch & 0x03) << 24);
250 puch += 5;
251 cch -= 6;
252 }
253 else
254 {
255 Assert(!(uch & RT_BIT(1)));
256 *pCp++ = (puch[5] & 0x3f)
257 | ((RTUNICP)(puch[4] & 0x3f) << 6)
258 | ((RTUNICP)(puch[3] & 0x3f) << 12)
259 | ((RTUNICP)(puch[2] & 0x3f) << 18)
260 | ((RTUNICP)(puch[1] & 0x3f) << 24)
261 | ((RTUNICP)(uch & 0x01) << 30);
262 puch += 6;
263 cch -= 6;
264 }
265 }
266
267 /* done */
268 *pCp = 0;
269 return rc;
270}
271
272
273RTDECL(size_t) RTStrUniLen(const char *psz)
274{
275 size_t cCodePoints;
276 int rc = rtUtf8Length(psz, RTSTR_MAX, &cCodePoints, NULL);
277 return RT_SUCCESS(rc) ? cCodePoints : 0;
278}
279RT_EXPORT_SYMBOL(RTStrUniLen);
280
281
282RTDECL(int) RTStrUniLenEx(const char *psz, size_t cch, size_t *pcCps)
283{
284 size_t cCodePoints;
285 int rc = rtUtf8Length(psz, cch, &cCodePoints, NULL);
286 if (pcCps)
287 *pcCps = RT_SUCCESS(rc) ? cCodePoints : 0;
288 return rc;
289}
290RT_EXPORT_SYMBOL(RTStrUniLenEx);
291
292
293RTDECL(int) RTStrValidateEncoding(const char *psz)
294{
295 return RTStrValidateEncodingEx(psz, RTSTR_MAX, 0);
296}
297RT_EXPORT_SYMBOL(RTStrValidateEncoding);
298
299
300RTDECL(int) RTStrValidateEncodingEx(const char *psz, size_t cch, uint32_t fFlags)
301{
302 AssertReturn(!(fFlags & ~(RTSTR_VALIDATE_ENCODING_ZERO_TERMINATED)), VERR_INVALID_PARAMETER);
303 AssertPtr(psz);
304
305 /*
306 * Use rtUtf8Length for the job.
307 */
308 size_t cchActual;
309 size_t cCpsIgnored;
310 int rc = rtUtf8Length(psz, cch, &cCpsIgnored, &cchActual);
311 if (RT_SUCCESS(rc))
312 {
313 if ( (fFlags & RTSTR_VALIDATE_ENCODING_ZERO_TERMINATED)
314 && cchActual >= cch)
315 rc = VERR_BUFFER_OVERFLOW;
316 }
317 return rc;
318}
319RT_EXPORT_SYMBOL(RTStrValidateEncodingEx);
320
321
322RTDECL(bool) RTStrIsValidEncoding(const char *psz)
323{
324 int rc = RTStrValidateEncodingEx(psz, RTSTR_MAX, 0);
325 return RT_SUCCESS(rc);
326}
327RT_EXPORT_SYMBOL(RTStrIsValidEncoding);
328
329
330RTDECL(size_t) RTStrPurgeEncoding(char *psz)
331{
332 size_t cErrors = 0;
333 for (;;)
334 {
335 RTUNICP Cp;
336 int rc = RTStrGetCpEx((const char **)&psz, &Cp);
337 if (RT_SUCCESS(rc))
338 {
339 if (!Cp)
340 break;
341 }
342 else
343 {
344 psz[-1] = '?';
345 cErrors++;
346 }
347 }
348 return cErrors;
349}
350RT_EXPORT_SYMBOL(RTStrPurgeEncoding);
351
352
353RTDECL(int) RTStrToUni(const char *pszString, PRTUNICP *ppaCps)
354{
355 /*
356 * Validate input.
357 */
358 Assert(VALID_PTR(pszString));
359 Assert(VALID_PTR(ppaCps));
360 *ppaCps = NULL;
361
362 /*
363 * Validate the UTF-8 input and count its code points.
364 */
365 size_t cCps;
366 int rc = rtUtf8Length(pszString, RTSTR_MAX, &cCps, NULL);
367 if (RT_SUCCESS(rc))
368 {
369 /*
370 * Allocate buffer.
371 */
372 PRTUNICP paCps = (PRTUNICP)RTMemAlloc((cCps + 1) * sizeof(RTUNICP));
373 if (paCps)
374 {
375 /*
376 * Decode the string.
377 */
378 rc = rtUtf8Decode(pszString, RTSTR_MAX, paCps, cCps);
379 if (RT_SUCCESS(rc))
380 {
381 *ppaCps = paCps;
382 return rc;
383 }
384 RTMemFree(paCps);
385 }
386 else
387 rc = VERR_NO_CODE_POINT_MEMORY;
388 }
389 return rc;
390}
391RT_EXPORT_SYMBOL(RTStrToUni);
392
393
394RTDECL(int) RTStrToUniEx(const char *pszString, size_t cchString, PRTUNICP *ppaCps, size_t cCps, size_t *pcCps)
395{
396 /*
397 * Validate input.
398 */
399 Assert(VALID_PTR(pszString));
400 Assert(VALID_PTR(ppaCps));
401 Assert(!pcCps || VALID_PTR(pcCps));
402
403 /*
404 * Validate the UTF-8 input and count the code points.
405 */
406 size_t cCpsResult;
407 int rc = rtUtf8Length(pszString, cchString, &cCpsResult, NULL);
408 if (RT_SUCCESS(rc))
409 {
410 if (pcCps)
411 *pcCps = cCpsResult;
412
413 /*
414 * Check buffer size / Allocate buffer.
415 */
416 bool fShouldFree;
417 PRTUNICP paCpsResult;
418 if (cCps > 0 && *ppaCps)
419 {
420 fShouldFree = false;
421 if (cCps <= cCpsResult)
422 return VERR_BUFFER_OVERFLOW;
423 paCpsResult = *ppaCps;
424 }
425 else
426 {
427 *ppaCps = NULL;
428 fShouldFree = true;
429 cCps = RT_MAX(cCpsResult + 1, cCps);
430 paCpsResult = (PRTUNICP)RTMemAlloc(cCps * sizeof(RTUNICP));
431 }
432 if (paCpsResult)
433 {
434 /*
435 * Encode the UTF-16 string.
436 */
437 rc = rtUtf8Decode(pszString, cchString, paCpsResult, cCps - 1);
438 if (RT_SUCCESS(rc))
439 {
440 *ppaCps = paCpsResult;
441 return rc;
442 }
443 if (fShouldFree)
444 RTMemFree(paCpsResult);
445 }
446 else
447 rc = VERR_NO_CODE_POINT_MEMORY;
448 }
449 return rc;
450}
451RT_EXPORT_SYMBOL(RTStrToUniEx);
452
453
454/**
455 * Calculates the UTF-16 length of a string, validating the encoding while doing so.
456 *
457 * @returns IPRT status code.
458 * @param psz Pointer to the UTF-8 string.
459 * @param cch The max length of the string. (btw cch = cb)
460 * Use RTSTR_MAX if all of the string is to be examined.
461 * @param pcwc Where to store the length of the UTF-16 string as a number of RTUTF16 characters.
462 */
463static int rtUtf8CalcUtf16Length(const char *psz, size_t cch, size_t *pcwc)
464{
465 const unsigned char *puch = (const unsigned char *)psz;
466 size_t cwc = 0;
467 while (cch > 0)
468 {
469 const unsigned char uch = *puch;
470 if (!uch)
471 break;
472 if (!(uch & RT_BIT(7)))
473 {
474 /* one ASCII byte */
475 cwc++;
476 puch++;
477 cch--;
478 }
479 else
480 {
481 /* figure sequence length and validate the first byte */
482 unsigned cb;
483 if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5))) == (RT_BIT(7) | RT_BIT(6)))
484 cb = 2;
485 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5)))
486 cb = 3;
487 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4)))
488 cb = 4;
489 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3) | RT_BIT(2))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3)))
490 cb = 5;
491 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3) | RT_BIT(2) | RT_BIT(1))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3) | RT_BIT(2)))
492 cb = 6;
493 else
494 {
495 RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.*Rhxs\n", RT_MIN(cch, 10), puch));
496 return VERR_INVALID_UTF8_ENCODING;
497 }
498
499 /* check length */
500 if (cb > cch)
501 {
502 RTStrAssertMsgFailed(("Invalid UTF-8 length: cb=%d cch=%d (%.*Rhxs)\n", cb, cch, RT_MIN(cch, 10), puch));
503 return VERR_INVALID_UTF8_ENCODING;
504 }
505
506 /* validate the rest */
507 switch (cb)
508 {
509 case 6:
510 RTStrAssertMsgReturn((puch[5] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("6/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
511 case 5:
512 RTStrAssertMsgReturn((puch[4] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("5/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
513 case 4:
514 RTStrAssertMsgReturn((puch[3] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("4/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
515 case 3:
516 RTStrAssertMsgReturn((puch[2] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("3/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
517 case 2:
518 RTStrAssertMsgReturn((puch[1] & (RT_BIT(7) | RT_BIT(6))) == RT_BIT(7), ("2/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
519 break;
520 }
521
522 /* validate the code point. */
523 RTUNICP uc;
524 switch (cb)
525 {
526 case 6:
527 uc = (puch[5] & 0x3f)
528 | ((RTUNICP)(puch[4] & 0x3f) << 6)
529 | ((RTUNICP)(puch[3] & 0x3f) << 12)
530 | ((RTUNICP)(puch[2] & 0x3f) << 18)
531 | ((RTUNICP)(puch[1] & 0x3f) << 24)
532 | ((RTUNICP)(uch & 0x01) << 30);
533 RTStrAssertMsgReturn(uc >= 0x04000000 && uc <= 0x7fffffff,
534 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
535 RTStrAssertMsgFailed(("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch));
536 return VERR_CANT_RECODE_AS_UTF16;
537 case 5:
538 uc = (puch[4] & 0x3f)
539 | ((RTUNICP)(puch[3] & 0x3f) << 6)
540 | ((RTUNICP)(puch[2] & 0x3f) << 12)
541 | ((RTUNICP)(puch[1] & 0x3f) << 18)
542 | ((RTUNICP)(uch & 0x03) << 24);
543 RTStrAssertMsgReturn(uc >= 0x00200000 && uc <= 0x03ffffff,
544 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
545 RTStrAssertMsgFailed(("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch));
546 return VERR_CANT_RECODE_AS_UTF16;
547 case 4:
548 uc = (puch[3] & 0x3f)
549 | ((RTUNICP)(puch[2] & 0x3f) << 6)
550 | ((RTUNICP)(puch[1] & 0x3f) << 12)
551 | ((RTUNICP)(uch & 0x07) << 18);
552 RTStrAssertMsgReturn(uc >= 0x00010000 && uc <= 0x001fffff,
553 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
554 RTStrAssertMsgReturn(uc <= 0x0010ffff,
555 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_CANT_RECODE_AS_UTF16);
556 cwc++;
557 break;
558 case 3:
559 uc = (puch[2] & 0x3f)
560 | ((RTUNICP)(puch[1] & 0x3f) << 6)
561 | ((RTUNICP)(uch & 0x0f) << 12);
562 RTStrAssertMsgReturn(uc >= 0x00000800 && uc <= 0x0000fffd,
563 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch),
564 uc == 0xffff || uc == 0xfffe ? VERR_CODE_POINT_ENDIAN_INDICATOR : VERR_INVALID_UTF8_ENCODING);
565 RTStrAssertMsgReturn(uc < 0xd800 || uc > 0xdfff,
566 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_CODE_POINT_SURROGATE);
567 break;
568 case 2:
569 uc = (puch[1] & 0x3f)
570 | ((RTUNICP)(uch & 0x1f) << 6);
571 RTStrAssertMsgReturn(uc >= 0x00000080 && uc <= 0x000007ff,
572 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, cch), puch), VERR_INVALID_UTF8_ENCODING);
573 break;
574 }
575
576 /* advance */
577 cch -= cb;
578 puch += cb;
579 cwc++;
580 }
581 }
582
583 /* done */
584 *pcwc = cwc;
585 return VINF_SUCCESS;
586}
587
588
589/**
590 * Recodes a valid UTF-8 string as UTF-16.
591 *
592 * Since we know the input is valid, we do *not* perform encoding or length checks.
593 *
594 * @returns iprt status code.
595 * @param psz The UTF-8 string to recode. This is a valid encoding.
596 * @param cch The number of chars (the type char, so bytes if you like) to process of the UTF-8 string.
597 * The recoding will stop when cch or '\\0' is reached. Pass RTSTR_MAX to process up to '\\0'.
598 * @param pwsz Where to store the UTF-16 string.
599 * @param cwc The number of RTUTF16 items the pwsz buffer can hold, excluding the terminator ('\\0').
600 */
601static int rtUtf8RecodeAsUtf16(const char *psz, size_t cch, PRTUTF16 pwsz, size_t cwc)
602{
603 int rc = VINF_SUCCESS;
604 const unsigned char *puch = (const unsigned char *)psz;
605 PRTUTF16 pwc = pwsz;
606 while (cch > 0)
607 {
608 /* read the next char and check for terminator. */
609 const unsigned char uch = *puch;
610 if (!uch)
611 break;
612
613 /* check for output overflow */
614 if (RT_UNLIKELY(cwc < 1))
615 {
616 rc = VERR_BUFFER_OVERFLOW;
617 break;
618 }
619 cwc--;
620
621 /* decode and recode the code point */
622 if (!(uch & RT_BIT(7)))
623 {
624 *pwc++ = uch;
625 puch++;
626 cch--;
627 }
628 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5))) == (RT_BIT(7) | RT_BIT(6)))
629 {
630 uint16_t uc = (puch[1] & 0x3f)
631 | ((uint16_t)(uch & 0x1f) << 6);
632 *pwc++ = uc;
633 puch += 2;
634 cch -= 2;
635 }
636 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5)))
637 {
638 uint16_t uc = (puch[2] & 0x3f)
639 | ((uint16_t)(puch[1] & 0x3f) << 6)
640 | ((uint16_t)(uch & 0x0f) << 12);
641 *pwc++ = uc;
642 puch += 3;
643 cch -= 3;
644 }
645 else
646 {
647 /* generate surrugate pair */
648 Assert((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4) | RT_BIT(3))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4)));
649 RTUNICP uc = (puch[3] & 0x3f)
650 | ((RTUNICP)(puch[2] & 0x3f) << 6)
651 | ((RTUNICP)(puch[1] & 0x3f) << 12)
652 | ((RTUNICP)(uch & 0x07) << 18);
653 if (RT_UNLIKELY(cwc < 1))
654 {
655 rc = VERR_BUFFER_OVERFLOW;
656 break;
657 }
658 cwc--;
659
660 uc -= 0x10000;
661 *pwc++ = 0xd800 | (uc >> 10);
662 *pwc++ = 0xdc00 | (uc & 0x3ff);
663 puch += 4;
664 cch -= 4;
665 }
666 }
667
668 /* done */
669 *pwc = '\0';
670 return rc;
671}
672
673
674RTDECL(int) RTStrToUtf16Tag(const char *pszString, PRTUTF16 *ppwszString, const char *pszTag)
675{
676 /*
677 * Validate input.
678 */
679 Assert(VALID_PTR(ppwszString));
680 Assert(VALID_PTR(pszString));
681 *ppwszString = NULL;
682
683 /*
684 * Validate the UTF-8 input and calculate the length of the UTF-16 string.
685 */
686 size_t cwc;
687 int rc = rtUtf8CalcUtf16Length(pszString, RTSTR_MAX, &cwc);
688 if (RT_SUCCESS(rc))
689 {
690 /*
691 * Allocate buffer.
692 */
693 PRTUTF16 pwsz = (PRTUTF16)RTMemAllocTag((cwc + 1) * sizeof(RTUTF16), pszTag);
694 if (pwsz)
695 {
696 /*
697 * Encode the UTF-16 string.
698 */
699 rc = rtUtf8RecodeAsUtf16(pszString, RTSTR_MAX, pwsz, cwc);
700 if (RT_SUCCESS(rc))
701 {
702 *ppwszString = pwsz;
703 return rc;
704 }
705 RTMemFree(pwsz);
706 }
707 else
708 rc = VERR_NO_UTF16_MEMORY;
709 }
710 return rc;
711}
712RT_EXPORT_SYMBOL(RTStrToUtf16Tag);
713
714
715RTDECL(int) RTStrToUtf16ExTag(const char *pszString, size_t cchString,
716 PRTUTF16 *ppwsz, size_t cwc, size_t *pcwc, const char *pszTag)
717{
718 /*
719 * Validate input.
720 */
721 Assert(VALID_PTR(pszString));
722 Assert(VALID_PTR(ppwsz));
723 Assert(!pcwc || VALID_PTR(pcwc));
724
725 /*
726 * Validate the UTF-8 input and calculate the length of the UTF-16 string.
727 */
728 size_t cwcResult;
729 int rc = rtUtf8CalcUtf16Length(pszString, cchString, &cwcResult);
730 if (RT_SUCCESS(rc))
731 {
732 if (pcwc)
733 *pcwc = cwcResult;
734
735 /*
736 * Check buffer size / Allocate buffer.
737 */
738 bool fShouldFree;
739 PRTUTF16 pwszResult;
740 if (cwc > 0 && *ppwsz)
741 {
742 fShouldFree = false;
743 if (cwc <= cwcResult)
744 return VERR_BUFFER_OVERFLOW;
745 pwszResult = *ppwsz;
746 }
747 else
748 {
749 *ppwsz = NULL;
750 fShouldFree = true;
751 cwc = RT_MAX(cwcResult + 1, cwc);
752 pwszResult = (PRTUTF16)RTMemAllocTag(cwc * sizeof(RTUTF16), pszTag);
753 }
754 if (pwszResult)
755 {
756 /*
757 * Encode the UTF-16 string.
758 */
759 rc = rtUtf8RecodeAsUtf16(pszString, cchString, pwszResult, cwc - 1);
760 if (RT_SUCCESS(rc))
761 {
762 *ppwsz = pwszResult;
763 return rc;
764 }
765 if (fShouldFree)
766 RTMemFree(pwszResult);
767 }
768 else
769 rc = VERR_NO_UTF16_MEMORY;
770 }
771 return rc;
772}
773RT_EXPORT_SYMBOL(RTStrToUtf16ExTag);
774
775
776RTDECL(size_t) RTStrCalcUtf16Len(const char *psz)
777{
778 size_t cwc;
779 int rc = rtUtf8CalcUtf16Length(psz, RTSTR_MAX, &cwc);
780 return RT_SUCCESS(rc) ? cwc : 0;
781}
782RT_EXPORT_SYMBOL(RTStrCalcUtf16Len);
783
784
785RTDECL(int) RTStrCalcUtf16LenEx(const char *psz, size_t cch, size_t *pcwc)
786{
787 size_t cwc;
788 int rc = rtUtf8CalcUtf16Length(psz, cch, &cwc);
789 if (pcwc)
790 *pcwc = RT_SUCCESS(rc) ? cwc : ~(size_t)0;
791 return rc;
792}
793RT_EXPORT_SYMBOL(RTStrCalcUtf16LenEx);
794
795
796/**
797 * Calculates the length of the UTF-8 encoding of a Latin-1 string.
798 *
799 * @returns iprt status code.
800 * @param psz The Latin-1 string.
801 * @param cchIn The max length of the Latin-1 string to consider.
802 * @param pcch Where to store the length (excluding '\\0') of the UTF-8 string. (cch == cb, btw)
803 */
804static int rtLatin1CalcUtf8Length(const char *psz, size_t cchIn, size_t *pcch)
805{
806 size_t cch = 0;
807 while (cchIn > 0)
808 {
809 char ch = *psz++; cchIn--;
810 if (!ch)
811 break;
812 if (!(ch & 0x80))
813 cch++;
814 else
815 cch += 2;
816 }
817
818
819 /* done */
820 *pcch = cch;
821 return VINF_SUCCESS;
822}
823
824
825/**
826 * Recodes a Latin-1 string as UTF-8.
827 *
828 * @returns iprt status code.
829 * @param psz The Latin-1 string.
830 * @param cchIn The number of characters to process from psz. The recoding
831 * will stop when cch or '\\0' is reached.
832 * @param psz Where to store the UTF-8 string.
833 * @param cch The size of the UTF-8 buffer, excluding the terminator.
834 * @param pcch Where to store the number of octets actually encoded.
835 */
836static int rtLatin1RecodeAsUtf8(const char *pszIn, size_t cchIn, char *psz, size_t cch, size_t *pcch)
837{
838 unsigned char *puch = (unsigned char *)psz;
839 int rc = VINF_SUCCESS;
840 while (cchIn > 0)
841 {
842 char ch = *pszIn++; cchIn--;
843 if (!ch)
844 break;
845 if (!(ch & 0x80))
846 {
847 if (RT_UNLIKELY(cch < 1))
848 {
849 RTStrAssertMsgFailed(("Buffer overflow! 1\n"));
850 rc = VERR_BUFFER_OVERFLOW;
851 break;
852 }
853 cch--;
854 *puch++ = (unsigned char)ch;
855 }
856 else
857 {
858 if (RT_UNLIKELY(cch < 2))
859 {
860 RTStrAssertMsgFailed(("Buffer overflow! 2\n"));
861 rc = VERR_BUFFER_OVERFLOW;
862 break;
863 }
864 cch -= 2;
865 *puch++ = 0xc0 | (ch >> 6);
866 *puch++ = 0x80 | (ch & 0x3f);
867 }
868 }
869
870 /* done */
871 *puch = '\0';
872 *pcch = (char *)puch - psz;
873 return rc;
874}
875
876
877
878RTDECL(int) RTLatin1ToUtf8Tag(const char *pszString, char **ppszString, const char *pszTag)
879{
880 /*
881 * Validate input.
882 */
883 Assert(VALID_PTR(ppszString));
884 Assert(VALID_PTR(pszString));
885 *ppszString = NULL;
886
887 /*
888 * Calculate the length of the UTF-8 encoding of the Latin-1 string.
889 */
890 size_t cch;
891 int rc = rtLatin1CalcUtf8Length(pszString, RTSTR_MAX, &cch);
892 if (RT_SUCCESS(rc))
893 {
894 /*
895 * Allocate buffer and recode it.
896 */
897 char *pszResult = (char *)RTMemAllocTag(cch + 1, pszTag);
898 if (pszResult)
899 {
900 rc = rtLatin1RecodeAsUtf8(pszString, RTSTR_MAX, pszResult, cch, &cch);
901 if (RT_SUCCESS(rc))
902 {
903 *ppszString = pszResult;
904 return rc;
905 }
906
907 RTMemFree(pszResult);
908 }
909 else
910 rc = VERR_NO_STR_MEMORY;
911 }
912 return rc;
913}
914RT_EXPORT_SYMBOL(RTLatin1ToUtf8Tag);
915
916
917RTDECL(int) RTLatin1ToUtf8ExTag(const char *pszString, size_t cchString, char **ppsz, size_t cch, size_t *pcch, const char *pszTag)
918{
919 /*
920 * Validate input.
921 */
922 Assert(VALID_PTR(pszString));
923 Assert(VALID_PTR(ppsz));
924 Assert(!pcch || VALID_PTR(pcch));
925
926 /*
927 * Calculate the length of the UTF-8 encoding of the Latin-1 string.
928 */
929 size_t cchResult;
930 int rc = rtLatin1CalcUtf8Length(pszString, cchString, &cchResult);
931 if (RT_SUCCESS(rc))
932 {
933 if (pcch)
934 *pcch = cchResult;
935
936 /*
937 * Check buffer size / Allocate buffer and recode it.
938 */
939 bool fShouldFree;
940 char *pszResult;
941 if (cch > 0 && *ppsz)
942 {
943 fShouldFree = false;
944 if (RT_UNLIKELY(cch <= cchResult))
945 return VERR_BUFFER_OVERFLOW;
946 pszResult = *ppsz;
947 }
948 else
949 {
950 *ppsz = NULL;
951 fShouldFree = true;
952 cch = RT_MAX(cch, cchResult + 1);
953 pszResult = (char *)RTStrAllocTag(cch, pszTag);
954 }
955 if (pszResult)
956 {
957 rc = rtLatin1RecodeAsUtf8(pszString, cchString, pszResult, cch - 1, &cch);
958 if (RT_SUCCESS(rc))
959 {
960 *ppsz = pszResult;
961 return rc;
962 }
963
964 if (fShouldFree)
965 RTStrFree(pszResult);
966 }
967 else
968 rc = VERR_NO_STR_MEMORY;
969 }
970 return rc;
971}
972RT_EXPORT_SYMBOL(RTLatin1ToUtf8ExTag);
973
974
975RTDECL(size_t) RTLatin1CalcUtf8Len(const char *psz)
976{
977 size_t cch;
978 int rc = rtLatin1CalcUtf8Length(psz, RTSTR_MAX, &cch);
979 return RT_SUCCESS(rc) ? cch : 0;
980}
981RT_EXPORT_SYMBOL(RTLatin1CalcUtf8Len);
982
983
984RTDECL(int) RTLatin1CalcUtf8LenEx(const char *psz, size_t cchIn, size_t *pcch)
985{
986 size_t cch;
987 int rc = rtLatin1CalcUtf8Length(psz, cchIn, &cch);
988 if (pcch)
989 *pcch = RT_SUCCESS(rc) ? cch : ~(size_t)0;
990 return rc;
991}
992RT_EXPORT_SYMBOL(RTLatin1CalcUtf8LenEx);
993
994
995/**
996 * Calculates the Latin-1 length of a string, validating the encoding while doing so.
997 *
998 * @returns IPRT status code.
999 * @param psz Pointer to the UTF-8 string.
1000 * @param cch The max length of the string. (btw cch = cb)
1001 * Use RTSTR_MAX if all of the string is to be examined.
1002 * @param pcwc Where to store the length of the Latin-1 string in bytes.
1003 */
1004static int rtUtf8CalcLatin1Length(const char *psz, size_t cch, size_t *pcch)
1005{
1006 /* We re-encode to one byte per unicode code point. */
1007 return RTStrUniLenEx(psz, cch, pcch);
1008}
1009
1010
1011/**
1012 * Recodes a valid UTF-8 string as Latin-1.
1013 *
1014 * Since we know the input is valid, we do *not* perform encoding or length checks.
1015 *
1016 * @returns iprt status code.
1017 * @param psz The UTF-8 string to recode. This is a valid encoding.
1018 * @param cch The number of chars (the type char, so bytes if you like) to process of the UTF-8 string.
1019 * The recoding will stop when cch or '\\0' is reached. Pass RTSTR_MAX to process up to '\\0'.
1020 * @param pszOut Where to store the Latin-1 string.
1021 * @param cchOut The number of characters the pszOut buffer can hold, excluding the terminator ('\\0').
1022 */
1023static int rtUtf8RecodeAsLatin1(const char *psz, size_t cch, char *pszOut, size_t cchOut)
1024{
1025 int rc = VINF_SUCCESS;
1026 const unsigned char *puch = (const unsigned char *)psz;
1027 unsigned char *puchOut = (unsigned char *)pszOut;
1028 while (cch > 0)
1029 {
1030 /* read the next char and check for terminator. */
1031 const unsigned char uch = *puch;
1032 if (!uch)
1033 break;
1034
1035 /* check for output overflow */
1036 if (RT_UNLIKELY(cchOut < 1))
1037 {
1038 rc = VERR_BUFFER_OVERFLOW;
1039 break;
1040 }
1041 cchOut--;
1042
1043 /* decode and recode the code point */
1044 if (!(uch & RT_BIT(7)))
1045 {
1046 *puchOut++ = uch;
1047 puch++;
1048 cch--;
1049 }
1050 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5))) == (RT_BIT(7) | RT_BIT(6)))
1051 {
1052 uint16_t uc = (puch[1] & 0x3f)
1053 | ((uint16_t)(uch & 0x1f) << 6);
1054 *puchOut++ = uc < 0x100 ? uc : '?';
1055 puch += 2;
1056 cch -= 2;
1057 }
1058 else if ((uch & (RT_BIT(7) | RT_BIT(6) | RT_BIT(5) | RT_BIT(4))) == (RT_BIT(7) | RT_BIT(6) | RT_BIT(5)))
1059 {
1060 *puchOut++ = '?';
1061 puch += 3;
1062 cch -= 3;
1063 }
1064 else
1065 {
1066 *puchOut++ = '?';
1067 puch += 4;
1068 cch -= 4;
1069 }
1070 }
1071
1072 /* done */
1073 *puchOut = '\0';
1074 return rc;
1075}
1076
1077
1078RTDECL(int) RTStrToLatin1Tag(const char *pszString, char **ppszString, const char *pszTag)
1079{
1080 /*
1081 * Validate input.
1082 */
1083 Assert(VALID_PTR(ppszString));
1084 Assert(VALID_PTR(pszString));
1085 *ppszString = NULL;
1086
1087 /*
1088 * Validate the UTF-8 input and calculate the length of the Latin-1 string.
1089 */
1090 size_t cch;
1091 int rc = rtUtf8CalcLatin1Length(pszString, RTSTR_MAX, &cch);
1092 if (RT_SUCCESS(rc))
1093 {
1094 /*
1095 * Allocate buffer.
1096 */
1097 char *psz = (char *)RTMemAllocTag(cch + 1, pszTag);
1098 if (psz)
1099 {
1100 /*
1101 * Encode the UTF-16 string.
1102 */
1103 rc = rtUtf8RecodeAsLatin1(pszString, RTSTR_MAX, psz, cch);
1104 if (RT_SUCCESS(rc))
1105 {
1106 *ppszString = psz;
1107 return rc;
1108 }
1109 RTMemFree(psz);
1110 }
1111 else
1112 rc = VERR_NO_STR_MEMORY;
1113 }
1114 return rc;
1115}
1116RT_EXPORT_SYMBOL(RTStrToLatin1Tag);
1117
1118
1119RTDECL(int) RTStrToLatin1ExTag(const char *pszString, size_t cchString,
1120 char **ppsz, size_t cch, size_t *pcch, const char *pszTag)
1121{
1122 /*
1123 * Validate input.
1124 */
1125 Assert(VALID_PTR(pszString));
1126 Assert(VALID_PTR(ppsz));
1127 Assert(!pcch || VALID_PTR(pcch));
1128
1129 /*
1130 * Validate the UTF-8 input and calculate the length of the UTF-16 string.
1131 */
1132 size_t cchResult;
1133 int rc = rtUtf8CalcLatin1Length(pszString, cchString, &cchResult);
1134 if (RT_SUCCESS(rc))
1135 {
1136 if (pcch)
1137 *pcch = cchResult;
1138
1139 /*
1140 * Check buffer size / Allocate buffer.
1141 */
1142 bool fShouldFree;
1143 char *pszResult;
1144 if (cch > 0 && *ppsz)
1145 {
1146 fShouldFree = false;
1147 if (cch <= cchResult)
1148 return VERR_BUFFER_OVERFLOW;
1149 pszResult = *ppsz;
1150 }
1151 else
1152 {
1153 *ppsz = NULL;
1154 fShouldFree = true;
1155 cch = RT_MAX(cchResult + 1, cch);
1156 pszResult = (char *)RTMemAllocTag(cch, pszTag);
1157 }
1158 if (pszResult)
1159 {
1160 /*
1161 * Encode the Latin-1 string.
1162 */
1163 rc = rtUtf8RecodeAsLatin1(pszString, cchString, pszResult, cch - 1);
1164 if (RT_SUCCESS(rc))
1165 {
1166 *ppsz = pszResult;
1167 return rc;
1168 }
1169 if (fShouldFree)
1170 RTMemFree(pszResult);
1171 }
1172 else
1173 rc = VERR_NO_STR_MEMORY;
1174 }
1175 return rc;
1176}
1177RT_EXPORT_SYMBOL(RTStrToLatin1Tag);
1178
1179
1180/**
1181 * Handle invalid encodings passed to RTStrGetCp() and RTStrGetCpEx().
1182 * @returns rc
1183 * @param ppsz The pointer to the string position point.
1184 * @param pCp Where to store RTUNICP_INVALID.
1185 * @param rc The iprt error code.
1186 */
1187static int rtStrGetCpExFailure(const char **ppsz, PRTUNICP pCp, int rc)
1188{
1189 /*
1190 * Try find a valid encoding.
1191 */
1192 (*ppsz)++; /** @todo code this! */
1193 *pCp = RTUNICP_INVALID;
1194 return rc;
1195}
1196
1197
1198RTDECL(RTUNICP) RTStrGetCpInternal(const char *psz)
1199{
1200 RTUNICP Cp;
1201 RTStrGetCpExInternal(&psz, &Cp);
1202 return Cp;
1203}
1204RT_EXPORT_SYMBOL(RTStrGetCpInternal);
1205
1206
1207RTDECL(int) RTStrGetCpExInternal(const char **ppsz, PRTUNICP pCp)
1208{
1209 const unsigned char *puch = (const unsigned char *)*ppsz;
1210 const unsigned char uch = *puch;
1211 RTUNICP uc;
1212
1213 /* ASCII ? */
1214 if (!(uch & RT_BIT(7)))
1215 {
1216 uc = uch;
1217 puch++;
1218 }
1219 else if (uch & RT_BIT(6))
1220 {
1221 /* figure the length and validate the first octet. */
1222 unsigned cb;
1223 if (!(uch & RT_BIT(5)))
1224 cb = 2;
1225 else if (!(uch & RT_BIT(4)))
1226 cb = 3;
1227 else if (!(uch & RT_BIT(3)))
1228 cb = 4;
1229 else if (!(uch & RT_BIT(2)))
1230 cb = 5;
1231 else if (!(uch & RT_BIT(1)))
1232 cb = 6;
1233 else
1234 {
1235 RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.*Rhxs\n", RT_MIN(strlen((char *)puch), 10), puch));
1236 return rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING);
1237 }
1238
1239 /* validate the rest */
1240 switch (cb)
1241 {
1242 case 6:
1243 RTStrAssertMsgReturn((puch[5] & 0xc0) == 0x80, ("6/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1244 rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
1245 case 5:
1246 RTStrAssertMsgReturn((puch[4] & 0xc0) == 0x80, ("5/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1247 rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
1248 case 4:
1249 RTStrAssertMsgReturn((puch[3] & 0xc0) == 0x80, ("4/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1250 rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
1251 case 3:
1252 RTStrAssertMsgReturn((puch[2] & 0xc0) == 0x80, ("3/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1253 rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
1254 case 2:
1255 RTStrAssertMsgReturn((puch[1] & 0xc0) == 0x80, ("2/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1256 rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
1257 break;
1258 }
1259
1260 /* get and validate the code point. */
1261 switch (cb)
1262 {
1263 case 6:
1264 uc = (puch[5] & 0x3f)
1265 | ((RTUNICP)(puch[4] & 0x3f) << 6)
1266 | ((RTUNICP)(puch[3] & 0x3f) << 12)
1267 | ((RTUNICP)(puch[2] & 0x3f) << 18)
1268 | ((RTUNICP)(puch[1] & 0x3f) << 24)
1269 | ((RTUNICP)(uch & 0x01) << 30);
1270 RTStrAssertMsgReturn(uc >= 0x04000000 && uc <= 0x7fffffff,
1271 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1272 rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
1273 break;
1274 case 5:
1275 uc = (puch[4] & 0x3f)
1276 | ((RTUNICP)(puch[3] & 0x3f) << 6)
1277 | ((RTUNICP)(puch[2] & 0x3f) << 12)
1278 | ((RTUNICP)(puch[1] & 0x3f) << 18)
1279 | ((RTUNICP)(uch & 0x03) << 24);
1280 RTStrAssertMsgReturn(uc >= 0x00200000 && uc <= 0x03ffffff,
1281 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1282 rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
1283 break;
1284 case 4:
1285 uc = (puch[3] & 0x3f)
1286 | ((RTUNICP)(puch[2] & 0x3f) << 6)
1287 | ((RTUNICP)(puch[1] & 0x3f) << 12)
1288 | ((RTUNICP)(uch & 0x07) << 18);
1289 RTStrAssertMsgReturn(uc >= 0x00010000 && uc <= 0x001fffff,
1290 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1291 rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
1292 break;
1293 case 3:
1294 uc = (puch[2] & 0x3f)
1295 | ((RTUNICP)(puch[1] & 0x3f) << 6)
1296 | ((RTUNICP)(uch & 0x0f) << 12);
1297 RTStrAssertMsgReturn(uc >= 0x00000800 && uc <= 0x0000fffd,
1298 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1299 rtStrGetCpExFailure(ppsz, pCp, uc == 0xffff || uc == 0xfffe ? VERR_CODE_POINT_ENDIAN_INDICATOR : VERR_INVALID_UTF8_ENCODING));
1300 RTStrAssertMsgReturn(uc < 0xd800 || uc > 0xdfff,
1301 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1302 rtStrGetCpExFailure(ppsz, pCp, VERR_CODE_POINT_SURROGATE));
1303 break;
1304 case 2:
1305 uc = (puch[1] & 0x3f)
1306 | ((RTUNICP)(uch & 0x1f) << 6);
1307 RTStrAssertMsgReturn(uc >= 0x00000080 && uc <= 0x000007ff,
1308 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1309 rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING));
1310 break;
1311 default: /* impossible, but GCC is bitching. */
1312 uc = RTUNICP_INVALID;
1313 break;
1314 }
1315 puch += cb;
1316 }
1317 else
1318 {
1319 /* 6th bit is always set. */
1320 RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.*Rhxs\n", RT_MIN(strlen((char *)puch), 10), puch));
1321 return rtStrGetCpExFailure(ppsz, pCp, VERR_INVALID_UTF8_ENCODING);
1322 }
1323 *pCp = uc;
1324 *ppsz = (const char *)puch;
1325 return VINF_SUCCESS;
1326}
1327RT_EXPORT_SYMBOL(RTStrGetCpExInternal);
1328
1329
1330/**
1331 * Handle invalid encodings passed to RTStrGetCpNEx().
1332 * @returns rc
1333 * @param ppsz The pointer to the string position point.
1334 * @param pcch Pointer to the string length.
1335 * @param pCp Where to store RTUNICP_INVALID.
1336 * @param rc The iprt error code.
1337 */
1338static int rtStrGetCpNExFailure(const char **ppsz, size_t *pcch, PRTUNICP pCp, int rc)
1339{
1340 /*
1341 * Try find a valid encoding.
1342 */
1343 (*ppsz)++; /** @todo code this! */
1344 (*pcch)--;
1345 *pCp = RTUNICP_INVALID;
1346 return rc;
1347}
1348
1349
1350RTDECL(int) RTStrGetCpNExInternal(const char **ppsz, size_t *pcch, PRTUNICP pCp)
1351{
1352 const unsigned char *puch = (const unsigned char *)*ppsz;
1353 const unsigned char uch = *puch;
1354 size_t cch = *pcch;
1355 RTUNICP uc;
1356
1357 if (cch == 0)
1358 {
1359 *pCp = RTUNICP_INVALID;
1360 return VERR_END_OF_STRING;
1361 }
1362
1363 /* ASCII ? */
1364 if (!(uch & RT_BIT(7)))
1365 {
1366 uc = uch;
1367 puch++;
1368 cch--;
1369 }
1370 else if (uch & RT_BIT(6))
1371 {
1372 /* figure the length and validate the first octet. */
1373 unsigned cb;
1374 if (!(uch & RT_BIT(5)))
1375 cb = 2;
1376 else if (!(uch & RT_BIT(4)))
1377 cb = 3;
1378 else if (!(uch & RT_BIT(3)))
1379 cb = 4;
1380 else if (!(uch & RT_BIT(2)))
1381 cb = 5;
1382 else if (!(uch & RT_BIT(1)))
1383 cb = 6;
1384 else
1385 {
1386 RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.*Rhxs\n", RT_MIN(strlen((char *)puch), 10), puch));
1387 return rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING);
1388 }
1389
1390 if (cb > cch)
1391 return rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING);
1392
1393 /* validate the rest */
1394 switch (cb)
1395 {
1396 case 6:
1397 RTStrAssertMsgReturn((puch[5] & 0xc0) == 0x80, ("6/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1398 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1399 case 5:
1400 RTStrAssertMsgReturn((puch[4] & 0xc0) == 0x80, ("5/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1401 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1402 case 4:
1403 RTStrAssertMsgReturn((puch[3] & 0xc0) == 0x80, ("4/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1404 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1405 case 3:
1406 RTStrAssertMsgReturn((puch[2] & 0xc0) == 0x80, ("3/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1407 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1408 case 2:
1409 RTStrAssertMsgReturn((puch[1] & 0xc0) == 0x80, ("2/%u: %.*Rhxs\n", cb, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1410 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1411 break;
1412 }
1413
1414 /* get and validate the code point. */
1415 switch (cb)
1416 {
1417 case 6:
1418 uc = (puch[5] & 0x3f)
1419 | ((RTUNICP)(puch[4] & 0x3f) << 6)
1420 | ((RTUNICP)(puch[3] & 0x3f) << 12)
1421 | ((RTUNICP)(puch[2] & 0x3f) << 18)
1422 | ((RTUNICP)(puch[1] & 0x3f) << 24)
1423 | ((RTUNICP)(uch & 0x01) << 30);
1424 RTStrAssertMsgReturn(uc >= 0x04000000 && uc <= 0x7fffffff,
1425 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1426 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1427 break;
1428 case 5:
1429 uc = (puch[4] & 0x3f)
1430 | ((RTUNICP)(puch[3] & 0x3f) << 6)
1431 | ((RTUNICP)(puch[2] & 0x3f) << 12)
1432 | ((RTUNICP)(puch[1] & 0x3f) << 18)
1433 | ((RTUNICP)(uch & 0x03) << 24);
1434 RTStrAssertMsgReturn(uc >= 0x00200000 && uc <= 0x03ffffff,
1435 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1436 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1437 break;
1438 case 4:
1439 uc = (puch[3] & 0x3f)
1440 | ((RTUNICP)(puch[2] & 0x3f) << 6)
1441 | ((RTUNICP)(puch[1] & 0x3f) << 12)
1442 | ((RTUNICP)(uch & 0x07) << 18);
1443 RTStrAssertMsgReturn(uc >= 0x00010000 && uc <= 0x001fffff,
1444 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1445 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1446 break;
1447 case 3:
1448 uc = (puch[2] & 0x3f)
1449 | ((RTUNICP)(puch[1] & 0x3f) << 6)
1450 | ((RTUNICP)(uch & 0x0f) << 12);
1451 RTStrAssertMsgReturn(uc >= 0x00000800 && uc <= 0x0000fffd,
1452 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1453 rtStrGetCpNExFailure(ppsz, pcch, pCp, uc == 0xffff || uc == 0xfffe ? VERR_CODE_POINT_ENDIAN_INDICATOR : VERR_INVALID_UTF8_ENCODING));
1454 RTStrAssertMsgReturn(uc < 0xd800 || uc > 0xdfff,
1455 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1456 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_CODE_POINT_SURROGATE));
1457 break;
1458 case 2:
1459 uc = (puch[1] & 0x3f)
1460 | ((RTUNICP)(uch & 0x1f) << 6);
1461 RTStrAssertMsgReturn(uc >= 0x00000080 && uc <= 0x000007ff,
1462 ("%u: cp=%#010RX32: %.*Rhxs\n", cb, uc, RT_MIN(cb + 10, strlen((char *)puch)), puch),
1463 rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING));
1464 break;
1465 default: /* impossible, but GCC is bitching. */
1466 uc = RTUNICP_INVALID;
1467 break;
1468 }
1469 puch += cb;
1470 cch -= cb;
1471 }
1472 else
1473 {
1474 /* 6th bit is always set. */
1475 RTStrAssertMsgFailed(("Invalid UTF-8 first byte: %.*Rhxs\n", RT_MIN(strlen((char *)puch), 10), puch));
1476 return rtStrGetCpNExFailure(ppsz, pcch, pCp, VERR_INVALID_UTF8_ENCODING);
1477 }
1478 *pCp = uc;
1479 *ppsz = (const char *)puch;
1480 (*pcch) = cch;
1481 return VINF_SUCCESS;
1482}
1483RT_EXPORT_SYMBOL(RTStrGetCpNExInternal);
1484
1485
1486RTDECL(char *) RTStrPutCpInternal(char *psz, RTUNICP uc)
1487{
1488 unsigned char *puch = (unsigned char *)psz;
1489 if (uc < 0x80)
1490 *puch++ = (unsigned char )uc;
1491 else if (uc < 0x00000800)
1492 {
1493 *puch++ = 0xc0 | (uc >> 6);
1494 *puch++ = 0x80 | (uc & 0x3f);
1495 }
1496 else if (uc < 0x00010000)
1497 {
1498 if ( uc < 0x0000d8000
1499 || ( uc > 0x0000dfff
1500 && uc < 0x0000fffe))
1501 {
1502 *puch++ = 0xe0 | (uc >> 12);
1503 *puch++ = 0x80 | ((uc >> 6) & 0x3f);
1504 *puch++ = 0x80 | (uc & 0x3f);
1505 }
1506 else
1507 {
1508 AssertMsgFailed(("Invalid code point U+%05x!\n", uc));
1509 *puch++ = 0x7f;
1510 }
1511 }
1512 else if (uc < 0x00200000)
1513 {
1514 *puch++ = 0xf0 | (uc >> 18);
1515 *puch++ = 0x80 | ((uc >> 12) & 0x3f);
1516 *puch++ = 0x80 | ((uc >> 6) & 0x3f);
1517 *puch++ = 0x80 | (uc & 0x3f);
1518 }
1519 else if (uc < 0x04000000)
1520 {
1521 *puch++ = 0xf8 | (uc >> 24);
1522 *puch++ = 0x80 | ((uc >> 18) & 0x3f);
1523 *puch++ = 0x80 | ((uc >> 12) & 0x3f);
1524 *puch++ = 0x80 | ((uc >> 6) & 0x3f);
1525 *puch++ = 0x80 | (uc & 0x3f);
1526 }
1527 else if (uc <= 0x7fffffff)
1528 {
1529 *puch++ = 0xfc | (uc >> 30);
1530 *puch++ = 0x80 | ((uc >> 24) & 0x3f);
1531 *puch++ = 0x80 | ((uc >> 18) & 0x3f);
1532 *puch++ = 0x80 | ((uc >> 12) & 0x3f);
1533 *puch++ = 0x80 | ((uc >> 6) & 0x3f);
1534 *puch++ = 0x80 | (uc & 0x3f);
1535 }
1536 else
1537 {
1538 AssertMsgFailed(("Invalid code point U+%08x!\n", uc));
1539 *puch++ = 0x7f;
1540 }
1541
1542 return (char *)puch;
1543}
1544RT_EXPORT_SYMBOL(RTStrPutCpInternal);
1545
1546
1547RTDECL(char *) RTStrPrevCp(const char *pszStart, const char *psz)
1548{
1549 if (pszStart < psz)
1550 {
1551 /* simple char? */
1552 const unsigned char *puch = (const unsigned char *)psz;
1553 unsigned uch = *--puch;
1554 if (!(uch & RT_BIT(7)))
1555 return (char *)puch;
1556 RTStrAssertMsgReturn(!(uch & RT_BIT(6)), ("uch=%#x\n", uch), (char *)pszStart);
1557
1558 /* two or more. */
1559 uint32_t uMask = 0xffffffc0;
1560 while ( (const unsigned char *)pszStart < puch
1561 && !(uMask & 1))
1562 {
1563 uch = *--puch;
1564 if ((uch & 0xc0) != 0x80)
1565 {
1566 RTStrAssertMsgReturn((uch & (uMask >> 1)) == (uMask & 0xff),
1567 ("Invalid UTF-8 encoding: %.*Rhxs puch=%p psz=%p\n", psz - (char *)puch, puch, psz),
1568 (char *)pszStart);
1569 return (char *)puch;
1570 }
1571 uMask >>= 1;
1572 }
1573 RTStrAssertMsgFailed(("Invalid UTF-8 encoding: %.*Rhxs puch=%p psz=%p\n", psz - (char *)puch, puch, psz));
1574 }
1575 return (char *)pszStart;
1576}
1577RT_EXPORT_SYMBOL(RTStrPrevCp);
1578
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette