CW Cyrix's Git Repository Mirror
Browse code

Lots of changes in regards to strings, as follows:

* Removed my custom String class, as well as removed the really old
Unicode ConvertUTF, the UTF Converter, and the UTF Encode/Decode
functions.
* Replaced the above with using standard C++ std::wstring_convert and
std::codecvt_utf8.
* Added headers adapted from llvm's libc++ project for allowing the
above C++ classes to exist with GCC and Clang when libc++ isn't being
used.
* Used std::string over std::wstring whenever possible.
* Added header adapted from llvm's libc++ project to create special
versions of the ifstream and ofstream classes that would utilize _wfopen
on non-MSVC Windows compilers, so those compilers could access files on
Windows that use characters outside the current codepage.
* Removed the -static-libgcc and -static-libstdc++ flags from the
Makefile for non-MSVC builds. The generated DLLs will require extra DLLs
from either Cygwin or MinGW (whichever is used to compile), I've only
tested with MinGW and for some reason they crash Winamp on exit, I have
no idea why.

Naram Qashat authored on 2014/09/24 13:58:55
Showing 1 changed files
src/in_xsf_framework/ConvertUTF.cpp
History View file @ fe3d6ae
1 1 
deleted file mode 100644
... ... 
@@ -1,602 +0,0 @@
1 
-/*
2 
- * Copyright 2001-2004 Unicode, Inc.
3 
- *
4 
- * Disclaimer
5 
- *
6 
- * This source code is provided as is by Unicode, Inc. No claims are
7 
- * made as to fitness for any particular purpose. No warranties of any
8 
- * kind are expressed or implied. The recipient agrees to determine
9 
- * applicability of information provided. If this file has been
10 
- * purchased on magnetic or optical media from Unicode, Inc., the
11 
- * sole remedy for any claim will be exchange of defective media
12 
- * within 90 days of receipt.
13 
- *
14 
- * Limitations on Rights to Redistribute This Code
15 
- *
16 
- * Unicode, Inc. hereby grants the right to freely use the information
17 
- * supplied in this file in the creation of products supporting the
18 
- * Unicode Standard, and to make copies of this file in any form
19 
- * for internal or external distribution as long as this notice
20 
- * remains attached.
21 
- */
22 
-
23 
-/* ---------------------------------------------------------------------
24 
-
25 
-    Conversions between UTF32, UTF-16, and UTF-8. Source code file.
26 
-    Author: Mark E. Davis, 1994.
27 
-    Rev History: Rick McGowan, fixes & updates May 2001.
28 
-    Sept 2001: fixed const & error conditions per
29 
-	mods suggested by S. Parent & A. Lillich.
30 
-    June 2002: Tim Dodd added detection and handling of incomplete
31 
-	source sequences, enhanced error detection, added casts
32 
-	to eliminate compiler warnings.
33 
-    July 2003: slight mods to back out aggressive FFFE detection.
34 
-    Jan 2004: updated switches in from-UTF8 conversions.
35 
-    Oct 2004: updated to use UNI_MAX_LEGAL_UTF32 in UTF-32 conversions.
36 
-
37 
-    See the header file "ConvertUTF.h" for complete documentation.
38 
-
39 
-
40 
-#include "ConvertUTF.h"
41 
-
42 
-static const int halfShift  = 10; /* used for shifting by 10 bits */
43 
-
44 
-static const UTF32 halfBase = 0x0010000UL;
45 
-static const UTF32 halfMask = 0x3FFUL;
46 
-
47 
-static const UTF32 UNI_SUR_HIGH_START = 0xD800;
48 
-static const UTF32 UNI_SUR_HIGH_END = 0xDBFF;
49 
-static const UTF32 UNI_SUR_LOW_START = 0xDC00;
50 
-static const UTF32 UNI_SUR_LOW_END = 0xDFFF;
51 
-
52 
-/* --------------------------------------------------------------------- */
53 
-
54 
-ConversionResult ConvertUTF32toUTF16(const UTF32 **sourceStart, const UTF32 *sourceEnd, UTF16 **targetStart, UTF16 *targetEnd, ConversionFlags flags)
55 
-{
56 
-	ConversionResult result = conversionOK;
57 
-	const UTF32 *source = *sourceStart;
58 
-	UTF16 *target = *targetStart;
59 
-	while (source < sourceEnd)
60 
-	{
61 
-		if (target >= targetEnd)
62 
-		{
63 
-			result = targetExhausted;
64 
-			break;
65 
-		}
66 
-		UTF32 ch = *source++;
67 
-		if (ch <= UNI_MAX_BMP) /* Target is a character <= 0xFFFF */
68 
-		{
69 
-			/* UTF-16 surrogate values are illegal in UTF-32; 0xffff or 0xfffe are both reserved values */
70 
-			if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END)
71 
-			{
72 
-				if (flags == strictConversion)
73 
-				{
74 
-					--source; /* return to the illegal value itself */
75 
-					result = sourceIllegal;
76 
-					break;
77 
-				}
78 
-				else
79 
-					*target++ = UNI_REPLACEMENT_CHAR;
80 
-			}
81 
-			else
82 
-				*target++ = static_cast<UTF16>(ch); /* normal case */
83 
-		}
84 
-		else if (ch > UNI_MAX_LEGAL_UTF32)
85 
-		{
86 
-			if (flags == strictConversion)
87 
-				result = sourceIllegal;
88 
-			else
89 
-				*target++ = UNI_REPLACEMENT_CHAR;
90 
-		}
91 
-		else
92 
-		{
93 
-			/* target is a character in range 0xFFFF - 0x10FFFF. */
94 
-			if (target + 1 >= targetEnd)
95 
-			{
96 
-				--source; /* Back up source pointer! */
97 
-				result = targetExhausted;
98 
-				break;
99 
-			}
100 
-			ch -= halfBase;
101 
-			*target++ = static_cast<UTF16>((ch >> halfShift) + UNI_SUR_HIGH_START);
102 
-			*target++ = static_cast<UTF16>((ch & halfMask) + UNI_SUR_LOW_START);
103 
-		}
104 
-	}
105 
-	*sourceStart = source;
106 
-	*targetStart = target;
107 
-	return result;
108 
-}
109 
-
110 
-/* --------------------------------------------------------------------- */
111 
-
112 
-ConversionResult ConvertUTF16toUTF32(const UTF16 **sourceStart, const UTF16 *sourceEnd, UTF32 **targetStart, UTF32 *targetEnd, ConversionFlags flags)
113 
-{
114 
-	ConversionResult result = conversionOK;
115 
-	const UTF16 *source = *sourceStart;
116 
-	UTF32 *target = *targetStart;
117 
-	while (source < sourceEnd)
118 
-	{
119 
-		const UTF16 *oldSource = source; /* In case we have to back up because of target overflow. */
120 
-		UTF32 ch = *source++;
121 
-		/* If we have a surrogate pair, convert to UTF32 first. */
122 
-		if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END)
123 
-		{
124 
-			/* If the 16 bits following the high surrogate are in the source buffer... */
125 
-			if (source < sourceEnd)
126 
-			{
127 
-				UTF32 ch2 = *source;
128 
-				/* If it's a low surrogate, convert to UTF32. */
129 
-				if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END)
130 
-				{
131 
-					ch = ((ch - UNI_SUR_HIGH_START) << halfShift) + (ch2 - UNI_SUR_LOW_START) + halfBase;
132 
-					++source;
133 
-				}
134 
-				else if (flags == strictConversion) /* it's an unpaired high surrogate */
135 
-				{
136 
-					--source; /* return to the illegal value itself */
137 
-					result = sourceIllegal;
138 
-					break;
139 
-				}
140 
-			}
141 
-			else /* We don't have the 16 bits following the high surrogate. */
142 
-			{
143 
-				--source; /* return to the high surrogate */
144 
-				result = sourceExhausted;
145 
-				break;
146 
-			}
147 
-		}
148 
-		else if (flags == strictConversion)
149 
-		{
150 
-			/* UTF-16 surrogate values are illegal in UTF-32 */
151 
-			if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END)
152 
-			{
153 
-				--source; /* return to the illegal value itself */
154 
-				result = sourceIllegal;
155 
-				break;
156 
-			}
157 
-		}
158 
-		if (target >= targetEnd)
159 
-		{
160 
-			source = oldSource; /* Back up source pointer! */
161 
-			result = targetExhausted;
162 
-			break;
163 
-		}
164 
-		*target++ = ch;
165 
-	}
166 
-	*sourceStart = source;
167 
-	*targetStart = target;
168 
-	return result;
169 
-}
170 
-
171 
-/* --------------------------------------------------------------------- */
172 
-
173 
-/*
174 
- * Index into the table below with the first byte of a UTF-8 sequence to
175 
- * get the number of trailing bytes that are supposed to follow it.
176 
- * Note that *legal* UTF-8 values can't have 4 or 5-bytes. The table is
177 
- * left as-is for anyone who may want to do such conversion, which was
178 
- * allowed in earlier algorithms.
179 
- */
180 
-static const char trailingBytesForUTF8[256] = {
181 
-	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
182 
-	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
183 
-	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
184 
-	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
185 
-	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
186 
-	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
187 
-	1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
188 
-	2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5
189 
-};
190 
-
191 
-/*
192 
- * Magic values subtracted from a buffer value during UTF8 conversion.
193 
- * This table contains as many values as there might be trailing bytes
194 
- * in a UTF-8 sequence.
195 
- */
196 
-static const UTF32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL, 0x03C82080UL, 0xFA082080UL, 0x82082080UL };
197 
-
198 
-/*
199 
- * Once the bits are split out into bytes of UTF-8, this is a mask OR-ed
200 
- * into the first byte, depending on how many bytes follow.  There are
201 
- * as many entries in this table as there are UTF-8 sequence types.
202 
- * (I.e., one byte sequence, two byte... etc.). Remember that sequencs
203 
- * for *legal* UTF-8 will be 4 or fewer bytes total.
204 
- */
205 
-static const UTF8 firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
206 
-
207 
-/* --------------------------------------------------------------------- */
208 
-
209 
-/* The interface converts a whole buffer to avoid function-call overhead.
210 
- * Constants have been gathered. Loops & conditionals have been removed as
211 
- * much as possible for efficiency, in favor of drop-through switches.
212 
- * (See "Note A" at the bottom of the file for equivalent code.)
213 
- * If your compiler supports it, the "isLegalUTF8" call can be turned
214 
- * into an inline function.
215 
- */
216 
-
217 
-/* --------------------------------------------------------------------- */
218 
-
219 
-ConversionResult ConvertUTF16toUTF8(const UTF16 **sourceStart, const UTF16 *sourceEnd, UTF8 **targetStart, UTF8 *targetEnd, ConversionFlags flags)
220 
-{
221 
-	ConversionResult result = conversionOK;
222 
-	const UTF16 *source = *sourceStart;
223 
-	UTF8 *target = *targetStart;
224 
-	while (source < sourceEnd)
225 
-	{
226 
-		const UTF32 byteMask = 0xBF, byteMark = 0x80;
227 
-		const UTF16 *oldSource = source; /* In case we have to back up because of target overflow. */
228 
-		UTF32 ch = *source++;
229 
-		/* If we have a surrogate pair, convert to UTF32 first. */
230 
-		if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END)
231 
-		{
232 
-			/* If the 16 bits following the high surrogate are in the source buffer... */
233 
-			if (source < sourceEnd)
234 
-			{
235 
-				UTF32 ch2 = *source;
236 
-				/* If it's a low surrogate, convert to UTF32. */
237 
-				if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END)
238 
-				{
239 
-					ch = ((ch - UNI_SUR_HIGH_START) << halfShift) + (ch2 - UNI_SUR_LOW_START) + halfBase;
240 
-					++source;
241 
-				}
242 
-				else if (flags == strictConversion) /* it's an unpaired high surrogate */
243 
-				{
244 
-					--source; /* return to the illegal value itself */
245 
-					result = sourceIllegal;
246 
-					break;
247 
-				}
248 
-			}
249 
-			else /* We don't have the 16 bits following the high surrogate. */
250 
-			{
251 
-				--source; /* return to the high surrogate */
252 
-				result = sourceExhausted;
253 
-				break;
254 
-			}
255 
-		}
256 
-		else if (flags == strictConversion)
257 
-		{
258 
-			/* UTF-16 surrogate values are illegal in UTF-32 */
259 
-			if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END)
260 
-			{
261 
-				--source; /* return to the illegal value itself */
262 
-				result = sourceIllegal;
263 
-				break;
264 
-			}
265 
-		}
266 
-		/* Figure out how many bytes the result will require */
267 
-		unsigned short bytesToWrite = 0;
268 
-		if (ch < 0x80)
269 
-			bytesToWrite = 1;
270 
-		else if (ch < 0x800)
271 
-			bytesToWrite = 2;
272 
-		else if (ch < 0x10000)
273 
-			bytesToWrite = 3;
274 
-		else if (ch < 0x110000)
275 
-			bytesToWrite = 4;
276 
-		else
277 
-		{
278 
-			bytesToWrite = 3;
279 
-			ch = UNI_REPLACEMENT_CHAR;
280 
-		}
281 
-		target += bytesToWrite;
282 
-		if (target > targetEnd)
283 
-		{
284 
-			source = oldSource; /* Back up source pointer! */
285 
-			target -= bytesToWrite;
286 
-			result = targetExhausted;
287 
-			break;
288 
-		}
289 
-		switch (bytesToWrite) /* note: everything falls through. */
290 
-		{
291 
-			case 4: *--target = static_cast<UTF8>((ch | byteMark) & byteMask); ch >>= 6;
292 
-			case 3: *--target = static_cast<UTF8>((ch | byteMark) & byteMask); ch >>= 6;
293 
-			case 2: *--target = static_cast<UTF8>((ch | byteMark) & byteMask); ch >>= 6;
294 
-			case 1: *--target = static_cast<UTF8>(ch | firstByteMark[bytesToWrite]);
295 
-		}
296 
-		target += bytesToWrite;
297 
-	}
298 
-	*sourceStart = source;
299 
-	*targetStart = target;
300 
-	return result;
301 
-}
302 
-
303 
-/* --------------------------------------------------------------------- */
304 
-
305 
-/*
306 
- * Utility routine to tell whether a sequence of bytes is legal UTF-8.
307 
- * This must be called with the length pre-determined by the first byte.
308 
- * If not calling this from ConvertUTF8to*, then the length can be set by:
309 
- *  length = trailingBytesForUTF8[*source]+1;
310 
- * and the sequence is illegal right away if there aren't that many bytes
311 
- * available.
312 
- * If presented with a length > 4, this returns false.  The Unicode
313 
- * definition of UTF-8 goes up to 4-byte sequences.
314 
- */
315 
-
316 
-static bool isLegalUTF8(const UTF8 *source, int length)
317 
-{
318 
-	const UTF8 *srcptr = source + length;
319 
-	UTF8 a;
320 
-	switch (length)
321 
-	{
322 
-		default: return false;
323 
-		/* Everything else falls through when "true"... */
324 
-		case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
325 
-		case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
326 
-		case 2: if ((a = (*--srcptr)) > 0xBF) return false;
327 
-
328 
-		switch (*source)
329 
-		{
330 
-			/* no fall-through in this inner switch */
331 
-			case 0xE0: if (a < 0xA0) return false; break;
332 
-			case 0xED: if (a > 0x9F) return false; break;
333 
-			case 0xF0: if (a < 0x90) return false; break;
334 
-			case 0xF4: if (a > 0x8F) return false; break;
335 
-			default: if (a < 0x80) return false;
336 
-		}
337 
-
338 
-		case 1: if (*source >= 0x80 && *source < 0xC2) return false;
339 
-	}
340 
-	if (*source > 0xF4)
341 
-		return false;
342 
-	return true;
343 
-}
344 
-
345 
-/* --------------------------------------------------------------------- */
346 
-
347 
-/*
348 
- * Exported function to return whether a UTF-8 sequence is legal or not.
349 
- * This is not used here; it's just exported.
350 
- */
351 
-bool isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd)
352 
-{
353 
-	int length = trailingBytesForUTF8[*source] + 1;
354 
-	if (source + length > sourceEnd)
355 
-		return false;
356 
-	return isLegalUTF8(source, length);
357 
-}
358 
-
359 
-/* --------------------------------------------------------------------- */
360 
-
361 
-ConversionResult ConvertUTF8toUTF16(const UTF8 **sourceStart, const UTF8 *sourceEnd, UTF16 **targetStart, UTF16 *targetEnd, ConversionFlags flags)
362 
-{
363 
-	ConversionResult result = conversionOK;
364 
-	const UTF8 *source = *sourceStart;
365 
-	UTF16 *target = *targetStart;
366 
-	while (source < sourceEnd)
367 
-	{
368 
-		UTF32 ch = 0;
369 
-		unsigned short extraBytesToRead = trailingBytesForUTF8[*source];
370 
-		if (source + extraBytesToRead >= sourceEnd)
371 
-		{
372 
-			result = sourceExhausted;
373 
-			break;
374 
-		}
375 
-		/* Do this check whether lenient or strict */
376 
-		if (!isLegalUTF8(source, extraBytesToRead + 1))
377 
-		{
378 
-			result = sourceIllegal;
379 
-			break;
380 
-		}
381 
-		/*
382 
-		 * The cases all fall through. See "Note A" below.
383 
-		 */
384 
-		switch (extraBytesToRead)
385 
-		{
386 
-			case 5: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */
387 
-			case 4: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */
388 
-			case 3: ch += *source++; ch <<= 6;
389 
-			case 2: ch += *source++; ch <<= 6;
390 
-			case 1: ch += *source++; ch <<= 6;
391 
-			case 0: ch += *source++;
392 
-		}
393 
-		ch -= offsetsFromUTF8[extraBytesToRead];
394 
-
395 
-		if (target >= targetEnd)
396 
-		{
397 
-			source -= extraBytesToRead + 1; /* Back up source pointer! */
398 
-			result = targetExhausted;
399 
-			break;
400 
-		}
401 
-		if (ch <= UNI_MAX_BMP) /* Target is a character <= 0xFFFF */
402 
-		{
403 
-			/* UTF-16 surrogate values are illegal in UTF-32 */
404 
-			if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END)
405 
-			{
406 
-				if (flags == strictConversion)
407 
-				{
408 
-					source -= extraBytesToRead + 1; /* return to the illegal value itself */
409 
-					result = sourceIllegal;
410 
-					break;
411 
-				}
412 
-				else
413 
-					*target++ = UNI_REPLACEMENT_CHAR;
414 
-			}
415 
-			else
416 
-				*target++ = static_cast<UTF16>(ch); /* normal case */
417 
-		}
418 
-		else if (ch > UNI_MAX_UTF16)
419 
-		{
420 
-			if (flags == strictConversion)
421 
-			{
422 
-				result = sourceIllegal;
423 
-				source -= extraBytesToRead + 1; /* return to the start */
424 
-				break; /* Bail out; shouldn't continue */
425 
-			}
426 
-			else
427 
-				*target++ = UNI_REPLACEMENT_CHAR;
428 
-		}
429 
-		else
430 
-		{
431 
-			/* target is a character in range 0xFFFF - 0x10FFFF. */
432 
-			if (target + 1 >= targetEnd)
433 
-			{
434 
-				source -= extraBytesToRead + 1; /* Back up source pointer! */
435 
-				result = targetExhausted;
436 
-				break;
437 
-			}
438 
-			ch -= halfBase;
439 
-			*target++ = static_cast<UTF16>((ch >> halfShift) + UNI_SUR_HIGH_START);
440 
-			*target++ = static_cast<UTF16>((ch & halfMask) + UNI_SUR_LOW_START);
441 
-		}
442 
-	}
443 
-	*sourceStart = source;
444 
-	*targetStart = target;
445 
-	return result;
446 
-}
447 
-
448 
-/* --------------------------------------------------------------------- */
449 
-
450 
-ConversionResult ConvertUTF32toUTF8(const UTF32 **sourceStart, const UTF32 *sourceEnd, UTF8 **targetStart, UTF8 *targetEnd, ConversionFlags flags)
451 
-{
452 
-	ConversionResult result = conversionOK;
453 
-	const UTF32 *source = *sourceStart;
454 
-	UTF8 *target = *targetStart;
455 
-	while (source < sourceEnd)
456 
-	{
457 
-		const UTF32 byteMask = 0xBF, byteMark = 0x80;
458 
-		UTF32 ch = *source++;
459 
-		if (flags == strictConversion)
460 
-		{
461 
-			/* UTF-16 surrogate values are illegal in UTF-32 */
462 
-			if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END)
463 
-			{
464 
-				--source; /* return to the illegal value itself */
465 
-				result = sourceIllegal;
466 
-				break;
467 
-			}
468 
-		}
469 
-		/*
470 
-		 * Figure out how many bytes the result will require. Turn any
471 
-		 * illegally large UTF32 things (> Plane 17) into replacement chars.
472 
-		 */
473 
-		unsigned short bytesToWrite = 0;
474 
-		if (ch < 0x80)
475 
-			bytesToWrite = 1;
476 
-		else if (ch < 0x800)
477 
-			bytesToWrite = 2;
478 
-		else if (ch < 0x10000)
479 
-			bytesToWrite = 3;
480 
-		else if (ch <= UNI_MAX_LEGAL_UTF32)
481 
-			bytesToWrite = 4;
482 
-		else
483 
-		{
484 
-			bytesToWrite = 3;
485 
-			ch = UNI_REPLACEMENT_CHAR;
486 
-			result = sourceIllegal;
487 
-		}
488 
-
489 
-		target += bytesToWrite;
490 
-		if (target > targetEnd)
491 
-		{
492 
-			--source; /* Back up source pointer! */
493 
-			target -= bytesToWrite;
494 
-			result = targetExhausted;
495 
-			break;
496 
-		}
497 
-		switch (bytesToWrite) /* note: everything falls through. */
498 
-		{
499 
-			case 4: *--target = static_cast<UTF8>((ch | byteMark) & byteMask); ch >>= 6;
500 
-			case 3: *--target = static_cast<UTF8>((ch | byteMark) & byteMask); ch >>= 6;
501 
-			case 2: *--target = static_cast<UTF8>((ch | byteMark) & byteMask); ch >>= 6;
502 
-			case 1: *--target = static_cast<UTF8>(ch | firstByteMark[bytesToWrite]);
503 
-		}
504 
-		target += bytesToWrite;
505 
-	}
506 
-	*sourceStart = source;
507 
-	*targetStart = target;
508 
-	return result;
509 
-}
510 
-
511 
-/* --------------------------------------------------------------------- */
512 
-
513 
-ConversionResult ConvertUTF8toUTF32(const UTF8 **sourceStart, const UTF8 *sourceEnd, UTF32 **targetStart, UTF32 *targetEnd, ConversionFlags flags)
514 
-{
515 
-	ConversionResult result = conversionOK;
516 
-	const UTF8 *source = *sourceStart;
517 
-	UTF32 *target = *targetStart;
518 
-	while (source < sourceEnd)
519 
-	{
520 
-		UTF32 ch = 0;
521 
-		unsigned short extraBytesToRead = trailingBytesForUTF8[*source];
522 
-		if (source + extraBytesToRead >= sourceEnd)
523 
-		{
524 
-			result = sourceExhausted;
525 
-			break;
526 
-		}
527 
-		/* Do this check whether lenient or strict */
528 
-		if (!isLegalUTF8(source, extraBytesToRead + 1))
529 
-		{
530 
-			result = sourceIllegal;
531 
-			break;
532 
-		}
533 
-		/*
534 
-		 * The cases all fall through. See "Note A" below.
535 
-		 */
536 
-		switch (extraBytesToRead)
537 
-		{
538 
-			case 5: ch += *source++; ch <<= 6;
539 
-			case 4: ch += *source++; ch <<= 6;
540 
-			case 3: ch += *source++; ch <<= 6;
541 
-			case 2: ch += *source++; ch <<= 6;
542 
-			case 1: ch += *source++; ch <<= 6;
543 
-			case 0: ch += *source++;
544 
-		}
545 
-		ch -= offsetsFromUTF8[extraBytesToRead];
546 
-
547 
-		if (target >= targetEnd)
548 
-		{
549 
-			source -= extraBytesToRead + 1; /* Back up the source pointer! */
550 
-			result = targetExhausted;
551 
-			break;
552 
-		}
553 
-		if (ch <= UNI_MAX_LEGAL_UTF32)
554 
-		{
555 
-			/*
556 
-			 * UTF-16 surrogate values are illegal in UTF-32, and anything
557 
-			 * over Plane 17 (> 0x10FFFF) is illegal.
558 
-			 */
559 
-			if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END)
560 
-			{
561 
-				if (flags == strictConversion)
562 
-				{
563 
-					source -= extraBytesToRead + 1; /* return to the illegal value itself */
564 
-					result = sourceIllegal;
565 
-					break;
566 
-				}
567 
-				else
568 
-					*target++ = UNI_REPLACEMENT_CHAR;
569 
-			}
570 
-			else
571 
-				*target++ = ch;
572 
-		}
573 
-		else /* i.e., ch > UNI_MAX_LEGAL_UTF32 */
574 
-		{
575 
-			result = sourceIllegal;
576 
-			*target++ = UNI_REPLACEMENT_CHAR;
577 
-		}
578 
-	}
579 
-	*sourceStart = source;
580 
-	*targetStart = target;
581 
-	return result;
582 
-}
583 
-
584 
-/* ---------------------------------------------------------------------
585 
-
586 
-    Note A.
587 
-    The fall-through switches in UTF-8 reading code save a
588 
-    temp variable, some decrements & conditionals.  The switches
589 
-    are equivalent to the following loop:
590 
-	{
591 
-	    int tmpBytesToRead = extraBytesToRead+1;
592 
-	    do {
593 
-		ch += *source++;
594 
-		--tmpBytesToRead;
595 
-		if (tmpBytesToRead) ch <<= 6;
596 
-	    } while (tmpBytesToRead > 0);
597 
-	}
598 
-    In UTF-8 writing code, the switches on "bytesToWrite" are
599 
-    similarly unrolled loops.
600 
-
601 
-   --------------------------------------------------------------------- */
Browse code

Import actual code.

Naram Qashat authored on 2013/03/26 02:41:19
Showing 1 changed files
src/in_xsf_framework/ConvertUTF.cpp
History View file @ bc076aa
1 1 
new file mode 100644
... ... 
@@ -0,0 +1,602 @@
1 
+/*
2 
+ * Copyright 2001-2004 Unicode, Inc.
3 
+ *
4 
+ * Disclaimer
5 
+ *
6 
+ * This source code is provided as is by Unicode, Inc. No claims are
7 
+ * made as to fitness for any particular purpose. No warranties of any
8 
+ * kind are expressed or implied. The recipient agrees to determine
9 
+ * applicability of information provided. If this file has been
10 
+ * purchased on magnetic or optical media from Unicode, Inc., the
11 
+ * sole remedy for any claim will be exchange of defective media
12 
+ * within 90 days of receipt.
13 
+ *
14 
+ * Limitations on Rights to Redistribute This Code
15 
+ *
16 
+ * Unicode, Inc. hereby grants the right to freely use the information
17 
+ * supplied in this file in the creation of products supporting the
18 
+ * Unicode Standard, and to make copies of this file in any form
19 
+ * for internal or external distribution as long as this notice
20 
+ * remains attached.
21 
+ */
22 
+
23 
+/* ---------------------------------------------------------------------
24 
+
25 
+    Conversions between UTF32, UTF-16, and UTF-8. Source code file.
26 
+    Author: Mark E. Davis, 1994.
27 
+    Rev History: Rick McGowan, fixes & updates May 2001.
28 
+    Sept 2001: fixed const & error conditions per
29 
+	mods suggested by S. Parent & A. Lillich.
30 
+    June 2002: Tim Dodd added detection and handling of incomplete
31 
+	source sequences, enhanced error detection, added casts
32 
+	to eliminate compiler warnings.
33 
+    July 2003: slight mods to back out aggressive FFFE detection.
34 
+    Jan 2004: updated switches in from-UTF8 conversions.
35 
+    Oct 2004: updated to use UNI_MAX_LEGAL_UTF32 in UTF-32 conversions.
36 
+
37 
+    See the header file "ConvertUTF.h" for complete documentation.
38 
+
39 
+------------------------------------------------------------------------ */
40 
+
41 
+#include "ConvertUTF.h"
42 
+
43 
+static const int halfShift  = 10; /* used for shifting by 10 bits */
44 
+
45 
+static const UTF32 halfBase = 0x0010000UL;
46 
+static const UTF32 halfMask = 0x3FFUL;
47 
+
48 
+static const UTF32 UNI_SUR_HIGH_START = 0xD800;
49 
+static const UTF32 UNI_SUR_HIGH_END = 0xDBFF;
50 
+static const UTF32 UNI_SUR_LOW_START = 0xDC00;
51 
+static const UTF32 UNI_SUR_LOW_END = 0xDFFF;
52 
+
53 
+/* --------------------------------------------------------------------- */
54 
+
55 
+ConversionResult ConvertUTF32toUTF16(const UTF32 **sourceStart, const UTF32 *sourceEnd, UTF16 **targetStart, UTF16 *targetEnd, ConversionFlags flags)
56 
+{
57 
+	ConversionResult result = conversionOK;
58 
+	const UTF32 *source = *sourceStart;
59 
+	UTF16 *target = *targetStart;
60 
+	while (source < sourceEnd)
61 
+	{
62 
+		if (target >= targetEnd)
63 
+		{
64 
+			result = targetExhausted;
65 
+			break;
66 
+		}
67 
+		UTF32 ch = *source++;
68 
+		if (ch <= UNI_MAX_BMP) /* Target is a character <= 0xFFFF */
69 
+		{
70 
+			/* UTF-16 surrogate values are illegal in UTF-32; 0xffff or 0xfffe are both reserved values */
71 
+			if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END)
72 
+			{
73 
+				if (flags == strictConversion)
74 
+				{
75 
+					--source; /* return to the illegal value itself */
76 
+					result = sourceIllegal;
77 
+					break;
78 
+				}
79 
+				else
80 
+					*target++ = UNI_REPLACEMENT_CHAR;
81 
+			}
82 
+			else
83 
+				*target++ = static_cast<UTF16>(ch); /* normal case */
84 
+		}
85 
+		else if (ch > UNI_MAX_LEGAL_UTF32)
86 
+		{
87 
+			if (flags == strictConversion)
88 
+				result = sourceIllegal;
89 
+			else
90 
+				*target++ = UNI_REPLACEMENT_CHAR;
91 
+		}
92 
+		else
93 
+		{
94 
+			/* target is a character in range 0xFFFF - 0x10FFFF. */
95 
+			if (target + 1 >= targetEnd)
96 
+			{
97 
+				--source; /* Back up source pointer! */
98 
+				result = targetExhausted;
99 
+				break;
100 
+			}
101 
+			ch -= halfBase;
102 
+			*target++ = static_cast<UTF16>((ch >> halfShift) + UNI_SUR_HIGH_START);
103 
+			*target++ = static_cast<UTF16>((ch & halfMask) + UNI_SUR_LOW_START);
104 
+		}
105 
+	}
106 
+	*sourceStart = source;
107 
+	*targetStart = target;
108 
+	return result;
109 
+}
110 
+
111 
+/* --------------------------------------------------------------------- */
112 
+
113 
+ConversionResult ConvertUTF16toUTF32(const UTF16 **sourceStart, const UTF16 *sourceEnd, UTF32 **targetStart, UTF32 *targetEnd, ConversionFlags flags)
114 
+{
115 
+	ConversionResult result = conversionOK;
116 
+	const UTF16 *source = *sourceStart;
117 
+	UTF32 *target = *targetStart;
118 
+	while (source < sourceEnd)
119 
+	{
120 
+		const UTF16 *oldSource = source; /* In case we have to back up because of target overflow. */
121 
+		UTF32 ch = *source++;
122 
+		/* If we have a surrogate pair, convert to UTF32 first. */
123 
+		if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END)
124 
+		{
125 
+			/* If the 16 bits following the high surrogate are in the source buffer... */
126 
+			if (source < sourceEnd)
127 
+			{
128 
+				UTF32 ch2 = *source;
129 
+				/* If it's a low surrogate, convert to UTF32. */
130 
+				if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END)
131 
+				{
132 
+					ch = ((ch - UNI_SUR_HIGH_START) << halfShift) + (ch2 - UNI_SUR_LOW_START) + halfBase;
133 
+					++source;
134 
+				}
135 
+				else if (flags == strictConversion) /* it's an unpaired high surrogate */
136 
+				{
137 
+					--source; /* return to the illegal value itself */
138 
+					result = sourceIllegal;
139 
+					break;
140 
+				}
141 
+			}
142 
+			else /* We don't have the 16 bits following the high surrogate. */
143 
+			{
144 
+				--source; /* return to the high surrogate */
145 
+				result = sourceExhausted;
146 
+				break;
147 
+			}
148 
+		}
149 
+		else if (flags == strictConversion)
150 
+		{
151 
+			/* UTF-16 surrogate values are illegal in UTF-32 */
152 
+			if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END)
153 
+			{
154 
+				--source; /* return to the illegal value itself */
155 
+				result = sourceIllegal;
156 
+				break;
157 
+			}
158 
+		}
159 
+		if (target >= targetEnd)
160 
+		{
161 
+			source = oldSource; /* Back up source pointer! */
162 
+			result = targetExhausted;
163 
+			break;
164 
+		}
165 
+		*target++ = ch;
166 
+	}
167 
+	*sourceStart = source;
168 
+	*targetStart = target;
169 
+	return result;
170 
+}
171 
+
172 
+/* --------------------------------------------------------------------- */
173 
+
174 
+/*
175 
+ * Index into the table below with the first byte of a UTF-8 sequence to
176 
+ * get the number of trailing bytes that are supposed to follow it.
177 
+ * Note that *legal* UTF-8 values can't have 4 or 5-bytes. The table is
178 
+ * left as-is for anyone who may want to do such conversion, which was
179 
+ * allowed in earlier algorithms.
180 
+ */
181 
+static const char trailingBytesForUTF8[256] = {
182 
+	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
183 
+	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
184 
+	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
185 
+	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
186 
+	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
187 
+	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
188 
+	1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
189 
+	2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5
190 
+};
191 
+
192 
+/*
193 
+ * Magic values subtracted from a buffer value during UTF8 conversion.
194 
+ * This table contains as many values as there might be trailing bytes
195 
+ * in a UTF-8 sequence.
196 
+ */
197 
+static const UTF32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL, 0x03C82080UL, 0xFA082080UL, 0x82082080UL };
198 
+
199 
+/*
200 
+ * Once the bits are split out into bytes of UTF-8, this is a mask OR-ed
201 
+ * into the first byte, depending on how many bytes follow.  There are
202 
+ * as many entries in this table as there are UTF-8 sequence types.
203 
+ * (I.e., one byte sequence, two byte... etc.). Remember that sequencs
204 
+ * for *legal* UTF-8 will be 4 or fewer bytes total.
205 
+ */
206 
+static const UTF8 firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
207 
+
208 
+/* --------------------------------------------------------------------- */
209 
+
210 
+/* The interface converts a whole buffer to avoid function-call overhead.
211 
+ * Constants have been gathered. Loops & conditionals have been removed as
212 
+ * much as possible for efficiency, in favor of drop-through switches.
213 
+ * (See "Note A" at the bottom of the file for equivalent code.)
214 
+ * If your compiler supports it, the "isLegalUTF8" call can be turned
215 
+ * into an inline function.
216 
+ */
217 
+
218 
+/* --------------------------------------------------------------------- */
219 
+
220 
+ConversionResult ConvertUTF16toUTF8(const UTF16 **sourceStart, const UTF16 *sourceEnd, UTF8 **targetStart, UTF8 *targetEnd, ConversionFlags flags)
221 
+{
222 
+	ConversionResult result = conversionOK;
223 
+	const UTF16 *source = *sourceStart;
224 
+	UTF8 *target = *targetStart;
225 
+	while (source < sourceEnd)
226 
+	{
227 
+		const UTF32 byteMask = 0xBF, byteMark = 0x80;
228 
+		const UTF16 *oldSource = source; /* In case we have to back up because of target overflow. */
229 
+		UTF32 ch = *source++;
230 
+		/* If we have a surrogate pair, convert to UTF32 first. */
231 
+		if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END)
232 
+		{
233 
+			/* If the 16 bits following the high surrogate are in the source buffer... */
234 
+			if (source < sourceEnd)
235 
+			{
236 
+				UTF32 ch2 = *source;
237 
+				/* If it's a low surrogate, convert to UTF32. */
238 
+				if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END)
239 
+				{
240 
+					ch = ((ch - UNI_SUR_HIGH_START) << halfShift) + (ch2 - UNI_SUR_LOW_START) + halfBase;
241 
+					++source;
242 
+				}
243 
+				else if (flags == strictConversion) /* it's an unpaired high surrogate */
244 
+				{
245 
+					--source; /* return to the illegal value itself */
246 
+					result = sourceIllegal;
247 
+					break;
248 
+				}
249 
+			}
250 
+			else /* We don't have the 16 bits following the high surrogate. */
251 
+			{
252 
+				--source; /* return to the high surrogate */
253 
+				result = sourceExhausted;
254 
+				break;
255 
+			}
256 
+		}
257 
+		else if (flags == strictConversion)
258 
+		{
259 
+			/* UTF-16 surrogate values are illegal in UTF-32 */
260 
+			if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END)
261 
+			{
262 
+				--source; /* return to the illegal value itself */
263 
+				result = sourceIllegal;
264 
+				break;
265 
+			}
266 
+		}
267 
+		/* Figure out how many bytes the result will require */
268 
+		unsigned short bytesToWrite = 0;
269 
+		if (ch < 0x80)
270 
+			bytesToWrite = 1;
271 
+		else if (ch < 0x800)
272 
+			bytesToWrite = 2;
273 
+		else if (ch < 0x10000)
274 
+			bytesToWrite = 3;
275 
+		else if (ch < 0x110000)
276 
+			bytesToWrite = 4;
277 
+		else
278 
+		{
279 
+			bytesToWrite = 3;
280 
+			ch = UNI_REPLACEMENT_CHAR;
281 
+		}
282 
+		target += bytesToWrite;
283 
+		if (target > targetEnd)
284 
+		{
285 
+			source = oldSource; /* Back up source pointer! */
286 
+			target -= bytesToWrite;
287 
+			result = targetExhausted;
288 
+			break;
289 
+		}
290 
+		switch (bytesToWrite) /* note: everything falls through. */
291 
+		{
292 
+			case 4: *--target = static_cast<UTF8>((ch | byteMark) & byteMask); ch >>= 6;
293 
+			case 3: *--target = static_cast<UTF8>((ch | byteMark) & byteMask); ch >>= 6;
294 
+			case 2: *--target = static_cast<UTF8>((ch | byteMark) & byteMask); ch >>= 6;
295 
+			case 1: *--target = static_cast<UTF8>(ch | firstByteMark[bytesToWrite]);
296 
+		}
297 
+		target += bytesToWrite;
298 
+	}
299 
+	*sourceStart = source;
300 
+	*targetStart = target;
301 
+	return result;
302 
+}
303 
+
304 
+/* --------------------------------------------------------------------- */
305 
+
306 
+/*
307 
+ * Utility routine to tell whether a sequence of bytes is legal UTF-8.
308 
+ * This must be called with the length pre-determined by the first byte.
309 
+ * If not calling this from ConvertUTF8to*, then the length can be set by:
310 
+ *  length = trailingBytesForUTF8[*source]+1;
311 
+ * and the sequence is illegal right away if there aren't that many bytes
312 
+ * available.
313 
+ * If presented with a length > 4, this returns false.  The Unicode
314 
+ * definition of UTF-8 goes up to 4-byte sequences.
315 
+ */
316 
+
317 
+static bool isLegalUTF8(const UTF8 *source, int length)
318 
+{
319 
+	const UTF8 *srcptr = source + length;
320 
+	UTF8 a;
321 
+	switch (length)
322 
+	{
323 
+		default: return false;
324 
+		/* Everything else falls through when "true"... */
325 
+		case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
326 
+		case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
327 
+		case 2: if ((a = (*--srcptr)) > 0xBF) return false;
328 
+
329 
+		switch (*source)
330 
+		{
331 
+			/* no fall-through in this inner switch */
332 
+			case 0xE0: if (a < 0xA0) return false; break;
333 
+			case 0xED: if (a > 0x9F) return false; break;
334 
+			case 0xF0: if (a < 0x90) return false; break;
335 
+			case 0xF4: if (a > 0x8F) return false; break;
336 
+			default: if (a < 0x80) return false;
337 
+		}
338 
+
339 
+		case 1: if (*source >= 0x80 && *source < 0xC2) return false;
340 
+	}
341 
+	if (*source > 0xF4)
342 
+		return false;
343 
+	return true;
344 
+}
345 
+
346 
+/* --------------------------------------------------------------------- */
347 
+
348 
+/*
349 
+ * Exported function to return whether a UTF-8 sequence is legal or not.
350 
+ * This is not used here; it's just exported.
351 
+ */
352 
+bool isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd)
353 
+{
354 
+	int length = trailingBytesForUTF8[*source] + 1;
355 
+	if (source + length > sourceEnd)
356 
+		return false;
357 
+	return isLegalUTF8(source, length);
358 
+}
359 
+
360 
+/* --------------------------------------------------------------------- */
361 
+
362 
+ConversionResult ConvertUTF8toUTF16(const UTF8 **sourceStart, const UTF8 *sourceEnd, UTF16 **targetStart, UTF16 *targetEnd, ConversionFlags flags)
363 
+{
364 
+	ConversionResult result = conversionOK;
365 
+	const UTF8 *source = *sourceStart;
366 
+	UTF16 *target = *targetStart;
367 
+	while (source < sourceEnd)
368 
+	{
369 
+		UTF32 ch = 0;
370 
+		unsigned short extraBytesToRead = trailingBytesForUTF8[*source];
371 
+		if (source + extraBytesToRead >= sourceEnd)
372 
+		{
373 
+			result = sourceExhausted;
374 
+			break;
375 
+		}
376 
+		/* Do this check whether lenient or strict */
377 
+		if (!isLegalUTF8(source, extraBytesToRead + 1))
378 
+		{
379 
+			result = sourceIllegal;
380 
+			break;
381 
+		}
382 
+		/*
383 
+		 * The cases all fall through. See "Note A" below.
384 
+		 */
385 
+		switch (extraBytesToRead)
386 
+		{
387 
+			case 5: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */
388 
+			case 4: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */
389 
+			case 3: ch += *source++; ch <<= 6;
390 
+			case 2: ch += *source++; ch <<= 6;
391 
+			case 1: ch += *source++; ch <<= 6;
392 
+			case 0: ch += *source++;
393 
+		}
394 
+		ch -= offsetsFromUTF8[extraBytesToRead];
395 
+
396 
+		if (target >= targetEnd)
397 
+		{
398 
+			source -= extraBytesToRead + 1; /* Back up source pointer! */
399 
+			result = targetExhausted;
400 
+			break;
401 
+		}
402 
+		if (ch <= UNI_MAX_BMP) /* Target is a character <= 0xFFFF */
403 
+		{
404 
+			/* UTF-16 surrogate values are illegal in UTF-32 */
405 
+			if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END)
406 
+			{
407 
+				if (flags == strictConversion)
408 
+				{
409 
+					source -= extraBytesToRead + 1; /* return to the illegal value itself */
410 
+					result = sourceIllegal;
411 
+					break;
412 
+				}
413 
+				else
414 
+					*target++ = UNI_REPLACEMENT_CHAR;
415 
+			}
416 
+			else
417 
+				*target++ = static_cast<UTF16>(ch); /* normal case */
418 
+		}
419 
+		else if (ch > UNI_MAX_UTF16)
420 
+		{
421 
+			if (flags == strictConversion)
422 
+			{
423 
+				result = sourceIllegal;
424 
+				source -= extraBytesToRead + 1; /* return to the start */
425 
+				break; /* Bail out; shouldn't continue */
426 
+			}
427 
+			else
428 
+				*target++ = UNI_REPLACEMENT_CHAR;
429 
+		}
430 
+		else
431 
+		{
432 
+			/* target is a character in range 0xFFFF - 0x10FFFF. */
433 
+			if (target + 1 >= targetEnd)
434 
+			{
435 
+				source -= extraBytesToRead + 1; /* Back up source pointer! */
436 
+				result = targetExhausted;
437 
+				break;
438 
+			}
439 
+			ch -= halfBase;
440 
+			*target++ = static_cast<UTF16>((ch >> halfShift) + UNI_SUR_HIGH_START);
441 
+			*target++ = static_cast<UTF16>((ch & halfMask) + UNI_SUR_LOW_START);
442 
+		}
443 
+	}
444 
+	*sourceStart = source;
445 
+	*targetStart = target;
446 
+	return result;
447 
+}
448 
+
449 
+/* --------------------------------------------------------------------- */
450 
+
451 
+ConversionResult ConvertUTF32toUTF8(const UTF32 **sourceStart, const UTF32 *sourceEnd, UTF8 **targetStart, UTF8 *targetEnd, ConversionFlags flags)
452 
+{
453 
+	ConversionResult result = conversionOK;
454 
+	const UTF32 *source = *sourceStart;
455 
+	UTF8 *target = *targetStart;
456 
+	while (source < sourceEnd)
457 
+	{
458 
+		const UTF32 byteMask = 0xBF, byteMark = 0x80;
459 
+		UTF32 ch = *source++;
460 
+		if (flags == strictConversion)
461 
+		{
462 
+			/* UTF-16 surrogate values are illegal in UTF-32 */
463 
+			if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END)
464 
+			{
465 
+				--source; /* return to the illegal value itself */
466 
+				result = sourceIllegal;
467 
+				break;
468 
+			}
469 
+		}
470 
+		/*
471 
+		 * Figure out how many bytes the result will require. Turn any
472 
+		 * illegally large UTF32 things (> Plane 17) into replacement chars.
473 
+		 */
474 
+		unsigned short bytesToWrite = 0;
475 
+		if (ch < 0x80)
476 
+			bytesToWrite = 1;
477 
+		else if (ch < 0x800)
478 
+			bytesToWrite = 2;
479 
+		else if (ch < 0x10000)
480 
+			bytesToWrite = 3;
481 
+		else if (ch <= UNI_MAX_LEGAL_UTF32)
482 
+			bytesToWrite = 4;
483 
+		else
484 
+		{
485 
+			bytesToWrite = 3;
486 
+			ch = UNI_REPLACEMENT_CHAR;
487 
+			result = sourceIllegal;
488 
+		}
489 
+
490 
+		target += bytesToWrite;
491 
+		if (target > targetEnd)
492 
+		{
493 
+			--source; /* Back up source pointer! */
494 
+			target -= bytesToWrite;
495 
+			result = targetExhausted;
496 
+			break;
497 
+		}
498 
+		switch (bytesToWrite) /* note: everything falls through. */
499 
+		{
500 
+			case 4: *--target = static_cast<UTF8>((ch | byteMark) & byteMask); ch >>= 6;
501 
+			case 3: *--target = static_cast<UTF8>((ch | byteMark) & byteMask); ch >>= 6;
502 
+			case 2: *--target = static_cast<UTF8>((ch | byteMark) & byteMask); ch >>= 6;
503 
+			case 1: *--target = static_cast<UTF8>(ch | firstByteMark[bytesToWrite]);
504 
+		}
505 
+		target += bytesToWrite;
506 
+	}
507 
+	*sourceStart = source;
508 
+	*targetStart = target;
509 
+	return result;
510 
+}
511 
+
512 
+/* --------------------------------------------------------------------- */
513 
+
514 
+ConversionResult ConvertUTF8toUTF32(const UTF8 **sourceStart, const UTF8 *sourceEnd, UTF32 **targetStart, UTF32 *targetEnd, ConversionFlags flags)
515 
+{
516 
+	ConversionResult result = conversionOK;
517 
+	const UTF8 *source = *sourceStart;
518 
+	UTF32 *target = *targetStart;
519 
+	while (source < sourceEnd)
520 
+	{
521 
+		UTF32 ch = 0;
522 
+		unsigned short extraBytesToRead = trailingBytesForUTF8[*source];
523 
+		if (source + extraBytesToRead >= sourceEnd)
524 
+		{
525 
+			result = sourceExhausted;
526 
+			break;
527 
+		}
528 
+		/* Do this check whether lenient or strict */
529 
+		if (!isLegalUTF8(source, extraBytesToRead + 1))
530 
+		{
531 
+			result = sourceIllegal;
532 
+			break;
533 
+		}
534 
+		/*
535 
+		 * The cases all fall through. See "Note A" below.
536 
+		 */
537 
+		switch (extraBytesToRead)
538 
+		{
539 
+			case 5: ch += *source++; ch <<= 6;
540 
+			case 4: ch += *source++; ch <<= 6;
541 
+			case 3: ch += *source++; ch <<= 6;
542 
+			case 2: ch += *source++; ch <<= 6;
543 
+			case 1: ch += *source++; ch <<= 6;
544 
+			case 0: ch += *source++;
545 
+		}
546 
+		ch -= offsetsFromUTF8[extraBytesToRead];
547 
+
548 
+		if (target >= targetEnd)
549 
+		{
550 
+			source -= extraBytesToRead + 1; /* Back up the source pointer! */
551 
+			result = targetExhausted;
552 
+			break;
553 
+		}
554 
+		if (ch <= UNI_MAX_LEGAL_UTF32)
555 
+		{
556 
+			/*
557 
+			 * UTF-16 surrogate values are illegal in UTF-32, and anything
558 
+			 * over Plane 17 (> 0x10FFFF) is illegal.
559 
+			 */
560 
+			if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END)
561 
+			{
562 
+				if (flags == strictConversion)
563 
+				{
564 
+					source -= extraBytesToRead + 1; /* return to the illegal value itself */
565 
+					result = sourceIllegal;
566 
+					break;
567 
+				}
568 
+				else
569 
+					*target++ = UNI_REPLACEMENT_CHAR;
570 
+			}
571 
+			else
572 
+				*target++ = ch;
573 
+		}
574 
+		else /* i.e., ch > UNI_MAX_LEGAL_UTF32 */
575 
+		{
576 
+			result = sourceIllegal;
577 
+			*target++ = UNI_REPLACEMENT_CHAR;
578 
+		}
579 
+	}
580 
+	*sourceStart = source;
581 
+	*targetStart = target;
582 
+	return result;
583 
+}
584 
+
585 
+/* ---------------------------------------------------------------------
586 
+
587 
+    Note A.
588 
+    The fall-through switches in UTF-8 reading code save a
589 
+    temp variable, some decrements & conditionals.  The switches
590 
+    are equivalent to the following loop:
591 
+	{
592 
+	    int tmpBytesToRead = extraBytesToRead+1;
593 
+	    do {
594 
+		ch += *source++;
595 
+		--tmpBytesToRead;
596 
+		if (tmpBytesToRead) ch <<= 6;
597 
+	    } while (tmpBytesToRead > 0);
598 
+	}
599 
+    In UTF-8 writing code, the switches on "bytesToWrite" are
600 
+    similarly unrolled loops.
601 
+
602 
+   --------------------------------------------------------------------- */