xref: /netbsd-src/external/cddl/osnet/dist/common/unicode/u8_textprep.c (revision 3227e6cf668bd374971740bd6660f43cee4417ac)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 
27 
28 /*
29  * UTF-8 text preparation functions (PSARC/2007/149, PSARC/2007/458).
30  *
31  * Man pages: u8_textprep_open(9F), u8_textprep_buf(9F), u8_textprep_close(9F),
32  * u8_textprep_str(9F), u8_strcmp(9F), and u8_validate(9F). See also
33  * the section 3C man pages.
34  * Interface stability: Committed.
35  */
36 
37 #include <sys/types.h>
38 #ifdef	_KERNEL
39 #include <sys/param.h>
40 #include <sys/sysmacros.h>
41 #include <sys/systm.h>
42 #include <sys/debug.h>
43 #include <sys/kmem.h>
44 #include <sys/sunddi.h>
45 #else
46 #include <strings.h>
47 #endif	/* _KERNEL */
48 #include <sys/byteorder.h>
49 #include <sys/errno.h>
50 #include <sys/u8_textprep.h>
51 #include <sys/u8_textprep_data.h>
52 
53 
54 /* The maximum possible number of bytes in a UTF-8 character. */
55 #define	U8_MB_CUR_MAX			(4)
56 
57 /*
58  * The maximum number of bytes needed for a UTF-8 character to cover
59  * U+0000 - U+FFFF, i.e., the coding space of now deprecated UCS-2.
60  */
61 #define	U8_MAX_BYTES_UCS2		(3)
62 
63 /* The maximum possible number of bytes in a Stream-Safe Text. */
64 #define	U8_STREAM_SAFE_TEXT_MAX		(128)
65 
66 /*
67  * The maximum number of characters in a combining/conjoining sequence and
68  * the actual upperbound limit of a combining/conjoining sequence.
69  */
70 #define	U8_MAX_CHARS_A_SEQ		(32)
71 #define	U8_UPPER_LIMIT_IN_A_SEQ		(31)
72 
73 /* The combining class value for Starter. */
74 #define	U8_COMBINING_CLASS_STARTER	(0)
75 
76 /*
77  * Some Hangul related macros at below.
78  *
79  * The first and the last of Hangul syllables, Hangul Jamo Leading consonants,
80  * Vowels, and optional Trailing consonants in Unicode scalar values.
81  *
82  * Please be noted that the U8_HANGUL_JAMO_T_FIRST is 0x11A7 at below not
83  * the actual U+11A8. This is due to that the trailing consonant is optional
84  * and thus we are doing a pre-calculation of subtracting one.
85  *
86  * Each of 19 modern leading consonants has total 588 possible syllables since
87  * Hangul has 21 modern vowels and 27 modern trailing consonants plus 1 for
88  * no trailing consonant case, i.e., 21 x 28 = 588.
89  *
90  * We also have bunch of Hangul related macros at below. Please bear in mind
91  * that the U8_HANGUL_JAMO_1ST_BYTE can be used to check whether it is
92  * a Hangul Jamo or not but the value does not guarantee that it is a Hangul
93  * Jamo; it just guarantee that it will be most likely.
94  */
95 #define	U8_HANGUL_SYL_FIRST		(0xAC00U)
96 #define	U8_HANGUL_SYL_LAST		(0xD7A3U)
97 
98 #define	U8_HANGUL_JAMO_L_FIRST		(0x1100U)
99 #define	U8_HANGUL_JAMO_L_LAST		(0x1112U)
100 #define	U8_HANGUL_JAMO_V_FIRST		(0x1161U)
101 #define	U8_HANGUL_JAMO_V_LAST		(0x1175U)
102 #define	U8_HANGUL_JAMO_T_FIRST		(0x11A7U)
103 #define	U8_HANGUL_JAMO_T_LAST		(0x11C2U)
104 
105 #define	U8_HANGUL_V_COUNT		(21)
106 #define	U8_HANGUL_VT_COUNT		(588)
107 #define	U8_HANGUL_T_COUNT		(28)
108 
109 #define	U8_HANGUL_JAMO_1ST_BYTE		(0xE1U)
110 
111 #define	U8_SAVE_HANGUL_AS_UTF8(s, i, j, k, b) \
112 	(s)[(i)] = (uchar_t)(0xE0U | ((uint32_t)(b) & 0xF000U) >> 12); \
113 	(s)[(j)] = (uchar_t)(0x80U | ((uint32_t)(b) & 0x0FC0U) >> 6); \
114 	(s)[(k)] = (uchar_t)(0x80U | ((uint32_t)(b) & 0x003FU));
115 
116 #define	U8_HANGUL_JAMO_L(u) \
117 	((u) >= U8_HANGUL_JAMO_L_FIRST && (u) <= U8_HANGUL_JAMO_L_LAST)
118 
119 #define	U8_HANGUL_JAMO_V(u) \
120 	((u) >= U8_HANGUL_JAMO_V_FIRST && (u) <= U8_HANGUL_JAMO_V_LAST)
121 
122 #define	U8_HANGUL_JAMO_T(u) \
123 	((u) > U8_HANGUL_JAMO_T_FIRST && (u) <= U8_HANGUL_JAMO_T_LAST)
124 
125 #define	U8_HANGUL_JAMO(u) \
126 	((u) >= U8_HANGUL_JAMO_L_FIRST && (u) <= U8_HANGUL_JAMO_T_LAST)
127 
128 #define	U8_HANGUL_SYLLABLE(u) \
129 	((u) >= U8_HANGUL_SYL_FIRST && (u) <= U8_HANGUL_SYL_LAST)
130 
131 #define	U8_HANGUL_COMPOSABLE_L_V(s, u) \
132 	((s) == U8_STATE_HANGUL_L && U8_HANGUL_JAMO_V((u)))
133 
134 #define	U8_HANGUL_COMPOSABLE_LV_T(s, u) \
135 	((s) == U8_STATE_HANGUL_LV && U8_HANGUL_JAMO_T((u)))
136 
137 /* The types of decomposition mappings. */
138 #define	U8_DECOMP_BOTH			(0xF5U)
139 #define	U8_DECOMP_CANONICAL		(0xF6U)
140 
141 /* The indicator for 16-bit table. */
142 #define	U8_16BIT_TABLE_INDICATOR	(0x8000U)
143 
144 /* The following are some convenience macros. */
145 #define	U8_PUT_3BYTES_INTO_UTF32(u, b1, b2, b3)  \
146 	(u) = ((((uint32_t)(b1) & 0x0F) << 12) | \
147 		(((uint32_t)(b2) & 0x3F) << 6)  | \
148 		((uint32_t)(b3) & 0x3F));
149 #define	U8_SIMPLE_SWAP(a, b, t) \
150 	(t) = (a); \
151 	(a) = (b); \
152 	(b) = (t);
153 
154 #define	U8_ASCII_TOUPPER(c) \
155 	(((c) >= 'a' && (c) <= 'z') ? (c) - 'a' + 'A' : (c))
156 
157 #define	U8_ASCII_TOLOWER(c) \
158 	(((c) >= 'A' && (c) <= 'Z') ? (c) - 'A' + 'a' : (c))
159 
160 #define	U8_ISASCII(c)			(((uchar_t)(c)) < 0x80U)
161 /*
162  * The following macro assumes that the two characters that are to be
163  * swapped are adjacent to each other and 'a' comes before 'b'.
164  *
165  * If the assumptions are not met, then, the macro will fail.
166  */
167 #define	U8_SWAP_COMB_MARKS(a, b) \
168 	for (k = 0; k < disp[(a)]; k++) \
169 		u8t[k] = u8s[start[(a)] + k]; \
170 	for (k = 0; k < disp[(b)]; k++) \
171 		u8s[start[(a)] + k] = u8s[start[(b)] + k]; \
172 	start[(b)] = start[(a)] + disp[(b)]; \
173 	for (k = 0; k < disp[(a)]; k++) \
174 		u8s[start[(b)] + k] = u8t[k]; \
175 	U8_SIMPLE_SWAP(comb_class[(a)], comb_class[(b)], tc); \
176 	U8_SIMPLE_SWAP(disp[(a)], disp[(b)], tc);
177 
178 /* The possible states during normalization. */
179 typedef enum {
180 	U8_STATE_START = 0,
181 	U8_STATE_HANGUL_L = 1,
182 	U8_STATE_HANGUL_LV = 2,
183 	U8_STATE_HANGUL_LVT = 3,
184 	U8_STATE_HANGUL_V = 4,
185 	U8_STATE_HANGUL_T = 5,
186 	U8_STATE_COMBINING_MARK = 6
187 } u8_normalization_states_t;
188 
189 /*
190  * The three vectors at below are used to check bytes of a given UTF-8
191  * character are valid and not containing any malformed byte values.
192  *
193  * We used to have a quite relaxed UTF-8 binary representation but then there
194  * was some security related issues and so the Unicode Consortium defined
195  * and announced the UTF-8 Corrigendum at Unicode 3.1 and then refined it
196  * one more time at the Unicode 3.2. The following three tables are based on
197  * that.
198  */
199 
200 #define	U8_ILLEGAL_NEXT_BYTE_COMMON(c)	((c) < 0x80 || (c) > 0xBF)
201 
202 #define	I_				U8_ILLEGAL_CHAR
203 #define	O_				U8_OUT_OF_RANGE_CHAR
204 
205 const int8_t u8_number_of_bytes[0x100] = {
206 	1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,
207 	1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,
208 	1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,
209 	1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,
210 	1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,
211 	1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,
212 	1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,
213 	1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,
214 
215 /*	80  81  82  83  84  85  86  87  88  89  8A  8B  8C  8D  8E  8F  */
216 	I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_,
217 
218 /*	90  91  92  93  94  95  96  97  98  99  9A  9B  9C  9D  9E  9F  */
219 	I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_,
220 
221 /*	A0  A1  A2  A3  A4  A5  A6  A7  A8  A9  AA  AB  AC  AD  AE  AF  */
222 	I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_,
223 
224 /*	B0  B1  B2  B3  B4  B5  B6  B7  B8  B9  BA  BB  BC  BD  BE  BF  */
225 	I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_,
226 
227 /*	C0  C1  C2  C3  C4  C5  C6  C7  C8  C9  CA  CB  CC  CD  CE  CF  */
228 	I_, I_, 2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
229 
230 /*	D0  D1  D2  D3  D4  D5  D6  D7  D8  D9  DA  DB  DC  DD  DE  DF  */
231 	2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,
232 
233 /*	E0  E1  E2  E3  E4  E5  E6  E7  E8  E9  EA  EB  EC  ED  EE  EF  */
234 	3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,
235 
236 /*	F0  F1  F2  F3  F4  F5  F6  F7  F8  F9  FA  FB  FC  FD  FE  FF  */
237 	4,  4,  4,  4,  4,  O_, O_, O_, O_, O_, O_, O_, O_, O_, O_, O_,
238 };
239 
240 #undef	I_
241 #undef	O_
242 
243 const uint8_t u8_valid_min_2nd_byte[0x100] = {
244 	0,    0,    0,    0,    0,    0,    0,    0,
245 	0,    0,    0,    0,    0,    0,    0,    0,
246 	0,    0,    0,    0,    0,    0,    0,    0,
247 	0,    0,    0,    0,    0,    0,    0,    0,
248 	0,    0,    0,    0,    0,    0,    0,    0,
249 	0,    0,    0,    0,    0,    0,    0,    0,
250 	0,    0,    0,    0,    0,    0,    0,    0,
251 	0,    0,    0,    0,    0,    0,    0,    0,
252 	0,    0,    0,    0,    0,    0,    0,    0,
253 	0,    0,    0,    0,    0,    0,    0,    0,
254 	0,    0,    0,    0,    0,    0,    0,    0,
255 	0,    0,    0,    0,    0,    0,    0,    0,
256 	0,    0,    0,    0,    0,    0,    0,    0,
257 	0,    0,    0,    0,    0,    0,    0,    0,
258 	0,    0,    0,    0,    0,    0,    0,    0,
259 	0,    0,    0,    0,    0,    0,    0,    0,
260 	0,    0,    0,    0,    0,    0,    0,    0,
261 	0,    0,    0,    0,    0,    0,    0,    0,
262 	0,    0,    0,    0,    0,    0,    0,    0,
263 	0,    0,    0,    0,    0,    0,    0,    0,
264 	0,    0,    0,    0,    0,    0,    0,    0,
265 	0,    0,    0,    0,    0,    0,    0,    0,
266 	0,    0,    0,    0,    0,    0,    0,    0,
267 	0,    0,    0,    0,    0,    0,    0,    0,
268 /*	C0    C1    C2    C3    C4    C5    C6    C7    */
269 	0,    0,    0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
270 /*	C8    C9    CA    CB    CC    CD    CE    CF    */
271 	0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
272 /*	D0    D1    D2    D3    D4    D5    D6    D7    */
273 	0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
274 /*	D8    D9    DA    DB    DC    DD    DE    DF    */
275 	0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
276 /*	E0    E1    E2    E3    E4    E5    E6    E7    */
277 	0xa0, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
278 /*	E8    E9    EA    EB    EC    ED    EE    EF    */
279 	0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
280 /*	F0    F1    F2    F3    F4    F5    F6    F7    */
281 	0x90, 0x80, 0x80, 0x80, 0x80, 0,    0,    0,
282 	0,    0,    0,    0,    0,    0,    0,    0,
283 };
284 
285 const uint8_t u8_valid_max_2nd_byte[0x100] = {
286 	0,    0,    0,    0,    0,    0,    0,    0,
287 	0,    0,    0,    0,    0,    0,    0,    0,
288 	0,    0,    0,    0,    0,    0,    0,    0,
289 	0,    0,    0,    0,    0,    0,    0,    0,
290 	0,    0,    0,    0,    0,    0,    0,    0,
291 	0,    0,    0,    0,    0,    0,    0,    0,
292 	0,    0,    0,    0,    0,    0,    0,    0,
293 	0,    0,    0,    0,    0,    0,    0,    0,
294 	0,    0,    0,    0,    0,    0,    0,    0,
295 	0,    0,    0,    0,    0,    0,    0,    0,
296 	0,    0,    0,    0,    0,    0,    0,    0,
297 	0,    0,    0,    0,    0,    0,    0,    0,
298 	0,    0,    0,    0,    0,    0,    0,    0,
299 	0,    0,    0,    0,    0,    0,    0,    0,
300 	0,    0,    0,    0,    0,    0,    0,    0,
301 	0,    0,    0,    0,    0,    0,    0,    0,
302 	0,    0,    0,    0,    0,    0,    0,    0,
303 	0,    0,    0,    0,    0,    0,    0,    0,
304 	0,    0,    0,    0,    0,    0,    0,    0,
305 	0,    0,    0,    0,    0,    0,    0,    0,
306 	0,    0,    0,    0,    0,    0,    0,    0,
307 	0,    0,    0,    0,    0,    0,    0,    0,
308 	0,    0,    0,    0,    0,    0,    0,    0,
309 	0,    0,    0,    0,    0,    0,    0,    0,
310 /*	C0    C1    C2    C3    C4    C5    C6    C7    */
311 	0,    0,    0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf,
312 /*	C8    C9    CA    CB    CC    CD    CE    CF    */
313 	0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf,
314 /*	D0    D1    D2    D3    D4    D5    D6    D7    */
315 	0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf,
316 /*	D8    D9    DA    DB    DC    DD    DE    DF    */
317 	0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf,
318 /*	E0    E1    E2    E3    E4    E5    E6    E7    */
319 	0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf,
320 /*	E8    E9    EA    EB    EC    ED    EE    EF    */
321 	0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0x9f, 0xbf, 0xbf,
322 /*	F0    F1    F2    F3    F4    F5    F6    F7    */
323 	0xbf, 0xbf, 0xbf, 0xbf, 0x8f, 0,    0,    0,
324 	0,    0,    0,    0,    0,    0,    0,    0,
325 };
326 
327 
328 /*
329  * The u8_validate() validates on the given UTF-8 character string and
330  * calculate the byte length. It is quite similar to mblen(3C) except that
331  * this will validate against the list of characters if required and
332  * specific to UTF-8 and Unicode.
333  */
334 int
u8_validate(char * u8str,size_t n,char ** list,int flag,int * errnum)335 u8_validate(char *u8str, size_t n, char **list, int flag, int *errnum)
336 {
337 	uchar_t *ib;
338 	uchar_t *ibtail;
339 	uchar_t **p;
340 	uchar_t *s1;
341 	uchar_t *s2;
342 	uchar_t f;
343 	int sz;
344 	size_t i;
345 	int ret_val;
346 	boolean_t second;
347 	boolean_t no_need_to_validate_entire;
348 	boolean_t check_additional;
349 	boolean_t validate_ucs2_range_only;
350 
351 	if (! u8str)
352 		return (0);
353 
354 	ib = (uchar_t *)u8str;
355 	ibtail = ib + n;
356 
357 	ret_val = 0;
358 
359 	no_need_to_validate_entire = ! (flag & U8_VALIDATE_ENTIRE);
360 	check_additional = flag & U8_VALIDATE_CHECK_ADDITIONAL;
361 	validate_ucs2_range_only = flag & U8_VALIDATE_UCS2_RANGE;
362 
363 	while (ib < ibtail) {
364 		/*
365 		 * The first byte of a UTF-8 character tells how many
366 		 * bytes will follow for the character. If the first byte
367 		 * is an illegal byte value or out of range value, we just
368 		 * return -1 with an appropriate error number.
369 		 */
370 		sz = u8_number_of_bytes[*ib];
371 		if (sz == U8_ILLEGAL_CHAR) {
372 			*errnum = EILSEQ;
373 			return (-1);
374 		}
375 
376 		if (sz == U8_OUT_OF_RANGE_CHAR ||
377 		    (validate_ucs2_range_only && sz > U8_MAX_BYTES_UCS2)) {
378 			*errnum = ERANGE;
379 			return (-1);
380 		}
381 
382 		/*
383 		 * If we don't have enough bytes to check on, that's also
384 		 * an error. As you can see, we give illegal byte sequence
385 		 * checking higher priority then EINVAL cases.
386 		 */
387 		if ((ibtail - ib) < sz) {
388 			*errnum = EINVAL;
389 			return (-1);
390 		}
391 
392 		if (sz == 1) {
393 			ib++;
394 			ret_val++;
395 		} else {
396 			/*
397 			 * Check on the multi-byte UTF-8 character. For more
398 			 * details on this, see comment added for the used
399 			 * data structures at the beginning of the file.
400 			 */
401 			f = *ib++;
402 			ret_val++;
403 			second = B_TRUE;
404 			for (i = 1; i < sz; i++) {
405 				if (second) {
406 					if (*ib < u8_valid_min_2nd_byte[f] ||
407 					    *ib > u8_valid_max_2nd_byte[f]) {
408 						*errnum = EILSEQ;
409 						return (-1);
410 					}
411 					second = B_FALSE;
412 				} else if (U8_ILLEGAL_NEXT_BYTE_COMMON(*ib)) {
413 					*errnum = EILSEQ;
414 					return (-1);
415 				}
416 				ib++;
417 				ret_val++;
418 			}
419 		}
420 
421 		if (check_additional) {
422 			for (p = (uchar_t **)list, i = 0; p[i]; i++) {
423 				s1 = ib - sz;
424 				s2 = p[i];
425 				while (s1 < ib) {
426 					if (*s1 != *s2 || *s2 == '\0')
427 						break;
428 					s1++;
429 					s2++;
430 				}
431 
432 				if (s1 >= ib && *s2 == '\0') {
433 					*errnum = EBADF;
434 					return (-1);
435 				}
436 			}
437 		}
438 
439 		if (no_need_to_validate_entire)
440 			break;
441 	}
442 
443 	return (ret_val);
444 }
445 
446 /*
447  * The do_case_conv() looks at the mapping tables and returns found
448  * bytes if any. If not found, the input bytes are returned. The function
449  * always terminate the return bytes with a null character assuming that
450  * there are plenty of room to do so.
451  *
452  * The case conversions are simple case conversions mapping a character to
453  * another character as specified in the Unicode data. The byte size of
454  * the mapped character could be different from that of the input character.
455  *
456  * The return value is the byte length of the returned character excluding
457  * the terminating null byte.
458  */
459 static size_t
do_case_conv(int uv,uchar_t * u8s,uchar_t * s,int sz,boolean_t is_it_toupper)460 do_case_conv(int uv, uchar_t *u8s, uchar_t *s, int sz, boolean_t is_it_toupper)
461 {
462 	size_t i;
463 	uint16_t b1 = 0;
464 	uint16_t b2 = 0;
465 	uint16_t b3 = 0;
466 	uint16_t b3_tbl;
467 	uint16_t b3_base;
468 	uint16_t b4 = 0;
469 	size_t start_id;
470 	size_t end_id;
471 
472 	/*
473 	 * At this point, the only possible values for sz are 2, 3, and 4.
474 	 * The u8s should point to a vector that is well beyond the size of
475 	 * 5 bytes.
476 	 */
477 	if (sz == 2) {
478 		b3 = u8s[0] = s[0];
479 		b4 = u8s[1] = s[1];
480 	} else if (sz == 3) {
481 		b2 = u8s[0] = s[0];
482 		b3 = u8s[1] = s[1];
483 		b4 = u8s[2] = s[2];
484 	} else if (sz == 4) {
485 		b1 = u8s[0] = s[0];
486 		b2 = u8s[1] = s[1];
487 		b3 = u8s[2] = s[2];
488 		b4 = u8s[3] = s[3];
489 	} else {
490 		/* This is not possible but just in case as a fallback. */
491 		if (is_it_toupper)
492 			*u8s = U8_ASCII_TOUPPER(*s);
493 		else
494 			*u8s = U8_ASCII_TOLOWER(*s);
495 		u8s[1] = '\0';
496 
497 		return (1);
498 	}
499 	u8s[sz] = '\0';
500 
501 	/*
502 	 * Let's find out if we have a corresponding character.
503 	 */
504 	b1 = u8_common_b1_tbl[uv][b1];
505 	if (b1 == U8_TBL_ELEMENT_NOT_DEF)
506 		return ((size_t)sz);
507 
508 	b2 = u8_case_common_b2_tbl[uv][b1][b2];
509 	if (b2 == U8_TBL_ELEMENT_NOT_DEF)
510 		return ((size_t)sz);
511 
512 	if (is_it_toupper) {
513 		b3_tbl = u8_toupper_b3_tbl[uv][b2][b3].tbl_id;
514 		if (b3_tbl == U8_TBL_ELEMENT_NOT_DEF)
515 			return ((size_t)sz);
516 
517 		start_id = u8_toupper_b4_tbl[uv][b3_tbl][b4];
518 		end_id = u8_toupper_b4_tbl[uv][b3_tbl][b4 + 1];
519 
520 		/* Either there is no match or an error at the table. */
521 		if (start_id >= end_id || (end_id - start_id) > U8_MB_CUR_MAX)
522 			return ((size_t)sz);
523 
524 		b3_base = u8_toupper_b3_tbl[uv][b2][b3].base;
525 
526 		for (i = 0; start_id < end_id; start_id++)
527 			u8s[i++] = u8_toupper_final_tbl[uv][b3_base + start_id];
528 	} else {
529 		b3_tbl = u8_tolower_b3_tbl[uv][b2][b3].tbl_id;
530 		if (b3_tbl == U8_TBL_ELEMENT_NOT_DEF)
531 			return ((size_t)sz);
532 
533 		start_id = u8_tolower_b4_tbl[uv][b3_tbl][b4];
534 		end_id = u8_tolower_b4_tbl[uv][b3_tbl][b4 + 1];
535 
536 		if (start_id >= end_id || (end_id - start_id) > U8_MB_CUR_MAX)
537 			return ((size_t)sz);
538 
539 		b3_base = u8_tolower_b3_tbl[uv][b2][b3].base;
540 
541 		for (i = 0; start_id < end_id; start_id++)
542 			u8s[i++] = u8_tolower_final_tbl[uv][b3_base + start_id];
543 	}
544 
545 	/*
546 	 * If i is still zero, that means there is no corresponding character.
547 	 */
548 	if (i == 0)
549 		return ((size_t)sz);
550 
551 	u8s[i] = '\0';
552 
553 	return (i);
554 }
555 
556 /*
557  * The do_case_compare() function compares the two input strings, s1 and s2,
558  * one character at a time doing case conversions if applicable and return
559  * the comparison result as like strcmp().
560  *
561  * Since, in empirical sense, most of text data are 7-bit ASCII characters,
562  * we treat the 7-bit ASCII characters as a special case trying to yield
563  * faster processing time.
564  */
565 static int
do_case_compare(size_t uv,uchar_t * s1,uchar_t * s2,size_t n1,size_t n2,boolean_t is_it_toupper,int * errnum)566 do_case_compare(size_t uv, uchar_t *s1, uchar_t *s2, size_t n1,
567 	size_t n2, boolean_t is_it_toupper, int *errnum)
568 {
569 	int f;
570 	int sz1;
571 	int sz2;
572 	size_t j;
573 	size_t i1;
574 	size_t i2;
575 	uchar_t u8s1[U8_MB_CUR_MAX + 1];
576 	uchar_t u8s2[U8_MB_CUR_MAX + 1];
577 
578 	i1 = i2 = 0;
579 	while (i1 < n1 && i2 < n2) {
580 		/*
581 		 * Find out what would be the byte length for this UTF-8
582 		 * character at string s1 and also find out if this is
583 		 * an illegal start byte or not and if so, issue a proper
584 		 * error number and yet treat this byte as a character.
585 		 */
586 		sz1 = u8_number_of_bytes[*s1];
587 		if (sz1 < 0) {
588 			*errnum = EILSEQ;
589 			sz1 = 1;
590 		}
591 
592 		/*
593 		 * For 7-bit ASCII characters mainly, we do a quick case
594 		 * conversion right at here.
595 		 *
596 		 * If we don't have enough bytes for this character, issue
597 		 * an EINVAL error and use what are available.
598 		 *
599 		 * If we have enough bytes, find out if there is
600 		 * a corresponding uppercase character and if so, copy over
601 		 * the bytes for a comparison later. If there is no
602 		 * corresponding uppercase character, then, use what we have
603 		 * for the comparison.
604 		 */
605 		if (sz1 == 1) {
606 			if (is_it_toupper)
607 				u8s1[0] = U8_ASCII_TOUPPER(*s1);
608 			else
609 				u8s1[0] = U8_ASCII_TOLOWER(*s1);
610 			s1++;
611 			u8s1[1] = '\0';
612 		} else if ((i1 + sz1) > n1) {
613 			*errnum = EINVAL;
614 			for (j = 0; (i1 + j) < n1; )
615 				u8s1[j++] = *s1++;
616 			u8s1[j] = '\0';
617 		} else {
618 			(void) do_case_conv(uv, u8s1, s1, sz1, is_it_toupper);
619 			s1 += sz1;
620 		}
621 
622 		/* Do the same for the string s2. */
623 		sz2 = u8_number_of_bytes[*s2];
624 		if (sz2 < 0) {
625 			*errnum = EILSEQ;
626 			sz2 = 1;
627 		}
628 
629 		if (sz2 == 1) {
630 			if (is_it_toupper)
631 				u8s2[0] = U8_ASCII_TOUPPER(*s2);
632 			else
633 				u8s2[0] = U8_ASCII_TOLOWER(*s2);
634 			s2++;
635 			u8s2[1] = '\0';
636 		} else if ((i2 + sz2) > n2) {
637 			*errnum = EINVAL;
638 			for (j = 0; (i2 + j) < n2; )
639 				u8s2[j++] = *s2++;
640 			u8s2[j] = '\0';
641 		} else {
642 			(void) do_case_conv(uv, u8s2, s2, sz2, is_it_toupper);
643 			s2 += sz2;
644 		}
645 
646 		/* Now compare the two characters. */
647 		if (sz1 == 1 && sz2 == 1) {
648 			if (*u8s1 > *u8s2)
649 				return (1);
650 			if (*u8s1 < *u8s2)
651 				return (-1);
652 		} else {
653 			f = strcmp((const char *)u8s1, (const char *)u8s2);
654 			if (f != 0)
655 				return (f);
656 		}
657 
658 		/*
659 		 * They were the same. Let's move on to the next
660 		 * characters then.
661 		 */
662 		i1 += sz1;
663 		i2 += sz2;
664 	}
665 
666 	/*
667 	 * We compared until the end of either or both strings.
668 	 *
669 	 * If we reached to or went over the ends for the both, that means
670 	 * they are the same.
671 	 *
672 	 * If we reached only one of the two ends, that means the other string
673 	 * has something which then the fact can be used to determine
674 	 * the return value.
675 	 */
676 	if (i1 >= n1) {
677 		if (i2 >= n2)
678 			return (0);
679 		return (-1);
680 	}
681 	return (1);
682 }
683 
684 /*
685  * The combining_class() function checks on the given bytes and find out
686  * the corresponding Unicode combining class value. The return value 0 means
687  * it is a Starter. Any illegal UTF-8 character will also be treated as
688  * a Starter.
689  */
690 static uchar_t
combining_class(size_t uv,uchar_t * s,size_t sz)691 combining_class(size_t uv, uchar_t *s, size_t sz)
692 {
693 	uint16_t b1 = 0;
694 	uint16_t b2 = 0;
695 	uint16_t b3 = 0;
696 	uint16_t b4 = 0;
697 
698 	if (sz == 1 || sz > 4)
699 		return (0);
700 
701 	if (sz == 2) {
702 		b3 = s[0];
703 		b4 = s[1];
704 	} else if (sz == 3) {
705 		b2 = s[0];
706 		b3 = s[1];
707 		b4 = s[2];
708 	} else if (sz == 4) {
709 		b1 = s[0];
710 		b2 = s[1];
711 		b3 = s[2];
712 		b4 = s[3];
713 	}
714 
715 	b1 = u8_common_b1_tbl[uv][b1];
716 	if (b1 == U8_TBL_ELEMENT_NOT_DEF)
717 		return (0);
718 
719 	b2 = u8_combining_class_b2_tbl[uv][b1][b2];
720 	if (b2 == U8_TBL_ELEMENT_NOT_DEF)
721 		return (0);
722 
723 	b3 = u8_combining_class_b3_tbl[uv][b2][b3];
724 	if (b3 == U8_TBL_ELEMENT_NOT_DEF)
725 		return (0);
726 
727 	return (u8_combining_class_b4_tbl[uv][b3][b4]);
728 }
729 
730 /*
731  * The do_decomp() function finds out a matching decomposition if any
732  * and return. If there is no match, the input bytes are copied and returned.
733  * The function also checks if there is a Hangul, decomposes it if necessary
734  * and returns.
735  *
736  * To save time, a single byte 7-bit ASCII character should be handled by
737  * the caller.
738  *
739  * The function returns the number of bytes returned sans always terminating
740  * the null byte. It will also return a state that will tell if there was
741  * a Hangul character decomposed which then will be used by the caller.
742  */
743 static size_t
do_decomp(size_t uv,uchar_t * u8s,uchar_t * s,int sz,boolean_t canonical_decomposition,u8_normalization_states_t * state)744 do_decomp(size_t uv, uchar_t *u8s, uchar_t *s, int sz,
745 	boolean_t canonical_decomposition, u8_normalization_states_t *state)
746 {
747 	uint16_t b1 = 0;
748 	uint16_t b2 = 0;
749 	uint16_t b3 = 0;
750 	uint16_t b3_tbl;
751 	uint16_t b3_base;
752 	uint16_t b4 = 0;
753 	size_t start_id;
754 	size_t end_id;
755 	size_t i;
756 	uint32_t u1;
757 
758 	if (sz == 2) {
759 		b3 = u8s[0] = s[0];
760 		b4 = u8s[1] = s[1];
761 		u8s[2] = '\0';
762 	} else if (sz == 3) {
763 		/* Convert it to a Unicode scalar value. */
764 		U8_PUT_3BYTES_INTO_UTF32(u1, s[0], s[1], s[2]);
765 
766 		/*
767 		 * If this is a Hangul syllable, we decompose it into
768 		 * a leading consonant, a vowel, and an optional trailing
769 		 * consonant and then return.
770 		 */
771 		if (U8_HANGUL_SYLLABLE(u1)) {
772 			u1 -= U8_HANGUL_SYL_FIRST;
773 
774 			b1 = U8_HANGUL_JAMO_L_FIRST + u1 / U8_HANGUL_VT_COUNT;
775 			b2 = U8_HANGUL_JAMO_V_FIRST + (u1 % U8_HANGUL_VT_COUNT)
776 			    / U8_HANGUL_T_COUNT;
777 			b3 = u1 % U8_HANGUL_T_COUNT;
778 
779 			U8_SAVE_HANGUL_AS_UTF8(u8s, 0, 1, 2, b1);
780 			U8_SAVE_HANGUL_AS_UTF8(u8s, 3, 4, 5, b2);
781 			if (b3) {
782 				b3 += U8_HANGUL_JAMO_T_FIRST;
783 				U8_SAVE_HANGUL_AS_UTF8(u8s, 6, 7, 8, b3);
784 
785 				u8s[9] = '\0';
786 				*state = U8_STATE_HANGUL_LVT;
787 				return (9);
788 			}
789 
790 			u8s[6] = '\0';
791 			*state = U8_STATE_HANGUL_LV;
792 			return (6);
793 		}
794 
795 		b2 = u8s[0] = s[0];
796 		b3 = u8s[1] = s[1];
797 		b4 = u8s[2] = s[2];
798 		u8s[3] = '\0';
799 
800 		/*
801 		 * If this is a Hangul Jamo, we know there is nothing
802 		 * further that we can decompose.
803 		 */
804 		if (U8_HANGUL_JAMO_L(u1)) {
805 			*state = U8_STATE_HANGUL_L;
806 			return (3);
807 		}
808 
809 		if (U8_HANGUL_JAMO_V(u1)) {
810 			if (*state == U8_STATE_HANGUL_L)
811 				*state = U8_STATE_HANGUL_LV;
812 			else
813 				*state = U8_STATE_HANGUL_V;
814 			return (3);
815 		}
816 
817 		if (U8_HANGUL_JAMO_T(u1)) {
818 			if (*state == U8_STATE_HANGUL_LV)
819 				*state = U8_STATE_HANGUL_LVT;
820 			else
821 				*state = U8_STATE_HANGUL_T;
822 			return (3);
823 		}
824 	} else if (sz == 4) {
825 		b1 = u8s[0] = s[0];
826 		b2 = u8s[1] = s[1];
827 		b3 = u8s[2] = s[2];
828 		b4 = u8s[3] = s[3];
829 		u8s[4] = '\0';
830 	} else {
831 		/*
832 		 * This is a fallback and should not happen if the function
833 		 * was called properly.
834 		 */
835 		u8s[0] = s[0];
836 		u8s[1] = '\0';
837 		*state = U8_STATE_START;
838 		return (1);
839 	}
840 
841 	/*
842 	 * At this point, this rountine does not know what it would get.
843 	 * The caller should sort it out if the state isn't a Hangul one.
844 	 */
845 	*state = U8_STATE_START;
846 
847 	/* Try to find matching decomposition mapping byte sequence. */
848 	b1 = u8_common_b1_tbl[uv][b1];
849 	if (b1 == U8_TBL_ELEMENT_NOT_DEF)
850 		return ((size_t)sz);
851 
852 	b2 = u8_decomp_b2_tbl[uv][b1][b2];
853 	if (b2 == U8_TBL_ELEMENT_NOT_DEF)
854 		return ((size_t)sz);
855 
856 	b3_tbl = u8_decomp_b3_tbl[uv][b2][b3].tbl_id;
857 	if (b3_tbl == U8_TBL_ELEMENT_NOT_DEF)
858 		return ((size_t)sz);
859 
860 	/*
861 	 * If b3_tbl is bigger than or equal to U8_16BIT_TABLE_INDICATOR
862 	 * which is 0x8000, this means we couldn't fit the mappings into
863 	 * the cardinality of a unsigned byte.
864 	 */
865 	if (b3_tbl >= U8_16BIT_TABLE_INDICATOR) {
866 		b3_tbl -= U8_16BIT_TABLE_INDICATOR;
867 		start_id = u8_decomp_b4_16bit_tbl[uv][b3_tbl][b4];
868 		end_id = u8_decomp_b4_16bit_tbl[uv][b3_tbl][b4 + 1];
869 	} else {
870 		start_id = u8_decomp_b4_tbl[uv][b3_tbl][b4];
871 		end_id = u8_decomp_b4_tbl[uv][b3_tbl][b4 + 1];
872 	}
873 
874 	/* This also means there wasn't any matching decomposition. */
875 	if (start_id >= end_id)
876 		return ((size_t)sz);
877 
878 	/*
879 	 * The final table for decomposition mappings has three types of
880 	 * byte sequences depending on whether a mapping is for compatibility
881 	 * decomposition, canonical decomposition, or both like the following:
882 	 *
883 	 * (1) Compatibility decomposition mappings:
884 	 *
885 	 *	+---+---+-...-+---+
886 	 *	| B0| B1| ... | Bm|
887 	 *	+---+---+-...-+---+
888 	 *
889 	 *	The first byte, B0, is always less then 0xF5 (U8_DECOMP_BOTH).
890 	 *
891 	 * (2) Canonical decomposition mappings:
892 	 *
893 	 *	+---+---+---+-...-+---+
894 	 *	| T | b0| b1| ... | bn|
895 	 *	+---+---+---+-...-+---+
896 	 *
897 	 *	where the first byte, T, is 0xF6 (U8_DECOMP_CANONICAL).
898 	 *
899 	 * (3) Both mappings:
900 	 *
901 	 *	+---+---+---+---+-...-+---+---+---+-...-+---+
902 	 *	| T | D | b0| b1| ... | bn| B0| B1| ... | Bm|
903 	 *	+---+---+---+---+-...-+---+---+---+-...-+---+
904 	 *
905 	 *	where T is 0xF5 (U8_DECOMP_BOTH) and D is a displacement
906 	 *	byte, b0 to bn are canonical mapping bytes and B0 to Bm are
907 	 *	compatibility mapping bytes.
908 	 *
909 	 * Note that compatibility decomposition means doing recursive
910 	 * decompositions using both compatibility decomposition mappings and
911 	 * canonical decomposition mappings. On the other hand, canonical
912 	 * decomposition means doing recursive decompositions using only
913 	 * canonical decomposition mappings. Since the table we have has gone
914 	 * through the recursions already, we do not need to do so during
915 	 * runtime, i.e., the table has been completely flattened out
916 	 * already.
917 	 */
918 
919 	b3_base = u8_decomp_b3_tbl[uv][b2][b3].base;
920 
921 	/* Get the type, T, of the byte sequence. */
922 	b1 = u8_decomp_final_tbl[uv][b3_base + start_id];
923 
924 	/*
925 	 * If necessary, adjust start_id, end_id, or both. Note that if
926 	 * this is compatibility decomposition mapping, there is no
927 	 * adjustment.
928 	 */
929 	if (canonical_decomposition) {
930 		/* Is the mapping only for compatibility decomposition? */
931 		if (b1 < U8_DECOMP_BOTH)
932 			return ((size_t)sz);
933 
934 		start_id++;
935 
936 		if (b1 == U8_DECOMP_BOTH) {
937 			end_id = start_id +
938 			    u8_decomp_final_tbl[uv][b3_base + start_id];
939 			start_id++;
940 		}
941 	} else {
942 		/*
943 		 * Unless this is a compatibility decomposition mapping,
944 		 * we adjust the start_id.
945 		 */
946 		if (b1 == U8_DECOMP_BOTH) {
947 			start_id++;
948 			start_id += u8_decomp_final_tbl[uv][b3_base + start_id];
949 		} else if (b1 == U8_DECOMP_CANONICAL) {
950 			start_id++;
951 		}
952 	}
953 
954 	for (i = 0; start_id < end_id; start_id++)
955 		u8s[i++] = u8_decomp_final_tbl[uv][b3_base + start_id];
956 	u8s[i] = '\0';
957 
958 	return (i);
959 }
960 
961 /*
962  * The find_composition_start() function uses the character bytes given and
963  * find out the matching composition mappings if any and return the address
964  * to the composition mappings as explained in the do_composition().
965  */
966 static uchar_t *
find_composition_start(size_t uv,uchar_t * s,size_t sz)967 find_composition_start(size_t uv, uchar_t *s, size_t sz)
968 {
969 	uint16_t b1 = 0;
970 	uint16_t b2 = 0;
971 	uint16_t b3 = 0;
972 	uint16_t b3_tbl;
973 	uint16_t b3_base;
974 	uint16_t b4 = 0;
975 	size_t start_id;
976 	size_t end_id;
977 
978 	if (sz == 1) {
979 		b4 = s[0];
980 	} else if (sz == 2) {
981 		b3 = s[0];
982 		b4 = s[1];
983 	} else if (sz == 3) {
984 		b2 = s[0];
985 		b3 = s[1];
986 		b4 = s[2];
987 	} else if (sz == 4) {
988 		b1 = s[0];
989 		b2 = s[1];
990 		b3 = s[2];
991 		b4 = s[3];
992 	} else {
993 		/*
994 		 * This is a fallback and should not happen if the function
995 		 * was called properly.
996 		 */
997 		return (NULL);
998 	}
999 
1000 	b1 = u8_composition_b1_tbl[uv][b1];
1001 	if (b1 == U8_TBL_ELEMENT_NOT_DEF)
1002 		return (NULL);
1003 
1004 	b2 = u8_composition_b2_tbl[uv][b1][b2];
1005 	if (b2 == U8_TBL_ELEMENT_NOT_DEF)
1006 		return (NULL);
1007 
1008 	b3_tbl = u8_composition_b3_tbl[uv][b2][b3].tbl_id;
1009 	if (b3_tbl == U8_TBL_ELEMENT_NOT_DEF)
1010 		return (NULL);
1011 
1012 	if (b3_tbl >= U8_16BIT_TABLE_INDICATOR) {
1013 		b3_tbl -= U8_16BIT_TABLE_INDICATOR;
1014 		start_id = u8_composition_b4_16bit_tbl[uv][b3_tbl][b4];
1015 		end_id = u8_composition_b4_16bit_tbl[uv][b3_tbl][b4 + 1];
1016 	} else {
1017 		start_id = u8_composition_b4_tbl[uv][b3_tbl][b4];
1018 		end_id = u8_composition_b4_tbl[uv][b3_tbl][b4 + 1];
1019 	}
1020 
1021 	if (start_id >= end_id)
1022 		return (NULL);
1023 
1024 	b3_base = u8_composition_b3_tbl[uv][b2][b3].base;
1025 
1026 	return ((uchar_t *)&(u8_composition_final_tbl[uv][b3_base + start_id]));
1027 }
1028 
1029 /*
1030  * The blocked() function checks on the combining class values of previous
1031  * characters in this sequence and return whether it is blocked or not.
1032  */
1033 static boolean_t
blocked(uchar_t * comb_class,size_t last)1034 blocked(uchar_t *comb_class, size_t last)
1035 {
1036 	uchar_t my_comb_class;
1037 	size_t i;
1038 
1039 	my_comb_class = comb_class[last];
1040 	for (i = 1; i < last; i++)
1041 		if (comb_class[i] >= my_comb_class ||
1042 		    comb_class[i] == U8_COMBINING_CLASS_STARTER)
1043 			return (B_TRUE);
1044 
1045 	return (B_FALSE);
1046 }
1047 
1048 /*
1049  * The do_composition() reads the character string pointed by 's' and
1050  * do necessary canonical composition and then copy over the result back to
1051  * the 's'.
1052  *
1053  * The input argument 's' cannot contain more than 32 characters.
1054  */
1055 static size_t
do_composition(size_t uv,uchar_t * s,uchar_t * comb_class,uchar_t * start,uchar_t * disp,size_t last,uchar_t ** os,uchar_t * oslast)1056 do_composition(size_t uv, uchar_t *s, uchar_t *comb_class, uchar_t *start,
1057 	uchar_t *disp, size_t last, uchar_t **os, uchar_t *oslast)
1058 {
1059 	uchar_t t[U8_STREAM_SAFE_TEXT_MAX + 1];
1060 	uchar_t tc[U8_MB_CUR_MAX];
1061 	uint8_t saved_marks[U8_MAX_CHARS_A_SEQ];
1062 	size_t saved_marks_count;
1063 	uchar_t *p;
1064 	uchar_t *saved_p;
1065 	uchar_t *q;
1066 	size_t i;
1067 	size_t saved_i;
1068 	size_t j;
1069 	size_t k;
1070 	size_t l;
1071 	size_t C;
1072 	size_t saved_l;
1073 	size_t size;
1074 	uint32_t u1;
1075 	uint32_t u2;
1076 	boolean_t match_not_found = B_TRUE;
1077 
1078 	/*
1079 	 * This should never happen unless the callers are doing some strange
1080 	 * and unexpected things.
1081 	 *
1082 	 * The "last" is the index pointing to the last character not last + 1.
1083 	 */
1084 	if (last >= U8_MAX_CHARS_A_SEQ)
1085 		last = U8_UPPER_LIMIT_IN_A_SEQ;
1086 
1087 	for (i = l = 0; i <= last; i++) {
1088 		/*
1089 		 * The last or any non-Starters at the beginning, we don't
1090 		 * have any chance to do composition and so we just copy them
1091 		 * to the temporary buffer.
1092 		 */
1093 		if (i >= last || comb_class[i] != U8_COMBINING_CLASS_STARTER) {
1094 SAVE_THE_CHAR:
1095 			p = s + start[i];
1096 			size = disp[i];
1097 			for (k = 0; k < size; k++)
1098 				t[l++] = *p++;
1099 			continue;
1100 		}
1101 
1102 		/*
1103 		 * If this could be a start of Hangul Jamos, then, we try to
1104 		 * conjoin them.
1105 		 */
1106 		if (s[start[i]] == U8_HANGUL_JAMO_1ST_BYTE) {
1107 			U8_PUT_3BYTES_INTO_UTF32(u1, s[start[i]],
1108 			    s[start[i] + 1], s[start[i] + 2]);
1109 			U8_PUT_3BYTES_INTO_UTF32(u2, s[start[i] + 3],
1110 			    s[start[i] + 4], s[start[i] + 5]);
1111 
1112 			if (U8_HANGUL_JAMO_L(u1) && U8_HANGUL_JAMO_V(u2)) {
1113 				u1 -= U8_HANGUL_JAMO_L_FIRST;
1114 				u2 -= U8_HANGUL_JAMO_V_FIRST;
1115 				u1 = U8_HANGUL_SYL_FIRST +
1116 				    (u1 * U8_HANGUL_V_COUNT + u2) *
1117 				    U8_HANGUL_T_COUNT;
1118 
1119 				i += 2;
1120 				if (i <= last) {
1121 					U8_PUT_3BYTES_INTO_UTF32(u2,
1122 					    s[start[i]], s[start[i] + 1],
1123 					    s[start[i] + 2]);
1124 
1125 					if (U8_HANGUL_JAMO_T(u2)) {
1126 						u1 += u2 -
1127 						    U8_HANGUL_JAMO_T_FIRST;
1128 						i++;
1129 					}
1130 				}
1131 
1132 				U8_SAVE_HANGUL_AS_UTF8(t + l, 0, 1, 2, u1);
1133 				i--;
1134 				l += 3;
1135 				continue;
1136 			}
1137 		}
1138 
1139 		/*
1140 		 * Let's then find out if this Starter has composition
1141 		 * mapping.
1142 		 */
1143 		p = find_composition_start(uv, s + start[i], disp[i]);
1144 		if (p == NULL)
1145 			goto SAVE_THE_CHAR;
1146 
1147 		/*
1148 		 * We have a Starter with composition mapping and the next
1149 		 * character is a non-Starter. Let's try to find out if
1150 		 * we can do composition.
1151 		 */
1152 
1153 		saved_p = p;
1154 		saved_i = i;
1155 		saved_l = l;
1156 		saved_marks_count = 0;
1157 
1158 TRY_THE_NEXT_MARK:
1159 		q = s + start[++i];
1160 		size = disp[i];
1161 
1162 		/*
1163 		 * The next for() loop compares the non-Starter pointed by
1164 		 * 'q' with the possible (joinable) characters pointed by 'p'.
1165 		 *
1166 		 * The composition final table entry pointed by the 'p'
1167 		 * looks like the following:
1168 		 *
1169 		 * +---+---+---+-...-+---+---+---+---+-...-+---+---+
1170 		 * | C | b0| b2| ... | bn| F | B0| B1| ... | Bm| F |
1171 		 * +---+---+---+-...-+---+---+---+---+-...-+---+---+
1172 		 *
1173 		 * where C is the count byte indicating the number of
1174 		 * mapping pairs where each pair would be look like
1175 		 * (b0-bn F, B0-Bm F). The b0-bn are the bytes of the second
1176 		 * character of a canonical decomposition and the B0-Bm are
1177 		 * the bytes of a matching composite character. The F is
1178 		 * a filler byte after each character as the separator.
1179 		 */
1180 
1181 		match_not_found = B_TRUE;
1182 
1183 		for (C = *p++; C > 0; C--) {
1184 			for (k = 0; k < size; p++, k++)
1185 				if (*p != q[k])
1186 					break;
1187 
1188 			/* Have we found it? */
1189 			if (k >= size && *p == U8_TBL_ELEMENT_FILLER) {
1190 				match_not_found = B_FALSE;
1191 
1192 				l = saved_l;
1193 
1194 				while (*++p != U8_TBL_ELEMENT_FILLER)
1195 					t[l++] = *p;
1196 
1197 				break;
1198 			}
1199 
1200 			/* We didn't find; skip to the next pair. */
1201 			if (*p != U8_TBL_ELEMENT_FILLER)
1202 				while (*++p != U8_TBL_ELEMENT_FILLER)
1203 					;
1204 			while (*++p != U8_TBL_ELEMENT_FILLER)
1205 				;
1206 			p++;
1207 		}
1208 
1209 		/*
1210 		 * If there was no match, we will need to save the combining
1211 		 * mark for later appending. After that, if the next one
1212 		 * is a non-Starter and not blocked, then, we try once
1213 		 * again to do composition with the next non-Starter.
1214 		 *
1215 		 * If there was no match and this was a Starter, then,
1216 		 * this is a new start.
1217 		 *
1218 		 * If there was a match and a composition done and we have
1219 		 * more to check on, then, we retrieve a new composition final
1220 		 * table entry for the composite and then try to do the
1221 		 * composition again.
1222 		 */
1223 
1224 		if (match_not_found) {
1225 			if (comb_class[i] == U8_COMBINING_CLASS_STARTER) {
1226 				i--;
1227 				goto SAVE_THE_CHAR;
1228 			}
1229 
1230 			saved_marks[saved_marks_count++] = i;
1231 		}
1232 
1233 		if (saved_l == l) {
1234 			while (i < last) {
1235 				if (blocked(comb_class, i + 1))
1236 					saved_marks[saved_marks_count++] = ++i;
1237 				else
1238 					break;
1239 			}
1240 			if (i < last) {
1241 				p = saved_p;
1242 				goto TRY_THE_NEXT_MARK;
1243 			}
1244 		} else if (i < last) {
1245 			p = find_composition_start(uv, t + saved_l,
1246 			    l - saved_l);
1247 			if (p != NULL) {
1248 				saved_p = p;
1249 				goto TRY_THE_NEXT_MARK;
1250 			}
1251 		}
1252 
1253 		/*
1254 		 * There is no more composition possible.
1255 		 *
1256 		 * If there was no composition what so ever then we copy
1257 		 * over the original Starter and then append any non-Starters
1258 		 * remaining at the target string sequentially after that.
1259 		 */
1260 
1261 		if (saved_l == l) {
1262 			p = s + start[saved_i];
1263 			size = disp[saved_i];
1264 			for (j = 0; j < size; j++)
1265 				t[l++] = *p++;
1266 		}
1267 
1268 		for (k = 0; k < saved_marks_count; k++) {
1269 			p = s + start[saved_marks[k]];
1270 			size = disp[saved_marks[k]];
1271 			for (j = 0; j < size; j++)
1272 				t[l++] = *p++;
1273 		}
1274 	}
1275 
1276 	/*
1277 	 * If the last character is a Starter and if we have a character
1278 	 * (possibly another Starter) that can be turned into a composite,
1279 	 * we do so and we do so until there is no more of composition
1280 	 * possible.
1281 	 */
1282 	if (comb_class[last] == U8_COMBINING_CLASS_STARTER) {
1283 		p = *os;
1284 		saved_l = l - disp[last];
1285 
1286 		while (p < oslast) {
1287 			size = u8_number_of_bytes[*p];
1288 			if (size <= 1 || (p + size) > oslast)
1289 				break;
1290 
1291 			saved_p = p;
1292 
1293 			for (i = 0; i < size; i++)
1294 				tc[i] = *p++;
1295 
1296 			q = find_composition_start(uv, t + saved_l,
1297 			    l - saved_l);
1298 			if (q == NULL) {
1299 				p = saved_p;
1300 				break;
1301 			}
1302 
1303 			match_not_found = B_TRUE;
1304 
1305 			for (C = *q++; C > 0; C--) {
1306 				for (k = 0; k < size; q++, k++)
1307 					if (*q != tc[k])
1308 						break;
1309 
1310 				if (k >= size && *q == U8_TBL_ELEMENT_FILLER) {
1311 					match_not_found = B_FALSE;
1312 
1313 					l = saved_l;
1314 
1315 					while (*++q != U8_TBL_ELEMENT_FILLER) {
1316 						/*
1317 						 * This is practically
1318 						 * impossible but we don't
1319 						 * want to take any chances.
1320 						 */
1321 						if (l >=
1322 						    U8_STREAM_SAFE_TEXT_MAX) {
1323 							p = saved_p;
1324 							goto SAFE_RETURN;
1325 						}
1326 						t[l++] = *q;
1327 					}
1328 
1329 					break;
1330 				}
1331 
1332 				if (*q != U8_TBL_ELEMENT_FILLER)
1333 					while (*++q != U8_TBL_ELEMENT_FILLER)
1334 						;
1335 				while (*++q != U8_TBL_ELEMENT_FILLER)
1336 					;
1337 				q++;
1338 			}
1339 
1340 			if (match_not_found) {
1341 				p = saved_p;
1342 				break;
1343 			}
1344 		}
1345 SAFE_RETURN:
1346 		*os = p;
1347 	}
1348 
1349 	/*
1350 	 * Now we copy over the temporary string to the target string.
1351 	 * Since composition always reduces the number of characters or
1352 	 * the number of characters stay, we don't need to worry about
1353 	 * the buffer overflow here.
1354 	 */
1355 	for (i = 0; i < l; i++)
1356 		s[i] = t[i];
1357 	s[l] = '\0';
1358 
1359 	return (l);
1360 }
1361 
1362 /*
1363  * The collect_a_seq() function checks on the given string s, collect
1364  * a sequence of characters at u8s, and return the sequence. While it collects
1365  * a sequence, it also applies case conversion, canonical or compatibility
1366  * decomposition, canonical decomposition, or some or all of them and
1367  * in that order.
1368  *
1369  * The collected sequence cannot be bigger than 32 characters since if
1370  * it is having more than 31 characters, the sequence will be terminated
1371  * with a U+034F COMBINING GRAPHEME JOINER (CGJ) character and turned into
1372  * a Stream-Safe Text. The collected sequence is always terminated with
1373  * a null byte and the return value is the byte length of the sequence
1374  * including 0. The return value does not include the terminating
1375  * null byte.
1376  */
1377 static size_t
collect_a_seq(size_t uv,uchar_t * u8s,uchar_t ** source,uchar_t * slast,boolean_t is_it_toupper,boolean_t is_it_tolower,boolean_t canonical_decomposition,boolean_t compatibility_decomposition,boolean_t canonical_composition,int * errnum,u8_normalization_states_t * state)1378 collect_a_seq(size_t uv, uchar_t *u8s, uchar_t **source, uchar_t *slast,
1379 	boolean_t is_it_toupper,
1380 	boolean_t is_it_tolower,
1381 	boolean_t canonical_decomposition,
1382 	boolean_t compatibility_decomposition,
1383 	boolean_t canonical_composition,
1384 	int *errnum, u8_normalization_states_t *state)
1385 {
1386 	uchar_t *s;
1387 	int sz;
1388 	int saved_sz;
1389 	size_t i;
1390 	size_t j;
1391 	size_t k;
1392 	size_t l;
1393 	uchar_t comb_class[U8_MAX_CHARS_A_SEQ];
1394 	uchar_t disp[U8_MAX_CHARS_A_SEQ];
1395 	uchar_t start[U8_MAX_CHARS_A_SEQ];
1396 	uchar_t u8t[U8_MB_CUR_MAX];
1397 	uchar_t uts[U8_STREAM_SAFE_TEXT_MAX + 1];
1398 	uchar_t tc;
1399 	size_t last;
1400 	size_t saved_last;
1401 	uint32_t u1;
1402 
1403 	/*
1404 	 * Save the source string pointer which we will return a changed
1405 	 * pointer if we do processing.
1406 	 */
1407 	s = *source;
1408 
1409 	/*
1410 	 * The following is a fallback for just in case callers are not
1411 	 * checking the string boundaries before the calling.
1412 	 */
1413 	if (s >= slast) {
1414 		u8s[0] = '\0';
1415 
1416 		return (0);
1417 	}
1418 
1419 	/*
1420 	 * As the first thing, let's collect a character and do case
1421 	 * conversion if necessary.
1422 	 */
1423 
1424 	sz = u8_number_of_bytes[*s];
1425 
1426 	if (sz < 0) {
1427 		*errnum = EILSEQ;
1428 
1429 		u8s[0] = *s++;
1430 		u8s[1] = '\0';
1431 
1432 		*source = s;
1433 
1434 		return (1);
1435 	}
1436 
1437 	if (sz == 1) {
1438 		if (is_it_toupper)
1439 			u8s[0] = U8_ASCII_TOUPPER(*s);
1440 		else if (is_it_tolower)
1441 			u8s[0] = U8_ASCII_TOLOWER(*s);
1442 		else
1443 			u8s[0] = *s;
1444 		s++;
1445 		u8s[1] = '\0';
1446 	} else if ((s + sz) > slast) {
1447 		*errnum = EINVAL;
1448 
1449 		for (i = 0; s < slast; )
1450 			u8s[i++] = *s++;
1451 		u8s[i] = '\0';
1452 
1453 		*source = s;
1454 
1455 		return (i);
1456 	} else {
1457 		if (is_it_toupper || is_it_tolower) {
1458 			i = do_case_conv(uv, u8s, s, sz, is_it_toupper);
1459 			s += sz;
1460 			sz = i;
1461 		} else {
1462 			for (i = 0; i < sz; )
1463 				u8s[i++] = *s++;
1464 			u8s[i] = '\0';
1465 		}
1466 	}
1467 
1468 	/*
1469 	 * And then canonical/compatibility decomposition followed by
1470 	 * an optional canonical composition. Please be noted that
1471 	 * canonical composition is done only when a decomposition is
1472 	 * done.
1473 	 */
1474 	if (canonical_decomposition || compatibility_decomposition) {
1475 		if (sz == 1) {
1476 			*state = U8_STATE_START;
1477 
1478 			saved_sz = 1;
1479 
1480 			comb_class[0] = 0;
1481 			start[0] = 0;
1482 			disp[0] = 1;
1483 
1484 			last = 1;
1485 		} else {
1486 			saved_sz = do_decomp(uv, u8s, u8s, sz,
1487 			    canonical_decomposition, state);
1488 
1489 			last = 0;
1490 
1491 			for (i = 0; i < saved_sz; ) {
1492 				sz = u8_number_of_bytes[u8s[i]];
1493 
1494 				comb_class[last] = combining_class(uv,
1495 				    u8s + i, sz);
1496 				start[last] = i;
1497 				disp[last] = sz;
1498 
1499 				last++;
1500 				i += sz;
1501 			}
1502 
1503 			/*
1504 			 * Decomposition yields various Hangul related
1505 			 * states but not on combining marks. We need to
1506 			 * find out at here by checking on the last
1507 			 * character.
1508 			 */
1509 			if (*state == U8_STATE_START) {
1510 				if (comb_class[last - 1])
1511 					*state = U8_STATE_COMBINING_MARK;
1512 			}
1513 		}
1514 
1515 		saved_last = last;
1516 
1517 		while (s < slast) {
1518 			sz = u8_number_of_bytes[*s];
1519 
1520 			/*
1521 			 * If this is an illegal character, an incomplete
1522 			 * character, or an 7-bit ASCII Starter character,
1523 			 * then we have collected a sequence; break and let
1524 			 * the next call deal with the two cases.
1525 			 *
1526 			 * Note that this is okay only if you are using this
1527 			 * function with a fixed length string, not on
1528 			 * a buffer with multiple calls of one chunk at a time.
1529 			 */
1530 			if (sz <= 1) {
1531 				break;
1532 			} else if ((s + sz) > slast) {
1533 				break;
1534 			} else {
1535 				/*
1536 				 * If the previous character was a Hangul Jamo
1537 				 * and this character is a Hangul Jamo that
1538 				 * can be conjoined, we collect the Jamo.
1539 				 */
1540 				if (*s == U8_HANGUL_JAMO_1ST_BYTE) {
1541 					U8_PUT_3BYTES_INTO_UTF32(u1,
1542 					    *s, *(s + 1), *(s + 2));
1543 
1544 					if (U8_HANGUL_COMPOSABLE_L_V(*state,
1545 					    u1)) {
1546 						i = 0;
1547 						*state = U8_STATE_HANGUL_LV;
1548 						goto COLLECT_A_HANGUL;
1549 					}
1550 
1551 					if (U8_HANGUL_COMPOSABLE_LV_T(*state,
1552 					    u1)) {
1553 						i = 0;
1554 						*state = U8_STATE_HANGUL_LVT;
1555 						goto COLLECT_A_HANGUL;
1556 					}
1557 				}
1558 
1559 				/*
1560 				 * Regardless of whatever it was, if this is
1561 				 * a Starter, we don't collect the character
1562 				 * since that's a new start and we will deal
1563 				 * with it at the next time.
1564 				 */
1565 				i = combining_class(uv, s, sz);
1566 				if (i == U8_COMBINING_CLASS_STARTER)
1567 					break;
1568 
1569 				/*
1570 				 * We know the current character is a combining
1571 				 * mark. If the previous character wasn't
1572 				 * a Starter (not Hangul) or a combining mark,
1573 				 * then, we don't collect this combining mark.
1574 				 */
1575 				if (*state != U8_STATE_START &&
1576 				    *state != U8_STATE_COMBINING_MARK)
1577 					break;
1578 
1579 				*state = U8_STATE_COMBINING_MARK;
1580 COLLECT_A_HANGUL:
1581 				/*
1582 				 * If we collected a Starter and combining
1583 				 * marks up to 30, i.e., total 31 characters,
1584 				 * then, we terminate this degenerately long
1585 				 * combining sequence with a U+034F COMBINING
1586 				 * GRAPHEME JOINER (CGJ) which is 0xCD 0x8F in
1587 				 * UTF-8 and turn this into a Stream-Safe
1588 				 * Text. This will be extremely rare but
1589 				 * possible.
1590 				 *
1591 				 * The following will also guarantee that
1592 				 * we are not writing more than 32 characters
1593 				 * plus a NULL at u8s[].
1594 				 */
1595 				if (last >= U8_UPPER_LIMIT_IN_A_SEQ) {
1596 TURN_STREAM_SAFE:
1597 					*state = U8_STATE_START;
1598 					comb_class[last] = 0;
1599 					start[last] = saved_sz;
1600 					disp[last] = 2;
1601 					last++;
1602 
1603 					u8s[saved_sz++] = 0xCD;
1604 					u8s[saved_sz++] = 0x8F;
1605 
1606 					break;
1607 				}
1608 
1609 				/*
1610 				 * Some combining marks also do decompose into
1611 				 * another combining mark or marks.
1612 				 */
1613 				if (*state == U8_STATE_COMBINING_MARK) {
1614 					k = last;
1615 					l = sz;
1616 					i = do_decomp(uv, uts, s, sz,
1617 					    canonical_decomposition, state);
1618 					for (j = 0; j < i; ) {
1619 						sz = u8_number_of_bytes[uts[j]];
1620 
1621 						comb_class[last] =
1622 						    combining_class(uv,
1623 						    uts + j, sz);
1624 						start[last] = saved_sz + j;
1625 						disp[last] = sz;
1626 
1627 						last++;
1628 						if (last >=
1629 						    U8_UPPER_LIMIT_IN_A_SEQ) {
1630 							last = k;
1631 							goto TURN_STREAM_SAFE;
1632 						}
1633 						j += sz;
1634 					}
1635 
1636 					*state = U8_STATE_COMBINING_MARK;
1637 					sz = i;
1638 					s += l;
1639 
1640 					for (i = 0; i < sz; i++)
1641 						u8s[saved_sz++] = uts[i];
1642 				} else {
1643 					comb_class[last] = i;
1644 					start[last] = saved_sz;
1645 					disp[last] = sz;
1646 					last++;
1647 
1648 					for (i = 0; i < sz; i++)
1649 						u8s[saved_sz++] = *s++;
1650 				}
1651 
1652 				/*
1653 				 * If this is U+0345 COMBINING GREEK
1654 				 * YPOGEGRAMMENI (0xCD 0x85 in UTF-8), a.k.a.,
1655 				 * iota subscript, and need to be converted to
1656 				 * uppercase letter, convert it to U+0399 GREEK
1657 				 * CAPITAL LETTER IOTA (0xCE 0x99 in UTF-8),
1658 				 * i.e., convert to capital adscript form as
1659 				 * specified in the Unicode standard.
1660 				 *
1661 				 * This is the only special case of (ambiguous)
1662 				 * case conversion at combining marks and
1663 				 * probably the standard will never have
1664 				 * anything similar like this in future.
1665 				 */
1666 				if (is_it_toupper && sz >= 2 &&
1667 				    u8s[saved_sz - 2] == 0xCD &&
1668 				    u8s[saved_sz - 1] == 0x85) {
1669 					u8s[saved_sz - 2] = 0xCE;
1670 					u8s[saved_sz - 1] = 0x99;
1671 				}
1672 			}
1673 		}
1674 
1675 		/*
1676 		 * Let's try to ensure a canonical ordering for the collected
1677 		 * combining marks. We do this only if we have collected
1678 		 * at least one more non-Starter. (The decomposition mapping
1679 		 * data tables have fully (and recursively) expanded and
1680 		 * canonically ordered decompositions.)
1681 		 *
1682 		 * The U8_SWAP_COMB_MARKS() convenience macro has some
1683 		 * assumptions and we are meeting the assumptions.
1684 		 */
1685 		last--;
1686 		if (last >= saved_last) {
1687 			for (i = 0; i < last; i++)
1688 				for (j = last; j > i; j--)
1689 					if (comb_class[j] &&
1690 					    comb_class[j - 1] > comb_class[j]) {
1691 						U8_SWAP_COMB_MARKS(j - 1, j);
1692 					}
1693 		}
1694 
1695 		*source = s;
1696 
1697 		if (! canonical_composition) {
1698 			u8s[saved_sz] = '\0';
1699 			return (saved_sz);
1700 		}
1701 
1702 		/*
1703 		 * Now do the canonical composition. Note that we do this
1704 		 * only after a canonical or compatibility decomposition to
1705 		 * finish up NFC or NFKC.
1706 		 */
1707 		sz = do_composition(uv, u8s, comb_class, start, disp, last,
1708 		    &s, slast);
1709 	}
1710 
1711 	*source = s;
1712 
1713 	return ((size_t)sz);
1714 }
1715 
1716 /*
1717  * The do_norm_compare() function does string comparion based on Unicode
1718  * simple case mappings and Unicode Normalization definitions.
1719  *
1720  * It does so by collecting a sequence of character at a time and comparing
1721  * the collected sequences from the strings.
1722  *
1723  * The meanings on the return values are the same as the usual strcmp().
1724  */
1725 static int
do_norm_compare(size_t uv,uchar_t * s1,uchar_t * s2,size_t n1,size_t n2,int flag,int * errnum)1726 do_norm_compare(size_t uv, uchar_t *s1, uchar_t *s2, size_t n1, size_t n2,
1727 	int flag, int *errnum)
1728 {
1729 	int result;
1730 	size_t sz1;
1731 	size_t sz2;
1732 	uchar_t u8s1[U8_STREAM_SAFE_TEXT_MAX + 1];
1733 	uchar_t u8s2[U8_STREAM_SAFE_TEXT_MAX + 1];
1734 	uchar_t *s1last;
1735 	uchar_t *s2last;
1736 	boolean_t is_it_toupper;
1737 	boolean_t is_it_tolower;
1738 	boolean_t canonical_decomposition;
1739 	boolean_t compatibility_decomposition;
1740 	boolean_t canonical_composition;
1741 	u8_normalization_states_t state;
1742 
1743 	s1last = s1 + n1;
1744 	s2last = s2 + n2;
1745 
1746 	is_it_toupper = flag & U8_TEXTPREP_TOUPPER;
1747 	is_it_tolower = flag & U8_TEXTPREP_TOLOWER;
1748 	canonical_decomposition = flag & U8_CANON_DECOMP;
1749 	compatibility_decomposition = flag & U8_COMPAT_DECOMP;
1750 	canonical_composition = flag & U8_CANON_COMP;
1751 
1752 	while (s1 < s1last && s2 < s2last) {
1753 		/*
1754 		 * If the current character is a 7-bit ASCII and the last
1755 		 * character, or, if the current character and the next
1756 		 * character are both some 7-bit ASCII characters then
1757 		 * we treat the current character as a sequence.
1758 		 *
1759 		 * In any other cases, we need to call collect_a_seq().
1760 		 */
1761 
1762 		if (U8_ISASCII(*s1) && ((s1 + 1) >= s1last ||
1763 		    ((s1 + 1) < s1last && U8_ISASCII(*(s1 + 1))))) {
1764 			if (is_it_toupper)
1765 				u8s1[0] = U8_ASCII_TOUPPER(*s1);
1766 			else if (is_it_tolower)
1767 				u8s1[0] = U8_ASCII_TOLOWER(*s1);
1768 			else
1769 				u8s1[0] = *s1;
1770 			u8s1[1] = '\0';
1771 			sz1 = 1;
1772 			s1++;
1773 		} else {
1774 			state = U8_STATE_START;
1775 			sz1 = collect_a_seq(uv, u8s1, &s1, s1last,
1776 			    is_it_toupper, is_it_tolower,
1777 			    canonical_decomposition,
1778 			    compatibility_decomposition,
1779 			    canonical_composition, errnum, &state);
1780 		}
1781 
1782 		if (U8_ISASCII(*s2) && ((s2 + 1) >= s2last ||
1783 		    ((s2 + 1) < s2last && U8_ISASCII(*(s2 + 1))))) {
1784 			if (is_it_toupper)
1785 				u8s2[0] = U8_ASCII_TOUPPER(*s2);
1786 			else if (is_it_tolower)
1787 				u8s2[0] = U8_ASCII_TOLOWER(*s2);
1788 			else
1789 				u8s2[0] = *s2;
1790 			u8s2[1] = '\0';
1791 			sz2 = 1;
1792 			s2++;
1793 		} else {
1794 			state = U8_STATE_START;
1795 			sz2 = collect_a_seq(uv, u8s2, &s2, s2last,
1796 			    is_it_toupper, is_it_tolower,
1797 			    canonical_decomposition,
1798 			    compatibility_decomposition,
1799 			    canonical_composition, errnum, &state);
1800 		}
1801 
1802 		/*
1803 		 * Now compare the two characters. If they are the same,
1804 		 * we move on to the next character sequences.
1805 		 */
1806 		if (sz1 == 1 && sz2 == 1) {
1807 			if (*u8s1 > *u8s2)
1808 				return (1);
1809 			if (*u8s1 < *u8s2)
1810 				return (-1);
1811 		} else {
1812 			result = strcmp((const char *)u8s1, (const char *)u8s2);
1813 			if (result != 0)
1814 				return (result);
1815 		}
1816 	}
1817 
1818 	/*
1819 	 * We compared until the end of either or both strings.
1820 	 *
1821 	 * If we reached to or went over the ends for the both, that means
1822 	 * they are the same.
1823 	 *
1824 	 * If we reached only one end, that means the other string has
1825 	 * something which then can be used to determine the return value.
1826 	 */
1827 	if (s1 >= s1last) {
1828 		if (s2 >= s2last)
1829 			return (0);
1830 		return (-1);
1831 	}
1832 	return (1);
1833 }
1834 
1835 /*
1836  * The u8_strcmp() function compares two UTF-8 strings quite similar to
1837  * the strcmp(). For the comparison, however, Unicode Normalization specific
1838  * equivalency and Unicode simple case conversion mappings based equivalency
1839  * can be requested and checked against.
1840  */
1841 int
u8_strcmp(const char * s1,const char * s2,size_t n,int flag,size_t uv,int * errnum)1842 u8_strcmp(const char *s1, const char *s2, size_t n, int flag, size_t uv,
1843 		int *errnum)
1844 {
1845 	int f;
1846 	size_t n1;
1847 	size_t n2;
1848 
1849 	*errnum = 0;
1850 
1851 	/*
1852 	 * Check on the requested Unicode version, case conversion, and
1853 	 * normalization flag values.
1854 	 */
1855 
1856 	if (uv > U8_UNICODE_LATEST) {
1857 		*errnum = ERANGE;
1858 		uv = U8_UNICODE_LATEST;
1859 	}
1860 
1861 	if (flag == 0) {
1862 		flag = U8_STRCMP_CS;
1863 	} else {
1864 		f = flag & (U8_STRCMP_CS | U8_STRCMP_CI_UPPER |
1865 		    U8_STRCMP_CI_LOWER);
1866 		if (f == 0) {
1867 			flag |= U8_STRCMP_CS;
1868 		} else if (f != U8_STRCMP_CS && f != U8_STRCMP_CI_UPPER &&
1869 		    f != U8_STRCMP_CI_LOWER) {
1870 			*errnum = EBADF;
1871 			flag = U8_STRCMP_CS;
1872 		}
1873 
1874 		f = flag & (U8_CANON_DECOMP | U8_COMPAT_DECOMP | U8_CANON_COMP);
1875 		if (f && f != U8_STRCMP_NFD && f != U8_STRCMP_NFC &&
1876 		    f != U8_STRCMP_NFKD && f != U8_STRCMP_NFKC) {
1877 			*errnum = EBADF;
1878 			flag = U8_STRCMP_CS;
1879 		}
1880 	}
1881 
1882 	if (flag == U8_STRCMP_CS) {
1883 		return (n == 0 ? strcmp(s1, s2) : strncmp(s1, s2, n));
1884 	}
1885 
1886 	n1 = strlen(s1);
1887 	n2 = strlen(s2);
1888 	if (n != 0) {
1889 		if (n < n1)
1890 			n1 = n;
1891 		if (n < n2)
1892 			n2 = n;
1893 	}
1894 
1895 	/*
1896 	 * Simple case conversion can be done much faster and so we do
1897 	 * them separately here.
1898 	 */
1899 	if (flag == U8_STRCMP_CI_UPPER) {
1900 		return (do_case_compare(uv, (uchar_t *)s1, (uchar_t *)s2,
1901 		    n1, n2, B_TRUE, errnum));
1902 	} else if (flag == U8_STRCMP_CI_LOWER) {
1903 		return (do_case_compare(uv, (uchar_t *)s1, (uchar_t *)s2,
1904 		    n1, n2, B_FALSE, errnum));
1905 	}
1906 
1907 	return (do_norm_compare(uv, (uchar_t *)s1, (uchar_t *)s2, n1, n2,
1908 	    flag, errnum));
1909 }
1910 
1911 size_t
u8_textprep_str(char * inarray,size_t * inlen,char * outarray,size_t * outlen,int flag,size_t unicode_version,int * errnum)1912 u8_textprep_str(char *inarray, size_t *inlen, char *outarray, size_t *outlen,
1913 	int flag, size_t unicode_version, int *errnum)
1914 {
1915 	int f;
1916 	int sz;
1917 	uchar_t *ib;
1918 	uchar_t *ibtail;
1919 	uchar_t *ob;
1920 	uchar_t *obtail;
1921 	boolean_t do_not_ignore_null;
1922 	boolean_t do_not_ignore_invalid;
1923 	boolean_t is_it_toupper;
1924 	boolean_t is_it_tolower;
1925 	boolean_t canonical_decomposition;
1926 	boolean_t compatibility_decomposition;
1927 	boolean_t canonical_composition;
1928 	size_t ret_val;
1929 	size_t i;
1930 	size_t j;
1931 	uchar_t u8s[U8_STREAM_SAFE_TEXT_MAX + 1];
1932 	u8_normalization_states_t state;
1933 
1934 	if (unicode_version > U8_UNICODE_LATEST) {
1935 		*errnum = ERANGE;
1936 		return ((size_t)-1);
1937 	}
1938 
1939 	f = flag & (U8_TEXTPREP_TOUPPER | U8_TEXTPREP_TOLOWER);
1940 	if (f == (U8_TEXTPREP_TOUPPER | U8_TEXTPREP_TOLOWER)) {
1941 		*errnum = EBADF;
1942 		return ((size_t)-1);
1943 	}
1944 
1945 	f = flag & (U8_CANON_DECOMP | U8_COMPAT_DECOMP | U8_CANON_COMP);
1946 	if (f && f != U8_TEXTPREP_NFD && f != U8_TEXTPREP_NFC &&
1947 	    f != U8_TEXTPREP_NFKD && f != U8_TEXTPREP_NFKC) {
1948 		*errnum = EBADF;
1949 		return ((size_t)-1);
1950 	}
1951 
1952 	if (inarray == NULL || *inlen == 0)
1953 		return (0);
1954 
1955 	if (outarray == NULL) {
1956 		*errnum = E2BIG;
1957 		return ((size_t)-1);
1958 	}
1959 
1960 	ib = (uchar_t *)inarray;
1961 	ob = (uchar_t *)outarray;
1962 	ibtail = ib + *inlen;
1963 	obtail = ob + *outlen;
1964 
1965 	do_not_ignore_null = !(flag & U8_TEXTPREP_IGNORE_NULL);
1966 	do_not_ignore_invalid = !(flag & U8_TEXTPREP_IGNORE_INVALID);
1967 	is_it_toupper = flag & U8_TEXTPREP_TOUPPER;
1968 	is_it_tolower = flag & U8_TEXTPREP_TOLOWER;
1969 
1970 	ret_val = 0;
1971 
1972 	/*
1973 	 * If we don't have a normalization flag set, we do the simple case
1974 	 * conversion based text preparation separately below. Text
1975 	 * preparation involving Normalization will be done in the false task
1976 	 * block, again, separately since it will take much more time and
1977 	 * resource than doing simple case conversions.
1978 	 */
1979 	if (f == 0) {
1980 		while (ib < ibtail) {
1981 			if (*ib == '\0' && do_not_ignore_null)
1982 				break;
1983 
1984 			sz = u8_number_of_bytes[*ib];
1985 
1986 			if (sz < 0) {
1987 				if (do_not_ignore_invalid) {
1988 					*errnum = EILSEQ;
1989 					ret_val = (size_t)-1;
1990 					break;
1991 				}
1992 
1993 				sz = 1;
1994 				ret_val++;
1995 			}
1996 
1997 			if (sz == 1) {
1998 				if (ob >= obtail) {
1999 					*errnum = E2BIG;
2000 					ret_val = (size_t)-1;
2001 					break;
2002 				}
2003 
2004 				if (is_it_toupper)
2005 					*ob = U8_ASCII_TOUPPER(*ib);
2006 				else if (is_it_tolower)
2007 					*ob = U8_ASCII_TOLOWER(*ib);
2008 				else
2009 					*ob = *ib;
2010 				ib++;
2011 				ob++;
2012 			} else if ((ib + sz) > ibtail) {
2013 				if (do_not_ignore_invalid) {
2014 					*errnum = EINVAL;
2015 					ret_val = (size_t)-1;
2016 					break;
2017 				}
2018 
2019 				if ((obtail - ob) < (ibtail - ib)) {
2020 					*errnum = E2BIG;
2021 					ret_val = (size_t)-1;
2022 					break;
2023 				}
2024 
2025 				/*
2026 				 * We treat the remaining incomplete character
2027 				 * bytes as a character.
2028 				 */
2029 				ret_val++;
2030 
2031 				while (ib < ibtail)
2032 					*ob++ = *ib++;
2033 			} else {
2034 				if (is_it_toupper || is_it_tolower) {
2035 					i = do_case_conv(unicode_version, u8s,
2036 					    ib, sz, is_it_toupper);
2037 
2038 					if ((obtail - ob) < i) {
2039 						*errnum = E2BIG;
2040 						ret_val = (size_t)-1;
2041 						break;
2042 					}
2043 
2044 					ib += sz;
2045 
2046 					for (sz = 0; sz < i; sz++)
2047 						*ob++ = u8s[sz];
2048 				} else {
2049 					if ((obtail - ob) < sz) {
2050 						*errnum = E2BIG;
2051 						ret_val = (size_t)-1;
2052 						break;
2053 					}
2054 
2055 					for (i = 0; i < sz; i++)
2056 						*ob++ = *ib++;
2057 				}
2058 			}
2059 		}
2060 	} else {
2061 		canonical_decomposition = flag & U8_CANON_DECOMP;
2062 		compatibility_decomposition = flag & U8_COMPAT_DECOMP;
2063 		canonical_composition = flag & U8_CANON_COMP;
2064 
2065 		while (ib < ibtail) {
2066 			if (*ib == '\0' && do_not_ignore_null)
2067 				break;
2068 
2069 			/*
2070 			 * If the current character is a 7-bit ASCII
2071 			 * character and it is the last character, or,
2072 			 * if the current character is a 7-bit ASCII
2073 			 * character and the next character is also a 7-bit
2074 			 * ASCII character, then, we copy over this
2075 			 * character without going through collect_a_seq().
2076 			 *
2077 			 * In any other cases, we need to look further with
2078 			 * the collect_a_seq() function.
2079 			 */
2080 			if (U8_ISASCII(*ib) && ((ib + 1) >= ibtail ||
2081 			    ((ib + 1) < ibtail && U8_ISASCII(*(ib + 1))))) {
2082 				if (ob >= obtail) {
2083 					*errnum = E2BIG;
2084 					ret_val = (size_t)-1;
2085 					break;
2086 				}
2087 
2088 				if (is_it_toupper)
2089 					*ob = U8_ASCII_TOUPPER(*ib);
2090 				else if (is_it_tolower)
2091 					*ob = U8_ASCII_TOLOWER(*ib);
2092 				else
2093 					*ob = *ib;
2094 				ib++;
2095 				ob++;
2096 			} else {
2097 				*errnum = 0;
2098 				state = U8_STATE_START;
2099 
2100 				j = collect_a_seq(unicode_version, u8s,
2101 				    &ib, ibtail,
2102 				    is_it_toupper,
2103 				    is_it_tolower,
2104 				    canonical_decomposition,
2105 				    compatibility_decomposition,
2106 				    canonical_composition,
2107 				    errnum, &state);
2108 
2109 				if (*errnum && do_not_ignore_invalid) {
2110 					ret_val = (size_t)-1;
2111 					break;
2112 				}
2113 
2114 				if ((obtail - ob) < j) {
2115 					*errnum = E2BIG;
2116 					ret_val = (size_t)-1;
2117 					break;
2118 				}
2119 
2120 				for (i = 0; i < j; i++)
2121 					*ob++ = u8s[i];
2122 			}
2123 		}
2124 	}
2125 
2126 	*inlen = ibtail - ib;
2127 	*outlen = obtail - ob;
2128 
2129 	return (ret_val);
2130 }
2131