1 /* $NetBSD: lutf8lib.c,v 1.2 2015/02/02 14:03:05 lneto Exp $ */ 2 3 /* 4 ** Id: lutf8lib.c,v 1.13 2014/11/02 19:19:04 roberto Exp 5 ** Standard library for UTF-8 manipulation 6 ** See Copyright Notice in lua.h 7 */ 8 9 #define lutf8lib_c 10 #define LUA_LIB 11 12 #include "lprefix.h" 13 14 15 #ifndef _KERNEL 16 #include <assert.h> 17 #include <stdlib.h> 18 #include <string.h> 19 #endif 20 21 #include "lua.h" 22 23 #include "lauxlib.h" 24 #include "lualib.h" 25 26 #define MAXUNICODE 0x10FFFF 27 28 #define iscont(p) ((*(p) & 0xC0) == 0x80) 29 30 31 /* from strlib */ 32 /* translate a relative string position: negative means back from end */ 33 static lua_Integer u_posrelat (lua_Integer pos, size_t len) { 34 if (pos >= 0) return pos; 35 else if (0u - (size_t)pos > len) return 0; 36 else return (lua_Integer)len + pos + 1; 37 } 38 39 40 /* 41 ** Decode one UTF-8 sequence, returning NULL if byte sequence is invalid. 42 */ 43 static const char *utf8_decode (const char *o, int *val) { 44 static unsigned int limits[] = {0xFF, 0x7F, 0x7FF, 0xFFFF}; 45 const unsigned char *s = (const unsigned char *)o; 46 unsigned int c = s[0]; 47 unsigned int res = 0; /* final result */ 48 if (c < 0x80) /* ascii? */ 49 res = c; 50 else { 51 int count = 0; /* to count number of continuation bytes */ 52 while (c & 0x40) { /* still have continuation bytes? */ 53 int cc = s[++count]; /* read next byte */ 54 if ((cc & 0xC0) != 0x80) /* not a continuation byte? */ 55 return NULL; /* invalid byte sequence */ 56 res = (res << 6) | (cc & 0x3F); /* add lower 6 bits from cont. byte */ 57 c <<= 1; /* to test next bit */ 58 } 59 res |= ((c & 0x7F) << (count * 5)); /* add first byte */ 60 if (count > 3 || res > MAXUNICODE || res <= limits[count]) 61 return NULL; /* invalid byte sequence */ 62 s += count; /* skip continuation bytes read */ 63 } 64 if (val) *val = res; 65 return (const char *)s + 1; /* +1 to include first byte */ 66 } 67 68 69 /* 70 ** utf8len(s [, i [, j]]) --> number of characters that start in the 71 ** range [i,j], or nil + current position if 's' is not well formed in 72 ** that interval 73 */ 74 static int utflen (lua_State *L) { 75 int n = 0; 76 size_t len; 77 const char *s = luaL_checklstring(L, 1, &len); 78 lua_Integer posi = u_posrelat(luaL_optinteger(L, 2, 1), len); 79 lua_Integer posj = u_posrelat(luaL_optinteger(L, 3, -1), len); 80 luaL_argcheck(L, 1 <= posi && --posi <= (lua_Integer)len, 2, 81 "initial position out of string"); 82 luaL_argcheck(L, --posj < (lua_Integer)len, 3, 83 "final position out of string"); 84 while (posi <= posj) { 85 const char *s1 = utf8_decode(s + posi, NULL); 86 if (s1 == NULL) { /* conversion error? */ 87 lua_pushnil(L); /* return nil ... */ 88 lua_pushinteger(L, posi + 1); /* ... and current position */ 89 return 2; 90 } 91 posi = s1 - s; 92 n++; 93 } 94 lua_pushinteger(L, n); 95 return 1; 96 } 97 98 99 /* 100 ** codepoint(s, [i, [j]]) -> returns codepoints for all characters 101 ** that start in the range [i,j] 102 */ 103 static int codepoint (lua_State *L) { 104 size_t len; 105 const char *s = luaL_checklstring(L, 1, &len); 106 lua_Integer posi = u_posrelat(luaL_optinteger(L, 2, 1), len); 107 lua_Integer pose = u_posrelat(luaL_optinteger(L, 3, posi), len); 108 int n; 109 const char *se; 110 luaL_argcheck(L, posi >= 1, 2, "out of range"); 111 luaL_argcheck(L, pose <= (lua_Integer)len, 3, "out of range"); 112 if (posi > pose) return 0; /* empty interval; return no values */ 113 n = (int)(pose - posi + 1); 114 if (posi + n <= pose) /* (lua_Integer -> int) overflow? */ 115 return luaL_error(L, "string slice too long"); 116 luaL_checkstack(L, n, "string slice too long"); 117 n = 0; 118 se = s + pose; 119 for (s += posi - 1; s < se;) { 120 int code; 121 s = utf8_decode(s, &code); 122 if (s == NULL) 123 return luaL_error(L, "invalid UTF-8 code"); 124 lua_pushinteger(L, code); 125 n++; 126 } 127 return n; 128 } 129 130 131 static void pushutfchar (lua_State *L, int arg) { 132 lua_Integer code = luaL_checkinteger(L, arg); 133 luaL_argcheck(L, 0 <= code && code <= MAXUNICODE, arg, "value out of range"); 134 lua_pushfstring(L, "%U", (long)code); 135 } 136 137 138 /* 139 ** utfchar(n1, n2, ...) -> char(n1)..char(n2)... 140 */ 141 static int utfchar (lua_State *L) { 142 int n = lua_gettop(L); /* number of arguments */ 143 if (n == 1) /* optimize common case of single char */ 144 pushutfchar(L, 1); 145 else { 146 int i; 147 luaL_Buffer b; 148 luaL_buffinit(L, &b); 149 for (i = 1; i <= n; i++) { 150 pushutfchar(L, i); 151 luaL_addvalue(&b); 152 } 153 luaL_pushresult(&b); 154 } 155 return 1; 156 } 157 158 159 /* 160 ** offset(s, n, [i]) -> index where n-th character counting from 161 ** position 'i' starts; 0 means character at 'i'. 162 */ 163 static int byteoffset (lua_State *L) { 164 size_t len; 165 const char *s = luaL_checklstring(L, 1, &len); 166 lua_Integer n = luaL_checkinteger(L, 2); 167 lua_Integer posi = (n >= 0) ? 1 : len + 1; 168 posi = u_posrelat(luaL_optinteger(L, 3, posi), len); 169 luaL_argcheck(L, 1 <= posi && --posi <= (lua_Integer)len, 3, 170 "position out of range"); 171 if (n == 0) { 172 /* find beginning of current byte sequence */ 173 while (posi > 0 && iscont(s + posi)) posi--; 174 } 175 else { 176 if (iscont(s + posi)) 177 luaL_error(L, "initial position is a continuation byte"); 178 if (n < 0) { 179 while (n < 0 && posi > 0) { /* move back */ 180 do { /* find beginning of previous character */ 181 posi--; 182 } while (posi > 0 && iscont(s + posi)); 183 n++; 184 } 185 } 186 else { 187 n--; /* do not move for 1st character */ 188 while (n > 0 && posi < (lua_Integer)len) { 189 do { /* find beginning of next character */ 190 posi++; 191 } while (iscont(s + posi)); /* (cannot pass final '\0') */ 192 n--; 193 } 194 } 195 } 196 if (n == 0) /* did it find given character? */ 197 lua_pushinteger(L, posi + 1); 198 else /* no such character */ 199 lua_pushnil(L); 200 return 1; 201 } 202 203 204 static int iter_aux (lua_State *L) { 205 size_t len; 206 const char *s = luaL_checklstring(L, 1, &len); 207 lua_Integer n = lua_tointeger(L, 2) - 1; 208 if (n < 0) /* first iteration? */ 209 n = 0; /* start from here */ 210 else if (n < (lua_Integer)len) { 211 n++; /* skip current byte */ 212 while (iscont(s + n)) n++; /* and its continuations */ 213 } 214 if (n >= (lua_Integer)len) 215 return 0; /* no more codepoints */ 216 else { 217 int code; 218 const char *next = utf8_decode(s + n, &code); 219 if (next == NULL || iscont(next)) 220 return luaL_error(L, "invalid UTF-8 code"); 221 lua_pushinteger(L, n + 1); 222 lua_pushinteger(L, code); 223 return 2; 224 } 225 } 226 227 228 static int iter_codes (lua_State *L) { 229 luaL_checkstring(L, 1); 230 lua_pushcfunction(L, iter_aux); 231 lua_pushvalue(L, 1); 232 lua_pushinteger(L, 0); 233 return 3; 234 } 235 236 237 /* pattern to match a single UTF-8 character */ 238 #define UTF8PATT "[\0-\x7F\xC2-\xF4][\x80-\xBF]*" 239 240 241 static struct luaL_Reg funcs[] = { 242 {"offset", byteoffset}, 243 {"codepoint", codepoint}, 244 {"char", utfchar}, 245 {"len", utflen}, 246 {"codes", iter_codes}, 247 /* placeholders */ 248 {"charpattern", NULL}, 249 {NULL, NULL} 250 }; 251 252 253 LUAMOD_API int luaopen_utf8 (lua_State *L) { 254 luaL_newlib(L, funcs); 255 lua_pushliteral(L, UTF8PATT); 256 lua_setfield(L, -2, "charpattern"); 257 return 1; 258 } 259 260