1 /* $NetBSD: tokenizer.c,v 1.21 2011/08/16 16:25:15 christos Exp $ */
2
3 /*-
4 * Copyright (c) 1992, 1993
5 * The Regents of the University of California. All rights reserved.
6 *
7 * This code is derived from software contributed to Berkeley by
8 * Christos Zoulas of Cornell University.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 * 3. Neither the name of the University nor the names of its contributors
19 * may be used to endorse or promote products derived from this software
20 * without specific prior written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 */
34
35 #include "config.h"
36 #if !defined(lint) && !defined(SCCSID)
37 #if 0
38 static char sccsid[] = "@(#)tokenizer.c 8.1 (Berkeley) 6/4/93";
39 #else
40 __RCSID("$NetBSD: tokenizer.c,v 1.21 2011/08/16 16:25:15 christos Exp $");
41 #endif
42 #endif /* not lint && not SCCSID */
43
44 /* We build this file twice, once as NARROW, once as WIDE. */
45 /*
46 * tokenize.c: Bourne shell like tokenizer
47 */
48 #include <string.h>
49 #include <stdlib.h>
50 #include "histedit.h"
51 #include "chartype.h"
52
53 typedef enum {
54 Q_none, Q_single, Q_double, Q_one, Q_doubleone
55 } quote_t;
56
57 #define TOK_KEEP 1
58 #define TOK_EAT 2
59
60 #define WINCR 20
61 #define AINCR 10
62
63 #define IFS STR("\t \n")
64
65 #define tok_malloc(a) malloc(a)
66 #define tok_free(a) free(a)
67 #define tok_realloc(a, b) realloc(a, b)
68 #define tok_strdup(a) Strdup(a)
69
70
TYPE(tokenizer)71 struct TYPE(tokenizer) {
72 Char *ifs; /* In field separator */
73 size_t argc, amax; /* Current and maximum number of args */
74 Char **argv; /* Argument list */
75 Char *wptr, *wmax; /* Space and limit on the word buffer */
76 Char *wstart; /* Beginning of next word */
77 Char *wspace; /* Space of word buffer */
78 quote_t quote; /* Quoting state */
79 int flags; /* flags; */
80 };
81
82
83 private void FUN(tok,finish)(TYPE(Tokenizer) *);
84
85
86 /* FUN(tok,finish)():
87 * Finish a word in the tokenizer.
88 */
89 private void
FUN(tok,finish)90 FUN(tok,finish)(TYPE(Tokenizer) *tok)
91 {
92
93 *tok->wptr = '\0';
94 if ((tok->flags & TOK_KEEP) || tok->wptr != tok->wstart) {
95 tok->argv[tok->argc++] = tok->wstart;
96 tok->argv[tok->argc] = NULL;
97 tok->wstart = ++tok->wptr;
98 }
99 tok->flags &= ~TOK_KEEP;
100 }
101
102
103 /* FUN(tok,init)():
104 * Initialize the tokenizer
105 */
TYPE(Tokenizer)106 public TYPE(Tokenizer) *
107 FUN(tok,init)(const Char *ifs)
108 {
109 TYPE(Tokenizer) *tok = tok_malloc(sizeof(*tok));
110
111 if (tok == NULL)
112 return NULL;
113 tok->ifs = tok_strdup(ifs ? ifs : IFS);
114 if (tok->ifs == NULL) {
115 tok_free(tok);
116 return NULL;
117 }
118 tok->argc = 0;
119 tok->amax = AINCR;
120 tok->argv = tok_malloc(sizeof(*tok->argv) * tok->amax);
121 if (tok->argv == NULL) {
122 tok_free(tok->ifs);
123 tok_free(tok);
124 return NULL;
125 }
126 tok->argv[0] = NULL;
127 tok->wspace = tok_malloc(WINCR * sizeof(*tok->wspace));
128 if (tok->wspace == NULL) {
129 tok_free(tok->argv);
130 tok_free(tok->ifs);
131 tok_free(tok);
132 return NULL;
133 }
134 tok->wmax = tok->wspace + WINCR;
135 tok->wstart = tok->wspace;
136 tok->wptr = tok->wspace;
137 tok->flags = 0;
138 tok->quote = Q_none;
139
140 return tok;
141 }
142
143
144 /* FUN(tok,reset)():
145 * Reset the tokenizer
146 */
147 public void
FUN(tok,reset)148 FUN(tok,reset)(TYPE(Tokenizer) *tok)
149 {
150
151 tok->argc = 0;
152 tok->wstart = tok->wspace;
153 tok->wptr = tok->wspace;
154 tok->flags = 0;
155 tok->quote = Q_none;
156 }
157
158
159 /* FUN(tok,end)():
160 * Clean up
161 */
162 public void
FUN(tok,end)163 FUN(tok,end)(TYPE(Tokenizer) *tok)
164 {
165
166 tok_free(tok->ifs);
167 tok_free(tok->wspace);
168 tok_free(tok->argv);
169 tok_free(tok);
170 }
171
172
173
174 /* FUN(tok,line)():
175 * Bourne shell (sh(1)) like tokenizing
176 * Arguments:
177 * tok current tokenizer state (setup with FUN(tok,init)())
178 * line line to parse
179 * Returns:
180 * -1 Internal error
181 * 3 Quoted return
182 * 2 Unmatched double quote
183 * 1 Unmatched single quote
184 * 0 Ok
185 * Modifies (if return value is 0):
186 * argc number of arguments
187 * argv argument array
188 * cursorc if !NULL, argv element containing cursor
189 * cursorv if !NULL, offset in argv[cursorc] of cursor
190 */
191 public int
FUN(tok,line)192 FUN(tok,line)(TYPE(Tokenizer) *tok, const TYPE(LineInfo) *line,
193 int *argc, const Char ***argv, int *cursorc, int *cursoro)
194 {
195 const Char *ptr;
196 int cc, co;
197
198 cc = co = -1;
199 ptr = line->buffer;
200 for (ptr = line->buffer; ;ptr++) {
201 if (ptr >= line->lastchar)
202 ptr = STR("");
203 if (ptr == line->cursor) {
204 cc = (int)tok->argc;
205 co = (int)(tok->wptr - tok->wstart);
206 }
207 switch (*ptr) {
208 case '\'':
209 tok->flags |= TOK_KEEP;
210 tok->flags &= ~TOK_EAT;
211 switch (tok->quote) {
212 case Q_none:
213 tok->quote = Q_single; /* Enter single quote
214 * mode */
215 break;
216
217 case Q_single: /* Exit single quote mode */
218 tok->quote = Q_none;
219 break;
220
221 case Q_one: /* Quote this ' */
222 tok->quote = Q_none;
223 *tok->wptr++ = *ptr;
224 break;
225
226 case Q_double: /* Stay in double quote mode */
227 *tok->wptr++ = *ptr;
228 break;
229
230 case Q_doubleone: /* Quote this ' */
231 tok->quote = Q_double;
232 *tok->wptr++ = *ptr;
233 break;
234
235 default:
236 return -1;
237 }
238 break;
239
240 case '"':
241 tok->flags &= ~TOK_EAT;
242 tok->flags |= TOK_KEEP;
243 switch (tok->quote) {
244 case Q_none: /* Enter double quote mode */
245 tok->quote = Q_double;
246 break;
247
248 case Q_double: /* Exit double quote mode */
249 tok->quote = Q_none;
250 break;
251
252 case Q_one: /* Quote this " */
253 tok->quote = Q_none;
254 *tok->wptr++ = *ptr;
255 break;
256
257 case Q_single: /* Stay in single quote mode */
258 *tok->wptr++ = *ptr;
259 break;
260
261 case Q_doubleone: /* Quote this " */
262 tok->quote = Q_double;
263 *tok->wptr++ = *ptr;
264 break;
265
266 default:
267 return -1;
268 }
269 break;
270
271 case '\\':
272 tok->flags |= TOK_KEEP;
273 tok->flags &= ~TOK_EAT;
274 switch (tok->quote) {
275 case Q_none: /* Quote next character */
276 tok->quote = Q_one;
277 break;
278
279 case Q_double: /* Quote next character */
280 tok->quote = Q_doubleone;
281 break;
282
283 case Q_one: /* Quote this, restore state */
284 *tok->wptr++ = *ptr;
285 tok->quote = Q_none;
286 break;
287
288 case Q_single: /* Stay in single quote mode */
289 *tok->wptr++ = *ptr;
290 break;
291
292 case Q_doubleone: /* Quote this \ */
293 tok->quote = Q_double;
294 *tok->wptr++ = *ptr;
295 break;
296
297 default:
298 return -1;
299 }
300 break;
301
302 case '\n':
303 tok->flags &= ~TOK_EAT;
304 switch (tok->quote) {
305 case Q_none:
306 goto tok_line_outok;
307
308 case Q_single:
309 case Q_double:
310 *tok->wptr++ = *ptr; /* Add the return */
311 break;
312
313 case Q_doubleone: /* Back to double, eat the '\n' */
314 tok->flags |= TOK_EAT;
315 tok->quote = Q_double;
316 break;
317
318 case Q_one: /* No quote, more eat the '\n' */
319 tok->flags |= TOK_EAT;
320 tok->quote = Q_none;
321 break;
322
323 default:
324 return 0;
325 }
326 break;
327
328 case '\0':
329 switch (tok->quote) {
330 case Q_none:
331 /* Finish word and return */
332 if (tok->flags & TOK_EAT) {
333 tok->flags &= ~TOK_EAT;
334 return 3;
335 }
336 goto tok_line_outok;
337
338 case Q_single:
339 return 1;
340
341 case Q_double:
342 return 2;
343
344 case Q_doubleone:
345 tok->quote = Q_double;
346 *tok->wptr++ = *ptr;
347 break;
348
349 case Q_one:
350 tok->quote = Q_none;
351 *tok->wptr++ = *ptr;
352 break;
353
354 default:
355 return -1;
356 }
357 break;
358
359 default:
360 tok->flags &= ~TOK_EAT;
361 switch (tok->quote) {
362 case Q_none:
363 if (Strchr(tok->ifs, *ptr) != NULL)
364 FUN(tok,finish)(tok);
365 else
366 *tok->wptr++ = *ptr;
367 break;
368
369 case Q_single:
370 case Q_double:
371 *tok->wptr++ = *ptr;
372 break;
373
374
375 case Q_doubleone:
376 *tok->wptr++ = '\\';
377 tok->quote = Q_double;
378 *tok->wptr++ = *ptr;
379 break;
380
381 case Q_one:
382 tok->quote = Q_none;
383 *tok->wptr++ = *ptr;
384 break;
385
386 default:
387 return -1;
388
389 }
390 break;
391 }
392
393 if (tok->wptr >= tok->wmax - 4) {
394 size_t size = (size_t)(tok->wmax - tok->wspace + WINCR);
395 Char *s = tok_realloc(tok->wspace,
396 size * sizeof(*s));
397 if (s == NULL)
398 return -1;
399
400 if (s != tok->wspace) {
401 size_t i;
402 for (i = 0; i < tok->argc; i++) {
403 tok->argv[i] =
404 (tok->argv[i] - tok->wspace) + s;
405 }
406 tok->wptr = (tok->wptr - tok->wspace) + s;
407 tok->wstart = (tok->wstart - tok->wspace) + s;
408 tok->wspace = s;
409 }
410 tok->wmax = s + size;
411 }
412 if (tok->argc >= tok->amax - 4) {
413 Char **p;
414 tok->amax += AINCR;
415 p = tok_realloc(tok->argv, tok->amax * sizeof(*p));
416 if (p == NULL)
417 return -1;
418 tok->argv = p;
419 }
420 }
421 tok_line_outok:
422 if (cc == -1 && co == -1) {
423 cc = (int)tok->argc;
424 co = (int)(tok->wptr - tok->wstart);
425 }
426 if (cursorc != NULL)
427 *cursorc = cc;
428 if (cursoro != NULL)
429 *cursoro = co;
430 FUN(tok,finish)(tok);
431 *argv = (const Char **)tok->argv;
432 *argc = (int)tok->argc;
433 return 0;
434 }
435
436 /* FUN(tok,str)():
437 * Simpler version of tok_line, taking a NUL terminated line
438 * and splitting into words, ignoring cursor state.
439 */
440 public int
FUN(tok,str)441 FUN(tok,str)(TYPE(Tokenizer) *tok, const Char *line, int *argc,
442 const Char ***argv)
443 {
444 TYPE(LineInfo) li;
445
446 memset(&li, 0, sizeof(li));
447 li.buffer = line;
448 li.cursor = li.lastchar = Strchr(line, '\0');
449 return FUN(tok,line(tok, &li, argc, argv, NULL, NULL));
450 }
451