1*f3bd4955Schristos /* $NetBSD: tokenizer.c,v 1.29 2023/05/30 11:53:40 christos Exp $ */
22543e3e6Slukem
36dc2f1dbScgd /*-
46dc2f1dbScgd * Copyright (c) 1992, 1993
56dc2f1dbScgd * The Regents of the University of California. All rights reserved.
66dc2f1dbScgd *
76dc2f1dbScgd * This code is derived from software contributed to Berkeley by
86dc2f1dbScgd * Christos Zoulas of Cornell University.
96dc2f1dbScgd *
106dc2f1dbScgd * Redistribution and use in source and binary forms, with or without
116dc2f1dbScgd * modification, are permitted provided that the following conditions
126dc2f1dbScgd * are met:
136dc2f1dbScgd * 1. Redistributions of source code must retain the above copyright
146dc2f1dbScgd * notice, this list of conditions and the following disclaimer.
156dc2f1dbScgd * 2. Redistributions in binary form must reproduce the above copyright
166dc2f1dbScgd * notice, this list of conditions and the following disclaimer in the
176dc2f1dbScgd * documentation and/or other materials provided with the distribution.
18eb7c1594Sagc * 3. Neither the name of the University nor the names of its contributors
196dc2f1dbScgd * may be used to endorse or promote products derived from this software
206dc2f1dbScgd * without specific prior written permission.
216dc2f1dbScgd *
226dc2f1dbScgd * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
236dc2f1dbScgd * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
246dc2f1dbScgd * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
256dc2f1dbScgd * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
266dc2f1dbScgd * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
276dc2f1dbScgd * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
286dc2f1dbScgd * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
296dc2f1dbScgd * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
306dc2f1dbScgd * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
316dc2f1dbScgd * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
326dc2f1dbScgd * SUCH DAMAGE.
336dc2f1dbScgd */
346dc2f1dbScgd
350e0ac6b7Schristos #include "config.h"
366dc2f1dbScgd #if !defined(lint) && !defined(SCCSID)
372543e3e6Slukem #if 0
386dc2f1dbScgd static char sccsid[] = "@(#)tokenizer.c 8.1 (Berkeley) 6/4/93";
392543e3e6Slukem #else
40*f3bd4955Schristos __RCSID("$NetBSD: tokenizer.c,v 1.29 2023/05/30 11:53:40 christos Exp $");
412543e3e6Slukem #endif
426dc2f1dbScgd #endif /* not lint && not SCCSID */
436dc2f1dbScgd
4434e53048Schristos /* We build this file twice, once as NARROW, once as WIDE. */
456dc2f1dbScgd /*
466dc2f1dbScgd * tokenize.c: Bourne shell like tokenizer
476dc2f1dbScgd */
486dc2f1dbScgd #include <stdlib.h>
4922383670Schristos #include <string.h>
5022383670Schristos
51a9beb0e4Slukem #include "histedit.h"
526dc2f1dbScgd
53d30d584aSlukem typedef enum {
54d30d584aSlukem Q_none, Q_single, Q_double, Q_one, Q_doubleone
55d30d584aSlukem } quote_t;
566dc2f1dbScgd
576dc2f1dbScgd #define TOK_KEEP 1
586dc2f1dbScgd #define TOK_EAT 2
596dc2f1dbScgd
606dc2f1dbScgd #define WINCR 20
616dc2f1dbScgd #define AINCR 10
626dc2f1dbScgd
6334e53048Schristos #define IFS STR("\t \n")
6434e53048Schristos
656dc2f1dbScgd #define tok_malloc(a) malloc(a)
666dc2f1dbScgd #define tok_free(a) free(a)
676dc2f1dbScgd #define tok_realloc(a, b) realloc(a, b)
686dc2f1dbScgd
690aefc7f9Schristos #ifdef NARROWCHAR
700594af80Schristos #define Char char
710aefc7f9Schristos #define FUN(prefix, rest) prefix ## _ ## rest
720aefc7f9Schristos #define TYPE(type) type
730aefc7f9Schristos #define STR(x) x
740aefc7f9Schristos #define Strchr(s, c) strchr(s, c)
750aefc7f9Schristos #define tok_strdup(s) strdup(s)
760aefc7f9Schristos #else
770594af80Schristos #define Char wchar_t
780aefc7f9Schristos #define FUN(prefix, rest) prefix ## _w ## rest
790aefc7f9Schristos #define TYPE(type) type ## W
800aefc7f9Schristos #define STR(x) L ## x
810aefc7f9Schristos #define Strchr(s, c) wcschr(s, c)
820aefc7f9Schristos #define tok_strdup(s) wcsdup(s)
830aefc7f9Schristos #endif
846dc2f1dbScgd
TYPE(tokenizer)8573eda9feSchristos struct TYPE(tokenizer) {
8634e53048Schristos Char *ifs; /* In field separator */
873d802cf5Schristos size_t argc, amax; /* Current and maximum number of args */
88*f3bd4955Schristos const Char **argv; /* Argument list */
8934e53048Schristos Char *wptr, *wmax; /* Space and limit on the word buffer */
9034e53048Schristos Char *wstart; /* Beginning of next word */
9134e53048Schristos Char *wspace; /* Space of word buffer */
926dc2f1dbScgd quote_t quote; /* Quoting state */
936dc2f1dbScgd int flags; /* flags; */
946dc2f1dbScgd };
956dc2f1dbScgd
966dc2f1dbScgd
97469d44f8Schristos static void FUN(tok,finish)(TYPE(Tokenizer) *);
986dc2f1dbScgd
996dc2f1dbScgd
10034e53048Schristos /* FUN(tok,finish)():
1016dc2f1dbScgd * Finish a word in the tokenizer.
1026dc2f1dbScgd */
103469d44f8Schristos static void
FUN(tok,finish)10434e53048Schristos FUN(tok,finish)(TYPE(Tokenizer) *tok)
1056dc2f1dbScgd {
106d30d584aSlukem
1076dc2f1dbScgd *tok->wptr = '\0';
1086dc2f1dbScgd if ((tok->flags & TOK_KEEP) || tok->wptr != tok->wstart) {
1096dc2f1dbScgd tok->argv[tok->argc++] = tok->wstart;
1106dc2f1dbScgd tok->argv[tok->argc] = NULL;
1116dc2f1dbScgd tok->wstart = ++tok->wptr;
1126dc2f1dbScgd }
1136dc2f1dbScgd tok->flags &= ~TOK_KEEP;
1146dc2f1dbScgd }
1156dc2f1dbScgd
1166dc2f1dbScgd
11734e53048Schristos /* FUN(tok,init)():
1186dc2f1dbScgd * Initialize the tokenizer
1196dc2f1dbScgd */
TYPE(Tokenizer)120469d44f8Schristos TYPE(Tokenizer) *
12134e53048Schristos FUN(tok,init)(const Char *ifs)
1226dc2f1dbScgd {
123a13cd756Schristos TYPE(Tokenizer) *tok = tok_malloc(sizeof(*tok));
1246dc2f1dbScgd
125e6ee0301Schristos if (tok == NULL)
126e6ee0301Schristos return NULL;
1276e782349Schristos tok->ifs = tok_strdup(ifs ? ifs : IFS);
128e6ee0301Schristos if (tok->ifs == NULL) {
129a13cd756Schristos tok_free(tok);
130e6ee0301Schristos return NULL;
131e6ee0301Schristos }
1326dc2f1dbScgd tok->argc = 0;
1336dc2f1dbScgd tok->amax = AINCR;
13434e53048Schristos tok->argv = tok_malloc(sizeof(*tok->argv) * tok->amax);
135e6ee0301Schristos if (tok->argv == NULL) {
136a13cd756Schristos tok_free(tok->ifs);
137a13cd756Schristos tok_free(tok);
138e6ee0301Schristos return NULL;
139e6ee0301Schristos }
1406dc2f1dbScgd tok->argv[0] = NULL;
14134e53048Schristos tok->wspace = tok_malloc(WINCR * sizeof(*tok->wspace));
142e6ee0301Schristos if (tok->wspace == NULL) {
143a13cd756Schristos tok_free(tok->argv);
144a13cd756Schristos tok_free(tok->ifs);
145a13cd756Schristos tok_free(tok);
146e6ee0301Schristos return NULL;
147e6ee0301Schristos }
1486dc2f1dbScgd tok->wmax = tok->wspace + WINCR;
1496dc2f1dbScgd tok->wstart = tok->wspace;
1506dc2f1dbScgd tok->wptr = tok->wspace;
1516dc2f1dbScgd tok->flags = 0;
1526dc2f1dbScgd tok->quote = Q_none;
1536dc2f1dbScgd
154b71bed95Schristos return tok;
1556dc2f1dbScgd }
1566dc2f1dbScgd
1576dc2f1dbScgd
15834e53048Schristos /* FUN(tok,reset)():
1596dc2f1dbScgd * Reset the tokenizer
1606dc2f1dbScgd */
161469d44f8Schristos void
FUN(tok,reset)16234e53048Schristos FUN(tok,reset)(TYPE(Tokenizer) *tok)
1636dc2f1dbScgd {
164d30d584aSlukem
1656dc2f1dbScgd tok->argc = 0;
1666dc2f1dbScgd tok->wstart = tok->wspace;
1676dc2f1dbScgd tok->wptr = tok->wspace;
1686dc2f1dbScgd tok->flags = 0;
1696dc2f1dbScgd tok->quote = Q_none;
1706dc2f1dbScgd }
1716dc2f1dbScgd
1726dc2f1dbScgd
17334e53048Schristos /* FUN(tok,end)():
1746dc2f1dbScgd * Clean up
1756dc2f1dbScgd */
176469d44f8Schristos void
FUN(tok,end)17734e53048Schristos FUN(tok,end)(TYPE(Tokenizer) *tok)
1786dc2f1dbScgd {
179d30d584aSlukem
180a13cd756Schristos tok_free(tok->ifs);
181a13cd756Schristos tok_free(tok->wspace);
182a13cd756Schristos tok_free(tok->argv);
183a13cd756Schristos tok_free(tok);
1846dc2f1dbScgd }
1856dc2f1dbScgd
1866dc2f1dbScgd
1876dc2f1dbScgd
18834e53048Schristos /* FUN(tok,line)():
189a9beb0e4Slukem * Bourne shell (sh(1)) like tokenizing
190a9beb0e4Slukem * Arguments:
19134e53048Schristos * tok current tokenizer state (setup with FUN(tok,init)())
192a9beb0e4Slukem * line line to parse
193a9beb0e4Slukem * Returns:
194a9beb0e4Slukem * -1 Internal error
195a9beb0e4Slukem * 3 Quoted return
196a9beb0e4Slukem * 2 Unmatched double quote
197a9beb0e4Slukem * 1 Unmatched single quote
198a9beb0e4Slukem * 0 Ok
199a9beb0e4Slukem * Modifies (if return value is 0):
200a9beb0e4Slukem * argc number of arguments
201a9beb0e4Slukem * argv argument array
202a9beb0e4Slukem * cursorc if !NULL, argv element containing cursor
203a9beb0e4Slukem * cursorv if !NULL, offset in argv[cursorc] of cursor
2046dc2f1dbScgd */
205469d44f8Schristos int
FUN(tok,line)2060b9ae3fdSchristos FUN(tok,line)(TYPE(Tokenizer) *tok, const TYPE(LineInfo) *line,
20734e53048Schristos int *argc, const Char ***argv, int *cursorc, int *cursoro)
2086dc2f1dbScgd {
20934e53048Schristos const Char *ptr;
210a9beb0e4Slukem int cc, co;
2116dc2f1dbScgd
212a9beb0e4Slukem cc = co = -1;
213a9beb0e4Slukem ptr = line->buffer;
214a9beb0e4Slukem for (ptr = line->buffer; ;ptr++) {
215a9beb0e4Slukem if (ptr >= line->lastchar)
21634e53048Schristos ptr = STR("");
217a9beb0e4Slukem if (ptr == line->cursor) {
2183d802cf5Schristos cc = (int)tok->argc;
2195c894153Schristos co = (int)(tok->wptr - tok->wstart);
220a9beb0e4Slukem }
221a9beb0e4Slukem switch (*ptr) {
2226dc2f1dbScgd case '\'':
2236dc2f1dbScgd tok->flags |= TOK_KEEP;
2246dc2f1dbScgd tok->flags &= ~TOK_EAT;
2256dc2f1dbScgd switch (tok->quote) {
2266dc2f1dbScgd case Q_none:
227d30d584aSlukem tok->quote = Q_single; /* Enter single quote
228d30d584aSlukem * mode */
2296dc2f1dbScgd break;
2306dc2f1dbScgd
2316dc2f1dbScgd case Q_single: /* Exit single quote mode */
2326dc2f1dbScgd tok->quote = Q_none;
2336dc2f1dbScgd break;
2346dc2f1dbScgd
2356dc2f1dbScgd case Q_one: /* Quote this ' */
2366dc2f1dbScgd tok->quote = Q_none;
2376dc2f1dbScgd *tok->wptr++ = *ptr;
2386dc2f1dbScgd break;
2396dc2f1dbScgd
2406dc2f1dbScgd case Q_double: /* Stay in double quote mode */
2416dc2f1dbScgd *tok->wptr++ = *ptr;
2426dc2f1dbScgd break;
2436dc2f1dbScgd
2446dc2f1dbScgd case Q_doubleone: /* Quote this ' */
2456dc2f1dbScgd tok->quote = Q_double;
2466dc2f1dbScgd *tok->wptr++ = *ptr;
2476dc2f1dbScgd break;
2486dc2f1dbScgd
2496dc2f1dbScgd default:
250b71bed95Schristos return -1;
2516dc2f1dbScgd }
2526dc2f1dbScgd break;
2536dc2f1dbScgd
2546dc2f1dbScgd case '"':
2556dc2f1dbScgd tok->flags &= ~TOK_EAT;
2566dc2f1dbScgd tok->flags |= TOK_KEEP;
2576dc2f1dbScgd switch (tok->quote) {
2586dc2f1dbScgd case Q_none: /* Enter double quote mode */
2596dc2f1dbScgd tok->quote = Q_double;
2606dc2f1dbScgd break;
2616dc2f1dbScgd
262d30d584aSlukem case Q_double: /* Exit double quote mode */
263d30d584aSlukem tok->quote = Q_none;
2646dc2f1dbScgd break;
2656dc2f1dbScgd
2666dc2f1dbScgd case Q_one: /* Quote this " */
2676dc2f1dbScgd tok->quote = Q_none;
2686dc2f1dbScgd *tok->wptr++ = *ptr;
2696dc2f1dbScgd break;
2706dc2f1dbScgd
2716dc2f1dbScgd case Q_single: /* Stay in single quote mode */
2726dc2f1dbScgd *tok->wptr++ = *ptr;
2736dc2f1dbScgd break;
2746dc2f1dbScgd
2756dc2f1dbScgd case Q_doubleone: /* Quote this " */
2766dc2f1dbScgd tok->quote = Q_double;
2776dc2f1dbScgd *tok->wptr++ = *ptr;
2786dc2f1dbScgd break;
2796dc2f1dbScgd
2806dc2f1dbScgd default:
281b71bed95Schristos return -1;
2826dc2f1dbScgd }
2836dc2f1dbScgd break;
2846dc2f1dbScgd
2856dc2f1dbScgd case '\\':
2866dc2f1dbScgd tok->flags |= TOK_KEEP;
2876dc2f1dbScgd tok->flags &= ~TOK_EAT;
2886dc2f1dbScgd switch (tok->quote) {
2896dc2f1dbScgd case Q_none: /* Quote next character */
2906dc2f1dbScgd tok->quote = Q_one;
2916dc2f1dbScgd break;
2926dc2f1dbScgd
293d30d584aSlukem case Q_double: /* Quote next character */
294d30d584aSlukem tok->quote = Q_doubleone;
2956dc2f1dbScgd break;
2966dc2f1dbScgd
297d30d584aSlukem case Q_one: /* Quote this, restore state */
2986dc2f1dbScgd *tok->wptr++ = *ptr;
299d30d584aSlukem tok->quote = Q_none;
3006dc2f1dbScgd break;
3016dc2f1dbScgd
3026dc2f1dbScgd case Q_single: /* Stay in single quote mode */
3036dc2f1dbScgd *tok->wptr++ = *ptr;
3046dc2f1dbScgd break;
3056dc2f1dbScgd
3066dc2f1dbScgd case Q_doubleone: /* Quote this \ */
3076dc2f1dbScgd tok->quote = Q_double;
3086dc2f1dbScgd *tok->wptr++ = *ptr;
3096dc2f1dbScgd break;
3106dc2f1dbScgd
3116dc2f1dbScgd default:
312b71bed95Schristos return -1;
3136dc2f1dbScgd }
3146dc2f1dbScgd break;
3156dc2f1dbScgd
3166dc2f1dbScgd case '\n':
3176dc2f1dbScgd tok->flags &= ~TOK_EAT;
3186dc2f1dbScgd switch (tok->quote) {
3196dc2f1dbScgd case Q_none:
320a9beb0e4Slukem goto tok_line_outok;
3216dc2f1dbScgd
3226dc2f1dbScgd case Q_single:
3236dc2f1dbScgd case Q_double:
3246dc2f1dbScgd *tok->wptr++ = *ptr; /* Add the return */
3256dc2f1dbScgd break;
3266dc2f1dbScgd
327d30d584aSlukem case Q_doubleone: /* Back to double, eat the '\n' */
3286dc2f1dbScgd tok->flags |= TOK_EAT;
329d30d584aSlukem tok->quote = Q_double;
3306dc2f1dbScgd break;
3316dc2f1dbScgd
332d30d584aSlukem case Q_one: /* No quote, more eat the '\n' */
3336dc2f1dbScgd tok->flags |= TOK_EAT;
334d30d584aSlukem tok->quote = Q_none;
3356dc2f1dbScgd break;
3366dc2f1dbScgd
3376dc2f1dbScgd default:
338b71bed95Schristos return 0;
3396dc2f1dbScgd }
3406dc2f1dbScgd break;
3416dc2f1dbScgd
3426dc2f1dbScgd case '\0':
3436dc2f1dbScgd switch (tok->quote) {
3446dc2f1dbScgd case Q_none:
3456dc2f1dbScgd /* Finish word and return */
3466dc2f1dbScgd if (tok->flags & TOK_EAT) {
3476dc2f1dbScgd tok->flags &= ~TOK_EAT;
348b71bed95Schristos return 3;
3496dc2f1dbScgd }
350a9beb0e4Slukem goto tok_line_outok;
3516dc2f1dbScgd
3526dc2f1dbScgd case Q_single:
353b71bed95Schristos return 1;
3546dc2f1dbScgd
3556dc2f1dbScgd case Q_double:
356b71bed95Schristos return 2;
3576dc2f1dbScgd
3586dc2f1dbScgd case Q_doubleone:
3596dc2f1dbScgd tok->quote = Q_double;
3606dc2f1dbScgd *tok->wptr++ = *ptr;
3616dc2f1dbScgd break;
3626dc2f1dbScgd
3636dc2f1dbScgd case Q_one:
3646dc2f1dbScgd tok->quote = Q_none;
3656dc2f1dbScgd *tok->wptr++ = *ptr;
3666dc2f1dbScgd break;
3676dc2f1dbScgd
3686dc2f1dbScgd default:
369b71bed95Schristos return -1;
3706dc2f1dbScgd }
3716dc2f1dbScgd break;
3726dc2f1dbScgd
3736dc2f1dbScgd default:
3746dc2f1dbScgd tok->flags &= ~TOK_EAT;
3756dc2f1dbScgd switch (tok->quote) {
3766dc2f1dbScgd case Q_none:
37734e53048Schristos if (Strchr(tok->ifs, *ptr) != NULL)
37834e53048Schristos FUN(tok,finish)(tok);
3796dc2f1dbScgd else
3806dc2f1dbScgd *tok->wptr++ = *ptr;
3816dc2f1dbScgd break;
3826dc2f1dbScgd
3836dc2f1dbScgd case Q_single:
3846dc2f1dbScgd case Q_double:
3856dc2f1dbScgd *tok->wptr++ = *ptr;
3866dc2f1dbScgd break;
3876dc2f1dbScgd
3886dc2f1dbScgd
3896dc2f1dbScgd case Q_doubleone:
3906dc2f1dbScgd *tok->wptr++ = '\\';
3916dc2f1dbScgd tok->quote = Q_double;
3926dc2f1dbScgd *tok->wptr++ = *ptr;
3936dc2f1dbScgd break;
3946dc2f1dbScgd
3956dc2f1dbScgd case Q_one:
3966dc2f1dbScgd tok->quote = Q_none;
3976dc2f1dbScgd *tok->wptr++ = *ptr;
3986dc2f1dbScgd break;
3996dc2f1dbScgd
4006dc2f1dbScgd default:
401b71bed95Schristos return -1;
4026dc2f1dbScgd
4036dc2f1dbScgd }
4046dc2f1dbScgd break;
4056dc2f1dbScgd }
4066dc2f1dbScgd
4076dc2f1dbScgd if (tok->wptr >= tok->wmax - 4) {
4083d802cf5Schristos size_t size = (size_t)(tok->wmax - tok->wspace + WINCR);
40934e53048Schristos Char *s = tok_realloc(tok->wspace,
41034e53048Schristos size * sizeof(*s));
4112f3389ceSchristos if (s == NULL)
412b71bed95Schristos return -1;
4136dc2f1dbScgd
414a4b0370fSchristos if (s != tok->wspace) {
4153d802cf5Schristos size_t i;
416a4b0370fSchristos for (i = 0; i < tok->argc; i++) {
417a4b0370fSchristos tok->argv[i] =
418a4b0370fSchristos (tok->argv[i] - tok->wspace) + s;
419a4b0370fSchristos }
420a4b0370fSchristos tok->wptr = (tok->wptr - tok->wspace) + s;
421a4b0370fSchristos tok->wstart = (tok->wstart - tok->wspace) + s;
4226dc2f1dbScgd tok->wspace = s;
4236dc2f1dbScgd }
424910773a8Schristos tok->wmax = s + size;
4256dc2f1dbScgd }
4266dc2f1dbScgd if (tok->argc >= tok->amax - 4) {
427*f3bd4955Schristos const Char **p;
4286dc2f1dbScgd tok->amax += AINCR;
42934e53048Schristos p = tok_realloc(tok->argv, tok->amax * sizeof(*p));
430c0d16449Schristos if (p == NULL) {
431c0d16449Schristos tok->amax -= AINCR;
432b71bed95Schristos return -1;
433c0d16449Schristos }
4342f3389ceSchristos tok->argv = p;
4356dc2f1dbScgd }
4366dc2f1dbScgd }
437a9beb0e4Slukem tok_line_outok:
438a9beb0e4Slukem if (cc == -1 && co == -1) {
4393d802cf5Schristos cc = (int)tok->argc;
4405c894153Schristos co = (int)(tok->wptr - tok->wstart);
441a9beb0e4Slukem }
442a9beb0e4Slukem if (cursorc != NULL)
443a9beb0e4Slukem *cursorc = cc;
444a9beb0e4Slukem if (cursoro != NULL)
445a9beb0e4Slukem *cursoro = co;
44634e53048Schristos FUN(tok,finish)(tok);
447*f3bd4955Schristos *argv = tok->argv;
4483d802cf5Schristos *argc = (int)tok->argc;
449b71bed95Schristos return 0;
450a9beb0e4Slukem }
451a9beb0e4Slukem
45234e53048Schristos /* FUN(tok,str)():
453a9beb0e4Slukem * Simpler version of tok_line, taking a NUL terminated line
454a9beb0e4Slukem * and splitting into words, ignoring cursor state.
455a9beb0e4Slukem */
456469d44f8Schristos int
FUN(tok,str)45734e53048Schristos FUN(tok,str)(TYPE(Tokenizer) *tok, const Char *line, int *argc,
45834e53048Schristos const Char ***argv)
459a9beb0e4Slukem {
4600b9ae3fdSchristos TYPE(LineInfo) li;
461a9beb0e4Slukem
462a9beb0e4Slukem memset(&li, 0, sizeof(li));
463a9beb0e4Slukem li.buffer = line;
46434e53048Schristos li.cursor = li.lastchar = Strchr(line, '\0');
4657ce9f672Schristos return FUN(tok,line)(tok, &li, argc, argv, NULL, NULL);
4666dc2f1dbScgd }
467