xref: /netbsd-src/lib/libedit/tokenizer.c (revision f3bd4955436e61d927bd002023c24d99f9d50336)
1*f3bd4955Schristos /*	$NetBSD: tokenizer.c,v 1.29 2023/05/30 11:53:40 christos Exp $	*/
22543e3e6Slukem 
36dc2f1dbScgd /*-
46dc2f1dbScgd  * Copyright (c) 1992, 1993
56dc2f1dbScgd  *	The Regents of the University of California.  All rights reserved.
66dc2f1dbScgd  *
76dc2f1dbScgd  * This code is derived from software contributed to Berkeley by
86dc2f1dbScgd  * Christos Zoulas of Cornell University.
96dc2f1dbScgd  *
106dc2f1dbScgd  * Redistribution and use in source and binary forms, with or without
116dc2f1dbScgd  * modification, are permitted provided that the following conditions
126dc2f1dbScgd  * are met:
136dc2f1dbScgd  * 1. Redistributions of source code must retain the above copyright
146dc2f1dbScgd  *    notice, this list of conditions and the following disclaimer.
156dc2f1dbScgd  * 2. Redistributions in binary form must reproduce the above copyright
166dc2f1dbScgd  *    notice, this list of conditions and the following disclaimer in the
176dc2f1dbScgd  *    documentation and/or other materials provided with the distribution.
18eb7c1594Sagc  * 3. Neither the name of the University nor the names of its contributors
196dc2f1dbScgd  *    may be used to endorse or promote products derived from this software
206dc2f1dbScgd  *    without specific prior written permission.
216dc2f1dbScgd  *
226dc2f1dbScgd  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
236dc2f1dbScgd  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
246dc2f1dbScgd  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
256dc2f1dbScgd  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
266dc2f1dbScgd  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
276dc2f1dbScgd  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
286dc2f1dbScgd  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
296dc2f1dbScgd  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
306dc2f1dbScgd  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
316dc2f1dbScgd  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
326dc2f1dbScgd  * SUCH DAMAGE.
336dc2f1dbScgd  */
346dc2f1dbScgd 
350e0ac6b7Schristos #include "config.h"
366dc2f1dbScgd #if !defined(lint) && !defined(SCCSID)
372543e3e6Slukem #if 0
386dc2f1dbScgd static char sccsid[] = "@(#)tokenizer.c	8.1 (Berkeley) 6/4/93";
392543e3e6Slukem #else
40*f3bd4955Schristos __RCSID("$NetBSD: tokenizer.c,v 1.29 2023/05/30 11:53:40 christos Exp $");
412543e3e6Slukem #endif
426dc2f1dbScgd #endif /* not lint && not SCCSID */
436dc2f1dbScgd 
4434e53048Schristos /* We build this file twice, once as NARROW, once as WIDE. */
456dc2f1dbScgd /*
466dc2f1dbScgd  * tokenize.c: Bourne shell like tokenizer
476dc2f1dbScgd  */
486dc2f1dbScgd #include <stdlib.h>
4922383670Schristos #include <string.h>
5022383670Schristos 
51a9beb0e4Slukem #include "histedit.h"
526dc2f1dbScgd 
53d30d584aSlukem typedef enum {
54d30d584aSlukem 	Q_none, Q_single, Q_double, Q_one, Q_doubleone
55d30d584aSlukem } quote_t;
566dc2f1dbScgd 
576dc2f1dbScgd #define	TOK_KEEP	1
586dc2f1dbScgd #define	TOK_EAT		2
596dc2f1dbScgd 
606dc2f1dbScgd #define	WINCR		20
616dc2f1dbScgd #define	AINCR		10
626dc2f1dbScgd 
6334e53048Schristos #define	IFS		STR("\t \n")
6434e53048Schristos 
656dc2f1dbScgd #define	tok_malloc(a)		malloc(a)
666dc2f1dbScgd #define	tok_free(a)		free(a)
676dc2f1dbScgd #define	tok_realloc(a, b)	realloc(a, b)
686dc2f1dbScgd 
690aefc7f9Schristos #ifdef NARROWCHAR
700594af80Schristos #define	Char			char
710aefc7f9Schristos #define	FUN(prefix, rest)	prefix ## _ ## rest
720aefc7f9Schristos #define	TYPE(type)		type
730aefc7f9Schristos #define	STR(x)			x
740aefc7f9Schristos #define	Strchr(s, c)		strchr(s, c)
750aefc7f9Schristos #define	tok_strdup(s)		strdup(s)
760aefc7f9Schristos #else
770594af80Schristos #define	Char			wchar_t
780aefc7f9Schristos #define	FUN(prefix, rest)	prefix ## _w ## rest
790aefc7f9Schristos #define	TYPE(type)		type ## W
800aefc7f9Schristos #define	STR(x)			L ## x
810aefc7f9Schristos #define	Strchr(s, c)		wcschr(s, c)
820aefc7f9Schristos #define	tok_strdup(s)		wcsdup(s)
830aefc7f9Schristos #endif
846dc2f1dbScgd 
TYPE(tokenizer)8573eda9feSchristos struct TYPE(tokenizer) {
8634e53048Schristos 	Char	*ifs;		/* In field separator			 */
873d802cf5Schristos 	size_t	 argc, amax;	/* Current and maximum number of args	 */
88*f3bd4955Schristos 	const Char   **argv;	/* Argument list			 */
8934e53048Schristos 	Char	*wptr, *wmax;	/* Space and limit on the word buffer	 */
9034e53048Schristos 	Char	*wstart;	/* Beginning of next word		 */
9134e53048Schristos 	Char	*wspace;	/* Space of word buffer			 */
926dc2f1dbScgd 	quote_t	 quote;		/* Quoting state			 */
936dc2f1dbScgd 	int	 flags;		/* flags;				 */
946dc2f1dbScgd };
956dc2f1dbScgd 
966dc2f1dbScgd 
97469d44f8Schristos static void FUN(tok,finish)(TYPE(Tokenizer) *);
986dc2f1dbScgd 
996dc2f1dbScgd 
10034e53048Schristos /* FUN(tok,finish)():
1016dc2f1dbScgd  *	Finish a word in the tokenizer.
1026dc2f1dbScgd  */
103469d44f8Schristos static void
FUN(tok,finish)10434e53048Schristos FUN(tok,finish)(TYPE(Tokenizer) *tok)
1056dc2f1dbScgd {
106d30d584aSlukem 
1076dc2f1dbScgd 	*tok->wptr = '\0';
1086dc2f1dbScgd 	if ((tok->flags & TOK_KEEP) || tok->wptr != tok->wstart) {
1096dc2f1dbScgd 		tok->argv[tok->argc++] = tok->wstart;
1106dc2f1dbScgd 		tok->argv[tok->argc] = NULL;
1116dc2f1dbScgd 		tok->wstart = ++tok->wptr;
1126dc2f1dbScgd 	}
1136dc2f1dbScgd 	tok->flags &= ~TOK_KEEP;
1146dc2f1dbScgd }
1156dc2f1dbScgd 
1166dc2f1dbScgd 
11734e53048Schristos /* FUN(tok,init)():
1186dc2f1dbScgd  *	Initialize the tokenizer
1196dc2f1dbScgd  */
TYPE(Tokenizer)120469d44f8Schristos TYPE(Tokenizer) *
12134e53048Schristos FUN(tok,init)(const Char *ifs)
1226dc2f1dbScgd {
123a13cd756Schristos 	TYPE(Tokenizer) *tok = tok_malloc(sizeof(*tok));
1246dc2f1dbScgd 
125e6ee0301Schristos 	if (tok == NULL)
126e6ee0301Schristos 		return NULL;
1276e782349Schristos 	tok->ifs = tok_strdup(ifs ? ifs : IFS);
128e6ee0301Schristos 	if (tok->ifs == NULL) {
129a13cd756Schristos 		tok_free(tok);
130e6ee0301Schristos 		return NULL;
131e6ee0301Schristos 	}
1326dc2f1dbScgd 	tok->argc = 0;
1336dc2f1dbScgd 	tok->amax = AINCR;
13434e53048Schristos 	tok->argv = tok_malloc(sizeof(*tok->argv) * tok->amax);
135e6ee0301Schristos 	if (tok->argv == NULL) {
136a13cd756Schristos 		tok_free(tok->ifs);
137a13cd756Schristos 		tok_free(tok);
138e6ee0301Schristos 		return NULL;
139e6ee0301Schristos 	}
1406dc2f1dbScgd 	tok->argv[0] = NULL;
14134e53048Schristos 	tok->wspace = tok_malloc(WINCR * sizeof(*tok->wspace));
142e6ee0301Schristos 	if (tok->wspace == NULL) {
143a13cd756Schristos 		tok_free(tok->argv);
144a13cd756Schristos 		tok_free(tok->ifs);
145a13cd756Schristos 		tok_free(tok);
146e6ee0301Schristos 		return NULL;
147e6ee0301Schristos 	}
1486dc2f1dbScgd 	tok->wmax = tok->wspace + WINCR;
1496dc2f1dbScgd 	tok->wstart = tok->wspace;
1506dc2f1dbScgd 	tok->wptr = tok->wspace;
1516dc2f1dbScgd 	tok->flags = 0;
1526dc2f1dbScgd 	tok->quote = Q_none;
1536dc2f1dbScgd 
154b71bed95Schristos 	return tok;
1556dc2f1dbScgd }
1566dc2f1dbScgd 
1576dc2f1dbScgd 
15834e53048Schristos /* FUN(tok,reset)():
1596dc2f1dbScgd  *	Reset the tokenizer
1606dc2f1dbScgd  */
161469d44f8Schristos void
FUN(tok,reset)16234e53048Schristos FUN(tok,reset)(TYPE(Tokenizer) *tok)
1636dc2f1dbScgd {
164d30d584aSlukem 
1656dc2f1dbScgd 	tok->argc = 0;
1666dc2f1dbScgd 	tok->wstart = tok->wspace;
1676dc2f1dbScgd 	tok->wptr = tok->wspace;
1686dc2f1dbScgd 	tok->flags = 0;
1696dc2f1dbScgd 	tok->quote = Q_none;
1706dc2f1dbScgd }
1716dc2f1dbScgd 
1726dc2f1dbScgd 
17334e53048Schristos /* FUN(tok,end)():
1746dc2f1dbScgd  *	Clean up
1756dc2f1dbScgd  */
176469d44f8Schristos void
FUN(tok,end)17734e53048Schristos FUN(tok,end)(TYPE(Tokenizer) *tok)
1786dc2f1dbScgd {
179d30d584aSlukem 
180a13cd756Schristos 	tok_free(tok->ifs);
181a13cd756Schristos 	tok_free(tok->wspace);
182a13cd756Schristos 	tok_free(tok->argv);
183a13cd756Schristos 	tok_free(tok);
1846dc2f1dbScgd }
1856dc2f1dbScgd 
1866dc2f1dbScgd 
1876dc2f1dbScgd 
18834e53048Schristos /* FUN(tok,line)():
189a9beb0e4Slukem  *	Bourne shell (sh(1)) like tokenizing
190a9beb0e4Slukem  *	Arguments:
19134e53048Schristos  *		tok	current tokenizer state (setup with FUN(tok,init)())
192a9beb0e4Slukem  *		line	line to parse
193a9beb0e4Slukem  *	Returns:
194a9beb0e4Slukem  *		-1	Internal error
195a9beb0e4Slukem  *		 3	Quoted return
196a9beb0e4Slukem  *		 2	Unmatched double quote
197a9beb0e4Slukem  *		 1	Unmatched single quote
198a9beb0e4Slukem  *		 0	Ok
199a9beb0e4Slukem  *	Modifies (if return value is 0):
200a9beb0e4Slukem  *		argc	number of arguments
201a9beb0e4Slukem  *		argv	argument array
202a9beb0e4Slukem  *		cursorc	if !NULL, argv element containing cursor
203a9beb0e4Slukem  *		cursorv	if !NULL, offset in argv[cursorc] of cursor
2046dc2f1dbScgd  */
205469d44f8Schristos int
FUN(tok,line)2060b9ae3fdSchristos FUN(tok,line)(TYPE(Tokenizer) *tok, const TYPE(LineInfo) *line,
20734e53048Schristos     int *argc, const Char ***argv, int *cursorc, int *cursoro)
2086dc2f1dbScgd {
20934e53048Schristos 	const Char *ptr;
210a9beb0e4Slukem 	int cc, co;
2116dc2f1dbScgd 
212a9beb0e4Slukem 	cc = co = -1;
213a9beb0e4Slukem 	ptr = line->buffer;
214a9beb0e4Slukem 	for (ptr = line->buffer; ;ptr++) {
215a9beb0e4Slukem 		if (ptr >= line->lastchar)
21634e53048Schristos 			ptr = STR("");
217a9beb0e4Slukem 		if (ptr == line->cursor) {
2183d802cf5Schristos 			cc = (int)tok->argc;
2195c894153Schristos 			co = (int)(tok->wptr - tok->wstart);
220a9beb0e4Slukem 		}
221a9beb0e4Slukem 		switch (*ptr) {
2226dc2f1dbScgd 		case '\'':
2236dc2f1dbScgd 			tok->flags |= TOK_KEEP;
2246dc2f1dbScgd 			tok->flags &= ~TOK_EAT;
2256dc2f1dbScgd 			switch (tok->quote) {
2266dc2f1dbScgd 			case Q_none:
227d30d584aSlukem 				tok->quote = Q_single;	/* Enter single quote
228d30d584aSlukem 							 * mode */
2296dc2f1dbScgd 				break;
2306dc2f1dbScgd 
2316dc2f1dbScgd 			case Q_single:	/* Exit single quote mode */
2326dc2f1dbScgd 				tok->quote = Q_none;
2336dc2f1dbScgd 				break;
2346dc2f1dbScgd 
2356dc2f1dbScgd 			case Q_one:	/* Quote this ' */
2366dc2f1dbScgd 				tok->quote = Q_none;
2376dc2f1dbScgd 				*tok->wptr++ = *ptr;
2386dc2f1dbScgd 				break;
2396dc2f1dbScgd 
2406dc2f1dbScgd 			case Q_double:	/* Stay in double quote mode */
2416dc2f1dbScgd 				*tok->wptr++ = *ptr;
2426dc2f1dbScgd 				break;
2436dc2f1dbScgd 
2446dc2f1dbScgd 			case Q_doubleone:	/* Quote this ' */
2456dc2f1dbScgd 				tok->quote = Q_double;
2466dc2f1dbScgd 				*tok->wptr++ = *ptr;
2476dc2f1dbScgd 				break;
2486dc2f1dbScgd 
2496dc2f1dbScgd 			default:
250b71bed95Schristos 				return -1;
2516dc2f1dbScgd 			}
2526dc2f1dbScgd 			break;
2536dc2f1dbScgd 
2546dc2f1dbScgd 		case '"':
2556dc2f1dbScgd 			tok->flags &= ~TOK_EAT;
2566dc2f1dbScgd 			tok->flags |= TOK_KEEP;
2576dc2f1dbScgd 			switch (tok->quote) {
2586dc2f1dbScgd 			case Q_none:	/* Enter double quote mode */
2596dc2f1dbScgd 				tok->quote = Q_double;
2606dc2f1dbScgd 				break;
2616dc2f1dbScgd 
262d30d584aSlukem 			case Q_double:	/* Exit double quote mode */
263d30d584aSlukem 				tok->quote = Q_none;
2646dc2f1dbScgd 				break;
2656dc2f1dbScgd 
2666dc2f1dbScgd 			case Q_one:	/* Quote this " */
2676dc2f1dbScgd 				tok->quote = Q_none;
2686dc2f1dbScgd 				*tok->wptr++ = *ptr;
2696dc2f1dbScgd 				break;
2706dc2f1dbScgd 
2716dc2f1dbScgd 			case Q_single:	/* Stay in single quote mode */
2726dc2f1dbScgd 				*tok->wptr++ = *ptr;
2736dc2f1dbScgd 				break;
2746dc2f1dbScgd 
2756dc2f1dbScgd 			case Q_doubleone:	/* Quote this " */
2766dc2f1dbScgd 				tok->quote = Q_double;
2776dc2f1dbScgd 				*tok->wptr++ = *ptr;
2786dc2f1dbScgd 				break;
2796dc2f1dbScgd 
2806dc2f1dbScgd 			default:
281b71bed95Schristos 				return -1;
2826dc2f1dbScgd 			}
2836dc2f1dbScgd 			break;
2846dc2f1dbScgd 
2856dc2f1dbScgd 		case '\\':
2866dc2f1dbScgd 			tok->flags |= TOK_KEEP;
2876dc2f1dbScgd 			tok->flags &= ~TOK_EAT;
2886dc2f1dbScgd 			switch (tok->quote) {
2896dc2f1dbScgd 			case Q_none:	/* Quote next character */
2906dc2f1dbScgd 				tok->quote = Q_one;
2916dc2f1dbScgd 				break;
2926dc2f1dbScgd 
293d30d584aSlukem 			case Q_double:	/* Quote next character */
294d30d584aSlukem 				tok->quote = Q_doubleone;
2956dc2f1dbScgd 				break;
2966dc2f1dbScgd 
297d30d584aSlukem 			case Q_one:	/* Quote this, restore state */
2986dc2f1dbScgd 				*tok->wptr++ = *ptr;
299d30d584aSlukem 				tok->quote = Q_none;
3006dc2f1dbScgd 				break;
3016dc2f1dbScgd 
3026dc2f1dbScgd 			case Q_single:	/* Stay in single quote mode */
3036dc2f1dbScgd 				*tok->wptr++ = *ptr;
3046dc2f1dbScgd 				break;
3056dc2f1dbScgd 
3066dc2f1dbScgd 			case Q_doubleone:	/* Quote this \ */
3076dc2f1dbScgd 				tok->quote = Q_double;
3086dc2f1dbScgd 				*tok->wptr++ = *ptr;
3096dc2f1dbScgd 				break;
3106dc2f1dbScgd 
3116dc2f1dbScgd 			default:
312b71bed95Schristos 				return -1;
3136dc2f1dbScgd 			}
3146dc2f1dbScgd 			break;
3156dc2f1dbScgd 
3166dc2f1dbScgd 		case '\n':
3176dc2f1dbScgd 			tok->flags &= ~TOK_EAT;
3186dc2f1dbScgd 			switch (tok->quote) {
3196dc2f1dbScgd 			case Q_none:
320a9beb0e4Slukem 				goto tok_line_outok;
3216dc2f1dbScgd 
3226dc2f1dbScgd 			case Q_single:
3236dc2f1dbScgd 			case Q_double:
3246dc2f1dbScgd 				*tok->wptr++ = *ptr;	/* Add the return */
3256dc2f1dbScgd 				break;
3266dc2f1dbScgd 
327d30d584aSlukem 			case Q_doubleone:   /* Back to double, eat the '\n' */
3286dc2f1dbScgd 				tok->flags |= TOK_EAT;
329d30d584aSlukem 				tok->quote = Q_double;
3306dc2f1dbScgd 				break;
3316dc2f1dbScgd 
332d30d584aSlukem 			case Q_one:	/* No quote, more eat the '\n' */
3336dc2f1dbScgd 				tok->flags |= TOK_EAT;
334d30d584aSlukem 				tok->quote = Q_none;
3356dc2f1dbScgd 				break;
3366dc2f1dbScgd 
3376dc2f1dbScgd 			default:
338b71bed95Schristos 				return 0;
3396dc2f1dbScgd 			}
3406dc2f1dbScgd 			break;
3416dc2f1dbScgd 
3426dc2f1dbScgd 		case '\0':
3436dc2f1dbScgd 			switch (tok->quote) {
3446dc2f1dbScgd 			case Q_none:
3456dc2f1dbScgd 				/* Finish word and return */
3466dc2f1dbScgd 				if (tok->flags & TOK_EAT) {
3476dc2f1dbScgd 					tok->flags &= ~TOK_EAT;
348b71bed95Schristos 					return 3;
3496dc2f1dbScgd 				}
350a9beb0e4Slukem 				goto tok_line_outok;
3516dc2f1dbScgd 
3526dc2f1dbScgd 			case Q_single:
353b71bed95Schristos 				return 1;
3546dc2f1dbScgd 
3556dc2f1dbScgd 			case Q_double:
356b71bed95Schristos 				return 2;
3576dc2f1dbScgd 
3586dc2f1dbScgd 			case Q_doubleone:
3596dc2f1dbScgd 				tok->quote = Q_double;
3606dc2f1dbScgd 				*tok->wptr++ = *ptr;
3616dc2f1dbScgd 				break;
3626dc2f1dbScgd 
3636dc2f1dbScgd 			case Q_one:
3646dc2f1dbScgd 				tok->quote = Q_none;
3656dc2f1dbScgd 				*tok->wptr++ = *ptr;
3666dc2f1dbScgd 				break;
3676dc2f1dbScgd 
3686dc2f1dbScgd 			default:
369b71bed95Schristos 				return -1;
3706dc2f1dbScgd 			}
3716dc2f1dbScgd 			break;
3726dc2f1dbScgd 
3736dc2f1dbScgd 		default:
3746dc2f1dbScgd 			tok->flags &= ~TOK_EAT;
3756dc2f1dbScgd 			switch (tok->quote) {
3766dc2f1dbScgd 			case Q_none:
37734e53048Schristos 				if (Strchr(tok->ifs, *ptr) != NULL)
37834e53048Schristos 					FUN(tok,finish)(tok);
3796dc2f1dbScgd 				else
3806dc2f1dbScgd 					*tok->wptr++ = *ptr;
3816dc2f1dbScgd 				break;
3826dc2f1dbScgd 
3836dc2f1dbScgd 			case Q_single:
3846dc2f1dbScgd 			case Q_double:
3856dc2f1dbScgd 				*tok->wptr++ = *ptr;
3866dc2f1dbScgd 				break;
3876dc2f1dbScgd 
3886dc2f1dbScgd 
3896dc2f1dbScgd 			case Q_doubleone:
3906dc2f1dbScgd 				*tok->wptr++ = '\\';
3916dc2f1dbScgd 				tok->quote = Q_double;
3926dc2f1dbScgd 				*tok->wptr++ = *ptr;
3936dc2f1dbScgd 				break;
3946dc2f1dbScgd 
3956dc2f1dbScgd 			case Q_one:
3966dc2f1dbScgd 				tok->quote = Q_none;
3976dc2f1dbScgd 				*tok->wptr++ = *ptr;
3986dc2f1dbScgd 				break;
3996dc2f1dbScgd 
4006dc2f1dbScgd 			default:
401b71bed95Schristos 				return -1;
4026dc2f1dbScgd 
4036dc2f1dbScgd 			}
4046dc2f1dbScgd 			break;
4056dc2f1dbScgd 		}
4066dc2f1dbScgd 
4076dc2f1dbScgd 		if (tok->wptr >= tok->wmax - 4) {
4083d802cf5Schristos 			size_t size = (size_t)(tok->wmax - tok->wspace + WINCR);
40934e53048Schristos 			Char *s = tok_realloc(tok->wspace,
41034e53048Schristos 			    size * sizeof(*s));
4112f3389ceSchristos 			if (s == NULL)
412b71bed95Schristos 				return -1;
4136dc2f1dbScgd 
414a4b0370fSchristos 			if (s != tok->wspace) {
4153d802cf5Schristos 				size_t i;
416a4b0370fSchristos 				for (i = 0; i < tok->argc; i++) {
417a4b0370fSchristos 				    tok->argv[i] =
418a4b0370fSchristos 					(tok->argv[i] - tok->wspace) + s;
419a4b0370fSchristos 				}
420a4b0370fSchristos 				tok->wptr = (tok->wptr - tok->wspace) + s;
421a4b0370fSchristos 				tok->wstart = (tok->wstart - tok->wspace) + s;
4226dc2f1dbScgd 				tok->wspace = s;
4236dc2f1dbScgd 			}
424910773a8Schristos 			tok->wmax = s + size;
4256dc2f1dbScgd 		}
4266dc2f1dbScgd 		if (tok->argc >= tok->amax - 4) {
427*f3bd4955Schristos 			const Char **p;
4286dc2f1dbScgd 			tok->amax += AINCR;
42934e53048Schristos 			p = tok_realloc(tok->argv, tok->amax * sizeof(*p));
430c0d16449Schristos 			if (p == NULL) {
431c0d16449Schristos 				tok->amax -= AINCR;
432b71bed95Schristos 				return -1;
433c0d16449Schristos 			}
4342f3389ceSchristos 			tok->argv = p;
4356dc2f1dbScgd 		}
4366dc2f1dbScgd 	}
437a9beb0e4Slukem  tok_line_outok:
438a9beb0e4Slukem 	if (cc == -1 && co == -1) {
4393d802cf5Schristos 		cc = (int)tok->argc;
4405c894153Schristos 		co = (int)(tok->wptr - tok->wstart);
441a9beb0e4Slukem 	}
442a9beb0e4Slukem 	if (cursorc != NULL)
443a9beb0e4Slukem 		*cursorc = cc;
444a9beb0e4Slukem 	if (cursoro != NULL)
445a9beb0e4Slukem 		*cursoro = co;
44634e53048Schristos 	FUN(tok,finish)(tok);
447*f3bd4955Schristos 	*argv = tok->argv;
4483d802cf5Schristos 	*argc = (int)tok->argc;
449b71bed95Schristos 	return 0;
450a9beb0e4Slukem }
451a9beb0e4Slukem 
45234e53048Schristos /* FUN(tok,str)():
453a9beb0e4Slukem  *	Simpler version of tok_line, taking a NUL terminated line
454a9beb0e4Slukem  *	and splitting into words, ignoring cursor state.
455a9beb0e4Slukem  */
456469d44f8Schristos int
FUN(tok,str)45734e53048Schristos FUN(tok,str)(TYPE(Tokenizer) *tok, const Char *line, int *argc,
45834e53048Schristos     const Char ***argv)
459a9beb0e4Slukem {
4600b9ae3fdSchristos 	TYPE(LineInfo) li;
461a9beb0e4Slukem 
462a9beb0e4Slukem 	memset(&li, 0, sizeof(li));
463a9beb0e4Slukem 	li.buffer = line;
46434e53048Schristos 	li.cursor = li.lastchar = Strchr(line, '\0');
4657ce9f672Schristos 	return FUN(tok,line)(tok, &li, argc, argv, NULL, NULL);
4666dc2f1dbScgd }
467