xref: /netbsd-src/usr.bin/indent/lexi.c (revision 82d56013d7b633d116a93943de88e08335357a7c)
1 /*	$NetBSD: lexi.c,v 1.41 2021/03/14 00:22:16 rillig Exp $	*/
2 
3 /*-
4  * SPDX-License-Identifier: BSD-4-Clause
5  *
6  * Copyright (c) 1985 Sun Microsystems, Inc.
7  * Copyright (c) 1980, 1993
8  *	The Regents of the University of California.  All rights reserved.
9  * All rights reserved.
10  *
11  * Redistribution and use in source and binary forms, with or without
12  * modification, are permitted provided that the following conditions
13  * are met:
14  * 1. Redistributions of source code must retain the above copyright
15  *    notice, this list of conditions and the following disclaimer.
16  * 2. Redistributions in binary form must reproduce the above copyright
17  *    notice, this list of conditions and the following disclaimer in the
18  *    documentation and/or other materials provided with the distribution.
19  * 3. All advertising materials mentioning features or use of this software
20  *    must display the following acknowledgement:
21  *	This product includes software developed by the University of
22  *	California, Berkeley and its contributors.
23  * 4. Neither the name of the University nor the names of its contributors
24  *    may be used to endorse or promote products derived from this software
25  *    without specific prior written permission.
26  *
27  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
28  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
29  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
30  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
31  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
32  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
33  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
34  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
35  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
36  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
37  * SUCH DAMAGE.
38  */
39 
40 #if 0
41 #ifndef lint
42 static char sccsid[] = "@(#)lexi.c	8.1 (Berkeley) 6/6/93";
43 #endif /* not lint */
44 #endif
45 
46 #include <sys/cdefs.h>
47 #ifndef lint
48 #if defined(__NetBSD__)
49 __RCSID("$NetBSD: lexi.c,v 1.41 2021/03/14 00:22:16 rillig Exp $");
50 #elif defined(__FreeBSD__)
51 __FBSDID("$FreeBSD: head/usr.bin/indent/lexi.c 337862 2018-08-15 18:19:45Z pstef $");
52 #endif
53 #endif
54 
55 /*
56  * Here we have the token scanner for indent.  It scans off one token and puts
57  * it in the global variable "token".  It returns a code, indicating the type
58  * of token scanned.
59  */
60 
61 #include <assert.h>
62 #include <err.h>
63 #include <stdio.h>
64 #include <ctype.h>
65 #include <stdlib.h>
66 #include <string.h>
67 #include <sys/param.h>
68 
69 #include "indent.h"
70 
71 struct templ {
72     const char *rwd;
73     enum rwcode rwcode;
74 };
75 
76 /*
77  * This table has to be sorted alphabetically, because it'll be used in binary
78  * search.
79  */
80 const struct templ specials[] =
81 {
82     {"_Bool", rw_type},
83     {"_Complex", rw_type},
84     {"_Imaginary", rw_type},
85     {"auto", rw_storage_class},
86     {"bool", rw_type},
87     {"break", rw_jump},
88     {"case", rw_case_or_default},
89     {"char", rw_type},
90     {"complex", rw_type},
91     {"const", rw_type},
92     {"continue", rw_jump},
93     {"default", rw_case_or_default},
94     {"do", rw_do_or_else},
95     {"double", rw_type},
96     {"else", rw_do_or_else},
97     {"enum", rw_struct_or_union_or_enum},
98     {"extern", rw_storage_class},
99     {"float", rw_type},
100     {"for", rw_for_or_if_or_while},
101     {"global", rw_type},
102     {"goto", rw_jump},
103     {"if", rw_for_or_if_or_while},
104     {"imaginary", rw_type},
105     {"inline", rw_inline_or_restrict},
106     {"int", rw_type},
107     {"long", rw_type},
108     {"offsetof", rw_offsetof},
109     {"register", rw_storage_class},
110     {"restrict", rw_inline_or_restrict},
111     {"return", rw_jump},
112     {"short", rw_type},
113     {"signed", rw_type},
114     {"sizeof", rw_sizeof},
115     {"static", rw_storage_class},
116     {"struct", rw_struct_or_union_or_enum},
117     {"switch", rw_switch},
118     {"typedef", rw_typedef},
119     {"union", rw_struct_or_union_or_enum},
120     {"unsigned", rw_type},
121     {"void", rw_type},
122     {"volatile", rw_type},
123     {"while", rw_for_or_if_or_while}
124 };
125 
126 const char **typenames;
127 int         typename_count;
128 int         typename_top = -1;
129 
130 /*
131  * The transition table below was rewritten by hand from lx's output, given
132  * the following definitions. lx is Katherine Flavel's lexer generator.
133  *
134  * O  = /[0-7]/;        D  = /[0-9]/;          NZ = /[1-9]/;
135  * H  = /[a-f0-9]/i;    B  = /[0-1]/;          HP = /0x/i;
136  * BP = /0b/i;          E  = /e[+\-]?/i D+;    P  = /p[+\-]?/i D+;
137  * FS = /[fl]/i;        IS = /u/i /(l|L|ll|LL)/? | /(l|L|ll|LL)/ /u/i?;
138  *
139  * D+           E  FS? -> $float;
140  * D*    "." D+ E? FS? -> $float;
141  * D+    "."    E? FS? -> $float;    HP H+           IS? -> $int;
142  * HP H+        P  FS? -> $float;    NZ D*           IS? -> $int;
143  * HP H* "." H+ P  FS? -> $float;    "0" O*          IS? -> $int;
144  * HP H+ "."    P  FS  -> $float;    BP B+           IS? -> $int;
145  */
146 static char const *table[] = {
147     /*                examples:
148                                      00
149              s                      0xx
150              t                    00xaa
151              a     11       101100xxa..
152              r   11ee0001101lbuuxx.a.pp
153              t.01.e+008bLuxll0Ll.aa.p+0
154     states:  ABCDEFGHIJKLMNOPQRSTUVWXYZ */
155     ['0'] = "CEIDEHHHIJQ  U  Q  VUVVZZZ",
156     ['1'] = "DEIDEHHHIJQ  U  Q  VUVVZZZ",
157     ['7'] = "DEIDEHHHIJ   U     VUVVZZZ",
158     ['9'] = "DEJDEHHHJJ   U     VUVVZZZ",
159     ['a'] = "             U     VUVV   ",
160     ['b'] = "  K          U     VUVV   ",
161     ['e'] = "  FFF   FF   U     VUVV   ",
162     ['f'] = "    f  f     U     VUVV  f",
163     ['u'] = "  MM    M  i  iiM   M     ",
164     ['x'] = "  N                       ",
165     ['p'] = "                    FFX   ",
166     ['L'] = "  LLf  fL  PR   Li  L    f",
167     ['l'] = "  OOf  fO   S P O i O    f",
168     ['+'] = "     G                 Y  ",
169     ['.'] = "B EE    EE   T      W     ",
170     /*       ABCDEFGHIJKLMNOPQRSTUVWXYZ */
171     [0]   = "uuiifuufiuuiiuiiiiiuiuuuuu",
172 };
173 
174 /* Initialize constant transition table */
175 void
176 init_constant_tt(void)
177 {
178     table['-'] = table['+'];
179     table['8'] = table['9'];
180     table['2'] = table['3'] = table['4'] = table['5'] = table['6'] = table['7'];
181     table['A'] = table['C'] = table['D'] = table['c'] = table['d'] = table['a'];
182     table['B'] = table['b'];
183     table['E'] = table['e'];
184     table['U'] = table['u'];
185     table['X'] = table['x'];
186     table['P'] = table['p'];
187     table['F'] = table['f'];
188 }
189 
190 static char
191 inbuf_peek(void)
192 {
193     return *buf_ptr;
194 }
195 
196 static void
197 inbuf_skip(void)
198 {
199     buf_ptr++;
200     if (buf_ptr >= buf_end)
201 	fill_buffer();
202 }
203 
204 static char
205 inbuf_next(void)
206 {
207     char ch = inbuf_peek();
208     inbuf_skip();
209     return ch;
210 }
211 
212 static void
213 check_size_token(size_t desired_size)
214 {
215     if (e_token + (desired_size) < l_token)
216         return;
217 
218     size_t nsize = l_token - s_token + 400 + desired_size;
219     size_t token_len = e_token - s_token;
220     tokenbuf = realloc(tokenbuf, nsize);
221     if (tokenbuf == NULL)
222 	err(1, NULL);
223     e_token = tokenbuf + token_len + 1;
224     l_token = tokenbuf + nsize - 5;
225     s_token = tokenbuf + 1;
226 }
227 
228 static int
229 compare_templ_array(const void *key, const void *elem)
230 {
231     return strcmp(key, ((const struct templ *)elem)->rwd);
232 }
233 
234 static int
235 compare_string_array(const void *key, const void *elem)
236 {
237     return strcmp(key, *((const char *const *)elem));
238 }
239 
240 #ifdef debug
241 const char *
242 token_type_name(token_type tk)
243 {
244     static const char *const name[] = {
245 	"end_of_file", "newline", "lparen", "rparen", "unary_op",
246 	"binary_op", "postfix_op", "question", "case_label", "colon",
247 	"semicolon", "lbrace", "rbrace", "ident", "comma",
248 	"comment", "switch_expr", "preprocessing", "form_feed", "decl",
249 	"keyword_for_if_while", "keyword_do_else",
250 	"if_expr", "while_expr", "for_exprs",
251 	"stmt", "stmt_list", "keyword_else", "keyword_do", "do_stmt",
252 	"if_expr_stmt", "if_expr_stmt_else", "period", "string_prefix",
253 	"storage_class", "funcname", "type_def", "keyword_struct_union_enum"
254     };
255 
256     assert(0 <= tk && tk < sizeof name / sizeof name[0]);
257 
258     return name[tk];
259 }
260 
261 static void
262 print_buf(const char *name, const char *s, const char *e)
263 {
264     if (s < e) {
265 	debug_printf(" %s ", name);
266 	debug_vis_range("\"", s, e, "\"");
267     }
268 }
269 
270 static token_type
271 lexi_end(token_type code)
272 {
273     debug_printf("in line %d, lexi returns '%s'",
274 	line_no, token_type_name(code));
275     print_buf("token", s_token, e_token);
276     print_buf("label", s_lab, e_lab);
277     print_buf("code", s_code, e_code);
278     print_buf("comment", s_com, e_com);
279     debug_printf("\n");
280 
281     return code;
282 }
283 #else
284 #  define lexi_end(tk) (tk)
285 #endif
286 
287 token_type
288 lexi(struct parser_state *state)
289 {
290     int         unary_delim;	/* this is set to 1 if the current token
291 				 * forces a following operator to be unary */
292     token_type  code;		/* internal code to be returned */
293     char        qchar;		/* the delimiter character for a string */
294 
295     e_token = s_token;		/* point to start of place to save token */
296     unary_delim = false;
297     state->col_1 = state->last_nl;	/* tell world that this token started
298 					 * in column 1 iff the last thing
299 					 * scanned was a newline */
300     state->last_nl = false;
301 
302     while (*buf_ptr == ' ' || *buf_ptr == '\t') {	/* get rid of blanks */
303 	state->col_1 = false;	/* leading blanks imply token is not in column
304 				 * 1 */
305 	inbuf_skip();
306     }
307 
308     /* Scan an alphanumeric token */
309     if (isalnum((unsigned char)*buf_ptr) ||
310 	*buf_ptr == '_' || *buf_ptr == '$' ||
311 	(buf_ptr[0] == '.' && isdigit((unsigned char)buf_ptr[1]))) {
312 	/*
313 	 * we have a character or number
314 	 */
315 	struct templ *p;
316 
317 	if (isdigit((unsigned char)*buf_ptr) ||
318 	    (buf_ptr[0] == '.' && isdigit((unsigned char)buf_ptr[1]))) {
319 	    char s;
320 	    unsigned char i;
321 
322 	    for (s = 'A'; s != 'f' && s != 'i' && s != 'u'; ) {
323 		i = (unsigned char)*buf_ptr;
324 		if (i >= nitems(table) || table[i] == NULL ||
325 		    table[i][s - 'A'] == ' ') {
326 		    s = table[0][s - 'A'];
327 		    break;
328 		}
329 		s = table[i][s - 'A'];
330 		check_size_token(1);
331 		*e_token++ = inbuf_next();
332 	    }
333 	    /* s now indicates the type: f(loating), i(integer), u(nknown) */
334 	} else {
335 	    while (isalnum((unsigned char)*buf_ptr) ||
336 		   *buf_ptr == '\\' ||
337 		   *buf_ptr == '_' || *buf_ptr == '$') {
338 		/* fill_buffer() terminates buffer with newline */
339 		if (*buf_ptr == '\\') {
340 		    if (buf_ptr[1] == '\n') {
341 			buf_ptr += 2;
342 			if (buf_ptr >= buf_end)
343 			    fill_buffer();
344 		    } else
345 			break;
346 		}
347 		check_size_token(1);
348 		*e_token++ = inbuf_next();
349 	    }
350 	}
351 	*e_token = '\0';
352 
353 	if (s_token[0] == 'L' && s_token[1] == '\0' &&
354 	      (*buf_ptr == '"' || *buf_ptr == '\''))
355 	    return lexi_end(string_prefix);
356 
357 	while (*buf_ptr == ' ' || *buf_ptr == '\t')	/* get rid of blanks */
358 	    inbuf_next();
359 	state->keyword = rw_0;
360 	if (state->last_token == keyword_struct_union_enum &&
361 	    !state->p_l_follow) {
362 	    /* if last token was 'struct' and we're not in parentheses, then
363 	     * this token should be treated as a declaration */
364 	    state->last_u_d = true;
365 	    return lexi_end(decl);
366 	}
367 	/*
368 	 * Operator after identifier is binary unless last token was 'struct'
369 	 */
370 	state->last_u_d = (state->last_token == keyword_struct_union_enum);
371 
372 	p = bsearch(s_token, specials, sizeof specials / sizeof specials[0],
373 	    sizeof specials[0], compare_templ_array);
374 	if (p == NULL) {	/* not a special keyword... */
375 	    char *u;
376 
377 	    /* ... so maybe a type_t or a typedef */
378 	    if ((opt.auto_typedefs && ((u = strrchr(s_token, '_')) != NULL) &&
379 	        strcmp(u, "_t") == 0) || (typename_top >= 0 &&
380 		  bsearch(s_token, typenames, typename_top + 1,
381 		    sizeof typenames[0], compare_string_array))) {
382 		state->keyword = rw_type;
383 		state->last_u_d = true;
384 	        goto found_typename;
385 	    }
386 	} else {			/* we have a keyword */
387 	    state->keyword = p->rwcode;
388 	    state->last_u_d = true;
389 	    switch (p->rwcode) {
390 	    case rw_switch:
391 		return lexi_end(switch_expr);
392 	    case rw_case_or_default:
393 		return lexi_end(case_label);
394 	    case rw_struct_or_union_or_enum:
395 	    case rw_type:
396 	    found_typename:
397 		if (state->p_l_follow) {
398 		    /* inside parens: cast, param list, offsetof or sizeof */
399 		    state->cast_mask |= (1 << state->p_l_follow) & ~state->not_cast_mask;
400 		}
401 		if (state->last_token == period || state->last_token == unary_op) {
402 		    state->keyword = rw_0;
403 		    break;
404 		}
405 		if (p != NULL && p->rwcode == rw_struct_or_union_or_enum)
406 		    return lexi_end(keyword_struct_union_enum);
407 		if (state->p_l_follow)
408 		    break;
409 		return lexi_end(decl);
410 
411 	    case rw_for_or_if_or_while:
412 		return lexi_end(keyword_for_if_while);
413 
414 	    case rw_do_or_else:
415 		return lexi_end(keyword_do_else);
416 
417 	    case rw_storage_class:
418 		return lexi_end(storage_class);
419 
420 	    case rw_typedef:
421 		return lexi_end(type_def);
422 
423 	    default:		/* all others are treated like any other
424 				 * identifier */
425 		return lexi_end(ident);
426 	    }			/* end of switch */
427 	}			/* end of if (found_it) */
428 	if (*buf_ptr == '(' && state->tos <= 1 && state->ind_level == 0 &&
429 	    state->in_parameter_declaration == 0 && state->block_init == 0) {
430 	    char *tp = buf_ptr;
431 	    while (tp < buf_end)
432 		if (*tp++ == ')' && (*tp == ';' || *tp == ','))
433 		    goto not_proc;
434 	    strncpy(state->procname, token, sizeof state->procname - 1);
435 	    if (state->in_decl)
436 		state->in_parameter_declaration = 1;
437 	    return lexi_end(funcname);
438     not_proc:;
439 	}
440 	/*
441 	 * The following hack attempts to guess whether or not the current
442 	 * token is in fact a declaration keyword -- one that has been
443 	 * typedefd
444 	 */
445 	else if (!state->p_l_follow && !state->block_init &&
446 	    !state->in_stmt &&
447 	    ((*buf_ptr == '*' && buf_ptr[1] != '=') ||
448 		isalpha((unsigned char)*buf_ptr)) &&
449 	    (state->last_token == semicolon || state->last_token == lbrace ||
450 		state->last_token == rbrace)) {
451 	    state->keyword = rw_type;
452 	    state->last_u_d = true;
453 	    return lexi_end(decl);
454 	}
455 	if (state->last_token == decl)	/* if this is a declared variable,
456 					 * then following sign is unary */
457 	    state->last_u_d = true;	/* will make "int a -1" work */
458 	return lexi_end(ident);		/* the ident is not in the list */
459     }				/* end of procesing for alpanum character */
460 
461     /* Scan a non-alphanumeric token */
462 
463     check_size_token(3);	/* things like "<<=" */
464     *e_token++ = inbuf_next();	/* if it is only a one-character token, it is
465 				 * moved here */
466     *e_token = '\0';
467 
468     switch (*token) {
469     case '\n':
470 	unary_delim = state->last_u_d;
471 	state->last_nl = true;	/* remember that we just had a newline */
472 	code = (had_eof ? end_of_file : newline);
473 
474 	/*
475 	 * if data has been exhausted, the newline is a dummy, and we should
476 	 * return code to stop
477 	 */
478 	break;
479 
480     case '\'':			/* start of quoted character */
481     case '"':			/* start of string */
482 	qchar = *token;
483 	do {			/* copy the string */
484 	    for (;;) {		/* move one character or [/<char>]<char> */
485 		if (*buf_ptr == '\n') {
486 		    diag(1, "Unterminated literal");
487 		    goto stop_lit;
488 		}
489 		check_size_token(2);
490 		*e_token = inbuf_next();
491 		if (*e_token == '\\') {		/* if escape, copy extra char */
492 		    if (*buf_ptr == '\n')	/* check for escaped newline */
493 			++line_no;
494 		    *++e_token = inbuf_next();
495 		    ++e_token;	/* we must increment this again because we
496 				 * copied two chars */
497 		} else
498 		    break;	/* we copied one character */
499 	    }			/* end of while (1) */
500 	} while (*e_token++ != qchar);
501 stop_lit:
502 	code = ident;
503 	break;
504 
505     case '(':
506     case '[':
507 	unary_delim = true;
508 	code = lparen;
509 	break;
510 
511     case ')':
512     case ']':
513 	code = rparen;
514 	break;
515 
516     case '#':
517 	unary_delim = state->last_u_d;
518 	code = preprocessing;
519 	break;
520 
521     case '?':
522 	unary_delim = true;
523 	code = question;
524 	break;
525 
526     case ':':
527 	code = colon;
528 	unary_delim = true;
529 	break;
530 
531     case ';':
532 	unary_delim = true;
533 	code = semicolon;
534 	break;
535 
536     case '{':
537 	unary_delim = true;
538 
539 	/*
540 	 * if (state->in_or_st) state->block_init = 1;
541 	 */
542 	/* ?	code = state->block_init ? lparen : lbrace; */
543 	code = lbrace;
544 	break;
545 
546     case '}':
547 	unary_delim = true;
548 	/* ?	code = state->block_init ? rparen : rbrace; */
549 	code = rbrace;
550 	break;
551 
552     case 014:			/* a form feed */
553 	unary_delim = state->last_u_d;
554 	state->last_nl = true;	/* remember this so we can set 'state->col_1'
555 				 * right */
556 	code = form_feed;
557 	break;
558 
559     case ',':
560 	unary_delim = true;
561 	code = comma;
562 	break;
563 
564     case '.':
565 	unary_delim = false;
566 	code = period;
567 	break;
568 
569     case '-':
570     case '+':			/* check for -, +, --, ++ */
571 	code = (state->last_u_d ? unary_op : binary_op);
572 	unary_delim = true;
573 
574 	if (*buf_ptr == token[0]) {
575 	    /* check for doubled character */
576 	    *e_token++ = *buf_ptr++;
577 	    /* buffer overflow will be checked at end of loop */
578 	    if (state->last_token == ident || state->last_token == rparen) {
579 		code = (state->last_u_d ? unary_op : postfix_op);
580 		/* check for following ++ or -- */
581 		unary_delim = false;
582 	    }
583 	} else if (*buf_ptr == '=')
584 	    /* check for operator += */
585 	    *e_token++ = *buf_ptr++;
586 	else if (*buf_ptr == '>') {
587 	    /* check for operator -> */
588 	    *e_token++ = *buf_ptr++;
589 	    unary_delim = false;
590 	    code = unary_op;
591 	    state->want_blank = false;
592 	}
593 	break;			/* buffer overflow will be checked at end of
594 				 * switch */
595 
596     case '=':
597 	if (state->in_or_st)
598 	    state->block_init = 1;
599 	if (*buf_ptr == '=') {	/* == */
600 	    *e_token++ = '=';	/* Flip =+ to += */
601 	    buf_ptr++;
602 	    *e_token = 0;
603 	}
604 	code = binary_op;
605 	unary_delim = true;
606 	break;
607 	/* can drop thru!!! */
608 
609     case '>':
610     case '<':
611     case '!':			/* ops like <, <<, <=, !=, etc */
612 	if (*buf_ptr == '>' || *buf_ptr == '<' || *buf_ptr == '=')
613 	    *e_token++ = inbuf_next();
614 	if (*buf_ptr == '=')
615 	    *e_token++ = *buf_ptr++;
616 	code = (state->last_u_d ? unary_op : binary_op);
617 	unary_delim = true;
618 	break;
619 
620     case '*':
621 	unary_delim = true;
622 	if (!state->last_u_d) {
623 	    if (*buf_ptr == '=')
624 		*e_token++ = *buf_ptr++;
625 	    code = binary_op;
626 	    break;
627 	}
628 	while (*buf_ptr == '*' || isspace((unsigned char)*buf_ptr)) {
629 	    if (*buf_ptr == '*') {
630 		check_size_token(1);
631 		*e_token++ = *buf_ptr;
632 	    }
633 	    inbuf_skip();
634 	}
635 	if (ps.in_decl) {
636 	    char *tp = buf_ptr;
637 
638 	    while (isalpha((unsigned char)*tp) ||
639 		   isspace((unsigned char)*tp)) {
640 		if (++tp >= buf_end)
641 		    fill_buffer();
642 	    }
643 	    if (*tp == '(')
644 		ps.procname[0] = ' ';
645 	}
646 	code = unary_op;
647 	break;
648 
649     default:
650 	if (token[0] == '/' && (*buf_ptr == '*' || *buf_ptr == '/')) {
651 	    /* it is start of comment */
652 	    *e_token++ = inbuf_next();
653 
654 	    code = comment;
655 	    unary_delim = state->last_u_d;
656 	    break;
657 	}
658 	while (e_token[-1] == *buf_ptr || *buf_ptr == '=') {
659 	    /*
660 	     * handle ||, &&, etc, and also things as in int *****i
661 	     */
662 	    check_size_token(1);
663 	    *e_token++ = inbuf_next();
664 	}
665 	code = (state->last_u_d ? unary_op : binary_op);
666 	unary_delim = true;
667 
668 
669     }				/* end of switch */
670     if (buf_ptr >= buf_end)	/* check for input buffer empty */
671 	fill_buffer();
672     state->last_u_d = unary_delim;
673     check_size_token(1);
674     *e_token = '\0';		/* null terminate the token */
675     return lexi_end(code);
676 }
677 
678 void
679 alloc_typenames(void)
680 {
681 
682     typenames = malloc(sizeof(typenames[0]) * (typename_count = 16));
683     if (typenames == NULL)
684 	err(1, NULL);
685 }
686 
687 void
688 add_typename(const char *key)
689 {
690     int comparison;
691     const char *copy;
692 
693     if (typename_top + 1 >= typename_count) {
694 	typenames = realloc((void *)typenames,
695 	    sizeof(typenames[0]) * (typename_count *= 2));
696 	if (typenames == NULL)
697 	    err(1, NULL);
698     }
699     if (typename_top == -1)
700 	typenames[++typename_top] = copy = strdup(key);
701     else if ((comparison = strcmp(key, typenames[typename_top])) >= 0) {
702 	/* take advantage of sorted input */
703 	if (comparison == 0)	/* remove duplicates */
704 	    return;
705 	typenames[++typename_top] = copy = strdup(key);
706     } else {
707 	int p;
708 
709 	for (p = 0; (comparison = strcmp(key, typenames[p])) > 0; p++)
710 	    /* find place for the new key */;
711 	if (comparison == 0)	/* remove duplicates */
712 	    return;
713 	memmove(&typenames[p + 1], &typenames[p],
714 	    sizeof(typenames[0]) * (++typename_top - p));
715 	typenames[p] = copy = strdup(key);
716     }
717 
718     if (copy == NULL)
719 	err(1, NULL);
720 }
721