1 /* CPP Library - lexical analysis. 2 Copyright (C) 2000-2019 Free Software Foundation, Inc. 3 Contributed by Per Bothner, 1994-95. 4 Based on CCCP program by Paul Rubin, June 1986 5 Adapted to ANSI C, Richard Stallman, Jan 1987 6 Broken out to separate file, Zack Weinberg, Mar 2000 7 8 This program is free software; you can redistribute it and/or modify it 9 under the terms of the GNU General Public License as published by the 10 Free Software Foundation; either version 3, or (at your option) any 11 later version. 12 13 This program is distributed in the hope that it will be useful, 14 but WITHOUT ANY WARRANTY; without even the implied warranty of 15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 GNU General Public License for more details. 17 18 You should have received a copy of the GNU General Public License 19 along with this program; see the file COPYING3. If not see 20 <http://www.gnu.org/licenses/>. */ 21 22 #include "config.h" 23 #include "system.h" 24 #include "cpplib.h" 25 #include "internal.h" 26 27 enum spell_type 28 { 29 SPELL_OPERATOR = 0, 30 SPELL_IDENT, 31 SPELL_LITERAL, 32 SPELL_NONE 33 }; 34 35 struct token_spelling 36 { 37 enum spell_type category; 38 const unsigned char *name; 39 }; 40 41 static const unsigned char *const digraph_spellings[] = 42 { UC"%:", UC"%:%:", UC"<:", UC":>", UC"<%", UC"%>" }; 43 44 #define OP(e, s) { SPELL_OPERATOR, UC s }, 45 #define TK(e, s) { SPELL_ ## s, UC #e }, 46 static const struct token_spelling token_spellings[N_TTYPES] = { TTYPE_TABLE }; 47 #undef OP 48 #undef TK 49 50 #define TOKEN_SPELL(token) (token_spellings[(token)->type].category) 51 #define TOKEN_NAME(token) (token_spellings[(token)->type].name) 52 53 static void add_line_note (cpp_buffer *, const uchar *, unsigned int); 54 static int skip_line_comment (cpp_reader *); 55 static void skip_whitespace (cpp_reader *, cppchar_t); 56 static void lex_string (cpp_reader *, cpp_token *, const uchar *); 57 static void save_comment (cpp_reader *, cpp_token *, const uchar *, cppchar_t); 58 static void store_comment (cpp_reader *, cpp_token *); 59 static void create_literal (cpp_reader *, cpp_token *, const uchar *, 60 unsigned int, enum cpp_ttype); 61 static bool warn_in_comment (cpp_reader *, _cpp_line_note *); 62 static int name_p (cpp_reader *, const cpp_string *); 63 static tokenrun *next_tokenrun (tokenrun *); 64 65 static _cpp_buff *new_buff (size_t); 66 67 68 /* Utility routine: 69 70 Compares, the token TOKEN to the NUL-terminated string STRING. 71 TOKEN must be a CPP_NAME. Returns 1 for equal, 0 for unequal. */ 72 int 73 cpp_ideq (const cpp_token *token, const char *string) 74 { 75 if (token->type != CPP_NAME) 76 return 0; 77 78 return !ustrcmp (NODE_NAME (token->val.node.node), (const uchar *) string); 79 } 80 81 /* Record a note TYPE at byte POS into the current cleaned logical 82 line. */ 83 static void 84 add_line_note (cpp_buffer *buffer, const uchar *pos, unsigned int type) 85 { 86 if (buffer->notes_used == buffer->notes_cap) 87 { 88 buffer->notes_cap = buffer->notes_cap * 2 + 200; 89 buffer->notes = XRESIZEVEC (_cpp_line_note, buffer->notes, 90 buffer->notes_cap); 91 } 92 93 buffer->notes[buffer->notes_used].pos = pos; 94 buffer->notes[buffer->notes_used].type = type; 95 buffer->notes_used++; 96 } 97 98 99 /* Fast path to find line special characters using optimized character 100 scanning algorithms. Anything complicated falls back to the slow 101 path below. Since this loop is very hot it's worth doing these kinds 102 of optimizations. 103 104 One of the paths through the ifdefs should provide 105 106 const uchar *search_line_fast (const uchar *s, const uchar *end); 107 108 Between S and END, search for \n, \r, \\, ?. Return a pointer to 109 the found character. 110 111 Note that the last character of the buffer is *always* a newline, 112 as forced by _cpp_convert_input. This fact can be used to avoid 113 explicitly looking for the end of the buffer. */ 114 115 /* Configure gives us an ifdef test. */ 116 #ifndef WORDS_BIGENDIAN 117 #define WORDS_BIGENDIAN 0 118 #endif 119 120 /* We'd like the largest integer that fits into a register. There's nothing 121 in <stdint.h> that gives us that. For most hosts this is unsigned long, 122 but MS decided on an LLP64 model. Thankfully when building with GCC we 123 can get the "real" word size. */ 124 #ifdef __GNUC__ 125 typedef unsigned int word_type __attribute__((__mode__(__word__))); 126 #else 127 typedef unsigned long word_type; 128 #endif 129 130 /* The code below is only expecting sizes 4 or 8. 131 Die at compile-time if this expectation is violated. */ 132 typedef char check_word_type_size 133 [(sizeof(word_type) == 8 || sizeof(word_type) == 4) * 2 - 1]; 134 135 /* Return X with the first N bytes forced to values that won't match one 136 of the interesting characters. Note that NUL is not interesting. */ 137 138 static inline word_type 139 acc_char_mask_misalign (word_type val, unsigned int n) 140 { 141 word_type mask = -1; 142 if (WORDS_BIGENDIAN) 143 mask >>= n * 8; 144 else 145 mask <<= n * 8; 146 return val & mask; 147 } 148 149 /* Return X replicated to all byte positions within WORD_TYPE. */ 150 151 static inline word_type 152 acc_char_replicate (uchar x) 153 { 154 word_type ret; 155 156 ret = (x << 24) | (x << 16) | (x << 8) | x; 157 if (sizeof(word_type) == 8) 158 ret = (ret << 16 << 16) | ret; 159 return ret; 160 } 161 162 /* Return non-zero if some byte of VAL is (probably) C. */ 163 164 static inline word_type 165 acc_char_cmp (word_type val, word_type c) 166 { 167 #if defined(__GNUC__) && defined(__alpha__) 168 /* We can get exact results using a compare-bytes instruction. 169 Get (val == c) via (0 >= (val ^ c)). */ 170 return __builtin_alpha_cmpbge (0, val ^ c); 171 #else 172 word_type magic = 0x7efefefeU; 173 if (sizeof(word_type) == 8) 174 magic = (magic << 16 << 16) | 0xfefefefeU; 175 magic |= 1; 176 177 val ^= c; 178 return ((val + magic) ^ ~val) & ~magic; 179 #endif 180 } 181 182 /* Given the result of acc_char_cmp is non-zero, return the index of 183 the found character. If this was a false positive, return -1. */ 184 185 static inline int 186 acc_char_index (word_type cmp ATTRIBUTE_UNUSED, 187 word_type val ATTRIBUTE_UNUSED) 188 { 189 #if defined(__GNUC__) && defined(__alpha__) && !WORDS_BIGENDIAN 190 /* The cmpbge instruction sets *bits* of the result corresponding to 191 matches in the bytes with no false positives. */ 192 return __builtin_ctzl (cmp); 193 #else 194 unsigned int i; 195 196 /* ??? It would be nice to force unrolling here, 197 and have all of these constants folded. */ 198 for (i = 0; i < sizeof(word_type); ++i) 199 { 200 uchar c; 201 if (WORDS_BIGENDIAN) 202 c = (val >> (sizeof(word_type) - i - 1) * 8) & 0xff; 203 else 204 c = (val >> i * 8) & 0xff; 205 206 if (c == '\n' || c == '\r' || c == '\\' || c == '?') 207 return i; 208 } 209 210 return -1; 211 #endif 212 } 213 214 /* A version of the fast scanner using bit fiddling techniques. 215 216 For 32-bit words, one would normally perform 16 comparisons and 217 16 branches. With this algorithm one performs 24 arithmetic 218 operations and one branch. Whether this is faster with a 32-bit 219 word size is going to be somewhat system dependent. 220 221 For 64-bit words, we eliminate twice the number of comparisons 222 and branches without increasing the number of arithmetic operations. 223 It's almost certainly going to be a win with 64-bit word size. */ 224 225 static const uchar * search_line_acc_char (const uchar *, const uchar *) 226 ATTRIBUTE_UNUSED; 227 228 static const uchar * 229 search_line_acc_char (const uchar *s, const uchar *end ATTRIBUTE_UNUSED) 230 { 231 const word_type repl_nl = acc_char_replicate ('\n'); 232 const word_type repl_cr = acc_char_replicate ('\r'); 233 const word_type repl_bs = acc_char_replicate ('\\'); 234 const word_type repl_qm = acc_char_replicate ('?'); 235 236 unsigned int misalign; 237 const word_type *p; 238 word_type val, t; 239 240 /* Align the buffer. Mask out any bytes from before the beginning. */ 241 p = (word_type *)((uintptr_t)s & -sizeof(word_type)); 242 val = *p; 243 misalign = (uintptr_t)s & (sizeof(word_type) - 1); 244 if (misalign) 245 val = acc_char_mask_misalign (val, misalign); 246 247 /* Main loop. */ 248 while (1) 249 { 250 t = acc_char_cmp (val, repl_nl); 251 t |= acc_char_cmp (val, repl_cr); 252 t |= acc_char_cmp (val, repl_bs); 253 t |= acc_char_cmp (val, repl_qm); 254 255 if (__builtin_expect (t != 0, 0)) 256 { 257 int i = acc_char_index (t, val); 258 if (i >= 0) 259 return (const uchar *)p + i; 260 } 261 262 val = *++p; 263 } 264 } 265 266 /* Disable on Solaris 2/x86 until the following problem can be properly 267 autoconfed: 268 269 The Solaris 10+ assembler tags objects with the instruction set 270 extensions used, so SSE4.2 executables cannot run on machines that 271 don't support that extension. */ 272 273 #if (GCC_VERSION >= 4005) && (__GNUC__ >= 5 || !defined(__PIC__)) && (defined(__i386__) || defined(__x86_64__)) && !(defined(__sun__) && defined(__svr4__)) 274 275 /* Replicated character data to be shared between implementations. 276 Recall that outside of a context with vector support we can't 277 define compatible vector types, therefore these are all defined 278 in terms of raw characters. */ 279 static const char repl_chars[4][16] __attribute__((aligned(16))) = { 280 { '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n', 281 '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n' }, 282 { '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r', 283 '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r' }, 284 { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', 285 '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }, 286 { '?', '?', '?', '?', '?', '?', '?', '?', 287 '?', '?', '?', '?', '?', '?', '?', '?' }, 288 }; 289 290 /* A version of the fast scanner using MMX vectorized byte compare insns. 291 292 This uses the PMOVMSKB instruction which was introduced with "MMX2", 293 which was packaged into SSE1; it is also present in the AMD MMX 294 extension. Mark the function as using "sse" so that we emit a real 295 "emms" instruction, rather than the 3dNOW "femms" instruction. */ 296 297 static const uchar * 298 #ifndef __SSE__ 299 __attribute__((__target__("sse"))) 300 #endif 301 search_line_mmx (const uchar *s, const uchar *end ATTRIBUTE_UNUSED) 302 { 303 typedef char v8qi __attribute__ ((__vector_size__ (8))); 304 typedef int __m64 __attribute__ ((__vector_size__ (8), __may_alias__)); 305 306 const v8qi repl_nl = *(const v8qi *)repl_chars[0]; 307 const v8qi repl_cr = *(const v8qi *)repl_chars[1]; 308 const v8qi repl_bs = *(const v8qi *)repl_chars[2]; 309 const v8qi repl_qm = *(const v8qi *)repl_chars[3]; 310 311 unsigned int misalign, found, mask; 312 const v8qi *p; 313 v8qi data, t, c; 314 315 /* Align the source pointer. While MMX doesn't generate unaligned data 316 faults, this allows us to safely scan to the end of the buffer without 317 reading beyond the end of the last page. */ 318 misalign = (uintptr_t)s & 7; 319 p = (const v8qi *)((uintptr_t)s & -8); 320 data = *p; 321 322 /* Create a mask for the bytes that are valid within the first 323 16-byte block. The Idea here is that the AND with the mask 324 within the loop is "free", since we need some AND or TEST 325 insn in order to set the flags for the branch anyway. */ 326 mask = -1u << misalign; 327 328 /* Main loop processing 8 bytes at a time. */ 329 goto start; 330 do 331 { 332 data = *++p; 333 mask = -1; 334 335 start: 336 t = __builtin_ia32_pcmpeqb(data, repl_nl); 337 c = __builtin_ia32_pcmpeqb(data, repl_cr); 338 t = (v8qi) __builtin_ia32_por ((__m64)t, (__m64)c); 339 c = __builtin_ia32_pcmpeqb(data, repl_bs); 340 t = (v8qi) __builtin_ia32_por ((__m64)t, (__m64)c); 341 c = __builtin_ia32_pcmpeqb(data, repl_qm); 342 t = (v8qi) __builtin_ia32_por ((__m64)t, (__m64)c); 343 found = __builtin_ia32_pmovmskb (t); 344 found &= mask; 345 } 346 while (!found); 347 348 __builtin_ia32_emms (); 349 350 /* FOUND contains 1 in bits for which we matched a relevant 351 character. Conversion to the byte index is trivial. */ 352 found = __builtin_ctz(found); 353 return (const uchar *)p + found; 354 } 355 356 /* A version of the fast scanner using SSE2 vectorized byte compare insns. */ 357 358 static const uchar * 359 #ifndef __SSE2__ 360 __attribute__((__target__("sse2"))) 361 #endif 362 search_line_sse2 (const uchar *s, const uchar *end ATTRIBUTE_UNUSED) 363 { 364 typedef char v16qi __attribute__ ((__vector_size__ (16))); 365 366 const v16qi repl_nl = *(const v16qi *)repl_chars[0]; 367 const v16qi repl_cr = *(const v16qi *)repl_chars[1]; 368 const v16qi repl_bs = *(const v16qi *)repl_chars[2]; 369 const v16qi repl_qm = *(const v16qi *)repl_chars[3]; 370 371 unsigned int misalign, found, mask; 372 const v16qi *p; 373 v16qi data, t; 374 375 /* Align the source pointer. */ 376 misalign = (uintptr_t)s & 15; 377 p = (const v16qi *)((uintptr_t)s & -16); 378 data = *p; 379 380 /* Create a mask for the bytes that are valid within the first 381 16-byte block. The Idea here is that the AND with the mask 382 within the loop is "free", since we need some AND or TEST 383 insn in order to set the flags for the branch anyway. */ 384 mask = -1u << misalign; 385 386 /* Main loop processing 16 bytes at a time. */ 387 goto start; 388 do 389 { 390 data = *++p; 391 mask = -1; 392 393 start: 394 t = __builtin_ia32_pcmpeqb128(data, repl_nl); 395 t |= __builtin_ia32_pcmpeqb128(data, repl_cr); 396 t |= __builtin_ia32_pcmpeqb128(data, repl_bs); 397 t |= __builtin_ia32_pcmpeqb128(data, repl_qm); 398 found = __builtin_ia32_pmovmskb128 (t); 399 found &= mask; 400 } 401 while (!found); 402 403 /* FOUND contains 1 in bits for which we matched a relevant 404 character. Conversion to the byte index is trivial. */ 405 found = __builtin_ctz(found); 406 return (const uchar *)p + found; 407 } 408 409 #ifdef HAVE_SSE4 410 /* A version of the fast scanner using SSE 4.2 vectorized string insns. */ 411 412 static const uchar * 413 #ifndef __SSE4_2__ 414 __attribute__((__target__("sse4.2"))) 415 #endif 416 search_line_sse42 (const uchar *s, const uchar *end) 417 { 418 typedef char v16qi __attribute__ ((__vector_size__ (16))); 419 static const v16qi search = { '\n', '\r', '?', '\\' }; 420 421 uintptr_t si = (uintptr_t)s; 422 uintptr_t index; 423 424 /* Check for unaligned input. */ 425 if (si & 15) 426 { 427 v16qi sv; 428 429 if (__builtin_expect (end - s < 16, 0) 430 && __builtin_expect ((si & 0xfff) > 0xff0, 0)) 431 { 432 /* There are less than 16 bytes left in the buffer, and less 433 than 16 bytes left on the page. Reading 16 bytes at this 434 point might generate a spurious page fault. Defer to the 435 SSE2 implementation, which already handles alignment. */ 436 return search_line_sse2 (s, end); 437 } 438 439 /* ??? The builtin doesn't understand that the PCMPESTRI read from 440 memory need not be aligned. */ 441 sv = __builtin_ia32_loaddqu ((const char *) s); 442 index = __builtin_ia32_pcmpestri128 (search, 4, sv, 16, 0); 443 444 if (__builtin_expect (index < 16, 0)) 445 goto found; 446 447 /* Advance the pointer to an aligned address. We will re-scan a 448 few bytes, but we no longer need care for reading past the 449 end of a page, since we're guaranteed a match. */ 450 s = (const uchar *)((si + 15) & -16); 451 } 452 453 /* Main loop, processing 16 bytes at a time. */ 454 #ifdef __GCC_ASM_FLAG_OUTPUTS__ 455 while (1) 456 { 457 char f; 458 459 /* By using inline assembly instead of the builtin, 460 we can use the result, as well as the flags set. */ 461 __asm ("%vpcmpestri\t$0, %2, %3" 462 : "=c"(index), "=@ccc"(f) 463 : "m"(*s), "x"(search), "a"(4), "d"(16)); 464 if (f) 465 break; 466 467 s += 16; 468 } 469 #else 470 s -= 16; 471 /* By doing the whole loop in inline assembly, 472 we can make proper use of the flags set. */ 473 __asm ( ".balign 16\n" 474 "0: add $16, %1\n" 475 " %vpcmpestri\t$0, (%1), %2\n" 476 " jnc 0b" 477 : "=&c"(index), "+r"(s) 478 : "x"(search), "a"(4), "d"(16)); 479 #endif 480 481 found: 482 return s + index; 483 } 484 485 #else 486 /* Work around out-dated assemblers without sse4 support. */ 487 #define search_line_sse42 search_line_sse2 488 #endif 489 490 /* Check the CPU capabilities. */ 491 492 #include "../gcc/config/i386/cpuid.h" 493 494 typedef const uchar * (*search_line_fast_type) (const uchar *, const uchar *); 495 static search_line_fast_type search_line_fast; 496 497 #define HAVE_init_vectorized_lexer 1 498 static inline void 499 init_vectorized_lexer (void) 500 { 501 unsigned dummy, ecx = 0, edx = 0; 502 search_line_fast_type impl = search_line_acc_char; 503 int minimum = 0; 504 505 #if defined(__SSE4_2__) 506 minimum = 3; 507 #elif defined(__SSE2__) 508 minimum = 2; 509 #elif defined(__SSE__) 510 minimum = 1; 511 #endif 512 513 if (minimum == 3) 514 impl = search_line_sse42; 515 else if (__get_cpuid (1, &dummy, &dummy, &ecx, &edx) || minimum == 2) 516 { 517 if (minimum == 3 || (ecx & bit_SSE4_2)) 518 impl = search_line_sse42; 519 else if (minimum == 2 || (edx & bit_SSE2)) 520 impl = search_line_sse2; 521 else if (minimum == 1 || (edx & bit_SSE)) 522 impl = search_line_mmx; 523 } 524 else if (__get_cpuid (0x80000001, &dummy, &dummy, &dummy, &edx)) 525 { 526 if (minimum == 1 527 || (edx & (bit_MMXEXT | bit_CMOV)) == (bit_MMXEXT | bit_CMOV)) 528 impl = search_line_mmx; 529 } 530 531 search_line_fast = impl; 532 } 533 534 #elif defined(_ARCH_PWR8) && defined(__ALTIVEC__) 535 536 /* A vection of the fast scanner using AltiVec vectorized byte compares 537 and VSX unaligned loads (when VSX is available). This is otherwise 538 the same as the pre-GCC 5 version. */ 539 540 ATTRIBUTE_NO_SANITIZE_UNDEFINED 541 static const uchar * 542 search_line_fast (const uchar *s, const uchar *end ATTRIBUTE_UNUSED) 543 { 544 typedef __attribute__((altivec(vector))) unsigned char vc; 545 546 const vc repl_nl = { 547 '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n', 548 '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n' 549 }; 550 const vc repl_cr = { 551 '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r', 552 '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r' 553 }; 554 const vc repl_bs = { 555 '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', 556 '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' 557 }; 558 const vc repl_qm = { 559 '?', '?', '?', '?', '?', '?', '?', '?', 560 '?', '?', '?', '?', '?', '?', '?', '?', 561 }; 562 const vc zero = { 0 }; 563 564 vc data, t; 565 566 /* Main loop processing 16 bytes at a time. */ 567 do 568 { 569 vc m_nl, m_cr, m_bs, m_qm; 570 571 data = __builtin_vec_vsx_ld (0, s); 572 s += 16; 573 574 m_nl = (vc) __builtin_vec_cmpeq(data, repl_nl); 575 m_cr = (vc) __builtin_vec_cmpeq(data, repl_cr); 576 m_bs = (vc) __builtin_vec_cmpeq(data, repl_bs); 577 m_qm = (vc) __builtin_vec_cmpeq(data, repl_qm); 578 t = (m_nl | m_cr) | (m_bs | m_qm); 579 580 /* T now contains 0xff in bytes for which we matched one of the relevant 581 characters. We want to exit the loop if any byte in T is non-zero. 582 Below is the expansion of vec_any_ne(t, zero). */ 583 } 584 while (!__builtin_vec_vcmpeq_p(/*__CR6_LT_REV*/3, t, zero)); 585 586 /* Restore s to to point to the 16 bytes we just processed. */ 587 s -= 16; 588 589 { 590 #define N (sizeof(vc) / sizeof(long)) 591 592 union { 593 vc v; 594 /* Statically assert that N is 2 or 4. */ 595 unsigned long l[(N == 2 || N == 4) ? N : -1]; 596 } u; 597 unsigned long l, i = 0; 598 599 u.v = t; 600 601 /* Find the first word of T that is non-zero. */ 602 switch (N) 603 { 604 case 4: 605 l = u.l[i++]; 606 if (l != 0) 607 break; 608 s += sizeof(unsigned long); 609 l = u.l[i++]; 610 if (l != 0) 611 break; 612 s += sizeof(unsigned long); 613 /* FALLTHRU */ 614 case 2: 615 l = u.l[i++]; 616 if (l != 0) 617 break; 618 s += sizeof(unsigned long); 619 l = u.l[i]; 620 } 621 622 /* L now contains 0xff in bytes for which we matched one of the 623 relevant characters. We can find the byte index by finding 624 its bit index and dividing by 8. */ 625 #ifdef __BIG_ENDIAN__ 626 l = __builtin_clzl(l) >> 3; 627 #else 628 l = __builtin_ctzl(l) >> 3; 629 #endif 630 return s + l; 631 632 #undef N 633 } 634 } 635 636 #elif (GCC_VERSION >= 4005) && defined(__ALTIVEC__) && defined (__BIG_ENDIAN__) 637 638 /* A vection of the fast scanner using AltiVec vectorized byte compares. 639 This cannot be used for little endian because vec_lvsl/lvsr are 640 deprecated for little endian and the code won't work properly. */ 641 /* ??? Unfortunately, attribute(target("altivec")) is not yet supported, 642 so we can't compile this function without -maltivec on the command line 643 (or implied by some other switch). */ 644 645 static const uchar * 646 search_line_fast (const uchar *s, const uchar *end ATTRIBUTE_UNUSED) 647 { 648 typedef __attribute__((altivec(vector))) unsigned char vc; 649 650 const vc repl_nl = { 651 '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n', 652 '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n' 653 }; 654 const vc repl_cr = { 655 '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r', 656 '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r' 657 }; 658 const vc repl_bs = { 659 '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', 660 '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' 661 }; 662 const vc repl_qm = { 663 '?', '?', '?', '?', '?', '?', '?', '?', 664 '?', '?', '?', '?', '?', '?', '?', '?', 665 }; 666 const vc ones = { 667 -1, -1, -1, -1, -1, -1, -1, -1, 668 -1, -1, -1, -1, -1, -1, -1, -1, 669 }; 670 const vc zero = { 0 }; 671 672 vc data, mask, t; 673 674 /* Altivec loads automatically mask addresses with -16. This lets us 675 issue the first load as early as possible. */ 676 data = __builtin_vec_ld(0, (const vc *)s); 677 678 /* Discard bytes before the beginning of the buffer. Do this by 679 beginning with all ones and shifting in zeros according to the 680 mis-alignment. The LVSR instruction pulls the exact shift we 681 want from the address. */ 682 mask = __builtin_vec_lvsr(0, s); 683 mask = __builtin_vec_perm(zero, ones, mask); 684 data &= mask; 685 686 /* While altivec loads mask addresses, we still need to align S so 687 that the offset we compute at the end is correct. */ 688 s = (const uchar *)((uintptr_t)s & -16); 689 690 /* Main loop processing 16 bytes at a time. */ 691 goto start; 692 do 693 { 694 vc m_nl, m_cr, m_bs, m_qm; 695 696 s += 16; 697 data = __builtin_vec_ld(0, (const vc *)s); 698 699 start: 700 m_nl = (vc) __builtin_vec_cmpeq(data, repl_nl); 701 m_cr = (vc) __builtin_vec_cmpeq(data, repl_cr); 702 m_bs = (vc) __builtin_vec_cmpeq(data, repl_bs); 703 m_qm = (vc) __builtin_vec_cmpeq(data, repl_qm); 704 t = (m_nl | m_cr) | (m_bs | m_qm); 705 706 /* T now contains 0xff in bytes for which we matched one of the relevant 707 characters. We want to exit the loop if any byte in T is non-zero. 708 Below is the expansion of vec_any_ne(t, zero). */ 709 } 710 while (!__builtin_vec_vcmpeq_p(/*__CR6_LT_REV*/3, t, zero)); 711 712 { 713 #define N (sizeof(vc) / sizeof(long)) 714 715 union { 716 vc v; 717 /* Statically assert that N is 2 or 4. */ 718 unsigned long l[(N == 2 || N == 4) ? N : -1]; 719 } u; 720 unsigned long l, i = 0; 721 722 u.v = t; 723 724 /* Find the first word of T that is non-zero. */ 725 switch (N) 726 { 727 case 4: 728 l = u.l[i++]; 729 if (l != 0) 730 break; 731 s += sizeof(unsigned long); 732 l = u.l[i++]; 733 if (l != 0) 734 break; 735 s += sizeof(unsigned long); 736 /* FALLTHROUGH */ 737 case 2: 738 l = u.l[i++]; 739 if (l != 0) 740 break; 741 s += sizeof(unsigned long); 742 l = u.l[i]; 743 } 744 745 /* L now contains 0xff in bytes for which we matched one of the 746 relevant characters. We can find the byte index by finding 747 its bit index and dividing by 8. */ 748 l = __builtin_clzl(l) >> 3; 749 return s + l; 750 751 #undef N 752 } 753 } 754 755 #elif defined (__ARM_NEON) && defined (__ARM_64BIT_STATE) 756 #include "arm_neon.h" 757 758 /* This doesn't have to be the exact page size, but no system may use 759 a size smaller than this. ARMv8 requires a minimum page size of 760 4k. The impact of being conservative here is a small number of 761 cases will take the slightly slower entry path into the main 762 loop. */ 763 764 #define AARCH64_MIN_PAGE_SIZE 4096 765 766 static const uchar * 767 search_line_fast (const uchar *s, const uchar *end ATTRIBUTE_UNUSED) 768 { 769 const uint8x16_t repl_nl = vdupq_n_u8 ('\n'); 770 const uint8x16_t repl_cr = vdupq_n_u8 ('\r'); 771 const uint8x16_t repl_bs = vdupq_n_u8 ('\\'); 772 const uint8x16_t repl_qm = vdupq_n_u8 ('?'); 773 const uint8x16_t xmask = (uint8x16_t) vdupq_n_u64 (0x8040201008040201ULL); 774 775 #ifdef __ARM_BIG_ENDIAN 776 const int16x8_t shift = {8, 8, 8, 8, 0, 0, 0, 0}; 777 #else 778 const int16x8_t shift = {0, 0, 0, 0, 8, 8, 8, 8}; 779 #endif 780 781 unsigned int found; 782 const uint8_t *p; 783 uint8x16_t data; 784 uint8x16_t t; 785 uint16x8_t m; 786 uint8x16_t u, v, w; 787 788 /* Align the source pointer. */ 789 p = (const uint8_t *)((uintptr_t)s & -16); 790 791 /* Assuming random string start positions, with a 4k page size we'll take 792 the slow path about 0.37% of the time. */ 793 if (__builtin_expect ((AARCH64_MIN_PAGE_SIZE 794 - (((uintptr_t) s) & (AARCH64_MIN_PAGE_SIZE - 1))) 795 < 16, 0)) 796 { 797 /* Slow path: the string starts near a possible page boundary. */ 798 uint32_t misalign, mask; 799 800 misalign = (uintptr_t)s & 15; 801 mask = (-1u << misalign) & 0xffff; 802 data = vld1q_u8 (p); 803 t = vceqq_u8 (data, repl_nl); 804 u = vceqq_u8 (data, repl_cr); 805 v = vorrq_u8 (t, vceqq_u8 (data, repl_bs)); 806 w = vorrq_u8 (u, vceqq_u8 (data, repl_qm)); 807 t = vorrq_u8 (v, w); 808 t = vandq_u8 (t, xmask); 809 m = vpaddlq_u8 (t); 810 m = vshlq_u16 (m, shift); 811 found = vaddvq_u16 (m); 812 found &= mask; 813 if (found) 814 return (const uchar*)p + __builtin_ctz (found); 815 } 816 else 817 { 818 data = vld1q_u8 ((const uint8_t *) s); 819 t = vceqq_u8 (data, repl_nl); 820 u = vceqq_u8 (data, repl_cr); 821 v = vorrq_u8 (t, vceqq_u8 (data, repl_bs)); 822 w = vorrq_u8 (u, vceqq_u8 (data, repl_qm)); 823 t = vorrq_u8 (v, w); 824 if (__builtin_expect (vpaddd_u64 ((uint64x2_t)t) != 0, 0)) 825 goto done; 826 } 827 828 do 829 { 830 p += 16; 831 data = vld1q_u8 (p); 832 t = vceqq_u8 (data, repl_nl); 833 u = vceqq_u8 (data, repl_cr); 834 v = vorrq_u8 (t, vceqq_u8 (data, repl_bs)); 835 w = vorrq_u8 (u, vceqq_u8 (data, repl_qm)); 836 t = vorrq_u8 (v, w); 837 } while (!vpaddd_u64 ((uint64x2_t)t)); 838 839 done: 840 /* Now that we've found the terminating substring, work out precisely where 841 we need to stop. */ 842 t = vandq_u8 (t, xmask); 843 m = vpaddlq_u8 (t); 844 m = vshlq_u16 (m, shift); 845 found = vaddvq_u16 (m); 846 return (((((uintptr_t) p) < (uintptr_t) s) ? s : (const uchar *)p) 847 + __builtin_ctz (found)); 848 } 849 850 #elif defined (__ARM_NEON) 851 #include "arm_neon.h" 852 853 static const uchar * 854 search_line_fast (const uchar *s, const uchar *end ATTRIBUTE_UNUSED) 855 { 856 const uint8x16_t repl_nl = vdupq_n_u8 ('\n'); 857 const uint8x16_t repl_cr = vdupq_n_u8 ('\r'); 858 const uint8x16_t repl_bs = vdupq_n_u8 ('\\'); 859 const uint8x16_t repl_qm = vdupq_n_u8 ('?'); 860 const uint8x16_t xmask = (uint8x16_t) vdupq_n_u64 (0x8040201008040201ULL); 861 862 unsigned int misalign, found, mask; 863 const uint8_t *p; 864 uint8x16_t data; 865 866 /* Align the source pointer. */ 867 misalign = (uintptr_t)s & 15; 868 p = (const uint8_t *)((uintptr_t)s & -16); 869 data = vld1q_u8 (p); 870 871 /* Create a mask for the bytes that are valid within the first 872 16-byte block. The Idea here is that the AND with the mask 873 within the loop is "free", since we need some AND or TEST 874 insn in order to set the flags for the branch anyway. */ 875 mask = (-1u << misalign) & 0xffff; 876 877 /* Main loop, processing 16 bytes at a time. */ 878 goto start; 879 880 do 881 { 882 uint8x8_t l; 883 uint16x4_t m; 884 uint32x2_t n; 885 uint8x16_t t, u, v, w; 886 887 p += 16; 888 data = vld1q_u8 (p); 889 mask = 0xffff; 890 891 start: 892 t = vceqq_u8 (data, repl_nl); 893 u = vceqq_u8 (data, repl_cr); 894 v = vorrq_u8 (t, vceqq_u8 (data, repl_bs)); 895 w = vorrq_u8 (u, vceqq_u8 (data, repl_qm)); 896 t = vandq_u8 (vorrq_u8 (v, w), xmask); 897 l = vpadd_u8 (vget_low_u8 (t), vget_high_u8 (t)); 898 m = vpaddl_u8 (l); 899 n = vpaddl_u16 (m); 900 901 found = vget_lane_u32 ((uint32x2_t) vorr_u64 ((uint64x1_t) n, 902 vshr_n_u64 ((uint64x1_t) n, 24)), 0); 903 found &= mask; 904 } 905 while (!found); 906 907 /* FOUND contains 1 in bits for which we matched a relevant 908 character. Conversion to the byte index is trivial. */ 909 found = __builtin_ctz (found); 910 return (const uchar *)p + found; 911 } 912 913 #else 914 915 /* We only have one accelerated alternative. Use a direct call so that 916 we encourage inlining. */ 917 918 #define search_line_fast search_line_acc_char 919 920 #endif 921 922 /* Initialize the lexer if needed. */ 923 924 void 925 _cpp_init_lexer (void) 926 { 927 #ifdef HAVE_init_vectorized_lexer 928 init_vectorized_lexer (); 929 #endif 930 } 931 932 /* Returns with a logical line that contains no escaped newlines or 933 trigraphs. This is a time-critical inner loop. */ 934 void 935 _cpp_clean_line (cpp_reader *pfile) 936 { 937 cpp_buffer *buffer; 938 const uchar *s; 939 uchar c, *d, *p; 940 941 buffer = pfile->buffer; 942 buffer->cur_note = buffer->notes_used = 0; 943 buffer->cur = buffer->line_base = buffer->next_line; 944 buffer->need_line = false; 945 s = buffer->next_line; 946 947 if (!buffer->from_stage3) 948 { 949 const uchar *pbackslash = NULL; 950 951 /* Fast path. This is the common case of an un-escaped line with 952 no trigraphs. The primary win here is by not writing any 953 data back to memory until we have to. */ 954 while (1) 955 { 956 /* Perform an optimized search for \n, \r, \\, ?. */ 957 s = search_line_fast (s, buffer->rlimit); 958 959 c = *s; 960 if (c == '\\') 961 { 962 /* Record the location of the backslash and continue. */ 963 pbackslash = s++; 964 } 965 else if (__builtin_expect (c == '?', 0)) 966 { 967 if (__builtin_expect (s[1] == '?', false) 968 && _cpp_trigraph_map[s[2]]) 969 { 970 /* Have a trigraph. We may or may not have to convert 971 it. Add a line note regardless, for -Wtrigraphs. */ 972 add_line_note (buffer, s, s[2]); 973 if (CPP_OPTION (pfile, trigraphs)) 974 { 975 /* We do, and that means we have to switch to the 976 slow path. */ 977 d = (uchar *) s; 978 *d = _cpp_trigraph_map[s[2]]; 979 s += 2; 980 goto slow_path; 981 } 982 } 983 /* Not a trigraph. Continue on fast-path. */ 984 s++; 985 } 986 else 987 break; 988 } 989 990 /* This must be \r or \n. We're either done, or we'll be forced 991 to write back to the buffer and continue on the slow path. */ 992 d = (uchar *) s; 993 994 if (__builtin_expect (s == buffer->rlimit, false)) 995 goto done; 996 997 /* DOS line ending? */ 998 if (__builtin_expect (c == '\r', false) && s[1] == '\n') 999 { 1000 s++; 1001 if (s == buffer->rlimit) 1002 goto done; 1003 } 1004 1005 if (__builtin_expect (pbackslash == NULL, true)) 1006 goto done; 1007 1008 /* Check for escaped newline. */ 1009 p = d; 1010 while (is_nvspace (p[-1])) 1011 p--; 1012 if (p - 1 != pbackslash) 1013 goto done; 1014 1015 /* Have an escaped newline; process it and proceed to 1016 the slow path. */ 1017 add_line_note (buffer, p - 1, p != d ? ' ' : '\\'); 1018 d = p - 2; 1019 buffer->next_line = p - 1; 1020 1021 slow_path: 1022 while (1) 1023 { 1024 c = *++s; 1025 *++d = c; 1026 1027 if (c == '\n' || c == '\r') 1028 { 1029 /* Handle DOS line endings. */ 1030 if (c == '\r' && s != buffer->rlimit && s[1] == '\n') 1031 s++; 1032 if (s == buffer->rlimit) 1033 break; 1034 1035 /* Escaped? */ 1036 p = d; 1037 while (p != buffer->next_line && is_nvspace (p[-1])) 1038 p--; 1039 if (p == buffer->next_line || p[-1] != '\\') 1040 break; 1041 1042 add_line_note (buffer, p - 1, p != d ? ' ': '\\'); 1043 d = p - 2; 1044 buffer->next_line = p - 1; 1045 } 1046 else if (c == '?' && s[1] == '?' && _cpp_trigraph_map[s[2]]) 1047 { 1048 /* Add a note regardless, for the benefit of -Wtrigraphs. */ 1049 add_line_note (buffer, d, s[2]); 1050 if (CPP_OPTION (pfile, trigraphs)) 1051 { 1052 *d = _cpp_trigraph_map[s[2]]; 1053 s += 2; 1054 } 1055 } 1056 } 1057 } 1058 else 1059 { 1060 while (*s != '\n' && *s != '\r') 1061 s++; 1062 d = (uchar *) s; 1063 1064 /* Handle DOS line endings. */ 1065 if (*s == '\r' && s != buffer->rlimit && s[1] == '\n') 1066 s++; 1067 } 1068 1069 done: 1070 *d = '\n'; 1071 /* A sentinel note that should never be processed. */ 1072 add_line_note (buffer, d + 1, '\n'); 1073 buffer->next_line = s + 1; 1074 } 1075 1076 /* Return true if the trigraph indicated by NOTE should be warned 1077 about in a comment. */ 1078 static bool 1079 warn_in_comment (cpp_reader *pfile, _cpp_line_note *note) 1080 { 1081 const uchar *p; 1082 1083 /* Within comments we don't warn about trigraphs, unless the 1084 trigraph forms an escaped newline, as that may change 1085 behavior. */ 1086 if (note->type != '/') 1087 return false; 1088 1089 /* If -trigraphs, then this was an escaped newline iff the next note 1090 is coincident. */ 1091 if (CPP_OPTION (pfile, trigraphs)) 1092 return note[1].pos == note->pos; 1093 1094 /* Otherwise, see if this forms an escaped newline. */ 1095 p = note->pos + 3; 1096 while (is_nvspace (*p)) 1097 p++; 1098 1099 /* There might have been escaped newlines between the trigraph and the 1100 newline we found. Hence the position test. */ 1101 return (*p == '\n' && p < note[1].pos); 1102 } 1103 1104 /* Process the notes created by add_line_note as far as the current 1105 location. */ 1106 void 1107 _cpp_process_line_notes (cpp_reader *pfile, int in_comment) 1108 { 1109 cpp_buffer *buffer = pfile->buffer; 1110 1111 for (;;) 1112 { 1113 _cpp_line_note *note = &buffer->notes[buffer->cur_note]; 1114 unsigned int col; 1115 1116 if (note->pos > buffer->cur) 1117 break; 1118 1119 buffer->cur_note++; 1120 col = CPP_BUF_COLUMN (buffer, note->pos + 1); 1121 1122 if (note->type == '\\' || note->type == ' ') 1123 { 1124 if (note->type == ' ' && !in_comment) 1125 cpp_error_with_line (pfile, CPP_DL_WARNING, pfile->line_table->highest_line, col, 1126 "backslash and newline separated by space"); 1127 1128 if (buffer->next_line > buffer->rlimit) 1129 { 1130 cpp_error_with_line (pfile, CPP_DL_PEDWARN, pfile->line_table->highest_line, col, 1131 "backslash-newline at end of file"); 1132 /* Prevent "no newline at end of file" warning. */ 1133 buffer->next_line = buffer->rlimit; 1134 } 1135 1136 buffer->line_base = note->pos; 1137 CPP_INCREMENT_LINE (pfile, 0); 1138 } 1139 else if (_cpp_trigraph_map[note->type]) 1140 { 1141 if (CPP_OPTION (pfile, warn_trigraphs) 1142 && (!in_comment || warn_in_comment (pfile, note))) 1143 { 1144 if (CPP_OPTION (pfile, trigraphs)) 1145 cpp_warning_with_line (pfile, CPP_W_TRIGRAPHS, 1146 pfile->line_table->highest_line, col, 1147 "trigraph ??%c converted to %c", 1148 note->type, 1149 (int) _cpp_trigraph_map[note->type]); 1150 else 1151 { 1152 cpp_warning_with_line 1153 (pfile, CPP_W_TRIGRAPHS, 1154 pfile->line_table->highest_line, col, 1155 "trigraph ??%c ignored, use -trigraphs to enable", 1156 note->type); 1157 } 1158 } 1159 } 1160 else if (note->type == 0) 1161 /* Already processed in lex_raw_string. */; 1162 else 1163 abort (); 1164 } 1165 } 1166 1167 /* Skip a C-style block comment. We find the end of the comment by 1168 seeing if an asterisk is before every '/' we encounter. Returns 1169 nonzero if comment terminated by EOF, zero otherwise. 1170 1171 Buffer->cur points to the initial asterisk of the comment. */ 1172 bool 1173 _cpp_skip_block_comment (cpp_reader *pfile) 1174 { 1175 cpp_buffer *buffer = pfile->buffer; 1176 const uchar *cur = buffer->cur; 1177 uchar c; 1178 1179 cur++; 1180 if (*cur == '/') 1181 cur++; 1182 1183 for (;;) 1184 { 1185 /* People like decorating comments with '*', so check for '/' 1186 instead for efficiency. */ 1187 c = *cur++; 1188 1189 if (c == '/') 1190 { 1191 if (cur[-2] == '*') 1192 break; 1193 1194 /* Warn about potential nested comments, but not if the '/' 1195 comes immediately before the true comment delimiter. 1196 Don't bother to get it right across escaped newlines. */ 1197 if (CPP_OPTION (pfile, warn_comments) 1198 && cur[0] == '*' && cur[1] != '/') 1199 { 1200 buffer->cur = cur; 1201 cpp_warning_with_line (pfile, CPP_W_COMMENTS, 1202 pfile->line_table->highest_line, 1203 CPP_BUF_COL (buffer), 1204 "\"/*\" within comment"); 1205 } 1206 } 1207 else if (c == '\n') 1208 { 1209 unsigned int cols; 1210 buffer->cur = cur - 1; 1211 _cpp_process_line_notes (pfile, true); 1212 if (buffer->next_line >= buffer->rlimit) 1213 return true; 1214 _cpp_clean_line (pfile); 1215 1216 cols = buffer->next_line - buffer->line_base; 1217 CPP_INCREMENT_LINE (pfile, cols); 1218 1219 cur = buffer->cur; 1220 } 1221 } 1222 1223 buffer->cur = cur; 1224 _cpp_process_line_notes (pfile, true); 1225 return false; 1226 } 1227 1228 /* Skip a C++ line comment, leaving buffer->cur pointing to the 1229 terminating newline. Handles escaped newlines. Returns nonzero 1230 if a multiline comment. */ 1231 static int 1232 skip_line_comment (cpp_reader *pfile) 1233 { 1234 cpp_buffer *buffer = pfile->buffer; 1235 location_t orig_line = pfile->line_table->highest_line; 1236 1237 while (*buffer->cur != '\n') 1238 buffer->cur++; 1239 1240 _cpp_process_line_notes (pfile, true); 1241 return orig_line != pfile->line_table->highest_line; 1242 } 1243 1244 /* Skips whitespace, saving the next non-whitespace character. */ 1245 static void 1246 skip_whitespace (cpp_reader *pfile, cppchar_t c) 1247 { 1248 cpp_buffer *buffer = pfile->buffer; 1249 bool saw_NUL = false; 1250 1251 do 1252 { 1253 /* Horizontal space always OK. */ 1254 if (c == ' ' || c == '\t') 1255 ; 1256 /* Just \f \v or \0 left. */ 1257 else if (c == '\0') 1258 saw_NUL = true; 1259 else if (pfile->state.in_directive && CPP_PEDANTIC (pfile)) 1260 cpp_error_with_line (pfile, CPP_DL_PEDWARN, pfile->line_table->highest_line, 1261 CPP_BUF_COL (buffer), 1262 "%s in preprocessing directive", 1263 c == '\f' ? "form feed" : "vertical tab"); 1264 1265 c = *buffer->cur++; 1266 } 1267 /* We only want non-vertical space, i.e. ' ' \t \f \v \0. */ 1268 while (is_nvspace (c)); 1269 1270 if (saw_NUL) 1271 cpp_error (pfile, CPP_DL_WARNING, "null character(s) ignored"); 1272 1273 buffer->cur--; 1274 } 1275 1276 /* See if the characters of a number token are valid in a name (no 1277 '.', '+' or '-'). */ 1278 static int 1279 name_p (cpp_reader *pfile, const cpp_string *string) 1280 { 1281 unsigned int i; 1282 1283 for (i = 0; i < string->len; i++) 1284 if (!is_idchar (string->text[i])) 1285 return 0; 1286 1287 return 1; 1288 } 1289 1290 /* After parsing an identifier or other sequence, produce a warning about 1291 sequences not in NFC/NFKC. */ 1292 static void 1293 warn_about_normalization (cpp_reader *pfile, 1294 const cpp_token *token, 1295 const struct normalize_state *s) 1296 { 1297 if (CPP_OPTION (pfile, warn_normalize) < NORMALIZE_STATE_RESULT (s) 1298 && !pfile->state.skipping) 1299 { 1300 /* Make sure that the token is printed using UCNs, even 1301 if we'd otherwise happily print UTF-8. */ 1302 unsigned char *buf = XNEWVEC (unsigned char, cpp_token_len (token)); 1303 size_t sz; 1304 1305 sz = cpp_spell_token (pfile, token, buf, false) - buf; 1306 if (NORMALIZE_STATE_RESULT (s) == normalized_C) 1307 cpp_warning_with_line (pfile, CPP_W_NORMALIZE, token->src_loc, 0, 1308 "`%.*s' is not in NFKC", (int) sz, buf); 1309 else 1310 cpp_warning_with_line (pfile, CPP_W_NORMALIZE, token->src_loc, 0, 1311 "`%.*s' is not in NFC", (int) sz, buf); 1312 free (buf); 1313 } 1314 } 1315 1316 /* Returns TRUE if the sequence starting at buffer->cur is invalid in 1317 an identifier. FIRST is TRUE if this starts an identifier. */ 1318 static bool 1319 forms_identifier_p (cpp_reader *pfile, int first, 1320 struct normalize_state *state) 1321 { 1322 cpp_buffer *buffer = pfile->buffer; 1323 1324 if (*buffer->cur == '$') 1325 { 1326 if (!CPP_OPTION (pfile, dollars_in_ident)) 1327 return false; 1328 1329 buffer->cur++; 1330 if (CPP_OPTION (pfile, warn_dollars) && !pfile->state.skipping) 1331 { 1332 CPP_OPTION (pfile, warn_dollars) = 0; 1333 cpp_error (pfile, CPP_DL_PEDWARN, "'$' in identifier or number"); 1334 } 1335 1336 return true; 1337 } 1338 1339 /* Is this a syntactically valid UCN? */ 1340 if (CPP_OPTION (pfile, extended_identifiers) 1341 && *buffer->cur == '\\' 1342 && (buffer->cur[1] == 'u' || buffer->cur[1] == 'U')) 1343 { 1344 cppchar_t s; 1345 buffer->cur += 2; 1346 if (_cpp_valid_ucn (pfile, &buffer->cur, buffer->rlimit, 1 + !first, 1347 state, &s, NULL, NULL)) 1348 return true; 1349 buffer->cur -= 2; 1350 } 1351 1352 return false; 1353 } 1354 1355 /* Helper function to issue error about improper __VA_OPT__ use. */ 1356 static void 1357 maybe_va_opt_error (cpp_reader *pfile) 1358 { 1359 if (CPP_PEDANTIC (pfile) && !CPP_OPTION (pfile, va_opt)) 1360 { 1361 /* __VA_OPT__ should not be accepted at all, but allow it in 1362 system headers. */ 1363 if (!cpp_in_system_header (pfile)) 1364 cpp_error (pfile, CPP_DL_PEDWARN, 1365 "__VA_OPT__ is not available until C++2a"); 1366 } 1367 else if (!pfile->state.va_args_ok) 1368 { 1369 /* __VA_OPT__ should only appear in the replacement list of a 1370 variadic macro. */ 1371 cpp_error (pfile, CPP_DL_PEDWARN, 1372 "__VA_OPT__ can only appear in the expansion" 1373 " of a C++2a variadic macro"); 1374 } 1375 } 1376 1377 /* Helper function to get the cpp_hashnode of the identifier BASE. */ 1378 static cpp_hashnode * 1379 lex_identifier_intern (cpp_reader *pfile, const uchar *base) 1380 { 1381 cpp_hashnode *result; 1382 const uchar *cur; 1383 unsigned int len; 1384 unsigned int hash = HT_HASHSTEP (0, *base); 1385 1386 cur = base + 1; 1387 while (ISIDNUM (*cur)) 1388 { 1389 hash = HT_HASHSTEP (hash, *cur); 1390 cur++; 1391 } 1392 len = cur - base; 1393 hash = HT_HASHFINISH (hash, len); 1394 result = CPP_HASHNODE (ht_lookup_with_hash (pfile->hash_table, 1395 base, len, hash, HT_ALLOC)); 1396 1397 /* Rarely, identifiers require diagnostics when lexed. */ 1398 if (__builtin_expect ((result->flags & NODE_DIAGNOSTIC) 1399 && !pfile->state.skipping, 0)) 1400 { 1401 /* It is allowed to poison the same identifier twice. */ 1402 if ((result->flags & NODE_POISONED) && !pfile->state.poisoned_ok) 1403 cpp_error (pfile, CPP_DL_ERROR, "attempt to use poisoned \"%s\"", 1404 NODE_NAME (result)); 1405 1406 /* Constraint 6.10.3.5: __VA_ARGS__ should only appear in the 1407 replacement list of a variadic macro. */ 1408 if (result == pfile->spec_nodes.n__VA_ARGS__ 1409 && !pfile->state.va_args_ok) 1410 { 1411 if (CPP_OPTION (pfile, cplusplus)) 1412 cpp_error (pfile, CPP_DL_PEDWARN, 1413 "__VA_ARGS__ can only appear in the expansion" 1414 " of a C++11 variadic macro"); 1415 else 1416 cpp_error (pfile, CPP_DL_PEDWARN, 1417 "__VA_ARGS__ can only appear in the expansion" 1418 " of a C99 variadic macro"); 1419 } 1420 1421 if (result == pfile->spec_nodes.n__VA_OPT__) 1422 maybe_va_opt_error (pfile); 1423 1424 /* For -Wc++-compat, warn about use of C++ named operators. */ 1425 if (result->flags & NODE_WARN_OPERATOR) 1426 cpp_warning (pfile, CPP_W_CXX_OPERATOR_NAMES, 1427 "identifier \"%s\" is a special operator name in C++", 1428 NODE_NAME (result)); 1429 } 1430 1431 return result; 1432 } 1433 1434 /* Get the cpp_hashnode of an identifier specified by NAME in 1435 the current cpp_reader object. If none is found, NULL is returned. */ 1436 cpp_hashnode * 1437 _cpp_lex_identifier (cpp_reader *pfile, const char *name) 1438 { 1439 cpp_hashnode *result; 1440 result = lex_identifier_intern (pfile, (uchar *) name); 1441 return result; 1442 } 1443 1444 /* Lex an identifier starting at BUFFER->CUR - 1. */ 1445 static cpp_hashnode * 1446 lex_identifier (cpp_reader *pfile, const uchar *base, bool starts_ucn, 1447 struct normalize_state *nst, cpp_hashnode **spelling) 1448 { 1449 cpp_hashnode *result; 1450 const uchar *cur; 1451 unsigned int len; 1452 unsigned int hash = HT_HASHSTEP (0, *base); 1453 1454 cur = pfile->buffer->cur; 1455 if (! starts_ucn) 1456 { 1457 while (ISIDNUM (*cur)) 1458 { 1459 hash = HT_HASHSTEP (hash, *cur); 1460 cur++; 1461 } 1462 NORMALIZE_STATE_UPDATE_IDNUM (nst, *(cur - 1)); 1463 } 1464 pfile->buffer->cur = cur; 1465 if (starts_ucn || forms_identifier_p (pfile, false, nst)) 1466 { 1467 /* Slower version for identifiers containing UCNs (or $). */ 1468 do { 1469 while (ISIDNUM (*pfile->buffer->cur)) 1470 { 1471 NORMALIZE_STATE_UPDATE_IDNUM (nst, *pfile->buffer->cur); 1472 pfile->buffer->cur++; 1473 } 1474 } while (forms_identifier_p (pfile, false, nst)); 1475 result = _cpp_interpret_identifier (pfile, base, 1476 pfile->buffer->cur - base); 1477 *spelling = cpp_lookup (pfile, base, pfile->buffer->cur - base); 1478 } 1479 else 1480 { 1481 len = cur - base; 1482 hash = HT_HASHFINISH (hash, len); 1483 1484 result = CPP_HASHNODE (ht_lookup_with_hash (pfile->hash_table, 1485 base, len, hash, HT_ALLOC)); 1486 *spelling = result; 1487 } 1488 1489 /* Rarely, identifiers require diagnostics when lexed. */ 1490 if (__builtin_expect ((result->flags & NODE_DIAGNOSTIC) 1491 && !pfile->state.skipping, 0)) 1492 { 1493 /* It is allowed to poison the same identifier twice. */ 1494 if ((result->flags & NODE_POISONED) && !pfile->state.poisoned_ok) 1495 cpp_error (pfile, CPP_DL_ERROR, "attempt to use poisoned \"%s\"", 1496 NODE_NAME (result)); 1497 1498 /* Constraint 6.10.3.5: __VA_ARGS__ should only appear in the 1499 replacement list of a variadic macro. */ 1500 if (result == pfile->spec_nodes.n__VA_ARGS__ 1501 && !pfile->state.va_args_ok) 1502 { 1503 if (CPP_OPTION (pfile, cplusplus)) 1504 cpp_error (pfile, CPP_DL_PEDWARN, 1505 "__VA_ARGS__ can only appear in the expansion" 1506 " of a C++11 variadic macro"); 1507 else 1508 cpp_error (pfile, CPP_DL_PEDWARN, 1509 "__VA_ARGS__ can only appear in the expansion" 1510 " of a C99 variadic macro"); 1511 } 1512 1513 /* __VA_OPT__ should only appear in the replacement list of a 1514 variadic macro. */ 1515 if (result == pfile->spec_nodes.n__VA_OPT__) 1516 maybe_va_opt_error (pfile); 1517 1518 /* For -Wc++-compat, warn about use of C++ named operators. */ 1519 if (result->flags & NODE_WARN_OPERATOR) 1520 cpp_warning (pfile, CPP_W_CXX_OPERATOR_NAMES, 1521 "identifier \"%s\" is a special operator name in C++", 1522 NODE_NAME (result)); 1523 } 1524 1525 return result; 1526 } 1527 1528 /* Lex a number to NUMBER starting at BUFFER->CUR - 1. */ 1529 static void 1530 lex_number (cpp_reader *pfile, cpp_string *number, 1531 struct normalize_state *nst) 1532 { 1533 const uchar *cur; 1534 const uchar *base; 1535 uchar *dest; 1536 1537 base = pfile->buffer->cur - 1; 1538 do 1539 { 1540 cur = pfile->buffer->cur; 1541 1542 /* N.B. ISIDNUM does not include $. */ 1543 while (ISIDNUM (*cur) || *cur == '.' || DIGIT_SEP (*cur) 1544 || VALID_SIGN (*cur, cur[-1])) 1545 { 1546 NORMALIZE_STATE_UPDATE_IDNUM (nst, *cur); 1547 cur++; 1548 } 1549 /* A number can't end with a digit separator. */ 1550 while (cur > pfile->buffer->cur && DIGIT_SEP (cur[-1])) 1551 --cur; 1552 1553 pfile->buffer->cur = cur; 1554 } 1555 while (forms_identifier_p (pfile, false, nst)); 1556 1557 number->len = cur - base; 1558 dest = _cpp_unaligned_alloc (pfile, number->len + 1); 1559 memcpy (dest, base, number->len); 1560 dest[number->len] = '\0'; 1561 number->text = dest; 1562 } 1563 1564 /* Create a token of type TYPE with a literal spelling. */ 1565 static void 1566 create_literal (cpp_reader *pfile, cpp_token *token, const uchar *base, 1567 unsigned int len, enum cpp_ttype type) 1568 { 1569 uchar *dest = _cpp_unaligned_alloc (pfile, len + 1); 1570 1571 memcpy (dest, base, len); 1572 dest[len] = '\0'; 1573 token->type = type; 1574 token->val.str.len = len; 1575 token->val.str.text = dest; 1576 } 1577 1578 /* Subroutine of lex_raw_string: Append LEN chars from BASE to the buffer 1579 sequence from *FIRST_BUFF_P to LAST_BUFF_P. */ 1580 1581 static void 1582 bufring_append (cpp_reader *pfile, const uchar *base, size_t len, 1583 _cpp_buff **first_buff_p, _cpp_buff **last_buff_p) 1584 { 1585 _cpp_buff *first_buff = *first_buff_p; 1586 _cpp_buff *last_buff = *last_buff_p; 1587 1588 if (first_buff == NULL) 1589 first_buff = last_buff = _cpp_get_buff (pfile, len); 1590 else if (len > BUFF_ROOM (last_buff)) 1591 { 1592 size_t room = BUFF_ROOM (last_buff); 1593 memcpy (BUFF_FRONT (last_buff), base, room); 1594 BUFF_FRONT (last_buff) += room; 1595 base += room; 1596 len -= room; 1597 last_buff = _cpp_append_extend_buff (pfile, last_buff, len); 1598 } 1599 1600 memcpy (BUFF_FRONT (last_buff), base, len); 1601 BUFF_FRONT (last_buff) += len; 1602 1603 *first_buff_p = first_buff; 1604 *last_buff_p = last_buff; 1605 } 1606 1607 1608 /* Returns true if a macro has been defined. 1609 This might not work if compile with -save-temps, 1610 or preprocess separately from compilation. */ 1611 1612 static bool 1613 is_macro(cpp_reader *pfile, const uchar *base) 1614 { 1615 const uchar *cur = base; 1616 if (! ISIDST (*cur)) 1617 return false; 1618 unsigned int hash = HT_HASHSTEP (0, *cur); 1619 ++cur; 1620 while (ISIDNUM (*cur)) 1621 { 1622 hash = HT_HASHSTEP (hash, *cur); 1623 ++cur; 1624 } 1625 hash = HT_HASHFINISH (hash, cur - base); 1626 1627 cpp_hashnode *result = CPP_HASHNODE (ht_lookup_with_hash (pfile->hash_table, 1628 base, cur - base, hash, HT_NO_INSERT)); 1629 1630 return result && cpp_macro_p (result); 1631 } 1632 1633 /* Returns true if a literal suffix does not have the expected form 1634 and is defined as a macro. */ 1635 1636 static bool 1637 is_macro_not_literal_suffix(cpp_reader *pfile, const uchar *base) 1638 { 1639 /* User-defined literals outside of namespace std must start with a single 1640 underscore, so assume anything of that form really is a UDL suffix. 1641 We don't need to worry about UDLs defined inside namespace std because 1642 their names are reserved, so cannot be used as macro names in valid 1643 programs. */ 1644 if (base[0] == '_' && base[1] != '_') 1645 return false; 1646 return is_macro (pfile, base); 1647 } 1648 1649 /* Lexes a raw string. The stored string contains the spelling, including 1650 double quotes, delimiter string, '(' and ')', any leading 1651 'L', 'u', 'U' or 'u8' and 'R' modifier. It returns the type of the 1652 literal, or CPP_OTHER if it was not properly terminated. 1653 1654 The spelling is NUL-terminated, but it is not guaranteed that this 1655 is the first NUL since embedded NULs are preserved. */ 1656 1657 static void 1658 lex_raw_string (cpp_reader *pfile, cpp_token *token, const uchar *base, 1659 const uchar *cur) 1660 { 1661 uchar raw_prefix[17]; 1662 uchar temp_buffer[18]; 1663 const uchar *orig_base; 1664 unsigned int raw_prefix_len = 0, raw_suffix_len = 0; 1665 enum raw_str_phase { RAW_STR_PREFIX, RAW_STR, RAW_STR_SUFFIX }; 1666 raw_str_phase phase = RAW_STR_PREFIX; 1667 enum cpp_ttype type; 1668 size_t total_len = 0; 1669 /* Index into temp_buffer during phases other than RAW_STR, 1670 during RAW_STR phase 17 to tell BUF_APPEND that nothing should 1671 be appended to temp_buffer. */ 1672 size_t temp_buffer_len = 0; 1673 _cpp_buff *first_buff = NULL, *last_buff = NULL; 1674 size_t raw_prefix_start; 1675 _cpp_line_note *note = &pfile->buffer->notes[pfile->buffer->cur_note]; 1676 1677 type = (*base == 'L' ? CPP_WSTRING : 1678 *base == 'U' ? CPP_STRING32 : 1679 *base == 'u' ? (base[1] == '8' ? CPP_UTF8STRING : CPP_STRING16) 1680 : CPP_STRING); 1681 1682 #define BUF_APPEND(STR,LEN) \ 1683 do { \ 1684 bufring_append (pfile, (const uchar *)(STR), (LEN), \ 1685 &first_buff, &last_buff); \ 1686 total_len += (LEN); \ 1687 if (__builtin_expect (temp_buffer_len < 17, 0) \ 1688 && (const uchar *)(STR) != base \ 1689 && (LEN) <= 2) \ 1690 { \ 1691 memcpy (temp_buffer + temp_buffer_len, \ 1692 (const uchar *)(STR), (LEN)); \ 1693 temp_buffer_len += (LEN); \ 1694 } \ 1695 } while (0) 1696 1697 orig_base = base; 1698 ++cur; 1699 raw_prefix_start = cur - base; 1700 for (;;) 1701 { 1702 cppchar_t c; 1703 1704 /* If we previously performed any trigraph or line splicing 1705 transformations, undo them in between the opening and closing 1706 double quote. */ 1707 while (note->pos < cur) 1708 ++note; 1709 for (; note->pos == cur; ++note) 1710 { 1711 switch (note->type) 1712 { 1713 case '\\': 1714 case ' ': 1715 /* Restore backslash followed by newline. */ 1716 BUF_APPEND (base, cur - base); 1717 base = cur; 1718 BUF_APPEND ("\\", 1); 1719 after_backslash: 1720 if (note->type == ' ') 1721 { 1722 /* GNU backslash whitespace newline extension. FIXME 1723 could be any sequence of non-vertical space. When we 1724 can properly restore any such sequence, we should mark 1725 this note as handled so _cpp_process_line_notes 1726 doesn't warn. */ 1727 BUF_APPEND (" ", 1); 1728 } 1729 1730 BUF_APPEND ("\n", 1); 1731 break; 1732 1733 case 0: 1734 /* Already handled. */ 1735 break; 1736 1737 default: 1738 if (_cpp_trigraph_map[note->type]) 1739 { 1740 /* Don't warn about this trigraph in 1741 _cpp_process_line_notes, since trigraphs show up as 1742 trigraphs in raw strings. */ 1743 uchar type = note->type; 1744 note->type = 0; 1745 1746 if (!CPP_OPTION (pfile, trigraphs)) 1747 /* If we didn't convert the trigraph in the first 1748 place, don't do anything now either. */ 1749 break; 1750 1751 BUF_APPEND (base, cur - base); 1752 base = cur; 1753 BUF_APPEND ("??", 2); 1754 1755 /* ??/ followed by newline gets two line notes, one for 1756 the trigraph and one for the backslash/newline. */ 1757 if (type == '/' && note[1].pos == cur) 1758 { 1759 if (note[1].type != '\\' 1760 && note[1].type != ' ') 1761 abort (); 1762 BUF_APPEND ("/", 1); 1763 ++note; 1764 goto after_backslash; 1765 } 1766 else 1767 { 1768 /* Skip the replacement character. */ 1769 base = ++cur; 1770 BUF_APPEND (&type, 1); 1771 c = type; 1772 goto check_c; 1773 } 1774 } 1775 else 1776 abort (); 1777 break; 1778 } 1779 } 1780 c = *cur++; 1781 if (__builtin_expect (temp_buffer_len < 17, 0)) 1782 temp_buffer[temp_buffer_len++] = c; 1783 1784 check_c: 1785 if (phase == RAW_STR_PREFIX) 1786 { 1787 while (raw_prefix_len < temp_buffer_len) 1788 { 1789 raw_prefix[raw_prefix_len] = temp_buffer[raw_prefix_len]; 1790 switch (raw_prefix[raw_prefix_len]) 1791 { 1792 case ' ': case '(': case ')': case '\\': case '\t': 1793 case '\v': case '\f': case '\n': default: 1794 break; 1795 /* Basic source charset except the above chars. */ 1796 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': 1797 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': 1798 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': 1799 case 's': case 't': case 'u': case 'v': case 'w': case 'x': 1800 case 'y': case 'z': 1801 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': 1802 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': 1803 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': 1804 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': 1805 case 'Y': case 'Z': 1806 case '0': case '1': case '2': case '3': case '4': case '5': 1807 case '6': case '7': case '8': case '9': 1808 case '_': case '{': case '}': case '#': case '[': case ']': 1809 case '<': case '>': case '%': case ':': case ';': case '.': 1810 case '?': case '*': case '+': case '-': case '/': case '^': 1811 case '&': case '|': case '~': case '!': case '=': case ',': 1812 case '"': case '\'': 1813 if (raw_prefix_len < 16) 1814 { 1815 raw_prefix_len++; 1816 continue; 1817 } 1818 break; 1819 } 1820 1821 if (raw_prefix[raw_prefix_len] != '(') 1822 { 1823 int col = CPP_BUF_COLUMN (pfile->buffer, cur) + 1; 1824 if (raw_prefix_len == 16) 1825 cpp_error_with_line (pfile, CPP_DL_ERROR, token->src_loc, 1826 col, "raw string delimiter longer " 1827 "than 16 characters"); 1828 else if (raw_prefix[raw_prefix_len] == '\n') 1829 cpp_error_with_line (pfile, CPP_DL_ERROR, token->src_loc, 1830 col, "invalid new-line in raw " 1831 "string delimiter"); 1832 else 1833 cpp_error_with_line (pfile, CPP_DL_ERROR, token->src_loc, 1834 col, "invalid character '%c' in " 1835 "raw string delimiter", 1836 (int) raw_prefix[raw_prefix_len]); 1837 pfile->buffer->cur = orig_base + raw_prefix_start - 1; 1838 create_literal (pfile, token, orig_base, 1839 raw_prefix_start - 1, CPP_OTHER); 1840 if (first_buff) 1841 _cpp_release_buff (pfile, first_buff); 1842 return; 1843 } 1844 raw_prefix[raw_prefix_len] = '"'; 1845 phase = RAW_STR; 1846 /* Nothing should be appended to temp_buffer during 1847 RAW_STR phase. */ 1848 temp_buffer_len = 17; 1849 break; 1850 } 1851 continue; 1852 } 1853 else if (phase == RAW_STR_SUFFIX) 1854 { 1855 while (raw_suffix_len <= raw_prefix_len 1856 && raw_suffix_len < temp_buffer_len 1857 && temp_buffer[raw_suffix_len] == raw_prefix[raw_suffix_len]) 1858 raw_suffix_len++; 1859 if (raw_suffix_len > raw_prefix_len) 1860 break; 1861 if (raw_suffix_len == temp_buffer_len) 1862 continue; 1863 phase = RAW_STR; 1864 /* Nothing should be appended to temp_buffer during 1865 RAW_STR phase. */ 1866 temp_buffer_len = 17; 1867 } 1868 if (c == ')') 1869 { 1870 phase = RAW_STR_SUFFIX; 1871 raw_suffix_len = 0; 1872 temp_buffer_len = 0; 1873 } 1874 else if (c == '\n') 1875 { 1876 if (pfile->state.in_directive 1877 || (pfile->state.parsing_args 1878 && pfile->buffer->next_line >= pfile->buffer->rlimit)) 1879 { 1880 cur--; 1881 type = CPP_OTHER; 1882 cpp_error_with_line (pfile, CPP_DL_ERROR, token->src_loc, 0, 1883 "unterminated raw string"); 1884 break; 1885 } 1886 1887 BUF_APPEND (base, cur - base); 1888 1889 if (pfile->buffer->cur < pfile->buffer->rlimit) 1890 CPP_INCREMENT_LINE (pfile, 0); 1891 pfile->buffer->need_line = true; 1892 1893 pfile->buffer->cur = cur-1; 1894 _cpp_process_line_notes (pfile, false); 1895 if (!_cpp_get_fresh_line (pfile)) 1896 { 1897 location_t src_loc = token->src_loc; 1898 token->type = CPP_EOF; 1899 /* Tell the compiler the line number of the EOF token. */ 1900 token->src_loc = pfile->line_table->highest_line; 1901 token->flags = BOL; 1902 if (first_buff != NULL) 1903 _cpp_release_buff (pfile, first_buff); 1904 cpp_error_with_line (pfile, CPP_DL_ERROR, src_loc, 0, 1905 "unterminated raw string"); 1906 return; 1907 } 1908 1909 cur = base = pfile->buffer->cur; 1910 note = &pfile->buffer->notes[pfile->buffer->cur_note]; 1911 } 1912 } 1913 1914 if (CPP_OPTION (pfile, user_literals)) 1915 { 1916 /* If a string format macro, say from inttypes.h, is placed touching 1917 a string literal it could be parsed as a C++11 user-defined string 1918 literal thus breaking the program. */ 1919 if (is_macro_not_literal_suffix (pfile, cur)) 1920 { 1921 /* Raise a warning, but do not consume subsequent tokens. */ 1922 if (CPP_OPTION (pfile, warn_literal_suffix) && !pfile->state.skipping) 1923 cpp_warning_with_line (pfile, CPP_W_LITERAL_SUFFIX, 1924 token->src_loc, 0, 1925 "invalid suffix on literal; C++11 requires " 1926 "a space between literal and string macro"); 1927 } 1928 /* Grab user defined literal suffix. */ 1929 else if (ISIDST (*cur)) 1930 { 1931 type = cpp_userdef_string_add_type (type); 1932 ++cur; 1933 1934 while (ISIDNUM (*cur)) 1935 ++cur; 1936 } 1937 } 1938 1939 pfile->buffer->cur = cur; 1940 if (first_buff == NULL) 1941 create_literal (pfile, token, base, cur - base, type); 1942 else 1943 { 1944 uchar *dest = _cpp_unaligned_alloc (pfile, total_len + (cur - base) + 1); 1945 1946 token->type = type; 1947 token->val.str.len = total_len + (cur - base); 1948 token->val.str.text = dest; 1949 last_buff = first_buff; 1950 while (last_buff != NULL) 1951 { 1952 memcpy (dest, last_buff->base, 1953 BUFF_FRONT (last_buff) - last_buff->base); 1954 dest += BUFF_FRONT (last_buff) - last_buff->base; 1955 last_buff = last_buff->next; 1956 } 1957 _cpp_release_buff (pfile, first_buff); 1958 memcpy (dest, base, cur - base); 1959 dest[cur - base] = '\0'; 1960 } 1961 } 1962 1963 /* Lexes a string, character constant, or angle-bracketed header file 1964 name. The stored string contains the spelling, including opening 1965 quote and any leading 'L', 'u', 'U' or 'u8' and optional 1966 'R' modifier. It returns the type of the literal, or CPP_OTHER 1967 if it was not properly terminated, or CPP_LESS for an unterminated 1968 header name which must be relexed as normal tokens. 1969 1970 The spelling is NUL-terminated, but it is not guaranteed that this 1971 is the first NUL since embedded NULs are preserved. */ 1972 static void 1973 lex_string (cpp_reader *pfile, cpp_token *token, const uchar *base) 1974 { 1975 bool saw_NUL = false; 1976 const uchar *cur; 1977 cppchar_t terminator; 1978 enum cpp_ttype type; 1979 1980 cur = base; 1981 terminator = *cur++; 1982 if (terminator == 'L' || terminator == 'U') 1983 terminator = *cur++; 1984 else if (terminator == 'u') 1985 { 1986 terminator = *cur++; 1987 if (terminator == '8') 1988 terminator = *cur++; 1989 } 1990 if (terminator == 'R') 1991 { 1992 lex_raw_string (pfile, token, base, cur); 1993 return; 1994 } 1995 if (terminator == '"') 1996 type = (*base == 'L' ? CPP_WSTRING : 1997 *base == 'U' ? CPP_STRING32 : 1998 *base == 'u' ? (base[1] == '8' ? CPP_UTF8STRING : CPP_STRING16) 1999 : CPP_STRING); 2000 else if (terminator == '\'') 2001 type = (*base == 'L' ? CPP_WCHAR : 2002 *base == 'U' ? CPP_CHAR32 : 2003 *base == 'u' ? (base[1] == '8' ? CPP_UTF8CHAR : CPP_CHAR16) 2004 : CPP_CHAR); 2005 else 2006 terminator = '>', type = CPP_HEADER_NAME; 2007 2008 for (;;) 2009 { 2010 cppchar_t c = *cur++; 2011 2012 /* In #include-style directives, terminators are not escapable. */ 2013 if (c == '\\' && !pfile->state.angled_headers && *cur != '\n') 2014 cur++; 2015 else if (c == terminator) 2016 break; 2017 else if (c == '\n') 2018 { 2019 cur--; 2020 /* Unmatched quotes always yield undefined behavior, but 2021 greedy lexing means that what appears to be an unterminated 2022 header name may actually be a legitimate sequence of tokens. */ 2023 if (terminator == '>') 2024 { 2025 token->type = CPP_LESS; 2026 return; 2027 } 2028 type = CPP_OTHER; 2029 break; 2030 } 2031 else if (c == '\0') 2032 saw_NUL = true; 2033 } 2034 2035 if (saw_NUL && !pfile->state.skipping) 2036 cpp_error (pfile, CPP_DL_WARNING, 2037 "null character(s) preserved in literal"); 2038 2039 if (type == CPP_OTHER && CPP_OPTION (pfile, lang) != CLK_ASM) 2040 cpp_error (pfile, CPP_DL_PEDWARN, "missing terminating %c character", 2041 (int) terminator); 2042 2043 if (CPP_OPTION (pfile, user_literals)) 2044 { 2045 /* If a string format macro, say from inttypes.h, is placed touching 2046 a string literal it could be parsed as a C++11 user-defined string 2047 literal thus breaking the program. */ 2048 if (is_macro_not_literal_suffix (pfile, cur)) 2049 { 2050 /* Raise a warning, but do not consume subsequent tokens. */ 2051 if (CPP_OPTION (pfile, warn_literal_suffix) && !pfile->state.skipping) 2052 cpp_warning_with_line (pfile, CPP_W_LITERAL_SUFFIX, 2053 token->src_loc, 0, 2054 "invalid suffix on literal; C++11 requires " 2055 "a space between literal and string macro"); 2056 } 2057 /* Grab user defined literal suffix. */ 2058 else if (ISIDST (*cur)) 2059 { 2060 type = cpp_userdef_char_add_type (type); 2061 type = cpp_userdef_string_add_type (type); 2062 ++cur; 2063 2064 while (ISIDNUM (*cur)) 2065 ++cur; 2066 } 2067 } 2068 else if (CPP_OPTION (pfile, cpp_warn_cxx11_compat) 2069 && is_macro (pfile, cur) 2070 && !pfile->state.skipping) 2071 cpp_warning_with_line (pfile, CPP_W_CXX11_COMPAT, 2072 token->src_loc, 0, "C++11 requires a space " 2073 "between string literal and macro"); 2074 2075 pfile->buffer->cur = cur; 2076 create_literal (pfile, token, base, cur - base, type); 2077 } 2078 2079 /* Return the comment table. The client may not make any assumption 2080 about the ordering of the table. */ 2081 cpp_comment_table * 2082 cpp_get_comments (cpp_reader *pfile) 2083 { 2084 return &pfile->comments; 2085 } 2086 2087 /* Append a comment to the end of the comment table. */ 2088 static void 2089 store_comment (cpp_reader *pfile, cpp_token *token) 2090 { 2091 int len; 2092 2093 if (pfile->comments.allocated == 0) 2094 { 2095 pfile->comments.allocated = 256; 2096 pfile->comments.entries = (cpp_comment *) xmalloc 2097 (pfile->comments.allocated * sizeof (cpp_comment)); 2098 } 2099 2100 if (pfile->comments.count == pfile->comments.allocated) 2101 { 2102 pfile->comments.allocated *= 2; 2103 pfile->comments.entries = (cpp_comment *) xrealloc 2104 (pfile->comments.entries, 2105 pfile->comments.allocated * sizeof (cpp_comment)); 2106 } 2107 2108 len = token->val.str.len; 2109 2110 /* Copy comment. Note, token may not be NULL terminated. */ 2111 pfile->comments.entries[pfile->comments.count].comment = 2112 (char *) xmalloc (sizeof (char) * (len + 1)); 2113 memcpy (pfile->comments.entries[pfile->comments.count].comment, 2114 token->val.str.text, len); 2115 pfile->comments.entries[pfile->comments.count].comment[len] = '\0'; 2116 2117 /* Set source location. */ 2118 pfile->comments.entries[pfile->comments.count].sloc = token->src_loc; 2119 2120 /* Increment the count of entries in the comment table. */ 2121 pfile->comments.count++; 2122 } 2123 2124 /* The stored comment includes the comment start and any terminator. */ 2125 static void 2126 save_comment (cpp_reader *pfile, cpp_token *token, const unsigned char *from, 2127 cppchar_t type) 2128 { 2129 unsigned char *buffer; 2130 unsigned int len, clen, i; 2131 int convert_to_c = (pfile->state.in_directive || pfile->state.parsing_args) 2132 && type == '/'; 2133 2134 len = pfile->buffer->cur - from + 1; /* + 1 for the initial '/'. */ 2135 2136 /* C++ comments probably (not definitely) have moved past a new 2137 line, which we don't want to save in the comment. */ 2138 if (is_vspace (pfile->buffer->cur[-1])) 2139 len--; 2140 2141 /* If we are currently in a directive or in argument parsing, then 2142 we need to store all C++ comments as C comments internally, and 2143 so we need to allocate a little extra space in that case. 2144 2145 Note that the only time we encounter a directive here is 2146 when we are saving comments in a "#define". */ 2147 clen = convert_to_c ? len + 2 : len; 2148 2149 buffer = _cpp_unaligned_alloc (pfile, clen); 2150 2151 token->type = CPP_COMMENT; 2152 token->val.str.len = clen; 2153 token->val.str.text = buffer; 2154 2155 buffer[0] = '/'; 2156 memcpy (buffer + 1, from, len - 1); 2157 2158 /* Finish conversion to a C comment, if necessary. */ 2159 if (convert_to_c) 2160 { 2161 buffer[1] = '*'; 2162 buffer[clen - 2] = '*'; 2163 buffer[clen - 1] = '/'; 2164 /* As there can be in a C++ comments illegal sequences for C comments 2165 we need to filter them out. */ 2166 for (i = 2; i < (clen - 2); i++) 2167 if (buffer[i] == '/' && (buffer[i - 1] == '*' || buffer[i + 1] == '*')) 2168 buffer[i] = '|'; 2169 } 2170 2171 /* Finally store this comment for use by clients of libcpp. */ 2172 store_comment (pfile, token); 2173 } 2174 2175 /* Returns true if comment at COMMENT_START is a recognized FALLTHROUGH 2176 comment. */ 2177 2178 static bool 2179 fallthrough_comment_p (cpp_reader *pfile, const unsigned char *comment_start) 2180 { 2181 const unsigned char *from = comment_start + 1; 2182 2183 switch (CPP_OPTION (pfile, cpp_warn_implicit_fallthrough)) 2184 { 2185 /* For both -Wimplicit-fallthrough=0 and -Wimplicit-fallthrough=5 we 2186 don't recognize any comments. The latter only checks attributes, 2187 the former doesn't warn. */ 2188 case 0: 2189 default: 2190 return false; 2191 /* -Wimplicit-fallthrough=1 considers any comment, no matter what 2192 content it has. */ 2193 case 1: 2194 return true; 2195 case 2: 2196 /* -Wimplicit-fallthrough=2 looks for (case insensitive) 2197 .*falls?[ \t-]*thr(u|ough).* regex. */ 2198 for (; (size_t) (pfile->buffer->cur - from) >= sizeof "fallthru" - 1; 2199 from++) 2200 { 2201 /* Is there anything like strpbrk with upper boundary, or 2202 memchr looking for 2 characters rather than just one? */ 2203 if (from[0] != 'f' && from[0] != 'F') 2204 continue; 2205 if (from[1] != 'a' && from[1] != 'A') 2206 continue; 2207 if (from[2] != 'l' && from[2] != 'L') 2208 continue; 2209 if (from[3] != 'l' && from[3] != 'L') 2210 continue; 2211 from += sizeof "fall" - 1; 2212 if (from[0] == 's' || from[0] == 'S') 2213 from++; 2214 while (*from == ' ' || *from == '\t' || *from == '-') 2215 from++; 2216 if (from[0] != 't' && from[0] != 'T') 2217 continue; 2218 if (from[1] != 'h' && from[1] != 'H') 2219 continue; 2220 if (from[2] != 'r' && from[2] != 'R') 2221 continue; 2222 if (from[3] == 'u' || from[3] == 'U') 2223 return true; 2224 if (from[3] != 'o' && from[3] != 'O') 2225 continue; 2226 if (from[4] != 'u' && from[4] != 'U') 2227 continue; 2228 if (from[5] != 'g' && from[5] != 'G') 2229 continue; 2230 if (from[6] != 'h' && from[6] != 'H') 2231 continue; 2232 return true; 2233 } 2234 return false; 2235 case 3: 2236 case 4: 2237 break; 2238 } 2239 2240 /* Whole comment contents: 2241 -fallthrough 2242 @fallthrough@ 2243 */ 2244 if (*from == '-' || *from == '@') 2245 { 2246 size_t len = sizeof "fallthrough" - 1; 2247 if ((size_t) (pfile->buffer->cur - from - 1) < len) 2248 return false; 2249 if (memcmp (from + 1, "fallthrough", len)) 2250 return false; 2251 if (*from == '@') 2252 { 2253 if (from[len + 1] != '@') 2254 return false; 2255 len++; 2256 } 2257 from += 1 + len; 2258 } 2259 /* Whole comment contents (regex): 2260 lint -fallthrough[ \t]* 2261 */ 2262 else if (*from == 'l') 2263 { 2264 size_t len = sizeof "int -fallthrough" - 1; 2265 if ((size_t) (pfile->buffer->cur - from - 1) < len) 2266 return false; 2267 if (memcmp (from + 1, "int -fallthrough", len)) 2268 return false; 2269 from += 1 + len; 2270 while (*from == ' ' || *from == '\t') 2271 from++; 2272 } 2273 /* Whole comment contents (regex): 2274 [ \t]*FALLTHR(U|OUGH)[ \t]* 2275 */ 2276 else if (CPP_OPTION (pfile, cpp_warn_implicit_fallthrough) == 4) 2277 { 2278 while (*from == ' ' || *from == '\t') 2279 from++; 2280 if ((size_t) (pfile->buffer->cur - from) < sizeof "FALLTHRU" - 1) 2281 return false; 2282 if (memcmp (from, "FALLTHR", sizeof "FALLTHR" - 1)) 2283 return false; 2284 from += sizeof "FALLTHR" - 1; 2285 if (*from == 'U') 2286 from++; 2287 else if ((size_t) (pfile->buffer->cur - from) < sizeof "OUGH" - 1) 2288 return false; 2289 else if (memcmp (from, "OUGH", sizeof "OUGH" - 1)) 2290 return false; 2291 else 2292 from += sizeof "OUGH" - 1; 2293 while (*from == ' ' || *from == '\t') 2294 from++; 2295 } 2296 /* Whole comment contents (regex): 2297 [ \t.!]*(ELSE,? |INTENTIONAL(LY)? )?FALL(S | |-)?THR(OUGH|U)[ \t.!]*(-[^\n\r]*)? 2298 [ \t.!]*(Else,? |Intentional(ly)? )?Fall((s | |-)[Tt]|t)hr(ough|u)[ \t.!]*(-[^\n\r]*)? 2299 [ \t.!]*([Ee]lse,? |[Ii]ntentional(ly)? )?fall(s | |-)?thr(ough|u)[ \t.!]*(-[^\n\r]*)? 2300 */ 2301 else 2302 { 2303 while (*from == ' ' || *from == '\t' || *from == '.' || *from == '!') 2304 from++; 2305 unsigned char f = *from; 2306 bool all_upper = false; 2307 if (f == 'E' || f == 'e') 2308 { 2309 if ((size_t) (pfile->buffer->cur - from) 2310 < sizeof "else fallthru" - 1) 2311 return false; 2312 if (f == 'E' && memcmp (from + 1, "LSE", sizeof "LSE" - 1) == 0) 2313 all_upper = true; 2314 else if (memcmp (from + 1, "lse", sizeof "lse" - 1)) 2315 return false; 2316 from += sizeof "else" - 1; 2317 if (*from == ',') 2318 from++; 2319 if (*from != ' ') 2320 return false; 2321 from++; 2322 if (all_upper && *from == 'f') 2323 return false; 2324 if (f == 'e' && *from == 'F') 2325 return false; 2326 f = *from; 2327 } 2328 else if (f == 'I' || f == 'i') 2329 { 2330 if ((size_t) (pfile->buffer->cur - from) 2331 < sizeof "intentional fallthru" - 1) 2332 return false; 2333 if (f == 'I' && memcmp (from + 1, "NTENTIONAL", 2334 sizeof "NTENTIONAL" - 1) == 0) 2335 all_upper = true; 2336 else if (memcmp (from + 1, "ntentional", 2337 sizeof "ntentional" - 1)) 2338 return false; 2339 from += sizeof "intentional" - 1; 2340 if (*from == ' ') 2341 { 2342 from++; 2343 if (all_upper && *from == 'f') 2344 return false; 2345 } 2346 else if (all_upper) 2347 { 2348 if (memcmp (from, "LY F", sizeof "LY F" - 1)) 2349 return false; 2350 from += sizeof "LY " - 1; 2351 } 2352 else 2353 { 2354 if (memcmp (from, "ly ", sizeof "ly " - 1)) 2355 return false; 2356 from += sizeof "ly " - 1; 2357 } 2358 if (f == 'i' && *from == 'F') 2359 return false; 2360 f = *from; 2361 } 2362 if (f != 'F' && f != 'f') 2363 return false; 2364 if ((size_t) (pfile->buffer->cur - from) < sizeof "fallthru" - 1) 2365 return false; 2366 if (f == 'F' && memcmp (from + 1, "ALL", sizeof "ALL" - 1) == 0) 2367 all_upper = true; 2368 else if (all_upper) 2369 return false; 2370 else if (memcmp (from + 1, "all", sizeof "all" - 1)) 2371 return false; 2372 from += sizeof "fall" - 1; 2373 if (*from == (all_upper ? 'S' : 's') && from[1] == ' ') 2374 from += 2; 2375 else if (*from == ' ' || *from == '-') 2376 from++; 2377 else if (*from != (all_upper ? 'T' : 't')) 2378 return false; 2379 if ((f == 'f' || *from != 'T') && (all_upper || *from != 't')) 2380 return false; 2381 if ((size_t) (pfile->buffer->cur - from) < sizeof "thru" - 1) 2382 return false; 2383 if (memcmp (from + 1, all_upper ? "HRU" : "hru", sizeof "hru" - 1)) 2384 { 2385 if ((size_t) (pfile->buffer->cur - from) < sizeof "through" - 1) 2386 return false; 2387 if (memcmp (from + 1, all_upper ? "HROUGH" : "hrough", 2388 sizeof "hrough" - 1)) 2389 return false; 2390 from += sizeof "through" - 1; 2391 } 2392 else 2393 from += sizeof "thru" - 1; 2394 while (*from == ' ' || *from == '\t' || *from == '.' || *from == '!') 2395 from++; 2396 if (*from == '-') 2397 { 2398 from++; 2399 if (*comment_start == '*') 2400 { 2401 do 2402 { 2403 while (*from && *from != '*' 2404 && *from != '\n' && *from != '\r') 2405 from++; 2406 if (*from != '*' || from[1] == '/') 2407 break; 2408 from++; 2409 } 2410 while (1); 2411 } 2412 else 2413 while (*from && *from != '\n' && *from != '\r') 2414 from++; 2415 } 2416 } 2417 /* C block comment. */ 2418 if (*comment_start == '*') 2419 { 2420 if (*from != '*' || from[1] != '/') 2421 return false; 2422 } 2423 /* C++ line comment. */ 2424 else if (*from != '\n') 2425 return false; 2426 2427 return true; 2428 } 2429 2430 /* Allocate COUNT tokens for RUN. */ 2431 void 2432 _cpp_init_tokenrun (tokenrun *run, unsigned int count) 2433 { 2434 run->base = XNEWVEC (cpp_token, count); 2435 run->limit = run->base + count; 2436 run->next = NULL; 2437 } 2438 2439 /* Returns the next tokenrun, or creates one if there is none. */ 2440 static tokenrun * 2441 next_tokenrun (tokenrun *run) 2442 { 2443 if (run->next == NULL) 2444 { 2445 run->next = XNEW (tokenrun); 2446 run->next->prev = run; 2447 _cpp_init_tokenrun (run->next, 250); 2448 } 2449 2450 return run->next; 2451 } 2452 2453 /* Return the number of not yet processed token in a given 2454 context. */ 2455 int 2456 _cpp_remaining_tokens_num_in_context (cpp_context *context) 2457 { 2458 if (context->tokens_kind == TOKENS_KIND_DIRECT) 2459 return (LAST (context).token - FIRST (context).token); 2460 else if (context->tokens_kind == TOKENS_KIND_INDIRECT 2461 || context->tokens_kind == TOKENS_KIND_EXTENDED) 2462 return (LAST (context).ptoken - FIRST (context).ptoken); 2463 else 2464 abort (); 2465 } 2466 2467 /* Returns the token present at index INDEX in a given context. If 2468 INDEX is zero, the next token to be processed is returned. */ 2469 static const cpp_token* 2470 _cpp_token_from_context_at (cpp_context *context, int index) 2471 { 2472 if (context->tokens_kind == TOKENS_KIND_DIRECT) 2473 return &(FIRST (context).token[index]); 2474 else if (context->tokens_kind == TOKENS_KIND_INDIRECT 2475 || context->tokens_kind == TOKENS_KIND_EXTENDED) 2476 return FIRST (context).ptoken[index]; 2477 else 2478 abort (); 2479 } 2480 2481 /* Look ahead in the input stream. */ 2482 const cpp_token * 2483 cpp_peek_token (cpp_reader *pfile, int index) 2484 { 2485 cpp_context *context = pfile->context; 2486 const cpp_token *peektok; 2487 int count; 2488 2489 /* First, scan through any pending cpp_context objects. */ 2490 while (context->prev) 2491 { 2492 ptrdiff_t sz = _cpp_remaining_tokens_num_in_context (context); 2493 2494 if (index < (int) sz) 2495 return _cpp_token_from_context_at (context, index); 2496 index -= (int) sz; 2497 context = context->prev; 2498 } 2499 2500 /* We will have to read some new tokens after all (and do so 2501 without invalidating preceding tokens). */ 2502 count = index; 2503 pfile->keep_tokens++; 2504 2505 /* For peeked tokens temporarily disable line_change reporting, 2506 until the tokens are parsed for real. */ 2507 void (*line_change) (cpp_reader *, const cpp_token *, int) 2508 = pfile->cb.line_change; 2509 pfile->cb.line_change = NULL; 2510 2511 do 2512 { 2513 peektok = _cpp_lex_token (pfile); 2514 if (peektok->type == CPP_EOF) 2515 { 2516 index--; 2517 break; 2518 } 2519 } 2520 while (index--); 2521 2522 _cpp_backup_tokens_direct (pfile, count - index); 2523 pfile->keep_tokens--; 2524 pfile->cb.line_change = line_change; 2525 2526 return peektok; 2527 } 2528 2529 /* Allocate a single token that is invalidated at the same time as the 2530 rest of the tokens on the line. Has its line and col set to the 2531 same as the last lexed token, so that diagnostics appear in the 2532 right place. */ 2533 cpp_token * 2534 _cpp_temp_token (cpp_reader *pfile) 2535 { 2536 cpp_token *old, *result; 2537 ptrdiff_t sz = pfile->cur_run->limit - pfile->cur_token; 2538 ptrdiff_t la = (ptrdiff_t) pfile->lookaheads; 2539 2540 old = pfile->cur_token - 1; 2541 /* Any pre-existing lookaheads must not be clobbered. */ 2542 if (la) 2543 { 2544 if (sz <= la) 2545 { 2546 tokenrun *next = next_tokenrun (pfile->cur_run); 2547 2548 if (sz < la) 2549 memmove (next->base + 1, next->base, 2550 (la - sz) * sizeof (cpp_token)); 2551 2552 next->base[0] = pfile->cur_run->limit[-1]; 2553 } 2554 2555 if (sz > 1) 2556 memmove (pfile->cur_token + 1, pfile->cur_token, 2557 MIN (la, sz - 1) * sizeof (cpp_token)); 2558 } 2559 2560 if (!sz && pfile->cur_token == pfile->cur_run->limit) 2561 { 2562 pfile->cur_run = next_tokenrun (pfile->cur_run); 2563 pfile->cur_token = pfile->cur_run->base; 2564 } 2565 2566 result = pfile->cur_token++; 2567 result->src_loc = old->src_loc; 2568 return result; 2569 } 2570 2571 /* Lex a token into RESULT (external interface). Takes care of issues 2572 like directive handling, token lookahead, multiple include 2573 optimization and skipping. */ 2574 const cpp_token * 2575 _cpp_lex_token (cpp_reader *pfile) 2576 { 2577 cpp_token *result; 2578 2579 for (;;) 2580 { 2581 if (pfile->cur_token == pfile->cur_run->limit) 2582 { 2583 pfile->cur_run = next_tokenrun (pfile->cur_run); 2584 pfile->cur_token = pfile->cur_run->base; 2585 } 2586 /* We assume that the current token is somewhere in the current 2587 run. */ 2588 if (pfile->cur_token < pfile->cur_run->base 2589 || pfile->cur_token >= pfile->cur_run->limit) 2590 abort (); 2591 2592 if (pfile->lookaheads) 2593 { 2594 pfile->lookaheads--; 2595 result = pfile->cur_token++; 2596 } 2597 else 2598 result = _cpp_lex_direct (pfile); 2599 2600 if (result->flags & BOL) 2601 { 2602 /* Is this a directive. If _cpp_handle_directive returns 2603 false, it is an assembler #. */ 2604 if (result->type == CPP_HASH 2605 /* 6.10.3 p 11: Directives in a list of macro arguments 2606 gives undefined behavior. This implementation 2607 handles the directive as normal. */ 2608 && pfile->state.parsing_args != 1) 2609 { 2610 if (_cpp_handle_directive (pfile, result->flags & PREV_WHITE)) 2611 { 2612 if (pfile->directive_result.type == CPP_PADDING) 2613 continue; 2614 result = &pfile->directive_result; 2615 } 2616 } 2617 else if (pfile->state.in_deferred_pragma) 2618 result = &pfile->directive_result; 2619 2620 if (pfile->cb.line_change && !pfile->state.skipping) 2621 pfile->cb.line_change (pfile, result, pfile->state.parsing_args); 2622 } 2623 2624 /* We don't skip tokens in directives. */ 2625 if (pfile->state.in_directive || pfile->state.in_deferred_pragma) 2626 break; 2627 2628 /* Outside a directive, invalidate controlling macros. At file 2629 EOF, _cpp_lex_direct takes care of popping the buffer, so we never 2630 get here and MI optimization works. */ 2631 pfile->mi_valid = false; 2632 2633 if (!pfile->state.skipping || result->type == CPP_EOF) 2634 break; 2635 } 2636 2637 return result; 2638 } 2639 2640 /* Returns true if a fresh line has been loaded. */ 2641 bool 2642 _cpp_get_fresh_line (cpp_reader *pfile) 2643 { 2644 int return_at_eof; 2645 2646 /* We can't get a new line until we leave the current directive. */ 2647 if (pfile->state.in_directive) 2648 return false; 2649 2650 for (;;) 2651 { 2652 cpp_buffer *buffer = pfile->buffer; 2653 2654 if (!buffer->need_line) 2655 return true; 2656 2657 if (buffer->next_line < buffer->rlimit) 2658 { 2659 _cpp_clean_line (pfile); 2660 return true; 2661 } 2662 2663 /* First, get out of parsing arguments state. */ 2664 if (pfile->state.parsing_args) 2665 return false; 2666 2667 /* End of buffer. Non-empty files should end in a newline. */ 2668 if (buffer->buf != buffer->rlimit 2669 && buffer->next_line > buffer->rlimit 2670 && !buffer->from_stage3) 2671 { 2672 /* Clip to buffer size. */ 2673 buffer->next_line = buffer->rlimit; 2674 } 2675 2676 return_at_eof = buffer->return_at_eof; 2677 _cpp_pop_buffer (pfile); 2678 if (pfile->buffer == NULL || return_at_eof) 2679 return false; 2680 } 2681 } 2682 2683 #define IF_NEXT_IS(CHAR, THEN_TYPE, ELSE_TYPE) \ 2684 do \ 2685 { \ 2686 result->type = ELSE_TYPE; \ 2687 if (*buffer->cur == CHAR) \ 2688 buffer->cur++, result->type = THEN_TYPE; \ 2689 } \ 2690 while (0) 2691 2692 /* Lex a token into pfile->cur_token, which is also incremented, to 2693 get diagnostics pointing to the correct location. 2694 2695 Does not handle issues such as token lookahead, multiple-include 2696 optimization, directives, skipping etc. This function is only 2697 suitable for use by _cpp_lex_token, and in special cases like 2698 lex_expansion_token which doesn't care for any of these issues. 2699 2700 When meeting a newline, returns CPP_EOF if parsing a directive, 2701 otherwise returns to the start of the token buffer if permissible. 2702 Returns the location of the lexed token. */ 2703 cpp_token * 2704 _cpp_lex_direct (cpp_reader *pfile) 2705 { 2706 cppchar_t c; 2707 cpp_buffer *buffer; 2708 const unsigned char *comment_start; 2709 bool fallthrough_comment = false; 2710 cpp_token *result = pfile->cur_token++; 2711 2712 fresh_line: 2713 result->flags = 0; 2714 buffer = pfile->buffer; 2715 if (buffer->need_line) 2716 { 2717 if (pfile->state.in_deferred_pragma) 2718 { 2719 result->type = CPP_PRAGMA_EOL; 2720 pfile->state.in_deferred_pragma = false; 2721 if (!pfile->state.pragma_allow_expansion) 2722 pfile->state.prevent_expansion--; 2723 return result; 2724 } 2725 if (!_cpp_get_fresh_line (pfile)) 2726 { 2727 result->type = CPP_EOF; 2728 if (!pfile->state.in_directive) 2729 { 2730 /* Tell the compiler the line number of the EOF token. */ 2731 result->src_loc = pfile->line_table->highest_line; 2732 result->flags = BOL; 2733 } 2734 return result; 2735 } 2736 if (buffer != pfile->buffer) 2737 fallthrough_comment = false; 2738 if (!pfile->keep_tokens) 2739 { 2740 pfile->cur_run = &pfile->base_run; 2741 result = pfile->base_run.base; 2742 pfile->cur_token = result + 1; 2743 } 2744 result->flags = BOL; 2745 if (pfile->state.parsing_args == 2) 2746 result->flags |= PREV_WHITE; 2747 } 2748 buffer = pfile->buffer; 2749 update_tokens_line: 2750 result->src_loc = pfile->line_table->highest_line; 2751 2752 skipped_white: 2753 if (buffer->cur >= buffer->notes[buffer->cur_note].pos 2754 && !pfile->overlaid_buffer) 2755 { 2756 _cpp_process_line_notes (pfile, false); 2757 result->src_loc = pfile->line_table->highest_line; 2758 } 2759 c = *buffer->cur++; 2760 2761 if (pfile->forced_token_location) 2762 result->src_loc = pfile->forced_token_location; 2763 else 2764 result->src_loc = linemap_position_for_column (pfile->line_table, 2765 CPP_BUF_COLUMN (buffer, buffer->cur)); 2766 2767 switch (c) 2768 { 2769 case ' ': case '\t': case '\f': case '\v': case '\0': 2770 result->flags |= PREV_WHITE; 2771 skip_whitespace (pfile, c); 2772 goto skipped_white; 2773 2774 case '\n': 2775 if (buffer->cur < buffer->rlimit) 2776 CPP_INCREMENT_LINE (pfile, 0); 2777 buffer->need_line = true; 2778 goto fresh_line; 2779 2780 case '0': case '1': case '2': case '3': case '4': 2781 case '5': case '6': case '7': case '8': case '9': 2782 { 2783 struct normalize_state nst = INITIAL_NORMALIZE_STATE; 2784 result->type = CPP_NUMBER; 2785 lex_number (pfile, &result->val.str, &nst); 2786 warn_about_normalization (pfile, result, &nst); 2787 break; 2788 } 2789 2790 case 'L': 2791 case 'u': 2792 case 'U': 2793 case 'R': 2794 /* 'L', 'u', 'U', 'u8' or 'R' may introduce wide characters, 2795 wide strings or raw strings. */ 2796 if (c == 'L' || CPP_OPTION (pfile, rliterals) 2797 || (c != 'R' && CPP_OPTION (pfile, uliterals))) 2798 { 2799 if ((*buffer->cur == '\'' && c != 'R') 2800 || *buffer->cur == '"' 2801 || (*buffer->cur == 'R' 2802 && c != 'R' 2803 && buffer->cur[1] == '"' 2804 && CPP_OPTION (pfile, rliterals)) 2805 || (*buffer->cur == '8' 2806 && c == 'u' 2807 && ((buffer->cur[1] == '"' || (buffer->cur[1] == '\'' 2808 && CPP_OPTION (pfile, utf8_char_literals))) 2809 || (buffer->cur[1] == 'R' && buffer->cur[2] == '"' 2810 && CPP_OPTION (pfile, rliterals))))) 2811 { 2812 lex_string (pfile, result, buffer->cur - 1); 2813 break; 2814 } 2815 } 2816 /* Fall through. */ 2817 2818 case '_': 2819 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': 2820 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': 2821 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': 2822 case 's': case 't': case 'v': case 'w': case 'x': 2823 case 'y': case 'z': 2824 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': 2825 case 'G': case 'H': case 'I': case 'J': case 'K': 2826 case 'M': case 'N': case 'O': case 'P': case 'Q': 2827 case 'S': case 'T': case 'V': case 'W': case 'X': 2828 case 'Y': case 'Z': 2829 result->type = CPP_NAME; 2830 { 2831 struct normalize_state nst = INITIAL_NORMALIZE_STATE; 2832 result->val.node.node = lex_identifier (pfile, buffer->cur - 1, false, 2833 &nst, 2834 &result->val.node.spelling); 2835 warn_about_normalization (pfile, result, &nst); 2836 } 2837 2838 /* Convert named operators to their proper types. */ 2839 if (result->val.node.node->flags & NODE_OPERATOR) 2840 { 2841 result->flags |= NAMED_OP; 2842 result->type = (enum cpp_ttype) result->val.node.node->directive_index; 2843 } 2844 2845 /* Signal FALLTHROUGH comment followed by another token. */ 2846 if (fallthrough_comment) 2847 result->flags |= PREV_FALLTHROUGH; 2848 break; 2849 2850 case '\'': 2851 case '"': 2852 lex_string (pfile, result, buffer->cur - 1); 2853 break; 2854 2855 case '/': 2856 /* A potential block or line comment. */ 2857 comment_start = buffer->cur; 2858 c = *buffer->cur; 2859 2860 if (c == '*') 2861 { 2862 if (_cpp_skip_block_comment (pfile)) 2863 cpp_error (pfile, CPP_DL_ERROR, "unterminated comment"); 2864 } 2865 else if (c == '/' && ! CPP_OPTION (pfile, traditional)) 2866 { 2867 /* Don't warn for system headers. */ 2868 if (cpp_in_system_header (pfile)) 2869 ; 2870 /* Warn about comments if pedantically GNUC89, and not 2871 in system headers. */ 2872 else if (CPP_OPTION (pfile, lang) == CLK_GNUC89 2873 && CPP_PEDANTIC (pfile) 2874 && ! buffer->warned_cplusplus_comments) 2875 { 2876 if (cpp_error (pfile, CPP_DL_PEDWARN, 2877 "C++ style comments are not allowed in ISO C90")) 2878 cpp_error (pfile, CPP_DL_NOTE, 2879 "(this will be reported only once per input file)"); 2880 buffer->warned_cplusplus_comments = 1; 2881 } 2882 /* Or if specifically desired via -Wc90-c99-compat. */ 2883 else if (CPP_OPTION (pfile, cpp_warn_c90_c99_compat) > 0 2884 && ! CPP_OPTION (pfile, cplusplus) 2885 && ! buffer->warned_cplusplus_comments) 2886 { 2887 if (cpp_error (pfile, CPP_DL_WARNING, 2888 "C++ style comments are incompatible with C90")) 2889 cpp_error (pfile, CPP_DL_NOTE, 2890 "(this will be reported only once per input file)"); 2891 buffer->warned_cplusplus_comments = 1; 2892 } 2893 /* In C89/C94, C++ style comments are forbidden. */ 2894 else if ((CPP_OPTION (pfile, lang) == CLK_STDC89 2895 || CPP_OPTION (pfile, lang) == CLK_STDC94)) 2896 { 2897 /* But don't be confused about valid code such as 2898 - // immediately followed by *, 2899 - // in a preprocessing directive, 2900 - // in an #if 0 block. */ 2901 if (buffer->cur[1] == '*' 2902 || pfile->state.in_directive 2903 || pfile->state.skipping) 2904 { 2905 result->type = CPP_DIV; 2906 break; 2907 } 2908 else if (! buffer->warned_cplusplus_comments) 2909 { 2910 if (cpp_error (pfile, CPP_DL_ERROR, 2911 "C++ style comments are not allowed in " 2912 "ISO C90")) 2913 cpp_error (pfile, CPP_DL_NOTE, 2914 "(this will be reported only once per input " 2915 "file)"); 2916 buffer->warned_cplusplus_comments = 1; 2917 } 2918 } 2919 if (skip_line_comment (pfile) && CPP_OPTION (pfile, warn_comments)) 2920 cpp_warning (pfile, CPP_W_COMMENTS, "multi-line comment"); 2921 } 2922 else if (c == '=') 2923 { 2924 buffer->cur++; 2925 result->type = CPP_DIV_EQ; 2926 break; 2927 } 2928 else 2929 { 2930 result->type = CPP_DIV; 2931 break; 2932 } 2933 2934 if (fallthrough_comment_p (pfile, comment_start)) 2935 fallthrough_comment = true; 2936 2937 if (pfile->cb.comment) 2938 { 2939 size_t len = pfile->buffer->cur - comment_start; 2940 pfile->cb.comment (pfile, result->src_loc, comment_start - 1, 2941 len + 1); 2942 } 2943 2944 if (!pfile->state.save_comments) 2945 { 2946 result->flags |= PREV_WHITE; 2947 goto update_tokens_line; 2948 } 2949 2950 if (fallthrough_comment) 2951 result->flags |= PREV_FALLTHROUGH; 2952 2953 /* Save the comment as a token in its own right. */ 2954 save_comment (pfile, result, comment_start, c); 2955 break; 2956 2957 case '<': 2958 if (pfile->state.angled_headers) 2959 { 2960 lex_string (pfile, result, buffer->cur - 1); 2961 if (result->type != CPP_LESS) 2962 break; 2963 } 2964 2965 result->type = CPP_LESS; 2966 if (*buffer->cur == '=') 2967 buffer->cur++, result->type = CPP_LESS_EQ; 2968 else if (*buffer->cur == '<') 2969 { 2970 buffer->cur++; 2971 IF_NEXT_IS ('=', CPP_LSHIFT_EQ, CPP_LSHIFT); 2972 } 2973 else if (CPP_OPTION (pfile, digraphs)) 2974 { 2975 if (*buffer->cur == ':') 2976 { 2977 /* C++11 [2.5/3 lex.pptoken], "Otherwise, if the next 2978 three characters are <:: and the subsequent character 2979 is neither : nor >, the < is treated as a preprocessor 2980 token by itself". */ 2981 if (CPP_OPTION (pfile, cplusplus) 2982 && CPP_OPTION (pfile, lang) != CLK_CXX98 2983 && CPP_OPTION (pfile, lang) != CLK_GNUCXX 2984 && buffer->cur[1] == ':' 2985 && buffer->cur[2] != ':' && buffer->cur[2] != '>') 2986 break; 2987 2988 buffer->cur++; 2989 result->flags |= DIGRAPH; 2990 result->type = CPP_OPEN_SQUARE; 2991 } 2992 else if (*buffer->cur == '%') 2993 { 2994 buffer->cur++; 2995 result->flags |= DIGRAPH; 2996 result->type = CPP_OPEN_BRACE; 2997 } 2998 } 2999 break; 3000 3001 case '>': 3002 result->type = CPP_GREATER; 3003 if (*buffer->cur == '=') 3004 buffer->cur++, result->type = CPP_GREATER_EQ; 3005 else if (*buffer->cur == '>') 3006 { 3007 buffer->cur++; 3008 IF_NEXT_IS ('=', CPP_RSHIFT_EQ, CPP_RSHIFT); 3009 } 3010 break; 3011 3012 case '%': 3013 result->type = CPP_MOD; 3014 if (*buffer->cur == '=') 3015 buffer->cur++, result->type = CPP_MOD_EQ; 3016 else if (CPP_OPTION (pfile, digraphs)) 3017 { 3018 if (*buffer->cur == ':') 3019 { 3020 buffer->cur++; 3021 result->flags |= DIGRAPH; 3022 result->type = CPP_HASH; 3023 if (*buffer->cur == '%' && buffer->cur[1] == ':') 3024 buffer->cur += 2, result->type = CPP_PASTE, result->val.token_no = 0; 3025 } 3026 else if (*buffer->cur == '>') 3027 { 3028 buffer->cur++; 3029 result->flags |= DIGRAPH; 3030 result->type = CPP_CLOSE_BRACE; 3031 } 3032 } 3033 break; 3034 3035 case '.': 3036 result->type = CPP_DOT; 3037 if (ISDIGIT (*buffer->cur)) 3038 { 3039 struct normalize_state nst = INITIAL_NORMALIZE_STATE; 3040 result->type = CPP_NUMBER; 3041 lex_number (pfile, &result->val.str, &nst); 3042 warn_about_normalization (pfile, result, &nst); 3043 } 3044 else if (*buffer->cur == '.' && buffer->cur[1] == '.') 3045 buffer->cur += 2, result->type = CPP_ELLIPSIS; 3046 else if (*buffer->cur == '*' && CPP_OPTION (pfile, cplusplus)) 3047 buffer->cur++, result->type = CPP_DOT_STAR; 3048 break; 3049 3050 case '+': 3051 result->type = CPP_PLUS; 3052 if (*buffer->cur == '+') 3053 buffer->cur++, result->type = CPP_PLUS_PLUS; 3054 else if (*buffer->cur == '=') 3055 buffer->cur++, result->type = CPP_PLUS_EQ; 3056 break; 3057 3058 case '-': 3059 result->type = CPP_MINUS; 3060 if (*buffer->cur == '>') 3061 { 3062 buffer->cur++; 3063 result->type = CPP_DEREF; 3064 if (*buffer->cur == '*' && CPP_OPTION (pfile, cplusplus)) 3065 buffer->cur++, result->type = CPP_DEREF_STAR; 3066 } 3067 else if (*buffer->cur == '-') 3068 buffer->cur++, result->type = CPP_MINUS_MINUS; 3069 else if (*buffer->cur == '=') 3070 buffer->cur++, result->type = CPP_MINUS_EQ; 3071 break; 3072 3073 case '&': 3074 result->type = CPP_AND; 3075 if (*buffer->cur == '&') 3076 buffer->cur++, result->type = CPP_AND_AND; 3077 else if (*buffer->cur == '=') 3078 buffer->cur++, result->type = CPP_AND_EQ; 3079 break; 3080 3081 case '|': 3082 result->type = CPP_OR; 3083 if (*buffer->cur == '|') 3084 buffer->cur++, result->type = CPP_OR_OR; 3085 else if (*buffer->cur == '=') 3086 buffer->cur++, result->type = CPP_OR_EQ; 3087 break; 3088 3089 case ':': 3090 result->type = CPP_COLON; 3091 if (*buffer->cur == ':' && CPP_OPTION (pfile, cplusplus)) 3092 buffer->cur++, result->type = CPP_SCOPE; 3093 else if (*buffer->cur == '>' && CPP_OPTION (pfile, digraphs)) 3094 { 3095 buffer->cur++; 3096 result->flags |= DIGRAPH; 3097 result->type = CPP_CLOSE_SQUARE; 3098 } 3099 break; 3100 3101 case '*': IF_NEXT_IS ('=', CPP_MULT_EQ, CPP_MULT); break; 3102 case '=': IF_NEXT_IS ('=', CPP_EQ_EQ, CPP_EQ); break; 3103 case '!': IF_NEXT_IS ('=', CPP_NOT_EQ, CPP_NOT); break; 3104 case '^': IF_NEXT_IS ('=', CPP_XOR_EQ, CPP_XOR); break; 3105 case '#': IF_NEXT_IS ('#', CPP_PASTE, CPP_HASH); result->val.token_no = 0; break; 3106 3107 case '?': result->type = CPP_QUERY; break; 3108 case '~': result->type = CPP_COMPL; break; 3109 case ',': result->type = CPP_COMMA; break; 3110 case '(': result->type = CPP_OPEN_PAREN; break; 3111 case ')': result->type = CPP_CLOSE_PAREN; break; 3112 case '[': result->type = CPP_OPEN_SQUARE; break; 3113 case ']': result->type = CPP_CLOSE_SQUARE; break; 3114 case '{': result->type = CPP_OPEN_BRACE; break; 3115 case '}': result->type = CPP_CLOSE_BRACE; break; 3116 case ';': result->type = CPP_SEMICOLON; break; 3117 3118 /* @ is a punctuator in Objective-C. */ 3119 case '@': result->type = CPP_ATSIGN; break; 3120 3121 case '$': 3122 case '\\': 3123 { 3124 const uchar *base = --buffer->cur; 3125 struct normalize_state nst = INITIAL_NORMALIZE_STATE; 3126 3127 if (forms_identifier_p (pfile, true, &nst)) 3128 { 3129 result->type = CPP_NAME; 3130 result->val.node.node = lex_identifier (pfile, base, true, &nst, 3131 &result->val.node.spelling); 3132 warn_about_normalization (pfile, result, &nst); 3133 break; 3134 } 3135 buffer->cur++; 3136 } 3137 /* FALLTHRU */ 3138 3139 default: 3140 create_literal (pfile, result, buffer->cur - 1, 1, CPP_OTHER); 3141 break; 3142 } 3143 3144 /* Potentially convert the location of the token to a range. */ 3145 if (result->src_loc >= RESERVED_LOCATION_COUNT 3146 && result->type != CPP_EOF) 3147 { 3148 /* Ensure that any line notes are processed, so that we have the 3149 correct physical line/column for the end-point of the token even 3150 when a logical line is split via one or more backslashes. */ 3151 if (buffer->cur >= buffer->notes[buffer->cur_note].pos 3152 && !pfile->overlaid_buffer) 3153 _cpp_process_line_notes (pfile, false); 3154 3155 source_range tok_range; 3156 tok_range.m_start = result->src_loc; 3157 tok_range.m_finish 3158 = linemap_position_for_column (pfile->line_table, 3159 CPP_BUF_COLUMN (buffer, buffer->cur)); 3160 3161 result->src_loc = COMBINE_LOCATION_DATA (pfile->line_table, 3162 result->src_loc, 3163 tok_range, NULL); 3164 } 3165 3166 return result; 3167 } 3168 3169 /* An upper bound on the number of bytes needed to spell TOKEN. 3170 Does not include preceding whitespace. */ 3171 unsigned int 3172 cpp_token_len (const cpp_token *token) 3173 { 3174 unsigned int len; 3175 3176 switch (TOKEN_SPELL (token)) 3177 { 3178 default: len = 6; break; 3179 case SPELL_LITERAL: len = token->val.str.len; break; 3180 case SPELL_IDENT: len = NODE_LEN (token->val.node.node) * 10; break; 3181 } 3182 3183 return len; 3184 } 3185 3186 /* Parse UTF-8 out of NAMEP and place a \U escape in BUFFER. 3187 Return the number of bytes read out of NAME. (There are always 3188 10 bytes written to BUFFER.) */ 3189 3190 static size_t 3191 utf8_to_ucn (unsigned char *buffer, const unsigned char *name) 3192 { 3193 int j; 3194 int ucn_len = 0; 3195 int ucn_len_c; 3196 unsigned t; 3197 unsigned long utf32; 3198 3199 /* Compute the length of the UTF-8 sequence. */ 3200 for (t = *name; t & 0x80; t <<= 1) 3201 ucn_len++; 3202 3203 utf32 = *name & (0x7F >> ucn_len); 3204 for (ucn_len_c = 1; ucn_len_c < ucn_len; ucn_len_c++) 3205 { 3206 utf32 = (utf32 << 6) | (*++name & 0x3F); 3207 3208 /* Ill-formed UTF-8. */ 3209 if ((*name & ~0x3F) != 0x80) 3210 abort (); 3211 } 3212 3213 *buffer++ = '\\'; 3214 *buffer++ = 'U'; 3215 for (j = 7; j >= 0; j--) 3216 *buffer++ = "0123456789abcdef"[(utf32 >> (4 * j)) & 0xF]; 3217 return ucn_len; 3218 } 3219 3220 /* Given a token TYPE corresponding to a digraph, return a pointer to 3221 the spelling of the digraph. */ 3222 static const unsigned char * 3223 cpp_digraph2name (enum cpp_ttype type) 3224 { 3225 return digraph_spellings[(int) type - (int) CPP_FIRST_DIGRAPH]; 3226 } 3227 3228 /* Write the spelling of an identifier IDENT, using UCNs, to BUFFER. 3229 The buffer must already contain the enough space to hold the 3230 token's spelling. Returns a pointer to the character after the 3231 last character written. */ 3232 unsigned char * 3233 _cpp_spell_ident_ucns (unsigned char *buffer, cpp_hashnode *ident) 3234 { 3235 size_t i; 3236 const unsigned char *name = NODE_NAME (ident); 3237 3238 for (i = 0; i < NODE_LEN (ident); i++) 3239 if (name[i] & ~0x7F) 3240 { 3241 i += utf8_to_ucn (buffer, name + i) - 1; 3242 buffer += 10; 3243 } 3244 else 3245 *buffer++ = name[i]; 3246 3247 return buffer; 3248 } 3249 3250 /* Write the spelling of a token TOKEN to BUFFER. The buffer must 3251 already contain the enough space to hold the token's spelling. 3252 Returns a pointer to the character after the last character written. 3253 FORSTRING is true if this is to be the spelling after translation 3254 phase 1 (with the original spelling of extended identifiers), false 3255 if extended identifiers should always be written using UCNs (there is 3256 no option for always writing them in the internal UTF-8 form). 3257 FIXME: Would be nice if we didn't need the PFILE argument. */ 3258 unsigned char * 3259 cpp_spell_token (cpp_reader *pfile, const cpp_token *token, 3260 unsigned char *buffer, bool forstring) 3261 { 3262 switch (TOKEN_SPELL (token)) 3263 { 3264 case SPELL_OPERATOR: 3265 { 3266 const unsigned char *spelling; 3267 unsigned char c; 3268 3269 if (token->flags & DIGRAPH) 3270 spelling = cpp_digraph2name (token->type); 3271 else if (token->flags & NAMED_OP) 3272 goto spell_ident; 3273 else 3274 spelling = TOKEN_NAME (token); 3275 3276 while ((c = *spelling++) != '\0') 3277 *buffer++ = c; 3278 } 3279 break; 3280 3281 spell_ident: 3282 case SPELL_IDENT: 3283 if (forstring) 3284 { 3285 memcpy (buffer, NODE_NAME (token->val.node.spelling), 3286 NODE_LEN (token->val.node.spelling)); 3287 buffer += NODE_LEN (token->val.node.spelling); 3288 } 3289 else 3290 buffer = _cpp_spell_ident_ucns (buffer, token->val.node.node); 3291 break; 3292 3293 case SPELL_LITERAL: 3294 memcpy (buffer, token->val.str.text, token->val.str.len); 3295 buffer += token->val.str.len; 3296 break; 3297 3298 case SPELL_NONE: 3299 cpp_error (pfile, CPP_DL_ICE, 3300 "unspellable token %s", TOKEN_NAME (token)); 3301 break; 3302 } 3303 3304 return buffer; 3305 } 3306 3307 /* Returns TOKEN spelt as a null-terminated string. The string is 3308 freed when the reader is destroyed. Useful for diagnostics. */ 3309 unsigned char * 3310 cpp_token_as_text (cpp_reader *pfile, const cpp_token *token) 3311 { 3312 unsigned int len = cpp_token_len (token) + 1; 3313 unsigned char *start = _cpp_unaligned_alloc (pfile, len), *end; 3314 3315 end = cpp_spell_token (pfile, token, start, false); 3316 end[0] = '\0'; 3317 3318 return start; 3319 } 3320 3321 /* Returns a pointer to a string which spells the token defined by 3322 TYPE and FLAGS. Used by C front ends, which really should move to 3323 using cpp_token_as_text. */ 3324 const char * 3325 cpp_type2name (enum cpp_ttype type, unsigned char flags) 3326 { 3327 if (flags & DIGRAPH) 3328 return (const char *) cpp_digraph2name (type); 3329 else if (flags & NAMED_OP) 3330 return cpp_named_operator2name (type); 3331 3332 return (const char *) token_spellings[type].name; 3333 } 3334 3335 /* Writes the spelling of token to FP, without any preceding space. 3336 Separated from cpp_spell_token for efficiency - to avoid stdio 3337 double-buffering. */ 3338 void 3339 cpp_output_token (const cpp_token *token, FILE *fp) 3340 { 3341 switch (TOKEN_SPELL (token)) 3342 { 3343 case SPELL_OPERATOR: 3344 { 3345 const unsigned char *spelling; 3346 int c; 3347 3348 if (token->flags & DIGRAPH) 3349 spelling = cpp_digraph2name (token->type); 3350 else if (token->flags & NAMED_OP) 3351 goto spell_ident; 3352 else 3353 spelling = TOKEN_NAME (token); 3354 3355 c = *spelling; 3356 do 3357 putc (c, fp); 3358 while ((c = *++spelling) != '\0'); 3359 } 3360 break; 3361 3362 spell_ident: 3363 case SPELL_IDENT: 3364 { 3365 size_t i; 3366 const unsigned char * name = NODE_NAME (token->val.node.node); 3367 3368 for (i = 0; i < NODE_LEN (token->val.node.node); i++) 3369 if (name[i] & ~0x7F) 3370 { 3371 unsigned char buffer[10]; 3372 i += utf8_to_ucn (buffer, name + i) - 1; 3373 fwrite (buffer, 1, 10, fp); 3374 } 3375 else 3376 fputc (NODE_NAME (token->val.node.node)[i], fp); 3377 } 3378 break; 3379 3380 case SPELL_LITERAL: 3381 fwrite (token->val.str.text, 1, token->val.str.len, fp); 3382 break; 3383 3384 case SPELL_NONE: 3385 /* An error, most probably. */ 3386 break; 3387 } 3388 } 3389 3390 /* Compare two tokens. */ 3391 int 3392 _cpp_equiv_tokens (const cpp_token *a, const cpp_token *b) 3393 { 3394 if (a->type == b->type && a->flags == b->flags) 3395 switch (TOKEN_SPELL (a)) 3396 { 3397 default: /* Keep compiler happy. */ 3398 case SPELL_OPERATOR: 3399 /* token_no is used to track where multiple consecutive ## 3400 tokens were originally located. */ 3401 return (a->type != CPP_PASTE || a->val.token_no == b->val.token_no); 3402 case SPELL_NONE: 3403 return (a->type != CPP_MACRO_ARG 3404 || (a->val.macro_arg.arg_no == b->val.macro_arg.arg_no 3405 && a->val.macro_arg.spelling == b->val.macro_arg.spelling)); 3406 case SPELL_IDENT: 3407 return (a->val.node.node == b->val.node.node 3408 && a->val.node.spelling == b->val.node.spelling); 3409 case SPELL_LITERAL: 3410 return (a->val.str.len == b->val.str.len 3411 && !memcmp (a->val.str.text, b->val.str.text, 3412 a->val.str.len)); 3413 } 3414 3415 return 0; 3416 } 3417 3418 /* Returns nonzero if a space should be inserted to avoid an 3419 accidental token paste for output. For simplicity, it is 3420 conservative, and occasionally advises a space where one is not 3421 needed, e.g. "." and ".2". */ 3422 int 3423 cpp_avoid_paste (cpp_reader *pfile, const cpp_token *token1, 3424 const cpp_token *token2) 3425 { 3426 enum cpp_ttype a = token1->type, b = token2->type; 3427 cppchar_t c; 3428 3429 if (token1->flags & NAMED_OP) 3430 a = CPP_NAME; 3431 if (token2->flags & NAMED_OP) 3432 b = CPP_NAME; 3433 3434 c = EOF; 3435 if (token2->flags & DIGRAPH) 3436 c = digraph_spellings[(int) b - (int) CPP_FIRST_DIGRAPH][0]; 3437 else if (token_spellings[b].category == SPELL_OPERATOR) 3438 c = token_spellings[b].name[0]; 3439 3440 /* Quickly get everything that can paste with an '='. */ 3441 if ((int) a <= (int) CPP_LAST_EQ && c == '=') 3442 return 1; 3443 3444 switch (a) 3445 { 3446 case CPP_GREATER: return c == '>'; 3447 case CPP_LESS: return c == '<' || c == '%' || c == ':'; 3448 case CPP_PLUS: return c == '+'; 3449 case CPP_MINUS: return c == '-' || c == '>'; 3450 case CPP_DIV: return c == '/' || c == '*'; /* Comments. */ 3451 case CPP_MOD: return c == ':' || c == '>'; 3452 case CPP_AND: return c == '&'; 3453 case CPP_OR: return c == '|'; 3454 case CPP_COLON: return c == ':' || c == '>'; 3455 case CPP_DEREF: return c == '*'; 3456 case CPP_DOT: return c == '.' || c == '%' || b == CPP_NUMBER; 3457 case CPP_HASH: return c == '#' || c == '%'; /* Digraph form. */ 3458 case CPP_NAME: return ((b == CPP_NUMBER 3459 && name_p (pfile, &token2->val.str)) 3460 || b == CPP_NAME 3461 || b == CPP_CHAR || b == CPP_STRING); /* L */ 3462 case CPP_NUMBER: return (b == CPP_NUMBER || b == CPP_NAME 3463 || c == '.' || c == '+' || c == '-'); 3464 /* UCNs */ 3465 case CPP_OTHER: return ((token1->val.str.text[0] == '\\' 3466 && b == CPP_NAME) 3467 || (CPP_OPTION (pfile, objc) 3468 && token1->val.str.text[0] == '@' 3469 && (b == CPP_NAME || b == CPP_STRING))); 3470 case CPP_STRING: 3471 case CPP_WSTRING: 3472 case CPP_UTF8STRING: 3473 case CPP_STRING16: 3474 case CPP_STRING32: return (CPP_OPTION (pfile, user_literals) 3475 && (b == CPP_NAME 3476 || (TOKEN_SPELL (token2) == SPELL_LITERAL 3477 && ISIDST (token2->val.str.text[0])))); 3478 3479 default: break; 3480 } 3481 3482 return 0; 3483 } 3484 3485 /* Output all the remaining tokens on the current line, and a newline 3486 character, to FP. Leading whitespace is removed. If there are 3487 macros, special token padding is not performed. */ 3488 void 3489 cpp_output_line (cpp_reader *pfile, FILE *fp) 3490 { 3491 const cpp_token *token; 3492 3493 token = cpp_get_token (pfile); 3494 while (token->type != CPP_EOF) 3495 { 3496 cpp_output_token (token, fp); 3497 token = cpp_get_token (pfile); 3498 if (token->flags & PREV_WHITE) 3499 putc (' ', fp); 3500 } 3501 3502 putc ('\n', fp); 3503 } 3504 3505 /* Return a string representation of all the remaining tokens on the 3506 current line. The result is allocated using xmalloc and must be 3507 freed by the caller. */ 3508 unsigned char * 3509 cpp_output_line_to_string (cpp_reader *pfile, const unsigned char *dir_name) 3510 { 3511 const cpp_token *token; 3512 unsigned int out = dir_name ? ustrlen (dir_name) : 0; 3513 unsigned int alloced = 120 + out; 3514 unsigned char *result = (unsigned char *) xmalloc (alloced); 3515 3516 /* If DIR_NAME is empty, there are no initial contents. */ 3517 if (dir_name) 3518 { 3519 sprintf ((char *) result, "#%s ", dir_name); 3520 out += 2; 3521 } 3522 3523 token = cpp_get_token (pfile); 3524 while (token->type != CPP_EOF) 3525 { 3526 unsigned char *last; 3527 /* Include room for a possible space and the terminating nul. */ 3528 unsigned int len = cpp_token_len (token) + 2; 3529 3530 if (out + len > alloced) 3531 { 3532 alloced *= 2; 3533 if (out + len > alloced) 3534 alloced = out + len; 3535 result = (unsigned char *) xrealloc (result, alloced); 3536 } 3537 3538 last = cpp_spell_token (pfile, token, &result[out], 0); 3539 out = last - result; 3540 3541 token = cpp_get_token (pfile); 3542 if (token->flags & PREV_WHITE) 3543 result[out++] = ' '; 3544 } 3545 3546 result[out] = '\0'; 3547 return result; 3548 } 3549 3550 /* Memory buffers. Changing these three constants can have a dramatic 3551 effect on performance. The values here are reasonable defaults, 3552 but might be tuned. If you adjust them, be sure to test across a 3553 range of uses of cpplib, including heavy nested function-like macro 3554 expansion. Also check the change in peak memory usage (NJAMD is a 3555 good tool for this). */ 3556 #define MIN_BUFF_SIZE 8000 3557 #define BUFF_SIZE_UPPER_BOUND(MIN_SIZE) (MIN_BUFF_SIZE + (MIN_SIZE) * 3 / 2) 3558 #define EXTENDED_BUFF_SIZE(BUFF, MIN_EXTRA) \ 3559 (MIN_EXTRA + ((BUFF)->limit - (BUFF)->cur) * 2) 3560 3561 #if MIN_BUFF_SIZE > BUFF_SIZE_UPPER_BOUND (0) 3562 #error BUFF_SIZE_UPPER_BOUND must be at least as large as MIN_BUFF_SIZE! 3563 #endif 3564 3565 /* Create a new allocation buffer. Place the control block at the end 3566 of the buffer, so that buffer overflows will cause immediate chaos. */ 3567 static _cpp_buff * 3568 new_buff (size_t len) 3569 { 3570 _cpp_buff *result; 3571 unsigned char *base; 3572 3573 if (len < MIN_BUFF_SIZE) 3574 len = MIN_BUFF_SIZE; 3575 len = CPP_ALIGN (len); 3576 3577 #ifdef ENABLE_VALGRIND_ANNOTATIONS 3578 /* Valgrind warns about uses of interior pointers, so put _cpp_buff 3579 struct first. */ 3580 size_t slen = CPP_ALIGN2 (sizeof (_cpp_buff), 2 * DEFAULT_ALIGNMENT); 3581 base = XNEWVEC (unsigned char, len + slen); 3582 result = (_cpp_buff *) base; 3583 base += slen; 3584 #else 3585 base = XNEWVEC (unsigned char, len + sizeof (_cpp_buff)); 3586 result = (_cpp_buff *) (base + len); 3587 #endif 3588 result->base = base; 3589 result->cur = base; 3590 result->limit = base + len; 3591 result->next = NULL; 3592 return result; 3593 } 3594 3595 /* Place a chain of unwanted allocation buffers on the free list. */ 3596 void 3597 _cpp_release_buff (cpp_reader *pfile, _cpp_buff *buff) 3598 { 3599 _cpp_buff *end = buff; 3600 3601 while (end->next) 3602 end = end->next; 3603 end->next = pfile->free_buffs; 3604 pfile->free_buffs = buff; 3605 } 3606 3607 /* Return a free buffer of size at least MIN_SIZE. */ 3608 _cpp_buff * 3609 _cpp_get_buff (cpp_reader *pfile, size_t min_size) 3610 { 3611 _cpp_buff *result, **p; 3612 3613 for (p = &pfile->free_buffs;; p = &(*p)->next) 3614 { 3615 size_t size; 3616 3617 if (*p == NULL) 3618 return new_buff (min_size); 3619 result = *p; 3620 size = result->limit - result->base; 3621 /* Return a buffer that's big enough, but don't waste one that's 3622 way too big. */ 3623 if (size >= min_size && size <= BUFF_SIZE_UPPER_BOUND (min_size)) 3624 break; 3625 } 3626 3627 *p = result->next; 3628 result->next = NULL; 3629 result->cur = result->base; 3630 return result; 3631 } 3632 3633 /* Creates a new buffer with enough space to hold the uncommitted 3634 remaining bytes of BUFF, and at least MIN_EXTRA more bytes. Copies 3635 the excess bytes to the new buffer. Chains the new buffer after 3636 BUFF, and returns the new buffer. */ 3637 _cpp_buff * 3638 _cpp_append_extend_buff (cpp_reader *pfile, _cpp_buff *buff, size_t min_extra) 3639 { 3640 size_t size = EXTENDED_BUFF_SIZE (buff, min_extra); 3641 _cpp_buff *new_buff = _cpp_get_buff (pfile, size); 3642 3643 buff->next = new_buff; 3644 memcpy (new_buff->base, buff->cur, BUFF_ROOM (buff)); 3645 return new_buff; 3646 } 3647 3648 /* Creates a new buffer with enough space to hold the uncommitted 3649 remaining bytes of the buffer pointed to by BUFF, and at least 3650 MIN_EXTRA more bytes. Copies the excess bytes to the new buffer. 3651 Chains the new buffer before the buffer pointed to by BUFF, and 3652 updates the pointer to point to the new buffer. */ 3653 void 3654 _cpp_extend_buff (cpp_reader *pfile, _cpp_buff **pbuff, size_t min_extra) 3655 { 3656 _cpp_buff *new_buff, *old_buff = *pbuff; 3657 size_t size = EXTENDED_BUFF_SIZE (old_buff, min_extra); 3658 3659 new_buff = _cpp_get_buff (pfile, size); 3660 memcpy (new_buff->base, old_buff->cur, BUFF_ROOM (old_buff)); 3661 new_buff->next = old_buff; 3662 *pbuff = new_buff; 3663 } 3664 3665 /* Free a chain of buffers starting at BUFF. */ 3666 void 3667 _cpp_free_buff (_cpp_buff *buff) 3668 { 3669 _cpp_buff *next; 3670 3671 for (; buff; buff = next) 3672 { 3673 next = buff->next; 3674 #ifdef ENABLE_VALGRIND_ANNOTATIONS 3675 free (buff); 3676 #else 3677 free (buff->base); 3678 #endif 3679 } 3680 } 3681 3682 /* Allocate permanent, unaligned storage of length LEN. */ 3683 unsigned char * 3684 _cpp_unaligned_alloc (cpp_reader *pfile, size_t len) 3685 { 3686 _cpp_buff *buff = pfile->u_buff; 3687 unsigned char *result = buff->cur; 3688 3689 if (len > (size_t) (buff->limit - result)) 3690 { 3691 buff = _cpp_get_buff (pfile, len); 3692 buff->next = pfile->u_buff; 3693 pfile->u_buff = buff; 3694 result = buff->cur; 3695 } 3696 3697 buff->cur = result + len; 3698 return result; 3699 } 3700 3701 /* Allocate permanent, unaligned storage of length LEN from a_buff. 3702 That buffer is used for growing allocations when saving macro 3703 replacement lists in a #define, and when parsing an answer to an 3704 assertion in #assert, #unassert or #if (and therefore possibly 3705 whilst expanding macros). It therefore must not be used by any 3706 code that they might call: specifically the lexer and the guts of 3707 the macro expander. 3708 3709 All existing other uses clearly fit this restriction: storing 3710 registered pragmas during initialization. */ 3711 unsigned char * 3712 _cpp_aligned_alloc (cpp_reader *pfile, size_t len) 3713 { 3714 _cpp_buff *buff = pfile->a_buff; 3715 unsigned char *result = buff->cur; 3716 3717 if (len > (size_t) (buff->limit - result)) 3718 { 3719 buff = _cpp_get_buff (pfile, len); 3720 buff->next = pfile->a_buff; 3721 pfile->a_buff = buff; 3722 result = buff->cur; 3723 } 3724 3725 buff->cur = result + len; 3726 return result; 3727 } 3728 3729 /* Commit or allocate storage from a buffer. */ 3730 3731 void * 3732 _cpp_commit_buff (cpp_reader *pfile, size_t size) 3733 { 3734 void *ptr = BUFF_FRONT (pfile->a_buff); 3735 3736 if (pfile->hash_table->alloc_subobject) 3737 { 3738 void *copy = pfile->hash_table->alloc_subobject (size); 3739 memcpy (copy, ptr, size); 3740 ptr = copy; 3741 } 3742 else 3743 BUFF_FRONT (pfile->a_buff) += size; 3744 3745 return ptr; 3746 } 3747 3748 /* Say which field of TOK is in use. */ 3749 3750 enum cpp_token_fld_kind 3751 cpp_token_val_index (const cpp_token *tok) 3752 { 3753 switch (TOKEN_SPELL (tok)) 3754 { 3755 case SPELL_IDENT: 3756 return CPP_TOKEN_FLD_NODE; 3757 case SPELL_LITERAL: 3758 return CPP_TOKEN_FLD_STR; 3759 case SPELL_OPERATOR: 3760 if (tok->type == CPP_PASTE) 3761 return CPP_TOKEN_FLD_TOKEN_NO; 3762 else 3763 return CPP_TOKEN_FLD_NONE; 3764 case SPELL_NONE: 3765 if (tok->type == CPP_MACRO_ARG) 3766 return CPP_TOKEN_FLD_ARG_NO; 3767 else if (tok->type == CPP_PADDING) 3768 return CPP_TOKEN_FLD_SOURCE; 3769 else if (tok->type == CPP_PRAGMA) 3770 return CPP_TOKEN_FLD_PRAGMA; 3771 /* fall through */ 3772 default: 3773 return CPP_TOKEN_FLD_NONE; 3774 } 3775 } 3776 3777 /* All tokens lexed in R after calling this function will be forced to 3778 have their location_t to be P, until 3779 cpp_stop_forcing_token_locations is called for R. */ 3780 3781 void 3782 cpp_force_token_locations (cpp_reader *r, location_t loc) 3783 { 3784 r->forced_token_location = loc; 3785 } 3786 3787 /* Go back to assigning locations naturally for lexed tokens. */ 3788 3789 void 3790 cpp_stop_forcing_token_locations (cpp_reader *r) 3791 { 3792 r->forced_token_location = 0; 3793 } 3794