1 /**
2 * Defines lexical tokens.
3 *
4 * Specification: $(LINK2 https://dlang.org/spec/lex.html#tokens, Tokens)
5 *
6 * Copyright: Copyright (C) 1999-2022 by The D Language Foundation, All Rights Reserved
7 * Authors: $(LINK2 https://www.digitalmars.com, Walter Bright)
8 * License: $(LINK2 https://www.boost.org/LICENSE_1_0.txt, Boost License 1.0)
9 * Source: $(LINK2 https://github.com/dlang/dmd/blob/master/src/dmd/tokens.d, _tokens.d)
10 * Documentation: https://dlang.org/phobos/dmd_tokens.html
11 * Coverage: https://codecov.io/gh/dlang/dmd/src/master/src/dmd/tokens.d
12 */
13
14 module dmd.tokens;
15
16 import core.stdc.ctype;
17 import core.stdc.stdio;
18 import core.stdc.string;
19 import dmd.globals;
20 import dmd.identifier;
21 import dmd.root.ctfloat;
22 import dmd.common.outbuffer;
23 import dmd.root.rmem;
24 import dmd.root.utf;
25
26 enum TOK : ubyte
27 {
28 reserved,
29
30 // Other
31 leftParenthesis,
32 rightParenthesis,
33 leftBracket,
34 rightBracket,
35 leftCurly,
36 rightCurly,
37 colon,
38 semicolon,
39 dotDotDot,
40 endOfFile,
41 cast_,
42 null_,
43 assert_,
44 true_,
45 false_,
46 throw_,
47 new_,
48 delete_,
49 variable,
50 slice,
51 version_,
52 module_,
53 dollar,
54 template_,
55 typeof_,
56 pragma_,
57 typeid_,
58 comment,
59
60 // Operators
61 lessThan,
62 greaterThan,
63 lessOrEqual,
64 greaterOrEqual,
65 equal,
66 notEqual,
67 identity,
68 notIdentity,
69 is_,
70
71 leftShift,
72 rightShift,
73 leftShiftAssign,
74 rightShiftAssign,
75 unsignedRightShift,
76 unsignedRightShiftAssign,
77 concatenateAssign, // ~=
78 add,
79 min,
80 addAssign,
81 minAssign,
82 mul,
83 div,
84 mod,
85 mulAssign,
86 divAssign,
87 modAssign,
88 and,
89 or,
90 xor,
91 andAssign,
92 orAssign,
93 xorAssign,
94 assign,
95 not,
96 tilde,
97 plusPlus,
98 minusMinus,
99 dot,
100 comma,
101 question,
102 andAnd,
103 orOr,
104
105 // Numeric literals
106 int32Literal,
107 uns32Literal,
108 int64Literal,
109 uns64Literal,
110 int128Literal,
111 uns128Literal,
112 float32Literal,
113 float64Literal,
114 float80Literal,
115 imaginary32Literal,
116 imaginary64Literal,
117 imaginary80Literal,
118
119 // Char constants
120 charLiteral,
121 wcharLiteral,
122 dcharLiteral,
123
124 // Leaf operators
125 identifier,
126 string_,
127 this_,
128 super_,
129 error,
130
131 // Basic types
132 void_,
133 int8,
134 uns8,
135 int16,
136 uns16,
137 int32,
138 uns32,
139 int64,
140 uns64,
141 int128,
142 uns128,
143 float32,
144 float64,
145 float80,
146 imaginary32,
147 imaginary64,
148 imaginary80,
149 complex32,
150 complex64,
151 complex80,
152 char_,
153 wchar_,
154 dchar_,
155 bool_,
156
157 // Aggregates
158 struct_,
159 class_,
160 interface_,
161 union_,
162 enum_,
163 import_,
164 alias_,
165 override_,
166 delegate_,
167 function_,
168 mixin_,
169 align_,
170 extern_,
171 private_,
172 protected_,
173 public_,
174 export_,
175 static_,
176 final_,
177 const_,
178 abstract_,
179 debug_,
180 deprecated_,
181 in_,
182 out_,
183 inout_,
184 lazy_,
185 auto_,
186 package_,
187 immutable_,
188
189 // Statements
190 if_,
191 else_,
192 while_,
193 for_,
194 do_,
195 switch_,
196 case_,
197 default_,
198 break_,
199 continue_,
200 with_,
201 synchronized_,
202 return_,
203 goto_,
204 try_,
205 catch_,
206 finally_,
207 asm_,
208 foreach_,
209 foreach_reverse_,
210 scope_,
211 onScopeExit,
212 onScopeFailure,
213 onScopeSuccess,
214
215 // Contracts
216 invariant_,
217
218 // Testing
219 unittest_,
220
221 // Added after 1.0
222 argumentTypes,
223 ref_,
224 macro_,
225
226 parameters,
227 traits,
228 pure_,
229 nothrow_,
230 gshared,
231 line,
232 file,
233 fileFullPath,
234 moduleString, // __MODULE__
235 functionString, // __FUNCTION__
236 prettyFunction, // __PRETTY_FUNCTION__
237 shared_,
238 at,
239 pow,
240 powAssign,
241 goesTo,
242 vector,
243 pound,
244
245 arrow, // ->
246 colonColon, // ::
247 wchar_tLiteral,
248 endOfLine, // \n, \r, \u2028, \u2029
249 whitespace,
250
251 // C only keywords
252 inline,
253 register,
254 restrict,
255 signed,
256 sizeof_,
257 typedef_,
258 unsigned,
259 volatile,
260 _Alignas,
261 _Alignof,
262 _Atomic,
263 _Bool,
264 _Complex,
265 _Generic,
266 _Imaginary,
267 _Noreturn,
268 _Static_assert,
269 _Thread_local,
270
271 // C only extended keywords
272 _import,
273 __cdecl,
274 __declspec,
275 __stdcall,
276 __attribute__,
277 }
278
279 /// Expression nodes
280 enum EXP : ubyte
281 {
282 reserved,
283
284 // Other
285 negate,
286 cast_,
287 null_,
288 assert_,
289 true_,
290 false_,
291 array,
292 call,
293 address,
294 type,
295 throw_,
296 new_,
297 delete_,
298 star,
299 symbolOffset,
300 variable,
301 dotVariable,
302 dotIdentifier,
303 dotTemplateInstance,
304 dotType,
305 slice,
306 arrayLength,
307 version_,
308 dollar,
309 template_,
310 dotTemplateDeclaration,
311 declaration,
312 typeof_,
313 pragma_,
314 dSymbol,
315 typeid_,
316 uadd,
317 remove,
318 newAnonymousClass,
319 arrayLiteral,
320 assocArrayLiteral,
321 structLiteral,
322 classReference,
323 thrownException,
324 delegatePointer,
325 delegateFunctionPointer,
326
327 // Operators
328 lessThan,
329 greaterThan,
330 lessOrEqual,
331 greaterOrEqual,
332 equal,
333 notEqual,
334 identity,
335 notIdentity,
336 index,
337 is_,
338
339 leftShift,
340 rightShift,
341 leftShiftAssign,
342 rightShiftAssign,
343 unsignedRightShift,
344 unsignedRightShiftAssign,
345 concatenate,
346 concatenateAssign, // ~=
347 concatenateElemAssign,
348 concatenateDcharAssign,
349 add,
350 min,
351 addAssign,
352 minAssign,
353 mul,
354 div,
355 mod,
356 mulAssign,
357 divAssign,
358 modAssign,
359 and,
360 or,
361 xor,
362 andAssign,
363 orAssign,
364 xorAssign,
365 assign,
366 not,
367 tilde,
368 plusPlus,
369 minusMinus,
370 construct,
371 blit,
372 dot,
373 comma,
374 question,
375 andAnd,
376 orOr,
377 prePlusPlus,
378 preMinusMinus,
379
380 // Leaf operators
381 identifier,
382 string_,
383 this_,
384 super_,
385 halt,
386 tuple,
387 error,
388
389 // Basic types
390 void_,
391 int64,
392 float64,
393 complex80,
394 char_,
395 import_,
396 delegate_,
397 function_,
398 mixin_,
399 in_,
400 default_,
401 break_,
402 continue_,
403 goto_,
404 scope_,
405
406 traits,
407 overloadSet,
408 line,
409 file,
410 fileFullPath,
411 moduleString, // __MODULE__
412 functionString, // __FUNCTION__
413 prettyFunction, // __PRETTY_FUNCTION__
414 shared_,
415 pow,
416 powAssign,
417 vector,
418
419 voidExpression,
420 cantExpression,
421 showCtfeContext,
422 objcClassReference,
423 vectorArray,
424 arrow, // ->
425 compoundLiteral, // ( type-name ) { initializer-list }
426 _Generic,
427 interval,
428 }
429
430 enum FirstCKeyword = TOK.inline;
431
432 // Assert that all token enum members have consecutive values and
433 // that none of them overlap
434 static assert(() {
435 foreach (idx, enumName; __traits(allMembers, TOK)) {
436 static if (idx != __traits(getMember, TOK, enumName)) {
437 pragma(msg, "Error: Expected TOK.", enumName, " to be ", idx, " but is ", __traits(getMember, TOK, enumName));
438 static assert(0);
439 }
440 }
441 return true;
442 }());
443
444 /****************************************
445 */
446
447 private immutable TOK[] keywords =
448 [
449 TOK.this_,
450 TOK.super_,
451 TOK.assert_,
452 TOK.null_,
453 TOK.true_,
454 TOK.false_,
455 TOK.cast_,
456 TOK.new_,
457 TOK.delete_,
458 TOK.throw_,
459 TOK.module_,
460 TOK.pragma_,
461 TOK.typeof_,
462 TOK.typeid_,
463 TOK.template_,
464 TOK.void_,
465 TOK.int8,
466 TOK.uns8,
467 TOK.int16,
468 TOK.uns16,
469 TOK.int32,
470 TOK.uns32,
471 TOK.int64,
472 TOK.uns64,
473 TOK.int128,
474 TOK.uns128,
475 TOK.float32,
476 TOK.float64,
477 TOK.float80,
478 TOK.bool_,
479 TOK.char_,
480 TOK.wchar_,
481 TOK.dchar_,
482 TOK.imaginary32,
483 TOK.imaginary64,
484 TOK.imaginary80,
485 TOK.complex32,
486 TOK.complex64,
487 TOK.complex80,
488 TOK.delegate_,
489 TOK.function_,
490 TOK.is_,
491 TOK.if_,
492 TOK.else_,
493 TOK.while_,
494 TOK.for_,
495 TOK.do_,
496 TOK.switch_,
497 TOK.case_,
498 TOK.default_,
499 TOK.break_,
500 TOK.continue_,
501 TOK.synchronized_,
502 TOK.return_,
503 TOK.goto_,
504 TOK.try_,
505 TOK.catch_,
506 TOK.finally_,
507 TOK.with_,
508 TOK.asm_,
509 TOK.foreach_,
510 TOK.foreach_reverse_,
511 TOK.scope_,
512 TOK.struct_,
513 TOK.class_,
514 TOK.interface_,
515 TOK.union_,
516 TOK.enum_,
517 TOK.import_,
518 TOK.mixin_,
519 TOK.static_,
520 TOK.final_,
521 TOK.const_,
522 TOK.alias_,
523 TOK.override_,
524 TOK.abstract_,
525 TOK.debug_,
526 TOK.deprecated_,
527 TOK.in_,
528 TOK.out_,
529 TOK.inout_,
530 TOK.lazy_,
531 TOK.auto_,
532 TOK.align_,
533 TOK.extern_,
534 TOK.private_,
535 TOK.package_,
536 TOK.protected_,
537 TOK.public_,
538 TOK.export_,
539 TOK.invariant_,
540 TOK.unittest_,
541 TOK.version_,
542 TOK.argumentTypes,
543 TOK.parameters,
544 TOK.ref_,
545 TOK.macro_,
546 TOK.pure_,
547 TOK.nothrow_,
548 TOK.gshared,
549 TOK.traits,
550 TOK.vector,
551 TOK.file,
552 TOK.fileFullPath,
553 TOK.line,
554 TOK.moduleString,
555 TOK.functionString,
556 TOK.prettyFunction,
557 TOK.shared_,
558 TOK.immutable_,
559
560 // C only keywords
561 TOK.inline,
562 TOK.register,
563 TOK.restrict,
564 TOK.signed,
565 TOK.sizeof_,
566 TOK.typedef_,
567 TOK.unsigned,
568 TOK.volatile,
569 TOK._Alignas,
570 TOK._Alignof,
571 TOK._Atomic,
572 TOK._Bool,
573 TOK._Complex,
574 TOK._Generic,
575 TOK._Imaginary,
576 TOK._Noreturn,
577 TOK._Static_assert,
578 TOK._Thread_local,
579
580 // C only extended keywords
581 TOK._import,
582 TOK.__cdecl,
583 TOK.__declspec,
584 TOK.__stdcall,
585 TOK.__attribute__,
586 ];
587
588 // Initialize the identifier pool
this()589 shared static this() nothrow
590 {
591 Identifier.initTable();
592 foreach (kw; keywords)
593 {
594 //printf("keyword[%d] = '%s'\n",kw, Token.tochars[kw].ptr);
595 Identifier.idPool(Token.tochars[kw].ptr, Token.tochars[kw].length, cast(uint)kw);
596 }
597 }
598
599 /************************************
600 * This is used to pick the C keywords out of the tokens.
601 * If it's not a C keyword, then it's an identifier.
602 */
603 static immutable TOK[TOK.max + 1] Ckeywords =
604 () {
with(TOK)605 with (TOK)
606 {
607 TOK[TOK.max + 1] tab = identifier; // default to identifier
608 enum Ckwds = [ auto_, break_, case_, char_, const_, continue_, default_, do_, float64, else_,
609 enum_, extern_, float32, for_, goto_, if_, inline, int32, int64, register,
610 restrict, return_, int16, signed, sizeof_, static_, struct_, switch_, typedef_,
611 union_, unsigned, void_, volatile, while_, asm_,
612 _Alignas, _Alignof, _Atomic, _Bool, _Complex, _Generic, _Imaginary, _Noreturn,
613 _Static_assert, _Thread_local, _import, __cdecl, __declspec, __stdcall, __attribute__ ];
614
615 foreach (kw; Ckwds)
616 tab[kw] = cast(TOK) kw;
617
618 return tab;
619 }
620 } ();
621
622
623 /***********************************************************
624 */
625 extern (C++) struct Token
626 {
627 Token* next;
628 Loc loc;
629 const(char)* ptr; // pointer to first character of this token within buffer
630 TOK value;
631 const(char)[] blockComment; // doc comment string prior to this token
632 const(char)[] lineComment; // doc comment for previous token
633
634 union
635 {
636 // Integers
637 sinteger_t intvalue;
638 uinteger_t unsvalue;
639 // Floats
640 real_t floatvalue;
641
642 struct
643 {
644 const(char)* ustring; // UTF8 string
645 uint len;
646 ubyte postfix; // 'c', 'w', 'd'
647 }
648
649 Identifier ident;
650 }
651
652 extern (D) private static immutable string[TOK.max + 1] tochars =
653 [
654 // Keywords
655 TOK.this_: "this",
656 TOK.super_: "super",
657 TOK.assert_: "assert",
658 TOK.null_: "null",
659 TOK.true_: "true",
660 TOK.false_: "false",
661 TOK.cast_: "cast",
662 TOK.new_: "new",
663 TOK.delete_: "delete",
664 TOK.throw_: "throw",
665 TOK.module_: "module",
666 TOK.pragma_: "pragma",
667 TOK.typeof_: "typeof",
668 TOK.typeid_: "typeid",
669 TOK.template_: "template",
670 TOK.void_: "void",
671 TOK.int8: "byte",
672 TOK.uns8: "ubyte",
673 TOK.int16: "short",
674 TOK.uns16: "ushort",
675 TOK.int32: "int",
676 TOK.uns32: "uint",
677 TOK.int64: "long",
678 TOK.uns64: "ulong",
679 TOK.int128: "cent",
680 TOK.uns128: "ucent",
681 TOK.float32: "float",
682 TOK.float64: "double",
683 TOK.float80: "real",
684 TOK.bool_: "bool",
685 TOK.char_: "char",
686 TOK.wchar_: "wchar",
687 TOK.dchar_: "dchar",
688 TOK.imaginary32: "ifloat",
689 TOK.imaginary64: "idouble",
690 TOK.imaginary80: "ireal",
691 TOK.complex32: "cfloat",
692 TOK.complex64: "cdouble",
693 TOK.complex80: "creal",
694 TOK.delegate_: "delegate",
695 TOK.function_: "function",
696 TOK.is_: "is",
697 TOK.if_: "if",
698 TOK.else_: "else",
699 TOK.while_: "while",
700 TOK.for_: "for",
701 TOK.do_: "do",
702 TOK.switch_: "switch",
703 TOK.case_: "case",
704 TOK.default_: "default",
705 TOK.break_: "break",
706 TOK.continue_: "continue",
707 TOK.synchronized_: "synchronized",
708 TOK.return_: "return",
709 TOK.goto_: "goto",
710 TOK.try_: "try",
711 TOK.catch_: "catch",
712 TOK.finally_: "finally",
713 TOK.with_: "with",
714 TOK.asm_: "asm",
715 TOK.foreach_: "foreach",
716 TOK.foreach_reverse_: "foreach_reverse",
717 TOK.scope_: "scope",
718 TOK.struct_: "struct",
719 TOK.class_: "class",
720 TOK.interface_: "interface",
721 TOK.union_: "union",
722 TOK.enum_: "enum",
723 TOK.import_: "import",
724 TOK.mixin_: "mixin",
725 TOK.static_: "static",
726 TOK.final_: "final",
727 TOK.const_: "const",
728 TOK.alias_: "alias",
729 TOK.override_: "override",
730 TOK.abstract_: "abstract",
731 TOK.debug_: "debug",
732 TOK.deprecated_: "deprecated",
733 TOK.in_: "in",
734 TOK.out_: "out",
735 TOK.inout_: "inout",
736 TOK.lazy_: "lazy",
737 TOK.auto_: "auto",
738 TOK.align_: "align",
739 TOK.extern_: "extern",
740 TOK.private_: "private",
741 TOK.package_: "package",
742 TOK.protected_: "protected",
743 TOK.public_: "public",
744 TOK.export_: "export",
745 TOK.invariant_: "invariant",
746 TOK.unittest_: "unittest",
747 TOK.version_: "version",
748 TOK.argumentTypes: "__argTypes",
749 TOK.parameters: "__parameters",
750 TOK.ref_: "ref",
751 TOK.macro_: "macro",
752 TOK.pure_: "pure",
753 TOK.nothrow_: "nothrow",
754 TOK.gshared: "__gshared",
755 TOK.traits: "__traits",
756 TOK.vector: "__vector",
757 TOK.file: "__FILE__",
758 TOK.fileFullPath: "__FILE_FULL_PATH__",
759 TOK.line: "__LINE__",
760 TOK.moduleString: "__MODULE__",
761 TOK.functionString: "__FUNCTION__",
762 TOK.prettyFunction: "__PRETTY_FUNCTION__",
763 TOK.shared_: "shared",
764 TOK.immutable_: "immutable",
765
766 TOK.endOfFile: "End of File",
767 TOK.leftCurly: "{",
768 TOK.rightCurly: "}",
769 TOK.leftParenthesis: "(",
770 TOK.rightParenthesis: ")",
771 TOK.leftBracket: "[",
772 TOK.rightBracket: "]",
773 TOK.semicolon: ";",
774 TOK.colon: ":",
775 TOK.comma: ",",
776 TOK.dot: ".",
777 TOK.xor: "^",
778 TOK.xorAssign: "^=",
779 TOK.assign: "=",
780 TOK.lessThan: "<",
781 TOK.greaterThan: ">",
782 TOK.lessOrEqual: "<=",
783 TOK.greaterOrEqual: ">=",
784 TOK.equal: "==",
785 TOK.notEqual: "!=",
786 TOK.not: "!",
787 TOK.leftShift: "<<",
788 TOK.rightShift: ">>",
789 TOK.unsignedRightShift: ">>>",
790 TOK.add: "+",
791 TOK.min: "-",
792 TOK.mul: "*",
793 TOK.div: "/",
794 TOK.mod: "%",
795 TOK.slice: "..",
796 TOK.dotDotDot: "...",
797 TOK.and: "&",
798 TOK.andAnd: "&&",
799 TOK.or: "|",
800 TOK.orOr: "||",
801 TOK.tilde: "~",
802 TOK.dollar: "$",
803 TOK.plusPlus: "++",
804 TOK.minusMinus: "--",
805 TOK.question: "?",
806 TOK.variable: "var",
807 TOK.addAssign: "+=",
808 TOK.minAssign: "-=",
809 TOK.mulAssign: "*=",
810 TOK.divAssign: "/=",
811 TOK.modAssign: "%=",
812 TOK.leftShiftAssign: "<<=",
813 TOK.rightShiftAssign: ">>=",
814 TOK.unsignedRightShiftAssign: ">>>=",
815 TOK.andAssign: "&=",
816 TOK.orAssign: "|=",
817 TOK.concatenateAssign: "~=",
818 TOK.identity: "is",
819 TOK.notIdentity: "!is",
820 TOK.identifier: "identifier",
821 TOK.at: "@",
822 TOK.pow: "^^",
823 TOK.powAssign: "^^=",
824 TOK.goesTo: "=>",
825 TOK.pound: "#",
826 TOK.arrow: "->",
827 TOK.colonColon: "::",
828
829 // For debugging
830 TOK.error: "error",
831 TOK.string_: "string",
832 TOK.onScopeExit: "scope(exit)",
833 TOK.onScopeSuccess: "scope(success)",
834 TOK.onScopeFailure: "scope(failure)",
835
836 // Finish up
837 TOK.reserved: "reserved",
838 TOK.comment: "comment",
839 TOK.int32Literal: "int32v",
840 TOK.uns32Literal: "uns32v",
841 TOK.int64Literal: "int64v",
842 TOK.uns64Literal: "uns64v",
843 TOK.int128Literal: "int128v",
844 TOK.uns128Literal: "uns128v",
845 TOK.float32Literal: "float32v",
846 TOK.float64Literal: "float64v",
847 TOK.float80Literal: "float80v",
848 TOK.imaginary32Literal: "imaginary32v",
849 TOK.imaginary64Literal: "imaginary64v",
850 TOK.imaginary80Literal: "imaginary80v",
851 TOK.charLiteral: "charv",
852 TOK.wcharLiteral: "wcharv",
853 TOK.dcharLiteral: "dcharv",
854 TOK.wchar_tLiteral: "wchar_tv",
855 TOK.endOfLine: "\\n",
856 TOK.whitespace: "whitespace",
857
858 // C only keywords
859 TOK.inline : "inline",
860 TOK.register : "register",
861 TOK.restrict : "restrict",
862 TOK.signed : "signed",
863 TOK.sizeof_ : "sizeof",
864 TOK.typedef_ : "typedef",
865 TOK.unsigned : "unsigned",
866 TOK.volatile : "volatile",
867 TOK._Alignas : "_Alignas",
868 TOK._Alignof : "_Alignof",
869 TOK._Atomic : "_Atomic",
870 TOK._Bool : "_Bool",
871 TOK._Complex : "_Complex",
872 TOK._Generic : "_Generic",
873 TOK._Imaginary: "_Imaginary",
874 TOK._Noreturn : "_Noreturn",
875 TOK._Static_assert : "_Static_assert",
876 TOK._Thread_local : "_Thread_local",
877
878 // C only extended keywords
879 TOK._import : "__import",
880 TOK.__cdecl : "__cdecl",
881 TOK.__declspec : "__declspec",
882 TOK.__stdcall : "__stdcall",
883 TOK.__attribute__ : "__attribute__",
884 ];
885
886 static assert(() {
887 foreach (s; tochars)
888 assert(s.length);
889 return true;
890 }());
891
892 nothrow:
893
isKeywordToken894 int isKeyword() const
895 {
896 foreach (kw; keywords)
897 {
898 if (kw == value)
899 return 1;
900 }
901 return 0;
902 }
903
904 /****
905 * Set to contents of ptr[0..length]
906 * Params:
907 * ptr = pointer to string
908 * length = length of string
909 */
setStringToken910 void setString(const(char)* ptr, size_t length)
911 {
912 auto s = cast(char*)mem.xmalloc_noscan(length + 1);
913 memcpy(s, ptr, length);
914 s[length] = 0;
915 ustring = s;
916 len = cast(uint)length;
917 postfix = 0;
918 }
919
920 /****
921 * Set to contents of buf
922 * Params:
923 * buf = string (not zero terminated)
924 */
setStringToken925 void setString(const ref OutBuffer buf)
926 {
927 setString(cast(const(char)*)buf[].ptr, buf.length);
928 }
929
930 /****
931 * Set to empty string
932 */
setStringToken933 void setString()
934 {
935 ustring = "";
936 len = 0;
937 postfix = 0;
938 }
939
toCharsToken940 extern (C++) const(char)* toChars() const
941 {
942 __gshared char[3 + 3 * floatvalue.sizeof + 1] buffer;
943 const(char)* p = &buffer[0];
944 switch (value)
945 {
946 case TOK.int32Literal:
947 sprintf(&buffer[0], "%d", cast(int)intvalue);
948 break;
949 case TOK.uns32Literal:
950 case TOK.wchar_tLiteral:
951 sprintf(&buffer[0], "%uU", cast(uint)unsvalue);
952 break;
953 case TOK.wcharLiteral:
954 case TOK.dcharLiteral:
955 case TOK.charLiteral:
956 {
957 OutBuffer buf;
958 buf.writeSingleCharLiteral(cast(dchar) intvalue);
959 buf.writeByte('\0');
960 p = buf.extractSlice().ptr;
961 }
962 break;
963 case TOK.int64Literal:
964 sprintf(&buffer[0], "%lldL", cast(long)intvalue);
965 break;
966 case TOK.uns64Literal:
967 sprintf(&buffer[0], "%lluUL", cast(ulong)unsvalue);
968 break;
969 case TOK.float32Literal:
970 CTFloat.sprint(&buffer[0], 'g', floatvalue);
971 strcat(&buffer[0], "f");
972 break;
973 case TOK.float64Literal:
974 CTFloat.sprint(&buffer[0], 'g', floatvalue);
975 break;
976 case TOK.float80Literal:
977 CTFloat.sprint(&buffer[0], 'g', floatvalue);
978 strcat(&buffer[0], "L");
979 break;
980 case TOK.imaginary32Literal:
981 CTFloat.sprint(&buffer[0], 'g', floatvalue);
982 strcat(&buffer[0], "fi");
983 break;
984 case TOK.imaginary64Literal:
985 CTFloat.sprint(&buffer[0], 'g', floatvalue);
986 strcat(&buffer[0], "i");
987 break;
988 case TOK.imaginary80Literal:
989 CTFloat.sprint(&buffer[0], 'g', floatvalue);
990 strcat(&buffer[0], "Li");
991 break;
992 case TOK.string_:
993 {
994 OutBuffer buf;
995 buf.writeByte('"');
996 for (size_t i = 0; i < len;)
997 {
998 dchar c;
999 utf_decodeChar(ustring[0 .. len], i, c);
1000 writeCharLiteral(buf, c);
1001 }
1002 buf.writeByte('"');
1003 if (postfix)
1004 buf.writeByte(postfix);
1005 buf.writeByte(0);
1006 p = buf.extractSlice().ptr;
1007 }
1008 break;
1009 case TOK.identifier:
1010 case TOK.enum_:
1011 case TOK.struct_:
1012 case TOK.import_:
1013 case TOK.wchar_:
1014 case TOK.dchar_:
1015 case TOK.bool_:
1016 case TOK.char_:
1017 case TOK.int8:
1018 case TOK.uns8:
1019 case TOK.int16:
1020 case TOK.uns16:
1021 case TOK.int32:
1022 case TOK.uns32:
1023 case TOK.int64:
1024 case TOK.uns64:
1025 case TOK.int128:
1026 case TOK.uns128:
1027 case TOK.float32:
1028 case TOK.float64:
1029 case TOK.float80:
1030 case TOK.imaginary32:
1031 case TOK.imaginary64:
1032 case TOK.imaginary80:
1033 case TOK.complex32:
1034 case TOK.complex64:
1035 case TOK.complex80:
1036 case TOK.void_:
1037 p = ident.toChars();
1038 break;
1039 default:
1040 p = toChars(value);
1041 break;
1042 }
1043 return p;
1044 }
1045
toCharsToken1046 static const(char)* toChars(TOK value)
1047 {
1048 return toString(value).ptr;
1049 }
1050
toStringToken1051 extern (D) static string toString(TOK value) pure nothrow @nogc @safe
1052 {
1053 return tochars[value];
1054 }
1055 }
1056
1057 /**
1058 * Write a character, using a readable escape sequence if needed
1059 *
1060 * Useful for printing "" string literals in e.g. error messages, ddoc, or the `.stringof` property
1061 *
1062 * Params:
1063 * buf = buffer to append character in
1064 * c = code point to write
1065 */
1066 nothrow
writeCharLiteral(ref OutBuffer buf,dchar c)1067 void writeCharLiteral(ref OutBuffer buf, dchar c)
1068 {
1069 switch (c)
1070 {
1071 case '\0':
1072 buf.writestring("\\0");
1073 break;
1074 case '\n':
1075 buf.writestring("\\n");
1076 break;
1077 case '\r':
1078 buf.writestring("\\r");
1079 break;
1080 case '\t':
1081 buf.writestring("\\t");
1082 break;
1083 case '\b':
1084 buf.writestring("\\b");
1085 break;
1086 case '\f':
1087 buf.writestring("\\f");
1088 break;
1089 case '"':
1090 case '\\':
1091 buf.writeByte('\\');
1092 goto default;
1093 default:
1094 if (c <= 0xFF)
1095 {
1096 if (isprint(c))
1097 buf.writeByte(c);
1098 else
1099 buf.printf("\\x%02x", c);
1100 }
1101 else if (c <= 0xFFFF)
1102 buf.printf("\\u%04x", c);
1103 else
1104 buf.printf("\\U%08x", c);
1105 break;
1106 }
1107 }
1108
1109 unittest
1110 {
1111 OutBuffer buf;
1112 foreach(dchar d; "a\n\r\t\b\f\0\x11\u7233\U00017233"d)
1113 {
1114 writeCharLiteral(buf, d);
1115 }
1116 assert(buf.extractSlice() == `a\n\r\t\b\f\0\x11\u7233\U00017233`);
1117 }
1118
1119 /**
1120 * Write a single-quoted character literal
1121 *
1122 * Useful for printing '' char literals in e.g. error messages, ddoc, or the `.stringof` property
1123 *
1124 * Params:
1125 * buf = buffer to append character in
1126 * c = code point to write
1127 */
1128 nothrow
writeSingleCharLiteral(ref OutBuffer buf,dchar c)1129 void writeSingleCharLiteral(ref OutBuffer buf, dchar c)
1130 {
1131 buf.writeByte('\'');
1132 if (c == '\'')
1133 buf.writeByte('\\');
1134
1135 if (c == '"')
1136 buf.writeByte('"');
1137 else
1138 writeCharLiteral(buf, c);
1139
1140 buf.writeByte('\'');
1141 }
1142
1143 unittest
1144 {
1145 OutBuffer buf;
1146 writeSingleCharLiteral(buf, '\'');
1147 assert(buf.extractSlice() == `'\''`);
1148 buf.reset();
1149 writeSingleCharLiteral(buf, '"');
1150 assert(buf.extractSlice() == `'"'`);
1151 buf.reset();
1152 writeSingleCharLiteral(buf, '\n');
1153 assert(buf.extractSlice() == `'\n'`);
1154 }
1155