xref: /netbsd-src/external/gpl3/gcc.old/dist/gcc/d/dmd/tokens.c (revision 627f7eb200a4419d89b531d55fccd2ee3ffdcde0)

/* Compiler implementation of the D programming language
 * Copyright (C) 1999-2019 by The D Language Foundation, All Rights Reserved
 * written by Walter Bright
 * http://www.digitalmars.com
 * Distributed under the Boost Software License, Version 1.0.
 * http://www.boost.org/LICENSE_1_0.txt
 * https://github.com/D-Programming-Language/dmd/blob/master/src/lexer.c
 */

#include "root/dsystem.h"

#include "tokens.h"
#include "root/rmem.h"
#include "root/outbuffer.h"
#include "id.h"
#include "identifier.h"
#include "utf.h"

/************************* Token **********************************************/

Token *Token::freelist = NULL;

const char *Token::tochars[TOKMAX];

Token *Token::alloc()
{
    if (Token::freelist)
    {
        Token *t = freelist;
        freelist = t->next;
        t->next = NULL;
        return t;
    }

    return new Token();
}

void Token::free()
{
    next = freelist;
    freelist = this;
}

const char *Token::toChars() const
{
    static char buffer[3 + 3 * sizeof(floatvalue) + 1];

    const char *p = &buffer[0];
    switch (value)
    {
        case TOKint32v:
            sprintf(&buffer[0],"%d",(d_int32)int64value);
            break;

        case TOKuns32v:
        case TOKcharv:
        case TOKwcharv:
        case TOKdcharv:
            sprintf(&buffer[0],"%uU",(d_uns32)uns64value);
            break;

        case TOKint64v:
            sprintf(&buffer[0],"%lldL",(longlong)int64value);
            break;

        case TOKuns64v:
            sprintf(&buffer[0],"%lluUL",(ulonglong)uns64value);
            break;

        case TOKfloat32v:
            CTFloat::sprint(&buffer[0], 'g', floatvalue);
            strcat(&buffer[0], "f");
            break;

        case TOKfloat64v:
            CTFloat::sprint(&buffer[0], 'g', floatvalue);
            break;

        case TOKfloat80v:
            CTFloat::sprint(&buffer[0], 'g', floatvalue);
            strcat(&buffer[0], "L");
            break;

        case TOKimaginary32v:
            CTFloat::sprint(&buffer[0], 'g', floatvalue);
            strcat(&buffer[0], "fi");
            break;

        case TOKimaginary64v:
            CTFloat::sprint(&buffer[0], 'g', floatvalue);
            strcat(&buffer[0], "i");
            break;

        case TOKimaginary80v:
            CTFloat::sprint(&buffer[0], 'g', floatvalue);
            strcat(&buffer[0], "Li");
            break;

        case TOKstring:
        {
            OutBuffer buf;
            buf.writeByte('"');
            for (size_t i = 0; i < len; )
            {
                unsigned c;
                utf_decodeChar((utf8_t *)ustring, len, &i, &c);
                switch (c)
                {
                    case 0:
                        break;

                    case '"':
                    case '\\':
                        buf.writeByte('\\');
                        /* fall through */
                    default:
                        if (c <= 0x7F)
                        {
                            if (isprint(c))
                                buf.writeByte(c);
                            else
                                buf.printf("\\x%02x", c);
                        }
                        else if (c <= 0xFFFF)
                            buf.printf("\\u%04x", c);
                        else
                            buf.printf("\\U%08x", c);
                        continue;
                }
                break;
            }
            buf.writeByte('"');
            if (postfix)
                buf.writeByte(postfix);
            p = buf.extractString();
        }
            break;

        case TOKxstring:
        {
            OutBuffer buf;
            buf.writeByte('x');
            buf.writeByte('"');
            for (size_t i = 0; i < len; i++)
            {
                if (i)
                    buf.writeByte(' ');
                buf.printf("%02x", ustring[i]);
            }
            buf.writeByte('"');
            if (postfix)
                buf.writeByte(postfix);
            buf.writeByte(0);
            p = (char *)buf.extractData();
            break;
        }

        case TOKidentifier:
        case TOKenum:
        case TOKstruct:
        case TOKimport:
        case TOKwchar: case TOKdchar:
        case TOKbool: case TOKchar:
        case TOKint8: case TOKuns8:
        case TOKint16: case TOKuns16:
        case TOKint32: case TOKuns32:
        case TOKint64: case TOKuns64:
        case TOKint128: case TOKuns128:
        case TOKfloat32: case TOKfloat64: case TOKfloat80:
        case TOKimaginary32: case TOKimaginary64: case TOKimaginary80:
        case TOKcomplex32: case TOKcomplex64: case TOKcomplex80:
        case TOKvoid:
            p = ident->toChars();
            break;

        default:
            p = toChars(value);
            break;
    }
    return p;
}

const char *Token::toChars(TOK value)
{
    static char buffer[3 + 3 * sizeof(value) + 1];

    const char *p = tochars[value];
    if (!p)
    {
        sprintf(&buffer[0],"TOK%d",value);
        p = &buffer[0];
    }
    return p;
}

/****************************************
 */

struct Keyword
{
    const char *name;
    TOK value;
};

static size_t nkeywords;
static Keyword keywords[] =
{
    {   "this",         TOKthis         },
    {   "super",        TOKsuper        },
    {   "assert",       TOKassert       },
    {   "null",         TOKnull         },
    {   "true",         TOKtrue         },
    {   "false",        TOKfalse        },
    {   "cast",         TOKcast         },
    {   "new",          TOKnew          },
    {   "delete",       TOKdelete       },
    {   "throw",        TOKthrow        },
    {   "module",       TOKmodule       },
    {   "pragma",       TOKpragma       },
    {   "typeof",       TOKtypeof       },
    {   "typeid",       TOKtypeid       },

    {   "template",     TOKtemplate     },

    {   "void",         TOKvoid         },
    {   "byte",         TOKint8         },
    {   "ubyte",        TOKuns8         },
    {   "short",        TOKint16        },
    {   "ushort",       TOKuns16        },
    {   "int",          TOKint32        },
    {   "uint",         TOKuns32        },
    {   "long",         TOKint64        },
    {   "ulong",        TOKuns64        },
    {   "cent",         TOKint128,      },
    {   "ucent",        TOKuns128,      },
    {   "float",        TOKfloat32      },
    {   "double",       TOKfloat64      },
    {   "real",         TOKfloat80      },

    {   "bool",         TOKbool         },
    {   "char",         TOKchar         },
    {   "wchar",        TOKwchar        },
    {   "dchar",        TOKdchar        },

    {   "ifloat",       TOKimaginary32  },
    {   "idouble",      TOKimaginary64  },
    {   "ireal",        TOKimaginary80  },

    {   "cfloat",       TOKcomplex32    },
    {   "cdouble",      TOKcomplex64    },
    {   "creal",        TOKcomplex80    },

    {   "delegate",     TOKdelegate     },
    {   "function",     TOKfunction     },

    {   "is",           TOKis           },
    {   "if",           TOKif           },
    {   "else",         TOKelse         },
    {   "while",        TOKwhile        },
    {   "for",          TOKfor          },
    {   "do",           TOKdo           },
    {   "switch",       TOKswitch       },
    {   "case",         TOKcase         },
    {   "default",      TOKdefault      },
    {   "break",        TOKbreak        },
    {   "continue",     TOKcontinue     },
    {   "synchronized", TOKsynchronized },
    {   "return",       TOKreturn       },
    {   "goto",         TOKgoto         },
    {   "try",          TOKtry          },
    {   "catch",        TOKcatch        },
    {   "finally",      TOKfinally      },
    {   "with",         TOKwith         },
    {   "asm",          TOKasm          },
    {   "foreach",      TOKforeach      },
    {   "foreach_reverse",      TOKforeach_reverse      },
    {   "scope",        TOKscope        },

    {   "struct",       TOKstruct       },
    {   "class",        TOKclass        },
    {   "interface",    TOKinterface    },
    {   "union",        TOKunion        },
    {   "enum",         TOKenum         },
    {   "import",       TOKimport       },
    {   "mixin",        TOKmixin        },
    {   "static",       TOKstatic       },
    {   "final",        TOKfinal        },
    {   "const",        TOKconst        },
    {   "alias",        TOKalias        },
    {   "override",     TOKoverride     },
    {   "abstract",     TOKabstract     },
    {   "debug",        TOKdebug        },
    {   "deprecated",   TOKdeprecated   },
    {   "in",           TOKin           },
    {   "out",          TOKout          },
    {   "inout",        TOKinout        },
    {   "lazy",         TOKlazy         },
    {   "auto",         TOKauto         },

    {   "align",        TOKalign        },
    {   "extern",       TOKextern       },
    {   "private",      TOKprivate      },
    {   "package",      TOKpackage      },
    {   "protected",    TOKprotected    },
    {   "public",       TOKpublic       },
    {   "export",       TOKexport       },

    {   "invariant",    TOKinvariant    },
    {   "unittest",     TOKunittest     },
    {   "version",      TOKversion      },

    {   "__argTypes",   TOKargTypes     },
    {   "__parameters", TOKparameters   },
    {   "ref",          TOKref          },
    {   "macro",        TOKmacro        },

    {   "pure",         TOKpure         },
    {   "nothrow",      TOKnothrow      },
    {   "__gshared",    TOKgshared      },
    {   "__traits",     TOKtraits       },
    {   "__vector",     TOKvector       },
    {   "__overloadset", TOKoverloadset },
    {   "__FILE__",     TOKfile         },
    {   "__FILE_FULL_PATH__", TOKfilefullpath  },
    {   "__LINE__",     TOKline         },
    {   "__MODULE__",   TOKmodulestring },
    {   "__FUNCTION__", TOKfuncstring   },
    {   "__PRETTY_FUNCTION__", TOKprettyfunc   },
    {   "shared",       TOKshared       },
    {   "immutable",    TOKimmutable    },
    {   NULL,           TOKreserved     }
};

int Token::isKeyword()
{
    for (size_t u = 0; u < nkeywords; u++)
    {
        if (keywords[u].value == value)
            return 1;
    }
    return 0;
}

struct TokenInitializer
{
    TokenInitializer();
};

static TokenInitializer tokeninitializer;

TokenInitializer::TokenInitializer()
{
    Identifier::initTable();
    for (nkeywords = 0; keywords[nkeywords].name; nkeywords++)
    {
        //printf("keyword[%d] = '%s'\n",u, keywords[u].name);
        const char *s = keywords[nkeywords].name;
        size_t len = strlen(s);
        TOK v = keywords[nkeywords].value;
        Identifier::idPool(s, len, v);

        //printf("tochars[%d] = '%s'\n",v, s);
        Token::tochars[v] = s;
    }

    Token::tochars[TOKeof]              = "EOF";
    Token::tochars[TOKlcurly]           = "{";
    Token::tochars[TOKrcurly]           = "}";
    Token::tochars[TOKlparen]           = "(";
    Token::tochars[TOKrparen]           = ")";
    Token::tochars[TOKlbracket]         = "[";
    Token::tochars[TOKrbracket]         = "]";
    Token::tochars[TOKsemicolon]        = ";";
    Token::tochars[TOKcolon]            = ":";
    Token::tochars[TOKcomma]            = ",";
    Token::tochars[TOKdot]              = ".";
    Token::tochars[TOKxor]              = "^";
    Token::tochars[TOKxorass]           = "^=";
    Token::tochars[TOKassign]           = "=";
    Token::tochars[TOKconstruct]        = "=";
    Token::tochars[TOKblit]             = "=";
    Token::tochars[TOKlt]               = "<";
    Token::tochars[TOKgt]               = ">";
    Token::tochars[TOKle]               = "<=";
    Token::tochars[TOKge]               = ">=";
    Token::tochars[TOKequal]            = "==";
    Token::tochars[TOKnotequal]         = "!=";
    Token::tochars[TOKnotidentity]      = "!is";

    Token::tochars[TOKunord]            = "!<>=";
    Token::tochars[TOKue]               = "!<>";
    Token::tochars[TOKlg]               = "<>";
    Token::tochars[TOKleg]              = "<>=";
    Token::tochars[TOKule]              = "!>";
    Token::tochars[TOKul]               = "!>=";
    Token::tochars[TOKuge]              = "!<";
    Token::tochars[TOKug]               = "!<=";

    Token::tochars[TOKnot]              = "!";
    Token::tochars[TOKshl]              = "<<";
    Token::tochars[TOKshr]              = ">>";
    Token::tochars[TOKushr]             = ">>>";
    Token::tochars[TOKadd]              = "+";
    Token::tochars[TOKmin]              = "-";
    Token::tochars[TOKmul]              = "*";
    Token::tochars[TOKdiv]              = "/";
    Token::tochars[TOKmod]              = "%";
    Token::tochars[TOKslice]            = "..";
    Token::tochars[TOKdotdotdot]        = "...";
    Token::tochars[TOKand]              = "&";
    Token::tochars[TOKandand]           = "&&";
    Token::tochars[TOKor]               = "|";
    Token::tochars[TOKoror]             = "||";
    Token::tochars[TOKarray]            = "[]";
    Token::tochars[TOKindex]            = "[i]";
    Token::tochars[TOKaddress]          = "&";
    Token::tochars[TOKstar]             = "*";
    Token::tochars[TOKtilde]            = "~";
    Token::tochars[TOKdollar]           = "$";
    Token::tochars[TOKcast]             = "cast";
    Token::tochars[TOKplusplus]         = "++";
    Token::tochars[TOKminusminus]       = "--";
    Token::tochars[TOKpreplusplus]      = "++";
    Token::tochars[TOKpreminusminus]    = "--";
    Token::tochars[TOKtype]             = "type";
    Token::tochars[TOKquestion]         = "?";
    Token::tochars[TOKneg]              = "-";
    Token::tochars[TOKuadd]             = "+";
    Token::tochars[TOKvar]              = "var";
    Token::tochars[TOKaddass]           = "+=";
    Token::tochars[TOKminass]           = "-=";
    Token::tochars[TOKmulass]           = "*=";
    Token::tochars[TOKdivass]           = "/=";
    Token::tochars[TOKmodass]           = "%=";
    Token::tochars[TOKshlass]           = "<<=";
    Token::tochars[TOKshrass]           = ">>=";
    Token::tochars[TOKushrass]          = ">>>=";
    Token::tochars[TOKandass]           = "&=";
    Token::tochars[TOKorass]            = "|=";
    Token::tochars[TOKcatass]           = "~=";
    Token::tochars[TOKcat]              = "~";
    Token::tochars[TOKcall]             = "call";
    Token::tochars[TOKidentity]         = "is";
    Token::tochars[TOKnotidentity]      = "!is";

    Token::tochars[TOKorass]            = "|=";
    Token::tochars[TOKidentifier]       = "identifier";
    Token::tochars[TOKat]               = "@";
    Token::tochars[TOKpow]              = "^^";
    Token::tochars[TOKpowass]           = "^^=";
    Token::tochars[TOKgoesto]           = "=>";
    Token::tochars[TOKpound]            = "#";

     // For debugging
    Token::tochars[TOKerror]            = "error";
    Token::tochars[TOKdotid]            = "dotid";
    Token::tochars[TOKdottd]            = "dottd";
    Token::tochars[TOKdotti]            = "dotti";
    Token::tochars[TOKdotvar]           = "dotvar";
    Token::tochars[TOKdottype]          = "dottype";
    Token::tochars[TOKsymoff]           = "symoff";
    Token::tochars[TOKarraylength]      = "arraylength";
    Token::tochars[TOKarrayliteral]     = "arrayliteral";
    Token::tochars[TOKassocarrayliteral] = "assocarrayliteral";
    Token::tochars[TOKstructliteral]    = "structliteral";
    Token::tochars[TOKstring]           = "string";
    Token::tochars[TOKdsymbol]          = "symbol";
    Token::tochars[TOKtuple]            = "tuple";
    Token::tochars[TOKdeclaration]      = "declaration";
    Token::tochars[TOKon_scope_exit]    = "scope(exit)";
    Token::tochars[TOKon_scope_success] = "scope(success)";
    Token::tochars[TOKon_scope_failure] = "scope(failure)";
    Token::tochars[TOKdelegateptr]      = "delegateptr";
    Token::tochars[TOKvectorarray]      = "vectorarray";
}