Lines Matching +full:- +full:- +full:token
1 //===- Lexer.cpp - MLIR Lexer Implementation ------------------------------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
11 //===----------------------------------------------------------------------===//
14 #include "Token.h"
30 // Returns true if 'c' is an allowable punctuation character: [$._-]
33 return c == '$' || c == '.' || c == '_' || c == '-'; in isPunct()
40 curBuffer = sourceMgr.getMemoryBuffer(bufferID)->getBuffer(); in Lexer()
45 codeCompleteLoc = codeCompleteContext->getCodeCompleteLoc().getPointer(); in Lexer()
59 (loc.getPointer() - bufferInfo.getPointerForLineNumber(lineNo)) + 1; in getEncodedSourceLocation()
62 return FileLineColLoc::get(context, buffer->getBufferIdentifier(), lineNo, in getEncodedSourceLocation()
66 /// emitError - Emit an error message and return an Token::error token.
67 Token Lexer::emitError(const char *loc, const Twine &message) { in emitError()
70 return formToken(Token::error, loc); in emitError()
73 Token Lexer::lexToken() { in lexToken()
77 // Check to see if the current token is at the code completion location. in lexToken()
79 return formToken(Token::code_complete, tokStart); in lexToken()
81 // Lex the next token. in lexToken()
85 if (isalpha(curPtr[-1])) in lexToken()
105 if (curPtr - 1 == curBuffer.end()) in lexToken()
106 return formToken(Token::eof, tokStart); in lexToken()
110 return formToken(Token::colon, tokStart); in lexToken()
112 return formToken(Token::comma, tokStart); in lexToken()
116 return formToken(Token::l_paren, tokStart); in lexToken()
118 return formToken(Token::r_paren, tokStart); in lexToken()
120 if (*curPtr == '-' && *(curPtr + 1) == '#') { in lexToken()
122 return formToken(Token::file_metadata_begin, tokStart); in lexToken()
124 return formToken(Token::l_brace, tokStart); in lexToken()
126 return formToken(Token::r_brace, tokStart); in lexToken()
128 return formToken(Token::l_square, tokStart); in lexToken()
130 return formToken(Token::r_square, tokStart); in lexToken()
132 return formToken(Token::less, tokStart); in lexToken()
134 return formToken(Token::greater, tokStart); in lexToken()
136 return formToken(Token::equal, tokStart); in lexToken()
139 return formToken(Token::plus, tokStart); in lexToken()
141 return formToken(Token::star, tokStart); in lexToken()
142 case '-': in lexToken()
145 return formToken(Token::arrow, tokStart); in lexToken()
147 return formToken(Token::minus, tokStart); in lexToken()
150 return formToken(Token::question, tokStart); in lexToken()
153 return formToken(Token::vertical_bar, tokStart); in lexToken()
166 if (*curPtr == '-' && *(curPtr + 1) == '}') { in lexToken()
168 return formToken(Token::file_metadata_end, tokStart); in lexToken()
195 /// symbol-ref-id ::= `@` (bare-id | string-literal)
197 Token Lexer::lexAtIdentifier(const char *tokStart) { in lexAtIdentifier()
202 Token stringIdentifier = lexString(curPtr); in lexAtIdentifier()
203 if (stringIdentifier.is(Token::error)) in lexAtIdentifier()
205 return formToken(Token::at_identifier, tokStart); in lexAtIdentifier()
210 return emitError(curPtr - 1, in lexAtIdentifier()
216 return formToken(Token::at_identifier, tokStart); in lexAtIdentifier()
221 /// bare-id ::= (letter|[_]) (letter|digit|[_$.])*
222 /// integer-type ::= `[su]?i[1-9][0-9]*`
224 Token Lexer::lexBareIdentifierOrKeyword(const char *tokStart) { in lexBareIdentifierOrKeyword()
225 // Match the rest of the identifier regex: [0-9a-zA-Z_.$]* in lexBareIdentifierOrKeyword()
231 StringRef spelling(tokStart, curPtr - tokStart); in lexBareIdentifierOrKeyword()
243 return Token(Token::inttype, spelling); in lexBareIdentifierOrKeyword()
245 Token::Kind kind = StringSwitch<Token::Kind>(spelling) in lexBareIdentifierOrKeyword()
246 #define TOK_KEYWORD(SPELLING) .Case(#SPELLING, Token::kw_##SPELLING) in lexBareIdentifierOrKeyword()
248 .Default(Token::bare_identifier); in lexBareIdentifierOrKeyword()
250 return Token(kind, spelling); in lexBareIdentifierOrKeyword()
270 if (curPtr - 1 == curBuffer.end()) { in skipComment()
271 --curPtr; in skipComment()
286 Token Lexer::lexEllipsis(const char *tokStart) { in lexEllipsis()
287 assert(curPtr[-1] == '.'); in lexEllipsis()
293 return formToken(Token::ellipsis, tokStart); in lexEllipsis()
298 /// integer-literal ::= digit+ | `0x` hex_digit+
299 /// float-literal ::= [-+]?[0-9]+[.][0-9]*([eE][-+]?[0-9]+)?
301 Token Lexer::lexNumber(const char *tokStart) { in lexNumber()
302 assert(isdigit(curPtr[-1])); in lexNumber()
305 if (curPtr[-1] == '0' && *curPtr == 'x') { in lexNumber()
309 return formToken(Token::integer, tokStart); in lexNumber()
315 return formToken(Token::integer, tokStart); in lexNumber()
323 return formToken(Token::integer, tokStart); in lexNumber()
326 // Skip over [0-9]*([eE][-+]?[0-9]+)? in lexNumber()
332 ((curPtr[1] == '-' || curPtr[1] == '+') && in lexNumber()
339 return formToken(Token::floatliteral, tokStart); in lexNumber()
342 /// Lex an identifier that starts with a prefix followed by suffix-id.
344 /// attribute-id ::= `#` suffix-id
345 /// ssa-id ::= '%' suffix-id
346 /// block-id ::= '^' suffix-id
347 /// type-id ::= '!' suffix-id
348 /// suffix-id ::= digit+ | (letter|id-punct) (letter|id-punct|digit)*
349 /// id-punct ::= `$` | `.` | `_` | `-`
351 Token Lexer::lexPrefixedIdentifier(const char *tokStart) { in lexPrefixedIdentifier()
352 Token::Kind kind; in lexPrefixedIdentifier()
356 kind = Token::hash_identifier; in lexPrefixedIdentifier()
360 kind = Token::percent_identifier; in lexPrefixedIdentifier()
364 kind = Token::caret_identifier; in lexPrefixedIdentifier()
368 kind = Token::exclamation_identifier; in lexPrefixedIdentifier()
375 // Parse suffix-id. in lexPrefixedIdentifier()
377 // If suffix-id starts with a digit, the rest must be digits. in lexPrefixedIdentifier()
385 return formToken(Token::code_complete, tokStart); in lexPrefixedIdentifier()
387 return emitError(curPtr - 1, errorKind); in lexPrefixedIdentifier()
393 return Token(Token::code_complete, in lexPrefixedIdentifier()
394 StringRef(tokStart, codeCompleteLoc - tokStart)); in lexPrefixedIdentifier()
402 /// string-literal ::= '"' [^"\n\f\v\r]* '"'
405 Token Lexer::lexString(const char *tokStart) { in lexString()
406 assert(curPtr[-1] == '"'); in lexString()
411 // lexed string within the token. This allows for the parser to use the in lexString()
414 return formToken(Token::code_complete, tokStart); in lexString()
418 return formToken(Token::string, tokStart); in lexString()
422 if (curPtr - 1 != curBuffer.end()) in lexString()
428 return emitError(curPtr - 1, "expected '\"' in string literal"); in lexString()
437 return emitError(curPtr - 1, "unknown escape in string literal"); in lexString()