New lexer for mojom written in go.

mojom/mojom_parser/lexer/tokens.go

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+// TokenKinds is a type which describes the kinds of tokens which can be
+// encountered in a mojom file.
+
+package lexer
+
+import (
+        "fmt"
+)
+
+type TokenKind int
+
+// TokenKinds
+const (
+        // An error of an unknown nature has occured.
+        ErrorUnknown TokenKind = iota
+        // A character was found which is not part of a valid token.
+        ErrorIllegalChar
+        // A quoted string was opened but not closed.
+        ErrorUnterminatedStringLiteral
+        // A multiline comment was opened but not closed.
+        ErrorUnterminatedComment
+        // Indicates the end of a stream of tokens.
+        EOF
+
+        // Punctuators and Separators
+        LParen
+        RParen
+        LBracket
+        RBracket
+        LBrace
+        RBrace
+        LAngle
+        RAngle
+        Semi
+        Comma
+        Dot
+        Minus
+        Plus
+        Amp
+        Qstn
+        Equals
+        Response
+
+        // Names
+        Name
+
+        // Keywords
+        Import
+        Module
+        Struct
+        Union
+        Interface
+        Enum
+        Const
+        True
+        False
+        Default
+
+        // Constants
+        IntConstDec
+        IntConstHex
+        FloatConst
+        Ordinal
+        StringLiteral
+)
+
+// This method is used to generate user-facing strings in compilation error
+// messages. For example for LBrace we produce the string "'{'". Notice the
+// single-quotes. This will be used for example in an error message that looks
+// like the following:
+// Unexpected token at line 5, column 6: '###'. Expecting '{'.
+func (tokenKind TokenKind) String() string {
+        switch tokenKind {
+        // Errors
+        case ErrorUnknown:
+                return "unknown token"
+        case ErrorIllegalChar:
+                return "illegal token"
+        case ErrorUnterminatedStringLiteral:
+                return "unterminated string literal"
+        case ErrorUnterminatedComment:
+                return "unterminated comment"
+
+        // End of file
+        case EOF:
+                return "eof"
+
+        // Punctuators and Separators
+        case LParen:
+                return "'('"
+        case RParen:
+                return "')'"
+        case LBracket:
+                return "'['"
+        case RBracket:
+                return "']'"
+        case LBrace:
+                return "'{'"
+        case RBrace:
+                return "'}'"
+        case LAngle:
+                return "'<'"
+        case RAngle:
+                return "'>'"
+        case Semi:
+                return "';'"
+        case Comma:
+                return "','"
+        case Dot:
+                return "'.'"
+        case Minus:
+                return "'-'"
+        case Plus:
+                return "'+'"
+        case Amp:
+                return "'&'"
+        case Qstn:
+                return "'?'"
+        case Equals:
+                return "'='"
+        case Response:
+                return "'=>'"
+
+        // Names
+        case Name:
+                return "a name"
+
+        // Keywords
+        case Import:
+                return "'import'"
+        case Module:
+                return "'module'"
+        case Struct:
+                return "'struct'"
+        case Union:
+                return "'union'"
+        case Interface:
+                return "'interface'"
+        case Enum:
+                return "'enum'"
+        case Const:
+                return "'const'"
+        case True:
+                return "'true'"
+        case False:
+                return "'false'"
+        case Default:
+                return "'default'"
+
+        // Constants
+        case IntConstDec:
+                return "decimal integer literal"
+        case IntConstHex:
+                return "hex integer literal"
+        case FloatConst:
+                return "float literal"
+        case Ordinal:
+                return "an ordinal"
+        case StringLiteral:
+                return "a string literal"
+
+        default:
+                // Note(rudominer) It is important to use %d below so as to avoid
+                // re-invoking this method and causing an infinite recursion.
+                return fmt.Sprintf("%d", tokenKind)
+        }
+}
+
+type Token struct {
+        Kind TokenKind
+        Text string
+        // CharPos is the number of runes preceeding the token.
+        CharPos int
+        // LineNo is the line on which the token is found. (First line is 0.)
+        LineNo int
+        // LinePost is the number of runes preceeding the token on its line.
+        LinePos int
+}
+
+// ShortLocationString is used to generate user-facing strings in compilation
+// error messages. This will be used for example in an error message that looks
+// like the following:
+// Unexpected token at line 5, column 6: '###'. Expecting '{'.
+func (t Token) ShortLocationString() string {
+        return fmt.Sprintf("%d,%d", t.LineNo+1, t.LinePos+1)
+}
+
+func (t Token) LongLocationString() string {
+        return fmt.Sprintf("line %d, column %d", t.LineNo+1, t.LinePos+1)
+}
+
+// EOF returns true if the token on which it is called represents the end of the
+// token string.
+func (t Token) EOF() bool {
+        return t.Kind == EOF
+}
+
+// String is used to generate user-facing strings in compilation error
+// messages. For many token kinds the TokenKind.String() method will produce
+// good results for representing the token. But for other TokenKinds we will
+// want to include some information besides a representation of the kind.
+// For example for an ErrorUnknown kind we wnat to show the text.
+// This will be used for example in an error message that looks
+// like the following:
+// Unexpected token at line 5, column 6: '###'. Expecting '{'.
+func (token Token) String() string {
+        switch token.Kind {
+        case ErrorUnknown, Name, StringLiteral, IntConstDec, IntConstHex, FloatConst, Ordinal:
+                return fmt.Sprintf("'%s'", token.Text)
+
+        default:
+                return token.Kind.String()
+        }
+}