New lexer for mojom written in go.

mojom/mojom_parser/lexer/tokens.go

Index: mojom/mojom_parser/lexer/tokens.go
diff --git a/mojom/mojom_parser/lexer/tokens.go b/mojom/mojom_parser/lexer/tokens.go
new file mode 100644
index 0000000000000000000000000000000000000000..2ed9b5c894cb968698d62f6b42a5bea6e28d9ca0
--- /dev/null
+++ b/mojom/mojom_parser/lexer/tokens.go
@@ -0,0 +1,218 @@
+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+// TokenKinds is a type which describes the kinds of tokens which can be
+// encountered in a mojom file.
+
+package lexer
+
+import (
+	"fmt"
+)
+
+type TokenKind int
+
+// TokenKinds
+const (
+	// An error of an unknown nature has occured.
+	ErrorUnknown TokenKind = iota
+	// A character was found which is not part of a valid token.
+	ErrorIllegalChar
+	// A quoted string was opened but not closed.
+	ErrorUnterminatedStringLiteral
+	// A multiline comment was opened but not closed.
+	ErrorUnterminatedComment
+	// Indicates the end of a stream of tokens.
+	EOF
+
+	// Punctuators and Separators
+	LParen
+	RParen
+	LBracket
+	RBracket
+	LBrace
+	RBrace
+	LAngle
+	RAngle
+	Semi
+	Comma
+	Dot
+	Minus
+	Plus
+	Amp
+	Qstn
+	Equals
+	Response
+
+	// Names
+	Name
+
+	// Keywords
+	Import
+	Module
+	Struct
+	Union
+	Interface
+	Enum
+	Const
+	True
+	False
+	Default
+
+	// Constants
+	IntConstDec
+	IntConstHex
+	FloatConst
+	Ordinal
+	StringLiteral
+)
+
+// This method is used to generate user-facing strings in compilation error
+// messages. For example for LBrace we produce the string "'{'". Notice the
+// single-quotes. This will be used for example in an error message that looks
+// like the following:
+// Unexpected token at line 5, column 6: '###'. Expecting '{'.
+func (tokenKind TokenKind) String() string {
+	switch tokenKind {
+	// Errors
+	case ErrorUnknown:
+		return "unknown token"
+	case ErrorIllegalChar:
+		return "illegal token"
+	case ErrorUnterminatedStringLiteral:
+		return "unterminated string literal"
+	case ErrorUnterminatedComment:
+		return "unterminated comment"
+
+	// End of file
+	case EOF:
+		return "eof"
+
+	// Punctuators and Separators
+	case LParen:
+		return "'('"
+	case RParen:
+		return "')'"
+	case LBracket:
+		return "'['"
+	case RBracket:
+		return "']'"
+	case LBrace:
+		return "'{'"
+	case RBrace:
+		return "'}'"
+	case LAngle:
+		return "'<'"
+	case RAngle:
+		return "'>'"
+	case Semi:
+		return "';'"
+	case Comma:
+		return "','"
+	case Dot:
+		return "'.'"
+	case Minus:
+		return "'-'"
+	case Plus:
+		return "'+'"
+	case Amp:
+		return "'&'"
+	case Qstn:
+		return "'?'"
+	case Equals:
+		return "'='"
+	case Response:
+		return "'=>'"
+
+	// Names
+	case Name:
+		return "a name"
+
+	// Keywords
+	case Import:
+		return "'import'"
+	case Module:
+		return "'module'"
+	case Struct:
+		return "'struct'"
+	case Union:
+		return "'union'"
+	case Interface:
+		return "'interface'"
+	case Enum:
+		return "'enum'"
+	case Const:
+		return "'const'"
+	case True:
+		return "'true'"
+	case False:
+		return "'false'"
+	case Default:
+		return "'default'"
+
+	// Constants
+	case IntConstDec:
+		return "decimal integer literal"
+	case IntConstHex:
+		return "hex integer literal"
+	case FloatConst:
+		return "float literal"
+	case Ordinal:
+		return "an ordinal"
+	case StringLiteral:
+		return "a string literal"
+
+	default:
+		// Note(rudominer) It is important to use %d below so as to avoid
+		// re-invoking this method and causing an infinite recursion.
+		return fmt.Sprintf("%d", tokenKind)
+	}
+}
+
+type Token struct {
+	Kind TokenKind
+	Text string
+	// CharPos is the number of runes preceeding the token.
+	CharPos int
+	// LineNo is the line on which the token is found. (First line is 0.)
+	LineNo int
+	// LinePost is the number of runes preceeding the token on its line.
+	LinePos int
+}
+
+// ShortLocationString is used to generate user-facing strings in compilation
+// error messages. This will be used for example in an error message that looks
+// like the following:
+// Unexpected token at line 5, column 6: '###'. Expecting '{'.
+func (t Token) ShortLocationString() string {
+	return fmt.Sprintf("%d,%d", t.LineNo+1, t.LinePos+1)
+}
+
+func (t Token) LongLocationString() string {
+	return fmt.Sprintf("line %d, column %d", t.LineNo+1, t.LinePos+1)
+}
+
+// EOF returns true if the token on which it is called represents the end of the
+// token string.
+func (t Token) EOF() bool {
+	return t.Kind == EOF
+}
+
+// String is used to generate user-facing strings in compilation error
+// messages. For many token kinds the TokenKind.String() method will produce
+// good results for representing the token. But for other TokenKinds we will
+// want to include some information besides a representation of the kind.
+// For example for an ErrorUnknown kind we wnat to show the text.
+// This will be used for example in an error message that looks
+// like the following:
+// Unexpected token at line 5, column 6: '###'. Expecting '{'.
+func (token Token) String() string {
+	switch token.Kind {
+	case ErrorUnknown, Name, StringLiteral, IntConstDec, IntConstHex, FloatConst, Ordinal:
+		return fmt.Sprintf("'%s'", token.Text)
+
+	default:
+		return token.Kind.String()
+	}
+}