New Mojom parser: Fix up error messages for lexing errors.

mojom/mojom_parser/lexer/tokens.go

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 //
 // TokenKinds is a type which describes the kinds of tokens which can be
 // encountered in a mojom file.
 
 package lexer
 
 import (
         "bytes"
         "fmt"
         "strings"
         "unicode/utf8"
 )
 
 type TokenKind int
 
 // TokenKinds
 const (
-        // An error of an unknown nature has occured.
-        ErrorUnknown TokenKind = iota
         // A character was found which is not part of a valid token.
-        ErrorIllegalChar
+        ErrorIllegalChar TokenKind = iota
         // A quoted string was opened but not closed.
         ErrorUnterminatedStringLiteral
         // A multiline comment was opened but not closed.
         ErrorUnterminatedComment
         // Indicates the end of a stream of tokens.
         EOF
 
         // Punctuators and Separators
         LParen
         RParen
@@ skipping 37 matching lines @@
 )
 
 // This method is used to generate user-facing strings in compilation error
 // messages. For example for LBrace we produce the string "'{'". Notice the
 // single-quotes. This will be used for example in an error message that looks
 // like the following:
 // Unexpected token at line 5, column 6: '###'. Expecting '{'.
 func (tokenKind TokenKind) String() string {
         switch tokenKind {
         // Errors
-        case ErrorUnknown:
-                return "unknown token"
         case ErrorIllegalChar:
                 return "illegal token"
         case ErrorUnterminatedStringLiteral:
                 return "unterminated string literal"
         case ErrorUnterminatedComment:
                 return "unterminated comment"
 
         // End of file
         case EOF:
                 return "eof"
@@ skipping 109 matching lines @@
 // EOF returns true if the token on which it is called represents the end of the
 // token string.
 func (t Token) EOF() bool {
         return t.Kind == EOF
 }
 
 // String is used to generate user-facing strings in compilation error
 // messages. For many token kinds the TokenKind.String() method will produce
 // good results for representing the token. But for other TokenKinds we will
 // want to include some information besides a representation of the kind.
-// For example for an ErrorUnknown kind we wnat to show the text.
+// For example for an ErrorIllegalChar kind we wnat to show the text.
 // This will be used for example in an error message that looks
 // like the following:
 // Unexpected token at line 5, column 6: '###'. Expecting '{'.
 func (token Token) String() string {
         switch token.Kind {
-        case ErrorUnknown, Name, StringLiteral, IntConstDec, IntConstHex, FloatConst, Ordinal:
-                return fmt.Sprintf("'%s'", token.Text)
+        case StringLiteral:
+                return token.Text
+        case Name, IntConstDec, IntConstHex, FloatConst, Ordinal, ErrorIllegalChar:
+                return fmt.Sprintf("%q", token.Text)
 
         default:
                 return token.Kind.String()
         }
 }
 
 // Snippet is used to generate a user-facing string in compilation error
 // messages. It displays the token's text as well as the surrounding line for
 // context. It also includes a line with some carets to highlight the token.
 // source is the source code where token was found.
@@ skipping 39 matching lines @@
                 if rune == '\t' {
                         snippetBuffer.WriteRune('\t')
                 } else {
                         snippetBuffer.WriteRune(' ')
                 }
         }
 
         // We don't want too big of a caret line as that may be distracting. So we
         // limit it to 20 runes at most.
         tokenSize := utf8.RuneCountInString(token.Text)
-        if tokenSize > 20 {
+        if token.Kind == ErrorUnterminatedComment {
+                // In the case of an unterminated comment, the token text will contain
+                // the entire rest of the source after the opening comment. We really
+                // only want to highlight the "/*" characters marking the beginning of
+                // the comment.
+                tokenSize = 2
+        } else if tokenSize > 20 {
                 tokenSize = 20
         }
         if color {
                 // Set the caret characters to green.
                 snippetBuffer.WriteString("\x1b[32;1m")
         }
         snippetBuffer.WriteString(strings.Repeat("^", tokenSize))
         if color {
                 // Reset all printing attributes.
                 snippetBuffer.WriteString("\x1b[0m")
         }
         snippet = snippetBuffer.String()
         return
 }