improve fasta unterminated string recovery

pkg/front_end/lib/src/fasta/scanner/abstract_scanner.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library fasta.scanner.abstract_scanner;
 
 import 'dart:collection' show ListMixin;
 
 import 'dart:typed_data' show Uint16List, Uint32List;
 
@@ skipping 144 matching lines @@
    * [:scanOffset:] plus the [:extraOffset:]. For example, if the current
    * scanOffset is 10, then [:appendSubstringToken(5, -1):] will append the
    * substring string [5,9).
    *
    * Note that [extraOffset] can only be used if the covered character(s) are
    * known to be ASCII.
    */
   void appendSubstringToken(TokenType type, int start, bool asciiOnly,
       [int extraOffset]);
 
+  /**
+   * Appends a substring from the scan offset [start] to the current
+   * [scanOffset] plus [closingQuotes]. The closing quote(s) will be added
+   * to the unterminated string literal's lexeme but the returned
+   * token's length will *not* include those closing quotes
+   * so as to be true to the original source.
+   */
+  void appendSyntheticSubstringToken(
+      TokenType type, int start, bool asciiOnly, String closingQuotes);
+
   /** Documentation in subclass [ArrayBasedScanner]. */
   void appendPrecedenceToken(TokenType type);
 
   /** Documentation in subclass [ArrayBasedScanner]. */
   int select(int choice, TokenType yes, TokenType no);
 
   /** Documentation in subclass [ArrayBasedScanner]. */
   void appendKeywordToken(Keyword keyword);
 
   /** Documentation in subclass [ArrayBasedScanner]. */
@@ skipping 842 matching lines @@
         next = tokenizeStringInterpolation(start, asciiOnly);
         start = scanOffset;
         asciiOnly = true;
         continue;
       }
       if (next <= $CR &&
           (identical(next, $LF) ||
               identical(next, $CR) ||
               identical(next, $EOF))) {
         if (!asciiOnly) handleUnicode(start);
-        return unterminatedString(quoteChar);
+        return unterminatedString(quoteChar, start, asciiOnly);
       }
       if (next > 127) asciiOnly = false;
       next = advance();
     }
     if (!asciiOnly) handleUnicode(start);
     // Advance past the quote character.
     next = advance();
     appendSubstringToken(TokenType.STRING, start, asciiOnly);
     return next;
   }
@@ skipping 44 matching lines @@
   int tokenizeSingleLineRawString(int next, int quoteChar, int start) {
     bool asciiOnly = true;
     while (next != $EOF) {
       if (identical(next, quoteChar)) {
         if (!asciiOnly) handleUnicode(start);
         next = advance();
         appendSubstringToken(TokenType.STRING, start, asciiOnly);
         return next;
       } else if (identical(next, $LF) || identical(next, $CR)) {
         if (!asciiOnly) handleUnicode(start);
-        return unterminatedRawString(quoteChar);
+        return unterminatedRawString(quoteChar, start, asciiOnly);
       } else if (next > 127) {
         asciiOnly = false;
       }
       next = advance();
     }
     if (!asciiOnly) handleUnicode(start);
-    return unterminatedRawString(quoteChar);
+    return unterminatedRawString(quoteChar, start, asciiOnly);
   }
 
   int tokenizeMultiLineRawString(int quoteChar, int start) {
     bool asciiOnlyString = true;
     bool asciiOnlyLine = true;
     int unicodeStart = start;
     int next = advance(); // Advance past the (last) quote (of three).
     outer:
     while (!identical(next, $EOF)) {
       while (!identical(next, quoteChar)) {
@@ skipping 17 matching lines @@
         next = advance();
         if (identical(next, quoteChar)) {
           if (!asciiOnlyLine) handleUnicode(unicodeStart);
           next = advance();
           appendSubstringToken(TokenType.STRING, start, asciiOnlyString);
           return next;
         }
       }
     }
     if (!asciiOnlyLine) handleUnicode(unicodeStart);
-    return unterminatedRawMultiLineString(quoteChar);
+    return unterminatedRawMultiLineString(quoteChar, start, asciiOnlyLine);
   }
 
   int tokenizeMultiLineString(int quoteChar, int start, bool raw) {
     if (raw) return tokenizeMultiLineRawString(quoteChar, start);
     bool asciiOnlyString = true;
     bool asciiOnlyLine = true;
     int unicodeStart = start;
     int next = advance(); // Advance past the (last) quote (of three).
     while (!identical(next, $EOF)) {
       if (identical(next, $$)) {
@@ skipping 30 matching lines @@
           unicodeStart = scanOffset;
         }
         lineFeedInMultiline();
       } else if (next > 127) {
         asciiOnlyString = false;
         asciiOnlyLine = false;
       }
       next = advance();
     }
     if (!asciiOnlyLine) handleUnicode(unicodeStart);
-    return unterminatedMultiLineString(quoteChar);
+    return unterminatedMultiLineString(quoteChar, start, asciiOnlyString);
   }
 
   int unexpected(int character) {
     appendErrorToken(buildUnexpectedCharacterToken(character, tokenStart));
     return advanceAfterError(true);
   }
 
   int unterminated(String prefix, {bool shouldAdvance: true}) {
     appendErrorToken(new UnterminatedToken(prefix, tokenStart, stringOffset));
     return advanceAfterError(shouldAdvance);
   }
 
-  int unterminatedString(int quoteChar) {
-    return unterminated(new String.fromCharCodes([quoteChar]));
+  void terminateUnterminatedString(
+      int start, bool asciiOnlyString, String closingQuotes) {
+    appendSyntheticSubstringToken(
+        TokenType.STRING, start, asciiOnlyString, closingQuotes);
+    beginToken();
   }
 
-  int unterminatedRawString(int quoteChar) {
-    return unterminated('r${new String.fromCharCodes([quoteChar])}');
+  int unterminatedString(int quoteChar, int start, bool asciiOnlyString) {
+    String suffix = new String.fromCharCodes([quoteChar]);
+    terminateUnterminatedString(start, asciiOnlyString, suffix);
+    return unterminated(suffix);
   }
 
-  int unterminatedMultiLineString(int quoteChar) {
-    return unterminated(
-        new String.fromCharCodes([quoteChar, quoteChar, quoteChar]));
+  int unterminatedRawString(int quoteChar, int start, bool asciiOnlyString) {
+    String suffix = new String.fromCharCodes([quoteChar]);
+    terminateUnterminatedString(start, asciiOnlyString, suffix);
+    return unterminated('r$suffix');
   }
 
-  int unterminatedRawMultiLineString(int quoteChar) {
-    return unterminated(
-        'r${new String.fromCharCodes([quoteChar, quoteChar, quoteChar])}');
+  int unterminatedMultiLineString(
+      int quoteChar, int start, bool asciiOnlyString) {
+    String suffix = new String.fromCharCodes([quoteChar, quoteChar, quoteChar]);
+    terminateUnterminatedString(start, asciiOnlyString, suffix);
+    return unterminated(suffix);
+  }
+
+  int unterminatedRawMultiLineString(
+      int quoteChar, int start, bool asciiOnlyString) {

[ahe, 2017/06/02 12:26:43]

Since these are for error handling, you could possibly collapse them into one
method with these named parameters:

{bool asciiOnlyString, bool isRaw, bool isMultiLine}

Normally, I avoid using names parameters in performance critical code, but this
isn't that.

[danrubel, 2017/06/07 15:51:41]

Sounds reasonable. Fixed in https://codereview.chromium.org/2919163002/

+    String suffix = new String.fromCharCodes([quoteChar, quoteChar, quoteChar]);
+    terminateUnterminatedString(start, asciiOnlyString, suffix);
+    return unterminated('r$suffix');
   }
 
   int advanceAfterError(bool shouldAdvance) {
     if (atEndOfFile()) return $EOF;
     if (shouldAdvance) {
       return advance(); // Ensure progress.
     } else {
       return -1;
     }
   }
@@ skipping 78 matching lines @@
       switchToUint32(newLength);
     }
   }
 
   void switchToUint32(int newLength) {
     final newArray = new Uint32List(newLength);
     newArray.setRange(0, arrayLength, array);
     array = newArray;
   }
 }