Issue 8742. Preserve leading line comments during java2dart translation.

pkg/analyzer/lib/src/generated/scanner.dart

 // Copyright (c) 2014, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 // This code was auto-generated, is not intended to be edited, and is subject to
 // significant change. Please see the README file for more information.
 
 library engine.scanner;
 
 import 'dart:collection';
@@ skipping 632 matching lines @@
    * specification of the modification that was made, return a stream of tokens scanned from the
    * modified source. The original stream of tokens will not be modified.
    *
    * @param originalStream the stream of tokens scanned from the original source
    * @param index the index of the first character in both the original and modified source that was
    *          affected by the modification
    * @param removedLength the number of characters removed from the original source
    * @param insertedLength the number of characters added to the modified source
    */
   Token rescan(Token originalStream, int index, int removedLength, int insertedLength) {
+    //
+    // Copy all of the tokens in the originalStream whose end is less than the replacement start.
+    // (If the replacement start is equal to the end of an existing token, then it means that the
+    // existing token might have been modified, so we need to rescan it.)
+    //
     while (originalStream.type != TokenType.EOF && originalStream.end < index) {
       originalStream = copyAndAdvance(originalStream, 0);
     }
     Token oldFirst = originalStream;
     Token oldLeftToken = originalStream.previous;
     _leftToken = tail;
+    //
+    // Skip tokens in the original stream until we find a token whose offset is greater than the end
+    // of the removed region. (If the end of the removed region is equal to the beginning of an
+    // existing token, then it means that the existing token might have been modified, so we need to
+    // rescan it.)
+    //
     int removedEnd = index + (removedLength == 0 ? 0 : removedLength - 1);
     while (originalStream.type != TokenType.EOF && originalStream.offset <= removedEnd) {
       originalStream = originalStream.next;
     }
     Token oldLast;
     Token oldRightToken;
     if (originalStream.type != TokenType.EOF && removedEnd + 1 == originalStream.offset) {
       oldLast = originalStream;
       originalStream = originalStream.next;
       oldRightToken = originalStream;
     } else {
       oldLast = originalStream.previous;
       oldRightToken = originalStream;
     }
+    //
+    // Compute the delta between the character index of characters after the modified region in the
+    // original source and the index of the corresponding character in the modified source.
+    //
     int delta = insertedLength - removedLength;
+    //
+    // Compute the range of characters that are known to need to be rescanned. If the index is
+    // within an existing token, then we need to start at the beginning of the token.
+    //
     int scanStart = Math.min(oldFirst.offset, index);
     int oldEnd = oldLast.end + delta - 1;
     int newEnd = index + insertedLength - 1;
     int scanEnd = Math.max(newEnd, oldEnd);
+    //
+    // Starting at the start of the scan region, scan tokens from the modifiedSource until the end
+    // of the just scanned token is greater than or equal to end of the scan region in the modified
+    // source. Include trailing characters of any token that was split as a result of inserted text,
+    // as in "ab" --> "a.b".
+    //
     _reader.offset = scanStart - 1;
     int next = _reader.advance();
     while (next != -1 && _reader.offset <= scanEnd) {
       next = bigSwitch(next);
     }
+    //
+    // Copy the remaining tokens in the original stream, but apply the delta to the token's offset.
+    //
     if (identical(originalStream.type, TokenType.EOF)) {
       copyAndAdvance(originalStream, delta);
       _rightToken = tail;
       _rightToken.setNextWithoutSettingPrevious(_rightToken);
     } else {
       originalStream = copyAndAdvance(originalStream, delta);
       _rightToken = tail;
       while (originalStream.type != TokenType.EOF) {
         originalStream = copyAndAdvance(originalStream, delta);
       }
       Token eof = copyAndAdvance(originalStream, delta);
       eof.setNextWithoutSettingPrevious(eof);
     }
+    //
+    // If the index is immediately after an existing token and the inserted characters did not
+    // change that original token, then adjust the leftToken to be the next token. For example, in
+    // "a; c;" --> "a;b c;", the leftToken was ";", but this code advances it to "b" since "b" is
+    // the first new token.
+    //
     Token newFirst = _leftToken.next;
     while (newFirst != _rightToken && oldFirst != oldRightToken && newFirst.type != TokenType.EOF && equals3(oldFirst, newFirst)) {
       _tokenMap.put(oldFirst, newFirst);
       oldLeftToken = oldFirst;
       oldFirst = oldFirst.next;
       _leftToken = newFirst;
       newFirst = newFirst.next;
     }
     Token newLast = _rightToken.previous;
     while (newLast != _leftToken && oldLast != oldLeftToken && newLast.type != TokenType.EOF && equals3(oldLast, newLast)) {
       _tokenMap.put(oldLast, newLast);
       oldRightToken = oldLast;
       oldLast = oldLast.previous;
       _rightToken = newLast;
       newLast = newLast.previous;
     }
     _hasNonWhitespaceChange2 = _leftToken.next != _rightToken || oldLeftToken.next != oldRightToken;
+    //
+    // TODO(brianwilkerson) Begin tokens are not getting associated with the corresponding end
+    //     tokens (because the end tokens have not been copied when we're copying the begin tokens).
+    //     This could have implications for parsing.
+    // TODO(brianwilkerson) Update the lineInfo.
+    //
     return firstToken;
   }
 
   Token copyAndAdvance(Token originalToken, int delta) {
     Token copiedToken = originalToken.copy();
     _tokenMap.put(originalToken, copiedToken);
     copiedToken.applyDelta(delta);
     appendToken(copiedToken);
     Token originalComment = originalToken.precedingComments;
     Token copiedComment = originalToken.precedingComments;
@@ skipping 359 matching lines @@
       token = new BeginTokenWithComment(type, _tokenStart, _firstComment);
       _firstComment = null;
       _lastComment = null;
     }
     _tail = _tail.setNext(token);
     _groupingStack.add(token);
     _stackEnd++;
   }
 
   void appendCommentToken(TokenType type, String value) {
+    // Ignore comment tokens if client specified that it doesn't need them.
     if (!_preserveComments) {
       return;
     }
+    // OK, remember comment tokens.
     if (_firstComment == null) {
       _firstComment = new StringToken(type, value, _tokenStart);
       _lastComment = _firstComment;
     } else {
       _lastComment = _lastComment.setNext(new StringToken(type, value, _tokenStart));
     }
   }
 
   void appendEndToken(TokenType type, TokenType beginType) {
     Token token;
@@ skipping 16 matching lines @@
 
   void appendEofToken() {
     Token eofToken;
     if (_firstComment == null) {
       eofToken = new Token(TokenType.EOF, _reader.offset + 1);
     } else {
       eofToken = new TokenWithComment(TokenType.EOF, _reader.offset + 1, _firstComment);
       _firstComment = null;
       _lastComment = null;
     }
+    // The EOF token points to itself so that there is always infinite look-ahead.
     eofToken.setNext(eofToken);
     _tail = _tail.setNext(eofToken);
     if (_stackEnd >= 0) {
       _hasUnmatchedGroups2 = true;
     }
   }
 
   void appendKeywordToken(Keyword keyword) {
     if (_firstComment == null) {
       _tail = _tail.setNext(new KeywordToken(keyword, _tokenStart));
@@ skipping 57 matching lines @@
    */
   BeginToken findTokenMatchingClosingBraceInInterpolationExpression() {
     while (_stackEnd >= 0) {
       BeginToken begin = _groupingStack[_stackEnd];
       if (identical(begin.type, TokenType.OPEN_CURLY_BRACKET) || identical(begin.type, TokenType.STRING_INTERPOLATION_EXPRESSION)) {
         return begin;
       }
       _hasUnmatchedGroups2 = true;
       _groupingStack.removeAt(_stackEnd--);
     }
+    //
+    // We should never get to this point because we wouldn't be inside a string interpolation
+    // expression unless we had previously found the start of the expression.
+    //
     return null;
   }
 
   /**
    * Report an error at the current offset.
    *
    * @param errorCode the error code indicating the nature of the error
    * @param arguments any arguments needed to complete the error message
    */
   void reportError(ScannerErrorCode errorCode, List<Object> arguments) {
@@ skipping 16 matching lines @@
     if (next == choice) {
       appendToken3(yesType, offset);
       return _reader.advance();
     } else {
       appendToken3(noType, offset);
       return next;
     }
   }
 
   int tokenizeAmpersand(int next) {
+    // && &= &
     next = _reader.advance();
     if (next == 0x26) {
       appendToken2(TokenType.AMPERSAND_AMPERSAND);
       return _reader.advance();
     } else if (next == 0x3D) {
       appendToken2(TokenType.AMPERSAND_EQ);
       return _reader.advance();
     } else {
       appendToken2(TokenType.AMPERSAND);
       return next;
     }
   }
 
   int tokenizeBar(int next) {
+    // | || |=
     next = _reader.advance();
     if (next == 0x7C) {
       appendToken2(TokenType.BAR_BAR);
       return _reader.advance();
     } else if (next == 0x3D) {
       appendToken2(TokenType.BAR_EQ);
       return _reader.advance();
     } else {
       appendToken2(TokenType.BAR);
       return next;
     }
   }
 
   int tokenizeCaret(int next) => select(0x3D, TokenType.CARET_EQ, TokenType.CARET);
 
   int tokenizeDotOrNumber(int next) {
     int start = _reader.offset;
     next = _reader.advance();
     if (0x30 <= next && next <= 0x39) {
       return tokenizeFractionPart(next, start);
     } else if (0x2E == next) {
       return select(0x2E, TokenType.PERIOD_PERIOD_PERIOD, TokenType.PERIOD_PERIOD);
     } else {
       appendToken2(TokenType.PERIOD);
       return next;
     }
   }
 
   int tokenizeEquals(int next) {
+    // = == =>
     next = _reader.advance();
     if (next == 0x3D) {
       appendToken2(TokenType.EQ_EQ);
       return _reader.advance();
     } else if (next == 0x3E) {
       appendToken2(TokenType.FUNCTION);
       return _reader.advance();
     }
     appendToken2(TokenType.EQ);
     return next;
   }
 
   int tokenizeExclamation(int next) {
+    // ! !=
     next = _reader.advance();
     if (next == 0x3D) {
       appendToken2(TokenType.BANG_EQ);
       return _reader.advance();
     }
     appendToken2(TokenType.BANG);
     return next;
   }
 
   int tokenizeExponent(int next) {
@@ skipping 37 matching lines @@
         return select2(0x2E, TokenType.PERIOD_PERIOD_PERIOD, TokenType.PERIOD_PERIOD, _reader.offset - 1);
       }
       appendToken3(TokenType.PERIOD, _reader.offset - 1);
       return bigSwitch(next);
     }
     appendStringToken(TokenType.DOUBLE, _reader.getString(start, next < 0 ? 0 : -1));
     return next;
   }
 
   int tokenizeGreaterThan(int next) {
+    // > >= >> >>=
     next = _reader.advance();
     if (0x3D == next) {
       appendToken2(TokenType.GT_EQ);
       return _reader.advance();
     } else if (0x3E == next) {
       next = _reader.advance();
       if (0x3D == next) {
         appendToken2(TokenType.GT_GT_EQ);
         return _reader.advance();
       } else {
@@ skipping 100 matching lines @@
       return tokenizeIdentifier(next, start, allowDollar);
     } else if (next < 128) {
       appendKeywordToken(state.keyword());
       return next;
     } else {
       return tokenizeIdentifier(next, start, allowDollar);
     }
   }
 
   int tokenizeLessThan(int next) {
+    // < <= << <<=
     next = _reader.advance();
     if (0x3D == next) {
       appendToken2(TokenType.LT_EQ);
       return _reader.advance();
     } else if (0x3C == next) {
       return select(0x3D, TokenType.LT_LT_EQ, TokenType.LT_LT);
     } else {
       appendToken2(TokenType.LT);
       return next;
     }
   }
 
   int tokenizeMinus(int next) {
+    // - -- -=
     next = _reader.advance();
     if (next == 0x2D) {
       appendToken2(TokenType.MINUS_MINUS);
       return _reader.advance();
     } else if (next == 0x3D) {
       appendToken2(TokenType.MINUS_EQ);
       return _reader.advance();
     } else {
       appendToken2(TokenType.MINUS);
       return next;
@@ skipping 144 matching lines @@
       } else if (next == 0x65 || next == 0x45) {
         return tokenizeFractionPart(next, start);
       } else {
         appendStringToken(TokenType.INT, _reader.getString(start, next < 0 ? 0 : -1));
         return next;
       }
     }
   }
 
   int tokenizeOpenSquareBracket(int next) {
+    // [ []  []=
     next = _reader.advance();
     if (next == 0x5D) {
       return select(0x3D, TokenType.INDEX_EQ, TokenType.INDEX);
     } else {
       appendBeginToken(TokenType.OPEN_SQUARE_BRACKET);
       return next;
     }
   }
 
   int tokenizePercent(int next) => select(0x3D, TokenType.PERCENT_EQ, TokenType.PERCENT);
 
   int tokenizePlus(int next) {
+    // + ++ +=
     next = _reader.advance();
     if (0x2B == next) {
       appendToken2(TokenType.PLUS_PLUS);
       return _reader.advance();
     } else if (0x3D == next) {
       appendToken2(TokenType.PLUS_EQ);
       return _reader.advance();
     } else {
       appendToken2(TokenType.PLUS);
       return next;
@@ skipping 67 matching lines @@
       return next;
     }
   }
 
   int tokenizeString(int next, int start, bool raw) {
     int quoteChar = next;
     next = _reader.advance();
     if (quoteChar == next) {
       next = _reader.advance();
       if (quoteChar == next) {
+        // Multiline string.
         return tokenizeMultiLineString(quoteChar, start, raw);
       } else {
+        // Empty string.
         appendStringToken(TokenType.STRING, _reader.getString(start, -1));
         return next;
       }
     }
     if (raw) {
       return tokenizeSingleLineRawString(next, quoteChar, start);
     } else {
       return tokenizeSingleLineString(next, quoteChar, start);
     }
   }
 
   int tokenizeStringInterpolation(int start) {
     beginToken();
     int next = _reader.advance();
     if (next == 0x7B) {
       return tokenizeInterpolatedExpression(next, start);
     } else {
       return tokenizeInterpolatedIdentifier(next, start);
     }
   }
 
   int tokenizeTag(int next) {
+    // # or #!.*[\n\r]
     if (_reader.offset == 0) {
       if (_reader.peek() == 0x21) {
         do {
           next = _reader.advance();
         } while (next != 0xA && next != 0xD && next > 0);
         appendStringToken(TokenType.SCRIPT_TAG, _reader.getString(_tokenStart, 0));
         return next;
       }
     }
     appendToken2(TokenType.HASH);
     return _reader.advance();
   }
 
   int tokenizeTilde(int next) {
+    // ~ ~/ ~/=
     next = _reader.advance();
     if (next == 0x2F) {
       return select(0x3D, TokenType.TILDE_SLASH_EQ, TokenType.TILDE_SLASH);
     } else {
       appendToken2(TokenType.TILDE);
       return next;
     }
   }
 }
 
@@ skipping 866 matching lines @@
    * @return `true` if this token type represents an operator that can be defined by users
    */
   bool get isUserDefinableOperator => identical(_lexeme, "==") || identical(_lexeme, "~") || identical(_lexeme, "[]") || identical(_lexeme, "[]=") || identical(_lexeme, "*") || identical(_lexeme, "/") || identical(_lexeme, "%") || identical(_lexeme, "~/") || identical(_lexeme, "+") || identical(_lexeme, "-") || identical(_lexeme, "<<") || identical(_lexeme, ">>") || identical(_lexeme, ">=") || identical(_lexeme, ">") || identical(_lexeme, "<=") || identical(_lexeme, "<") || identical(_lexeme, "&") || identical(_lexeme, "^") || identical(_lexeme, "|");
 }
 
 class TokenType_EOF extends TokenType {
   TokenType_EOF(String name, int ordinal, TokenClass arg0, String arg1) : super.con2(name, ordinal, arg0, arg1);
 
   String toString() => "-eof-";
 }