[V8] Reland https://codereview.chromium.org/1100993003/

src/scanner.cc

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 // Features shared by parsing and pre-parsing scanners.
 
 #include <stdint.h>
 
 #include <cmath>
 
@@ skipping 207 matching lines @@
   Token::ILLEGAL,
   Token::LBRACE,       // 0x7b
   Token::ILLEGAL,
   Token::RBRACE,       // 0x7d
   Token::BIT_NOT,      // 0x7e
   Token::ILLEGAL
 };
 
 
 Token::Value Scanner::Next() {
+  if (next_.token == Token::EOS) {
+    next_.location.beg_pos = current_.location.beg_pos;
+    next_.location.end_pos = current_.location.end_pos;
+  }
   current_ = next_;
   has_line_terminator_before_next_ = false;
   has_multiline_comment_before_next_ = false;
   if (static_cast<unsigned>(c0_) <= 0x7f) {
     Token::Value token = static_cast<Token::Value>(one_char_tokens[c0_]);
     if (token != Token::ILLEGAL) {
       int pos = source_pos();
       next_.token = token;
       next_.location.beg_pos = pos;
       next_.location.end_pos = pos + 1;
@@ skipping 1321 matching lines @@
     }
     backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 7) | 0x80u));
   }
   backing_store_.Add(static_cast<uint8_t>(one_byte_length & 0x7f));
 
   backing_store_.AddBlock(bytes);
   return backing_store_.EndSequence().start();
 }
 
 } }  // namespace v8::internal