Make Script::GetTokenLocation() and Script::TokenRangeAtLine() use token streams where possible.

runtime/vm/object.cc

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/object.h"
 
 #include "include/dart_api.h"
 #include "platform/assert.h"
 #include "vm/assembler.h"
 #include "vm/cpu.h"
@@ skipping 6346 matching lines @@
   ASSERT(line_offset >= 0);
   ASSERT(col_offset >= 0);
   raw_ptr()->line_offset_ = line_offset;
   raw_ptr()->col_offset_ = col_offset;
 }
 
 
 void Script::GetTokenLocation(intptr_t token_pos,
                               intptr_t* line,
                               intptr_t* column) const {
-  const String& src = String::Handle(Source());
   const TokenStream& tkns = TokenStream::Handle(tokens());
-  intptr_t src_pos = tkns.ComputeSourcePosition(token_pos);
-  Scanner scanner(src, Symbols::Empty());
-  scanner.ScanTo(src_pos);
-  intptr_t relative_line = scanner.CurrentPosition().line;
-  *line = relative_line + line_offset();
-  *column = scanner.CurrentPosition().column;
-  // On the first line of the script we must add the column offset.
-  if (relative_line == 1) {
-    *column += col_offset();
+  if (column == NULL) {
+    TokenStream::Iterator tkit(tkns, 0, TokenStream::Iterator::kAllTokens);
+    intptr_t relative_line = 1;
+    while (tkit.CurrentPosition() < token_pos) {

[hausner, 2013/09/23 18:54:49]

Does this terminate properly if I give a token position that is past the end of
the token stream?

[Michael Lippautz (Google), 2013/09/23 21:15:03]

Fixed.

+      if (tkit.CurrentTokenKind() == Token::kNEWLINE) {
+        relative_line++;
+      }
+      tkit.Advance();
+    }
+    *line = relative_line + line_offset();
+  } else {
+    const String& src = String::Handle(Source());
+    intptr_t src_pos = tkns.ComputeSourcePosition(token_pos);
+    Scanner scanner(src, Symbols::Empty());
+    scanner.ScanTo(src_pos);
+    intptr_t relative_line = scanner.CurrentPosition().line;
+    *line = relative_line + line_offset();
+    *column = scanner.CurrentPosition().column;
+    // On the first line of the script we must add the column offset.
+    if (relative_line == 1) {
+      *column += col_offset();
+    }
   }
 }
 
 
 void Script::TokenRangeAtLine(intptr_t line_number,
                               intptr_t* first_token_index,
                               intptr_t* last_token_index) const {
-  const String& src = String::Handle(Source());
   const TokenStream& tkns = TokenStream::Handle(tokens());
   line_number -= line_offset();
   if (line_number < 1) line_number = 1;
-  Scanner scanner(src, Symbols::Empty());
-  scanner.TokenRangeAtLine(line_number, first_token_index, last_token_index);
-  if (*first_token_index >= 0) {
-    *first_token_index = tkns.ComputeTokenPosition(*first_token_index);
+  TokenStream::Iterator tkit(tkns, 0, TokenStream::Iterator::kAllTokens);
+  // Scan through the token stream to the required line.
+  while (line_number > 1) {
+    if (tkit.CurrentTokenKind() == Token::kNEWLINE) {
+      line_number--;
+    }
+    tkit.Advance();
   }
-  if (*last_token_index >= 0) {
-    *last_token_index = tkns.ComputeTokenPosition(*last_token_index);
+  *first_token_index = tkit.CurrentPosition();
+  // We cannot do "CurrentPosition() - 1" for the last token, because we do not
+  // know whether the previous token is a simple or not.
+  intptr_t end_pos = *first_token_index;
+  while (tkit.CurrentTokenKind() != Token::kNEWLINE &&
+         tkit.CurrentTokenKind() != Token::kEOS) {
+    end_pos = tkit.CurrentPosition();
+    tkit.Advance();
   }
+  *last_token_index = end_pos;

[hausner, 2013/09/23 18:54:49]

The old code only assigned to the out parameters if they are non-null.

[Michael Lippautz (Google), 2013/09/23 21:15:03]

The out parameters are required to be non-NULL. Moved the assert (which was
hidden in scanner.TokenRangeAtLine) into this function.

 }
 
 
 RawString* Script::GetLine(intptr_t line_number) const {
   const String& src = String::Handle(Source());
   intptr_t relative_line_number = line_number - line_offset();
   intptr_t current_line = 1;
   intptr_t line_start_idx = -1;
   intptr_t last_char_idx = -1;
   for (intptr_t ix = 0;
@@ skipping 8630 matching lines @@
   return "_MirrorReference";
 }
 
 
 void MirrorReference::PrintToJSONStream(JSONStream* stream, bool ref) const {
   JSONObject jsobj(stream);
 }
 
 
 }  // namespace dart