Make Script::GetTokenLocation() and Script::TokenRangeAtLine() use token streams where possible.

runtime/vm/object.cc

Index: runtime/vm/object.cc
diff --git a/runtime/vm/object.cc b/runtime/vm/object.cc
index 290273aa9f39e4e685a69f4980d67aee5efa9258..5f783b4d8763de13749b6f42618bc746cf5b2469 100644
--- a/runtime/vm/object.cc
+++ b/runtime/vm/object.cc
@@ -6364,17 +6364,29 @@ void Script::SetLocationOffset(intptr_t line_offset,
 void Script::GetTokenLocation(intptr_t token_pos,
                               intptr_t* line,
                               intptr_t* column) const {
-  const String& src = String::Handle(Source());
   const TokenStream& tkns = TokenStream::Handle(tokens());
-  intptr_t src_pos = tkns.ComputeSourcePosition(token_pos);
-  Scanner scanner(src, Symbols::Empty());
-  scanner.ScanTo(src_pos);
-  intptr_t relative_line = scanner.CurrentPosition().line;
-  *line = relative_line + line_offset();
-  *column = scanner.CurrentPosition().column;
-  // On the first line of the script we must add the column offset.
-  if (relative_line == 1) {
-    *column += col_offset();
+  if (column == NULL) {
+    TokenStream::Iterator tkit(tkns, 0, TokenStream::Iterator::kAllTokens);
+    intptr_t relative_line = 1;
+    while (tkit.CurrentPosition() < token_pos) {

[hausner, 2013/09/23 18:54:49]

Does this terminate properly if I give a token position that is past the end of
the token stream?

[Michael Lippautz (Google), 2013/09/23 21:15:03]

Fixed.

+      if (tkit.CurrentTokenKind() == Token::kNEWLINE) {
+        relative_line++;
+      }
+      tkit.Advance();
+    }
+    *line = relative_line + line_offset();
+  } else {
+    const String& src = String::Handle(Source());
+    intptr_t src_pos = tkns.ComputeSourcePosition(token_pos);
+    Scanner scanner(src, Symbols::Empty());
+    scanner.ScanTo(src_pos);
+    intptr_t relative_line = scanner.CurrentPosition().line;
+    *line = relative_line + line_offset();
+    *column = scanner.CurrentPosition().column;
+    // On the first line of the script we must add the column offset.
+    if (relative_line == 1) {
+      *column += col_offset();
+    }
   }
 }
 
@@ -6382,18 +6394,27 @@ void Script::GetTokenLocation(intptr_t token_pos,
 void Script::TokenRangeAtLine(intptr_t line_number,
                               intptr_t* first_token_index,
                               intptr_t* last_token_index) const {
-  const String& src = String::Handle(Source());
   const TokenStream& tkns = TokenStream::Handle(tokens());
   line_number -= line_offset();
   if (line_number < 1) line_number = 1;
-  Scanner scanner(src, Symbols::Empty());
-  scanner.TokenRangeAtLine(line_number, first_token_index, last_token_index);
-  if (*first_token_index >= 0) {
-    *first_token_index = tkns.ComputeTokenPosition(*first_token_index);
-  }
-  if (*last_token_index >= 0) {
-    *last_token_index = tkns.ComputeTokenPosition(*last_token_index);
-  }
+  TokenStream::Iterator tkit(tkns, 0, TokenStream::Iterator::kAllTokens);
+  // Scan through the token stream to the required line.
+  while (line_number > 1) {
+    if (tkit.CurrentTokenKind() == Token::kNEWLINE) {
+      line_number--;
+    }
+    tkit.Advance();
+  }
+  *first_token_index = tkit.CurrentPosition();
+  // We cannot do "CurrentPosition() - 1" for the last token, because we do not
+  // know whether the previous token is a simple or not.
+  intptr_t end_pos = *first_token_index;
+  while (tkit.CurrentTokenKind() != Token::kNEWLINE &&
+         tkit.CurrentTokenKind() != Token::kEOS) {
+    end_pos = tkit.CurrentPosition();
+    tkit.Advance();
+  }
+  *last_token_index = end_pos;

[hausner, 2013/09/23 18:54:49]

The old code only assigned to the out parameters if they are non-null.

[Michael Lippautz (Google), 2013/09/23 21:15:03]

The out parameters are required to be non-NULL. Moved the assert (which was
hidden in scanner.TokenRangeAtLine) into this function.

 }