Replace intptr_t with TokenDescriptor

runtime/vm/debugger.cc

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/debugger.h"
 
 #include "include/dart_api.h"
 
 #include "vm/code_generator.h"
 #include "vm/code_patcher.h"
 #include "vm/compiler.h"
 #include "vm/dart_entry.h"
 #include "vm/deopt_instructions.h"
 #include "vm/flags.h"
 #include "vm/globals.h"
 #include "vm/longjump.h"
 #include "vm/json_stream.h"
 #include "vm/message_handler.h"
 #include "vm/object.h"
 #include "vm/object_store.h"
 #include "vm/os.h"
 #include "vm/port.h"
 #include "vm/service_event.h"
 #include "vm/service_isolate.h"
 #include "vm/service.h"
 #include "vm/stack_frame.h"
 #include "vm/stub_code.h"
 #include "vm/symbols.h"
 #include "vm/thread_interrupter.h"
+#include "vm/token_descriptor.h"
 #include "vm/visitor.h"
 
 
 namespace dart {
 
 DEFINE_FLAG(bool, show_invisible_frames, false,
             "Show invisible frames in debugger stack traces");
 DEFINE_FLAG(bool, trace_debugger_stacktrace, false,
             "Trace debugger stacktrace collection");
 DEFINE_FLAG(bool, verbose_debug, false, "Verbose debugger messages");
@@ skipping 20 matching lines @@
 
  private:
   GrowableObjectArray* objs_;
 
   DISALLOW_COPY_AND_ASSIGN(RemoteObjectCache);
 };
 
 
 // Create an unresolved breakpoint in given token range and script.
 BreakpointLocation::BreakpointLocation(const Script& script,
-                                       intptr_t token_pos,
-                                       intptr_t end_token_pos,
+                                       TokenDescriptor token_pos,
+                                       TokenDescriptor end_token_pos,
                                        intptr_t requested_line_number,
                                        intptr_t requested_column_number)
     : script_(script.raw()),
       url_(script.url()),
       token_pos_(token_pos),
       end_token_pos_(end_token_pos),
       is_resolved_(false),
       next_(NULL),
       conditions_(NULL),
       requested_line_number_(requested_line_number),
       requested_column_number_(requested_column_number),
       function_(Function::null()),
       line_number_(-1),
       column_number_(-1) {
   ASSERT(!script.IsNull());
-  ASSERT(Token::IsReal(token_pos_));
+  ASSERT(TokenDescriptor(token_pos_).IsReal());
 }
 
 // Create a latent breakpoint at given url and line number.
 BreakpointLocation::BreakpointLocation(const String& url,
                                        intptr_t requested_line_number,
                                        intptr_t requested_column_number)
     : script_(Script::null()),
       url_(url.raw()),
-      token_pos_(Token::kNoSourcePos),
-      end_token_pos_(Token::kNoSourcePos),
+      token_pos_(TokenDescriptor::kNoSource),
+      end_token_pos_(TokenDescriptor::kNoSource),
       is_resolved_(false),
       next_(NULL),
       conditions_(NULL),
       requested_line_number_(requested_line_number),
       requested_column_number_(requested_column_number),
       function_(Function::null()),
       line_number_(-1),
       column_number_(-1) {
   ASSERT(requested_line_number_ >= 0);
 }
 
 
 BreakpointLocation::~BreakpointLocation() {
   Breakpoint* bpt = breakpoints();
   while (bpt != NULL) {
     Breakpoint* temp = bpt;
     bpt = bpt->next();
     delete temp;
   }
 }
 
 
 bool BreakpointLocation::AnyEnabled() const {
   return breakpoints() != NULL;
 }
 
 
-void BreakpointLocation::SetResolved(const Function& func, intptr_t token_pos) {
+void BreakpointLocation::SetResolved(const Function& func,
+                                     TokenDescriptor token_pos) {
   ASSERT(!IsLatent());
   ASSERT(func.script() == script_);
   ASSERT((func.token_pos() <= token_pos) &&
          (token_pos <= func.end_token_pos()));
   ASSERT(func.is_debuggable());
   function_ = func.raw();
   token_pos_ = token_pos;
   end_token_pos_ = token_pos;
   is_resolved_ = true;
 }
 
 
 // TODO(hausner): Get rid of library parameter. A source breakpoint location
 // does not imply a library, since the same source code can be included
 // in more than one library, e.g. the text location of mixin functions.
 void BreakpointLocation::GetCodeLocation(Library* lib,
                                          Script* script,
-                                         intptr_t* pos) const {
+                                         TokenDescriptor* pos) const {
   if (IsLatent()) {
     *lib = Library::null();
     *script = Script::null();
-    *pos = -1;
+    *pos = TokenDescriptor::kNoSource;
   } else {
     *script = this->script();
     *pos = token_pos_;
     if (IsResolved()) {
       const Function& func = Function::Handle(function_);
       ASSERT(!func.IsNull());
       const Class& cls = Class::Handle(func.origin());
       *lib = cls.library();
     } else {
       *lib = Library::null();
@@ skipping 74 matching lines @@
     uword fp,
     uword sp,
     const Code& code,
     const Array& deopt_frame,
     intptr_t deopt_frame_offset)
     : pc_(pc), fp_(fp), sp_(sp),
       ctx_(Context::ZoneHandle()),
       code_(Code::ZoneHandle(code.raw())),
       function_(Function::ZoneHandle(code.function())),
       token_pos_initialized_(false),
-      token_pos_(Token::kNoSourcePos),
+      token_pos_(TokenDescriptor::kNoSource),
       try_index_(-1),
       line_number_(-1),
       column_number_(-1),
       context_level_(-1),
       deopt_frame_(Array::ZoneHandle(deopt_frame.raw())),
       deopt_frame_offset_(deopt_frame_offset),
       vars_initialized_(false),
       var_descriptors_(LocalVarDescriptors::ZoneHandle()),
       desc_indices_(8),
       pc_desc_(PcDescriptors::ZoneHandle()) {
@@ skipping 191 matching lines @@
   return OS::SCreate(Thread::Current()->zone(),
     "%s%s%s", func_class.IsTopLevel() ? "" : class_name.ToCString(),
               func_class.IsTopLevel() ? "" : ".",
               func_name.ToCString());
 }
 
 
 // Returns true if function contains the token position in the given script.
 static bool FunctionContains(const Function& func,
                              const Script& script,
-                             intptr_t token_pos) {
+                             TokenDescriptor token_pos) {
   if ((func.token_pos() <= token_pos) && (token_pos <= func.end_token_pos())) {
     // Check script equality second because it allocates
     // handles as a side effect.
     return func.script() == script.raw();
   }
   return false;
 }
 
 
 bool Debugger::HasBreakpoint(const Function& func, Zone* zone) {
@@ skipping 91 matching lines @@
 
 void ActivationFrame::GetPcDescriptors() {
   if (pc_desc_.IsNull()) {
     pc_desc_ = code().pc_descriptors();
     ASSERT(!pc_desc_.IsNull());
   }
 }
 
 
 // Compute token_pos_ and try_index_ and token_pos_initialized_.
-intptr_t ActivationFrame::TokenPos() {
+TokenDescriptor ActivationFrame::TokenPos() {
   if (!token_pos_initialized_) {
     token_pos_initialized_ = true;
-    token_pos_ = Token::kNoSourcePos;
+    token_pos_ = TokenDescriptor::kNoSource;
     GetPcDescriptors();
     PcDescriptors::Iterator iter(pc_desc_, RawPcDescriptors::kAnyKind);
     uword pc_offset = pc_ - code().EntryPoint();
     while (iter.MoveNext()) {
       if (iter.PcOffset() == pc_offset) {
         try_index_ = iter.TryIndex();
         token_pos_ = iter.TokenPos();
         break;
       }
     }
   }
   return token_pos_;
 }
 
 
 intptr_t ActivationFrame::TryIndex() {
   if (!token_pos_initialized_) {
     TokenPos();  // Side effect: computes token_pos_initialized_, try_index_.
   }
   return try_index_;
 }
 
 
 intptr_t ActivationFrame::LineNumber() {
   // Compute line number lazily since it causes scanning of the script.
-  if ((line_number_ < 0) && Token::IsReal(TokenPos())) {
-    const intptr_t token_pos = TokenPos();
+  if ((line_number_ < 0) && TokenPos().IsReal()) {
+    const TokenDescriptor token_pos = TokenPos();
     const Script& script = Script::Handle(SourceScript());
     script.GetTokenLocation(token_pos, &line_number_, NULL);
   }
   return line_number_;
 }
 
 
 intptr_t ActivationFrame::ColumnNumber() {
   // Compute column number lazily since it causes scanning of the script.
-  if ((column_number_ < 0) && Token::IsReal(TokenPos())) {
-    const intptr_t token_pos = TokenPos();
+  if ((column_number_ < 0) && TokenPos().IsReal()) {
+    const TokenDescriptor token_pos = TokenPos();
     const Script& script = Script::Handle(SourceScript());
     if (script.HasSource()) {
       script.GetTokenLocation(token_pos, &line_number_, &column_number_);
     } else {
       column_number_ = -1;
     }
   }
   return column_number_;
 }
 
@@ skipping 24 matching lines @@
 // Calculate the context level at the current token index of the frame.
 intptr_t ActivationFrame::ContextLevel() {
   const Context& ctx = GetSavedCurrentContext();
   if (context_level_ < 0 && !ctx.IsNull()) {
     ASSERT(!code_.is_optimized());
     context_level_ = 0;
     // TODO(hausner): What to do if there is no descriptor entry
     // for the code position of the frame? For now say we are at context
     // level 0.
     TokenPos();
-    if (token_pos_ == -1) {
+    if (token_pos_ == TokenDescriptor::kNoSource) {
       // No PcDescriptor.
       return context_level_;
     }
     ASSERT(!pc_desc_.IsNull());
-    intptr_t innermost_begin_pos = 0;
-    intptr_t activation_token_pos = TokenPos();
-    ASSERT(Token::IsReal(activation_token_pos));
+    TokenDescriptor innermost_begin_pos = TokenDescriptor::kMinSource;
+    TokenDescriptor activation_token_pos = TokenPos();
+    ASSERT(activation_token_pos.IsReal());
     GetVarDescriptors();
     intptr_t var_desc_len = var_descriptors_.Length();
     for (intptr_t cur_idx = 0; cur_idx < var_desc_len; cur_idx++) {
       RawLocalVarDescriptors::VarInfo var_info;
       var_descriptors_.GetInfo(cur_idx, &var_info);
       const int8_t kind = var_info.kind();
       if ((kind == RawLocalVarDescriptors::kContextLevel) &&
           (var_info.begin_pos <= activation_token_pos) &&
           (activation_token_pos < var_info.end_pos)) {
         // This var_descriptors_ entry is a context scope which is in scope
@@ skipping 91 matching lines @@
   return NULL;
 }
 
 
 void ActivationFrame::GetDescIndices() {
   if (vars_initialized_) {
     return;
   }
   GetVarDescriptors();
 
-  intptr_t activation_token_pos = TokenPos();
-  if (!Token::IsDebugPause(activation_token_pos)) {
+  TokenDescriptor activation_token_pos = TokenPos();
+  if (!activation_token_pos.IsDebugPause()) {
     // We don't have a token position for this frame, so can't determine
     // which variables are visible.
     vars_initialized_ = true;
     return;
   }
 
   GrowableArray<String*> var_names(8);
   intptr_t var_desc_len = var_descriptors_.Length();
   for (intptr_t cur_idx = 0; cur_idx < var_desc_len; cur_idx++) {
     ASSERT(var_names.length() == desc_indices_.length());
@@ skipping 139 matching lines @@
   intptr_t num = 0;
   while ((frame != NULL)) {
     OS::PrintErr("#%04" Pd " %s\n", num++, frame->ToCString());
     frame = iterator.NextFrame();
   }
 }
 
 
 void ActivationFrame::VariableAt(intptr_t i,
                                  String* name,
-                                 intptr_t* token_pos,
-                                 intptr_t* end_pos,
+                                 TokenDescriptor* token_pos,
+                                 TokenDescriptor* end_pos,
                                  Object* value) {
   GetDescIndices();
   ASSERT(i < desc_indices_.length());
   intptr_t desc_index = desc_indices_[i];
   ASSERT(name != NULL);
 
   *name = var_descriptors_.GetName(desc_index);
 
   RawLocalVarDescriptors::VarInfo var_info;
   var_descriptors_.GetInfo(desc_index, &var_info);
@@ skipping 47 matching lines @@
 }
 
 
 RawArray* ActivationFrame::GetLocalVariables() {
   GetDescIndices();
   intptr_t num_variables = desc_indices_.length();
   String& var_name = String::Handle();
   Object& value = Instance::Handle();
   const Array& list = Array::Handle(Array::New(2 * num_variables));
   for (intptr_t i = 0; i < num_variables; i++) {
-    intptr_t ignore;
+    TokenDescriptor ignore;
     VariableAt(i, &var_name, &ignore, &ignore, &value);
     list.SetAt(2 * i, var_name);
     list.SetAt((2 * i) + 1, value);
   }
   return list.raw();
 }
 
 
 RawObject* ActivationFrame::GetReceiver() {
   GetDescIndices();
   intptr_t num_variables = desc_indices_.length();
   String& var_name = String::Handle();
   Instance& value = Instance::Handle();
   for (intptr_t i = 0; i < num_variables; i++) {
-    intptr_t ignore;
+    TokenDescriptor ignore;
     VariableAt(i, &var_name, &ignore, &ignore, &value);
     if (var_name.Equals(Symbols::This())) {
       return value.raw();
     }
   }
   return Symbols::OptimizedOut().raw();
 }
 
 
 bool IsPrivateVariableName(const String& var_name) {
   return (var_name.Length() >= 1) && (var_name.CharAt(0) == '_');
 }
 
 
 RawObject* ActivationFrame::Evaluate(const String& expr) {
   GetDescIndices();
   const GrowableObjectArray& param_names =
       GrowableObjectArray::Handle(GrowableObjectArray::New());
   const GrowableObjectArray& param_values =
       GrowableObjectArray::Handle(GrowableObjectArray::New());
   String& name = String::Handle();
   Object& value = Instance::Handle();
   intptr_t num_variables = desc_indices_.length();
   for (intptr_t i = 0; i < num_variables; i++) {
-    intptr_t ignore;
+    TokenDescriptor ignore;
     VariableAt(i, &name, &ignore, &ignore, &value);
     if (!name.Equals(Symbols::This())) {
       if (IsPrivateVariableName(name)) {
         name = String::IdentifierPrettyName(name);
       }
       param_names.Add(name);
       param_values.Add(value);
     }
   }
 
@@ skipping 50 matching lines @@
     // in the world where we pass the script as part of the
     // location.
     jsobj->AddProperty("script", script, !full);
   }
   {
     JSONArray jsvars(jsobj, "vars");
     const int num_vars = NumLocalVariables();
     for (intptr_t v = 0; v < num_vars; v++) {
       String& var_name = String::Handle();
       Instance& var_value = Instance::Handle();
-      intptr_t token_pos;
-      intptr_t end_token_pos;
+      TokenDescriptor token_pos;
+      TokenDescriptor end_token_pos;
       VariableAt(v, &var_name, &token_pos, &end_token_pos, &var_value);
       if (var_name.raw() != Symbols::AsyncOperation().raw()) {
         JSONObject jsvar(&jsvars);
         jsvar.AddProperty("type", "BoundVariable");
         var_name = String::IdentifierPrettyName(var_name);
         jsvar.AddProperty("name", var_name.ToCString());
         jsvar.AddProperty("value", var_value, !full);
         // TODO(turnidge): Do we really want to provide this on every
         // stack dump?  Should be associated with the function object, I
         // think, and not the stack frame.
-        jsvar.AddProperty("_tokenPos", token_pos);
-        jsvar.AddProperty("_endTokenPos", end_token_pos);
+        jsvar.AddProperty("_tokenPos", token_pos.value());
+        jsvar.AddProperty("_endTokenPos", end_token_pos.value());
       }
     }
   }
 }
 
 
 
 void DebuggerStackTrace::AddActivation(ActivationFrame* frame) {
   if (FLAG_show_invisible_frames || frame->function().is_visible()) {
     trace_.Add(frame);
   }
 }
 
 
 const uint8_t kSafepointKind = RawPcDescriptors::kIcCall
                              | RawPcDescriptors::kUnoptStaticCall
                              | RawPcDescriptors::kRuntimeCall;
 
 
 CodeBreakpoint::CodeBreakpoint(const Code& code,
-                               intptr_t token_pos,
+                               TokenDescriptor token_pos,
                                uword pc,
                                RawPcDescriptors::Kind kind)
     : code_(code.raw()),
       token_pos_(token_pos),
       pc_(pc),
       line_number_(-1),
       is_enabled_(false),
       bpt_location_(NULL),
       next_(NULL),
       breakpoint_kind_(kind),
       saved_value_(Code::null()) {
   ASSERT(!code.IsNull());
-  ASSERT(token_pos_ > 0);
+  ASSERT(token_pos_.IsReal());
   ASSERT(pc_ != 0);
   ASSERT((breakpoint_kind_ & kSafepointKind) != 0);
 }
 
 
 CodeBreakpoint::~CodeBreakpoint() {
   // Make sure we don't leave patched code behind.
   ASSERT(!IsEnabled());
   // Poison the data so we catch use after free errors.
 #ifdef DEBUG
@@ skipping 470 matching lines @@
   event.set_exception(&exc);
   ASSERT(stack_trace->Length() > 0);
   event.set_top_frame(stack_trace->FrameAt(0));
   ASSERT(stack_trace_ == NULL);
   stack_trace_ = stack_trace;
   Pause(&event);
   stack_trace_ = NULL;
 }
 
 
-static intptr_t LastTokenOnLine(const TokenStream& tokens, intptr_t pos) {
-  TokenStream::Iterator iter(tokens, pos, TokenStream::Iterator::kAllTokens);
+static TokenDescriptor LastTokenOnLine(const TokenStream& tokens,
+                                       TokenDescriptor pos) {
+  TokenStream::Iterator iter(tokens,
+                             pos.value(),
+                             TokenStream::Iterator::kAllTokens);
   ASSERT(iter.IsValid());
-  intptr_t last_pos = pos;
+  TokenDescriptor last_pos = pos;
   while ((iter.CurrentTokenKind() != Token::kNEWLINE) &&
-      (iter.CurrentTokenKind() != Token::kEOS)) {
-    last_pos = iter.CurrentPosition();
+         (iter.CurrentTokenKind() != Token::kEOS)) {
+    last_pos = TokenDescriptor(iter.CurrentPosition());
     iter.Advance();
   }
   return last_pos;
 }
 
 
 // Returns the best fit token position for a breakpoint.
 //
 // Takes a range of tokens [requested_token_pos, last_token_pos] and
 // an optional column (requested_column).  The range of tokens usually
@@ skipping 42 matching lines @@
 //
 // If no best fit token can be found, the search is expanded,
 // searching through the rest of the current function by calling this
 // function recursively.
 //
 // TODO(turnidge): Given that we usually call this function with a
 // token range restricted to a single line, this could be a one-pass
 // algorithm, which would be simpler.  I believe that it only needs
 // two passes to support the recursive try-the-whole-function case.
 // Rewrite this later, once there are more tests in place.
-intptr_t Debugger::ResolveBreakpointPos(const Function& func,
-                                        intptr_t requested_token_pos,
-                                        intptr_t last_token_pos,
-                                        intptr_t requested_column) {
+TokenDescriptor Debugger::ResolveBreakpointPos(
+    const Function& func,
+    TokenDescriptor requested_token_pos,
+    TokenDescriptor last_token_pos,
+    intptr_t requested_column) {
   ASSERT(func.HasCode());
   ASSERT(!func.HasOptimizedCode());
 
   if (requested_token_pos < func.token_pos()) {
     requested_token_pos = func.token_pos();
   }
   if (last_token_pos > func.end_token_pos()) {
     last_token_pos = func.end_token_pos();
   }
 
   Script& script = Script::Handle(func.script());
   Code& code = Code::Handle(func.unoptimized_code());
   ASSERT(!code.IsNull());
   PcDescriptors& desc = PcDescriptors::Handle(code.pc_descriptors());
 
   // First pass: find the safe point which is closest to the beginning
   // of the given token range.
-  intptr_t best_fit_pos = INT_MAX;
+  TokenDescriptor best_fit_pos = TokenDescriptor::kMaxSource;
   intptr_t best_column = INT_MAX;
   PcDescriptors::Iterator iter(desc, kSafepointKind);
   while (iter.MoveNext()) {
-    const intptr_t pos = iter.TokenPos();
-    if ((!Token::IsReal(pos)) ||
+    const TokenDescriptor pos = iter.TokenPos();
+    if ((!pos.IsReal()) ||
         (pos < requested_token_pos) ||
         (pos > last_token_pos)) {
       // Token is not in the target range.
       continue;
     }
 
     intptr_t token_start_column = -1;
     if (requested_column >= 0) {
       intptr_t ignored = -1;
       intptr_t token_len = -1;
@@ skipping 16 matching lines @@
     // Prefer the lowest (first) token pos.
     if (pos < best_fit_pos) {
       best_fit_pos = pos;
       best_column = token_start_column;
     }
   }
 
   // Second pass (if we found a safe point in the first pass).  Find
   // the token on the line which is at the best fit column (if column
   // was specified) and has the lowest code address.
-  if (best_fit_pos != INT_MAX) {
+  if (best_fit_pos != TokenDescriptor::kMaxSource) {
     const Script& script = Script::Handle(func.script());
     const TokenStream& tokens = TokenStream::Handle(script.tokens());
-    const intptr_t begin_pos = best_fit_pos;
-    const intptr_t end_of_line_pos = LastTokenOnLine(tokens, begin_pos);
+    const TokenDescriptor begin_pos = best_fit_pos;
+    const TokenDescriptor end_of_line_pos = LastTokenOnLine(tokens, begin_pos);
     uword lowest_pc_offset = kUwordMax;
     PcDescriptors::Iterator iter(desc, kSafepointKind);
     while (iter.MoveNext()) {
-      const intptr_t pos = iter.TokenPos();
-      if ((!Token::IsReal(pos)) ||
+      const TokenDescriptor pos = iter.TokenPos();
+      if ((!TokenDescriptor(pos).IsReal()) ||
           (pos < begin_pos) ||
           (pos > end_of_line_pos)) {
         // Token is not on same line as best fit.
         continue;
       }
 
       if (requested_column >= 0) {
         intptr_t ignored = -1;
         intptr_t token_start_column = -1;
         // We look for other tokens at the best column in case there
@@ skipping 14 matching lines @@
   }
 
   // We didn't find a safe point in the given token range. Try and
   // find a safe point in the remaining source code of the function.
   // Since we have moved to the next line of the function, we no
   // longer are requesting a specific column number.
   if (last_token_pos < func.end_token_pos()) {
     return ResolveBreakpointPos(func, last_token_pos, func.end_token_pos(),
                                 -1 /* no column */);
   }
-  return Token::kNoSourcePos;
+  return TokenDescriptor::kNoSource;
 }
 
 
 void Debugger::MakeCodeBreakpointAt(const Function& func,
                                     BreakpointLocation* loc) {
-  ASSERT(Token::IsReal(loc->token_pos_));
+  ASSERT(TokenDescriptor(loc->token_pos_).IsReal());
   ASSERT((loc != NULL) && loc->IsResolved());
   ASSERT(!func.HasOptimizedCode());
   Code& code = Code::Handle(func.unoptimized_code());
   ASSERT(!code.IsNull());
   PcDescriptors& desc = PcDescriptors::Handle(code.pc_descriptors());
   uword lowest_pc_offset = kUwordMax;
   RawPcDescriptors::Kind lowest_kind = RawPcDescriptors::kAnyKind;
   // Find the safe point with the lowest compiled code address
   // that maps to the token position of the source breakpoint.
   PcDescriptors::Iterator iter(desc, kSafepointKind);
@@ skipping 17 matching lines @@
     RegisterCodeBreakpoint(code_bpt);
   }
   code_bpt->set_bpt_location(loc);
   if (loc->AnyEnabled()) {
     code_bpt->Enable();
   }
 }
 
 
 void Debugger::FindCompiledFunctions(const Script& script,
-                                     intptr_t start_pos,
-                                     intptr_t end_pos,
+                                     TokenDescriptor start_pos,
+                                     TokenDescriptor end_pos,
                                      GrowableObjectArray* function_list) {
   Zone* zone = Thread::Current()->zone();
   Class& cls = Class::Handle(zone);
   Array& functions = Array::Handle(zone);
   GrowableObjectArray& closures = GrowableObjectArray::Handle(zone);
   Function& function = Function::Handle(zone);
 
   closures = isolate_->object_store()->closure_functions();
   const intptr_t num_closures = closures.Length();
   for (intptr_t pos = 0; pos < num_closures; pos++) {
@@ skipping 63 matching lines @@
   } else {
     if ((func->token_pos() > best_fit->token_pos()) &&
         ((func->end_token_pos() <= best_fit->end_token_pos()))) {
       *best_fit = func->raw();
     }
   }
 }
 
 
 RawFunction* Debugger::FindBestFit(const Script& script,
-                                   intptr_t token_pos) {
+                                   TokenDescriptor token_pos) {
   Zone* zone = Thread::Current()->zone();
   Class& cls = Class::Handle(zone);
   Array& functions = Array::Handle(zone);
   GrowableObjectArray& closures = GrowableObjectArray::Handle(zone);
   Function& function = Function::Handle(zone);
   Function& best_fit = Function::Handle(zone);
   Error& error = Error::Handle(zone);
 
   closures = isolate_->object_store()->closure_functions();
   const intptr_t num_closures = closures.Length();
@@ skipping 38 matching lines @@
           }
         }
       }
     }
   }
   return best_fit.raw();
 }
 
 
 BreakpointLocation* Debugger::SetBreakpoint(const Script& script,
-                                            intptr_t token_pos,
-                                            intptr_t last_token_pos,
+                                            TokenDescriptor token_pos,
+                                            TokenDescriptor last_token_pos,
                                             intptr_t requested_line,
                                             intptr_t requested_column) {
   Function& func = Function::Handle();
   func = FindBestFit(script, token_pos);
   if (func.IsNull()) {
     return NULL;
   }
   // There may be more than one function object for a given function
   // in source code. There may be implicit closure functions, and
   // there may be copies of mixin functions. Collect all compiled
   // functions whose source code range matches exactly the best fit
   // function we found.
   GrowableObjectArray& functions =
       GrowableObjectArray::Handle(GrowableObjectArray::New());
   FindCompiledFunctions(script,
                         func.token_pos(),
                         func.end_token_pos(),
                         &functions);
 
   if (functions.Length() > 0) {
     // One or more function object containing this breakpoint location
     // have already been compiled. We can resolve the breakpoint now.
     DeoptimizeWorld();
     func ^= functions.At(0);
-    intptr_t breakpoint_pos =
+    TokenDescriptor breakpoint_pos =
         ResolveBreakpointPos(func, token_pos, last_token_pos, requested_column);
-    if (breakpoint_pos >= 0) {
+    if (breakpoint_pos >= TokenDescriptor::kMinSource) {
       BreakpointLocation* bpt =
           GetBreakpointLocation(script, breakpoint_pos, requested_column);
       if (bpt != NULL) {
         // A source breakpoint for this location already exists.
         return bpt;
       }
       bpt = new BreakpointLocation(script, token_pos, last_token_pos,
                                    requested_line, requested_column);
       bpt->SetResolved(func, breakpoint_pos);
       RegisterBreakpointLocation(bpt);
@@ skipping 182 matching lines @@
     }
     return latent_bpt;
   }
   if (scripts.Length() > 1) {
     if (FLAG_verbose_debug) {
       OS::Print("Multiple scripts match url '%s'\n", script_url.ToCString());
     }
     return NULL;
   }
   script ^= scripts.At(0);
-  intptr_t first_token_idx, last_token_idx;
+  TokenDescriptor first_token_idx, last_token_idx;
   script.TokenRangeAtLine(line_number, &first_token_idx, &last_token_idx);
-  if (!Token::IsReal(first_token_idx)) {
+  if (!TokenDescriptor(first_token_idx).IsReal()) {
     // Script does not contain the given line number.
     if (FLAG_verbose_debug) {
       OS::Print("Script '%s' does not contain line number %" Pd "\n",
                 script_url.ToCString(), line_number);
     }
     return NULL;
-  } else if (!Token::IsReal(last_token_idx)) {
+  } else if (!TokenDescriptor(last_token_idx).IsReal()) {
     // Line does not contain any tokens.
     if (FLAG_verbose_debug) {
       OS::Print("No executable code at line %" Pd " in '%s'\n",
                 line_number, script_url.ToCString());
     }
     return NULL;
   }
 
   BreakpointLocation* bpt = NULL;
   ASSERT(first_token_idx <= last_token_idx);
   while ((bpt == NULL) && (first_token_idx <= last_token_idx)) {
     bpt = SetBreakpoint(script, first_token_idx, last_token_idx,
                         line_number, column_number);
-    first_token_idx++;
+    first_token_idx = TokenDescriptor(first_token_idx.value() + 1);
   }
   if ((bpt == NULL) && FLAG_verbose_debug) {
     OS::Print("No executable code at line %" Pd " in '%s'\n",
                 line_number, script_url.ToCString());
   }
   return bpt;
 }
 
 
 intptr_t Debugger::CacheObject(const Object& obj) {
@@ skipping 325 matching lines @@
   DebuggerEvent event(isolate_, DebuggerEvent::kBreakpointReached);
   event.set_top_frame(top_frame);
   event.set_breakpoint(bpt);
   Object& closure_or_null = Object::Handle(top_frame->GetAsyncOperation());
   if (!closure_or_null.IsNull()) {
     ASSERT(closure_or_null.IsInstance());
     ASSERT(Instance::Cast(closure_or_null).IsClosure());
     event.set_async_continuation(&closure_or_null);
     const Script& script = Script::Handle(top_frame->SourceScript());
     const TokenStream& tokens = TokenStream::Handle(script.tokens());
-    TokenStream::Iterator iter(tokens, top_frame->TokenPos());
+    TokenStream::Iterator iter(tokens, top_frame->TokenPos().value());
     if ((iter.CurrentTokenKind() == Token::kIDENT) &&
         ((iter.CurrentLiteral() == Symbols::Await().raw()) ||
          (iter.CurrentLiteral() == Symbols::YieldKw().raw()))) {
       event.set_at_async_jump(true);
     }
   }
   Pause(&event);
 }
 
 
@@ skipping 27 matching lines @@
       // We returned from the "interesting frame", there can be no more
       // stepping breaks for it. Pause at the next appropriate location
       // and let the user set the "interesting" frame again.
       stepping_fp_ = 0;
     }
   }
 
   if (!frame->IsDebuggable()) {
     return Error::null();
   }
-  if (!Token::IsDebugPause(frame->TokenPos())) {
+  if (!TokenDescriptor(frame->TokenPos()).IsDebugPause()) {
     return Error::null();
   }
 
   // If there is an active breakpoint at this pc, then we should have
   // already bailed out of this function in the skip_next_step_ test
   // above.
   ASSERT(!HasActiveBreakpoint(frame->pc()));
 
   if (FLAG_verbose_debug) {
     OS::Print(">>> single step break at %s:%" Pd " (func %s token %" Pd ")\n",
               String::Handle(frame->SourceUrl()).ToCString(),
               frame->LineNumber(),
               String::Handle(frame->QualifiedFunctionName()).ToCString(),
-              frame->TokenPos());
+              frame->TokenPos().value());
   }
 
   ASSERT(stack_trace_ == NULL);
   stack_trace_ = CollectStackTrace();
   SignalPausedEvent(frame, NULL);
   HandleSteppingRequest(stack_trace_);
   stack_trace_ = NULL;
 
   // If any error occurred while in the debug message loop, return it here.
   const Error& error =
@@ skipping 64 matching lines @@
   if (bpt_hit == NULL) {
     return Error::null();
   }
 
   if (FLAG_verbose_debug) {
     OS::Print(">>> hit %s breakpoint at %s:%" Pd " "
               "(token %" Pd ") (address %#" Px ")\n",
               cbpt->IsInternal() ? "internal" : "user",
               String::Handle(cbpt->SourceUrl()).ToCString(),
               cbpt->LineNumber(),
-              cbpt->token_pos(),
+              cbpt->token_pos().value(),
               top_frame->pc());
   }
 
   ASSERT(stack_trace_ == NULL);
   stack_trace_ = stack_trace;
   SignalPausedEvent(top_frame, bpt_hit);
   // When we single step from a user breakpoint, our next stepping
   // point will be at the exact same pc.  Skip it.
   HandleSteppingRequest(stack_trace_, true /* skip next step */);
   stack_trace_ = NULL;
@@ skipping 58 matching lines @@
       !ServiceIsolate::IsServiceIsolateDescendant(isolate_)) {
     SignalIsolateEvent(DebuggerEvent::kIsolateCreated);
     creation_message_sent_ = true;
   }
 }
 
 
 // Return innermost closure contained in 'function' that contains
 // the given token position.
 RawFunction* Debugger::FindInnermostClosure(const Function& function,
-                                            intptr_t token_pos) {
+                                            TokenDescriptor token_pos) {
   Zone* zone = Thread::Current()->zone();
   const Script& outer_origin = Script::Handle(zone, function.script());
   const GrowableObjectArray& closures =
      GrowableObjectArray::Handle(zone,
          Isolate::Current()->object_store()->closure_functions());
   const intptr_t num_closures = closures.Length();
   Function& closure = Function::Handle(zone);
   Function& best_fit = Function::Handle(zone);
   for (intptr_t i = 0; i < num_closures; i++) {
     closure ^= closures.At(i);
@@ skipping 44 matching lines @@
 
       // TODO(hausner): What should we do if function is optimized?
       // Can we deoptimize the function?
       ASSERT(!func.HasOptimizedCode());
 
       // There is no local function within func that contains the
       // breakpoint token position. Resolve the breakpoint if necessary
       // and set the code breakpoints.
       if (!loc->IsResolved()) {
         // Resolve source breakpoint in the newly compiled function.
-        intptr_t bp_pos =
+        TokenDescriptor bp_pos =
             ResolveBreakpointPos(func, loc->token_pos(), loc->end_token_pos(),
                                  loc->requested_column_number());
-        if (!Token::IsDebugPause(bp_pos)) {
+        if (!TokenDescriptor(bp_pos).IsDebugPause()) {
           if (FLAG_verbose_debug) {
             OS::Print("Failed resolving breakpoint for function '%s'\n",
                       String::Handle(func.name()).ToCString());
           }
           continue;
         }
-        intptr_t requested_pos = loc->token_pos();
-        intptr_t requested_end_pos = loc->end_token_pos();
+        TokenDescriptor requested_pos = loc->token_pos();
+        TokenDescriptor requested_end_pos = loc->end_token_pos();
         loc->SetResolved(func, bp_pos);
         Breakpoint* bpt = loc->breakpoints();
         while (bpt != NULL) {
           if (FLAG_verbose_debug) {
             OS::Print("Resolved BP %" Pd " to pos %" Pd ", "
                       "line %" Pd " col %" Pd ", "
                       "function '%s' (requested range %" Pd "-%" Pd ", "
                       "requested col %" Pd ")\n",
                       bpt->id(),
-                      loc->token_pos(),
+                      loc->token_pos().value(),
                       loc->LineNumber(),
                       loc->ColumnNumber(),
                       func.ToFullyQualifiedCString(),
-                      requested_pos,
-                      requested_end_pos,
+                      requested_pos.value(),
+                      requested_end_pos.value(),
                       loc->requested_column_number());
           }
           SignalBpResolved(bpt);
           SendServiceBreakpointEvent(ServiceEvent::kBreakpointResolved, bpt);
           bpt = bpt->next();
         }
       }
       ASSERT(loc->IsResolved());
       if (FLAG_verbose_debug) {
         Breakpoint* bpt = loc->breakpoints();
@@ skipping 42 matching lines @@
         if (prev_loc == NULL) {
           latent_locations_ = loc;
         } else {
           prev_loc->set_next(loc);
         }
         // Now find the token range at the requested line and make a
         // new unresolved source breakpoint.
         intptr_t line_number = matched_loc->requested_line_number();
         intptr_t column_number = matched_loc->requested_column_number();
         ASSERT(line_number >= 0);
-        intptr_t first_token_pos, last_token_pos;
+        TokenDescriptor first_token_pos, last_token_pos;
         script.TokenRangeAtLine(line_number, &first_token_pos, &last_token_pos);
-        if (!Token::IsDebugPause(first_token_pos) ||
-            !Token::IsDebugPause(last_token_pos)) {
+        if (!first_token_pos.IsDebugPause() || !last_token_pos.IsDebugPause()) {
           // Script does not contain the given line number or there are no
           // tokens on the line. Drop the breakpoint silently.
           Breakpoint* bpt = matched_loc->breakpoints();
           while (bpt != NULL) {
             if (FLAG_verbose_debug) {
               OS::Print("No code found at line %" Pd ": "
                         "dropping latent breakpoint %" Pd " in '%s'\n",
                         line_number,
                         bpt->id(),
                         url.ToCString());
@@ skipping 187 matching lines @@
       delete temp_bpt;
     } else {
       prev_bpt = curr_bpt;
       curr_bpt = curr_bpt->next();
     }
   }
 }
 
 
 BreakpointLocation* Debugger::GetBreakpointLocation(const Script& script,
-                                                    intptr_t token_pos,
+                                                    TokenDescriptor token_pos,
                                                     intptr_t requested_column) {
   BreakpointLocation* bpt = breakpoint_locations_;
   while (bpt != NULL) {
     if ((bpt->script_ == script.raw()) &&
         (bpt->token_pos_ == token_pos) &&
         (bpt->requested_column_number_ == requested_column)) {
       return bpt;
     }
     bpt = bpt->next();
   }
@@ skipping 46 matching lines @@
 }
 
 
 void Debugger::RegisterCodeBreakpoint(CodeBreakpoint* bpt) {
   ASSERT(bpt->next() == NULL);
   bpt->set_next(code_breakpoints_);
   code_breakpoints_ = bpt;
 }
 
 }  // namespace dart