Replace intptr_t with TokenDescriptor

runtime/vm/object.cc

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/object.h"
 
 #include "include/dart_api.h"
 #include "platform/assert.h"
 #include "vm/assembler.h"
 #include "vm/cpu.h"
@@ skipping 1166 matching lines @@
   cls = object_store->array_class();  // Was allocated above.
   RegisterPrivateClass(cls, Symbols::_List(), core_lib);
   pending_classes.Add(cls);
   // We cannot use NewNonParameterizedType(cls), because Array is parameterized.
   // Warning: class _List has not been patched yet. Its declared number of type
   // parameters is still 0. It will become 1 after patching. The array type
   // allocated below represents the raw type _List and not _List<E> as we
   // could expect. Use with caution.
   type ^= Type::New(Object::Handle(zone, cls.raw()),
                     TypeArguments::Handle(zone),
-                    Token::kNoSourcePos);
+                    TokenDescriptor::kNoSource);
   type.SetIsFinalized();
   type ^= type.Canonicalize();
   object_store->set_array_type(type);
 
   cls = object_store->growable_object_array_class();  // Was allocated above.
   RegisterPrivateClass(cls, Symbols::_GrowableList(), core_lib);
   pending_classes.Add(cls);
 
   cls = Class::New<Array>(kImmutableArrayCid);
   object_store->set_immutable_array_class(cls);
@@ skipping 380 matching lines @@
   Smi& value = Smi::Handle(zone);
   String& field_name = String::Handle(zone);
 
 #define CLASS_LIST_WITH_NULL(V)                                                \
   V(Null)                                                                      \
   CLASS_LIST_NO_OBJECT(V)
 
 #define ADD_SET_FIELD(clazz)                                                   \
   field_name = Symbols::New("cid"#clazz);                                      \
   field = Field::New(field_name, true, false, true, false, cls,                \
-      Type::Handle(Type::IntType()), 0);                                       \
+      Type::Handle(Type::IntType()), TokenDescriptor::kMinSource);             \
   value = Smi::New(k##clazz##Cid);                                             \
   field.SetStaticValue(value, true);                                           \
   cls.AddField(field);                                                         \
 
   CLASS_LIST_WITH_NULL(ADD_SET_FIELD)
 #undef ADD_SET_FIELD
 
   isolate->object_store()->InitKnownObjects();
 
   return Error::null();
@@ skipping 365 matching lines @@
 bool Class::IsInFullSnapshot() const {
   NoSafepointScope no_safepoint;
   return raw_ptr()->library_->ptr()->is_in_fullsnapshot_;
 }
 
 
 RawAbstractType* Class::RareType() const {
   const Type& type = Type::Handle(Type::New(
       *this,
       Object::null_type_arguments(),
-      Token::kNoSourcePos));
+      TokenDescriptor::kNoSource));
   return ClassFinalizer::FinalizeType(*this,
                                       type,
                                       ClassFinalizer::kCanonicalize);
 }
 
 
 RawAbstractType* Class::DeclarationType() const {
   const TypeArguments& args = TypeArguments::Handle(type_parameters());
   const Type& type = Type::Handle(Type::New(
       *this,
       args,
-      Token::kNoSourcePos));
+      TokenDescriptor::kNoSource));
   return ClassFinalizer::FinalizeType(*this,
                                       type,
                                       ClassFinalizer::kCanonicalize);
 }
 
 
 template <class FakeObject>
 RawClass* Class::New() {
   ASSERT(Object::class_class() != Class::null());
   Class& result = Class::Handle();
@@ skipping 17 matching lines @@
     result.set_is_finalized();
   } else {
     // VM backed classes are almost ready: run checks and resolve class
     // references, but do not recompute size.
     result.set_is_prefinalized();
   }
   result.set_type_arguments_field_offset_in_words(kNoTypeArguments);
   result.set_num_type_arguments(0);
   result.set_num_own_type_arguments(0);
   result.set_num_native_fields(0);
-  result.set_token_pos(Token::kNoSourcePos);
+  result.set_token_pos(TokenDescriptor::kNoSource);
   result.InitEmptyFields();
   Isolate::Current()->RegisterClass(result);
   return result.raw();
 }
 
 
 
 static void ReportTooManyTypeArguments(const Class& cls) {
   Report::MessageF(Report::kError,
                    Script::Handle(cls.script()),
@@ skipping 579 matching lines @@
                                                RawFunction::Kind kind) const {
   Function& invocation = Function::Handle(
       Function::New(String::Handle(Symbols::New(target_name)),
                     kind,
                     false,  // Not static.
                     false,  // Not const.
                     false,  // Not abstract.
                     false,  // Not external.
                     false,  // Not native.
                     *this,
-                    0));    // token_pos
+                    TokenDescriptor::kMinSource));
   ArgumentsDescriptor desc(args_desc);
   invocation.set_num_fixed_parameters(desc.PositionalCount());
   invocation.SetNumOptionalParameters(desc.NamedCount(),
                                       false);  // Not positional.
   invocation.set_parameter_types(Array::Handle(Array::New(desc.Count(),
                                                           Heap::kOld)));
   invocation.set_parameter_names(Array::Handle(Array::New(desc.Count(),
                                                           Heap::kOld)));
   // Receiver.
   invocation.SetParameterTypeAt(0, Object::dynamic_type());
@@ skipping 36 matching lines @@
   const Class& owner = Class::Handle(closure_function.Owner());
   Function& extractor = Function::Handle(
     Function::New(String::Handle(Symbols::New(getter_name)),
                   RawFunction::kMethodExtractor,
                   false,  // Not static.
                   false,  // Not const.
                   false,  // Not abstract.
                   false,  // Not external.
                   false,  // Not native.
                   owner,
-                  ClassifyingTokenPositions::kMethodExtractor));  // token_pos
+                  TokenDescriptor::kMethodExtractor));
 
   // Initialize signature: receiver is a single fixed parameter.
   const intptr_t kNumParameters = 1;
   extractor.set_num_fixed_parameters(kNumParameters);
   extractor.SetNumOptionalParameters(0, 0);
   extractor.set_parameter_types(Object::extractor_parameter_types());
   extractor.set_parameter_names(Object::extractor_parameter_names());
   extractor.set_result_type(Object::dynamic_type());
 
   extractor.set_extracted_method_closure(closure_function);
@@ skipping 392 matching lines @@
   ASSERT(fake.IsInstance());
   result.set_handle_vtable(fake.vtable());
   result.set_instance_size(FakeInstance::InstanceSize());
   result.set_next_field_offset(FakeInstance::NextFieldOffset());
   result.set_id(index);
   result.set_state_bits(0);
   result.set_type_arguments_field_offset_in_words(kNoTypeArguments);
   result.set_num_type_arguments(kUnknownNumTypeArguments);
   result.set_num_own_type_arguments(kUnknownNumTypeArguments);
   result.set_num_native_fields(0);
-  result.set_token_pos(Token::kNoSourcePos);
+  result.set_token_pos(TokenDescriptor::kNoSource);
   result.InitEmptyFields();
   Isolate::Current()->RegisterClass(result);
   return result.raw();
 }
 
 
 RawClass* Class::New(const String& name,
                      const Script& script,
-                     intptr_t token_pos) {
+                     TokenDescriptor token_pos) {
   Class& result = Class::Handle(New<Instance>(kIllegalCid));
   result.set_name(name);
   result.set_script(script);
   result.set_token_pos(token_pos);
   return result.raw();
 }
 
 
 RawClass* Class::NewNativeWrapper(const Library& library,
                                   const String& name,
                                   int field_count) {
   Class& cls = Class::Handle(library.LookupClass(name));
   if (cls.IsNull()) {
-    cls = New(name, Script::Handle(), Token::kNoSourcePos);
+    cls = New(name, Script::Handle(), TokenDescriptor::kNoSource);
     cls.SetFields(Object::empty_array());
     cls.SetFunctions(Object::empty_array());
     // Set super class to Object.
     cls.set_super_type(Type::Handle(Type::ObjectType()));
     // Compute instance size. First word contains a pointer to a properly
     // sized typed array once the first native field has been set.
     intptr_t instance_size = sizeof(RawInstance) + kWordSize;
     cls.set_instance_size(RoundedAllocationSize(instance_size));
     cls.set_next_field_offset(instance_size);
     cls.set_num_native_fields(field_count);
@@ skipping 224 matching lines @@
   }
   UNREACHABLE();
 }
 
 
 void Class::set_script(const Script& value) const {
   StorePointer(&raw_ptr()->script_, value.raw());
 }
 
 
-void Class::set_token_pos(intptr_t token_pos) const {
-  ASSERT(!Token::IsClassifying(token_pos));
+void Class::set_token_pos(TokenDescriptor token_pos) const {
+  ASSERT(!TokenDescriptor(token_pos).IsClassifying());
   StoreNonPointer(&raw_ptr()->token_pos_, token_pos);
 }
 
 
-intptr_t Class::ComputeEndTokenPos() const {
+TokenDescriptor Class::ComputeEndTokenPos() const {
   // Return the begin token for synthetic classes.
   if (IsMixinApplication() || IsTopLevel()) {
     return token_pos();
   }
   const Script& scr = Script::Handle(script());
   ASSERT(!scr.IsNull());
   const TokenStream& tkns = TokenStream::Handle(scr.tokens());
-  TokenStream::Iterator tkit(
-      tkns, token_pos(), TokenStream::Iterator::kNoNewlines);
+  TokenStream::Iterator tkit(tkns,
+                             token_pos().value(),
+                             TokenStream::Iterator::kNoNewlines);
   intptr_t level = 0;
   while (tkit.CurrentTokenKind() != Token::kEOS) {
     if (tkit.CurrentTokenKind() == Token::kLBRACE) {
       level++;
     } else if (tkit.CurrentTokenKind() == Token::kRBRACE) {
       if (--level == 0) {
-        return tkit.CurrentPosition();
+        return TokenDescriptor(tkit.CurrentPosition());
       }
     }
     tkit.Advance();
   }
   UNREACHABLE();
-  return 0;
+  return TokenDescriptor::kNoSource;
 }
 
 
 void Class::set_is_implemented() const {
   set_state_bits(ImplementedBit::update(true, raw_ptr()->state_bits_));
 }
 
 
 void Class::set_is_abstract() const {
   set_state_bits(AbstractBit::update(true, raw_ptr()->state_bits_));
@@ skipping 838 matching lines @@
     set_constants(new_canonical_list);
     new_canonical_list.SetAt(index, constant);
   } else {
     canonical_list.SetAt(index, constant);
   }
 }
 
 
 RawUnresolvedClass* UnresolvedClass::New(const LibraryPrefix& library_prefix,
                                          const String& ident,
-                                         intptr_t token_pos) {
+                                         TokenDescriptor token_pos) {
   const UnresolvedClass& type = UnresolvedClass::Handle(UnresolvedClass::New());
   type.set_library_prefix(library_prefix);
   type.set_ident(ident);
   type.set_token_pos(token_pos);
   return type.raw();
 }
 
 
 RawUnresolvedClass* UnresolvedClass::New() {
   ASSERT(Object::unresolved_class_class() != Class::null());
   RawObject* raw = Object::Allocate(UnresolvedClass::kClassId,
                                     UnresolvedClass::InstanceSize(),
                                     Heap::kOld);
   return reinterpret_cast<RawUnresolvedClass*>(raw);
 }
 
 
-void UnresolvedClass::set_token_pos(intptr_t token_pos) const {
-  ASSERT(!Token::IsClassifying(token_pos));
+void UnresolvedClass::set_token_pos(TokenDescriptor token_pos) const {
+  ASSERT(!TokenDescriptor(token_pos).IsClassifying());
   StoreNonPointer(&raw_ptr()->token_pos_, token_pos);
 }
 
 
 void UnresolvedClass::set_ident(const String& ident) const {
   StorePointer(&raw_ptr()->ident_, ident.raw());
 }
 
 
 void UnresolvedClass::set_library_prefix(
@@ skipping 973 matching lines @@
 RawScript* Function::eval_script() const {
   const Object& obj = Object::Handle(raw_ptr()->data_);
   if (obj.IsScript()) {
     return Script::Cast(obj).raw();
   }
   return Script::null();
 }
 
 
 void Function::set_eval_script(const Script& script) const {
-  ASSERT(token_pos() == 0);
+  ASSERT(token_pos() == TokenDescriptor::kMinSource);
   ASSERT(raw_ptr()->data_ == Object::null());
   set_data(script);
 }
 
 
 RawFunction* Function::extracted_method_closure() const {
   ASSERT(kind() == RawFunction::kMethodExtractor);
   const Object& obj = Object::Handle(raw_ptr()->data_);
   ASSERT(obj.IsFunction());
   return Function::Cast(obj).raw();
@@ skipping 129 matching lines @@
     // class may be. In this case, the scope class of the function type is reset
     // to _Closure class as well as the owner of the signature function.
     Class& scope_class = Class::Handle(Owner());
     if (!scope_class.IsTypedefClass() &&
         (is_static() ||
          !scope_class.IsGeneric() ||
          HasInstantiatedSignature())) {
       scope_class = Isolate::Current()->object_store()->closure_class();
       if (IsSignatureFunction()) {
         set_owner(scope_class);
-        set_token_pos(Token::kNoSourcePos);
+        set_token_pos(TokenDescriptor::kNoSource);
       }
     }
     const TypeArguments& signature_type_arguments =
         TypeArguments::Handle(scope_class.type_parameters());
     // Return the still unfinalized signature type.
     type = FunctionType::New(scope_class,
                              signature_type_arguments,
                              *this,
                              token_pos());
 
@@ skipping 271 matching lines @@
 }
 
 
 void Function::set_recognized_kind(MethodRecognizer::Kind value) const {
   // Prevent multiple settings of kind.
   ASSERT((value == MethodRecognizer::kUnknown) || !IsRecognized());
   set_kind_tag(RecognizedBits::update(value, raw_ptr()->kind_tag_));
 }
 
 
-void Function::set_token_pos(intptr_t token_pos) const {
-  ASSERT(!Token::IsClassifying(token_pos) || IsMethodExtractor());
+void Function::set_token_pos(TokenDescriptor token_pos) const {
+  ASSERT(!TokenDescriptor(token_pos).IsClassifying() || IsMethodExtractor());
   StoreNonPointer(&raw_ptr()->token_pos_, token_pos);
 }
 
 
 void Function::set_kind_tag(intptr_t value) const {
   StoreNonPointer(&raw_ptr()->kind_tag_, static_cast<uint32_t>(value));
 }
 
 
 void Function::set_num_fixed_parameters(intptr_t value) const {
@@ skipping 23 matching lines @@
 
 bool Function::IsOptimizable() const {
   if (FLAG_coverage_dir != NULL) {
     // Do not optimize if collecting coverage data.
     return false;
   }
   if (is_native()) {
     // Native methods don't need to be optimized.
     return false;
   }
+  const intptr_t function_length =
+      end_token_pos().value() - token_pos().value();
   if (is_optimizable() && (script() != Script::null()) &&
-      ((end_token_pos() - token_pos()) < FLAG_huge_method_cutoff_in_tokens)) {
+      (function_length < FLAG_huge_method_cutoff_in_tokens)) {
     // Additional check needed for implicit getters.
     return (unoptimized_code() == Object::null()) ||
         (Code::Handle(unoptimized_code()).Size() <
             FLAG_huge_method_cutoff_in_code_size);
   }
   return false;
 }
 
 
 bool Function::IsNativeAutoSetupScope() const {
@@ skipping 553 matching lines @@
 
 
 RawFunction* Function::New(const String& name,
                            RawFunction::Kind kind,
                            bool is_static,
                            bool is_const,
                            bool is_abstract,
                            bool is_external,
                            bool is_native,
                            const Object& owner,
-                           intptr_t token_pos) {
+                           TokenDescriptor token_pos) {
   ASSERT(!owner.IsNull());
   const Function& result = Function::Handle(Function::New());
   result.set_parameter_types(Object::empty_array());
   result.set_parameter_names(Object::empty_array());
   result.set_name(name);
   result.set_kind(kind);
   result.set_recognized_kind(MethodRecognizer::kUnknown);
   result.set_modifier(RawFunction::kNoModifier);
   result.set_is_static(is_static);
   result.set_is_const(is_const);
@@ skipping 59 matching lines @@
       type ^= type.CloneUninstantiated(new_owner);
       clone.SetParameterTypeAt(i, type);
     }
   }
   return clone.raw();
 }
 
 
 RawFunction* Function::NewClosureFunction(const String& name,
                                           const Function& parent,
-                                          intptr_t token_pos) {
+                                          TokenDescriptor token_pos) {
   ASSERT(!parent.IsNull());
   // Use the owner defining the parent function and not the class containing it.
   const Object& parent_owner = Object::Handle(parent.raw_ptr()->owner_);
   ASSERT(!parent_owner.IsNull());
   const Function& result = Function::Handle(
       Function::New(name,
                     RawFunction::kClosureFunction,
                     /* is_static = */ parent.is_static(),
                     /* is_const = */ false,
                     /* is_abstract = */ false,
                     /* is_external = */ false,
                     parent.is_native(),
                     parent_owner,
                     token_pos));
   result.set_parent_function(parent);
   return result.raw();
 }
 
 
 RawFunction* Function::NewSignatureFunction(const Class& owner,
-                                            intptr_t token_pos) {
+                                            TokenDescriptor token_pos) {
   const Function& result = Function::Handle(Function::New(
       Symbols::AnonymousSignature(),
       RawFunction::kSignatureFunction,
       /* is_static = */ false,
       /* is_const = */ false,
       /* is_abstract = */ false,
       /* is_external = */ false,
       /* is_native = */ false,
       owner,  // Same as function type scope class.
       token_pos));
   result.set_is_reflectable(false);
   result.set_is_visible(false);
   result.set_is_debuggable(false);
   return result.raw();
 }
 
 
 RawFunction* Function::NewEvalFunction(const Class& owner,
                                        const Script& script,
                                        bool is_static) {
   const Function& result = Function::Handle(
       Function::New(String::Handle(Symbols::New(":Eval")),
                     RawFunction::kRegularFunction,
                     is_static,
                     /* is_const = */ false,
                     /* is_abstract = */ false,
                     /* is_external = */ false,
                     /* is_native = */ false,
                     owner,
-                    /* token_pos = */ 0));
+                    TokenDescriptor::kMinSource));
   ASSERT(!script.IsNull());
   result.set_is_debuggable(false);
   result.set_is_visible(true);
   result.set_eval_script(script);
   return result.raw();
 }
 
 RawFunction* Function::ImplicitClosureFunction() const {
   // Return the existing implicit closure function if any.
   if (implicit_closure_function() != Function::null()) {
@@ skipping 270 matching lines @@
   const Object& obj = Object::Handle(raw_ptr()->owner_);
   if (obj.IsClass()) {
     return Class::Cast(obj).raw();
   }
   ASSERT(obj.IsPatchClass());
   return PatchClass::Cast(obj).origin_class();
 }
 
 
 RawScript* Function::script() const {
-  if (token_pos() == 0) {
+  if (token_pos() == TokenDescriptor::kMinSource) {
     // Testing for position 0 is an optimization that relies on temporary
     // eval functions having token position 0.
     const Script& script = Script::Handle(eval_script());
     if (!script.IsNull()) {
       return script.raw();
     }
   }
   if (IsClosureFunction()) {
     return Function::Handle(parent_function()).script();
   }
@@ skipping 82 matching lines @@
     // relaxed. In particular we might start associating some source with the
     // forwarding constructors when it becomes possible to specify a particular
     // constructor from the mixin to use.
     return String::null();
   }
   const Script& func_script = Script::Handle(script());
   const TokenStream& stream = TokenStream::Handle(func_script.tokens());
   if (!func_script.HasSource()) {
     // When source is not available, avoid printing the whole token stream and
     // doing expensive position calculations.
-    return stream.GenerateSource(token_pos(), end_token_pos() + 1);
+    return stream.GenerateSource(token_pos(),
+                                 TokenDescriptor(end_token_pos().value() + 1));
   }
 
-  const TokenStream::Iterator tkit(stream, end_token_pos());
+  const TokenStream::Iterator tkit(stream, end_token_pos().value());
   intptr_t from_line;
   intptr_t from_col;
   intptr_t to_line;
   intptr_t to_col;
   func_script.GetTokenLocation(token_pos(), &from_line, &from_col);
   func_script.GetTokenLocation(end_token_pos(), &to_line, &to_col);
   intptr_t last_tok_len = String::Handle(tkit.CurrentLiteral()).Length();
   // Handle special cases for end tokens of closures (where we exclude the last
   // token):
   // (1) "foo(() => null, bar);": End token is `,', but we don't print it.
@@ skipping 16 matching lines @@
 // Construct fingerprint from token stream. The token stream contains also
 // arguments.
 int32_t Function::SourceFingerprint() const {
   uint32_t result = IsImplicitClosureFunction()
       ? String::Handle(Function::Handle(parent_function()).Signature()).Hash()
       : String::Handle(Signature()).Hash();
   TokenStream::Iterator tokens_iterator(TokenStream::Handle(
       Script::Handle(script()).tokens()), token_pos());
   Object& obj = Object::Handle();
   String& literal = String::Handle();
-  while (tokens_iterator.CurrentPosition() < end_token_pos()) {
+  while (tokens_iterator.CurrentPosition() < end_token_pos().value()) {
     uint32_t val = 0;
     obj = tokens_iterator.CurrentToken();
     if (obj.IsSmi()) {
       val = Smi::Cast(obj).Value();
     } else {
       literal = tokens_iterator.MakeLiteralToken(obj);
       val = literal.Hash();
     }
     result = 31 * result + val;
     tokens_iterator.Advance();
@@ skipping 446 matching lines @@
 }
 
 
 RawField* Field::New(const String& name,
                      bool is_static,
                      bool is_final,
                      bool is_const,
                      bool is_reflectable,
                      const Class& owner,
                      const AbstractType& type,
-                     intptr_t token_pos) {
+                     TokenDescriptor token_pos) {
   ASSERT(!owner.IsNull());
   const Field& result = Field::Handle(Field::New());
   result.set_name(name);
   result.set_is_static(is_static);
   if (!is_static) {
     result.SetOffset(0);
   }
   result.set_is_final(is_final);
   result.set_is_const(is_const);
   result.set_is_reflectable(is_reflectable);
@@ skipping 13 matching lines @@
     result.set_guarded_list_length(Field::kNoFixedLength);
   }
   return result.raw();
 }
 
 
 RawField* Field::NewTopLevel(const String& name,
                              bool is_final,
                              bool is_const,
                              const Object& owner,
-                             intptr_t token_pos) {
+                             TokenDescriptor token_pos) {
   ASSERT(!owner.IsNull());
   const Field& result = Field::Handle(Field::New());
   result.set_name(name);
   result.set_is_static(true);
   result.set_is_final(is_final);
   result.set_is_const(is_const);
   result.set_is_reflectable(true);
   result.set_is_double_initialized(false);
   result.set_owner(owner);
   result.set_token_pos(token_pos);
@@ skipping 578 matching lines @@
 RawString* TokenStream::PrivateKey() const {
   return raw_ptr()->private_key_;
 }
 
 
 void TokenStream::SetPrivateKey(const String& value) const {
   StorePointer(&raw_ptr()->private_key_, value.raw());
 }
 
 RawString* TokenStream::GenerateSource() const {
-  return GenerateSource(0, kMaxElements);
+  return GenerateSource(TokenDescriptor::kMinSource,
+                        TokenDescriptor::kMaxSource);
 }
 
-RawString* TokenStream::GenerateSource(intptr_t start_pos,
-                                       intptr_t end_pos) const {
+RawString* TokenStream::GenerateSource(TokenDescriptor start_pos,
+                                       TokenDescriptor end_pos) const {
   Iterator iterator(*this, start_pos, Iterator::kAllTokens);
   const ExternalTypedData& data = ExternalTypedData::Handle(GetStream());
   const GrowableObjectArray& literals =
       GrowableObjectArray::Handle(GrowableObjectArray::New(data.Length()));
   const String& private_key = String::Handle(PrivateKey());
   intptr_t private_len = private_key.Length();
 
   Token::Kind curr = iterator.CurrentTokenKind();
   Token::Kind prev = Token::kILLEGAL;
   // Handles used in the loop.
   Object& obj = Object::Handle();
   String& literal = String::Handle();
   // Current indentation level.
   int indent = 0;
 
-  while ((curr != Token::kEOS) && (iterator.CurrentPosition() < end_pos)) {
+  while ((curr != Token::kEOS) &&
+         (iterator.CurrentPosition() < end_pos.value())) {
     // Remember current values for this token.
     obj = iterator.CurrentToken();
     literal = iterator.MakeLiteralToken(obj);
     // Advance to be able to use next token kind.
     iterator.Advance();
     Token::Kind next = iterator.CurrentTokenKind();
 
     // Handle the current token.
     if (curr == Token::kSTRING) {
       bool escape_characters = false;
@@ skipping 137 matching lines @@
 
     // Setup for next iteration.
     prev = curr;
     curr = next;
   }
   const Array& source = Array::Handle(Array::MakeArray(literals));
   return String::ConcatAll(source);
 }
 
 
-intptr_t TokenStream::ComputeSourcePosition(intptr_t tok_pos) const {
-  Iterator iterator(*this, 0, Iterator::kAllTokens);
-  intptr_t src_pos = 0;
+TokenDescriptor TokenStream::ComputeSourcePosition(
+    TokenDescriptor tok_pos) const {
+  Iterator iterator(*this, TokenDescriptor::kMinSource, Iterator::kAllTokens);
+  TokenDescriptor src_pos = TokenDescriptor::kMinSource;
   Token::Kind kind = iterator.CurrentTokenKind();
-  while (iterator.CurrentPosition() < tok_pos && kind != Token::kEOS) {
+  while (iterator.CurrentPosition() < tok_pos.value() && kind != Token::kEOS) {
     iterator.Advance();
     kind = iterator.CurrentTokenKind();
-    src_pos += 1;
+    src_pos.Next();
   }
   return src_pos;
 }
 
 
 RawTokenStream* TokenStream::New() {
   ASSERT(Object::token_stream_class() != Class::null());
   RawObject* raw = Object::Allocate(TokenStream::kClassId,
                                     TokenStream::InstanceSize(),
                                     Heap::kOld);
@@ skipping 264 matching lines @@
     : tokens_(TokenStream::Handle(tokens.raw())),
       data_(ExternalTypedData::Handle(tokens.GetStream())),
       stream_(reinterpret_cast<uint8_t*>(data_.DataAddr(0)), data_.Length()),
       token_objects_(Array::Handle(
           GrowableObjectArray::Handle(tokens.TokenObjects()).data())),
       obj_(Object::Handle()),
       cur_token_pos_(token_pos),
       cur_token_kind_(Token::kILLEGAL),
       cur_token_obj_index_(-1),
       stream_type_(stream_type) {
-  ASSERT(token_pos != Token::kNoSourcePos);
-  if (token_pos >= 0) {
+  ASSERT(token_pos != TokenDescriptor::kNoSource.value());
+  if (TokenDescriptor(token_pos).IsReal()) {
     SetCurrentPosition(token_pos);
   }
 }
 
 
+TokenStream::Iterator::Iterator(const TokenStream& tokens,
+                                TokenDescriptor token_pos,
+                                Iterator::StreamType stream_type)
+    : tokens_(TokenStream::Handle(tokens.raw())),
+      data_(ExternalTypedData::Handle(tokens.GetStream())),
+      stream_(reinterpret_cast<uint8_t*>(data_.DataAddr(0)), data_.Length()),
+      token_objects_(Array::Handle(
+          GrowableObjectArray::Handle(tokens.TokenObjects()).data())),
+      obj_(Object::Handle()),
+      cur_token_pos_(token_pos.value()),
+      cur_token_kind_(Token::kILLEGAL),
+      cur_token_obj_index_(-1),
+      stream_type_(stream_type) {
+  ASSERT(token_pos != TokenDescriptor::kNoSource);
+  if (token_pos.IsReal()) {
+    SetCurrentPosition(token_pos.value());
+  }
+}
+
+
 void TokenStream::Iterator::SetStream(const TokenStream& tokens,
                                       intptr_t token_pos) {
   tokens_ = tokens.raw();
   data_ = tokens.GetStream();
   stream_.SetStream(reinterpret_cast<uint8_t*>(data_.DataAddr(0)),
                     data_.Length());
   token_objects_ = GrowableObjectArray::Handle(tokens.TokenObjects()).data();
   obj_ = Object::null();
   cur_token_pos_ = token_pos;
   cur_token_kind_ = Token::kILLEGAL;
   cur_token_obj_index_ = -1;
   SetCurrentPosition(token_pos);
 }
 
 
+void TokenStream::Iterator::SetStream(const TokenStream& tokens,
+                                      TokenDescriptor token_pos) {
+  SetStream(tokens, token_pos.value());
+}
+
+
 bool TokenStream::Iterator::IsValid() const {
   return !tokens_.IsNull();
 }
 
 
 Token::Kind TokenStream::Iterator::LookaheadTokenKind(intptr_t num_tokens) {
   intptr_t saved_position = stream_.Position();
   Token::Kind kind = Token::kILLEGAL;
   intptr_t value = -1;
   intptr_t count = 0;
@@ skipping 26 matching lines @@
   return cur_token_pos_;
 }
 
 
 void TokenStream::Iterator::SetCurrentPosition(intptr_t value) {
   stream_.SetPosition(value);
   Advance();
 }
 
 
+void TokenStream::Iterator::SetCurrentPosition(TokenDescriptor token_pos) {
+  SetCurrentPosition(token_pos.value());
+}
+
+
 void TokenStream::Iterator::Advance() {
   intptr_t value;
   do {
     cur_token_pos_ = stream_.Position();
     value = ReadToken();
   } while ((stream_type_ == kNoNewlines) &&
            (static_cast<Token::Kind>(value) == Token::kNEWLINE));
   if (value < Token::kNumTokens) {
     cur_token_kind_ = static_cast<Token::Kind>(value);
     cur_token_obj_index_ = -1;
@@ skipping 215 matching lines @@
 
 void Script::SetLocationOffset(intptr_t line_offset,
                                intptr_t col_offset) const {
   ASSERT(line_offset >= 0);
   ASSERT(col_offset >= 0);
   StoreNonPointer(&raw_ptr()->line_offset_, line_offset);
   StoreNonPointer(&raw_ptr()->col_offset_, col_offset);
 }
 
 
-void Script::GetTokenLocation(intptr_t token_pos,
+void Script::GetTokenLocation(TokenDescriptor token_pos,
                               intptr_t* line,
                               intptr_t* column,
                               intptr_t* token_len) const {
   ASSERT(line != NULL);
   const TokenStream& tkns = TokenStream::Handle(tokens());
   if (tkns.IsNull()) {
     ASSERT(Dart::IsRunningPrecompiledCode());
     *line = -1;
     if (column != NULL) {
       *column = -1;
     }
     if (token_len != NULL) {
       *token_len = 1;
     }
     return;
   }
   if (column == NULL) {
     TokenStream::Iterator tkit(tkns, 0, TokenStream::Iterator::kAllTokens);
     intptr_t cur_line = line_offset() + 1;
-    while (tkit.CurrentPosition() < token_pos &&
-           tkit.CurrentTokenKind() != Token::kEOS) {
+    while ((tkit.CurrentPosition() < token_pos.value()) &&
+           (tkit.CurrentTokenKind() != Token::kEOS)) {
       if (tkit.CurrentTokenKind() == Token::kNEWLINE) {
         cur_line++;
       }
       tkit.Advance();
     }
     *line = cur_line;
   } else {
     const String& src = String::Handle(Source());
-    intptr_t src_pos = tkns.ComputeSourcePosition(token_pos);
+    TokenDescriptor src_pos = tkns.ComputeSourcePosition(token_pos);
     Scanner scanner(src, Symbols::Empty());
     scanner.ScanTo(src_pos);
     intptr_t relative_line = scanner.CurrentPosition().line;
     *line = relative_line + line_offset();
     *column = scanner.CurrentPosition().column;
     if (token_len != NULL) {
       if (scanner.current_token().literal != NULL) {
         *token_len = scanner.current_token().literal->Length();
       } else {
         *token_len = 1;
       }
     }
     // On the first line of the script we must add the column offset.
     if (relative_line == 1) {
       *column += col_offset();
     }
   }
 }
 
 
 void Script::TokenRangeAtLine(intptr_t line_number,
-                              intptr_t* first_token_index,
-                              intptr_t* last_token_index) const {
+                              TokenDescriptor* first_token_index,
+                              TokenDescriptor* last_token_index) const {
   ASSERT(first_token_index != NULL && last_token_index != NULL);
   ASSERT(line_number > 0);
-  *first_token_index = -1;
-  *last_token_index = -1;
+  *first_token_index = TokenDescriptor::kNoSource;
+  *last_token_index = TokenDescriptor::kNoSource;
   const TokenStream& tkns = TokenStream::Handle(tokens());
   line_number -= line_offset();
   if (line_number < 1) line_number = 1;
   TokenStream::Iterator tkit(tkns, 0, TokenStream::Iterator::kAllTokens);
   // Scan through the token stream to the required line.
   intptr_t cur_line = 1;
   while (cur_line < line_number && tkit.CurrentTokenKind() != Token::kEOS) {
     if (tkit.CurrentTokenKind() == Token::kNEWLINE) {
       cur_line++;
     }
     tkit.Advance();
   }
   if (tkit.CurrentTokenKind() == Token::kEOS) {
     // End of token stream before reaching required line.
     return;
   }
   if (tkit.CurrentTokenKind() == Token::kNEWLINE) {
     // No tokens on the current line. If there is a valid token afterwards, put
     // it into first_token_index.
     while (tkit.CurrentTokenKind() == Token::kNEWLINE &&
            tkit.CurrentTokenKind() != Token::kEOS) {
       tkit.Advance();
     }
     if (tkit.CurrentTokenKind() != Token::kEOS) {
-      *first_token_index = tkit.CurrentPosition();
+      *first_token_index = TokenDescriptor(tkit.CurrentPosition());
     }
     return;
   }
-  *first_token_index = tkit.CurrentPosition();
+  *first_token_index = TokenDescriptor(tkit.CurrentPosition());
   // We cannot do "CurrentPosition() - 1" for the last token, because we do not
   // know whether the previous token is a simple one or not.
-  intptr_t end_pos = *first_token_index;
+  TokenDescriptor end_pos = *first_token_index;
   while (tkit.CurrentTokenKind() != Token::kNEWLINE &&
          tkit.CurrentTokenKind() != Token::kEOS) {
-    end_pos = tkit.CurrentPosition();
+    end_pos = TokenDescriptor(tkit.CurrentPosition());
     tkit.Advance();
   }
   *last_token_index = end_pos;
 }
 
 
 RawString* Script::GetLine(intptr_t line_number, Heap::Space space) const {
   const String& src = String::Handle(Source());
   intptr_t relative_line_number = line_number - line_offset();
   intptr_t current_line = 1;
@@ skipping 279 matching lines @@
     next_ix_++;
     obj = array_.At(next_ix_);
   }
 }
 
 
 static void ReportTooManyImports(const Library& lib) {
   const String& url = String::Handle(lib.url());
   Report::MessageF(Report::kError,
                    Script::Handle(lib.LookupScript(url)),
-                   Token::kNoSourcePos,
+                   TokenDescriptor::kNoSource,
                    Report::AtLocation,
                    "too many imports in library '%s'",
                    url.ToCString());
   UNREACHABLE();
 }
 
 
 void Library::set_num_imports(intptr_t value) const {
   if (!Utils::IsUint(16, value)) {
     ReportTooManyImports(*this);
@@ skipping 166 matching lines @@
   GrowableHandlePtrArray<const String> pieces(Thread::Current()->zone(), 3);
   pieces.Add(cname);
   pieces.Add(Symbols::At());
   pieces.Add(String::Handle(param.name()));
   return Symbols::FromConcatAll(pieces);
 }
 
 
 void Library::AddMetadata(const Object& owner,
                           const String& name,
-                          intptr_t token_pos) const {
+                          TokenDescriptor token_pos) const {
   const String& metaname = String::Handle(Symbols::New(name));
   const Field& field = Field::Handle(
       Field::NewTopLevel(metaname,
                          false,  // is_final
                          false,  // is_const
                          owner,
                          token_pos));
   field.SetFieldType(Object::dynamic_type());
   field.set_is_reflectable(false);
   field.SetStaticValue(Array::empty_array(), true);
   GrowableObjectArray& metadata =
       GrowableObjectArray::Handle(this->metadata());
   metadata.Add(field, Heap::kOld);
 }
 
 
 void Library::AddClassMetadata(const Class& cls,
                                const Object& tl_owner,
-                               intptr_t token_pos) const {
+                               TokenDescriptor token_pos) const {
   // We use the toplevel class as the owner of a class's metadata field because
   // a class's metadata is in scope of the library, not the class.
   AddMetadata(tl_owner,
               String::Handle(MakeClassMetaName(cls)),
               token_pos);
 }
 
 
 void Library::AddFieldMetadata(const Field& field,
-                               intptr_t token_pos) const {
+                               TokenDescriptor token_pos) const {
   AddMetadata(Object::Handle(field.RawOwner()),
               String::Handle(MakeFieldMetaName(field)),
               token_pos);
 }
 
 
 void Library::AddFunctionMetadata(const Function& func,
-                                  intptr_t token_pos) const {
+                                  TokenDescriptor token_pos) const {
   AddMetadata(Object::Handle(func.RawOwner()),
               String::Handle(MakeFunctionMetaName(func)),
               token_pos);
 }
 
 
 void Library::AddTypeParameterMetadata(const TypeParameter& param,
-                                       intptr_t token_pos) const {
+                                       TokenDescriptor token_pos) const {
   AddMetadata(Class::Handle(param.parameterized_class()),
               String::Handle(MakeTypeParameterMetaName(param)),
               token_pos);
 }
 
 
 void Library::AddLibraryMetadata(const Object& tl_owner,
-                                 intptr_t token_pos) const {
+                                 TokenDescriptor token_pos) const {
   AddMetadata(tl_owner, Symbols::TopLevel(), token_pos);
 }
 
 
 RawString* Library::MakeMetadataName(const Object& obj) const {
   if (obj.IsClass()) {
     return MakeClassMetaName(Class::Cast(obj));
   } else if (obj.IsField()) {
     return MakeFieldMetaName(Field::Cast(obj));
   } else if (obj.IsFunction()) {
@@ skipping 1428 matching lines @@
 void LibraryPrefix::PrintJSONImpl(JSONStream* stream, bool ref) const {
   Object::PrintJSONImpl(stream, ref);
 }
 
 
 void Namespace::set_metadata_field(const Field& value) const {
   StorePointer(&raw_ptr()->metadata_field_, value.raw());
 }
 
 
-void Namespace::AddMetadata(const Object& owner, intptr_t token_pos) {
+void Namespace::AddMetadata(const Object& owner, TokenDescriptor token_pos) {
   ASSERT(Field::Handle(metadata_field()).IsNull());
   Field& field = Field::Handle(Field::NewTopLevel(Symbols::TopLevel(),
                                           false,  // is_final
                                           false,  // is_const
                                           owner,
                                           token_pos));
   field.set_is_reflectable(false);
   field.SetFieldType(Object::dynamic_type());
   field.SetStaticValue(Array::empty_array(), true);
   set_metadata_field(field);
@@ skipping 563 matching lines @@
   const int addr_width = kBitsPerWord / 4;
   // First compute the buffer size required.
   intptr_t len = 1;  // Trailing '\0'.
   {
     Iterator iter(*this, RawPcDescriptors::kAnyKind);
     while (iter.MoveNext()) {
       len += OS::SNPrint(NULL, 0, FORMAT, addr_width,
                          iter.PcOffset(),
                          KindAsStr(iter.Kind()),
                          iter.DeoptId(),
-                         iter.TokenPos(),
+                         iter.TokenPos().value(),
                          iter.TryIndex());
     }
   }
   // Allocate the buffer.
   char* buffer = Thread::Current()->zone()->Alloc<char>(len);
   // Layout the fields in the buffer.
   intptr_t index = 0;
   Iterator iter(*this, RawPcDescriptors::kAnyKind);
   while (iter.MoveNext()) {
     index += OS::SNPrint((buffer + index), (len - index), FORMAT, addr_width,
                          iter.PcOffset(),
                          KindAsStr(iter.Kind()),
                          iter.DeoptId(),
-                         iter.TokenPos(),
+                         iter.TokenPos().value(),
                          iter.TryIndex());
   }
   return buffer;
 #undef FORMAT
 }
 
 
 void PcDescriptors::PrintToJSONObject(JSONObject* jsobj, bool ref) const {
   AddCommonObjectProperties(jsobj, "Object", ref);
   // TODO(johnmccutchan): Generate a stable id. PcDescriptors hang off a Code
   // object but do not have a back reference to generate an ID.
   jsobj->AddServiceId(*this);
   if (ref) {
     return;
   }
   JSONArray members(jsobj, "members");
   Iterator iter(*this, RawPcDescriptors::kAnyKind);
   while (iter.MoveNext()) {
     JSONObject descriptor(&members);
     descriptor.AddPropertyF("pcOffset", "%" Px "", iter.PcOffset());
     descriptor.AddProperty("kind", KindAsStr(iter.Kind()));
     descriptor.AddProperty("deoptId", iter.DeoptId());
     // TODO(turnidge): Use AddLocation instead.
-    descriptor.AddProperty("tokenPos", iter.TokenPos());
+    descriptor.AddProperty("tokenPos", iter.TokenPos().value());
     descriptor.AddProperty("tryIndex", iter.TryIndex());
   }
 }
 
 
 void PcDescriptors::PrintJSONImpl(JSONStream* stream, bool ref) const {
   JSONObject jsobj(stream);
   PrintToJSONObject(&jsobj, ref);
 }
 
@@ skipping 110 matching lines @@
   }
   // 4 bits per hex digit.
   const int addr_width = kBitsPerWord / 4;
   // First compute the buffer size required.
   intptr_t len = 1;  // Trailing '\0'.
   {
     Iterator iter(*this);
     while (iter.MoveNext()) {
       len += OS::SNPrint(NULL, 0, FORMAT, addr_width,
                          iter.PcOffset(),
-                         iter.TokenPos());
+                         iter.TokenPos().value());
     }
   }
   // Allocate the buffer.
   char* buffer = Thread::Current()->zone()->Alloc<char>(len);
   // Layout the fields in the buffer.
   intptr_t index = 0;
   Iterator iter(*this);
   while (iter.MoveNext()) {
     index += OS::SNPrint((buffer + index), (len - index), FORMAT, addr_width,
                          iter.PcOffset(),
-                         iter.TokenPos());
+                         iter.TokenPos().value());
   }
   return buffer;
 #undef FORMAT
 }
 
 
 // Encode integer in SLEB128 format.
 void CodeSourceMap::EncodeInteger(GrowableArray<uint8_t>* data,
                                   intptr_t value) {
   return EncodeSLEB128(data, value);
@@ skipping 169 matching lines @@
   const RawLocalVarDescriptors::VarInfoKind kind = info.kind();
   const int32_t index = info.index();
   if (kind == RawLocalVarDescriptors::kContextLevel) {
     return OS::SNPrint(buffer, len,
                        "%2" Pd " %-13s level=%-3d scope=%-3d"
                        " begin=%-3d end=%d\n",
                        i,
                        LocalVarDescriptors::KindToCString(kind),
                        index,
                        info.scope_id,
-                       info.begin_pos,
-                       info.end_pos);
+                       static_cast<int>(info.begin_pos.value()),
+                       static_cast<int>(info.end_pos.value()));
   } else if (kind == RawLocalVarDescriptors::kContextVar) {
     return OS::SNPrint(buffer, len,
                        "%2" Pd " %-13s level=%-3d index=%-3d"
                        " begin=%-3d end=%-3d name=%s\n",
                        i,
                        LocalVarDescriptors::KindToCString(kind),
                        info.scope_id,
                        index,
-                       info.begin_pos,
-                       info.end_pos,
+                       static_cast<int>(info.begin_pos.value()),
+                       static_cast<int>(info.end_pos.value()),
                        var_name.ToCString());
   } else {
     return OS::SNPrint(buffer, len,
                        "%2" Pd " %-13s scope=%-3d index=%-3d"
                        " begin=%-3d end=%-3d name=%s\n",
                        i,
                        LocalVarDescriptors::KindToCString(kind),
                        info.scope_id,
                        index,
-                       info.begin_pos,
-                       info.end_pos,
+                       static_cast<int>(info.begin_pos.value()),
+                       static_cast<int>(info.end_pos.value()),
                        var_name.ToCString());
   }
 }
 
 
 const char* LocalVarDescriptors::ToCString() const {
   if (IsNull()) {
     return "LocalVarDescriptors(NULL)";
   }
   if (Length() == 0) {
@@ skipping 34 matching lines @@
   }
   JSONArray members(&jsobj, "members");
   String& var_name = String::Handle();
   for (intptr_t i = 0; i < Length(); i++) {
     RawLocalVarDescriptors::VarInfo info;
     var_name = GetName(i);
     GetInfo(i, &info);
     JSONObject var(&members);
     var.AddProperty("name", var_name.ToCString());
     var.AddProperty("index", static_cast<intptr_t>(info.index()));
-    var.AddProperty("beginPos", static_cast<intptr_t>(info.begin_pos));
-    var.AddProperty("endPos", static_cast<intptr_t>(info.end_pos));
+    var.AddProperty("beginPos", info.begin_pos.value());
+    var.AddProperty("endPos", info.end_pos.value());
     var.AddProperty("scopeId", static_cast<intptr_t>(info.scope_id));
     var.AddProperty("kind", KindToCString(info.kind()));
   }
 }
 
 
 const char* LocalVarDescriptors::KindToCString(
     RawLocalVarDescriptors::VarInfoKind kind) {
   switch (kind) {
     case RawLocalVarDescriptors::kStackVar:
@@ skipping 1056 matching lines @@
   if (ref) {
     return;
   }
   jsobj.AddProperty("_argumentsDescriptor",
                     Object::Handle(arguments_descriptor()));
   jsobj.AddProperty("_entries", Object::Handle(ic_data()));
 }
 
 
 void ICData::PrintToJSONArray(const JSONArray& jsarray,
-                              intptr_t token_pos,
+                              TokenDescriptor token_pos,
                               bool is_static_call) const {
   Isolate* isolate = Isolate::Current();
   Class& cls = Class::Handle();
   Function& func = Function::Handle();
 
   JSONObject jsobj(&jsarray);
   jsobj.AddProperty("name", String::Handle(target_name()).ToCString());
-  jsobj.AddProperty("tokenPos", token_pos);
+  jsobj.AddProperty("tokenPos", token_pos.value());
   // TODO(rmacnak): Figure out how to stringify DeoptReasons().
   // jsobj.AddProperty("deoptReasons", ...);
 
   JSONArray cache_entries(&jsobj, "cacheEntries");
   for (intptr_t i = 0; i < NumberOfChecks(); i++) {
     func = GetTargetAt(i);
     if (is_static_call) {
       cls ^= func.Owner();
     } else {
       intptr_t cid = GetReceiverClassIdAt(i);
       cls ^= isolate->class_table()->At(cid);
     }
     intptr_t count = GetCountAt(i);
     JSONObject cache_entry(&cache_entries);
     if (cls.IsTopLevel()) {
       cache_entry.AddProperty("receiverContainer",
                               Library::Handle(cls.library()));
     } else {
       cache_entry.AddProperty("receiverContainer", cls);
     }
     cache_entry.AddProperty("count", count);
     cache_entry.AddProperty("target", func);
   }
 }
 
 
 void ICData::PrintToJSONArrayNew(const JSONArray& jsarray,
-                                 intptr_t token_pos,
+                                 TokenDescriptor token_pos,
                                  bool is_static_call) const {
   Isolate* isolate = Isolate::Current();
   Class& cls = Class::Handle();
   Function& func = Function::Handle();
 
   JSONObject jsobj(&jsarray);
   jsobj.AddProperty("name", String::Handle(target_name()).ToCString());
-  jsobj.AddProperty("tokenPos", token_pos);
+  jsobj.AddProperty("tokenPos", token_pos.value());
   // TODO(rmacnak): Figure out how to stringify DeoptReasons().
   // jsobj.AddProperty("deoptReasons", ...);
 
   JSONArray cache_entries(&jsobj, "cacheEntries");
   for (intptr_t i = 0; i < NumberOfChecks(); i++) {
     JSONObject cache_entry(&cache_entries);
     func = GetTargetAt(i);
     intptr_t count = GetCountAt(i);
     if (!is_static_call) {
       intptr_t cid = GetReceiverClassIdAt(i);
@@ skipping 599 matching lines @@
   code ^= Code::LookupCodeInVmIsolate(pc);
   if (!code.IsNull() && (code.compile_timestamp() == timestamp) &&
       (code.EntryPoint() == pc)) {
     // Found code in VM isolate.
     return code.raw();
   }
   return Code::null();
 }
 
 
-intptr_t Code::GetTokenIndexOfPC(uword pc) const {
+TokenDescriptor Code::GetTokenIndexOfPC(uword pc) const {
   uword pc_offset = pc - EntryPoint();
   const PcDescriptors& descriptors = PcDescriptors::Handle(pc_descriptors());
   PcDescriptors::Iterator iter(descriptors, RawPcDescriptors::kAnyKind);
   while (iter.MoveNext()) {
     if (iter.PcOffset() == pc_offset) {
       return iter.TokenPos();
     }
   }
-  return -1;
+  return TokenDescriptor::kNoSource;
 }
 
 
 uword Code::GetPcForDeoptId(intptr_t deopt_id,
                             RawPcDescriptors::Kind kind) const {
   const PcDescriptors& descriptors = PcDescriptors::Handle(pc_descriptors());
   PcDescriptors::Iterator iter(descriptors, kind);
   while (iter.MoveNext()) {
     if (iter.DeoptId() == deopt_id) {
       uword pc_offset = iter.PcOffset();
@@ skipping 477 matching lines @@
                                       Heap::kOld);
     NoSafepointScope no_safepoint;
     result ^= raw;
     result.set_num_variables(num_variables);
     result.set_is_implicit(is_implicit);
   }
   return result.raw();
 }
 
 
-intptr_t ContextScope::TokenIndexAt(intptr_t scope_index) const {
-  return Smi::Value(VariableDescAddr(scope_index)->token_pos);
+TokenDescriptor ContextScope::TokenIndexAt(intptr_t scope_index) const {
+  return TokenDescriptor(Smi::Value(VariableDescAddr(scope_index)->token_pos));
 }
 
 
 void ContextScope::SetTokenIndexAt(intptr_t scope_index,
-                                   intptr_t token_pos) const {
+                                   TokenDescriptor token_pos) const {
   StoreSmi(&VariableDescAddr(scope_index)->token_pos,
-           Smi::New(token_pos));
+           Smi::New(token_pos.value()));
 }
 
 
 RawString* ContextScope::NameAt(intptr_t scope_index) const {
   return VariableDescAddr(scope_index)->name;
 }
 
 
 void ContextScope::SetNameAt(intptr_t scope_index, const String& name) const {
   StorePointer(&(VariableDescAddr(scope_index)->name), name.raw());
@@ skipping 70 matching lines @@
            Smi::New(context_level));
 }
 
 
 const char* ContextScope::ToCString() const {
   const char* prev_cstr = "ContextScope:";
   String& name = String::Handle();
   for (int i = 0; i < num_variables(); i++) {
     name = NameAt(i);
     const char* cname = name.ToCString();
-    intptr_t pos = TokenIndexAt(i);
+    TokenDescriptor pos = TokenIndexAt(i);
     intptr_t idx = ContextIndexAt(i);
     intptr_t lvl = ContextLevelAt(i);
     char* chars = OS::SCreate(Thread::Current()->zone(),
         "%s\nvar %s  token-pos %" Pd "  ctx lvl %" Pd "  index %" Pd "",
-        prev_cstr, cname, pos, lvl, idx);
+        prev_cstr, cname, pos.value(), lvl, idx);
     prev_cstr = chars;
   }
   return prev_cstr;
 }
 
 
 void ContextScope::PrintJSONImpl(JSONStream* stream, bool ref) const {
   Object::PrintJSONImpl(stream, ref);
 }
 
@@ skipping 300 matching lines @@
   ASSERT(Object::language_error_class() != Class::null());
   RawObject* raw = Object::Allocate(LanguageError::kClassId,
                                     LanguageError::InstanceSize(),
                                     Heap::kOld);
   return reinterpret_cast<RawLanguageError*>(raw);
 }
 
 
 RawLanguageError* LanguageError::NewFormattedV(const Error& prev_error,
                                                const Script& script,
-                                               intptr_t token_pos,
+                                               TokenDescriptor token_pos,
                                                bool report_after_token,
                                                Report::Kind kind,
                                                Heap::Space space,
                                                const char* format,
                                                va_list args) {
   ASSERT(Object::language_error_class() != Class::null());
   LanguageError& result = LanguageError::Handle();
   {
     RawObject* raw = Object::Allocate(LanguageError::kClassId,
                                       LanguageError::InstanceSize(),
                                       space);
     NoSafepointScope no_safepoint;
     result ^= raw;
   }
   result.set_previous_error(prev_error);
   result.set_script(script);
   result.set_token_pos(token_pos);
   result.set_report_after_token(report_after_token);
   result.set_kind(kind);
   result.set_message(String::Handle(
       String::NewFormattedV(format, args, space)));
   return result.raw();
 }
 
 
 RawLanguageError* LanguageError::NewFormatted(const Error& prev_error,
                                               const Script& script,
-                                              intptr_t token_pos,
+                                              TokenDescriptor token_pos,
                                               bool report_after_token,
                                               Report::Kind kind,
                                               Heap::Space space,
                                               const char* format, ...) {
   va_list args;
   va_start(args, format);
   RawLanguageError* result = LanguageError::NewFormattedV(
       prev_error, script, token_pos, report_after_token,
       kind, space, format, args);
   NoSafepointScope no_safepoint;
@@ skipping 23 matching lines @@
 void LanguageError::set_previous_error(const Error& value) const {
   StorePointer(&raw_ptr()->previous_error_, value.raw());
 }
 
 
 void LanguageError::set_script(const Script& value) const {
   StorePointer(&raw_ptr()->script_, value.raw());
 }
 
 
-void LanguageError::set_token_pos(intptr_t token_pos) const {
-  ASSERT(!Token::IsClassifying(token_pos));
+void LanguageError::set_token_pos(TokenDescriptor token_pos) const {
+  ASSERT(!TokenDescriptor(token_pos).IsClassifying());
   StoreNonPointer(&raw_ptr()->token_pos_, token_pos);
 }
 
 
 void LanguageError::set_report_after_token(bool value) {
   StoreNonPointer(&raw_ptr()->report_after_token_, value);
 }
 
 
 void LanguageError::set_kind(uint8_t value) const {
@@ skipping 390 matching lines @@
       if (!type.IsCanonical()) {
         type ^= type.Canonicalize();
         signature.SetSignatureType(type);
       }
       return type.raw();
     }
     const Class& scope_cls = Class::Handle(type.scope_class());
     ASSERT(scope_cls.NumTypeArguments() > 0);
     TypeArguments& type_arguments = TypeArguments::Handle(GetTypeArguments());
     type = FunctionType::New(
-        scope_cls, type_arguments, signature, Token::kNoSourcePos);
+        scope_cls, type_arguments, signature, TokenDescriptor::kNoSource);
     type.SetIsFinalized();
     type ^= type.Canonicalize();
     return type.raw();
   }
   Type& type = Type::Handle();
   if (!cls.IsGeneric()) {
     type = cls.CanonicalType();
   }
   if (type.IsNull()) {
     TypeArguments& type_arguments = TypeArguments::Handle();
     if (cls.NumTypeArguments() > 0) {
       type_arguments = GetTypeArguments();
     }
-    type = Type::New(cls, type_arguments, Token::kNoSourcePos);
+    type = Type::New(cls, type_arguments, TokenDescriptor::kNoSource);
     type.SetIsFinalized();
     type ^= type.Canonicalize();
   }
   return type.raw();
 }
 
 
 RawTypeArguments* Instance::GetTypeArguments() const {
   const Class& cls = Class::Handle(clazz());
   intptr_t field_offset = cls.type_arguments_field_offset();
@@ skipping 306 matching lines @@
   } else {
     if (IsClosure()) {
       return Closure::Cast(*this).ToCString();
     }
     const Class& cls = Class::Handle(clazz());
     TypeArguments& type_arguments = TypeArguments::Handle();
     const intptr_t num_type_arguments = cls.NumTypeArguments();
     if (num_type_arguments > 0) {
       type_arguments = GetTypeArguments();
     }
-    const Type& type =
-        Type::Handle(Type::New(cls, type_arguments, Token::kNoSourcePos));
+    const Type& type = Type::Handle(
+        Type::New(cls, type_arguments, TokenDescriptor::kNoSource));
     const String& type_name = String::Handle(type.UserVisibleName());
     return OS::SCreate(Thread::Current()->zone(),
         "Instance of '%s'", type_name.ToCString());
   }
 }
 
 
 void Instance::PrintSharedInstanceJSON(JSONObject* jsobj,
                                        bool ref) const {
   AddCommonObjectProperties(jsobj, "Instance", ref);
@@ skipping 118 matching lines @@
   UNREACHABLE();
   return NULL;
 }
 
 
 void AbstractType::set_arguments(const TypeArguments& value) const {
   // AbstractType is an abstract class.
   UNREACHABLE();
 }
 
-intptr_t AbstractType::token_pos() const {
+TokenDescriptor AbstractType::token_pos() const {
   // AbstractType is an abstract class.
   UNREACHABLE();
-  return -1;
+  return TokenDescriptor::kNoSource;
 }
 
 
 bool AbstractType::IsInstantiated(TrailPtr trail) const {
   // AbstractType is an abstract class.
   UNREACHABLE();
   return false;
 }
 
 
@@ skipping 572 matching lines @@
 }
 
 
 RawType* Type::NewNonParameterizedType(const Class& type_class) {
   ASSERT(type_class.NumTypeArguments() == 0);
   Type& type = Type::Handle(type_class.CanonicalType());
   if (type.IsNull()) {
     const TypeArguments& no_type_arguments = TypeArguments::Handle();
     type ^= Type::New(Object::Handle(type_class.raw()),
                       no_type_arguments,
-                      Token::kNoSourcePos);
+                      TokenDescriptor::kNoSource);
     type.SetIsFinalized();
     type ^= type.Canonicalize();
   }
   ASSERT(type.IsFinalized());
   return type.raw();
 }
 
 
 void Type::SetIsFinalized() const {
   ASSERT(!IsFinalized());
@@ skipping 437 matching lines @@
 RawType* Type::New(Heap::Space space) {
   RawObject* raw = Object::Allocate(Type::kClassId,
                                     Type::InstanceSize(),
                                     space);
   return reinterpret_cast<RawType*>(raw);
 }
 
 
 RawType* Type::New(const Object& clazz,
                    const TypeArguments& arguments,
-                   intptr_t token_pos,
+                   TokenDescriptor token_pos,
                    Heap::Space space) {
   const Type& result = Type::Handle(Type::New(space));
   result.set_type_class(clazz);
   result.set_arguments(arguments);
   result.set_token_pos(token_pos);
   result.StoreNonPointer(&result.raw_ptr()->type_state_, RawType::kAllocated);
   return result.raw();
 }
 
 
-void Type::set_token_pos(intptr_t token_pos) const {
-  ASSERT(!Token::IsClassifying(token_pos));
+void Type::set_token_pos(TokenDescriptor token_pos) const {
+  ASSERT(!TokenDescriptor(token_pos).IsClassifying());
   StoreNonPointer(&raw_ptr()->token_pos_, token_pos);
 }
 
 
 void Type::set_type_state(int8_t state) const {
   ASSERT((state >= RawType::kAllocated) &&
          (state <= RawType::kFinalizedUninstantiated));
   StoreNonPointer(&raw_ptr()->type_state_, state);
 }
 
@@ skipping 397 matching lines @@
   // In case the type is first canonicalized at runtime, its type argument
   // vector may be longer than necessary. This is not an issue.
   ASSERT(type_args.IsNull() ||
          (type_args.Length() >= scope_cls.NumTypeArguments()));
   type_args = type_args.Canonicalize(trail);
   set_arguments(type_args);
 
   // Replace the actual function by a signature function.
   const Function& fun = Function::Handle(zone, signature());
   if (!fun.IsSignatureFunction()) {
-    Function& sig_fun =
-        Function::Handle(zone,
-                         Function::NewSignatureFunction(scope_cls,
-                                                        Token::kNoSourcePos));
+    Function& sig_fun = Function::Handle(zone,
+        Function::NewSignatureFunction(scope_cls, TokenDescriptor::kNoSource));
     type = fun.result_type();
     type = type.Canonicalize(trail);
     sig_fun.set_result_type(type);
     const intptr_t num_params = fun.NumParameters();
     sig_fun.set_num_fixed_parameters(fun.num_fixed_parameters());
     sig_fun.SetNumOptionalParameters(fun.NumOptionalParameters(),
                                      fun.HasOptionalPositionalParameters());
     sig_fun.set_parameter_types(Array::Handle(Array::New(num_params,
                                                          Heap::kOld)));
     for (intptr_t i = 0; i < num_params; i++) {
@@ skipping 90 matching lines @@
   RawObject* raw = Object::Allocate(FunctionType::kClassId,
                                     FunctionType::InstanceSize(),
                                     space);
   return reinterpret_cast<RawFunctionType*>(raw);
 }
 
 
 RawFunctionType* FunctionType::New(const Class& clazz,
                                    const TypeArguments& arguments,
                                    const Function& signature,
-                                   intptr_t token_pos,
+                                   TokenDescriptor token_pos,
                                    Heap::Space space) {
   const FunctionType& result = FunctionType::Handle(FunctionType::New(space));
   result.set_scope_class(clazz);
   result.set_arguments(arguments);
   result.set_signature(signature);
   result.set_token_pos(token_pos);
   result.StoreNonPointer(&result.raw_ptr()->type_state_,
                          RawFunctionType::kAllocated);
   return result.raw();
 }
 
 
-void FunctionType::set_token_pos(intptr_t token_pos) const {
-  ASSERT(!Token::IsClassifying(token_pos));
+void FunctionType::set_token_pos(TokenDescriptor token_pos) const {
+  ASSERT(!TokenDescriptor(token_pos).IsClassifying());
   StoreNonPointer(&raw_ptr()->token_pos_, token_pos);
 }
 
 
 void FunctionType::set_type_state(int8_t state) const {
   ASSERT((state >= RawFunctionType::kAllocated) &&
          (state <= RawFunctionType::kFinalizedUninstantiated));
   StoreNonPointer(&raw_ptr()->type_state_, state);
 }
 
@@ skipping 397 matching lines @@
                                     TypeParameter::InstanceSize(),
                                     Heap::kOld);
   return reinterpret_cast<RawTypeParameter*>(raw);
 }
 
 
 RawTypeParameter* TypeParameter::New(const Class& parameterized_class,
                                      intptr_t index,
                                      const String& name,
                                      const AbstractType& bound,
-                                     intptr_t token_pos) {
+                                     TokenDescriptor token_pos) {
   const TypeParameter& result = TypeParameter::Handle(TypeParameter::New());
   result.set_parameterized_class(parameterized_class);
   result.set_index(index);
   result.set_name(name);
   result.set_bound(bound);
   result.set_token_pos(token_pos);
   result.StoreNonPointer(&result.raw_ptr()->type_state_,
                          RawTypeParameter::kAllocated);
   return result.raw();
 }
 
 
-void TypeParameter::set_token_pos(intptr_t token_pos) const {
-  ASSERT(!Token::IsClassifying(token_pos));
+void TypeParameter::set_token_pos(TokenDescriptor token_pos) const {
+  ASSERT(!TokenDescriptor(token_pos).IsClassifying());
   StoreNonPointer(&raw_ptr()->token_pos_, token_pos);
 }
 
 
 void TypeParameter::set_type_state(int8_t state) const {
   ASSERT((state == RawTypeParameter::kAllocated) ||
          (state == RawTypeParameter::kBeingFinalized) ||
          (state == RawTypeParameter::kFinalizedUninstantiated));
   StoreNonPointer(&raw_ptr()->type_state_, state);
 }
@@ skipping 256 matching lines @@
   const String& vm_name = String::Handle(Name());
   AddNameProperties(&jsobj, user_name, vm_name);
   if (ref) {
     return;
   }
   jsobj.AddProperty("targetType", AbstractType::Handle(type()));
   jsobj.AddProperty("bound", AbstractType::Handle(bound()));
 }
 
 
-intptr_t MixinAppType::token_pos() const {
+TokenDescriptor MixinAppType::token_pos() const {
   return AbstractType::Handle(MixinTypeAt(0)).token_pos();
 }
 
 
 intptr_t MixinAppType::Depth() const {
   return Array::Handle(mixin_types()).Length();
 }
 
 
 RawString* MixinAppType::Name() const {
@@ skipping 4477 matching lines @@
   jsobj.AddProperty("valueAsString", ToCStringInternal(&idx));
 }
 
 
 static intptr_t PrintOneStacktrace(Zone* zone,
                                    GrowableArray<char*>* frame_strings,
                                    uword pc,
                                    const Function& function,
                                    const Code& code,
                                    intptr_t frame_index) {
-  const intptr_t token_pos = code.GetTokenIndexOfPC(pc);
+  const TokenDescriptor token_pos = code.GetTokenIndexOfPC(pc);
   const Script& script = Script::Handle(zone, function.script());
   const String& function_name =
       String::Handle(zone, function.QualifiedUserVisibleName());
   const String& url = String::Handle(zone, script.url());
   intptr_t line = -1;
   intptr_t column = -1;
-  if (token_pos >= 0) {
+  if (token_pos.IsReal()) {
     if (script.HasSource()) {
       script.GetTokenLocation(token_pos, &line, &column);
     } else {
       script.GetTokenLocation(token_pos, &line, NULL);
     }
   }
   char* chars = NULL;
   if (column >= 0) {
     chars = OS::SCreate(zone,
         "#%-6" Pd " %s (%s:%" Pd ":%" Pd ")\n",
@@ skipping 487 matching lines @@
   return tag_label.ToCString();
 }
 
 
 void UserTag::PrintJSONImpl(JSONStream* stream, bool ref) const {
   Instance::PrintJSONImpl(stream, ref);
 }
 
 
 }  // namespace dart