Various token issues fixed in the compiler

lib/compiler/implementation/compiler.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 
 /**
  * If true, print a warning for each method that was resolved, but not
  * compiled.
  */
 final bool REPORT_EXCESS_RESOLUTION = false;
@@ skipping 763 matching lines @@
       // URI.
       throw 'Cannot find tokens to produce error message.';
     }
     if (uri === null) {
       uri = currentElement.getCompilationUnit().script.uri;
     }
     return SourceSpan.withOffsets(begin, end, (beginOffset, endOffset) =>
         new SourceSpan(uri, beginOffset, endOffset));
   }
 
-  SourceSpan spanFromNode(Node node) {
-    return spanFromTokens(node.getBeginToken(), node.getEndToken());
+  SourceSpan spanFromNode(Node node, [Uri uri]) {
+    return spanFromTokens(node.getBeginToken(), node.getEndToken(), uri);
   }
 
   SourceSpan spanFromElement(Element element) {
     if (element.position() === null) {
       // Sometimes, the backend fakes up elements that have no
       // position. So we use the enclosing element instead. It is
       // not a good error location, but cancel really is "internal
       // error" or "not implemented yet", so the vicinity is good
       // enough for now.
       element = element.enclosingElement;
@@ skipping 73 matching lines @@
   }
 }
 
 class SourceSpan {
   final Uri uri;
   final int begin;
   final int end;
 
   const SourceSpan(this.uri, this.begin, this.end);
 
   static withOffsets(Token begin, Token end,

[Lasse Reichstein Nielsen, 2012/06/27 09:22:16]

withOffsets -> withCharacterOffsets ?

                      f(int beginOffset, int endOffset)) {
     final beginOffset = begin.charOffset;
-    // TODO(ahe): Compute proper end offset in token. Right now we use
-    // the position of the next token. We want to preserve the
-    // invariant that endOffset > beginOffset, but for EOF the
-    // charoffset of the next token may be [beginOffset]. This can
-    // also happen for synthetized tokens that are produced during
-    // error handling.
-    final endOffset =
-      Math.max((end.next !== null) ? end.next.charOffset : 0, beginOffset + 1);
-    assert(endOffset > beginOffset);
+    final endOffset = end.charOffset + end.slowCharLength;

[Lasse Reichstein Nielsen, 2012/06/27 09:22:16]

slowCharLength -> slowCharCount

+
+    // [begin] and [end] might be the same empty token. This happens for

[Lasse Reichstein Nielsen, 2012/06/27 09:22:16]

"the same empty token" -> "the same for the empty token"

+    // instance when parsing '$$'.

[Lasse Reichstein Nielsen, 2012/06/27 09:22:16]

parsing -> scanning.

+    assert(endOffset >= beginOffset);
     return f(beginOffset, endOffset);
   }
 }