Scanner can include comments in the token stream.

tests/compiler/dart2js/scanner_offset_length_test.dart

Index: tests/compiler/dart2js/scanner_offset_length_test.dart
diff --git a/tests/compiler/dart2js/scanner_offset_length_test.dart b/tests/compiler/dart2js/scanner_offset_length_test.dart
new file mode 100644
index 0000000000000000000000000000000000000000..cf4d919d9cbb2497a6dc89dc5d0e6993f1edbbfb
--- /dev/null
+++ b/tests/compiler/dart2js/scanner_offset_length_test.dart
@@ -0,0 +1,47 @@
+// Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+#import('../../../lib/compiler/implementation/scanner/scannerlib.dart');
+#import('../../../lib/compiler/implementation/scanner/scanner_implementation.dart');
+
+Token scan(String text) => new StringScanner(text, true).tokenize();
+
+check(String text) {
+  Token token = scan(text);
+  while (token.kind != EOF_TOKEN) {
+    Expect.equals(token.slowToString().length, token.slowCharLength);
+
+    var start = token.charOffset;
+    var end = token.charOffset+token.slowCharLength;
+
+    Expect.isTrue(start < text.length,
+        'start=$start < text.length=${text.length}: $text');
+
+    Expect.isTrue(end <= text.length,
+        'end=$end <= text.length=${text.length}: $text');
+
+    Expect.isTrue(start <= end, 'start=$end <= end=$end: $text');
+
+    var substring = text.substring(start, end);
+
+    Expect.stringEquals(token.slowToString(), substring,
+        'token.slowToString()=${token.slowToString()} == '
+        'text.substring(start,end)=${substring}: $text');
+
+    print('$text: [$start,$end]:$token');
+
+    token = token.next;
+  }
+}
+
+main() {
+    check('foo'); // identifier

[ahe, 2012/06/25 10:55:52]

Comments still aren't proper sentences.

+    check('\'\''); // empty string
+    check('\'foo\''); // simple string
+    check('\'\$foo\''); // interpolation, identifier
+    check('\'\${foo}\''); // interpolation, expression
+
+    check('//'); // single line comment
+    check('/**/'); // multi line comment
+}