Tokenize one line at a time.

dart/site/try/src/interaction_manager.dart

Index: dart/site/try/src/interaction_manager.dart
diff --git a/dart/site/try/src/interaction_manager.dart b/dart/site/try/src/interaction_manager.dart
index 1422e1968d1ef805f92e589a619f2aa97ec71563..b3bbe29b9291094f686a8cd94b8d5dbc7b3b9325 100644
--- a/dart/site/try/src/interaction_manager.dart
+++ b/dart/site/try/src/interaction_manager.dart
@@ -21,7 +21,9 @@ import 'package:compiler/implementation/scanner/scannerlib.dart' show
     EOF_TOKEN,
     ErrorToken,
     StringScanner,
-    Token;
+    Token,
+    UnmatchedToken,
+    UnterminatedToken;
 
 import 'package:compiler/implementation/source_file.dart' show
     StringSourceFile;
@@ -274,7 +276,16 @@ class InitialState extends InteractionState {
     int offset = 0;
     List<Node> nodes = <Node>[];
 
-    tokenizeAndHighlight(currentText, offset, trySelection, nodes);
+    String state = '';
+    for (String line in currentText.split(new RegExp('^', multiLine: true))) {
+      List<Node> lineNodes = <Node>[];
+      state =
+          tokenizeAndHighlight(line, state, offset, trySelection, lineNodes);
+      offset += line.length;
+      nodes.add(new SpanElement()
+          ..nodes.addAll(lineNodes)
+          ..classes.add('lineNumber'));
+    }
 
     mainEditorPane
         ..nodes.clear()
@@ -700,50 +711,76 @@ bool computeHasModifier(KeyboardEvent event) {
       event.getModifierState("OS");
 }
 
-void tokenizeAndHighlight(String currentText,
-                          int offset,
-                          TrySelection trySelection,
-                          List<Node> nodes) {
+String tokenizeAndHighlight(String line,
+                            String state,
+                            int start,
+                            TrySelection trySelection,
+                            List<Node> nodes) {
+  String newState = '';

[kasperl, 2014/05/02 08:01:06]

Consider using a StringBuffer or += to build the newState string? 

[ahe, 2014/05/02 10:01:45]

I knew the other version would generate nice JS code on dart2js. But your
suggestion works equally great.

+  int offset = state.length;
   //   + offset  + charOffset  + globalOffset   + (charOffset + charCount)
   //   v         v             v                v
   // do          identifier_abcdefghijklmnopqrst
-  for (Token token = tokenize(currentText);
+  for (Token token = tokenize('$state$line');
        token.kind != EOF_TOKEN;
        token = token.next) {
     int charOffset = token.charOffset;
     int charCount = token.charCount;
 
-    if (charOffset < offset) continue; // Happens for scanner errors.
-
-    Decoration decoration = editor.getDecoration(token);
-
-    Token follow = token.next;
-    if (token is BeginGroupToken) {
-      follow = token.endGroup.next;
+    Token tokenToDecorate = token;
+    if (token is UnterminatedToken && isUnterminatedMultiLineToken(token)) {
+      newState = '$newState${token.start}';

[ahe, 2014/05/02 10:01:45]

Using += compiles to:

        newState += t1.get$start(token);

This is good, but not great (there shouldn't be a need to go through the
interceptor.

+      continue; // This might not be an error.
+    } else {
+      Token follow = token.next;
+      if (token is BeginGroupToken && token.endGroup != null) {
+        follow = token.endGroup.next;
+      }
+      if (follow is ErrorToken && follow.charOffset == token.charOffset) {
+        if (follow is UnmatchedToken) {
+          newState = '$newState${follow.begin.value}';

[ahe, 2014/05/02 10:01:45]

Compiles to:

            newState += H.S(t1.get$value(t1));

Again, good but not great (interceptor problems).

+        } else {
+          tokenToDecorate = follow;
+        }
+      }
     }
-    if (follow is ErrorToken) {
-      decoration = editor.getDecoration(follow);
+
+    if (charOffset < offset) {
+      // Happens for scanner errors, or for the [state] prefix.
+      continue;
     }
 
+    Decoration decoration = editor.getDecoration(tokenToDecorate);
+
     if (decoration == null) continue;
 
     // Add a node for text before current token.
-    trySelection.addNodeFromSubstring(offset, charOffset, nodes);
+    trySelection.addNodeFromSubstring(

[kasperl, 2014/05/02 08:01:06]

Compue (start - state.length) only once?

[ahe, 2014/05/02 10:01:45]

Done.

+        start + offset - state.length, start + charOffset - state.length,
+        nodes);
 
     // Add a node for current token.
     trySelection.addNodeFromSubstring(
-        charOffset, charOffset + charCount, nodes, decoration);
+        start + charOffset - state.length,
+        start + charOffset + charCount - state.length, nodes, decoration);
 
     offset = charOffset + charCount;
   }
 
   // Add a node for anything after the last (decorated) token.
-  trySelection.addNodeFromSubstring(offset, currentText.length, nodes);
+  trySelection.addNodeFromSubstring(
+      start + offset - state.length, start + line.length, nodes);
 
-  // Ensure text always ends with a newline.
-  if (!currentText.endsWith('\n')) {
-    nodes.add(new Text('\n'));
-  }
+  return newState;
+}
+
+bool isUnterminatedMultiLineToken(UnterminatedToken token) {
+  return
+      token.start == '/*' ||
+      token.start == "'''" ||
+      token.start == '"""' ||
+      token.start == "r'''" ||
+      token.start == 'r"""';
 }
 
 void normalizeMutationRecord(MutationRecord record, TrySelection selection) {