Tokenize one line at a time.

dart/site/try/src/interaction_manager.dart

 // Copyright (c) 2014, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library trydart.interaction_manager;
 
 import 'dart:html';
 
 import 'dart:convert' show
     JSON;
 
 import 'dart:math' show
     max,
     min;
 
 import 'dart:async' show
     Future;
 
 import 'package:compiler/implementation/scanner/scannerlib.dart' show
     BeginGroupToken,
     EOF_TOKEN,
     ErrorToken,
     StringScanner,
-    Token;
+    Token,
+    UnmatchedToken,
+    UnterminatedToken;
 
 import 'package:compiler/implementation/source_file.dart' show
     StringSourceFile;
 
 import 'compilation.dart' show
     currentSource,
     scheduleCompilation;
 
 import 'ui.dart' show
     currentTheme,
@@ skipping 232 matching lines @@
     trySelection.updateText(currentText);
 
     context.currentCompilationUnit.content = currentText;
 
     editor.seenIdentifiers = new Set<String>.from(mock.identifiers);
 
     editor.isMalformedInput = false;
     int offset = 0;
     List<Node> nodes = <Node>[];
 
-    tokenizeAndHighlight(currentText, offset, trySelection, nodes);
+    String state = '';
+    for (String line in currentText.split(new RegExp('^', multiLine: true))) {
+      List<Node> lineNodes = <Node>[];
+      state =
+          tokenizeAndHighlight(line, state, offset, trySelection, lineNodes);
+      offset += line.length;
+      nodes.add(new SpanElement()
+          ..nodes.addAll(lineNodes)
+          ..classes.add('lineNumber'));
+    }
 
     mainEditorPane
         ..nodes.clear()
         ..nodes.addAll(nodes);
     trySelection.adjust(selection);
 
     // Discard highlighting mutations.
     observer.takeRecords();
   }
 
@@ skipping 405 matching lines @@
       event.getModifierState("Meta") ||
       event.getModifierState("NumLock") ||
       event.getModifierState("ScrollLock") ||
       event.getModifierState("Scroll") ||
       event.getModifierState("Win") ||
       event.getModifierState("Shift") ||
       event.getModifierState("SymbolLock") ||
       event.getModifierState("OS");
 }
 
-void tokenizeAndHighlight(String currentText,
-                          int offset,
-                          TrySelection trySelection,
-                          List<Node> nodes) {
+String tokenizeAndHighlight(String line,
+                            String state,
+                            int start,
+                            TrySelection trySelection,
+                            List<Node> nodes) {
+  String newState = '';

[kasperl, 2014/05/02 08:01:06]

Consider using a StringBuffer or += to build the newState string? 

[ahe, 2014/05/02 10:01:45]

I knew the other version would generate nice JS code on dart2js. But your
suggestion works equally great.

+  int offset = state.length;
   //   + offset  + charOffset  + globalOffset   + (charOffset + charCount)
   //   v         v             v                v
   // do          identifier_abcdefghijklmnopqrst
-  for (Token token = tokenize(currentText);
+  for (Token token = tokenize('$state$line');
        token.kind != EOF_TOKEN;
        token = token.next) {
     int charOffset = token.charOffset;
     int charCount = token.charCount;
 
-    if (charOffset < offset) continue; // Happens for scanner errors.
+    Token tokenToDecorate = token;
+    if (token is UnterminatedToken && isUnterminatedMultiLineToken(token)) {
+      newState = '$newState${token.start}';

[ahe, 2014/05/02 10:01:45]

Using += compiles to:

        newState += t1.get$start(token);

This is good, but not great (there shouldn't be a need to go through the
interceptor.

+      continue; // This might not be an error.
+    } else {
+      Token follow = token.next;
+      if (token is BeginGroupToken && token.endGroup != null) {
+        follow = token.endGroup.next;
+      }
+      if (follow is ErrorToken && follow.charOffset == token.charOffset) {
+        if (follow is UnmatchedToken) {
+          newState = '$newState${follow.begin.value}';

[ahe, 2014/05/02 10:01:45]

Compiles to:

            newState += H.S(t1.get$value(t1));

Again, good but not great (interceptor problems).

+        } else {
+          tokenToDecorate = follow;
+        }
+      }
+    }
 
-    Decoration decoration = editor.getDecoration(token);
+    if (charOffset < offset) {
+      // Happens for scanner errors, or for the [state] prefix.
+      continue;
+    }
 
-    Token follow = token.next;
-    if (token is BeginGroupToken) {
-      follow = token.endGroup.next;
-    }
-    if (follow is ErrorToken) {
-      decoration = editor.getDecoration(follow);
-    }
+    Decoration decoration = editor.getDecoration(tokenToDecorate);
 
     if (decoration == null) continue;
 
     // Add a node for text before current token.
-    trySelection.addNodeFromSubstring(offset, charOffset, nodes);
+    trySelection.addNodeFromSubstring(

[kasperl, 2014/05/02 08:01:06]

Compue (start - state.length) only once?

[ahe, 2014/05/02 10:01:45]

Done.

+        start + offset - state.length, start + charOffset - state.length,
+        nodes);
 
     // Add a node for current token.
     trySelection.addNodeFromSubstring(
-        charOffset, charOffset + charCount, nodes, decoration);
+        start + charOffset - state.length,
+        start + charOffset + charCount - state.length, nodes, decoration);
 
     offset = charOffset + charCount;
   }
 
   // Add a node for anything after the last (decorated) token.
-  trySelection.addNodeFromSubstring(offset, currentText.length, nodes);
+  trySelection.addNodeFromSubstring(
+      start + offset - state.length, start + line.length, nodes);
 
-  // Ensure text always ends with a newline.
-  if (!currentText.endsWith('\n')) {
-    nodes.add(new Text('\n'));
-  }
+  return newState;
+}
+
+bool isUnterminatedMultiLineToken(UnterminatedToken token) {
+  return
+      token.start == '/*' ||
+      token.start == "'''" ||
+      token.start == '"""' ||
+      token.start == "r'''" ||
+      token.start == 'r"""';
 }
 
 void normalizeMutationRecord(MutationRecord record, TrySelection selection) {
   if (record.addedNodes.isEmpty) return;
   for (Node node in record.addedNodes) {
     if (node.parent == null) continue;
     StringBuffer buffer = new StringBuffer();
     int selectionOffset = htmlToText(node, buffer, selection);
     Text newNode = new Text('$buffer');
     node.replaceWith(newNode);
     if (selectionOffset != -1) {
       selection.anchorNode = newNode;
       selection.anchorOffset = selectionOffset;
     }
   }
 }