test.py: Sending JSON between test_controller.js <-> browser_controller

dart/tools/testing/dart/test_controller.js

Index: dart/tools/testing/dart/test_controller.js
diff --git a/dart/pkg/unittest/lib/test_controller.js b/dart/tools/testing/dart/test_controller.js
similarity index 54%
copy from dart/pkg/unittest/lib/test_controller.js
copy to dart/tools/testing/dart/test_controller.js
index 6f604cd50f734ba4ca1a30c6eb75005dc74b399a..1c6a379d7fa00f4d5800eb25d9b99d814d78ad76 100644
--- a/dart/pkg/unittest/lib/test_controller.js
+++ b/dart/tools/testing/dart/test_controller.js
@@ -7,11 +7,62 @@
  * conent shell.
  */
 
-// Clear the console before every test run - this is Firebug specific code.
-if (typeof console == "object" && typeof console.clear == "function") {
-  console.clear();
+/*
+ * We will collect testing driver specific events here instead of printing
+ * them to the DOM.
+ * Every entry will look like this:
+ *   {
+ *     'type' : 'test_outcome' / 'print' /  'debug' / 'message_received' / 'dom'
+ *     'value' : 'some content',
+ *     'timestamp' : TimestampInMs,
+ *   }
+ */
+var EVENTS = [];
+var FIRST_TIMESTAMP = null;
+
+function getCurrentTimestamp() {
+  if (FIRST_TIMESTAMP == null) {
+    FIRST_TIMESTAMP = new Date().getTime();

[ricow1, 2013/11/15 07:26:33]

shouldn't we just initialize this when we start the script?

[kustermann, 2013/11/19 10:06:10]

I don't want to do it here. I added a 'recordEvent()' call in "notifyStart()"
[which is called when this script is loaded].

+  }
+  return (new Date().getTime() - FIRST_TIMESTAMP)/1000.0;
+}
+
+function stringifyEvent(event) {
+  return JSON.stringify(event, null, 2);
+}
+
+function recordEvent(type, value) {
+  var event = {
+    type: type,
+    value: value,
+    timestamp: getCurrentTimestamp()
+  };
+  EVENTS.push(event);
+  printToConsole(stringifyEvent(event));
+}
+
+function clearConsole() {
+  // Clear the console before every test run - this is Firebug specific code.
+  if (typeof console == 'object' && typeof console.clear == 'function') {
+    console.clear();
+  }
 }
 
+function printToConsole(message) {
+  var consoleAvailable = typeof console === 'object';
+
+  if (!consoleAvailable || isContentShell) {
+    var pre = document.createElement('pre');
+    pre.appendChild(document.createTextNode(String(message)));

[ricow1, 2013/11/15 07:26:33]

so on content shell we will interleave this with actual output?

[kustermann, 2013/11/19 10:06:10]

This is not much different from what we have now.

The only two communication channels we have on drt are:
 - DOM
 - console
=> We don't have the possibility to send that as JSON string.

We could print it to the console and try to parse it from there, but it's not so
nice (the application could also print to there).

Ideally we should also use the browser controller for content shell -- but still
run one content shell per test, but post the result via HTTP instead of the
console/DOM.

+    document.body.appendChild(pre);
+    document.body.appendChild(document.createTextNode('\n'));
+  } else if (consoleAvailable) {

[Bill Hesse, 2013/11/15 11:24:03]

consoleAvailable is always true here.

[kustermann, 2013/11/19 10:06:10]

Correct.

+    console.log(message);
+  }
+}
+
+clearConsole();
+
 // Some tests may expect and have no way to suppress global errors.
 var testExpectsGlobalError = false;
 var testSuppressedGlobalErrors = [];

[ricow1, 2013/11/15 07:26:33]

here we use camel case for global variables, our new ones are all caps, please
be consistent

[kustermann, 2013/11/19 10:06:10]

Done.

@@ -19,19 +70,16 @@ var testSuppressedGlobalErrors = [];
 // Set window onerror to make sure that we catch test harness errors across all
 // browsers.
 window.onerror = function (message, url, lineNumber) {
+  if (url) {
+    message = "window.onerror called: \n\n" + url + ":" + lineNumber + ":\n" + message + "\n\n";

[ricow1, 2013/11/15 07:26:33]

long line

[kustermann, 2013/11/19 10:06:10]

Done.

+  }
   if (testExpectsGlobalError) {
     testSuppressedGlobalErrors.push({
       message: message
     });
     return;
   }
-  if (url) {
-    showErrorAndExit(
-        "\n\n" + url + ":" + lineNumber + ":\n" + message + "\n\n");
-  } else {
-    showErrorAndExit(message);
-  }
-  window.postMessage('unittest-suite-external-error', '*');
+  showErrorAndExit(message);
 };
 
 // Start Dartium/content_shell, unless we are waiting for HTML Imports to load.
@@ -47,8 +95,9 @@ if (navigator.webkitStartDart && !window.HTMLImports) {
 }
 
 // testRunner is provided by content shell.
-// It is not available in browser tests.
+// It is not available in selenium tests.

[Bill Hesse, 2013/11/15 11:24:03]

Stray revert.

[kustermann, 2013/11/19 10:06:10]

Done.

 var testRunner = window.testRunner || window.layoutTestController;
+var isContentShell = testRunner;
 
 var waitForDone = false;
 
@@ -64,6 +113,10 @@ function getDriverWindow() {
   return null;
 }
 
+function usingBrowserController() {
+  return getDriverWindow() != null;
+}
+
 function notifyStart() {
   var driver = getDriverWindow();
   if (driver) {
@@ -73,33 +126,58 @@ function notifyStart() {
 // We call notifyStart here to notify the encapsulating browser.
 notifyStart();
 
-function notifyDone() {
-  if (testRunner) testRunner.notifyDone();
+function notifyDone(test_outcome) {
+  recordEvent('debug', 'Test outcome: ' + test_outcome);
 
-  // TODO(ricow): REMOVE, debug info, see issue 13292
-  if (!testRunner) {
-    printMessage('Calling notifyDone()');
-  }
-  // To support in browser launching of tests we post back start and result
-  // messages to the window.opener.
-  var driver = getDriverWindow();
-  if (driver) {
-    driver.postMessage(window.document.body.innerHTML, "*");
+  var dom = '' + window.document.documentElement.innerHTML;
+  var domEvent = {
+    type: 'dom',
+    value: dom,
+    timestamp: getCurrentTimestamp()
+  };
+  var outcomeEvent = {
+    type: 'test_outcome',
+    value: test_outcome,
+    timestamp: getCurrentTimestamp()
+  };
+
+  // If we are not using the browser controller (i.e. in the none-drt

[ricow1, 2013/11/15 07:26:33]

not just none-drt I guess, also in dart2js-drt right?

[kustermann, 2013/11/19 10:06:10]

Done.

+  // configuration), we need to print 'test_outcome' as it is.
+  if (!usingBrowserController()) {
+    printToConsole(stringifyEvent(outcomeEvent));

[Bill Hesse, 2013/11/15 11:24:03]

Isn't it just confusing to print the outcome twice on ContentShell?
Could we skip the printing of outcomeEvent?

[kustermann, 2013/11/19 10:06:10]

Sure we could do that. But the "outcomeEvent" contains also the timestamp
information.

The "printToConsole(test_outcome)" on line 151 is necessary, because test.dart
will only recognize this.

+    if (isContentShell) {
+      // We need this, since test.dart is looking for 'FAIL\n', 'PASS\n' in the
+      // console output of drt.
+      printToConsole(test_outcome);
+    }
+    if (!isContentShell) {
+      // If we're running in content shell, the events will be recorded to the
+      // DOM and it will print the text representation of the DOM. So there is

[Bill Hesse, 2013/11/15 11:24:03]

Comment applies to other case.  Also, use "else" instead?

[kustermann, 2013/11/19 10:06:10]

Done.

+      // no need to have the DOM twice.
+      printToConsole(dom);
+    }
+  } else {
+    // To support in browser launching of tests we post back start and result
+    // messages to the window.opener.
+    var driver = getDriverWindow();
+
+    // Post the DOM, the test outcome and all events that happened.
+    var events = EVENTS.slice(0);
+    events.push(domEvent);
+    events.push(outcomeEvent);
+
+    driver.postMessage(JSON.stringify(events), '*');
   }
+  if (testRunner) testRunner.notifyDone();
 }
 
 function processMessage(msg) {
-  if (typeof msg != 'string') return;
-  // TODO(ricow): REMOVE, debug info, see issue 13292
-  if (!testRunner) {
-    // Filter out ShadowDOM polyfill messages which are random floats.
-    if (msg != parseFloat(msg)) {
-      printMessage('processMessage(): ' + msg);
-    }
+  // Filter out ShadowDOM polyfill messages which are random floats.
+  if (msg != parseFloat(msg)) {
+    recordEvent('message_received', '' + msg);
   }
-  if (msg == 'unittest-suite-done') {
-    notifyDone();
-  } else if (msg == 'unittest-suite-wait-for-done') {
+  if (typeof msg != 'string') return;
+  if (msg == 'unittest-suite-wait-for-done') {
     waitForDone = true;
     if (testRunner) {
       testRunner.startedDartTest = true;
@@ -110,14 +188,12 @@ function processMessage(msg) {
     }
   } else if (msg == 'dart-main-done') {
     if (!waitForDone) {
-      printMessage('PASS');
-      notifyDone();
+      notifyDone('PASS');
     }
   } else if (msg == 'unittest-suite-success') {
-    printMessage('PASS');
-    notifyDone();
+    notifyDone('PASS');
   } else if (msg == 'unittest-suite-fail') {
-    showErrorAndExit('Some tests failed.');
+    notifyDone('FAIL');
   }
 }
 
@@ -133,18 +209,15 @@ window.addEventListener("message", onReceive, false);
 
 function showErrorAndExit(message) {
   if (message) {
-    printMessage('Error: ' + String(message));
+    recordEvent('debug', 'Error: ' + String(message));
   }
-  // dart/tools/testing/test_runner.dart is looking for either PASS or
-  // FAIL in a browser test's output.
-  printMessage('FAIL');
-  notifyDone();
+  notifyDone('FAIL');
 }
 
 function onLoad(e) {
   // needed for dartium compilation errors.
   if (window.compilationError) {
-    showErrorAndExit(window.compilationError);
+    showErrorAndExit("DOMContentLoaded event: window.compilationError = " + calledwindow.compilationError);

[ricow1, 2013/11/15 07:26:33]

long line

[kustermann, 2013/11/19 10:06:10]

Done.

   }
 }
 
@@ -153,10 +226,9 @@ window.addEventListener("DOMContentLoaded", onLoad, false);
 // Note: before renaming this function, note that it is also included in an
 // inlined error handler in the HTML files that wrap DRT tests.
 // See: tools/testing/dart/browser_test.dart
-function externalError(e) {
-  // needed for dartium compilation errors.
-  showErrorAndExit(e && e.message);
-  window.postMessage('unittest-suite-external-error', '*');
+function scriptTagOnErrorCallback(e) {
+  var message = e && e.message;
+  showErrorAndExit('debug', 'script.onError called: ' + message);
 }
 
 document.addEventListener('readystatechange', function () {
@@ -170,7 +242,7 @@ document.addEventListener('readystatechange', function () {
     // posted message.
     setTimeout(function() {
       if (testRunner && !testRunner.startedDartTest) {
-        notifyDone();
+        notifyDone('NOT_STARTED');
       }
     }, 0);
   }, 50);
@@ -195,26 +267,17 @@ document.addEventListener('readystatechange', function () {
 //
 // These messages are used to communicate with the test and will be posted so
 // [processMessage] above can see it.
-function dartPrint(msg) {
-  if ((msg === 'unittest-suite-success')
-      || (msg === 'unittest-suite-done')
-      || (msg === 'unittest-suite-wait-for-done')
-      || (msg === 'dart-calling-main')
-      || (msg === 'dart-main-done')) {
-    window.postMessage(msg, '*');
+function dartPrint(message) {
+  recordEvent('print', message);
+  if ((message === 'unittest-suite-success')
+      || (message === 'unittest-suite-wait-for-done')
+      || (message === 'dart-calling-main')
+      || (message === 'dart-main-done')) {
+    // We have to do this asynchronously, in case error messages are
+    // already in the message queue.
+    window.postMessage(message, '*');
     return;
   }
-  printMessage(msg);
-}
-
-// Prints 'msg' to the console (if available) and to the body of the html
-// document.
-function printMessage(msg) {
-  if (typeof console === 'object') console.warn(msg);
-  var pre = document.createElement('pre');
-  pre.appendChild(document.createTextNode(String(msg)));
-  document.body.appendChild(pre);
-  document.body.appendChild(document.createTextNode('\n'));
 }
 
 // dart2js will generate code to call this function instead of calling
@@ -224,9 +287,7 @@ function dartMainRunner(main) {
   try {
     main();
   } catch (e) {
-    dartPrint(e);
-    if (e.stack) dartPrint(e.stack);
-    window.postMessage('unittest-suite-fail', '*');
+    showErrorAndExit('Exception: ' + e + '\nStack: ' + e.stack);
     return;
   }
   dartPrint('dart-main-done');