Make Safari tests more robust.

tools/testing/dart/browser_controller.dart

 // Copyright (c) 2013, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 library browser;
 
 import "dart:async";
 import "dart:convert" show LineSplitter, UTF8, JSON;
 import "dart:core";
 import "dart:io";
 import "dart:math" show max, min;
 
 import 'android.dart';
 import 'http_server.dart';
 import 'path.dart';
 import 'utils.dart';
 
+import 'reset_safari.dart' show
+    killAndResetSafari;
+
 class BrowserOutput {
   final StringBuffer stdout = new StringBuffer();
   final StringBuffer stderr = new StringBuffer();
   final StringBuffer eventLog = new StringBuffer();
 }
 
 /** Class describing the interface for communicating with browsers. */
 abstract class Browser {
   BrowserOutput _allBrowserOutput = new BrowserOutput();
   BrowserOutput _testBrowserOutput = new BrowserOutput();
@@ skipping 19 matching lines @@
   Process process;
 
   Function logger;
 
   /**
    * Id of the browser
    */
   String id;
 
   /**
-   * Delete the browser specific caches on startup.
+   * Reset the browser to a known configuration on start-up.
    * Browser specific implementations are free to ignore this.
    */
-  static bool deleteCache = false;
+  static bool resetBrowserConfiguration = false;
 
   /** Print everything (stdout, stderr, usageLog) whenever we add to it */
   bool debugPrint = false;
 
   // This future returns when the process exits. It is also the return value
   // of close()
   Future done;
 
   Browser();
 
@@ skipping 28 matching lines @@
     'ie10',
     'ie11',
     'dartium'
   ];
 
   static const List<String> BROWSERS_WITH_WINDOW_SUPPORT = const [
     'ie11',
     'ie10'
   ];
 
+  /// If [browserName] doesn't support Window.open, we use iframes instead.
+  static bool requiresIframe(String browserName) {
+    return !BROWSERS_WITH_WINDOW_SUPPORT.contains(browserName);
+  }
+
+  static bool requiresFocus(String browserName) {
+    return browserName == "safari";
+  }
+
   // TODO(kustermann): add standard support for chrome on android
   static bool supportedBrowser(String name) {
     return SUPPORTED_BROWSERS.contains(name);
   }
 
   void _logEvent(String event) {
     String toLog = "$this ($id) - $event \n";
     if (debugPrint) print("usageLog: $toLog");
     if (logger != null) logger(toLog);
 
@@ skipping 144 matching lines @@
    * Add useful info about the browser to the _testBrowserOutput.stdout,
    * where it will be reported for failing tests.  Used to report which
    * android device a failing test is running on.
    */
   void logBrowserInfoToTestBrowserOutput() {}
 
   String toString();
 
   /** Starts the browser loading the given url */
   Future<bool> start(String url);
+
+  Future<Null> notifyGetDriverPage() => new Future<Null>.value();

[Bill Hesse, 2016/06/09 17:35:42]

This could use a comment as explanation, that this signals when the driver page
is requested, so actions (like foregrounding the browser) can be taken.  Would a
better name be onDriverPageRequested? 

[ahe, 2016/06/15 08:16:54]

Done.

 }
 
 class Safari extends Browser {
   /**
    * We get the safari version by parsing a version file
    */
   static const String versionFile =
       "/Applications/Safari.app/Contents/version.plist";
 
-  /**
-   * Directories where safari stores state. We delete these if the deleteCache
-   * is set
-   */
-  static const List<String> CACHE_DIRECTORIES = const [
-    "Library/Caches/com.apple.Safari",
-    "Library/Safari",
-    "Library/Saved Application State/com.apple.Safari.savedState",
-    "Library/Caches/Metadata/Safari"
-  ];
+  static const String safariBundleLocation = "/Applications/Safari.app/";
 
-  Future<bool> allowPopUps() {
-    var command = "defaults";
-    var args = [
-      "write",
-      "com.apple.safari",
-      "com.apple.Safari.ContentPageGroupIdentifier."
-          "WebKit2JavaScriptCanOpenWindowsAutomatically",
-      "1"
-    ];
-    return Process.run(command, args).then((result) {
-      if (result.exitCode != 0) {
-        _logEvent("Could not disable pop-up blocking for safari");
-        return false;
+  // Clears the cache if the static resetBrowserConfiguration flag is set.
+  // Returns false if the command to actually clear the cache did not complete.
+  Future<bool> resetConfiguration() async {
+    if (!Browser.resetBrowserConfiguration) return true;
+
+    Completer completer = new Completer();
+    handleUncaughtError(error, StackTrace stackTrace) {
+      if (!completer.isCompleted) {
+        completer.completeError(error, stackTrace);
+      } else {
+        throw new AsyncError(error, stackTrace);
       }
+    }
+    Zone parent = Zone.current;
+    ZoneSpecification specification = new ZoneSpecification(
+        print: (Zone self, ZoneDelegate delegate, Zone zone, String line) {
+          delegate.run(parent, () {
+            _logEvent(line);
+          });
+        });
+    Future zoneWrapper() {
+      Uri safariUri = Uri.base.resolve(safariBundleLocation);
+      return new Future(() => killAndResetSafari(bundle: safariUri))
+          .then(completer.complete);
+    }
+
+    // We run killAndResetSafari in a Zone as opposed to running an external
+    // process. The Zone allows us to collect its output, and protect the rest
+    // of the test infrastructure against errors in it.
+    runZoned(
+        zoneWrapper, zoneSpecification: specification,
+        onError: handleUncaughtError);
+
+    try {
+      await completer.future;
       return true;
-    });
-  }
-
-  Future<bool> deleteIfExists(Iterator<String> paths) {
-    if (!paths.moveNext()) return new Future.value(true);
-    Directory directory = new Directory(paths.current);
-    return directory.exists().then((exists) {
-      if (exists) {
-        _logEvent("Deleting ${paths.current}");
-        return directory
-            .delete(recursive: true)
-            .then((_) => deleteIfExists(paths))
-            .catchError((error) {
-          _logEvent("Failure trying to delete ${paths.current}: $error");
-          return false;
-        });
-      } else {
-        _logEvent("${paths.current} is not present");
-        return deleteIfExists(paths);
-      }
-    });
-  }
-
-  // Clears the cache if the static deleteCache flag is set.
-  // Returns false if the command to actually clear the cache did not complete.
-  Future<bool> clearCache() {
-    if (!Browser.deleteCache) return new Future.value(true);
-    var home = Platform.environment['HOME'];
-    Iterator iterator = CACHE_DIRECTORIES.map((s) => "$home/$s").iterator;
-    return deleteIfExists(iterator);
+    } catch (error, st) {
+      _logEvent("Unable to reset Safari: $error$st");
+      return false;
+    }
   }
 
   Future<String> getVersion() {
     /**
      * Example of the file:
      * <?xml version="1.0" encoding="UTF-8"?>
      * <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
      * <plist version="1.0">
      * <dict>
      *       <key>BuildVersion</key>
@@ skipping 15 matching lines @@
       for (var line in content) {
         if (versionOnNextLine) return line;
         if (line.contains("CFBundleShortVersionString")) {
           versionOnNextLine = true;
         }
       }
       return null;
     });
   }
 
-  void _createLaunchHTML(var path, var url) {
+  Future<Null> _createLaunchHTML(var path, var url) async {
     var file = new File("${path}/launch.html");
-    var randomFile = file.openSync(mode: FileMode.WRITE);
+    var randomFile = await file.open(mode: FileMode.WRITE);
     var content = '<script language="JavaScript">location = "$url"</script>';
-    randomFile.writeStringSync(content);
-    randomFile.close();
+    await randomFile.writeString(content);
+    await randomFile.close();
   }
 
-  Future<bool> start(String url) {
+  Future<bool> start(String url) async {
     _logEvent("Starting Safari browser on: $url");
-    return allowPopUps().then((success) {
-      if (!success) {
-        return false;
-      }
-      return clearCache().then((cleared) {
-        if (!cleared) {
-          _logEvent("Could not clear cache");
-          return false;
-        }
-        // Get the version and log that.
-        return getVersion().then((version) {
-          _logEvent("Got version: $version");
-          return Directory.systemTemp.createTemp().then((userDir) {
-            _cleanup = () {
-              userDir.deleteSync(recursive: true);
-            };
-            _createLaunchHTML(userDir.path, url);
-            var args = ["${userDir.path}/launch.html"];
-            return startBrowserProcess(_binary, args);
-          });
-        }).catchError((error) {
-          _logEvent("Running $_binary --version failed with $error");
-          return false;
-        });
-      });
-    });
+    if (!await resetConfiguration()) {
+      _logEvent("Could not clear cache");
+      return false;
+    }
+    String version;
+    try {
+      version = await getVersion();
+    } catch (error) {
+      _logEvent("Running $_binary --version failed with $error");
+      return false;
+    }
+    _logEvent("Got version: $version");
+    Directory userDir;
+    try {
+      userDir = await Directory.systemTemp.createTemp();
+    } catch (error) {
+      _logEvent("Error creating temporary directory: $error");
+      return false;
+    }
+    _cleanup = () {
+      userDir.deleteSync(recursive: true);
+    };
+    try {
+      await _createLaunchHTML(userDir.path, url);
+    } catch (error) {
+      _logEvent("Error creating launch HTML: $error");
+      return false;
+    }
+    var args = [
+        "-d", "-i", "-m", "-s", "-u", _binary,
+        "${userDir.path}/launch.html"];
+    try {
+      return startBrowserProcess("/usr/bin/caffeinate", args);
+    } catch (error) {
+      _logEvent("Error starting browser process: $error");
+      return false;
+    }
+  }
+
+  Future<Null> notifyGetDriverPage() async {
+    await Process.run("/usr/bin/osascript",
+        ['-e', 'tell application "Safari" to activate']);
   }
 
   String toString() => "Safari";
 }
 
 class Chrome extends Browser {
   String _version = "Version not found yet";
 
   Map<String, String> _getEnvironment() => null;
 
@@ skipping 52 matching lines @@
       return false;
     });
   }
 
   String toString() => "Chrome";
 }
 
 class SafariMobileSimulator extends Safari {
   /**
    * Directories where safari simulator stores state. We delete these if the
-   * deleteCache is set
+   * resetBrowserConfiguration is set
    */
   static const List<String> CACHE_DIRECTORIES = const [
     "Library/Application Support/iPhone Simulator/7.1/Applications"
   ];
 
-  // Clears the cache if the static deleteCache flag is set.
+  // Clears the cache if the static resetBrowserConfiguration flag is set.
   // Returns false if the command to actually clear the cache did not complete.
-  Future<bool> clearCache() {
-    if (!Browser.deleteCache) return new Future.value(true);
+  Future<bool> resetConfiguration() {
+    if (!Browser.resetBrowserConfiguration) return new Future.value(true);
     var home = Platform.environment['HOME'];
     Iterator iterator = CACHE_DIRECTORIES.map((s) => "$home/$s").iterator;
     return deleteIfExists(iterator);
   }
 
   Future<bool> start(String url) {
     _logEvent("Starting safari mobile simulator browser on: $url");
-    return clearCache().then((success) {
+    return resetConfiguration().then((success) {
       if (!success) {
         _logEvent("Could not clear cache, exiting");
         return false;
       }
       var args = [
         "-SimulateApplication",
         "/Applications/Xcode.app/Contents/Developer/Platforms/"
             "iPhoneSimulator.platform/Developer/SDKs/"
             "iPhoneSimulator7.1.sdk/Applications/MobileSafari.app/"
             "MobileSafari",
@@ skipping 46 matching lines @@
         var findString = "REG_SZ";
         var index = result.stdout.indexOf(findString);
         if (index > 0) {
           return result.stdout.substring(index + findString.length).trim();
         }
       }
       return "Could not get the version of internet explorer";
     });
   }
 
-  // Clears the recovery cache if the static deleteCache flag is set.
-  Future<bool> clearCache() {
-    if (!Browser.deleteCache) return new Future.value(true);
+  // Clears the recovery cache if the static resetBrowserConfiguration flag is
+  // set.
+  Future<bool> resetConfiguration() {
+    if (!Browser.resetBrowserConfiguration) return new Future.value(true);
     var localAppData = Platform.environment['LOCALAPPDATA'];
 
     Directory dir = new Directory("$localAppData\\Microsoft\\"
         "Internet Explorer\\Recovery");
     return dir.delete(recursive: true).then((_) {
       return true;
     }).catchError((error) {
       _logEvent("Deleting recovery dir failed with $error");
       return false;
     });
   }
 
   Future<bool> start(String url) {
     _logEvent("Starting ie browser on: $url");
-    return clearCache().then((_) => getVersion()).then((version) {
+    return resetConfiguration().then((_) => getVersion()).then((version) {
       _logEvent("Got version: $version");
       return startBrowserProcess(_binary, [url]);
     });
   }
 
   String toString() => "IE";
 }
 
 class AndroidBrowserConfig {
   final String name;
@@ skipping 277 matching lines @@
 /// requests back from the browsers.
 class BrowserTestRunner {
   static const int MAX_NEXT_TEST_TIMEOUTS = 10;
   static const Duration NEXT_TEST_TIMEOUT = const Duration(seconds: 60);
   static const Duration RESTART_BROWSER_INTERVAL = const Duration(seconds: 60);
 
   /// If the queue was recently empty, don't start another browser.
   static const Duration MIN_NONEMPTY_QUEUE_TIME = const Duration(seconds: 1);
 
   final Map configuration;
-  BrowserTestingServer testingServer;
+  final BrowserTestingServer testingServer;
 
   final String localIp;
-  String browserName;
-  int maxNumBrowsers;
-  bool checkedMode;
+  final String browserName;
+  final int maxNumBrowsers;
+  final bool checkedMode;
   int numBrowsers = 0;
   // Used to send back logs from the browser (start, stop etc)
   Function logger;
 
   int browserIdCounter = 1;
 
   bool testingServerStarted = false;
   bool underTermination = false;
   int numBrowserGetTestTimeouts = 0;
   DateTime lastEmptyTestQueueTime = new DateTime.now();
@@ skipping 26 matching lines @@
   // When no browser is currently starting, _currentStartingBrowserId is null.
   bool get aBrowserIsCurrentlyStarting => _currentStartingBrowserId != null;
   void markCurrentlyStarting(String id) {
     _currentStartingBrowserId = id;
   }
 
   void markNotCurrentlyStarting(String id) {
     if (_currentStartingBrowserId == id) _currentStartingBrowserId = null;
   }
 
-  // If [browserName] doesn't support opening new windows, we use new iframes
-  // instead.
-  bool get useIframe =>
-      !Browser.BROWSERS_WITH_WINDOW_SUPPORT.contains(browserName);
-
-  /// The optional testingServer parameter allows callers to pass in
-  /// a testing server with different behavior than the  default
-  /// BrowserTestServer. The url handlers of the testingServer are
-  /// overwritten, so an existing handler can't be shared between instances.
   BrowserTestRunner(
-      this.configuration, this.localIp, this.browserName, this.maxNumBrowsers,
-      {BrowserTestingServer this.testingServer}) {
-    checkedMode = configuration['checked'];
-    if (browserName == 'ff') browserName = 'firefox';
+      Map configuration,
+      String localIp,
+      String browserName,
+      this.maxNumBrowsers)
+      : configuration = configuration,
+        localIp = localIp,
+        browserName = (browserName == 'ff') ? 'firefox' : browserName,
+        checkedMode = configuration['checked'],
+        testingServer = new BrowserTestingServer(
+            configuration, localIp,
+            Browser.requiresIframe(browserName),
+            Browser.requiresFocus(browserName)) {
+    testingServer.testRunner = this;
   }
 
   Future start() async {
-    if (testingServer == null) {
-      testingServer =
-          new BrowserTestingServer(configuration, localIp, useIframe);
-    }
     await testingServer.start();
     testingServer
       ..testDoneCallBack = handleResults
       ..testStatusUpdateCallBack = handleStatusUpdate
       ..testStartedCallBack = handleStarted
       ..nextTestCallBack = getNextTest;
     if (browserName == 'chromeOnAndroid') {
       var idbNames = await AdbHelper.listDevices();
       idleAdbDevices = new List.from(idbNames.map((id) => new AdbDevice(id)));
       maxNumBrowsers = min(maxNumBrowsers, idleAdbDevices.length);
@@ skipping 316 matching lines @@
   ///                           and run tests ...
   /// GET /next_test/BROWSER_ID -- returns "WAIT" "TERMINATE" or "url#id"
   /// where url is the test to run, and id is the id of the test.
   /// If there are currently no available tests the waitSignal is send
   /// back. If we are in the process of terminating the terminateSignal
   /// is send back and the browser will stop requesting new tasks.
   /// POST /report/BROWSER_ID?id=NUM -- sends back the dom of the executed
   ///                                   test
 
   final String localIp;
+  final bool useIframe;
+  final bool requiresFocus;
+  BrowserTestRunner testRunner;
 
   static const String driverPath = "/driver";
   static const String nextTestPath = "/next_test";
   static const String reportPath = "/report";
   static const String statusUpdatePath = "/status_update";
   static const String startedPath = "/started";
   static const String waitSignal = "WAIT";
   static const String terminateSignal = "TERMINATE";
 
   var testCount = 0;
   var errorReportingServer;
   bool underTermination = false;
-  bool useIframe = false;
 
   Function testDoneCallBack;
   Function testStatusUpdateCallBack;
   Function testStartedCallBack;
   Function nextTestCallBack;
 
-  BrowserTestingServer(this.configuration, this.localIp, this.useIframe);
+  BrowserTestingServer(
+      this.configuration, this.localIp, this.useIframe, this.requiresFocus);
 
   Future start() {
     var test_driver_error_port = configuration['test_driver_error_port'];
     return HttpServer
         .bind(localIp, test_driver_error_port)
         .then(setupErrorServer)
         .then(setupDispatchingServer);
   }
 
   void setupErrorServer(HttpServer server) {
@@ skipping 41 matching lines @@
       noCache(request);
       handleReport(
           request, browserId(request, statusUpdatePath), testId(request),
           isStatusUpdate: true);
     });
     server.addHandler(startedPath, (HttpRequest request) {
       noCache(request);
       handleStarted(request, browserId(request, startedPath), testId(request));
     });
 
-    makeSendPageHandler(String prefix) => (HttpRequest request) {
-          noCache(request);
-          var textResponse = "";
-          if (prefix == driverPath) {
-            textResponse = getDriverPage(browserId(request, prefix));
-            request.response.headers.set('Content-Type', 'text/html');
-          }
-          if (prefix == nextTestPath) {
-            textResponse = getNextTest(browserId(request, prefix));
-            request.response.headers.set('Content-Type', 'text/plain');
-          }
-          request.response.write(textResponse);
-          request.listen((_) {}, onDone: request.response.close);
-          request.response.done.catchError((error) {
-            if (!underTermination) {
-              print("URI ${request.uri}");
-              print("Textresponse $textResponse");
-              throw "Error returning content to browser: $error";
-            }
+    void sendPageHandler(HttpRequest request) {
+      // Do NOT make this method async. We need to call catchError below
+      // synchronously to avoid unhandled asynchronous errors.
+      noCache(request);
+      Future<String> textResponse;
+      if (request.uri.path.startsWith(driverPath)) {
+        textResponse = getDriverPage(browserId(request, driverPath));
+        request.response.headers.set('Content-Type', 'text/html');
+      } else if (request.uri.path.startsWith(nextTestPath)) {
+        textResponse = new Future<String>.value(
+            getNextTest(browserId(request, nextTestPath)));
+        request.response.headers.set('Content-Type', 'text/plain');
+      } else {
+        textResponse = new Future<String>.value("");
+      }
+      request.response.done.catchError((error) {
+        if (!underTermination) {
+          return textResponse.then((String text) {
+            print("URI ${request.uri}");
+            print("textResponse $textResponse");
+            throw "Error returning content to browser: $error";
           });
-        };
-    server.addHandler(driverPath, makeSendPageHandler(driverPath));
-    server.addHandler(nextTestPath, makeSendPageHandler(nextTestPath));
+        }
+      });
+      textResponse.then((String text) async {
+        request.response.write(text);
+        await request.listen(null).asFuture();
+        // Ignoring the returned closure as it returns the 'done' future
+        // which alread has catchError installed above.
+        request.response.close();
+      });
+    }
+
+    server.addHandler(driverPath, sendPageHandler);
+    server.addHandler(nextTestPath, sendPageHandler);
   }
 
   void handleReport(HttpRequest request, String browserId, var testId,
       {bool isStatusUpdate}) {
     StringBuffer buffer = new StringBuffer();
     request.transform(UTF8.decoder).listen((data) {
       buffer.write(data);
     }, onDone: () {
       String back = buffer.toString();
       request.response.close();
@@ skipping 38 matching lines @@
     if (errorReportingServer == null) {
       print("Bad browser testing server, you are not started yet. Can't "
           "produce driver url");
       exit(1);
       // This should never happen - exit immediately;
     }
     var port = configuration['_servers_'].port;
     return "http://$localIp:$port/driver/$browserId";
   }
 
-  String getDriverPage(String browserId) {
+  Future<String> getDriverPage(String browserId) async {
+    await testRunner.browserStatus[browserId].browser.notifyGetDriverPage();
     var errorReportingUrl =
         "http://$localIp:${errorReportingServer.port}/$browserId";
     String driverContent = """
 <!DOCTYPE html><html>
 <head>
+  <title>Driving page</title>
   <style>
-    body {
-      margin: 0;
-    }
-    .box {
-      overflow: hidden;
-      overflow-y: auto;
-      position: absolute;
-      left: 0;
-      right: 0;
-    }
-    .controller.box {
-      height: 75px;
-      top: 0;
-    }
-    .test.box {
-      top: 75px;
-      bottom: 0;
-    }
+.big-notice {
+  background-color: red;
+  color: white;
+  font-weight: bold;
+  font-size: xx-large;
+  text-align: center;
+}
+.controller.box {
+  white-space: nowrap;
+  overflow: scroll;
+  height: 6em;
+}
+body {
+  font-family: sans-serif;
+}
+body div {
+  padding-top: 10px;
+}
   </style>
-  <title>Driving page</title>
   <script type='text/javascript'>
     var STATUS_UPDATE_INTERVAL = 10000;
 
     function startTesting() {
       var number_of_tests = 0;
       var current_id;
       var next_id;
 
       // Has the test in the current iframe reported that it is done?
       var test_completed = true;
@@ skipping 109 matching lines @@
         if (use_iframe) {
           if (html_test) {
             window.addEventListener('detect_errors', setChildHandlers, false);
             embedded_iframe.onload = checkChildHandlersInstalled;
           } else {
             embedded_iframe.onload = null;
           }
           embedded_iframe_div.removeChild(embedded_iframe);
           embedded_iframe = document.createElement('iframe');
           embedded_iframe.id = "embedded_iframe";
-          embedded_iframe.style="width:100%;height:100%";
+          embedded_iframe.width='800px';
+          embedded_iframe.height='600px';
           embedded_iframe_div.appendChild(embedded_iframe);
           embedded_iframe.src = url;
         } else {
           if (typeof testing_window != 'undefined') {
             testing_window.close();
           }
           testing_window = window.open(url);
         }
         test_started = false;
         test_completed = false;
@@ skipping 151 matching lines @@
 
       if (!html_test) {
         window.addEventListener('message', messageHandler, false);
         waitForDone = false;
       }
       getNextTask();
     }
   </script>
 </head>
   <body onload="startTesting()">
+
+    <div class='big-notice'>
+      Please keep this window in focus at all times.
+    </div>
+
+    <div>
+      Some browsers, Safari, in particular, may pause JavaScript when not
+      visible to conserve power consumption and CPU resources. In addition,
+      some tests of focus events will not work correctly if this window doesn't
+      have focus. It's also advisable to close any other programs that may open
+      modal dialogs, for example, Chrome with Calendar open.
+    </div>
+
     <div class="controller box">
     Dart test driver, number of tests: <span id="number"></span><br>
     Currently executing: <span id="currently_executing"></span><br>
     Unhandled error: <span id="unhandled_error"></span>
     </div>
     <div id="embedded_iframe_div" class="test box">
-      <iframe style="width:100%;height:100%;" id="embedded_iframe"></iframe>
+      <iframe id="embedded_iframe"></iframe>
     </div>
   </body>
 </html>
 """;
     return driverContent;
   }
 }