Add an additional content_shell per worker for running virtual tests.

Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py

Index: Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py
diff --git a/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py b/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py
index 230eb71d742375681a142143b816527ef633cb54..d2e56e5913ebf9273de5800d1a62aa341d08690b 100644
--- a/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py
+++ b/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py
@@ -43,8 +43,11 @@ from webkitpy.layout_tests.models import testharness_results
 _log = logging.getLogger(__name__)
 
 
-def run_single_test(port, options, results_directory, worker_name, driver, test_input, stop_when_done):
-    runner = SingleTestRunner(port, options, results_directory, worker_name, driver, test_input, stop_when_done)
+def run_single_test(port, options, results_directory, worker_name,
+                    default_driver, virtual_driver, test_input, stop_when_done):
+    runner = SingleTestRunner(
+        port, options, results_directory, worker_name, default_driver,
+        virtual_driver, test_input, stop_when_done)
     try:
         return runner.run()
     except DeviceFailure as e:
@@ -55,12 +58,14 @@ def run_single_test(port, options, results_directory, worker_name, driver, test_
 class SingleTestRunner(object):
     (ALONGSIDE_TEST, PLATFORM_DIR, VERSION_DIR, UPDATE) = ('alongside', 'platform', 'version', 'update')
 
-    def __init__(self, port, options, results_directory, worker_name, driver, test_input, stop_when_done):
+    def __init__(self, port, options, results_directory, worker_name,
+                 default_driver, virtual_driver, test_input, stop_when_done):
         self._port = port
         self._filesystem = port.host.filesystem
         self._options = options
         self._results_directory = results_directory
-        self._driver = driver
+        self._driver = default_driver
+        self._reference_driver = default_driver
         self._timeout = test_input.timeout
         self._worker_name = worker_name
         self._test_name = test_input.test_name
@@ -69,6 +74,14 @@ class SingleTestRunner(object):
         self._should_add_missing_baselines = test_input.should_add_missing_baselines
         self._stop_when_done = stop_when_done
 
+        # If this is a virtual test, run it on the virtual driver so that the
+        # main driver does not need to be restarted. If the test uses the
+        # default flags for it's references, render them using the main driver.
+        if self._port.is_virtual_test(self._test_name):
+            self._driver = virtual_driver
+            if self._port.lookup_virtual_reference_args(self._test_name):
+                self._reference_driver = virtual_driver

[Dirk Pranke, 2015/06/01 22:06:20]

If I'm following this change correctly, this means that any worker that ever
runs both regular (non-virtual) tests and virtual tests will start up and use
two workers, rather than restarting when we switch from a non-virtual shard to a
virtual shard.

This means, I think, that we will have fewer restarts of the regular
content_shell, but we will more often be switching between the two different
drivers (in addition to switching back and forth) during a virtual suite when
the the reftest doesn't use args, we'll switch back and forth between shards).

Do you have any sense of the perf impact of this? I don't really know how much
of a hit we pay for restarts between shards compared to the hit of having more
warm content_shells.

[ojan, 2015/06/01 23:19:54]

Why would switching between different drivers have a perf impact?
You're saying that maybe we should kill the extra driver when we finish with a
shard?

+
         if self._reference_files:
             # Detect and report a test which has a wrong combination of expectation files.
             # For example, if 'foo.html' has two expectation files, 'foo-expected.html' and
@@ -372,7 +385,7 @@ class SingleTestRunner(object):
             reference_test_name = self._port.relative_test_filename(reference_filename)
             reference_test_names.append(reference_test_name)
             driver_input = DriverInput(reference_test_name, self._timeout, image_hash=None, should_run_pixel_test=True, args=args)
-            reference_output = self._driver.run_test(driver_input, self._stop_when_done)
+            reference_output = self._reference_driver.run_test(driver_input, self._stop_when_done)
             test_result = self._compare_output_with_reference(reference_output, test_output, reference_filename, expectation == '!=')
 
             if (expectation == '!=' and test_result.failures) or (expectation == '==' and not test_result.failures):