[test] Use generator to accelerate test runner startup.

tools/testrunner/local/execution.py

 # Copyright 2012 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 196 matching lines @@
     self.indicator.Starting()
     self._RunInternal(jobs)
     self.indicator.Done()
     if self.failed or self.remaining:
       return 1
     return 0
 
   def _RunInternal(self, jobs):
     pool = Pool(jobs)
     test_map = {}
-    # TODO(machenbach): Instead of filling the queue completely before
-    # pool.imap_unordered, make this a generator that already starts testing
-    # while the queue is filled.
-    queue = []
-    queued_exception = None
-    for test in self.tests:
-      assert test.id >= 0
-      test_map[test.id] = test
-      try:
-        queue.append([self._GetJob(test)])
-      except Exception, e:
-        # If this failed, save the exception and re-raise it later (after
-        # all other tests have had a chance to run).
-        queued_exception = e
-        continue
+    queued_exception = [None]
+    def gen_tests():
+      for test in self.tests:
+        assert test.id >= 0
+        test_map[test.id] = test
+        try:
+          yield [self._GetJob(test)]
+        except Exception, e:
+          # If this failed, save the exception and re-raise it later (after
+          # all other tests have had a chance to run).
+          queued_exception[0] = e
+          continue
     try:
-      it = pool.imap_unordered(RunTest, queue)
+      it = pool.imap_unordered(RunTest, gen_tests())
       for result in it:
         if result.heartbeat:
           self.indicator.Heartbeat()
           continue
         test = test_map[result.value[0]]
         if self.context.predictable:
           update_perf = self._ProcessTestPredictable(test, result.value, pool)
         else:
           update_perf = self._ProcessTestNormal(test, result.value, pool)
         if update_perf:
           self._RunPerfSafe(lambda: self.perfdata.UpdatePerfData(test))
     finally:
       self._VerbosePrint("Closing process pool.")
       pool.terminate()
       self._VerbosePrint("Closing database connection.")
       self._RunPerfSafe(lambda: self.perf_data_manager.close())
       if self.perf_failures:
         # Nuke perf data in case of failures. This might not work on windows as
         # some files might still be open.
         print "Deleting perf test data due to db corruption."
         shutil.rmtree(self.datapath)
-    if queued_exception:
-      raise queued_exception
+    if queued_exception[0]:
+      raise queued_exception[0]
 
     # Make sure that any allocations were printed in predictable mode (if we
     # ran any tests).
     assert (
         not self.total or
         not self.context.predictable or
         self.printed_allocations
     )
 
   def _VerbosePrint(self, text):
@@ skipping 15 matching lines @@
            test.suite.GetFlagsForTestCase(test, self.context) +
            self.context.extra_flags)
     return cmd
 
 
 class BreakNowException(Exception):
   def __init__(self, value):
     self.value = value
   def __str__(self):
     return repr(self.value)