Add flag to test harness to stop sorting test cases.

tools/testrunner/local/execution.py

 # Copyright 2012 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 47 matching lines @@
   return (job.id, output, time.time() - start_time)
 
 class Runner(object):
 
   def __init__(self, suites, progress_indicator, context):
     datapath = os.path.join("out", "testrunner_data")
     self.perf_data_manager = perfdata.PerfDataManager(datapath)
     self.perfdata = self.perf_data_manager.GetStore(context.arch, context.mode)
     self.tests = [ t for s in suites for t in s.tests ]
     for t in self.tests:
       t.duration = self.perfdata.FetchPerfData(t) or 1.0

[Michael Achenbach, 2014/05/28 11:08:32]

If perf data is conditionally not used, maybe it should also not be harvested?

[Michael Starzinger, 2014/05/28 11:24:56]

IMHO we should still harvest perf-data (assuming by harvest you mean "measure
and store it into the DB") because of the following reasons: (1) When the
test-harness is used without the flag from time to time, then the data will
already by populated from previous runs and (2) should we ever decide to use the
perf-data for something else besides sorting, then the pure data won't hinge on
the sorting flag.

     self._CommonInit(len(self.tests), progress_indicator, context)
 
   def _CommonInit(self, num_tests, progress_indicator, context):
     self.indicator = progress_indicator
     progress_indicator.runner = self
     self.context = context
     self.succeeded = 0
     self.total = num_tests
     self.remaining = num_tests
     self.failed = []
     self.crashed = 0
 
   def Run(self, jobs):
     self.indicator.Starting()
     self._RunInternal(jobs)
     self.indicator.Done()
     if self.failed or self.remaining:
       return 1
     return 0
 
   def _RunInternal(self, jobs):
     pool = Pool(jobs)
     test_map = {}
     # TODO(machenbach): Instead of filling the queue completely before
     # pool.imap_unordered, make this a generator that already starts testing
     # while the queue is filled.
     queue = []
     queued_exception = None
-    for test in sorted(self.tests, key=lambda t: t.duration, reverse=True):
+    if not self.context.no_sorting:
+      self.tests = sorted(self.tests, key=lambda t: t.duration, reverse=True)

[Michael Achenbach, 2014/05/28 11:08:32]

Not totally happy with replacing generators with copied lists. We might loose a
few micro seconds :P

[Jakob Kummerow, 2014/05/28 11:12:16]

Right, should probably use self.tests.sort(key=...) instead. My bad.

[Michael Starzinger, 2014/05/28 11:24:56]

Will upload a follow-up CL to address this.

+    for test in self.tests:
       assert test.id >= 0
       test_map[test.id] = test
       try:
         command = self.GetCommand(test)
       except Exception, e:
         # If this failed, save the exception and re-raise it later (after
         # all other tests have had a chance to run).
         queued_exception = e
         continue
       timeout = self.context.timeout
@@ skipping 16 matching lines @@
         test.duration = result[2]
         has_unexpected_output = test.suite.HasUnexpectedOutput(test)
         if has_unexpected_output:
           self.failed.append(test)
           if test.output.HasCrashed():
             self.crashed += 1
         else:
           self.succeeded += 1
         self.remaining -= 1
         try:
           self.perfdata.UpdatePerfData(test)

[Michael Achenbach, 2014/05/28 11:08:32]

Also here...

[Jakob Kummerow, 2014/05/28 11:12:16]

I disagree. Updating the perfstore should be fast enough that we can do it
anyways. More data points = better data :-)

         except Exception, e:
           print("UpdatePerfData exception: %s" % e)
           pass  # Just keep working.
         self.indicator.HasRun(test, has_unexpected_output)
     finally:
       pool.terminate()
       self.perf_data_manager.close()
     if queued_exception:
       raise queued_exception
 
@@ skipping 12 matching lines @@
            test.suite.GetFlagsForTestCase(test, self.context) +
            self.context.extra_flags)
     return cmd
 
 
 class BreakNowException(Exception):
   def __init__(self, value):
     self.value = value
   def __str__(self):
     return repr(self.value)