Android perf tests: only store the retry results if it's better.

build/android/pylib/perf/test_runner.py

 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Runs a perf test on a single device.

[Dominik Grewe, 2013/10/16 13:51:43]

Is this still accurate? According to the comment below we're running the tests
across multiple devices. Or am I missing something?

[bulach, 2013/10/16 17:03:01]

good point! fixed..

 
 Our buildbot infrastructure requires each slave to run steps serially.
 This is sub-optimal for android, where these steps can run independently on
 multiple connected devices.
 
 The buildbots will run this script multiple times per cycle:
 - First: all steps listed in --steps in will be executed in parallel using all
 connected devices. Step results will be pickled to disk. Each step has a unique
 name. The result code will be ignored if the step name is listed in
 --flaky-steps.
@@ skipping 24 matching lines @@
 options:
   --device: the serial number to be passed to all adb commands.
   --keep_test_server_ports: indicates it's being run as a shard, and shouldn't
   reset test server port allocation.
 """
 
 import datetime
 import logging
 import pickle
 import os
+import re

[Dominik Grewe, 2013/10/16 13:51:43]

Is this needed for anything?

[bulach, 2013/10/16 17:03:01]

nope, removed..

 import sys
 
 from pylib import constants
 from pylib import pexpect
 from pylib.base import base_test_result
 from pylib.base import base_test_runner
 
 
 def PrintTestOutput(test_name):
   """Helper method to print the output of previously executed test_name.
@@ skipping 29 matching lines @@
       device: Device to run the tests.
       tests: a dict mapping test_name to command.
       flaky_tests: a list of flaky test_name.
     """
     super(TestRunner, self).__init__(device, None, 'Release')
     self._options = test_options
     self._tests = tests
     self._flaky_tests = flaky_tests
 
   @staticmethod
+  def _IsBetter(result):
+    if result['actual_exit_code'] == 0:
+      return True
+    pickled = os.path.join(constants.PERF_OUTPUT_DIR,
+                           result['name'])
+    if not os.path.exists(pickled):
+      return True
+    with file(pickled, 'r') as f:
+      previous = pickle.loads(f.read())
+    return result['actual_exit_code'] < previous['actual_exit_code']
+
+  @staticmethod
   def _SaveResult(result):
-    with file(os.path.join(constants.PERF_OUTPUT_DIR,
-                           result['name']), 'w') as f:
-      f.write(pickle.dumps(result))
+    if TestRunner._IsBetter(result):
+      with file(os.path.join(constants.PERF_OUTPUT_DIR,
+                             result['name']), 'w') as f:
+        f.write(pickle.dumps(result))
 
   def _LaunchPerfTest(self, test_name):
     """Runs a perf test.
 
     Args:
       test_name: the name of the test to be executed.
 
     Returns:
       A tuple containing (Output, base_test_result.ResultType)
     """
@@ skipping 15 matching lines @@
         env=os.environ)
     end_time = datetime.datetime.now()
     if exit_code is None:
       exit_code = -1
     logging.info('%s : exit_code=%d in %d secs at %s',
                  test_name, exit_code, (end_time - start_time).seconds,
                  self.device)
     result_type = base_test_result.ResultType.FAIL
     if exit_code == 0:
       result_type = base_test_result.ResultType.PASS
+    actual_exit_code = exit_code
     if test_name in self._flaky_tests:
       # The exit_code is used at the second stage when printing the
       # test output. If the test is flaky, force to "0" to get that step green
       # whilst still gathering data to the perf dashboards.
       # The result_type is used by the test_dispatcher to retry the test.
       exit_code = 0
 
     persisted_result = {
         'name': test_name,
         'output': output,
         'exit_code': exit_code,
+        'actual_exit_code': actual_exit_code,
         'result_type': result_type,
         'total_time': (end_time - start_time).seconds,
         'device': self.device,
         'cmd': cmd,
     }
     self._SaveResult(persisted_result)
 
     return (output, result_type)
 
   def RunTest(self, test_name):
     """Run a perf test on the device.
 
     Args:
       test_name: String to use for logging the test result.
 
     Returns:
       A tuple of (TestRunResults, retry).
     """
     output, result_type = self._LaunchPerfTest(test_name)
     results = base_test_result.TestRunResults()
     results.AddResult(base_test_result.BaseTestResult(test_name, result_type))
     retry = None
     if not results.DidRunPass():
       retry = test_name
     return results, retry