Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(356)

Side by Side Diff: build/android/pylib/perf/test_runner.py

Issue 1148873007: Fix last_devices to be quieter, and improve device affinity. (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Created 5 years, 7 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 # Copyright 2013 The Chromium Authors. All rights reserved. 1 # Copyright 2013 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 """Runs perf tests. 5 """Runs perf tests.
6 6
7 Our buildbot infrastructure requires each slave to run steps serially. 7 Our buildbot infrastructure requires each slave to run steps serially.
8 This is sub-optimal for android, where these steps can run independently on 8 This is sub-optimal for android, where these steps can run independently on
9 multiple connected devices. 9 multiple connected devices.
10 10
(...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after
58 import threading 58 import threading
59 import time 59 import time
60 60
61 from pylib import cmd_helper 61 from pylib import cmd_helper
62 from pylib import constants 62 from pylib import constants
63 from pylib import forwarder 63 from pylib import forwarder
64 from pylib.base import base_test_result 64 from pylib.base import base_test_result
65 from pylib.base import base_test_runner 65 from pylib.base import base_test_runner
66 from pylib.device import device_errors 66 from pylib.device import device_errors
67 67
68 NUM_DEVICE_AFFINITIES = 8
jbudorick 2015/05/23 01:06:49 What if the number of devices is higher than 8? I
luqui 2015/05/27 20:01:12 As far as I could tell there are no specs of devic
shatch 2015/05/28 13:59:17 This isn't a thing yet, but we did ask for some in
jbudorick 2015/05/28 14:03:16 Currently there aren't, and I don't think we have
69
68 70
69 def OutputJsonList(json_input, json_output): 71 def OutputJsonList(json_input, json_output):
70 with file(json_input, 'r') as i: 72 with file(json_input, 'r') as i:
71 all_steps = json.load(i) 73 all_steps = json.load(i)
72 step_names = all_steps['steps'].keys() 74 step_names = all_steps['steps'].keys()
73 with file(json_output, 'w') as o: 75 with file(json_output, 'w') as o:
74 o.write(json.dumps(step_names)) 76 o.write(json.dumps(step_names))
75 return 0 77 return 0
76 78
77 79
(...skipping 72 matching lines...) Expand 10 before | Expand all | Expand 10 after
150 if now - self._tick >= _HeartBeatLogger._PRINT_INTERVAL: 152 if now - self._tick >= _HeartBeatLogger._PRINT_INTERVAL:
151 self._tick = now 153 self._tick = now
152 print '--single-step output length %d' % self._len 154 print '--single-step output length %d' % self._len
153 sys.stdout.flush() 155 sys.stdout.flush()
154 156
155 def stop(self): 157 def stop(self):
156 self._stopped.set() 158 self._stopped.set()
157 159
158 160
159 class TestRunner(base_test_runner.BaseTestRunner): 161 class TestRunner(base_test_runner.BaseTestRunner):
160 def __init__(self, test_options, device, shard_index, max_shard, tests, 162 def __init__(self, test_options, device, affinities, tests, flaky_tests):
161 flaky_tests):
162 """A TestRunner instance runs a perf test on a single device. 163 """A TestRunner instance runs a perf test on a single device.
163 164
164 Args: 165 Args:
165 test_options: A PerfOptions object. 166 test_options: A PerfOptions object.
166 device: Device to run the tests. 167 device: Device to run the tests.
167 shard_index: the index of this device. 168 affinities: the list of affinities to run on this shard.
168 max_shards: the maximum shard index.
169 tests: a dict mapping test_name to command. 169 tests: a dict mapping test_name to command.
170 flaky_tests: a list of flaky test_name. 170 flaky_tests: a list of flaky test_name.
171 """ 171 """
172 super(TestRunner, self).__init__(device, None, 'Release') 172 super(TestRunner, self).__init__(device, None, 'Release')
173 self._options = test_options 173 self._options = test_options
174 self._shard_index = shard_index 174 self._affinities = affinities
175 self._max_shard = max_shard
176 self._tests = tests 175 self._tests = tests
177 self._flaky_tests = flaky_tests 176 self._flaky_tests = flaky_tests
178 self._output_dir = None 177 self._output_dir = None
179 178
180 @staticmethod 179 @staticmethod
181 def _IsBetter(result): 180 def _IsBetter(result):
182 if result['actual_exit_code'] == 0: 181 if result['actual_exit_code'] == 0:
183 return True 182 return True
184 pickled = os.path.join(constants.PERF_OUTPUT_DIR, 183 pickled = os.path.join(constants.PERF_OUTPUT_DIR,
185 result['name']) 184 result['name'])
186 if not os.path.exists(pickled): 185 if not os.path.exists(pickled):
187 return True 186 return True
188 with file(pickled, 'r') as f: 187 with file(pickled, 'r') as f:
189 previous = pickle.loads(f.read()) 188 previous = pickle.loads(f.read())
190 return result['actual_exit_code'] < previous['actual_exit_code'] 189 return result['actual_exit_code'] < previous['actual_exit_code']
191 190
192 @staticmethod 191 @staticmethod
193 def _SaveResult(result): 192 def _SaveResult(result):
194 if TestRunner._IsBetter(result): 193 if TestRunner._IsBetter(result):
195 with file(os.path.join(constants.PERF_OUTPUT_DIR, 194 with file(os.path.join(constants.PERF_OUTPUT_DIR,
196 result['name']), 'w') as f: 195 result['name']), 'w') as f:
197 f.write(pickle.dumps(result)) 196 f.write(pickle.dumps(result))
198 197
199 def _CheckDeviceAffinity(self, test_name): 198 def _CheckDeviceAffinity(self, test_name):
200 """Returns True if test_name has affinity for this shard.""" 199 """Returns True if test_name has affinity for this shard."""
201 affinity = (self._tests['steps'][test_name]['device_affinity'] % 200 affinity = self._tests['steps'][test_name]['device_affinity']
202 self._max_shard) 201 assert 0 <= affinity and affinity < NUM_DEVICE_AFFINITIES, (
jbudorick 2015/05/23 01:06:49 This should be an exception. We generally avoid as
luqui 2015/05/27 20:01:12 Done.
203 if self._shard_index == affinity: 202 'Got out-of-range device affinity %s' % affinity)
203 if affinity in self._affinities:
204 return True 204 return True
205 logging.info('Skipping %s on %s (affinity is %s, device is %s)', 205 logging.info(
206 test_name, self.device_serial, affinity, self._shard_index) 206 'Skipping %s on %s (affinity is %s, allowed affinities are %s)',
207 test_name, self.device_serial, affinity, self._affinities)
207 return False 208 return False
208 209
209 def _CleanupOutputDirectory(self): 210 def _CleanupOutputDirectory(self):
210 if self._output_dir: 211 if self._output_dir:
211 shutil.rmtree(self._output_dir, ignore_errors=True) 212 shutil.rmtree(self._output_dir, ignore_errors=True)
212 self._output_dir = None 213 self._output_dir = None
213 214
214 def _ReadChartjsonOutput(self): 215 def _ReadChartjsonOutput(self):
215 if not self._output_dir: 216 if not self._output_dir:
216 return '' 217 return ''
(...skipping 110 matching lines...) Expand 10 before | Expand all | Expand 10 after
327 Returns: 328 Returns:
328 A tuple of (TestRunResults, retry). 329 A tuple of (TestRunResults, retry).
329 """ 330 """
330 _, result_type = self._LaunchPerfTest(test_name) 331 _, result_type = self._LaunchPerfTest(test_name)
331 results = base_test_result.TestRunResults() 332 results = base_test_result.TestRunResults()
332 results.AddResult(base_test_result.BaseTestResult(test_name, result_type)) 333 results.AddResult(base_test_result.BaseTestResult(test_name, result_type))
333 retry = None 334 retry = None
334 if not results.DidRunPass(): 335 if not results.DidRunPass():
335 retry = test_name 336 retry = test_name
336 return results, retry 337 return results, retry
OLDNEW
« build/android/pylib/perf/setup.py ('K') | « build/android/pylib/perf/setup.py ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698