Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(3970)

Unified Diff: build/android/pylib/perf/setup.py

Issue 1148873007: Fix last_devices to be quieter, and improve device affinity. (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Created 5 years, 7 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
Index: build/android/pylib/perf/setup.py
diff --git a/build/android/pylib/perf/setup.py b/build/android/pylib/perf/setup.py
index 99c3e19a643150651dcc424325172cd2024617af..9ee65bf6771747cd0b529f05bd8149fa19aecf9e 100644
--- a/build/android/pylib/perf/setup.py
+++ b/build/android/pylib/perf/setup.py
@@ -6,6 +6,7 @@
import json
import fnmatch
+import hashlib
import logging
import os
import shutil
@@ -18,17 +19,6 @@ from pylib.perf import test_runner
from pylib.utils import test_environment
-def _GetAllDevices():
- devices_path = os.path.join(os.environ.get('CHROMIUM_OUT_DIR', 'out'),
- device_list.LAST_DEVICES_FILENAME)
- try:
- devices = device_list.GetPersistentDeviceList(devices_path)
- except IOError as e:
- logging.error('Unable to find %s [%s]', devices_path, e)
- devices = android_commands.GetAttachedDevices()
jbudorick 2015/05/23 01:06:49 How is this still around...
luqui 2015/05/27 20:01:12 Looks like I forgot to sync before starting work o
- return sorted(devices)
-
-
def _GetStepsDictFromSingleStep(test_options):
# Running a single command, build the tests structure.
steps_dict = {
@@ -55,6 +45,39 @@ def _GetStepsDict(test_options):
return steps
+def DistributeAffinities(devices):
+ """Create a mapping from device to affinity using a round-robin hash scheme.
+ This is a way to make sure that affinity -> device mapping is relatively
jbudorick 2015/05/23 01:06:49 nit: line break after the first line of the docstr
luqui 2015/05/27 20:01:12 Done.
+ stable while still maximizing the use of resources."""
jbudorick 2015/05/23 01:06:49 I'm not sure this is stable if you lose one of the
luqui 2015/05/27 20:01:12 IIUC the persistent device list basically messes u
+ if not devices:
+ return {}
+
+ device_cycle = []
+ while len(device_cycle) < test_runner.NUM_DEVICE_AFFINITIES:
+ device_cycle.extend(devices)
+ device_cycle = device_cycle[:test_runner.NUM_DEVICE_AFFINITIES]
+
+ affinity_to_device = {}
+
+ # Hash and rehash each device ID until we get a unique affinity.
+ for device in device_cycle:
+ cur_hash = device
+ while True:
+ cur_hash = hashlib.md5(cur_hash).hexdigest()
+ affinity = int(int(cur_hash, 16) % test_runner.NUM_DEVICE_AFFINITIES)
+ if affinity not in affinity_to_device:
+ affinity_to_device[affinity] = device
+ break
+
+ # Invert the dictionary we just built.
+ device_to_affinity = {}
+ for k, v in affinity_to_device.iteritems():
+ if v not in device_to_affinity:
+ device_to_affinity[v] = []
+ device_to_affinity[v].append(k)
+
+ return device_to_affinity
+
def Setup(test_options):
"""Create and return the test runner factory and tests.
@@ -74,8 +97,8 @@ def Setup(test_options):
test_environment.CleanupLeftoverProcesses()
forwarder.Forwarder.UseMultiprocessing()
- # We want to keep device affinity, so return all devices ever seen.
- all_devices = _GetAllDevices()
+ all_devices = android_commands.GetAttachedDevices()
jbudorick 2015/05/23 01:06:49 device_utils.DeviceUtils.HealthyDevices() if you
luqui 2015/05/27 20:01:12 Done.
+ affinity_map = DistributeAffinities(all_devices)
steps_dict = _GetStepsDict(test_options)
sorted_step_names = sorted(steps_dict['steps'].keys())
@@ -89,9 +112,9 @@ def Setup(test_options):
with file(test_options.flaky_steps, 'r') as f:
flaky_steps = json.load(f)
- def TestRunnerFactory(device, shard_index):
+ def TestRunnerFactory(device, _):
+ affinities = affinity_map[device]
return test_runner.TestRunner(
- test_options, device, shard_index, len(all_devices),
- steps_dict, flaky_steps)
+ test_options, device, affinities, steps_dict, flaky_steps)
return (TestRunnerFactory, sorted_step_names, all_devices)

Powered by Google App Engine
This is Rietveld 408576698