OLD | NEW |
1 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved. | 1 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 | 5 |
6 # DESCRIPTION : | 6 # DESCRIPTION : |
7 # | 7 # |
8 # This library provides common types and routines for the factory ui | 8 # This library provides common types and routines for the factory ui |
9 # infrastructure. This library explicitly does not import gtk, to | 9 # infrastructure. This library explicitly does not import gtk, to |
10 # allow its use by the autotest control process. | 10 # allow its use by the autotest control process. |
11 | 11 |
12 | 12 |
13 import subprocess | 13 import subprocess |
14 import sys | 14 import sys |
15 import time | 15 import time |
16 | 16 |
17 | 17 |
| 18 ACTIVE = 'ACTIVE' |
| 19 PASSED = 'PASS' |
| 20 FAILED = 'FAIL' |
| 21 UNTESTED = 'UNTESTED' |
| 22 |
| 23 STATUS_CODE_MAP = { |
| 24 'START': ACTIVE, |
| 25 'GOOD': PASSED, |
| 26 'FAIL': FAILED, |
| 27 'ERROR': FAILED} |
| 28 |
| 29 |
18 LOG_PATH = '/var/log/factory.log' | 30 LOG_PATH = '/var/log/factory.log' |
19 RESULT_FILE_PATH = '/var/run/factory_test_result' | 31 DATA_PREFIX = 'FACTORY_DATA:' |
20 | |
21 | 32 |
22 def log(s): | 33 def log(s): |
23 print >> sys.stderr, 'FACTORY: ' + s | 34 print >> sys.stderr, 'FACTORY: ' + s |
24 | 35 |
| 36 def log_shared_data(key, value): |
| 37 print >> sys.stderr, '%s %s=%s' % (DATA_PREFIX, key, repr(value)) |
25 | 38 |
26 class TestData: | |
27 '''Factory-specific information on the tests to be run. The label | |
28 and trigger fields contain the description strings to be shown in | |
29 the test control list of the UI. The trigger field specifies the | |
30 keyboard shortcut to allow on-demain out-of-order test activation. | |
31 The dargs field allows test specific extra arguments.''' | |
32 | 39 |
33 def __init__(self, label_en='', label_zw='', formal_name=None, | 40 class FactoryTest: |
34 tag_prefix=None, trigger=None, automated_seq=[], dargs={}, | |
35 repeat_forever=False): | |
36 self.__dict__.update(vars()) | |
37 | |
38 def __repr__(self): | 41 def __repr__(self): |
39 d = ['%s=%s' % (l, repr(v)) | 42 d = ['%s=%s' % (l, repr(v)) |
40 for l, v in self.__dict__.items() | 43 for l, v in self.__dict__.items() |
41 if l != 'self'] | 44 if l != 'self'] |
42 c = ('%s' % self.__class__).rpartition('.')[2] | 45 c = ('%s' % self.__class__).rpartition('.')[2] |
43 return '%s(%s)' % (c, ','.join(d)) | 46 return '%s(%s)' % (c, ','.join(d)) |
44 | 47 |
45 | 48 class FactoryAutotestTest(FactoryTest): |
46 def test_map_index(formal_name, tag_prefix): | 49 # Placeholder parent for tests with autotest_name fields. |
47 return formal_name + '.' + tag_prefix | 50 pass |
48 | 51 |
49 | 52 class OperatorTest(FactoryAutotestTest): |
50 def make_test_map(test_list): | 53 def __init__(self, label_en='', label_zw='', autotest_name=None, |
51 return dict((test_map_index(test.formal_name, test.tag_prefix), test) | 54 kbd_shortcut=None, dargs={}, drop_caches=False): |
52 for test in test_list) | 55 self.__dict__.update(vars()) |
53 | 56 |
54 | 57 class InformationScreen(OperatorTest): |
55 def make_trigger_set(test_list): | 58 # These tests never pass or fail, just return to untested state. |
56 trigger_map = dict((test.trigger, test) for test in test_list) | 59 pass |
57 delta = set(test_list) - set(trigger_map.values()) | 60 |
58 for test in delta: | 61 class AutomatedSequence(FactoryTest): |
59 collision = trigger_map[test.trigger] | 62 def __init__(self, label_en='', label_zw='', subtest_tag_prefix=None, |
60 log('ERROR: tests %s and %s both have trigger %s' % | 63 kbd_shortcut=None, subtest_list=[]): |
61 (test.label_en, collision.label_en, test.trigger)) | 64 self.__dict__.update(vars()) |
62 assert not delta | 65 |
63 return set(trigger_map) | 66 class AutomatedSubTest(FactoryAutotestTest): |
| 67 def __init__(self, label_en='', label_zw='', autotest_name=None, |
| 68 dargs={}, drop_caches=False): |
| 69 self.__dict__.update(vars()) |
| 70 |
| 71 class AutomatedRebootSubTest(FactoryAutotestTest): |
| 72 def __init__(self, label_en='', label_zw='', iterations=None, |
| 73 autotest_name='factory_RebootStub', dargs={}, |
| 74 drop_caches=False): |
| 75 self.__dict__.update(vars()) |
| 76 |
| 77 |
| 78 class TestDatabase: |
| 79 |
| 80 def __init__(self, test_list): |
| 81 self.test_queue = [t for t in reversed(test_list)] |
| 82 self._subtest_parent_map = {} |
| 83 self._tag_prefix_map = {} |
| 84 for test in test_list: |
| 85 if not isinstance(test, AutomatedSequence): |
| 86 self._tag_prefix_map[test] = test.kbd_shortcut |
| 87 continue |
| 88 step_count = 1 |
| 89 for subtest in test.subtest_list: |
| 90 self._subtest_parent_map[subtest] = test |
| 91 if not isinstance(subtest, FactoryAutotestTest): |
| 92 continue |
| 93 tag_prefix = ('%s_s%d' % (test.subtest_tag_prefix, step_count)) |
| 94 self._tag_prefix_map[subtest] = tag_prefix |
| 95 step_count += 1 |
| 96 self.seq_test_set = set(test for test in test_list |
| 97 if isinstance(test, AutomatedSequence)) |
| 98 self.subtest_set = set(reduce(lambda x, y: x + y, |
| 99 [test.subtest_list for test in |
| 100 self.seq_test_set], [])) |
| 101 self._subtest_map = dict((self._tag_prefix_map[st], st) |
| 102 for st in self.subtest_set) |
| 103 self._unique_name_map = dict((self.get_unique_name(t), t) |
| 104 for t in self._tag_prefix_map) |
| 105 self._kbd_shortcut_map = dict((test.kbd_shortcut, test) |
| 106 for test in test_list) |
| 107 self.kbd_shortcut_set = set(self._kbd_shortcut_map) |
| 108 |
| 109 # Validate keyboard shortcut uniqueness. |
| 110 assert(None not in self.kbd_shortcut_set) |
| 111 delta = set(test_list) - set(self._kbd_shortcut_map.values()) |
| 112 for test in delta: |
| 113 collision = kbd_shortcut_map[test.kbd_shortcut] |
| 114 log('ERROR: tests %s and %s both have kbd_shortcut %s' % |
| 115 (test.label_en, collision.label_en, test.kbd_shortcut)) |
| 116 assert not delta |
| 117 |
| 118 def get_test_by_details(self, autotest_name, tag_prefix): |
| 119 unique_name = '%s.%s' % (autotest_name, tag_prefix) |
| 120 return self._unique_name_map.get(unique_name) |
| 121 |
| 122 def get_test_by_kbd_shortcut(self, kbd_shortcut): |
| 123 return self._kbd_shortcut_map.get(kbd_shortcut) |
| 124 |
| 125 def get_unique_name(self, test): |
| 126 if isinstance(test, AutomatedSequence): |
| 127 return test.subtest_tag_prefix |
| 128 return '%s.%s' % (test.autotest_name, self._tag_prefix_map[test]) |
| 129 |
| 130 def get_tag_prefix(self, test): |
| 131 return self._tag_prefix_map[test] |
| 132 |
| 133 def get_all_tests(self): |
| 134 return set(self.test_queue) | self.subtest_set |
| 135 |
| 136 def get_subtest_parent(self, test): |
| 137 return self._subtest_parent_map.get(test) |
| 138 |
| 139 def get_subtest_by_tag_prefix(self, tag_prefix): |
| 140 return self._subtest_map.get(tag_prefix) |
| 141 |
| 142 |
| 143 class StatusMap: |
| 144 |
| 145 class Entry: |
| 146 |
| 147 def __init__(self): |
| 148 self.status = UNTESTED |
| 149 self.count = 0 |
| 150 self.label_box = None |
| 151 self.error_msg = None |
| 152 |
| 153 def __init__(self, test_list, status_file_path): |
| 154 self.test_db = TestDatabase(test_list) |
| 155 all_tests = self.test_db.get_all_tests() |
| 156 self._status_map = dict((t, StatusMap.Entry()) for t in all_tests) |
| 157 self._status_file_path = status_file_path |
| 158 self._status_file_pos = 0 |
| 159 self.read_new_data() |
| 160 |
| 161 def lookup_status(self, test): |
| 162 return self._status_map[test].status |
| 163 |
| 164 def lookup_count(self, test): |
| 165 return self._status_map[test].count |
| 166 |
| 167 def lookup_label_box(self, test): |
| 168 return self._status_map[test].label_box |
| 169 |
| 170 def lookup_error_msg(self, test): |
| 171 return self._status_map[test].error_msg |
| 172 |
| 173 def lookup_tag(self, test): |
| 174 tag_prefix = self.test_db.get_tag_prefix(test) |
| 175 count = self._status_map[test].count |
| 176 return '%s_%s' % (tag_prefix, count) |
| 177 |
| 178 def incr_count(self, test): |
| 179 self._status_map[test].count += 1 |
| 180 |
| 181 def filter(self, target_status): |
| 182 comp = (isinstance(target_status, list) and |
| 183 (lambda s: s in target_status) or |
| 184 (lambda s: s == target_status)) |
| 185 return [t for t in self.test_db.test_queue |
| 186 if comp(self.lookup_status(t))] |
| 187 |
| 188 def next_untested(self): |
| 189 remaining = self.filter(UNTESTED) |
| 190 unique_names = [self.test_db.get_unique_name(t) for t in remaining] |
| 191 log('remaining untested = [%s]' % ', '.join(unique_names)) |
| 192 return remaining is not [] and remaining.pop() or None |
| 193 |
| 194 def read_new_data(self): |
| 195 with open(self._status_file_path) as file: |
| 196 file.seek(self._status_file_pos) |
| 197 for line in file: |
| 198 cols = line.strip().split('\t') + [''] |
| 199 code = cols[0] |
| 200 test_id = cols[1] |
| 201 if code not in STATUS_CODE_MAP or test_id == '----': |
| 202 continue |
| 203 status = STATUS_CODE_MAP[code] |
| 204 error_msg = status == FAILED and cols[len(cols) - 2] or None |
| 205 log('reading code = %s, test_id = %s, error_msg = "%s"' |
| 206 % (code, test_id, error_msg)) |
| 207 autotest_name, _, tag = test_id.rpartition('.') |
| 208 tag_prefix, _, count = tag.rpartition('_') |
| 209 test = self.test_db.get_test_by_details( |
| 210 autotest_name, tag_prefix) |
| 211 if test is None: |
| 212 log('ignoring update (%s) for test "%s" "%s"' % |
| 213 (status, autotest_name, tag_prefix)) |
| 214 continue |
| 215 self.update(test, status, int(count), error_msg) |
| 216 map(self.update_seq_test, self.test_db.seq_test_set) |
| 217 self._status_file_pos = file.tell() |
| 218 |
| 219 def get_active_top_level_test(self): |
| 220 active_tests = set(self.filter(ACTIVE)) - self.test_db.subtest_set |
| 221 return active_tests and active_tests.pop() or None |
| 222 |
| 223 def get_active_subtest(self): |
| 224 active_subtests = set(self.filter(ACTIVE)) & self.test_db.subtest_set |
| 225 return active_subtests and active_subtests.pop() or None |
| 226 |
| 227 def register_active(self, test): |
| 228 active_tests = set(self.filter(ACTIVE)) |
| 229 assert(test not in active_tests) |
| 230 if test in self.test_db.subtest_set: |
| 231 parent_seq_test = self.test_db.get_subtest_parent(test) |
| 232 active_tests -= set([parent_seq_test]) |
| 233 for bad_test in active_tests: |
| 234 unique_name = self.test_db.get_unique_name(bad_test) |
| 235 log('WARNING: assuming test %s FAILED (status log has no data)' % |
| 236 unique_name) |
| 237 self.update(bad_test, FAILED, self.lookup_count(bad_test), |
| 238 'assumed FAILED (status log has no data)') |
| 239 |
| 240 def update(self, test, status, count, error_msg): |
| 241 entry = self._status_map[test] |
| 242 unique_name = self.test_db.get_unique_name(test) |
| 243 if count < entry.count: |
| 244 log('ERROR: count regression for %s (%d -> %d)' % |
| 245 (unique_name, entry.count, count)) |
| 246 if isinstance(test, InformationScreen) and status in [PASSED, FAILED]: |
| 247 status = UNTESTED |
| 248 if status != entry.status: |
| 249 log('status change for %s : %s/%s -> %s/%s' % |
| 250 (unique_name, entry.status, entry.count, status, count)) |
| 251 if entry.label_box is not None: |
| 252 entry.label_box.update(status) |
| 253 if status == ACTIVE: |
| 254 self.register_active(test) |
| 255 entry.status = status |
| 256 entry.count = count |
| 257 entry.error_msg = error_msg |
| 258 log('%s new status = %s' % (unique_name, self._status_map[test].status)) |
| 259 |
| 260 def update_seq_test(self, test): |
| 261 subtest_status_set = set(map(self.lookup_status, test.subtest_list)) |
| 262 max_count = max(map(self.lookup_count, test.subtest_list)) |
| 263 if len(subtest_status_set) == 1: |
| 264 status = subtest_status_set.pop() |
| 265 else: |
| 266 status = ACTIVE in subtest_status_set and ACTIVE or FAILED |
| 267 self.update(test, status, max_count, None) |
| 268 |
| 269 def set_label_box(self, test, label_box): |
| 270 entry = self._status_map[test] |
| 271 entry.label_box = label_box |
| 272 label_box.update(entry.status) |
| 273 |
| 274 |
| 275 class LogData: |
| 276 |
| 277 def __init__(self): |
| 278 self._log_file_pos = 0 |
| 279 self.shared_dict = {} |
| 280 self.read_new_data() |
| 281 |
| 282 def read_new_data(self): |
| 283 with open(LOG_PATH) as file: |
| 284 file.seek(self._log_file_pos) |
| 285 for line in file: |
| 286 parts = line.rsplit(DATA_PREFIX, 1) |
| 287 if not len(parts) == 2: |
| 288 continue |
| 289 key, raw_value = parts.pop().strip().split('=', 1) |
| 290 log('updating shared_dict[%s]=%s' % (key, raw_value)) |
| 291 self.shared_dict[key] = eval(raw_value) |
| 292 self._log_file_pos = file.tell() |
64 | 293 |
65 | 294 |
66 class UiClient: | 295 class UiClient: |
67 '''Support communication with the factory_ui process. To simplify | 296 '''Support communication with the factory_ui process. To simplify |
68 surrounding code, this communication is an exchange of well formed | 297 surrounding code, this communication is an exchange of well formed |
69 python expressions. Basically send wraps its arguments in a call | 298 python expressions. Basically send wraps its arguments in a call |
70 to repr() and recv calls eval() to re-generate the python data.''' | 299 to repr() and recv calls eval() to re-generate the python data.''' |
71 | 300 |
72 def __init__(self, factory_ui_path): | 301 def __init__(self, test_list, factory_ui_path, status_file_path): |
73 self._proc = subprocess.Popen(factory_ui_path, | 302 self._proc = subprocess.Popen(factory_ui_path, |
74 stdin=subprocess.PIPE, | 303 stdin=subprocess.PIPE, |
75 stdout=subprocess.PIPE) | 304 stdout=subprocess.PIPE) |
| 305 self.send(test_list) |
| 306 self.send(status_file_path) |
| 307 self.test_widget_size = self.recv() |
| 308 log('control received test_widget_size = %s' % |
| 309 repr(self.test_widget_size)) |
76 | 310 |
77 def __del__(self): | 311 def __del__(self): |
78 log('control deleting factory_ui subprocess') | 312 log('control deleting factory_ui subprocess') |
79 self._proc.terminate() | 313 self._proc.terminate() |
80 time.sleep(1) | 314 time.sleep(1) |
81 if self._proc.poll() is None: | 315 if self._proc.poll() is None: |
82 self._proc.kill() | 316 self._proc.kill() |
83 | 317 |
84 def send(self, x=None): | 318 def send(self, x=None): |
85 print >> self._proc.stdin, repr(x) | 319 print >> self._proc.stdin, repr(x) |
86 self._proc.stdin.flush() | 320 self._proc.stdin.flush() |
87 | 321 |
88 def send_cmd_next_test(self): | |
89 self.send(('next_test', None)) | |
90 | |
91 def send_cmd_switch_to(self, trigger): | |
92 self.send(('switch_to', trigger)) | |
93 | |
94 def recv(self): | 322 def recv(self): |
95 return eval(self._proc.stdout.readline().rstrip()) | 323 return eval(self._proc.stdout.readline().rstrip()) |
96 | 324 |
97 def recv_target_test_update(self, test_map): | 325 |
98 update = self.recv() | 326 class ControlState: |
99 log('control recv target test %s' % repr(update)) | 327 |
100 formal_name, tag_prefix, count = update | 328 def __init__(self, job, test_list, ui, status_map, status_file_path): |
101 test = test_map.get(test_map_index(formal_name, tag_prefix), None) | 329 self._job = job |
102 return (test, count) | 330 self._status_map = status_map |
| 331 self._log_data = LogData() |
| 332 self._std_dargs = { |
| 333 'test_widget_size': ui.test_widget_size, |
| 334 'trigger_set': status_map.test_db.kbd_shortcut_set, |
| 335 'status_file_path' : status_file_path, |
| 336 'test_list': test_list} |
| 337 |
| 338 def run_test(self, test): |
| 339 self._status_map.incr_count(test) |
| 340 dargs = test.dargs |
| 341 dargs.update(self._std_dargs) |
| 342 test_tag = self._status_map.lookup_tag(test) |
| 343 dargs.update({'tag': test_tag, |
| 344 'subtest_tag': test_tag, |
| 345 'shared_dict': self._log_data.shared_dict}) |
| 346 if test.drop_caches: |
| 347 self._job.drop_caches_between_iterations = True |
| 348 self._job.run_test(test.autotest_name, **dargs) |
| 349 self._job.drop_caches_between_iterations = False |
| 350 self._log_data.read_new_data() |
| 351 activated_ks = self._log_data.shared_dict.pop('activated_kbd_shortcut') |
| 352 lookup = self._status_map.test_db.get_test_by_kbd_shortcut |
| 353 self.activated_kbd_shortcut_test = ( |
| 354 activated_ks and lookup(activated_ks) or None) |
OLD | NEW |