OLD | NEW |
| (Empty) |
1 #!/usr/bin/env python | |
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | |
3 # Use of this source code is governed by a BSD-style license that can be | |
4 # found in the LICENSE file. | |
5 | |
6 """Runs the whole set unit tests on swarm.""" | |
7 | |
8 import datetime | |
9 import glob | |
10 import getpass | |
11 import hashlib | |
12 import logging | |
13 import optparse | |
14 import os | |
15 import shutil | |
16 import subprocess | |
17 import sys | |
18 import tempfile | |
19 import time | |
20 | |
21 BASE_DIR = os.path.dirname(os.path.abspath(__file__)) | |
22 ROOT_DIR = os.path.dirname(BASE_DIR) | |
23 | |
24 sys.path.insert(0, ROOT_DIR) | |
25 | |
26 from utils import threading_utils | |
27 | |
28 | |
29 # Mapping of the sys.platform value into Swarm OS value. | |
30 OSES = {'win32': 'win', 'linux2': 'linux', 'darwin': 'mac'} | |
31 | |
32 | |
33 class Runner(object): | |
34 def __init__(self, isolate_server, swarm_server, add_task, progress, tempdir): | |
35 self.isolate_server = isolate_server | |
36 self.swarm_server = swarm_server | |
37 self.add_task = add_task | |
38 self.progress = progress | |
39 self.tempdir = tempdir | |
40 self.prefix = ( | |
41 getpass.getuser() + '-' + datetime.datetime.now().isoformat() + '-') | |
42 | |
43 @staticmethod | |
44 def _call(args): | |
45 start = time.time() | |
46 proc = subprocess.Popen( | |
47 [sys.executable] + args, | |
48 stdout=subprocess.PIPE, | |
49 stderr=subprocess.STDOUT, | |
50 cwd=ROOT_DIR) | |
51 stdout = proc.communicate()[0] | |
52 return proc.returncode, stdout, time.time() - start | |
53 | |
54 def archive(self, test, platform): | |
55 # Put the .isolated files in a temporary directory. This is simply done so | |
56 # the current directory doesn't have the following files created: | |
57 # - swarm_client_tests.isolated | |
58 # - swarm_client_tests.isolated.state | |
59 test_name = os.path.basename(test) | |
60 handle, isolated = tempfile.mkstemp( | |
61 dir=self.tempdir, prefix='run_swarm_tests_on_swarm_', | |
62 suffix='.isolated') | |
63 os.close(handle) | |
64 try: | |
65 returncode, stdout, duration = self._call( | |
66 [ | |
67 'isolate.py', | |
68 'archive', | |
69 '--isolate', os.path.join(BASE_DIR, 'run_a_test.isolate'), | |
70 '--isolated', isolated, | |
71 '--outdir', self.isolate_server, | |
72 '--variable', 'TEST_EXECUTABLE', test, | |
73 '--variable', 'OS', OSES[platform], | |
74 ]) | |
75 step_name = '%s/%s (%3.2fs)' % (platform, test_name, duration) | |
76 if returncode: | |
77 self.progress.update_item( | |
78 'Failed to archive %s\n%s' % (step_name, stdout), index=1) | |
79 else: | |
80 hash_value = hashlib.sha1(open(isolated, 'rb').read()).hexdigest() | |
81 logging.info('%s: %s', step_name, hash_value) | |
82 self.progress.update_item('Archived %s' % step_name, index=1) | |
83 self.add_task(0, self.trigger, test, platform, hash_value) | |
84 finally: | |
85 try: | |
86 os.remove(isolated) | |
87 except OSError: | |
88 logging.debug('%s was already deleted', isolated) | |
89 return None | |
90 | |
91 def trigger(self, test, platform, hash_value): | |
92 test_name = os.path.basename(test) | |
93 returncode, stdout, duration = self._call( | |
94 [ | |
95 'swarming.py', | |
96 'trigger', | |
97 '--os', platform, | |
98 '--swarming', self.swarm_server, | |
99 '--task-prefix', self.prefix, | |
100 '--isolate-server', self.isolate_server, | |
101 '--task', | |
102 # Isolated hash. | |
103 hash_value, | |
104 # Test name. | |
105 'swarm_client_tests_%s_%s' % (platform, test_name), | |
106 # Number of shards. | |
107 '1', | |
108 # test filter. | |
109 '*', | |
110 ]) | |
111 step_name = '%s/%s (%3.2fs)' % (platform, test_name, duration) | |
112 if returncode: | |
113 self.progress.update_item( | |
114 'Failed to trigger %s\n%s' % (step_name, stdout), index=1) | |
115 else: | |
116 self.progress.update_item('Triggered %s' % step_name, index=1) | |
117 self.add_task(0, self.get_result, test, platform) | |
118 return None | |
119 | |
120 def get_result(self, test, platform): | |
121 test_name = os.path.basename(test) | |
122 name = '%s_%s' % (platform, test_name) | |
123 returncode, stdout, duration = self._call( | |
124 [ | |
125 'swarming.py', | |
126 'collect', | |
127 '--swarming', self.swarm_server, | |
128 self.prefix + 'swarm_client_tests_' + name, | |
129 ]) | |
130 step_name = '%s/%s (%3.2fs)' % (platform, test_name, duration) | |
131 # Only print the output for failures, successes are unexciting. | |
132 if returncode: | |
133 self.progress.update_item( | |
134 'Failed %s:\n%s' % (step_name, stdout), index=1) | |
135 return (test_name, platform, stdout) | |
136 self.progress.update_item('Passed %s' % step_name, index=1) | |
137 return None | |
138 | |
139 | |
140 def run_swarm_tests_on_swarm(oses, tests, logs, isolate_server, swarm_server): | |
141 runs = len(tests) * len(oses) | |
142 total = 3 * runs | |
143 columns = [('index', 0), ('size', total)] | |
144 progress = threading_utils.Progress(columns) | |
145 progress.use_cr_only = False | |
146 tempdir = tempfile.mkdtemp(prefix='swarm_client_tests') | |
147 try: | |
148 with threading_utils.ThreadPoolWithProgress( | |
149 progress, runs, runs, total) as pool: | |
150 start = time.time() | |
151 runner = Runner( | |
152 isolate_server, swarm_server, pool.add_task, progress, tempdir) | |
153 for test in tests: | |
154 for platform in oses: | |
155 pool.add_task(0, runner.archive, test, platform) | |
156 | |
157 failed_tests = pool.join() | |
158 duration = time.time() - start | |
159 print('') | |
160 finally: | |
161 shutil.rmtree(tempdir) | |
162 | |
163 if logs: | |
164 os.makedirs(logs) | |
165 for test, platform, stdout in failed_tests: | |
166 name = '%s_%s' % (platform, os.path.basename(test)) | |
167 with open(os.path.join(logs, name + '.log'), 'wb') as f: | |
168 f.write(stdout) | |
169 | |
170 print('Completed in %3.2fs' % duration) | |
171 if failed_tests: | |
172 failed_tests_per_os = {} | |
173 for test, platform, _ in failed_tests: | |
174 failed_tests_per_os.setdefault(test, []).append(platform) | |
175 print('Detected the following failures:') | |
176 for test, platforms in failed_tests_per_os.iteritems(): | |
177 print(' %s on %s' % (test, ', '.join(sorted(platforms)))) | |
178 return bool(failed_tests) | |
179 | |
180 | |
181 def main(): | |
182 parser = optparse.OptionParser(description=sys.modules[__name__].__doc__) | |
183 parser.add_option( | |
184 '-I', '--isolate-server', | |
185 metavar='URL', default='', | |
186 help='Isolate server to use') | |
187 parser.add_option( | |
188 '-S', '--swarming', | |
189 metavar='URL', default='', | |
190 help='Swarming server to use') | |
191 parser.add_option( | |
192 '-l', '--logs', | |
193 help='Destination where to store the failure logs (recommended)') | |
194 parser.add_option('-o', '--os', help='Run tests only on this OS') | |
195 parser.add_option('-t', '--test', help='Run only this test') | |
196 parser.add_option('-v', '--verbose', action='store_true') | |
197 options, args = parser.parse_args() | |
198 if args: | |
199 parser.error('Unsupported argument %s' % args) | |
200 if options.verbose: | |
201 os.environ['ISOLATE_DEBUG'] = '1' | |
202 | |
203 if not options.isolate_server: | |
204 parser.error('--isolate-server is required.') | |
205 if not options.swarming: | |
206 parser.error('--swarming is required.') | |
207 | |
208 logging.basicConfig(level=logging.DEBUG if options.verbose else logging.ERROR) | |
209 | |
210 # Note that the swarm and the isolate code use different strings for the | |
211 # different oses. | |
212 oses = OSES.keys() | |
213 tests = [ | |
214 os.path.relpath(i, BASE_DIR) | |
215 for i in glob.iglob(os.path.join(ROOT_DIR, 'tests', '*_test.py')) | |
216 ] | |
217 | |
218 if options.test: | |
219 valid_tests = sorted(map(os.path.basename, tests)) | |
220 if not options.test in valid_tests: | |
221 parser.error( | |
222 '--test %s is unknown. Valid values are:\n%s' % ( | |
223 options.test, '\n'.join(' ' + i for i in valid_tests))) | |
224 tests = [t for t in tests if t.endswith(os.path.sep + options.test)] | |
225 | |
226 if options.os: | |
227 if options.os not in oses: | |
228 parser.error( | |
229 '--os %s is unknown. Valid values are %s' % ( | |
230 options.os, ', '.join(sorted(oses)))) | |
231 oses = [options.os] | |
232 | |
233 if sys.platform in ('win32', 'cygwin'): | |
234 # If we are on Windows, don't generate the tests for Linux and Mac since | |
235 # they use symlinks and we can't create symlinks on windows. | |
236 oses = ['win32'] | |
237 if options.os != 'win32': | |
238 print('Linux and Mac tests skipped since running on Windows.') | |
239 | |
240 return run_swarm_tests_on_swarm( | |
241 oses, | |
242 tests, | |
243 options.logs, | |
244 options.isolate_server, | |
245 options.swarming) | |
246 | |
247 | |
248 if __name__ == '__main__': | |
249 sys.exit(main()) | |
OLD | NEW |