OLD | NEW |
| (Empty) |
1 #!/usr/bin/env python | |
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | |
3 # Use of this source code is governed by a BSD-style license that can be | |
4 # found in the LICENSE file. | |
5 | |
6 """Generate and process code coverage. | |
7 | |
8 TODO(jrg): rename this from coverage_posix.py to coverage_all.py! | |
9 | |
10 Written for and tested on Mac, Linux, and Windows. To use this script | |
11 to generate coverage numbers, please run from within a gyp-generated | |
12 project. | |
13 | |
14 All platforms, to set up coverage: | |
15 cd ...../chromium ; src/tools/gyp/gyp_dogfood -Dcoverage=1 src/build/all.gyp | |
16 | |
17 Run coverage on... | |
18 Mac: | |
19 ( cd src/chrome ; xcodebuild -configuration Debug -target coverage ) | |
20 Linux: | |
21 ( cd src/chrome ; hammer coverage ) | |
22 # In particular, don't try and run 'coverage' from src/build | |
23 | |
24 | |
25 --directory=DIR: specify directory that contains gcda files, and where | |
26 a "coverage" directory will be created containing the output html. | |
27 Example name: ..../chromium/src/xcodebuild/Debug. | |
28 If not specified (e.g. buildbot) we will try and figure it out based on | |
29 other options (e.g. --target and --build-dir; see below). | |
30 | |
31 --genhtml: generate html output. If not specified only lcov is generated. | |
32 | |
33 --all_unittests: if present, run all files named *_unittests that we | |
34 can find. | |
35 | |
36 --fast_test: make the tests run real fast (just for testing) | |
37 | |
38 --strict: if a test fails, we continue happily. --strict will cause | |
39 us to die immediately. | |
40 | |
41 --trim=False: by default we trim away tests known to be problematic on | |
42 specific platforms. If set to false we do NOT trim out tests. | |
43 | |
44 --xvfb=True: By default we use Xvfb to make sure DISPLAY is valid | |
45 (Linux only). if set to False, do not use Xvfb. TODO(jrg): convert | |
46 this script from the compile stage of a builder to a | |
47 RunPythonCommandInBuildDir() command to avoid the need for this | |
48 step. | |
49 | |
50 --timeout=SECS: if a subprocess doesn't have output within SECS, | |
51 assume it's a hang. Kill it and give up. | |
52 | |
53 --bundles=BUNDLEFILE: a file containing a python list of coverage | |
54 bundles to be eval'd. Example contents of the bundlefile: | |
55 ['../base/base.gyp:base_unittests'] | |
56 This is used as part of the coverage bot. | |
57 If no other bundlefile-finding args are used (--target, | |
58 --build-dir), this is assumed to be an absolute path. | |
59 If those args are used, find BUNDLEFILE in a way consistent with | |
60 other scripts launched by buildbot. Example of another script | |
61 launched by buildbot: | |
62 http://src.chromium.org/viewvc/chrome/trunk/tools/buildbot/scripts/slave/runte
st.py | |
63 | |
64 --target=NAME: specify the build target (e.g. 'Debug' or 'Release'). | |
65 This is used by buildbot scripts to help us find the output directory. | |
66 Must be used with --build-dir. | |
67 | |
68 --build-dir=DIR: According to buildbot comments, this is the name of | |
69 the directory within the buildbot working directory in which the | |
70 solution, Debug, and Release directories are found. | |
71 It's usually "src/build", but on mac it's $DIR/../xcodebuild and on | |
72 Linux it's $DIR/out. | |
73 This is used by buildbot scripts to help us find the output directory. | |
74 Must be used with --target. | |
75 | |
76 --no_exclusions: Do NOT use the exclusion list. This script keeps a | |
77 list of tests known to be problematic under coverage. For example, | |
78 ProcessUtilTest.SpawnChild will crash inside __gcov_fork() when | |
79 using the MacOS 10.6 SDK. Use of --no_exclusions prevents the use | |
80 of this exclusion list. | |
81 | |
82 --dont-clear-coverage-data: Normally we clear coverage data from | |
83 previous runs. If this arg is used we do NOT clear the coverage | |
84 data. | |
85 | |
86 Strings after all options are considered tests to run. Test names | |
87 have all text before a ':' stripped to help with gyp compatibility. | |
88 For example, ../base/base.gyp:base_unittests is interpreted as a test | |
89 named "base_unittests". | |
90 """ | |
91 | |
92 import glob | |
93 import logging | |
94 import optparse | |
95 import os | |
96 import Queue | |
97 import re | |
98 import shutil | |
99 import signal | |
100 import subprocess | |
101 import sys | |
102 import tempfile | |
103 import threading | |
104 import time | |
105 import traceback | |
106 | |
107 """Global list of child PIDs to kill when we die.""" | |
108 gChildPIDs = [] | |
109 | |
110 """Exclusion list. Format is | |
111 { platform: { testname: (exclusion1, exclusion2, ... ), ... } } | |
112 | |
113 Platform is a match for sys.platform and can be a list. | |
114 Matching code does an 'if sys.platform in (the key):'. | |
115 Similarly, matching does an 'if testname in thefulltestname:' | |
116 | |
117 The Chromium convention has traditionally been to place the | |
118 exclusion list in a distinct file. Unlike valgrind (which has | |
119 frequent changes when things break and are fixed), the expectation | |
120 here is that exclusions remain relatively constant (e.g. OS bugs). | |
121 If that changes, revisit the decision to place inclusions in this | |
122 script. | |
123 | |
124 Details: | |
125 ProcessUtilTest.SpawnChild: chokes in __gcov_fork on 10.6 | |
126 IPCFuzzingTest.MsgBadPayloadArgs: ditto | |
127 PanelBrowserNavigatorTest.NavigateFromCrashedPanel: Fails on coverage bot. | |
128 WebGLConformanceTests.conformance_attribs_gl_enable_vertex_attrib: Fails | |
129 with timeout (45000 ms) exceeded error. crbug.com/143248 | |
130 WebGLConformanceTests.conformance_attribs_gl_disabled_vertex_attrib: | |
131 ditto. | |
132 WebGLConformanceTests.conformance_attribs_gl_vertex_attrib_zero_issues: | |
133 ditto. | |
134 WebGLConformanceTests.conformance_attribs_gl_vertex_attrib: ditto. | |
135 WebGLConformanceTests.conformance_attribs_gl_vertexattribpointer_offsets: | |
136 ditto. | |
137 WebGLConformanceTests.conformance_attribs_gl_vertexattribpointer: ditto. | |
138 WebGLConformanceTests.conformance_buffers_buffer_bind_test: After | |
139 disabling WebGLConformanceTests specified above, this test fails when run | |
140 on local machine. | |
141 WebGLConformanceTests.conformance_buffers_buffer_data_array_buffer: ditto. | |
142 WebGLConformanceTests.conformance_buffers_index_validation_copies_indices: | |
143 ditto. | |
144 WebGLConformanceTests. | |
145 conformance_buffers_index_validation_crash_with_buffer_sub_data: ditto. | |
146 WebGLConformanceTests. | |
147 conformance_buffers_index_validation_verifies_too_many_indices: ditto. | |
148 WebGLConformanceTests. | |
149 conformance_buffers_index_validation_with_resized_buffer: ditto. | |
150 WebGLConformanceTests.conformance_canvas_buffer_offscreen_test: ditto. | |
151 WebGLConformanceTests.conformance_canvas_buffer_preserve_test: ditto. | |
152 WebGLConformanceTests.conformance_canvas_canvas_test: ditto. | |
153 WebGLConformanceTests.conformance_canvas_canvas_zero_size: ditto. | |
154 WebGLConformanceTests. | |
155 conformance_canvas_drawingbuffer_static_canvas_test: ditto. | |
156 WebGLConformanceTests.conformance_canvas_drawingbuffer_test: ditto. | |
157 PageCycler*.*: Fails on coverage bot with "Missing test directory | |
158 /....../slave/coverage-dbg-linux/build/src/data/page_cycler/moz" error. | |
159 *FrameRateCompositingTest.*: Fails with | |
160 "FATAL:chrome_content_browser_client.cc(893)] Check failed: | |
161 command_line->HasSwitch(switches::kEnableStatsTable)." | |
162 *FrameRateNoVsyncCanvasInternalTest.*: ditto. | |
163 *FrameRateGpuCanvasInternalTest.*: ditto. | |
164 IndexedDBTest.Perf: Fails with 'Timeout reached in WaitUntilCookieValue' | |
165 error. | |
166 TwoClientPasswordsSyncTest.DeleteAll: Fails on coverage bot. | |
167 MigrationTwoClientTest.MigrationHellWithoutNigori: Fails with timeout | |
168 (45000 ms) exceeded error. | |
169 TwoClientSessionsSyncTest.DeleteActiveSession: ditto. | |
170 MultipleClientSessionsSyncTest.EncryptedAndChanged: ditto. | |
171 MigrationSingleClientTest.AllTypesIndividuallyTriggerNotification: ditto. | |
172 *OldPanelResizeBrowserTest.*: crbug.com/143247 | |
173 *OldPanelDragBrowserTest.*: ditto. | |
174 *OldPanelBrowserTest.*: ditto. | |
175 *OldPanelAndDesktopNotificationTest.*: ditto. | |
176 *OldDockedPanelBrowserTest.*: ditto. | |
177 *OldDetachedPanelBrowserTest.*: ditto. | |
178 PanelDragBrowserTest.AttachWithSqueeze: ditto. | |
179 *PanelBrowserTest.*: ditto. | |
180 *DockedPanelBrowserTest.*: ditto. | |
181 *DetachedPanelBrowserTest.*: ditto. | |
182 AutomatedUITest.TheOneAndOnlyTest: crbug.com/143419 | |
183 AutomatedUITestBase.DragOut: ditto | |
184 | |
185 """ | |
186 gTestExclusions = { | |
187 'darwin2': { 'base_unittests': ('ProcessUtilTest.SpawnChild',), | |
188 'ipc_tests': ('IPCFuzzingTest.MsgBadPayloadArgs',), }, | |
189 'linux2': { | |
190 'gpu_tests': | |
191 ('WebGLConformanceTests.conformance_attribs_gl_enable_vertex_attrib', | |
192 'WebGLConformanceTests.' | |
193 'conformance_attribs_gl_disabled_vertex_attrib', | |
194 'WebGLConformanceTests.' | |
195 'conformance_attribs_gl_vertex_attrib_zero_issues', | |
196 'WebGLConformanceTests.conformance_attribs_gl_vertex_attrib', | |
197 'WebGLConformanceTests.' | |
198 'conformance_attribs_gl_vertexattribpointer_offsets', | |
199 'WebGLConformanceTests.conformance_attribs_gl_vertexattribpointer', | |
200 'WebGLConformanceTests.conformance_buffers_buffer_bind_test', | |
201 'WebGLConformanceTests.' | |
202 'conformance_buffers_buffer_data_array_buffer', | |
203 'WebGLConformanceTests.' | |
204 'conformance_buffers_index_validation_copies_indices', | |
205 'WebGLConformanceTests.' | |
206 'conformance_buffers_index_validation_crash_with_buffer_sub_data', | |
207 'WebGLConformanceTests.' | |
208 'conformance_buffers_index_validation_verifies_too_many_indices', | |
209 'WebGLConformanceTests.' | |
210 'conformance_buffers_index_validation_with_resized_buffer', | |
211 'WebGLConformanceTests.conformance_canvas_buffer_offscreen_test', | |
212 'WebGLConformanceTests.conformance_canvas_buffer_preserve_test', | |
213 'WebGLConformanceTests.conformance_canvas_canvas_test', | |
214 'WebGLConformanceTests.conformance_canvas_canvas_zero_size', | |
215 'WebGLConformanceTests.' | |
216 'conformance_canvas_drawingbuffer_static_canvas_test', | |
217 'WebGLConformanceTests.conformance_canvas_drawingbuffer_test',), | |
218 'performance_ui_tests': | |
219 ('*PageCycler*.*', | |
220 '*FrameRateCompositingTest.*', | |
221 '*FrameRateNoVsyncCanvasInternalTest.*', | |
222 '*FrameRateGpuCanvasInternalTest.*', | |
223 'IndexedDBTest.Perf',), | |
224 'sync_integration_tests': | |
225 ('TwoClientPasswordsSyncTest.DeleteAll', | |
226 'MigrationTwoClientTest.MigrationHellWithoutNigori', | |
227 'TwoClientSessionsSyncTest.DeleteActiveSession', | |
228 'MultipleClientSessionsSyncTest.EncryptedAndChanged', | |
229 'MigrationSingleClientTest.' | |
230 'AllTypesIndividuallyTriggerNotification',), | |
231 'interactive_ui_tests': | |
232 ('*OldPanelResizeBrowserTest.*', | |
233 '*OldPanelDragBrowserTest.*', | |
234 '*OldPanelBrowserTest.*', | |
235 '*OldPanelAndDesktopNotificationTest.*', | |
236 '*OldDockedPanelBrowserTest.*', | |
237 '*OldDetachedPanelBrowserTest.*', | |
238 'PanelDragBrowserTest.AttachWithSqueeze', | |
239 '*PanelBrowserTest.*', | |
240 '*DockedPanelBrowserTest.*', | |
241 '*DetachedPanelBrowserTest.*',), | |
242 'automated_ui_tests': | |
243 ('AutomatedUITest.TheOneAndOnlyTest', | |
244 'AutomatedUITestBase.DragOut',), }, | |
245 } | |
246 | |
247 """Since random tests are failing/hanging on coverage bot, we are enabling | |
248 tests feature by feature. crbug.com/159748 | |
249 """ | |
250 gTestInclusions = { | |
251 'linux2': { | |
252 'browser_tests': | |
253 (# 'src/chrome/browser/downloads' | |
254 'SavePageBrowserTest.*', | |
255 'SavePageAsMHTMLBrowserTest.*', | |
256 'DownloadQueryTest.*', | |
257 'DownloadDangerPromptTest.*', | |
258 'DownloadTest.*', | |
259 # 'src/chrome/browser/net' | |
260 'CookiePolicyBrowserTest.*', | |
261 'FtpBrowserTest.*', | |
262 'LoadTimingObserverTest.*', | |
263 'PredictorBrowserTest.*', | |
264 'ProxyBrowserTest.*', | |
265 # 'src/chrome/browser/extensions' | |
266 'Extension*.*', | |
267 'WindowOpenPanelDisabledTest.*', | |
268 'WindowOpenPanelTest.*', | |
269 'WebstoreStandalone*.*', | |
270 'CommandLineWebstoreInstall.*', | |
271 'WebViewTest.*', | |
272 'RequirementsCheckerBrowserTest.*', | |
273 'ProcessManagementTest.*', | |
274 'PlatformAppBrowserTest.*', | |
275 'PlatformAppDevToolsBrowserTest.*', | |
276 'LazyBackgroundPageApiTest.*', | |
277 'IsolatedAppTest.*', | |
278 'PanelMessagingTest.*', | |
279 'GeolocationApiTest.*', | |
280 'ClipboardApiTest.*', | |
281 'ExecuteScriptApiTest.*', | |
282 'CalculatorBrowserTest.*', | |
283 'ChromeAppAPITest.*', | |
284 'AppApiTest.*', | |
285 'BlockedAppApiTest.*', | |
286 'AppBackgroundPageApiTest.*', | |
287 'WebNavigationApiTest.*', | |
288 'UsbApiTest.*', | |
289 'TabCaptureApiTest.*', | |
290 'SystemInfo*.*', | |
291 'SyncFileSystemApiTest.*', | |
292 'SocketApiTest.*', | |
293 'SerialApiTest.*', | |
294 'RecordApiTest.*', | |
295 'PushMessagingApiTest.*', | |
296 'ProxySettingsApiTest.*', | |
297 'ExperimentalApiTest.*', | |
298 'OmniboxApiTest.*', | |
299 'OffscreenTabsApiTest.*', | |
300 'NotificationApiTest.*', | |
301 'MediaGalleriesPrivateApiTest.*', | |
302 'PlatformAppMediaGalleriesBrowserTest.*', | |
303 'GetAuthTokenFunctionTest.*', | |
304 'LaunchWebAuthFlowFunctionTest.*', | |
305 'FileSystemApiTest.*', | |
306 'ScriptBadgeApiTest.*', | |
307 'PageAsBrowserActionApiTest.*', | |
308 'PageActionApiTest.*', | |
309 'BrowserActionApiTest.*', | |
310 'DownloadExtensionTest.*', | |
311 'DnsApiTest.*', | |
312 'DeclarativeApiTest.*', | |
313 'BluetoothApiTest.*', | |
314 'AllUrlsApiTest.*', | |
315 # 'src/chrome/browser/nacl_host' | |
316 'nacl_host.*', | |
317 # 'src/chrome/browser/automation' | |
318 'AutomationMiscBrowserTest.*', | |
319 # 'src/chrome/browser/autofill' | |
320 'FormStructureBrowserTest.*', | |
321 'AutofillPopupViewBrowserTest.*', | |
322 'AutofillTest.*', | |
323 # 'src/chrome/browser/autocomplete' | |
324 'AutocompleteBrowserTest.*', | |
325 # 'src/chrome/browser/captive_portal' | |
326 'CaptivePortalBrowserTest.*', | |
327 # 'src/chrome/browser/geolocation' | |
328 'GeolocationAccessTokenStoreTest.*', | |
329 'GeolocationBrowserTest.*', | |
330 # 'src/chrome/browser/nacl_host' | |
331 'NaClGdbTest.*', | |
332 # 'src/chrome/browser/devtools' | |
333 'DevToolsSanityTest.*', | |
334 'DevToolsExtensionTest.*', | |
335 'DevToolsExperimentalExtensionTest.*', | |
336 'WorkerDevToolsSanityTest.*', | |
337 # 'src/chrome/browser/first_run' | |
338 'FirstRunBrowserTest.*', | |
339 # 'src/chrome/browser/importer' | |
340 'ToolbarImporterUtilsTest.*', | |
341 # 'src/chrome/browser/page_cycler' | |
342 'PageCyclerBrowserTest.*', | |
343 'PageCyclerCachedBrowserTest.*', | |
344 # 'src/chrome/browser/performance_monitor' | |
345 'PerformanceMonitorBrowserTest.*', | |
346 'PerformanceMonitorUncleanExitBrowserTest.*', | |
347 'PerformanceMonitorSessionRestoreBrowserTest.*', | |
348 # 'src/chrome/browser/prerender' | |
349 'PrerenderBrowserTest.*', | |
350 'PrerenderBrowserTestWithNaCl.*', | |
351 'PrerenderBrowserTestWithExtensions.*', | |
352 'PrefetchBrowserTest.*', | |
353 'PrefetchBrowserTestNoPrefetching.*', ), | |
354 }, | |
355 } | |
356 | |
357 | |
358 def TerminateSignalHandler(sig, stack): | |
359 """When killed, try and kill our child processes.""" | |
360 signal.signal(sig, signal.SIG_DFL) | |
361 for pid in gChildPIDs: | |
362 if 'kill' in os.__all__: # POSIX | |
363 os.kill(pid, sig) | |
364 else: | |
365 subprocess.call(['taskkill.exe', '/PID', str(pid)]) | |
366 sys.exit(0) | |
367 | |
368 | |
369 class RunTooLongException(Exception): | |
370 """Thrown when a command runs too long without output.""" | |
371 pass | |
372 | |
373 class BadUserInput(Exception): | |
374 """Thrown when arguments from the user are incorrectly formatted.""" | |
375 pass | |
376 | |
377 | |
378 class RunProgramThread(threading.Thread): | |
379 """A thread to run a subprocess. | |
380 | |
381 We want to print the output of our subprocess in real time, but also | |
382 want a timeout if there has been no output for a certain amount of | |
383 time. Normal techniques (e.g. loop in select()) aren't cross | |
384 platform enough. the function seems simple: "print output of child, kill it | |
385 if there is no output by timeout. But it was tricky to get this right | |
386 in a x-platform way (see warnings about deadlock on the python | |
387 subprocess doc page). | |
388 | |
389 """ | |
390 # Constants in our queue | |
391 PROGRESS = 0 | |
392 DONE = 1 | |
393 | |
394 def __init__(self, cmd): | |
395 super(RunProgramThread, self).__init__() | |
396 self._cmd = cmd | |
397 self._process = None | |
398 self._queue = Queue.Queue() | |
399 self._retcode = None | |
400 | |
401 def run(self): | |
402 if sys.platform in ('win32', 'cygwin'): | |
403 return self._run_windows() | |
404 else: | |
405 self._run_posix() | |
406 | |
407 def _run_windows(self): | |
408 # We need to save stdout to a temporary file because of a bug on the | |
409 # windows implementation of python which can deadlock while waiting | |
410 # for the IO to complete while writing to the PIPE and the pipe waiting | |
411 # on us and us waiting on the child process. | |
412 stdout_file = tempfile.TemporaryFile() | |
413 try: | |
414 self._process = subprocess.Popen(self._cmd, | |
415 stdin=subprocess.PIPE, | |
416 stdout=stdout_file, | |
417 stderr=subprocess.STDOUT) | |
418 gChildPIDs.append(self._process.pid) | |
419 try: | |
420 # To make sure that the buildbot don't kill us if we run too long | |
421 # without any activity on the console output, we look for progress in | |
422 # the length of the temporary file and we print what was accumulated so | |
423 # far to the output console to make the buildbot know we are making some | |
424 # progress. | |
425 previous_tell = 0 | |
426 # We will poll the process until we get a non-None return code. | |
427 self._retcode = None | |
428 while self._retcode is None: | |
429 self._retcode = self._process.poll() | |
430 current_tell = stdout_file.tell() | |
431 if current_tell > previous_tell: | |
432 # Report progress to our main thread so we don't timeout. | |
433 self._queue.put(RunProgramThread.PROGRESS) | |
434 # And print what was accumulated to far. | |
435 stdout_file.seek(previous_tell) | |
436 print stdout_file.read(current_tell - previous_tell), | |
437 previous_tell = current_tell | |
438 # Don't be selfish, let other threads do stuff while we wait for | |
439 # the process to complete. | |
440 time.sleep(0.5) | |
441 # OK, the child process has exited, let's print its output to our | |
442 # console to create debugging logs in case they get to be needed. | |
443 stdout_file.flush() | |
444 stdout_file.seek(previous_tell) | |
445 print stdout_file.read(stdout_file.tell() - previous_tell) | |
446 except IOError, e: | |
447 logging.exception('%s', e) | |
448 pass | |
449 finally: | |
450 stdout_file.close() | |
451 | |
452 # If we get here the process is done. | |
453 gChildPIDs.remove(self._process.pid) | |
454 self._queue.put(RunProgramThread.DONE) | |
455 | |
456 def _run_posix(self): | |
457 """No deadlock problem so use the simple answer. The windows solution | |
458 appears to add extra buffering which we don't want on other platforms.""" | |
459 self._process = subprocess.Popen(self._cmd, | |
460 stdout=subprocess.PIPE, | |
461 stderr=subprocess.STDOUT) | |
462 gChildPIDs.append(self._process.pid) | |
463 try: | |
464 while True: | |
465 line = self._process.stdout.readline() | |
466 if not line: # EOF | |
467 break | |
468 print line, | |
469 self._queue.put(RunProgramThread.PROGRESS, True) | |
470 except IOError: | |
471 pass | |
472 # If we get here the process is done. | |
473 gChildPIDs.remove(self._process.pid) | |
474 self._queue.put(RunProgramThread.DONE) | |
475 | |
476 def stop(self): | |
477 self.kill() | |
478 | |
479 def kill(self): | |
480 """Kill our running process if needed. Wait for kill to complete. | |
481 | |
482 Should be called in the PARENT thread; we do not self-kill. | |
483 Returns the return code of the process. | |
484 Safe to call even if the process is dead. | |
485 """ | |
486 if not self._process: | |
487 return self.retcode() | |
488 if 'kill' in os.__all__: # POSIX | |
489 os.kill(self._process.pid, signal.SIGKILL) | |
490 else: | |
491 subprocess.call(['taskkill.exe', '/PID', str(self._process.pid)]) | |
492 return self.retcode() | |
493 | |
494 def retcode(self): | |
495 """Return the return value of the subprocess. | |
496 | |
497 Waits for process to die but does NOT kill it explicitly. | |
498 """ | |
499 if self._retcode == None: # must be none, not 0/False | |
500 self._retcode = self._process.wait() | |
501 return self._retcode | |
502 | |
503 def RunUntilCompletion(self, timeout): | |
504 """Run thread until completion or timeout (in seconds). | |
505 | |
506 Start the thread. Let it run until completion, or until we've | |
507 spent TIMEOUT without seeing output. On timeout throw | |
508 RunTooLongException. | |
509 """ | |
510 self.start() | |
511 while True: | |
512 try: | |
513 x = self._queue.get(True, timeout) | |
514 if x == RunProgramThread.DONE: | |
515 return self.retcode() | |
516 except Queue.Empty, e: # timed out | |
517 logging.info('TIMEOUT (%d seconds exceeded with no output): killing' % | |
518 timeout) | |
519 self.kill() | |
520 raise RunTooLongException() | |
521 | |
522 | |
523 class Coverage(object): | |
524 """Doitall class for code coverage.""" | |
525 | |
526 def __init__(self, options, args): | |
527 super(Coverage, self).__init__() | |
528 logging.basicConfig(level=logging.DEBUG) | |
529 self.directory = options.directory | |
530 self.options = options | |
531 self.args = args | |
532 self.ConfirmDirectory() | |
533 self.directory_parent = os.path.dirname(self.directory) | |
534 self.output_directory = os.path.join(self.directory, 'coverage') | |
535 if not os.path.exists(self.output_directory): | |
536 os.mkdir(self.output_directory) | |
537 # The "final" lcov-format file | |
538 self.coverage_info_file = os.path.join(self.directory, 'coverage.info') | |
539 # If needed, an intermediate VSTS-format file | |
540 self.vsts_output = os.path.join(self.directory, 'coverage.vsts') | |
541 # Needed for Windows. | |
542 self.src_root = options.src_root | |
543 self.FindPrograms() | |
544 self.ConfirmPlatformAndPaths() | |
545 self.tests = [] # This can be a list of strings, lists or both. | |
546 self.xvfb_pid = 0 | |
547 self.test_files = [] # List of files with test specifications. | |
548 self.test_filters = {} # Mapping from testname->--gtest_filter arg. | |
549 logging.info('self.directory: ' + self.directory) | |
550 logging.info('self.directory_parent: ' + self.directory_parent) | |
551 | |
552 def FindInPath(self, program): | |
553 """Find program in our path. Return abs path to it, or None.""" | |
554 if not 'PATH' in os.environ: | |
555 logging.fatal('No PATH environment variable?') | |
556 sys.exit(1) | |
557 paths = os.environ['PATH'].split(os.pathsep) | |
558 for path in paths: | |
559 fullpath = os.path.join(path, program) | |
560 if os.path.exists(fullpath): | |
561 return fullpath | |
562 return None | |
563 | |
564 def FindPrograms(self): | |
565 """Find programs we may want to run.""" | |
566 if self.IsPosix(): | |
567 self.lcov_directory = os.path.join(sys.path[0], | |
568 '../../third_party/lcov/bin') | |
569 self.lcov = os.path.join(self.lcov_directory, 'lcov') | |
570 self.mcov = os.path.join(self.lcov_directory, 'mcov') | |
571 self.genhtml = os.path.join(self.lcov_directory, 'genhtml') | |
572 self.programs = [self.lcov, self.mcov, self.genhtml] | |
573 else: | |
574 # Hack to get the buildbot working. | |
575 os.environ['PATH'] += r';c:\coverage\coverage_analyzer' | |
576 os.environ['PATH'] += r';c:\coverage\performance_tools' | |
577 # (end hack) | |
578 commands = ['vsperfcmd.exe', 'vsinstr.exe', 'coverage_analyzer.exe'] | |
579 self.perf = self.FindInPath('vsperfcmd.exe') | |
580 self.instrument = self.FindInPath('vsinstr.exe') | |
581 self.analyzer = self.FindInPath('coverage_analyzer.exe') | |
582 if not self.perf or not self.instrument or not self.analyzer: | |
583 logging.fatal('Could not find Win performance commands.') | |
584 logging.fatal('Commands needed in PATH: ' + str(commands)) | |
585 sys.exit(1) | |
586 self.programs = [self.perf, self.instrument, self.analyzer] | |
587 | |
588 def PlatformBuildPrefix(self): | |
589 """Return a platform specific build directory prefix. | |
590 | |
591 This prefix is prepended to the build target (Debug, Release) to | |
592 identify output as relative to the build directory. | |
593 These values are specific to Chromium's use of gyp. | |
594 """ | |
595 if self.IsMac(): | |
596 return '../xcodebuild' | |
597 if self.IsWindows(): | |
598 return '' | |
599 else: # Linux | |
600 return '../out' # assumes make, unlike runtest.py | |
601 | |
602 def ConfirmDirectory(self): | |
603 """Confirm correctness of self.directory. | |
604 | |
605 If it exists, happiness. If not, try and figure it out in a | |
606 manner similar to FindBundlesFile(). The 'figure it out' case | |
607 happens with buildbot where the directory isn't specified | |
608 explicitly. | |
609 """ | |
610 if (not self.directory and | |
611 not (self.options.target and self.options.build_dir)): | |
612 logging.fatal('Must use --directory or (--target and --build-dir)') | |
613 sys.exit(1) | |
614 | |
615 if not self.directory: | |
616 self.directory = os.path.join(self.options.build_dir, | |
617 self.PlatformBuildPrefix(), | |
618 self.options.target) | |
619 | |
620 if os.path.exists(self.directory): | |
621 logging.info('Directory: ' + self.directory) | |
622 return | |
623 else: | |
624 logging.fatal('Directory ' + | |
625 self.directory + ' doesn\'t exist') | |
626 sys.exit(1) | |
627 | |
628 | |
629 def FindBundlesFile(self): | |
630 """Find the bundlesfile. | |
631 | |
632 The 'bundles' file can be either absolute path, or (if we are run | |
633 from buildbot) we need to find it based on other hints (--target, | |
634 --build-dir, etc). | |
635 """ | |
636 # If no bundle file, no problem! | |
637 if not self.options.bundles: | |
638 return | |
639 # If true, we're buildbot. Form a path. | |
640 # Else assume absolute. | |
641 if self.options.target and self.options.build_dir: | |
642 fullpath = os.path.join(self.options.build_dir, | |
643 self.PlatformBuildPrefix(), | |
644 self.options.target, | |
645 self.options.bundles) | |
646 self.options.bundles = fullpath | |
647 | |
648 if os.path.exists(self.options.bundles): | |
649 logging.info('BundlesFile: ' + self.options.bundles) | |
650 return | |
651 else: | |
652 logging.fatal('bundlefile ' + | |
653 self.options.bundles + ' doesn\'t exist') | |
654 sys.exit(1) | |
655 | |
656 | |
657 def FindTests(self): | |
658 """Find unit tests to run; set self.tests to this list. | |
659 | |
660 Assume all non-option items in the arg list are tests to be run. | |
661 """ | |
662 # Before we begin, find the bundles file if not an absolute path. | |
663 self.FindBundlesFile() | |
664 | |
665 # Small tests: can be run in the "chromium" directory. | |
666 # If asked, run all we can find. | |
667 if self.options.all_unittests: | |
668 self.tests += glob.glob(os.path.join(self.directory, '*_unittests')) | |
669 self.tests += glob.glob(os.path.join(self.directory, '*unit_tests')) | |
670 elif self.options.all_browsertests: | |
671 # Run all tests in browser_tests and content_browsertests. | |
672 self.tests += glob.glob(os.path.join(self.directory, 'browser_tests')) | |
673 self.tests += glob.glob(os.path.join(self.directory, | |
674 'content_browsertests')) | |
675 | |
676 # Tests can come in as args directly, indirectly (through a file | |
677 # of test lists) or as a file of bundles. | |
678 all_testnames = self.args[:] # Copy since we might modify | |
679 | |
680 for test_file in self.options.test_files: | |
681 f = open(test_file) | |
682 for line in f: | |
683 line = re.sub(r"#.*$", "", line) | |
684 line = re.sub(r"\s*", "", line) | |
685 if re.match("\s*$"): | |
686 continue | |
687 all_testnames.append(line) | |
688 f.close() | |
689 | |
690 tests_from_bundles = None | |
691 if self.options.bundles: | |
692 try: | |
693 tests_from_bundles = eval(open(self.options.bundles).read()) | |
694 except IOError: | |
695 logging.fatal('IO error in bundle file ' + | |
696 self.options.bundles + ' (doesn\'t exist?)') | |
697 except (NameError, SyntaxError): | |
698 logging.fatal('Parse or syntax error in bundle file ' + | |
699 self.options.bundles) | |
700 if hasattr(tests_from_bundles, '__iter__'): | |
701 all_testnames += tests_from_bundles | |
702 else: | |
703 logging.fatal('Fatal error with bundle file; could not get list from' + | |
704 self.options.bundles) | |
705 sys.exit(1) | |
706 | |
707 # If told explicit tests, run those (after stripping the name as | |
708 # appropriate) | |
709 for testname in all_testnames: | |
710 mo = re.search(r"(.*)\[(.*)\]$", testname) | |
711 gtest_filter = None | |
712 if mo: | |
713 gtest_filter = mo.group(2) | |
714 testname = mo.group(1) | |
715 if ':' in testname: | |
716 testname = testname.split(':')[1] | |
717 # We need 'pyautolib' to run pyauto tests and 'pyautolib' itself is not an | |
718 # executable. So skip this test from adding into coverage_bundles.py. | |
719 if testname == 'pyautolib': | |
720 continue | |
721 self.tests += [os.path.join(self.directory, testname)] | |
722 if gtest_filter: | |
723 self.test_filters[testname] = gtest_filter | |
724 | |
725 # Add 'src/test/functional/pyauto_functional.py' to self.tests. | |
726 # This file with '-v --suite=CODE_COVERAGE' arguments runs all pyauto tests. | |
727 # Pyauto tests are failing randomly on coverage bots. So excluding them. | |
728 # self.tests += [['src/chrome/test/functional/pyauto_functional.py', | |
729 # '-v', | |
730 # '--suite=CODE_COVERAGE']] | |
731 | |
732 # Medium tests? | |
733 # Not sure all of these work yet (e.g. page_cycler_tests) | |
734 # self.tests += glob.glob(os.path.join(self.directory, '*_tests')) | |
735 | |
736 # If needed, append .exe to tests since vsinstr.exe likes it that | |
737 # way. | |
738 if self.IsWindows(): | |
739 for ind in range(len(self.tests)): | |
740 test = self.tests[ind] | |
741 test_exe = test + '.exe' | |
742 if not test.endswith('.exe') and os.path.exists(test_exe): | |
743 self.tests[ind] = test_exe | |
744 | |
745 def TrimTests(self): | |
746 """Trim specific tests for each platform.""" | |
747 if self.IsWindows(): | |
748 return | |
749 # TODO(jrg): remove when not needed | |
750 inclusion = ['unit_tests'] | |
751 keep = [] | |
752 for test in self.tests: | |
753 for i in inclusion: | |
754 if i in test: | |
755 keep.append(test) | |
756 self.tests = keep | |
757 logging.info('After trimming tests we have ' + ' '.join(self.tests)) | |
758 return | |
759 if self.IsLinux(): | |
760 # self.tests = filter(lambda t: t.endswith('base_unittests'), self.tests) | |
761 return | |
762 if self.IsMac(): | |
763 exclusion = ['automated_ui_tests'] | |
764 punted = [] | |
765 for test in self.tests: | |
766 for e in exclusion: | |
767 if test.endswith(e): | |
768 punted.append(test) | |
769 self.tests = filter(lambda t: t not in punted, self.tests) | |
770 if punted: | |
771 logging.info('Tests trimmed out: ' + str(punted)) | |
772 | |
773 def ConfirmPlatformAndPaths(self): | |
774 """Confirm OS and paths (e.g. lcov).""" | |
775 for program in self.programs: | |
776 if not os.path.exists(program): | |
777 logging.fatal('Program missing: ' + program) | |
778 sys.exit(1) | |
779 | |
780 def Run(self, cmdlist, ignore_error=False, ignore_retcode=None, | |
781 explanation=None): | |
782 """Run the command list; exit fatally on error. | |
783 | |
784 Args: | |
785 cmdlist: a list of commands (e.g. to pass to subprocess.call) | |
786 ignore_error: if True log an error; if False then exit. | |
787 ignore_retcode: if retcode is non-zero, exit unless we ignore. | |
788 | |
789 Returns: process return code. | |
790 Throws: RunTooLongException if the process does not produce output | |
791 within TIMEOUT seconds; timeout is specified as a command line | |
792 option to the Coverage class and is set on init. | |
793 """ | |
794 logging.info('Running ' + str(cmdlist)) | |
795 t = RunProgramThread(cmdlist) | |
796 retcode = t.RunUntilCompletion(self.options.timeout) | |
797 | |
798 if retcode: | |
799 if ignore_error or retcode == ignore_retcode: | |
800 logging.warning('COVERAGE: %s unhappy but errors ignored %s' % | |
801 (str(cmdlist), explanation or '')) | |
802 else: | |
803 logging.fatal('COVERAGE: %s failed; return code: %d' % | |
804 (str(cmdlist), retcode)) | |
805 sys.exit(retcode) | |
806 return retcode | |
807 | |
808 def IsPosix(self): | |
809 """Return True if we are POSIX.""" | |
810 return self.IsMac() or self.IsLinux() | |
811 | |
812 def IsMac(self): | |
813 return sys.platform == 'darwin' | |
814 | |
815 def IsLinux(self): | |
816 return sys.platform.startswith('linux') | |
817 | |
818 def IsWindows(self): | |
819 """Return True if we are Windows.""" | |
820 return sys.platform in ('win32', 'cygwin') | |
821 | |
822 def ClearData(self): | |
823 """Clear old gcda files and old coverage info files.""" | |
824 if self.options.dont_clear_coverage_data: | |
825 print 'Clearing of coverage data NOT performed.' | |
826 return | |
827 print 'Clearing coverage data from previous runs.' | |
828 if os.path.exists(self.coverage_info_file): | |
829 os.remove(self.coverage_info_file) | |
830 if self.IsPosix(): | |
831 subprocess.call([self.lcov, | |
832 '--directory', self.directory_parent, | |
833 '--zerocounters']) | |
834 shutil.rmtree(os.path.join(self.directory, 'coverage')) | |
835 if self.options.all_unittests: | |
836 if os.path.exists(os.path.join(self.directory, 'unittests_coverage')): | |
837 shutil.rmtree(os.path.join(self.directory, 'unittests_coverage')) | |
838 elif self.options.all_browsertests: | |
839 if os.path.exists(os.path.join(self.directory, | |
840 'browsertests_coverage')): | |
841 shutil.rmtree(os.path.join(self.directory, 'browsertests_coverage')) | |
842 else: | |
843 if os.path.exists(os.path.join(self.directory, 'total_coverage')): | |
844 shutil.rmtree(os.path.join(self.directory, 'total_coverage')) | |
845 | |
846 def BeforeRunOneTest(self, testname): | |
847 """Do things before running each test.""" | |
848 if not self.IsWindows(): | |
849 return | |
850 # Stop old counters if needed | |
851 cmdlist = [self.perf, '-shutdown'] | |
852 self.Run(cmdlist, ignore_error=True) | |
853 # Instrument binaries | |
854 for fulltest in self.tests: | |
855 if os.path.exists(fulltest): | |
856 # See http://support.microsoft.com/kb/939818 for details on args | |
857 cmdlist = [self.instrument, '/d:ignorecverr', '/COVERAGE', fulltest] | |
858 self.Run(cmdlist, ignore_retcode=4, | |
859 explanation='OK with a multiple-instrument') | |
860 # Start new counters | |
861 cmdlist = [self.perf, '-start:coverage', '-output:' + self.vsts_output] | |
862 self.Run(cmdlist) | |
863 | |
864 def BeforeRunAllTests(self): | |
865 """Called right before we run all tests.""" | |
866 if self.IsLinux() and self.options.xvfb: | |
867 self.StartXvfb() | |
868 | |
869 def GtestFilter(self, fulltest, excl=None): | |
870 """Return a --gtest_filter=BLAH for this test. | |
871 | |
872 Args: | |
873 fulltest: full name of test executable | |
874 exclusions: the exclusions list. Only set in a unit test; | |
875 else uses gTestExclusions. | |
876 Returns: | |
877 String of the form '--gtest_filter=BLAH', or None. | |
878 """ | |
879 positive_gfilter_list = [] | |
880 negative_gfilter_list = [] | |
881 | |
882 # Exclude all flaky, failing, disabled and maybe tests; | |
883 # they don't count for code coverage. | |
884 negative_gfilter_list += ('*.FLAKY_*', '*.FAILS_*', | |
885 '*.DISABLED_*', '*.MAYBE_*') | |
886 | |
887 if not self.options.no_exclusions: | |
888 exclusions = excl or gTestExclusions | |
889 excldict = exclusions.get(sys.platform) | |
890 if excldict: | |
891 for test in excldict.keys(): | |
892 # example: if base_unittests in ../blah/blah/base_unittests.exe | |
893 if test in fulltest: | |
894 negative_gfilter_list += excldict[test] | |
895 | |
896 inclusions = gTestInclusions | |
897 include_dict = inclusions.get(sys.platform) | |
898 if include_dict: | |
899 for test in include_dict.keys(): | |
900 if test in fulltest: | |
901 positive_gfilter_list += include_dict[test] | |
902 | |
903 fulltest_basename = os.path.basename(fulltest) | |
904 if fulltest_basename in self.test_filters: | |
905 specific_test_filters = self.test_filters[fulltest_basename].split('-') | |
906 if len(specific_test_filters) > 2: | |
907 logging.error('Multiple "-" symbols in filter list: %s' % | |
908 self.test_filters[fulltest_basename]) | |
909 raise BadUserInput() | |
910 if len(specific_test_filters) == 2: | |
911 # Remove trailing ':' | |
912 specific_test_filters[0] = specific_test_filters[0][:-1] | |
913 | |
914 if specific_test_filters[0]: # Test for no positive filters. | |
915 positive_gfilter_list += specific_test_filters[0].split(':') | |
916 if len(specific_test_filters) > 1: | |
917 negative_gfilter_list += specific_test_filters[1].split(':') | |
918 | |
919 if not positive_gfilter_list and not negative_gfilter_list: | |
920 return None | |
921 | |
922 result = '--gtest_filter=' | |
923 if positive_gfilter_list: | |
924 result += ':'.join(positive_gfilter_list) | |
925 if negative_gfilter_list: | |
926 if positive_gfilter_list: result += ':' | |
927 result += '-' + ':'.join(negative_gfilter_list) | |
928 return result | |
929 | |
930 def RunTests(self): | |
931 """Run all unit tests and generate appropriate lcov files.""" | |
932 self.BeforeRunAllTests() | |
933 for fulltest in self.tests: | |
934 if type(fulltest) is str: | |
935 if not os.path.exists(fulltest): | |
936 logging.info(fulltest + ' does not exist') | |
937 if self.options.strict: | |
938 sys.exit(2) | |
939 else: | |
940 logging.info('%s path exists' % fulltest) | |
941 cmdlist = [fulltest, '--gtest_print_time'] | |
942 | |
943 # If asked, make this REAL fast for testing. | |
944 if self.options.fast_test: | |
945 logging.info('Running as a FAST test for testing') | |
946 # cmdlist.append('--gtest_filter=RenderWidgetHost*') | |
947 # cmdlist.append('--gtest_filter=CommandLine*') | |
948 cmdlist.append('--gtest_filter=C*') | |
949 | |
950 # Possibly add a test-specific --gtest_filter | |
951 filter = self.GtestFilter(fulltest) | |
952 if filter: | |
953 cmdlist.append(filter) | |
954 elif type(fulltest) is list: | |
955 cmdlist = fulltest | |
956 | |
957 self.BeforeRunOneTest(fulltest) | |
958 logging.info('Running test ' + str(cmdlist)) | |
959 try: | |
960 retcode = self.Run(cmdlist, ignore_retcode=True) | |
961 except SystemExit: # e.g. sys.exit() was called somewhere in here | |
962 raise | |
963 except: # can't "except WindowsError" since script runs on non-Windows | |
964 logging.info('EXCEPTION while running a unit test') | |
965 logging.info(traceback.format_exc()) | |
966 retcode = 999 | |
967 self.AfterRunOneTest(fulltest) | |
968 | |
969 if retcode: | |
970 logging.info('COVERAGE: test %s failed; return code: %d.' % | |
971 (fulltest, retcode)) | |
972 if self.options.strict: | |
973 logging.fatal('Test failure is fatal.') | |
974 sys.exit(retcode) | |
975 self.AfterRunAllTests() | |
976 | |
977 def AfterRunOneTest(self, testname): | |
978 """Do things right after running each test.""" | |
979 if not self.IsWindows(): | |
980 return | |
981 # Stop counters | |
982 cmdlist = [self.perf, '-shutdown'] | |
983 self.Run(cmdlist) | |
984 full_output = self.vsts_output + '.coverage' | |
985 shutil.move(full_output, self.vsts_output) | |
986 # generate lcov! | |
987 self.GenerateLcovWindows(testname) | |
988 | |
989 def AfterRunAllTests(self): | |
990 """Do things right after running ALL tests.""" | |
991 # On POSIX we can do it all at once without running out of memory. | |
992 # This contrasts with Windows where we must do it after each test. | |
993 if self.IsPosix(): | |
994 self.GenerateLcovPosix() | |
995 # Only on Linux do we have the Xvfb step. | |
996 if self.IsLinux() and self.options.xvfb: | |
997 self.StopXvfb() | |
998 | |
999 def StartXvfb(self): | |
1000 """Start Xvfb and set an appropriate DISPLAY environment. Linux only. | |
1001 | |
1002 Copied from http://src.chromium.org/viewvc/chrome/trunk/tools/buildbot/ | |
1003 scripts/slave/slave_utils.py?view=markup | |
1004 with some simplifications (e.g. no need to use xdisplaycheck, save | |
1005 pid in var not file, etc) | |
1006 """ | |
1007 logging.info('Xvfb: starting') | |
1008 proc = subprocess.Popen(["Xvfb", ":9", "-screen", "0", "1024x768x24", | |
1009 "-ac"], | |
1010 stdout=subprocess.PIPE, stderr=subprocess.STDOUT) | |
1011 self.xvfb_pid = proc.pid | |
1012 if not self.xvfb_pid: | |
1013 logging.info('Could not start Xvfb') | |
1014 return | |
1015 os.environ['DISPLAY'] = ":9" | |
1016 # Now confirm, giving a chance for it to start if needed. | |
1017 logging.info('Xvfb: confirming') | |
1018 for test in range(10): | |
1019 proc = subprocess.Popen('xdpyinfo >/dev/null', shell=True) | |
1020 pid, retcode = os.waitpid(proc.pid, 0) | |
1021 if retcode == 0: | |
1022 break | |
1023 time.sleep(0.5) | |
1024 if retcode != 0: | |
1025 logging.info('Warning: could not confirm Xvfb happiness') | |
1026 else: | |
1027 logging.info('Xvfb: OK') | |
1028 | |
1029 def StopXvfb(self): | |
1030 """Stop Xvfb if needed. Linux only.""" | |
1031 if self.xvfb_pid: | |
1032 logging.info('Xvfb: killing') | |
1033 try: | |
1034 os.kill(self.xvfb_pid, signal.SIGKILL) | |
1035 except: | |
1036 pass | |
1037 del os.environ['DISPLAY'] | |
1038 self.xvfb_pid = 0 | |
1039 | |
1040 def CopyCoverageFileToDestination(self, coverage_folder): | |
1041 coverage_dir = os.path.join(self.directory, coverage_folder) | |
1042 if not os.path.exists(coverage_dir): | |
1043 os.makedirs(coverage_dir) | |
1044 shutil.copyfile(self.coverage_info_file, os.path.join(coverage_dir, | |
1045 'coverage.info')) | |
1046 | |
1047 def GenerateLcovPosix(self): | |
1048 """Convert profile data to lcov on Mac or Linux.""" | |
1049 start_dir = os.getcwd() | |
1050 logging.info('GenerateLcovPosix: start_dir=' + start_dir) | |
1051 if self.IsLinux(): | |
1052 # With Linux/make (e.g. the coverage_run target), the current | |
1053 # directory for this command is .../build/src/chrome but we need | |
1054 # to be in .../build/src for the relative path of source files | |
1055 # to be correct. However, when run from buildbot, the current | |
1056 # directory is .../build. Accommodate. | |
1057 # On Mac source files are compiled with abs paths so this isn't | |
1058 # a problem. | |
1059 # This is a bit of a hack. The best answer is to require this | |
1060 # script be run in a specific directory for all cases (from | |
1061 # Makefile or from buildbot). | |
1062 if start_dir.endswith('chrome'): | |
1063 logging.info('coverage_posix.py: doing a "cd .." ' | |
1064 'to accomodate Linux/make PWD') | |
1065 os.chdir('..') | |
1066 elif start_dir.endswith('build'): | |
1067 logging.info('coverage_posix.py: doing a "cd src" ' | |
1068 'to accomodate buildbot PWD') | |
1069 os.chdir('src') | |
1070 else: | |
1071 logging.info('coverage_posix.py: NOT changing directory.') | |
1072 elif self.IsMac(): | |
1073 pass | |
1074 | |
1075 command = [self.mcov, | |
1076 '--directory', | |
1077 os.path.join(start_dir, self.directory_parent), | |
1078 '--output', | |
1079 os.path.join(start_dir, self.coverage_info_file)] | |
1080 logging.info('Assembly command: ' + ' '.join(command)) | |
1081 retcode = subprocess.call(command) | |
1082 if retcode: | |
1083 logging.fatal('COVERAGE: %s failed; return code: %d' % | |
1084 (command[0], retcode)) | |
1085 if self.options.strict: | |
1086 sys.exit(retcode) | |
1087 if self.IsLinux(): | |
1088 os.chdir(start_dir) | |
1089 | |
1090 # Copy the unittests coverage information to a different folder. | |
1091 if self.options.all_unittests: | |
1092 self.CopyCoverageFileToDestination('unittests_coverage') | |
1093 elif self.options.all_browsertests: | |
1094 # Save browsertests only coverage information. | |
1095 self.CopyCoverageFileToDestination('browsertests_coverage') | |
1096 else: | |
1097 # Save the overall coverage information. | |
1098 self.CopyCoverageFileToDestination('total_coverage') | |
1099 | |
1100 if not os.path.exists(self.coverage_info_file): | |
1101 logging.fatal('%s was not created. Coverage run failed.' % | |
1102 self.coverage_info_file) | |
1103 sys.exit(1) | |
1104 | |
1105 def GenerateLcovWindows(self, testname=None): | |
1106 """Convert VSTS format to lcov. Appends coverage data to sum file.""" | |
1107 lcov_file = self.vsts_output + '.lcov' | |
1108 if os.path.exists(lcov_file): | |
1109 os.remove(lcov_file) | |
1110 # generates the file (self.vsts_output + ".lcov") | |
1111 | |
1112 cmdlist = [self.analyzer, | |
1113 '-sym_path=' + self.directory, | |
1114 '-src_root=' + self.src_root, | |
1115 '-noxml', | |
1116 self.vsts_output] | |
1117 self.Run(cmdlist) | |
1118 if not os.path.exists(lcov_file): | |
1119 logging.fatal('Output file %s not created' % lcov_file) | |
1120 sys.exit(1) | |
1121 logging.info('Appending lcov for test %s to %s' % | |
1122 (testname, self.coverage_info_file)) | |
1123 size_before = 0 | |
1124 if os.path.exists(self.coverage_info_file): | |
1125 size_before = os.stat(self.coverage_info_file).st_size | |
1126 src = open(lcov_file, 'r') | |
1127 dst = open(self.coverage_info_file, 'a') | |
1128 dst.write(src.read()) | |
1129 src.close() | |
1130 dst.close() | |
1131 size_after = os.stat(self.coverage_info_file).st_size | |
1132 logging.info('Lcov file growth for %s: %d --> %d' % | |
1133 (self.coverage_info_file, size_before, size_after)) | |
1134 | |
1135 def GenerateHtml(self): | |
1136 """Convert lcov to html.""" | |
1137 # TODO(jrg): This isn't happy when run with unit_tests since V8 has a | |
1138 # different "base" so V8 includes can't be found in ".". Fix. | |
1139 command = [self.genhtml, | |
1140 self.coverage_info_file, | |
1141 '--output-directory', | |
1142 self.output_directory] | |
1143 print >>sys.stderr, 'html generation command: ' + ' '.join(command) | |
1144 retcode = subprocess.call(command) | |
1145 if retcode: | |
1146 logging.fatal('COVERAGE: %s failed; return code: %d' % | |
1147 (command[0], retcode)) | |
1148 if self.options.strict: | |
1149 sys.exit(retcode) | |
1150 | |
1151 def CoverageOptionParser(): | |
1152 """Return an optparse.OptionParser() suitable for Coverage object creation.""" | |
1153 parser = optparse.OptionParser() | |
1154 parser.add_option('-d', | |
1155 '--directory', | |
1156 dest='directory', | |
1157 default=None, | |
1158 help='Directory of unit test files') | |
1159 parser.add_option('-a', | |
1160 '--all_unittests', | |
1161 dest='all_unittests', | |
1162 default=False, | |
1163 help='Run all tests we can find (*_unittests)') | |
1164 parser.add_option('-b', | |
1165 '--all_browsertests', | |
1166 dest='all_browsertests', | |
1167 default=False, | |
1168 help='Run all tests in browser_tests ' | |
1169 'and content_browsertests') | |
1170 parser.add_option('-g', | |
1171 '--genhtml', | |
1172 dest='genhtml', | |
1173 default=False, | |
1174 help='Generate html from lcov output') | |
1175 parser.add_option('-f', | |
1176 '--fast_test', | |
1177 dest='fast_test', | |
1178 default=False, | |
1179 help='Make the tests run REAL fast by doing little.') | |
1180 parser.add_option('-s', | |
1181 '--strict', | |
1182 dest='strict', | |
1183 default=False, | |
1184 help='Be strict and die on test failure.') | |
1185 parser.add_option('-S', | |
1186 '--src_root', | |
1187 dest='src_root', | |
1188 default='.', | |
1189 help='Source root (only used on Windows)') | |
1190 parser.add_option('-t', | |
1191 '--trim', | |
1192 dest='trim', | |
1193 default=True, | |
1194 help='Trim out tests? Default True.') | |
1195 parser.add_option('-x', | |
1196 '--xvfb', | |
1197 dest='xvfb', | |
1198 default=True, | |
1199 help='Use Xvfb for tests? Default True.') | |
1200 parser.add_option('-T', | |
1201 '--timeout', | |
1202 dest='timeout', | |
1203 default=5.0 * 60.0, | |
1204 type="int", | |
1205 help='Timeout before bailing if a subprocess has no output.' | |
1206 ' Default is 5min (Buildbot is 10min.)') | |
1207 parser.add_option('-B', | |
1208 '--bundles', | |
1209 dest='bundles', | |
1210 default=None, | |
1211 help='Filename of bundles for coverage.') | |
1212 parser.add_option('--build-dir', | |
1213 dest='build_dir', | |
1214 default=None, | |
1215 help=('Working directory for buildbot build.' | |
1216 'used for finding bundlefile.')) | |
1217 parser.add_option('--target', | |
1218 dest='target', | |
1219 default=None, | |
1220 help=('Buildbot build target; ' | |
1221 'used for finding bundlefile (e.g. Debug)')) | |
1222 parser.add_option('--no_exclusions', | |
1223 dest='no_exclusions', | |
1224 default=None, | |
1225 help=('Disable the exclusion list.')) | |
1226 parser.add_option('--dont-clear-coverage-data', | |
1227 dest='dont_clear_coverage_data', | |
1228 default=False, | |
1229 action='store_true', | |
1230 help=('Turn off clearing of cov data from a prev run')) | |
1231 parser.add_option('-F', | |
1232 '--test-file', | |
1233 dest="test_files", | |
1234 default=[], | |
1235 action='append', | |
1236 help=('Specify a file from which tests to be run will ' + | |
1237 'be extracted')) | |
1238 return parser | |
1239 | |
1240 | |
1241 def main(): | |
1242 # Print out the args to help someone do it by hand if needed | |
1243 print >>sys.stderr, sys.argv | |
1244 | |
1245 # Try and clean up nice if we're killed by buildbot, Ctrl-C, ... | |
1246 signal.signal(signal.SIGINT, TerminateSignalHandler) | |
1247 signal.signal(signal.SIGTERM, TerminateSignalHandler) | |
1248 | |
1249 parser = CoverageOptionParser() | |
1250 (options, args) = parser.parse_args() | |
1251 if options.all_unittests and options.all_browsertests: | |
1252 print 'Error! Can not have all_unittests and all_browsertests together!' | |
1253 sys.exit(1) | |
1254 coverage = Coverage(options, args) | |
1255 coverage.ClearData() | |
1256 coverage.FindTests() | |
1257 if options.trim: | |
1258 coverage.TrimTests() | |
1259 coverage.RunTests() | |
1260 if options.genhtml: | |
1261 coverage.GenerateHtml() | |
1262 return 0 | |
1263 | |
1264 | |
1265 if __name__ == '__main__': | |
1266 sys.exit(main()) | |
OLD | NEW |