| OLD | NEW |
| (Empty) |
| 1 #!/usr/bin/env python | |
| 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | |
| 3 # Use of this source code is governed by a BSD-style license that can be | |
| 4 # found in the LICENSE file. | |
| 5 | |
| 6 """Generate and process code coverage. | |
| 7 | |
| 8 TODO(jrg): rename this from coverage_posix.py to coverage_all.py! | |
| 9 | |
| 10 Written for and tested on Mac, Linux, and Windows. To use this script | |
| 11 to generate coverage numbers, please run from within a gyp-generated | |
| 12 project. | |
| 13 | |
| 14 All platforms, to set up coverage: | |
| 15 cd ...../chromium ; src/tools/gyp/gyp_dogfood -Dcoverage=1 src/build/all.gyp | |
| 16 | |
| 17 Run coverage on... | |
| 18 Mac: | |
| 19 ( cd src/chrome ; xcodebuild -configuration Debug -target coverage ) | |
| 20 Linux: | |
| 21 ( cd src/chrome ; hammer coverage ) | |
| 22 # In particular, don't try and run 'coverage' from src/build | |
| 23 | |
| 24 | |
| 25 --directory=DIR: specify directory that contains gcda files, and where | |
| 26 a "coverage" directory will be created containing the output html. | |
| 27 Example name: ..../chromium/src/xcodebuild/Debug. | |
| 28 If not specified (e.g. buildbot) we will try and figure it out based on | |
| 29 other options (e.g. --target and --build-dir; see below). | |
| 30 | |
| 31 --genhtml: generate html output. If not specified only lcov is generated. | |
| 32 | |
| 33 --all_unittests: if present, run all files named *_unittests that we | |
| 34 can find. | |
| 35 | |
| 36 --fast_test: make the tests run real fast (just for testing) | |
| 37 | |
| 38 --strict: if a test fails, we continue happily. --strict will cause | |
| 39 us to die immediately. | |
| 40 | |
| 41 --trim=False: by default we trim away tests known to be problematic on | |
| 42 specific platforms. If set to false we do NOT trim out tests. | |
| 43 | |
| 44 --xvfb=True: By default we use Xvfb to make sure DISPLAY is valid | |
| 45 (Linux only). if set to False, do not use Xvfb. TODO(jrg): convert | |
| 46 this script from the compile stage of a builder to a | |
| 47 RunPythonCommandInBuildDir() command to avoid the need for this | |
| 48 step. | |
| 49 | |
| 50 --timeout=SECS: if a subprocess doesn't have output within SECS, | |
| 51 assume it's a hang. Kill it and give up. | |
| 52 | |
| 53 --bundles=BUNDLEFILE: a file containing a python list of coverage | |
| 54 bundles to be eval'd. Example contents of the bundlefile: | |
| 55 ['../base/base.gyp:base_unittests'] | |
| 56 This is used as part of the coverage bot. | |
| 57 If no other bundlefile-finding args are used (--target, | |
| 58 --build-dir), this is assumed to be an absolute path. | |
| 59 If those args are used, find BUNDLEFILE in a way consistent with | |
| 60 other scripts launched by buildbot. Example of another script | |
| 61 launched by buildbot: | |
| 62 http://src.chromium.org/viewvc/chrome/trunk/tools/buildbot/scripts/slave/runte
st.py | |
| 63 | |
| 64 --target=NAME: specify the build target (e.g. 'Debug' or 'Release'). | |
| 65 This is used by buildbot scripts to help us find the output directory. | |
| 66 Must be used with --build-dir. | |
| 67 | |
| 68 --build-dir=DIR: According to buildbot comments, this is the name of | |
| 69 the directory within the buildbot working directory in which the | |
| 70 solution, Debug, and Release directories are found. | |
| 71 It's usually "src/build", but on mac it's $DIR/../xcodebuild and on | |
| 72 Linux it's $DIR/out. | |
| 73 This is used by buildbot scripts to help us find the output directory. | |
| 74 Must be used with --target. | |
| 75 | |
| 76 --no_exclusions: Do NOT use the exclusion list. This script keeps a | |
| 77 list of tests known to be problematic under coverage. For example, | |
| 78 ProcessUtilTest.SpawnChild will crash inside __gcov_fork() when | |
| 79 using the MacOS 10.6 SDK. Use of --no_exclusions prevents the use | |
| 80 of this exclusion list. | |
| 81 | |
| 82 --dont-clear-coverage-data: Normally we clear coverage data from | |
| 83 previous runs. If this arg is used we do NOT clear the coverage | |
| 84 data. | |
| 85 | |
| 86 Strings after all options are considered tests to run. Test names | |
| 87 have all text before a ':' stripped to help with gyp compatibility. | |
| 88 For example, ../base/base.gyp:base_unittests is interpreted as a test | |
| 89 named "base_unittests". | |
| 90 """ | |
| 91 | |
| 92 import glob | |
| 93 import logging | |
| 94 import optparse | |
| 95 import os | |
| 96 import Queue | |
| 97 import re | |
| 98 import shutil | |
| 99 import signal | |
| 100 import subprocess | |
| 101 import sys | |
| 102 import tempfile | |
| 103 import threading | |
| 104 import time | |
| 105 import traceback | |
| 106 | |
| 107 """Global list of child PIDs to kill when we die.""" | |
| 108 gChildPIDs = [] | |
| 109 | |
| 110 """Exclusion list. Format is | |
| 111 { platform: { testname: (exclusion1, exclusion2, ... ), ... } } | |
| 112 | |
| 113 Platform is a match for sys.platform and can be a list. | |
| 114 Matching code does an 'if sys.platform in (the key):'. | |
| 115 Similarly, matching does an 'if testname in thefulltestname:' | |
| 116 | |
| 117 The Chromium convention has traditionally been to place the | |
| 118 exclusion list in a distinct file. Unlike valgrind (which has | |
| 119 frequent changes when things break and are fixed), the expectation | |
| 120 here is that exclusions remain relatively constant (e.g. OS bugs). | |
| 121 If that changes, revisit the decision to place inclusions in this | |
| 122 script. | |
| 123 | |
| 124 Details: | |
| 125 ProcessUtilTest.SpawnChild: chokes in __gcov_fork on 10.6 | |
| 126 IPCFuzzingTest.MsgBadPayloadArgs: ditto | |
| 127 PanelBrowserNavigatorTest.NavigateFromCrashedPanel: Fails on coverage bot. | |
| 128 WebGLConformanceTests.conformance_attribs_gl_enable_vertex_attrib: Fails | |
| 129 with timeout (45000 ms) exceeded error. crbug.com/143248 | |
| 130 WebGLConformanceTests.conformance_attribs_gl_disabled_vertex_attrib: | |
| 131 ditto. | |
| 132 WebGLConformanceTests.conformance_attribs_gl_vertex_attrib_zero_issues: | |
| 133 ditto. | |
| 134 WebGLConformanceTests.conformance_attribs_gl_vertex_attrib: ditto. | |
| 135 WebGLConformanceTests.conformance_attribs_gl_vertexattribpointer_offsets: | |
| 136 ditto. | |
| 137 WebGLConformanceTests.conformance_attribs_gl_vertexattribpointer: ditto. | |
| 138 WebGLConformanceTests.conformance_buffers_buffer_bind_test: After | |
| 139 disabling WebGLConformanceTests specified above, this test fails when run | |
| 140 on local machine. | |
| 141 WebGLConformanceTests.conformance_buffers_buffer_data_array_buffer: ditto. | |
| 142 WebGLConformanceTests.conformance_buffers_index_validation_copies_indices: | |
| 143 ditto. | |
| 144 WebGLConformanceTests. | |
| 145 conformance_buffers_index_validation_crash_with_buffer_sub_data: ditto. | |
| 146 WebGLConformanceTests. | |
| 147 conformance_buffers_index_validation_verifies_too_many_indices: ditto. | |
| 148 WebGLConformanceTests. | |
| 149 conformance_buffers_index_validation_with_resized_buffer: ditto. | |
| 150 WebGLConformanceTests.conformance_canvas_buffer_offscreen_test: ditto. | |
| 151 WebGLConformanceTests.conformance_canvas_buffer_preserve_test: ditto. | |
| 152 WebGLConformanceTests.conformance_canvas_canvas_test: ditto. | |
| 153 WebGLConformanceTests.conformance_canvas_canvas_zero_size: ditto. | |
| 154 WebGLConformanceTests. | |
| 155 conformance_canvas_drawingbuffer_static_canvas_test: ditto. | |
| 156 WebGLConformanceTests.conformance_canvas_drawingbuffer_test: ditto. | |
| 157 PageCycler*.*: Fails on coverage bot with "Missing test directory | |
| 158 /....../slave/coverage-dbg-linux/build/src/data/page_cycler/moz" error. | |
| 159 *FrameRateCompositingTest.*: Fails with | |
| 160 "FATAL:chrome_content_browser_client.cc(893)] Check failed: | |
| 161 command_line->HasSwitch(switches::kEnableStatsTable)." | |
| 162 *FrameRateNoVsyncCanvasInternalTest.*: ditto. | |
| 163 *FrameRateGpuCanvasInternalTest.*: ditto. | |
| 164 IndexedDBTest.Perf: Fails with 'Timeout reached in WaitUntilCookieValue' | |
| 165 error. | |
| 166 TwoClientPasswordsSyncTest.DeleteAll: Fails on coverage bot. | |
| 167 MigrationTwoClientTest.MigrationHellWithoutNigori: Fails with timeout | |
| 168 (45000 ms) exceeded error. | |
| 169 TwoClientSessionsSyncTest.DeleteActiveSession: ditto. | |
| 170 MultipleClientSessionsSyncTest.EncryptedAndChanged: ditto. | |
| 171 MigrationSingleClientTest.AllTypesIndividuallyTriggerNotification: ditto. | |
| 172 *OldPanelResizeBrowserTest.*: crbug.com/143247 | |
| 173 *OldPanelDragBrowserTest.*: ditto. | |
| 174 *OldPanelBrowserTest.*: ditto. | |
| 175 *OldPanelAndDesktopNotificationTest.*: ditto. | |
| 176 *OldDockedPanelBrowserTest.*: ditto. | |
| 177 *OldDetachedPanelBrowserTest.*: ditto. | |
| 178 PanelDragBrowserTest.AttachWithSqueeze: ditto. | |
| 179 *PanelBrowserTest.*: ditto. | |
| 180 *DockedPanelBrowserTest.*: ditto. | |
| 181 *DetachedPanelBrowserTest.*: ditto. | |
| 182 AutomatedUITest.TheOneAndOnlyTest: crbug.com/143419 | |
| 183 AutomatedUITestBase.DragOut: ditto | |
| 184 | |
| 185 """ | |
| 186 gTestExclusions = { | |
| 187 'darwin2': { 'base_unittests': ('ProcessUtilTest.SpawnChild',), | |
| 188 'ipc_tests': ('IPCFuzzingTest.MsgBadPayloadArgs',), }, | |
| 189 'linux2': { | |
| 190 'gpu_tests': | |
| 191 ('WebGLConformanceTests.conformance_attribs_gl_enable_vertex_attrib', | |
| 192 'WebGLConformanceTests.' | |
| 193 'conformance_attribs_gl_disabled_vertex_attrib', | |
| 194 'WebGLConformanceTests.' | |
| 195 'conformance_attribs_gl_vertex_attrib_zero_issues', | |
| 196 'WebGLConformanceTests.conformance_attribs_gl_vertex_attrib', | |
| 197 'WebGLConformanceTests.' | |
| 198 'conformance_attribs_gl_vertexattribpointer_offsets', | |
| 199 'WebGLConformanceTests.conformance_attribs_gl_vertexattribpointer', | |
| 200 'WebGLConformanceTests.conformance_buffers_buffer_bind_test', | |
| 201 'WebGLConformanceTests.' | |
| 202 'conformance_buffers_buffer_data_array_buffer', | |
| 203 'WebGLConformanceTests.' | |
| 204 'conformance_buffers_index_validation_copies_indices', | |
| 205 'WebGLConformanceTests.' | |
| 206 'conformance_buffers_index_validation_crash_with_buffer_sub_data', | |
| 207 'WebGLConformanceTests.' | |
| 208 'conformance_buffers_index_validation_verifies_too_many_indices', | |
| 209 'WebGLConformanceTests.' | |
| 210 'conformance_buffers_index_validation_with_resized_buffer', | |
| 211 'WebGLConformanceTests.conformance_canvas_buffer_offscreen_test', | |
| 212 'WebGLConformanceTests.conformance_canvas_buffer_preserve_test', | |
| 213 'WebGLConformanceTests.conformance_canvas_canvas_test', | |
| 214 'WebGLConformanceTests.conformance_canvas_canvas_zero_size', | |
| 215 'WebGLConformanceTests.' | |
| 216 'conformance_canvas_drawingbuffer_static_canvas_test', | |
| 217 'WebGLConformanceTests.conformance_canvas_drawingbuffer_test',), | |
| 218 'performance_ui_tests': | |
| 219 ('*PageCycler*.*', | |
| 220 '*FrameRateCompositingTest.*', | |
| 221 '*FrameRateNoVsyncCanvasInternalTest.*', | |
| 222 '*FrameRateGpuCanvasInternalTest.*', | |
| 223 'IndexedDBTest.Perf',), | |
| 224 'sync_integration_tests': | |
| 225 ('TwoClientPasswordsSyncTest.DeleteAll', | |
| 226 'MigrationTwoClientTest.MigrationHellWithoutNigori', | |
| 227 'TwoClientSessionsSyncTest.DeleteActiveSession', | |
| 228 'MultipleClientSessionsSyncTest.EncryptedAndChanged', | |
| 229 'MigrationSingleClientTest.' | |
| 230 'AllTypesIndividuallyTriggerNotification',), | |
| 231 'interactive_ui_tests': | |
| 232 ('*OldPanelResizeBrowserTest.*', | |
| 233 '*OldPanelDragBrowserTest.*', | |
| 234 '*OldPanelBrowserTest.*', | |
| 235 '*OldPanelAndDesktopNotificationTest.*', | |
| 236 '*OldDockedPanelBrowserTest.*', | |
| 237 '*OldDetachedPanelBrowserTest.*', | |
| 238 'PanelDragBrowserTest.AttachWithSqueeze', | |
| 239 '*PanelBrowserTest.*', | |
| 240 '*DockedPanelBrowserTest.*', | |
| 241 '*DetachedPanelBrowserTest.*',), | |
| 242 'automated_ui_tests': | |
| 243 ('AutomatedUITest.TheOneAndOnlyTest', | |
| 244 'AutomatedUITestBase.DragOut',), }, | |
| 245 } | |
| 246 | |
| 247 """Since random tests are failing/hanging on coverage bot, we are enabling | |
| 248 tests feature by feature. crbug.com/159748 | |
| 249 """ | |
| 250 gTestInclusions = { | |
| 251 'linux2': { | |
| 252 'browser_tests': | |
| 253 (# 'src/chrome/browser/downloads' | |
| 254 'SavePageBrowserTest.*', | |
| 255 'SavePageAsMHTMLBrowserTest.*', | |
| 256 'DownloadQueryTest.*', | |
| 257 'DownloadDangerPromptTest.*', | |
| 258 'DownloadTest.*', | |
| 259 # 'src/chrome/browser/net' | |
| 260 'CookiePolicyBrowserTest.*', | |
| 261 'FtpBrowserTest.*', | |
| 262 'LoadTimingObserverTest.*', | |
| 263 'PredictorBrowserTest.*', | |
| 264 'ProxyBrowserTest.*', | |
| 265 # 'src/chrome/browser/extensions' | |
| 266 'Extension*.*', | |
| 267 'WindowOpenPanelDisabledTest.*', | |
| 268 'WindowOpenPanelTest.*', | |
| 269 'WebstoreStandalone*.*', | |
| 270 'CommandLineWebstoreInstall.*', | |
| 271 'WebViewTest.*', | |
| 272 'RequirementsCheckerBrowserTest.*', | |
| 273 'ProcessManagementTest.*', | |
| 274 'PlatformAppBrowserTest.*', | |
| 275 'PlatformAppDevToolsBrowserTest.*', | |
| 276 'LazyBackgroundPageApiTest.*', | |
| 277 'IsolatedAppTest.*', | |
| 278 'PanelMessagingTest.*', | |
| 279 'GeolocationApiTest.*', | |
| 280 'ClipboardApiTest.*', | |
| 281 'ExecuteScriptApiTest.*', | |
| 282 'CalculatorBrowserTest.*', | |
| 283 'ChromeAppAPITest.*', | |
| 284 'AppApiTest.*', | |
| 285 'BlockedAppApiTest.*', | |
| 286 'AppBackgroundPageApiTest.*', | |
| 287 'WebNavigationApiTest.*', | |
| 288 'UsbApiTest.*', | |
| 289 'TabCaptureApiTest.*', | |
| 290 'SystemInfo*.*', | |
| 291 'SyncFileSystemApiTest.*', | |
| 292 'SocketApiTest.*', | |
| 293 'SerialApiTest.*', | |
| 294 'RecordApiTest.*', | |
| 295 'PushMessagingApiTest.*', | |
| 296 'ProxySettingsApiTest.*', | |
| 297 'ExperimentalApiTest.*', | |
| 298 'OmniboxApiTest.*', | |
| 299 'OffscreenTabsApiTest.*', | |
| 300 'NotificationApiTest.*', | |
| 301 'MediaGalleriesPrivateApiTest.*', | |
| 302 'PlatformAppMediaGalleriesBrowserTest.*', | |
| 303 'GetAuthTokenFunctionTest.*', | |
| 304 'LaunchWebAuthFlowFunctionTest.*', | |
| 305 'FileSystemApiTest.*', | |
| 306 'ScriptBadgeApiTest.*', | |
| 307 'PageAsBrowserActionApiTest.*', | |
| 308 'PageActionApiTest.*', | |
| 309 'BrowserActionApiTest.*', | |
| 310 'DownloadExtensionTest.*', | |
| 311 'DnsApiTest.*', | |
| 312 'DeclarativeApiTest.*', | |
| 313 'BluetoothApiTest.*', | |
| 314 'AllUrlsApiTest.*', | |
| 315 # 'src/chrome/browser/nacl_host' | |
| 316 'nacl_host.*', | |
| 317 # 'src/chrome/browser/automation' | |
| 318 'AutomationMiscBrowserTest.*', | |
| 319 # 'src/chrome/browser/autofill' | |
| 320 'FormStructureBrowserTest.*', | |
| 321 'AutofillPopupViewBrowserTest.*', | |
| 322 'AutofillTest.*', | |
| 323 # 'src/chrome/browser/autocomplete' | |
| 324 'AutocompleteBrowserTest.*', | |
| 325 # 'src/chrome/browser/captive_portal' | |
| 326 'CaptivePortalBrowserTest.*', | |
| 327 # 'src/chrome/browser/geolocation' | |
| 328 'GeolocationAccessTokenStoreTest.*', | |
| 329 'GeolocationBrowserTest.*', | |
| 330 # 'src/chrome/browser/nacl_host' | |
| 331 'NaClGdbTest.*', | |
| 332 # 'src/chrome/browser/devtools' | |
| 333 'DevToolsSanityTest.*', | |
| 334 'DevToolsExtensionTest.*', | |
| 335 'DevToolsExperimentalExtensionTest.*', | |
| 336 'WorkerDevToolsSanityTest.*', | |
| 337 # 'src/chrome/browser/first_run' | |
| 338 'FirstRunBrowserTest.*', | |
| 339 # 'src/chrome/browser/importer' | |
| 340 'ToolbarImporterUtilsTest.*', | |
| 341 # 'src/chrome/browser/page_cycler' | |
| 342 'PageCyclerBrowserTest.*', | |
| 343 'PageCyclerCachedBrowserTest.*', | |
| 344 # 'src/chrome/browser/performance_monitor' | |
| 345 'PerformanceMonitorBrowserTest.*', | |
| 346 'PerformanceMonitorUncleanExitBrowserTest.*', | |
| 347 'PerformanceMonitorSessionRestoreBrowserTest.*', | |
| 348 # 'src/chrome/browser/prerender' | |
| 349 'PrerenderBrowserTest.*', | |
| 350 'PrerenderBrowserTestWithNaCl.*', | |
| 351 'PrerenderBrowserTestWithExtensions.*', | |
| 352 'PrefetchBrowserTest.*', | |
| 353 'PrefetchBrowserTestNoPrefetching.*', ), | |
| 354 }, | |
| 355 } | |
| 356 | |
| 357 | |
| 358 def TerminateSignalHandler(sig, stack): | |
| 359 """When killed, try and kill our child processes.""" | |
| 360 signal.signal(sig, signal.SIG_DFL) | |
| 361 for pid in gChildPIDs: | |
| 362 if 'kill' in os.__all__: # POSIX | |
| 363 os.kill(pid, sig) | |
| 364 else: | |
| 365 subprocess.call(['taskkill.exe', '/PID', str(pid)]) | |
| 366 sys.exit(0) | |
| 367 | |
| 368 | |
| 369 class RunTooLongException(Exception): | |
| 370 """Thrown when a command runs too long without output.""" | |
| 371 pass | |
| 372 | |
| 373 class BadUserInput(Exception): | |
| 374 """Thrown when arguments from the user are incorrectly formatted.""" | |
| 375 pass | |
| 376 | |
| 377 | |
| 378 class RunProgramThread(threading.Thread): | |
| 379 """A thread to run a subprocess. | |
| 380 | |
| 381 We want to print the output of our subprocess in real time, but also | |
| 382 want a timeout if there has been no output for a certain amount of | |
| 383 time. Normal techniques (e.g. loop in select()) aren't cross | |
| 384 platform enough. the function seems simple: "print output of child, kill it | |
| 385 if there is no output by timeout. But it was tricky to get this right | |
| 386 in a x-platform way (see warnings about deadlock on the python | |
| 387 subprocess doc page). | |
| 388 | |
| 389 """ | |
| 390 # Constants in our queue | |
| 391 PROGRESS = 0 | |
| 392 DONE = 1 | |
| 393 | |
| 394 def __init__(self, cmd): | |
| 395 super(RunProgramThread, self).__init__() | |
| 396 self._cmd = cmd | |
| 397 self._process = None | |
| 398 self._queue = Queue.Queue() | |
| 399 self._retcode = None | |
| 400 | |
| 401 def run(self): | |
| 402 if sys.platform in ('win32', 'cygwin'): | |
| 403 return self._run_windows() | |
| 404 else: | |
| 405 self._run_posix() | |
| 406 | |
| 407 def _run_windows(self): | |
| 408 # We need to save stdout to a temporary file because of a bug on the | |
| 409 # windows implementation of python which can deadlock while waiting | |
| 410 # for the IO to complete while writing to the PIPE and the pipe waiting | |
| 411 # on us and us waiting on the child process. | |
| 412 stdout_file = tempfile.TemporaryFile() | |
| 413 try: | |
| 414 self._process = subprocess.Popen(self._cmd, | |
| 415 stdin=subprocess.PIPE, | |
| 416 stdout=stdout_file, | |
| 417 stderr=subprocess.STDOUT) | |
| 418 gChildPIDs.append(self._process.pid) | |
| 419 try: | |
| 420 # To make sure that the buildbot don't kill us if we run too long | |
| 421 # without any activity on the console output, we look for progress in | |
| 422 # the length of the temporary file and we print what was accumulated so | |
| 423 # far to the output console to make the buildbot know we are making some | |
| 424 # progress. | |
| 425 previous_tell = 0 | |
| 426 # We will poll the process until we get a non-None return code. | |
| 427 self._retcode = None | |
| 428 while self._retcode is None: | |
| 429 self._retcode = self._process.poll() | |
| 430 current_tell = stdout_file.tell() | |
| 431 if current_tell > previous_tell: | |
| 432 # Report progress to our main thread so we don't timeout. | |
| 433 self._queue.put(RunProgramThread.PROGRESS) | |
| 434 # And print what was accumulated to far. | |
| 435 stdout_file.seek(previous_tell) | |
| 436 print stdout_file.read(current_tell - previous_tell), | |
| 437 previous_tell = current_tell | |
| 438 # Don't be selfish, let other threads do stuff while we wait for | |
| 439 # the process to complete. | |
| 440 time.sleep(0.5) | |
| 441 # OK, the child process has exited, let's print its output to our | |
| 442 # console to create debugging logs in case they get to be needed. | |
| 443 stdout_file.flush() | |
| 444 stdout_file.seek(previous_tell) | |
| 445 print stdout_file.read(stdout_file.tell() - previous_tell) | |
| 446 except IOError, e: | |
| 447 logging.exception('%s', e) | |
| 448 pass | |
| 449 finally: | |
| 450 stdout_file.close() | |
| 451 | |
| 452 # If we get here the process is done. | |
| 453 gChildPIDs.remove(self._process.pid) | |
| 454 self._queue.put(RunProgramThread.DONE) | |
| 455 | |
| 456 def _run_posix(self): | |
| 457 """No deadlock problem so use the simple answer. The windows solution | |
| 458 appears to add extra buffering which we don't want on other platforms.""" | |
| 459 self._process = subprocess.Popen(self._cmd, | |
| 460 stdout=subprocess.PIPE, | |
| 461 stderr=subprocess.STDOUT) | |
| 462 gChildPIDs.append(self._process.pid) | |
| 463 try: | |
| 464 while True: | |
| 465 line = self._process.stdout.readline() | |
| 466 if not line: # EOF | |
| 467 break | |
| 468 print line, | |
| 469 self._queue.put(RunProgramThread.PROGRESS, True) | |
| 470 except IOError: | |
| 471 pass | |
| 472 # If we get here the process is done. | |
| 473 gChildPIDs.remove(self._process.pid) | |
| 474 self._queue.put(RunProgramThread.DONE) | |
| 475 | |
| 476 def stop(self): | |
| 477 self.kill() | |
| 478 | |
| 479 def kill(self): | |
| 480 """Kill our running process if needed. Wait for kill to complete. | |
| 481 | |
| 482 Should be called in the PARENT thread; we do not self-kill. | |
| 483 Returns the return code of the process. | |
| 484 Safe to call even if the process is dead. | |
| 485 """ | |
| 486 if not self._process: | |
| 487 return self.retcode() | |
| 488 if 'kill' in os.__all__: # POSIX | |
| 489 os.kill(self._process.pid, signal.SIGKILL) | |
| 490 else: | |
| 491 subprocess.call(['taskkill.exe', '/PID', str(self._process.pid)]) | |
| 492 return self.retcode() | |
| 493 | |
| 494 def retcode(self): | |
| 495 """Return the return value of the subprocess. | |
| 496 | |
| 497 Waits for process to die but does NOT kill it explicitly. | |
| 498 """ | |
| 499 if self._retcode == None: # must be none, not 0/False | |
| 500 self._retcode = self._process.wait() | |
| 501 return self._retcode | |
| 502 | |
| 503 def RunUntilCompletion(self, timeout): | |
| 504 """Run thread until completion or timeout (in seconds). | |
| 505 | |
| 506 Start the thread. Let it run until completion, or until we've | |
| 507 spent TIMEOUT without seeing output. On timeout throw | |
| 508 RunTooLongException. | |
| 509 """ | |
| 510 self.start() | |
| 511 while True: | |
| 512 try: | |
| 513 x = self._queue.get(True, timeout) | |
| 514 if x == RunProgramThread.DONE: | |
| 515 return self.retcode() | |
| 516 except Queue.Empty, e: # timed out | |
| 517 logging.info('TIMEOUT (%d seconds exceeded with no output): killing' % | |
| 518 timeout) | |
| 519 self.kill() | |
| 520 raise RunTooLongException() | |
| 521 | |
| 522 | |
| 523 class Coverage(object): | |
| 524 """Doitall class for code coverage.""" | |
| 525 | |
| 526 def __init__(self, options, args): | |
| 527 super(Coverage, self).__init__() | |
| 528 logging.basicConfig(level=logging.DEBUG) | |
| 529 self.directory = options.directory | |
| 530 self.options = options | |
| 531 self.args = args | |
| 532 self.ConfirmDirectory() | |
| 533 self.directory_parent = os.path.dirname(self.directory) | |
| 534 self.output_directory = os.path.join(self.directory, 'coverage') | |
| 535 if not os.path.exists(self.output_directory): | |
| 536 os.mkdir(self.output_directory) | |
| 537 # The "final" lcov-format file | |
| 538 self.coverage_info_file = os.path.join(self.directory, 'coverage.info') | |
| 539 # If needed, an intermediate VSTS-format file | |
| 540 self.vsts_output = os.path.join(self.directory, 'coverage.vsts') | |
| 541 # Needed for Windows. | |
| 542 self.src_root = options.src_root | |
| 543 self.FindPrograms() | |
| 544 self.ConfirmPlatformAndPaths() | |
| 545 self.tests = [] # This can be a list of strings, lists or both. | |
| 546 self.xvfb_pid = 0 | |
| 547 self.test_files = [] # List of files with test specifications. | |
| 548 self.test_filters = {} # Mapping from testname->--gtest_filter arg. | |
| 549 logging.info('self.directory: ' + self.directory) | |
| 550 logging.info('self.directory_parent: ' + self.directory_parent) | |
| 551 | |
| 552 def FindInPath(self, program): | |
| 553 """Find program in our path. Return abs path to it, or None.""" | |
| 554 if not 'PATH' in os.environ: | |
| 555 logging.fatal('No PATH environment variable?') | |
| 556 sys.exit(1) | |
| 557 paths = os.environ['PATH'].split(os.pathsep) | |
| 558 for path in paths: | |
| 559 fullpath = os.path.join(path, program) | |
| 560 if os.path.exists(fullpath): | |
| 561 return fullpath | |
| 562 return None | |
| 563 | |
| 564 def FindPrograms(self): | |
| 565 """Find programs we may want to run.""" | |
| 566 if self.IsPosix(): | |
| 567 self.lcov_directory = os.path.join(sys.path[0], | |
| 568 '../../third_party/lcov/bin') | |
| 569 self.lcov = os.path.join(self.lcov_directory, 'lcov') | |
| 570 self.mcov = os.path.join(self.lcov_directory, 'mcov') | |
| 571 self.genhtml = os.path.join(self.lcov_directory, 'genhtml') | |
| 572 self.programs = [self.lcov, self.mcov, self.genhtml] | |
| 573 else: | |
| 574 # Hack to get the buildbot working. | |
| 575 os.environ['PATH'] += r';c:\coverage\coverage_analyzer' | |
| 576 os.environ['PATH'] += r';c:\coverage\performance_tools' | |
| 577 # (end hack) | |
| 578 commands = ['vsperfcmd.exe', 'vsinstr.exe', 'coverage_analyzer.exe'] | |
| 579 self.perf = self.FindInPath('vsperfcmd.exe') | |
| 580 self.instrument = self.FindInPath('vsinstr.exe') | |
| 581 self.analyzer = self.FindInPath('coverage_analyzer.exe') | |
| 582 if not self.perf or not self.instrument or not self.analyzer: | |
| 583 logging.fatal('Could not find Win performance commands.') | |
| 584 logging.fatal('Commands needed in PATH: ' + str(commands)) | |
| 585 sys.exit(1) | |
| 586 self.programs = [self.perf, self.instrument, self.analyzer] | |
| 587 | |
| 588 def PlatformBuildPrefix(self): | |
| 589 """Return a platform specific build directory prefix. | |
| 590 | |
| 591 This prefix is prepended to the build target (Debug, Release) to | |
| 592 identify output as relative to the build directory. | |
| 593 These values are specific to Chromium's use of gyp. | |
| 594 """ | |
| 595 if self.IsMac(): | |
| 596 return '../xcodebuild' | |
| 597 if self.IsWindows(): | |
| 598 return '' | |
| 599 else: # Linux | |
| 600 return '../out' # assumes make, unlike runtest.py | |
| 601 | |
| 602 def ConfirmDirectory(self): | |
| 603 """Confirm correctness of self.directory. | |
| 604 | |
| 605 If it exists, happiness. If not, try and figure it out in a | |
| 606 manner similar to FindBundlesFile(). The 'figure it out' case | |
| 607 happens with buildbot where the directory isn't specified | |
| 608 explicitly. | |
| 609 """ | |
| 610 if (not self.directory and | |
| 611 not (self.options.target and self.options.build_dir)): | |
| 612 logging.fatal('Must use --directory or (--target and --build-dir)') | |
| 613 sys.exit(1) | |
| 614 | |
| 615 if not self.directory: | |
| 616 self.directory = os.path.join(self.options.build_dir, | |
| 617 self.PlatformBuildPrefix(), | |
| 618 self.options.target) | |
| 619 | |
| 620 if os.path.exists(self.directory): | |
| 621 logging.info('Directory: ' + self.directory) | |
| 622 return | |
| 623 else: | |
| 624 logging.fatal('Directory ' + | |
| 625 self.directory + ' doesn\'t exist') | |
| 626 sys.exit(1) | |
| 627 | |
| 628 | |
| 629 def FindBundlesFile(self): | |
| 630 """Find the bundlesfile. | |
| 631 | |
| 632 The 'bundles' file can be either absolute path, or (if we are run | |
| 633 from buildbot) we need to find it based on other hints (--target, | |
| 634 --build-dir, etc). | |
| 635 """ | |
| 636 # If no bundle file, no problem! | |
| 637 if not self.options.bundles: | |
| 638 return | |
| 639 # If true, we're buildbot. Form a path. | |
| 640 # Else assume absolute. | |
| 641 if self.options.target and self.options.build_dir: | |
| 642 fullpath = os.path.join(self.options.build_dir, | |
| 643 self.PlatformBuildPrefix(), | |
| 644 self.options.target, | |
| 645 self.options.bundles) | |
| 646 self.options.bundles = fullpath | |
| 647 | |
| 648 if os.path.exists(self.options.bundles): | |
| 649 logging.info('BundlesFile: ' + self.options.bundles) | |
| 650 return | |
| 651 else: | |
| 652 logging.fatal('bundlefile ' + | |
| 653 self.options.bundles + ' doesn\'t exist') | |
| 654 sys.exit(1) | |
| 655 | |
| 656 | |
| 657 def FindTests(self): | |
| 658 """Find unit tests to run; set self.tests to this list. | |
| 659 | |
| 660 Assume all non-option items in the arg list are tests to be run. | |
| 661 """ | |
| 662 # Before we begin, find the bundles file if not an absolute path. | |
| 663 self.FindBundlesFile() | |
| 664 | |
| 665 # Small tests: can be run in the "chromium" directory. | |
| 666 # If asked, run all we can find. | |
| 667 if self.options.all_unittests: | |
| 668 self.tests += glob.glob(os.path.join(self.directory, '*_unittests')) | |
| 669 self.tests += glob.glob(os.path.join(self.directory, '*unit_tests')) | |
| 670 elif self.options.all_browsertests: | |
| 671 # Run all tests in browser_tests and content_browsertests. | |
| 672 self.tests += glob.glob(os.path.join(self.directory, 'browser_tests')) | |
| 673 self.tests += glob.glob(os.path.join(self.directory, | |
| 674 'content_browsertests')) | |
| 675 | |
| 676 # Tests can come in as args directly, indirectly (through a file | |
| 677 # of test lists) or as a file of bundles. | |
| 678 all_testnames = self.args[:] # Copy since we might modify | |
| 679 | |
| 680 for test_file in self.options.test_files: | |
| 681 f = open(test_file) | |
| 682 for line in f: | |
| 683 line = re.sub(r"#.*$", "", line) | |
| 684 line = re.sub(r"\s*", "", line) | |
| 685 if re.match("\s*$"): | |
| 686 continue | |
| 687 all_testnames.append(line) | |
| 688 f.close() | |
| 689 | |
| 690 tests_from_bundles = None | |
| 691 if self.options.bundles: | |
| 692 try: | |
| 693 tests_from_bundles = eval(open(self.options.bundles).read()) | |
| 694 except IOError: | |
| 695 logging.fatal('IO error in bundle file ' + | |
| 696 self.options.bundles + ' (doesn\'t exist?)') | |
| 697 except (NameError, SyntaxError): | |
| 698 logging.fatal('Parse or syntax error in bundle file ' + | |
| 699 self.options.bundles) | |
| 700 if hasattr(tests_from_bundles, '__iter__'): | |
| 701 all_testnames += tests_from_bundles | |
| 702 else: | |
| 703 logging.fatal('Fatal error with bundle file; could not get list from' + | |
| 704 self.options.bundles) | |
| 705 sys.exit(1) | |
| 706 | |
| 707 # If told explicit tests, run those (after stripping the name as | |
| 708 # appropriate) | |
| 709 for testname in all_testnames: | |
| 710 mo = re.search(r"(.*)\[(.*)\]$", testname) | |
| 711 gtest_filter = None | |
| 712 if mo: | |
| 713 gtest_filter = mo.group(2) | |
| 714 testname = mo.group(1) | |
| 715 if ':' in testname: | |
| 716 testname = testname.split(':')[1] | |
| 717 # We need 'pyautolib' to run pyauto tests and 'pyautolib' itself is not an | |
| 718 # executable. So skip this test from adding into coverage_bundles.py. | |
| 719 if testname == 'pyautolib': | |
| 720 continue | |
| 721 self.tests += [os.path.join(self.directory, testname)] | |
| 722 if gtest_filter: | |
| 723 self.test_filters[testname] = gtest_filter | |
| 724 | |
| 725 # Add 'src/test/functional/pyauto_functional.py' to self.tests. | |
| 726 # This file with '-v --suite=CODE_COVERAGE' arguments runs all pyauto tests. | |
| 727 # Pyauto tests are failing randomly on coverage bots. So excluding them. | |
| 728 # self.tests += [['src/chrome/test/functional/pyauto_functional.py', | |
| 729 # '-v', | |
| 730 # '--suite=CODE_COVERAGE']] | |
| 731 | |
| 732 # Medium tests? | |
| 733 # Not sure all of these work yet (e.g. page_cycler_tests) | |
| 734 # self.tests += glob.glob(os.path.join(self.directory, '*_tests')) | |
| 735 | |
| 736 # If needed, append .exe to tests since vsinstr.exe likes it that | |
| 737 # way. | |
| 738 if self.IsWindows(): | |
| 739 for ind in range(len(self.tests)): | |
| 740 test = self.tests[ind] | |
| 741 test_exe = test + '.exe' | |
| 742 if not test.endswith('.exe') and os.path.exists(test_exe): | |
| 743 self.tests[ind] = test_exe | |
| 744 | |
| 745 def TrimTests(self): | |
| 746 """Trim specific tests for each platform.""" | |
| 747 if self.IsWindows(): | |
| 748 return | |
| 749 # TODO(jrg): remove when not needed | |
| 750 inclusion = ['unit_tests'] | |
| 751 keep = [] | |
| 752 for test in self.tests: | |
| 753 for i in inclusion: | |
| 754 if i in test: | |
| 755 keep.append(test) | |
| 756 self.tests = keep | |
| 757 logging.info('After trimming tests we have ' + ' '.join(self.tests)) | |
| 758 return | |
| 759 if self.IsLinux(): | |
| 760 # self.tests = filter(lambda t: t.endswith('base_unittests'), self.tests) | |
| 761 return | |
| 762 if self.IsMac(): | |
| 763 exclusion = ['automated_ui_tests'] | |
| 764 punted = [] | |
| 765 for test in self.tests: | |
| 766 for e in exclusion: | |
| 767 if test.endswith(e): | |
| 768 punted.append(test) | |
| 769 self.tests = filter(lambda t: t not in punted, self.tests) | |
| 770 if punted: | |
| 771 logging.info('Tests trimmed out: ' + str(punted)) | |
| 772 | |
| 773 def ConfirmPlatformAndPaths(self): | |
| 774 """Confirm OS and paths (e.g. lcov).""" | |
| 775 for program in self.programs: | |
| 776 if not os.path.exists(program): | |
| 777 logging.fatal('Program missing: ' + program) | |
| 778 sys.exit(1) | |
| 779 | |
| 780 def Run(self, cmdlist, ignore_error=False, ignore_retcode=None, | |
| 781 explanation=None): | |
| 782 """Run the command list; exit fatally on error. | |
| 783 | |
| 784 Args: | |
| 785 cmdlist: a list of commands (e.g. to pass to subprocess.call) | |
| 786 ignore_error: if True log an error; if False then exit. | |
| 787 ignore_retcode: if retcode is non-zero, exit unless we ignore. | |
| 788 | |
| 789 Returns: process return code. | |
| 790 Throws: RunTooLongException if the process does not produce output | |
| 791 within TIMEOUT seconds; timeout is specified as a command line | |
| 792 option to the Coverage class and is set on init. | |
| 793 """ | |
| 794 logging.info('Running ' + str(cmdlist)) | |
| 795 t = RunProgramThread(cmdlist) | |
| 796 retcode = t.RunUntilCompletion(self.options.timeout) | |
| 797 | |
| 798 if retcode: | |
| 799 if ignore_error or retcode == ignore_retcode: | |
| 800 logging.warning('COVERAGE: %s unhappy but errors ignored %s' % | |
| 801 (str(cmdlist), explanation or '')) | |
| 802 else: | |
| 803 logging.fatal('COVERAGE: %s failed; return code: %d' % | |
| 804 (str(cmdlist), retcode)) | |
| 805 sys.exit(retcode) | |
| 806 return retcode | |
| 807 | |
| 808 def IsPosix(self): | |
| 809 """Return True if we are POSIX.""" | |
| 810 return self.IsMac() or self.IsLinux() | |
| 811 | |
| 812 def IsMac(self): | |
| 813 return sys.platform == 'darwin' | |
| 814 | |
| 815 def IsLinux(self): | |
| 816 return sys.platform.startswith('linux') | |
| 817 | |
| 818 def IsWindows(self): | |
| 819 """Return True if we are Windows.""" | |
| 820 return sys.platform in ('win32', 'cygwin') | |
| 821 | |
| 822 def ClearData(self): | |
| 823 """Clear old gcda files and old coverage info files.""" | |
| 824 if self.options.dont_clear_coverage_data: | |
| 825 print 'Clearing of coverage data NOT performed.' | |
| 826 return | |
| 827 print 'Clearing coverage data from previous runs.' | |
| 828 if os.path.exists(self.coverage_info_file): | |
| 829 os.remove(self.coverage_info_file) | |
| 830 if self.IsPosix(): | |
| 831 subprocess.call([self.lcov, | |
| 832 '--directory', self.directory_parent, | |
| 833 '--zerocounters']) | |
| 834 shutil.rmtree(os.path.join(self.directory, 'coverage')) | |
| 835 if self.options.all_unittests: | |
| 836 if os.path.exists(os.path.join(self.directory, 'unittests_coverage')): | |
| 837 shutil.rmtree(os.path.join(self.directory, 'unittests_coverage')) | |
| 838 elif self.options.all_browsertests: | |
| 839 if os.path.exists(os.path.join(self.directory, | |
| 840 'browsertests_coverage')): | |
| 841 shutil.rmtree(os.path.join(self.directory, 'browsertests_coverage')) | |
| 842 else: | |
| 843 if os.path.exists(os.path.join(self.directory, 'total_coverage')): | |
| 844 shutil.rmtree(os.path.join(self.directory, 'total_coverage')) | |
| 845 | |
| 846 def BeforeRunOneTest(self, testname): | |
| 847 """Do things before running each test.""" | |
| 848 if not self.IsWindows(): | |
| 849 return | |
| 850 # Stop old counters if needed | |
| 851 cmdlist = [self.perf, '-shutdown'] | |
| 852 self.Run(cmdlist, ignore_error=True) | |
| 853 # Instrument binaries | |
| 854 for fulltest in self.tests: | |
| 855 if os.path.exists(fulltest): | |
| 856 # See http://support.microsoft.com/kb/939818 for details on args | |
| 857 cmdlist = [self.instrument, '/d:ignorecverr', '/COVERAGE', fulltest] | |
| 858 self.Run(cmdlist, ignore_retcode=4, | |
| 859 explanation='OK with a multiple-instrument') | |
| 860 # Start new counters | |
| 861 cmdlist = [self.perf, '-start:coverage', '-output:' + self.vsts_output] | |
| 862 self.Run(cmdlist) | |
| 863 | |
| 864 def BeforeRunAllTests(self): | |
| 865 """Called right before we run all tests.""" | |
| 866 if self.IsLinux() and self.options.xvfb: | |
| 867 self.StartXvfb() | |
| 868 | |
| 869 def GtestFilter(self, fulltest, excl=None): | |
| 870 """Return a --gtest_filter=BLAH for this test. | |
| 871 | |
| 872 Args: | |
| 873 fulltest: full name of test executable | |
| 874 exclusions: the exclusions list. Only set in a unit test; | |
| 875 else uses gTestExclusions. | |
| 876 Returns: | |
| 877 String of the form '--gtest_filter=BLAH', or None. | |
| 878 """ | |
| 879 positive_gfilter_list = [] | |
| 880 negative_gfilter_list = [] | |
| 881 | |
| 882 # Exclude all flaky, failing, disabled and maybe tests; | |
| 883 # they don't count for code coverage. | |
| 884 negative_gfilter_list += ('*.FLAKY_*', '*.FAILS_*', | |
| 885 '*.DISABLED_*', '*.MAYBE_*') | |
| 886 | |
| 887 if not self.options.no_exclusions: | |
| 888 exclusions = excl or gTestExclusions | |
| 889 excldict = exclusions.get(sys.platform) | |
| 890 if excldict: | |
| 891 for test in excldict.keys(): | |
| 892 # example: if base_unittests in ../blah/blah/base_unittests.exe | |
| 893 if test in fulltest: | |
| 894 negative_gfilter_list += excldict[test] | |
| 895 | |
| 896 inclusions = gTestInclusions | |
| 897 include_dict = inclusions.get(sys.platform) | |
| 898 if include_dict: | |
| 899 for test in include_dict.keys(): | |
| 900 if test in fulltest: | |
| 901 positive_gfilter_list += include_dict[test] | |
| 902 | |
| 903 fulltest_basename = os.path.basename(fulltest) | |
| 904 if fulltest_basename in self.test_filters: | |
| 905 specific_test_filters = self.test_filters[fulltest_basename].split('-') | |
| 906 if len(specific_test_filters) > 2: | |
| 907 logging.error('Multiple "-" symbols in filter list: %s' % | |
| 908 self.test_filters[fulltest_basename]) | |
| 909 raise BadUserInput() | |
| 910 if len(specific_test_filters) == 2: | |
| 911 # Remove trailing ':' | |
| 912 specific_test_filters[0] = specific_test_filters[0][:-1] | |
| 913 | |
| 914 if specific_test_filters[0]: # Test for no positive filters. | |
| 915 positive_gfilter_list += specific_test_filters[0].split(':') | |
| 916 if len(specific_test_filters) > 1: | |
| 917 negative_gfilter_list += specific_test_filters[1].split(':') | |
| 918 | |
| 919 if not positive_gfilter_list and not negative_gfilter_list: | |
| 920 return None | |
| 921 | |
| 922 result = '--gtest_filter=' | |
| 923 if positive_gfilter_list: | |
| 924 result += ':'.join(positive_gfilter_list) | |
| 925 if negative_gfilter_list: | |
| 926 if positive_gfilter_list: result += ':' | |
| 927 result += '-' + ':'.join(negative_gfilter_list) | |
| 928 return result | |
| 929 | |
| 930 def RunTests(self): | |
| 931 """Run all unit tests and generate appropriate lcov files.""" | |
| 932 self.BeforeRunAllTests() | |
| 933 for fulltest in self.tests: | |
| 934 if type(fulltest) is str: | |
| 935 if not os.path.exists(fulltest): | |
| 936 logging.info(fulltest + ' does not exist') | |
| 937 if self.options.strict: | |
| 938 sys.exit(2) | |
| 939 else: | |
| 940 logging.info('%s path exists' % fulltest) | |
| 941 cmdlist = [fulltest, '--gtest_print_time'] | |
| 942 | |
| 943 # If asked, make this REAL fast for testing. | |
| 944 if self.options.fast_test: | |
| 945 logging.info('Running as a FAST test for testing') | |
| 946 # cmdlist.append('--gtest_filter=RenderWidgetHost*') | |
| 947 # cmdlist.append('--gtest_filter=CommandLine*') | |
| 948 cmdlist.append('--gtest_filter=C*') | |
| 949 | |
| 950 # Possibly add a test-specific --gtest_filter | |
| 951 filter = self.GtestFilter(fulltest) | |
| 952 if filter: | |
| 953 cmdlist.append(filter) | |
| 954 elif type(fulltest) is list: | |
| 955 cmdlist = fulltest | |
| 956 | |
| 957 self.BeforeRunOneTest(fulltest) | |
| 958 logging.info('Running test ' + str(cmdlist)) | |
| 959 try: | |
| 960 retcode = self.Run(cmdlist, ignore_retcode=True) | |
| 961 except SystemExit: # e.g. sys.exit() was called somewhere in here | |
| 962 raise | |
| 963 except: # can't "except WindowsError" since script runs on non-Windows | |
| 964 logging.info('EXCEPTION while running a unit test') | |
| 965 logging.info(traceback.format_exc()) | |
| 966 retcode = 999 | |
| 967 self.AfterRunOneTest(fulltest) | |
| 968 | |
| 969 if retcode: | |
| 970 logging.info('COVERAGE: test %s failed; return code: %d.' % | |
| 971 (fulltest, retcode)) | |
| 972 if self.options.strict: | |
| 973 logging.fatal('Test failure is fatal.') | |
| 974 sys.exit(retcode) | |
| 975 self.AfterRunAllTests() | |
| 976 | |
| 977 def AfterRunOneTest(self, testname): | |
| 978 """Do things right after running each test.""" | |
| 979 if not self.IsWindows(): | |
| 980 return | |
| 981 # Stop counters | |
| 982 cmdlist = [self.perf, '-shutdown'] | |
| 983 self.Run(cmdlist) | |
| 984 full_output = self.vsts_output + '.coverage' | |
| 985 shutil.move(full_output, self.vsts_output) | |
| 986 # generate lcov! | |
| 987 self.GenerateLcovWindows(testname) | |
| 988 | |
| 989 def AfterRunAllTests(self): | |
| 990 """Do things right after running ALL tests.""" | |
| 991 # On POSIX we can do it all at once without running out of memory. | |
| 992 # This contrasts with Windows where we must do it after each test. | |
| 993 if self.IsPosix(): | |
| 994 self.GenerateLcovPosix() | |
| 995 # Only on Linux do we have the Xvfb step. | |
| 996 if self.IsLinux() and self.options.xvfb: | |
| 997 self.StopXvfb() | |
| 998 | |
| 999 def StartXvfb(self): | |
| 1000 """Start Xvfb and set an appropriate DISPLAY environment. Linux only. | |
| 1001 | |
| 1002 Copied from http://src.chromium.org/viewvc/chrome/trunk/tools/buildbot/ | |
| 1003 scripts/slave/slave_utils.py?view=markup | |
| 1004 with some simplifications (e.g. no need to use xdisplaycheck, save | |
| 1005 pid in var not file, etc) | |
| 1006 """ | |
| 1007 logging.info('Xvfb: starting') | |
| 1008 proc = subprocess.Popen(["Xvfb", ":9", "-screen", "0", "1024x768x24", | |
| 1009 "-ac"], | |
| 1010 stdout=subprocess.PIPE, stderr=subprocess.STDOUT) | |
| 1011 self.xvfb_pid = proc.pid | |
| 1012 if not self.xvfb_pid: | |
| 1013 logging.info('Could not start Xvfb') | |
| 1014 return | |
| 1015 os.environ['DISPLAY'] = ":9" | |
| 1016 # Now confirm, giving a chance for it to start if needed. | |
| 1017 logging.info('Xvfb: confirming') | |
| 1018 for test in range(10): | |
| 1019 proc = subprocess.Popen('xdpyinfo >/dev/null', shell=True) | |
| 1020 pid, retcode = os.waitpid(proc.pid, 0) | |
| 1021 if retcode == 0: | |
| 1022 break | |
| 1023 time.sleep(0.5) | |
| 1024 if retcode != 0: | |
| 1025 logging.info('Warning: could not confirm Xvfb happiness') | |
| 1026 else: | |
| 1027 logging.info('Xvfb: OK') | |
| 1028 | |
| 1029 def StopXvfb(self): | |
| 1030 """Stop Xvfb if needed. Linux only.""" | |
| 1031 if self.xvfb_pid: | |
| 1032 logging.info('Xvfb: killing') | |
| 1033 try: | |
| 1034 os.kill(self.xvfb_pid, signal.SIGKILL) | |
| 1035 except: | |
| 1036 pass | |
| 1037 del os.environ['DISPLAY'] | |
| 1038 self.xvfb_pid = 0 | |
| 1039 | |
| 1040 def CopyCoverageFileToDestination(self, coverage_folder): | |
| 1041 coverage_dir = os.path.join(self.directory, coverage_folder) | |
| 1042 if not os.path.exists(coverage_dir): | |
| 1043 os.makedirs(coverage_dir) | |
| 1044 shutil.copyfile(self.coverage_info_file, os.path.join(coverage_dir, | |
| 1045 'coverage.info')) | |
| 1046 | |
| 1047 def GenerateLcovPosix(self): | |
| 1048 """Convert profile data to lcov on Mac or Linux.""" | |
| 1049 start_dir = os.getcwd() | |
| 1050 logging.info('GenerateLcovPosix: start_dir=' + start_dir) | |
| 1051 if self.IsLinux(): | |
| 1052 # With Linux/make (e.g. the coverage_run target), the current | |
| 1053 # directory for this command is .../build/src/chrome but we need | |
| 1054 # to be in .../build/src for the relative path of source files | |
| 1055 # to be correct. However, when run from buildbot, the current | |
| 1056 # directory is .../build. Accommodate. | |
| 1057 # On Mac source files are compiled with abs paths so this isn't | |
| 1058 # a problem. | |
| 1059 # This is a bit of a hack. The best answer is to require this | |
| 1060 # script be run in a specific directory for all cases (from | |
| 1061 # Makefile or from buildbot). | |
| 1062 if start_dir.endswith('chrome'): | |
| 1063 logging.info('coverage_posix.py: doing a "cd .." ' | |
| 1064 'to accomodate Linux/make PWD') | |
| 1065 os.chdir('..') | |
| 1066 elif start_dir.endswith('build'): | |
| 1067 logging.info('coverage_posix.py: doing a "cd src" ' | |
| 1068 'to accomodate buildbot PWD') | |
| 1069 os.chdir('src') | |
| 1070 else: | |
| 1071 logging.info('coverage_posix.py: NOT changing directory.') | |
| 1072 elif self.IsMac(): | |
| 1073 pass | |
| 1074 | |
| 1075 command = [self.mcov, | |
| 1076 '--directory', | |
| 1077 os.path.join(start_dir, self.directory_parent), | |
| 1078 '--output', | |
| 1079 os.path.join(start_dir, self.coverage_info_file)] | |
| 1080 logging.info('Assembly command: ' + ' '.join(command)) | |
| 1081 retcode = subprocess.call(command) | |
| 1082 if retcode: | |
| 1083 logging.fatal('COVERAGE: %s failed; return code: %d' % | |
| 1084 (command[0], retcode)) | |
| 1085 if self.options.strict: | |
| 1086 sys.exit(retcode) | |
| 1087 if self.IsLinux(): | |
| 1088 os.chdir(start_dir) | |
| 1089 | |
| 1090 # Copy the unittests coverage information to a different folder. | |
| 1091 if self.options.all_unittests: | |
| 1092 self.CopyCoverageFileToDestination('unittests_coverage') | |
| 1093 elif self.options.all_browsertests: | |
| 1094 # Save browsertests only coverage information. | |
| 1095 self.CopyCoverageFileToDestination('browsertests_coverage') | |
| 1096 else: | |
| 1097 # Save the overall coverage information. | |
| 1098 self.CopyCoverageFileToDestination('total_coverage') | |
| 1099 | |
| 1100 if not os.path.exists(self.coverage_info_file): | |
| 1101 logging.fatal('%s was not created. Coverage run failed.' % | |
| 1102 self.coverage_info_file) | |
| 1103 sys.exit(1) | |
| 1104 | |
| 1105 def GenerateLcovWindows(self, testname=None): | |
| 1106 """Convert VSTS format to lcov. Appends coverage data to sum file.""" | |
| 1107 lcov_file = self.vsts_output + '.lcov' | |
| 1108 if os.path.exists(lcov_file): | |
| 1109 os.remove(lcov_file) | |
| 1110 # generates the file (self.vsts_output + ".lcov") | |
| 1111 | |
| 1112 cmdlist = [self.analyzer, | |
| 1113 '-sym_path=' + self.directory, | |
| 1114 '-src_root=' + self.src_root, | |
| 1115 '-noxml', | |
| 1116 self.vsts_output] | |
| 1117 self.Run(cmdlist) | |
| 1118 if not os.path.exists(lcov_file): | |
| 1119 logging.fatal('Output file %s not created' % lcov_file) | |
| 1120 sys.exit(1) | |
| 1121 logging.info('Appending lcov for test %s to %s' % | |
| 1122 (testname, self.coverage_info_file)) | |
| 1123 size_before = 0 | |
| 1124 if os.path.exists(self.coverage_info_file): | |
| 1125 size_before = os.stat(self.coverage_info_file).st_size | |
| 1126 src = open(lcov_file, 'r') | |
| 1127 dst = open(self.coverage_info_file, 'a') | |
| 1128 dst.write(src.read()) | |
| 1129 src.close() | |
| 1130 dst.close() | |
| 1131 size_after = os.stat(self.coverage_info_file).st_size | |
| 1132 logging.info('Lcov file growth for %s: %d --> %d' % | |
| 1133 (self.coverage_info_file, size_before, size_after)) | |
| 1134 | |
| 1135 def GenerateHtml(self): | |
| 1136 """Convert lcov to html.""" | |
| 1137 # TODO(jrg): This isn't happy when run with unit_tests since V8 has a | |
| 1138 # different "base" so V8 includes can't be found in ".". Fix. | |
| 1139 command = [self.genhtml, | |
| 1140 self.coverage_info_file, | |
| 1141 '--output-directory', | |
| 1142 self.output_directory] | |
| 1143 print >>sys.stderr, 'html generation command: ' + ' '.join(command) | |
| 1144 retcode = subprocess.call(command) | |
| 1145 if retcode: | |
| 1146 logging.fatal('COVERAGE: %s failed; return code: %d' % | |
| 1147 (command[0], retcode)) | |
| 1148 if self.options.strict: | |
| 1149 sys.exit(retcode) | |
| 1150 | |
| 1151 def CoverageOptionParser(): | |
| 1152 """Return an optparse.OptionParser() suitable for Coverage object creation.""" | |
| 1153 parser = optparse.OptionParser() | |
| 1154 parser.add_option('-d', | |
| 1155 '--directory', | |
| 1156 dest='directory', | |
| 1157 default=None, | |
| 1158 help='Directory of unit test files') | |
| 1159 parser.add_option('-a', | |
| 1160 '--all_unittests', | |
| 1161 dest='all_unittests', | |
| 1162 default=False, | |
| 1163 help='Run all tests we can find (*_unittests)') | |
| 1164 parser.add_option('-b', | |
| 1165 '--all_browsertests', | |
| 1166 dest='all_browsertests', | |
| 1167 default=False, | |
| 1168 help='Run all tests in browser_tests ' | |
| 1169 'and content_browsertests') | |
| 1170 parser.add_option('-g', | |
| 1171 '--genhtml', | |
| 1172 dest='genhtml', | |
| 1173 default=False, | |
| 1174 help='Generate html from lcov output') | |
| 1175 parser.add_option('-f', | |
| 1176 '--fast_test', | |
| 1177 dest='fast_test', | |
| 1178 default=False, | |
| 1179 help='Make the tests run REAL fast by doing little.') | |
| 1180 parser.add_option('-s', | |
| 1181 '--strict', | |
| 1182 dest='strict', | |
| 1183 default=False, | |
| 1184 help='Be strict and die on test failure.') | |
| 1185 parser.add_option('-S', | |
| 1186 '--src_root', | |
| 1187 dest='src_root', | |
| 1188 default='.', | |
| 1189 help='Source root (only used on Windows)') | |
| 1190 parser.add_option('-t', | |
| 1191 '--trim', | |
| 1192 dest='trim', | |
| 1193 default=True, | |
| 1194 help='Trim out tests? Default True.') | |
| 1195 parser.add_option('-x', | |
| 1196 '--xvfb', | |
| 1197 dest='xvfb', | |
| 1198 default=True, | |
| 1199 help='Use Xvfb for tests? Default True.') | |
| 1200 parser.add_option('-T', | |
| 1201 '--timeout', | |
| 1202 dest='timeout', | |
| 1203 default=5.0 * 60.0, | |
| 1204 type="int", | |
| 1205 help='Timeout before bailing if a subprocess has no output.' | |
| 1206 ' Default is 5min (Buildbot is 10min.)') | |
| 1207 parser.add_option('-B', | |
| 1208 '--bundles', | |
| 1209 dest='bundles', | |
| 1210 default=None, | |
| 1211 help='Filename of bundles for coverage.') | |
| 1212 parser.add_option('--build-dir', | |
| 1213 dest='build_dir', | |
| 1214 default=None, | |
| 1215 help=('Working directory for buildbot build.' | |
| 1216 'used for finding bundlefile.')) | |
| 1217 parser.add_option('--target', | |
| 1218 dest='target', | |
| 1219 default=None, | |
| 1220 help=('Buildbot build target; ' | |
| 1221 'used for finding bundlefile (e.g. Debug)')) | |
| 1222 parser.add_option('--no_exclusions', | |
| 1223 dest='no_exclusions', | |
| 1224 default=None, | |
| 1225 help=('Disable the exclusion list.')) | |
| 1226 parser.add_option('--dont-clear-coverage-data', | |
| 1227 dest='dont_clear_coverage_data', | |
| 1228 default=False, | |
| 1229 action='store_true', | |
| 1230 help=('Turn off clearing of cov data from a prev run')) | |
| 1231 parser.add_option('-F', | |
| 1232 '--test-file', | |
| 1233 dest="test_files", | |
| 1234 default=[], | |
| 1235 action='append', | |
| 1236 help=('Specify a file from which tests to be run will ' + | |
| 1237 'be extracted')) | |
| 1238 return parser | |
| 1239 | |
| 1240 | |
| 1241 def main(): | |
| 1242 # Print out the args to help someone do it by hand if needed | |
| 1243 print >>sys.stderr, sys.argv | |
| 1244 | |
| 1245 # Try and clean up nice if we're killed by buildbot, Ctrl-C, ... | |
| 1246 signal.signal(signal.SIGINT, TerminateSignalHandler) | |
| 1247 signal.signal(signal.SIGTERM, TerminateSignalHandler) | |
| 1248 | |
| 1249 parser = CoverageOptionParser() | |
| 1250 (options, args) = parser.parse_args() | |
| 1251 if options.all_unittests and options.all_browsertests: | |
| 1252 print 'Error! Can not have all_unittests and all_browsertests together!' | |
| 1253 sys.exit(1) | |
| 1254 coverage = Coverage(options, args) | |
| 1255 coverage.ClearData() | |
| 1256 coverage.FindTests() | |
| 1257 if options.trim: | |
| 1258 coverage.TrimTests() | |
| 1259 coverage.RunTests() | |
| 1260 if options.genhtml: | |
| 1261 coverage.GenerateHtml() | |
| 1262 return 0 | |
| 1263 | |
| 1264 | |
| 1265 if __name__ == '__main__': | |
| 1266 sys.exit(main()) | |
| OLD | NEW |