| OLD | NEW |
|---|
| (Empty) |
| 1 # Copyright (c) 2013 The Chromium Authors. All rights reserved. | |
| 2 # Use of this source code is governed by a BSD-style license that can be | |
| 3 # found in the LICENSE file. | |
| 4 | |
| 5 | |
| 6 """Utility class to build the Skia master BuildFactory's. | |
| 7 | |
| 8 Based on gclient_factory.py and adds Skia-specific steps.""" | |
| 9 | |
| 10 | |
| 11 from buildbot.process.properties import WithProperties | |
| 12 from buildbot.status import builder | |
| 13 from config_private import SKIA_GIT_URL | |
| 14 from master.factory import gclient_factory | |
| 15 from master.factory.build_factory import BuildFactory | |
| 16 from skia_master_scripts import commands as skia_commands | |
| 17 | |
| 18 import builder_name_schema | |
| 19 import config | |
| 20 import config_private | |
| 21 import ntpath | |
| 22 import os | |
| 23 import posixpath | |
| 24 import skia_vars | |
| 25 import utils | |
| 26 | |
| 27 | |
| 28 # TODO(epoger): My intent is to make the build steps identical on all platforms | |
| 29 # and thus remove the need for the whole target_platform parameter. | |
| 30 # For now, these must match the target_platform values used in | |
| 31 # third_party/chromium_buildbot/scripts/master/factory/gclient_factory.py , | |
| 32 # because we pass these values into GClientFactory.__init__() . | |
| 33 TARGET_PLATFORM_LINUX = 'linux' | |
| 34 TARGET_PLATFORM_MAC = 'mac' | |
| 35 TARGET_PLATFORM_WIN32 = 'win32' | |
| 36 | |
| 37 CONFIG_DEBUG = 'Debug' | |
| 38 CONFIG_RELEASE = 'Release' | |
| 39 CONFIGURATIONS = [CONFIG_DEBUG, CONFIG_RELEASE] | |
| 40 | |
| 41 | |
| 42 _RUNGYP_STEP_DESCRIPTION = 'RunGYP' | |
| 43 _COMPILE_STEP_PREFIX = 'Build' | |
| 44 _COMPILE_RETRY_PREFIX = 'Retry_' + _COMPILE_STEP_PREFIX | |
| 45 _COMPILE_NO_WERR_PREFIX = 'Retry_NoWarningsAsErrors_' + _COMPILE_STEP_PREFIX | |
| 46 | |
| 47 | |
| 48 class SkiaFactory(BuildFactory): | |
| 49 """Encapsulates data and methods common to the Skia master.cfg files.""" | |
| 50 | |
| 51 def __init__(self, other_repos=None, do_upload_render_results=False, | |
| 52 do_upload_bench_results=False, do_patch_step=False, | |
| 53 build_subdir='skia', target_platform=None, | |
| 54 configuration=CONFIG_DEBUG, default_timeout=8*60*60, | |
| 55 deps_target_os=None, environment_variables=None, | |
| 56 perf_output_basedir=None, builder_name=None, flavor=None, | |
| 57 make_flags=None, test_args=None, gm_args=None, bench_args=None, | |
| 58 bench_pictures_cfg='default', compile_warnings_as_errors=False, | |
| 59 gyp_defines=None, build_targets=None): | |
| 60 """Instantiates a SkiaFactory as appropriate for this target_platform. | |
| 61 do_upload_render_results: whether we should upload render results | |
| 62 do_upload_bench_results: whether we should upload bench results | |
| 63 do_patch_step: whether the build should include a step which applies a | |
| 64 patch. This is only applicable for trybots. | |
| 65 build_subdir: subdirectory to check out and then build within | |
| 66 other_repos: list of other repositories to also check out (or None). Each | |
| 67 repo is specified as a tuple: (name, url), where "name" is the target | |
| 68 directory and "url" is the source code url. | |
| 69 target_platform: a string such as TARGET_PLATFORM_LINUX | |
| 70 configuration: 'Debug' or 'Release' | |
| 71 default_timeout: default timeout for each command, in seconds | |
| 72 deps_target_os: string; the target_os to be specified in the gclient config. | |
| 73 environment_variables: dictionary of environment variables that should | |
| 74 be passed to all commands | |
| 75 perf_output_basedir: path to directory under which to store performance | |
| 76 data, or None if we don't want to store performance data | |
| 77 builder_name: name of the builder associated with this factory | |
| 78 flavor: which "flavor" of slave-side scripts this factory should use | |
| 79 make_flags: list of extra flags to pass to the compile step | |
| 80 test_args: list of extra flags to pass to the 'tests' executable | |
| 81 gm_args: list of extra flags to pass to the 'gm' executable | |
| 82 bench_args: list of extra flags to pass to the 'bench' executable | |
| 83 bench_pictures_cfg: config name to use for bench_pictures | |
| 84 compile_warnings_as_errors: boolean; whether to build with "-Werror" or | |
| 85 some equivalent. | |
| 86 gyp_defines: optional dict; GYP_DEFINES to be used in the build. | |
| 87 build_targets: optional list; the targets to build. Default is set depending | |
| 88 on which Build() function is called. | |
| 89 """ | |
| 90 properties = {} | |
| 91 | |
| 92 self.skipsteps = utils.GetListFromEnvVar( | |
| 93 config_private.SKIPSTEPS_ENVIRONMENT_VARIABLE) | |
| 94 self.dontskipsteps = utils.GetListFromEnvVar( | |
| 95 config_private.DONTSKIPSTEPS_ENVIRONMENT_VARIABLE) | |
| 96 | |
| 97 if not make_flags: | |
| 98 make_flags = [] | |
| 99 self._make_flags = make_flags | |
| 100 # Platform-specific stuff. | |
| 101 if target_platform == TARGET_PLATFORM_WIN32: | |
| 102 self.TargetPath = ntpath | |
| 103 else: | |
| 104 self.TargetPath = posixpath | |
| 105 | |
| 106 # Create gclient solutions corresponding to the main build_subdir | |
| 107 # and other directories we also wish to check out. | |
| 108 self._gclient_solutions = [gclient_factory.GClientSolution( | |
| 109 svn_url=SKIA_GIT_URL, name=build_subdir | |
| 110 ).GetSpec()] | |
| 111 | |
| 112 if not other_repos: | |
| 113 other_repos = [] | |
| 114 repos_to_checkout = set(other_repos) | |
| 115 | |
| 116 for other_repo in repos_to_checkout: | |
| 117 self._gclient_solutions.append(gclient_factory.GClientSolution( | |
| 118 svn_url=other_repo[1], name=other_repo[0]).GetSpec()) | |
| 119 | |
| 120 self._deps_target_os = deps_target_os | |
| 121 | |
| 122 # Set _default_clobber based on config.Master | |
| 123 self._default_clobber = getattr(config.Master, 'default_clobber', False) | |
| 124 | |
| 125 self._do_upload_render_results = do_upload_render_results | |
| 126 self._do_upload_bench_results = (do_upload_bench_results and | |
| 127 perf_output_basedir != None) | |
| 128 self._do_patch_step = do_patch_step | |
| 129 | |
| 130 if not environment_variables: | |
| 131 self._env_vars = {} | |
| 132 else: | |
| 133 self._env_vars = dict(environment_variables) | |
| 134 | |
| 135 self._gyp_defines = dict(gyp_defines or {}) | |
| 136 self._gyp_defines['skia_warnings_as_errors'] = \ | |
| 137 '%d' % int(compile_warnings_as_errors) | |
| 138 | |
| 139 self._build_targets = list(build_targets or []) | |
| 140 | |
| 141 # Get an implementation of SkiaCommands as appropriate for | |
| 142 # this target_platform. | |
| 143 self._workdir = self.TargetPath.join('build', build_subdir) | |
| 144 self._skia_cmd_obj = skia_commands.SkiaCommands( | |
| 145 target_platform=target_platform, factory=self, | |
| 146 configuration=configuration, workdir=self._workdir, | |
| 147 target_arch=None, default_timeout=default_timeout, | |
| 148 environment_variables=self._env_vars) | |
| 149 | |
| 150 self._perf_output_basedir = perf_output_basedir | |
| 151 | |
| 152 self._configuration = configuration | |
| 153 if self._configuration not in CONFIGURATIONS: | |
| 154 raise ValueError('Invalid configuration %s. Must be one of: %s' % ( | |
| 155 self._configuration, CONFIGURATIONS)) | |
| 156 | |
| 157 self._skia_svn_username_file = '.skia_svn_username' | |
| 158 self._skia_svn_password_file = '.skia_svn_password' | |
| 159 self._builder_name = builder_name | |
| 160 self._flavor = flavor | |
| 161 | |
| 162 def _DetermineRevision(build): | |
| 163 """ Get the 'revision' property at build time. WithProperties returns the | |
| 164 empty string if 'revision' is not defined, which causes failures when we | |
| 165 try to pass the revision over a command line, so we use the string "None" | |
| 166 to indicate that the revision is not defined. | |
| 167 | |
| 168 build: instance of Build for the current build. | |
| 169 """ | |
| 170 props = build.getProperties().asDict() | |
| 171 if props.has_key('revision'): | |
| 172 if props['revision'][0]: | |
| 173 return props['revision'][0] | |
| 174 return 'None' | |
| 175 | |
| 176 if not test_args: | |
| 177 test_args = [] | |
| 178 if not gm_args: | |
| 179 gm_args = [] | |
| 180 if not bench_args: | |
| 181 bench_args = [] | |
| 182 | |
| 183 self._common_args = [ | |
| 184 # See http://skbug.com/2663 before deleting the next line. | |
| 185 '--autogen_svn_baseurl', 'bogusValueToFixSkBug2663', | |
| 186 '--configuration', configuration, | |
| 187 '--deps_target_os', self._deps_target_os or 'None', | |
| 188 '--builder_name', builder_name, | |
| 189 '--build_number', WithProperties('%(buildnumber)s'), | |
| 190 '--target_platform', target_platform, | |
| 191 '--revision', WithProperties('%(rev)s', rev=_DetermineRevision), | |
| 192 '--got_revision', WithProperties('%(got_revision:-None)s'), | |
| 193 '--perf_output_basedir', perf_output_basedir or 'None', | |
| 194 '--make_flags', '"%s"' % ' '.join(self._make_flags), | |
| 195 '--test_args', '"%s' % ' '.join(test_args), | |
| 196 '--gm_args', '"%s"' % ' '.join(gm_args), | |
| 197 '--bench_args', '"%s"' % ' '.join(bench_args), | |
| 198 '--is_try', str(self._do_patch_step), | |
| 199 '--bench_pictures_cfg', bench_pictures_cfg, | |
| 200 '--issue_number', WithProperties('%(issue:-None)s'), | |
| 201 ] | |
| 202 BuildFactory.__init__(self, build_factory_properties=properties) | |
| 203 | |
| 204 def Validate(self): | |
| 205 """ Validate the Factory against the known good configuration. """ | |
| 206 test_dir = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, | |
| 207 'tools', 'tests', 'factory_configuration') | |
| 208 | |
| 209 # Write the actual configuration. | |
| 210 actual_dir = os.path.join(test_dir, 'actual') | |
| 211 if not os.path.exists(actual_dir): | |
| 212 os.makedirs(actual_dir) | |
| 213 self_as_string = utils.ToString(self.__dict__) | |
| 214 with open(os.path.join(actual_dir, self._builder_name), 'w') as f: | |
| 215 f.write(self_as_string) | |
| 216 | |
| 217 # Read the expected configuration. | |
| 218 expected_dir = os.path.join(test_dir, 'expected') | |
| 219 try: | |
| 220 expectation = open(os.path.join(expected_dir, self._builder_name)).read() | |
| 221 except IOError: | |
| 222 msg = 'No expected factory configuration for %s in %s.' % ( | |
| 223 self._builder_name, expected_dir) | |
| 224 if config_private.die_on_validation_failure: | |
| 225 raise Exception(msg) | |
| 226 else: | |
| 227 print 'Warning: %s' % msg | |
| 228 return | |
| 229 | |
| 230 # Compare actual to expected. | |
| 231 if self_as_string != expectation: | |
| 232 if config_private.die_on_validation_failure: | |
| 233 raise ValueError('Factory configuration for %s does not match ' | |
| 234 'expectation in %s! Here\'s the diff:\n%s\n' % | |
| 235 (self._builder_name, expected_dir, | |
| 236 utils.StringDiff(expectation, self_as_string))) | |
| 237 else: | |
| 238 # We don't print the full diff in this case because: | |
| 239 # a. It's generally too long to be easily read in a terminal | |
| 240 # b. All of the printing can noticeably slow down the master startup | |
| 241 # c. The master prints so much output that it would be easy to miss the | |
| 242 # diff if we did print it. | |
| 243 print 'Warning: Factory configuration for %s does not match ' \ | |
| 244 'expectation!' % self._builder_name | |
| 245 | |
| 246 # TODO(borenet): Can kwargs be used to simplify this function declaration? | |
| 247 def AddSlaveScript(self, script, description, args=None, timeout=None, | |
| 248 halt_on_failure=False, | |
| 249 is_upload_render_step=False, is_upload_bench_step=False, | |
| 250 is_rebaseline_step=False, get_props_from_stdout=None, | |
| 251 workdir=None, do_step_if=None, always_run=False, | |
| 252 flunk_on_failure=True, exception_on_failure=False): | |
| 253 """ Add a BuildStep consisting of a python script. | |
| 254 | |
| 255 script: which slave-side python script to run. | |
| 256 description: string briefly describing the BuildStep; if this description | |
| 257 is in the self.skipsteps list, this BuildStep will be skipped--unless | |
| 258 it's in the self.dontskipsteps list, in which case we run it! | |
| 259 args: optional list of strings; arguments to pass to the script. | |
| 260 timeout: optional integer; maximum time for the BuildStep to complete. | |
| 261 halt_on_failure: boolean indicating whether to continue the build if this | |
| 262 step fails. | |
| 263 is_upload_render_step: boolean; if true, only run if | |
| 264 self._do_upload_render_results is True | |
| 265 is_upload_bench_step: boolean; if true, only run if | |
| 266 self._do_upload_bench_results is True | |
| 267 is_rebaseline_step: boolean indicating whether this step is required for | |
| 268 rebaseline-only builds. | |
| 269 get_props_from_stdout: optional dictionary. Keys are strings indicating | |
| 270 build properties to set based on the output of this step. Values are | |
| 271 strings containing regular expressions for parsing the property from | |
| 272 the output of the step. | |
| 273 workdir: optional string indicating the working directory in which to run | |
| 274 the script. If this is provided, then the script must be given relative | |
| 275 to this directory. | |
| 276 do_step_if: optional, function which determines whether or not to run the | |
| 277 step. The function is not evaluated until runtime. | |
| 278 always_run: boolean indicating whether this step should run even if a | |
| 279 previous step which had halt_on_failure has failed. | |
| 280 flunk_on_failure: boolean indicating whether the whole build fails if this | |
| 281 step fails. | |
| 282 exception_on_failure: boolean indicating whether to raise an exception if | |
| 283 this step fails. This causes the step to go purple instead of red, and | |
| 284 causes the build to stop. Should be used if the build step's failure is | |
| 285 typically transient or results from an infrastructure failure rather | |
| 286 than a code change. | |
| 287 """ | |
| 288 if description not in self.dontskipsteps: | |
| 289 if description in self.skipsteps: | |
| 290 return | |
| 291 if is_upload_render_step and not self._do_upload_render_results: | |
| 292 return | |
| 293 if is_upload_bench_step and not self._do_upload_bench_results: | |
| 294 return | |
| 295 | |
| 296 arguments = list(self._common_args) | |
| 297 if args: | |
| 298 arguments += args | |
| 299 self._skia_cmd_obj.AddSlaveScript( | |
| 300 script=script, | |
| 301 args=arguments, | |
| 302 description=description, | |
| 303 timeout=timeout, | |
| 304 halt_on_failure=halt_on_failure, | |
| 305 is_upload_step=is_upload_render_step or is_upload_bench_step, | |
| 306 is_rebaseline_step=is_rebaseline_step, | |
| 307 get_props_from_stdout=get_props_from_stdout, | |
| 308 workdir=workdir, | |
| 309 do_step_if=do_step_if, | |
| 310 always_run=always_run, | |
| 311 flunk_on_failure=flunk_on_failure, | |
| 312 exception_on_failure=exception_on_failure) | |
| 313 | |
| 314 def AddFlavoredSlaveScript(self, script, args=None, **kwargs): | |
| 315 """ Add a flavor-specific BuildStep. | |
| 316 | |
| 317 Finds a script to run by concatenating the flavor of this BuildFactory with | |
| 318 the provided script name. | |
| 319 """ | |
| 320 flavor_args = ['--flavor', self._flavor or 'default'] | |
| 321 self.AddSlaveScript(script, args=list(args or []) + flavor_args, **kwargs) | |
| 322 | |
| 323 def RunGYP(self, description=_RUNGYP_STEP_DESCRIPTION, do_step_if=None): | |
| 324 """ Run GYP to generate build files. | |
| 325 | |
| 326 description: string; description of this BuildStep. | |
| 327 do_step_if: optional, function which determines whether or not to run this | |
| 328 step. | |
| 329 """ | |
| 330 self.AddFlavoredSlaveScript(script='run_gyp.py', description=description, | |
| 331 halt_on_failure=True, do_step_if=do_step_if, | |
| 332 args=['--gyp_defines', | |
| 333 ' '.join('%s=%s' % (k, v) for k, v in | |
| 334 self._gyp_defines.items())]) | |
| 335 | |
| 336 def Make(self, target, description, is_rebaseline_step=False, do_step_if=None, | |
| 337 always_run=False, flunk_on_failure=True, halt_on_failure=True): | |
| 338 """ Build a single target. | |
| 339 | |
| 340 target: string; the target to build. | |
| 341 description: string; description of this BuildStep. | |
| 342 is_rebaseline_step: optional boolean; whether or not this step is required | |
| 343 for rebaseline-only builds. | |
| 344 do_step_if: optional, function which determines whether or not to run this | |
| 345 step. | |
| 346 always_run: boolean indicating whether this step should run even if a | |
| 347 previous step which had halt_on_failure has failed. | |
| 348 flunk_on_failure: boolean indicating whether the whole build fails if this | |
| 349 step fails. | |
| 350 halt_on_failure: boolean indicating whether to continue the build if this | |
| 351 step fails. | |
| 352 """ | |
| 353 args = ['--target', target, | |
| 354 '--gyp_defines', | |
| 355 ' '.join('%s=%s' % (k, v) for k, v in self._gyp_defines.items())] | |
| 356 self.AddFlavoredSlaveScript(script='compile.py', args=args, | |
| 357 description=description, | |
| 358 halt_on_failure=halt_on_failure, | |
| 359 is_rebaseline_step=is_rebaseline_step, | |
| 360 do_step_if=do_step_if, | |
| 361 always_run=always_run, | |
| 362 flunk_on_failure=flunk_on_failure) | |
| 363 | |
| 364 def Compile(self, clobber=None, retry_with_clobber_on_failure=True, | |
| 365 retry_without_werr_on_failure=False): | |
| 366 """ Compile step. Build everything. | |
| 367 | |
| 368 clobber: optional boolean; whether to 'clean' before building. | |
| 369 retry_with_clobber_on_failure: optional boolean; if the build fails, clean | |
| 370 and try again, with the same configuration as before. | |
| 371 retry_without_werr_on_failure: optional boolean; if the build fails, clean | |
| 372 and try again *without* warnings-as-errors. | |
| 373 """ | |
| 374 if clobber is None: | |
| 375 clobber = self._default_clobber | |
| 376 | |
| 377 | |
| 378 if clobber: | |
| 379 self.AddFlavoredSlaveScript(script='clean.py', description='Clean', | |
| 380 halt_on_failure=True) | |
| 381 | |
| 382 # Only retry with clobber if we've requested it AND we aren't clobbering on | |
| 383 # the first build. | |
| 384 maybe_retry_with_clobber = retry_with_clobber_on_failure and not clobber | |
| 385 | |
| 386 def ShouldRetryWithClobber(step): | |
| 387 """ Determine whether the retry step should run. """ | |
| 388 if not maybe_retry_with_clobber: | |
| 389 return False | |
| 390 gyp_or_compile_failed = False | |
| 391 retry_failed = False | |
| 392 for build_step in step.build.getStatus().getSteps(): | |
| 393 if (build_step.isFinished() and | |
| 394 build_step.getResults()[0] == builder.FAILURE): | |
| 395 if build_step.getName().startswith(_COMPILE_STEP_PREFIX): | |
| 396 gyp_or_compile_failed = True | |
| 397 elif build_step.getName() == _RUNGYP_STEP_DESCRIPTION: | |
| 398 gyp_or_compile_failed = True | |
| 399 elif build_step.getName().startswith(_COMPILE_RETRY_PREFIX): | |
| 400 retry_failed = True | |
| 401 return gyp_or_compile_failed and not retry_failed | |
| 402 | |
| 403 def ShouldRetryWithoutWarnings(step): | |
| 404 """ Determine whether the retry-without-warnings-as-errors step should | |
| 405 run. """ | |
| 406 if not retry_without_werr_on_failure: | |
| 407 return False | |
| 408 gyp_or_compile_failed = False | |
| 409 retry_failed = False | |
| 410 no_warning_retry_failed = False | |
| 411 for build_step in step.build.getStatus().getSteps(): | |
| 412 if (build_step.isFinished() and | |
| 413 build_step.getResults()[0] == builder.FAILURE): | |
| 414 if build_step.getName().startswith(_COMPILE_STEP_PREFIX): | |
| 415 gyp_or_compile_failed = True | |
| 416 elif build_step.getName().startswith(_COMPILE_RETRY_PREFIX): | |
| 417 retry_failed = True | |
| 418 elif build_step.getName().startswith( | |
| 419 _COMPILE_NO_WERR_PREFIX): | |
| 420 no_warning_retry_failed = True | |
| 421 # If we've already failed a previous retry without warnings, just give up. | |
| 422 if no_warning_retry_failed: | |
| 423 return False | |
| 424 # If we're retrying with clobber, only retry without warnings if a clobber | |
| 425 # retry has failed. | |
| 426 if maybe_retry_with_clobber: | |
| 427 return retry_failed | |
| 428 # Only run the retry if the initial compile has failed. | |
| 429 return gyp_or_compile_failed | |
| 430 | |
| 431 for build_target in self._build_targets: | |
| 432 self.Make(target=build_target, | |
| 433 description=_COMPILE_STEP_PREFIX + \ | |
| 434 utils.UnderscoresToCapWords(build_target), | |
| 435 flunk_on_failure=not maybe_retry_with_clobber, | |
| 436 halt_on_failure=(not maybe_retry_with_clobber and | |
| 437 not retry_without_werr_on_failure)) | |
| 438 | |
| 439 # Try again with a clean build. | |
| 440 self.AddFlavoredSlaveScript(script='clean.py', description='Clean', | |
| 441 do_step_if=ShouldRetryWithClobber) | |
| 442 for build_target in self._build_targets: | |
| 443 self.Make(target=build_target, | |
| 444 description=_COMPILE_RETRY_PREFIX + \ | |
| 445 utils.UnderscoresToCapWords(build_target), | |
| 446 flunk_on_failure=True, | |
| 447 halt_on_failure=not retry_without_werr_on_failure, | |
| 448 do_step_if=ShouldRetryWithClobber) | |
| 449 | |
| 450 # Try again without warnings-as-errors. | |
| 451 self._gyp_defines['skia_warnings_as_errors'] = '0' | |
| 452 self.AddFlavoredSlaveScript(script='clean.py', description='Clean', | |
| 453 always_run=True, | |
| 454 do_step_if=ShouldRetryWithoutWarnings) | |
| 455 for build_target in self._build_targets: | |
| 456 self.Make(target=build_target, | |
| 457 description=_COMPILE_NO_WERR_PREFIX + \ | |
| 458 utils.UnderscoresToCapWords(build_target), | |
| 459 flunk_on_failure=True, | |
| 460 halt_on_failure=True, | |
| 461 do_step_if=ShouldRetryWithoutWarnings) | |
| 462 | |
| 463 def Install(self): | |
| 464 """ Install the compiled executables. """ | |
| 465 self.AddFlavoredSlaveScript(script='install.py', description='Install', | |
| 466 halt_on_failure=True, exception_on_failure=True) | |
| 467 | |
| 468 def DownloadSKPs(self): | |
| 469 """ Download the SKPs. """ | |
| 470 self.AddSlaveScript(script='download_skps.py', description='DownloadSKPs', | |
| 471 halt_on_failure=True, exception_on_failure=True) | |
| 472 | |
| 473 def DownloadSKImageFiles(self): | |
| 474 """ Download image files for running skimage. """ | |
| 475 self.AddSlaveScript(script='download_skimage_files.py', | |
| 476 description='DownloadSKImageFiles', | |
| 477 halt_on_failure=True, exception_on_failure=True) | |
| 478 | |
| 479 def RunDecodingTests(self): | |
| 480 """ Run tests of image decoders. """ | |
| 481 self.AddFlavoredSlaveScript(script='run_decoding_tests.py', | |
| 482 description='RunDecodingTests') | |
| 483 | |
| 484 def RunDM(self): | |
| 485 """Run DM.""" | |
| 486 self.AddFlavoredSlaveScript('run_dm.py', description='RunDM') | |
| 487 | |
| 488 def RunGM(self): | |
| 489 """ Run the "GM" tool, saving the images to disk. """ | |
| 490 self.AddFlavoredSlaveScript(script='run_gm.py', description='GenerateGMs', | |
| 491 is_rebaseline_step=True) | |
| 492 | |
| 493 def PreRender(self): | |
| 494 """ Step to run before the render steps. """ | |
| 495 self.AddFlavoredSlaveScript(script='prerender.py', description='PreRender', | |
| 496 exception_on_failure=True) | |
| 497 | |
| 498 def RenderSKPs(self): | |
| 499 """ Generate images from .skp's. """ | |
| 500 self.AddFlavoredSlaveScript(script='render_skps.py', | |
| 501 description='RenderSKPs') | |
| 502 | |
| 503 def RenderPdfs(self): | |
| 504 """ Run the "render_pdfs" tool to generate pdfs from .skp's. """ | |
| 505 self.AddFlavoredSlaveScript(script='render_pdfs.py', | |
| 506 description='RenderPdfs') | |
| 507 | |
| 508 def PostRender(self): | |
| 509 """ Step to run after the render steps. """ | |
| 510 self.AddFlavoredSlaveScript(script='postrender.py', | |
| 511 description='PostRender', | |
| 512 exception_on_failure=True) | |
| 513 | |
| 514 def PreBench(self): | |
| 515 """ Step to run before the benchmarking steps. """ | |
| 516 self.AddFlavoredSlaveScript(script='prebench.py', | |
| 517 description='PreBench', | |
| 518 exception_on_failure=True) | |
| 519 | |
| 520 def PostBench(self): | |
| 521 """ Step to run after the benchmarking steps. """ | |
| 522 self.AddFlavoredSlaveScript(script='postbench.py', | |
| 523 description='PostBench', | |
| 524 exception_on_failure=True) | |
| 525 | |
| 526 def CompareGMs(self): | |
| 527 """Compare the actually-generated GM images to the checked-in baselines.""" | |
| 528 self.AddSlaveScript(script='compare_gms.py', | |
| 529 description='CompareGMs', | |
| 530 get_props_from_stdout={ | |
| 531 'latest_gm_failures_url': | |
| 532 '%s([^\n]*)\n' % skia_vars.GetGlobalVariable( | |
| 533 'latest_gm_failures_preamble')}, | |
| 534 is_rebaseline_step=True) | |
| 535 | |
| 536 def CompareRenderedSKPs(self): | |
| 537 """Compare the actual image results of SKP rendering to expectations.""" | |
| 538 self.AddSlaveScript(script='compare_rendered_skps.py', | |
| 539 description='CompareRenderedSKPs', | |
| 540 is_rebaseline_step=True) | |
| 541 | |
| 542 def RunNanobench(self): | |
| 543 """ Run "nanobench" """ | |
| 544 self.AddFlavoredSlaveScript(script='run_nanobench.py', | |
| 545 description='RunNanobench') | |
| 546 | |
| 547 def UpdateScripts(self): | |
| 548 """ Update the buildbot scripts on the build slave. | |
| 549 | |
| 550 Only runs in production. See http://skbug.com/2432 | |
| 551 """ | |
| 552 description = 'UpdateScripts' | |
| 553 if ((config_private.Master.get_active_master().is_production_host) or | |
| 554 (description in self.dontskipsteps)): | |
| 555 self.AddSlaveScript( | |
| 556 script=self.TargetPath.join('..', '..', '..', '..', | |
| 557 '..', 'slave', | |
| 558 'skia_slave_scripts', | |
| 559 'update_scripts.py'), | |
| 560 description=description, | |
| 561 halt_on_failure=True, | |
| 562 get_props_from_stdout={'buildbot_revision': | |
| 563 'Skiabot scripts updated to (\w+)'}, | |
| 564 workdir='build', | |
| 565 exception_on_failure=True) | |
| 566 | |
| 567 def Update(self): | |
| 568 """ Update the Skia code on the build slave. """ | |
| 569 args = ['--gclient_solutions', '"%s"' % self._gclient_solutions] | |
| 570 self.AddSlaveScript( | |
| 571 script=self.TargetPath.join('..', '..', '..', '..', '..', 'slave', | |
| 572 'skia_slave_scripts', 'update.py'), | |
| 573 description='Update', | |
| 574 args=args, | |
| 575 timeout=None, | |
| 576 halt_on_failure=True, | |
| 577 is_rebaseline_step=True, | |
| 578 get_props_from_stdout={'got_revision':'Skia updated to (\w+)'}, | |
| 579 workdir='build', | |
| 580 exception_on_failure=True) | |
| 581 | |
| 582 def ApplyPatch(self, alternate_workdir=None, alternate_script=None): | |
| 583 """ Apply a patch to the Skia code on the build slave. """ | |
| 584 def _GetPatch(build): | |
| 585 """Find information about the patch (if any) to apply. | |
| 586 | |
| 587 Returns: | |
| 588 An encoded string containing a tuple of the form (level, url) which | |
| 589 indicates where to go to download the patch. | |
| 590 """ | |
| 591 if build.getSourceStamp().patch and \ | |
| 592 'patch_file_url' in build.getProperties(): | |
| 593 # The presence of a patch attached to the Source Stamp and the | |
| 594 # 'patch_file_url' build property indicate that the patch came from the | |
| 595 # skia_try repo, and was probably submitted using the submit_try script | |
| 596 # or "gcl/git-cl try". | |
| 597 patch = (build.getSourceStamp().patch[0], | |
| 598 build.getProperty('patch_file_url')) | |
| 599 return str(patch).encode() | |
| 600 elif 'issue' in build.getProperties() and \ | |
| 601 'patchset' in build.getProperties(): | |
| 602 # The presence of the 'issue' and 'patchset' build properties indicates | |
| 603 # that the patch came from Rietveld. | |
| 604 # Don't assume that the 'issue' and 'patchset' build properties are | |
| 605 # always integer type, even though they should contain integers. | |
| 606 patch = '%s/download/issue%s_%s.diff' % ( | |
| 607 config_private.CODE_REVIEW_SITE.rstrip('/'), | |
| 608 build.getProperty('issue'), | |
| 609 build.getProperty('patchset')) | |
| 610 # If the patch came from Rietveld, assume it came from a git repo and | |
| 611 # therefore it has a patch level of 1. If this isn't the case, the | |
| 612 # slave-side script should detect it and use level 0 instead. | |
| 613 return str((1, patch)).encode() | |
| 614 else: | |
| 615 patch = 'None' | |
| 616 return patch | |
| 617 | |
| 618 if not bool(alternate_workdir) == bool(alternate_script): | |
| 619 raise ValueError('alternate_workdir and alternate_script must be provided' | |
| 620 ' together.') | |
| 621 args = ['--patch', WithProperties('%(patch)s', patch=_GetPatch)] | |
| 622 if alternate_script: | |
| 623 self.AddSlaveScript(script=alternate_script, | |
| 624 description='ApplyPatch', | |
| 625 args=args, | |
| 626 halt_on_failure=True, | |
| 627 workdir=alternate_workdir, | |
| 628 exception_on_failure=True) | |
| 629 else: | |
| 630 self.AddSlaveScript(script='apply_patch.py', | |
| 631 description='ApplyPatch', | |
| 632 args=args, | |
| 633 halt_on_failure=True, | |
| 634 exception_on_failure=True) | |
| 635 | |
| 636 def UpdateSteps(self): | |
| 637 """ Update the Skia sources. """ | |
| 638 self.UpdateScripts() | |
| 639 self.Update() | |
| 640 if self._do_patch_step: | |
| 641 self.ApplyPatch() | |
| 642 | |
| 643 def UploadBenchResults(self): | |
| 644 """ Upload bench results (performance data). """ | |
| 645 self.AddSlaveScript(script='upload_bench_results.py', | |
| 646 description='UploadBenchResults', | |
| 647 exception_on_failure=True, | |
| 648 is_upload_bench_step=True) | |
| 649 | |
| 650 def UploadBenchResultsToAppEngine(self): | |
| 651 """ Upload bench results (performance data) to AppEngine. """ | |
| 652 self.AddSlaveScript(script='upload_bench_results_appengine.py', | |
| 653 description='UploadBenchResultsToAppengine', | |
| 654 exception_on_failure=True) | |
| 655 | |
| 656 def UploadWebpagePictureBenchResults(self): | |
| 657 """ Upload webpage picture bench results (performance data). """ | |
| 658 self.AddSlaveScript(script='upload_webpage_picture_bench_results.py', | |
| 659 description='UploadWebpagePictureBenchResults', | |
| 660 exception_on_failure=True) | |
| 661 | |
| 662 | |
| 663 def UploadGMResults(self): | |
| 664 """ Upload the images generated by GM """ | |
| 665 self.AddSlaveScript(script='upload_gm_results.py', | |
| 666 description='UploadGMResults', timeout=5400, | |
| 667 is_upload_render_step=True, is_rebaseline_step=True, | |
| 668 exception_on_failure=True) | |
| 669 | |
| 670 def UploadRenderedSKPs(self): | |
| 671 """Upload the actual image results of SKP rendering.""" | |
| 672 self.AddSlaveScript(script='upload_rendered_skps.py', | |
| 673 description='UploadRenderedSKPs', | |
| 674 is_upload_render_step=True, is_rebaseline_step=True) | |
| 675 | |
| 676 def UploadSKImageResults(self): | |
| 677 self.AddSlaveScript(script='upload_skimage_results.py', | |
| 678 description='UploadSKImageResults', | |
| 679 is_upload_render_step=True, | |
| 680 exception_on_failure=True) | |
| 681 | |
| 682 def CommonSteps(self, clobber=None): | |
| 683 """ Steps which are run at the beginning of all builds. """ | |
| 684 self.UpdateSteps() | |
| 685 self.DownloadSKPs() | |
| 686 self.Compile(clobber) | |
| 687 self.Install() | |
| 688 | |
| 689 def NonPerfSteps(self): | |
| 690 """ Add correctness testing BuildSteps. """ | |
| 691 self.DownloadSKImageFiles() | |
| 692 self.PreRender() | |
| 693 self.RunDM() | |
| 694 self.RunGM() | |
| 695 self.RenderSKPs() | |
| 696 self.RenderPdfs() | |
| 697 self.RunDecodingTests() | |
| 698 self.PostRender() | |
| 699 self.UploadGMResults() | |
| 700 self.UploadRenderedSKPs() | |
| 701 self.UploadSKImageResults() | |
| 702 self.CompareGMs() | |
| 703 self.CompareRenderedSKPs() | |
| 704 | |
| 705 def PerfSteps(self): | |
| 706 """ Add performance testing BuildSteps. """ | |
| 707 self.PreBench() | |
| 708 self.RunNanobench() | |
| 709 self.PostBench() | |
| 710 self.UploadBenchResults() | |
| 711 | |
| 712 def Build(self, role=None, clobber=None): | |
| 713 """Build and return the complete BuildFactory. | |
| 714 | |
| 715 role: string; the intended role of this builder. The role affects which | |
| 716 steps are run. Known values are given in the utils module. | |
| 717 clobber: boolean indicating whether we should clean before building | |
| 718 """ | |
| 719 # Special case: for the ZeroGPUCache bot, we only run GM. | |
| 720 if 'ZeroGPUCache' in self._builder_name: | |
| 721 self._build_targets = ['gm'] | |
| 722 self.UpdateSteps() | |
| 723 self.Compile(clobber) | |
| 724 self.Install() | |
| 725 self.PreRender() | |
| 726 self.RunGM() | |
| 727 self.PostRender() | |
| 728 elif ('TSAN' in self._builder_name and | |
| 729 role == builder_name_schema.BUILDER_ROLE_TEST): | |
| 730 self._build_targets = ['dm'] | |
| 731 self.UpdateSteps() | |
| 732 self.Compile(clobber) | |
| 733 self.Install() | |
| 734 self.RunDM() | |
| 735 elif ('Valgrind' in self._builder_name and | |
| 736 role == builder_name_schema.BUILDER_ROLE_TEST): | |
| 737 if not self._build_targets: | |
| 738 self._build_targets = ['most'] | |
| 739 self.CommonSteps(clobber) | |
| 740 # TODO(borenet):When https://code.google.com//p/skia/issues/detail?id=1711 | |
| 741 # is fixed, run self.NonPerfSteps() instead of the below steps. | |
| 742 self.DownloadSKImageFiles() | |
| 743 self.PreRender() | |
| 744 self.RunDM() | |
| 745 self.RunGM() | |
| 746 self.RenderSKPs() | |
| 747 self.RenderPdfs() | |
| 748 self.RunDecodingTests() | |
| 749 self.PostRender() | |
| 750 # (end steps which need to be replaced once #1711 is fixed) | |
| 751 self.PreBench() | |
| 752 self.RunNanobench() | |
| 753 self.PostBench() | |
| 754 elif not role: | |
| 755 # If no role is provided, just run everything. | |
| 756 if not self._build_targets: | |
| 757 self._build_targets = ['most'] | |
| 758 self.CommonSteps(clobber) | |
| 759 self.NonPerfSteps() | |
| 760 self.PerfSteps() | |
| 761 elif role == builder_name_schema.BUILDER_ROLE_BUILD: | |
| 762 # Compile-only builder. | |
| 763 self.UpdateSteps() | |
| 764 if not self._build_targets: | |
| 765 self._build_targets = [] | |
| 766 if (('Win' in self._builder_name) or | |
| 767 ('Ubuntu' in self._builder_name and 'x86-' in self._builder_name) or | |
| 768 'Mac10.6' in self._builder_name or 'Mac10.7' in self._builder_name): | |
| 769 # Don't compile the debugger on Windows, Mac 10.6, Mac 10.7, or | |
| 770 # 32-bit Linux since the Qt SDK doesn't include libraries for those | |
| 771 # platforms. | |
| 772 self._build_targets.append('most') | |
| 773 else: | |
| 774 self._build_targets.append('everything') | |
| 775 self.Compile(clobber=clobber, | |
| 776 retry_without_werr_on_failure=True) | |
| 777 else: | |
| 778 if not self._build_targets: | |
| 779 self._build_targets = ['most'] | |
| 780 self.CommonSteps(clobber) | |
| 781 if role == builder_name_schema.BUILDER_ROLE_TEST: | |
| 782 # Test-running builder. | |
| 783 self.NonPerfSteps() | |
| 784 if self._configuration == CONFIG_DEBUG: | |
| 785 # Debug-mode testers run all steps, but release-mode testers don't. | |
| 786 self.PerfSteps() | |
| 787 elif role == builder_name_schema.BUILDER_ROLE_PERF: | |
| 788 # Perf-only builder. | |
| 789 if not self._perf_output_basedir: | |
| 790 raise ValueError( | |
| 791 'BuildPerfOnly requires perf_output_basedir to be defined.') | |
| 792 if self._configuration != CONFIG_RELEASE: | |
| 793 raise ValueError('BuildPerfOnly should run in %s configuration.' % | |
| 794 CONFIG_RELEASE) | |
| 795 self.PerfSteps() | |
| 796 self.Validate() | |
| 797 return self | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 648353002:
Remove Skia's forked buildbot code (Closed)
Base URL: https://skia.googlesource.com/buildbot.git@master