| OLD | NEW |
|---|
| (Empty) |
| 1 # | |
| 2 # Copyright 2015 Google Inc. | |
| 3 # | |
| 4 # Use of this source code is governed by a BSD-style license that can be | |
| 5 # found in the LICENSE file. | |
| 6 # | |
| 7 | |
| 8 #!/usr/bin/env python | |
| 9 | |
| 10 usage = ''' | |
| 11 Write buildbot spec to outfile based on the bot name: | |
| 12 $ python buildbot_spec.py outfile Test-Ubuntu-GCC-GCE-CPU-AVX2-x86-Debug | |
| 13 Or run self-tests: | |
| 14 $ python buildbot_spec.py test | |
| 15 ''' | |
| 16 | |
| 17 import inspect | |
| 18 import json | |
| 19 import os | |
| 20 import sys | |
| 21 | |
| 22 import builder_name_schema | |
| 23 import dm_flags | |
| 24 import nanobench_flags | |
| 25 | |
| 26 | |
| 27 CONFIG_COVERAGE = 'Coverage' | |
| 28 CONFIG_DEBUG = 'Debug' | |
| 29 CONFIG_RELEASE = 'Release' | |
| 30 | |
| 31 | |
| 32 def lineno(): | |
| 33 caller = inspect.stack()[1] # Up one level to our caller. | |
| 34 return inspect.getframeinfo(caller[0]).lineno | |
| 35 | |
| 36 # Since we don't actually start coverage until we're in the self-test, | |
| 37 # some function def lines aren't reported as covered. Add them to this | |
| 38 # list so that we can ignore them. | |
| 39 cov_skip = [] | |
| 40 | |
| 41 cov_start = lineno()+1 # We care about coverage starting just past this def. | |
| 42 def gyp_defines(builder_dict): | |
| 43 gyp_defs = {} | |
| 44 | |
| 45 # skia_arch_type. | |
| 46 if builder_dict['role'] == builder_name_schema.BUILDER_ROLE_BUILD: | |
| 47 arch = builder_dict['target_arch'] | |
| 48 elif builder_dict['role'] in (builder_name_schema.BUILDER_ROLE_HOUSEKEEPER, | |
| 49 builder_name_schema.BUILDER_ROLE_INFRA): | |
| 50 arch = None | |
| 51 else: | |
| 52 arch = builder_dict['arch'] | |
| 53 | |
| 54 arch_types = { | |
| 55 'x86': 'x86', | |
| 56 'x86_64': 'x86_64', | |
| 57 'Arm7': 'arm', | |
| 58 'Arm64': 'arm64', | |
| 59 'Mips': 'mips32', | |
| 60 'Mips64': 'mips64', | |
| 61 'MipsDSP2': 'mips32', | |
| 62 } | |
| 63 if arch in arch_types: | |
| 64 gyp_defs['skia_arch_type'] = arch_types[arch] | |
| 65 | |
| 66 # housekeeper: build shared lib. | |
| 67 if builder_dict['role'] == builder_name_schema.BUILDER_ROLE_HOUSEKEEPER: | |
| 68 gyp_defs['skia_shared_lib'] = '1' | |
| 69 | |
| 70 # skia_gpu. | |
| 71 if builder_dict.get('cpu_or_gpu') == 'CPU': | |
| 72 gyp_defs['skia_gpu'] = '0' | |
| 73 | |
| 74 # skia_warnings_as_errors. | |
| 75 werr = False | |
| 76 if builder_dict['role'] == builder_name_schema.BUILDER_ROLE_BUILD: | |
| 77 if 'Win' in builder_dict.get('os', ''): | |
| 78 if not ('GDI' in builder_dict.get('extra_config', '') or | |
| 79 'Exceptions' in builder_dict.get('extra_config', '')): | |
| 80 werr = True | |
| 81 elif ('Mac' in builder_dict.get('os', '') and | |
| 82 'Android' in builder_dict.get('extra_config', '')): | |
| 83 werr = False | |
| 84 elif 'Fast' in builder_dict.get('extra_config', ''): | |
| 85 # See https://bugs.chromium.org/p/skia/issues/detail?id=5257 | |
| 86 werr = False | |
| 87 else: | |
| 88 werr = True | |
| 89 gyp_defs['skia_warnings_as_errors'] = str(int(werr)) # True/False -> '1'/'0' | |
| 90 | |
| 91 # Win debugger. | |
| 92 if 'Win' in builder_dict.get('os', ''): | |
| 93 gyp_defs['skia_win_debuggers_path'] = 'c:/DbgHelp' | |
| 94 | |
| 95 # Qt SDK (Win). | |
| 96 if 'Win' in builder_dict.get('os', ''): | |
| 97 if builder_dict.get('os') == 'Win8': | |
| 98 gyp_defs['qt_sdk'] = 'C:/Qt/Qt5.1.0/5.1.0/msvc2012_64/' | |
| 99 else: | |
| 100 gyp_defs['qt_sdk'] = 'C:/Qt/4.8.5/' | |
| 101 | |
| 102 # ANGLE. | |
| 103 if builder_dict.get('extra_config') == 'ANGLE': | |
| 104 gyp_defs['skia_angle'] = '1' | |
| 105 if builder_dict.get('os', '') in ('Ubuntu', 'Linux'): | |
| 106 gyp_defs['use_x11'] = '1' | |
| 107 gyp_defs['chromeos'] = '0' | |
| 108 | |
| 109 # GDI. | |
| 110 if builder_dict.get('extra_config') == 'GDI': | |
| 111 gyp_defs['skia_gdi'] = '1' | |
| 112 | |
| 113 # Build with Exceptions on Windows. | |
| 114 if ('Win' in builder_dict.get('os', '') and | |
| 115 builder_dict.get('extra_config') == 'Exceptions'): | |
| 116 gyp_defs['skia_win_exceptions'] = '1' | |
| 117 | |
| 118 # iOS. | |
| 119 if (builder_dict.get('os') == 'iOS' or | |
| 120 builder_dict.get('extra_config') == 'iOS'): | |
| 121 gyp_defs['skia_os'] = 'ios' | |
| 122 | |
| 123 # Shared library build. | |
| 124 if builder_dict.get('extra_config') == 'Shared': | |
| 125 gyp_defs['skia_shared_lib'] = '1' | |
| 126 | |
| 127 # Build fastest Skia possible. | |
| 128 if builder_dict.get('extra_config') == 'Fast': | |
| 129 gyp_defs['skia_fast'] = '1' | |
| 130 | |
| 131 # PDF viewer in GM. | |
| 132 if (builder_dict.get('os') == 'Mac10.8' and | |
| 133 builder_dict.get('arch') == 'x86_64' and | |
| 134 builder_dict.get('configuration') == 'Release'): | |
| 135 gyp_defs['skia_run_pdfviewer_in_gm'] = '1' | |
| 136 | |
| 137 # Clang. | |
| 138 if builder_dict.get('compiler') == 'Clang': | |
| 139 gyp_defs['skia_clang_build'] = '1' | |
| 140 | |
| 141 # Valgrind. | |
| 142 if 'Valgrind' in builder_dict.get('extra_config', ''): | |
| 143 gyp_defs['skia_release_optimization_level'] = '1' | |
| 144 | |
| 145 # Link-time code generation just wastes time on compile-only bots. | |
| 146 if (builder_dict.get('role') == builder_name_schema.BUILDER_ROLE_BUILD and | |
| 147 builder_dict.get('compiler') == 'MSVC'): | |
| 148 gyp_defs['skia_win_ltcg'] = '0' | |
| 149 | |
| 150 # Mesa. | |
| 151 if (builder_dict.get('extra_config') == 'Mesa' or | |
| 152 builder_dict.get('cpu_or_gpu_value') == 'Mesa'): | |
| 153 gyp_defs['skia_mesa'] = '1' | |
| 154 | |
| 155 # VisualBench | |
| 156 if builder_dict.get('extra_config') == 'VisualBench': | |
| 157 gyp_defs['skia_use_sdl'] = '1' | |
| 158 | |
| 159 # skia_use_android_framework_defines. | |
| 160 if builder_dict.get('extra_config') == 'Android_FrameworkDefs': | |
| 161 gyp_defs['skia_use_android_framework_defines'] = '1' | |
| 162 | |
| 163 # Skia dump stats for perf tests and gpu | |
| 164 if (builder_dict.get('cpu_or_gpu') == 'GPU' and | |
| 165 builder_dict.get('role') == 'Perf'): | |
| 166 gyp_defs['skia_dump_stats'] = '1' | |
| 167 | |
| 168 # CommandBuffer. | |
| 169 if builder_dict.get('extra_config') == 'CommandBuffer': | |
| 170 gyp_defs['skia_command_buffer'] = '1' | |
| 171 | |
| 172 # Vulkan. | |
| 173 if builder_dict.get('extra_config') == 'Vulkan': | |
| 174 gyp_defs['skia_vulkan'] = '1' | |
| 175 gyp_defs['skia_vulkan_debug_layers'] = '0' | |
| 176 | |
| 177 return gyp_defs | |
| 178 | |
| 179 | |
| 180 cov_skip.extend([lineno(), lineno() + 1]) | |
| 181 def get_extra_env_vars(builder_dict): | |
| 182 env = {} | |
| 183 if builder_dict.get('configuration') == CONFIG_COVERAGE: | |
| 184 # We have to use Clang 3.6 because earlier versions do not support the | |
| 185 # compile flags we use and 3.7 and 3.8 hit asserts during compilation. | |
| 186 env['CC'] = '/usr/bin/clang-3.6' | |
| 187 env['CXX'] = '/usr/bin/clang++-3.6' | |
| 188 elif builder_dict.get('compiler') == 'Clang': | |
| 189 env['CC'] = '/usr/bin/clang' | |
| 190 env['CXX'] = '/usr/bin/clang++' | |
| 191 | |
| 192 # SKNX_NO_SIMD, SK_USE_DISCARDABLE_SCALEDIMAGECACHE, etc. | |
| 193 extra_config = builder_dict.get('extra_config', '') | |
| 194 if extra_config.startswith('SK') and extra_config.isupper(): | |
| 195 env['CPPFLAGS'] = '-D' + extra_config | |
| 196 | |
| 197 return env | |
| 198 | |
| 199 | |
| 200 cov_skip.extend([lineno(), lineno() + 1]) | |
| 201 def build_targets_from_builder_dict(builder_dict, do_test_steps, do_perf_steps): | |
| 202 """Return a list of targets to build, depending on the builder type.""" | |
| 203 if builder_dict['role'] in ('Test', 'Perf') and builder_dict['os'] == 'iOS': | |
| 204 return ['iOSShell'] | |
| 205 if builder_dict.get('extra_config') == 'Appurify': | |
| 206 return ['VisualBenchTest_APK'] | |
| 207 if 'SAN' in builder_dict.get('extra_config', ''): | |
| 208 # 'most' does not compile under MSAN. | |
| 209 return ['dm', 'nanobench'] | |
| 210 t = [] | |
| 211 if do_test_steps: | |
| 212 t.append('dm') | |
| 213 if builder_dict.get('extra_config') == 'VisualBench': | |
| 214 t.append('visualbench') | |
| 215 elif do_perf_steps: | |
| 216 t.append('nanobench') | |
| 217 if t: | |
| 218 return t | |
| 219 else: | |
| 220 return ['most'] | |
| 221 | |
| 222 | |
| 223 cov_skip.extend([lineno(), lineno() + 1]) | |
| 224 def device_cfg(builder_dict): | |
| 225 # Android. | |
| 226 if 'Android' in builder_dict.get('extra_config', ''): | |
| 227 if 'NoNeon' in builder_dict['extra_config']: | |
| 228 return 'arm_v7' | |
| 229 return { | |
| 230 'Arm64': 'arm64', | |
| 231 'x86': 'x86', | |
| 232 'x86_64': 'x86_64', | |
| 233 'Mips': 'mips', | |
| 234 'Mips64': 'mips64', | |
| 235 'MipsDSP2': 'mips_dsp2', | |
| 236 }.get(builder_dict['target_arch'], 'arm_v7_neon') | |
| 237 elif builder_dict.get('os') == 'Android': | |
| 238 return { | |
| 239 'AndroidOne': 'arm_v7_neon', | |
| 240 'GalaxyS3': 'arm_v7_neon', | |
| 241 'GalaxyS4': 'arm_v7_neon', | |
| 242 'NVIDIA_Shield': 'arm64', | |
| 243 'Nexus10': 'arm_v7_neon', | |
| 244 'Nexus5': 'arm_v7_neon', | |
| 245 'Nexus6': 'arm_v7_neon', | |
| 246 'Nexus7': 'arm_v7_neon', | |
| 247 'Nexus7v2': 'arm_v7_neon', | |
| 248 'Nexus9': 'arm64', | |
| 249 'NexusPlayer': 'x86', | |
| 250 }[builder_dict['model']] | |
| 251 | |
| 252 # iOS. | |
| 253 if 'iOS' in builder_dict.get('os', ''): | |
| 254 return { | |
| 255 'iPad4': 'iPad4,1', | |
| 256 }[builder_dict['model']] | |
| 257 | |
| 258 return None | |
| 259 | |
| 260 | |
| 261 cov_skip.extend([lineno(), lineno() + 1]) | |
| 262 def product_board(builder_dict): | |
| 263 if 'Android' in builder_dict.get('os', ''): | |
| 264 return { | |
| 265 'AndroidOne': 'sprout', | |
| 266 'GalaxyS3': 'm0', #'smdk4x12', Detected incorrectly by swarming? | |
| 267 'GalaxyS4': None, # TODO(borenet,kjlubick) | |
| 268 'NVIDIA_Shield': 'foster', | |
| 269 'Nexus10': 'manta', | |
| 270 'Nexus5': 'hammerhead', | |
| 271 'Nexus6': 'shamu', | |
| 272 'Nexus7': 'grouper', | |
| 273 'Nexus7v2': 'flo', | |
| 274 'Nexus9': 'flounder', | |
| 275 'NexusPlayer': 'fugu', | |
| 276 }[builder_dict['model']] | |
| 277 return None | |
| 278 | |
| 279 | |
| 280 cov_skip.extend([lineno(), lineno() + 1]) | |
| 281 def get_builder_spec(builder_name): | |
| 282 builder_dict = builder_name_schema.DictForBuilderName(builder_name) | |
| 283 env = get_extra_env_vars(builder_dict) | |
| 284 gyp_defs = gyp_defines(builder_dict) | |
| 285 gyp_defs_list = ['%s=%s' % (k, v) for k, v in gyp_defs.iteritems()] | |
| 286 gyp_defs_list.sort() | |
| 287 env['GYP_DEFINES'] = ' '.join(gyp_defs_list) | |
| 288 rv = { | |
| 289 'builder_cfg': builder_dict, | |
| 290 'dm_flags': dm_flags.get_args(builder_name), | |
| 291 'env': env, | |
| 292 'nanobench_flags': nanobench_flags.get_args(builder_name), | |
| 293 } | |
| 294 device = device_cfg(builder_dict) | |
| 295 if device: | |
| 296 rv['device_cfg'] = device | |
| 297 board = product_board(builder_dict) | |
| 298 if board: | |
| 299 rv['product.board'] = board | |
| 300 | |
| 301 role = builder_dict['role'] | |
| 302 if role == builder_name_schema.BUILDER_ROLE_HOUSEKEEPER: | |
| 303 configuration = CONFIG_RELEASE | |
| 304 else: | |
| 305 configuration = builder_dict.get( | |
| 306 'configuration', CONFIG_DEBUG) | |
| 307 arch = (builder_dict.get('arch') or builder_dict.get('target_arch')) | |
| 308 if ('Win' in builder_dict.get('os', '') and arch == 'x86_64'): | |
| 309 configuration += '_x64' | |
| 310 rv['configuration'] = configuration | |
| 311 if configuration == CONFIG_COVERAGE: | |
| 312 rv['do_compile_steps'] = False | |
| 313 rv['do_test_steps'] = role == builder_name_schema.BUILDER_ROLE_TEST | |
| 314 rv['do_perf_steps'] = role == builder_name_schema.BUILDER_ROLE_PERF | |
| 315 | |
| 316 rv['build_targets'] = build_targets_from_builder_dict( | |
| 317 builder_dict, rv['do_test_steps'], rv['do_perf_steps']) | |
| 318 | |
| 319 # Do we upload perf results? | |
| 320 upload_perf_results = False | |
| 321 if (role == builder_name_schema.BUILDER_ROLE_PERF and | |
| 322 CONFIG_RELEASE in configuration): | |
| 323 upload_perf_results = True | |
| 324 rv['upload_perf_results'] = upload_perf_results | |
| 325 | |
| 326 # Do we upload correctness results? | |
| 327 skip_upload_bots = [ | |
| 328 'ASAN', | |
| 329 'Coverage', | |
| 330 'MSAN', | |
| 331 'TSAN', | |
| 332 'UBSAN', | |
| 333 'Valgrind', | |
| 334 ] | |
| 335 upload_dm_results = True | |
| 336 for s in skip_upload_bots: | |
| 337 if s in builder_name: | |
| 338 upload_dm_results = False | |
| 339 break | |
| 340 rv['upload_dm_results'] = upload_dm_results | |
| 341 | |
| 342 return rv | |
| 343 | |
| 344 | |
| 345 cov_end = lineno() # Don't care about code coverage past here. | |
| 346 | |
| 347 | |
| 348 def self_test(): | |
| 349 import coverage # This way the bots don't need coverage.py to be installed. | |
| 350 args = {} | |
| 351 cases = [ | |
| 352 'Build-Mac10.8-Clang-Arm7-Debug-Android', | |
| 353 'Build-Win-MSVC-x86-Debug', | |
| 354 'Build-Win-MSVC-x86-Debug-GDI', | |
| 355 'Build-Win-MSVC-x86-Debug-Exceptions', | |
| 356 'Build-Ubuntu-GCC-Arm7-Debug-Android_FrameworkDefs', | |
| 357 'Build-Ubuntu-GCC-Arm7-Debug-Android_NoNeon', | |
| 358 'Build-Ubuntu-GCC-x86_64-Release-ANGLE', | |
| 359 'Build-Ubuntu-GCC-x64_64-Release-Fast', | |
| 360 'Build-Ubuntu-GCC-x86_64-Release-Mesa', | |
| 361 'Housekeeper-PerCommit', | |
| 362 'Perf-Win8-MSVC-ShuttleB-GPU-HD4600-x86_64-Release-Trybot', | |
| 363 'Perf-Ubuntu-GCC-ShuttleA-GPU-GTX660-x86_64-Release-VisualBench', | |
| 364 'Test-Android-GCC-GalaxyS4-GPU-SGX544-Arm7-Debug', | |
| 365 'Perf-Android-GCC-Nexus5-GPU-Adreno330-Arm7-Release-Appurify', | |
| 366 'Test-Android-GCC-Nexus6-GPU-Adreno420-Arm7-Debug', | |
| 367 'Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Debug', | |
| 368 'Test-Mac-Clang-MacMini6.2-GPU-HD4000-x86_64-Debug-CommandBuffer', | |
| 369 'Test-Mac10.8-Clang-MacMini4.1-GPU-GeForce320M-x86_64-Release', | |
| 370 'Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Coverage', | |
| 371 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-MSAN', | |
| 372 ('Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-' | |
| 373 'SK_USE_DISCARDABLE_SCALEDIMAGECACHE'), | |
| 374 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-SKNX_NO_SIMD', | |
| 375 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Fast', | |
| 376 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared', | |
| 377 'Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind', | |
| 378 'Test-Win10-MSVC-ShuttleA-GPU-GTX660-x86_64-Debug-Vulkan', | |
| 379 'Test-Win8-MSVC-ShuttleB-GPU-HD4600-x86-Release-ANGLE', | |
| 380 'Test-Win8-MSVC-ShuttleA-CPU-AVX-x86_64-Debug', | |
| 381 ] | |
| 382 | |
| 383 cov = coverage.coverage() | |
| 384 cov.start() | |
| 385 for case in cases: | |
| 386 args[case] = get_builder_spec(case) | |
| 387 cov.stop() | |
| 388 | |
| 389 this_file = os.path.basename(__file__) | |
| 390 _, _, not_run, _ = cov.analysis(this_file) | |
| 391 filtered = [line for line in not_run if | |
| 392 line > cov_start and line < cov_end and line not in cov_skip] | |
| 393 if filtered: | |
| 394 print 'Lines not covered by test cases: ', filtered | |
| 395 sys.exit(1) | |
| 396 | |
| 397 golden = this_file.replace('.py', '.json') | |
| 398 with open(os.path.join(os.path.dirname(__file__), golden), 'w') as f: | |
| 399 json.dump(args, f, indent=2, sort_keys=True) | |
| 400 | |
| 401 | |
| 402 if __name__ == '__main__': | |
| 403 if len(sys.argv) == 2 and sys.argv[1] == 'test': | |
| 404 self_test() | |
| 405 sys.exit(0) | |
| 406 | |
| 407 if len(sys.argv) != 3: | |
| 408 print usage | |
| 409 sys.exit(1) | |
| 410 | |
| 411 with open(sys.argv[1], 'w') as out: | |
| 412 json.dump(get_builder_spec(sys.argv[2]), out) | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2215803002:
Move builder_spec, [dm|nanobench]_flags, builder_name_schema to recipes (Closed)
Base URL: https://skia.googlesource.com/skia.git@master