| OLD | NEW |
|---|
| (Empty) |
| 1 #!/usr/bin/env python | |
| 2 # Copyright (c) 2014 The Chromium Authors. All rights reserved. | |
| 3 # Use of this source code is governed by a BSD-style license that can be | |
| 4 # found in the LICENSE file. | |
| 5 | |
| 6 """Module that downloads SKPs from Google Storage into a specified dir. | |
| 7 | |
| 8 Downloads SKPs which were generated by Cluster Telemetry's GCE slaves using: | |
| 9 http://skia-tree-status.appspot.com/skia-telemetry | |
| 10 """ | |
| 11 | |
| 12 import optparse | |
| 13 import os | |
| 14 import posixpath | |
| 15 import sys | |
| 16 | |
| 17 # This number is unlikely to ever change. | |
| 18 # TODO(rmistry): Read NUMBER_OF_GCE_SLAVES and DEST_GSBASE from a Cluster | |
| 19 # Telemetry global_variables.json | |
| 20 NUMBER_OF_GCE_SLAVES = 100 | |
| 21 # This bucket is also unlikely to ever change (though it would be great if it | |
| 22 # did). | |
| 23 DEST_GSBASE = 'gs://chromium-skia-gm' | |
| 24 | |
| 25 PARENT_DIR = os.path.dirname(os.path.realpath(__file__)) | |
| 26 CHROMIUM_SITE_CONFIG_DIR = os.path.join(PARENT_DIR, os.pardir, 'third_party', | |
| 27 'chromium_buildbot', 'site_config') | |
| 28 | |
| 29 # Set the PYTHONPATH for this script to include chromium_buildbot scripts | |
| 30 # and site_config. Also include skia buildbot's slave utils directory. | |
| 31 sys.path.append( | |
| 32 os.path.join(PARENT_DIR, os.pardir, 'third_party', 'chromium_buildbot', | |
| 33 'scripts')) | |
| 34 sys.path.append(CHROMIUM_SITE_CONFIG_DIR) | |
| 35 sys.path.append( | |
| 36 os.path.join(PARENT_DIR, os.pardir, 'slave', 'skia_slave_scripts', 'utils')) | |
| 37 | |
| 38 import gs_utils | |
| 39 | |
| 40 | |
| 41 def DownloadSKPs(chromium_build, pageset_type, dest_dir): | |
| 42 | |
| 43 # Validate that the .boto file exists. | |
| 44 if not os.path.isfile(os.path.join(CHROMIUM_SITE_CONFIG_DIR, '.boto')): | |
| 45 raise Exception( | |
| 46 'Please copy your .boto file to %s.' % CHROMIUM_SITE_CONFIG_DIR) | |
| 47 | |
| 48 gs_skp_slave_template = posixpath.join( | |
| 49 DEST_GSBASE, 'telemetry', 'skps', 'slave%s', pageset_type, chromium_build) | |
| 50 | |
| 51 # Validate that the requested SKP directory exists for worker1. | |
| 52 if not gs_utils.does_storage_object_exist(gs_skp_slave_template % '1'): | |
| 53 raise Exception( | |
| 54 'There are no SKPs generated for the pageset: %s and the ' | |
| 55 'chromium_build: %s' % (pageset_type, chromium_build)) | |
| 56 | |
| 57 # Create destination directory if it does not exist. | |
| 58 if not os.path.isdir(dest_dir): | |
| 59 os.makedirs(dest_dir) | |
| 60 | |
| 61 # Ensure that the specified destination directory is empty. | |
| 62 if os.listdir(dest_dir): | |
| 63 raise Exception('%s is not empty. Please provide an empty dir.' % dest_dir) | |
| 64 | |
| 65 # Transfer one slave's generated SKPs at a time to the specified directory. | |
| 66 for slave_num in range(1, NUMBER_OF_GCE_SLAVES + 1): | |
| 67 # Create a directory for the slave to download to. | |
| 68 local_slave_dir = os.path.join(dest_dir, chromium_build, | |
| 69 'slave%s' % slave_num) | |
| 70 os.makedirs(local_slave_dir) | |
| 71 # Download from Google Storage to the local slave directory. | |
| 72 gs_utils.copy_storage_directory( | |
| 73 src_dir=posixpath.join(gs_skp_slave_template % slave_num, '*'), | |
| 74 dest_dir=local_slave_dir) | |
| 75 | |
| 76 | |
| 77 if '__main__' == __name__: | |
| 78 option_parser = optparse.OptionParser() | |
| 79 option_parser.add_option( | |
| 80 '-b', '--chromium_build', | |
| 81 help='Specifies which chromium build was used to generate the SKPs. ' | |
| 82 'Eg: e69f2ce-e2b1569. The list of all builds are at: ' | |
| 83 'http://skia-tree-status.appspot.com/skia-telemetry/chromium_builds') | |
| 84 option_parser.add_option( | |
| 85 '-p', '--pageset_type', | |
| 86 help='Specifies the pageset type we want to download. Options are ' | |
| 87 'All/100k/10k/IndexSample10k. Defaults to All.', | |
| 88 default='All') | |
| 89 option_parser.add_option( | |
| 90 '-d', '--dest_dir', | |
| 91 help='Specifies the directory where the SKPs will be downloaded too. %s ' | |
| 92 'slave directories will be created each containing their own ' | |
| 93 'corresponding SKPs.' % NUMBER_OF_GCE_SLAVES) | |
| 94 options, unused_args = option_parser.parse_args() | |
| 95 | |
| 96 # Validate required parameters. | |
| 97 assert options.chromium_build, 'Must specify --chromium_build' | |
| 98 assert options.dest_dir, 'Must specify --dest_dir' | |
| 99 | |
| 100 DownloadSKPs(options.chromium_build, options.pageset_type, options.dest_dir) | |
| 101 | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 648353002:
Remove Skia's forked buildbot code (Closed)
Base URL: https://skia.googlesource.com/buildbot.git@master