| OLD | NEW |
| (Empty) |
| 1 # Copyright 2014 The Chromium Authors. All rights reserved. | |
| 2 # Use of this source code is governed by a BSD-style license that can be | |
| 3 # found in the LICENSE file. | |
| 4 | |
| 5 """Bootstrap Chrome Telemetry by downloading all its files from SVN servers. | |
| 6 | |
| 7 Requires a DEPS file to specify which directories on which SVN servers | |
| 8 are required to run Telemetry. Format of that DEPS file is a subset of the | |
| 9 normal DEPS file format[1]; currently only only the "deps" dictionary is | |
| 10 supported and nothing else. | |
| 11 | |
| 12 Fetches all files in the specified directories using WebDAV (SVN is WebDAV under | |
| 13 the hood). | |
| 14 | |
| 15 [1] http://dev.chromium.org/developers/how-tos/depottools#TOC-DEPS-file | |
| 16 """ | |
| 17 | |
| 18 import imp | |
| 19 import logging | |
| 20 import os | |
| 21 import urllib | |
| 22 import urlparse | |
| 23 | |
| 24 # Dummy module for DAVclient. | |
| 25 davclient = None | |
| 26 | |
| 27 | |
| 28 # TODO(eakuefner): Switch this link to tools/perf version after verifying. | |
| 29 # Link to file containing the 'davclient' WebDAV client library. | |
| 30 _DAVCLIENT_URL = ('https://src.chromium.org/chrome/trunk/src/tools/' | |
| 31 'telemetry/third_party/davclient/davclient.py') | |
| 32 | |
| 33 | |
| 34 def _DownloadAndImportDAVClientModule(): | |
| 35 """Dynamically import davclient helper library.""" | |
| 36 global davclient | |
| 37 davclient_src = urllib.urlopen(_DAVCLIENT_URL).read() | |
| 38 davclient = imp.new_module('davclient') | |
| 39 exec davclient_src in davclient.__dict__ # pylint: disable=exec-used | |
| 40 | |
| 41 | |
| 42 class DAVClientWrapper(object): | |
| 43 """Knows how to retrieve subdirectories and files from WebDAV/SVN servers.""" | |
| 44 | |
| 45 def __init__(self, root_url): | |
| 46 """Initialize SVN server root_url, save files to local dest_dir. | |
| 47 | |
| 48 Args: | |
| 49 root_url: string url of SVN/WebDAV server | |
| 50 """ | |
| 51 self.root_url = root_url | |
| 52 self.client = davclient.DAVClient(root_url) | |
| 53 | |
| 54 @staticmethod | |
| 55 def __norm_path_keys(dict_with_path_keys): | |
| 56 """Returns a dictionary with os.path.normpath called on every key.""" | |
| 57 return dict((os.path.normpath(k), v) for (k, v) in | |
| 58 dict_with_path_keys.items()) | |
| 59 | |
| 60 def GetDirList(self, path): | |
| 61 """Returns string names of all files and subdirs of path on the server.""" | |
| 62 props = self.__norm_path_keys(self.client.propfind(path, depth=1)) | |
| 63 # remove this path | |
| 64 del props[os.path.normpath(path)] | |
| 65 return [os.path.basename(p) for p in props.keys()] | |
| 66 | |
| 67 def IsFile(self, path): | |
| 68 """Returns True if the path is a file on the server, False if directory.""" | |
| 69 props = self.__norm_path_keys(self.client.propfind(path, depth=1)) | |
| 70 return props[os.path.normpath(path)]['resourcetype'] is None | |
| 71 | |
| 72 def Traverse(self, src_path, dst_path): | |
| 73 """Walks the directory hierarchy pointed to by src_path download all files. | |
| 74 | |
| 75 Recursively walks src_path and saves all files and subfolders into | |
| 76 dst_path. | |
| 77 | |
| 78 Args: | |
| 79 src_path: string path on SVN server to save (absolute path on server). | |
| 80 dest_path: string local path (relative or absolute) to save to. | |
| 81 """ | |
| 82 if self.IsFile(src_path): | |
| 83 if not os.path.exists(os.path.dirname(dst_path)): | |
| 84 logging.info('Creating %s', os.path.dirname(dst_path)) | |
| 85 os.makedirs(os.path.dirname(dst_path)) | |
| 86 if os.path.isfile(dst_path): | |
| 87 logging.info('Skipping %s', dst_path) | |
| 88 else: | |
| 89 logging.info('Saving %s to %s', self.root_url + src_path, dst_path) | |
| 90 urllib.urlretrieve(self.root_url + src_path, dst_path) | |
| 91 return | |
| 92 else: | |
| 93 for subdir in self.GetDirList(src_path): | |
| 94 self.Traverse(os.path.join(src_path, subdir), | |
| 95 os.path.join(dst_path, subdir)) | |
| 96 | |
| 97 | |
| 98 def ListAllDepsPaths(deps_file): | |
| 99 """Recursively returns a list of all paths indicated in this deps file. | |
| 100 | |
| 101 Note that this discards information about where path dependencies come from, | |
| 102 so this is only useful in the context of a Chromium source checkout that has | |
| 103 already fetched all dependencies. | |
| 104 | |
| 105 Args: | |
| 106 deps_file: File containing deps information to be evaluated, in the | |
| 107 format given in the header of this file. | |
| 108 Returns: | |
| 109 A list of string paths starting under src that are required by the | |
| 110 given deps file, and all of its sub-dependencies. This amounts to | |
| 111 the keys of the 'deps' dictionary. | |
| 112 """ | |
| 113 deps = {} | |
| 114 deps_includes = {} | |
| 115 | |
| 116 chrome_root = os.path.dirname(__file__) | |
| 117 while os.path.basename(chrome_root) != 'src': | |
| 118 chrome_root = os.path.abspath(os.path.join(chrome_root, '..')) | |
| 119 | |
| 120 exec open(deps_file).read() # pylint: disable=exec-used | |
| 121 | |
| 122 deps_paths = deps.keys() | |
| 123 | |
| 124 for path in deps_includes.keys(): | |
| 125 # Need to localize the paths. | |
| 126 path = os.path.join(chrome_root, '..', path) | |
| 127 deps_paths += ListAllDepsPaths(path) | |
| 128 | |
| 129 return deps_paths | |
| 130 | |
| 131 | |
| 132 def DownloadDeps(destination_dir, url): | |
| 133 """Saves all the dependencies in deps_path. | |
| 134 | |
| 135 Opens and reads url, assuming the contents are in the simple DEPS-like file | |
| 136 format specified in the header of this file, then download all | |
| 137 files/directories listed to the destination_dir. | |
| 138 | |
| 139 Args: | |
| 140 destination_dir: String path to directory to download files into. | |
| 141 url: URL containing deps information to be evaluated. | |
| 142 """ | |
| 143 logging.warning('Downloading deps from %s...', url) | |
| 144 # TODO(wiltzius): Add a parameter for which revision to pull. | |
| 145 _DownloadAndImportDAVClientModule() | |
| 146 | |
| 147 deps = {} | |
| 148 deps_includes = {} | |
| 149 | |
| 150 exec urllib.urlopen(url).read() # pylint: disable=exec-used | |
| 151 | |
| 152 for dst_path, src_path in deps.iteritems(): | |
| 153 full_dst_path = os.path.join(destination_dir, dst_path) | |
| 154 parsed_url = urlparse.urlparse(src_path) | |
| 155 root_url = parsed_url.scheme + '://' + parsed_url.netloc | |
| 156 | |
| 157 dav_client = DAVClientWrapper(root_url) | |
| 158 dav_client.Traverse(parsed_url.path, full_dst_path) | |
| 159 | |
| 160 for url in deps_includes.values(): | |
| 161 DownloadDeps(destination_dir, url) | |
| OLD | NEW |