| OLD | NEW | 
 | (Empty) | 
|    1 # Copyright 2014 The Chromium Authors. All rights reserved. |  | 
|    2 # Use of this source code is governed by a BSD-style license that can be |  | 
|    3 # found in the LICENSE file. |  | 
|    4  |  | 
|    5 """Bootstrap Chrome Telemetry by downloading all its files from SVN servers. |  | 
|    6  |  | 
|    7 Requires a DEPS file to specify which directories on which SVN servers |  | 
|    8 are required to run Telemetry. Format of that DEPS file is a subset of the |  | 
|    9 normal DEPS file format[1]; currently only only the "deps" dictionary is |  | 
|   10 supported and nothing else. |  | 
|   11  |  | 
|   12 Fetches all files in the specified directories using WebDAV (SVN is WebDAV under |  | 
|   13 the hood). |  | 
|   14  |  | 
|   15 [1] http://dev.chromium.org/developers/how-tos/depottools#TOC-DEPS-file |  | 
|   16 """ |  | 
|   17  |  | 
|   18 import imp |  | 
|   19 import logging |  | 
|   20 import os |  | 
|   21 import urllib |  | 
|   22 import urlparse |  | 
|   23  |  | 
|   24 # Dummy module for DAVclient. |  | 
|   25 davclient = None |  | 
|   26  |  | 
|   27  |  | 
|   28 # TODO(eakuefner): Switch this link to tools/perf version after verifying. |  | 
|   29 # Link to file containing the 'davclient' WebDAV client library. |  | 
|   30 _DAVCLIENT_URL = ('https://src.chromium.org/chrome/trunk/src/tools/' |  | 
|   31                   'telemetry/third_party/davclient/davclient.py') |  | 
|   32  |  | 
|   33  |  | 
|   34 def _DownloadAndImportDAVClientModule(): |  | 
|   35   """Dynamically import davclient helper library.""" |  | 
|   36   global davclient |  | 
|   37   davclient_src = urllib.urlopen(_DAVCLIENT_URL).read() |  | 
|   38   davclient = imp.new_module('davclient') |  | 
|   39   exec davclient_src in davclient.__dict__  # pylint: disable=exec-used |  | 
|   40  |  | 
|   41  |  | 
|   42 class DAVClientWrapper(object): |  | 
|   43   """Knows how to retrieve subdirectories and files from WebDAV/SVN servers.""" |  | 
|   44  |  | 
|   45   def __init__(self, root_url): |  | 
|   46     """Initialize SVN server root_url, save files to local dest_dir. |  | 
|   47  |  | 
|   48     Args: |  | 
|   49       root_url: string url of SVN/WebDAV server |  | 
|   50     """ |  | 
|   51     self.root_url = root_url |  | 
|   52     self.client = davclient.DAVClient(root_url) |  | 
|   53  |  | 
|   54   @staticmethod |  | 
|   55   def __norm_path_keys(dict_with_path_keys): |  | 
|   56     """Returns a dictionary with os.path.normpath called on every key.""" |  | 
|   57     return dict((os.path.normpath(k), v) for (k, v) in |  | 
|   58                 dict_with_path_keys.items()) |  | 
|   59  |  | 
|   60   def GetDirList(self, path): |  | 
|   61     """Returns string names of all files and subdirs of path on the server.""" |  | 
|   62     props = self.__norm_path_keys(self.client.propfind(path, depth=1)) |  | 
|   63     # remove this path |  | 
|   64     del props[os.path.normpath(path)] |  | 
|   65     return [os.path.basename(p) for p in props.keys()] |  | 
|   66  |  | 
|   67   def IsFile(self, path): |  | 
|   68     """Returns True if the path is a file on the server, False if directory.""" |  | 
|   69     props = self.__norm_path_keys(self.client.propfind(path, depth=1)) |  | 
|   70     return props[os.path.normpath(path)]['resourcetype'] is None |  | 
|   71  |  | 
|   72   def Traverse(self, src_path, dst_path): |  | 
|   73     """Walks the directory hierarchy pointed to by src_path download all files. |  | 
|   74  |  | 
|   75     Recursively walks src_path and saves all files and subfolders into |  | 
|   76     dst_path. |  | 
|   77  |  | 
|   78     Args: |  | 
|   79       src_path: string path on SVN server to save (absolute path on server). |  | 
|   80       dest_path: string local path (relative or absolute) to save to. |  | 
|   81     """ |  | 
|   82     if self.IsFile(src_path): |  | 
|   83       if not os.path.exists(os.path.dirname(dst_path)): |  | 
|   84         logging.info('Creating %s', os.path.dirname(dst_path)) |  | 
|   85         os.makedirs(os.path.dirname(dst_path)) |  | 
|   86       if os.path.isfile(dst_path): |  | 
|   87         logging.info('Skipping %s', dst_path) |  | 
|   88       else: |  | 
|   89         logging.info('Saving %s to %s', self.root_url + src_path, dst_path) |  | 
|   90         urllib.urlretrieve(self.root_url + src_path, dst_path) |  | 
|   91       return |  | 
|   92     else: |  | 
|   93       for subdir in self.GetDirList(src_path): |  | 
|   94         self.Traverse(os.path.join(src_path, subdir), |  | 
|   95                       os.path.join(dst_path, subdir)) |  | 
|   96  |  | 
|   97  |  | 
|   98 def ListAllDepsPaths(deps_file): |  | 
|   99   """Recursively returns a list of all paths indicated in this deps file. |  | 
|  100  |  | 
|  101   Note that this discards information about where path dependencies come from, |  | 
|  102   so this is only useful in the context of a Chromium source checkout that has |  | 
|  103   already fetched all dependencies. |  | 
|  104  |  | 
|  105   Args: |  | 
|  106     deps_file: File containing deps information to be evaluated, in the |  | 
|  107                format given in the header of this file. |  | 
|  108   Returns: |  | 
|  109     A list of string paths starting under src that are required by the |  | 
|  110     given deps file, and all of its sub-dependencies. This amounts to |  | 
|  111     the keys of the 'deps' dictionary. |  | 
|  112   """ |  | 
|  113   deps = {} |  | 
|  114   deps_includes = {} |  | 
|  115  |  | 
|  116   chrome_root = os.path.dirname(__file__) |  | 
|  117   while os.path.basename(chrome_root) != 'src': |  | 
|  118     chrome_root = os.path.abspath(os.path.join(chrome_root, '..')) |  | 
|  119  |  | 
|  120   exec open(deps_file).read()  # pylint: disable=exec-used |  | 
|  121  |  | 
|  122   deps_paths = deps.keys() |  | 
|  123  |  | 
|  124   for path in deps_includes.keys(): |  | 
|  125     # Need to localize the paths. |  | 
|  126     path = os.path.join(chrome_root, '..', path) |  | 
|  127     deps_paths += ListAllDepsPaths(path) |  | 
|  128  |  | 
|  129   return deps_paths |  | 
|  130  |  | 
|  131  |  | 
|  132 def DownloadDeps(destination_dir, url): |  | 
|  133   """Saves all the dependencies in deps_path. |  | 
|  134  |  | 
|  135   Opens and reads url, assuming the contents are in the simple DEPS-like file |  | 
|  136   format specified in the header of this file, then download all |  | 
|  137   files/directories listed to the destination_dir. |  | 
|  138  |  | 
|  139   Args: |  | 
|  140     destination_dir: String path to directory to download files into. |  | 
|  141     url: URL containing deps information to be evaluated. |  | 
|  142   """ |  | 
|  143   logging.warning('Downloading deps from %s...', url) |  | 
|  144   # TODO(wiltzius): Add a parameter for which revision to pull. |  | 
|  145   _DownloadAndImportDAVClientModule() |  | 
|  146  |  | 
|  147   deps = {} |  | 
|  148   deps_includes = {} |  | 
|  149  |  | 
|  150   exec urllib.urlopen(url).read()  # pylint: disable=exec-used |  | 
|  151  |  | 
|  152   for dst_path, src_path in deps.iteritems(): |  | 
|  153     full_dst_path = os.path.join(destination_dir, dst_path) |  | 
|  154     parsed_url = urlparse.urlparse(src_path) |  | 
|  155     root_url = parsed_url.scheme + '://' + parsed_url.netloc |  | 
|  156  |  | 
|  157     dav_client = DAVClientWrapper(root_url) |  | 
|  158     dav_client.Traverse(parsed_url.path, full_dst_path) |  | 
|  159  |  | 
|  160   for url in deps_includes.values(): |  | 
|  161     DownloadDeps(destination_dir, url) |  | 
| OLD | NEW |