Add telemetry_bootstrap module

tools/telemetry/telemetry_bootstrap.py

+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Bootstrap Chrome Telemetry by downloading all its files from SVN servers.
+
+Requires a DEPS file to specify which directories on which SVN servers
+are required to run Telemetry. Format of that DEPS file is a subset of the
+normal DEPS file format[1]; currently only only the "deps" dictionary is
+supported and nothing else.
+
+DEPS can be specified with --deps or defaults to ./DEPS
+
+Fetches all files in the specified directories using WebDAV (SVN is WebDAV under
+the hood).
+
+[1] http://dev.chromium.org/developers/how-tos/depottools#TOC-DEPS-file
+"""
+
+import imp
+import logging
+from optparse import OptionParser

[nduca, 2013/01/05 01:28:19]

I think you can ditch main() entirely and thus OptionParser.

[wiltzius, 2013/01/08 02:53:34]

Done.

+import os
+import urllib
+import urlparse
+
+# default DEPS filename
+DEPS_FILE = "DEPS"

[nduca, 2013/01/05 01:28:19]

This can go away

[wiltzius, 2013/01/08 02:53:34]

Done.

[wiltzius, 2013/01/08 02:53:34]

Done.

+# link to file containing the 'davclient' WebDAV client library

[nduca, 2013/01/05 01:28:19]

# Comments start with capitals end and end with a period.

^ here and elsewhere

[wiltzius, 2013/01/08 02:53:34]

Done.

+#TODO(wiltzius) change this to point at Chromium SVN server after checkin

[nduca, 2013/01/05 01:28:19]

# TODO(wiltzius): 

+DAVCLIENT_URL = 'http://svn.osafoundation.org/tools/davclient/trunk/src/davclient/davclient.py'

[nduca, 2013/01/05 01:28:19]

_DAVCLIENT_URL since its private to the module. _-prefix all the methods and
vars in this file that aren't intended to be visible to users of the file.

[wiltzius, 2013/01/08 02:53:34]

Done.

+
+def bootstrap_davclient():

[nduca, 2013/01/05 01:28:19]

_bootstrap_davclient since this shouldn't be called by users of the module

[wiltzius, 2013/01/08 02:53:34]

Done.

+  """Dynamically import davclient helper library."""

[nduca, 2013/01/05 01:28:19]

_download_and_import_davclient_module()?

[wiltzius, 2013/01/08 02:53:34]

Done.

+  global davclient
+  davclient_txt = urllib.urlopen(DAVCLIENT_URL).read()

[nduca, 2013/01/05 01:28:19]

txt -> source?

[wiltzius, 2013/01/08 02:53:34]

Done.

[wiltzius, 2013/01/08 02:53:34]

Done.

+  davclient = imp.new_module('davclient')
+  exec davclient_txt in davclient.__dict__
+
+
+class DAVClientWrapper():
+  """Knows how to retrieve subdirectories and files from WebDAV/SVN servers."""
+
+  def __init__(self, root_url):
+    """Initialize SVN server root_url, save files to local dest_dir.
+
+    Args:
+      root_url: string url of SVN/WebDAV server
+    """
+    self.root_url = root_url
+    self.client = davclient.DAVClient(root_url)
+
+  def GetSubdirs(self, path):
+    """Returns string names of all subdirs of this path on the SVN server."""
+    props = self.client.propfind(path, depth=1)
+    return map(os.path.basename, props.keys())
+
+  def IsFile(self, path):
+    """Returns True if the path is a file on the server, False if directory."""
+    props = self.client.propfind(path, depth=1)
+    # build up normalized path list since paths to directories may or may not
+    # have trailing slashes
+    norm_keys = {}
+    for entry in props.keys():
+      norm_keys[os.path.normpath(entry)] = entry
+    return props[norm_keys[os.path.normpath(path)]]['resourcetype'] is None
+
+  def Traverse(self, src_path, dst_path):
+    """Walks the directory hierarchy pointed to by src_path download all files.
+
+    Recursively walks src_path and saves all files and subfolders into
+    dst_path.
+
+    Args:
+      src_path: string path on SVN server to save (absolute path on server).
+      dest_path: string local path (relative or absolute) to save to.
+    """
+    if self.IsFile(src_path):
+      if not os.path.exists(os.path.dirname(dst_path)):
+        logging.info("creating %s", os.path.dirname(dst_path))
+        os.makedirs(os.path.dirname(dst_path))
+      logging.info("Saving %s to %s", self.root_url + src_path, dst_path)
+      urllib.urlretrieve(self.root_url + src_path, dst_path)
+      return
+    else:
+      for subdir in self.GetSubdirs(src_path):
+        if subdir:
+          self.Traverse(os.path.join(src_path, subdir),
+                        os.path.join(dst_path, subdir))
+
+
+def ParseOptions():

[nduca, 2013/01/05 01:28:19]

I think this can go away

[wiltzius, 2013/01/08 02:53:34]

Done.

+  """Parses command-line options for the bootstrap."""
+  #TODO(wiltzius) have a --revision flag that pulls at a specific rev
+  parser = OptionParser()
+  parser.add_option("-v", "--verbosity", dest="verbosity", default=1,
+            type="int", help="logging verbosity level (0, 1, 2, 3)")
+  parser.add_option("--deps", dest="deps_filename", default=DEPS_FILE,
+            help="look for dependency manifest in this file")
+  return parser.parse_args()
+
+
+def DownloadDEPS(deps_path):

[nduca, 2013/01/05 01:28:19]

Should we have a TODO here for what revision of telemetry to pull?

[wiltzius, 2013/01/08 02:53:34]

Done.

+  """Saves all the dependencies in deps_path."""
+  # dynamically import davclient library
+  bootstrap_davclient()
+  with open(deps_path) as deps_file:
+    deps = imp.new_module('deps')
+    exec deps_file.read() in deps.__dict__

[nduca, 2013/01/05 01:28:19]

unindent the exec stuff

so you say

with blah as blah:
  deps_contents = deps_file.read()

deps = ...
exec deps_contents in deps.__dict() 

[wiltzius, 2013/01/08 02:53:34]

Done.

+    #TODO(wiltzius) in the future, make the destination directory configurable

[nduca, 2013/01/05 01:28:19]

formatting

[wiltzius, 2013/01/08 02:53:34]

Done.

+    # as something other than the cwd

[nduca, 2013/01/05 01:28:19]

this function should take in destination dir. E.g. the caller of it should give
the destination directory, which could be cwd, but not done here.

[wiltzius, 2013/01/08 02:53:34]

Done.

+    destination_dir = os.getcwd()
+    for dst_path, src_path in deps.deps.iteritems():
+      parsed_url = urlparse.urlparse(src_path)
+      root_url = parsed_url.scheme + '://' + parsed_url.netloc

[nduca, 2013/01/05 01:28:19]

where will we handle "foo@181" type stuff?

[wiltzius, 2013/01/08 02:53:34]

That's the "fetch at revision" TODO. This will need to go here somehow, but will
require changes to the DAVClient wrapper too.

On 2013/01/05 01:28:19, nduca wrote:

+      dav_client = DAVClientWrapper(root_url)
+      dav_client.Traverse(parsed_url.path,
+                          os.path.join(destination_dir, dst_path))

[nduca, 2013/01/05 01:28:19]

after we clone dst_path, what if dst_path has a DEPS file?

+
+
+def Main():

[nduca, 2013/01/05 01:28:19]

not needed I think.

[wiltzius, 2013/01/08 02:53:34]

Done.

+  """Fetches files specified in dependency manifest from SVN/WebDAV server."""
+  options = ParseOptions()[0]
+  # set logging level per verbosity
+  logging_levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG]
+  logging.basicConfig(level=logging_levels[options.verbosity])
+  DownloadDEPS(options.deps_filename)
+
+
+if __name__ == "__main__":

[nduca, 2013/01/05 01:28:19]

Ditch the __name__ bit as well as the #! bit. If someone wants to run this
directly, we should add a script up one level. Not clear if it makes sense,
anyway.

[wiltzius, 2013/01/08 02:53:34]

Done.

+  Main()

[nduca, 2013/01/05 01:28:19]

Lets add the stuff that you had in __main__ into
tools/perf/run_multipage_benchmark --- eg that should have the script to
download telemetry_bootstrap.py

[nduca, 2013/01/05 01:28:19]

Lets move this to tools/telemetry/tools/telemetry_bootstrap.py --- that will
ensure that this can't reference any parts of telemetry itself and vice versa.

[wiltzius, 2013/01/08 02:53:34]

I will do this in a follow-up patch after this bootstrap lands, don't want to
risk messing up run_multipage_benchmark before I'm sure this script works :)

On 2013/01/05 01:28:19, nduca wrote: