new tool: download all GM images for a given builder, ready for skdiff

gm/rebaseline_server/url_or_path.py

+#!/usr/bin/python
+
+"""
+Copyright 2014 Google Inc.
+
+Use of this source code is governed by a BSD-style license that can be
+found in the LICENSE file.
+
+Utility module for handling paths that may be either URL paths or local
+filepaths.
+
+TODO(epoger): Move this module to some common place so we can use it more widely
+"""
+
+# System-level imports
+import contextlib
+import os
+import posixpath
+import shutil
+import urllib
+import urllib2
+import urlparse
+
+def is_url(path):
+  """ Returns true if path looks like a URL. """
+  return True if urlparse.urlparse(path).scheme else False
+
+def join(first, *remaining):
+  """ Joins two or more URL or pathname components.  If the first component
+  looks like a valid URL, we assemble a URL; otherwise, we assemble it as
+  a pathname to a local file.
+
+  Args:
+    first: string; first component of the URL/path
+    remaining: variable number of strings; remaining URL/path components
+
+  Returns:
+    The complete URL or filename path.
+  """
+  if is_url(first):
+    return posixpath.join(first, *remaining)
+  else:
+    return os.path.join(first, *remaining)
+
+def create_filepath_url(filepath):
+  """ Returns a file:/// URL pointing at the given filepath on local disk.
+
+  Args:
+    filepath: string; path to a file on local disk (may be absolute or relative,
+        and the file does not need to exist)
+
+  Returns:
+    A file:/// URL pointing at the file.  Regardless of whether filepath was
+        specified as a relative or absolute path, the URL will contain an
+        absolute path to the file.
+
+  Raises:
+    An Exception, if filepath is already a URL.
+  """
+  if is_url(filepath):
+    raise Exception('"%s" is already a URL' % filepath)
+  return urlparse.urljoin(
+      'file:', urllib.pathname2url(os.path.abspath(filepath)))
+
+def read_as_string(path):
+  """ Reads in the full contents of the URL or filepath 'path', returning those
+  contents as a single string.
+
+  Args:
+    path: string; complete URL or filepath (if filepath, may be absolute or
+        relative)
+
+  Returns:
+    Complete contents of the URL or filepath as a string.
+
+  Raises:
+    Some subclass of Exception if unable to read the contents (URL/file not
+    found, etc.)
+  """
+  if is_url(path):
+    try:
+      return urllib2.urlopen(path).read()
+    except Exception, e:
+      e.msg += ' for URL ' + path
+      raise
+  else:
+    return open(path, 'r').read()
+
+def copy_contents(source_path, dest_path, create_subdirs_if_needed=False):
+  """ Copies the full contents of the URL or filepath 'source_path' into
+  filepath 'dest_path'.
+
+  Args:
+    source_path: string; complete URL or filepath to read from (if filepath,
+        may be absolute or relative)
+    dest_path: string; complete filepath to write to (may be absolute or
+        relative)
+    create_subdirs_if_needed: boolean; whether to create subdirectories as
+        needed to create dest_path
+
+  Raises:
+    Some subclass of Exception if unable to read source_path (URL/file not
+    found, etc.) or write dest_path.
+  """
+  if create_subdirs_if_needed:
+    dest_dir = os.path.dirname(dest_path)
+    if not os.path.exists(dest_dir):
+      os.makedirs(dest_dir)
+  if is_url(source_path):
+    with contextlib.closing(urllib.urlopen(source_path)) as source_handle:
+      with open(dest_path, 'wb') as dest_handle:
+        shutil.copyfileobj(fsrc=source_handle, fdst=dest_handle)
+  else:
+    shutil.copyfile(source_path, dest_path)