Chrome updater test framework

pyautolib/fetch_prebuilt_pyauto.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Fetch prebuilt binaries to run PyAuto.
 
 Sets up Chrome and PyAuto binaries using prebuilt binaries from the
 continuous build archives.  Works on mac, win, linux (32 & 64 bit).
 
@@ skipping 10 matching lines @@
 import glob
 import httplib
 import optparse
 import os
 import platform
 import shutil
 import sys
 import urllib
 import urllib2
 import urlparse
+import socket
+import stat
+import zipfile
 
 import pyauto_utils
 
+            
+class FetchPrebuilt(object):
+  """Util class to fetch prebuilt binaries to run PyAuto."""  
+  
+  def __init__(self, url, outdir, platform):  
+    self._url = url
+    if not pyauto_utils.DoesUrlExist(self._url):
+      raise httplib.error('%s is not a valid URL.' % self._url)
+    self._outdir = outdir    
+    self._platform = platform   
+    self._InitZipName()    
+    self._InitDownloadUrls()
 
-class FetchPrebuilt(object):
-  """Util class to fetch prebuilt binaries to run PyAuto."""
-
-  def _ParseArgs(self):
-    parser = optparse.OptionParser()
-    parser.add_option(
-        '-d', '--outdir', type='string', default=None,
-        help='Directory in which to setup. This is typically the directory '
-        'where the binaries would go when compiled from source.')
-    parser.add_option(
-        '-p', '--platform', type='string',
-        default=pyauto_utils.GetCurrentPlatform(),
-        help='Platform. Valid options: win, mac, linux32, linux64. '
-        'Default: current platform (%s)' % pyauto_utils.GetCurrentPlatform())
-    parser.add_option(
-        '-l', '--latest', action='store_true', default=False,
-        help='Download the latest chromium build from commondatastorage. '
-        '[default=False]')
-    self._options, self._args = parser.parse_args()
-    if self._options.latest:
-      self._url = self._GetLastestDownloadURL(self._options.platform)
-    elif not self._args:
-      print >>sys.stderr, 'Need download url'
-      sys.exit(2)
-    else:
-      self._url = self._args[0]
-    if not self._options.outdir:
-      print >>sys.stderr, 'Need output directory: -d/--outdir'
-      sys.exit(1)
-    self._outdir = self._options.outdir
-    # Chromium continuous build archive has a non-standard format.
-    if 'index.html?path=' in self._url:
-      self._url = self._url.replace('index.html?path=', '')
-    self._url = self._url.rstrip('/')
-    # Determine name of zip.
-    if not self._options.platform.startswith('linux'):
-      self._chrome_zip_name = 'chrome-%s' % {'mac': 'mac',
-                                             'win': 'win32'
-                                            }[self._options.platform]
+  def _InitZipName(self):
+    """Determines the name of the zip file based on platform."""
+    if not self._platform.startswith('linux'):
+      self._chrome_zip_name = 'chrome-%s' % { 'mac' : 'mac',
+                                              'win' : 'win32'
+                                            }[self._platform]
     else:
       linux_32_names = ['linux', 'lucid32bit']
       linux_64_names = ['linux64', 'lucid64bit']
       linux_names = {'linux': linux_32_names + linux_64_names,
                      'linux32': linux_32_names,
                      'linux64': linux_64_names
-                    }[self._options.platform]
+                    }[self._platform]
       for name in linux_names:
         zip_name = 'chrome-' + name
-        if self._DoesURLExist('%s/%s.zip' % (self._url, zip_name)):
+        if pyauto_utils.DoesUrlExist('%s/%s.zip' % (self._url, zip_name)):
           self._chrome_zip_name = zip_name
           break
       else:
-        raise RuntimeError('Could not find chrome zip at ' + self._url)
+        raise RuntimeError('Could not find %s at ' % zip_name + self._url)
 
-    # Setup urls to download.
+  def _InitDownloadUrls(self):
+    """Generates URLs for files that are to be downloaded."""
     self._chrome_zip_url = '%s/%s.zip' % (self._url, self._chrome_zip_name)
     self._remoting_zip_url = self._url + '/' + 'remoting-webapp.zip'
     chrome_test_url = '%s/%s.test' % (self._url, self._chrome_zip_name)
     self._pyautolib_py_url = '%s/pyautolib.py' % chrome_test_url
-    if self._options.platform == 'win':
+    if self._platform == 'win':
       self._pyautolib_so_name = '_pyautolib.pyd'
       self._chromedriver_name = 'chromedriver.exe'
     else:
       self._pyautolib_so_name = '_pyautolib.so'
       self._chromedriver_name = 'chromedriver'
-    if self._options.platform == 'mac':
+    if self._platform == 'mac':
       self._ffmpegsumo_so_name = 'ffmpegsumo.so'
       self._ffmpegsumo_so_url = chrome_test_url + '/' + self._ffmpegsumo_so_name
     self._pyautolib_so_url = chrome_test_url + '/' + self._pyautolib_so_name
     self._chromedriver_url = chrome_test_url + '/' + self._chromedriver_name
 
-  def _GetLastestDownloadURL(self, os_platform):
+  @staticmethod
+  def GetLastestDownloadURL(os_platform):
     os_type = {'win': 'Win',
                'mac': 'Mac',
                'linux': 'Linux',
                'linux32': 'Linux',
                'linux64': 'Linux_x64'}[os_platform]
     if os_type == 'Linux' and platform.architecture()[0] == '64bit':
       os_type = 'Linux_x64'
     last_change_url = ('http://commondatastorage.googleapis.com/'
                        'chromium-browser-continuous/%s/LAST_CHANGE' % os_type)
     response = urllib2.urlopen(last_change_url)
     last_change = response.read()
     if not last_change:
       print >>sys.stderr, ('Unable to get latest from %s' % last_change_url)
       sys.exit(2)
     last_change_url = ('http://commondatastorage.googleapis.com/'
                        'chromium-browser-continuous/%s/%s' % (os_type,
                                                               last_change))
     return last_change_url
 
-  def _DoesURLExist(self, url):
-    """Determines whether a resource exists at the given URL."""
-    parsed = urlparse.urlparse(url)
-    conn = httplib.HTTPConnection(parsed.netloc)
-    conn.request('HEAD', parsed.path)
-    response = conn.getresponse()
-    if response.status == 302:  # Redirect; follow it.
-      return self._DoesURLExist(response.getheader('location'))
-    return response.status == 200
+  def _Cleanup(self):
+    """Remove old binaries, if any."""
+    
+    def _OnError(func, path, exc_info):
+      """Callback for shutil.rmtree."""    
+      if not os.access(path, os.W_OK):      
+        os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)      
+        func(path)    
+      else:
+        raise OSError('%s is not a read-only file, access denied.' % path)
+      
+    lst_dirs = ['PepperFlash', 'remoting', 'locales', 'resources', 'installer']
+    paths = [os.path.join(self._outdir, name) for name in lst_dirs]
+    for path in paths:
+      if os.path.exists(path):
+        shutil.rmtree(path, onerror=_OnError)
 
-  def Cleanup(self):
-    """Remove old binaries, if any."""
-    pass
-
-  def Run(self):
-    self._ParseArgs()
+  def Run(self):    
     if not os.path.isdir(self._outdir):
       os.makedirs(self._outdir)
-    get_remoting = self._DoesURLExist(self._remoting_zip_url)
-
+    else:      
+      print 'Cleaning up %s' % self._outdir
+      self._Cleanup()
+    get_remoting = pyauto_utils.DoesUrlExist(self._remoting_zip_url)
     # Fetch chrome & pyauto binaries
     print 'Fetching', self._chrome_zip_url
     chrome_zip = urllib.urlretrieve(self._chrome_zip_url)[0]
 
     if get_remoting:
       print 'Fetching', self._remoting_zip_url
       remoting_zip = urllib.urlretrieve(self._remoting_zip_url)[0]
     else:
       print 'Warning: %s does not exist.' % self._remoting_zip_url
 
     print 'Fetching', self._pyautolib_py_url
     pyautolib_py = urllib.urlretrieve(self._pyautolib_py_url)[0]
 
     print 'Fetching', self._pyautolib_so_url
     pyautolib_so = urllib.urlretrieve(self._pyautolib_so_url)[0]
 
-    if self._options.platform == 'mac':
+    if self._platform == 'mac':
       print 'Fetching', self._ffmpegsumo_so_url
       ffmpegsumo_so = urllib.urlretrieve(self._ffmpegsumo_so_url)[0]
 
     print 'Fetching', self._chromedriver_url
     chromedriver = urllib.urlretrieve(self._chromedriver_url)[0]
 
     chrome_unzip_dir = os.path.join(self._outdir, self._chrome_zip_name)
     if os.path.exists(chrome_unzip_dir):
       print 'Cleaning', chrome_unzip_dir
       pyauto_utils.RemovePath(chrome_unzip_dir)
     print 'Unzipping'
-    pyauto_utils.UnzipFilenameToDir(chrome_zip, self._outdir)
+    try:
+      pyauto_utils.UnzipFilenameToDir(chrome_zip, self._outdir)
+    except zipfile.BadZipfile:
+      raise RuntimeError('Specified zip file does not exist or is corrupted.')
     if get_remoting:
-      pyauto_utils.UnzipFilenameToDir(remoting_zip, self._outdir)
+      pyauto_utils.UnzipFilenameToDir(remoting_zip, self._outdir)       
       shutil.move(self._outdir + '/remoting-webapp',
                   self._outdir + '/remoting/remoting.webapp')
-
-    # Copy over the binaries to outdir
+    # Copy over the binaries to destination directory.
     items_to_copy = {
       pyautolib_py: os.path.join(self._outdir, 'pyautolib.py'),
       pyautolib_so: os.path.join(self._outdir, self._pyautolib_so_name),
       chromedriver: os.path.join(self._outdir, self._chromedriver_name)
     }
-    if self._options.platform == 'mac':
+    if self._platform == 'mac':
       items_to_copy[ffmpegsumo_so] = \
           os.path.join(self._outdir, self._ffmpegsumo_so_name)
 
     unzip_dir_contents = glob.glob(os.path.join(chrome_unzip_dir, '*'))
     for item in unzip_dir_contents:
       name = os.path.basename(item)
       items_to_copy[item] = os.path.join(self._outdir, name)
 
     for src, dest in items_to_copy.iteritems():
       pyauto_utils.RemovePath(dest)
       print '%s ==> %s' % (os.path.basename(src), dest)
       shutil.move(src, dest)
     pyauto_utils.RemovePath(chrome_unzip_dir)
 
     # Final setup (if any)
     # Set executable bit on chromedriver binary.
-    if not self._options.platform == 'win':
+    if not self._platform == 'win':
       os.chmod(items_to_copy[chromedriver], 0700)
 
     # Create symlink to .framework on Mac
-    if self._options.platform == 'mac':
+    if self._platform == 'mac':
       mac_app_name = os.path.basename([x for x in unzip_dir_contents
                                        if x.endswith('.app')][0])
       os.chdir(self._outdir)
       framework = glob.glob(os.path.join(
           mac_app_name, 'Contents', 'Versions', '*', '*.framework'))[0]
       print framework
       dest = os.path.basename(framework)
       os.path.lexists(dest) and os.remove(dest)
       print 'Creating symlink "%s"' % dest
       os.symlink(framework, dest)
 
     print 'Prepared binaries in "%s"' % self._outdir
     return 0
 
 
-if __name__ == '__main__':
-  sys.exit(FetchPrebuilt().Run())
+class Main:

[Nirnimesh, 2012/08/22 07:06:35]

derive from object

[nkang, 2012/08/24 22:45:26]

Done.

+  """Main program for fetching a Chrome build."""
+
+  def __init__(self):
+    self._ParseArgs()
+    FetchPrebuilt(self._url, self._outdir, self._platform).Run()  
+
+  def _ParseArgs(self):
+    """Parses command line arguments."""
+    parser = optparse.OptionParser()
+    parser.add_option(
+        '-d', '--outdir', type='string', default=None,
+        help='Directory in which to setup. This is typically the directory '
+        'where the binaries would go when compiled from source.')
+    parser.add_option(
+        '-p', '--platform', type='string',
+        default=pyauto_utils.GetCurrentPlatform(),
+        help='Platform. Valid options: win, mac, linux32, linux64. '
+        'Default: current platform (%s)' % pyauto_utils.GetCurrentPlatform())
+    parser.add_option(
+        '-l', '--latest', action='store_true', default=False,
+        help='Download the latest chromium build from commondatastorage. '
+        '[default=False]')
+    options, args = parser.parse_args()
+    if not options.outdir:
+      print >>sys.stderr, 'Need output directory: -d/--outdir'
+      sys.exit(1)
+    if options.latest:
+      self._url = FetchPrebuilt.GetLastestDownloadURL(options.platform)
+    elif not args:
+      print >>sys.stderr, 'Need download url'
+      sys.exit(2)
+    else:
+      self._url = args[0]
+
+    self._platform = options.platform
+    self._outdir = options.outdir
+    # Chromium continuous build archive has a non-standard format.
+    if 'index.html?path=' in self._url:
+      self._url = self._url.replace('index.html?path=', '')
+    self._url = self._url.rstrip('/')
+    
+
+if __name__ == '__main__':  
+  sys.exit(Main())

[Nirnimesh, 2012/08/22 07:06:35]

Please ensure that this script continues to work for the current use case.

[nkang, 2012/08/24 22:45:26]

Yes it does. We just separated the command parsing from the rest of the
functionality because we wanted to create a FetchPrebuilt object in InstallTest
and use that object to 'fetch' the build, rather than using this script as a
standalone script. So by separating the command parsing we can do both. Now this
script can be run as standalone, just as it is done for the current use case, or
you can instantiate FetchPrebuilt and use the instance to fetch the build.