update_nacl_manifest improvements

native_client_sdk/src/build_tools/update_nacl_manifest.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Script that reads omahaproxy and gsutil to determine version of SDK to put
 in manifest.
 """
 
 # pylint is convinced the email module is missing attributes
@@ skipping 149 matching lines @@
       stdin: If src is '-', this is used as the stdin to give to gsutil. The
           effect is that text in stdin is copied to |dest|."""
     raise NotImplementedError()
 
   def Print(self, *args):
     """Print a message."""
     raise NotImplementedError()
 
 
 class RealDelegate(Delegate):
-  def __init__(self, dryrun=False, gsutil=None):
+  def __init__(self, dryrun=False, gsutil=None, verbose=False):
     super(RealDelegate, self).__init__()
     self.dryrun = dryrun
+    self.verbose = verbose
     if gsutil:
       self.gsutil = gsutil
     else:
       self.gsutil = buildbot_common.GetGsutil()
 
   def GetRepoManifest(self):
     """See Delegate.GetRepoManifest"""
     with open(REPO_MANIFEST, 'r') as sdk_stream:
       sdk_json_string = sdk_stream.read()
 
     manifest = manifest_util.SDKManifest()
-    manifest.LoadDataFromString(sdk_json_string)
+    manifest.LoadDataFromString(sdk_json_string, add_missing_info=True)
     return manifest
 
   def GetHistory(self):
     """See Delegate.GetHistory"""
     url_stream = urllib2.urlopen('https://omahaproxy.appspot.com/history')
     return [(platform, channel, version, date)
         for platform, channel, version, date in csv.reader(url_stream)]
 
   def GetTrunkRevision(self, version):
     """See Delegate.GetTrunkRevision"""
@@ skipping 10 matching lines @@
     # filter out empty lines
     return filter(None, stdout.split('\n'))
 
   def GsUtil_cat(self, url):
     """See Delegate.GsUtil_cat"""
     return self._RunGsUtil(None, 'cat', url)
 
   def GsUtil_cp(self, src, dest, stdin=None):
     """See Delegate.GsUtil_cp"""
     if self.dryrun:
+      self.Trace("Skipping upload: %s -> %s" % (src, dest))
       return
 
     # -p ensures we keep permissions when copying "in-the-cloud".
     return self._RunGsUtil(stdin, 'cp', '-p', '-a', 'public-read', src, dest)
 
   def Print(self, *args):
     sys.stdout.write(' '.join(map(str, args)) + '\n')
 
+  def Trace(self, *args):
+    if self.verbose:
+      self.Print(*args)
+
   def _RunGsUtil(self, stdin, *args):
     """Run gsutil as a subprocess.
 
     Args:
       stdin: If non-None, used as input to the process.
       *args: Arguments to pass to gsutil. The first argument should be an
           operation such as ls, cp or cat.
     Returns:
       The stdout from the process."""
     cmd = [self.gsutil] + list(args)
+    self.Trace("Running: %s" % str(cmd))
     if stdin:
       stdin_pipe = subprocess.PIPE
     else:
       stdin_pipe = None
 
-    process = subprocess.Popen(cmd, stdin=stdin_pipe, stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE)
-    stdout, stderr = process.communicate(stdin)
+    try:
+      process = subprocess.Popen(cmd, stdin=stdin_pipe, stdout=subprocess.PIPE,
+          stderr=subprocess.PIPE)
+      stdout, stderr = process.communicate(stdin)
+    except OSError as e:
+      raise manifest_util.Error("Unable to run '%s': %s" % (cmd[0], str(e)))
 
-    if process.returncode != 0:
+    if process.returncode:
       sys.stderr.write(stderr)
       raise subprocess.CalledProcessError(process.returncode, ' '.join(cmd))
     return stdout
 
 
 class VersionFinder(object):
   """Finds a version of a pepper bundle that all desired platforms share."""
   def __init__(self, delegate):
     self.delegate = delegate
     self.history = delegate.GetHistory()
@@ skipping 258 matching lines @@
 
   def _UploadManifest(self, manifest):
     """Upload a serialized manifest_util.SDKManifest object.
 
     Upload one copy to gs://<BUCKET_PATH>/naclsdk_manifest2.json, and a copy to
     gs://<BUCKET_PATH>/manifest_backups/naclsdk_manifest2.<TIMESTAMP>.json.
 
     Args:
       manifest: The new manifest to upload.
     """
+    if self.delegate.dryrun:
+      name = MANIFEST_BASENAME + ".new"
+      self.delegate.Print("Writing new manifest: %s" % name)
+      with open(name, 'w') as f:
+        f.write(manifest.GetDataAsString())
+      stdout = self.delegate.GsUtil_cat(GS_SDK_MANIFEST)
+
+      online = MANIFEST_BASENAME + ".online"
+      self.delegate.Print("Writing online manifest: %s" % online)
+      with open(online, 'w') as f:
+        f.write(stdout)
+      os.system('diff -u %s %s' % (online, name))
+
     timestamp_manifest_path = GS_MANIFEST_BACKUP_DIR + \
         GetTimestampManifestName()
     self.delegate.GsUtil_cp('-', timestamp_manifest_path,
         stdin=manifest.GetDataAsString())
 
     # copy from timestampped copy over the official manifest.
     self.delegate.GsUtil_cp(timestamp_manifest_path, GS_SDK_MANIFEST)
 
 
 def Run(delegate, platforms):
@@ skipping 59 matching lines @@
     self.written.write(s)
     if self.passthrough:
       self.passthrough.write(s)
 
   def getvalue(self):
     return self.written.getvalue()
 
 
 def main(args):
   parser = optparse.OptionParser()
-  parser.add_option('--gsutil', help='path to gsutil', dest='gsutil',
-      default=None)
-  parser.add_option('--mailfrom', help='email address of sender',
-      dest='mailfrom', default=None)
-  parser.add_option('--mailto', help='send error mails to...', dest='mailto',
-      default=[], action='append')
-  parser.add_option('--dryrun', help='don\'t upload the manifest.',
-      dest='dryrun', action='store_true', default=False)
+  parser.add_option('--gsutil', help='path to gsutil.')
+  parser.add_option('-d', '--debug', help='run in debug mode.',
+      action='store_true')
+  parser.add_option('--mailfrom', help='email address of sender.')
+  parser.add_option('--mailto', help='send error mails to...', action='append')
+  parser.add_option('-n', '--dryrun', help="don't upload the manifest.",
+      action='store_true')
+  parser.add_option('-v', '--verbose', help='print more diagnotic messages.',
+      action='store_true')
   options, args = parser.parse_args(args[1:])
 
   if (options.mailfrom is None) != (not options.mailto):
     options.mailfrom = None
     options.mailto = None
     sys.stderr.write('warning: Disabling email, one of --mailto or --mailfrom '
         'was missing.\n')
 
   if options.mailfrom and options.mailto:
     # Capture stderr so it can be emailed, if necessary.
     sys.stderr = CapturedFile(sys.stderr)
 
   try:
-    delegate = RealDelegate(dryrun=options.dryrun, gsutil=options.gsutil)
-    Run(delegate, ('mac', 'win', 'linux'))
-  except Exception:
-    if options.mailfrom and options.mailto:
-      traceback.print_exc()
-      scriptname = os.path.basename(sys.argv[0])
-      subject = '[%s] Failed to update manifest' % (scriptname,)
-      text = '%s failed.\n\nSTDERR:\n%s\n' % (scriptname, sys.stderr.getvalue())
-      SendMail(options.mailfrom, options.mailto, subject, text)
-      sys.exit(1)
-    else:
+    try:
+      delegate = RealDelegate(options.dryrun, options.gsutil, options.verbose)
+      Run(delegate, ('mac', 'win', 'linux'))
+    except Exception:
+      if options.mailfrom and options.mailto:
+        traceback.print_exc()
+        scriptname = os.path.basename(sys.argv[0])
+        subject = '[%s] Failed to update manifest' % (scriptname,)
+        text = '%s failed.\n\nSTDERR:\n%s\n' % (scriptname,
+                                                sys.stderr.getvalue())
+        SendMail(options.mailfrom, options.mailto, subject, text)
+        sys.exit(1)
+      else:
+        raise
+  except manifest_util.Error as e:
+    if options.debug:
       raise
+    print e
+    sys.exit(1)
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv))