[NaCl SDK] update_nacl_manifest: Allow manual updating of a version.

native_client_sdk/src/build_tools/update_nacl_manifest.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Script that reads omahaproxy and gsutil to determine version of SDK to put
 in manifest.
 """
 
 # pylint is convinced the email module is missing attributes
 # pylint: disable=E1101
 
 import buildbot_common
 import csv
 import cStringIO
+import difflib
 import email
 import json
 import manifest_util
 import optparse
 import os
 import posixpath
 import re
 import smtplib
 import subprocess
 import sys
@@ skipping 275 matching lines @@
       "19.0.1084.41". The channel is always 'canary'. |archives| is a list of
       archive URLs."""
     # We don't ship canary on Linux, so it won't appear in self.history.
     # Instead, we can use the matching Linux trunk build for that version.
     shared_version_generator = self._FindNextSharedVersion(
         set(platforms) - set(('linux',)),
         self._GetPlatformCanaryHistory)
     return self._DoGetMostRecentSharedVersion(platforms,
         shared_version_generator, allow_trunk_revisions=True)
 
+  def GetAvailablePlatformArchivesFor(self, version, platforms,
+                                      allow_trunk_revisions):
+    """Returns a sequence of archives that exist for a given version, on the
+    given platforms.
+
+    The second element of the returned tuple is a list of all platforms that do
+    not have an archive for the given version.
+
+    Args:
+      version: The version to find archives for. (e.g. "18.0.1025.164")
+      platforms: A sequence of platforms to consider, e.g.
+          ('mac', 'linux', 'win')
+      allow_trunk_revisions: If True, will search for archives using the
+          trunk revision that matches the branch version.
+    Returns:
+      A tuple (archives, missing_platforms). |archives| is a list of archive
+      URLs, |missing_platforms| is a list of platform names.
+    """
+    archives = self._GetAvailableArchivesFor(version)
+    missing_platforms = set(platforms) - set(GetPlatformsFromArchives(archives))
+    if allow_trunk_revisions and missing_platforms:
+      # Try to find trunk archives for platforms that are missing archives.
+      trunk_version = self.delegate.GetTrunkRevision(version)
+      trunk_archives = self._GetAvailableArchivesFor(trunk_version)
+      for trunk_archive in trunk_archives:
+        trunk_archive_platform = GetPlatformFromArchiveUrl(trunk_archive)
+        if trunk_archive_platform in missing_platforms:
+          archives.append(trunk_archive)
+          missing_platforms.discard(trunk_archive_platform)
+
+    return archives, missing_platforms
+
   def _DoGetMostRecentSharedVersion(self, platforms, shared_version_generator,
       allow_trunk_revisions):
     """Returns the most recent version of a pepper bundle that exists on all
     given platforms.
 
     This function does the real work for the public GetMostRecentShared* above.
 
     Args:
       platforms: A sequence of platforms to consider, e.g.
           ('mac', 'linux', 'win')
       shared_version_generator: A generator that will yield (version, channel)
           tuples in order of most recent to least recent.
       allow_trunk_revisions: If True, will search for archives using the
-            trunk revision that matches the branch version.
+          trunk revision that matches the branch version.
     Returns:
       A tuple (version, channel, archives). The version is a string such as
       "19.0.1084.41". The channel is one of ('stable', 'beta', 'dev',
       'canary'). |archives| is a list of archive URLs."""
     version = None
     skipped_versions = []
     channel = ''
     while True:
       try:
         version, channel = shared_version_generator.next()
       except StopIteration:
         msg = 'No shared version for platforms: %s\n' % (', '.join(platforms))
         msg += 'Last version checked = %s.\n' % (version,)
         if skipped_versions:
           msg += 'Versions skipped due to missing archives:\n'
           for version, channel, archives in skipped_versions:
             if archives:
               archive_msg = '(%s available)' % (', '.join(archives))
             else:
               archive_msg = '(none available)'
           msg += '  %s (%s) %s\n' % (version, channel, archive_msg)
         raise Exception(msg)
 
-      archives = self._GetAvailableNaClSDKArchivesFor(version)
-      missing_platforms = set(platforms) - \
-          set(GetPlatformsFromArchives(archives))
-      if allow_trunk_revisions and missing_platforms:
-        # Try to find trunk archives for platforms that are missing archives.
-        trunk_version = self.delegate.GetTrunkRevision(version)
-        trunk_archives = self._GetAvailableNaClSDKArchivesFor(trunk_version)
-        for trunk_archive in trunk_archives:
-          trunk_archive_platform = GetPlatformFromArchiveUrl(trunk_archive)
-          if trunk_archive_platform in missing_platforms:
-            archives.append(trunk_archive)
-            missing_platforms.discard(trunk_archive_platform)
+      archives, missing_platforms = self.GetAvailablePlatformArchivesFor(
+          version, platforms, allow_trunk_revisions)
 
       if not missing_platforms:
         return version, channel, archives
 
       skipped_versions.append((version, channel, archives))
 
   def _GetPlatformMajorVersionHistory(self, with_major_version, with_platform):
     """Yields Chrome history for a given platform and major version.
 
     Args:
@@ skipping 61 matching lines @@
 
       if all(v == shared_version for c, v in platform_versions):
         # grab the channel from an arbitrary platform
         first_platform = platform_versions[0]
         channel = first_platform[0]
         yield JoinVersion(shared_version), channel
 
         # force increment to next version for all platforms
         shared_version = None
 
-  def _GetAvailableNaClSDKArchivesFor(self, version_string):
+  def _GetAvailableArchivesFor(self, version_string):
     """Downloads a list of all available archives for a given version.
 
     Args:
       version_string: The version to find archives for. (e.g. "18.0.1025.164")
     Returns:
       A list of strings, each of which is a platform-specific archive URL. (e.g.
       "gs://nativeclient_mirror/nacl/nacl_sdk/18.0.1025.164/"
       "naclsdk_linux.tar.bz2").
 
       All returned URLs will use the gs:// schema."""
     files = self.delegate.GsUtil_ls(GS_BUCKET_PATH + version_string)
 
     assert all(file.startswith('gs://') for file in files)
 
     archives = [f for f in files if not f.endswith('.json')]
     manifests = [f for f in files if f.endswith('.json')]
 
     # don't include any archives that don't have an associated manifest.
     return filter(lambda a: a + '.json' in manifests, archives)
 
 
 class Updater(object):
   def __init__(self, delegate):
     self.delegate = delegate
     self.versions_to_update = []
+    self.online_manifest = manifest_util.SDKManifest()
+    self._FetchOnlineManifest()
 
   def AddVersionToUpdate(self, bundle_name, version, channel, archives):
     """Add a pepper version to update in the uploaded manifest.
 
     Args:
       bundle_name: The name of the pepper bundle, e.g. 'pepper_18'
       version: The version of the pepper bundle, e.g. '18.0.1025.64'
       channel: The stability of the pepper bundle, e.g. 'beta'
       archives: A sequence of archive URLs for this bundle."""
     self.versions_to_update.append((bundle_name, version, channel, archives))
 
   def Update(self, manifest):
     """Update a manifest and upload it.
 
+    Note that bundles will not be updated if the current version is newer.
+    That is, the updater will never automatically update to an older version of
+    a bundle.
+
     Args:
       manifest: The manifest used as a template for updating. Only pepper
       bundles that contain no archives will be considered for auto-updating."""
     # Make sure there is only one stable branch: the one with the max version.
     # All others are post-stable.
     stable_major_versions = [SplitVersion(version)[0] for _, version, channel, _
                              in self.versions_to_update if channel == 'stable']
     # Add 0 in case there are no stable versions.
     max_stable_version = max([0] + stable_major_versions)
 
     for bundle_name, version, channel, archives in self.versions_to_update:
       self.delegate.Print('Updating %s to %s...' % (bundle_name, version))
       bundle = manifest.GetBundle(bundle_name)
       for archive in archives:
         platform_bundle = self._GetPlatformArchiveBundle(archive)
         # Normally the manifest snippet's bundle name matches our bundle name.
         # pepper_canary, however is called "pepper_###" in the manifest
         # snippet.
         platform_bundle.name = bundle_name
         bundle.MergeWithBundle(platform_bundle)
 
+      # Check to ensure this bundle is newer than the online bundle.
+      online_bundle = self.online_manifest.GetBundle(bundle_name)
+      if online_bundle and online_bundle.revision >= bundle.revision:
+        self.delegate.Print(
+            '  Revision %s is not newer than than online revision %s. '
+            'Skipping.' % (bundle.revision, online_bundle.revision))
+
+        manifest.MergeBundle(online_bundle)
+        continue
+
       major_version = SplitVersion(version)[0]
       if major_version < max_stable_version and channel == 'stable':
         bundle.stability = 'post_stable'
       else:
         bundle.stability = channel
       # We always recommend the stable version.
       if channel == 'stable':
         bundle.recommended = 'yes'
       else:
         bundle.recommended = 'no'
@@ skipping 22 matching lines @@
 
   def _UploadManifest(self, manifest):
     """Upload a serialized manifest_util.SDKManifest object.
 
     Upload one copy to gs://<BUCKET_PATH>/naclsdk_manifest2.json, and a copy to
     gs://<BUCKET_PATH>/manifest_backups/naclsdk_manifest2.<TIMESTAMP>.json.
 
     Args:
       manifest: The new manifest to upload.
     """
+    new_manifest_string = manifest.GetDataAsString()
+    online_manifest_string = self.online_manifest.GetDataAsString()
+
     if self.delegate.dryrun:
-      name = MANIFEST_BASENAME + ".new"
-      self.delegate.Print("Writing new manifest: %s" % name)
-      with open(name, 'w') as f:
-        f.write(manifest.GetDataAsString())
-      stdout = self.delegate.GsUtil_cat(GS_SDK_MANIFEST)
+      self.delegate.Print(''.join(list(difflib.unified_diff(
+          online_manifest_string.splitlines(1),
+          new_manifest_string.splitlines(1)))))
+    else:
+      online_manifest = manifest_util.SDKManifest()
+      online_manifest.LoadDataFromString(online_manifest_string)
 
-      online = MANIFEST_BASENAME + ".online"
-      self.delegate.Print("Writing online manifest: %s" % online)
-      with open(online, 'w') as f:
-        f.write(stdout)
-      os.system('diff -u %s %s' % (online, name))
+      if online_manifest == manifest:
+        self.delegate.Print('New manifest doesn\'t differ from online manifest.'
+            'Skipping upload.')
+        return
 
     timestamp_manifest_path = GS_MANIFEST_BACKUP_DIR + \
         GetTimestampManifestName()
     self.delegate.GsUtil_cp('-', timestamp_manifest_path,
         stdin=manifest.GetDataAsString())
 
     # copy from timestampped copy over the official manifest.
     self.delegate.GsUtil_cp(timestamp_manifest_path, GS_SDK_MANIFEST)
 
+  def _FetchOnlineManifest(self):
+    try:
+      online_manifest_string = self.delegate.GsUtil_cat(GS_SDK_MANIFEST)
+    except subprocess.CalledProcessError:
+      # It is not a failure if the online manifest doesn't exist.
+      online_manifest_string = ''
 
-def Run(delegate, platforms):
-  """Entry point for the auto-updater."""
+    if online_manifest_string:
+      self.online_manifest.LoadDataFromString(online_manifest_string)
+
+
+def Run(delegate, platforms, fixed_bundle_versions=None):
+  """Entry point for the auto-updater.
+
+  Args:
+    delegate: The Delegate object to use for reading Urls, files, etc.
+    platforms: A sequence of platforms to consider, e.g.
+        ('mac', 'linux', 'win')
+    fixed_bundle_versions: A sequence of tuples (bundle_name, version_string).
+        e.g. ('pepper_21', '21.0.1145.0')
+  """
+  if fixed_bundle_versions:
+    fixed_bundle_versions = dict(fixed_bundle_versions)
+  else:
+    fixed_bundle_versions = {}
+
   manifest = delegate.GetRepoManifest()
   auto_update_bundles = []
   for bundle in manifest.GetBundles():
     if not bundle.name.startswith('pepper_'):
       continue
     archives = bundle.GetArchives()
     if not archives:
       auto_update_bundles.append(bundle)
 
   if not auto_update_bundles:
     delegate.Print('No versions need auto-updating.')
     return
 
   version_finder = VersionFinder(delegate)
   updater = Updater(delegate)
 
   for bundle in auto_update_bundles:
     if bundle.name == CANARY_BUNDLE_NAME:
       version, channel, archives = version_finder.GetMostRecentSharedCanary(
           platforms)
     else:
       version, channel, archives = version_finder.GetMostRecentSharedVersion(
           bundle.version, platforms)
 
+    if bundle.name in fixed_bundle_versions:
+      # Ensure this version is valid for all platforms.
+      # If it is, use the channel found above (because the channel for this
+      # version may not be in the history.)
+      version = fixed_bundle_versions[bundle.name]
+      delegate.Print('Fixed bundle version: %s, %s' % (bundle.name, version))
+      allow_trunk_revisions = bundle.name == CANARY_BUNDLE_NAME
+      archives, missing = version_finder.GetAvailablePlatformArchivesFor(
+          version, platforms, allow_trunk_revisions)
+      if missing:
+        delegate.Print(
+            'An archive for version %s of bundle %s doesn\'t exist '
+            'for platform(s): %s' % (version, bundle.name, ', '.join(missing)))
+        return
+
     updater.AddVersionToUpdate(bundle.name, version, channel, archives)
 
   updater.Update(manifest)
 
 
 def SendMail(send_from, send_to, subject, text, smtp='localhost'):
   """Send an email.
 
   Args:
     send_from: The sender's email address.
@@ skipping 33 matching lines @@
   parser = optparse.OptionParser()
   parser.add_option('--gsutil', help='path to gsutil.')
   parser.add_option('-d', '--debug', help='run in debug mode.',
       action='store_true')
   parser.add_option('--mailfrom', help='email address of sender.')
   parser.add_option('--mailto', help='send error mails to...', action='append')
   parser.add_option('-n', '--dryrun', help="don't upload the manifest.",
       action='store_true')
   parser.add_option('-v', '--verbose', help='print more diagnotic messages.',
       action='store_true')
+  parser.add_option('--bundle-version',
+      help='Manually set a bundle version. This can be passed more than once. '
+      'format: --bundle-version pepper_24=24.0.1312.25', action='append')
   options, args = parser.parse_args(args[1:])
 
   if (options.mailfrom is None) != (not options.mailto):
     options.mailfrom = None
     options.mailto = None
     sys.stderr.write('warning: Disabling email, one of --mailto or --mailfrom '
         'was missing.\n')
 
+  # Parse bundle versions.
+  fixed_bundle_versions = {}
+  if options.bundle_version:
+    for bundle_version_string in options.bundle_version:
+      bundle_name, version = bundle_version_string.split('=')
+      fixed_bundle_versions[bundle_name] = version
+
   if options.mailfrom and options.mailto:
     # Capture stderr so it can be emailed, if necessary.
     sys.stderr = CapturedFile(sys.stderr)
 
   try:
     try:
       delegate = RealDelegate(options.dryrun, options.gsutil, options.verbose)
-      Run(delegate, ('mac', 'win', 'linux'))
+      Run(delegate, ('mac', 'win', 'linux'), fixed_bundle_versions)
     except Exception:
       if options.mailfrom and options.mailto:
         traceback.print_exc()
         scriptname = os.path.basename(sys.argv[0])
         subject = '[%s] Failed to update manifest' % (scriptname,)
         text = '%s failed.\n\nSTDERR:\n%s\n' % (scriptname,
                                                 sys.stderr.getvalue())
         SendMail(options.mailfrom, options.mailto, subject, text)
         sys.exit(1)
       else:
         raise
   except manifest_util.Error as e:
     if options.debug:
       raise
     print e
     sys.exit(1)
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv))