[NaCl SDK] Build naclmono packages based on the sdk manifest.

native_client_sdk/src/build_tools/nacl-mono-buildbot.py

Index: native_client_sdk/src/build_tools/nacl-mono-buildbot.py
===================================================================
--- native_client_sdk/src/build_tools/nacl-mono-buildbot.py	(revision 133727)
+++ native_client_sdk/src/build_tools/nacl-mono-buildbot.py	(working copy)
@@ -3,33 +3,179 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import hashlib
+import json
 import os
 import sys
 
 import buildbot_common
 import build_utils
 
+GS_MANIFEST_PATH = 'gs://nativeclient-mirror/nacl/nacl_sdk/'
+SDK_MANIFEST = 'naclsdk_manifest2.json'
+MONO_MANIFEST = 'naclmono_manifest.json' 
 
-def main(args):
-  args = args[1:]
-
-  buildbot_revision = os.environ.get('BUILDBOT_REVISION', '')
-  assert buildbot_revision
-  sdk_revision = buildbot_revision.split(':')[0]
-  pepper_revision = build_utils.ChromeMajorVersion()
-
+def build_and_upload_mono(sdk_revision, pepper_revision, sdk_url, upload_path, args):
   install_dir = 'naclmono'
   buildbot_common.RemoveDir(install_dir)
 
+  revision_opt = ['--sdk-revision', sdk_revision] if sdk_revision else []
+  url_opt = ['--sdk-url', sdk_url] if sdk_url else []
+
   buildbot_common.Run([sys.executable, 'nacl-mono-builder.py',
-                      '--arch', 'x86-32', '--install-dir', install_dir] + args)
+                      '--arch', 'x86-32', '--install-dir', install_dir] +
+                      revision_opt + url_opt + args)
   buildbot_common.Run([sys.executable, 'nacl-mono-builder.py',
-                      '--arch', 'x86-64', '--install-dir', install_dir] + args)
+                      '--arch', 'x86-64', '--install-dir', install_dir] +
+                      revision_opt + url_opt + args)
   buildbot_common.Run([sys.executable, 'nacl-mono-archive.py',
-                      '--sdk-revision', sdk_revision,
+                      '--upload-path', upload_path,
                       '--pepper-revision', pepper_revision,
                       '--install-dir', install_dir] + args)
 
+def get_sdk_build_info():
+  '''Returns a list of dictionaries for versions of NaCl Mono to build which are
+     out of date compared to the SDKs available to naclsdk'''
 
+  # Get a copy of the naclsdk manifest file
+  buildbot_common.Run([buildbot_common.GetGsutil(), 'cp',
+      GS_MANIFEST_PATH + SDK_MANIFEST, '.'])
+  manifest_file = open(SDK_MANIFEST, 'r')
+  sdk_manifest = json.loads(manifest_file.read())
+  manifest_file.close()
+
+  pepper_infos = []
+  for key, value in sdk_manifest.items():
+    if key == 'bundles':
+      stabilities = ['stable', 'beta', 'dev', 'post_stable']
+      # Pick pepper_* bundles, need pepper_19 or greater to build Mono
+      bundles = filter(lambda b: (b['stability'] in stabilities
+                                  and 'pepper_' in b['name'])
+                                  and b['version'] >= 19, value)
+      for b in bundles:
+        newdict = {}
+        newdict['pepper_revision'] = str(b['version'])
+        linux_arch = filter(lambda u: u['host_os'] == 'linux', b['archives'])
+        newdict['sdk_url'] = linux_arch[0]['url']
+        newdict['sdk_revision'] = b['revision']
+        newdict['stability'] = b['stability']
+        newdict['naclmono_name'] = 'naclmono_' + newdict['pepper_revision']
+        pepper_infos.append(newdict)
+
+  # Get a copy of the naclmono manifest file
+  buildbot_common.Run([buildbot_common.GetGsutil(), 'cp',
+      GS_MANIFEST_PATH + MONO_MANIFEST, '.'])
+  manifest_file = open(MONO_MANIFEST, 'r')
+  mono_manifest = json.loads(manifest_file.read())
+  manifest_file.close()
+  
+  ret = []
+  mono_manifest_dirty = False
+  # Check to see if we need to rebuild mono based on sdk revision
+  for key, value in mono_manifest.items():
+    if key == 'bundles':
+      for info in pepper_infos:
+        bundle = filter(lambda b: b['name'] == info['naclmono_name'], value)
+        if len(bundle) == 0:
+          info['naclmono_rev'] = '1'
+          ret.append(info)
+        else:
+          if info['sdk_revision'] != bundle[0]['sdk_revision']:
+            # This bundle exists in the mono manifest, bump the revision
+            # for the new build we're about to make.
+            info['naclmono_rev'] = str(bundle[0]['revision'] + 1)
+            ret.append(info)
+          elif info['stability'] != bundle[0]['stability']:
+            # If all that happened was the SDK bundle was promoted in stability,
+            # change only that and re-write the manifest
+            mono_manifest_dirty = True
+            bundle[0]['stability'] = info['stability']
+
+  # re-write the manifest here because there are no bundles to build but
+  # the manifest has changed
+  if mono_manifest_dirty and len(ret) == 0:
+    manifest_file = open(MONO_MANIFEST, 'w')
+    manifest_file.write(json.dumps(mono_manifest, sort_keys=False, indent=2))
+    manifest_file.close()
+    buildbot_common.Run([buildbot_common.GetGsutil(), 'cp', '-a', 'public-read',
+        MONO_MANIFEST, GS_MANIFEST_PATH + MONO_MANIFEST])
+
+  return ret
+
+def update_mono_sdk_json(infos):
+  '''Update the naclmono manifest with the newly built packages'''
+  if len(infos) == 0:
+    return
+
+  manifest_file = open(MONO_MANIFEST, 'r')
+  mono_manifest = json.loads(manifest_file.read())
+  manifest_file.close()
+
+  newbundles = {}
+  for info in infos:
+    bundle = {}
+    bundle['name'] = info['naclmono_name']
+    bundle['description'] = 'Mono for Native Client'
+    bundle['stability'] = info['stability']
+    bundle['recommended'] = 'no'
+    bundle['version'] = 'experimental'
+    archive = {}
+    sha1_hash = hashlib.sha1()
+    f = open(info['naclmono_name'] + '.bz2', 'rb')
+    sha1_hash.update(f.read())
+    archive['size'] = f.tell()
+    f.close()
+    archive['checksum'] = { 'sha1': sha1_hash.hexdigest() }
+    archive['host_os'] = 'all'
+    archive['url'] = ('https://commondatastorage.googleapis.com/'
+                      'nativeclient-mirror/nacl/nacl_sdk/%s/%s/%s.bz2'
+                      % (info['naclmono_name'], info['naclmono_rev'],
+                        info['naclmono_name']))
+    bundle['archives'] = [archive]
+    bundle['revision'] = int(info['naclmono_rev'])
+    bundle['sdk_revision'] = int(info['sdk_revision'])
+
+    # Insert this new bundle into the manifest,
+    # probably overwriting an existing bundle.
+    for key, value in mono_manifest.items():
+      if key == 'bundles':
+        existing = filter(lambda b: b['name'] == info['naclmono_name'], value)
+        if len(existing) > 0:
+          loc = value.index(existing[0])
+          value[loc] = bundle
+        else:
+          value.append(bundle)
+    
+  # Write out the file locally, then upload to its known location.
+  manifest_file = open(MONO_MANIFEST, 'w')
+  manifest_file.write(json.dumps(mono_manifest, sort_keys=False, indent=2))
+  manifest_file.close()
+  buildbot_common.Run([buildbot_common.GetGsutil(), 'cp', '-a', 'public-read',
+      MONO_MANIFEST, GS_MANIFEST_PATH + MONO_MANIFEST])
+  
+
+def main(args):
+  args = args[1:]
+
+  buildbot_revision = os.environ.get('BUILDBOT_REVISION', '')
+  buildername = os.environ.get('BUILDBOT_BUILDERNAME', '')
+
+  if buildername == 'linux-sdk-mono32':
+    assert buildbot_revision
+    sdk_revision = buildbot_revision.split(':')[0]
+    pepper_revision = build_utils.ChromeMajorVersion()
+    build_and_upload_mono(sdk_revision, pepper_revision, None,
+                          'trunk.' + sdk_revision, args)
+  elif buildername == 'linux-sdk-mono64':
+    infos = get_sdk_build_info()
+    for info in infos:
+      # This will put the file in naclmono_19/1/naclmono_19.bz2 for example.
+      upload_path = info['naclmono_name'] + '/' + info['naclmono_rev']
+      build_and_upload_mono(None, info['pepper_revision'], info['sdk_url'],
+          upload_path, args)
+    update_mono_sdk_json(infos)
+    
+
+
 if __name__ == '__main__':
   sys.exit(main(sys.argv))