[NaCl SDK] Build naclmono packages based on the sdk manifest.

native_client_sdk/src/build_tools/nacl-mono-buildbot.py

Index: native_client_sdk/src/build_tools/nacl-mono-buildbot.py
===================================================================
--- native_client_sdk/src/build_tools/nacl-mono-buildbot.py	(revision 130135)
+++ native_client_sdk/src/build_tools/nacl-mono-buildbot.py	(working copy)
@@ -3,6 +3,8 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import hashlib
+import json
 import os
 import sys
 
@@ -10,26 +12,146 @@
 import build_utils
 
 
-def main(args):
-  args = args[1:]
-
-  buildbot_revision = os.environ.get('BUILDBOT_REVISION', '')
-  assert buildbot_revision
-  sdk_revision = buildbot_revision.split(':')[0]
-  pepper_revision = build_utils.ChromeMajorVersion()
-
+def build_and_upload_mono(sdk_revision, pepper_revision, sdk_url, upload_path):
   install_dir = 'naclmono'
   buildbot_common.RemoveDir(install_dir)
 
+  revision_opt = ['--sdk-revision', sdk_revision] if sdk_revision else []
+  url_opt = ['--sdk-url', sdk_url] if sdk_url else []
+
   buildbot_common.Run([sys.executable, 'nacl-mono-builder.py',
-                      '--arch', 'x86-32', '--install-dir', install_dir] + args)
+                      '--arch', 'x86-32', '--install-dir', install_dir] +
+                      revision_opt + url_opt + args)
   buildbot_common.Run([sys.executable, 'nacl-mono-builder.py',
-                      '--arch', 'x86-64', '--install-dir', install_dir] + args)
+                      '--arch', 'x86-64', '--install-dir', install_dir] +
+                      revision_opt + url_opt + args)
   buildbot_common.Run([sys.executable, 'nacl-mono-archive.py',
-                      '--sdk-revision', sdk_revision,
+                      '--upload-path', upload_path,
                       '--pepper-revision', pepper_revision,
                       '--install-dir', install_dir] + args)
 
+def get_sdk_build_info():
+  '''Returns a list of dictionaries for versions of NaCl Mono to build which are
+     out of date compared to the SDKs available to naclsdk'''
 
+  # Get a copy of the naclsdk manifest file
+  buildbot_common.Run([buildbot_common.GetGsutil(), 'cp',
+      'gs://nativeclient-mirror/nacl/nacl_sdk/naclsdk_manifest2.json', '.'])
+  manifest_file = open('naclsdk_manifest2.json', 'r')
+  sdk_manifest = json.loads(manifest_file.read())
+  manifest_file.close()
+
+  pepper_infos = []
+  for key, value in sdk_manifest.items():
+    if key == 'bundles':
+      # Pick current pepper_* bundles, need pepper_19 or greater to build Mono
+      bundles = filter(lambda b: (b['stability'] in ['stable', 'beta', 'dev']
+                                  and 'pepper_' in b['name'])
+                                  and b['version'] >= 19, value)
+      for b in bundles:
+        newdict = {}
+        newdict['pepper_revision'] = str(b['version'])
+        linux_arch = filter(lambda u: u['host_os'] == 'linux', b['archives'])
+        newdict['sdk_url'] = linux_arch[0]['url']
+        newdict['sdk_revision'] = b['revision']
+        newdict['stability'] = b['stability']
+        newdict['naclmono_name'] = 'naclmono_' + newdict['pepper_revision']
+        pepper_infos.append(newdict)
+
+  # Get a copy of the naclmono manifest file
+  buildbot_common.Run([buildbot_common.GetGsutil(), 'cp',
+      'gs://nativeclient-mirror/nacl/nacl_sdk/naclmono_manifest.json', '.'])

[binji, 2012/04/20 17:55:17]

could you make globals for these paths, I didn't realize they were different at
first.

BTW, I was going to look at this file, but it doesn't seem to allow public
access?

[elijahtaylor1, 2012/04/20 18:52:50]

Done.

+  manifest_file = open('naclmono_manifest.json', 'r')
+  mono_manifest = json.loads(manifest_file.read())
+  manifest_file.close()
+  
+  ret = []
+  # Check to see if we need to rebuild mono based on sdk revision
+  for key, value in mono_manifest.items():
+    if key == 'bundles':
+      for info in pepper_infos:
+        bundle = filter(lambda b: b['name'] == info['naclmono_name'], value)
+        if len(bundle) == 0:
+          info['naclmono_rev'] = '1'
+          ret.append(info)
+        else:
+          if info['sdk_revision'] != bundle[0]['sdk_revision']:
+            info['naclmono_rev'] = str(bundle[0]['revision']+1)

[binji, 2012/04/20 17:55:17]

why +1?

[elijahtaylor1, 2012/04/20 18:52:50]

Added a comment

+            ret.append(info)
+
+  return ret
+
+def update_mono_sdk_json(infos):
+  '''Update the naclmono manifest with the newly built packages'''
+  if len(infos) == 0:
+    return
+
+  manifest_file = open('naclmono_manifest.json', 'r')
+  mono_manifest = json.loads(manifest_file.read())
+  manifest_file.close()
+
+  newbundles = {}
+  for info in infos:
+    bundle = {}
+    bundle['name'] = info['naclmono_name']
+    bundle['description'] = 'Mono for Native Client'
+    bundle['stability'] = info['stability']
+    bundle['recommended'] = 'no'
+    bundle['version'] = 'experimental'
+    archive = {}
+    sha1_hash = hashlib.sha1()
+    f = open(info['naclmono_name'] + '.bz2', 'rb')
+    sha1_hash.update(f.read())
+    archive['size'] = f.tell()
+    f.close()
+    archive['checksum'] = { 'sha1': sha1_hash.hexdigest() }
+    archive['host_os'] = 'all'
+    archive['url'] = ('https://commondatastorage.googleapis.com/'

[binji, 2012/04/20 17:55:17]

I don't think this is the right path.
This will be:
  https://.../nacl/naclsdk/naclmono_20/<revision>/naclmono_20.bz2

But the archiver still puts it in:
  https://.../nacl/naclsdk/<revision>/naclmono_20.bz2

right?

[elijahtaylor1, 2012/04/20 18:52:50]

Added a comment below to show where this naclmono_20/<revision>/naclmono_20.bz2
construction comes from

+                      'nativeclient-mirror/nacl/nacl_sdk/%s/%s/%s.bz2'
+                      % (info['naclmono_name'], info['naclmono_rev'],
+                        info['naclmono_name']))
+    bundle['archives'] = [archive]
+    bundle['revision'] = int(info['naclmono_rev'])
+    bundle['sdk_revision'] = int(info['sdk_revision'])
+
+    # Insert this new bundle into the manifest,
+    # probably overwriting an existing bundle.
+    for key, value in mono_manifest.items():
+      if key == 'bundles':
+        existing = filter(lambda b: b['name'] == info['naclmono_name'], value)
+        if len(existing) > 0:
+          loc = value.index(existing)
+          value[loc] = bundle
+        else:
+          value.append(bundle)
+    
+  # Write out the file locally, then upload to its known location.
+  manifest_file = open('naclmono_manifest.json', 'w')
+  manifest_file.write(json.dumps(mono_manifest, sort_keys=False, indent=2))
+  manifest_file.close()
+  buildbot_common.Run([buildbot_common.GetGsutil(), 'cp',
+      'naclmono_manifest.json', 'gs://nativeclient-mirror/nacl/nacl_sdk/'])
+  
+
+def main(args):
+  args = args[1:]
+
+  buildbot_revision = os.environ.get('BUILDBOT_REVISION', '')
+  buildername = os.environ.get('BUILDBOT_BUILDERNAME', '')
+
+  if buildername == 'linux-sdk-mono32':
+    assert buildbot_revision
+    sdk_revision = buildbot_revision.split(':')[0]
+    pepper_revision = build_utils.ChromeMajorVersion()
+    build_and_upload_mono(sdk_revision, pepper_revision, None, sdk_revision)

[binji, 2012/04/20 17:55:17]

There is a discrepancy here:
the mono tarballs are uploaded to
  gs://nativeclient-mirror/nacl/nacl_sdk/<revision>

but the sdk tarballs are uploaded to
  gs://nativeclient-mirror/nacl/nacl_sdk/<pepper version>
or
  gs://nativeclient-mirror/nacl/nacl_sdk/trunk.<revision>

using build_utils.ChromeVersion()

Might be nice to be consistent here if we can.

[elijahtaylor1, 2012/04/20 18:52:50]

I could be wrong, but I think maybe trunk.<rev> is a newer construction than
these scripts which might explain why these are different.  Fixed to be
consistent with trunk.<rev> for the continuous builds

+  elif buildername == 'linux-sdk-mono64':
+    infos = get_sdk_build_info()
+    for info in infos:
+      build_and_upload_mono(None, info['pepper_revision'], info['sdk_url'],
+          info['naclmono_name'] + '/' + info['naclmono_rev'])
+    update_mono_sdk_json(infos)
+    
+
+
 if __name__ == '__main__':
   sys.exit(main(sys.argv))