Sysroot: Add build_and_upload.py

build/linux/sysroot_scripts/build_and_upload.py

+#!/usr/bin/env python
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Automates running BuildPackageLists, BuildSysroot, and
+UploadSysroot for each supported arch of each sysroot creator.
+"""
+
+import hashlib
+import json
+import multiprocessing
+import os
+import re
+import string
+import subprocess
+import sys
+
+def run_script(args):
+  fnull = open(os.devnull, 'w')
+  subprocess.check_call(args, stdout=fnull, stderr=fnull)
+
+def sha1sumfile(filename):
+  sha1 = hashlib.sha1()
+  with open(filename, 'rb') as f:
+    while True:
+      data = f.read(65536)
+      if not data:
+        break
+      sha1.update(data)
+  return sha1.hexdigest()
+
+def get_proc_output(args):
+  return subprocess.check_output(args).strip()
+
+def build_and_upload(script_path, distro, release, arch, lock):
+  # TODO(thomasanderson):  Find out which revision 'git-cl upload' uses to
+  # calculate the diff against and use that instead of HEAD.
+  script_dir = os.path.dirname(os.path.realpath(__file__))
+  revision = get_proc_output(['git', '-C', script_dir, 'rev-parse', 'HEAD'])
+
+  run_script([script_path, 'UpdatePackageLists%s' % arch])
+  run_script([script_path, 'BuildSysroot%s' % arch])
+  run_script([script_path, 'UploadSysroot%s' % arch, revision])
+
+  tarball = '%s_%s_%s_sysroot.tgz' % (distro, release, arch.lower())
+  tgz_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
+                          "..", "..", "..", "out", "sysroot-build",
+                          release, tarball)
+  sha1sum = sha1sumfile(tgz_path)
+  sysroot_dir = '%s_%s_%s-sysroot' % (distro, release, arch.lower())
+
+  sysroot_metadata = {
+      'Revision': revision,
+      'Tarball': tarball,
+      'Sha1Sum': sha1sum,
+      'SysrootDir': sysroot_dir
+  }
+  lock.acquire()
+  try:

[Sam Clegg, 2016/12/12 22:11:52]

I seems like "with lock:" should work here.

[Tom (Use chromium acct), 2016/12/13 00:06:19]

Done.

+    with open('sysroots.json', 'rw+') as f:
+      sysroots = json.load(f)
+      sysroots["%s_%s" % (release, arch.lower())] = sysroot_metadata
+      f.seek(0)
+      f.truncate()
+      f.write(json.dumps(sysroots, sort_keys=True, indent=4))
+      f.write('\n')
+      f.close()

[Sam Clegg, 2016/12/12 22:11:52]

No need with close here right?

[Tom (Use chromium acct), 2016/12/13 00:06:19]

Done.

+  finally:
+    lock.release()
+
+def build_and_upload_all():
+  script_dir = os.path.dirname(os.path.realpath(__file__))
+  procs = []
+  lock = multiprocessing.Lock()
+  for filename in os.listdir(script_dir):
+    if (not os.path.isfile(os.path.join(script_dir, filename)) or
+        not re.search(r'sysroot-creator-.+\.sh', filename)):
+      continue

[Sam Clegg, 2016/12/12 22:11:52]

How about just:

for filename in glob.glob(os.path.join(script_dir, 'sysroot-creator-*.sh')):

Or even just hardcode the list at the top of the file, since there should really
only be one at a given time (hopefully we will be dropping both trust and wheezy
ASAP)?

[Tom (Use chromium acct), 2016/12/13 00:06:19]

Done
Yeah, it seems with the switch to jessie some of the sysroots are going to
become obsolete.  I didn't hardcode the list so that we don't have to maintain
it

+    script_path = os.path.join(script_dir, filename)
+    distro = get_proc_output([script_path, 'PrintDistro'])
+    release = get_proc_output([script_path, 'PrintRelease'])
+    architectures = get_proc_output([script_path, 'PrintArchitectures'])
+    for arch in architectures.split('\n'):
+      proc = multiprocessing.Process(target=build_and_upload,
+                                     args=(script_path, distro, release, arch,
+                                           lock))
+      procs.append(("%s %s (%s)" % (distro, release, arch), proc))
+      proc.start()
+  for (_, proc) in procs:
+    proc.join()
+
+  print "SYSROOT CREATION SUMMARY"
+  for (name, proc) in procs:

[Sam Clegg, 2016/12/12 22:11:51]

nit: no need to braces when unpacking tuples like this (and 3 lines above)

[Tom (Use chromium acct), 2016/12/13 00:06:19]

Done.

+    status = "FAILURE" if proc.exitcode else "SUCCESS"
+    print "%s sysroot creation\t%s" % (name, status)
+
+if __name__ == '__main__':
+  build_and_upload_all()

[Sam Clegg, 2016/12/12 22:11:52]

The pattern I've prefer is "sys.exit(main())" here, and return the status code
from main.

[Tom (Use chromium acct), 2016/12/13 00:06:19]

Done.