Sysroot: Add build_and_upload.py

build/linux/sysroot_scripts/build_and_upload.py

+#!/usr/bin/env python
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Automates running BuildPackageLists, BuildSysroot, and
+UploadSysroot for each supported arch of each sysroot creator.
+"""
+
+import glob
+import hashlib
+import json
+import multiprocessing
+import os
+import re
+import string
+import subprocess
+import sys
+
+def run_script(args):
+  fnull = open(os.devnull, 'w')
+  subprocess.check_call(args, stdout=fnull, stderr=fnull)
+
+def sha1sumfile(filename):
+  sha1 = hashlib.sha1()
+  with open(filename, 'rb') as f:
+    while True:
+      data = f.read(65536)
+      if not data:
+        break
+      sha1.update(data)
+  return sha1.hexdigest()
+
+def get_proc_output(args):
+  return subprocess.check_output(args).strip()
+
+def build_and_upload(script_path, distro, release, arch, lock):
+  # TODO(thomasanderson):  Find out which revision 'git-cl upload' uses to
+  # calculate the diff against and use that instead of HEAD.
+  script_dir = os.path.dirname(os.path.realpath(__file__))
+  revision = get_proc_output(['git', '-C', script_dir, 'rev-parse', 'HEAD'])
+
+  run_script([script_path, 'UpdatePackageLists%s' % arch])
+  run_script([script_path, 'BuildSysroot%s' % arch])
+  run_script([script_path, 'UploadSysroot%s' % arch, revision])
+
+  tarball = '%s_%s_%s_sysroot.tgz' % (distro, release, arch.lower())
+  tgz_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
+                          "..", "..", "..", "out", "sysroot-build",
+                          release, tarball)
+  sha1sum = sha1sumfile(tgz_path)
+  sysroot_dir = '%s_%s_%s-sysroot' % (distro, release, arch.lower())
+
+  sysroot_metadata = {
+      'Revision': revision,
+      'Tarball': tarball,
+      'Sha1Sum': sha1sum,
+      'SysrootDir': sysroot_dir
+  }
+  with lock:
+    with open('sysroots.json', 'rw+') as f:
+      sysroots = json.load(f)
+      sysroots["%s_%s" % (release, arch.lower())] = sysroot_metadata
+      f.seek(0)
+      f.truncate()
+      f.write(json.dumps(sysroots, sort_keys=True, indent=4,
+                         separators=(',', ': ')))
+      f.write('\n')
+
+def build_and_upload_all():
+  script_dir = os.path.dirname(os.path.realpath(__file__))
+  procs = []
+  lock = multiprocessing.Lock()
+  for filename in glob.glob(os.path.join(script_dir, 'sysroot-creator-*.sh')):
+    script_path = os.path.join(script_dir, filename)
+    distro = get_proc_output([script_path, 'PrintDistro'])
+    release = get_proc_output([script_path, 'PrintRelease'])
+    architectures = get_proc_output([script_path, 'PrintArchitectures'])
+    for arch in architectures.split('\n'):
+      proc = multiprocessing.Process(target=build_and_upload,
+                                     args=(script_path, distro, release, arch,
+                                           lock))
+      procs.append(("%s %s (%s)" % (distro, release, arch), proc))
+      proc.start()
+  for _, proc in procs:
+    proc.join()
+
+  print "SYSROOT CREATION SUMMARY"
+  for name, proc in procs:
+    status = "FAILURE" if proc.exitcode else "SUCCESS"
+    print "%s sysroot creation\t%s" % (name, status)

[Sam Clegg, 2016/12/13 00:30:37]

Shouldn't function return 1 if any proc.exitcode is non-zero? (and maybe just be
called main?)

[Tom (Use chromium acct), 2016/12/13 02:33:09]

Done.

+
+def main():
+  build_and_upload_all()
+  # If there were no exceptions, all went well.
+  return 0
+
+if __name__ == '__main__':
+  sys.exit(main())