tool: add script that uploads go tool binaries

tools/go/upload.py

+#!/usr/bin/python
+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+This script downloads GO_VERSION linux binaries from golang.org, build android
+binaries with NDK tool chain configured with NDK_PLATFORM and NDK_TOOLCHAIN
+parameters, zips the stuff and uploads it to Google Cloud Storage at
+gs://mojo/go/tool. It also produces VERSION file with sha1 code of the uploaded
+archive.
+
+In order to use it, you need:
+- depot_tools in your path
+- installed android build deps
+- WRITE access to GCS
+
+To update go tool binaries you need to
+1) run 'gsutil.py config' to update initialize gsutil's credentials
+2) run this script
+3) push new version of file 'VERSION'
+
+This script doesn't check if current version is already up to date, as the
+produced tar.gz archive is slightly different every time since it includes
+timestamps.
+"""
+
+import hashlib
+import os
+import shutil
+import subprocess
+import sys
+import tarfile
+import tempfile
+
+GO_VERSION = 'go1.4.2'
+NDK_PLATFORM = 'android-14'
+NDK_TOOLCHAIN = 'arm-linux-androideabi-4.9'
+
+# Path constants. (All of these should be absolute paths.)
+THIS_DIR = os.path.abspath(os.path.dirname(__file__))
+MOJO_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..'))
+
+
+def RunCommand(command, env=os.environ):

[viettrungluu, 2015/04/08 21:54:19]

The default value for subprocess.call/Popen's env argument is None, so you
should probably do that too.

Doing env=os.environ binds to the object named by os.environ "here", so it'd be
surprising if anyone ever did os.environ = ... elsewhere and then called this
(the default argument would be the old os.environ object, not the one at the
time of the call).

[rogulenko, 2015/04/08 22:33:19]

Done.

+  """Run command and return success (True) or failure."""
+
+  print 'Running %s' % (str(command))
+  if subprocess.call(command, shell=False, env=env) == 0:
+    return True
+  print 'Failed.'
+  return False
+
+def DownloadGoLinux(tmpdir):
+  """Downloads and go linux_amd64 go binaries from golang.org and extracts them
+     to provided tmpdir. The extracted files appear under
+     tmpdir/tool/linux_amd64."""
+
+  archive_name = '%s.linux-amd64.tar.gz' % GO_VERSION
+  tool_dir = os.path.join(tmpdir, 'tool')
+  os.mkdir(tool_dir)
+  archive_path = os.path.join(tool_dir, archive_name)
+  # Download the Linux x64 Go binaries from golang.org.
+  # '-C -': Resume transfer if possible.
+  # '--location': Follow Location: redirects.
+  # '-o': Output file.
+  curl_command = ['curl',
+                  '-C', '-',
+                  '--location',
+                  '-o', archive_path,
+                  'http://golang.org/dl/%s' % archive_name]
+  if not RunCommand(curl_command):
+    print "Failed to linux go binaries from server."
+    sys.exit(1)
+  with tarfile.open(archive_path) as arch:
+    arch.extractall(tool_dir)
+  os.remove(archive_path)
+  os.rename(os.path.join(tool_dir, 'go'),
+            os.path.join(tool_dir, 'linux_amd64'))
+
+def ConfigureAndroidNDK(tmpdir):
+  """Creates a standalone tool chain under tmpdir. Returns path to ndk cc
+     compiler."""
+
+  ndk_root = os.path.join(tmpdir, 'ndk-toolchain')
+  os.chdir(os.path.join(MOJO_DIR, 'third_party', 'android_tools', 'ndk'))
+  script = os.path.join('build', 'tools', 'make-standalone-toolchain.sh')
+  make_command = ['bash', script,
+                  '--platform=%s' % NDK_PLATFORM,
+                  '--toolchain=%s' % NDK_TOOLCHAIN,
+                  '--install-dir=%s' % ndk_root]
+  if not RunCommand(make_command):
+    print "Failed to generate NDK tool chain."
+    sys.exit(1)
+  return os.path.join(ndk_root, 'bin', 'arm-linux-androideabi-gcc')
+
+def BuildGoAndroid(tmpdir):
+  ndk_cc = ConfigureAndroidNDK(tmpdir)
+  go_linux = os.path.join(tmpdir, 'tool', 'linux_amd64')
+  go_android = os.path.join(tmpdir, 'tool', 'android_arm')
+  # Copy go sources and remove binaries to keep only that we generate.
+  shutil.copytree(go_linux, go_android)
+  shutil.rmtree(os.path.join(go_android, 'bin'))
+  shutil.rmtree(os.path.join(go_android, 'pkg'))
+  # Configure environment variables.
+  env = os.environ.copy()
+  env["GOPATH"] = go_linux
+  env["GOROOT"] = go_linux
+  env["CC_FOR_TARGET"] = ndk_cc
+  env["GOOS"] = 'android'
+  env["GOARCH"] = 'arm'
+  env["GOARM"] = '7'
+  # Build go tool.
+  os.chdir(os.path.join(go_android, 'src'))
+  make_command = ['bash', 'make.bash']
+  if not RunCommand(make_command, env=env):
+    print "Failed to make go tool for android."
+    sys.exit(1)
+
+def Compress(tmpdir):
+  """Compresses the go tool into tar.gz and generates sha1 code, renames the
+     archive to sha1.tar.gz and returns the sha1 code."""
+
+  print "Compressing go tool, this may take several minutes."
+  os.chdir(os.path.join(tmpdir, 'tool'))
+  with tarfile.open(os.path.join('..', 'a.tar.gz'), 'w|gz') as arch:
+    arch.add('android_arm')
+    arch.add('linux_amd64')
+
+  sha1 = ''
+  with open(os.path.join(tmpdir, 'a.tar.gz')) as f:
+    sha1 = hashlib.sha1(f.read()).hexdigest()
+  os.rename(os.path.join(tmpdir, 'a.tar.gz'),
+            os.path.join(tmpdir, '%s.tar.gz' % sha1))
+  return sha1
+
+def Upload(tmpdir, sha1):
+  """Uploads tmpdir/sha1.tar.gz to Google Cloud Storage under gs://mojo/go/tool
+     and writes sha1 to THIS_DIR/VERSION."""
+
+  stamp_file = os.path.join(THIS_DIR, 'VERSION')
+  with open(stamp_file, 'w+') as stamp:
+    stamp.write('%s\n' % sha1)
+  file_name = '%s.tar.gz' % sha1
+  upload_cmd = ['gsutil.py', 'cp',
+                '-n', # Do not upload if the file already exists.
+                os.path.join(tmpdir, file_name),
+                'gs://mojo/go/tool/%s' % file_name]
+
+  print "Uploading go tool to GCS."
+  if not RunCommand(upload_cmd):
+    print "Failed to upload go tool to GCS."
+    sys.exit(1)
+
+def main():
+  tmpdir = tempfile.mkdtemp(prefix='mojo')
+  print "Created temp dir %s" % tmpdir
+  DownloadGoLinux(tmpdir)
+  BuildGoAndroid(tmpdir)
+  sha1 = Compress(tmpdir)
+  Upload(tmpdir, sha1)
+  shutil.rmtree(tmpdir)
+  print "Done."
+
+if __name__ == '__main__':
+  sys.exit(main())