Scripts to upload and update the mac toolchain.

build/mac_toolchain.py

+#!/usr/bin/env python
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Download necessary mac toolchain files under certain conditions.  If
+xcode-select is already set and points to an external folder
+(e.g. /Application/Xcode.app), this script only runs if the GYP_DEFINE
+|force_mac_toolchain| is set.  To override the values in
+|TOOLCHAIN_REVISION|-|TOOLCHAIN_SUB_REVISION| below, GYP_DEFINE
+mac_toolchain_revision can be used instead.
+
+This script will only run on machines with the following programs added to a
+sudoers list:
+  /usr/bin/xcode-select
+  /usr/bin/xcodebuild
+
+Otherwise user input would be required to complete the script.  Perhaps future
+versions can be modified to allow for user input on developer machines.
+"""
+
+import os
+import shutil
+import subprocess
+import sys
+import tarfile
+import time
+import tempfile
+import urllib2
+
+# Update sys.path to import gyp.
+script_dir = os.path.dirname(os.path.realpath(__file__))
+chrome_src = os.path.abspath(os.path.join(script_dir, os.pardir))
+sys.path.insert(0, os.path.join(chrome_src, 'tools', 'gyp', 'pylib'))
+import gyp
+
+# This can be changed after running /build/package_mac_toolchain.py.
+TOOLCHAIN_REVISION = '7C68'
+TOOLCHAIN_SUB_REVISION = 1
+TOOLCHAIN_VERSION = "%s-%s" % (TOOLCHAIN_REVISION, TOOLCHAIN_SUB_REVISION)
+
+BASE_DIR = os.path.abspath(os.path.dirname(__file__))
+TOOLCHAIN_BUILD_DIR = os.path.join(BASE_DIR, 'mac_files', 'Xcode.app')
+STAMP_FILE = os.path.join(BASE_DIR, 'mac_files', 'toolchain_build_revision')
+TOOLCHAIN_URL = 'gs://chrome-mac-sdk/'
+
+def ReadStampFile():
+  """Return the contents of the stamp file, or '' if it doesn't exist."""
+  try:
+    with open(STAMP_FILE, 'r') as f:
+      return f.read().rstrip()
+  except IOError:
+    return ''
+
+
+def WriteStampFile(s):
+  """Write s to the stamp file."""
+  EnsureDirExists(os.path.dirname(STAMP_FILE))
+  with open(STAMP_FILE, 'w') as f:
+    f.write(s)
+    f.write('\n')
+
+
+def EnsureDirExists(path):
+  if not os.path.exists(path):
+    os.makedirs(path)
+
+
+def DownloadAndUnpack(url, output_dir):
+  """Decompresses |url| into a cleared |output_dir|."""
+  try:
+    temp_name = tempfile.mktemp(prefix='mac_toolchain')
+    print 'Downloading new toolchain.'
+    subprocess.check_call(['gsutil.py', 'cp', url, temp_name])
+    if os.path.exists(output_dir):
+      print 'Deleting old toolchain.'
+      shutil.rmtree(output_dir)
+    EnsureDirExists(output_dir)
+    print 'Unpacking new toolchain.'
+    tarfile.open(mode='r:gz', name=temp_name).extractall(path=output_dir)
+  finally:
+    if os.path.exists(temp_name):
+      os.unlink(temp_name)
+
+
+def CanAccessToolchainBucket():
+  """Checks whether the user has access to |TOOLCHAIN_URL|."""
+  proc = subprocess.Popen(['gsutil.py', 'ls', TOOLCHAIN_URL],
+                           stdout=subprocess.PIPE)
+  proc.communicate()
+  return proc.returncode == 0
+
+
+def SwitchToolchain(directory):
+  """Use xcode-select to select new toolchain and accept license.  This only
+  works if both xcode-select and xcodebuild are in sudoers."""
+  print 'Setting xcode-select to %s.' % directory
+  subprocess.check_call(['sudo', '/usr/bin/xcode-select', '-s', directory])

[Nico, 2016/03/08 18:48:46]

won't this require an auth password usually?

[justincohen, 2016/03/08 19:53:27]

Correct.  We check below that (sudo -l) to make sure we can sudo for
xcode-select and xcodebuild.  The bots can be configured to allow this without
entering a password.

+  subprocess.check_call(['sudo', '/usr/bin/xcodebuild', '-license', 'accept'])
+
+
+def main():
+  gyp_defines = gyp.NameValueListToDict(gyp.ShlexEnv('GYP_DEFINES'))

[Nico, 2016/03/08 18:48:46]

(https://bugs.chromium.org/p/chromium/issues/detail?id=570091 tracks on what to
do here wrt gn)

[justincohen, 2016/03/08 19:53:27]

Added a TODO

+  force_pull = 'force_mac_toolchain' in gyp_defines
+  toolchain_revision = gyp_defines.get('mac_toolchain_revision',
+                                       TOOLCHAIN_VERSION)
+
+  # Don't update the toolchain if there's already one installed outside of the
+  # expected location for a Chromium mac toolchain, unless |force_pull| is set.
+  proc = subprocess.Popen(['xcode-select', '-p'], stdout=subprocess.PIPE)
+  xcode_select_dir = proc.communicate()[0]
+  rc = proc.returncode
+  if not force_pull and rc == 0 and TOOLCHAIN_BUILD_DIR not in xcode_select_dir:

[Nico, 2016/03/08 18:48:46]

should this check that the local toolchain has the right version? as-is, from
what i understand this won't do anything if xcode 3.2 is installed, which seems
suboptimal

[justincohen, 2016/03/08 19:53:27]

I wasn't sure what the right functionality should be here.  Since xcode-select
will be system wide, last beyond reboot, etc. I didn't want to break old system.
 Ideally bots that are ready for this will set the GYP_DEFINE
force_mac_toolchain.

+    print 'Using local toolchain.'
+    return 0
+
+  if ReadStampFile() == toolchain_revision:
+    print 'Toolchain (%s) is already up to date.' % toolchain_revision
+    SwitchToolchain(TOOLCHAIN_BUILD_DIR)
+    return 0
+
+  # This script cannot complete if xcode-select and xcodebuild are not in
+  # sudoers.
+  proc = subprocess.Popen(['sudo', '-l'], stdout=subprocess.PIPE)
+  output = proc.communicate()[0]
+  cmds = ['xcode-select', 'xcodebuild']
+  if not all(x in output for x in cmds):
+    print 'xcode-select and xcodebuild must be present in sudoers.'

[justincohen, 2016/03/08 19:53:27]

Missed a return 0 here.

+
+  if not CanAccessToolchainBucket():
+    print 'Cannot access toolchain bucket.'
+    return 0
+
+  # Reset the stamp file in case the build is unsuccessful.
+  WriteStampFile('')
+
+  toolchain_file = "%s.tgz" % toolchain_revision
+  toolchain_full_url = TOOLCHAIN_URL + toolchain_file
+
+  print 'Updating toolchain to %s...' % toolchain_revision
+  try:
+    toolchain_file = "toolchain-%s.tgz" % toolchain_revision
+    toolchain_full_url = TOOLCHAIN_URL + toolchain_file
+    DownloadAndUnpack(toolchain_full_url, TOOLCHAIN_BUILD_DIR)
+    SwitchToolchain(TOOLCHAIN_BUILD_DIR)
+
+    print 'Toolchain %s unpacked.' % toolchain_revision
+    WriteStampFile(toolchain_revision)
+
+    return 0
+  except:
+    print 'Failed to download toolchain %s.' % toolchain_file
+    print 'Exiting.'
+    return 1
+
+if __name__ == '__main__':
+  sys.exit(main())