Pull toolchain isos from gs:// when on bots

win_toolchain/toolchain2013.py

 #!/usr/bin/env python
 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Extracts a Windows VS2013 toolchain from various downloadable pieces."""
 
 
 import ctypes
 import optparse
@@ skipping 38 matching lines @@
     sys.stdout.write('Cleaning up temporaries...\n')
   for temp in g_temp_dirs:
     # shutil.rmtree errors out on read only attributes.
     RunOrDie('rmdir /s/q "%s"' % temp)
   g_temp_dirs = []
 
 
 def GetIsoUrl(pro):
   """Gets the .iso URL.
 
-  If |pro| is False, downloads the Express edition.
+  If |pro| is False, downloads the Express edition. If |CHROME_HEADLESS| is
+  set in the environment, then we assume we're on an internal bot, and download
+  from internal google storage instead.
   """
   prefix = 'http://download.microsoft.com/download/'
   if pro:
     return (prefix +
         'A/F/1/AF128362-A6A8-4DB3-A39A-C348086472CC/VS2013_RTM_PRO_ENU.iso')
   else:
     return (prefix +
         '7/2/E/72E0F986-D247-4289-B9DC-C4FB07374894/VS2013_RTM_DskExp_ENU.iso')
 
 
@@ skipping 76 matching lines @@
     rc = os.system(target_path + ' /quiet '
                    '/features OptionId.WindowsDesktopDebuggers '
                    '/layout ' + standalone_path)
     if rc == 0:
       return standalone_path
     count += 1
     sys.stdout.write('Windows 8 SDK failed to download, retrying.\n')
   raise SystemExit("After multiple retries, couldn't download Win8 SDK")
 
 
+def DownloadUsingGsutil(filename):
+  """Downloads the given file from Google Storage chrome-wintoolchain bucket."""
+  temp_dir = TempDir()
+  assert os.path.basename(filename) == filename
+  target_path = os.path.join(temp_dir, filename)
+  RunOrDie([

[Ryan Tseng, 2014/01/28 00:19:43]

can use 
import download_from_google_storage
...

gsutil = download_from_google_storage.Gsutil(
    download_from_google_storage.GSUTIL_DEFAULT_PATH)
gsutil.check_call('cp', '...', '...')
code, _, err = gsutil.check_call('cp', '-q', file_url, output_filename)
if code != 0:
  raise SystemExit('Gsutil error: %s' % err)

That allows this to deal with some .boto.depot_tools shenanigans that we do on
some of our bots, and centralizes gsutil calls within depot_tools through that
method.

[scottmg, 2014/01/28 00:38:28]

Thanks, done.

Is os.devnull OK for boto_path? That's the best guess I could find here:
https://code.google.com/p/chromium/codesearch#search/&q=download_from_google_...


FWIW, I would have misused Gsutil.check_call without your example code, as it
matches subprocess.check_call in name, but doesn't check != 0 and raise
CalledProcessError as subprocess does.

+    sys.executable,
+    os.path.join(BASEDIR, '..', 'third_party', 'gsutil', 'gsutil'),
+    'cp',
+    'gs://chrome-wintoolchain/' + filename,
+    target_path])
+  return target_path
+
+
+def GetVSInternal():
+  """Uses gsutil to pull the toolchain from internal Google Storage bucket."""
+  return DownloadUsingGsutil('VS2013_RTM_PRO_ENU.iso')
+
+
+def GetSDKInternal():
+  """Downloads a zipped copy of the SDK from internal Google Storage bucket,
+  and extracts it."""
+  zip_file = DownloadUsingGsutil('Standalone.zip')
+  return ExtractIso(zip_file)
+
+
 class SourceImages(object):
   def __init__(self, vs_path, sdk8_path):
     self.vs_path = vs_path
     self.sdk8_path = sdk8_path
 
 
 def GetSourceImages(local_dir, pro):
   url = GetIsoUrl(pro)
-  if local_dir:
+  if os.environ.get('CHROME_HEADLESS'):
+    return SourceImages(GetVSInternal(), GetSDKInternal())
+  elif local_dir:
     return SourceImages(os.path.join(local_dir, os.path.basename(url)),
                         os.path.join(local_dir, 'Standalone'))
   else:
     # Note that we do the SDK first, as it might cause an elevation prompt.
     sdk8_path = DownloadSDK8()
     vs_path = DownloadMainIso(url)
     return SourceImages(vs_path, sdk8_path)
 
 
 def ExtractMsiList(root_dir, packages):
@@ skipping 159 matching lines @@
     CopyToFinalLocation(extracted, target_dir)
 
     GenerateSetEnvCmd(target_dir, not options.express)
   finally:
     if options.clean:
       DeleteAllTempDirs()
 
 
 if __name__ == '__main__':
   sys.exit(main())