Pull toolchain isos from gs:// when on bots

win_toolchain/toolchain2013.py

Index: win_toolchain/toolchain2013.py
diff --git a/win_toolchain/toolchain2013.py b/win_toolchain/toolchain2013.py
index b1b389f54882acf1025a4d7488f91327eb00a6dd..f90fb75e0bd770151f1dcda70ee6b4cadfa21b6b 100755
--- a/win_toolchain/toolchain2013.py
+++ b/win_toolchain/toolchain2013.py
@@ -56,7 +56,9 @@ def DeleteAllTempDirs():
 def GetIsoUrl(pro):
   """Gets the .iso URL.
 
-  If |pro| is False, downloads the Express edition.
+  If |pro| is False, downloads the Express edition. If |CHROME_HEADLESS| is
+  set in the environment, then we assume we're on an internal bot, and download
+  from internal google storage instead.
   """
   prefix = 'http://download.microsoft.com/download/'
   if pro:
@@ -153,6 +155,32 @@ def DownloadSDK8():
   raise SystemExit("After multiple retries, couldn't download Win8 SDK")
 
 
+def DownloadUsingGsutil(filename):
+  """Downloads the given file from Google Storage chrome-wintoolchain bucket."""
+  temp_dir = TempDir()
+  assert os.path.basename(filename) == filename
+  target_path = os.path.join(temp_dir, filename)
+  RunOrDie([

[Ryan Tseng, 2014/01/28 00:19:43]

can use 
import download_from_google_storage
...

gsutil = download_from_google_storage.Gsutil(
    download_from_google_storage.GSUTIL_DEFAULT_PATH)
gsutil.check_call('cp', '...', '...')
code, _, err = gsutil.check_call('cp', '-q', file_url, output_filename)
if code != 0:
  raise SystemExit('Gsutil error: %s' % err)

That allows this to deal with some .boto.depot_tools shenanigans that we do on
some of our bots, and centralizes gsutil calls within depot_tools through that
method.

[scottmg, 2014/01/28 00:38:28]

Thanks, done.

Is os.devnull OK for boto_path? That's the best guess I could find here:
https://code.google.com/p/chromium/codesearch#search/&q=download_from_google_...


FWIW, I would have misused Gsutil.check_call without your example code, as it
matches subprocess.check_call in name, but doesn't check != 0 and raise
CalledProcessError as subprocess does.

+    sys.executable,
+    os.path.join(BASEDIR, '..', 'third_party', 'gsutil', 'gsutil'),
+    'cp',
+    'gs://chrome-wintoolchain/' + filename,
+    target_path])
+  return target_path
+
+
+def GetVSInternal():
+  """Uses gsutil to pull the toolchain from internal Google Storage bucket."""
+  return DownloadUsingGsutil('VS2013_RTM_PRO_ENU.iso')
+
+
+def GetSDKInternal():
+  """Downloads a zipped copy of the SDK from internal Google Storage bucket,
+  and extracts it."""
+  zip_file = DownloadUsingGsutil('Standalone.zip')
+  return ExtractIso(zip_file)
+
+
 class SourceImages(object):
   def __init__(self, vs_path, sdk8_path):
     self.vs_path = vs_path
@@ -161,7 +189,9 @@ class SourceImages(object):
 
 def GetSourceImages(local_dir, pro):
   url = GetIsoUrl(pro)
-  if local_dir:
+  if os.environ.get('CHROME_HEADLESS'):
+    return SourceImages(GetVSInternal(), GetSDKInternal())
+  elif local_dir:
     return SourceImages(os.path.join(local_dir, os.path.basename(url)),
                         os.path.join(local_dir, 'Standalone'))
   else: