Pull toolchain isos from gs:// when on bots

win_toolchain/toolchain2013.py

Index: win_toolchain/toolchain2013.py
diff --git a/win_toolchain/toolchain2013.py b/win_toolchain/toolchain2013.py
index b1b389f54882acf1025a4d7488f91327eb00a6dd..4b14214f551274ae0c91c11d7818a9bb74bb0727 100755
--- a/win_toolchain/toolchain2013.py
+++ b/win_toolchain/toolchain2013.py
@@ -20,6 +20,10 @@ BASEDIR = os.path.dirname(os.path.abspath(__file__))
 g_temp_dirs = []
 
 
+sys.path.append(os.path.join(BASEDIR, '..'))

[iannucci, 2014/01/28 16:38:33]

ARGH! THE PAIN!

(no there's not a better alternative, but that doesn't make it less painful
:((((()

[scottmg, 2014/01/28 17:36:35]

Our 'environment' could use "some work". :(

+import download_from_google_storage
+
+
 def GetLongPathName(path):
   """Converts any 8dot3 names in the path to the full name."""
   buf = ctypes.create_unicode_buffer(260)
@@ -56,7 +60,9 @@ def DeleteAllTempDirs():
 def GetIsoUrl(pro):
   """Gets the .iso URL.
 
-  If |pro| is False, downloads the Express edition.
+  If |pro| is False, downloads the Express edition. If |CHROME_HEADLESS| is
+  set in the environment, then we assume we're on an internal bot, and download
+  from internal google storage instead.
   """
   prefix = 'http://download.microsoft.com/download/'
   if pro:
@@ -153,6 +159,31 @@ def DownloadSDK8():
   raise SystemExit("After multiple retries, couldn't download Win8 SDK")
 
 
+def DownloadUsingGsutil(filename):
+  """Downloads the given file from Google Storage chrome-wintoolchain bucket."""
+  temp_dir = TempDir()

[iannucci, 2014/01/28 16:38:33]

I really wish that TempDir was a contextlib.contextmanager that deleted the
directory... Right now this dir will always leak (especially b/c of the exit
later in this function).

Another way to do it would be to do:

TEMP_DIR = None

def TempDir():
  if TEMP_DIR is None:
    TEMP_DIR = mkdtemp
    atexit.register(nuke_it_dead, TEMP_DIR)
  return os.path.join(TEMP_DIR, <random junk>)

[scottmg, 2014/01/28 17:36:35]

Could you explain why it will leak? Any finally blocks (main) will still
execute, surely?

+  assert os.path.basename(filename) == filename
+  target_path = os.path.join(temp_dir, filename)
+  gsutil = download_from_google_storage.Gsutil(
+      download_from_google_storage.GSUTIL_DEFAULT_PATH, boto_path=None)
+  code = gsutil.call('cp', 'gs://chrome-wintoolchain/' + filename, target_path)
+  if code != 0:
+    raise SystemExit('gsutil failed')

[iannucci, 2014/01/28 16:38:33]

sys.exit('gsutil failed')

[scottmg, 2014/01/28 17:36:35]

I guess? https://codereview.chromium.org/148523009

http://hg.python.org/cpython/file/ca5431a434d6/Python/sysmodule.c#l207

http://docs.python.org/2/library/sys.html#sys.exit

+  return target_path
+
+
+def GetVSInternal():
+  """Uses gsutil to pull the toolchain from internal Google Storage bucket."""
+  return DownloadUsingGsutil('VS2013_RTM_PRO_ENU.iso')
+
+
+def GetSDKInternal():
+  """Downloads a zipped copy of the SDK from internal Google Storage bucket,
+  and extracts it."""
+  zip_file = DownloadUsingGsutil('Standalone.zip')
+  return ExtractIso(zip_file)
+
+
 class SourceImages(object):
   def __init__(self, vs_path, sdk8_path):
     self.vs_path = vs_path
@@ -161,7 +192,9 @@ class SourceImages(object):
 
 def GetSourceImages(local_dir, pro):
   url = GetIsoUrl(pro)
-  if local_dir:
+  if os.environ.get('CHROME_HEADLESS'):
+    return SourceImages(GetVSInternal(), GetSDKInternal())
+  elif local_dir:
     return SourceImages(os.path.join(local_dir, os.path.basename(url)),
                         os.path.join(local_dir, 'Standalone'))
   else: