Pull toolchain isos from gs:// when on bots

win_toolchain/toolchain2013.py

 #!/usr/bin/env python
 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Extracts a Windows VS2013 toolchain from various downloadable pieces."""
 
 
 import ctypes
 import optparse
 import os
 import shutil
 import subprocess
 import sys
 import tempfile
 import urllib2
 
 
 BASEDIR = os.path.dirname(os.path.abspath(__file__))
 g_temp_dirs = []
 
 
+sys.path.append(os.path.join(BASEDIR, '..'))

[iannucci, 2014/01/28 16:38:33]

ARGH! THE PAIN!

(no there's not a better alternative, but that doesn't make it less painful
:((((()

[scottmg, 2014/01/28 17:36:35]

Our 'environment' could use "some work". :(

+import download_from_google_storage
+
+
 def GetLongPathName(path):
   """Converts any 8dot3 names in the path to the full name."""
   buf = ctypes.create_unicode_buffer(260)
   size = ctypes.windll.kernel32.GetLongPathNameW(unicode(path), buf, 260)
   if (size > 260):
     sys.exit('Long form of path longer than 260 chars: %s' % path)
   return buf.value
 
 
 def RunOrDie(command):
@@ skipping 16 matching lines @@
     sys.stdout.write('Cleaning up temporaries...\n')
   for temp in g_temp_dirs:
     # shutil.rmtree errors out on read only attributes.
     RunOrDie('rmdir /s/q "%s"' % temp)
   g_temp_dirs = []
 
 
 def GetIsoUrl(pro):
   """Gets the .iso URL.
 
-  If |pro| is False, downloads the Express edition.
+  If |pro| is False, downloads the Express edition. If |CHROME_HEADLESS| is
+  set in the environment, then we assume we're on an internal bot, and download
+  from internal google storage instead.
   """
   prefix = 'http://download.microsoft.com/download/'
   if pro:
     return (prefix +
         'A/F/1/AF128362-A6A8-4DB3-A39A-C348086472CC/VS2013_RTM_PRO_ENU.iso')
   else:
     return (prefix +
         '7/2/E/72E0F986-D247-4289-B9DC-C4FB07374894/VS2013_RTM_DskExp_ENU.iso')
 
 
@@ skipping 76 matching lines @@
     rc = os.system(target_path + ' /quiet '
                    '/features OptionId.WindowsDesktopDebuggers '
                    '/layout ' + standalone_path)
     if rc == 0:
       return standalone_path
     count += 1
     sys.stdout.write('Windows 8 SDK failed to download, retrying.\n')
   raise SystemExit("After multiple retries, couldn't download Win8 SDK")
 
 
+def DownloadUsingGsutil(filename):
+  """Downloads the given file from Google Storage chrome-wintoolchain bucket."""
+  temp_dir = TempDir()

[iannucci, 2014/01/28 16:38:33]

I really wish that TempDir was a contextlib.contextmanager that deleted the
directory... Right now this dir will always leak (especially b/c of the exit
later in this function).

Another way to do it would be to do:

TEMP_DIR = None

def TempDir():
  if TEMP_DIR is None:
    TEMP_DIR = mkdtemp
    atexit.register(nuke_it_dead, TEMP_DIR)
  return os.path.join(TEMP_DIR, <random junk>)

[scottmg, 2014/01/28 17:36:35]

Could you explain why it will leak? Any finally blocks (main) will still
execute, surely?

+  assert os.path.basename(filename) == filename
+  target_path = os.path.join(temp_dir, filename)
+  gsutil = download_from_google_storage.Gsutil(
+      download_from_google_storage.GSUTIL_DEFAULT_PATH, boto_path=None)
+  code = gsutil.call('cp', 'gs://chrome-wintoolchain/' + filename, target_path)
+  if code != 0:
+    raise SystemExit('gsutil failed')

[iannucci, 2014/01/28 16:38:33]

sys.exit('gsutil failed')

[scottmg, 2014/01/28 17:36:35]

I guess? https://codereview.chromium.org/148523009

http://hg.python.org/cpython/file/ca5431a434d6/Python/sysmodule.c#l207

http://docs.python.org/2/library/sys.html#sys.exit

+  return target_path
+
+
+def GetVSInternal():
+  """Uses gsutil to pull the toolchain from internal Google Storage bucket."""
+  return DownloadUsingGsutil('VS2013_RTM_PRO_ENU.iso')
+
+
+def GetSDKInternal():
+  """Downloads a zipped copy of the SDK from internal Google Storage bucket,
+  and extracts it."""
+  zip_file = DownloadUsingGsutil('Standalone.zip')
+  return ExtractIso(zip_file)
+
+
 class SourceImages(object):
   def __init__(self, vs_path, sdk8_path):
     self.vs_path = vs_path
     self.sdk8_path = sdk8_path
 
 
 def GetSourceImages(local_dir, pro):
   url = GetIsoUrl(pro)
-  if local_dir:
+  if os.environ.get('CHROME_HEADLESS'):
+    return SourceImages(GetVSInternal(), GetSDKInternal())
+  elif local_dir:
     return SourceImages(os.path.join(local_dir, os.path.basename(url)),
                         os.path.join(local_dir, 'Standalone'))
   else:
     # Note that we do the SDK first, as it might cause an elevation prompt.
     sdk8_path = DownloadSDK8()
     vs_path = DownloadMainIso(url)
     return SourceImages(vs_path, sdk8_path)
 
 
 def ExtractMsiList(root_dir, packages):
@@ skipping 159 matching lines @@
     CopyToFinalLocation(extracted, target_dir)
 
     GenerateSetEnvCmd(target_dir, not options.express)
   finally:
     if options.clean:
       DeleteAllTempDirs()
 
 
 if __name__ == '__main__':
   sys.exit(main())