Add toolchain simple mirror option used for bots

win_toolchain/toolchain2013.py

 #!/usr/bin/env python
 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Extracts a Windows VS2013 toolchain from various downloadable pieces."""
 
 
 import ctypes
 import json
@@ skipping 230 matching lines @@
 
 class SourceImages(object):
   """Local paths for components. |wdk_path| may be None if it's unnecessary for
   the given configuration."""
   def __init__(self, vs_path, sdk8_path, wdk_path):
     self.vs_path = vs_path
     self.sdk8_path = sdk8_path
     self.wdk_path = wdk_path
 
 
-def GetSourceImages(local_dir, pro, bot_mode):
+def GetSourceImages(local_dir, pro):
   """Downloads the various sources that we need.
 
   Of note: Because Express does not include ATL, there's an additional download
   of the 7.1 WDK which is the latest publically accessible source for ATL. When
   |pro| this is not necessary (and CHROME_HEADLESS always implies Pro).
   """
   url = GetMainIsoUrl(pro)
-  if bot_mode:
-    return SourceImages(GetVSInternal(), GetSDKInternal(), wdk_path=None)
-  elif local_dir:
+  if local_dir:
     wdk_path = (os.path.join(local_dir, os.path.basename(WDK_ISO_URL))
                 if not pro else None)
     return SourceImages(os.path.join(local_dir, os.path.basename(url)),
                         os.path.join(local_dir, 'Standalone'),
                         wdk_path=wdk_path)
   else:
     # Note that we do the SDK first, as it might cause an elevation prompt.
     sdk8_path = DownloadSDK8()
     vs_path = DownloadMainIso(url)
     wdk_path = DownloadWDKIso() if not pro else None
@@ skipping 146 matching lines @@
       # x64 native.
       f.write(':x64\n'
               'set PATH=%~dp0..\\..\\win8sdk\\bin\\x64;'
                  '%~dp0..\\..\\VC\\bin\\amd64;'
                  '%PATH%\n')
     f.write('set LIB=%~dp0..\\..\\VC\\lib\\amd64;'
                '%~dp0..\\..\\win8sdk\\Lib\\win8\\um\\x64;'
                '%~dp0..\\..\\VC\\atlmfc\\lib\\amd64\n')
 
 
+def DoTreeMirror(target_dir, tree_sha1):
+  """In order to save temporary space on bots that do not have enough space to
+  download ISOs, unpack them, and copy to the target location, the whole tree
+  is uploaded as a zip to internal storage, and then mirrored here."""
+  local_zip = DownloadUsingGsutil(tree_sha1 + '.zip')
+  sys.stdout.write('Extracting %s...\n' % local_zip)
+  sys.stdout.flush()
+  RunOrDie('7z x "%s" -y "-o%s" >nul' % (local_zip, target_dir))
+
+
 def main():
   parser = optparse.OptionParser(description=sys.modules[__name__].__doc__)
   parser.add_option('--targetdir', metavar='DIR',
                     help='put toolchain into DIR',
                     default=os.path.join(BASEDIR, 'win_toolchain_2013'))
   parser.add_option('--noclean', action='store_false', dest='clean',
                     help='do not remove temp files',
                     default=True)
   parser.add_option('--local', metavar='DIR',
                     help='use downloaded files from DIR')
   parser.add_option('--express',
                     help='use VS Express instead of Pro', action='store_true')
+  parser.add_option('--sha1',
+                    help='tree sha1 that can be used to mirror an internal '
+                         'copy (used if --bot-mode)')
   parser.add_option('--bot-mode',
                     help='Use internal servers to pull isos',
                     default=bool(int(os.environ.get('CHROME_HEADLESS', 0))),
                     action='store_true')
   options, _ = parser.parse_args()
   try:
     target_dir = os.path.abspath(options.targetdir)
     if os.path.exists(target_dir):
       parser.error('%s already exists. Please [re]move it or use '
-                   '--targetdir to select a different target.\n' %
-                   target_dir)
+                  '--targetdir to select a different target.\n' %
+                  target_dir)
     # Set the working directory to 7z subdirectory. 7-zip doesn't find its
     # codec dll very well, so this is the simplest way to make sure it runs
     # correctly, as we don't otherwise care about working directory.
     os.chdir(os.path.join(BASEDIR, '7z'))
-    images = GetSourceImages(
-        options.local, not options.express, options.bot_mode)
-    extracted = ExtractComponents(images)
-    CopyToFinalLocation(extracted, target_dir)
+    if options.bot_mode and options.sha1:
+      DoTreeMirror(target_dir, options.sha1)
+    else:
+      images = GetSourceImages(options.local, not options.express)
+      extracted = ExtractComponents(images)
+      CopyToFinalLocation(extracted, target_dir)
+      GenerateSetEnvCmd(target_dir, not options.express)
 
-    GenerateSetEnvCmd(target_dir, not options.express)
     data = {
         'path': target_dir,
         'version': '2013e' if options.express else '2013',
         'win8sdk': os.path.join(target_dir, 'win8sdk'),
         'wdk': os.path.join(target_dir, 'wdk'),
         'runtime_dirs': [
           os.path.join(target_dir, 'sys64'),
           os.path.join(target_dir, 'sys32'),
         ],
     }
     with open(os.path.join(target_dir, '..', 'data.json'), 'w') as f:
       json.dump(data, f)
   finally:
     if options.clean:
       DeleteAllTempDirs()
 
 
 if __name__ == '__main__':
   sys.exit(main())