Make swarming work w/ GN (kinda)

tools/mb/mb.py

 #!/usr/bin/env python
 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """MB - the Meta-Build wrapper around GYP and GN
 
 MB is a wrapper script for GYP and GN that can be used to generate build files
 for sets of canned configurations and analyze them.
 """
 
 from __future__ import print_function
 
 import argparse
 import ast
 import json
 import os
 import pipes
+import pprint
 import shlex
 import shutil
 import sys
 import subprocess
 import tempfile
 
 
 def main(args):
   mbw = MetaBuildWrapper()
   mbw.ParseArgs(args)
@@ skipping 56 matching lines @@
                            'as a JSON object.')
     subp.set_defaults(func=self.CmdAnalyze)
 
     subp = subps.add_parser('gen',
                             help='generate a new set of build files')
     AddCommonOptions(subp)
     subp.add_argument('path', type=str, nargs=1,
                       help='path to generate build into')
     subp.set_defaults(func=self.CmdGen)
 
+    subp = subps.add_parser('isolate',
+                            help='build isolates')
+    AddCommonOptions(subp)
+    subp.add_argument('path', type=str, nargs=1,
+                      help='path build was generated into.')
+    subp.add_argument('input_path', nargs=1,
+                      help='path to a file containing the input arguments '
+                           'as a JSON object.')
+    subp.add_argument('output_path', nargs=1,
+                      help='path to a file containing the output arguments '
+                           'as a JSON object.')
+    subp.set_defaults(func=self.CmdIsolate)
+
     subp = subps.add_parser('lookup',
                             help='look up the command for a given config or '
                                  'builder')
     AddCommonOptions(subp)
     subp.set_defaults(func=self.CmdLookup)
 
     subp = subps.add_parser('validate',
                             help='validate the config file')
     AddCommonOptions(subp)
     subp.set_defaults(func=self.CmdValidate)
@@ skipping 17 matching lines @@
 
   def CmdGen(self):
     vals = self.GetConfig()
     if vals['type'] == 'gn':
       return self.RunGNGen(self.args.path[0], vals)
     if vals['type'] == 'gyp':
       return self.RunGYPGen(self.args.path[0], vals)
 
     raise MBErr('Unknown meta-build type "%s"' % vals['type'])
 
+  def CmdIsolate(self):
+    vals = self.GetConfig()
+    if vals['type'] == 'gn':
+      return self.RunGNIsolate(vals)
+    if vals['type'] == 'gyp':
+      return 0 # .isolated files are generated during the compile in GYP.

[M-A Ruel, 2015/06/03 15:47:18]

No, the .isolated.gen.json files are generated during the compile in GYP

[Dirk Pranke, 2015/06/03 17:15:49]

Will update. 

+    raise MBErr('Unknown meta-build type "%s"' % vals['type'])
+
   def CmdLookup(self):
     vals = self.GetConfig()
     if vals['type'] == 'gn':
       cmd = self.GNCmd('gen', '<path>', vals['gn_args'])
     elif vals['type'] == 'gyp':
       cmd = self.GYPCmd('<path>', vals['gyp_defines'], vals['gyp_config'])
     else:
       raise MBErr('Unknown meta-build type "%s"' % vals['type'])
 
     self.PrintCmd(cmd)
@@ skipping 218 matching lines @@
     ret, _, _ = self.Run(cmd)
     if not ret and self.args.verbose:
       outp = json.loads(self.ReadFile(self.args.output_path[0]))
       self.Print()
       self.Print('analyze output:')
       self.PrintJSON(inp)
       self.Print()
 
     return ret
 
+  def RunGNIsolate(self, vals):
+    build_path = self.args.path[0]
+    inp = self.GetIsolateInput()
+    if self.args.verbose:
+      self.Print()
+      self.Print('isolate input:')
+      self.PrintJSON(inp)
+      self.Print()
+    output_path = self.args.output_path[0]
+
+    for target in inp['targets']:
+      runtime_deps_path = self.ToAbsPath(build_path, target + '.runtime_deps')
+
+      if not self.Exists(runtime_deps_path):
+        self.WriteFailureAndRaise('"%s" does not exist' % runtime_deps_path,
+                                  output_path)
+
+      command, extra_files, read_only = self.GetIsolateCommand(target, vals)
+
+      runtime_deps = self.ReadFile(runtime_deps_path).splitlines()
+
+
+      isolate_path = self.ToAbsPath(build_path, target + '.isolate')
+      self.WriteFile(isolate_path,
+        pprint.pformat({
+          'variables': {
+            'command': command,
+            'files': sorted(runtime_deps + extra_files),
+            'read_only': read_only,
+          }
+        }) + '\n')
+
+      self.WriteJSON(
+        {
+          'args': [
+            '--isolated',
+            self.ToSrcRelPath('%s/%s.isolated' % (build_path, target)),
+            '--isolate',
+            self.ToSrcRelPath('%s/%s.isolate' % (build_path, target)),
+          ],
+          'dir': self.chromium_src_dir,
+          'version': 1,
+        },
+        isolate_path + '.gen.json',
+      )
+
+    return 0
+
+  def GetIsolateInput(self):

[M-A Ruel, 2015/06/03 15:47:19]

ReadIsolateInput ?
And it's not really an isolate, it's more LoadTargetsDescriptionFile() ?

[Dirk Pranke, 2015/06/03 17:15:50]

It was called GetIsolateInput because we're getting the input to the 'isolate'
command, but I can see the confusion. I'll see if I can come up w/ something
clearer.

+    path = self.args.input_path[0]
+    output_path = self.args.output_path[0]
+    if not self.Exists(path):
+      self.WriteFailureAndRaise('"%s" does not exist' % path, output_path)
+
+    try:
+      inp = json.loads(self.ReadFile(path))
+    except Exception as e:
+      self.WriteFailureAndRaise('Failed to read JSON input from "%s": %s' %
+                                (path, e), output_path)
+    if not 'targets' in inp:
+      self.WriteFailureAndRaise('input file is missing a "targets" key',
+                                output_path)
+
+    return inp
+
+  def GetIsolateCommand(self, target, vals):

[Dirk Pranke, 2015/06/03 02:24:54]

This whole function is dodgy and needs to be replaced with 
something more user friendly, but I don't know what that will
be (look like) until I can take a look at more .isolate files
and see what the appropriate set of command line templates
we need to support will be.

For now, I've only tested this w/ base_unittests on linux, and everything is
hardcoded, those in theory this *might* work as-is on win and mac (but probably
won't).

+    output_path = self.args.output_path[0]
+
+    extra_files = []
+
+    # TODO(dpranke): We should probably pull this from

[M-A Ruel, 2015/06/03 15:47:18]

Yes but obviously this only means that it works when driven through a recipe,
which is semi-fine. Is the recipe name/builder name available here? I guess not.

[Dirk Pranke, 2015/06/03 17:15:49]

Well, it would only work w/ things using the files in testing/buildbot, at
least. So far, 
using recipes is a requirement. I'm not sure yet if that'll stick all the way
through the 
migration.

The recipe name is not available, but the mastername and buildername are.

+    # the test list info in //testing/buildbot/*.json,
+    # and assert that the test has can_use_on_swarming_builders: True,
+    # but we hardcode it here for now.
+    test_type = {}.get(target, 'gtest_test')
+    read_only = {}.get(target, 1)

[M-A Ruel, 2015/06/03 15:47:18]

The only test that can't run read_only is unit_tests on Windows.
https://code.google.com/p/chromium/issues/detail?id=342913

Once fixed, you don't need this as a variable anymore. Probably worth fixing (or
disabling the test, whatever)

[Dirk Pranke, 2015/06/03 17:15:49]

Ok, I'll keep that in mind.

+
+    # This needs to mirror the settings in //build/config/ui.gni:
+    # use_x11 = is_linux && !use_ozone.
+    # TODO(dpranke): Figure out how to keep this in sync better.
+    use_x11 = (sys.platform == 'linux2' and
+               not 'target_os="android"' in vals['gn_args'] and
+               not 'use_ozone=true' in vals['gn_args'])
+
+    asan = 'is_asan=true' in vals['gn_args']
+    msan = 'is_msan=true' in vals['gn_args']
+    tsan = 'is_tsan=true' in vals['gn_args']
+
+    executable_suffix = '.exe' if sys.platform == 'win32' else ''
+
+    if test_type == 'gtest_test':
+      extra_files.append('../../testing/test_env.py')
+
+      if use_x11:
+        extra_files.append('xdisplaycheck')
+        extra_files.append('../../testing/xvfb.py')
+
+        cmdline = [
+          '../../testing/xvfb.py',
+          '.',
+          './' + str(target),
+          '--brave-new-test-launcher',
+          '--test-launcher-bot-mode',
+          '--asan=%d' % asan,
+          '--msan=%d' % msan,
+          '--tsan=%d' % tsan,
+        ]
+      else:
+        cmdline = [
+          '../../testing/test_env.py',
+          '.',
+          './' + str(target) + executable_suffix,
+          '--brave-new-test-launcher',
+          '--test-launcher-bot-mode',
+          '--asan=%d' % asan,
+          '--msan=%d' % msan,
+          '--tsan=%d' % tsan,
+        ]
+    else:
+      # TODO(dpranke): Handle script_tests and other types of
+      # swarmed tests.
+      self.WriteFailureAndRaise('unknown test type "%s" for %s' %
+                                (test_type, target),
+                                output_path)
+
+
+    return cmdline, extra_files, read_only
+
+  def ToAbsPath(self, build_path, relpath):
+    return os.path.join(self.chromium_src_dir,
+                        self.ToSrcRelPath(build_path),
+                        relpath)
+
   def ToSrcRelPath(self, path):
     """Returns a relative path from the top of the repo."""
     # TODO: Support normal paths in addition to source-absolute paths.
     assert(path.startswith('//'))
     return path[2:]
 
   def ParseGYPConfigPath(self, path):
     rpath = self.ToSrcRelPath(path)
     output_dir, _, config = rpath.rpartition('/')
     self.CheckGYPConfigIsSupported(config, path)
@@ skipping 196 matching lines @@
 
 if __name__ == '__main__':
   try:
     sys.exit(main(sys.argv[1:]))
   except MBErr as e:
     print(e)
     sys.exit(1)
   except KeyboardInterrupt:
     print("interrupted, exiting", stream=sys.stderr)
     sys.exit(130)