Add support for GN's --runtime-deps-list-file flag to MB.

tools/mb/mb.py

 #!/usr/bin/env python
 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """MB - the Meta-Build wrapper around GYP and GN
 
 MB is a wrapper script for GYP and GN that can be used to generate build files
 for sets of canned configurations and analyze them.
 """
@@ skipping 71 matching lines @@
                       help='path to a file containing the input arguments '
                            'as a JSON object.')
     subp.add_argument('output_path', nargs=1,
                       help='path to a file containing the output arguments '
                            'as a JSON object.')
     subp.set_defaults(func=self.CmdAnalyze)
 
     subp = subps.add_parser('gen',
                             help='generate a new set of build files')
     AddCommonOptions(subp)
+    subp.add_argument('--swarming-targets',
+                      help='save runtime dependencies for targets in file '
+                           '(GN only).')
     subp.add_argument('path', nargs=1,
                       help='path to generate build into')
     subp.set_defaults(func=self.CmdGen)
 
     subp = subps.add_parser('isolate',
                             help='build isolates')
     AddCommonOptions(subp)
     subp.add_argument('path', nargs=1,
                       help='path build was generated into.')
     subp.add_argument('input_path', nargs=1,
@@ skipping 227 matching lines @@
         if vals['gyp_defines']:
           vals['gyp_defines'] += ' ' + mixin_vals['gyp_defines']
         else:
           vals['gyp_defines'] = mixin_vals['gyp_defines']
       if 'mixins' in mixin_vals:
         self.FlattenMixins(mixin_vals['mixins'], vals, visited)
     return vals
 
   def RunGNGen(self, path, vals):
     cmd = self.GNCmd('gen', path, vals['gn_args'])
+
+    if self.args.swarming_targets:
+      # We need GN to generate the list of runtime dependencies for
+      # the compile targets listed (one per line) in the file so
+      # we can run them via swarming. We use ninja_to_gn.pyl to convert
+      # the compile targets to the matching GN labels.
+      contents = self.ReadFile(self.args.swarming_targets)
+      runtime_deps_targets = contents.splitlines()
+      ninja_targets_to_labels = ast.literal_eval(self.ReadFile(os.path.join(

[M-A Ruel, 2015/06/10 14:53:40]

I trick we used in isolate Go is to pre-process the file to remove all lines
that start with a #, roughly;

content = '\n'.join(
 l for l in self.ReadFile(os.path.join(...)).splitlines()
 if not l.lstrip().startswith('#'))

ninja_targets_to_labels = json.loads(content)

To be clear; that's a total abuse but it works fine enough. The only annoying
thing is the trailing comma but  we can live with it.

I don't mind either way. It's just that in Go, ast wasn't an option.

[Dirk Pranke, 2015/06/10 16:07:59]

That is a good suggestion, but since .pyl / ast.literal_eval() is an option,
I'll stick w/ that for now ...

+          self.chromium_src_dir, 'testing', 'buildbot', 'ninja_to_gn.pyl')))
+      gn_labels = []
+      for target in runtime_deps_targets:
+        if not target in ninja_targets_to_label:
+          raise MBErr('test target "%s"  not found in %s' %
+                      (target, "//testing/buildbot/ninja_to_gn.pyl"))

[M-A Ruel, 2015/06/10 14:53:40]

Use single quotes

[Dirk Pranke, 2015/06/10 16:07:59]

will fix.

+
+      gn_runtime_deps_path = self.ToAbsPath(os.path.join(path,
+                                                         'runtime_deps'))
+      self.WriteFile(gn_runtime_deps_path, '\n'.join(gn_labels) + '\n')
+      cmd.append('--runtime-deps-list-file=%s' % gn_runtime_deps_path)

[M-A Ruel, 2015/06/10 14:53:40]

use two separate args, it works better on Windows.

[Dirk Pranke, 2015/06/10 16:07:59]

I can't; GN only supports --foo=bar style arguments (argue w/ Brett about that
one).

+
     ret, _, _ = self.Run(cmd)
+
+    if self.args.swarming_targets:
+      for target in swarming_targets:

[M-A Ruel, 2015/06/10 14:53:40]

Where is this variable defined? I don't see how this code can work.

[Dirk Pranke, 2015/06/10 16:07:59]

Whoops, you're right. This should be looping over gn_labels, but apparently
even that isn't actually defined. Will fix.

+        output = os.path.join(path, target + '.runtime_deps')
+        deps_path = self.ToAbsPath(os.path.join(path,
+                                                target + '.runtime_deps'))
+        if not self.Exists(self.ToAbsPath(output)):
+          raise MBErr('did not generate %s' % output)
+
     return ret
 
   def GNCmd(self, subcommand, path, gn_args=''):
     if self.platform == 'linux2':
       gn_path = os.path.join(self.chromium_src_dir, 'buildtools', 'linux64',
                              'gn')
     elif self.platform == 'darwin':
       gn_path = os.path.join(self.chromium_src_dir, 'buildtools', 'mac',
                              'gn')
     else:
@@ skipping 30 matching lines @@
       self.Print()
 
     cmd = self.GYPCmd(output_dir, vals['gyp_defines'], config=gyp_config)
     cmd.extend(['-G', 'config_path=%s' % self.args.input_path[0],
                 '-G', 'analyzer_output_path=%s' % self.args.output_path[0]])
     ret, _, _ = self.Run(cmd)
     if not ret and self.args.verbose:
       outp = json.loads(self.ReadFile(self.args.output_path[0]))
       self.Print()
       self.Print('analyze output:')
-      self.PrintJSON(inp)
+      self.PrintJSON(outp)

[Dirk Pranke, 2015/06/10 06:43:17]

this is an unrelated bug that pylint revealed :).

       self.Print()
 
     return ret
 
   def RunGNIsolate(self, vals):
     build_path = self.args.path[0]
     inp = self.ReadInputJSON(['targets'])
     if self.args.verbose:
       self.Print()
       self.Print('isolate input:')
@@ skipping 316 matching lines @@
 
 if __name__ == '__main__':
   try:
     sys.exit(main(sys.argv[1:]))
   except MBErr as e:
     print(e)
     sys.exit(1)
   except KeyboardInterrupt:
     print("interrupted, exiting", stream=sys.stderr)
     sys.exit(130)