First pass at a meta-build wrapper for the gyp->gn migration

tools/mb/mb.py

+#!/usr/bin/env python
+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""MB - the Meta-Build wrapper around GYP and GN
+
+MB is a wrapper script for GYP and GN that can be used to generate build files
+for sets of canned configurations and analyze them.
+"""
+
+from __future__ import print_function
+
+import argparse
+import ast
+import json
+import os
+import pipes
+import shutil
+import sys
+import subprocess
+import tempfile
+
+
+DEFAULT_BUILDS = {
+  '//out': 'gyp'
+}
+
+
+def main(argv):
+  mb = MetaBuildWrapper()
+  mb.ParseArgs(argv)
+  return mb.args.func()
+
+
+class MetaBuildWrapper(object):
+  def __init__(self):
+    p = os.path
+    d = os.path.dirname
+    self.chromium_src_dir = p.normpath(p.abspath(d(d(d(__file__)))))
+    self.mb_conf_pyl = p.join(self.chromium_src_dir, 'build', 'mb_conf.pyl')
+    self.builds_pyl = p.normpath(p.join(self.chromium_src_dir, '..',
+                                        'builds.pyl'))
+    self.build_configs = {}
+    self.mixins = {}
+
+  def ParseArgs(self, argv):
+    parser = argparse.ArgumentParser(prog='mb')
+    subps = parser.add_subparsers()
+
+    subp = subps.add_parser('analyze',
+                            help='analyze whether changes to a set of files '
+                                 'will cause a set of binaries to be rebuilt.')
+    subp.add_argument('-i', '--json-input', action='store',
+                      help='path to a file containing the input arguments '
+                           'as a JSON object.')
+    subp.add_argument('-o', '--json-output', action='store',
+                      help='path to a file containing the output arguments '
+                           'as a JSON object.')
+    subp.add_argument('-n', '--dryrun', action='store_true',
+                      help='Do a dry run (i.e., do nothing, just print '
+                           'the commands')
+    subp.add_argument('-v', '--verbose', action='count',
+                      help='verbose logging (may specify multiple times.')
+
+    subp.add_argument('build_config', type=str, nargs='?',
+                      help='configuration to analyze')
+    subp.add_argument('paths', type=str, nargs='*',
+                      help='list of files or targets to analyze')
+    subp.set_defaults(func=self.CmdAnalyze)
+
+    subp = subps.add_parser('gen',
+                            help='generate a new set of build files')
+    subp.add_argument('-n', '--dryrun', action='store_true',
+                       help='Do a dry run (i.e., do nothing, just print '
+                            'the commands')
+    subp.add_argument('-v', '--verbose', action='count',
+                       help='verbose logging (may specify multiple times.')
+    subp.add_argument('path', type=str, nargs='?',
+                      help='path to generate build into')
+    subp.add_argument('build_config', type=str, nargs='?',
+                      help='build configuration to generate')
+    subp.set_defaults(func=self.CmdGen)
+
+    subp = subps.add_parser('help',
+                            help='Get help on a subcommand.')
+    subp.add_argument(nargs='?', action='store', dest='subcommand',
+                      help='The command to get help for.')
+    subp.set_defaults(func=self.CmdHelp)
+
+    self.args = parser.parse_args(argv)
+
+  def ReadConf(self):
+    if not self.PathExists(self.mwb_conf_pyl):
+      raise _Err('mwb conf file not found at %s' % self.mwb_conf_pyl)
+    contents = self.ReadPyl(self.mwb_conf_pyl, 'mwb conf file')
+    self.build_configs = contents['build_configs']
+    self.mixins = contents['mixins']
+
+  def PathExists(self, path):
+    return os.path.exists(path)
+
+  def ReadPyl(self, path, desc):
+    try:
+      return ast.literal_eval(open(path).read())
+    except SyntaxError as e:
+      raise _Err('Failed to parse %s at "%s": %s' % (desc, path, e))
+
+  def Call(self, cmd):
+    if self.args.dryrun or self.args.verbose:
+      if cmd[0] == sys.executable:
+        cmd = ['python'] + cmd[1:]
+      print(*[pipes.quote(c) for c in cmd])
+
+    if self.args.dryrun:
+      return 0, '', ''
+    p = subprocess.Popen(cmd, shell=False, cwd=self.chromium_src_dir,
+                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    out, err = p.communicate()
+    if self.args.verbose:
+      if out:
+        print(out)
+      if err:
+        print(err, stream=sys.stderr)
+    return p.returncode, out, err
+
+  def CmdHelp(self):
+    if self.args.subcommand:
+      self.ParseArgs([self.args.subcommand, '--help'])
+    else:
+      self.ParseArgs(['--help'])
+
+  def CmdGen(self):
+    self.ReadConf()
+
+    if self.args.build_config and self.args.path:
+      builds = self.BuildsFromArgs()
+    elif os.path.exists(self.builds_pyl):
+      builds = self.BuildsFromPyl()
+    else:
+      build = DEFAULT_BUILDS
+
+    for path, build_config in builds.items():
+      vals = self.FlattenBuildConfig(build_config)
+      if vals['type'] == 'gn':
+        self.RunGN(path, vals['gn_args'])
+      elif vals['type'] == 'gyp':
+        self.RunGYP(path, vals['gyp_defines'])
+      elif vals['type'] == 'gyp_one_config':
+        self.RunGYPOneConfig(path, vals['gyp_defines'])
+      else:
+        raise _Err('Unknown meta-build type "%s"' % vals['type'])
+    return 0
+
+  def BuildsFromArgs(self):
+    builds = {}
+    builds[self.args.path] = self.args.build_config
+    return builds
+
+  def BuildsFromPyl(self):
+    return self.ReadPyl(self.builds_pyl, "build file")
+
+  def RunGN(self, path, gn_args=''):
+    cmd = self.GNCmd(path, gn_args)
+    ret, _, _ = self.Call(cmd)
+    return ret
+
+  def FlattenBuildConfig(self, build_config):
+    if build_config not in self.build_configs:
+      raise _Err('Unknown build config "%s"' % build_config)
+    if self.build_configs[build_config] is None:
+      mixins = build_config.split('_')
+    else:
+      mixins = self.build_configs[build_config]
+
+    vals = {
+      'type': None,
+      'gn_args': [],
+      'gyp_config': [],
+      'gyp_defines': [],
+    }
+
+    visited = []
+    self.FlattenMixins(mixins, vals, visited)
+    return vals
+
+  def FlattenMixins(self, mixins, vals, visited):
+    for m in mixins:
+      if m not in self.mixins:
+        raise _Err('Unknown mixin "%s"' % m)
+      if m in visited:
+        raise _Err('Cycle in build configs for "%s": %s' % (config, visited))
+
+      visited.append(m)
+
+      mixin_vals = self.mixins[m]
+      if 'type' in mixin_vals:
+        vals['type'] = mixin_vals['type']
+      if 'gn_args' in mixin_vals:
+        vals['gn_args'].extend(mixin_vals['gn_args'])
+      if 'gyp_config' in mixin_vals:
+        vals['gyp_config'] = mixin_vals['gyp_config']
+      if 'gyp_defines' in mixin_vals:
+        vals['gyp_defines'].extend(mixin_vals['gyp_defines'])
+      if 'mixins' in mixin_vals:
+        self.FlattenMixins(mixin_vals['mixins'], vals, visited)
+    return vals
+
+  def GNCmd(self, path, gn_args):
+    # TODO(dpranke): Find gn explicitly in the path ...
+    cmd = ['gn', 'gen', path]
+    if gn_args:
+      cmd.append('--args=%s' % ' '.join(gn_args))
+    return cmd
+
+  def RunGYP(self, path, gyp_defines):
+    output_dir = self.ParseGYPOutputPath(path)
+    cmd = self.GYPCmd(output_dir, gyp_defines, config=None)
+    ret, _, _ = self.Call(cmd)
+    return ret
+
+  def RunGYPConfig(self, path, gyp_defines):
+    output_dir, gyp_config = self.ParseGYPConfigPath(path)
+    cmd = self.GYPCmd(output_dir, gyp_defines, config=gyp_config)
+    ret, _, _ = self.Call(cmd)
+    return ret
+
+  def ParseGYPOutputPath(self, path):
+    # Do we need to handle absolute paths or relative paths?
+    assert(path.startswith('//'))
+    return path[2:]
+
+  def ParseGYPConfigPath(self, path):
+    # Do we need to handle absolute paths or relative paths?
+    assert(path.startswith('//'))
+    output_dir, _, config = path[2:].rpartition('/')
+    self.CheckGYPConfigIsSupported(config, path)
+    return output_dir, config
+
+  def CheckGYPConfigIsSupported(self, config, path):
+    if config not in ('Debug', 'Release'):
+      if (sys.platform in ('win32', 'cygwin') and
+          config not in ('Debug_x64', 'Release_x64')):
+        raise _Err('Unknown or unsupported config type "%s" in "%s"' %
+                   config, path)
+
+  def GYPCmd(self, output_dir, gyp_defines, config):
+    cmd = [
+        sys.executable,
+        os.path.join('build', 'gyp_chromium'),
+        '-G',
+        'output_dir=' + output_dir
+    ]
+    if config:
+      cmd += ['-G', 'config='  + config]
+    for d in gyp_defines:
+      cmd += ['-D', d]
+    return cmd
+
+  def CmdAnalyze(self):
+    self.ReadConf()
+    vals = self.FlattenBuildConfig(self.args.build_config)
+    if vals['type'] == 'gn':
+      self.RunGNAnalyze(vals)
+    elif vals['type'] in ('gyp', 'gyp_one_config'):
+      self.RunGypAnalyze(vals)
+    else:
+      raise _Err('Unknown meta-build type "%s"' % vals['type'])
+
+  def RunGNAnalyze(self, vals):
+    if self.args.dryrun:
+      tmpdir = '//out/$tmpdir'
+    else:
+      tmpdir = tempfile.mkdtemp(prefix='analyze', suffix='tmp',
+                                dir=os.path.join(self.chromium_src_dir, 'out'))
+
+    files = self.args.paths
+    try:
+      idx = files.index('-')
+      targets = files[idx + 1:]
+      files = files[:idx]
+    except ValueError:
+      targets = []
+
+    self.RunGN(tmpdir, vals['gn_args'])
+
+    cmd = ['gn', 'refs', tmpdir] + files + ['--type=executable',
+           '--all', '--as=output']
+    needed_targets = []
+    ret, out, _ = self.Call(cmd)
+
+    # TODO: handle failures from 'gn refs'
+
+    if self.args.dryrun or self.args.verbose:
+      print('rm -fr', tmpdir)
+    if not self.args.dryrun:
+      shutil.rmtree(tmpdir, ignore_errors=True)
+
+    rpath = os.path.relpath(tmpdir, self.chromium_src_dir) + os.sep
+    needed_targets = [t.replace(rpath, '') for t in out.splitlines()]
+    if targets:
+      needed_targets = [nt for nt in needed_targets if nt in targets]
+
+    for nt in needed_targets:
+      print(nt)
+
+
+class _Err(Exception):
+  pass
+
+
+if __name__ == '__main__':
+  try:
+    sys.exit(main(sys.argv[1:]))
+  except _Err as e:
+    print(e)
+    sys.exit(1)
+  except KeyboardInterrupt:
+    print("interrupted, exiting", stream=sys.stderr)
+    sys.exit(130)