[tools] Build generator script

tools/dev/v8gen.py

+#!/usr/bin/env python
+# Copyright 2016 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Script to generate V8's gn arguments based on common developer defaults
+or builder configurations.
+
+Examples:
+
+# Generate the x64.release config in out.gn/x64.release.
+v8gen.py x64.release
+
+# Generate into out.gn/foo.
+v8gen.py -b x64.release foo

[jochen (gone - plz use gerrit), 2016/07/18 11:28:29]

shouldn't that be something like -b 'V8 Linux64 - builder'?

[Michael Achenbach, 2016/07/21 13:13:29]

No. All our standard configs have their own shortcuts. In most cases they are
equal to the respective bots, but sometimes the bots do a few more things, e.g.
additional things to pack in swarming isolates.

+
+# Pass additional gn arguments after -- (don't use spaces within gn args).
+v8gen.py x64.optdebug -- use_goma=true

[jochen (gone - plz use gerrit), 2016/07/18 11:28:29]

use_goma would be -g - maybe use another example here?

[Michael Achenbach, 2016/07/21 13:13:29]

I'll update this example with another flag than goma and add extra goma examples
below.

+
+# Print wrapped commands.
+v8gen.py x64.optdebug -v
+
+# Print output of wrapped commands.
+v8gen.py x64.optdebug -vv
+
+# Generate gn arguments of builder 'V8 Linux64 - builder' from master
+# 'client.v8' into out.gn/V8_Linux64___builder.
+v8gen.py -m client.v8 -b 'V8 Linux64 - builder'
+"""
+
+import argparse
+import os
+import re
+import subprocess
+import sys
+
+GOMA_DEFAULT = os.path.join(os.path.expanduser("~"), 'goma')
+OUT_DIR = 'out.gn'
+
+
+def _sanitize_nonalpha(text):
+  return ''.join(c if (c.isalnum() or c == '.') else '_' for c in text)

[vogelheim, 2016/07/18 18:28:26]

I think I'd find an re-based version more readable. Matter of taste, though.

  return re.sub(r'[^a-zA-Z0-9.]', '_', text)

[Michael Achenbach, 2016/07/21 13:13:29]

Not sure if I understand what you mean by re-based here. I added your
suggestion.

[vogelheim, 2016/07/21 13:19:54]

Sorry for being unclear.

What I meant: Based on regular expressions / the 're' module.

+
+
+class GenerateGnArgs(object):
+  def __init__(self, args):
+    # Split args into this script's arguments and gn args passed to the
+    # wrapped gn.
+    index = args.index('--') if '--' in args else len(args)
+    self._options = self._parse_arguments(args[:index])
+    self._gn_args = args[index + 1:]
+
+  def _parse_arguments(self, args):
+    parser = argparse.ArgumentParser(
+      description='Generator for V8\'s gn arguments. '
+                  'Additional gn args can be separated with -- '
+                  'from the arguments to this tool (don\'t use spaces or '
+                  'quote arguments like "use_goma = true").')
+    parser.add_argument(
+        'outdir', nargs='?',
+        help='optional gn output directory')
+    parser.add_argument(
+        '-b', '--builder',
+        help='build configuration or builder name from mb_config.pyl')
+    parser.add_argument(
+        '-m', '--master', default='developer_default',
+        help='config group or master from mb_config.pyl')
+    parser.add_argument(
+        '-p', '--pedantic', action='store_true',
+        help='run gn over command-line gn args to catch errors early')
+    parser.add_argument(
+        '-v', '--verbosity', action='count',
+        help='increase output verbosity (can be specified multiple times)')
+
+    goma = parser.add_mutually_exclusive_group()
+    goma.add_argument(
+        '-g' , '--goma',
+        action='store_true', default=None, dest='goma',
+        help='use goma')
+    goma.add_argument(
+        '--nogoma', '--no-goma',
+        action='store_false', default=None, dest='goma',
+        help='don\'t use goma')
+
+    options = parser.parse_args(args)
+
+    if not options.outdir and not options.builder:
+      parser.error('please specify either an output directory or '
+                   'a builder/config name (-b), e.g. x64.release')
+
+    if not options.outdir:
+      # Derive output directory from builder name.
+      options.outdir = _sanitize_nonalpha(options.builder)
+    else:
+      # Also, if this should work on windows, we might need to use \ where
+      # outdir is used as path, while using / if it's used in a gn context.
+      if options.outdir.startswith('/'):
+        parser.error(
+            'only output directories relative to %s are supported' % OUT_DIR)
+
+    if not options.builder:
+      # Derive builder from output directory.
+      options.builder = options.outdir
+
+    return options
+
+  def verbose_print_1(self, text):
+    if self._options.verbosity >= 1:
+      print '#' * 80
+      print text
+
+  def verbose_print_2(self, text):
+    if self._options.verbosity >= 2:
+      indent = ' ' * 2
+      for l in text.splitlines():
+        print indent + l
+
+  def _call_cmd(self, args):
+    self.verbose_print_1(' '.join(args))
+    try:
+      output = subprocess.check_output(
+        args=args,
+        stderr=subprocess.STDOUT,
+      )
+      self.verbose_print_2(output)
+    except subprocess.CalledProcessError as e:
+      self.verbose_print_2(e.output)
+      raise
+
+  def _find_work_dir(self, path):
+    """Find the closest v8 root to `path`."""
+    if os.path.exists(os.path.join(path, 'tools', 'dev', 'v8gen.py')):
+      # Approximate the v8 root dir by a folder where this script exists
+      # in the expected place.
+      return path
+    elif os.path.dirname(path) == path:
+      raise Exception(
+          'This appears to not be called from a recent v8 checkout')
+    else:
+      return self._find_work_dir(os.path.dirname(path))
+
+  @property
+  def _goma_dir(self):
+    return os.path.normpath(os.environ.get('GOMA_DIR') or GOMA_DEFAULT)
+
+  @property
+  def _need_goma_dir(self):
+    return self._goma_dir != GOMA_DEFAULT
+
+  @property
+  def _use_goma(self):
+    if self._options.goma is None:
+      # Auto-detect.
+      return os.path.exists(self._goma_dir) and os.path.isdir(self._goma_dir)
+    else:
+      return self._options.goma
+
+  @property
+  def _goma_args(self):
+    """Gn args for using goma."""
+    # Specify goma args if we want to use goma and if goma isn't specified
+    # via command line already. The command-line always has precedence over
+    # any other specification.
+    if (self._use_goma and
+        not any(re.match(r'use_goma\s*=.*', x) for x in self._gn_args)):
+      if self._need_goma_dir:
+        return 'use_goma=true\ngoma_dir="%s"' % self._goma_dir
+      else:
+        return 'use_goma=true'
+    else:
+      return ''
+
+  def _append_gn_args(self, type, gn_args_path, more_gn_args):
+    """Append extra gn arguments to the generated args.gn file."""
+    if not more_gn_args:
+      return False
+    self.verbose_print_1('Appending """\n%s\n""" to %s.' % (
+        more_gn_args, os.path.abspath(gn_args_path)))
+    with open(gn_args_path, 'a') as f:
+      f.write('\n# Additional %s args:\n' % type)
+      f.write(more_gn_args)
+      f.write('\n')
+    return True
+
+  def main(self):
+    # Always operate relative to the base directory for better relative-path
+    # handling. This script can be used in any v8 checkout.
+    workdir = self._find_work_dir(os.getcwd())
+    if workdir != os.getcwd():
+      self.verbose_print_1('cd ' + workdir)
+      os.chdir(workdir)
+
+    # The directories are separated with slashes in a gn context (platform
+    # independent).
+    gn_outdir = '/'.join([OUT_DIR, self._options.outdir])
+
+    # Call MB to generate the basic configuration.
+    self._call_cmd([
+      sys.executable,
+      '-u', os.path.join('tools', 'mb', 'mb.py'),
+      'gen',
+      '-f', os.path.join('infra', 'mb', 'mb_config.pyl'),
+      '-m', self._options.master,
+      '-b', self._options.builder,
+      gn_outdir,
+    ])
+
+    # Handle extra gn arguments.
+    gn_args_path = os.path.join(OUT_DIR, self._options.outdir, 'args.gn')
+
+    # Append command-line args.
+    modified = self._append_gn_args(
+        'command-line', gn_args_path, '\n'.join(self._gn_args))
+
+    # Append goma args.
+    # TODO(machenbach): We currently can't remove existing goma args from the
+    # original config. E.g. to build like a bot that uses goma, but switch
+    # goma off.
+    modified |= self._append_gn_args(
+        'goma', gn_args_path, self._goma_args)
+
+    # Regenerate ninja files to check for errors in the additional gn args.
+    if modified and self._options.pedantic:
+      self._call_cmd(['gn', 'gen', gn_outdir])
+    return 0
+
+if __name__ == "__main__":
+  gen = GenerateGnArgs(sys.argv[1:])
+  try:
+    sys.exit(gen.main())
+  except Exception:
+    if gen._options.verbosity < 2:
+      print ('\nHint: You can raise verbosity (-vv) to see the output of '
+             'failed commands.\n')
+    raise