Reland "Remove the 'gyp_config' concept from MB."

tools/mb/mb.py

 #!/usr/bin/env python
 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """MB - the Meta-Build wrapper around GYP and GN
 
 MB is a wrapper script for GYP and GN that can be used to generate build files
 for sets of canned configurations and analyze them.
 """
@@ skipping 19 matching lines @@
   return mbw.args.func()
 
 
 class MetaBuildWrapper(object):
   def __init__(self):
     p = os.path
     d = os.path.dirname
     self.chromium_src_dir = p.normpath(d(d(d(p.abspath(__file__)))))
     self.default_config = p.join(self.chromium_src_dir, 'tools', 'mb',
                                  'mb_config.pyl')
+    self.executable = sys.executable
     self.platform = sys.platform
+    self.sep = os.sep
     self.args = argparse.Namespace()
     self.configs = {}
     self.masters = {}
     self.mixins = {}
     self.private_configs = []
     self.common_dev_configs = []
     self.unsupported_configs = []
 
   def ParseArgs(self, argv):
     def AddCommonOptions(subp):
@@ skipping 88 matching lines @@
 
     raise MBErr('Unknown meta-build type "%s"' % vals['type'])
 
   def CmdLookup(self):
     vals = self.GetConfig()
     if vals['type'] == 'gn':
       cmd = self.GNCmd('gen', '<path>', vals['gn_args'])
     elif vals['type'] == 'gyp':
       if vals['gyp_crosscompile']:
         self.Print('GYP_CROSSCOMPILE=1')
-      cmd = self.GYPCmd('<path>', vals['gyp_defines'], vals['gyp_config'])
+      cmd = self.GYPCmd('<path>', vals['gyp_defines'])
     else:
       raise MBErr('Unknown meta-build type "%s"' % vals['type'])
 
     self.PrintCmd(cmd)
     return 0
 
   def CmdHelp(self):
     if self.args.subcommand:
       self.ParseArgs([self.args.subcommand, '--help'])
     else:
@@ skipping 118 matching lines @@
       raise MBErr('Builder name "%s"  not found under masters[%s] in "%s"' %
                   (self.args.builder, self.args.master, self.args.config_file))
 
     return self.masters[self.args.master][self.args.builder]
 
   def FlattenConfig(self, config):
     mixins = self.configs[config]
     vals = {
       'type': None,
       'gn_args': [],
-      'gyp_config': [],
       'gyp_defines': '',
       'gyp_crosscompile': False,
     }
 
     visited = []
     self.FlattenMixins(mixins, vals, visited)
     return vals
 
   def FlattenMixins(self, mixins, vals, visited):
     for m in mixins:
       if m not in self.mixins:
         raise MBErr('Unknown mixin "%s"' % m)
 
       # TODO: check for cycles in mixins.
 
       visited.append(m)
 
       mixin_vals = self.mixins[m]
       if 'type' in mixin_vals:
         vals['type'] = mixin_vals['type']
       if 'gn_args' in mixin_vals:
         if vals['gn_args']:
           vals['gn_args'] += ' ' + mixin_vals['gn_args']
         else:
           vals['gn_args'] = mixin_vals['gn_args']
-      if 'gyp_config' in mixin_vals:
-        vals['gyp_config'] = mixin_vals['gyp_config']
       if 'gyp_crosscompile' in mixin_vals:
         vals['gyp_crosscompile'] = mixin_vals['gyp_crosscompile']
       if 'gyp_defines' in mixin_vals:
         if vals['gyp_defines']:
           vals['gyp_defines'] += ' ' + mixin_vals['gyp_defines']
         else:
           vals['gyp_defines'] = mixin_vals['gyp_defines']
       if 'mixins' in mixin_vals:
         self.FlattenMixins(mixin_vals['mixins'], vals, visited)
     return vals
 
   def ClobberIfNeeded(self, vals):
     path = self.args.path[0]
     build_dir = self.ToAbsPath(path)
-    mb_type_path = os.path.join(build_dir, 'mb_type')
+    mb_type_path = self.PathJoin(build_dir, 'mb_type')
     needs_clobber = False
     new_mb_type = vals['type']
     if self.Exists(build_dir):
       if self.Exists(mb_type_path):
         old_mb_type = self.ReadFile(mb_type_path)
         if old_mb_type != new_mb_type:
           self.Print("Build type mismatch: was %s, will be %s, clobbering %s" %
                      (old_mb_type, new_mb_type, path))
           needs_clobber = True
       else:
@@ skipping 16 matching lines @@
     cmd = self.GNCmd('gen', path, vals['gn_args'])
 
     swarming_targets = []
     if self.args.swarming_targets_file:
       # We need GN to generate the list of runtime dependencies for
       # the compile targets listed (one per line) in the file so
       # we can run them via swarming. We use ninja_to_gn.pyl to convert
       # the compile targets to the matching GN labels.
       contents = self.ReadFile(self.args.swarming_targets_file)
       swarming_targets = contents.splitlines()
-      gn_isolate_map = ast.literal_eval(self.ReadFile(os.path.join(
+      gn_isolate_map = ast.literal_eval(self.ReadFile(self.PathJoin(
           self.chromium_src_dir, 'testing', 'buildbot', 'gn_isolate_map.pyl')))
       gn_labels = []
       for target in swarming_targets:
         if not target in gn_isolate_map:
           raise MBErr('test target "%s"  not found in %s' %
                       (target, '//testing/buildbot/gn_isolate_map.pyl'))
         gn_labels.append(gn_isolate_map[target]['label'])
 
       gn_runtime_deps_path = self.ToAbsPath(path, 'runtime_deps')
 
@@ skipping 14 matching lines @@
       if gn_isolate_map[target]['type'] == 'gpu_browser_test':
         runtime_deps_target = 'browser_tests'
       elif gn_isolate_map[target]['type'] == 'script':
         # For script targets, the build target is usually a group,
         # for which gn generates the runtime_deps next to the stamp file
         # for the label, which lives under the obj/ directory.
         label = gn_isolate_map[target]['label']
         runtime_deps_target = 'obj/%s.stamp' % label.replace(':', '/')
       else:
         runtime_deps_target = target
-      if sys.platform == 'win32':
+      if self.platform == 'win32':
         deps_path = self.ToAbsPath(path,
                                    runtime_deps_target + '.exe.runtime_deps')
       else:
         deps_path = self.ToAbsPath(path,
                                    runtime_deps_target + '.runtime_deps')
       if not self.Exists(deps_path):
           raise MBErr('did not generate %s' % deps_path)
 
       command, extra_files = self.GetIsolateCommand(target, vals,
                                                     gn_isolate_map)
 
       runtime_deps = self.ReadFile(deps_path).splitlines()
 
       isolate_path = self.ToAbsPath(path, target + '.isolate')
       self.WriteFile(isolate_path,
         pprint.pformat({
           'variables': {
             'command': command,
             'files': sorted(runtime_deps + extra_files),
           }
         }) + '\n')
 
       self.WriteJSON(
         {
           'args': [
             '--isolated',
-            self.ToSrcRelPath('%s%s%s.isolated' % (path, os.sep, target)),
+            self.ToSrcRelPath('%s%s%s.isolated' % (path, self.sep, target)),
             '--isolate',
-            self.ToSrcRelPath('%s%s%s.isolate' % (path, os.sep, target)),
+            self.ToSrcRelPath('%s%s%s.isolate' % (path, self.sep, target)),
           ],
           'dir': self.chromium_src_dir,
           'version': 1,
         },
         isolate_path + 'd.gen.json',
       )
 
     return ret
 
   def GNCmd(self, subcommand, path, gn_args=''):
     if self.platform == 'linux2':
-      gn_path = os.path.join(self.chromium_src_dir, 'buildtools', 'linux64',
-                             'gn')
+      subdir = 'linux64'
     elif self.platform == 'darwin':
-      gn_path = os.path.join(self.chromium_src_dir, 'buildtools', 'mac',
-                             'gn')
+      subdir = 'mac'
     else:
-      gn_path = os.path.join(self.chromium_src_dir, 'buildtools', 'win',
-                             'gn.exe')
+      subdir = 'win'
+    gn_path = self.PathJoin(self.chromium_src_dir, 'buildtools', subdir, 'gn')
 
     cmd = [gn_path, subcommand, path]
     gn_args = gn_args.replace("$(goma_dir)", self.args.goma_dir)
     if gn_args:
       cmd.append('--args=%s' % gn_args)
     return cmd
 
   def RunGYPGen(self, vals):
     path = self.args.path[0]
 
-    output_dir, gyp_config = self.ParseGYPConfigPath(path)
-    if gyp_config != vals['gyp_config']:
-      raise MBErr('The last component of the path (%s) must match the '
-                  'GYP configuration specified in the config (%s), and '
-                  'it does not.' % (gyp_config, vals['gyp_config']))
-    cmd = self.GYPCmd(output_dir, vals['gyp_defines'], config=gyp_config)
+    output_dir = self.ParseGYPConfigPath(path)
+    cmd = self.GYPCmd(output_dir, vals['gyp_defines'])
     env = None
     if vals['gyp_crosscompile']:
       if self.args.verbose:
         self.Print('Setting GYP_CROSSCOMPILE=1 in the environment')
       env = os.environ.copy()
       env['GYP_CROSSCOMPILE'] = '1'
     ret, _, _ = self.Run(cmd, env=env)
     return ret
 
   def RunGYPAnalyze(self, vals):
-    output_dir, gyp_config = self.ParseGYPConfigPath(self.args.path[0])
-    if gyp_config != vals['gyp_config']:
-      raise MBErr('The last component of the path (%s) must match the '
-                  'GYP configuration specified in the config (%s), and '
-                  'it does not.' % (gyp_config, vals['gyp_config']))
+    output_dir = self.ParseGYPConfigPath(self.args.path[0])
     if self.args.verbose:
       inp = self.ReadInputJSON(['files', 'targets'])
       self.Print()
       self.Print('analyze input:')
       self.PrintJSON(inp)
       self.Print()
 
-    cmd = self.GYPCmd(output_dir, vals['gyp_defines'], config=gyp_config)
+    cmd = self.GYPCmd(output_dir, vals['gyp_defines'])
     cmd.extend(['-f', 'analyzer',
                 '-G', 'config_path=%s' % self.args.input_path[0],
                 '-G', 'analyzer_output_path=%s' % self.args.output_path[0]])
     ret, _, _ = self.Run(cmd)
     if not ret and self.args.verbose:
       outp = json.loads(self.ReadFile(self.args.output_path[0]))
       self.Print()
       self.Print('analyze output:')
       self.PrintJSON(outp)
       self.Print()
 
     return ret
 
   def GetIsolateCommand(self, target, vals, gn_isolate_map):
     # This needs to mirror the settings in //build/config/ui.gni:
     # use_x11 = is_linux && !use_ozone.
     # TODO(dpranke): Figure out how to keep this in sync better.
-    use_x11 = (sys.platform == 'linux2' and
+    use_x11 = (self.platform == 'linux2' and
                not 'target_os="android"' in vals['gn_args'] and
                not 'use_ozone=true' in vals['gn_args'])
 
     asan = 'is_asan=true' in vals['gn_args']
     msan = 'is_msan=true' in vals['gn_args']
     tsan = 'is_tsan=true' in vals['gn_args']
 
-    executable_suffix = '.exe' if sys.platform == 'win32' else ''
+    executable_suffix = '.exe' if self.platform == 'win32' else ''
 
     test_type = gn_isolate_map[target]['type']
     cmdline = []
     extra_files = []
 
     if use_x11 and test_type == 'windowed_test_launcher':
       extra_files = [
           'xdisplaycheck',
           '../../testing/test_env.py',
           '../../testing/xvfb.py',
@@ skipping 47 matching lines @@
           './' + str(target) + executable_suffix,
       ] + gn_isolate_map[target].get('args')
 
     else:
       self.WriteFailureAndRaise('No command line for %s found (test type %s).'
                                 % (target, test_type), output_path=None)
 
     return cmdline, extra_files
 
   def ToAbsPath(self, build_path, *comps):
-    return os.path.join(self.chromium_src_dir,
-                        self.ToSrcRelPath(build_path),
-                        *comps)
+    return self.PathJoin(self.chromium_src_dir,
+                         self.ToSrcRelPath(build_path),
+                         *comps)
 
   def ToSrcRelPath(self, path):
     """Returns a relative path from the top of the repo."""
     # TODO: Support normal paths in addition to source-absolute paths.
     assert(path.startswith('//'))
-    return path[2:].replace('/', os.sep)
+    return path[2:].replace('/', self.sep)
 
   def ParseGYPConfigPath(self, path):
     rpath = self.ToSrcRelPath(path)
-    output_dir, _, config = rpath.rpartition('/')
-    self.CheckGYPConfigIsSupported(config, path)
-    return output_dir, config
+    output_dir, _, _ = rpath.rpartition(self.sep)
+    return output_dir
 
-  def CheckGYPConfigIsSupported(self, config, path):
-    if config not in ('Debug', 'Release'):
-      if (sys.platform in ('win32', 'cygwin') and
-          config not in ('Debug_x64', 'Release_x64')):
-        raise MBErr('Unknown or unsupported config type "%s" in "%s"' %
-                    config, path)
-
-  def GYPCmd(self, output_dir, gyp_defines, config):
+  def GYPCmd(self, output_dir, gyp_defines):
     gyp_defines = gyp_defines.replace("$(goma_dir)", self.args.goma_dir)
     cmd = [
-        sys.executable,
-        os.path.join('build', 'gyp_chromium'),
+        self.executable,
+        self.PathJoin('build', 'gyp_chromium'),
         '-G',
         'output_dir=' + output_dir,
-        '-G',
-        'config=' + config,
     ]
     for d in shlex.split(gyp_defines):
       cmd += ['-D', d]
     return cmd
 
   def RunGNAnalyze(self, vals):
     # analyze runs before 'gn gen' now, so we need to run gn gen
     # in order to ensure that we have a build directory.
     ret = self.RunGNGen(vals)
     if ret:
@@ skipping 35 matching lines @@
     response_file.close()
 
     matching_targets = []
     try:
       cmd = self.GNCmd('refs', self.args.path[0]) + [
           '@%s' % response_file.name, '--all', '--as=output']
       ret, out, _ = self.Run(cmd, force_verbose=False)
       if ret and not 'The input matches no targets' in out:
         self.WriteFailureAndRaise('gn refs returned %d: %s' % (ret, out),
                                   output_path)
-      build_dir = self.ToSrcRelPath(self.args.path[0]) + os.sep
+      build_dir = self.ToSrcRelPath(self.args.path[0]) + self.sep
       for output in out.splitlines():
         build_output = output.replace(build_dir, '')
         if build_output in inp['targets']:
           matching_targets.append(build_output)
 
       cmd = self.GNCmd('refs', self.args.path[0]) + [
           '@%s' % response_file.name, '--all']
       ret, out, _ = self.Run(cmd, force_verbose=False)
       if ret and not 'The input matches no targets' in out:
         self.WriteFailureAndRaise('gn refs returned %d: %s' % (ret, out),
@@ skipping 58 matching lines @@
 
   def WriteJSON(self, obj, path, force_verbose=False):
     try:
       self.WriteFile(path, json.dumps(obj, indent=2, sort_keys=True) + '\n',
                      force_verbose=force_verbose)
     except Exception as e:
       raise MBErr('Error %s writing to the output path "%s"' %
                  (e, path))
 
   def PrintCmd(self, cmd):
-    if cmd[0] == sys.executable:
+    if cmd[0] == self.executable:
       cmd = ['python'] + cmd[1:]
     self.Print(*[pipes.quote(c) for c in cmd])
 
   def PrintJSON(self, obj):
     self.Print(json.dumps(obj, indent=2, sort_keys=True))
 
   def Print(self, *args, **kwargs):
     # This function largely exists so it can be overridden for testing.
     print(*args, **kwargs)
 
@@ skipping 27 matching lines @@
     # This function largely exists so it can be overridden for testing.
     return os.path.exists(path)
 
   def MaybeMakeDirectory(self, path):
     try:
       os.makedirs(path)
     except OSError, e:
       if e.errno != errno.EEXIST:
         raise
 
+  def PathJoin(self, *comps):
+    # This function largely exists so it can be overriden for testing.
+    return os.path.join(*comps)
+
   def ReadFile(self, path):
     # This function largely exists so it can be overriden for testing.
     with open(path) as fp:
       return fp.read()
 
   def RemoveFile(self, path):
     # This function largely exists so it can be overriden for testing.
     os.remove(path)
 
   def RemoveDirectory(self, abs_path):
-    if sys.platform == 'win32':
+    if self.platform == 'win32':
       # In other places in chromium, we often have to retry this command
       # because we're worried about other processes still holding on to
       # file handles, but when MB is invoked, it will be early enough in the
       # build that their should be no other processes to interfere. We
       # can change this if need be.
       self.Run(['cmd.exe', '/c', 'rmdir', '/q', '/s', abs_path])
     else:
       shutil.rmtree(abs_path, ignore_errors=True)
 
   def TempFile(self, mode='w'):
@@ skipping 14 matching lines @@
 
 if __name__ == '__main__':
   try:
     sys.exit(main(sys.argv[1:]))
   except MBErr as e:
     print(e)
     sys.exit(1)
   except KeyboardInterrupt:
     print("interrupted, exiting", stream=sys.stderr)
     sys.exit(130)