Re-land "Update MB to use `gn analyze`."

tools/mb/mb.py

 #!/usr/bin/env python
 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """MB - the Meta-Build wrapper around GYP and GN
 
 MB is a wrapper script for GYP and GN that can be used to generate build files
 for sets of canned configurations and analyze them.
 """
@@ skipping 43 matching lines @@
     self.mixins = {}
 
   def Main(self, args):
     self.ParseArgs(args)
     try:
       ret = self.args.func()
       if ret:
         self.DumpInputFiles()
       return ret
     except KeyboardInterrupt:
-      self.Print('interrupted, exiting', stream=sys.stderr)
+      self.Print('interrupted, exiting')
       return 130
     except Exception:
       self.DumpInputFiles()
       s = traceback.format_exc()
       for l in s.splitlines():
         self.Print(l)
       return 1
 
   def ParseArgs(self, argv):
     def AddCommonOptions(subp):
@@ skipping 147 matching lines @@
                       help='The command to get help for.')
     subp.set_defaults(func=self.CmdHelp)
 
     self.args = parser.parse_args(argv)
 
   def DumpInputFiles(self):
 
     def DumpContentsOfFilePassedTo(arg_name, path):
       if path and self.Exists(path):
         self.Print("\n# To recreate the file passed to %s:" % arg_name)
-        self.Print("%% cat > %s <<EOF)" % path)
+        self.Print("%% cat > %s <<EOF" % path)
         contents = self.ReadFile(path)
         self.Print(contents)
         self.Print("EOF\n%\n")
 
     if getattr(self.args, 'input_path', None):
       DumpContentsOfFilePassedTo(
           'argv[0] (input_path)', self.args.input_path[0])
     if getattr(self.args, 'swarming_targets_file', None):
       DumpContentsOfFilePassedTo(
           '--swarming-targets-file', self.args.swarming_targets_file)
@@ skipping 393 matching lines @@
     try:
       contents = ast.literal_eval(self.ReadFile(self.args.config_file))
     except SyntaxError as e:
       raise MBErr('Failed to parse config file "%s": %s' %
                  (self.args.config_file, e))
 
     self.configs = contents['configs']
     self.masters = contents['masters']
     self.mixins = contents['mixins']
 
+  def ReadIsolateMap(self):
+    return ast.literal_eval(self.ReadFile(self.PathJoin(
+        self.chromium_src_dir, 'testing', 'buildbot', 'gn_isolate_map.pyl')))
+
   def ConfigFromArgs(self):
     if self.args.config:
       if self.args.master or self.args.builder:
         raise MBErr('Can not specific both -c/--config and -m/--master or '
                     '-b/--builder')
 
       return self.args.config
 
     if not self.args.master or not self.args.builder:
       raise MBErr('Must specify either -c/--config or '
@@ skipping 107 matching lines @@
     # Since GN hasn't run yet, the build directory may not even exist.
     self.MaybeMakeDirectory(self.ToAbsPath(build_dir))
 
     gn_args_path = self.ToAbsPath(build_dir, 'args.gn')
     self.WriteFile(gn_args_path, gn_args, force_verbose=True)
 
     swarming_targets = []
     if getattr(self.args, 'swarming_targets_file', None):
       # We need GN to generate the list of runtime dependencies for
       # the compile targets listed (one per line) in the file so
-      # we can run them via swarming. We use ninja_to_gn.pyl to convert
+      # we can run them via swarming. We use gn_isolate_map.pyl to convert
       # the compile targets to the matching GN labels.
       path = self.args.swarming_targets_file
       if not self.Exists(path):
         self.WriteFailureAndRaise('"%s" does not exist' % path,
                                   output_path=None)
       contents = self.ReadFile(path)
       swarming_targets = set(contents.splitlines())
-      gn_isolate_map = ast.literal_eval(self.ReadFile(self.PathJoin(
-          self.chromium_src_dir, 'testing', 'buildbot', 'gn_isolate_map.pyl')))
-      gn_labels = []
-      err = ''
-      for target in swarming_targets:
-        target_name = self.GNTargetName(target)
-        if not target_name in gn_isolate_map:
-          err += ('test target "%s" not found\n' % target_name)
-        elif gn_isolate_map[target_name]['type'] == 'unknown':
-          err += ('test target "%s" type is unknown\n' % target_name)
-        else:
-          gn_labels.append(gn_isolate_map[target_name]['label'])
 
+      isolate_map = self.ReadIsolateMap()
+      err, labels = self.MapTargetsToLabels(isolate_map, swarming_targets)
       if err:
-          raise MBErr('Error: Failed to match swarming targets to %s:\n%s' %
-                      ('//testing/buildbot/gn_isolate_map.pyl', err))
+          raise MBErr(err)
 
       gn_runtime_deps_path = self.ToAbsPath(build_dir, 'runtime_deps')
-      self.WriteFile(gn_runtime_deps_path, '\n'.join(gn_labels) + '\n')
+      self.WriteFile(gn_runtime_deps_path, '\n'.join(labels) + '\n')
       cmd.append('--runtime-deps-list-file=%s' % gn_runtime_deps_path)
 
     ret, _, _ = self.Run(cmd)
     if ret:
         # If `gn gen` failed, we should exit early rather than trying to
         # generate isolates. Run() will have already logged any error output.
         self.Print('GN gen failed: %d' % ret)
         return ret
 
     android = 'target_os="android"' in vals['gn_args']
     for target in swarming_targets:
       if android:
         # Android targets may be either android_apk or executable. The former
         # will result in runtime_deps associated with the stamp file, while the
         # latter will result in runtime_deps associated with the executable.
-        target_name = self.GNTargetName(target)
-        label = gn_isolate_map[target_name]['label']
+        label = isolate_map[target]['label']
         runtime_deps_targets = [
-            target_name + '.runtime_deps',
+            target + '.runtime_deps',
             'obj/%s.stamp.runtime_deps' % label.replace(':', '/')]
-      elif gn_isolate_map[target]['type'] == 'gpu_browser_test':
+      elif isolate_map[target]['type'] == 'gpu_browser_test':
         if self.platform == 'win32':
           runtime_deps_targets = ['browser_tests.exe.runtime_deps']
         else:
           runtime_deps_targets = ['browser_tests.runtime_deps']
-      elif (gn_isolate_map[target]['type'] == 'script' or
-            gn_isolate_map[target].get('label_type') == 'group'):
+      elif (isolate_map[target]['type'] == 'script' or
+            isolate_map[target].get('label_type') == 'group'):
         # For script targets, the build target is usually a group,
         # for which gn generates the runtime_deps next to the stamp file
         # for the label, which lives under the obj/ directory.
-        label = gn_isolate_map[target]['label']
+        label = isolate_map[target]['label']
         runtime_deps_targets = [
             'obj/%s.stamp.runtime_deps' % label.replace(':', '/')]
       elif self.platform == 'win32':
         runtime_deps_targets = [target + '.exe.runtime_deps']
       else:
         runtime_deps_targets = [target + '.runtime_deps']
 
       for r in runtime_deps_targets:
         runtime_deps_path = self.ToAbsPath(build_dir, r)
         if self.Exists(runtime_deps_path):
           break
       else:
         raise MBErr('did not generate any of %s' %
                     ', '.join(runtime_deps_targets))
 
-      command, extra_files = self.GetIsolateCommand(target, vals,
-                                                    gn_isolate_map)
+      command, extra_files = self.GetIsolateCommand(target, vals)
 
       runtime_deps = self.ReadFile(runtime_deps_path).splitlines()
 
       self.WriteIsolateFiles(build_dir, command, target, runtime_deps,
                              extra_files)
 
     return 0
 
   def RunGNIsolate(self, vals):
-    gn_isolate_map = ast.literal_eval(self.ReadFile(self.PathJoin(
-        self.chromium_src_dir, 'testing', 'buildbot', 'gn_isolate_map.pyl')))
+    target = self.args.target[0]
+    isolate_map = self.ReadIsolateMap()
+    err, labels = self.MapTargetsToLabels(isolate_map, [target])
+    if err:
+      raise MBErr(err)
+    label = labels[0]
 
     build_dir = self.args.path[0]
-    target = self.args.target[0]
-    target_name = self.GNTargetName(target)
-    command, extra_files = self.GetIsolateCommand(target, vals, gn_isolate_map)
+    command, extra_files = self.GetIsolateCommand(target, vals)
 
-    label = gn_isolate_map[target_name]['label']
     cmd = self.GNCmd('desc', build_dir, label, 'runtime_deps')
     ret, out, _ = self.Call(cmd)
     if ret:
       if out:
         self.Print(out)
       return ret
 
     runtime_deps = out.splitlines()
 
     self.WriteIsolateFiles(build_dir, command, target, runtime_deps,
@@ skipping 29 matching lines @@
           self.ToSrcRelPath('%s/%s.isolated' % (build_dir, target)),
           '--isolate',
           self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target)),
         ],
         'dir': self.chromium_src_dir,
         'version': 1,
       },
       isolate_path + 'd.gen.json',
     )
 
+  def MapTargetsToLabels(self, isolate_map, targets):
+    labels = []
+    err = ''
+
+    def StripTestSuffixes(target):
+      for suffix in ('_apk_run', '_apk', '_run'):
+        if target.endswith(suffix):
+          return target[:-len(suffix)], suffix
+      return None, None
+
+    for target in targets:
+      if target == 'all':
+        labels.append(target)
+      elif target.startswith('//'):
+        labels.append(target)
+      else:
+        if target in isolate_map:
+          stripped_target, suffix = target, ''
+        else:
+          stripped_target, suffix = StripTestSuffixes(target)
+        if stripped_target in isolate_map:
+          if isolate_map[stripped_target]['type'] == 'unknown':
+            err += ('test target "%s" type is unknown\n' % target)
+          else:
+            labels.append(isolate_map[stripped_target]['label'] + suffix)
+        else:
+          err += ('target "%s" not found in '
+                  '//testing/buildbot/gn_isolate_map.pyl\n' % target)
+
+    return err, labels
+
   def GNCmd(self, subcommand, path, *args):
     if self.platform == 'linux2':
       subdir, exe = 'linux64', 'gn'
     elif self.platform == 'darwin':
       subdir, exe = 'mac', 'gn'
     else:
       subdir, exe = 'win', 'gn.exe'
 
     gn_path = self.PathJoin(self.chromium_src_dir, 'buildtools', subdir, exe)
+    return [gn_path, subcommand, path] + list(args)
 
-    return [gn_path, subcommand, path] + list(args)
 
   def GNArgs(self, vals):
     if vals['cros_passthrough']:
       if not 'GN_ARGS' in os.environ:
         raise MBErr('MB is expecting GN_ARGS to be in the environment')
       gn_args = os.environ['GN_ARGS']
       if not re.search('target_os.*=.*"chromeos"', gn_args):
         raise MBErr('GN_ARGS is missing target_os = "chromeos": (GN_ARGS=%s)' %
                     gn_args)
     else:
@@ skipping 45 matching lines @@
     ret, _, _ = self.Run(cmd, env=env)
     if not ret and self.args.verbose:
       outp = json.loads(self.ReadFile(self.args.output_path[0]))
       self.Print()
       self.Print('analyze output:')
       self.PrintJSON(outp)
       self.Print()
 
     return ret
 
-  def GetIsolateCommand(self, target, vals, gn_isolate_map):
+  def GetIsolateCommand(self, target, vals):
     android = 'target_os="android"' in vals['gn_args']
 
     # This needs to mirror the settings in //build/config/ui.gni:
     # use_x11 = is_linux && !use_ozone.
     use_x11 = (self.platform == 'linux2' and
                not android and
                not 'use_ozone=true' in vals['gn_args'])
 
     asan = 'is_asan=true' in vals['gn_args']
     msan = 'is_msan=true' in vals['gn_args']
     tsan = 'is_tsan=true' in vals['gn_args']
 
-    target_name = self.GNTargetName(target)
-    test_type = gn_isolate_map[target_name]['type']
+    isolate_map = self.ReadIsolateMap()
+    test_type = isolate_map[target]['type']
 
-    executable = gn_isolate_map[target_name].get('executable', target_name)
+    executable = isolate_map[target].get('executable', target)
     executable_suffix = '.exe' if self.platform == 'win32' else ''
 
     cmdline = []
     extra_files = []
 
+    if test_type == 'nontest':
+      self.WriteFailureAndRaise('We should not be isolating %s.' % target,
+                                output_path=None)
+
     if android and test_type != "script":
       logdog_command = [
           '--logdog-bin-cmd', './../../bin/logdog_butler',
           '--project', 'chromium',
           '--service-account-json',
           '/creds/service_accounts/service-account-luci-logdog-publisher.json',
           '--prefix', 'android/swarming/logcats/${SWARMING_TASK_ID}',
           '--source', '${ISOLATED_OUTDIR}/logcats',
           '--name', 'unified_logcats',
       ]
       test_cmdline = [
-          self.PathJoin('bin', 'run_%s' % target_name),
+          self.PathJoin('bin', 'run_%s' % target),
           '--logcat-output-file', '${ISOLATED_OUTDIR}/logcats',
           '--target-devices-file', '${SWARMING_BOT_FILE}',
           '-v'
       ]
       cmdline = (['./../../build/android/test_wrapper/logdog_wrapper.py']
                  + logdog_command + test_cmdline)
     elif use_x11 and test_type == 'windowed_test_launcher':
       extra_files = [
           'xdisplaycheck',
           '../../testing/test_env.py',
@@ skipping 19 matching lines @@
           '--brave-new-test-launcher',
           '--test-launcher-bot-mode',
           '--asan=%d' % asan,
           '--msan=%d' % msan,
           '--tsan=%d' % tsan,
       ]
     elif test_type == 'gpu_browser_test':
       extra_files = [
           '../../testing/test_env.py'
       ]
-      gtest_filter = gn_isolate_map[target]['gtest_filter']
+      gtest_filter = isolate_map[target]['gtest_filter']
       cmdline = [
           '../../testing/test_env.py',
           './browser_tests' + executable_suffix,
           '--test-launcher-bot-mode',
           '--enable-gpu',
           '--test-launcher-jobs=1',
           '--gtest_filter=%s' % gtest_filter,
       ]
     elif test_type == 'script':
       extra_files = [
           '../../testing/test_env.py'
       ]
       cmdline = [
           '../../testing/test_env.py',
-          '../../' + self.ToSrcRelPath(gn_isolate_map[target]['script'])
+          '../../' + self.ToSrcRelPath(isolate_map[target]['script'])
       ]
     elif test_type in ('raw'):
       extra_files = []
       cmdline = [
           './' + str(target) + executable_suffix,
       ]
 
     else:
       self.WriteFailureAndRaise('No command line for %s found (test type %s).'
                                 % (target, test_type), output_path=None)
 
-    cmdline += gn_isolate_map[target_name].get('args', [])
+    cmdline += isolate_map[target].get('args', [])
 
     return cmdline, extra_files
 
   def ToAbsPath(self, build_path, *comps):
     return self.PathJoin(self.chromium_src_dir,
                          self.ToSrcRelPath(build_path),
                          *comps)
 
   def ToSrcRelPath(self, path):
     """Returns a relative path from the top of the repo."""
@@ skipping 75 matching lines @@
     if 'GYP_CHROMIUM_NO_ACTION' in env:
       del env['GYP_CHROMIUM_NO_ACTION']
     if 'GYP_CROSSCOMPILE' in env:
       del env['GYP_CROSSCOMPILE']
     env['GYP_DEFINES'] = gyp_defines
     if vals['gyp_crosscompile']:
       env['GYP_CROSSCOMPILE'] = '1'
     return cmd, env
 
   def RunGNAnalyze(self, vals):
-    # analyze runs before 'gn gen' now, so we need to run gn gen
+    # Analyze runs before 'gn gen' now, so we need to run gn gen
     # in order to ensure that we have a build directory.
     ret = self.RunGNGen(vals)
     if ret:
       return ret
 
+    build_path = self.args.path[0]
+    input_path = self.args.input_path[0]
+    gn_input_path = input_path + '.gn'
+    output_path = self.args.output_path[0]
+    gn_output_path = output_path + '.gn'
+
     inp = self.ReadInputJSON(['files', 'test_targets',
                               'additional_compile_targets'])
     if self.args.verbose:
       self.Print()
       self.Print('analyze input:')
       self.PrintJSON(inp)
       self.Print()
 
-    # TODO(crbug.com/555273) - currently GN treats targets and
-    # additional_compile_targets identically since we can't tell the
-    # difference between a target that is a group in GN and one that isn't.
-    # We should eventually fix this and treat the two types differently.
-    targets = (set(inp['test_targets']) |
-               set(inp['additional_compile_targets']))
-
-    output_path = self.args.output_path[0]
-
-    # Bail out early if a GN file was modified, since 'gn refs' won't know
-    # what to do about it. Also, bail out early if 'all' was asked for,
-    # since we can't deal with it yet.
-    if (any(f.endswith('.gn') or f.endswith('.gni') for f in inp['files']) or
-        'all' in targets):
-      self.WriteJSON({
-            'status': 'Found dependency (all)',
-            'compile_targets': sorted(targets),
-            'test_targets': sorted(targets & set(inp['test_targets'])),
-          }, output_path)
-      return 0
 
     # This shouldn't normally happen, but could due to unusual race conditions,
     # like a try job that gets scheduled before a patch lands but runs after
     # the patch has landed.
     if not inp['files']:
       self.Print('Warning: No files modified in patch, bailing out early.')
       self.WriteJSON({
             'status': 'No dependency',
             'compile_targets': [],
             'test_targets': [],
           }, output_path)
       return 0
 
-    ret = 0
-    response_file = self.TempFile()
-    response_file.write('\n'.join(inp['files']) + '\n')
-    response_file.close()
+    gn_inp = {}
+    gn_inp['files'] = ['//' + f for f in inp['files'] if not f.startswith('//')]
 
-    matching_targets = set()
+    isolate_map = self.ReadIsolateMap()
+    err, gn_inp['additional_compile_targets'] = self.MapTargetsToLabels(
+        isolate_map, inp['additional_compile_targets'])
+    if err:
+      raise MBErr(err)
+
+    err, gn_inp['test_targets'] = self.MapTargetsToLabels(
+        isolate_map, inp['test_targets'])
+    if err:
+      raise MBErr(err)
+    labels_to_targets = {}
+    for i, label in enumerate(gn_inp['test_targets']):
+      labels_to_targets[label] = inp['test_targets'][i]
+
     try:
-      cmd = self.GNCmd('refs',
-                       self.args.path[0],
-                       '@%s' % response_file.name,
-                       '--all',
-                       '--as=output')
-      ret, out, _ = self.Run(cmd, force_verbose=False)
-      if ret and not 'The input matches no targets' in out:
-        self.WriteFailureAndRaise('gn refs returned %d: %s' % (ret, out),
-                                  output_path)
-      build_dir = self.ToSrcRelPath(self.args.path[0]) + self.sep
-      for output in out.splitlines():
-        build_output = output.replace(build_dir, '')
-        if build_output in targets:
-          matching_targets.add(build_output)
+      self.WriteJSON(gn_inp, gn_input_path)
+      cmd = self.GNCmd('analyze', build_path, gn_input_path, gn_output_path)
+      ret, _, _ = self.Run(cmd, force_verbose=True)
+      if ret:
+        return ret
 
-      cmd = self.GNCmd('refs',
-                       self.args.path[0],
-                       '@%s' % response_file.name,
-                       '--all')
-      ret, out, _ = self.Run(cmd, force_verbose=False)
-      if ret and not 'The input matches no targets' in out:
-        self.WriteFailureAndRaise('gn refs returned %d: %s' % (ret, out),
-                                  output_path)
-      for label in out.splitlines():
-        build_target = label[2:]
-        # We want to accept 'chrome/android:chrome_public_apk' and
-        # just 'chrome_public_apk'. This may result in too many targets
-        # getting built, but we can adjust that later if need be.
-        for input_target in targets:
-          if (input_target == build_target or
-              build_target.endswith(':' + input_target)):
-            matching_targets.add(input_target)
+      gn_outp_str = self.ReadFile(gn_output_path)
+      try:
+        gn_outp = json.loads(gn_outp_str)
+      except Exception as e:
+        self.Print("Failed to parse the JSON string GN returned: %s\n%s"
+                   % (repr(gn_outp_str), str(e)))
+        raise
+
+      outp = {}
+      if 'status' in gn_outp:
+        outp['status'] = gn_outp['status']
+      if 'error' in gn_outp:
+        outp['error'] = gn_outp['error']
+      if 'invalid_targets' in gn_outp:
+        outp['invalid_targets'] = gn_outp['invalid_targets']
+      if 'compile_targets' in gn_outp:
+        outp['compile_targets'] = [
+          label.replace('//', '') for label in gn_outp['compile_targets']]
+      if 'test_targets' in gn_outp:
+        outp['test_targets'] = [
+          labels_to_targets[label] for label in gn_outp['test_targets']]
+
+      if self.args.verbose:
+        self.Print()
+        self.Print('analyze output:')
+        self.PrintJSON(outp)
+        self.Print()
+
+      self.WriteJSON(outp, output_path)
+
     finally:
-      self.RemoveFile(response_file.name)
-
-    if matching_targets:
-      self.WriteJSON({
-            'status': 'Found dependency',
-            'compile_targets': sorted(matching_targets),
-            'test_targets': sorted(matching_targets &
-                                   set(inp['test_targets'])),
-          }, output_path)
-    else:
-      self.WriteJSON({
-          'status': 'No dependency',
-          'compile_targets': [],
-          'test_targets': [],
-      }, output_path)
-
-    if self.args.verbose:
-      outp = json.loads(self.ReadFile(output_path))
-      self.Print()
-      self.Print('analyze output:')
-      self.PrintJSON(outp)
-      self.Print()
+      if self.Exists(gn_input_path):
+        self.RemoveFile(gn_input_path)
+      if self.Exists(gn_output_path):
+        self.RemoveFile(gn_output_path)
 
     return 0
 
   def ReadInputJSON(self, required_keys):
     path = self.args.input_path[0]
     output_path = self.args.output_path[0]
     if not self.Exists(path):
       self.WriteFailureAndRaise('"%s" does not exist' % path, output_path)
 
     try:
@@ skipping 62 matching lines @@
     print_env('GYP_LINK_CONCURRENCY')
     print_env('LLVM_FORCE_HEAD_REVISION')
 
     if cmd[0] == self.executable:
       cmd = ['python'] + cmd[1:]
     self.Print(*[shell_quoter(arg) for arg in cmd])
 
   def PrintJSON(self, obj):
     self.Print(json.dumps(obj, indent=2, sort_keys=True))
 
-  def GNTargetName(self, target):
-    return target
-
   def Build(self, target):
     build_dir = self.ToSrcRelPath(self.args.path[0])
     ninja_cmd = ['ninja', '-C', build_dir]
     if self.args.jobs:
       ninja_cmd.extend(['-j', '%d' % self.args.jobs])
     ninja_cmd.append(target)
     ret, _, _ = self.Run(ninja_cmd, force_verbose=False, buffer_output=False)
     return ret
 
   def Run(self, cmd, env=None, force_verbose=True, buffer_output=True):
@@ skipping 122 matching lines @@
   # Then check to see if the arg contains any metacharacters other than
   # double quotes; if it does, quote everything (including the double
   # quotes) for safety.
   if any(a in UNSAFE_FOR_CMD for a in arg):
     arg = ''.join('^' + a if a in ALL_META_CHARS else a for a in arg)
   return arg
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv[1:]))