Rework how isolates are generated in MB.

tools/mb/mb.py

 #!/usr/bin/env python
 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """MB - the Meta-Build wrapper around GYP and GN
 
 MB is a wrapper script for GYP and GN that can be used to generate build files
 for sets of canned configurations and analyze them.
 """
@@ skipping 318 matching lines @@
     cmd = self.GNCmd('gen', path, vals['gn_args'])
 
     swarming_targets = []
     if self.args.swarming_targets_file:
       # We need GN to generate the list of runtime dependencies for
       # the compile targets listed (one per line) in the file so
       # we can run them via swarming. We use ninja_to_gn.pyl to convert
       # the compile targets to the matching GN labels.
       contents = self.ReadFile(self.args.swarming_targets_file)
       swarming_targets = contents.splitlines()
-      ninja_targets_to_labels = ast.literal_eval(self.ReadFile(os.path.join(
-          self.chromium_src_dir, 'testing', 'buildbot', 'ninja_to_gn.pyl')))
+      gn_isolate_map = ast.literal_eval(self.ReadFile(os.path.join(
+          self.chromium_src_dir, 'testing', 'buildbot', 'gn_isolate_map.pyl')))
       gn_labels = []
       for target in swarming_targets:
-        if not target in ninja_targets_to_labels:
+        if not target in gn_isolate_map:
           raise MBErr('test target "%s"  not found in %s' %
-                      (target, '//testing/buildbot/ninja_to_gn.pyl'))
-        gn_labels.append(ninja_targets_to_labels[target])
+                      (target, '//testing/buildbot/gn_isolate_map.pyl'))
+        gn_labels.append(gn_isolate_map[target]['label'])
 
       gn_runtime_deps_path = self.ToAbsPath(path, 'runtime_deps')
 
       # Since GN hasn't run yet, the build directory may not even exist.
       self.MaybeMakeDirectory(self.ToAbsPath(path))
 
       self.WriteFile(gn_runtime_deps_path, '\n'.join(gn_labels) + '\n')
       cmd.append('--runtime-deps-list-file=%s' % gn_runtime_deps_path)
 
     ret, _, _ = self.Run(cmd)
 
     for target in swarming_targets:
       if sys.platform == 'win32':
         deps_path = self.ToAbsPath(path, target + '.exe.runtime_deps')
       else:
         deps_path = self.ToAbsPath(path, target + '.runtime_deps')
       if not self.Exists(deps_path):
           raise MBErr('did not generate %s' % deps_path)
 
-      command, extra_files = self.GetIsolateCommand(target, vals)
+      command, extra_files = self.GetIsolateCommand(target, vals,
+                                                    gn_isolate_map)
 
       runtime_deps = self.ReadFile(deps_path).splitlines()
 
       isolate_path = self.ToAbsPath(path, target + '.isolate')
       self.WriteFile(isolate_path,
         pprint.pformat({
           'variables': {
             'command': command,
             'files': sorted(runtime_deps + extra_files),
           }
@@ skipping 64 matching lines @@
     ret, _, _ = self.Run(cmd)
     if not ret and self.args.verbose:
       outp = json.loads(self.ReadFile(self.args.output_path[0]))
       self.Print()
       self.Print('analyze output:')
       self.PrintJSON(outp)
       self.Print()
 
     return ret
 
-  def GetIsolateCommand(self, target, vals):
-    extra_files = []
+  def RunGNIsolate(self, vals):
+    build_path = self.args.path[0]
+    inp = self.ReadInputJSON(['targets'])
+    if self.args.verbose:
+      self.Print()
+      self.Print('isolate input:')
+      self.PrintJSON(inp)
+      self.Print()
+    output_path = self.args.output_path[0]
 
-    # TODO(dpranke): We should probably pull this from
-    # the test list info in //testing/buildbot/*.json,
-    # and assert that the test has can_use_on_swarming_builders: True,
-    # but we hardcode it here for now.
-    test_type = {}.get(target, 'gtest_test')
+    for target in inp['targets']:
+      runtime_deps_path = self.ToAbsPath(build_path, target + '.runtime_deps')
 
+      if not self.Exists(runtime_deps_path):
+        self.WriteFailureAndRaise('"%s" does not exist' % runtime_deps_path,
+                                  output_path)
+
+      command, extra_files = self.GetIsolateCommand(target, vals, None)
+
+      runtime_deps = self.ReadFile(runtime_deps_path).splitlines()
+
+
+      isolate_path = self.ToAbsPath(build_path, target + '.isolate')
+      self.WriteFile(isolate_path,
+        pprint.pformat({
+          'variables': {
+            'command': command,
+            'files': sorted(runtime_deps + extra_files),
+            'read_only': 1,

[M-A Ruel, 2015/07/22 18:22:42]

Not needed

+          }
+        }) + '\n')
+
+      self.WriteJSON(
+        {
+          'args': [
+            '--isolated',
+            self.ToSrcRelPath('%s/%s.isolated' % (build_path, target)),
+            '--isolate',
+            self.ToSrcRelPath('%s/%s.isolate' % (build_path, target)),
+          ],
+          'dir': self.chromium_src_dir,
+          'version': 1,
+        },
+        isolate_path + 'd.gen.json',
+      )
+
+    return 0
+
+  def GetIsolateCommand(self, target, vals, gn_isolate_map):
     # This needs to mirror the settings in //build/config/ui.gni:
     # use_x11 = is_linux && !use_ozone.
     # TODO(dpranke): Figure out how to keep this in sync better.
     use_x11 = (sys.platform == 'linux2' and
                not 'target_os="android"' in vals['gn_args'] and
                not 'use_ozone=true' in vals['gn_args'])
 
     asan = 'is_asan=true' in vals['gn_args']
     msan = 'is_msan=true' in vals['gn_args']
     tsan = 'is_tsan=true' in vals['gn_args']
 
     executable_suffix = '.exe' if sys.platform == 'win32' else ''
 
-    if test_type == 'gtest_test':
-      extra_files.append('../../testing/test_env.py')
+    test_type = gn_isolate_map[target]['type']
+    cmdline = []
+    extra_files = []
 
-      if use_x11:
-        # TODO(dpranke): Figure out some way to figure out which
-        # test steps really need xvfb.
-        extra_files.append('xdisplaycheck')
-        extra_files.append('../../testing/xvfb.py')
-
-        cmdline = [
+    if use_x11 and test_type == 'windowed_test_launcher':
+      extra_files = [
+          'xdisplaycheck',
+          '../../testing/test_env.py',
           '../../testing/xvfb.py',
-          '.',
-          './' + str(target),
-          '--brave-new-test-launcher',
-          '--test-launcher-bot-mode',
-          '--asan=%d' % asan,
-          '--msan=%d' % msan,
-          '--tsan=%d' % tsan,
-        ]
-      else:
-        cmdline = [
+      ]
+      cmdline = [
+        '../../testing/xvfb.py',
+        '.',
+        './' + str(target),
+        '--brave-new-test-launcher',
+        '--test-launcher-bot-mode',
+        '--asan=%d' % asan,
+        '--msan=%d' % msan,
+        '--tsan=%d' % tsan,
+      ]
+    elif test_type in ('windowed_test_launcher', 'console_test_launcher'):
+      extra_files = [
+          '../../testing/test_env.py'
+      ]
+      cmdline = [
           '../../testing/test_env.py',
-          '.',
           './' + str(target) + executable_suffix,
           '--brave-new-test-launcher',
           '--test-launcher-bot-mode',
           '--asan=%d' % asan,
           '--msan=%d' % msan,
           '--tsan=%d' % tsan,
-        ]
+      ]
+    elif test_type in ('raw'):
+      extra_files = []
+      cmdline = [
+          './' + str(target) + executable_suffix,
+      ] + gn_isolate_map[target].get('args')
+
     else:
-      # TODO(dpranke): Handle script_tests and other types of swarmed tests.
-      self.WriteFailureAndRaise('unknown test type "%s" for %s' %
-                                (test_type, target), output_path=None)
-
+      self.WriteFailureAndRaise('No command line for %s found (test type %s).'
+                                % (target, test_type), output_path=None)
 
     return cmdline, extra_files
 
   def ToAbsPath(self, build_path, *comps):
     return os.path.join(self.chromium_src_dir,
                         self.ToSrcRelPath(build_path),
                         *comps)
 
   def ToSrcRelPath(self, path):
     """Returns a relative path from the top of the repo."""
@@ skipping 219 matching lines @@
 
 if __name__ == '__main__':
   try:
     sys.exit(main(sys.argv[1:]))
   except MBErr as e:
     print(e)
     sys.exit(1)
   except KeyboardInterrupt:
     print("interrupted, exiting", stream=sys.stderr)
     sys.exit(130)