Improve manage.py to automate CL creation further.

testing/buildbot/manage.py

 #!/usr/bin/env python
 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Toolbox to manage all the json files in this directory.
 
 It can reformat them in their canonical format or ensures they are well
 formatted.
 """
@@ skipping 75 matching lines @@
 class Error(Exception):
   """Processing error."""
 
 
 def get_isolates():
   """Returns the list of all isolate files."""
   files = subprocess.check_output(['git', 'ls-files'], cwd=SRC_DIR).splitlines()
   return [os.path.basename(f) for f in files if f.endswith('.isolate')]
 
 
-def process_builder_convert(data, filename, builder, test_name):
+def process_builder_convert(data, test_name):
   """Converts 'test_name' to run on Swarming in 'data'.
 
   Returns True if 'test_name' was found.
   """
   result = False
   for test in data['gtest_tests']:
     if test['test'] != test_name:
       continue
     test.setdefault('swarming', {})
     if not test['swarming'].get('can_use_on_swarming_builders'):
-      print('- %s: %s' % (filename, builder))
       test['swarming']['can_use_on_swarming_builders'] = True
     result = True
   return result
 
 
 def process_builder_remaining(data, filename, builder, tests_location):
   """Calculates tests_location when mode is --remaining."""
   for test in data['gtest_tests']:
     name = test['test']
     if test.get('swarming', {}).get('can_use_on_swarming_builders'):
       tests_location[name]['count_run_on_swarming'] += 1
     else:
       tests_location[name]['count_run_local'] += 1
       tests_location[name]['local_configs'].setdefault(
           filename, []).append(builder)
 
 
 def process_file(mode, test_name, tests_location, filepath, ninja_targets,
                  ninja_targets_seen):
-  """Processes a file.
+  """Processes a json file describing what tests should be run for each recipe.
 
   The action depends on mode. Updates tests_location.
 
   Return False if the process exit code should be 1.
   """
   filename = os.path.basename(filepath)
   with open(filepath) as f:
     content = f.read()
   try:
     config = json.loads(content)
@@ skipping 18 matching lines @@
     for d in data['gtest_tests']:
       if (d['test'] not in ninja_targets and
           d['test'] not in SKIP_NINJA_TO_GN_TARGETS):
         raise Error('%s: %s / %s is not listed in ninja_to_gn.pyl.' %
                     (filename, builder, d['test']))
       elif d['test'] in ninja_targets:
         ninja_targets_seen.add(d['test'])
 
     config[builder]['gtest_tests'] = sorted(
         data['gtest_tests'], key=lambda x: x['test'])
-    if mode == 'remaining':
+
+    # The trick here is that process_builder_remaining() is called before
+    # process_builder_convert() so tests_location can be used to know how many
+    # tests were converted.
+    if mode in ('convert', 'remaining'):
       process_builder_remaining(data, filename, builder, tests_location)
-    elif mode == 'convert':
-      process_builder_convert(data, filename, builder, test_name)
+    if mode == 'convert':
+      process_builder_convert(data, test_name)
 
   expected = json.dumps(
       config, sort_keys=True, indent=2, separators=(',', ': ')) + '\n'
   if content != expected:
     if mode in ('convert', 'write'):
       with open(filepath, 'wb') as f:
         f.write(expected)
       if mode == 'write':
         print('Updated %s' % filename)
     else:
       print('%s is not in canonical format' % filename)
       print('run `testing/buildbot/manage.py -w` to fix')
     return mode != 'check'
   return True
 
 
-def print_remaining(test_name,tests_location):
+def print_convert(test_name, tests_location):
+  """Prints statistics for a test being converted for use in a CL description.
+  """
+  data = tests_location[test_name]
+  print('Convert %s to run exclusively on Swarming' % test_name)
+  print('')
+  print(
+      '%d configs used to already run on Swarming' %

[Dirk Pranke, 2015/06/23 19:25:41]

"%d configs already ran on Swarming" maybe?

[M-A Ruel, 2015/06/23 19:29:51]

Done.

+      data['count_run_on_swarming'])

[Dirk Pranke, 2015/06/23 19:24:39]

this formatting is a bit odd; why isn't the string on the first line?

+  print('%d used to run locally and were converted:' % data['count_run_local'])
+  for master, builders in sorted(data['local_configs'].iteritems()):
+    for builder in builders:
+      print('- %s: %s' % (master, builder))
+  print('')
+  print('Ran:')
+  print('  ./manage.py --convert %s' % test_name)
+  print('')
+  print('R=')
+  print('BUG=98637')
+
+
+def print_remaining(test_name, tests_location):
   """Prints a visual summary of what tests are yet to be converted to run on
   Swarming.
   """
   if test_name:
     if test_name not in tests_location:
       raise Error('Unknown test %s' % test_name)
     for config, builders in sorted(
         tests_location[test_name]['local_configs'].iteritems()):
       print('%s:' % config)
       for builder in sorted(builders):
@@ skipping 79 matching lines @@
 
     extra_targets = set(ninja_targets) - ninja_targets_seen
     if extra_targets:
       if len(extra_targets) > 1:
         extra_targets_str = ', '.join(extra_targets) + ' are'
       else:
         extra_targets_str = list(extra_targets)[0] + ' is'
       raise Error('%s listed in ninja_to_gn.pyl but not in any .json files' %
                   extra_targets_str)
 
-    if args.mode == 'remaining':
+    if args.mode == 'convert':
+      print_convert(args.test_name, tests_location)
+    elif args.mode == 'remaining':
       print_remaining(args.test_name, tests_location)
     return result
   except Error as e:
     sys.stderr.write('%s\n' % e)
     return 1
 
 
 if __name__ == "__main__":
   sys.exit(main())