submit_try: Obtain the list of trybots from the checked-in slaves.cfg

tools/submit_try

 #!/usr/bin/python
 
 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 
 """
 submit_try: Submit a try request.
 
@@ skipping 17 matching lines @@
 
 # Alias which can be used to run a try on every builder.
 ALL_BUILDERS = 'all'
 # Alias which can be used to run a try on all compile builders.
 COMPILE_BUILDERS = 'compile'
 # Alias which can be used to run a try on all builders that are run in the CQ.
 CQ_BUILDERS = 'cq'
 # Alias which can be used to specify a regex to choose builders.
 REGEX = 'regex'
 
-ALL_ALIASES = [ALL_BUILDERS, COMPILE_BUILDERS, CQ_BUILDERS, REGEX]
-
-# Contact information for the build master.
-SKIA_BUILD_MASTER_HOST = str(buildbot_globals.Get('public_master_host'))
-SKIA_BUILD_MASTER_PORT = str(buildbot_globals.Get('public_external_port'))
+ALL_ALIASES = [ALL_BUILDERS, COMPILE_BUILDERS, REGEX, CQ_BUILDERS]

[rmistry, 2014/01/14 17:39:54]

Why rearrange this? it was sorted before.

[borenet, 2014/01/14 18:00:05]

See the format in the help string below.  I wanted CQ builders to come last so
that I didn't have to use a format dictionary.

 
 # All try builders have this suffix.
 TRYBOT_SUFFIX = '-Trybot'
 
 # String for matching the svn url of the try server inside codereview.settings.
 TRYSERVER_SVN_URL = 'TRYSERVER_SVN_URL: '
 
 # Strings used for matching svn config properties.
 URL_STR = 'URL'
 REPO_ROOT_STR = 'Repository Root'
@@ skipping 41 matching lines @@
   codereview_settings_file = os.path.join(os.path.dirname(__file__), os.pardir,
                                           'codereview.settings')
   with open(codereview_settings_file) as f:
     for line in f:
       if line.startswith(TRYSERVER_SVN_URL):
         return line[len(TRYSERVER_SVN_URL):].rstrip()
   raise Exception('Couldn\'t determine the TRYSERVER_SVN_URL. Make sure it is '
                   'defined in the %s file.' % codereview_settings_file)
 
 
-def RetrieveTrybotList(json_filename):
-  """ Retrieve the list of known trybots from the build master, stripping
-  TRYBOT_SUFFIX from the name. """
-  trybots = []
-  connection = httplib.HTTPConnection(SKIA_BUILD_MASTER_HOST,  
-                                      SKIA_BUILD_MASTER_PORT)
-  connection.request('GET', '/json/%s' % json_filename)
-  response = connection.getresponse()  
-  builders = json.load(response)
+def RetrieveTrybotList():
+  """Retrieve the list of known trybots from the checked-in buildbot
+  configuration."""
+  # Retrieve the slaves.cfg file from the repository.
+  slaves_cfg_url = ('https://skia.googlesource.com/buildbot/+/master/'
+                    'master/slaves.cfg')

[rmistry, 2014/01/14 17:39:54]

Can we make this a top-level constant.

[borenet, 2014/01/14 18:00:05]

Done.

+  slaves_cfg_text = buildbot_globals.retrieve_from_googlesource(slaves_cfg_url)
 
-  for builder in builders:
-    if builder.endswith(TRYBOT_SUFFIX):
-      trybots.append(builder[:-len(TRYBOT_SUFFIX)])
-  return trybots
+  # Execute the slaves.cfg file to obtain the list of slaves.
+  vars = {}
+  exec(slaves_cfg_text, vars)
+  slaves_cfg = vars['slaves']
+
+  # Pull the list of known builders from the slaves list.
+  trybots = set()
+  for slave in slaves_cfg:
+    for builder in slave['builder']:
+      if not builder.endswith(TRYBOT_SUFFIX):
+        trybots.add(builder)
+
+  return list(trybots), vars['cq_trybots']
 
 
-def ValidateArgs(argv, trybots, is_svn=True):
+def ValidateArgs(argv, trybots, cq_trybots, is_svn=True):
   """ Parse and validate command-line arguments. If the arguments are valid,
   returns a tuple of (<changelist name>, <list of trybots>).
 
-  trybots: A list of the known try builders.
+  trybots: list of strings; A list of the known try builders.
+  cq_trybots: list of strings; Trybots who get run by the commit queue.
+  is_svn: bool; whether or not we're in an svn checkout.
   """
 
   class CollectedArgs(object):
     def __init__(self, bots, changelist, revision):
       self._bots = bots
       self._changelist = changelist
       self._revision = revision
 
     @property
     def bots(self):
@@ skipping 27 matching lines @@
 
   using_bots = None
   changelist = None
   revision = None
 
   while argv:
     arg = argv.pop(0)
     if arg == '-h' or arg == '--help':
       Error()
     elif arg == '-l' or arg == '--list_bots':
-      format_args = ['\n  '.join(sorted(trybots))] + ALL_ALIASES
+      format_args = ['\n  '.join(sorted(trybots))] + \
+                    ALL_ALIASES + \
+                    ['\n    '.join(sorted(cq_trybots))]
       print (
 """
 submit_try: Available builders:\n  %s
 
 Can also use the following aliases to run on groups of builders-
   %s: Will run against all trybots.
   %s: Will run against all compile trybots.
-  %s: Will run against the same trybots as the commit queue.
   %s: You will be prompted to enter a regex to select builders with.
+  %s: Will run against the same trybots as the commit queue:\n    %s
 
 """ % tuple(format_args))
       sys.exit(0)
     elif arg == '-b' or arg == '--bot':
       if using_bots:
         Error('--bot specified multiple times.')
       if len(argv) < 1:
         Error('You must specify a builder with "--bot".')
       using_bots = []
       while argv and not argv[0].startswith('-'):
         for bot in argv.pop(0).split(','):
           if bot in ALL_ALIASES:
             if using_bots:
               Error('Cannot specify "%s" with additional builder names or '
                     'aliases.' % bot)
             if bot == ALL_BUILDERS:
               are_you_sure = raw_input('Running a try on every bot is very '
                                        'expensive. You may be able to get '
                                        'enough information by running on a '
                                        'smaller set of bots. Are you sure you '
                                        'want to run your try job on all of the '
                                        'trybots? [y,n]: ')
               if are_you_sure == 'y':
                 using_bots = trybots
             elif bot == COMPILE_BUILDERS:
               using_bots = [t for t in trybots if t.startswith('Build')]
             elif bot == CQ_BUILDERS:
-              using_bots = RetrieveTrybotList(json_filename='cqtrybots')
+              using_bots = cq_trybots
             elif bot == REGEX:
               while True:
                 regex = raw_input("Enter your trybot regex: ")
                 p = re.compile(regex)
                 using_bots = [t for t in trybots if p.match(t)]
                 print '\n\nTrybots that match your regex:\n%s\n\n' % '\n'.join(
                     using_bots)
                 if raw_input('Re-enter regex? [y,n]: ') == 'n':
                   break
             break
@@ skipping 72 matching lines @@
         try_args.extend(['-r', args.revision])
 
       # Submit the try request.
       trychange.TryChange(try_args, None, False)
     finally:
       shutil.rmtree(temp_dir)
 
 
 def main():
   # Retrieve the list of active try builders from the build master.
-  trybots = RetrieveTrybotList(json_filename='trybots')
+  trybots, cq_trybots = RetrieveTrybotList()
 
   # Determine if we're in an SVN checkout.
   is_svn = os.path.isdir('.svn')
 
   # Parse and validate the command-line arguments.
-  args = ValidateArgs(sys.argv[1:], trybots=trybots, is_svn=is_svn)
+  args = ValidateArgs(sys.argv[1:], trybots=trybots, cq_trybots=cq_trybots,
+                      is_svn=is_svn)
 
   # Submit the try request.
   SubmitTryRequest(args, is_svn=is_svn)
 
 
 if __name__ == '__main__':
   sys.exit(main())