| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # Copyright 2014 The Chromium Authors. All rights reserved. | 2 # Copyright 2014 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 | 5 |
| 6 import json | 6 import json |
| 7 import os | 7 import os |
| 8 import subprocess | 8 import subprocess |
| 9 import sys | 9 import sys |
| 10 import tempfile | 10 import tempfile |
| 11 | 11 |
| 12 | 12 |
| 13 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | 13 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
| 14 | 14 |
| 15 | 15 |
| 16 sys.path.insert(0, os.path.join(BASE_DIR, 'scripts')) |
| 17 sys.path.insert(0, os.path.join(BASE_DIR, 'scripts', 'slave', 'recipes')) |
| 18 sys.path.insert( |
| 19 0, os.path.join(BASE_DIR, 'scripts', 'slave', 'recipe_modules')) |
| 20 sys.path.insert(0, os.path.join(BASE_DIR, 'third_party')) |
| 21 |
| 22 import chromium_trybot |
| 23 from chromium_tests.builders import BUILDERS |
| 24 |
| 25 |
| 16 MAIN_WATERFALL_MASTERS = [ | 26 MAIN_WATERFALL_MASTERS = [ |
| 17 'master.chromium.chrome', | 27 'master.chromium.chrome', |
| 18 'master.chromium.chromiumos', | 28 'master.chromium.chromiumos', |
| 19 'master.chromium.gpu', | 29 'master.chromium.gpu', |
| 20 'master.chromium.linux', | 30 'master.chromium.linux', |
| 21 'master.chromium.mac', | 31 'master.chromium.mac', |
| 22 'master.chromium.memory', | 32 'master.chromium.memory', |
| 23 'master.chromium.win', | 33 'master.chromium.win', |
| 24 ] | 34 ] |
| 25 | 35 |
| (...skipping 59 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 85 ], | 95 ], |
| 86 } | 96 } |
| 87 | 97 |
| 88 UNUSED_MAIN_WATERFALL_BUILDERS = { | 98 UNUSED_MAIN_WATERFALL_BUILDERS = { |
| 89 'master.chromium.linux': [ | 99 'master.chromium.linux': [ |
| 90 'Android x86 Builder (dbg)', | 100 'Android x86 Builder (dbg)', |
| 91 ], | 101 ], |
| 92 } | 102 } |
| 93 | 103 |
| 94 | 104 |
| 95 def getBuilders(recipe_name): | |
| 96 """Asks the given recipe to dump its BUILDERS dictionary. | |
| 97 | |
| 98 This must be implemented by the recipe in question. | |
| 99 """ | |
| 100 (fh, builders_file) = tempfile.mkstemp('.json') | |
| 101 os.close(fh) | |
| 102 try: | |
| 103 subprocess.check_call([ | |
| 104 os.path.join(BASE_DIR, 'scripts', 'slave', 'recipes.py'), | |
| 105 'run', recipe_name, 'dump_builders=%s' % builders_file]) | |
| 106 with open(builders_file) as fh: | |
| 107 return json.load(fh) | |
| 108 finally: | |
| 109 os.remove(builders_file) | |
| 110 | |
| 111 | |
| 112 def getMasterConfig(master): | 105 def getMasterConfig(master): |
| 113 with tempfile.NamedTemporaryFile() as f: | 106 with tempfile.NamedTemporaryFile() as f: |
| 114 subprocess.check_call([ | 107 subprocess.check_call([ |
| 115 os.path.join(BASE_DIR, 'scripts', 'tools', 'runit.py'), | 108 os.path.join(BASE_DIR, 'scripts', 'tools', 'runit.py'), |
| 116 os.path.join(BASE_DIR, 'scripts', 'tools', 'dump_master_cfg.py'), | 109 os.path.join(BASE_DIR, 'scripts', 'tools', 'dump_master_cfg.py'), |
| 117 os.path.join(BASE_DIR, 'masters/%s' % master), | 110 os.path.join(BASE_DIR, 'masters/%s' % master), |
| 118 f.name]) | 111 f.name]) |
| 119 return json.load(f) | 112 return json.load(f) |
| 120 | 113 |
| 121 | 114 |
| 122 def getBuildersAndRecipes(master): | 115 def getBuildersAndRecipes(master): |
| 123 return { | 116 return { |
| 124 builder['name'] : builder['factory']['properties'].get( | 117 builder['name'] : builder['factory']['properties'].get( |
| 125 'recipe', [None])[0] | 118 'recipe', [None])[0] |
| 126 for builder in getMasterConfig(master)['builders'] | 119 for builder in getMasterConfig(master)['builders'] |
| 127 } | 120 } |
| 128 | 121 |
| 129 | 122 |
| 130 def mutualDifference(a, b): | 123 def mutualDifference(a, b): |
| 131 return a - b, b - a | 124 return a - b, b - a |
| 132 | 125 |
| 133 | 126 |
| 134 def main(argv): | 127 def main(argv): |
| 135 chromium_recipe_builders = {} | 128 chromium_recipe_builders = {} |
| 136 covered_builders = set() | 129 covered_builders = set() |
| 137 all_builders = set() | 130 all_builders = set() |
| 138 | 131 |
| 139 exit_code = 0 | 132 exit_code = 0 |
| 140 | 133 |
| 141 chromium_trybot_BUILDERS = getBuilders('chromium_trybot') | |
| 142 chromium_BUILDERS = getBuilders('chromium') | |
| 143 | |
| 144 for master in MAIN_WATERFALL_MASTERS: | 134 for master in MAIN_WATERFALL_MASTERS: |
| 145 builders = getBuildersAndRecipes(master) | 135 builders = getBuildersAndRecipes(master) |
| 146 all_builders.update((master, b) for b in builders) | 136 all_builders.update((master, b) for b in builders) |
| 147 | 137 |
| 148 # We only have a standardized way to mirror builders using the chromium | 138 # We only have a standardized way to mirror builders using the chromium |
| 149 # recipe on the tryserver. | 139 # recipe on the tryserver. |
| 150 chromium_recipe_builders[master] = [b for b in builders | 140 chromium_recipe_builders[master] = [b for b in builders |
| 151 if builders[b] == 'chromium'] | 141 if builders[b] == 'chromium'] |
| 152 | 142 |
| 153 # TODO(phajdan.jr): Also consider it an error if configured builders | 143 # TODO(phajdan.jr): Also consider it an error if configured builders |
| 154 # are not using chromium recipe. This might make it harder to experiment | 144 # are not using chromium recipe. This might make it harder to experiment |
| 155 # with switching bots over to chromium recipe though, so it may be better | 145 # with switching bots over to chromium recipe though, so it may be better |
| 156 # to just wait until the switch is done. | 146 # to just wait until the switch is done. |
| 157 recipe_side_builders = chromium_BUILDERS.get( | 147 recipe_side_builders = BUILDERS.get( |
| 158 master.replace('master.', ''), {}).get('builders') | 148 master.replace('master.', ''), {}).get('builders') |
| 159 if recipe_side_builders is not None: | 149 if recipe_side_builders is not None: |
| 160 bogus_builders = set(recipe_side_builders.keys()).difference( | 150 bogus_builders = set(recipe_side_builders.keys()).difference( |
| 161 set(builders.keys())) | 151 set(builders.keys())) |
| 162 bogus_builders, unused = mutualDifference( | 152 bogus_builders, unused = mutualDifference( |
| 163 bogus_builders, | 153 bogus_builders, |
| 164 set(UNUSED_MAIN_WATERFALL_BUILDERS.get(master, []))) | 154 set(UNUSED_MAIN_WATERFALL_BUILDERS.get(master, []))) |
| 165 # TODO(phajdan.jr): Clean up bogus chromiumos builders. | 155 # TODO(phajdan.jr): Clean up bogus chromiumos builders. |
| 166 if bogus_builders and master != 'master.chromium.chromiumos': | 156 if bogus_builders and master != 'master.chromium.chromiumos': |
| 167 exit_code = 1 | 157 exit_code = 1 |
| 168 print 'The following builders from chromium recipe' | 158 print 'The following builders from chromium recipe' |
| 169 print 'do not exist in master config for %s:' % master | 159 print 'do not exist in master config for %s:' % master |
| 170 print '\n'.join('\t%s' % b for b in sorted(bogus_builders)) | 160 print '\n'.join('\t%s' % b for b in sorted(bogus_builders)) |
| 171 if unused: | 161 if unused: |
| 172 exit_code = 1 | 162 exit_code = 1 |
| 173 print 'The following unused declarations are superfluous ' | 163 print 'The following unused declarations are superfluous ' |
| 174 print 'on %s' % master | 164 print 'on %s' % master |
| 175 print '\n'.join('\t%s' % b for b in sorted(unused)) | 165 print '\n'.join('\t%s' % b for b in sorted(unused)) |
| 176 | 166 |
| 177 | 167 |
| 178 for master in TRYSERVER_MASTERS: | 168 for master in TRYSERVER_MASTERS: |
| 179 builders = getBuildersAndRecipes(master) | 169 builders = getBuildersAndRecipes(master) |
| 180 recipe_side_builders = chromium_trybot_BUILDERS[ | 170 recipe_side_builders = chromium_trybot.BUILDERS[ |
| 181 master.replace('master.', '')]['builders'] | 171 master.replace('master.', '')]['builders'] |
| 182 | 172 |
| 183 bogus_builders = set(recipe_side_builders.keys()).difference( | 173 bogus_builders = set(recipe_side_builders.keys()).difference( |
| 184 set(builders.keys())) | 174 set(builders.keys())) |
| 185 if bogus_builders: | 175 if bogus_builders: |
| 186 exit_code = 1 | 176 exit_code = 1 |
| 187 print 'The following builders from chromium_trybot recipe' | 177 print 'The following builders from chromium_trybot recipe' |
| 188 print 'do not exist in master config for %s:' % master | 178 print 'do not exist in master config for %s:' % master |
| 189 print '\n'.join('\t%s' % b for b in sorted(bogus_builders)) | 179 print '\n'.join('\t%s' % b for b in sorted(bogus_builders)) |
| 190 | 180 |
| (...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 230 exit_code = 1 | 220 exit_code = 1 |
| 231 print 'Unused suppressions:' | 221 print 'Unused suppressions:' |
| 232 print '\n'.join(sorted( | 222 print '\n'.join(sorted( |
| 233 '\t%s:%s' % (b[0], b[1]) for b in unused_suppressions)) | 223 '\t%s:%s' % (b[0], b[1]) for b in unused_suppressions)) |
| 234 | 224 |
| 235 return exit_code | 225 return exit_code |
| 236 | 226 |
| 237 | 227 |
| 238 if __name__ == '__main__': | 228 if __name__ == '__main__': |
| 239 sys.exit(main(sys.argv[1:])) | 229 sys.exit(main(sys.argv[1:])) |
| OLD | NEW |