OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright 2014 The Chromium Authors. All rights reserved. | 2 # Copyright 2014 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 import json | 6 import json |
7 import os | 7 import os |
8 import subprocess | 8 import subprocess |
9 import sys | 9 import sys |
10 import tempfile | 10 import tempfile |
11 | 11 |
12 | 12 |
13 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | 13 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
14 | 14 |
15 | 15 |
| 16 sys.path.insert(0, os.path.join(BASE_DIR, 'scripts')) |
| 17 sys.path.insert(0, os.path.join(BASE_DIR, 'scripts', 'slave', 'recipes')) |
| 18 sys.path.insert( |
| 19 0, os.path.join(BASE_DIR, 'scripts', 'slave', 'recipe_modules')) |
| 20 sys.path.insert(0, os.path.join(BASE_DIR, 'third_party')) |
| 21 |
| 22 import chromium_trybot |
| 23 from chromium_tests.builders import BUILDERS |
| 24 |
| 25 |
16 MAIN_WATERFALL_MASTERS = [ | 26 MAIN_WATERFALL_MASTERS = [ |
17 'master.chromium.chrome', | 27 'master.chromium.chrome', |
18 'master.chromium.chromiumos', | 28 'master.chromium.chromiumos', |
19 'master.chromium.gpu', | 29 'master.chromium.gpu', |
20 'master.chromium.linux', | 30 'master.chromium.linux', |
21 'master.chromium.mac', | 31 'master.chromium.mac', |
22 'master.chromium.memory', | 32 'master.chromium.memory', |
23 'master.chromium.win', | 33 'master.chromium.win', |
24 ] | 34 ] |
25 | 35 |
(...skipping 59 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
85 ], | 95 ], |
86 } | 96 } |
87 | 97 |
88 UNUSED_MAIN_WATERFALL_BUILDERS = { | 98 UNUSED_MAIN_WATERFALL_BUILDERS = { |
89 'master.chromium.linux': [ | 99 'master.chromium.linux': [ |
90 'Android x86 Builder (dbg)', | 100 'Android x86 Builder (dbg)', |
91 ], | 101 ], |
92 } | 102 } |
93 | 103 |
94 | 104 |
95 def getBuilders(recipe_name): | |
96 """Asks the given recipe to dump its BUILDERS dictionary. | |
97 | |
98 This must be implemented by the recipe in question. | |
99 """ | |
100 (fh, builders_file) = tempfile.mkstemp('.json') | |
101 os.close(fh) | |
102 try: | |
103 subprocess.check_call([ | |
104 os.path.join(BASE_DIR, 'scripts', 'slave', 'recipes.py'), | |
105 'run', recipe_name, 'dump_builders=%s' % builders_file]) | |
106 with open(builders_file) as fh: | |
107 return json.load(fh) | |
108 finally: | |
109 os.remove(builders_file) | |
110 | |
111 | |
112 def getMasterConfig(master): | 105 def getMasterConfig(master): |
113 with tempfile.NamedTemporaryFile() as f: | 106 with tempfile.NamedTemporaryFile() as f: |
114 subprocess.check_call([ | 107 subprocess.check_call([ |
115 os.path.join(BASE_DIR, 'scripts', 'tools', 'runit.py'), | 108 os.path.join(BASE_DIR, 'scripts', 'tools', 'runit.py'), |
116 os.path.join(BASE_DIR, 'scripts', 'tools', 'dump_master_cfg.py'), | 109 os.path.join(BASE_DIR, 'scripts', 'tools', 'dump_master_cfg.py'), |
117 os.path.join(BASE_DIR, 'masters/%s' % master), | 110 os.path.join(BASE_DIR, 'masters/%s' % master), |
118 f.name]) | 111 f.name]) |
119 return json.load(f) | 112 return json.load(f) |
120 | 113 |
121 | 114 |
122 def getBuildersAndRecipes(master): | 115 def getBuildersAndRecipes(master): |
123 return { | 116 return { |
124 builder['name'] : builder['factory']['properties'].get( | 117 builder['name'] : builder['factory']['properties'].get( |
125 'recipe', [None])[0] | 118 'recipe', [None])[0] |
126 for builder in getMasterConfig(master)['builders'] | 119 for builder in getMasterConfig(master)['builders'] |
127 } | 120 } |
128 | 121 |
129 | 122 |
130 def mutualDifference(a, b): | 123 def mutualDifference(a, b): |
131 return a - b, b - a | 124 return a - b, b - a |
132 | 125 |
133 | 126 |
134 def main(argv): | 127 def main(argv): |
135 chromium_recipe_builders = {} | 128 chromium_recipe_builders = {} |
136 covered_builders = set() | 129 covered_builders = set() |
137 all_builders = set() | 130 all_builders = set() |
138 | 131 |
139 exit_code = 0 | 132 exit_code = 0 |
140 | 133 |
141 chromium_trybot_BUILDERS = getBuilders('chromium_trybot') | |
142 chromium_BUILDERS = getBuilders('chromium') | |
143 | |
144 for master in MAIN_WATERFALL_MASTERS: | 134 for master in MAIN_WATERFALL_MASTERS: |
145 builders = getBuildersAndRecipes(master) | 135 builders = getBuildersAndRecipes(master) |
146 all_builders.update((master, b) for b in builders) | 136 all_builders.update((master, b) for b in builders) |
147 | 137 |
148 # We only have a standardized way to mirror builders using the chromium | 138 # We only have a standardized way to mirror builders using the chromium |
149 # recipe on the tryserver. | 139 # recipe on the tryserver. |
150 chromium_recipe_builders[master] = [b for b in builders | 140 chromium_recipe_builders[master] = [b for b in builders |
151 if builders[b] == 'chromium'] | 141 if builders[b] == 'chromium'] |
152 | 142 |
153 # TODO(phajdan.jr): Also consider it an error if configured builders | 143 # TODO(phajdan.jr): Also consider it an error if configured builders |
154 # are not using chromium recipe. This might make it harder to experiment | 144 # are not using chromium recipe. This might make it harder to experiment |
155 # with switching bots over to chromium recipe though, so it may be better | 145 # with switching bots over to chromium recipe though, so it may be better |
156 # to just wait until the switch is done. | 146 # to just wait until the switch is done. |
157 recipe_side_builders = chromium_BUILDERS.get( | 147 recipe_side_builders = BUILDERS.get( |
158 master.replace('master.', ''), {}).get('builders') | 148 master.replace('master.', ''), {}).get('builders') |
159 if recipe_side_builders is not None: | 149 if recipe_side_builders is not None: |
160 bogus_builders = set(recipe_side_builders.keys()).difference( | 150 bogus_builders = set(recipe_side_builders.keys()).difference( |
161 set(builders.keys())) | 151 set(builders.keys())) |
162 bogus_builders, unused = mutualDifference( | 152 bogus_builders, unused = mutualDifference( |
163 bogus_builders, | 153 bogus_builders, |
164 set(UNUSED_MAIN_WATERFALL_BUILDERS.get(master, []))) | 154 set(UNUSED_MAIN_WATERFALL_BUILDERS.get(master, []))) |
165 # TODO(phajdan.jr): Clean up bogus chromiumos builders. | 155 # TODO(phajdan.jr): Clean up bogus chromiumos builders. |
166 if bogus_builders and master != 'master.chromium.chromiumos': | 156 if bogus_builders and master != 'master.chromium.chromiumos': |
167 exit_code = 1 | 157 exit_code = 1 |
168 print 'The following builders from chromium recipe' | 158 print 'The following builders from chromium recipe' |
169 print 'do not exist in master config for %s:' % master | 159 print 'do not exist in master config for %s:' % master |
170 print '\n'.join('\t%s' % b for b in sorted(bogus_builders)) | 160 print '\n'.join('\t%s' % b for b in sorted(bogus_builders)) |
171 if unused: | 161 if unused: |
172 exit_code = 1 | 162 exit_code = 1 |
173 print 'The following unused declarations are superfluous ' | 163 print 'The following unused declarations are superfluous ' |
174 print 'on %s' % master | 164 print 'on %s' % master |
175 print '\n'.join('\t%s' % b for b in sorted(unused)) | 165 print '\n'.join('\t%s' % b for b in sorted(unused)) |
176 | 166 |
177 | 167 |
178 for master in TRYSERVER_MASTERS: | 168 for master in TRYSERVER_MASTERS: |
179 builders = getBuildersAndRecipes(master) | 169 builders = getBuildersAndRecipes(master) |
180 recipe_side_builders = chromium_trybot_BUILDERS[ | 170 recipe_side_builders = chromium_trybot.BUILDERS[ |
181 master.replace('master.', '')]['builders'] | 171 master.replace('master.', '')]['builders'] |
182 | 172 |
183 bogus_builders = set(recipe_side_builders.keys()).difference( | 173 bogus_builders = set(recipe_side_builders.keys()).difference( |
184 set(builders.keys())) | 174 set(builders.keys())) |
185 if bogus_builders: | 175 if bogus_builders: |
186 exit_code = 1 | 176 exit_code = 1 |
187 print 'The following builders from chromium_trybot recipe' | 177 print 'The following builders from chromium_trybot recipe' |
188 print 'do not exist in master config for %s:' % master | 178 print 'do not exist in master config for %s:' % master |
189 print '\n'.join('\t%s' % b for b in sorted(bogus_builders)) | 179 print '\n'.join('\t%s' % b for b in sorted(bogus_builders)) |
190 | 180 |
(...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
230 exit_code = 1 | 220 exit_code = 1 |
231 print 'Unused suppressions:' | 221 print 'Unused suppressions:' |
232 print '\n'.join(sorted( | 222 print '\n'.join(sorted( |
233 '\t%s:%s' % (b[0], b[1]) for b in unused_suppressions)) | 223 '\t%s:%s' % (b[0], b[1]) for b in unused_suppressions)) |
234 | 224 |
235 return exit_code | 225 return exit_code |
236 | 226 |
237 | 227 |
238 if __name__ == '__main__': | 228 if __name__ == '__main__': |
239 sys.exit(main(sys.argv[1:])) | 229 sys.exit(main(sys.argv[1:])) |
OLD | NEW |