| OLD | NEW |
|---|
| (Empty) |
| 1 #!/usr/bin/env python | |
| 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | |
| 3 # Use of this source code is governed by a BSD-style license that can be | |
| 4 # found in the LICENSE file. | |
| 5 """Reduce the amount of data in ../test/data/*.json""" | |
| 6 | |
| 7 import json | |
| 8 import logging | |
| 9 import optparse | |
| 10 import os | |
| 11 import re | |
| 12 import sys | |
| 13 | |
| 14 | |
| 15 class Filterer(object): | |
| 16 def __init__(self): | |
| 17 self._deleted_builds = {} | |
| 18 self.max_cached_builds = 10 | |
| 19 self._allowed_builders = ('linux_clang', 'linux', 'linux_touch') | |
| 20 | |
| 21 def reset(self): | |
| 22 self._deleted_builds = {} | |
| 23 | |
| 24 def reduce_data(self, data): | |
| 25 """Reduces the amount of data sent from a server to simplify testing and the | |
| 26 amount of test data stored. | |
| 27 """ | |
| 28 self.reset() | |
| 29 for line in data[:]: | |
| 30 original_url, original_response = line | |
| 31 # Unpack and filter the response. | |
| 32 new_url, new_response = self.filter_response( | |
| 33 original_url, json.loads(original_response)) | |
| 34 if not new_url or new_response is None: | |
| 35 data.remove(line) | |
| 36 continue | |
| 37 # Repack the string. | |
| 38 line[1] = json.dumps(new_response, separators=(',',':')) | |
| 39 logging.info('%s length: %d -> %d' % ( | |
| 40 new_url, len(original_response), len(line[1]))) | |
| 41 if len(line[1]) > 20000: | |
| 42 logging.debug(line[1]) | |
| 43 return data | |
| 44 | |
| 45 def filter_response(self, url, response): | |
| 46 """Trims a single request. | |
| 47 | |
| 48 |response| must be decoded json. Decoded json will be returned. | |
| 49 """ | |
| 50 # Builders | |
| 51 match = re.match('.+/json/builders/(\w+)(|\?filter=1)$', url) | |
| 52 if match: | |
| 53 return url, self._filter_builder(match.group(1), response) | |
| 54 | |
| 55 match = re.match('.+/json/builders(|\?filter=1)$', url) | |
| 56 if match: | |
| 57 for builder in response.keys(): | |
| 58 value = self._filter_builder(builder, response[builder]) | |
| 59 if value is None: | |
| 60 del response[builder] | |
| 61 else: | |
| 62 response[builder] = value | |
| 63 return url, response | |
| 64 | |
| 65 # Pending | |
| 66 match = re.match('.+/json/builders/(\w+)/pendingBuilds(|\?filter=1)$', url) | |
| 67 if match: | |
| 68 assert match.group(1) in self._allowed_builders | |
| 69 return url, self._filter_pending(response) | |
| 70 | |
| 71 # Builds | |
| 72 match = re.match('.+/json/builders/(\w+)/builds/_all(|\?filter=1)$', url) | |
| 73 if match: | |
| 74 builder = match.group(1) | |
| 75 assert builder in self._allowed_builders, url | |
| 76 keys = response.keys() | |
| 77 keys = [int(k) for k in keys if int(k) not in self._deleted(builder)] | |
| 78 keys = sorted(keys)[:self.max_cached_builds] | |
| 79 response = dict((k, v) for k, v in response.iteritems() if int(k) in keys) | |
| 80 return url, response | |
| 81 | |
| 82 match = re.match('.+/json/builders/(\w+)/builds/\?(.+?)(|\&filter=1)$', url) | |
| 83 if match: | |
| 84 assert match.group(1) in self._allowed_builders | |
| 85 # Ignore the query, reconstruct it from what it kept. | |
| 86 for build in response.keys(): | |
| 87 value = self._filter_build(response[build]) | |
| 88 if value is None: | |
| 89 del response[build] | |
| 90 else: | |
| 91 response[build] = value | |
| 92 if not response: | |
| 93 return None, None | |
| 94 url = '%s?%s' % ( | |
| 95 url.split('?', 1)[0], | |
| 96 '&'.join('select=%s' % b for b in sorted(response))) | |
| 97 return url, response | |
| 98 | |
| 99 match = re.match('.+/json/builders/(\w+)/builds/(\d+)(|\?filter=1)$', url) | |
| 100 if match: | |
| 101 return url, self._filter_build(response) | |
| 102 | |
| 103 # Slaves | |
| 104 match = re.match('.+/json/slaves/([^/]+?)(|\?filter=1)$', url) | |
| 105 if match: | |
| 106 return url, self._filter_slave(match.group(1), response) | |
| 107 | |
| 108 match = re.match('.+/json/slaves(|\?filter=1)$', url) | |
| 109 if match: | |
| 110 for slave in response.keys(): | |
| 111 value = self._filter_slave(slave, response[slave]) | |
| 112 if value is None: | |
| 113 del response[slave] | |
| 114 else: | |
| 115 response[slave] = value | |
| 116 return url, response | |
| 117 | |
| 118 # Project | |
| 119 match = re.match('.+/json/project(|\?filter=1)$', url) | |
| 120 if match: | |
| 121 return url, response | |
| 122 | |
| 123 assert False, url | |
| 124 | |
| 125 @staticmethod | |
| 126 def _filter_pending(pending): | |
| 127 """Trim pendingBuilds.""" | |
| 128 return pending[:2] | |
| 129 | |
| 130 def _filter_builder(self, builder, response): | |
| 131 """Trims a builder. | |
| 132 | |
| 133 Reduces the number of cached builds. | |
| 134 """ | |
| 135 # TODO(maruel): Reduce the number of slaves. | |
| 136 if builder not in self._allowed_builders: | |
| 137 return None | |
| 138 builds_kept = response['cachedBuilds'][-self.max_cached_builds:] | |
| 139 builds_discarded = response['cachedBuilds'][:-self.max_cached_builds] | |
| 140 assert len(builds_kept) <= self.max_cached_builds | |
| 141 assert ( | |
| 142 len(builds_kept) + len(builds_discarded) == | |
| 143 len(response['cachedBuilds'])) | |
| 144 if builds_discarded: | |
| 145 assert min(builds_kept) > max(builds_discarded) | |
| 146 response['cachedBuilds'] = builds_kept | |
| 147 self._deleted(builder).union(int(b) for b in builds_discarded) | |
| 148 if response.get('currentBuilds'): | |
| 149 response['currentBuilds'] = [ | |
| 150 build for build in response['currentBuilds'] | |
| 151 if int(build) not in self._deleted(builder) | |
| 152 ] | |
| 153 if response.get('pendingBuilds', 0) > 2: | |
| 154 response['pendingBuilds'] = 2 | |
| 155 return response | |
| 156 | |
| 157 def _filter_build(self, response): | |
| 158 """Trims a build.""" | |
| 159 if response['builderName'] not in self._allowed_builders: | |
| 160 return None | |
| 161 if int(response['number']) in self._deleted(response['builderName']): | |
| 162 return None | |
| 163 # TODO(maruel): Fix StatusJson to not push that much logs data. | |
| 164 if 'logs' in response: | |
| 165 del response['logs'] | |
| 166 if response.get('currentStep') and response['currentStep'].get('logs'): | |
| 167 del response['currentStep']['logs'] | |
| 168 for step in response['steps']: | |
| 169 if 'logs' in step: | |
| 170 del step['logs'] | |
| 171 return response | |
| 172 | |
| 173 def _filter_slave(self, _slave, response): | |
| 174 """Trims a slave.""" | |
| 175 if response.get('builders'): | |
| 176 for builder in response['builders'].keys(): | |
| 177 if not builder in self._allowed_builders: | |
| 178 del response['builders'][builder] | |
| 179 if not response['builders']: | |
| 180 return None | |
| 181 if response.get('builderName'): | |
| 182 if not response['builderName'] in self._allowed_builders: | |
| 183 return None | |
| 184 if response.get('runningBuilds'): | |
| 185 for i, build in enumerate(response['runningBuilds'][:]): | |
| 186 value = self._filter_build(build) | |
| 187 if value is None: | |
| 188 response['runningBuilds'].remove(build) | |
| 189 else: | |
| 190 response['runningBuilds'][i] = value | |
| 191 return response | |
| 192 | |
| 193 def _deleted(self, builder): | |
| 194 return self._deleted_builds.get(builder, set()) | |
| 195 | |
| 196 | |
| 197 def main(): | |
| 198 parser = optparse.OptionParser( | |
| 199 description=sys.modules['__main__'].__doc__) | |
| 200 parser.add_option('-v', '--verbose', action='count', default=0) | |
| 201 parser.add_option('-d', '--dry-run', action='store_true') | |
| 202 options, args = parser.parse_args() | |
| 203 if args: | |
| 204 parser.error('Unsupported args: %s' % args) | |
| 205 logging.basicConfig( | |
| 206 level=[logging.WARNING, logging.INFO, logging.DEBUG][ | |
| 207 min(2, options.verbose)]) | |
| 208 | |
| 209 datadir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data') | |
| 210 for filename in os.listdir(datadir): | |
| 211 if not filename.endswith('.json') or filename.endswith('_expected.json'): | |
| 212 continue | |
| 213 filepath = os.path.join(datadir, filename) | |
| 214 print 'Processing %s' % filename | |
| 215 data = json.load(open(filepath)) | |
| 216 data = Filterer().reduce_data(data) | |
| 217 if not options.dry_run: | |
| 218 json.dump(data, open(filepath, 'w'), separators=(',',':')) | |
| 219 return 0 | |
| 220 | |
| 221 | |
| 222 if __name__ == '__main__': | |
| 223 sys.exit(main()) | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 135363007:
Delete public commit queue to avoid confusion after move to internal repo (Closed)
Base URL: svn://svn.chromium.org/chrome/trunk/tools/