| OLD | NEW |
|---|
| 1 #! /usr/bin/python | 1 #! /usr/bin/python |
| 2 # Copyright 2015 The Chromium Authors. All rights reserved. | 2 # Copyright 2015 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 | 5 |
| 6 import argparse | 6 import argparse |
| 7 import cgi | 7 import cgi |
| 8 import json | 8 import json |
| 9 import logging | 9 import logging |
| 10 import os | 10 import os |
| 11 import subprocess | 11 import subprocess |
| 12 import sys | 12 import sys |
| 13 import tempfile | 13 import tempfile |
| 14 import time | 14 import time |
| 15 | 15 |
| 16 _SRC_DIR = os.path.abspath(os.path.join( | 16 _SRC_DIR = os.path.abspath(os.path.join( |
| 17 os.path.dirname(__file__), '..', '..', '..')) | 17 os.path.dirname(__file__), '..', '..', '..')) |
| 18 | 18 |
| 19 sys.path.append(os.path.join(_SRC_DIR, 'third_party', 'catapult', 'devil')) | 19 sys.path.append(os.path.join(_SRC_DIR, 'third_party', 'catapult', 'devil')) |
| 20 from devil.android import device_utils | 20 from devil.android import device_utils |
| 21 from devil.android.sdk import intent | 21 from devil.android.sdk import intent |
| 22 | 22 |
| 23 sys.path.append(os.path.join(_SRC_DIR, 'build', 'android')) | 23 sys.path.append(os.path.join(_SRC_DIR, 'build', 'android')) |
| 24 import devil_chromium | 24 import devil_chromium |
| 25 from pylib import constants | 25 from pylib import constants |
| 26 | 26 |
| 27 import log_parser | 27 import device_setup |
| 28 import log_requests | |
| 29 import loading_model | 28 import loading_model |
| 29 import loading_trace |
| 30 import trace_recorder |
| 30 | 31 |
| 31 | 32 |
| 32 # TODO(mattcary): logging.info isn't that useful; we need something finer | 33 # TODO(mattcary): logging.info isn't that useful, as the whole (tools) world |
| 33 # grained. For now we just do logging.warning. | 34 # uses logging info; we need to introduce logging modules to get finer-grained |
| 35 # output. For now we just do logging.warning. |
| 34 | 36 |
| 35 | 37 |
| 36 # TODO(mattcary): probably we want this piped in through a flag. | 38 # TODO(mattcary): probably we want this piped in through a flag. |
| 37 CHROME = constants.PACKAGE_INFO['chrome'] | 39 CHROME = constants.PACKAGE_INFO['chrome'] |
| 38 | 40 |
| 39 | 41 |
| 40 def _SetupAndGetDevice(): | |
| 41 """Gets an android device, set up the way we like it. | |
| 42 | |
| 43 Returns: | |
| 44 An instance of DeviceUtils for the first device found. | |
| 45 """ | |
| 46 device = device_utils.DeviceUtils.HealthyDevices()[0] | |
| 47 device.EnableRoot() | |
| 48 device.KillAll(CHROME.package, quiet=True) | |
| 49 return device | |
| 50 | |
| 51 | |
| 52 def _LoadPage(device, url): | 42 def _LoadPage(device, url): |
| 53 """Load a page on chrome on our device. | 43 """Load a page on chrome on our device. |
| 54 | 44 |
| 55 Args: | 45 Args: |
| 56 device: an AdbWrapper for the device on which to load the page. | 46 device: an AdbWrapper for the device on which to load the page. |
| 57 url: url as a string to load. | 47 url: url as a string to load. |
| 58 """ | 48 """ |
| 59 load_intent = intent.Intent( | 49 load_intent = intent.Intent( |
| 60 package=CHROME.package, activity=CHROME.activity, data=url) | 50 package=CHROME.package, activity=CHROME.activity, data=url) |
| 61 logging.warning('Loading ' + url) | 51 logging.warning('Loading ' + url) |
| (...skipping 30 matching lines...) Expand all Loading... |
| 92 <html> | 82 <html> |
| 93 <head> | 83 <head> |
| 94 <title>%s</title> | 84 <title>%s</title> |
| 95 """ % title) | 85 """ % title) |
| 96 for info in graph.ResourceInfo(): | 86 for info in graph.ResourceInfo(): |
| 97 output.append('<link rel="prefetch" href="%s">\n' % info.Url()) | 87 output.append('<link rel="prefetch" href="%s">\n' % info.Url()) |
| 98 output.append("""</head> | 88 output.append("""</head> |
| 99 <body>%s</body> | 89 <body>%s</body> |
| 100 </html> | 90 </html> |
| 101 """ % title) | 91 """ % title) |
| 102 | |
| 103 return '\n'.join(output) | 92 return '\n'.join(output) |
| 104 | 93 |
| 105 | 94 |
| 106 def _LogRequests(url, clear_cache=True, local=False): | 95 def _LogRequests(url, clear_cache=True, local=False): |
| 107 """Log requests for a web page. | 96 """Log requests for a web page. |
| 108 | 97 |
| 109 TODO(mattcary): loading.log_requests probably needs to be refactored as we're | |
| 110 using private methods, also there's ugliness like _ResponseDataToJson return a | |
| 111 json.dumps that we immediately json.loads. | |
| 112 | |
| 113 Args: | 98 Args: |
| 114 url: url to log as string. | 99 url: url to log as string. |
| 115 clear_cache: optional flag to clear the cache. | 100 clear_cache: optional flag to clear the cache. |
| 116 local: log from local (desktop) chrome session. | 101 local: log from local (desktop) chrome session. |
| 117 | 102 |
| 118 Returns: | 103 Returns: |
| 119 JSON of logged information (ie, a dict that describes JSON). | 104 JSON dict of logged information (ie, a dict that describes JSON). |
| 120 """ | 105 """ |
| 121 device = _SetupAndGetDevice() if not local else None | 106 device = device_setup.GetFirstDevice() if not local else None |
| 122 request_logger = log_requests.AndroidRequestsLogger(device) | 107 with device_setup.DeviceConnection(device) as connection: |
| 123 logging.warning('Logging %scached %s' % ('un' if clear_cache else '', url)) | 108 trace = trace_recorder.MonitorUrl(connection, url, clear_cache=clear_cache) |
| 124 response_data = request_logger.LogPageLoad( | 109 return trace.ToJsonDict() |
| 125 url, clear_cache, 'chrome') | |
| 126 return json.loads(log_requests._ResponseDataToJson(response_data)) | |
| 127 | 110 |
| 128 | 111 |
| 129 def _FullFetch(url, json_output, prefetch, local, prefetch_delay_seconds): | 112 def _FullFetch(url, json_output, prefetch, local, prefetch_delay_seconds): |
| 130 """Do a full fetch with optional prefetching.""" | 113 """Do a full fetch with optional prefetching.""" |
| 131 if not url.startswith('http'): | 114 if not url.startswith('http'): |
| 132 url = 'http://' + url | 115 url = 'http://' + url |
| 133 logging.warning('Cold fetch') | 116 logging.warning('Cold fetch') |
| 134 cold_data = _LogRequests(url, local=local) | 117 cold_data = _LogRequests(url, local=local) |
| 135 assert cold_data, 'Cold fetch failed to produce data. Check your phone.' | 118 assert cold_data, 'Cold fetch failed to produce data. Check your phone.' |
| 136 if prefetch: | 119 if prefetch: |
| 137 assert not local | 120 assert not local |
| 138 logging.warning('Generating prefetch') | 121 logging.warning('Generating prefetch') |
| 139 prefetch_html = _GetPrefetchHtml(_ProcessJson(cold_data), name=url) | 122 prefetch_html = _GetPrefetchHtml( |
| 123 loading_model.ResourceGraph(cold_data), name=url) |
| 140 tmp = tempfile.NamedTemporaryFile() | 124 tmp = tempfile.NamedTemporaryFile() |
| 141 tmp.write(prefetch_html) | 125 tmp.write(prefetch_html) |
| 142 tmp.flush() | 126 tmp.flush() |
| 143 # We hope that the tmpfile name is unique enough for the device. | 127 # We hope that the tmpfile name is unique enough for the device. |
| 144 target = os.path.join('/sdcard/Download', os.path.basename(tmp.name)) | 128 target = os.path.join('/sdcard/Download', os.path.basename(tmp.name)) |
| 145 device = _SetupAndGetDevice() | 129 device = device_setup.GetFirstDevice() |
| 146 device.adb.Push(tmp.name, target) | 130 device.adb.Push(tmp.name, target) |
| 147 logging.warning('Pushed prefetch %s to device at %s' % (tmp.name, target)) | 131 logging.warning('Pushed prefetch %s to device at %s' % (tmp.name, target)) |
| 148 _LoadPage(device, 'file://' + target) | 132 _LoadPage(device, 'file://' + target) |
| 149 time.sleep(prefetch_delay_seconds) | 133 time.sleep(prefetch_delay_seconds) |
| 150 logging.warning('Warm fetch') | 134 logging.warning('Warm fetch') |
| 151 warm_data = _LogRequests(url, clear_cache=False) | 135 warm_data = _LogRequests(url, clear_cache=False) |
| 152 with open(json_output, 'w') as f: | 136 with open(json_output, 'w') as f: |
| 153 _WriteJson(f, warm_data) | 137 _WriteJson(f, warm_data) |
| 154 logging.warning('Wrote ' + json_output) | 138 logging.warning('Wrote ' + json_output) |
| 155 with open(json_output + '.cold', 'w') as f: | 139 with open(json_output + '.cold', 'w') as f: |
| 156 _WriteJson(f, cold_data) | 140 _WriteJson(f, cold_data) |
| 157 logging.warning('Wrote ' + json_output + '.cold') | 141 logging.warning('Wrote ' + json_output + '.cold') |
| 158 else: | 142 else: |
| 159 with open(json_output, 'w') as f: | 143 with open(json_output, 'w') as f: |
| 160 _WriteJson(f, cold_data) | 144 _WriteJson(f, cold_data) |
| 161 logging.warning('Wrote ' + json_output) | 145 logging.warning('Wrote ' + json_output) |
| 162 | 146 |
| 163 | 147 |
| 164 # TODO(mattcary): it would be nice to refactor so the --noads flag gets dealt | 148 # TODO(mattcary): it would be nice to refactor so the --noads flag gets dealt |
| 165 # with here. | 149 # with here. |
| 166 def _ProcessRequests(filename): | 150 def _ProcessRequests(filename): |
| 167 requests = log_parser.FilterRequests(log_parser.ParseJsonFile(filename)) | 151 with open(filename) as f: |
| 168 return loading_model.ResourceGraph(requests) | 152 return loading_model.ResourceGraph( |
| 169 | 153 loading_trace.LoadingTrace.FromJsonDict(json.load(f))) |
| 170 | |
| 171 def _ProcessJson(json_data): | |
| 172 assert json_data | |
| 173 return loading_model.ResourceGraph(log_parser.FilterRequests( | |
| 174 [log_parser.RequestData.FromDict(r) for r in json_data])) | |
| 175 | 154 |
| 176 | 155 |
| 177 def InvalidCommand(cmd): | 156 def InvalidCommand(cmd): |
| 178 sys.exit('Invalid command "%s"\nChoices are: %s' % | 157 sys.exit('Invalid command "%s"\nChoices are: %s' % |
| 179 (cmd, ' '.join(COMMAND_MAP.keys()))) | 158 (cmd, ' '.join(COMMAND_MAP.keys()))) |
| 180 | 159 |
| 181 | 160 |
| 182 def DoCost(arg_str): | 161 def DoCost(arg_str): |
| 183 parser = argparse.ArgumentParser(usage='cost [--parameter ...] REQUEST_JSON') | 162 parser = argparse.ArgumentParser(usage='cost [--parameter ...] REQUEST_JSON') |
| 184 parser.add_argument('request_json') | 163 parser.add_argument('request_json') |
| (...skipping 63 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 248 usage='prefetch_setup [--upload] REQUEST_JSON TARGET_HTML') | 227 usage='prefetch_setup [--upload] REQUEST_JSON TARGET_HTML') |
| 249 parser.add_argument('request_json') | 228 parser.add_argument('request_json') |
| 250 parser.add_argument('target_html') | 229 parser.add_argument('target_html') |
| 251 parser.add_argument('--upload', action='store_true') | 230 parser.add_argument('--upload', action='store_true') |
| 252 args = parser.parse_args(arg_str) | 231 args = parser.parse_args(arg_str) |
| 253 graph = _ProcessRequests(args.request_json) | 232 graph = _ProcessRequests(args.request_json) |
| 254 with open(args.target_html, 'w') as html: | 233 with open(args.target_html, 'w') as html: |
| 255 html.write(_GetPrefetchHtml( | 234 html.write(_GetPrefetchHtml( |
| 256 graph, name=os.path.basename(args.request_json))) | 235 graph, name=os.path.basename(args.request_json))) |
| 257 if args.upload: | 236 if args.upload: |
| 258 device = _SetupAndGetDevice() | 237 device = device_setup.GetFirstDevice() |
| 259 destination = os.path.join('/sdcard/Download', | 238 destination = os.path.join('/sdcard/Download', |
| 260 os.path.basename(args.target_html)) | 239 os.path.basename(args.target_html)) |
| 261 device.adb.Push(args.target_html, destination) | 240 device.adb.Push(args.target_html, destination) |
| 262 | 241 |
| 263 logging.warning( | 242 logging.warning( |
| 264 'Pushed %s to device at %s' % (args.target_html, destination)) | 243 'Pushed %s to device at %s' % (args.target_html, destination)) |
| 265 | 244 |
| 266 | 245 |
| 267 def DoLogRequests(arg_str): | 246 def DoLogRequests(arg_str): |
| 268 parser = argparse.ArgumentParser( | 247 parser = argparse.ArgumentParser( |
| (...skipping 26 matching lines...) Expand all Loading... |
| 295 args = parser.parse_args(arg_str) | 274 args = parser.parse_args(arg_str) |
| 296 if not os.path.exists(args.dir): | 275 if not os.path.exists(args.dir): |
| 297 os.makedirs(args.dir) | 276 os.makedirs(args.dir) |
| 298 _FullFetch(url=args.site, | 277 _FullFetch(url=args.site, |
| 299 json_output=os.path.join(args.dir, args.site + '.json'), | 278 json_output=os.path.join(args.dir, args.site + '.json'), |
| 300 prefetch=True, | 279 prefetch=True, |
| 301 prefetch_delay_seconds=args.prefetch_delay_seconds, | 280 prefetch_delay_seconds=args.prefetch_delay_seconds, |
| 302 local=False) | 281 local=False) |
| 303 | 282 |
| 304 | 283 |
| 305 def DoTracing(arg_str): | |
| 306 parser = argparse.ArgumentParser( | |
| 307 usage='tracing URL JSON_OUTPUT') | |
| 308 parser.add_argument('url') | |
| 309 parser.add_argument('json_output') | |
| 310 args = parser.parse_args(arg_str) | |
| 311 device = _SetupAndGetDevice() | |
| 312 request_logger = log_requests.AndroidRequestsLogger(device) | |
| 313 tracing = request_logger.LogTracing(args.url) | |
| 314 with open(args.json_output, 'w') as f: | |
| 315 _WriteJson(f, tracing) | |
| 316 logging.warning('Wrote ' + args.json_output) | |
| 317 | |
| 318 | |
| 319 def DoLongPole(arg_str): | 284 def DoLongPole(arg_str): |
| 320 parser = argparse.ArgumentParser(usage='longpole [--noads] REQUEST_JSON') | 285 parser = argparse.ArgumentParser(usage='longpole [--noads] REQUEST_JSON') |
| 321 parser.add_argument('request_json') | 286 parser.add_argument('request_json') |
| 322 parser.add_argument('--noads', action='store_true') | 287 parser.add_argument('--noads', action='store_true') |
| 323 args = parser.parse_args(arg_str) | 288 args = parser.parse_args(arg_str) |
| 324 graph = _ProcessRequests(args.request_json) | 289 graph = _ProcessRequests(args.request_json) |
| 325 if args.noads: | 290 if args.noads: |
| 326 graph.Set(node_filter=graph.FilterAds) | 291 graph.Set(node_filter=graph.FilterAds) |
| 327 path_list = [] | 292 path_list = [] |
| 328 cost = graph.Cost(path_list=path_list) | 293 cost = graph.Cost(path_list=path_list) |
| (...skipping 10 matching lines...) Expand all Loading... |
| 339 graph.Set(node_filter=graph.FilterAds) | 304 graph.Set(node_filter=graph.FilterAds) |
| 340 print sum((n.NodeCost() for n in graph.Nodes())) | 305 print sum((n.NodeCost() for n in graph.Nodes())) |
| 341 | 306 |
| 342 | 307 |
| 343 COMMAND_MAP = { | 308 COMMAND_MAP = { |
| 344 'cost': DoCost, | 309 'cost': DoCost, |
| 345 'png': DoPng, | 310 'png': DoPng, |
| 346 'compare': DoCompare, | 311 'compare': DoCompare, |
| 347 'prefetch_setup': DoPrefetchSetup, | 312 'prefetch_setup': DoPrefetchSetup, |
| 348 'log_requests': DoLogRequests, | 313 'log_requests': DoLogRequests, |
| 349 'tracing': DoTracing, | |
| 350 'longpole': DoLongPole, | 314 'longpole': DoLongPole, |
| 351 'nodecost': DoNodeCost, | 315 'nodecost': DoNodeCost, |
| 352 'fetch': DoFetch, | 316 'fetch': DoFetch, |
| 353 } | 317 } |
| 354 | 318 |
| 355 def main(): | 319 def main(): |
| 356 logging.basicConfig(level=logging.WARNING) | 320 logging.basicConfig(level=logging.WARNING) |
| 357 parser = argparse.ArgumentParser(usage=' '.join(COMMAND_MAP.keys())) | 321 parser = argparse.ArgumentParser(usage=' '.join(COMMAND_MAP.keys())) |
| 358 parser.add_argument('command') | 322 parser.add_argument('command') |
| 359 parser.add_argument('rest', nargs=argparse.REMAINDER) | 323 parser.add_argument('rest', nargs=argparse.REMAINDER) |
| 360 args = parser.parse_args() | 324 args = parser.parse_args() |
| 361 devil_chromium.Initialize() | 325 devil_chromium.Initialize() |
| 362 COMMAND_MAP.get(args.command, | 326 COMMAND_MAP.get(args.command, |
| 363 lambda _: InvalidCommand(args.command))(args.rest) | 327 lambda _: InvalidCommand(args.command))(args.rest) |
| 364 | 328 |
| 365 | 329 |
| 366 if __name__ == '__main__': | 330 if __name__ == '__main__': |
| 367 main() | 331 main() |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 1619713002:
Upgrade analyze.py and related scripts to new world order. (Closed)
Base URL: https://chromium.googlesource.com/chromium/src.git@master