OLD | NEW |
---|---|
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Archives or replays webpages and creates SKPs in a Google Storage location. | 6 """Archives or replays webpages and creates SKPs in a Google Storage location. |
7 | 7 |
8 To archive webpages and store SKP files (archives should be rarely updated): | 8 To archive webpages and store SKP files (archives should be rarely updated): |
9 | 9 |
10 cd skia | 10 cd skia |
(...skipping 89 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
100 CREDENTIALS_GS_PATH = '/playback/credentials/credentials.json' | 100 CREDENTIALS_GS_PATH = '/playback/credentials/credentials.json' |
101 | 101 |
102 X11_DISPLAY = os.getenv('DISPLAY', ':0') | 102 X11_DISPLAY = os.getenv('DISPLAY', ':0') |
103 | 103 |
104 GS_PREDEFINED_ACL = gs_utils.GSUtils.PredefinedACL.PRIVATE | 104 GS_PREDEFINED_ACL = gs_utils.GSUtils.PredefinedACL.PRIVATE |
105 GS_FINE_GRAINED_ACL_LIST = [ | 105 GS_FINE_GRAINED_ACL_LIST = [ |
106 (gs_utils.GSUtils.IdType.GROUP_BY_DOMAIN, 'google.com', | 106 (gs_utils.GSUtils.IdType.GROUP_BY_DOMAIN, 'google.com', |
107 gs_utils.GSUtils.Permission.READ), | 107 gs_utils.GSUtils.Permission.READ), |
108 ] | 108 ] |
109 | 109 |
110 # Path to Chromium's page sets. | |
111 CHROMIUM_PAGE_SETS_PATH = 'tools/perf/page_sets' | |
borenet
2015/03/30 19:31:57
os.path.join?
rmistry
2015/03/30 20:11:18
Done.
| |
112 | |
113 # Dictionary of supported Chromium page sets to their file prefixes. | |
114 CHROMIUM_PAGE_SETS_TO_PREFIX = { | |
115 'key_mobile_sites_smooth.py': 'keymobi', | |
116 # TODO(rmistry): Uncomment the below after it is verified that the above | |
117 # works. | |
118 # 'top_25_smooth.py': 'top25desk', | |
119 } | |
120 | |
121 | |
110 def remove_prefix(s, prefix): | 122 def remove_prefix(s, prefix): |
111 if s.startswith(prefix): | 123 if s.startswith(prefix): |
112 return s[len(prefix):] | 124 return s[len(prefix):] |
113 return s | 125 return s |
114 | 126 |
127 | |
115 class SkPicturePlayback(object): | 128 class SkPicturePlayback(object): |
116 """Class that archives or replays webpages and creates SKPs.""" | 129 """Class that archives or replays webpages and creates SKPs.""" |
117 | 130 |
118 def __init__(self, parse_options): | 131 def __init__(self, parse_options): |
119 """Constructs a SkPicturePlayback BuildStep instance.""" | 132 """Constructs a SkPicturePlayback BuildStep instance.""" |
120 assert parse_options.browser_executable, 'Must specify --browser_executable' | 133 assert parse_options.browser_executable, 'Must specify --browser_executable' |
121 self._browser_executable = parse_options.browser_executable | 134 self._browser_executable = parse_options.browser_executable |
122 | 135 |
136 self._chrome_page_sets_path = os.path.join(parse_options.chrome_src_path, | |
137 CHROMIUM_PAGE_SETS_PATH) | |
123 self._all_page_sets_specified = parse_options.page_sets == 'all' | 138 self._all_page_sets_specified = parse_options.page_sets == 'all' |
124 self._page_sets = self._ParsePageSets(parse_options.page_sets) | 139 self._page_sets = self._ParsePageSets(parse_options.page_sets) |
125 | 140 |
126 self._dest_gsbase = parse_options.dest_gsbase | 141 self._dest_gsbase = parse_options.dest_gsbase |
127 self._record = parse_options.record | 142 self._record = parse_options.record |
128 self._skia_tools = parse_options.skia_tools | 143 self._skia_tools = parse_options.skia_tools |
129 self._non_interactive = parse_options.non_interactive | 144 self._non_interactive = parse_options.non_interactive |
130 self._upload_to_gs = parse_options.upload_to_gs | 145 self._upload_to_gs = parse_options.upload_to_gs |
131 self._alternate_upload_dir = parse_options.alternate_upload_dir | 146 self._alternate_upload_dir = parse_options.alternate_upload_dir |
132 self._skip_all_gs_access = parse_options.skip_all_gs_access | 147 self._skip_all_gs_access = parse_options.skip_all_gs_access |
(...skipping 12 matching lines...) Expand all Loading... | |
145 if not page_sets: | 160 if not page_sets: |
146 raise ValueError('Must specify at least one page_set!') | 161 raise ValueError('Must specify at least one page_set!') |
147 elif self._all_page_sets_specified: | 162 elif self._all_page_sets_specified: |
148 # Get everything from the page_sets directory. | 163 # Get everything from the page_sets directory. |
149 page_sets_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), | 164 page_sets_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), |
150 'page_sets') | 165 'page_sets') |
151 ps = [os.path.join(page_sets_dir, page_set) | 166 ps = [os.path.join(page_sets_dir, page_set) |
152 for page_set in os.listdir(page_sets_dir) | 167 for page_set in os.listdir(page_sets_dir) |
153 if not os.path.isdir(os.path.join(page_sets_dir, page_set)) and | 168 if not os.path.isdir(os.path.join(page_sets_dir, page_set)) and |
154 page_set.endswith('.py')] | 169 page_set.endswith('.py')] |
170 chromium_ps = [ | |
171 os.path.join(self._chrome_page_sets_path, cr_page_set) | |
172 for cr_page_set in CHROMIUM_PAGE_SETS_TO_PREFIX] | |
173 ps.extend(chromium_ps) | |
155 elif '*' in page_sets: | 174 elif '*' in page_sets: |
156 # Explode and return the glob. | 175 # Explode and return the glob. |
157 ps = glob.glob(page_sets) | 176 ps = glob.glob(page_sets) |
158 else: | 177 else: |
159 ps = page_sets.split(',') | 178 ps = page_sets.split(',') |
160 ps.sort() | 179 ps.sort() |
161 return ps | 180 return ps |
162 | 181 |
182 def _IsChromiumPageSet(self, page_set): | |
183 """Returns true if the specified page set is a Chromium page set.""" | |
184 return page_set.startswith(self._chrome_page_sets_path) | |
185 | |
163 def Run(self): | 186 def Run(self): |
164 """Run the SkPicturePlayback BuildStep.""" | 187 """Run the SkPicturePlayback BuildStep.""" |
165 | 188 |
166 # Download the credentials file if it was not previously downloaded. | 189 # Download the credentials file if it was not previously downloaded. |
167 if self._skip_all_gs_access: | 190 if self._skip_all_gs_access: |
168 print """\n\nPlease create a %s file that contains: | 191 print """\n\nPlease create a %s file that contains: |
169 { | 192 { |
170 "google": { | 193 "google": { |
171 "username": "google_testing_account_username", | 194 "username": "google_testing_account_username", |
172 "password": "google_testing_account_password" | 195 "password": "google_testing_account_password" |
(...skipping 26 matching lines...) Expand all Loading... | |
199 start_time = time.time() | 222 start_time = time.time() |
200 | 223 |
201 # Loop through all page_sets. | 224 # Loop through all page_sets. |
202 for page_set in self._page_sets: | 225 for page_set in self._page_sets: |
203 | 226 |
204 page_set_basename = os.path.basename(page_set).split('.')[0] | 227 page_set_basename = os.path.basename(page_set).split('.')[0] |
205 page_set_json_name = page_set_basename + '.json' | 228 page_set_json_name = page_set_basename + '.json' |
206 wpr_data_file = page_set.split(os.path.sep)[-1].split('.')[0] + '_000.wpr' | 229 wpr_data_file = page_set.split(os.path.sep)[-1].split('.')[0] + '_000.wpr' |
207 page_set_dir = os.path.dirname(page_set) | 230 page_set_dir = os.path.dirname(page_set) |
208 | 231 |
209 if self._record: | 232 if self._IsChromiumPageSet(page_set): |
233 print 'Using Chromium\'s captured archives for Chromium\'s page sets.' | |
234 elif self._record: | |
210 # Create an archive of the specified webpages if '--record=True' is | 235 # Create an archive of the specified webpages if '--record=True' is |
211 # specified. | 236 # specified. |
212 record_wpr_cmd = ( | 237 record_wpr_cmd = ( |
213 'PYTHONPATH=%s:$PYTHONPATH' % page_set_dir, | 238 'PYTHONPATH=%s:$PYTHONPATH' % page_set_dir, |
214 'DISPLAY=%s' % X11_DISPLAY, | 239 'DISPLAY=%s' % X11_DISPLAY, |
215 os.path.join(self._telemetry_binaries_dir, 'record_wpr'), | 240 os.path.join(self._telemetry_binaries_dir, 'record_wpr'), |
216 '--extra-browser-args=--disable-setuid-sandbox', | 241 '--extra-browser-args=--disable-setuid-sandbox', |
217 '--browser=exact', | 242 '--browser=exact', |
218 '--browser-executable=%s' % self._browser_executable, | 243 '--browser-executable=%s' % self._browser_executable, |
219 '%s_page_set' % page_set_basename, | 244 '%s_page_set' % page_set_basename, |
(...skipping 110 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
330 | 355 |
331 print '\n\n=======New SKPs have been uploaded to %s =======\n\n' % ( | 356 print '\n\n=======New SKPs have been uploaded to %s =======\n\n' % ( |
332 posixpath.join(self._dest_gsbase, dest_dir_name, SKPICTURES_DIR_NAME)) | 357 posixpath.join(self._dest_gsbase, dest_dir_name, SKPICTURES_DIR_NAME)) |
333 else: | 358 else: |
334 print '\n\n=======Not Uploading to Google Storage=======\n\n' | 359 print '\n\n=======Not Uploading to Google Storage=======\n\n' |
335 print 'Generated resources are available in %s\n\n' % ( | 360 print 'Generated resources are available in %s\n\n' % ( |
336 LOCAL_PLAYBACK_ROOT_DIR) | 361 LOCAL_PLAYBACK_ROOT_DIR) |
337 | 362 |
338 return 0 | 363 return 0 |
339 | 364 |
365 def _GetSkiaSkpFileName(self, page_set): | |
366 """Returns the SKP file name for Skia page sets.""" | |
367 # /path/to/skia_yahooanswers_desktop.py -> skia_yahooanswers_desktop.py | |
368 _, ps_filename = os.path.split(page_set) | |
borenet
2015/03/30 19:31:57
Use os.path.basename here for a little extra clean
rmistry
2015/03/30 20:11:18
Done.
| |
369 # skia_yahooanswers_desktop.py -> skia_yahooanswers_desktop | |
370 ps_basename, _ = os.path.splitext(ps_filename) | |
371 # skia_yahooanswers_desktop -> skia, yahooanswers, desktop | |
372 _, page_name, device = ps_basename.split('_') | |
373 basename = '%s_%s' % (DEVICE_TO_PLATFORM_PREFIX[device], page_name) | |
374 return basename[:MAX_SKP_BASE_NAME_LEN] + '.skp' | |
375 | |
376 def _GetChromiumSkpFileName(self, page_set, site): | |
377 """Returns the SKP file name for Chromium page sets.""" | |
378 # /path/to/http___mobile_news_sandbox_pt0 -> http___mobile_news_sandbox_pt0 | |
379 _, webpage = os.path.split(site) | |
380 # http___mobile_news_sandbox_pt0 -> mobile_news_sandbox_pt0 | |
381 webpage = webpage.lstrip('http___').lstrip('https___') | |
382 # /path/to/skia_yahooanswers_desktop.py -> skia_yahooanswers_desktop.py | |
383 _, ps_filename = os.path.split(page_set) | |
384 # http___mobile_news_sandbox -> pagesetprefix_http___mobile_news_sandbox | |
385 basename = '%s_%s' % (CHROMIUM_PAGE_SETS_TO_PREFIX[ps_filename], webpage) | |
386 return basename[:MAX_SKP_BASE_NAME_LEN] + '.skp' | |
387 | |
340 def _RenameSkpFiles(self, page_set): | 388 def _RenameSkpFiles(self, page_set): |
341 """Rename generated SKP files into more descriptive names. | 389 """Rename generated SKP files into more descriptive names. |
342 | 390 |
343 Look into the subdirectory of TMP_SKP_DIR and find the most interesting | 391 Look into the subdirectory of TMP_SKP_DIR and find the most interesting |
344 .skp in there to be this page_set's representative .skp. | 392 .skp in there to be this page_set's representative .skp. |
345 """ | 393 """ |
346 # Here's where we're assuming there's one page per pageset. | 394 subdirs = glob.glob(os.path.join(TMP_SKP_DIR, '*')) |
347 # If there were more than one, we'd overwrite filename below. | 395 for site in subdirs: |
396 if self._IsChromiumPageSet(page_set): | |
397 filename = self._GetChromiumSkpFileName(page_set, site) | |
398 else: | |
399 filename = self._GetSkiaSkpFileName(page_set) | |
348 | 400 |
349 # /path/to/skia_yahooanswers_desktop.json -> skia_yahooanswers_desktop.json | |
350 _, ps_filename = os.path.split(page_set) | |
351 # skia_yahooanswers_desktop.json -> skia_yahooanswers_desktop | |
352 ps_basename, _ = os.path.splitext(ps_filename) | |
353 # skia_yahooanswers_desktop -> skia, yahooanswers, desktop | |
354 _, page_name, device = ps_basename.split('_') | |
355 | |
356 basename = '%s_%s' % (DEVICE_TO_PLATFORM_PREFIX[device], page_name) | |
357 filename = basename[:MAX_SKP_BASE_NAME_LEN] + '.skp' | |
358 | |
359 subdirs = glob.glob(os.path.join(TMP_SKP_DIR, '*')) | |
360 assert len(subdirs) == 1 | |
361 for site in subdirs: | |
362 # We choose the largest .skp as the most likely to be interesting. | 401 # We choose the largest .skp as the most likely to be interesting. |
363 largest_skp = max(glob.glob(os.path.join(site, '*.skp')), | 402 largest_skp = max(glob.glob(os.path.join(site, '*.skp')), |
364 key=lambda path: os.stat(path).st_size) | 403 key=lambda path: os.stat(path).st_size) |
365 dest = os.path.join(self._local_skp_dir, filename) | 404 dest = os.path.join(self._local_skp_dir, filename) |
366 print 'Moving', largest_skp, 'to', dest | 405 print 'Moving', largest_skp, 'to', dest |
367 shutil.move(largest_skp, dest) | 406 shutil.move(largest_skp, dest) |
368 self._skp_files.append(filename) | 407 self._skp_files.append(filename) |
369 shutil.rmtree(site) | 408 shutil.rmtree(site) |
370 | 409 |
371 def _CreateLocalStorageDirs(self): | 410 def _CreateLocalStorageDirs(self): |
(...skipping 79 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
451 default=None) | 490 default=None) |
452 option_parser.add_option( | 491 option_parser.add_option( |
453 '', '--non-interactive', action='store_true', | 492 '', '--non-interactive', action='store_true', |
454 help='Runs the script without any prompts. If this flag is specified and ' | 493 help='Runs the script without any prompts. If this flag is specified and ' |
455 '--skia_tools is specified then the debugger is not run.', | 494 '--skia_tools is specified then the debugger is not run.', |
456 default=False) | 495 default=False) |
457 options, unused_args = option_parser.parse_args() | 496 options, unused_args = option_parser.parse_args() |
458 | 497 |
459 playback = SkPicturePlayback(options) | 498 playback = SkPicturePlayback(options) |
460 sys.exit(playback.Run()) | 499 sys.exit(playback.Run()) |
OLD | NEW |