Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(176)

Side by Side Diff: tools/skp/webpages_playback.py

Issue 1047703003: Add ability to capture SKPs from Chromium's pagesets (Closed) Base URL: https://skia.googlesource.com/skia@master
Patch Set: More cleanup Created 5 years, 8 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 """Archives or replays webpages and creates SKPs in a Google Storage location. 6 """Archives or replays webpages and creates SKPs in a Google Storage location.
7 7
8 To archive webpages and store SKP files (archives should be rarely updated): 8 To archive webpages and store SKP files (archives should be rarely updated):
9 9
10 cd skia 10 cd skia
(...skipping 89 matching lines...) Expand 10 before | Expand all | Expand 10 after
100 CREDENTIALS_GS_PATH = '/playback/credentials/credentials.json' 100 CREDENTIALS_GS_PATH = '/playback/credentials/credentials.json'
101 101
102 X11_DISPLAY = os.getenv('DISPLAY', ':0') 102 X11_DISPLAY = os.getenv('DISPLAY', ':0')
103 103
104 GS_PREDEFINED_ACL = gs_utils.GSUtils.PredefinedACL.PRIVATE 104 GS_PREDEFINED_ACL = gs_utils.GSUtils.PredefinedACL.PRIVATE
105 GS_FINE_GRAINED_ACL_LIST = [ 105 GS_FINE_GRAINED_ACL_LIST = [
106 (gs_utils.GSUtils.IdType.GROUP_BY_DOMAIN, 'google.com', 106 (gs_utils.GSUtils.IdType.GROUP_BY_DOMAIN, 'google.com',
107 gs_utils.GSUtils.Permission.READ), 107 gs_utils.GSUtils.Permission.READ),
108 ] 108 ]
109 109
110 # Path to Chromium's page sets.
111 CHROMIUM_PAGE_SETS_PATH = 'tools/perf/page_sets'
borenet 2015/03/30 19:31:57 os.path.join?
rmistry 2015/03/30 20:11:18 Done.
112
113 # Dictionary of supported Chromium page sets to their file prefixes.
114 CHROMIUM_PAGE_SETS_TO_PREFIX = {
115 'key_mobile_sites_smooth.py': 'keymobi',
116 # TODO(rmistry): Uncomment the below after it is verified that the above
117 # works.
118 # 'top_25_smooth.py': 'top25desk',
119 }
120
121
110 def remove_prefix(s, prefix): 122 def remove_prefix(s, prefix):
111 if s.startswith(prefix): 123 if s.startswith(prefix):
112 return s[len(prefix):] 124 return s[len(prefix):]
113 return s 125 return s
114 126
127
115 class SkPicturePlayback(object): 128 class SkPicturePlayback(object):
116 """Class that archives or replays webpages and creates SKPs.""" 129 """Class that archives or replays webpages and creates SKPs."""
117 130
118 def __init__(self, parse_options): 131 def __init__(self, parse_options):
119 """Constructs a SkPicturePlayback BuildStep instance.""" 132 """Constructs a SkPicturePlayback BuildStep instance."""
120 assert parse_options.browser_executable, 'Must specify --browser_executable' 133 assert parse_options.browser_executable, 'Must specify --browser_executable'
121 self._browser_executable = parse_options.browser_executable 134 self._browser_executable = parse_options.browser_executable
122 135
136 self._chrome_page_sets_path = os.path.join(parse_options.chrome_src_path,
137 CHROMIUM_PAGE_SETS_PATH)
123 self._all_page_sets_specified = parse_options.page_sets == 'all' 138 self._all_page_sets_specified = parse_options.page_sets == 'all'
124 self._page_sets = self._ParsePageSets(parse_options.page_sets) 139 self._page_sets = self._ParsePageSets(parse_options.page_sets)
125 140
126 self._dest_gsbase = parse_options.dest_gsbase 141 self._dest_gsbase = parse_options.dest_gsbase
127 self._record = parse_options.record 142 self._record = parse_options.record
128 self._skia_tools = parse_options.skia_tools 143 self._skia_tools = parse_options.skia_tools
129 self._non_interactive = parse_options.non_interactive 144 self._non_interactive = parse_options.non_interactive
130 self._upload_to_gs = parse_options.upload_to_gs 145 self._upload_to_gs = parse_options.upload_to_gs
131 self._alternate_upload_dir = parse_options.alternate_upload_dir 146 self._alternate_upload_dir = parse_options.alternate_upload_dir
132 self._skip_all_gs_access = parse_options.skip_all_gs_access 147 self._skip_all_gs_access = parse_options.skip_all_gs_access
(...skipping 12 matching lines...) Expand all
145 if not page_sets: 160 if not page_sets:
146 raise ValueError('Must specify at least one page_set!') 161 raise ValueError('Must specify at least one page_set!')
147 elif self._all_page_sets_specified: 162 elif self._all_page_sets_specified:
148 # Get everything from the page_sets directory. 163 # Get everything from the page_sets directory.
149 page_sets_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 164 page_sets_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)),
150 'page_sets') 165 'page_sets')
151 ps = [os.path.join(page_sets_dir, page_set) 166 ps = [os.path.join(page_sets_dir, page_set)
152 for page_set in os.listdir(page_sets_dir) 167 for page_set in os.listdir(page_sets_dir)
153 if not os.path.isdir(os.path.join(page_sets_dir, page_set)) and 168 if not os.path.isdir(os.path.join(page_sets_dir, page_set)) and
154 page_set.endswith('.py')] 169 page_set.endswith('.py')]
170 chromium_ps = [
171 os.path.join(self._chrome_page_sets_path, cr_page_set)
172 for cr_page_set in CHROMIUM_PAGE_SETS_TO_PREFIX]
173 ps.extend(chromium_ps)
155 elif '*' in page_sets: 174 elif '*' in page_sets:
156 # Explode and return the glob. 175 # Explode and return the glob.
157 ps = glob.glob(page_sets) 176 ps = glob.glob(page_sets)
158 else: 177 else:
159 ps = page_sets.split(',') 178 ps = page_sets.split(',')
160 ps.sort() 179 ps.sort()
161 return ps 180 return ps
162 181
182 def _IsChromiumPageSet(self, page_set):
183 """Returns true if the specified page set is a Chromium page set."""
184 return page_set.startswith(self._chrome_page_sets_path)
185
163 def Run(self): 186 def Run(self):
164 """Run the SkPicturePlayback BuildStep.""" 187 """Run the SkPicturePlayback BuildStep."""
165 188
166 # Download the credentials file if it was not previously downloaded. 189 # Download the credentials file if it was not previously downloaded.
167 if self._skip_all_gs_access: 190 if self._skip_all_gs_access:
168 print """\n\nPlease create a %s file that contains: 191 print """\n\nPlease create a %s file that contains:
169 { 192 {
170 "google": { 193 "google": {
171 "username": "google_testing_account_username", 194 "username": "google_testing_account_username",
172 "password": "google_testing_account_password" 195 "password": "google_testing_account_password"
(...skipping 26 matching lines...) Expand all
199 start_time = time.time() 222 start_time = time.time()
200 223
201 # Loop through all page_sets. 224 # Loop through all page_sets.
202 for page_set in self._page_sets: 225 for page_set in self._page_sets:
203 226
204 page_set_basename = os.path.basename(page_set).split('.')[0] 227 page_set_basename = os.path.basename(page_set).split('.')[0]
205 page_set_json_name = page_set_basename + '.json' 228 page_set_json_name = page_set_basename + '.json'
206 wpr_data_file = page_set.split(os.path.sep)[-1].split('.')[0] + '_000.wpr' 229 wpr_data_file = page_set.split(os.path.sep)[-1].split('.')[0] + '_000.wpr'
207 page_set_dir = os.path.dirname(page_set) 230 page_set_dir = os.path.dirname(page_set)
208 231
209 if self._record: 232 if self._IsChromiumPageSet(page_set):
233 print 'Using Chromium\'s captured archives for Chromium\'s page sets.'
234 elif self._record:
210 # Create an archive of the specified webpages if '--record=True' is 235 # Create an archive of the specified webpages if '--record=True' is
211 # specified. 236 # specified.
212 record_wpr_cmd = ( 237 record_wpr_cmd = (
213 'PYTHONPATH=%s:$PYTHONPATH' % page_set_dir, 238 'PYTHONPATH=%s:$PYTHONPATH' % page_set_dir,
214 'DISPLAY=%s' % X11_DISPLAY, 239 'DISPLAY=%s' % X11_DISPLAY,
215 os.path.join(self._telemetry_binaries_dir, 'record_wpr'), 240 os.path.join(self._telemetry_binaries_dir, 'record_wpr'),
216 '--extra-browser-args=--disable-setuid-sandbox', 241 '--extra-browser-args=--disable-setuid-sandbox',
217 '--browser=exact', 242 '--browser=exact',
218 '--browser-executable=%s' % self._browser_executable, 243 '--browser-executable=%s' % self._browser_executable,
219 '%s_page_set' % page_set_basename, 244 '%s_page_set' % page_set_basename,
(...skipping 110 matching lines...) Expand 10 before | Expand all | Expand 10 after
330 355
331 print '\n\n=======New SKPs have been uploaded to %s =======\n\n' % ( 356 print '\n\n=======New SKPs have been uploaded to %s =======\n\n' % (
332 posixpath.join(self._dest_gsbase, dest_dir_name, SKPICTURES_DIR_NAME)) 357 posixpath.join(self._dest_gsbase, dest_dir_name, SKPICTURES_DIR_NAME))
333 else: 358 else:
334 print '\n\n=======Not Uploading to Google Storage=======\n\n' 359 print '\n\n=======Not Uploading to Google Storage=======\n\n'
335 print 'Generated resources are available in %s\n\n' % ( 360 print 'Generated resources are available in %s\n\n' % (
336 LOCAL_PLAYBACK_ROOT_DIR) 361 LOCAL_PLAYBACK_ROOT_DIR)
337 362
338 return 0 363 return 0
339 364
365 def _GetSkiaSkpFileName(self, page_set):
366 """Returns the SKP file name for Skia page sets."""
367 # /path/to/skia_yahooanswers_desktop.py -> skia_yahooanswers_desktop.py
368 _, ps_filename = os.path.split(page_set)
borenet 2015/03/30 19:31:57 Use os.path.basename here for a little extra clean
rmistry 2015/03/30 20:11:18 Done.
369 # skia_yahooanswers_desktop.py -> skia_yahooanswers_desktop
370 ps_basename, _ = os.path.splitext(ps_filename)
371 # skia_yahooanswers_desktop -> skia, yahooanswers, desktop
372 _, page_name, device = ps_basename.split('_')
373 basename = '%s_%s' % (DEVICE_TO_PLATFORM_PREFIX[device], page_name)
374 return basename[:MAX_SKP_BASE_NAME_LEN] + '.skp'
375
376 def _GetChromiumSkpFileName(self, page_set, site):
377 """Returns the SKP file name for Chromium page sets."""
378 # /path/to/http___mobile_news_sandbox_pt0 -> http___mobile_news_sandbox_pt0
379 _, webpage = os.path.split(site)
380 # http___mobile_news_sandbox_pt0 -> mobile_news_sandbox_pt0
381 webpage = webpage.lstrip('http___').lstrip('https___')
382 # /path/to/skia_yahooanswers_desktop.py -> skia_yahooanswers_desktop.py
383 _, ps_filename = os.path.split(page_set)
384 # http___mobile_news_sandbox -> pagesetprefix_http___mobile_news_sandbox
385 basename = '%s_%s' % (CHROMIUM_PAGE_SETS_TO_PREFIX[ps_filename], webpage)
386 return basename[:MAX_SKP_BASE_NAME_LEN] + '.skp'
387
340 def _RenameSkpFiles(self, page_set): 388 def _RenameSkpFiles(self, page_set):
341 """Rename generated SKP files into more descriptive names. 389 """Rename generated SKP files into more descriptive names.
342 390
343 Look into the subdirectory of TMP_SKP_DIR and find the most interesting 391 Look into the subdirectory of TMP_SKP_DIR and find the most interesting
344 .skp in there to be this page_set's representative .skp. 392 .skp in there to be this page_set's representative .skp.
345 """ 393 """
346 # Here's where we're assuming there's one page per pageset. 394 subdirs = glob.glob(os.path.join(TMP_SKP_DIR, '*'))
347 # If there were more than one, we'd overwrite filename below. 395 for site in subdirs:
396 if self._IsChromiumPageSet(page_set):
397 filename = self._GetChromiumSkpFileName(page_set, site)
398 else:
399 filename = self._GetSkiaSkpFileName(page_set)
348 400
349 # /path/to/skia_yahooanswers_desktop.json -> skia_yahooanswers_desktop.json
350 _, ps_filename = os.path.split(page_set)
351 # skia_yahooanswers_desktop.json -> skia_yahooanswers_desktop
352 ps_basename, _ = os.path.splitext(ps_filename)
353 # skia_yahooanswers_desktop -> skia, yahooanswers, desktop
354 _, page_name, device = ps_basename.split('_')
355
356 basename = '%s_%s' % (DEVICE_TO_PLATFORM_PREFIX[device], page_name)
357 filename = basename[:MAX_SKP_BASE_NAME_LEN] + '.skp'
358
359 subdirs = glob.glob(os.path.join(TMP_SKP_DIR, '*'))
360 assert len(subdirs) == 1
361 for site in subdirs:
362 # We choose the largest .skp as the most likely to be interesting. 401 # We choose the largest .skp as the most likely to be interesting.
363 largest_skp = max(glob.glob(os.path.join(site, '*.skp')), 402 largest_skp = max(glob.glob(os.path.join(site, '*.skp')),
364 key=lambda path: os.stat(path).st_size) 403 key=lambda path: os.stat(path).st_size)
365 dest = os.path.join(self._local_skp_dir, filename) 404 dest = os.path.join(self._local_skp_dir, filename)
366 print 'Moving', largest_skp, 'to', dest 405 print 'Moving', largest_skp, 'to', dest
367 shutil.move(largest_skp, dest) 406 shutil.move(largest_skp, dest)
368 self._skp_files.append(filename) 407 self._skp_files.append(filename)
369 shutil.rmtree(site) 408 shutil.rmtree(site)
370 409
371 def _CreateLocalStorageDirs(self): 410 def _CreateLocalStorageDirs(self):
(...skipping 79 matching lines...) Expand 10 before | Expand all | Expand 10 after
451 default=None) 490 default=None)
452 option_parser.add_option( 491 option_parser.add_option(
453 '', '--non-interactive', action='store_true', 492 '', '--non-interactive', action='store_true',
454 help='Runs the script without any prompts. If this flag is specified and ' 493 help='Runs the script without any prompts. If this flag is specified and '
455 '--skia_tools is specified then the debugger is not run.', 494 '--skia_tools is specified then the debugger is not run.',
456 default=False) 495 default=False)
457 options, unused_args = option_parser.parse_args() 496 options, unused_args = option_parser.parse_args()
458 497
459 playback = SkPicturePlayback(options) 498 playback = SkPicturePlayback(options)
460 sys.exit(playback.Run()) 499 sys.exit(playback.Run())
OLDNEW
« no previous file with comments | « no previous file | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698