Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(239)

Side by Side Diff: tools/skp/webpages_playback.py

Issue 655313003: Add recreate_skps script (Closed) Base URL: https://skia.googlesource.com/skia.git@master
Patch Set: Address comments Created 6 years, 2 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « tools/skp/recreate_skps.py ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 #!/usr/bin/env python
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
5
6 """Archives or replays webpages and creates SKPs in a Google Storage location.
7
8 To archive webpages and store SKP files (archives should be rarely updated):
9
10 cd ../buildbot/slave/skia_slave_scripts
11 python webpages_playback.py --dest_gsbase=gs://rmistry --record \
12 --page_sets=all --skia_tools=/home/default/trunk/out/Debug/ \
13 --browser_executable=/tmp/chromium/out/Release/chrome
14
15
16 To replay archived webpages and re-generate SKP files (should be run whenever
17 SkPicture.PICTURE_VERSION changes):
18
19 cd ../buildbot/slave/skia_slave_scripts
20 python webpages_playback.py --dest_gsbase=gs://rmistry \
21 --page_sets=all --skia_tools=/home/default/trunk/out/Debug/ \
22 --browser_executable=/tmp/chromium/out/Release/chrome
23
24
25 Specify the --page_sets flag (default value is 'all') to pick a list of which
26 webpages should be archived and/or replayed. Eg:
27
28 --page_sets=page_sets/skia_yahooanswers_desktop.json,\
29 page_sets/skia_wikipedia_galaxynexus.json
30
31 The --browser_executable flag should point to the browser binary you want to use
32 to capture archives and/or capture SKP files. Majority of the time it should be
33 a newly built chrome binary.
34
35 The --upload_to_gs flag controls whether generated artifacts will be uploaded
36 to Google Storage (default value is False if not specified).
37
38 The --non-interactive flag controls whether the script will prompt the user
39 (default value is False if not specified).
40
41 The --skia_tools flag if specified will allow this script to run
42 debugger, render_pictures, and render_pdfs on the captured
43 SKP(s). The tools are run after all SKPs are succesfully captured to make sure
44 they can be added to the buildbots with no breakages.
45 To preview the captured SKP before proceeding to the next page_set specify both
46 --skia_tools and --view_debugger_output.
47 """
48
49 import glob
50 import optparse
51 import os
52 import posixpath
53 import shutil
54 import subprocess
55 import sys
56 import tempfile
57 import time
58 import traceback
59
60 sys.path.insert(0, os.getcwd())
61
62 from common.py.utils import gs_utils
63 from common.py.utils import shell_utils
64
65 ROOT_PLAYBACK_DIR_NAME = 'playback'
66 SKPICTURES_DIR_NAME = 'skps'
67
68
69 # Local archive and SKP directories.
70 LOCAL_PLAYBACK_ROOT_DIR = os.path.join(
71 tempfile.gettempdir(), ROOT_PLAYBACK_DIR_NAME)
72 LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR = os.path.join(
73 os.path.abspath(os.path.dirname(__file__)), 'page_sets', 'data')
74 TMP_SKP_DIR = tempfile.mkdtemp()
75
76 # Stdout that signifies that a recording has failed.
77 RECORD_FAILURE_MSG = 'The recording has not been updated for these pages.'
78
79 # Name of the SKP benchmark
80 SKP_BENCHMARK = 'skpicture_printer'
81
82 # The max base name length of Skp files.
83 MAX_SKP_BASE_NAME_LEN = 31
84
85 # Dictionary of device to platform prefixes for SKP files.
86 DEVICE_TO_PLATFORM_PREFIX = {
87 'desktop': 'desk',
88 'galaxynexus': 'mobi',
89 'nexus10': 'tabl'
90 }
91
92 # How many times the record_wpr binary should be retried.
93 RETRY_RECORD_WPR_COUNT = 5
94 # How many times the run_measurement binary should be retried.
95 RETRY_RUN_MEASUREMENT_COUNT = 5
96
97 X11_DISPLAY = os.getenv('DISPLAY', ':0')
98
99 GS_PREDEFINED_ACL = gs_utils.GSUtils.PredefinedACL.PRIVATE
100 GS_FINE_GRAINED_ACL_LIST = [
101 (gs_utils.GSUtils.IdType.GROUP_BY_DOMAIN, 'google.com',
102 gs_utils.GSUtils.Permission.READ),
103 ]
104
105
106 class SkPicturePlayback(object):
107 """Class that archives or replays webpages and creates SKPs."""
108
109 def __init__(self, parse_options):
110 """Constructs a SkPicturePlayback BuildStep instance."""
111 assert parse_options.browser_executable, 'Must specify --browser_executable'
112 self._browser_executable = parse_options.browser_executable
113
114 self._all_page_sets_specified = parse_options.page_sets == 'all'
115 self._page_sets = self._ParsePageSets(parse_options.page_sets)
116
117 self._dest_gsbase = parse_options.dest_gsbase
118 self._record = parse_options.record
119 self._skia_tools = parse_options.skia_tools
120 self._non_interactive = parse_options.non_interactive
121 self._upload_to_gs = parse_options.upload_to_gs
122 self._alternate_upload_dir = parse_options.alternate_upload_dir
123 self._skip_all_gs_access = parse_options.skip_all_gs_access
124 self._telemetry_binaries_dir = os.path.join(parse_options.chrome_src_path,
125 'tools', 'perf')
126
127 self._local_skp_dir = os.path.join(
128 parse_options.output_dir, ROOT_PLAYBACK_DIR_NAME, SKPICTURES_DIR_NAME)
129 self._local_record_webpages_archive_dir = os.path.join(
130 parse_options.output_dir, ROOT_PLAYBACK_DIR_NAME, 'webpages_archive')
131
132 # List of SKP files generated by this script.
133 self._skp_files = []
134
135 def _ParsePageSets(self, page_sets):
136 if not page_sets:
137 raise ValueError('Must specify at least one page_set!')
138 elif self._all_page_sets_specified:
139 # Get everything from the page_sets directory.
140 page_sets_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)),
141 'page_sets')
142 ps = [os.path.join(page_sets_dir, page_set)
143 for page_set in os.listdir(page_sets_dir)
144 if not os.path.isdir(os.path.join(page_sets_dir, page_set)) and
145 page_set.endswith('.py')]
146 elif '*' in page_sets:
147 # Explode and return the glob.
148 ps = glob.glob(page_sets)
149 else:
150 ps = page_sets.split(',')
151 ps.sort()
152 return ps
153
154 def Run(self):
155 """Run the SkPicturePlayback BuildStep."""
156
157 # Delete any left over data files in the data directory.
158 for archive_file in glob.glob(
159 os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, 'skia_*')):
160 os.remove(archive_file)
161
162 # Delete the local root directory if it already exists.
163 if os.path.exists(LOCAL_PLAYBACK_ROOT_DIR):
164 shutil.rmtree(LOCAL_PLAYBACK_ROOT_DIR)
165
166 # Create the required local storage directories.
167 self._CreateLocalStorageDirs()
168
169 # Start the timer.
170 start_time = time.time()
171
172 # Loop through all page_sets.
173 for page_set in self._page_sets:
174
175 page_set_basename = os.path.basename(page_set).split('.')[0] + '.json'
176 wpr_data_file = page_set.split(os.path.sep)[-1].split('.')[0] + '_000.wpr'
177
178 if self._record:
179 # Create an archive of the specified webpages if '--record=True' is
180 # specified.
181 record_wpr_cmd = (
182 'DISPLAY=%s' % X11_DISPLAY,
183 os.path.join(self._telemetry_binaries_dir, 'record_wpr'),
184 '--extra-browser-args=--disable-setuid-sandbox',
185 '--browser=exact',
186 '--browser-executable=%s' % self._browser_executable,
187 page_set
188 )
189 for _ in range(RETRY_RECORD_WPR_COUNT):
190 output = shell_utils.run(' '.join(record_wpr_cmd), shell=True)
191 if RECORD_FAILURE_MSG in output:
192 print output
193 else:
194 # Break out of the retry loop since there were no errors.
195 break
196 else:
197 # If we get here then record_wpr did not succeed and thus did not
198 # break out of the loop.
199 raise Exception('record_wpr failed for page_set: %s' % page_set)
200
201 else:
202 if not self._skip_all_gs_access:
203 # Get the webpages archive so that it can be replayed.
204 self._DownloadWebpagesArchive(wpr_data_file, page_set_basename)
205
206 page_set_name = os.path.basename(page_set).split('.')[0]
207 run_measurement_cmd = (
208 'DISPLAY=%s' % X11_DISPLAY,
209 'timeout', '300',
210 os.path.join(self._telemetry_binaries_dir, 'run_measurement'),
211 '--extra-browser-args=--disable-setuid-sandbox',
212 '--browser=exact',
213 '--browser-executable=%s' % self._browser_executable,
214 SKP_BENCHMARK,
215 page_set_name,
216 '-o',
217 '/tmp/test.skp',
218 '--skp-outdir=%s' % TMP_SKP_DIR
219 )
220 page_set_dst = os.path.join(self._telemetry_binaries_dir, 'page_sets',
221 os.path.basename(page_set))
222 wpr_dst = os.path.join(self._telemetry_binaries_dir, 'page_sets', 'data',
223 wpr_data_file)
224 json_dst = os.path.join(self._telemetry_binaries_dir, 'page_sets', 'data',
225 page_set_basename)
226 copied_page_set = False
227 if not os.path.exists(page_set_dst):
228 print 'Copying %s to %s' % (page_set, page_set_dst)
229 shutil.copyfile(page_set, page_set_dst)
230 wpr_src = os.path.join(os.path.dirname(page_set), 'data',
231 wpr_data_file)
232 print 'Copying %s to %s' % (wpr_src, wpr_dst)
233 shutil.copyfile(wpr_src, wpr_dst)
234 json_src = os.path.join(os.path.dirname(page_set), 'data',
235 page_set_basename)
236 print 'Copying %s to %s' % (json_src, json_dst)
237 shutil.copyfile(json_src, json_dst)
238 copied_page_set = True
239
240 for _ in range(RETRY_RUN_MEASUREMENT_COUNT):
241 try:
242 print '\n\n=======Capturing SKP of %s=======\n\n' % page_set
243 shell_utils.run(' '.join(run_measurement_cmd), shell=True)
244 except shell_utils.CommandFailedException:
245 # skpicture_printer sometimes fails with AssertionError but the
246 # captured SKP is still valid. This is a known issue.
247 pass
248
249 if self._record:
250 # Move over the created archive into the local webpages archive
251 # directory.
252 shutil.move(
253 os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, wpr_data_file),
254 self._local_record_webpages_archive_dir)
255 shutil.move(
256 os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR,
257 page_set_basename),
258 self._local_record_webpages_archive_dir)
259
260 # Rename generated SKP files into more descriptive names.
261 try:
262 self._RenameSkpFiles(page_set)
263 # Break out of the retry loop since there were no errors.
264 break
265 except Exception:
266 # There was a failure continue with the loop.
267 traceback.print_exc()
268 print '\n\n=======Retrying %s=======\n\n' % page_set
269 time.sleep(10)
270 else:
271 if copied_page_set:
272 os.remove(page_set_dst)
273 os.remove(wpr_dst)
274 os.remove(json_dst)
275 # If we get here then run_measurement did not succeed and thus did not
276 # break out of the loop.
277 raise Exception('run_measurement failed for page_set: %s' % page_set)
278
279 if copied_page_set:
280 os.remove(page_set_dst)
281 os.remove(wpr_dst)
282 os.remove(json_dst)
283
284 print '\n\n=======Capturing SKP files took %s seconds=======\n\n' % (
285 time.time() - start_time)
286
287 if self._skia_tools:
288 render_pictures_cmd = [
289 os.path.join(self._skia_tools, 'render_pictures'),
290 '-r', self._local_skp_dir
291 ]
292 render_pdfs_cmd = [
293 os.path.join(self._skia_tools, 'render_pdfs'),
294 self._local_skp_dir
295 ]
296
297 for tools_cmd in (render_pictures_cmd, render_pdfs_cmd):
298 print '\n\n=======Running %s=======' % ' '.join(tools_cmd)
299 proc = subprocess.Popen(tools_cmd)
300 (code, output) = shell_utils.log_process_after_completion(proc,
301 echo=False)
302 if code != 0:
303 raise Exception('%s failed!' % ' '.join(tools_cmd))
304
305 if not self._non_interactive:
306 print '\n\n=======Running debugger======='
307 os.system('%s %s' % (os.path.join(self._skia_tools, 'debugger'),
308 os.path.join(self._local_skp_dir, '*')))
309
310 print '\n\n'
311
312 if not self._skip_all_gs_access and self._upload_to_gs:
313 print '\n\n=======Uploading to Google Storage=======\n\n'
314 # Copy the directory structure in the root directory into Google Storage.
315 dest_dir_name = ROOT_PLAYBACK_DIR_NAME
316 if self._alternate_upload_dir:
317 dest_dir_name = self._alternate_upload_dir
318
319 gs_bucket = self._dest_gsbase.lstrip(gs_utils.GS_PREFIX)
320 gs_utils.GSUtils().upload_dir_contents(
321 LOCAL_PLAYBACK_ROOT_DIR, gs_bucket, dest_dir_name,
322 upload_if=gs_utils.GSUtils.UploadIf.IF_MODIFIED,
323 predefined_acl=GS_PREDEFINED_ACL,
324 fine_grained_acl_list=GS_FINE_GRAINED_ACL_LIST)
325
326 print '\n\n=======New SKPs have been uploaded to %s =======\n\n' % (
327 posixpath.join(self._dest_gsbase, dest_dir_name, SKPICTURES_DIR_NAME))
328 else:
329 print '\n\n=======Not Uploading to Google Storage=======\n\n'
330 print 'Generated resources are available in %s\n\n' % (
331 LOCAL_PLAYBACK_ROOT_DIR)
332
333 return 0
334
335 def _RenameSkpFiles(self, page_set):
336 """Rename generated SKP files into more descriptive names.
337
338 Look into the subdirectory of TMP_SKP_DIR and find the most interesting
339 .skp in there to be this page_set's representative .skp.
340 """
341 # Here's where we're assuming there's one page per pageset.
342 # If there were more than one, we'd overwrite filename below.
343
344 # /path/to/skia_yahooanswers_desktop.json -> skia_yahooanswers_desktop.json
345 _, ps_filename = os.path.split(page_set)
346 # skia_yahooanswers_desktop.json -> skia_yahooanswers_desktop
347 ps_basename, _ = os.path.splitext(ps_filename)
348 # skia_yahooanswers_desktop -> skia, yahooanswers, desktop
349 _, page_name, device = ps_basename.split('_')
350
351 basename = '%s_%s' % (DEVICE_TO_PLATFORM_PREFIX[device], page_name)
352 filename = basename[:MAX_SKP_BASE_NAME_LEN] + '.skp'
353
354 subdirs = glob.glob(os.path.join(TMP_SKP_DIR, '*'))
355 assert len(subdirs) == 1
356 for site in subdirs:
357 # We choose the largest .skp as the most likely to be interesting.
358 largest_skp = max(glob.glob(os.path.join(site, '*.skp')),
359 key=lambda path: os.stat(path).st_size)
360 dest = os.path.join(self._local_skp_dir, filename)
361 print 'Moving', largest_skp, 'to', dest
362 shutil.move(largest_skp, dest)
363 self._skp_files.append(filename)
364 shutil.rmtree(site)
365
366 def _CreateLocalStorageDirs(self):
367 """Creates required local storage directories for this script."""
368 for d in (self._local_record_webpages_archive_dir,
369 self._local_skp_dir):
370 if os.path.exists(d):
371 shutil.rmtree(d)
372 os.makedirs(d)
373
374 def _DownloadWebpagesArchive(self, wpr_data_file, page_set_basename):
375 """Downloads the webpages archive and its required page set from GS."""
376 wpr_source = posixpath.join(ROOT_PLAYBACK_DIR_NAME, 'webpages_archive',
377 wpr_data_file)
378 page_set_source = posixpath.join(ROOT_PLAYBACK_DIR_NAME,
379 'webpages_archive',
380 page_set_basename)
381 gs = gs_utils.GSUtils()
382 gs_bucket = self._dest_gsbase.lstrip(gs_utils.GS_PREFIX)
383 if (gs.does_storage_object_exist(gs_bucket, wpr_source) and
384 gs.does_storage_object_exist(gs_bucket, page_set_source)):
385 gs.download_file(gs_bucket, wpr_source,
386 os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR,
387 wpr_data_file))
388 gs.download_file(gs_bucket, page_set_source,
389 os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR,
390 page_set_basename))
391 else:
392 raise Exception('%s and %s do not exist in Google Storage!' % (
393 wpr_source, page_set_source))
394
395
396 if '__main__' == __name__:
397 option_parser = optparse.OptionParser()
398 option_parser.add_option(
399 '', '--page_sets',
400 help='Specifies the page sets to use to archive. Supports globs.',
401 default='all')
402 option_parser.add_option(
403 '', '--skip_all_gs_access', action='store_true',
404 help='All Google Storage interactions will be skipped if this flag is '
405 'specified. This is useful for cases where the user does not have '
406 'the required .boto file but would like to generate webpage '
407 'archives and SKPs from the Skia page sets.',
408 default=False)
409 option_parser.add_option(
410 '', '--record', action='store_true',
411 help='Specifies whether a new website archive should be created.',
412 default=False)
413 option_parser.add_option(
414 '', '--dest_gsbase',
415 help='gs:// bucket_name, the bucket to upload the file to.',
416 default='gs://chromium-skia-gm')
417 option_parser.add_option(
418 '', '--skia_tools',
419 help=('Path to compiled Skia executable tools. '
420 'render_pictures/render_pdfs is run on the set '
421 'after all SKPs are captured. If the script is run without '
422 '--non-interactive then the debugger is also run at the end. Debug '
423 'builds are recommended because they seem to catch more failures '
424 'than Release builds.'),
425 default=None)
426 option_parser.add_option(
427 '', '--upload_to_gs', action='store_true',
428 help='Does not upload to Google Storage if this is False.',
429 default=False)
430 option_parser.add_option(
431 '', '--alternate_upload_dir',
432 help='Uploads to a different directory in Google Storage if this flag is '
433 'specified',
434 default=None)
435 option_parser.add_option(
436 '', '--output_dir',
437 help='Directory where SKPs and webpage archives will be outputted to.',
438 default=tempfile.gettempdir())
439 option_parser.add_option(
440 '', '--browser_executable',
441 help='The exact browser executable to run.',
442 default=None)
443 option_parser.add_option(
444 '', '--chrome_src_path',
445 help='Path to the chromium src directory.',
446 default=None)
447 option_parser.add_option(
448 '', '--non-interactive', action='store_true',
449 help='Runs the script without any prompts. If this flag is specified and '
450 '--skia_tools is specified then the debugger is not run.',
451 default=False)
452 options, unused_args = option_parser.parse_args()
453
454 playback = SkPicturePlayback(options)
455 sys.exit(playback.Run())
OLDNEW
« no previous file with comments | « tools/skp/recreate_skps.py ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698