Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(38)

Side by Side Diff: slave/skia_slave_scripts/webpages_playback.py

Issue 648353002: Remove Skia's forked buildbot code (Closed) Base URL: https://skia.googlesource.com/buildbot.git@master
Patch Set: Fix launch_slaves, remove more stuff Created 6 years, 2 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
(Empty)
1 #!/usr/bin/env python
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
5
6 """Archives or replays webpages and creates SKPs in a Google Storage location.
7
8 To archive webpages and store SKP files (archives should be rarely updated):
9
10 cd ../buildbot/slave/skia_slave_scripts
11 python webpages_playback.py --dest_gsbase=gs://rmistry --record \
12 --page_sets=all --skia_tools=/home/default/trunk/out/Debug/ \
13 --browser_executable=/tmp/chromium/out/Release/chrome
14
15
16 To replay archived webpages and re-generate SKP files (should be run whenever
17 SkPicture.PICTURE_VERSION changes):
18
19 cd ../buildbot/slave/skia_slave_scripts
20 python webpages_playback.py --dest_gsbase=gs://rmistry \
21 --page_sets=all --skia_tools=/home/default/trunk/out/Debug/ \
22 --browser_executable=/tmp/chromium/out/Release/chrome
23
24
25 Specify the --page_sets flag (default value is 'all') to pick a list of which
26 webpages should be archived and/or replayed. Eg:
27
28 --page_sets=page_sets/skia_yahooanswers_desktop.json,\
29 page_sets/skia_wikipedia_galaxynexus.json
30
31 The --browser_executable flag should point to the browser binary you want to use
32 to capture archives and/or capture SKP files. Majority of the time it should be
33 a newly built chrome binary.
34
35 The --upload_to_gs flag controls whether generated artifacts will be uploaded
36 to Google Storage (default value is False if not specified).
37
38 The --non-interactive flag controls whether the script will prompt the user
39 (default value is False if not specified).
40
41 The --skia_tools flag if specified will allow this script to run
42 debugger, render_pictures, and render_pdfs on the captured
43 SKP(s). The tools are run after all SKPs are succesfully captured to make sure
44 they can be added to the buildbots with no breakages.
45 To preview the captured SKP before proceeding to the next page_set specify both
46 --skia_tools and --view_debugger_output.
47 """
48
49 import glob
50 import optparse
51 import os
52 import posixpath
53 import shutil
54 import subprocess
55 import sys
56 import tempfile
57 import time
58 import traceback
59
60
61 # Set the PYTHONPATH for this script to include chromium_buildbot scripts,
62 # and site_config.
63 BUILDBOT_PATH = os.path.realpath(os.path.join(
64 os.path.dirname(os.path.abspath(__file__)), os.pardir, os.pardir))
65 sys.path.append(os.path.join(BUILDBOT_PATH, 'common'))
66 sys.path.append(os.path.join(BUILDBOT_PATH, 'third_party', 'chromium_buildbot',
67 'scripts'))
68 sys.path.append(os.path.join(BUILDBOT_PATH, 'third_party', 'chromium_buildbot',
69 'site_config'))
70
71 from utils import file_utils
72 from utils import gs_utils
73 from utils import old_gs_utils
74 from py.utils import misc
75 from py.utils import shell_utils
76
77 from slave import slave_utils
78
79 from playback_dirs import ROOT_PLAYBACK_DIR_NAME
80 from playback_dirs import SKPICTURES_DIR_NAME
81
82
83 # Local archive and SKP directories.
84 LOCAL_PLAYBACK_ROOT_DIR = os.path.join(
85 tempfile.gettempdir(), ROOT_PLAYBACK_DIR_NAME)
86 LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR = os.path.join(
87 os.path.abspath(os.path.dirname(__file__)), 'page_sets', 'data')
88 TMP_SKP_DIR = tempfile.mkdtemp()
89
90 # Location of Telemetry binaries (record_wpr, run_measurement).
91 TELEMETRY_BINARIES_DIR = os.path.join(misc.BUILDBOT_PATH, 'third_party',
92 'chromium_trunk', 'src', 'tools', 'perf')
93
94 # Location of the credentials.json file and the string that represents missing
95 # passwords.
96 CREDENTIALS_FILE_PATH = os.path.join(
97 os.path.abspath(os.path.dirname(__file__)), 'page_sets', 'data',
98 'credentials.json'
99 )
100
101 # Stdout that signifies that a recording has failed.
102 RECORD_FAILURE_MSG = 'The recording has not been updated for these pages.'
103
104 # Name of the SKP benchmark
105 SKP_BENCHMARK = 'skpicture_printer'
106
107 # The max base name length of Skp files.
108 MAX_SKP_BASE_NAME_LEN = 31
109
110 # Dictionary of device to platform prefixes for SKP files.
111 DEVICE_TO_PLATFORM_PREFIX = {
112 'desktop': 'desk',
113 'galaxynexus': 'mobi',
114 'nexus10': 'tabl'
115 }
116
117 # How many times the record_wpr binary should be retried.
118 RETRY_RECORD_WPR_COUNT = 5
119 # How many times the run_measurement binary should be retried.
120 RETRY_RUN_MEASUREMENT_COUNT = 30
121
122 # Location of the credentials.json file in Google Storage.
123 CREDENTIALS_GS_LOCATION = (
124 'gs://chromium-skia-gm/playback/credentials/credentials.json')
125
126 X11_DISPLAY = os.getenv('DISPLAY', ':0')
127
128
129 class SkPicturePlayback(object):
130 """Class that archives or replays webpages and creates SKPs."""
131
132 def __init__(self, parse_options):
133 """Constructs a SkPicturePlayback BuildStep instance."""
134 assert parse_options.browser_executable, 'Must specify --browser_executable'
135 self._browser_executable = parse_options.browser_executable
136
137 self._all_page_sets_specified = parse_options.page_sets == 'all'
138 self._page_sets = self._ParsePageSets(parse_options.page_sets)
139
140 self._dest_gsbase = parse_options.dest_gsbase
141 self._record = parse_options.record
142 self._skia_tools = parse_options.skia_tools
143 self._non_interactive = parse_options.non_interactive
144 self._upload_to_gs = parse_options.upload_to_gs
145 self._alternate_upload_dir = parse_options.alternate_upload_dir
146 self._skip_all_gs_access = parse_options.skip_all_gs_access
147
148 self._local_skp_dir = os.path.join(
149 parse_options.output_dir, ROOT_PLAYBACK_DIR_NAME, SKPICTURES_DIR_NAME)
150 self._local_record_webpages_archive_dir = os.path.join(
151 parse_options.output_dir, ROOT_PLAYBACK_DIR_NAME, 'webpages_archive')
152
153 # List of SKP files generated by this script.
154 self._skp_files = []
155
156 def _ParsePageSets(self, page_sets):
157 if not page_sets:
158 raise ValueError('Must specify atleast one page_set!')
159 elif self._all_page_sets_specified:
160 # Get everything from the page_sets directory.
161 page_sets_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)),
162 'page_sets')
163 return [os.path.join(page_sets_dir, page_set)
164 for page_set in os.listdir(page_sets_dir)
165 if not os.path.isdir(os.path.join(page_sets_dir, page_set)) and
166 page_set.endswith('.py')]
167 elif '*' in page_sets:
168 # Explode and return the glob.
169 return glob.glob(page_sets)
170 else:
171 return page_sets.split(',')
172
173 def Run(self):
174 """Run the SkPicturePlayback BuildStep."""
175
176 # Ensure the right .boto file is used by gsutil.
177 if not self._skip_all_gs_access and old_gs_utils.read_timestamp_file(
178 timestamp_file_name=old_gs_utils.TIMESTAMP_COMPLETED_FILENAME,
179 gs_base=self._dest_gsbase,
180 gs_relative_dir=posixpath.join(ROOT_PLAYBACK_DIR_NAME,
181 SKPICTURES_DIR_NAME)) == "0":
182 raise Exception(
183 'Missing .boto file or .boto does not have the right credentials.'
184 'Please see https://docs.google.com/a/google.com/document/d/1ZzHP6M5q'
185 'ACA9nJnLqOZr2Hl0rjYqE4yQsQWAfVjKCzs/edit '
186 '(may have to request access). The .boto file will need to be placed '
187 'under third_party/chromium_buildbot/site_config/')
188
189 # Download the credentials file if it was not previously downloaded.
190 if self._skip_all_gs_access:
191 print """\n\nPlease create a %s file that contains:
192 {
193 "google": {
194 "username": "google_testing_account_username",
195 "password": "google_testing_account_password"
196 },
197 "facebook": {
198 "username": "facebook_testing_account_username",
199 "password": "facebook_testing_account_password"
200 }
201 }\n\n""" % CREDENTIALS_FILE_PATH
202 raw_input("Please press a key when you are ready to proceed...")
203 elif not os.path.isfile(CREDENTIALS_FILE_PATH):
204 # Download the credentials.json file from Google Storage.
205 slave_utils.GSUtilDownloadFile(
206 src=CREDENTIALS_GS_LOCATION, dst=CREDENTIALS_FILE_PATH)
207
208 # Delete any left over data files in the data directory.
209 for archive_file in glob.glob(
210 os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, 'skia_*')):
211 os.remove(archive_file)
212
213 # Delete the local root directory if it already exists.
214 if os.path.exists(LOCAL_PLAYBACK_ROOT_DIR):
215 shutil.rmtree(LOCAL_PLAYBACK_ROOT_DIR)
216
217 # Create the required local storage directories.
218 self._CreateLocalStorageDirs()
219
220 # Start the timer.
221 start_time = time.time()
222
223 # Loop through all page_sets.
224 for page_set in self._page_sets:
225
226 page_set_basename = os.path.basename(page_set).split('.')[0] + '.json'
227 wpr_data_file = page_set.split(os.path.sep)[-1].split('.')[0] + '_000.wpr'
228
229 if self._record:
230 # Create an archive of the specified webpages if '--record=True' is
231 # specified.
232 record_wpr_cmd = (
233 'DISPLAY=%s' % X11_DISPLAY,
234 os.path.join(TELEMETRY_BINARIES_DIR, 'record_wpr'),
235 '--extra-browser-args=--disable-setuid-sandbox',
236 '--browser=exact',
237 '--browser-executable=%s' % self._browser_executable,
238 page_set
239 )
240 for _ in range(RETRY_RECORD_WPR_COUNT):
241 output = shell_utils.run(' '.join(record_wpr_cmd), shell=True)
242 if RECORD_FAILURE_MSG in output:
243 print output
244 else:
245 # Break out of the retry loop since there were no errors.
246 break
247 else:
248 # If we get here then record_wpr did not succeed and thus did not
249 # break out of the loop.
250 raise Exception('record_wpr failed for page_set: %s' % page_set)
251
252 else:
253 if not self._skip_all_gs_access:
254 # Get the webpages archive so that it can be replayed.
255 self._DownloadWebpagesArchive(wpr_data_file, page_set_basename)
256
257 run_measurement_cmd = (
258 'DISPLAY=%s' % X11_DISPLAY,
259 'timeout', '300',
260 os.path.join(TELEMETRY_BINARIES_DIR, 'run_measurement'),
261 '--extra-browser-args=--disable-setuid-sandbox',
262 '--browser=exact',
263 '--browser-executable=%s' % self._browser_executable,
264 SKP_BENCHMARK,
265 page_set,
266 '-o',
267 '/tmp/test.skp',
268 '--skp-outdir=%s' % TMP_SKP_DIR
269 )
270 for _ in range(RETRY_RUN_MEASUREMENT_COUNT):
271 try:
272 print '\n\n=======Capturing SKP of %s=======\n\n' % page_set
273 shell_utils.run(' '.join(run_measurement_cmd), shell=True)
274 except shell_utils.CommandFailedException:
275 # skpicture_printer sometimes fails with AssertionError but the
276 # captured SKP is still valid. This is a known issue.
277 pass
278
279 if self._record:
280 # Move over the created archive into the local webpages archive
281 # directory.
282 shutil.move(
283 os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, wpr_data_file),
284 self._local_record_webpages_archive_dir)
285 shutil.move(
286 os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR,
287 page_set_basename),
288 self._local_record_webpages_archive_dir)
289
290 # Rename generated SKP files into more descriptive names.
291 try:
292 self._RenameSkpFiles(page_set)
293 # Break out of the retry loop since there were no errors.
294 break
295 except Exception:
296 # There was a failure continue with the loop.
297 traceback.print_exc()
298 print '\n\n=======Retrying %s=======\n\n' % page_set
299 time.sleep(10)
300 else:
301 # If we get here then run_measurement did not succeed and thus did not
302 # break out of the loop.
303 raise Exception('run_measurement failed for page_set: %s' % page_set)
304
305 print '\n\n=======Capturing SKP files took %s seconds=======\n\n' % (
306 time.time() - start_time)
307
308 if self._skia_tools:
309 render_pictures_cmd = [
310 os.path.join(self._skia_tools, 'render_pictures'),
311 '-r', self._local_skp_dir
312 ]
313 render_pdfs_cmd = [
314 os.path.join(self._skia_tools, 'render_pdfs'),
315 self._local_skp_dir
316 ]
317
318 for tools_cmd in (render_pictures_cmd, render_pdfs_cmd):
319 print '\n\n=======Running %s=======' % ' '.join(tools_cmd)
320 proc = subprocess.Popen(tools_cmd)
321 (code, output) = shell_utils.log_process_after_completion(proc,
322 echo=False)
323 if code != 0:
324 raise Exception('%s failed!' % ' '.join(tools_cmd))
325
326 if not self._non_interactive:
327 print '\n\n=======Running debugger======='
328 os.system('%s %s' % (os.path.join(self._skia_tools, 'debugger'),
329 os.path.join(self._local_skp_dir, '*')))
330
331 print '\n\n'
332
333 if not self._skip_all_gs_access and self._upload_to_gs:
334 print '\n\n=======Uploading to Google Storage=======\n\n'
335 # Copy the directory structure in the root directory into Google Storage.
336 dest_dir_name = ROOT_PLAYBACK_DIR_NAME
337 if self._alternate_upload_dir:
338 dest_dir_name = self._alternate_upload_dir
339 gs_status = slave_utils.GSUtilCopyDir(
340 src_dir=LOCAL_PLAYBACK_ROOT_DIR,
341 gs_base=self._dest_gsbase,
342 dest_dir=dest_dir_name,
343 gs_acl=gs_utils.GSUtils.PLAYBACK_CANNED_ACL)
344 if gs_status != 0:
345 raise Exception(
346 'ERROR: GSUtilCopyDir error %d. "%s" -> "%s/%s"' % (
347 gs_status, LOCAL_PLAYBACK_ROOT_DIR, self._dest_gsbase,
348 ROOT_PLAYBACK_DIR_NAME))
349 self._SetGoogleReadACLs(
350 posixpath.join(self._dest_gsbase, dest_dir_name, SKPICTURES_DIR_NAME))
351
352 # Add a timestamp file to the SKP directory in Google Storage so we can
353 # use directory level rsync like functionality.
354 old_gs_utils.write_timestamp_file(
355 timestamp_file_name=old_gs_utils.TIMESTAMP_COMPLETED_FILENAME,
356 timestamp_value=time.time(),
357 gs_base=self._dest_gsbase,
358 gs_relative_dir=posixpath.join(dest_dir_name, SKPICTURES_DIR_NAME),
359 gs_acl=gs_utils.GSUtils.PLAYBACK_CANNED_ACL,
360 local_dir=LOCAL_PLAYBACK_ROOT_DIR)
361
362 print '\n\n=======New SKPs have been uploaded to %s =======\n\n' % (
363 posixpath.join(self._dest_gsbase, dest_dir_name, SKPICTURES_DIR_NAME))
364 else:
365 print '\n\n=======Not Uploading to Google Storage=======\n\n'
366 print 'Generated resources are available in %s\n\n' % (
367 LOCAL_PLAYBACK_ROOT_DIR)
368
369 return 0
370
371 def _SetGoogleReadACLs(self, gs_dir):
372 """Sets the ACLs of all objects in the directory to google read-only.
373
374 This method assumes that there is a gsutil in the system PATH that is recent
375 enough to run the 'acl ch' command. The gsutil in chromium_buildbot is old
376 and cannot run this command.
377 """
378 update_acls_cmd = ['gsutil', 'acl', 'ch', '-g', 'google.com:READ',
379 posixpath.join(gs_dir, '*')]
380 shell_utils.run(update_acls_cmd)
381
382 def _RenameSkpFiles(self, page_set):
383 """Rename generated SKP files into more descriptive names.
384
385 Look into the subdirectory of TMP_SKP_DIR and find the most interesting
386 .skp in there to be this page_set's representative .skp.
387 """
388 # Here's where we're assuming there's one page per pageset.
389 # If there were more than one, we'd overwrite filename below.
390
391 # /path/to/skia_yahooanswers_desktop.json -> skia_yahooanswers_desktop.json
392 _, ps_filename = os.path.split(page_set)
393 # skia_yahooanswers_desktop.json -> skia_yahooanswers_desktop
394 ps_basename, _ = os.path.splitext(ps_filename)
395 # skia_yahooanswers_desktop -> skia, yahooanswers, desktop
396 _, page_name, device = ps_basename.split('_')
397
398 basename = '%s_%s' % (DEVICE_TO_PLATFORM_PREFIX[device], page_name)
399 filename = basename[:MAX_SKP_BASE_NAME_LEN] + '.skp'
400
401 subdirs = glob.glob(os.path.join(TMP_SKP_DIR, '*'))
402 assert len(subdirs) == 1
403 for site in subdirs:
404 # We choose the largest .skp as the most likely to be interesting.
405 largest_skp = max(glob.glob(os.path.join(site, '*.skp')),
406 key=lambda path: os.stat(path).st_size)
407 dest = os.path.join(self._local_skp_dir, filename)
408 print 'Moving', largest_skp, 'to', dest
409 shutil.move(largest_skp, dest)
410 self._skp_files.append(filename)
411 shutil.rmtree(site)
412
413 def _CreateLocalStorageDirs(self):
414 """Creates required local storage directories for this script."""
415 file_utils.create_clean_local_dir(self._local_record_webpages_archive_dir)
416 file_utils.create_clean_local_dir(self._local_skp_dir)
417
418 def _DownloadWebpagesArchive(self, wpr_data_file, page_set_basename):
419 """Downloads the webpages archive and its required page set from GS."""
420 wpr_source = posixpath.join(
421 self._dest_gsbase, ROOT_PLAYBACK_DIR_NAME, 'webpages_archive',
422 wpr_data_file)
423 page_set_source = posixpath.join(
424 self._dest_gsbase, ROOT_PLAYBACK_DIR_NAME, 'webpages_archive',
425 page_set_basename)
426 if (old_gs_utils.does_storage_object_exist(wpr_source) and
427 old_gs_utils.does_storage_object_exist(page_set_source)):
428 slave_utils.GSUtilDownloadFile(
429 src=wpr_source, dst=LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR)
430 slave_utils.GSUtilDownloadFile(
431 src=page_set_source, dst=LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR)
432 else:
433 raise Exception('%s and %s do not exist in Google Storage!' % (
434 wpr_source, page_set_source))
435
436
437 if '__main__' == __name__:
438 option_parser = optparse.OptionParser()
439 option_parser.add_option(
440 '', '--page_sets',
441 help='Specifies the page sets to use to archive. Supports globs.',
442 default='all')
443 option_parser.add_option(
444 '', '--skip_all_gs_access', action='store_true',
445 help='All Google Storage interactions will be skipped if this flag is '
446 'specified. This is useful for cases where the user does not have '
447 'the required .boto file but would like to generate webpage '
448 'archives and SKPs from the Skia page sets.',
449 default=False)
450 option_parser.add_option(
451 '', '--record', action='store_true',
452 help='Specifies whether a new website archive should be created.',
453 default=False)
454 option_parser.add_option(
455 '', '--dest_gsbase',
456 help='gs:// bucket_name, the bucket to upload the file to.',
457 default=old_gs_utils.DEFAULT_DEST_GSBASE)
458 option_parser.add_option(
459 '', '--skia_tools',
460 help=('Path to compiled Skia executable tools. '
461 'render_pictures/render_pdfs is run on the set '
462 'after all SKPs are captured. If the script is run without '
463 '--non-interactive then the debugger is also run at the end. Debug '
464 'builds are recommended because they seem to catch more failures '
465 'than Release builds.'),
466 default=None)
467 option_parser.add_option(
468 '', '--upload_to_gs', action='store_true',
469 help='Does not upload to Google Storage if this is False.',
470 default=False)
471 option_parser.add_option(
472 '', '--alternate_upload_dir',
473 help='Uploads to a different directory in Google Storage if this flag is '
474 'specified',
475 default=None)
476 option_parser.add_option(
477 '', '--output_dir',
478 help='Directory where SKPs and webpage archives will be outputted to.',
479 default=tempfile.gettempdir())
480 option_parser.add_option(
481 '', '--browser_executable',
482 help='The exact browser executable to run.',
483 default=None)
484 option_parser.add_option(
485 '', '--non-interactive', action='store_true',
486 help='Runs the script without any prompts. If this flag is specified and '
487 '--skia_tools is specified then the debugger is not run.',
488 default=False)
489 options, unused_args = option_parser.parse_args()
490
491 playback = SkPicturePlayback(options)
492 sys.exit(playback.Run())
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698