Index: slave/skia_slave_scripts/upload_gm_results.py |
diff --git a/slave/skia_slave_scripts/upload_gm_results.py b/slave/skia_slave_scripts/upload_gm_results.py |
deleted file mode 100755 |
index 788086f50a29da7d90b339b65c407e3038f685af..0000000000000000000000000000000000000000 |
--- a/slave/skia_slave_scripts/upload_gm_results.py |
+++ /dev/null |
@@ -1,164 +0,0 @@ |
-#!/usr/bin/env python |
-# Copyright (c) 2012 The Chromium Authors. All rights reserved. |
-# Use of this source code is governed by a BSD-style license that can be |
-# found in the LICENSE file. |
- |
-""" Upload actual GM results to the cloud to allow for rebaselining.""" |
- |
-# TODO(mtklein): Delete most of this and s/GM/DM/g the rest when GM's gone. |
- |
-from build_step import BuildStep |
-from utils import gs_utils |
-from utils import old_gs_utils |
-import datetime |
-import os |
-import posixpath |
-import re |
-import shutil |
-import skia_vars |
-import sys |
-import tempfile |
- |
- |
-GS_SUMMARIES_BUCKET = 'gs://chromium-skia-gm-summaries' |
- |
- |
-class UploadGMResults(BuildStep): |
- def __init__(self, timeout=4800, **kwargs): |
- super(UploadGMResults, self).__init__(timeout=timeout, **kwargs) |
- |
- def _GSUploadAllImages(self, src_dir): |
- """Upload all image files from src_dir to Google Storage. |
- We know that GM wrote out these image files with a filename pattern we |
- can use to generate the checksum-based Google Storage paths.""" |
- all_files = sorted(os.listdir(src_dir)) |
- def filematcher(filename): |
- return filename.endswith('.png') |
- files_to_upload = filter(filematcher, all_files) |
- print 'Uploading %d GM-actual files to Google Storage...' % ( |
- len(files_to_upload)) |
- if not files_to_upload: |
- return |
- filename_pattern = re.compile('^([^_]+)_(.+)_([^_]+)\.png$') |
- |
- gm_actuals_subdir = 'gm' |
- temp_root = tempfile.mkdtemp() |
- try: |
- # Copy all of the desired files to a staging dir, with new filenames. |
- for filename in files_to_upload: |
- match = filename_pattern.match(filename) |
- if not match: |
- print 'Warning: found no images matching pattern "%s"' % filename |
- continue |
- (hashtype, test, hashvalue) = match.groups() |
- src_filepath = os.path.join(src_dir, filename) |
- temp_dir = os.path.join(temp_root, gm_actuals_subdir, hashtype, test) |
- if not os.path.isdir(temp_dir): |
- os.makedirs(temp_dir) |
- shutil.copy(src_filepath, os.path.join(temp_dir, hashvalue + '.png')) |
- |
- # Upload the entire staging dir to Google Storage. |
- # At present, this will merge the entire contents of [temp_root]/gm |
- # into the existing contents of gs://chromium-skia-gm/gm . |
- # |
- # TODO(epoger): Add a "noclobber" mode to gs_utils.upload_dir_contents() |
- # and use it here so we don't re-upload image files we already have |
- # in Google Storage. |
- bucket_url = gs_utils.GSUtils.with_gs_prefix( |
- skia_vars.GetGlobalVariable('googlestorage_bucket')) |
- old_gs_utils.upload_dir_contents( |
- local_src_dir=os.path.abspath( |
- os.path.join(temp_root, gm_actuals_subdir)), |
- remote_dest_dir=posixpath.join(bucket_url, gm_actuals_subdir), |
- gs_acl='public-read', |
- http_header_lines=['Cache-Control:public,max-age=3600']) |
- finally: |
- shutil.rmtree(temp_root) |
- |
- def _GSUploadJsonFiles(self, src_dir, step_name=None): |
- """Upload just the JSON files within src_dir to GS_SUMMARIES_BUCKET. |
- |
- Args: |
- src_dir: (string) directory to upload contents of |
- step_name: (string) name of the step that is performing this action; |
- defaults to self.__class__.__name__ |
- """ |
- if not step_name: |
- step_name = self.__class__.__name__ |
- all_files = sorted(os.listdir(src_dir)) |
- def filematcher(filename): |
- return filename.endswith('.json') |
- files_to_upload = filter(filematcher, all_files) |
- print 'Uploading %d JSON files to Google Storage: %s...' % ( |
- len(files_to_upload), files_to_upload) |
- gs_dest_dir = posixpath.join( |
- GS_SUMMARIES_BUCKET, self._args['builder_name']) |
- for filename in files_to_upload: |
- src_path = os.path.join(src_dir, filename) |
- gs_dest_path = posixpath.join(gs_dest_dir, filename) |
- old_gs_utils.upload_file( |
- local_src_path=src_path, remote_dest_path=gs_dest_path, |
- gs_acl='public-read', only_if_modified=True, |
- http_header_lines=['Cache-Control:public,max-age=3600']) |
- |
- def _UploadDMResults(self): |
- # self._dm_dir holds a bunch of <hash>.png files and one dm.json. |
- |
- # Upload the dm.json summary file. |
- summary_dir = tempfile.mkdtemp() |
- shutil.move(os.path.join(self._dm_dir, 'dm.json'), |
- os.path.join(summary_dir, 'dm.json')) |
- |
- # /dm-json-v1/year/month/day/hour/git-hash/builder/build-number |
- now = datetime.datetime.utcnow() |
- summary_dest_dir = '/'.join(['dm-json-v1', |
- str(now.year).zfill(4), |
- str(now.month).zfill(2), |
- str(now.day).zfill(2), |
- str(now.hour).zfill(2), |
- self._got_revision, |
- self._builder_name, |
- self._build_number]) |
- # Trybot results are further siloed by CL. |
- if self._is_try: |
- summary_dest_dir = '/'.join(['trybot', |
- summary_dest_dir, |
- self._args['issue_number']]) |
- |
- gs = gs_utils.GSUtils() |
- acl = gs.PLAYBACK_CANNED_ACL # Private, |
- fine_acls = gs.PLAYBACK_FINEGRAINED_ACL_LIST # but Google-visible. |
- |
- gs.upload_dir_contents( |
- source_dir=summary_dir, |
- dest_bucket=skia_vars.GetGlobalVariable('dm_summaries_bucket'), |
- dest_dir=summary_dest_dir, |
- upload_if=gs.UploadIf.ALWAYS, |
- predefined_acl=acl, |
- fine_grained_acl_list=fine_acls) |
- |
- # Now we can upload everything that's left, all the .pngs. |
- gs.upload_dir_contents( |
- source_dir=self._dm_dir, |
- dest_bucket=skia_vars.GetGlobalVariable('dm_images_bucket'), |
- dest_dir='dm-images-v1', |
- upload_if=gs.UploadIf.IF_NEW, |
- predefined_acl=acl, |
- fine_grained_acl_list=fine_acls) |
- |
- # Just for hygiene, put dm.json back. |
- shutil.move(os.path.join(summary_dir, 'dm.json'), |
- os.path.join(self._dm_dir, 'dm.json')) |
- os.rmdir(summary_dir) |
- |
- def _Run(self): |
- # TODO(epoger): Can't we get gm_output_dir from BuildStep._gm_actual_dir ? |
- gm_output_dir = os.path.join(os.pardir, os.pardir, 'gm', 'actual', |
- self._args['builder_name']) |
- self._GSUploadAllImages(src_dir=gm_output_dir) |
- self._GSUploadJsonFiles(src_dir=gm_output_dir) |
- self._UploadDMResults() |
- |
- |
-if '__main__' == __name__: |
- sys.exit(BuildStep.RunBuildStep(UploadGMResults)) |