Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(222)

Unified Diff: gm/rebaseline_server/download_actuals.py

Issue 855003006: delete old things! (Closed) Base URL: https://skia.googlesource.com/skia.git@master
Patch Set: Add makefiles back :( Created 5 years, 11 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
« no previous file with comments | « gm/rebaseline_server/compare_to_expectations_test.py ('k') | gm/rebaseline_server/download_actuals_test.py » ('j') | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: gm/rebaseline_server/download_actuals.py
diff --git a/gm/rebaseline_server/download_actuals.py b/gm/rebaseline_server/download_actuals.py
deleted file mode 100755
index 0e60289573ad2a5fa9b424c2c2f519f6c0e6c237..0000000000000000000000000000000000000000
--- a/gm/rebaseline_server/download_actuals.py
+++ /dev/null
@@ -1,289 +0,0 @@
-#!/usr/bin/python
-
-"""
-Copyright 2014 Google Inc.
-
-Use of this source code is governed by a BSD-style license that can be
-found in the LICENSE file.
-
-Download actual GM results for a particular builder.
-"""
-
-# System-level imports
-import httplib
-import logging
-import optparse
-import os
-import posixpath
-import re
-import urllib2
-
-# Must fix up PYTHONPATH before importing from within Skia
-import rs_fixpypath # pylint: disable=W0611
-
-# Imports from within Skia
-from py.utils import gs_utils
-from py.utils import url_utils
-import buildbot_globals
-import gm_json
-
-
-GM_SUMMARIES_BUCKET = buildbot_globals.Get('gm_summaries_bucket')
-DEFAULT_ACTUALS_BASE_URL = (
- 'http://storage.googleapis.com/%s' % GM_SUMMARIES_BUCKET)
-DEFAULT_JSON_FILENAME = 'actual-results.json'
-
-
-class Download(object):
-
- def __init__(self, actuals_base_url=DEFAULT_ACTUALS_BASE_URL,
- json_filename=DEFAULT_JSON_FILENAME,
- gm_actuals_root_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL):
- """
- Args:
- actuals_base_url: URL pointing at the root directory
- containing all actual-results.json files, e.g.,
- http://domain.name/path/to/dir OR
- file:///absolute/path/to/localdir
- json_filename: The JSON filename to read from within each directory.
- gm_actuals_root_url: Base URL under which the actually-generated-by-bots
- GM images are stored.
- """
- self._actuals_base_url = actuals_base_url
- self._json_filename = json_filename
- self._gm_actuals_root_url = gm_actuals_root_url
- self._image_filename_re = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
-
- def fetch(self, builder_name, dest_dir):
- """ Downloads actual GM results for a particular builder.
-
- Args:
- builder_name: which builder to download results of
- dest_dir: path to directory where the image files will be written;
- if the directory does not exist yet, it will be created
-
- TODO(epoger): Display progress info. Right now, it can take a long time
- to download all of the results, and there is no indication of progress.
-
- TODO(epoger): Download multiple images in parallel to speed things up.
- """
- json_url = posixpath.join(self._actuals_base_url, builder_name,
- self._json_filename)
- json_contents = urllib2.urlopen(json_url).read()
- results_dict = gm_json.LoadFromString(json_contents)
-
- actual_results_dict = results_dict[gm_json.JSONKEY_ACTUALRESULTS]
- for result_type in sorted(actual_results_dict.keys()):
- results_of_this_type = actual_results_dict[result_type]
- if not results_of_this_type:
- continue
- for image_name in sorted(results_of_this_type.keys()):
- (test, config) = self._image_filename_re.match(image_name).groups()
- (hash_type, hash_digest) = results_of_this_type[image_name]
- source_url = gm_json.CreateGmActualUrl(
- test_name=test, hash_type=hash_type, hash_digest=hash_digest,
- gm_actuals_root_url=self._gm_actuals_root_url)
- dest_path = os.path.join(dest_dir, config, test + '.png')
- url_utils.copy_contents(source_url=source_url, dest_path=dest_path,
- create_subdirs_if_needed=True)
-
-
-def get_builders_list(summaries_bucket=GM_SUMMARIES_BUCKET):
- """ Returns the list of builders we have actual results for.
-
- Args:
- summaries_bucket: Google Cloud Storage bucket containing the summary
- JSON files
- """
- dirs, _ = gs_utils.GSUtils().list_bucket_contents(bucket=GM_SUMMARIES_BUCKET)
- return dirs
-
-
-class ActualLocation(object):
- def __init__(self, bucket, path, generation):
- self.bucket = bucket
- self.path = path
- self.generation = generation
-
-
-class TipOfTreeActuals(object):
- def __init__(self, summaries_bucket=GM_SUMMARIES_BUCKET,
- json_filename=DEFAULT_JSON_FILENAME):
- """
- Args:
- summaries_bucket: URL pointing at the root directory
- containing all actual-results.json files, e.g.,
- http://domain.name/path/to/dir OR
- file:///absolute/path/to/localdir
- json_filename: The JSON filename to read from within each directory.
- """
- self._json_filename = json_filename
- self._summaries_bucket = summaries_bucket
-
- def description(self):
- return 'gm_summaries_bucket %s' % (self._summaries_bucket,)
-
- def get_builders(self):
- """ Returns the list of builders we have actual results for.
- {builder:string -> ActualLocation}
- """
- dirs = get_builders_list(self._summaries_bucket)
- result = dict()
- for builder in dirs:
- result[builder] = ActualLocation(
- self._summaries_bucket,
- "%s/%s" % (builder, self._json_filename),
- None)
- return result
-
-
-class RietveldIssueActuals(object):
- def __init__(self, issue, json_filename=DEFAULT_JSON_FILENAME):
- """
- Args:
- issue: The rietveld issue from which to obtain actuals.
- json_filename: The JSON filename to read from within each directory.
- """
- self._issue = issue
- self._json_filename = json_filename
-
- def description(self):
- return 'rietveld issue %s' % (self._issue,)
-
- def get_builders(self):
- """ Returns the actuals for the given rietveld issue's tryjobs.
- {builder:string -> ActualLocation}
-
- e.g.
- {'Test-Android-Xoom-Tegra2-Arm7-Release': (
- 'chromium-skia-gm-summaries',
- 'Test-Android-Xoom-Tegra2-Arm7-Release-Trybot/actual-results.json',
- '1415041165535000')}
- """
- result = dict()
- json_filename_re = re.compile(
- 'Created: gs://([^/]+)/((?:[^/]+/)+%s)#(\d+)'
- % re.escape(self._json_filename))
- codereview_api_url = 'https://codereview.chromium.org/api'
- upload_gm_step_url = '/steps/Upload GM Results/logs/stdio'
-
- logging.info('Fetching issue %s ...' % (self._issue,))
- json_issue_url = '%s/%s' % (codereview_api_url, self._issue)
- json_issue_data = urllib2.urlopen(json_issue_url).read()
- issue_dict = gm_json.LoadFromString(json_issue_data)
-
- patchsets = issue_dict.get("patchsets", [])
- patchset = patchsets[-1]
- if not patchset:
- logging.warning('No patchsets for rietveld issue %s.' % (self._issue,))
- return result
-
- logging.info('Fetching issue %s patch %s...' % (self._issue, patchset))
- json_patchset_url = '%s/%s/%s' % (codereview_api_url, self._issue, patchset)
- json_patchset_data = urllib2.urlopen(json_patchset_url).read()
- patchset_dict = gm_json.LoadFromString(json_patchset_data)
-
- # try_job_results is ordered reverse chronologically
- try_job_results = patchset_dict.get('try_job_results', [])
- for try_job_result in try_job_results:
- try_builder = try_job_result.get('builder', '<bad builder>')
- if not try_builder.endswith('-Trybot'):
- logging.warning('Builder %s is not a trybot?' % (try_builder,))
- continue
- builder = try_builder[:-len('-Trybot')]
- if builder in result:
- continue
-
- logging.info('Fetching issue %s patch %s try %s...' %
- (self._issue, patchset, try_builder))
- build_url = try_job_result.get('url', '<bad url>')
- if build_url is None:
- logging.warning('Builder %s has not started.' % (try_builder,))
- continue
- gm_upload_output_url = build_url + urllib2.quote(upload_gm_step_url)
- logging.info('Fetching %s ...' % (gm_upload_output_url,))
-
- # Tryjobs might not produce the step, but don't let that fail everything.
- gm_upload_output = None
- try:
- gm_upload_output = urllib2.urlopen(gm_upload_output_url).read()
- except (urllib2.HTTPError, urllib2.URLError, httplib.HTTPException) as e:
- logging.warning(e)
- except Exception:
- logging.exception('Error opening %s .' % (gm_upload_output_url,))
- if not gm_upload_output:
- logging.warning('Could not fetch %s .' % (gm_upload_output_url,))
- continue
-
- json_filename_match = json_filename_re.search(gm_upload_output)
- if json_filename_match:
- logging.info('Found issue %s patch %s try %s result gs://%s/%s#%s .' %
- (self._issue, patchset, builder,
- json_filename_match.group(1),
- json_filename_match.group(2),
- json_filename_match.group(3)))
- result[builder] = ActualLocation(json_filename_match.group(1),
- json_filename_match.group(2),
- json_filename_match.group(3))
- else:
- logging.warning('Did not find %s for issue %s patch %s try %s.' %
- (self._json_filename, self._issue, patchset, try_builder))
-
- return result
-
-
-def main():
- parser = optparse.OptionParser()
- required_params = []
- parser.add_option('--actuals-base-url',
- action='store', type='string',
- default=DEFAULT_ACTUALS_BASE_URL,
- help=('Base URL from which to read files containing JSON '
- 'summaries of actual GM results; defaults to '
- '"%default".'))
- required_params.append('builder')
- # TODO(epoger): Before https://codereview.chromium.org/309653005 , when this
- # tool downloaded the JSON summaries from skia-autogen, it had the ability
- # to get results as of a specific revision number. We should add similar
- # functionality when retrieving the summaries from Google Storage.
- parser.add_option('--builder',
- action='store', type='string',
- help=('REQUIRED: Which builder to download results for. '
- 'To see a list of builders, run with the '
- '--list-builders option set.'))
- required_params.append('dest_dir')
- parser.add_option('--dest-dir',
- action='store', type='string',
- help=('REQUIRED: Directory where all images should be '
- 'written. If this directory does not exist yet, it '
- 'will be created.'))
- parser.add_option('--json-filename',
- action='store', type='string',
- default=DEFAULT_JSON_FILENAME,
- help=('JSON summary filename to read for each builder; '
- 'defaults to "%default".'))
- parser.add_option('--list-builders', action='store_true',
- help=('List all available builders.'))
- (params, remaining_args) = parser.parse_args()
-
- if params.list_builders:
- print '\n'.join(get_builders_list())
- return
-
- # Make sure all required options were set,
- # and that there were no items left over in the command line.
- for required_param in required_params:
- if not getattr(params, required_param):
- raise Exception('required option \'%s\' was not set' % required_param)
- if len(remaining_args) is not 0:
- raise Exception('extra items specified in the command line: %s' %
- remaining_args)
-
- downloader = Download(actuals_base_url=params.actuals_base_url)
- downloader.fetch(builder_name=params.builder,
- dest_dir=params.dest_dir)
-
-
-
-if __name__ == '__main__':
- main()
« no previous file with comments | « gm/rebaseline_server/compare_to_expectations_test.py ('k') | gm/rebaseline_server/download_actuals_test.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698