tools/rebaseline.py - Issue 320443002: remove GM result-grooming tools that use skia-autogen

Unified Diff: tools/rebaseline.py

Issue 320443002: remove GM result-grooming tools that use skia-autogen (Closed) Base URL: https://skia.googlesource.com/skia.git@master

Patch Set: Created 6 years, 6 months ago

Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.

Jump to:

View side-by-side diff with in-line comments

Download patch

Index: tools/rebaseline.py

diff --git a/tools/rebaseline.py b/tools/rebaseline.py

deleted file mode 100755

index 9870b9003479287497d2b4a34cd7522cfe679ac6..0000000000000000000000000000000000000000

--- a/tools/rebaseline.py

+++ /dev/null

@@ -1,506 +0,0 @@

-#!/usr/bin/python

-'''

-Use of this source code is governed by a BSD-style license that can be

-found in the LICENSE file.

-'''

-Rebaselines the given GM tests, on all bots and all configurations.

-'''

-# System-level imports

-import argparse

-import json

-import os

-import re

-import subprocess

-import sys

-import urllib2

-# Imports from within Skia

-# We need to add the 'gm' directory, so that we can import gm_json.py within

-# that directory. That script allows us to parse the actual-results.json file

-# written out by the GM tool.

-# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end*

-# so any dirs that are already in the PYTHONPATH will be preferred.

-# This assumes that the 'gm' directory has been checked out as a sibling of

-# the 'tools' directory containing this script, which will be the case if

-# 'trunk' was checked out as a single unit.

-GM_DIRECTORY = os.path.realpath(

- os.path.join(os.path.dirname(os.path.dirname(__file__)), 'gm'))

-if GM_DIRECTORY not in sys.path:

- sys.path.append(GM_DIRECTORY)

-import gm_json

-# TODO(epoger): In the long run, we want to build this list automatically,

-# but for now we hard-code it until we can properly address

-# https://code.google.com/p/skia/issues/detail?id=1544

-# ('live query of builder list makes rebaseline.py slow to start up')

-TEST_BUILDERS = [

- 'Test-Android-GalaxyNexus-SGX540-Arm7-Debug',

- 'Test-Android-GalaxyNexus-SGX540-Arm7-Release',

- 'Test-Android-IntelRhb-SGX544-x86-Debug',

- 'Test-Android-IntelRhb-SGX544-x86-Release',

- 'Test-Android-Nexus10-MaliT604-Arm7-Debug',

- 'Test-Android-Nexus10-MaliT604-Arm7-Release',

- 'Test-Android-Nexus4-Adreno320-Arm7-Debug',

- 'Test-Android-Nexus4-Adreno320-Arm7-Release',

- 'Test-Android-Nexus7-Tegra3-Arm7-Debug',

- 'Test-Android-Nexus7-Tegra3-Arm7-Release',

- 'Test-Android-NexusS-SGX540-Arm7-Debug',

- 'Test-Android-NexusS-SGX540-Arm7-Release',

- 'Test-Android-Xoom-Tegra2-Arm7-Debug',

- 'Test-Android-Xoom-Tegra2-Arm7-Release',

- 'Test-ChromeOS-Alex-GMA3150-x86-Debug',

- 'Test-ChromeOS-Alex-GMA3150-x86-Release',

- 'Test-ChromeOS-Daisy-MaliT604-Arm7-Debug',

- 'Test-ChromeOS-Daisy-MaliT604-Arm7-Release',

- 'Test-ChromeOS-Link-HD4000-x86_64-Debug',

- 'Test-ChromeOS-Link-HD4000-x86_64-Release',

- 'Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug',

- 'Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release',

- 'Test-Mac10.6-MacMini4.1-GeForce320M-x86_64-Debug',

- 'Test-Mac10.6-MacMini4.1-GeForce320M-x86_64-Release',

- 'Test-Mac10.7-MacMini4.1-GeForce320M-x86-Debug',

- 'Test-Mac10.7-MacMini4.1-GeForce320M-x86-Release',

- 'Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug',

- 'Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Release',

- 'Test-Mac10.8-MacMini4.1-GeForce320M-x86-Debug',

- 'Test-Mac10.8-MacMini4.1-GeForce320M-x86-Release',

- 'Test-Mac10.8-MacMini4.1-GeForce320M-x86_64-Debug',

- 'Test-Mac10.8-MacMini4.1-GeForce320M-x86_64-Release',

- 'Test-Ubuntu12-ShuttleA-HD2000-x86_64-Release-Valgrind',

- 'Test-Ubuntu12-ShuttleA-GTX660-x86-Debug',

- 'Test-Ubuntu12-ShuttleA-GTX660-x86-Release',

- 'Test-Ubuntu12-ShuttleA-GTX660-x86_64-Debug',

- 'Test-Ubuntu12-ShuttleA-GTX660-x86_64-Release',

- 'Test-Ubuntu13.10-ShuttleA-NoGPU-x86_64-Debug',

- 'Test-Win7-ShuttleA-HD2000-x86-Debug',

- 'Test-Win7-ShuttleA-HD2000-x86-Debug-ANGLE',

- 'Test-Win7-ShuttleA-HD2000-x86-Debug-DirectWrite',

- 'Test-Win7-ShuttleA-HD2000-x86-Release',

- 'Test-Win7-ShuttleA-HD2000-x86-Release-ANGLE',

- 'Test-Win7-ShuttleA-HD2000-x86-Release-DirectWrite',

- 'Test-Win7-ShuttleA-HD2000-x86_64-Debug',

- 'Test-Win7-ShuttleA-HD2000-x86_64-Release',

- 'Test-Win8-ShuttleA-GTX660-x86-Debug',

- 'Test-Win8-ShuttleA-GTX660-x86-Release',

- 'Test-Win8-ShuttleA-GTX660-x86_64-Debug',

- 'Test-Win8-ShuttleA-GTX660-x86_64-Release',

- 'Test-Win8-ShuttleA-HD7770-x86-Debug',

- 'Test-Win8-ShuttleA-HD7770-x86-Release',

- 'Test-Win8-ShuttleA-HD7770-x86_64-Debug',

- 'Test-Win8-ShuttleA-HD7770-x86_64-Release',

-# TODO: Get this from builder_name_schema in buildbot.

-TRYBOT_SUFFIX = '-Trybot'

-class _InternalException(Exception):

- pass

-class ExceptionHandler(object):

- """ Object that handles exceptions, either raising them immediately or

- collecting them to display later on."""

- # params:

- def __init__(self, keep_going_on_failure=False):

- """

- params:

- keep_going_on_failure: if False, report failures and quit right away;

- if True, collect failures until

- ReportAllFailures() is called

- """

- self._keep_going_on_failure = keep_going_on_failure

- self._failures_encountered = []

- def RaiseExceptionOrContinue(self):

- """ We have encountered an exception; either collect the info and keep

- going, or exit the program right away."""

- # Get traceback information about the most recently raised exception.

- exc_info = sys.exc_info()

- if self._keep_going_on_failure:

- print >> sys.stderr, ('WARNING: swallowing exception %s' %

- repr(exc_info[1]))

- self._failures_encountered.append(exc_info)

- else:

- print >> sys.stderr, (

- '\nHalting at first exception.\n' +

- 'Please file a bug to epoger@google.com at ' +

- 'https://code.google.com/p/skia/issues/entry, containing the ' +

- 'command you ran and the following stack trace.\n\n' +

- 'Afterwards, you can re-run with the --keep-going-on-failure ' +

- 'option set.\n')

- raise exc_info[1], None, exc_info[2]

- def ReportAllFailures(self):

- if self._failures_encountered:

- print >> sys.stderr, ('Encountered %d failures (see above).' %

- len(self._failures_encountered))

- sys.exit(1)

-# Object that rebaselines a JSON expectations file (not individual image files).

-class JsonRebaseliner(object):

- # params:

- # expectations_root: root directory of all expectations JSON files

- # expectations_input_filename: filename (under expectations_root) of JSON

- # expectations file to read; typically

- # "expected-results.json"

- # expectations_output_filename: filename (under expectations_root) to

- # which updated expectations should be

- # written; typically the same as

- # expectations_input_filename, to overwrite

- # the old content

- # actuals_base_url: base URL from which to read actual-result JSON files

- # actuals_filename: filename (under actuals_base_url) from which to read a

- # summary of results; typically "actual-results.json"

- # exception_handler: reference to rebaseline.ExceptionHandler object

- # tests: list of tests to rebaseline, or None if we should rebaseline

- # whatever files the JSON results summary file tells us to

- # configs: which configs to run for each test, or None if we should

- # rebaseline whatever configs the JSON results summary file tells

- # us to

- # add_new: if True, add expectations for tests which don't have any yet

- # add_ignored: if True, add expectations for tests for which failures are

- # currently ignored

- # bugs: optional list of bug numbers which pertain to these expectations

- # notes: free-form text notes to add to all updated expectations

- # mark_unreviewed: if True, mark these expectations as NOT having been

- # reviewed by a human; otherwise, leave that field blank.

- # Currently, there is no way to make this script mark

- # expectations as reviewed-by-human=True.

- # TODO(epoger): Add that capability to a review tool.

- # mark_ignore_failure: if True, mark failures of a given test as being

- # ignored.

- # from_trybot: if True, read actual-result JSON files generated from a

- # trybot run rather than a waterfall run.

- def __init__(self, expectations_root, expectations_input_filename,

- expectations_output_filename, actuals_base_url,

- actuals_filename, exception_handler,

- tests=None, configs=None, add_new=False, add_ignored=False,

- bugs=None, notes=None, mark_unreviewed=None,

- mark_ignore_failure=False, from_trybot=False):

- self._expectations_root = expectations_root

- self._expectations_input_filename = expectations_input_filename

- self._expectations_output_filename = expectations_output_filename

- self._tests = tests

- self._configs = configs

- self._actuals_base_url = actuals_base_url

- self._actuals_filename = actuals_filename

- self._exception_handler = exception_handler

- self._add_new = add_new

- self._add_ignored = add_ignored

- self._bugs = bugs

- self._notes = notes

- self._mark_unreviewed = mark_unreviewed

- self._mark_ignore_failure = mark_ignore_failure;

- if self._tests or self._configs:

- self._image_filename_re = re.compile(gm_json.IMAGE_FILENAME_PATTERN)

- else:

- self._image_filename_re = None

- self._using_svn = os.path.isdir(os.path.join(expectations_root, '.svn'))

- self._from_trybot = from_trybot

- # Executes subprocess.call(cmd).

- # Raises an Exception if the command fails.

- def _Call(self, cmd):

- if subprocess.call(cmd) != 0:

- raise _InternalException('error running command: ' + ' '.join(cmd))

- # Returns the full contents of filepath, as a single string.

- # If filepath looks like a URL, try to read it that way instead of as

- # a path on local storage.

- #

- # Raises _InternalException if there is a problem.

- def _GetFileContents(self, filepath):

- if filepath.startswith('http:') or filepath.startswith('https:'):

- try:

- return urllib2.urlopen(filepath).read()

- except urllib2.HTTPError as e:

- raise _InternalException('unable to read URL %s: %s' % (

- filepath, e))

- else:

- return open(filepath, 'r').read()

- # Returns a dictionary of actual results from actual-results.json file.

- #

- # The dictionary returned has this format:

- # {

- # u'imageblur_565.png': [u'bitmap-64bitMD5', 3359963596899141322],

- # u'imageblur_8888.png': [u'bitmap-64bitMD5', 4217923806027861152],

- # u'shadertext3_8888.png': [u'bitmap-64bitMD5', 3713708307125704716]

- # }

- #

- # If the JSON actual result summary file cannot be loaded, logs a warning

- # message and returns None.

- # If the JSON actual result summary file can be loaded, but we have

- # trouble parsing it, raises an Exception.

- #

- # params:

- # json_url: URL pointing to a JSON actual result summary file

- # sections: a list of section names to include in the results, e.g.

- # [gm_json.JSONKEY_ACTUALRESULTS_FAILED,

- # gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON] ;

- # if None, then include ALL sections.

- def _GetActualResults(self, json_url, sections=None):

- try:

- json_contents = self._GetFileContents(json_url)

- except _InternalException:

- print >> sys.stderr, (

- 'could not read json_url %s ; skipping this platform.' %

- json_url)

- return None

- json_dict = gm_json.LoadFromString(json_contents)

- results_to_return = {}

- actual_results = json_dict[gm_json.JSONKEY_ACTUALRESULTS]

- if not sections:

- sections = actual_results.keys()

- for section in sections:

- section_results = actual_results[section]

- if section_results:

- results_to_return.update(section_results)

- return results_to_return

- # Rebaseline all tests/types we specified in the constructor,

- # within this builder's subdirectory in expectations/gm .

- #

- # params:

- # builder : e.g. 'Test-Win7-ShuttleA-HD2000-x86-Release'

- def RebaselineSubdir(self, builder):

- # Read in the actual result summary, and extract all the tests whose

- # results we need to update.

- results_builder = str(builder)

- if self._from_trybot:

- results_builder = results_builder + TRYBOT_SUFFIX

- actuals_url = '/'.join([self._actuals_base_url, results_builder,

- self._actuals_filename])

- # Only update results for tests that are currently failing.

- # We don't want to rewrite results for tests that are already succeeding,

- # because we don't want to add annotation fields (such as

- # JSONKEY_EXPECTEDRESULTS_BUGS) except for tests whose expectations we

- # are actually modifying.

- sections = [gm_json.JSONKEY_ACTUALRESULTS_FAILED]

- if self._add_new:

- sections.append(gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON)

- if self._add_ignored:

- sections.append(gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED)

- results_to_update = self._GetActualResults(json_url=actuals_url,

- sections=sections)

- # Read in current expectations.

- expectations_input_filepath = os.path.join(

- self._expectations_root, builder, self._expectations_input_filename)

- expectations_dict = gm_json.LoadFromFile(expectations_input_filepath)

- expected_results = expectations_dict.get(gm_json.JSONKEY_EXPECTEDRESULTS)

- if not expected_results:

- expected_results = {}

- expectations_dict[gm_json.JSONKEY_EXPECTEDRESULTS] = expected_results

- # Update the expectations in memory, skipping any tests/configs that

- # the caller asked to exclude.

- skipped_images = []

- if results_to_update:

- for (image_name, image_results) in results_to_update.iteritems():

- if self._image_filename_re:

- (test, config) = self._image_filename_re.match(image_name).groups()

- if self._tests:

- if test not in self._tests:

- skipped_images.append(image_name)

- continue

- if self._configs:

- if config not in self._configs:

- skipped_images.append(image_name)

- continue

- if not expected_results.get(image_name):

- expected_results[image_name] = {}