| Index: experimental/benchtools/greenify.py
|
| diff --git a/experimental/benchtools/greenify.py b/experimental/benchtools/greenify.py
|
| deleted file mode 100755
|
| index bd257d9889832d33f75cb3dbfb266d845ed5625d..0000000000000000000000000000000000000000
|
| --- a/experimental/benchtools/greenify.py
|
| +++ /dev/null
|
| @@ -1,205 +0,0 @@
|
| -#!/usr/bin/env python
|
| -
|
| -
|
| -# Copyright (c) 2014 The Chromium Authors. All rights reserved.
|
| -# Use of this source code is governed by a BSD-style license that can be
|
| -# found in the LICENSE file.
|
| -
|
| -
|
| -"""greenify.py: standalone script to correct flaky bench expectations.
|
| -
|
| - Requires Rietveld credentials on the running machine.
|
| -
|
| - Usage:
|
| - Copy script to a separate dir outside Skia repo. The script will create a
|
| - skia dir on the first run to host the repo, and will create/delete
|
| - temp dirs as needed.
|
| - ./greenify.py --url <the stdio url from failed CheckForRegressions step>
|
| -"""
|
| -
|
| -import argparse
|
| -import filecmp
|
| -import os
|
| -import re
|
| -import shutil
|
| -import subprocess
|
| -import time
|
| -import urllib2
|
| -
|
| -
|
| -# Regular expression for matching exception data.
|
| -EXCEPTION_RE = ('Bench (\S+) out of range \[(\d+.\d+), (\d+.\d+)\] \((\d+.\d+) '
|
| - 'vs (\d+.\d+), ')
|
| -EXCEPTION_RE_COMPILED = re.compile(EXCEPTION_RE)
|
| -
|
| -
|
| -def clean_dir(d):
|
| - if os.path.exists(d):
|
| - shutil.rmtree(d)
|
| - os.makedirs(d)
|
| -
|
| -def checkout_or_update_skia(repo_dir):
|
| - status = True
|
| - old_cwd = os.getcwd()
|
| - os.chdir(repo_dir)
|
| - print 'CHECK SKIA REPO...'
|
| - if subprocess.call(['git', 'pull'],
|
| - stderr=subprocess.PIPE):
|
| - print 'Checking out Skia from git, please be patient...'
|
| - os.chdir(old_cwd)
|
| - clean_dir(repo_dir)
|
| - os.chdir(repo_dir)
|
| - if subprocess.call(['git', 'clone', '-q', '--depth=50', '--single-branch',
|
| - 'https://skia.googlesource.com/skia.git', '.']):
|
| - status = False
|
| - subprocess.call(['git', 'checkout', 'master'])
|
| - subprocess.call(['git', 'pull'])
|
| - os.chdir(old_cwd)
|
| - return status
|
| -
|
| -def git_commit_expectations(repo_dir, exp_dir, bot, build, commit):
|
| - commit_msg = """Greenify bench bot %s at build %s
|
| -
|
| -TBR=bsalomon@google.com
|
| -
|
| -Bypassing trybots:
|
| -NOTRY=true""" % (bot, build)
|
| - old_cwd = os.getcwd()
|
| - os.chdir(repo_dir)
|
| - upload = ['git', 'cl', 'upload', '-f', '--bypass-hooks',
|
| - '--bypass-watchlists', '-m', commit_msg]
|
| - if commit:
|
| - upload.append('--use-commit-queue')
|
| - branch = exp_dir[exp_dir.rfind('/') + 1:]
|
| - filename = 'bench_expectations_%s.txt' % bot
|
| - cmds = ([['git', 'checkout', 'master'],
|
| - ['git', 'pull'],
|
| - ['git', 'checkout', '-b', branch, '-t', 'origin/master'],
|
| - ['cp', '%s/%s' % (exp_dir, filename), 'expectations/bench'],
|
| - ['git', 'add', 'expectations/bench/' + filename],
|
| - ['git', 'commit', '-m', commit_msg],
|
| - upload,
|
| - ['git', 'checkout', 'master'],
|
| - ['git', 'branch', '-D', branch],
|
| - ])
|
| - status = True
|
| - for cmd in cmds:
|
| - print 'Running ' + ' '.join(cmd)
|
| - if subprocess.call(cmd):
|
| - print 'FAILED. Please check if skia git repo is present.'
|
| - subprocess.call(['git', 'checkout', 'master'])
|
| - status = False
|
| - break
|
| - os.chdir(old_cwd)
|
| - return status
|
| -
|
| -def delete_dirs(li):
|
| - for d in li:
|
| - print 'Deleting directory %s' % d
|
| - shutil.rmtree(d)
|
| -
|
| -def widen_bench_ranges(url, bot, repo_dir, exp_dir):
|
| - fname = 'bench_expectations_%s.txt' % bot
|
| - src = os.path.join(repo_dir, 'expectations', 'bench', fname)
|
| - if not os.path.isfile(src):
|
| - print 'This bot has no expectations! %s' % bot
|
| - return False
|
| - row_dic = {}
|
| - for l in urllib2.urlopen(url).read().split('\n'):
|
| - data = EXCEPTION_RE_COMPILED.search(l)
|
| - if data:
|
| - row = data.group(1)
|
| - lb = float(data.group(2))
|
| - ub = float(data.group(3))
|
| - actual = float(data.group(4))
|
| - exp = float(data.group(5))
|
| - avg = (actual + exp) / 2
|
| - shift = avg - exp
|
| - lb = lb + shift
|
| - ub = ub + shift
|
| - # In case outlier really fluctuates a lot
|
| - if actual < lb:
|
| - lb = actual - abs(shift) * 0.1 + 0.5
|
| - elif actual > ub:
|
| - ub = actual + abs(shift) * 0.1 + 0.5
|
| - row_dic[row] = '%.2f,%.2f,%.2f' % (avg, lb, ub)
|
| - if not row_dic:
|
| - print 'NO out-of-range benches found at %s' % url
|
| - return False
|
| -
|
| - changed = 0
|
| - li = []
|
| - for l in open(src).readlines():
|
| - parts = l.strip().split(',')
|
| - if parts[0].startswith('#') or len(parts) != 5:
|
| - li.append(l.strip())
|
| - continue
|
| - if ','.join(parts[:2]) in row_dic:
|
| - li.append(','.join(parts[:2]) + ',' + row_dic[','.join(parts[:2])])
|
| - changed += 1
|
| - else:
|
| - li.append(l.strip())
|
| - if not changed:
|
| - print 'Not in source file:\n' + '\n'.join(row_dic.keys())
|
| - return False
|
| -
|
| - dst = os.path.join(exp_dir, fname)
|
| - with open(dst, 'w+') as f:
|
| - f.write('\n'.join(li))
|
| - return True
|
| -
|
| -
|
| -def main():
|
| - d = os.path.dirname(os.path.abspath(__file__))
|
| - os.chdir(d)
|
| - if not subprocess.call(['git', 'rev-parse'], stderr=subprocess.PIPE):
|
| - print 'Please copy script to a separate dir outside git repos to use.'
|
| - return
|
| - ts_str = '%s' % time.time()
|
| -
|
| - parser = argparse.ArgumentParser()
|
| - parser.add_argument('--url',
|
| - help='Broken bench build CheckForRegressions page url.')
|
| - parser.add_argument('--commit', action='store_true',
|
| - help='Whether to commit changes automatically.')
|
| - args = parser.parse_args()
|
| - repo_dir = os.path.join(d, 'skia')
|
| - if not os.path.exists(repo_dir):
|
| - os.makedirs(repo_dir)
|
| - if not checkout_or_update_skia(repo_dir):
|
| - print 'ERROR setting up Skia repo at %s' % repo_dir
|
| - return 1
|
| -
|
| - file_in_repo = os.path.join(d, 'skia/experimental/benchtools/greenify.py')
|
| - if not filecmp.cmp(__file__, file_in_repo):
|
| - shutil.copy(file_in_repo, __file__)
|
| - print 'Updated this script from repo; please run again.'
|
| - return
|
| -
|
| - if not args.url:
|
| - raise Exception('Please provide a url with broken CheckForRegressions.')
|
| - path = args.url.split('/')
|
| - if len(path) != 11 or not path[6].isdigit():
|
| - raise Exception('Unexpected url format: %s' % args.url)
|
| - bot = path[4]
|
| - build = path[6]
|
| - commit = False
|
| - if args.commit:
|
| - commit = True
|
| -
|
| - exp_dir = os.path.join(d, 'exp' + ts_str)
|
| - clean_dir(exp_dir)
|
| - if not widen_bench_ranges(args.url, bot, repo_dir, exp_dir):
|
| - print 'NO bench exceptions found! %s' % args.url
|
| - elif not git_commit_expectations(
|
| - repo_dir, exp_dir, bot, build, commit):
|
| - print 'ERROR uploading expectations using git.'
|
| - elif not commit:
|
| - print 'CL created. Please take a look at the link above.'
|
| - else:
|
| - print 'New bench baselines should be in CQ now.'
|
| - delete_dirs([exp_dir])
|
| -
|
| -
|
| -if __name__ == "__main__":
|
| - main()
|
|
|