Try 1 at rebaseline

bench/gen_bench_expectations.py

 #!/usr/bin/env python
 # Copyright (c) 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """ Generate bench_expectations file from a given set of bench data files. """
 
 import argparse
 import bench_util
 import os
@@ skipping 28 matching lines @@
 
 def compute_ranges(benches):
   """Given a list of bench numbers, calculate the alert range.
 
   Args:
     benches: a list of float bench values.
 
   Returns:
     a list of float [lower_bound, upper_bound].
   """
-  minimum = min(benches)
-  maximum = max(benches)
-  diff = maximum - minimum
   avg = sum(benches) / len(benches)
+  squared_avg = avg ** 2
+  avg_squared = sum([bench**2 for bench in benches])/len(benches)
+  std_dev = (avg_squared - squared_avg) ** 0.5
 
-  return [minimum - diff * RANGE_RATIO_LOWER - avg * ERR_RATIO - ERR_LB,
-          maximum + diff * RANGE_RATIO_UPPER + avg * ERR_RATIO + ERR_UB]
+  # If the results are normally distributed, 2 standard deviations
+  # captures something like ~95% of the possible range of results I think
+  return [avg - 2*std_dev, avg + 2*std_dev]

[benchen, 2014/06/10 19:59:23]

Sometimes for some combinations the results from this one point all have the
same value, which will make the  upper and lower bounds the same. I suggest at
least adding some padding like -0.5, +0.5 to avoid this situation. Could be done
in subsequent CLs though.

 
 
 def create_expectations_dict(revision_data_points, builder):
   """Convert list of bench data points into a dictionary of expectations data.
 
   Args:
     revision_data_points: a list of BenchDataPoint objects.
     builder: string of the corresponding buildbot builder name.
 
   Returns:
@@ skipping 64 matching lines @@
               'expected': expected,
               'lower_bound': lower_bound,
               'upper_bound': upper_bound})
 
     with open(args.output_file, 'w') as file_handle:
       file_handle.write('\n'.join(out_lines))
 
 
 if __name__ == "__main__":
     main()