Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(278)

Issues Search
    My Issues | Starred     Open | Closed | All

Unified Diff: tracing/tracing/metrics/compare_samples.html

Issue 2089833002: Entry point for bisect sample comparison. (Closed) Base URL: https://github.com/catapult-project/catapult.git@mann
Patch Set: Adding tests that pass Created 4 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
« tracing/bin/compare_samples ('K') | « tracing/bin/compare_samples ('k') | tracing/tracing/metrics/compare_samples.py » ('j') | tracing/tracing/metrics/compare_samples.py » ('J')
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: tracing/tracing/metrics/compare_samples.html
diff --git a/tracing/tracing/metrics/compare_samples.html b/tracing/tracing/metrics/compare_samples.html
new file mode 100644
index 0000000000000000000000000000000000000000..5fe336ffcece5fc852ee0193d6c67709fa09a271
--- /dev/null
+++ b/tracing/tracing/metrics/compare_samples.html
@@ -0,0 +1,254 @@
+<!DOCTYPE html>
+<!--
+Copyright 2016 The Chromium Authors. All rights reserved.
+Use of this source code is governed by a BSD-style license that can be
+found in the LICENSE file.
+-->
+
+<link rel="import" href="/tracing/base/statistics.html">
+<link rel="import" href="/tracing/base/xhr.html">
+<link rel="import" href="/tracing/value/value_set.html">
+
+<script>
+'use strict';
+
+tr.exportTo('tr.metrics', function() {
+ var BisectComparison = {
+ ENOUGH_SAMPLES: 18,
+ SIGNIFICANCE_LEVEL: 0.05,
+ escapeChars: function(s) {
eakuefner 2016/07/21 19:35:34 write this as an arrow, that is: s => s.replace..
RobertoCN 2016/09/02 21:19:30 Done.
+ return s.replace(/[\:|=\/#&,]/g, '_');
+ },
+ valuesFromCharts: function(listOfCharts, metricName) {
+ function geoMeanFromHistogram(h) {
+ if (!h.hasOwnProperty('buckets')) {
+ return 0.0;
+ }
+ var count = 0;
+ var sumOfLogs = 0;
+ for (var bucket in h.buckets) {
eakuefner 2016/07/21 19:35:34 you probably want for..of not for..in. for..in is
RobertoCN 2016/09/02 21:19:30 Done.
+ if (bucket.hasOwnProperty('high')) {
+ bucket.mean = (bucket.low + bucket.high) / 2.0;
+ } else {
+ bucket.mean = bucket.low;
+ }
+
+ if (bucket.mean > 0) {
+ sumOfLogs += Math.log(bucket.mean) * bucket.count;
+ count += bucket.count;
+ }
+ }
+ if (count == 0) {
+ return 0.0;
+ }
+ return Math.exp(sumOfLogs / count);
+ }
+ var all_values = [];
+ var charts;
+ for (var i = 0; i < listOfCharts.length; i++) {
eakuefner 2016/07/21 19:35:34 again a place where you could just use a for..of l
RobertoCN 2016/09/02 21:19:30 Done.
+ charts = listOfCharts[i];
+ var chartName, interactionName, traceName;
+ var parts = metricName.split('/');
+ if (parts.length == 3) {
+ chartName = parts[0];
+ interactionName = parts[1];
+ traceName = parts[2];
+ } else if (parts.length == 2) {
+ chartName = parts[0];
+ if (chartName != parts[1]) {
+ traceName = parts[1];
+ }
+ } else {
+ throw new Error('Could not parse metric name.');
+ }
+
+ if (interactionName) {
+ chartName = interactionName + '@@' + chartName;
+ }
+
+ if (!traceName) {
+ traceName = 'summary';
+ }
+
+
+ var chart, trace;
+
+ for (chart in charts.charts) {
+ if (charts.charts.hasOwnProperty(chart) &&
+ this.escapeChars(chart) == chartName) {
+ chartName = chart; // Unescaping
+ break;
+ }
+ }
+ for (trace in charts.charts[chartName]) {
+ if (charts.charts[chartName].hasOwnProperty(trace) &&
+ this.escapeChars(trace) == traceName) {
+ traceName = trace; // Unescaping
+ break;
+ }
+ }
+ if (charts.charts[chartName][traceName].type ==
+ 'list_of_scalar_values') {
+ all_values.push.apply(
+ all_values, charts.charts[chartName][traceName].values);
+ }
+
+ if (charts.charts[chartName][traceName].type == 'histogram') {
+ all_values.push(
+ geoMeanFromHistogam(charts.charts[chartName][traceName]));
+ }
+ }
+ return all_values;
+ },
+
+ compareValuesets: function(valueSetAPathList, valueSetBPathList, metric) {
+ var aPaths = valueSetAPathList.split(',');
+ var bPaths = valueSetBPathList.split(',');
+ var valueSetA = new tr.v.ValueSet();
+ var valueSetB = new tr.v.ValueSet();
+ var current;
+ for (var i = 0; i < aPaths.length; i++) {
+ try {
+ current = tr.b.getSync('file://' + aPaths[i]);
+ } catch (ex) {
+ var err = new Error('Could not open' + aPaths[i]);
+ err.name = 'File loading error';
+ throw err;
+ }
+ valueSetA.addValuesFromDicts(JSON.parse(current));
+ }
+ for (var i = 0; i < bPaths.length; i++) {
+ try {
+ current = tr.b.getSync('file://' + bPaths[i]);
+ } catch (ex) {
+ var err = new Error('Could not open' + bPaths[i]);
+ err.name = 'File loading error';
+ throw err;
+ }
+ valueSetB.addValuesFromDicts(JSON.parse(current));
+ }
+
+ function rawValuesByMetricName(valueSet, metricName, thisParam) {
+ var interactionRecord, valueName, story;
+ var metricNameParts = metricName.split('/');
+ if (metricNameParts[0] === metricNameParts[1]) {
+ story = 'summary';
+ } else {
+ story = metricNameParts[1];
+ }
+ var chartNameParts = metricNameParts[0].split('-');
+ valueName = chartNameParts[1];
+ if (chartNameParts.length == 2) {
+ interactionRecord = chartNameParts[0];
+ }
+ var values = valueSet.getValuesWithName(valueName);
+ if (!values || values.length == 0) {
+ // If there was a dash in the chart name, but it wasn't an
+ // interaction record.
+ valueName = metricNameParts[0];
+ values = valueSet.getValuesWithName(valueName);
+ interactionRecord = undefined;
+ if (!values || values.length == 0) {
+ throw new Error('No values with name ' + valueName);
+ }
+ }
+ var filtered = values.filter(function(value) {
+ if (value.name != valueName) {
+ return false;
+ }
+ var ii = tr.v.d.IterationInfo.getFromValue(value);
+ if (interactionRecord) {
+ var values = [];
+ var keys = Object.keys(ii.storyGroupingKeys);
+ keys.sort();
+ for (var i = 0; i < keys.length; i++) {
+ values.push(ii.storyGroupingKeys[keys[i]]);
+ }
+ if (interactionRecord == values.join('_')) {
+ return thisParam.escapeChars(ii.storyDisplayName) ==
+ thisParam.escapeChars(story);
+ }
+ return false;
+ }
+ return thisParam.escapeChars(ii.storyDisplayName) ==
+ thisParam.escapeChars(story);
+ });
+ var rawValues = [];
+ for (var i = 0; i < filtered.length; i++) {
+ if (filtered[i].numeric instanceof tr.v.Numeric) {
+ rawValues = rawValues.concat(filtered[i].numeric.sampleValues);
+ } else if (filtered[i].numeric instanceof tr.v.ScalarNumeric) {
+ rawValues.push(filtered[i].numeric.value);
+ }
+ }
+ return rawValues;
+ }
+ var sampleA = rawValuesByMetricName(valueSetA, metric, this);
+ var sampleB = rawValuesByMetricName(valueSetB, metric, this);
+ return this.compareSamples(sampleA, sampleB);
+ },
+
+ compareCharts: function(chartPathListA, chartPathListB, metric) {
+ var aPaths = chartPathListA.split(',');
+ var bPaths = chartPathListB.split(',');
+ var chartsA = new Array(aPaths.length);
+ var chartsB = new Array(bPaths.length);
+ var current;
+ for (var i = 0; i < aPaths.length; i++) {
+ try {
+ current = tr.b.getSync('file://' + aPaths[i]);
+ } catch (ex) {
+ var err = new Error('Could not open' + aPaths[i]);
+ err.name = 'File loading error';
+ throw err;
+ }
+ chartsA[i] = JSON.parse(current);
+ }
+ for (var i = 0; i < bPaths.length; i++) {
+ try {
+ current = tr.b.getSync('file://' + bPaths[i]);
+ } catch (ex) {
+ var err = new Error('Could not open' + bPaths[i]);
+ err.name = 'File loading error';
+ throw err;
+ }
+ chartsB[i] = JSON.parse(current);
+ }
+ var sampleA = this.valuesFromCharts(chartsA, metric);
+ var sampleB = this.valuesFromCharts(chartsB, metric);
+ return this.compareSamples(sampleA, sampleB);
+ },
+
+ compareSamples: function(sampleA, sampleB) {
+ var pValue = tr.b.Statistics.mwu.test(sampleA, sampleB);
+ // Diagnostics
+ var result = {
+ sample_a: {
+ std_dev: tr.b.Statistics.stddev(sampleA),
+ mean: tr.b.Statistics.mean(sampleA),
+ debug_values: sampleA
+ },
+ sample_b: {
+ std_dev: tr.b.Statistics.stddev(sampleB),
+ mean: tr.b.Statistics.mean(sampleB),
+ debug_values: sampleB
+ },
+ pValue: pValue.p,
+ UStatistic: pValue.U,
+ result: 'needMoreData',
+ };
+ if (pValue.p < this.SIGNIFICANCE_LEVEL) {
+ result.result = true; // Reject the null
+ } else if (sampleA.length > this.ENOUGH_SAMPLES &&
+ sampleB.length > this.ENOUGH_SAMPLES) {
+ result.result = false; // Fail to reject the null.
+ }
+ return result;
+ }
+ };
+
+ return {
+ BisectComparison: BisectComparison
+ };
+});
+</script>
« tracing/bin/compare_samples ('K') | « tracing/bin/compare_samples ('k') | tracing/tracing/metrics/compare_samples.py » ('j') | tracing/tracing/metrics/compare_samples.py » ('J')

Powered by Google App Engine
This is Rietveld 408576698