Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(299)

Issues Search
    My Issues | Starred     Open | Closed | All

Unified Diff: tracing/tracing/metrics/compare_samples_cmdline.html

Issue 2089833002: Entry point for bisect sample comparison. (Closed) Base URL: https://github.com/catapult-project/catapult.git@mann
Patch Set: Rebasing fixes. Created 4 years, 3 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
« tracing/bin/compare_samples ('K') | « tracing/tracing/metrics/compare_samples.py ('k') | tracing/tracing/metrics/compare_samples_unittest.py » ('j') | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: tracing/tracing/metrics/compare_samples_cmdline.html
diff --git a/tracing/tracing/metrics/compare_samples_cmdline.html b/tracing/tracing/metrics/compare_samples_cmdline.html
new file mode 100644
index 0000000000000000000000000000000000000000..e44d7b38b13dfe40fb5b0c05e700a72020f71422
--- /dev/null
+++ b/tracing/tracing/metrics/compare_samples_cmdline.html
@@ -0,0 +1,282 @@
+<!DOCTYPE html>
+<!--
+Copyright 2016 The Chromium Authors. All rights reserved.
+Use of this source code is governed by a BSD-style license that can be
+found in the LICENSE file.
+-->
+
+<link rel="import" href="/tracing/base/iteration_helpers.html">
+<link rel="import" href="/tracing/base/statistics.html">
+<link rel="import" href="/tracing/base/xhr.html">
+<link rel="import" href="/tracing/value/value_set.html">
+
+<script>
+'use strict';
+
+var escapeChars = s => s.replace(/[\:|=\/#&,]/g, '_');
+
+function findUnescapedKey(escaped, d) {
+ for (var k of tr.b.dictionaryKeys(d))
+ if (escapeChars(k) === escaped)
+ return k;
+ throw new Error('did not find key ' + escaped + ' in ' +
+ tr.b.dictionaryKeys(d));
+}
+
+function geoMeanFromHistogram(h) {
+ if (!h.hasOwnProperty('buckets'))
+ return 0.0;
+ var count = 0;
+ var sumOfLogs = 0;
+ for (var bucket of h.buckets) {
+ if (bucket.hasOwnProperty('high'))
+ bucket.mean = (bucket.low + bucket.high) / 2.0;
+ else
+ bucket.mean = bucket.low;
+
+ if (bucket.mean > 0) {
+ sumOfLogs += Math.log(bucket.mean) * bucket.count;
+ count += bucket.count;
+ }
+ }
+ if (count === 0)
+ return 0.0;
+ return Math.exp(sumOfLogs / count);
+}
+
+function splitMetric(metricName) {
+ var parts = metricName.split('/');
benjhayden 2016/09/27 06:25:54 too much indentation
RobertoCN 2016/09/27 22:58:49 Done.
+ var interactionName;
+ var traceName = 'summary';
+ var chartName = parts[0];
+ if (parts.length === 3) {
+ // parts[1] is the interactionName
+ if (parts[1])
+ chartName = parts[1] + '@@' + chartName;
+ traceName = parts[2];
+ } else if (parts.length === 2) {
+ if (chartName !== parts[1])
+ traceName = parts[1];
+ } else
+ throw new Error('Could not parse metric name.');
+ return [chartName, traceName];
+}
+
+function valuesFromCharts(listOfCharts, metricName) {
+ var allValues = [];
+ var chartAndTrace = splitMetric(metricName);
+ for (var charts of listOfCharts) {
+ var chartName = findUnescapedKey(chartAndTrace[0], charts.charts);
+ if (chartName) {
+ var traceName = findUnescapedKey(
+ chartAndTrace[1], charts.charts[chartName]);
+ if (traceName) {
+ if (charts.charts[chartName][traceName].type ===
+ 'list_of_scalar_values')
+ allValues.push(...charts.charts[chartName][traceName].values);
+ if (charts.charts[chartName][traceName].type === 'histogram')
+ allValues.push(
+ geoMeanFromHistogram(charts.charts[chartName][traceName]));
+ }
+ }
+ }
+ return allValues;
+}
+
+function rawValuesByMetricName(valueSet, metricName) {
+ var interactionRecord, valueName;
+ var [itrPlusChart, story] = splitMetric(metricName);
+ if (itrPlusChart.indexOf('@@') > -1)
+ [interactionRecord, valueName] = itrPlusChart.split('@@');
+ else if (itrPlusChart.indexOf('-') > -1) {
+ [interactionRecord, ...valueName] = itrPlusChart.split('-');
+ valueName = valueName.join('');
+ } else
+ valueName = itrPlusChart;
+ var values = valueSet.getValuesNamed(valueName);
+ if (!values || values.length === 0) {
+ // If there was a dash in the chart name, but it wasn't an
+ // interaction record.
+ valueName = itrPlusChart;
+ values = valueSet.getValuesNamed(valueName);
+ interactionRecord = undefined;
+ if (!values || values.length === 0)
+ throw new Error('No values with name ' + valueName);
+ }
+ var filtered = [];
+ for (var value of values) {
+ if (value.name !== valueName)
+ continue;
+ var ii = tr.v.d.IterationInfo.getFromValue(value);
+ if (interactionRecord) {
+ var IRParts = [];
+ var keys = Object.keys(ii.storyGroupingKeys);
+ keys.sort();
+ for (var key of keys)
+ IRParts.push(ii.storyGroupingKeys[key]);
+ if (interactionRecord === IRParts.join('_') &&
+ escapeChars(ii.storyDisplayName) ===
+ escapeChars(story)) {
+ filtered.push(value);
+ }
+ } else if (escapeChars(ii.storyDisplayName) ==
+ escapeChars(story))
+ filtered.push(value);
+ }
+
+ var rawValues = [];
+ for (var val of filtered) {
+ if (val instanceof tr.v.Histogram)
+ rawValues = rawValues.concat(val.sampleValues);
+ else
+ throw new Error('Only tr.v.Histogram values are supported');
+ }
+ return rawValues;
+}
+
+function parseFiles(files) {
+ var results = [];
+ for (var path of files) {
+ try {
+ var current = tr.b.getSync('file://' + path);
+ } catch (ex) {
+ var err = new Error('Could not open' + path);
+ err.name = 'File loading error';
+ throw err;
+ }
+ results.push(JSON.parse(current));
+ }
+ return results;
+}
+
+var escapeForRegExp = s => s.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&');
+
+var strFromRE = re => re.toString().split('/')[1];
+
+function valuesFromBuildbotOutput(out, metric) {
+ if (!out)
+ return [];
+ var stringVals = [];
+ var floatVals = [];
+ var chartAndTrace = splitMetric(metric);
+ var metricRE = escapeForRegExp(
+ 'RESULT ' + chartAndTrace[0] + ': ' + chartAndTrace[1] + '=');
+ var singleResultRE = new RegExp(metricRE +
+ strFromRE(/\s*([-]?[\d\.]+)/), 'g');
+ var multiResultsRE = new RegExp(metricRE +
+ strFromRE(/\s*\[\s*([\d\., -]+)\s*\]/), 'g');
+ var meanStdDevRE = new RegExp(metricRE +
+ strFromRE(/\s*\{\s*([-]?\d*(?:\.\d*)?),\s*([-]?\d*(?:\.\d*)?)\}/), 'g');
+ for (var line of out.split(/\r?\n/)) {
+ var singleResultMatch = singleResultRE.exec(line);
+ var multiResultsMatch = multiResultsRE.exec(line);
+ var meanStdDevMatch = meanStdDevRE.exec(line);
+ if (singleResultMatch && singleResultMatch.length > 1)
+ stringVals.push(singleResultMatch[1]);
+ else if (multiResultsMatch && multiResultsMatch.length > 1) {
+ var values = multiResultsMatch[1].split(',');
+ stringVals = stringVals.concat(values);
+ } else if (meanStdDevMatch && meanStdDevMatch.length > 1)
+ stringVals.push(meanStdDevMatch[1]);
+ }
+ for (var val of stringVals) {
+ var f = parseFloat(val);
+ if (!isNaN(f))
+ floatVals.push(f);
+ }
+ return floatVals;
+}
+
+function parseMultipleBuildbotStreams(files, metric) {
+ var allValues = [];
+ for (var path of files) {
+ try {
+ var contents = tr.b.getSync('file://' + path);
+ }
+ catch (ex) {
+ var err = new Error('Could not open' + path);
+ err.name = 'File loading error';
+ throw err;
+ }
+ allValues = allValues.concat(valuesFromBuildbotOutput(contents, metric));
+ }
+ return allValues;
+}
+
+var SampleComparison = {
+ ENOUGH_SAMPLES: 18,
+ SIGNIFICANCE_LEVEL: 0.01,
+ REJECT_THE_NULL: 'REJECT_THE_NULL',
+ FAIL_TO_REJECT_THE_NULL: 'FAIL_TO_REJECT_THE_NULL',
+ NEED_MORE_DATA: 'NEED_MORE_DATA',
+
+ compareBuildbotOutputs: function(
+ buildbotOutputAPathList, buildbotOutputBPathList, metric) {
+ var aPaths = buildbotOutputAPathList.split(',');
+ var bPaths = buildbotOutputBPathList.split(',');
+ var sampleA = parseMultipleBuildbotStreams(aPaths, metric);
+ var sampleB = parseMultipleBuildbotStreams(bPaths, metric);
+ return SampleComparison.compareSamples(sampleA, sampleB);
+ },
+
+ compareValuesets: function(valueSetAPathList, valueSetBPathList, metric) {
+ var aPaths = valueSetAPathList.split(',');
+ var bPaths = valueSetBPathList.split(',');
+ var valueSetA = new tr.v.ValueSet();
+ var valueSetB = new tr.v.ValueSet();
+ var dictsA = parseFiles(aPaths);
+ var dictsB = parseFiles(bPaths);
+ for (var d of dictsA)
+ valueSetA.addValuesFromDicts(d);
+ for (var d of dictsB)
+ valueSetB.addValuesFromDicts(d);
+
+ var sampleA = rawValuesByMetricName(valueSetA, metric);
+ var sampleB = rawValuesByMetricName(valueSetB, metric);
+ return SampleComparison.compareSamples(sampleA, sampleB);
+ },
+
+ compareCharts: function(chartPathListA, chartPathListB, metric) {
+ var aPaths = chartPathListA.split(',');
+ var bPaths = chartPathListB.split(',');
+ var chartsA = parseFiles(aPaths);
+ var chartsB = parseFiles(bPaths);
+ var sampleA = valuesFromCharts(chartsA, metric);
+ var sampleB = valuesFromCharts(chartsB, metric);
+ return SampleComparison.compareSamples(sampleA, sampleB);
+ },
+
+ compareSamples: function(sampleA, sampleB) {
+ var cmp = tr.b.Statistics.mwu(sampleA, sampleB,
+ SampleComparison.SIGNIFICANCE_LEVEL);
+ var result = {
+ sample_a: sampleA,
+ sample_b: sampleB,
+ result: SampleComparison.NEED_MORE_DATA
+ };
+ if (cmp.significance === tr.b.Statistics.Significance.SIGNIFICANT)
+ result.result = SampleComparison.REJECT_THE_NULL;
+ else if (sampleA.length > SampleComparison.ENOUGH_SAMPLES &&
+ sampleB.length > SampleComparison.ENOUGH_SAMPLES)
+ result.result = SampleComparison.FAIL_TO_REJECT_THE_NULL;
+ return result;
+ }
+};
+
+if (tr.isHeadless) {
+ var method, rest;
+ [method, ...rest] = sys.argv.slice(1);
+ switch (method) {
benjhayden 2016/09/27 06:25:54 Why not SampleComparison[method](...rest)?
RobertoCN 2016/09/27 22:58:49 Right! I wanted to do something like that, I just
+ case 'compareCharts':
+ console.log(JSON.stringify(SampleComparison.compareCharts(...rest)));
+ break;
+ case 'compareValuesets':
+ console.log(JSON.stringify(SampleComparison.compareValuesets(...rest)));
+ break;
+ case 'compareBuildbotOutputs':
+ console.log(JSON.stringify(
+ SampleComparison.compareBuildbotOutputs(...rest)));
+ break;
+ }
+}
+</script>
« tracing/bin/compare_samples ('K') | « tracing/tracing/metrics/compare_samples.py ('k') | tracing/tracing/metrics/compare_samples_unittest.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698