Chromium Code Reviews Chromium Code Reviews
Issue 2089833002:
Entry point for bisect sample comparison. (Closed)
Base URL: https://github.com/catapult-project/catapult.git@mann| OLD | NEW |
|---|---|
| (Empty) | |
| 1 <!DOCTYPE html> | |
| 2 <!-- | |
| 3 Copyright 2016 The Chromium Authors. All rights reserved. | |
| 4 Use of this source code is governed by a BSD-style license that can be | |
| 5 found in the LICENSE file. | |
| 6 --> | |
| 7 | |
| 8 <link rel="import" href="/tracing/base/statistics.html"> | |
| 9 <link rel="import" href="/tracing/base/xhr.html"> | |
| 10 <link rel="import" href="/tracing/value/value_set.html"> | |
| 11 | |
| 12 <script> | |
| 13 'use strict'; | |
| 14 | |
| 15 tr.exportTo('tr.metrics', function() { | |
| 16 var BisectComparison = { | |
| 17 ENOUGH_SAMPLES: 18, | |
| 18 SIGNIFICANCE_LEVEL: 0.05, | |
| 19 escapeChars: function(s) { | |
|
eakuefner
2016/07/21 19:35:34
write this as an arrow, that is:
s => s.replace..
RobertoCN
2016/09/02 21:19:30
Done.
| |
| 20 return s.replace(/[\:|=\/#&,]/g, '_'); | |
| 21 }, | |
| 22 valuesFromCharts: function(listOfCharts, metricName) { | |
| 23 function geoMeanFromHistogram(h) { | |
| 24 if (!h.hasOwnProperty('buckets')) { | |
| 25 return 0.0; | |
| 26 } | |
| 27 var count = 0; | |
| 28 var sumOfLogs = 0; | |
| 29 for (var bucket in h.buckets) { | |
|
eakuefner
2016/07/21 19:35:34
you probably want for..of not for..in. for..in is
RobertoCN
2016/09/02 21:19:30
Done.
| |
| 30 if (bucket.hasOwnProperty('high')) { | |
| 31 bucket.mean = (bucket.low + bucket.high) / 2.0; | |
| 32 } else { | |
| 33 bucket.mean = bucket.low; | |
| 34 } | |
| 35 | |
| 36 if (bucket.mean > 0) { | |
| 37 sumOfLogs += Math.log(bucket.mean) * bucket.count; | |
| 38 count += bucket.count; | |
| 39 } | |
| 40 } | |
| 41 if (count == 0) { | |
| 42 return 0.0; | |
| 43 } | |
| 44 return Math.exp(sumOfLogs / count); | |
| 45 } | |
| 46 var all_values = []; | |
| 47 var charts; | |
| 48 for (var i = 0; i < listOfCharts.length; i++) { | |
|
eakuefner
2016/07/21 19:35:34
again a place where you could just use a for..of l
RobertoCN
2016/09/02 21:19:30
Done.
| |
| 49 charts = listOfCharts[i]; | |
| 50 var chartName, interactionName, traceName; | |
| 51 var parts = metricName.split('/'); | |
| 52 if (parts.length == 3) { | |
| 53 chartName = parts[0]; | |
| 54 interactionName = parts[1]; | |
| 55 traceName = parts[2]; | |
| 56 } else if (parts.length == 2) { | |
| 57 chartName = parts[0]; | |
| 58 if (chartName != parts[1]) { | |
| 59 traceName = parts[1]; | |
| 60 } | |
| 61 } else { | |
| 62 throw new Error('Could not parse metric name.'); | |
| 63 } | |
| 64 | |
| 65 if (interactionName) { | |
| 66 chartName = interactionName + '@@' + chartName; | |
| 67 } | |
| 68 | |
| 69 if (!traceName) { | |
| 70 traceName = 'summary'; | |
| 71 } | |
| 72 | |
| 73 | |
| 74 var chart, trace; | |
| 75 | |
| 76 for (chart in charts.charts) { | |
| 77 if (charts.charts.hasOwnProperty(chart) && | |
| 78 this.escapeChars(chart) == chartName) { | |
| 79 chartName = chart; // Unescaping | |
| 80 break; | |
| 81 } | |
| 82 } | |
| 83 for (trace in charts.charts[chartName]) { | |
| 84 if (charts.charts[chartName].hasOwnProperty(trace) && | |
| 85 this.escapeChars(trace) == traceName) { | |
| 86 traceName = trace; // Unescaping | |
| 87 break; | |
| 88 } | |
| 89 } | |
| 90 if (charts.charts[chartName][traceName].type == | |
| 91 'list_of_scalar_values') { | |
| 92 all_values.push.apply( | |
| 93 all_values, charts.charts[chartName][traceName].values); | |
| 94 } | |
| 95 | |
| 96 if (charts.charts[chartName][traceName].type == 'histogram') { | |
| 97 all_values.push( | |
| 98 geoMeanFromHistogam(charts.charts[chartName][traceName])); | |
| 99 } | |
| 100 } | |
| 101 return all_values; | |
| 102 }, | |
| 103 | |
| 104 compareValuesets: function(valueSetAPathList, valueSetBPathList, metric) { | |
| 105 var aPaths = valueSetAPathList.split(','); | |
| 106 var bPaths = valueSetBPathList.split(','); | |
| 107 var valueSetA = new tr.v.ValueSet(); | |
| 108 var valueSetB = new tr.v.ValueSet(); | |
| 109 var current; | |
| 110 for (var i = 0; i < aPaths.length; i++) { | |
| 111 try { | |
| 112 current = tr.b.getSync('file://' + aPaths[i]); | |
| 113 } catch (ex) { | |
| 114 var err = new Error('Could not open' + aPaths[i]); | |
| 115 err.name = 'File loading error'; | |
| 116 throw err; | |
| 117 } | |
| 118 valueSetA.addValuesFromDicts(JSON.parse(current)); | |
| 119 } | |
| 120 for (var i = 0; i < bPaths.length; i++) { | |
| 121 try { | |
| 122 current = tr.b.getSync('file://' + bPaths[i]); | |
| 123 } catch (ex) { | |
| 124 var err = new Error('Could not open' + bPaths[i]); | |
| 125 err.name = 'File loading error'; | |
| 126 throw err; | |
| 127 } | |
| 128 valueSetB.addValuesFromDicts(JSON.parse(current)); | |
| 129 } | |
| 130 | |
| 131 function rawValuesByMetricName(valueSet, metricName, thisParam) { | |
| 132 var interactionRecord, valueName, story; | |
| 133 var metricNameParts = metricName.split('/'); | |
| 134 if (metricNameParts[0] === metricNameParts[1]) { | |
| 135 story = 'summary'; | |
| 136 } else { | |
| 137 story = metricNameParts[1]; | |
| 138 } | |
| 139 var chartNameParts = metricNameParts[0].split('-'); | |
| 140 valueName = chartNameParts[1]; | |
| 141 if (chartNameParts.length == 2) { | |
| 142 interactionRecord = chartNameParts[0]; | |
| 143 } | |
| 144 var values = valueSet.getValuesWithName(valueName); | |
| 145 if (!values || values.length == 0) { | |
| 146 // If there was a dash in the chart name, but it wasn't an | |
| 147 // interaction record. | |
| 148 valueName = metricNameParts[0]; | |
| 149 values = valueSet.getValuesWithName(valueName); | |
| 150 interactionRecord = undefined; | |
| 151 if (!values || values.length == 0) { | |
| 152 throw new Error('No values with name ' + valueName); | |
| 153 } | |
| 154 } | |
| 155 var filtered = values.filter(function(value) { | |
| 156 if (value.name != valueName) { | |
| 157 return false; | |
| 158 } | |
| 159 var ii = tr.v.d.IterationInfo.getFromValue(value); | |
| 160 if (interactionRecord) { | |
| 161 var values = []; | |
| 162 var keys = Object.keys(ii.storyGroupingKeys); | |
| 163 keys.sort(); | |
| 164 for (var i = 0; i < keys.length; i++) { | |
| 165 values.push(ii.storyGroupingKeys[keys[i]]); | |
| 166 } | |
| 167 if (interactionRecord == values.join('_')) { | |
| 168 return thisParam.escapeChars(ii.storyDisplayName) == | |
| 169 thisParam.escapeChars(story); | |
| 170 } | |
| 171 return false; | |
| 172 } | |
| 173 return thisParam.escapeChars(ii.storyDisplayName) == | |
| 174 thisParam.escapeChars(story); | |
| 175 }); | |
| 176 var rawValues = []; | |
| 177 for (var i = 0; i < filtered.length; i++) { | |
| 178 if (filtered[i].numeric instanceof tr.v.Numeric) { | |
| 179 rawValues = rawValues.concat(filtered[i].numeric.sampleValues); | |
| 180 } else if (filtered[i].numeric instanceof tr.v.ScalarNumeric) { | |
| 181 rawValues.push(filtered[i].numeric.value); | |
| 182 } | |
| 183 } | |
| 184 return rawValues; | |
| 185 } | |
| 186 var sampleA = rawValuesByMetricName(valueSetA, metric, this); | |
| 187 var sampleB = rawValuesByMetricName(valueSetB, metric, this); | |
| 188 return this.compareSamples(sampleA, sampleB); | |
| 189 }, | |
| 190 | |
| 191 compareCharts: function(chartPathListA, chartPathListB, metric) { | |
| 192 var aPaths = chartPathListA.split(','); | |
| 193 var bPaths = chartPathListB.split(','); | |
| 194 var chartsA = new Array(aPaths.length); | |
| 195 var chartsB = new Array(bPaths.length); | |
| 196 var current; | |
| 197 for (var i = 0; i < aPaths.length; i++) { | |
| 198 try { | |
| 199 current = tr.b.getSync('file://' + aPaths[i]); | |
| 200 } catch (ex) { | |
| 201 var err = new Error('Could not open' + aPaths[i]); | |
| 202 err.name = 'File loading error'; | |
| 203 throw err; | |
| 204 } | |
| 205 chartsA[i] = JSON.parse(current); | |
| 206 } | |
| 207 for (var i = 0; i < bPaths.length; i++) { | |
| 208 try { | |
| 209 current = tr.b.getSync('file://' + bPaths[i]); | |
| 210 } catch (ex) { | |
| 211 var err = new Error('Could not open' + bPaths[i]); | |
| 212 err.name = 'File loading error'; | |
| 213 throw err; | |
| 214 } | |
| 215 chartsB[i] = JSON.parse(current); | |
| 216 } | |
| 217 var sampleA = this.valuesFromCharts(chartsA, metric); | |
| 218 var sampleB = this.valuesFromCharts(chartsB, metric); | |
| 219 return this.compareSamples(sampleA, sampleB); | |
| 220 }, | |
| 221 | |
| 222 compareSamples: function(sampleA, sampleB) { | |
| 223 var pValue = tr.b.Statistics.mwu.test(sampleA, sampleB); | |
| 224 // Diagnostics | |
| 225 var result = { | |
| 226 sample_a: { | |
| 227 std_dev: tr.b.Statistics.stddev(sampleA), | |
| 228 mean: tr.b.Statistics.mean(sampleA), | |
| 229 debug_values: sampleA | |
| 230 }, | |
| 231 sample_b: { | |
| 232 std_dev: tr.b.Statistics.stddev(sampleB), | |
| 233 mean: tr.b.Statistics.mean(sampleB), | |
| 234 debug_values: sampleB | |
| 235 }, | |
| 236 pValue: pValue.p, | |
| 237 UStatistic: pValue.U, | |
| 238 result: 'needMoreData', | |
| 239 }; | |
| 240 if (pValue.p < this.SIGNIFICANCE_LEVEL) { | |
| 241 result.result = true; // Reject the null | |
| 242 } else if (sampleA.length > this.ENOUGH_SAMPLES && | |
| 243 sampleB.length > this.ENOUGH_SAMPLES) { | |
| 244 result.result = false; // Fail to reject the null. | |
| 245 } | |
| 246 return result; | |
| 247 } | |
| 248 }; | |
| 249 | |
| 250 return { | |
| 251 BisectComparison: BisectComparison | |
| 252 }; | |
| 253 }); | |
| 254 </script> | |
| OLD | NEW |
