Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(68)

Issues Search
    My Issues | Starred     Open | Closed | All

Side by Side Diff: chrome/test/data/third_party/kraken/hosted/compare-results.js

Issue 11348021: Automate Kraken benchmark with Chrome Remote Control. (Closed) Base URL: svn://svn.chromium.org/chrome/trunk/src
Patch Set: Created 8 years, 1 month ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « chrome/test/data/third_party/kraken/hosted/analyze-results.js ('k') | chrome/test/data/third_party/kraken/hosted/explanations/astar.html » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 /*
2 * Copyright (C) 2007 Apple Inc. All rights reserved.
3 * Copyright (C) 2010 Mozilla Foundation
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY APPLE COMPUTER, INC. ``AS IS'' AND ANY
15 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE COMPUTER, INC. OR
18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
21 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
22 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
24 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27 function sunspiderCompareResults(output1, output2)
28 {
29 var count1 = output1.length;
30 var count2 = output2.length;
31
32 var itemTotals1 = {};
33 itemTotals1.length = count1;
34
35 var total1 = 0;
36 var categoryTotals1 = {};
37 var testTotalsByCategory1 = {};
38
39 var mean1 = 0;
40 var categoryMeans1 = {};
41 var testMeansByCategory1 = {};
42
43 var stdDev1 = 0;
44 var categoryStdDevs1 = {};
45 var testStdDevsByCategory1 = {};
46
47 var stdErr1 = 0;
48 var categoryStdErrs1 = {};
49 var testStdErrsByCategory1 = {};
50
51 var itemTotals2 = {};
52 itemTotals2.length = count2;
53
54 var total2 = 0;
55 var categoryTotals2 = {};
56 var testTotalsByCategory2 = {};
57
58 var mean2 = 0;
59 var categoryMeans2 = {};
60 var testMeansByCategory2 = {};
61
62 var stdDev2 = 0;
63 var categoryStdDevs2 = {};
64 var testStdDevsByCategory2 = {};
65
66 var stdErr2 = 0;
67 var categoryStdErrs2 = {};
68 var testStdErrsByCategory2 = {};
69
70 function initialize()
71 {
72 itemTotals1 = {total: []};
73
74 for (var i = 0; i < categories.length; i++) {
75 var category = categories[i];
76 itemTotals1[category] = [];
77 categoryTotals1[category] = 0;
78 testTotalsByCategory1[category] = {};
79 categoryMeans1[category] = 0;
80 testMeansByCategory1[category] = {};
81 categoryStdDevs1[category] = 0;
82 testStdDevsByCategory1[category] = {};
83 categoryStdErrs1[category] = 0;
84 testStdErrsByCategory1[category] = {};
85 }
86
87 for (var i = 0; i < tests.length; i++) {
88 var test = tests[i];
89 itemTotals1[test] = [];
90 var category = test.replace(/-.*/, "");
91 testTotalsByCategory1[category][test] = 0;
92 testMeansByCategory1[category][test] = 0;
93 testStdDevsByCategory1[category][test] = 0;
94 testStdErrsByCategory1[category][test] = 0;
95 }
96
97 for (var i = 0; i < count1; i++) {
98 itemTotals1["total"][i] = 0;
99 for (var category in categoryTotals1) {
100 itemTotals1[category][i] = 0;
101 for (var test in testTotalsByCategory1[category]) {
102 itemTotals1[test][i] = 0;
103 }
104 }
105 }
106
107 itemTotals2 = {total: []};
108
109 for (var i = 0; i < categories.length; i++) {
110 var category = categories[i];
111 itemTotals2[category] = [];
112 categoryTotals2[category] = 0;
113 testTotalsByCategory2[category] = {};
114 categoryMeans2[category] = 0;
115 testMeansByCategory2[category] = {};
116 categoryStdDevs2[category] = 0;
117 testStdDevsByCategory2[category] = {};
118 categoryStdErrs2[category] = 0;
119 testStdErrsByCategory2[category] = {};
120 }
121
122 for (var i = 0; i < tests.length; i++) {
123 var test = tests[i];
124 itemTotals2[test] = [];
125 var category = test.replace(/-.*/, "");
126 testTotalsByCategory2[category][test] = 0;
127 testMeansByCategory2[category][test] = 0;
128 testStdDevsByCategory2[category][test] = 0;
129 testStdErrsByCategory2[category][test] = 0;
130 }
131
132 for (var i = 0; i < count2; i++) {
133 itemTotals2["total"][i] = 0;
134 for (var category in categoryTotals2) {
135 itemTotals2[category][i] = 0;
136 for (var test in testTotalsByCategory2[category]) {
137 itemTotals2[test][i] = 0;
138 }
139 }
140 }
141
142 }
143
144 function computeItemTotals(output, itemTotals)
145 {
146 for (var i = 0; i < output.length; i++) {
147 var result = output[i];
148 for (var test in result) {
149 var time = result[test];
150 var category = test.replace(/-.*/, "");
151 itemTotals["total"][i] += time;
152 itemTotals[category][i] += time;
153 itemTotals[test][i] += time;
154 }
155 }
156 }
157
158 function computeTotals(output, categoryTotals, testTotalsByCategory)
159 {
160 var total = 0;
161
162 for (var i = 0; i < output.length; i++) {
163 var result = output[i];
164 for (var test in result) {
165 var time = result[test];
166 var category = test.replace(/-.*/, "");
167 total += time;
168 categoryTotals[category] += time;
169 testTotalsByCategory[category][test] += time;
170 }
171 }
172
173 return total;
174 }
175
176 function computeMeans(count, total, categoryTotals, categoryMeans, testTotal sByCategory, testMeansByCategory)
177 {
178 var mean = total / count;
179 for (var category in categoryTotals) {
180 categoryMeans[category] = categoryTotals[category] / count;
181 for (var test in testTotalsByCategory[category]) {
182 testMeansByCategory[category][test] = testTotalsByCategory[categ ory][test] / count;
183 }
184 }
185 return mean;
186 }
187
188 function standardDeviation(mean, items)
189 {
190 var deltaSquaredSum = 0;
191 for (var i = 0; i < items.length; i++) {
192 var delta = items[i] - mean;
193 deltaSquaredSum += delta * delta;
194 }
195 variance = deltaSquaredSum / (items.length - 1);
196 return Math.sqrt(variance);
197 }
198
199 function computeStdDevs(mean, itemTotals, categoryStdDevs, categoryMeans, te stStdDevsByCategory, testMeansByCategory)
200 {
201 var stdDev = standardDeviation(mean, itemTotals["total"]);
202 for (var category in categoryStdDevs) {
203 categoryStdDevs[category] = standardDeviation(categoryMeans[category ], itemTotals[category]);
204 }
205 for (var category in categoryStdDevs) {
206 for (var test in testStdDevsByCategory[category]) {
207 testStdDevsByCategory[category][test] = standardDeviation(testMe ansByCategory[category][test], itemTotals[test]);
208 }
209 }
210 return stdDev;
211 }
212
213 function computeStdErrors(count, stdDev, categoryStdErrs, categoryStdDevs, t estStdErrsByCategory, testStdDevsByCategory)
214 {
215 var sqrtCount = Math.sqrt(count);
216
217 var stdErr = stdDev / sqrtCount;
218 for (var category in categoryStdErrs) {
219 categoryStdErrs[category] = categoryStdDevs[category] / sqrtCount;
220 }
221 for (var category in categoryStdDevs) {
222 for (var test in testStdErrsByCategory[category]) {
223 testStdErrsByCategory[category][test] = testStdDevsByCategory[ca tegory][test] / sqrtCount;
224 }
225 }
226
227 return stdErr;
228 }
229
230 var tDistribution = [NaN, NaN, 12.71, 4.30, 3.18, 2.78, 2.57, 2.45, 2.36, 2. 31, 2.26, 2.23, 2.20, 2.18, 2.16, 2.14, 2.13, 2.12, 2.11, 2.10, 2.09, 2.09, 2.08 , 2.07, 2.07, 2.06, 2.06, 2.06, 2.05, 2.05, 2.05, 2.04, 2.04, 2.04, 2.03, 2.03, 2.03, 2.03, 2.03, 2.02, 2.02, 2.02, 2.02, 2.02, 2.02, 2.02, 2.01, 2.01, 2.01, 2. 01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00 , 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1. 99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98 , 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1. 98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98 , 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1. 97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97 , 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1. 97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97 , 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1. 97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97 , 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1. 97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97 , 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1. 97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97 , 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1. 97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97 , 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1. 97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97 , 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1. 97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97 , 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.96];
231 var tMax = tDistribution.length;
232 var tLimit = 1.96;
233
234 function tDist(n)
235 {
236 if (n > tMax)
237 return tLimit;
238 return tDistribution[n];
239 }
240
241
242 function formatMean(meanWidth, mean, stdErr, count)
243 {
244 var meanString = mean.toFixed(1).toString();
245 while (meanString.length < meanWidth) {
246 meanString = " " + meanString;
247 }
248
249 var error = "+/- " + ((tDist(count) * stdErr / mean) * 100).toFixed(1) + "% ";
250
251 return meanString + "ms " + error;
252 }
253
254 function computeLabelWidth()
255 {
256 var width = "Total".length;
257 for (var category in categoryMeans1) {
258 if (category.length + 2 > width)
259 width = category.length + 2;
260 }
261 for (var i = 0; i < tests.length; i++) {
262 var shortName = tests[i].replace(/^[^-]*-/, "");
263 if (shortName.length + 4 > width)
264 width = shortName.length + 4;
265 }
266
267 return width;
268 }
269
270 function computeMeanWidth(mean, categoryMeans, testMeansByCategory)
271 {
272 var width = mean.toFixed(1).toString().length;
273 for (var category in categoryMeans) {
274 var candidate = categoryMeans[category].toFixed(1).toString().length ;
275 if (candidate > width)
276 width = candidate;
277 for (var test in testMeansByCategory[category]) {
278 var candidate = testMeansByCategory[category][test].toFixed(1).t oString().length;
279 if (candidate > width)
280 width = candidate;
281 }
282 }
283
284 return width;
285 }
286
287 function pad(str, n)
288 {
289 while (str.length < n) {
290 str += " ";
291 }
292 return str;
293 }
294
295 function resultLine(labelWidth, indent, label, meanWidth1, mean1, stdErr1, m eanWidth2, mean2, stdErr2)
296 {
297 result = pad("", indent);
298 result += label + ": ";
299 result = pad(result, labelWidth + 2);
300
301 var t = (mean1 - mean2) / (Math.sqrt((stdErr1 * stdErr1) + (stdErr2 * st dErr2)));
302 var df = count1 + count2 - 2;
303
304 var statisticallySignificant = (Math.abs(t) > tDist(df+1));
305 var diff = mean2 - mean1;
306 var percentage = 100 * diff / mean1;
307 var isFaster = diff < 0;
308 var probablySame = (percentage < 0.1) && !statisticallySignificant;
309 var ratio = isFaster ? (mean1 / mean2) : (mean2 / mean1);
310 var fixedRatio = (ratio < 1.2) ? ratio.toFixed(3).toString() : ((ratio < 10) ? ratio.toFixed(2).toString() : ratio.toFixed(1).toString());
311 var formattedRatio = isFaster ? fixedRatio + "x as fast" : "*" + fixedRa tio + "x as slow*";
312
313 var diffSummary;
314 var diffDetail;
315
316 if (probablySame) {
317 diffSummary = "-";
318 diffDetail = "";
319 } else if (!statisticallySignificant) {
320 diffSummary = "??";
321 diffDetail = " might be " + formattedRatio;
322 } else {
323 diffSummary = formattedRatio;
324 diffDetail = " significant";
325 }
326
327 return result + pad(diffSummary, 18) + formatMean(meanWidth1, mean1, std Err1, count1) + " " + formatMean(meanWidth2, mean2, stdErr2, count2) + diffDeta il;
328 }
329
330 function printOutput()
331 {
332 var labelWidth = computeLabelWidth();
333 var meanWidth1 = computeMeanWidth(mean1, categoryMeans1, testMeansByCate gory1);
334 var meanWidth2 = computeMeanWidth(mean2, categoryMeans2, testMeansByCate gory2);
335
336 print("\n");
337 var header = "TEST";
338 while (header.length < labelWidth)
339 header += " ";
340 header += " COMPARISON FROM TO DETAILS";
341 print(header);
342 print("");
343 print("================================================================= ===================");
344 print("");
345 print(resultLine(labelWidth, 0, "** TOTAL **", meanWidth1, mean1, stdErr 1, meanWidth2, mean2, stdErr2));
346 print("");
347 print("================================================================= ===================");
348
349 for (var category in categoryMeans1) {
350 print("");
351 print(resultLine(labelWidth, 2, category,
352 meanWidth1, categoryMeans1[category], categoryStdEr rs1[category],
353 meanWidth2, categoryMeans2[category], categoryStdEr rs2[category]));
354 for (var test in testMeansByCategory1[category]) {
355 var shortName = test.replace(/^[^-]*-/, "");
356 print(resultLine(labelWidth, 4, shortName,
357 meanWidth1, testMeansByCategory1[category][test ], testStdErrsByCategory1[category][test],
358 meanWidth2, testMeansByCategory2[category][test ], testStdErrsByCategory2[category][test]));
359 }
360 }
361 }
362
363 initialize();
364
365 computeItemTotals(output1, itemTotals1);
366 computeItemTotals(output2, itemTotals2);
367
368 total1 = computeTotals(output1, categoryTotals1, testTotalsByCategory1);
369 total2 = computeTotals(output2, categoryTotals2, testTotalsByCategory2);
370
371 mean1 = computeMeans(count1, total1, categoryTotals1, categoryMeans1, testTo talsByCategory1, testMeansByCategory1);
372 mean2 = computeMeans(count2, total2, categoryTotals2, categoryMeans2, testTo talsByCategory2, testMeansByCategory2);
373
374 stdDev1 = computeStdDevs(mean1, itemTotals1, categoryStdDevs1, categoryMeans 1, testStdDevsByCategory1, testMeansByCategory1);
375 stdDev2 = computeStdDevs(mean2, itemTotals2, categoryStdDevs2, categoryMeans 2, testStdDevsByCategory2, testMeansByCategory2);
376
377 stdErr1 = computeStdErrors(count1, stdDev1, categoryStdErrs1, categoryStdDev s1, testStdErrsByCategory1, testStdDevsByCategory1);
378 stdErr2 = computeStdErrors(count2, stdDev2, categoryStdErrs2, categoryStdDev s2, testStdErrsByCategory2, testStdDevsByCategory2);
379
380 printOutput();
381 }
OLDNEW
« no previous file with comments | « chrome/test/data/third_party/kraken/hosted/analyze-results.js ('k') | chrome/test/data/third_party/kraken/hosted/explanations/astar.html » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698