| OLD | NEW |
| (Empty) | |
| 1 #!/usr/bin/env node |
| 2 |
| 3 var _ = require("lodash"); |
| 4 var Benchmark = require("benchmark"); |
| 5 var exec = require("child_process").exec; |
| 6 var execSync = require("child_process").execSync; |
| 7 var fs = require("fs"); |
| 8 var path = require("path"); |
| 9 var mkdirp = require("mkdirp"); |
| 10 var async = require("../"); |
| 11 var suiteConfigs = require("./suites"); |
| 12 var semver = require("semver"); |
| 13 |
| 14 var args = require("yargs") |
| 15 .usage("Usage: $0 [options] [tag1] [tag2]") |
| 16 .describe("g", "run only benchmarks whose names match this regex") |
| 17 .alias("g", "grep") |
| 18 .default("g", ".*") |
| 19 .describe("i", "skip benchmarks whose names match this regex") |
| 20 .alias("i", "reject") |
| 21 .default("i", "^$") |
| 22 .describe("l", "maximum running time per test (in seconds)") |
| 23 .alias("l", "limit") |
| 24 .default("l", 2) |
| 25 .help("h") |
| 26 .alias("h", "help") |
| 27 .example("$0 0.9.2 0.9.0", "Compare v0.9.2 with v0.9.0") |
| 28 .example("$0 0.9.2", "Compare v0.9.2 with the current working version") |
| 29 .example("$0", "Compare the latest tag with the current working version") |
| 30 .example("$0 -g each", "only run the each(), eachLimit() and " + |
| 31 "eachSeries() benchmarks") |
| 32 .example("") |
| 33 .argv; |
| 34 |
| 35 var grep = new RegExp(args.g, "i"); |
| 36 var reject = new RegExp(args.i, "i"); |
| 37 |
| 38 function getLatestVersion() { |
| 39 var tags = execSync("git tag"); |
| 40 var latest = _(tags).split("\n") |
| 41 .compact() |
| 42 .sort(semver.gt) |
| 43 .last(); |
| 44 console.log("Latest tag is ", latest); |
| 45 return latest; |
| 46 } |
| 47 |
| 48 var version0 = args._[0] || getLatestVersion(); |
| 49 var version1 = args._[1] || "current"; |
| 50 var versionNames = [version0, version1]; |
| 51 var benchOptions = {defer: true, minSamples: 1, maxTime: +args.l}; |
| 52 var versions; |
| 53 var wins = {}; |
| 54 var totalTime = {}; |
| 55 totalTime[version0] = wins[version0] = 0; |
| 56 totalTime[version1] = wins[version1] = 0; |
| 57 |
| 58 console.log("Comparing " + version0 + " with " + version1 + |
| 59 " on Node " + process.version); |
| 60 console.log("--------------------------------------"); |
| 61 |
| 62 |
| 63 async.eachSeries(versionNames, cloneVersion, function (err) { |
| 64 if (err) { throw err; } |
| 65 versions = versionNames.map(requireVersion); |
| 66 |
| 67 var suites = suiteConfigs |
| 68 .map(setDefaultOptions) |
| 69 .reduce(handleMultipleArgs, []) |
| 70 .map(setName) |
| 71 .filter(matchesGrep) |
| 72 .filter(doesNotMatch) |
| 73 .map(createSuite); |
| 74 |
| 75 async.eachSeries(suites, runSuite, function () { |
| 76 var totalTime0 = +totalTime[version0].toPrecision(3); |
| 77 var totalTime1 = +totalTime[version1].toPrecision(3); |
| 78 |
| 79 var wins0 = wins[version0]; |
| 80 var wins1 = wins[version1]; |
| 81 |
| 82 if ( Math.abs((totalTime0 / totalTime1) - 1) < 0.01) { |
| 83 // if < 1% difference, we're likely within the margins of error |
| 84 console.log("Both versions are about equal " + |
| 85 "(" + totalTime0 + "ms total vs. " + totalTime1 + "ms total)"); |
| 86 } else if (totalTime0 < totalTime1) { |
| 87 console.log(version0 + " faster overall " + |
| 88 "(" + totalTime0 + "ms total vs. " + totalTime1 + "ms total)"); |
| 89 } else if (totalTime1 < totalTime0) { |
| 90 console.log(version1 + " faster overall " + |
| 91 "(" + totalTime1 + "ms total vs. " + totalTime0 + "ms total)"); |
| 92 } |
| 93 |
| 94 if (wins0 > wins1) { |
| 95 console.log(version0 + " won more benchmarks " + |
| 96 "(" + wins0 + " vs. " + wins1 + ")"); |
| 97 } else if (wins1 > wins0) { |
| 98 console.log(version1 + " won more benchmarks " + |
| 99 "(" + wins1 + " vs. " + wins0 + ")"); |
| 100 } else { |
| 101 console.log("Both versions won the same number of benchmarks " + |
| 102 "(" + wins0 + " vs. " + wins1 + ")"); |
| 103 } |
| 104 }); |
| 105 }); |
| 106 |
| 107 function runSuite(suite, callback) { |
| 108 suite.on("complete", function () { |
| 109 callback(); |
| 110 }).run({async: true}); |
| 111 } |
| 112 |
| 113 function setDefaultOptions(suiteConfig) { |
| 114 suiteConfig.args = suiteConfig.args || [[]]; |
| 115 suiteConfig.setup = suiteConfig.setup || function () {}; |
| 116 return suiteConfig; |
| 117 } |
| 118 |
| 119 function handleMultipleArgs(list, suiteConfig) { |
| 120 return list.concat(suiteConfig.args.map(function (args) { |
| 121 return _.defaults({args: args}, suiteConfig); |
| 122 })); |
| 123 } |
| 124 |
| 125 function setName(suiteConfig) { |
| 126 suiteConfig.name = suiteConfig.name + "(" + suiteConfig.args.join(",") + ")"
; |
| 127 return suiteConfig; |
| 128 } |
| 129 |
| 130 function matchesGrep(suiteConfig) { |
| 131 return !!grep.exec(suiteConfig.name); |
| 132 } |
| 133 |
| 134 function doesNotMatch(suiteConfig) { |
| 135 return !reject.exec(suiteConfig.name); |
| 136 } |
| 137 |
| 138 function createSuite(suiteConfig) { |
| 139 var suite = new Benchmark.Suite(); |
| 140 var args = suiteConfig.args; |
| 141 var errored = false; |
| 142 |
| 143 function addBench(version, versionName) { |
| 144 var name = suiteConfig.name + " " + versionName; |
| 145 |
| 146 try { |
| 147 suiteConfig.setup(1); |
| 148 suiteConfig.fn(version, function () {}); |
| 149 } catch (e) { |
| 150 console.error(name + " Errored"); |
| 151 errored = true; |
| 152 return; |
| 153 } |
| 154 |
| 155 suite.add(name, function (deferred) { |
| 156 suiteConfig.fn(version, function () { |
| 157 deferred.resolve(); |
| 158 }); |
| 159 }, _.extend({ |
| 160 versionName: versionName, |
| 161 setup: _.partial.apply(null, [suiteConfig.setup].concat(args)), |
| 162 onError: function (err) { |
| 163 console.log(err.stack); |
| 164 } |
| 165 }, benchOptions)); |
| 166 } |
| 167 |
| 168 addBench(versions[0], versionNames[0]); |
| 169 addBench(versions[1], versionNames[1]); |
| 170 |
| 171 |
| 172 return suite.on('cycle', function(event) { |
| 173 var mean = event.target.stats.mean * 1000; |
| 174 console.log(event.target + ", " + mean.toPrecision(3) + "ms per run"); |
| 175 var version = event.target.options.versionName; |
| 176 if (errored) return; |
| 177 totalTime[version] += mean; |
| 178 }) |
| 179 .on('error', function (err) { console.error(err); }) |
| 180 .on('complete', function() { |
| 181 if (!errored) { |
| 182 var fastest = this.filter('fastest'); |
| 183 if (fastest.length === 2) { |
| 184 console.log("Tie"); |
| 185 } else { |
| 186 var winner = fastest[0].options.versionName; |
| 187 console.log(winner + ' is faster'); |
| 188 wins[winner]++; |
| 189 } |
| 190 } |
| 191 console.log("--------------------------------------"); |
| 192 }); |
| 193 |
| 194 } |
| 195 |
| 196 function requireVersion(tag) { |
| 197 if (tag === "current") { |
| 198 return async; |
| 199 } |
| 200 |
| 201 return require("./versions/" + tag + "/"); |
| 202 } |
| 203 |
| 204 function cloneVersion(tag, callback) { |
| 205 if (tag === "current") return callback(); |
| 206 |
| 207 var versionDir = __dirname + "/versions/" + tag; |
| 208 mkdirp.sync(versionDir); |
| 209 fs.open(versionDir + "/package.json", "r", function (err, handle) { |
| 210 if (!err) { |
| 211 // version has already been cloned |
| 212 fs.close(handle); |
| 213 return callback(); |
| 214 } |
| 215 |
| 216 var repoPath = path.join(__dirname, ".."); |
| 217 |
| 218 var cmd = "git clone --branch " + tag + " " + repoPath + " " + versionDi
r; |
| 219 |
| 220 exec(cmd, function (err) { |
| 221 if (err) { |
| 222 throw err; |
| 223 } |
| 224 callback(); |
| 225 }); |
| 226 |
| 227 }); |
| 228 } |
| OLD | NEW |