| OLD | NEW |
|---|
| (Empty) |
| 1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file | |
| 2 // for details. All rights reserved. Use of this source code is governed by a | |
| 3 // BSD-style license that can be found in the LICENSE file. | |
| 4 | |
| 5 import 'dart:io'; | |
| 6 import 'dart:async'; | |
| 7 import 'dart:convert'; | |
| 8 import 'package:http/http.dart' as http; | |
| 9 import 'package:html/parser.dart' show parse; | |
| 10 import 'try.dart'; | |
| 11 import 'cache_new.dart'; | |
| 12 | |
| 13 const String LUCI_HOST = "luci-milo.appspot.com"; | |
| 14 | |
| 15 typedef void ModifyRequestFunction(HttpClientRequest request); | |
| 16 | |
| 17 /// Base class for communicating with [Luci] | |
| 18 /// Some information is found through the api | |
| 19 /// <https://docs.google.com/document/d/1HbPp7Sy7ofC | |
| 20 /// U7C9USqcE91VubGg_IIET2GUj9iknev4/edit#> | |
| 21 /// and some information is found via screen-scraping. | |
| 22 class LuciApi { | |
| 23 final HttpClient _client = new HttpClient(); | |
| 24 | |
| 25 LuciApi(); | |
| 26 | |
| 27 /// [getBuildBots] fetches all build bots from luci (we cannot | |
| 28 /// get this from the api). The format is: | |
| 29 /// <li> | |
| 30 /// <a href="/buildbot/client.crashpad/crashpad_win_x86_wow64_rel"> | |
| 31 /// crashpad_win_x86_wow64_rel</a> | |
| 32 /// </li> | |
| 33 /// <h3> client.dart </h3> | |
| 34 /// <li> | |
| 35 /// <a href="/buildbot/client.dart/analyze-linux-be">analyze-linux-be</a> | |
| 36 /// </li> | |
| 37 /// <li> | |
| 38 /// <a href="/buildbot/client.dart/analyze-linux-stable"> | |
| 39 /// analyze-linux-stable</a> | |
| 40 /// </li> | |
| 41 /// <li> | |
| 42 /// <a href="/buildbot/client.dart/analyzer-linux-release-be"> | |
| 43 /// analyzer-linux-release-be</a> | |
| 44 /// </li> | |
| 45 /// | |
| 46 /// We look for the section header matching clients, then | |
| 47 /// if we are in the right section, we take the <li> element | |
| 48 /// and transform to a build bot | |
| 49 /// | |
| 50 Future<Try<List<LuciBuildBot>>> getAllBuildBots( | |
| 51 String client, WithCacheFunction withCache) async { | |
| 52 return await tryStartAsync(() => withCache( | |
| 53 () => _makeGetRequest( | |
| 54 new Uri(scheme: 'https', host: LUCI_HOST, path: "/")), | |
| 55 "all_buildbots")) | |
| 56 .then((Try<String> tryRes) => tryRes.bind(parse).bind((htmlDoc) { | |
| 57 // This is really dirty, but the structure of | |
| 58 // the document is not really suited for anything else. | |
| 59 var takeSection = false; | |
| 60 return htmlDoc.body.children.where((node) { | |
| 61 if (node.localName == "li") return takeSection; | |
| 62 if (node.localName != "h3") { | |
| 63 takeSection = false; | |
| 64 return false; | |
| 65 } | |
| 66 // Current node is <h3>. | |
| 67 takeSection = client == node.text.trim(); | |
| 68 return false; | |
| 69 }); | |
| 70 }).bind((elements) { | |
| 71 // Here we hold an iterable of buildbot elements | |
| 72 // <li> | |
| 73 // <a href="/buildbot/client.dart/analyzer-linux-release-be"> | |
| 74 // analyzer-linux-release-be</a> | |
| 75 // </li> | |
| 76 return elements.map((element) { | |
| 77 var name = element.children[0].text; | |
| 78 var url = element.children[0].attributes['href']; | |
| 79 return new LuciBuildBot(client, name, url); | |
| 80 }).toList(); | |
| 81 })); | |
| 82 } | |
| 83 | |
| 84 /// [getPrimaryBuilders] fetches all primary builders | |
| 85 /// (the ones usually used by gardeners) by not including buildbots with | |
| 86 /// the name -dev, -stable or -integration. | |
| 87 Future<Try<List<LuciBuildBot>>> getPrimaryBuilders( | |
| 88 String client, WithCacheFunction withCache) async { | |
| 89 return await getAllBuildBots(client, withCache) | |
| 90 .then((Try<List<LuciBuildBot>> tryRes) { | |
| 91 return tryRes | |
| 92 .bind((buildBots) => buildBots.where((LuciBuildBot buildBot) { | |
| 93 return !(buildBot.name.contains("-dev") || | |
| 94 buildBot.name.contains("-stable") || | |
| 95 buildBot.name.contains("-integration")); | |
| 96 })); | |
| 97 }); | |
| 98 } | |
| 99 | |
| 100 /// Calling the Milo Api to get latest builds for this bot, | |
| 101 /// where the field [amount] is the number of recent builds to fetch. | |
| 102 Future<Try<List<BuildDetail>>> getBuildBotDetails( | |
| 103 String client, String botName, WithCacheFunction withCache, | |
| 104 [int amount = 20]) async { | |
| 105 var uri = new Uri( | |
| 106 scheme: "https", | |
| 107 host: LUCI_HOST, | |
| 108 path: "prpc/milo.Buildbot/GetBuildbotBuildsJSON"); | |
| 109 var body = { | |
| 110 "master": client, | |
| 111 "builder": botName, | |
| 112 "limit": amount, | |
| 113 "includeCurrent": true | |
| 114 }; | |
| 115 var result = await tryStartAsync(() => withCache( | |
| 116 () => _makePostRequest(uri, JSON.encode(body), { | |
| 117 HttpHeaders.CONTENT_TYPE: "application/json", | |
| 118 HttpHeaders.ACCEPT: "application/json" | |
| 119 }), | |
| 120 '${uri.path}_${botName}_$amount')); | |
| 121 return result.bind(JSON.decode).bind((json) { | |
| 122 return json["builds"].map((b) { | |
| 123 var build = JSON.decode(UTF8.decode(BASE64.decode(b["data"]))); | |
| 124 return getBuildDetailFromJson(client, botName, build); | |
| 125 }).toList(); | |
| 126 }); | |
| 127 } | |
| 128 | |
| 129 /// Calling the Milo Api to get information about a specific build | |
| 130 /// where the field [buildNumber] is the build number to fetch. | |
| 131 Future<Try<BuildDetail>> getBuildBotBuildDetails(String client, | |
| 132 String botName, int buildNumber, WithCacheFunction withCache) async { | |
| 133 var uri = new Uri( | |
| 134 scheme: "https", | |
| 135 host: LUCI_HOST, | |
| 136 path: "prpc/milo.Buildbot/GetBuildbotBuildJSON"); | |
| 137 var body = {"master": client, "builder": botName, "buildNum": buildNumber}; | |
| 138 print(body); | |
| 139 var result = await tryStartAsync(() => withCache( | |
| 140 () => _makePostRequest(uri, JSON.encode(body), { | |
| 141 HttpHeaders.CONTENT_TYPE: "application/json", | |
| 142 HttpHeaders.ACCEPT: "application/json" | |
| 143 }), | |
| 144 '${uri.path}_${botName}_$buildNumber')); | |
| 145 return result.bind(JSON.decode).bind((json) { | |
| 146 var build = JSON.decode(UTF8.decode(BASE64.decode(json["data"]))); | |
| 147 return getBuildDetailFromJson(client, botName, build); | |
| 148 }); | |
| 149 } | |
| 150 | |
| 151 /// [_makeGetRequest] performs a get request to [uri]. | |
| 152 Future<String> _makeGetRequest(Uri uri) async { | |
| 153 var request = await _client.getUrl(uri); | |
| 154 var response = await request.close(); | |
| 155 if (response.statusCode != 200) { | |
| 156 response.drain(); | |
| 157 throw new HttpException(response.reasonPhrase, uri: uri); | |
| 158 } | |
| 159 return response.transform(UTF8.decoder).join(); | |
| 160 } | |
| 161 | |
| 162 /// [_makeGetRequest] performs a post request to [uri], where the posted | |
| 163 /// body is the string representation of [body]. For adding custom headers | |
| 164 /// use the map [headers]. | |
| 165 Future<String> _makePostRequest( | |
| 166 Uri uri, Object body, Map<String, String> headers) async { | |
| 167 var response = await http.post(uri, body: body, headers: headers); | |
| 168 if (response.statusCode != 200) { | |
| 169 throw new HttpException(response.reasonPhrase, uri: uri); | |
| 170 } | |
| 171 // Prpc outputs a prefix to combat vulnerability. | |
| 172 if (response.body.startsWith(")]}'")) { | |
| 173 return response.body.substring(4); | |
| 174 } | |
| 175 return response.body; | |
| 176 } | |
| 177 | |
| 178 /// Closes the Http client connection | |
| 179 void close() { | |
| 180 _client.close(); | |
| 181 } | |
| 182 } | |
| 183 | |
| 184 /// [getBuildDetailFromJson] parses json [build] to a class [BuildDetail] | |
| 185 BuildDetail getBuildDetailFromJson( | |
| 186 String client, String botName, dynamic build) { | |
| 187 List<GitCommit> changes = build["sourceStamp"]["changes"].map((change) { | |
| 188 return new GitCommit(change["revision"], change["revLink"], change["who"], | |
| 189 change["comments"], change["files"].map((file) => file["name"])); | |
| 190 }).toList(); | |
| 191 | |
| 192 List<BuildProperty> properties = build["properties"].map((prop) { | |
| 193 return new BuildProperty(prop[0], prop[1].toString(), prop[2]); | |
| 194 }).toList(); | |
| 195 | |
| 196 List<BuildStep> steps = build["steps"].map((step) { | |
| 197 var start = | |
| 198 new DateTime.fromMillisecondsSinceEpoch(step["times"][0] * 1000); | |
| 199 DateTime end = null; | |
| 200 if (step["times"][1] != null) { | |
| 201 end = new DateTime.fromMillisecondsSinceEpoch(step["times"][1] * 1000); | |
| 202 } | |
| 203 return new BuildStep( | |
| 204 step["name"], | |
| 205 step["text"], | |
| 206 step["results"].toString(), | |
| 207 start, | |
| 208 end, | |
| 209 step["step_number"], | |
| 210 step["isStarted"], | |
| 211 step["isFinished"], | |
| 212 step["logs"].map((log) => new BuildLog(log[0], log[1]))); | |
| 213 }).toList(); | |
| 214 | |
| 215 DateTime end = null; | |
| 216 if (build["times"][1] != null) { | |
| 217 end = new DateTime.fromMillisecondsSinceEpoch(build["times"][1] * 1000); | |
| 218 } | |
| 219 | |
| 220 Timing timing = new Timing( | |
| 221 new DateTime.fromMillisecondsSinceEpoch(build["times"][0] * 1000), end); | |
| 222 | |
| 223 return new BuildDetail( | |
| 224 client, | |
| 225 botName, | |
| 226 build["number"], | |
| 227 build["text"].join(' '), | |
| 228 build["finished"], | |
| 229 steps, | |
| 230 properties, | |
| 231 build["blame"], | |
| 232 timing, | |
| 233 changes); | |
| 234 } | |
| 235 | |
| 236 // Structured classes to relay information from api and web pages | |
| 237 | |
| 238 /// [LuciBuildBot] holds information about a build bot | |
| 239 class LuciBuildBot { | |
| 240 final String client; | |
| 241 final String name; | |
| 242 final String url; | |
| 243 | |
| 244 LuciBuildBot(this.client, this.name, this.url); | |
| 245 | |
| 246 @override | |
| 247 String toString() { | |
| 248 return "LuciBuildBot { client: $client, name: $name, url: $url }"; | |
| 249 } | |
| 250 } | |
| 251 | |
| 252 /// [BuildDetail] holds data detailing a specific build | |
| 253 class BuildDetail { | |
| 254 final String client; | |
| 255 final String botName; | |
| 256 final int buildNumber; | |
| 257 final String results; | |
| 258 final bool finished; | |
| 259 final List<BuildStep> steps; | |
| 260 final List<BuildProperty> buildProperties; | |
| 261 final List<String> blameList; | |
| 262 final Timing timing; | |
| 263 final List<GitCommit> allChanges; | |
| 264 | |
| 265 BuildDetail( | |
| 266 this.client, | |
| 267 this.botName, | |
| 268 this.buildNumber, | |
| 269 this.results, | |
| 270 this.finished, | |
| 271 this.steps, | |
| 272 this.buildProperties, | |
| 273 this.blameList, | |
| 274 this.timing, | |
| 275 this.allChanges); | |
| 276 | |
| 277 @override | |
| 278 String toString() { | |
| 279 StringBuffer buffer = new StringBuffer(); | |
| 280 buffer.writeln("--------------------------------------"); | |
| 281 buffer.writeln(results); | |
| 282 buffer.writeln(timing); | |
| 283 buffer.writeln("----------------STEPS-----------------"); | |
| 284 if (steps != null) steps.forEach(buffer.writeln); | |
| 285 buffer.writeln("----------BUILD PROPERTIES------------"); | |
| 286 if (buildProperties != null) buildProperties.forEach(buffer.writeln); | |
| 287 buffer.writeln("-------------BLAME LIST---------------"); | |
| 288 if (blameList != null) blameList.forEach(buffer.writeln); | |
| 289 buffer.writeln("------------ALL CHANGES---------------"); | |
| 290 if (allChanges != null) allChanges.forEach(buffer.writeln); | |
| 291 return buffer.toString(); | |
| 292 } | |
| 293 } | |
| 294 | |
| 295 /// [BuildStep] holds data detailing a specific build | |
| 296 class BuildStep { | |
| 297 final String name; | |
| 298 final String description; | |
| 299 final String result; | |
| 300 final DateTime start; | |
| 301 final DateTime end; | |
| 302 final int number; | |
| 303 final bool isStarted; | |
| 304 final bool isFinished; | |
| 305 final List<BuildLog> logs; | |
| 306 | |
| 307 BuildStep(this.name, this.description, this.result, this.start, this.end, | |
| 308 this.number, this.isStarted, this.isFinished, this.logs); | |
| 309 | |
| 310 @override | |
| 311 String toString() { | |
| 312 StringBuffer buffer = new StringBuffer(); | |
| 313 buffer.writeln("${result == '[0, []]' ? 'SUCCESS' : result}: " | |
| 314 "$name - $description ($start, $end)"); | |
| 315 logs.forEach((subLink) { | |
| 316 buffer.writeln("\t${subLink}"); | |
| 317 }); | |
| 318 return buffer.toString(); | |
| 319 } | |
| 320 } | |
| 321 | |
| 322 /// [BuildLog] holds log-information for a specific build. | |
| 323 class BuildLog { | |
| 324 final String name; | |
| 325 final String url; | |
| 326 | |
| 327 BuildLog(this.name, this.url); | |
| 328 | |
| 329 @override | |
| 330 String toString() { | |
| 331 return "$name | $url"; | |
| 332 } | |
| 333 } | |
| 334 | |
| 335 /// [BuildProperty] descibes build properties of a specific build. | |
| 336 class BuildProperty { | |
| 337 final String name; | |
| 338 final String value; | |
| 339 final String source; | |
| 340 | |
| 341 BuildProperty(this.name, this.value, this.source); | |
| 342 | |
| 343 @override | |
| 344 String toString() { | |
| 345 return "$name\t$value\t$source"; | |
| 346 } | |
| 347 } | |
| 348 | |
| 349 /// [Timing] is a class to hold timing information for builds and steps. | |
| 350 class Timing { | |
| 351 final DateTime start; | |
| 352 final DateTime end; | |
| 353 | |
| 354 Timing(this.start, this.end); | |
| 355 | |
| 356 @override | |
| 357 String toString() { | |
| 358 return "start: $start\tend: $end"; | |
| 359 } | |
| 360 } | |
| 361 | |
| 362 /// [GitCommit] holds data about a specific commit. | |
| 363 class GitCommit { | |
| 364 final String revision; | |
| 365 final String commitUrl; | |
| 366 final String changedBy; | |
| 367 final String comments; | |
| 368 final List<String> changedFiles; | |
| 369 | |
| 370 GitCommit(this.revision, this.commitUrl, this.changedBy, this.comments, | |
| 371 this.changedFiles); | |
| 372 | |
| 373 @override | |
| 374 String toString() { | |
| 375 StringBuffer buffer = new StringBuffer(); | |
| 376 buffer.writeln("revision: $revision"); | |
| 377 buffer.writeln("commitUrl: $commitUrl"); | |
| 378 buffer.writeln("changedBy: $changedBy"); | |
| 379 buffer.write("\n"); | |
| 380 buffer.writeln(comments); | |
| 381 buffer.write("\nfiles:\n"); | |
| 382 changedFiles.forEach(buffer.writeln); | |
| 383 return buffer.toString(); | |
| 384 } | |
| 385 } | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 3005723002:
Added ability to find builder-groups (Closed)
