OLD | NEW |
1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 import 'dart:io'; | 5 import 'dart:io'; |
6 import 'dart:async'; | 6 import 'dart:async'; |
7 import 'dart:convert'; | 7 import 'dart:convert'; |
8 import 'package:http/http.dart' as http; | 8 import 'package:http/http.dart' as http; |
9 import 'package:html/parser.dart' show parse; | 9 import 'package:archive/archive.dart'; |
10 import 'try.dart'; | 10 import 'try.dart'; |
11 import 'cache_new.dart'; | 11 import 'cache_new.dart'; |
12 | 12 |
13 const String LUCI_HOST = "luci-milo.appspot.com"; | 13 const String LUCI_HOST = "luci-milo.appspot.com"; |
| 14 const String CBE_HOST = "chrome-build-extract.appspot.com"; |
14 | 15 |
15 typedef void ModifyRequestFunction(HttpClientRequest request); | 16 typedef void ModifyRequestFunction(HttpClientRequest request); |
16 | 17 |
17 /// Base class for communicating with [Luci] | 18 /// Base class for communicating with [Luci] |
18 /// Some information is found through the api | 19 /// Some information is found through the api |
19 /// <https://docs.google.com/document/d/1HbPp7Sy7ofC | 20 /// <https://luci-milo.appspot.com/rpcexplorer/services/milo.Buildbot/>, |
20 /// U7C9USqcE91VubGg_IIET2GUj9iknev4/edit#> | 21 /// some information is found via Cbe |
21 /// and some information is found via screen-scraping. | 22 /// <https://chrome-build-extract.appspot.com/get_master/<client>>. |
22 class LuciApi { | 23 class Luci { |
23 final HttpClient _client = new HttpClient(); | 24 final HttpClient _client = new HttpClient(); |
24 | 25 |
25 LuciApi(); | 26 Luci(); |
26 | 27 |
27 /// [getBuildBots] fetches all build bots from luci (we cannot | 28 /// [getJsonFromChromeBuildExtract] gets json from Cbe, with information |
28 /// get this from the api). The format is: | 29 /// about all bots and current builds. |
29 /// <li> | 30 Future<Try<dynamic>> getJsonFromChromeBuildExtract( |
30 /// <a href="/buildbot/client.crashpad/crashpad_win_x86_wow64_rel"> | |
31 /// crashpad_win_x86_wow64_rel</a> | |
32 /// </li> | |
33 /// <h3> client.dart </h3> | |
34 /// <li> | |
35 /// <a href="/buildbot/client.dart/analyze-linux-be">analyze-linux-be</a> | |
36 /// </li> | |
37 /// <li> | |
38 /// <a href="/buildbot/client.dart/analyze-linux-stable"> | |
39 /// analyze-linux-stable</a> | |
40 /// </li> | |
41 /// <li> | |
42 /// <a href="/buildbot/client.dart/analyzer-linux-release-be"> | |
43 /// analyzer-linux-release-be</a> | |
44 /// </li> | |
45 /// | |
46 /// We look for the section header matching clients, then | |
47 /// if we are in the right section, we take the <li> element | |
48 /// and transform to a build bot | |
49 /// | |
50 Future<Try<List<LuciBuildBot>>> getAllBuildBots( | |
51 String client, WithCacheFunction withCache) async { | 31 String client, WithCacheFunction withCache) async { |
52 return await tryStartAsync(() => withCache( | 32 var result = await tryStartAsync(() => withCache( |
53 () => _makeGetRequest( | 33 () => _makeGetRequest(new Uri( |
54 new Uri(scheme: 'https', host: LUCI_HOST, path: "/")), | 34 scheme: 'https', host: CBE_HOST, path: "/get_master/${client}")), |
55 "all_buildbots")) | 35 "cbe")); |
56 .then((Try<String> tryRes) => tryRes.bind(parse).bind((htmlDoc) { | 36 return result.bind(JSON.decode); |
57 // This is really dirty, but the structure of | |
58 // the document is not really suited for anything else. | |
59 var takeSection = false; | |
60 return htmlDoc.body.children.where((node) { | |
61 if (node.localName == "li") return takeSection; | |
62 if (node.localName != "h3") { | |
63 takeSection = false; | |
64 return false; | |
65 } | |
66 // Current node is <h3>. | |
67 takeSection = client == node.text.trim(); | |
68 return false; | |
69 }); | |
70 }).bind((elements) { | |
71 // Here we hold an iterable of buildbot elements | |
72 // <li> | |
73 // <a href="/buildbot/client.dart/analyzer-linux-release-be"> | |
74 // analyzer-linux-release-be</a> | |
75 // </li> | |
76 return elements.map((element) { | |
77 var name = element.children[0].text; | |
78 var url = element.children[0].attributes['href']; | |
79 return new LuciBuildBot(client, name, url); | |
80 }).toList(); | |
81 })); | |
82 } | 37 } |
83 | 38 |
84 /// [getPrimaryBuilders] fetches all primary builders | 39 /// [getMaster] fetches master information for all bots. |
85 /// (the ones usually used by gardeners) by not including buildbots with | 40 Future<Try<Object>> getMaster( |
86 /// the name -dev, -stable or -integration. | |
87 Future<Try<List<LuciBuildBot>>> getPrimaryBuilders( | |
88 String client, WithCacheFunction withCache) async { | 41 String client, WithCacheFunction withCache) async { |
89 return await getAllBuildBots(client, withCache) | 42 var uri = new Uri( |
90 .then((Try<List<LuciBuildBot>> tryRes) { | 43 scheme: "https", |
91 return tryRes | 44 host: LUCI_HOST, |
92 .bind((buildBots) => buildBots.where((LuciBuildBot buildBot) { | 45 path: "prpc/milo.Buildbot/GetCompressedMasterJSON"); |
93 return !(buildBot.name.contains("-dev") || | 46 var body = {"name": client}; |
94 buildBot.name.contains("-stable") || | 47 var result = await tryStartAsync(() => withCache( |
95 buildBot.name.contains("-integration")); | 48 () => _makePostRequest(uri, JSON.encode(body), { |
96 })); | 49 HttpHeaders.CONTENT_TYPE: "application/json", |
| 50 HttpHeaders.ACCEPT: "application/json" |
| 51 }), |
| 52 '${uri.path}')); |
| 53 return result.bind(JSON.decode).bind((json) { |
| 54 var data = JSON.decode(UTF8 |
| 55 .decode(new GZipDecoder().decodeBytes(BASE64.decode(json["data"])))); |
| 56 return data; |
97 }); | 57 }); |
98 } | 58 } |
99 | 59 |
100 /// Calling the Milo Api to get latest builds for this bot, | 60 /// Calling the Milo Api to get latest builds for this bot, |
101 /// where the field [amount] is the number of recent builds to fetch. | 61 /// where the field [amount] is the number of recent builds to fetch. |
102 Future<Try<List<BuildDetail>>> getBuildBotDetails( | 62 Future<Try<List<BuildDetail>>> getBuildBotDetails( |
103 String client, String botName, WithCacheFunction withCache, | 63 String client, String botName, WithCacheFunction withCache, |
104 [int amount = 20]) async { | 64 [int amount = 20]) async { |
105 var uri = new Uri( | 65 var uri = new Uri( |
106 scheme: "https", | 66 scheme: "https", |
(...skipping 119 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
226 build["number"], | 186 build["number"], |
227 build["text"].join(' '), | 187 build["text"].join(' '), |
228 build["finished"], | 188 build["finished"], |
229 steps, | 189 steps, |
230 properties, | 190 properties, |
231 build["blame"], | 191 build["blame"], |
232 timing, | 192 timing, |
233 changes); | 193 changes); |
234 } | 194 } |
235 | 195 |
236 // Structured classes to relay information from api and web pages | |
237 | |
238 /// [LuciBuildBot] holds information about a build bot | |
239 class LuciBuildBot { | |
240 final String client; | |
241 final String name; | |
242 final String url; | |
243 | |
244 LuciBuildBot(this.client, this.name, this.url); | |
245 | |
246 @override | |
247 String toString() { | |
248 return "LuciBuildBot { client: $client, name: $name, url: $url }"; | |
249 } | |
250 } | |
251 | |
252 /// [BuildDetail] holds data detailing a specific build | 196 /// [BuildDetail] holds data detailing a specific build |
253 class BuildDetail { | 197 class BuildDetail { |
254 final String client; | 198 final String client; |
255 final String botName; | 199 final String botName; |
256 final int buildNumber; | 200 final int buildNumber; |
257 final String results; | 201 final String results; |
258 final bool finished; | 202 final bool finished; |
259 final List<BuildStep> steps; | 203 final List<BuildStep> steps; |
260 final List<BuildProperty> buildProperties; | 204 final List<BuildProperty> buildProperties; |
261 final List<String> blameList; | 205 final List<String> blameList; |
(...skipping 114 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
376 buffer.writeln("revision: $revision"); | 320 buffer.writeln("revision: $revision"); |
377 buffer.writeln("commitUrl: $commitUrl"); | 321 buffer.writeln("commitUrl: $commitUrl"); |
378 buffer.writeln("changedBy: $changedBy"); | 322 buffer.writeln("changedBy: $changedBy"); |
379 buffer.write("\n"); | 323 buffer.write("\n"); |
380 buffer.writeln(comments); | 324 buffer.writeln(comments); |
381 buffer.write("\nfiles:\n"); | 325 buffer.write("\nfiles:\n"); |
382 changedFiles.forEach(buffer.writeln); | 326 changedFiles.forEach(buffer.writeln); |
383 return buffer.toString(); | 327 return buffer.toString(); |
384 } | 328 } |
385 } | 329 } |
OLD | NEW |