Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(1879)

Side by Side Diff: tools/gardening/lib/src/luci_api.dart

Issue 3005443002: Additional tools for gardening. (Closed)
Patch Set: Added changes from johnniwinther Created 3 years, 3 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « tools/gardening/lib/src/logger.dart ('k') | tools/gardening/lib/src/luci_services.dart » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file.
4
5 import 'dart:io';
6 import 'dart:async';
7 import 'dart:convert';
8 import 'package:http/http.dart' as http;
9 import 'package:html/parser.dart' show parse;
10 import 'try.dart';
11 import 'cache_new.dart';
12
13 const String LUCI_HOST = "luci-milo.appspot.com";
14
15 typedef void ModifyRequestFunction(HttpClientRequest request);
16
17 /// Base class for communicating with [Luci]
18 /// Some information is found through the api
19 /// <https://docs.google.com/document/d/1HbPp7Sy7ofC
20 /// U7C9USqcE91VubGg_IIET2GUj9iknev4/edit#>
21 /// and some information is found via screen-scraping.
22 class LuciApi {
23 final HttpClient _client = new HttpClient();
24
25 LuciApi();
26
27 /// [getBuildBots] fetches all build bots from luci (we cannot
28 /// get this from the api). The format is:
29 /// <li>
30 /// <a href="/buildbot/client.crashpad/crashpad_win_x86_wow64_rel">
31 /// crashpad_win_x86_wow64_rel</a>
32 /// </li>
33 /// <h3> client.dart </h3>
34 /// <li>
35 /// <a href="/buildbot/client.dart/analyze-linux-be">analyze-linux-be</a>
36 /// </li>
37 /// <li>
38 /// <a href="/buildbot/client.dart/analyze-linux-stable">
39 /// analyze-linux-stable</a>
40 /// </li>
41 /// <li>
42 /// <a href="/buildbot/client.dart/analyzer-linux-release-be">
43 /// analyzer-linux-release-be</a>
44 /// </li>
45 ///
46 /// We look for the section header matching clients, then
47 /// if we are in the right section, we take the <li> element
48 /// and transform to a build bot
49 ///
50 Future<Try<List<LuciBuildBot>>> getAllBuildBots(
51 String client, WithCacheFunction withCache) async {
52 return await tryStartAsync(() => withCache(
53 () => _makeGetRequest(
54 new Uri(scheme: 'https', host: LUCI_HOST, path: "/")),
55 "all_buildbots"))
56 .then((Try<String> tryRes) => tryRes.bind(parse).bind((htmlDoc) {
57 // This is really dirty, but the structure of
58 // the document is not really suited for anything else.
59 var takeSection = false;
60 return htmlDoc.body.children.where((node) {
61 if (node.localName == "li") return takeSection;
62 if (node.localName != "h3") {
63 takeSection = false;
64 return false;
65 }
66 // Current node is <h3>.
67 takeSection = client == node.text.trim();
68 return false;
69 });
70 }).bind((elements) {
71 // Here we hold an iterable of buildbot elements
72 // <li>
73 // <a href="/buildbot/client.dart/analyzer-linux-release-be">
74 // analyzer-linux-release-be</a>
75 // </li>
76 return elements.map((element) {
77 var name = element.children[0].text;
78 var url = element.children[0].attributes['href'];
79 return new LuciBuildBot(client, name, url);
80 }).toList();
81 }));
82 }
83
84 /// [getPrimaryBuilders] fetches all primary builders
85 /// (the ones usually used by gardeners) by not including buildbots with
86 /// the name -dev, -stable or -integration.
87 Future<Try<List<LuciBuildBot>>> getPrimaryBuilders(
88 String client, WithCacheFunction withCache) async {
89 return await getAllBuildBots(client, withCache)
90 .then((Try<List<LuciBuildBot>> tryRes) {
91 return tryRes
92 .bind((buildBots) => buildBots.where((LuciBuildBot buildBot) {
93 return !(buildBot.name.contains("-dev") ||
94 buildBot.name.contains("-stable") ||
95 buildBot.name.contains("-integration"));
96 }));
97 });
98 }
99
100 /// Calling the Milo Api to get latest builds for this bot,
101 /// where the field [amount] is the number of recent builds to fetch.
102 Future<Try<List<BuildDetail>>> getBuildBotDetails(
103 String client, String botName, WithCacheFunction withCache,
104 [int amount = 20]) async {
105 var uri = new Uri(
106 scheme: "https",
107 host: LUCI_HOST,
108 path: "prpc/milo.Buildbot/GetBuildbotBuildsJSON");
109 var body = {
110 "master": client,
111 "builder": botName,
112 "limit": amount,
113 "includeCurrent": true
114 };
115 var result = await tryStartAsync(() => withCache(
116 () => _makePostRequest(uri, JSON.encode(body), {
117 HttpHeaders.CONTENT_TYPE: "application/json",
118 HttpHeaders.ACCEPT: "application/json"
119 }),
120 '${uri.path}_${botName}_$amount'));
121 return result.bind(JSON.decode).bind((json) {
122 return json["builds"].map((b) {
123 var build = JSON.decode(UTF8.decode(BASE64.decode(b["data"])));
124 return getBuildDetailFromJson(client, botName, build);
125 }).toList();
126 });
127 }
128
129 /// Calling the Milo Api to get information about a specific build
130 /// where the field [buildNumber] is the build number to fetch.
131 Future<Try<BuildDetail>> getBuildBotBuildDetails(String client,
132 String botName, int buildNumber, WithCacheFunction withCache) async {
133 var uri = new Uri(
134 scheme: "https",
135 host: LUCI_HOST,
136 path: "prpc/milo.Buildbot/GetBuildbotBuildJSON");
137 var body = {"master": client, "builder": botName, "buildNum": buildNumber};
138 print(body);
139 var result = await tryStartAsync(() => withCache(
140 () => _makePostRequest(uri, JSON.encode(body), {
141 HttpHeaders.CONTENT_TYPE: "application/json",
142 HttpHeaders.ACCEPT: "application/json"
143 }),
144 '${uri.path}_${botName}_$buildNumber'));
145 return result.bind(JSON.decode).bind((json) {
146 var build = JSON.decode(UTF8.decode(BASE64.decode(json["data"])));
147 return getBuildDetailFromJson(client, botName, build);
148 });
149 }
150
151 /// [_makeGetRequest] performs a get request to [uri].
152 Future<String> _makeGetRequest(Uri uri) async {
153 var request = await _client.getUrl(uri);
154 var response = await request.close();
155 if (response.statusCode != 200) {
156 response.drain();
157 throw new HttpException(response.reasonPhrase, uri: uri);
158 }
159 return response.transform(UTF8.decoder).join();
160 }
161
162 /// [_makeGetRequest] performs a post request to [uri], where the posted
163 /// body is the string representation of [body]. For adding custom headers
164 /// use the map [headers].
165 Future<String> _makePostRequest(
166 Uri uri, Object body, Map<String, String> headers) async {
167 var response = await http.post(uri, body: body, headers: headers);
168 if (response.statusCode != 200) {
169 throw new HttpException(response.reasonPhrase, uri: uri);
170 }
171 // Prpc outputs a prefix to combat vulnerability.
172 if (response.body.startsWith(")]}'")) {
173 return response.body.substring(4);
174 }
175 return response.body;
176 }
177
178 /// Closes the Http client connection
179 void close() {
180 _client.close();
181 }
182 }
183
184 /// [getBuildDetailFromJson] parses json [build] to a class [BuildDetail]
185 BuildDetail getBuildDetailFromJson(
186 String client, String botName, dynamic build) {
187 List<GitCommit> changes = build["sourceStamp"]["changes"].map((change) {
188 return new GitCommit(change["revision"], change["revLink"], change["who"],
189 change["comments"], change["files"].map((file) => file["name"]));
190 }).toList();
191
192 List<BuildProperty> properties = build["properties"].map((prop) {
193 return new BuildProperty(prop[0], prop[1].toString(), prop[2]);
194 }).toList();
195
196 List<BuildStep> steps = build["steps"].map((step) {
197 var start =
198 new DateTime.fromMillisecondsSinceEpoch(step["times"][0] * 1000);
199 DateTime end = null;
200 if (step["times"][1] != null) {
201 end = new DateTime.fromMillisecondsSinceEpoch(step["times"][1] * 1000);
202 }
203 return new BuildStep(
204 step["name"],
205 step["text"],
206 step["results"].toString(),
207 start,
208 end,
209 step["step_number"],
210 step["isStarted"],
211 step["isFinished"],
212 step["logs"].map((log) => new BuildLog(log[0], log[1])));
213 }).toList();
214
215 DateTime end = null;
216 if (build["times"][1] != null) {
217 end = new DateTime.fromMillisecondsSinceEpoch(build["times"][1] * 1000);
218 }
219
220 Timing timing = new Timing(
221 new DateTime.fromMillisecondsSinceEpoch(build["times"][0] * 1000), end);
222
223 return new BuildDetail(
224 client,
225 botName,
226 build["number"],
227 build["text"].join(' '),
228 build["finished"],
229 steps,
230 properties,
231 build["blame"],
232 timing,
233 changes);
234 }
235
236 // Structured classes to relay information from api and web pages
237
238 /// [LuciBuildBot] holds information about a build bot
239 class LuciBuildBot {
240 final String client;
241 final String name;
242 final String url;
243
244 LuciBuildBot(this.client, this.name, this.url);
245
246 @override
247 String toString() {
248 return "LuciBuildBot { client: $client, name: $name, url: $url }";
249 }
250 }
251
252 /// [BuildDetail] holds data detailing a specific build
253 class BuildDetail {
254 final String client;
255 final String botName;
256 final int buildNumber;
257 final String results;
258 final bool finished;
259 final List<BuildStep> steps;
260 final List<BuildProperty> buildProperties;
261 final List<String> blameList;
262 final Timing timing;
263 final List<GitCommit> allChanges;
264
265 BuildDetail(
266 this.client,
267 this.botName,
268 this.buildNumber,
269 this.results,
270 this.finished,
271 this.steps,
272 this.buildProperties,
273 this.blameList,
274 this.timing,
275 this.allChanges);
276
277 @override
278 String toString() {
279 StringBuffer buffer = new StringBuffer();
280 buffer.writeln("--------------------------------------");
281 buffer.writeln(results);
282 buffer.writeln(timing);
283 buffer.writeln("----------------STEPS-----------------");
284 if (steps != null) steps.forEach(buffer.writeln);
285 buffer.writeln("----------BUILD PROPERTIES------------");
286 if (buildProperties != null) buildProperties.forEach(buffer.writeln);
287 buffer.writeln("-------------BLAME LIST---------------");
288 if (blameList != null) blameList.forEach(buffer.writeln);
289 buffer.writeln("------------ALL CHANGES---------------");
290 if (allChanges != null) allChanges.forEach(buffer.writeln);
291 return buffer.toString();
292 }
293 }
294
295 /// [BuildStep] holds data detailing a specific build
296 class BuildStep {
297 final String name;
298 final String description;
299 final String result;
300 final DateTime start;
301 final DateTime end;
302 final int number;
303 final bool isStarted;
304 final bool isFinished;
305 final List<BuildLog> logs;
306
307 BuildStep(this.name, this.description, this.result, this.start, this.end,
308 this.number, this.isStarted, this.isFinished, this.logs);
309
310 @override
311 String toString() {
312 StringBuffer buffer = new StringBuffer();
313 buffer.writeln("${result == '[0, []]' ? 'SUCCESS' : result}: "
314 "$name - $description ($start, $end)");
315 logs.forEach((subLink) {
316 buffer.writeln("\t${subLink}");
317 });
318 return buffer.toString();
319 }
320 }
321
322 /// [BuildLog] holds log-information for a specific build.
323 class BuildLog {
324 final String name;
325 final String url;
326
327 BuildLog(this.name, this.url);
328
329 @override
330 String toString() {
331 return "$name | $url";
332 }
333 }
334
335 /// [BuildProperty] descibes build properties of a specific build.
336 class BuildProperty {
337 final String name;
338 final String value;
339 final String source;
340
341 BuildProperty(this.name, this.value, this.source);
342
343 @override
344 String toString() {
345 return "$name\t$value\t$source";
346 }
347 }
348
349 /// [Timing] is a class to hold timing information for builds and steps.
350 class Timing {
351 final DateTime start;
352 final DateTime end;
353
354 Timing(this.start, this.end);
355
356 @override
357 String toString() {
358 return "start: $start\tend: $end";
359 }
360 }
361
362 /// [GitCommit] holds data about a specific commit.
363 class GitCommit {
364 final String revision;
365 final String commitUrl;
366 final String changedBy;
367 final String comments;
368 final List<String> changedFiles;
369
370 GitCommit(this.revision, this.commitUrl, this.changedBy, this.comments,
371 this.changedFiles);
372
373 @override
374 String toString() {
375 StringBuffer buffer = new StringBuffer();
376 buffer.writeln("revision: $revision");
377 buffer.writeln("commitUrl: $commitUrl");
378 buffer.writeln("changedBy: $changedBy");
379 buffer.write("\n");
380 buffer.writeln(comments);
381 buffer.write("\nfiles:\n");
382 changedFiles.forEach(buffer.writeln);
383 return buffer.toString();
384 }
385 }
OLDNEW
« no previous file with comments | « tools/gardening/lib/src/logger.dart ('k') | tools/gardening/lib/src/luci_services.dart » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698