Additional tools for gardening.

tools/gardening/lib/src/luci_api.dart

+// Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:io';
+import 'dart:async';
+import 'dart:convert';
+import 'package:http/http.dart' as http;
+import 'package:html/parser.dart' show parse;
+import 'try.dart';
+import 'cache_new.dart';
+
+const String LUCI_HOST = "luci-milo.appspot.com";
+
+typedef void ModifyRequestFunction(HttpClientRequest request);
+
+/// Base class for communicating with [Luci]
+/// Some information is found through the api
+/// <https://docs.google.com/document/d/1HbPp7Sy7ofC
+/// U7C9USqcE91VubGg_IIET2GUj9iknev4/edit#>
+/// and some information is found via screen-scraping.
+class LuciApi {
+  final HttpClient _client = new HttpClient();
+
+  LuciApi();
+
+  /// [getBuildBots] fetches all build bots from luci (we cannot
+  /// get this from the api). The format is:
+  /// <li>
+  ///     <a href="/buildbot/client.crashpad/crashpad_win_x86_wow64_rel">
+  ///        crashpad_win_x86_wow64_rel</a>
+  /// </li>
+  /// <h3> client.dart </h3>
+  /// <li>
+  ///     <a href="/buildbot/client.dart/analyze-linux-be">analyze-linux-be</a>
+  /// </li>
+  /// <li>
+  ///     <a href="/buildbot/client.dart/analyze-linux-stable">
+  ///        analyze-linux-stable</a>
+  /// </li>
+  /// <li>
+  ///     <a href="/buildbot/client.dart/analyzer-linux-release-be">
+  ///        analyzer-linux-release-be</a>
+  /// </li>
+  ///
+  /// We look for the section header matching clients, then
+  /// if we are in the right section, we take the <li> element
+  /// and transform to a build bot
+  ///
+  Future<Try<List<LuciBuildBot>>> getAllBuildBots(
+      String client, WithCacheFunction withCache) async {
+    return await tryStartAsync(() => withCache(
+            () => _makeGetRequest(
+                new Uri(scheme: 'https', host: LUCI_HOST, path: "/")),
+            "all_buildbots"))
+        .then((Try<String> tryRes) => tryRes.bind(parse).bind((htmlDoc) {
+              // This is really dirty, but the structure of
+              // the document is not really suited for anything else.
+              var takeSection = false;
+              return htmlDoc.body.children.where((node) {
+                if (node.localName == "li") return takeSection;
+                if (node.localName != "h3") {
+                  takeSection = false;
+                  return false;
+                }
+                // Current node is <h3>.
+                takeSection = client == node.text.trim();
+                return false;
+              });
+            }).bind((elements) {
+              // Here we hold an iterable of buildbot elements
+              // <li>
+              //     <a href="/buildbot/client.dart/analyzer-linux-release-be">
+              //        analyzer-linux-release-be</a>
+              // </li>
+              return elements.map((element) {
+                var name = element.children[0].text;
+                var url = element.children[0].attributes['href'];
+                return new LuciBuildBot(client, name, url);
+              }).toList();
+            }));
+  }
+
+  /// [getPrimaryBuilders] fetches all primary builders
+  /// (the ones usually used by gardeners) by not including buildbots with
+  /// the name -dev, -stable or -integration.
+  Future<Try<List<LuciBuildBot>>> getPrimaryBuilders(
+      String client, WithCacheFunction withCache) async {
+    return await getAllBuildBots(client, withCache)
+        .then((Try<List<LuciBuildBot>> tryRes) {
+      return tryRes
+          .bind((buildBots) => buildBots.where((LuciBuildBot buildBot) {
+                return !(buildBot.name.contains("-dev") ||
+                    buildBot.name.contains("-stable") ||
+                    buildBot.name.contains("-integration"));
+              }));
+    });
+  }
+
+  /// Calling the Milo Api to get latest builds for this bot,
+  /// where the field [amount] is the number of recent builds to fetch.
+  Future<Try<List<BuildDetail>>> getBuildBotDetails(
+      String client, String botName, WithCacheFunction withCache,
+      [int amount = 20]) async {
+    var uri = new Uri(
+        scheme: "https",
+        host: LUCI_HOST,
+        path: "prpc/milo.Buildbot/GetBuildbotBuildsJSON");
+    var body = {
+      "master": client,
+      "builder": botName,
+      "limit": amount,
+      "includeCurrent": true
+    };
+    var result = await tryStartAsync(() => withCache(
+        () => _makePostRequest(uri, JSON.encode(body), {
+              HttpHeaders.CONTENT_TYPE: "application/json",
+              HttpHeaders.ACCEPT: "application/json"
+            }),
+        '${uri.path}_${botName}_$amount'));
+    return result.bind(JSON.decode).bind((json) {
+      return json["builds"].map((b) {
+        var build = JSON.decode(UTF8.decode(BASE64.decode(b["data"])));
+        return getBuildDetailFromJson(client, botName, build);
+      }).toList();
+    });
+  }
+
+  /// Calling the Milo Api to get information about a specific build
+  /// where the field [buildNumber] is the build number to fetch.
+  Future<Try<BuildDetail>> getBuildBotBuildDetails(String client,
+      String botName, int buildNumber, WithCacheFunction withCache) async {
+    var uri = new Uri(
+        scheme: "https",
+        host: LUCI_HOST,
+        path: "prpc/milo.Buildbot/GetBuildbotBuildJSON");
+    var body = {"master": client, "builder": botName, "buildNum": buildNumber};
+    print(body);
+    var result = await tryStartAsync(() => withCache(
+        () => _makePostRequest(uri, JSON.encode(body), {
+              HttpHeaders.CONTENT_TYPE: "application/json",
+              HttpHeaders.ACCEPT: "application/json"
+            }),
+        '${uri.path}_${botName}_$buildNumber'));
+    return result.bind(JSON.decode).bind((json) {
+      var build = JSON.decode(UTF8.decode(BASE64.decode(json["data"])));
+      return getBuildDetailFromJson(client, botName, build);
+    });
+  }
+
+  /// [_makeGetRequest] performs a get request to [uri].
+  Future<String> _makeGetRequest(Uri uri) async {
+    var request = await _client.getUrl(uri);
+    var response = await request.close();
+    if (response.statusCode != 200) {
+      response.drain();
+      throw new HttpException(response.reasonPhrase, uri: uri);
+    }
+    return response.transform(UTF8.decoder).join();
+  }
+
+  /// [_makeGetRequest] performs a post request to [uri], where the posted
+  /// body is the string representation of [body]. For adding custom headers
+  /// use the map [headers].
+  Future<String> _makePostRequest(
+      Uri uri, Object body, Map<String, String> headers) async {
+    var response = await http.post(uri, body: body, headers: headers);
+    if (response.statusCode != 200) {
+      throw new HttpException(response.reasonPhrase, uri: uri);
+    }
+    // Prpc outputs a prefix to combat vulnerability.
+    if (response.body.startsWith(")]}'")) {
+      return response.body.substring(4);
+    }
+    return response.body;
+  }
+
+  /// Closes the Http client connection
+  void close() {
+    _client.close();
+  }
+}
+
+/// [getBuildDetailFromJson] parses json [build] to a class [BuildDetail]
+BuildDetail getBuildDetailFromJson(
+    String client, String botName, dynamic build) {
+  List<GitCommit> changes = build["sourceStamp"]["changes"].map((change) {
+    return new GitCommit(change["revision"], change["revLink"], change["who"],
+        change["comments"], change["files"].map((file) => file["name"]));
+  }).toList();
+
+  List<BuildProperty> properties = build["properties"].map((prop) {
+    return new BuildProperty(prop[0], prop[1].toString(), prop[2]);
+  }).toList();
+
+  List<BuildStep> steps = build["steps"].map((step) {
+    var start =
+        new DateTime.fromMillisecondsSinceEpoch(step["times"][0] * 1000);
+    DateTime end = null;
+    if (step["times"][1] != null) {
+      end = new DateTime.fromMillisecondsSinceEpoch(step["times"][1] * 1000);
+    }
+    return new BuildStep(
+        step["name"],
+        step["text"],
+        step["results"].toString(),
+        start,
+        end,
+        step["step_number"],
+        step["isStarted"],
+        step["isFinished"],
+        step["logs"].map((log) => new BuildLog(log[0], log[1])));
+  }).toList();
+
+  DateTime end = null;
+  if (build["times"][1] != null) {
+    end = new DateTime.fromMillisecondsSinceEpoch(build["times"][1] * 1000);
+  }
+
+  Timing timing = new Timing(
+      new DateTime.fromMillisecondsSinceEpoch(build["times"][0] * 1000), end);
+
+  return new BuildDetail(
+      client,
+      botName,
+      build["number"],
+      build["text"].join(' '),
+      build["finished"],
+      steps,
+      properties,
+      build["blame"],
+      timing,
+      changes);
+}
+
+// Structured classes to relay information from api and web pages
+
+/// [LuciBuildBot] holds information about a build bot
+class LuciBuildBot {
+  final String client;
+  final String name;
+  final String url;
+
+  LuciBuildBot(this.client, this.name, this.url);
+
+  @override
+  String toString() {
+    return "LuciBuildBot { client: $client, name: $name, url: $url }";
+  }
+}
+
+/// [BuildDetail] holds data detailing a specific build
+class BuildDetail {
+  final String client;
+  final String botName;
+  final int buildNumber;
+  final String results;
+  final bool finished;
+  final List<BuildStep> steps;
+  final List<BuildProperty> buildProperties;
+  final List<String> blameList;
+  final Timing timing;
+  final List<GitCommit> allChanges;
+
+  BuildDetail(
+      this.client,
+      this.botName,
+      this.buildNumber,
+      this.results,
+      this.finished,
+      this.steps,
+      this.buildProperties,
+      this.blameList,
+      this.timing,
+      this.allChanges);
+
+  @override
+  String toString() {
+    StringBuffer buffer = new StringBuffer();
+    buffer.writeln("--------------------------------------");
+    buffer.writeln(results);
+    buffer.writeln(timing);
+    buffer.writeln("----------------STEPS-----------------");
+    if (steps != null) steps.forEach(buffer.writeln);
+    buffer.writeln("----------BUILD PROPERTIES------------");
+    if (buildProperties != null) buildProperties.forEach(buffer.writeln);
+    buffer.writeln("-------------BLAME LIST---------------");
+    if (blameList != null) blameList.forEach(buffer.writeln);
+    buffer.writeln("------------ALL CHANGES---------------");
+    if (allChanges != null) allChanges.forEach(buffer.writeln);
+    return buffer.toString();
+  }
+}
+
+/// [BuildStep] holds data detailing a specific build
+class BuildStep {
+  final String name;
+  final String description;
+  final String result;
+  final DateTime start;
+  final DateTime end;
+  final int number;
+  final bool isStarted;
+  final bool isFinished;
+  final List<BuildLog> logs;
+
+  BuildStep(this.name, this.description, this.result, this.start, this.end,
+      this.number, this.isStarted, this.isFinished, this.logs);
+
+  @override
+  String toString() {
+    StringBuffer buffer = new StringBuffer();
+    buffer.writeln("${result == '[0, []]' ? 'SUCCESS' : result}: "
+        "$name - $description ($start, $end)");
+    logs.forEach((subLink) {
+      buffer.writeln("\t${subLink}");
+    });
+    return buffer.toString();
+  }
+}
+
+/// [BuildLog] holds log-information for a specific build.
+class BuildLog {
+  final String name;
+  final String url;
+
+  BuildLog(this.name, this.url);
+
+  @override
+  String toString() {
+    return "$name | $url";
+  }
+}
+
+/// [BuildProperty] descibes build properties of a specific build.
+class BuildProperty {
+  final String name;
+  final String value;
+  final String source;
+
+  BuildProperty(this.name, this.value, this.source);
+
+  @override
+  String toString() {
+    return "$name\t$value\t$source";
+  }
+}
+
+/// [Timing] is a class to hold timing information for builds and steps.
+class Timing {
+  final DateTime start;
+  final DateTime end;
+
+  Timing(this.start, this.end);
+
+  @override
+  String toString() {
+    return "start: $start\tend: $end";
+  }
+}
+
+/// [GitCommit] holds data about a specific commit.
+class GitCommit {
+  final String revision;
+  final String commitUrl;
+  final String changedBy;
+  final String comments;
+  final List<String> changedFiles;
+
+  GitCommit(this.revision, this.commitUrl, this.changedBy, this.comments,
+      this.changedFiles);
+
+  @override
+  String toString() {
+    StringBuffer buffer = new StringBuffer();
+    buffer.writeln("revision: $revision");
+    buffer.writeln("commitUrl: $commitUrl");
+    buffer.writeln("changedBy: $changedBy");
+    buffer.write("\n");
+    buffer.writeln(comments);
+    buffer.write("\nfiles:\n");
+    changedFiles.forEach(buffer.writeln);
+    return buffer.toString();
+  }
+}