Additional tools for gardening.

tools/gardening/lib/src/_luci_api.dart

+import 'dart:io';

[Johnni Winther, 2017/08/28 07:14:03]

Remove this file.

+import 'dart:async';
+import 'dart:convert';
+import 'package:base_lib/base_lib.dart';
+import 'package:html/parser.dart' show parse;
+import 'package:html/dom.dart';
+
+const String LUCI_HOST = "luci-milo.appspot.com";
+const String BUILD_CHROMIUM_HOST = "build.chromium.org";
+
+/// Base class for communicating with Luci @ LogDog
+/// Since some results can take a long time to get
+/// and some results do not update that frequently,
+/// [LuciApi] requires a cache to be used. If no
+/// cache is needed, the [noCache] can be used.
+class LuciApi {
+  final HttpClient _client = new HttpClient();
+
+  LuciApi();
+
+  /// [getBuildBots] fetches all build bots from luci.
+  /// The format is:
+  /// <li>
+  ///     <a href="/buildbot/client.crashpad/crashpad_win_x86_wow64_rel">crashpad_win_x86_wow64_rel</a>
+  /// </li>
+  /// <h3> client.dart </h3>
+  /// <li>
+  ///     <a href="/buildbot/client.dart/analyze-linux-be">analyze-linux-be</a>
+  /// </li>
+  /// <li>
+  ///     <a href="/buildbot/client.dart/analyze-linux-dev">analyze-linux-dev</a>
+  /// </li>
+  /// <li>
+  ///     <a href="/buildbot/client.dart/analyze-linux-stable">analyze-linux-stable</a>
+  /// </li>
+  /// <li>
+  ///     <a href="/buildbot/client.dart/analyzer-linux-release-be">analyzer-linux-release-be</a>
+  /// </li>
+  ///
+  /// We look for the section header matching clients, then
+  /// if we are in the right section, we take the <li> element
+  /// and transform to a build bot
+  ///
+  Future<Try<List<LuciBuildBot>>> getAllBuildBots(
+      String client, WithCache withCache) async {
+    var reqResult = await tryStartAsync(() => withCache(
+        () => _makeGetRequest(
+            new Uri(scheme: 'https', host: LUCI_HOST, path: "/")),
+        "all_buildbots"));
+    return reqResult.bind(parse).bind((htmlDoc) {
+      var takeSection = false; // this is really dirty, but the structure of
+      // the document is not really suited for anything else
+      return htmlDoc.body.children.where((node) {
+        if (node.localName == "li") return takeSection;
+        if (node.localName != "h3") {
+          takeSection = false;
+          return false;
+        }
+        // current node is <h3>
+        takeSection = client == node.text.trim();
+        return false;
+      });
+    }).bind((elements) {
+      // here we hold an iterable of buildbot elements
+      // <li>
+      //     <a href="/buildbot/client.dart/analyzer-linux-release-be">analyzer-linux-release-be</a>
+      // </li>
+      return elements.map((element) {
+        var name = element.children[0].text;
+        var url = element.children[0].attributes['href'];
+        return new LuciBuildBot(client, name, url);
+      }).toList();
+    });
+  }
+
+  /// [getPrimaryBuilders] fetches all primary builders (the ones usually used by gardeners)
+  /// by just stopping when reaching the first -dev buildbot
+  Future<Try<List<LuciBuildBot>>> getPrimaryBuilders(
+      String client, WithCache withCache) async {
+    var reqResult = await tryStartAsync(() => withCache(
+        () => _makeGetRequest(new Uri(
+            scheme: 'https',
+            host: BUILD_CHROMIUM_HOST,
+            path: "/p/$client/builders")),
+        "primary_buildbots"));
+    return reqResult.bind(parse).bind((Document htmlDoc) {
+      // We take all <table><tbody><tr>
+      var mainTableRows =
+          htmlDoc.body.getElementsByTagName("table")[0].children[0].children;
+      // pick until we reach a dev builder bot
+      return mainTableRows.takeWhile((Element el) {
+        return !el.children[0].children[0].text.contains("-dev");
+      }).map((Element el) {
+        return new LuciBuildBot(client, el.children[0].children[0].text,
+            el.children[0].children[0].attributes["href"]);
+      });
+    });
+  }
+
+  /// Generates a suitable url to fetch information about a buildbot.
+  /// The field [amount] is the number of recent builds to fetch
+  Future<Try<LuciBuildBotDetail>> getBuildBotDetails(
+      String client, String botName, WithCache withCache,
+      [int amount = 25]) async {
+    var uri = new Uri(
+        scheme: "https",
+        host: LUCI_HOST,
+        path: "/buildbot/$client/$botName/",
+        query: "limit=$amount");
+    var reqResult = await tryStartAsync(
+        () => withCache(() => _makeGetRequest(uri), uri.path.toString()));
+    return reqResult.bind(parse).bind((Document document) {
+      var detail = new LuciBuildBotDetail()
+        ..client = client
+        ..botName = botName
+        ..currentBuild = getCurrentBuild(document)
+        ..latestBuilds = getBuildOverviews(document);
+      return detail;
+    });
+  }
+
+  /// Get the [BuildDetail] information for the requested build.
+  /// The url requested is on the form /buildbot/$client/$botName/$buildNumber.
+  Future<Try<BuildDetail>> getBuildDetails(String client, String botName,
+      int buildNumber, WithCache withCache) async {
+    var uri = new Uri(
+        scheme: "https",
+        host: LUCI_HOST,
+        path: "/buildbot/$client/$botName/$buildNumber");
+    var reqResult = await tryStartAsync(
+        () => withCache(() => _makeGetRequest(uri), uri.path.toString()));
+    return reqResult.bind(parse).bind((Document document) {
+      return new BuildDetail()
+        ..client = client
+        ..botName = botName
+        ..buildNumber = buildNumber
+        ..results = getBuildResults(document)
+        ..steps = getBuildSteps(document)
+        ..buildProperties = getBuildProperties(document)
+        ..blameList = getBuildBlameList(document)
+        ..timing = getBuildTiming(document)
+        ..allChanges = getBuildGitCommits(document);
+    });
+  }
+
+  Future<String> _makeGetRequest(Uri uri) async {
+    var request = await _client.getUrl(uri);
+    var response = await request.close();
+
+    if (response.statusCode != 200) {
+      response.drain();
+      throw new HttpException(response.reasonPhrase, uri: uri);
+    }
+
+    return response.transform(UTF8.decoder).join();
+  }
+
+  void close() {
+    _client.close();
+  }
+}
+
+// Functions to generate objects from the HTML representation
+
+CurrentBuild getCurrentBuild(Document document) {
+  var currentBuildColumn = document.body.getElementsByClassName("column")[1];
+  // <h2>Current Builds (1):</h2> <------ we are here
+  // <ul>
+  //   <li>
+  //     <a href="1277">#1277</a>
+  if (currentBuildColumn.children.length > 1) {
+    var li = currentBuildColumn.children[1].children[0];
+    return new CurrentBuild()
+      ..buildNumber = int.parse(li.children[0].attributes["href"])
+      ..duration = li.text
+          .substring(li.text.indexOf("[") + 1, li.text.indexOf("]"))
+          .trim();
+  }
+  return null;
+}
+
+List<BuildOverview> getBuildOverviews(Document document) {
+  var main = document.body.getElementsByClassName("main").first;
+  var tables = main.getElementsByClassName("info");
+  if (tables.length == 0) {
+    // happens when there is no builds at all
+    return new List<BuildOverview>();
+  }
+  // We want to iterate over all table rows
+  // <table> <tbody> <tr>
+  return tables[0]
+      .children[0]
+      .children
+      .skip(1)
+      .map(getBuildOverviewFromTableRow)
+      .toList();
+}
+
+BuildOverview getBuildOverviewFromTableRow(Element tr) {
+  var bo = new BuildOverview()
+    ..time = tr.children[0].firstChild.text
+    ..mainRevision = tr.children[1].text
+    ..result = tr.children[2].text
+    ..buildNumber = int.parse(tr.children[3].text.substring(1));
+  var changesText = tr.children[4].text;
+  bo.hasMultipleChanges =
+      changesText.contains(',') || changesText.contains("changes");
+  bo.info = tr.children[5].innerHtml
+      .split("<br>")
+      .map((info) => info.trim())
+      .toList();
+
+  return bo;
+}
+
+String getBuildResults(Document document) {
+  // <html> <--- we are here
+  // ...
+  // <div class="column">
+  //    <h2>Results:</h2>
+  //    <p class="success result">Build Successful
+  return document.getElementsByClassName("column")[0].children[1].text.trim();
+}
+
+List<Step> getBuildSteps(Document document) {
+  // <html> <--- we are here
+  // ...
+  // <ol id="steps" class="standard">
+  //  <li class="verbosity-Normal">
+  return document
+      .getElementsByClassName("standard")
+      .first
+      .children
+      .map(getBuildStep)
+      .toList();
+}
+
+Step getBuildStep(Element li) {
+  //  <li class="verbosity-Normal"> <--- we are here
+  //     <div class="status-Success result">
+  //         <span class="duration"
+  //             data-starttime="2017-08-21T17:01:41Z"
+  //             data-endtime="2017-08-21T19:40:20Z">
+  //           ( 2 hrs 38 mins )</span>
+  //         <b>steps</b>
+  //         <span>
+  //             <div class="step-text">running steps via annotated script</div>
+  //         </span>
+  //     </div>
+  //     <ul>
+  //         <li class="sublink">
+  var divResult = li.children[0];
+  return new Step(
+      divResult.children[1].text,
+      divResult.getElementsByClassName("step-text").first.text.trim(),
+      divResult.className.replaceAll("status-", "").replaceAll(" result", ""),
+      divResult.children.first.text.trim())
+    ..subLinks = li
+        .getElementsByClassName("sublink")
+        .map((liSub) => new SubLink(
+            liSub.firstChild.text, liSub.firstChild.attributes["href"]))
+        .toList();
+}
+
+List<BuildProperty> getBuildProperties(Document document) {
+  // <html> <-- we are here
+  // ...
+  // <div class="column">
+  // ...
+  // <div class="column">
+  //     <h2>Build Properties:</h2>
+  //     <table class="info BuildProperties" width="100%">
+  //       <tbody>
+  //         <tr>
+  //             <th>Name</th>
+  //             <th>Value</th>
+  //             <th>Source</th>
+  //         </tr>
+  var buildTable = document.getElementsByClassName("column")[1].children[1];
+  return buildTable.children.first.children.skip(1).map((tr) {
+    // here we hold a <tr> with cells of information
+    return new BuildProperty(tr.children[0].text.trim(),
+        tr.children[1].text.trim(), tr.children[2].text.trim());
+  }).toList();
+}
+
+List<String> getBuildBlameList(Document document) {
+  // <html> <-- we are here
+  // ...
+  // <div class="column">
+  // ...
+  // <div class="column">
+  //     <h2>Build Properties:</h2>
+  //     <table class="info BuildProperties" width="100%">
+  //     <h2>Blamelist:</h2>
+  //     <ol>
+  //       <li>
+  var blameList = document.getElementsByClassName("column")[1].children[3];
+  return blameList.children.map((li) {
+    return li.text.replaceAll("ohnoyoudont", "").trim();
+  }).toList();
+}
+
+Timing getBuildTiming(Document document) {
+  // <html> <-- we are here
+  // ...
+  // <div class="column">
+  // ...
+  // <div class="column">
+  //     <h2>Build Properties:</h2>
+  //     <table class="info BuildProperties" width="100%">
+  //     <h2>Blamelist:</h2>
+  //     <ol> ...
+  //     <h2>Timing:</h2>
+  //     <table..
+  var timingTableBody =
+      document.getElementsByClassName("column")[1].children[5].children.first;
+  var infos = timingTableBody.children
+      .map((tr) => tr.children.last.text.trim())
+      .toList();
+  return new Timing(infos[0], infos[1], infos[2]);
+}
+
+List<GitCommit> getBuildGitCommits(Document document) {
+  // <html> <-- we are here
+  // ...
+  // <div class="column">
+  // ...
+  // <div class="column">
+  // ...
+  // <div class="column">
+  // <h2>All Changes:</h2>
+  // <ol>
+  //     <li>
+  var olChanges = document.getElementsByClassName("column")[2].children[1];
+  return olChanges.children.map(getBuildGitCommit).toList();
+}
+
+GitCommit getBuildGitCommit(Element li) {
+  // <li> <--- we are here
+  //     <h3>Deprecate MethodElement.getReifiedType</h3>
+  //     <table class="info">
+  //         <tbody>
+  //             <tr>
+  String title = li.children[0].text;
+  Element revisionAnchor =
+      li.children[1].children[0].children.last.children[1].firstChild;
+  String commitUrl = revisionAnchor.attributes["href"];
+  String revision = revisionAnchor.text;
+  String changedBy = li.children[1].children[0].children.first.children[1].text
+      .replaceAll("ohnoyoudont", "")
+      .trim();
+  String comments = li.getElementsByClassName("comments").first.text;
+  List<String> files =
+      li.getElementsByClassName("file").map((liFile) => liFile.text).toList();
+
+  return new GitCommit(title, revision, commitUrl, changedBy, comments)
+    ..changedFiles = files;
+}
+
+// Structured classes to relay information scraped from the luci bot web pages
+
+/// [LuciBuildBot] holds information about a build bot
+class LuciBuildBot {
+  String client;
+  String name;
+  String url;
+
+  LuciBuildBot(this.client, this.name, this.url);
+
+  @override
+  String toString() {
+    return "LuciBuildBot { client: $client, name: $name, url: $url }";
+  }
+}
+
+/// [LuciBuildBotDetail] holds information about a bots current build,
+/// all latest builds (excluding the current)
+class LuciBuildBotDetail {
+  String client;
+  String botName;
+  CurrentBuild currentBuild;
+  List<BuildOverview> latestBuilds;
+
+  int latestBuildNumber() {
+    if (currentBuild == null && latestBuilds.length == 0) {
+      return -1;
+    }
+    return currentBuild != null
+        ? currentBuild.buildNumber
+        : latestBuilds[0].buildNumber;
+  }
+
+  @override
+  String toString() {
+    StringBuffer buffer = new StringBuffer();
+    buffer.writeln(currentBuild == null
+        ? "Current build: none\n"
+        : "Current build: ${currentBuild.buildNumber} (${currentBuild.duration})\n");
+
+    buffer.writeln(
+        "time\t\t\t\t\trevision\t\t\t\t\tresult\tbuild #\tmult. rev.\tinfo");
+
+    latestBuilds.forEach((build) {
+      if (build.time.length > 31) {
+        buffer.writeln(
+            "${build.time}\t${build.mainRevision}\t${build.result}\t${build.buildNumber}\t${build.hasMultipleChanges}\t\t${build.info.join(',')}");
+      } else {
+        buffer.writeln(
+            "${build.time}\t\t${build.mainRevision}\t${build.result}\t${build.buildNumber}\t${build.hasMultipleChanges}\t\t${build.info.join(',')}");
+      }
+    });
+
+    return buffer.toString();
+  }
+}
+
+/// [CurrentBuild] shows current build informaiton
+class CurrentBuild {
+  int buildNumber;
+  String duration;
+}
+
+/// [BuildOverview] has overview information about a build, such as time, mainRevision and result
+class BuildOverview {
+  String time;
+  String mainRevision;
+  String result;
+  int buildNumber;
+  bool hasMultipleChanges;
+  List<String> info;
+}
+
+class BuildDetail {
+  String client;
+  String botName;
+  int buildNumber;
+  String results;
+  List<Step> steps;
+  List<BuildProperty> buildProperties;
+  List<String> blameList;
+  Timing timing;
+  List<GitCommit> allChanges;
+
+  @override
+  String toString() {
+    StringBuffer buffer = new StringBuffer();
+    buffer.writeln("--------------------------------------");
+    buffer.writeln(results);
+    buffer.writeln(timing);
+    buffer.writeln("----------------STEPS-----------------");
+    if (steps != null) steps.forEach(buffer.writeln);
+    buffer.writeln("----------BUILD PROPERTIES------------");
+    if (buildProperties != null) buildProperties.forEach(buffer.writeln);
+    buffer.writeln("-------------BLAME LIST---------------");
+    if (blameList != null) blameList.forEach(buffer.writeln);
+    buffer.writeln("------------ALL CHANGES---------------");
+    if (allChanges != null) allChanges.forEach(buffer.writeln);
+    return buffer.toString();
+  }
+}
+
+class Step {
+  String name;
+  String description;
+  String result;
+  String time;
+  List<SubLink> subLinks;
+  Step(this.name, this.description, this.result, this.time);
+
+  @override
+  String toString() {
+    StringBuffer buffer = new StringBuffer();
+    buffer.writeln("$result: $name - $description ($time)");
+    subLinks.forEach((subLink) {
+      buffer.writeln("\t${subLink}");
+    });
+    return buffer.toString();
+  }
+}
+
+class SubLink {
+  String name;
+  String url;
+  SubLink(this.name, this.url);
+
+  @override
+  String toString() {
+    return "$name | $url";
+  }
+}
+
+class BuildProperty {
+  String name;
+  String value;
+  String source;
+  BuildProperty(this.name, this.value, this.source);
+
+  @override
+  String toString() {
+    return "$name\t$value\t$source";
+  }
+}
+
+class Timing {
+  String start;
+  String end;
+  String elapsed;
+  Timing(this.start, this.end, this.elapsed);
+
+  @override
+  String toString() {
+    return "start: $start\tend: $end\telapsed: $elapsed";
+  }
+}
+
+class GitCommit {
+  String title;
+  String revision;
+  String commitUrl;
+  String changedBy;
+  String comments;
+  List<String> changedFiles;
+  GitCommit(
+      this.title, this.revision, this.commitUrl, this.changedBy, this.comments);
+
+  @override
+  String toString() {
+    StringBuffer buffer = new StringBuffer();
+
+    buffer.writeln(title);
+    buffer.writeln("revision: $revision");
+    buffer.writeln("commitUrl: $commitUrl");
+    buffer.writeln("changedBy: $changedBy");
+    buffer.write("\n");
+    buffer.writeln(comments);
+    buffer.write("\nfiles:\n");
+    changedFiles.forEach(buffer.writeln);
+    return buffer.toString();
+  }
+}