[milo] Add an (uncached) method to get console rows.

milo/buildsource/console.go

+// Copyright 2017 The LUCI Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package buildsource
+
+import (
+        "encoding/hex"
+
+        "golang.org/x/net/context"
+
+        ds "github.com/luci/gae/service/datastore"

[Ryan Tseng, 2017/07/19 08:07:57]

nit: "datastore" is preferred in Milo

[iannucci, 2017/07/20 00:14:56]

Done.

+
+        "github.com/luci/luci-go/common/data/stringset"
+        "github.com/luci/luci-go/common/errors"
+        "github.com/luci/luci-go/common/sync/parallel"
+
+        "github.com/luci/luci-go/milo/api/config"
+        "github.com/luci/luci-go/milo/common/model"
+)
+
+// GetBuildSummariesForConsole returns a row-oriented collection of BuildSummary
+// objects. Each row corresponds to the similarly-indexed commit in the
+// `commits` slice.

[Ryan Tseng, 2017/07/19 08:07:57]

"row-oriented" could be little ambiguous.  Should say something more precise
like:
Returns a matrix of BuildSummery objects.  The first index always matches the
commit, while the second index always matches the builder.  The returned object
will always be size of [len(commits)][len(builders)].

[iannucci, 2017/07/20 00:14:56]

Done.

+func GetBuildSummariesForConsole(c context.Context, project string, console *config.Console, commits, builders []string) ([][]*model.BuildSummary, error) {
+        rawCommits := make([][]byte, len(commits))
+        for i, c := range commits {
+                var err error

[Ryan Tseng, 2017/07/19 08:07:57]

nit: this can be inlined

[iannucci, 2017/07/20 00:14:56]

can't because we're assigning to rawCommits[i]

+                if rawCommits[i], err = hex.DecodeString(c); err != nil {
+                        return nil, errors.Annotate(err, "bad commit[%d]: %q", i, c).Err()
+                }
+        }
+
+        builderSet := stringset.NewFromSlice(builders...)
+
+        ret := make([][]*model.BuildSummary, len(commits))

[Ryan Tseng, 2017/07/19 08:07:58]

might as well pre-allocate the whole matrix here

[iannucci, 2017/07/20 00:14:56]

Acknowledged.

+        url := console.RepoURL
+        // HACK(iannucci): This little hack should be removed when console definitions
+        // no longer use a manifest name of "REVISION". REVISION was used to index the
+        // 'got_revision' value before manifests were implemented.
+        if console.ManifestName == "REVISION" {
+                url = ""
+        }
+        prefix := model.BuildSummaryPrefix(project, console.Name, console.ManifestName, url)
+        q := ds.NewQuery("BuildSummary").KeysOnly(true)
+        err := parallel.WorkPool(4, func(ch chan<- func() error) {
+                for i := range rawCommits {
+                        i := i
+                        ch <- func() error {
+                                fullQ := q.Eq("ManifestRevisionIndex", prefix.AddRevision(rawCommits[i]))
+                                return ds.Run(c, fullQ, func(bs *model.BuildSummary) {
+                                        if builderSet.Has(bs.BuilderID) {
+                                                ret[i] = append(ret[i], bs)

[Ryan Tseng, 2017/07/19 08:07:58]

Pretty sure this is going to get mis-aligned.  The builds have to be inserted in
the correct slots (that matches the builder index).

[iannucci, 2017/07/20 00:14:56]

Acknowledged.

+                                        }
+                                })
+                        }
+                }
+        })
+
+        return ret, err
+}