test-results: builders and builderstate: bug fixes, tests

go/src/infra/appengine/test-results/frontend/builders.go

 package frontend
 
 import (
         "bytes"
         "encoding/json"
         "io"
         "net/http"
         "regexp"
         "sort"
         "strings"
@@ skipping 11 matching lines @@
         "infra/appengine/test-results/masters"
 )
 
 // BuilderData is the data returned from the GET "/builders"
 // endpoint.
 type BuilderData struct {
         Masters           []Master `json:"masters"`
         NoUploadTestTypes []string `json:"no_upload_test_types"`
 }
 
+// Master represents information about a build master.
 type Master struct {
         Name       string          `json:"name"`
         Identifier string          `json:"url_name"`
         Groups     []string        `json:"groups"`
         Tests      map[string]Test `json:"tests"`
 }
 
+// Test represents information about Tests in a master.
 type Test struct {
         Builders []string `json:"builders"`
 }
 
 const buildbotMemcacheKey = "buildbot_data"
 
 var nonTestStepNames = []string{
         "archive",
         "Run tests",
         "find isolated tests",
@@ skipping 15 matching lines @@
         "test_mini_installer",
         "webkit_python_tests",
 }
 
 func makeBuildExtractClient(ctx context.Context) *buildextract.Client {
         return buildextract.NewClient(&http.Client{
                 Transport: urlfetch.Get(ctx),
         })
 }
 
-func GetBuilders(ctx *router.Context) {
+func getBuildersHandler(ctx *router.Context) {
         c, w, r := ctx.Context, ctx.Writer, ctx.Request
         var res []byte
         item, err := memcache.Get(c).Get(buildbotMemcacheKey)
 
         switch err {
         case memcache.ErrCacheMiss:
                 start := time.Now()
                 data, err := getBuilderData(c, masters.Known, makeBuildExtractClient(c))
                 if err != nil {
                         logging.WithError(err).Errorf(c, "builders: GetBuilders: failed to fetch builders")
@@ skipping 28 matching lines @@
         n, err := io.Copy(w, out)
 
         if err != nil {
                 logging.Fields{
                         logging.ErrorKey: err,
                         "n":              n,
                 }.Errorf(c, "builders: GetBuilders: failed to write response")
         }
 }
 
-func UpdateBuilders(ctx *router.Context) {
+func updateBuildersHandler(ctx *router.Context) {
         c, w := ctx.Context, ctx.Writer
 
         start := time.Now()
         data, err := getBuilderData(c, masters.Known, makeBuildExtractClient(c))
         if err != nil {
                 logging.WithError(err).Errorf(c, "builders: UpdateBuilders: failed to fetch builders")
                 http.Error(w, err.Error(), http.StatusInternalServerError)
                 return
         }
         logging.Fields{"duration": time.Since(start)}.Infof(c, "builders: getBuilderData")
@@ skipping 45 matching lines @@
         }
         results := make(chan result, len(list))
         wg := sync.WaitGroup{}
         wg.Add(len(list))
 
         for _, master := range list {
                 m := Master{
                         Name:       master.Name,
                         Identifier: master.Identifier,
                         Groups:     master.Groups,
-                        Tests:      map[string]Test{},
+                        Tests:      make(map[string]Test),
                 }
 
                 go func() {
                         defer wg.Done()
                         builders, err := getBuilderNames(ctx, m.Identifier, client)
                         if err != nil {
                                 results <- result{Err: err}
                                 return
                         }
 
+                        if len(builders) == 0 {
+                                return
+                        }
+
                         for _, b := range builders {
                                 stepNames, err := getStepNames(ctx, m.Identifier, b, client)
                                 if err != nil {
                                         results <- result{Err: err}
                                         return
                                 }
                                 for _, s := range stepNames {
                                         t, ok := m.Tests[s]
                                         if !ok {
                                                 t = Test{}
                                         }
                                         t.Builders = append(t.Builders, b)
                                         m.Tests[s] = t
                                 }
                         }
+
                         results <- result{Master: m}
                 }()
         }
 
         wg.Wait()
         close(results)
         for res := range results {
                 if res.Err != nil {
                         return BuilderData{}, res.Err
                 }
@@ skipping 94 matching lines @@
 
 var ignoreTestNameRx = regexp.MustCompile(`_only|_ignore|_perf$`)
 var gtestUploaderStepRx = regexp.MustCompile(`Upload to test-results \[([^]]*)\]`)
 
 func cleanTestStep(name string) (clean string, ok bool) {
         if !strings.Contains(name, "test") {
                 return "", false
         }
 
         for _, n := range nonTestStepNames {
-                if strings.HasPrefix(name, n) {
+                if strings.Contains(name, n) {
                         return "", false
                 }
         }
 
         if ignoreTestNameRx.MatchString(name) {
                 return "", false
         }
 
         // Ignore triggering and collecting steps on swarming:
         // they are not actual tests.
@@ skipping 42 matching lines @@
         // data, we just keep everything before the first space.
         if i := strings.Index(name, " "); i != -1 {
                 name = name[:i]
         }
 
         if withPatch {
                 name += " (with patch)"
         }
         return name
 }