[alerts-dispatcher] update to build again :) also fixes a concurrent map access

go/src/infra/monitoring/dispatcher/dispatcher.go

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 // Dispatcher usage:
 // go run infra/monitoring/dispatcher
 // Expects gatekeeper.json to be in the current directory.
 
 package main
 
 import (
         "encoding/json"
         "expvar"
         "flag"
         "fmt"
         "io/ioutil"
         "net/http"
         "os"
+        "sort"
         "strings"
         "time"
 
         "github.com/luci/luci-go/common/auth"
         "github.com/luci/luci-go/common/clock"
         "github.com/luci/luci-go/common/logging"
         "github.com/luci/luci-go/common/logging/gologger"
 
         "golang.org/x/net/context"
 
@@ skipping 19 matching lines @@
         cycleStr            = flag.String("cycle", "1s", "Cycle time for loop")
         snapshot            = flag.String("record-snapshot", "", "Save a snapshot of infra responses to this path, which will be created if it does not already exist.")
         replay              = flag.String("replay-snapshot", "", "Replay a snapshot of infra responses from this path, which should have been created previously by running with --record-snapshot.")
         replayTime          = flag.String("replay-time", "", "Specify a simulated starting time for the replay in RFC3339 format, used with --replay-snapshot.")
         serviceAccountJSON  = flag.String("service-account", "", "Service account JSON file.")
 
         log             = gologger.Get()
         duration, cycle time.Duration
 
         // gk is the gatekeeper config.
-        gk = &struct {
-                // Weird. Are there ever multiple configs per master key?
-                Masters map[string][]messages.MasterConfig `json:"masters"`
-        }{}
+        gk = &messages.GatekeeperConfig{}
         // gkt is the gatekeeper trees config.
         gkt              = map[string]messages.TreeMasterConfig{}
         filteredFailures = uint64(0)
         expvars          = expvar.NewMap("dispatcher")
 )
 
 func init() {
         flag.Usage = func() {
                 fmt.Printf("By default runs a single check, saves any alerts to ./alerts.json and exits.")
                 flag.PrintDefaults()
@@ skipping 49 matching lines @@
 
         for range masterNames {
                 r := <-res
                 if r.be != nil {
                         bes[r.name] = r.be
                 }
         }
         return bes
 }
 
+type bySeverity []messages.Alert
+
+func (a bySeverity) Len() int      { return len(a) }
+func (a bySeverity) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a bySeverity) Less(i, j int) bool {
+        return a[i].Severity < a[j].Severity
+}
+
 func mainLoop(ctx context.Context, a *analyzer.Analyzer, trees map[string]bool) error {
         done := make(chan interface{})
         errs := make(chan error)
         for treeName := range trees {
                 go func(tree string) {
                         expvars.Add(fmt.Sprintf("Tree-%s", tree), 1)
                         defer expvars.Add(fmt.Sprintf("Tree-%s", tree), -1)
                         log.Infof("Checking tree: %s", tree)
                         masterNames := []string{}
                         t := gkt[tree]
                         for _, url := range t.Masters {
                                 masterNames = append(masterNames, masterFromURL(url))
                         }
 
                         // TODO(seanmccullough): Plumb ctx through the rest of these calls.
                         bes := fetchBuildExtracts(a.Reader, masterNames)
                         log.Infof("Build Extracts read: %d", len(bes))
 
                         alerts := &messages.Alerts{
                                 RevisionSummaries: map[string]messages.RevisionSummary{},
                         }
                         for masterName, be := range bes {
                                 alerts.Alerts = append(alerts.Alerts, analyzeBuildExtract(a, masterName, be)...)
                         }
 
+                        sort.Sort(bySeverity(alerts.Alerts))
+
                         // Make sure we have summaries for each revision implicated in a builder failure.
                         for _, alert := range alerts.Alerts {
                                 if bf, ok := alert.Extension.(messages.BuildFailure); ok {
                                         for _, r := range bf.RegressionRanges {
                                                 revs, err := a.GetRevisionSummaries(r.Revisions)
                                                 if err != nil {
                                                         log.Errorf("Couldn't get revision summaries: %v", err)
                                                         continue
                                                 }
                                                 for _, rev := range revs {
@@ skipping 94 matching lines @@
         }
 
         err = readJSONFile(*gatekeeperJSON, &gk)
         if err != nil {
                 log.Errorf("Error reading gatekeeper json: %v", err)
                 os.Exit(1)
         }
 
         err = readJSONFile(*gatekeeperTreesJSON, &gkt)
         if err != nil {
                 log.Errorf("Error reading gatekeeper json: %v", err)

[martiniss, 2016/04/11 21:33:34]

Maybe update this error to say "Error reading gatekeeper tree json" ?

[seanmccullough, 2016/04/11 23:11:08]

Done.

                 os.Exit(1)
         }
 
         if *snapshot != "" && *replay != "" {
                 log.Errorf("Cannot use snapshot and replay flags at the same time.")
                 os.Exit(1)
         }
 
         r := client.NewReader(transport)
 
@@ skipping 10 matching lines @@
                 t, err := time.Parse(time.RFC3339, *replayTime)
                 if err != nil {
                         log.Errorf("Couldn't parse replay-time: %s", err)
                         os.Exit(1)
                 }
                 start := time.Now()
                 a.Now = func() time.Time {
                         diff := time.Now().Sub(start)
                         return t.Add(diff)
                 }
+        } else if *replay != "" {
+                f, err := os.Open(*replay)
+                defer f.Close()
+                if err != nil {

[martiniss, 2016/04/11 21:33:34]

I think you need to handle the error before you close the result...?

[seanmccullough, 2016/04/11 23:11:08]

defer runs its argument right before the return of the function (not here). f
itself is evaluated on this line, which may sound risky since f can be nil if
Open fails. However, Close is defined for nil os.File pointers:
https://golang.org/src/os/file_unix.go?s=3525:3553#L118 (many standard library
functions handle nil receiver values like this without panicking. It makes the
calling code more robust and less likely to break in some common situations).

+                        log.Errorf("Couldn't open replay dir: %s", err)
+                        os.Exit(1)
+                }
+                stat, err := f.Stat()
+                if err != nil {
+                        log.Errorf("Couldn't stat replay dir: %s", err)
+                        os.Exit(1)
+                }
+                start := time.Now()
+                t := stat.ModTime()
+
+                a.Now = func() time.Time {
+                        diff := time.Now().Sub(start)
+                        return t.Add(diff)
+                }
         }
 
-        for masterURL, masterCfgs := range gk.Masters {
-                if len(masterCfgs) != 1 {
-                        log.Errorf("Multiple configs for master: %s", masterURL)
-                }
-                masterName := masterFromURL(masterURL)
-                a.MasterCfgs[masterName] = masterCfgs[0]
-        }
+        a.Gatekeeper = analyzer.NewGatekeeperRules(*gk)

[martiniss, 2016/04/11 21:33:34]

I can't seem to find this function... where is it?

[seanmccullough, 2016/04/11 23:11:08]

Forgot to add a file. Added.

 
         a.MasterOnly = *masterOnly
         a.BuilderOnly = *builderOnly
         a.BuildOnly = *buildOnly
 
         trees := map[string]bool{}
         if *treesOnly != "" {
                 for _, treeOnly := range strings.Split(*treesOnly, ",") {
                         trees[treeOnly] = true
                 }
@@ skipping 20 matching lines @@
         logging.Set(ctx, log)
         defer cancel()
 
         loopResults := looper.Run(ctx, f, cycle, *maxErrs, clock.GetSystemClock())
 
         if !loopResults.Success {
                 log.Errorf("Failed to run loop, %v errors", loopResults.Errs)
                 os.Exit(1)
         }
 }