[som] add tsmon to alerts-dispatcher

go/src/infra/monitoring/dispatcher/dispatcher.go

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 // Dispatcher usage:
 // go run infra/monitoring/dispatcher
 // Expects gatekeeper.json to be in the current directory.
 
 package main
 
 import (
         "encoding/json"
         "expvar"
         "flag"
         "fmt"
         "io/ioutil"
         "log"
         "net"
         "net/http"
         "net/url"
         "os"
         "sort"
         "strings"
         "time"
 
         "github.com/luci/luci-go/common/auth"
         "github.com/luci/luci-go/common/clock"
         "github.com/luci/luci-go/common/logging"
         "github.com/luci/luci-go/common/logging/gologger"
+        "github.com/luci/luci-go/common/tsmon"
+        "github.com/luci/luci-go/common/tsmon/field"
+        "github.com/luci/luci-go/common/tsmon/metric"
 
         "golang.org/x/net/context"
 
         "infra/monitoring/analyzer"
         "infra/monitoring/client"
         "infra/monitoring/looper"
         "infra/monitoring/messages"
 )
 
 type stringSlice []string
@@ skipping 29 matching lines @@
         duration, cycle time.Duration
 
         // gk is the gatekeeper config.
         gks = []*messages.GatekeeperConfig{}
         // gkt is the gatekeeper trees config.
         gkts             = map[string][]messages.TreeMasterConfig{}
         filteredFailures = uint64(0)
         expvars          = expvar.NewMap("dispatcher")
         errLog           = log.New(os.Stderr, "", log.Lshortfile|log.Ltime)
         infoLog          = log.New(os.Stdout, "", log.Lshortfile|log.Ltime)
+
+        // tsmon metrics
+        iterations = metric.NewCounter("alerts-dispatcher/iterations",

[Vadim Sh., 2016/07/28 23:08:37]

nit: I think we use '_' as separators in /chrome/infra/... namespace, can't find
a metric that uses '-'.

[seanmccullough1, 2016/07/29 00:47:55]

Done.

+                "Number if iterations of the main polling loop per run.", field.String("status"))

[Vadim Sh., 2016/07/28 23:08:37]

'per run' is misleading, there'll be no (easy) way to figure out when different
runs happen

This metrics would allow to graph stuff like "number of iterations per minute"
(regardless of what run they happened in).

[seanmccullough1, 2016/07/29 00:47:55]

Done.

+        postErrors = metric.NewCounter("alerts-dispatcher/post-errors",
+                "Number of posting errors per run.")
+        alertCount = metric.NewInt("alerts-dispatcher/alert-count",

[Vadim Sh., 2016/07/28 23:08:37]

I think this should be counter too. Will be easier to plot that way (as a rate
of alerts per time interval). Not sure... IIUC, iterations will keep
regenerating same alerts over and over again? Maybe NewInt is better in that
case...

[seanmccullough1, 2016/07/29 00:47:55]

Exactly the latter. The number of *new* alerts since the previous iteration is
not easily determined from the information we have here.

+                "Number of alerts generated in an iteration.", field.String("tree"))
 )
 
 func init() {
         flag.Usage = func() {
                 fmt.Printf("By default runs a single check, saves any alerts to ./alerts.json and exits.")
                 flag.PrintDefaults()
         }
 }
 
 func analyzeBuildExtract(ctx context.Context, a *analyzer.Analyzer, tree string, masterURL *messages.MasterLocation, b *messages.BuildExtract) []messages.Alert {
@@ skipping 102 matching lines @@
                                                         errLog.Printf("Couldn't get revision summaries: %v", err)
                                                         continue
                                                 }
                                                 for _, rev := range revs {
                                                         alerts.RevisionSummaries[rev.GitHash] = rev
                                                 }
                                         }
                                 }
                         }
                         alerts.Timestamp = messages.TimeToEpochTime(time.Now())
+                        alertCount.Set(ctx, int64(len(alerts.Alerts)), tree)
 
                         if *alertsBaseURL == "" {
                                 infoLog.Printf("No data_url provided. Writing to %s-alerts.json", tree)
 
                                 abytes, err := json.MarshalIndent(alerts, "", "\t")
                                 if err != nil {
                                         errLog.Printf("Couldn't marshal alerts json: %v", err)
                                         errs <- err
                                         return
                                 }
 
                                 if err := ioutil.WriteFile(fmt.Sprintf("%s-alerts.json", tree), abytes, 0644); err != nil {
                                         errLog.Printf("Couldn't write to alerts.json: %v", err)
                                         errs <- err
                                         return
                                 }
                         } else {
                                 alertsURL := fmt.Sprintf("%s/%s", *alertsBaseURL, tree)
                                 w := client.NewWriter(alertsURL, transport)
                                 infoLog.Printf("Posting alerts to %s", alertsURL)
                                 err := w.PostAlerts(alerts)
                                 if err != nil {
                                         errLog.Printf("Couldn't post alerts: %v", err)
+                                        postErrors.Add(ctx, 1)
                                         errs <- err
                                         return
                                 }
                         }
 
                         infoLog.Printf("Filtered failures: %v", filteredFailures)
                         done <- nil
                 }()
         }
 
@@ skipping 10 matching lines @@
 
 func main() {
         flag.Var(&gatekeeperJSON, "gatekeeper", "Location of gatekeeper json file. Can have multiple comma separated values.")
         flag.Var(&gatekeeperTreesJSON, "gatekeeper-trees", "Location of gatekeeper tree json file. Can have multiple comma separated values.")
 
         flag.Parse()
 
         ctx := context.Background()
         ctx = gologger.StdConfig.Use(ctx)
         logging.SetLevel(ctx, logging.Debug)
-
         authOptions := auth.Options{
                 ServiceAccountJSONPath: *serviceAccountJSON,
                 Scopes: []string{
                         auth.OAuthScopeEmail,
                         "https://www.googleapis.com/auth/projecthosting",
                 },
                 Method: auth.ServiceAccountMethod,
         }
 
         mode := auth.SilentLogin
         if *login {
                 mode = auth.InteractiveLogin
         }
 
         transport, err := auth.NewAuthenticator(ctx, mode, authOptions).Transport()
         if err != nil {
                 errLog.Printf("AuthenticatedTransport: %v", err)
                 if !*login {
                         errLog.Printf("Consider re-running with -login")
                 }
                 os.Exit(1)
         }
         ctx = context.Background()
 
+        tsFlags := tsmon.NewFlags()
+        tsFlags.Target.TargetType = "task"
+        tsFlags.Target.TaskServiceName = "alerts-dispatcher"
+        tsFlags.Flush = "auto"

[Vadim Sh., 2016/07/28 23:08:37]

"manual"? Since tsmon.Flush is used manually below.

(or remove tsmon.Flush there)

[seanmccullough1, 2016/07/29 00:47:55]

Done.

+        tsmon.InitializeFromFlags(ctx, &tsFlags)
+
         // Start serving expvars.
         go func() {
                 listener, err := net.Listen("tcp", "127.0.0.1:0")
                 if err != nil {
                         errLog.Printf("Listen: %s", err)
                         os.Exit(1)
                 }
 
                 infoLog.Printf("expvars listening on %v", listener.Addr())
 
@@ skipping 108 matching lines @@
         for tree := range trees {
                 if _, ok := gkts[tree]; !ok {
                         errLog.Printf("Unrecognized tree name: %s", tree)
                         os.Exit(1)
                 }
         }
 
         // This is the polling/analysis/alert posting function, which will run in a loop until
         // a timeout or max errors is reached.
         f := func(ctx context.Context) error {
-                return mainLoop(ctx, a, trees, transport)
+                err := mainLoop(ctx, a, trees, transport)
+                if err == nil {
+                        iterations.Add(ctx, 1, "success")
+                } else {
+                        iterations.Add(ctx, 1, "failure")
+                }
+                tsmon.Flush(ctx)

[Vadim Sh., 2016/07/28 23:08:37]

how often would this be called? I think tsmon doesn't like very frequent
flushes.

[seanmccullough1, 2016/07/29 00:47:55]

Done.

+                return err
         }
 
         ctx, cancel := context.WithTimeout(ctx, duration)
         defer cancel()
 
         loopResults := looper.Run(ctx, f, cycle, *maxErrs, clock.GetSystemClock())
 
+        tsmon.Shutdown(ctx)
+
         if !loopResults.Success {
                 errLog.Printf("Failed to run loop, %v errors", loopResults.Errs)
                 os.Exit(1)
         }
 }