Add monitoring for expired ignore rules.

golden/go/ignore/monitor.go

Index: golden/go/ignore/monitor.go
diff --git a/golden/go/ignore/monitor.go b/golden/go/ignore/monitor.go
new file mode 100644
index 0000000000000000000000000000000000000000..6721a5281f9977c2b7c19bde71cf5f86af8f881f
--- /dev/null
+++ b/golden/go/ignore/monitor.go
@@ -0,0 +1,50 @@
+package ignore
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/rcrowley/go-metrics"
+	"github.com/skia-dev/glog"
+	imetrics "go.skia.org/infra/go/metrics"
+)
+
+func oneStep(store IgnoreStore, metric metrics.Gauge) error {
+	list, err := store.List()
+	if err != nil {
+		return err
+	}
+	n := 0
+	for _, rule := range list {
+		if time.Now().After(rule.Expires) {
+			n += 1
+		}
+	}
+	metric.Update(int64(n))
+	return nil
+}
+
+// StartMonitoring starts a new monitoring routine for the given
+// ignore store that counts expired ignore rules and pushes
+// that info into a metric.
+func StartMonitoring(store IgnoreStore, r metrics.Registry) error {
+	numExpired := metrics.NewRegisteredGauge("num-expired-ignore-rules", r)
+	liveness := imetrics.NewLiveness("expired-ignore-rules-monitoring")
+
+	err := oneStep(store, numExpired)
+	if err != nil {
+		return fmt.Errorf("Unable to start monitoring ignore rules: %s", err)
+	}
+	go func() {
+		for _ = range time.Tick(time.Minute) {
+			err = oneStep(store, numExpired)
+			if err != nil {
+				glog.Errorf("Failed one step of monitoring ignore rules: %s", err)
+				continue
+			}
+			liveness.Update()
+		}
+	}()
+
+	return nil
+}