Deduplicate fuzz cases on the frontend before they are displayed.

fuzzer/go/frontend/gsloader/gsloader.go

 package gsloader
 
 import (
         "fmt"
         "sort"
         "sync"
         "sync/atomic"
 
         "github.com/skia-dev/glog"
         "go.skia.org/infra/fuzzer/go/common"
         "go.skia.org/infra/fuzzer/go/config"
+        "go.skia.org/infra/fuzzer/go/deduplicator"
         "go.skia.org/infra/fuzzer/go/frontend/data"
         "go.skia.org/infra/fuzzer/go/fuzzcache"
         "go.skia.org/infra/go/gs"
         "google.golang.org/cloud/storage"
 )
 
 // LoadFromBoltDB loads the data.FuzzReport from FuzzReportCache associated with the given hash.
 // The FuzzReport is first put into the staging fuzz cache, and then into the current.
 // If a cache for the commit does not exist, or there are other problems with the retrieval,
-// an error is returned.
+// an error is returned.  We do not need to deduplicate on extraction because
+// the fuzzes were deduplicated on storage.
 func LoadFromBoltDB(cache *fuzzcache.FuzzReportCache) error {
         glog.Infof("Looking into cache for revision %s", config.FrontEnd.SkiaVersion.Hash)
         for _, category := range common.FUZZ_CATEGORIES {
                 if staging, err := cache.LoadTree(category, config.FrontEnd.SkiaVersion.Hash); err != nil {
                         return fmt.Errorf("Problem decoding existing from bolt db: %s", err)
                 } else {
                         data.SetStaging(category, *staging)
                         glog.Infof("Successfully loaded %s fuzzes from bolt db cache", category)
                 }
         }
         data.StagingToCurrent()
         return nil
 }
 
 // GSLoader is a struct that handles downloading fuzzes from Google Storage.
 type GSLoader struct {
         storageClient *storage.Client
         Cache         *fuzzcache.FuzzReportCache
+        deduplicator  *deduplicator.Deduplicator
 
         // completedCounter is the number of fuzzes that have been downloaded from GCS, used for logging.
         completedCounter int32
 }
 
 // New creates a GSLoader and returns it.
 func New(s *storage.Client, c *fuzzcache.FuzzReportCache) *GSLoader {
         return &GSLoader{
                 storageClient: s,
                 Cache:         c,
+                deduplicator:  deduplicator.New(),
         }
 }
 
 // LoadFreshFromGoogleStorage pulls all fuzzes out of GCS and loads them into memory.
 // The "fresh" in the name refers to the fact that all other loaded fuzzes (if any)
 // are written over, including in the cache.
 // Upon completion, the full results are cached to a boltDB instance and moved from staging
 // to the current copy.
 func (g *GSLoader) LoadFreshFromGoogleStorage() error {
         revision := config.FrontEnd.SkiaVersion.Hash
         data.ClearStaging()
+        g.deduplicator.Clear()
         fuzzNames := make([]string, 0, 100)
         for _, cat := range common.FUZZ_CATEGORIES {
                 badPath := fmt.Sprintf("%s/%s/bad", cat, revision)
                 reports, err := g.getBinaryReportsFromGS(badPath, cat, nil)
                 if err != nil {
                         return err
                 }
-                n := 0
+                b := 0
+                d := 0
                 for report := range reports {
-                        data.NewFuzzFound(cat, report)
+                        // We always add the fuzzName, to avoid redownloading duplicates over and over again.
                         fuzzNames = append(fuzzNames, report.FuzzName)
-                        n++
+                        if g.deduplicator.IsUnique(report) {
+                                data.NewFuzzFound(cat, report)
+                                b++
+                        } else {
+                                d++
+                        }
+
                 }
-                glog.Infof("%d bad fuzzes freshly loaded from gs://%s/%s", n, config.GS.Bucket, badPath)
+                glog.Infof("%d bad fuzzes (%d duplicate) freshly loaded from gs://%s/%s", b, d, config.GS.Bucket, badPath)
+                data.StagingToCurrent()
         }
 
-        data.StagingToCurrent()
         for _, category := range common.FUZZ_CATEGORIES {
                 if err := g.Cache.StoreTree(data.StagingCopy(category), category, revision); err != nil {
                         glog.Errorf("Problem storing category %s to boltDB: %s", category, err)
                 }
         }
         return g.Cache.StoreFuzzNames(fuzzNames, revision)
 }
 
 // LoadBinaryFuzzesFromGoogleStorage pulls all fuzzes out of GCS that are on the given whitelist
 // and loads them into memory (as staging).  After loading them, it updates the cache
 // and moves them from staging to the current copy.
 func (g *GSLoader) LoadBinaryFuzzesFromGoogleStorage(whitelist []string) error {
         revision := config.FrontEnd.SkiaVersion.Hash
         data.StagingFromCurrent()
         sort.Strings(whitelist)
 
         fuzzNames := make([]string, 0, 100)
         for _, cat := range common.FUZZ_CATEGORIES {
                 badPath := fmt.Sprintf("%s/%s/bad", cat, revision)
                 reports, err := g.getBinaryReportsFromGS(badPath, cat, whitelist)
                 if err != nil {
                         return err
                 }
-                n := 0
+                b := 0
+                d := 0
                 for report := range reports {
-                        data.NewFuzzFound(cat, report)
+                        // We always add the fuzzName, to avoid redownloading duplicates over and over again.
                         fuzzNames = append(fuzzNames, report.FuzzName)
-                        n++
+                        if g.deduplicator.IsUnique(report) {
+                                data.NewFuzzFound(cat, report)
+                                b++
+                        } else {
+                                d++
+                        }
                 }
-                glog.Infof("%d bad fuzzes freshly loaded from gs://%s/%s", n, config.GS.Bucket, badPath)
+                glog.Infof("%d bad fuzzes (%d duplicate) incrementally loaded from gs://%s/%s", b, d, config.GS.Bucket, badPath)
         }
         data.StagingToCurrent()
 
         oldBinaryFuzzNames, err := g.Cache.LoadFuzzNames(revision)
         if err != nil {
                 glog.Warningf("Could not read old binary fuzz names from cache.  Continuing...", err)
                 oldBinaryFuzzNames = []string{}
         }
         for _, category := range common.FUZZ_CATEGORIES {
                 if err := g.Cache.StoreTree(data.StagingCopy(category), category, revision); err != nil {
@@ skipping 116 matching lines @@
                         },
                 }
 
                 reports <- data.ParseReport(p)
                 atomic.AddInt32(&g.completedCounter, 1)
                 if g.completedCounter%100 == 0 {
                         glog.Infof("%d fuzzes downloaded", g.completedCounter)
                 }
         }
 }