Fuzzer now deduplicates on the analysis side instead of the download side

fuzzer/go/fuzzer-be/main.go

 package main
 
 /*
 Runs the backend portions of the fuzzer.  This includes the generator and aggregator parts (see DESIGN.md)
 */
 
 import (
         "flag"
         "fmt"
         "os"
         "path/filepath"
         "time"
 
         "github.com/skia-dev/glog"
         "go.skia.org/infra/fuzzer/go/aggregator"
         "go.skia.org/infra/fuzzer/go/backend"
         fcommon "go.skia.org/infra/fuzzer/go/common"
         "go.skia.org/infra/fuzzer/go/config"
+        "go.skia.org/infra/fuzzer/go/data"
         "go.skia.org/infra/fuzzer/go/generator"
+        fstorage "go.skia.org/infra/fuzzer/go/storage"
         "go.skia.org/infra/go/auth"
         "go.skia.org/infra/go/common"
         "go.skia.org/infra/go/fileutil"
         "go.skia.org/infra/go/influxdb"
         "golang.org/x/net/context"
         "google.golang.org/cloud"
         "google.golang.org/cloud/storage"
 )
 
 var (
@@ skipping 49 matching lines @@
         }
         if err := fcommon.DownloadSkiaVersionForFuzzing(storageClient, config.Generator.SkiaRoot, &config.Generator); err != nil {
                 glog.Fatalf("Problem downloading Skia: %s", err)
         }
 
         fuzzPipelines := make([]backend.FuzzPipeline, 0, len(*fuzzesToRun))
 
         for _, category := range *fuzzesToRun {
                 gen := generator.New(category)
                 if err := gen.DownloadSeedFiles(storageClient); err != nil {
-                        glog.Fatalf("Problem downloading binary seed files: %s", err)
+                        glog.Fatalf("Problem downloading seed files: %s", err)
                 }
 
-                // If we are reanalyzing, no point in running the generator first, just to stop it.
+                var startingReports <-chan data.FuzzReport
+                // If we are reanalyzing, no point in running the generator first, just to stop it, nor
+                // is there reason to download all of the current fuzz reports.
                 if !*forceReanalysis {
                         glog.Infof("Starting %s generator with configuration %#v", category, config.Generator)
-                        if err := gen.Start(); err != nil {
-                                glog.Fatalf("Problem starting binary generator: %s", err)
+                        var err error
+                        if err = gen.Start(); err != nil {
+                                glog.Fatalf("Problem starting generator: %s", err)
+                        }
+                        glog.Infof("Downloading all bad %s fuzzes @%sto setup duplication detection", category, config.Generator.SkiaVersion.Hash)
+                        baseFolder := fmt.Sprintf("%s/%s/bad", category, config.Generator.SkiaVersion.Hash)
+                        if startingReports, err = fstorage.GetReportsFromGS(storageClient, baseFolder, category, nil, config.Generator.NumDownloadProcesses); err != nil {
+                                glog.Fatalf("Could not download previously found fuzzes for deduplication: %s", err)
                         }
                 } else {
-                        glog.Infof("Skipping %s generator because --skip_generation is enabled", category)
+                        glog.Infof("Skipping %s generator and deduplication setup because --force_reanalysis is enabled", category)
+
                 }
 
                 glog.Infof("Starting %s aggregator with configuration %#v", category, config.Aggregator)
-                agg, err := aggregator.StartAggregator(storageClient, category)
+                agg, err := aggregator.StartAggregator(storageClient, category, startingReports)
                 if err != nil {
                         glog.Fatalf("Could not start aggregator: %s", err)
                 }
                 fuzzPipelines = append(fuzzPipelines, backend.FuzzPipeline{
                         Category: category,
                         Agg:      agg,
                         Gen:      gen,
                 })
         }
 
@@ skipping 36 matching lines @@
         }
 
         config.Common.VerboseBuilds = *verboseBuilds
         config.Common.ClangPath = *clangPath
         config.Common.ClangPlusPlusPath = *clangPlusPlusPath
         config.Common.DepotToolsPath = *depotToolsPath
         config.Generator.NumFuzzProcesses = *numFuzzProcesses
         config.Generator.WatchAFL = *watchAFL
         config.Generator.VersionCheckPeriod = *versionCheckPeriod
         config.Generator.NumDownloadProcesses = *downloadProcesses
+        config.Generator.SkipGeneration = *skipGeneration
 
         config.GS.Bucket = *bucket
         config.Aggregator.FuzzPath, err = fileutil.EnsureDirExists(*fuzzPath)
         if err != nil {
                 return err
         }
         config.Aggregator.ExecutablePath, err = fileutil.EnsureDirExists(*executablePath)
         if err != nil {
                 return err
         }
@@ skipping 17 matching lines @@
         client, err := auth.NewDefaultJWTServiceAccountClient(auth.SCOPE_READ_WRITE)
         if err != nil {
                 return fmt.Errorf("Problem setting up client OAuth: %v", err)
         }
 
         if storageClient, err = storage.NewClient(context.Background(), cloud.WithBaseHTTP(client)); err != nil {
                 return fmt.Errorf("Problem authenticating: %v", err)
         }
         return nil
 }