Add AddressSanitizer to fuzzer analysis

fuzzer/go/aggregator/aggregator.go

 package aggregator
 
 import (
         "bytes"
         "crypto/sha1"
         "fmt"
         "io"
         "io/ioutil"
         "os"
         "path/filepath"
@@ skipping 54 matching lines @@
         aggregationShutdown  chan bool
         aggregationWaitGroup *sync.WaitGroup
         // These three counts are used to determine if there is any pending work.
         // There is no pending work if all three of these values are equal and the
         // work queues are empty.
         analysisCount  int64
         uploadCount    int64
         bugReportCount int64
 }
 
-// AnalysisPackage is a generic holder for the functions needed to analyze
-type AnalysisPackage struct {
-        Setup   func(workingDirPath string) error
-        Analyze func(workingDirPath, pathToFile string) (uploadPackage, error)
-}
-
 const (
         BAD_FUZZ  = "bad"
         GREY_FUZZ = "grey"
 )
 
+var (
+        CLANG_DEBUG   = common.TEST_HARNESS_NAME + "_clang_debug"
+        CLANG_RELEASE = common.TEST_HARNESS_NAME + "_clang_release"
+        ASAN_DEBUG    = common.TEST_HARNESS_NAME + "_asan_debug"
+        ASAN_RELEASE  = common.TEST_HARNESS_NAME + "_asan_release"
+)
+
 // uploadPackage is a struct containing all the pieces of a fuzz that need to be uploaded to GCS
 type uploadPackage struct {
-        Name        string
-        FilePath    string
-        DebugDump   string
-        DebugErr    string
-        ReleaseDump string
-        ReleaseErr  string
-        FileType    string
+        Data     data.GCSPackage
+        FilePath string
         // Must be BAD_FUZZ or GREY_FUZZ
         FuzzType string
 }
 
 // bugReportingPackage is a struct containing the pieces of a fuzz that may need to have
 // a bug filed or updated.
 type bugReportingPackage struct {
         FuzzName   string
         CommitHash string
         IsBadFuzz  bool
 }
 
 // StartAggregator creates and starts a Aggregator.
 // If there is a problem starting up, an error is returned.  Other errors will be logged.
 func StartAggregator(s *storage.Client, category string) (*Aggregator, error) {
         b := Aggregator{
                 Category:           category,
                 storageClient:      s,
                 fuzzPath:           filepath.Join(config.Aggregator.FuzzPath, category),
                 executablePath:     filepath.Join(config.Aggregator.ExecutablePath, category),
                 forAnalysis:        make(chan string, 10000),
                 forUpload:          make(chan uploadPackage, 100),
                 forBugReporting:    make(chan bugReportingPackage, 100),
                 MakeBugOnBadFuzz:   false,
+                UploadGreyFuzzes:   false,
                 monitoringShutdown: make(chan bool, 2),
                 // aggregationShutdown needs to be created with a calculated capacity in start
         }
 
         return &b, b.start()
 }
 
 // start starts up the Aggregator.  It refreshes all status it needs and builds a debug and a
 // release version of Skia for use in analysis.  It then spawns the aggregation pipeline and a
 // monitoring thread.
 func (agg *Aggregator) start() error {
         // Set the wait groups to fresh
         agg.monitoringWaitGroup = &sync.WaitGroup{}
         agg.aggregationWaitGroup = &sync.WaitGroup{}
         agg.analysisCount = 0
         agg.uploadCount = 0
         agg.bugReportCount = 0
-        if _, err := fileutil.EnsureDirExists(agg.fuzzPath); err != nil {
-                return err
-        }
-        if _, err := fileutil.EnsureDirExists(agg.executablePath); err != nil {
-                return err
-        }
-        if err := common.BuildClangHarness("Debug", true); err != nil {
-                return err
-        }
-        if err := common.BuildClangHarness("Release", true); err != nil {
+        if err := agg.buildAnalysisBinaries(); err != nil {
                 return err
         }
 
         agg.monitoringWaitGroup.Add(1)
         go agg.scanForNewCandidates()
 
         numAnalysisProcesses := config.Aggregator.NumAnalysisProcesses
         if numAnalysisProcesses <= 0 {
                 // TODO(kjlubick): Actually make this smart based on the number of cores
                 numAnalysisProcesses = 20
@@ skipping 15 matching lines @@
         }
         agg.aggregationWaitGroup.Add(1)
         go agg.waitForBugReporting()
         agg.aggregationShutdown = make(chan bool, numAnalysisProcesses+numUploadProcesses+1)
         // start background routine to monitor queue details
         agg.monitoringWaitGroup.Add(1)
         go agg.monitorStatus(numAnalysisProcesses, numUploadProcesses)
         return nil
 }
 
+// buildAnalysisBinaries creates the 4 executables we need to perform analysis and makes a copy of
+// them in the executablePath.  We need (Debug,Release) x (Clang,ASAN).  The copied binaries have
+// a suffix like _clang_debug
+func (agg *Aggregator) buildAnalysisBinaries() error {
+        if _, err := fileutil.EnsureDirExists(agg.fuzzPath); err != nil {
+                return err
+        }
+        if _, err := fileutil.EnsureDirExists(agg.executablePath); err != nil {
+                return err
+        }
+        if err := common.BuildClangHarness("Debug", true); err != nil {
+                return err
+        }
+        outPath := filepath.Join(config.Generator.SkiaRoot, "out")
+        if err := fileutil.CopyExecutable(filepath.Join(outPath, "Debug", common.TEST_HARNESS_NAME), filepath.Join(agg.executablePath, CLANG_DEBUG)); err != nil {
+                return err
+        }
+        if err := common.BuildClangHarness("Release", true); err != nil {
+                return err
+        }
+        if err := fileutil.CopyExecutable(filepath.Join(outPath, "Release", common.TEST_HARNESS_NAME), filepath.Join(agg.executablePath, CLANG_RELEASE)); err != nil {
+                return err
+        }
+        if err := common.BuildASANHarness("Debug", false); err != nil {
+                return err
+        }
+        if err := fileutil.CopyExecutable(filepath.Join(outPath, "Debug", common.TEST_HARNESS_NAME), filepath.Join(agg.executablePath, ASAN_DEBUG)); err != nil {
+                return err
+        }
+        if err := common.BuildASANHarness("Release", false); err != nil {
+                return err
+        }
+        if err := fileutil.CopyExecutable(filepath.Join(outPath, "Release", common.TEST_HARNESS_NAME), filepath.Join(agg.executablePath, ASAN_RELEASE)); err != nil {
+                return err
+        }
+        return nil
+}
+
 // scanForNewCandidates runs scanHelper once every config.Aggregator.RescanPeriod, which scans the
 // config.Generator.AflOutputPath for new fuzzes.  If scanHelper returns an error, this method
 // will terminate.
 func (agg *Aggregator) scanForNewCandidates() {
         defer agg.monitoringWaitGroup.Done()
 
         alreadyFoundFuzzes := &SortedStringSlice{}
         // time.Tick does not fire immediately, so we fire it manually once.
         if err := agg.scanHelper(alreadyFoundFuzzes); err != nil {
                 glog.Errorf("[%s] Scanner terminated due to error: %v", agg.Category, err)
@@ skipping 47 matching lines @@
 //                      -hangs/
 //                      -queue/
 //                      -fuzzer_stats
 //              -fuzzer1/
 //              ...
 func (agg *Aggregator) findBadFuzzPaths(alreadyFoundFuzzes *SortedStringSlice) ([]string, error) {
         badFuzzPaths := make([]string, 0)
 
         scanPath := filepath.Join(config.Generator.AflOutputPath, agg.Category)
         aflDir, err := os.Open(scanPath)
+        if os.IsNotExist(err) {
+                glog.Warningf("Path to scan %s does not exist.  Returning 0 found fuzzes", scanPath)
+                return []string{}, nil
+        }
         if err != nil {
                 return nil, err
         }
         defer util.Close(aflDir)
 
         fuzzerFolders, err := aflDir.Readdir(-1)
         if err != nil {
                 return nil, err
         }
 
@@ skipping 92 matching lines @@
 // in that directory.
 func (agg *Aggregator) setupAnalysis(workingDirPath string) error {
         // Delete all previous executables to get a clean start
         if err := os.RemoveAll(workingDirPath); err != nil && !os.IsNotExist(err) {
                 return err
         }
         if err := os.MkdirAll(workingDirPath, 0755); err != nil {
                 return err
         }
 
-        // make a copy of the debug and release executables
-        basePath := filepath.Join(config.Generator.SkiaRoot, "out")
-        if err := fileutil.CopyExecutable(filepath.Join(basePath, "Debug", common.TEST_HARNESS_NAME), filepath.Join(workingDirPath, common.TEST_HARNESS_NAME+"_debug")); err != nil {
+        // make a copy of the 4 executables that were made in buildAnalysisBinaries()
+        if err := fileutil.CopyExecutable(filepath.Join(agg.executablePath, CLANG_DEBUG), filepath.Join(workingDirPath, CLANG_DEBUG)); err != nil {
                 return err
         }
-        if err := fileutil.CopyExecutable(filepath.Join(basePath, "Release", common.TEST_HARNESS_NAME), filepath.Join(workingDirPath, common.TEST_HARNESS_NAME+"_release")); err != nil {
+        if err := fileutil.CopyExecutable(filepath.Join(agg.executablePath, CLANG_RELEASE), filepath.Join(workingDirPath, CLANG_RELEASE)); err != nil {
+                return err
+        }
+        if err := fileutil.CopyExecutable(filepath.Join(agg.executablePath, ASAN_DEBUG), filepath.Join(workingDirPath, ASAN_DEBUG)); err != nil {
+                return err
+        }
+        if err := fileutil.CopyExecutable(filepath.Join(agg.executablePath, ASAN_RELEASE), filepath.Join(workingDirPath, ASAN_RELEASE)); err != nil {
                 return err
         }
         return nil
 }
 
 // analyze simply invokes performAnalysis with a fuzz under both the Debug and Release build.  Upon
 // completion, it checks to see if the fuzz is a grey fuzz and sets the FuzzType accordingly.
 func (agg *Aggregator) analyze(workingDirPath, fileName string) (uploadPackage, error) {
         upload := uploadPackage{
-                Name:     fileName,
-                FileType: agg.Category,
+                Data: data.GCSPackage{
+                        Name:         fileName,
+                        FuzzCategory: agg.Category,
+                },
                 FuzzType: BAD_FUZZ,
                 FilePath: filepath.Join(agg.fuzzPath, fileName),
         }
 
-        if dump, stderr, err := agg.performAnalysis(workingDirPath, common.TEST_HARNESS_NAME, upload.FilePath, true); err != nil {
+        if dump, stderr, err := agg.performAnalysis(workingDirPath, CLANG_DEBUG, upload.FilePath); err != nil {
                 return upload, err
         } else {
-                upload.DebugDump = dump
-                upload.DebugErr = stderr
+                upload.Data.Debug.Dump = dump
+                upload.Data.Debug.StdErr = stderr
         }
-        if dump, stderr, err := agg.performAnalysis(workingDirPath, common.TEST_HARNESS_NAME, upload.FilePath, false); err != nil {
+        if dump, stderr, err := agg.performAnalysis(workingDirPath, CLANG_RELEASE, upload.FilePath); err != nil {
                 return upload, err
         } else {
-                upload.ReleaseDump = dump
-                upload.ReleaseErr = stderr
+                upload.Data.Release.Dump = dump
+                upload.Data.Release.StdErr = stderr
         }
-        if r := data.ParseFuzzResult(upload.DebugDump, upload.DebugErr, upload.ReleaseDump, upload.ReleaseErr); r.Flags == data.DebugFailedGracefully|data.ReleaseFailedGracefully {
+        // AddressSanitizer only outputs to stderr
+        if _, stderr, err := agg.performAnalysis(workingDirPath, ASAN_DEBUG, upload.FilePath); err != nil {
+                return upload, err
+        } else {
+                upload.Data.Debug.Asan = stderr
+        }
+        if _, stderr, err := agg.performAnalysis(workingDirPath, ASAN_RELEASE, upload.FilePath); err != nil {
+                return upload, err
+        } else {
+                upload.Data.Release.Asan = stderr
+        }
+        if r := data.ParseGCSPackage(upload.Data); r.Debug.Flags == data.TerminatedGracefully && r.Release.Flags == data.TerminatedGracefully {
                 upload.FuzzType = GREY_FUZZ
         }
         return upload, nil
 }
 
 // performAnalysis executes a command from the working dir specified using
 // AnalysisArgs for a given fuzz category. The crash dumps (which
 // come via standard out) and standard errors are recorded as strings.
-func (agg *Aggregator) performAnalysis(workingDirPath, baseExecutableName, pathToFile string, isDebug bool) (string, string, error) {
-        suffix := "_release"
-        if isDebug {
-                suffix = "_debug"
-        }
-
-        pathToExecutable := fmt.Sprintf("./%s%s", baseExecutableName, suffix)
+func (agg *Aggregator) performAnalysis(workingDirPath, executableName, pathToFile string) (string, string, error) {
 
         var dump bytes.Buffer
         var stdErr bytes.Buffer
 
         // GNU timeout is used instead of the option on exec.Command because experimentation with the
         // latter showed evidence of that way leaking processes, which led to OOM errors.
         cmd := &exec.Command{
-                Name:      "timeout",
-                Args:      common.AnalysisArgsFor(agg.Category, pathToExecutable, pathToFile),
-                LogStdout: false,
-                LogStderr: false,
-                Stdout:    &dump,
-                Stderr:    &stdErr,
-                Dir:       workingDirPath,
+                Name:        "timeout",
+                Args:        common.AnalysisArgsFor(agg.Category, "./"+executableName, pathToFile),
+                LogStdout:   false,
+                LogStderr:   false,
+                Stdout:      &dump,
+                Stderr:      &stdErr,
+                Dir:         workingDirPath,
+                InheritPath: true,
+                Env:         []string{common.ASAN_OPTIONS},
         }
 
         //errors are fine/expected from this, as we are dealing with bad fuzzes
         if err := exec.Run(cmd); err != nil {
                 return dump.String(), stdErr.String(), nil
         }
         return dump.String(), stdErr.String(), nil
 }
 
 // calcuateHash calculates the sha1 hash of a file, given its path.  It returns both the hash as a
@@ skipping 31 matching lines @@
 // them.  If any unrecoverable errors happen, this method terminates.
 func (agg *Aggregator) waitForUploads(identifier int) {
         defer agg.aggregationWaitGroup.Done()
         defer go_metrics.GetOrRegisterCounter("upload_process_count", go_metrics.DefaultRegistry).Dec(int64(1))
         glog.Infof("[%s] Spawning uploader %d", agg.Category, identifier)
         for {
                 select {
                 case p := <-agg.forUpload:
                         atomic.AddInt64(&agg.uploadCount, int64(1))
                         if !agg.UploadGreyFuzzes && p.FuzzType == GREY_FUZZ {
-                                glog.Infof("[%s] Skipping upload of grey fuzz %s", agg.Category, p.Name)
+                                glog.Infof("[%s] Skipping upload of grey fuzz %s", agg.Category, p.Data.Name)
                                 continue
                         }
                         if err := agg.upload(p); err != nil {
                                 glog.Errorf("[%s] Uploader %d terminated due to error: %s", agg.Category, identifier, err)
                                 return
                         }
                         agg.forBugReporting <- bugReportingPackage{
-                                FuzzName:   p.Name,
+                                FuzzName:   p.Data.Name,
                                 CommitHash: config.Generator.SkiaVersion.Hash,
                                 IsBadFuzz:  p.FuzzType == BAD_FUZZ,
                         }
                 case <-agg.aggregationShutdown:
                         glog.Infof("[%s] Uploader %d recieved shutdown signal", agg.Category, identifier)
                         return
                 }
         }
 }
 
 // upload breaks apart the uploadPackage into its constituant parts and uploads them to GCS using
 // some helper methods.
 func (agg *Aggregator) upload(p uploadPackage) error {
-        glog.Infof("[%s] uploading %s with file %s and analysis bytes %d;%d;%d;%d", agg.Category, p.Name, p.FilePath, len(p.DebugDump), len(p.DebugErr), len(p.ReleaseDump), len(p.ReleaseErr))
+        glog.Infof("[%s] uploading %s with file %s and analysis bytes %d;%d;%d|%d;%d;%d", agg.Category, p.Data.Name, p.FilePath, len(p.Data.Debug.Asan), len(p.Data.Debug.Dump), len(p.Data.Debug.StdErr), len(p.Data.Release.Asan), len(p.Data.Release.Dump), len(p.Data.Release.StdErr))
 
-        if err := agg.uploadBinaryFromDisk(p, p.Name, p.FilePath); err != nil {
+        if err := agg.uploadBinaryFromDisk(p, p.Data.Name, p.FilePath); err != nil {
                 return err
         }
-        if err := agg.uploadString(p, p.Name+"_debug.dump", p.DebugDump); err != nil {
+        if err := agg.uploadString(p, p.Data.Name+"_debug.asan", p.Data.Debug.Asan); err != nil {
                 return err
         }
-        if err := agg.uploadString(p, p.Name+"_debug.err", p.DebugErr); err != nil {
+        if err := agg.uploadString(p, p.Data.Name+"_debug.dump", p.Data.Debug.Dump); err != nil {
                 return err
         }
-        if err := agg.uploadString(p, p.Name+"_release.dump", p.ReleaseDump); err != nil {
+        if err := agg.uploadString(p, p.Data.Name+"_debug.err", p.Data.Debug.StdErr); err != nil {
                 return err
         }
-        return agg.uploadString(p, p.Name+"_release.err", p.ReleaseErr)
+        if err := agg.uploadString(p, p.Data.Name+"_release.asan", p.Data.Release.Asan); err != nil {
+                return err
+        }
+        if err := agg.uploadString(p, p.Data.Name+"_release.dump", p.Data.Release.Dump); err != nil {
+                return err
+        }
+        return agg.uploadString(p, p.Data.Name+"_release.err", p.Data.Release.StdErr)
 }
 
 // uploadBinaryFromDisk uploads a binary file on disk to GCS, returning an error if anything
 // goes wrong.
 func (agg *Aggregator) uploadBinaryFromDisk(p uploadPackage, fileName, filePath string) error {
-        name := fmt.Sprintf("%s/%s/%s/%s/%s", p.FileType, config.Generator.SkiaVersion.Hash, p.FuzzType, p.Name, fileName)
+        name := fmt.Sprintf("%s/%s/%s/%s/%s", p.Data.FuzzCategory, config.Generator.SkiaVersion.Hash, p.FuzzType, p.Data.Name, fileName)
         w := agg.storageClient.Bucket(config.GS.Bucket).Object(name).NewWriter(context.Background())
         defer util.Close(w)
         // We set the encoding to avoid accidental crashes if Chrome were to try to render a fuzzed png
         // or svg or something.
         w.ObjectAttrs.ContentEncoding = "application/octet-stream"
 
         f, err := os.Open(filePath)
         if err != nil {
                 return fmt.Errorf("There was a problem reading %s for uploading : %s", filePath, err)
         }
 
         if n, err := io.Copy(w, f); err != nil {
                 return fmt.Errorf("There was a problem uploading binary file %s.  Only uploaded %d bytes : %s", name, n, err)
         }
         return nil
 }
 
 // uploadBinaryFromDisk uploads the contents of a string as a file to GCS, returning an error if
 // anything goes wrong.
 func (agg *Aggregator) uploadString(p uploadPackage, fileName, contents string) error {
-        name := fmt.Sprintf("%s/%s/%s/%s/%s", p.FileType, config.Generator.SkiaVersion.Hash, p.FuzzType, p.Name, fileName)
+        name := fmt.Sprintf("%s/%s/%s/%s/%s", p.Data.FuzzCategory, config.Generator.SkiaVersion.Hash, p.FuzzType, p.Data.Name, fileName)
         w := agg.storageClient.Bucket(config.GS.Bucket).Object(name).NewWriter(context.Background())
         defer util.Close(w)
         w.ObjectAttrs.ContentEncoding = "text/plain"
 
         if n, err := w.Write([]byte(contents)); err != nil {
                 return fmt.Errorf("There was a problem uploading %s.  Only uploaded %d bytes: %s", name, n, err)
         }
         return nil
 }
 
@@ skipping 97 matching lines @@
                 }
                 glog.Infof("[%s] Waiting %s for the aggregator's queues to be empty", agg.Category, config.Aggregator.StatusPeriod)
         }
 }
 
 // ForceAnalysis directly adds the given path to the analysis queue, where it will be analyzed,
 // uploaded and possibly bug reported.
 func (agg *Aggregator) ForceAnalysis(path string) {
         agg.forAnalysis <- path
 }