Fuzzer now deduplicates on the analysis side instead of the download side

fuzzer/go/frontend/data/result.go

Index: fuzzer/go/frontend/data/result.go
diff --git a/fuzzer/go/frontend/data/result.go b/fuzzer/go/frontend/data/result.go
deleted file mode 100644
index 7dfb7cb5a67996eea5ea8c785a7ec1175b71e559..0000000000000000000000000000000000000000
--- a/fuzzer/go/frontend/data/result.go
+++ /dev/null
@@ -1,252 +0,0 @@
-package data
-
-import (
-	"fmt"
-	"regexp"
-	"sort"
-	"strings"
-
-	"go.skia.org/infra/fuzzer/go/common"
-)
-
-// Represents the metadata about a crash, hopefully easing debugging.
-type FuzzResult struct {
-	Debug   BuildData
-	Release BuildData
-}
-
-// BuildData represents the results of parsing a given skia build's output.
-type BuildData struct {
-	OutputFiles
-	StackTrace StackTrace
-	Flags      FuzzFlag
-}
-
-// OutputFiles are the files output by the analysis
-type OutputFiles struct {
-	Asan   string
-	Dump   string
-	StdErr string
-}
-
-// GCSPackage is a struct containing all the pieces of a fuzz that exist in Google Storage.
-type GCSPackage struct {
-	Name         string
-	FuzzCategory string
-	Debug        OutputFiles
-	Release      OutputFiles
-}
-
-// A bit mask representing what happened when a fuzz ran against Skia.
-type FuzzFlag int
-
-const (
-	TerminatedGracefully FuzzFlag = 1 << iota
-	ClangCrashed
-	ASANCrashed
-	AssertionViolated
-	BadAlloc
-	NoStackTrace
-	SKAbortHit
-	TimedOut
-	Other
-
-	ASAN_GlobalBufferOverflow
-	ASAN_HeapBufferOverflow
-	ASAN_StackBufferOverflow
-	ASAN_HeapUseAfterFree
-
-	SKPICTURE_DuringRendering
-)
-
-var flagNames = []string{
-	"FailedGracefully",
-	"ClangCrashed",
-	"ASANCrashed",
-	"AssertionViolated",
-	"BadAlloc",
-	"NoStackTrace",
-	"SKAbortHit",
-	"TimedOut",
-	"Other",
-
-	"ASAN_global-buffer-overflow",
-	"ASAN_heap-buffer-overflow",
-	"ASAN_stack-buffer-overflow",
-	"ASAN_heap-use-after-free",
-
-	"SKPICTURE_DuringRendering",
-}
-
-// ToHumanReadableFlags creates a sorted slice of strings that represents the flags.  The slice
-// is sorted by unicode points, as per sort.Strings().
-func (f FuzzFlag) ToHumanReadableFlags() []string {
-	flags := make([]string, 0)
-	i := 0
-	for mask := 1; mask < (2 << 16); mask *= 2 {
-		if int(f)&mask != 0 {
-			flags = append(flags, flagNames[i])
-		}
-		i++
-	}
-	// Front end filtering logic will expect the flags to be in alphabetical order.
-	sort.Strings(flags)
-	return flags
-}
-
-func (f FuzzFlag) String() string {
-	return fmt.Sprintf("FuzzFlag: %016b (%d) [%s]", f, f, strings.Join(f.ToHumanReadableFlags(), " | "))
-}
-
-// ParseGCSPackage parses the results of analysis of a fuzz and creates a FuzzResult with it.
-// This includes parsing the stacktraces and computing the flags about the fuzz.
-func ParseGCSPackage(g GCSPackage) FuzzResult {
-	result := FuzzResult{}
-	result.Debug.Asan = g.Debug.Asan
-	result.Debug.Dump = g.Debug.Dump
-	result.Debug.StdErr = g.Debug.StdErr
-	result.Debug.StackTrace = getStackTrace(g.Debug.Asan, g.Debug.Dump)
-	result.Release.Asan = g.Release.Asan
-	result.Release.Dump = g.Release.Dump
-	result.Release.StdErr = g.Release.StdErr
-	result.Release.StackTrace = getStackTrace(g.Release.Asan, g.Release.Dump)
-	result.computeFlags(g.FuzzCategory)
-
-	return result
-}
-
-// getStackTrace creates a StackTrace output from one of the two dumps given.  It first tries to
-// use the AddressSanitizer dump, with the Clang dump as a fallback.
-func getStackTrace(asan, dump string) StackTrace {
-	if asanCrashed(asan) {
-		return parseASANStackTrace(asan)
-	}
-	return parseCatchsegvStackTrace(dump)
-}
-
-// computeFlags parses the raw data to set both the Debug and Release flags.
-func (r *FuzzResult) computeFlags(category string) {
-	r.Debug.Flags = parseAll(category, &r.Debug)
-	r.Release.Flags = parseAll(category, &r.Release)
-}
-
-// parseAll looks at the three input files and parses the results, based on the category.  The
-// category allows for specialized flags, like SKPICTURE_DuringRendering.
-func parseAll(category string, data *BuildData) FuzzFlag {
-	f := FuzzFlag(0)
-	// Check for SKAbort message
-	if strings.Contains(data.Asan, "fatal error") {
-		f |= ASANCrashed
-		f |= SKAbortHit
-		if data.StackTrace.IsEmpty() {
-			data.StackTrace = extractSkAbortTrace(data.StdErr)
-		}
-	}
-	if strings.Contains(data.StdErr, "fatal error") {
-		f |= ClangCrashed
-		f |= SKAbortHit
-		if data.StackTrace.IsEmpty() {
-			data.StackTrace = extractSkAbortTrace(data.StdErr)
-		}
-	}
-	// If no sk abort message and no evidence of crashes, we either terminated gracefully or
-	// timed out.
-	if f == 0 && !asanCrashed(data.Asan) && !clangDumped(data.Dump) {
-		if strings.Contains(data.Asan, "[terminated]") && strings.Contains(data.StdErr, "[terminated]") {
-			return TerminatedGracefully
-		}
-		return TimedOut
-	}
-
-	// Look for clues from the various dumps.
-	f |= parseAsan(category, data.Asan)
-	f |= parseCatchsegv(category, data.Dump, data.StdErr)
-	if f == 0 {
-		// I don't know what this means (yet).
-		return Other
-	}
-	if data.StackTrace.IsEmpty() {
-		f |= NoStackTrace
-	}
-	return f
-}
-
-// parseAsan returns the flags discovered while looking through the AddressSanitizer output.  This
-// includes things like ASAN_GlobalBufferOverflow.
-func parseAsan(category, asan string) FuzzFlag {
-	f := FuzzFlag(0)
-	if !asanCrashed(asan) {
-		return f
-	}
-	f |= ASANCrashed
-	if strings.Contains(asan, "failed assertion") {
-		f |= AssertionViolated
-	}
-	if strings.Contains(asan, "global-buffer-overflow") {
-		f |= ASAN_GlobalBufferOverflow
-	}
-	if strings.Contains(asan, "heap-buffer-overflow") {
-		f |= ASAN_HeapBufferOverflow
-	}
-	if strings.Contains(asan, "stack-buffer-overflow") {
-		f |= ASAN_StackBufferOverflow
-	}
-	if strings.Contains(asan, "heap-use-after-free") {
-		f |= ASAN_HeapUseAfterFree
-	}
-	if strings.Contains(asan, "AddressSanitizer failed to allocate") {
-		f |= BadAlloc
-	}
-
-	// Split off the stderr that happened before the crash.
-	errs := strings.Split(asan, "=================")
-	if len(errs) > 0 {
-		err := errs[0]
-		if category == "skpicture" && strings.Contains(err, "Rendering") {
-			f |= SKPICTURE_DuringRendering
-		}
-	}
-	return f
-}
-
-// asanCrashed returns true if the asan output is consistent with a crash.
-func asanCrashed(asan string) bool {
-	return strings.Contains(asan, "ERROR: AddressSanitizer:")
-}
-
-// parseAsan returns the flags discovered while looking through the Clang dump and standard error.
-// This includes things like
-func parseCatchsegv(category, dump, err string) FuzzFlag {
-	f := FuzzFlag(0)
-	if !clangDumped(dump) && strings.Contains(err, "[terminated]") {
-		return f
-	}
-	f |= ClangCrashed
-	if strings.Contains(err, "failed assertion") {
-		f |= AssertionViolated
-	}
-	if category == "skpicture" && strings.Contains(err, "Rendering") {
-		f |= SKPICTURE_DuringRendering
-	}
-	if strings.Contains(err, "std::bad_alloc") {
-		f |= BadAlloc
-	}
-	return f
-}
-
-// clangDumped returns true if the clang output is consistent with a crash, that is, non empty.
-func clangDumped(dump string) bool {
-	return len(dump) != 0
-}
-
-var skAbortStackTraceLine = regexp.MustCompile(`(?:\.\./)+(?P<package>(?:\w+/)+)(?P<file>.+):(?P<line>\d+): fatal error`)
-
-// extractSkAbortTrace looks for the fatal error string indicative of the SKAbort termination
-// and tries to pull out the stacktrace frame on which it happened.
-func extractSkAbortTrace(err string) StackTrace {
-	st := StackTrace{}
-	if match := skAbortStackTraceLine.FindStringSubmatch(err); match != nil {
-		st.Frames = append(st.Frames, FullStackFrame(match[1], match[2], common.UNKNOWN_FUNCTION, safeParseInt(match[3])))
-	}
-	return st
-}