| OLD | NEW |
|---|
| (Empty) |
| 1 package data | |
| 2 | |
| 3 import ( | |
| 4 "bytes" | |
| 5 "encoding/gob" | |
| 6 "fmt" | |
| 7 "sort" | |
| 8 "sync" | |
| 9 | |
| 10 "github.com/skia-dev/glog" | |
| 11 "go.skia.org/infra/fuzzer/go/common" | |
| 12 ) | |
| 13 | |
| 14 type FuzzReportTree []FileFuzzReport | |
| 15 | |
| 16 type FileFuzzReport struct { | |
| 17 FileName string `json:"fileName"` | |
| 18 Count int `json:"count"` | |
| 19 Functions []FunctionFuzzReport `json:"byFunction"` | |
| 20 } | |
| 21 | |
| 22 type FunctionFuzzReport struct { | |
| 23 FunctionName string `json:"functionName"` | |
| 24 Count int `json:"count"` | |
| 25 LineNumbers []LineFuzzReport `json:"byLineNumber"` | |
| 26 } | |
| 27 | |
| 28 type LineFuzzReport struct { | |
| 29 LineNumber int `json:"lineNumber"` | |
| 30 Count int `json:"count"` | |
| 31 Details SortedFuzzReports `json:"reports"` | |
| 32 } | |
| 33 | |
| 34 type FuzzReport struct { | |
| 35 DebugStackTrace StackTrace `json:"debugStackTrace"` | |
| 36 ReleaseStackTrace StackTrace `json:"releaseStackTrace"` | |
| 37 DebugFlags []string `json:"debugFlags"` | |
| 38 ReleaseFlags []string `json:"releaseFlags"` | |
| 39 | |
| 40 FuzzName string `json:"fuzzName"` | |
| 41 FuzzCategory string `json:"category"` | |
| 42 } | |
| 43 | |
| 44 type SortedFuzzReports []FuzzReport | |
| 45 | |
| 46 // ParseReport creates a report given the raw materials passed in. | |
| 47 func ParseReport(g GCSPackage) FuzzReport { | |
| 48 result := ParseGCSPackage(g) | |
| 49 return FuzzReport{ | |
| 50 DebugStackTrace: result.Debug.StackTrace, | |
| 51 ReleaseStackTrace: result.Release.StackTrace, | |
| 52 DebugFlags: result.Debug.Flags.ToHumanReadableFlags(), | |
| 53 ReleaseFlags: result.Release.Flags.ToHumanReadableFlags(), | |
| 54 FuzzName: g.Name, | |
| 55 FuzzCategory: g.FuzzCategory, | |
| 56 } | |
| 57 } | |
| 58 | |
| 59 // treeReportBuilder is an in-memory structure that allows easy creation of a tr
ee of reports | |
| 60 // for use on the frontend. It has a fuzzReportCache for every fuzz type (e.g. s
kpicture, skcodec, etc) | |
| 61 type treeReportBuilder struct { | |
| 62 caches map[string]*fuzzReportCache | |
| 63 mutex sync.Mutex | |
| 64 } | |
| 65 | |
| 66 // newBuilder creates an initialized treeReportBuilder | |
| 67 func newBuilder() *treeReportBuilder { | |
| 68 return &treeReportBuilder{ | |
| 69 caches: map[string]*fuzzReportCache{}, | |
| 70 } | |
| 71 } | |
| 72 | |
| 73 // A fuzzReportCache holds three FuzzReportTrees - one for the raw data, a sorte
d version with | |
| 74 // all of the reports and an empty tree that holds no reports. These are used t
o procure data | |
| 75 // for the frontend. | |
| 76 type fuzzReportCache struct { | |
| 77 // All the data goes in here, in no particular order | |
| 78 rawData FuzzReportTree | |
| 79 // Generated, sorted cache | |
| 80 FullReport FuzzReportTree | |
| 81 | |
| 82 // If data is in rawData, but not in FullReport, the trees should be | |
| 83 // rebuilt | |
| 84 isDirty bool | |
| 85 } | |
| 86 | |
| 87 // currentData is the object that holds the cache of fuzz results. It is used b
y the frontend. | |
| 88 var currentData = newBuilder() | |
| 89 | |
| 90 // stagingData is the object that processes can write to to queue up new data | |
| 91 // without disturbing the data shown to users. | |
| 92 var stagingData = newBuilder() | |
| 93 | |
| 94 // FindFuzzDetails returns the detailed fuzz reports for a file name, function n
ame, and line number. | |
| 95 // If functionName is "" or lineNumber is -1, all reports are shown. | |
| 96 func FindFuzzDetails(category, fileName, functionName string, lineNumber int) (F
uzzReportTree, error) { | |
| 97 cache, found := currentData.caches[category] | |
| 98 if found { | |
| 99 if fileName == "" { | |
| 100 return cache.FullReport, nil | |
| 101 } | |
| 102 for _, file := range cache.FullReport { | |
| 103 if file.FileName == fileName { | |
| 104 if functionName == "" { | |
| 105 return FuzzReportTree{file}, nil | |
| 106 } | |
| 107 file.filterByFunctionName(functionName) | |
| 108 if lineNumber == common.UNKNOWN_LINE { | |
| 109 return FuzzReportTree{file}, nil | |
| 110 } | |
| 111 file.Functions[0].filterByLineNumber(lineNumber) | |
| 112 return FuzzReportTree{file}, nil | |
| 113 } | |
| 114 } | |
| 115 } | |
| 116 | |
| 117 return nil, fmt.Errorf("File %q not found", fileName) | |
| 118 } | |
| 119 | |
| 120 // filterByFunctionName removes all FuzzReportFunction except that which matches
functionName | |
| 121 func (file *FileFuzzReport) filterByFunctionName(functionName string) { | |
| 122 for _, function := range file.Functions { | |
| 123 if functionName == function.FunctionName { | |
| 124 file.Functions = []FunctionFuzzReport{function} | |
| 125 break | |
| 126 } | |
| 127 } | |
| 128 } | |
| 129 | |
| 130 // filterByLineNumber removes all FuzzReportLineNumber except that which matches
lineNumber | |
| 131 func (function *FunctionFuzzReport) filterByLineNumber(lineNumber int) { | |
| 132 for _, line := range function.LineNumbers { | |
| 133 if lineNumber == line.LineNumber { | |
| 134 function.LineNumbers = []LineFuzzReport{line} | |
| 135 } | |
| 136 } | |
| 137 } | |
| 138 | |
| 139 // FindFuzzDetailForFuzz returns a tree containing the single | |
| 140 // report with the given name, or an error, it it doesn't exist. | |
| 141 func FindFuzzDetailForFuzz(category, name string) (FuzzReportTree, error) { | |
| 142 if cache, found := currentData.caches[category]; found { | |
| 143 for _, file := range cache.FullReport { | |
| 144 if file.filterByFuzzName(name) { | |
| 145 return FuzzReportTree{file}, nil | |
| 146 } | |
| 147 } | |
| 148 } | |
| 149 return nil, fmt.Errorf("Fuzz with name %q not found", name) | |
| 150 } | |
| 151 | |
| 152 // filterByFuzzName filters out all functions that do not contain a fuzz with th
e given | |
| 153 // name and returns true. If such a fuzz does not exist, it returns false. | |
| 154 func (file *FileFuzzReport) filterByFuzzName(name string) bool { | |
| 155 for _, function := range file.Functions { | |
| 156 if function.filterByFuzzName(name) { | |
| 157 file.Functions = []FunctionFuzzReport{function} | |
| 158 return true | |
| 159 } | |
| 160 } | |
| 161 return false | |
| 162 } | |
| 163 | |
| 164 // filterByFuzzName filters out all lines that do not contain a fuzz with the gi
ven | |
| 165 // name and returns true. If such a fuzz does not exist, it returns false. | |
| 166 func (function *FunctionFuzzReport) filterByFuzzName(name string) bool { | |
| 167 for _, line := range function.LineNumbers { | |
| 168 if line.filterByFuzzName(name) { | |
| 169 function.LineNumbers = []LineFuzzReport{line} | |
| 170 return true | |
| 171 } | |
| 172 } | |
| 173 return false | |
| 174 } | |
| 175 | |
| 176 // filterByFuzzName filters out all fuzzes that do not have the given | |
| 177 // name and returns true. If such a fuzz does not exist, it returns false. | |
| 178 func (line *LineFuzzReport) filterByFuzzName(name string) bool { | |
| 179 if b, hasIt := line.Details.containsName(name); hasIt { | |
| 180 line.Details = SortedFuzzReports{b} | |
| 181 return true | |
| 182 } | |
| 183 return false | |
| 184 } | |
| 185 | |
| 186 func NewFuzzFound(category string, b FuzzReport) { | |
| 187 // set the category if it has not already been set | |
| 188 b.FuzzCategory = category | |
| 189 stagingData.addFuzzReport(category, b) | |
| 190 } | |
| 191 | |
| 192 // ClearStaging clears the staging representation. | |
| 193 func ClearStaging() { | |
| 194 stagingData.mutex.Lock() | |
| 195 defer stagingData.mutex.Unlock() | |
| 196 stagingData.caches = map[string]*fuzzReportCache{} | |
| 197 } | |
| 198 | |
| 199 // SetStaging replaces the staging representation with the given FuzzReport. | |
| 200 func SetStaging(category string, r FuzzReportTree) { | |
| 201 stagingData.mutex.Lock() | |
| 202 defer stagingData.mutex.Unlock() | |
| 203 cache, found := stagingData.caches[category] | |
| 204 if !found { | |
| 205 cache = &fuzzReportCache{} | |
| 206 stagingData.caches[category] = cache | |
| 207 } | |
| 208 cache.rawData = r | |
| 209 cache.rebuildSortedReports() | |
| 210 } | |
| 211 | |
| 212 // StagingToCurrent moves a copy of the staging data to the currentData. | |
| 213 func StagingToCurrent() { | |
| 214 currentData.mutex.Lock() | |
| 215 defer currentData.mutex.Unlock() | |
| 216 stagingData.mutex.Lock() | |
| 217 defer stagingData.mutex.Unlock() | |
| 218 | |
| 219 currentData.caches = map[string]*fuzzReportCache{} | |
| 220 for k, v := range stagingData.caches { | |
| 221 cache := fuzzReportCache{} | |
| 222 cache.rawData = cloneReport(v.rawData) | |
| 223 cache.rebuildSortedReports() | |
| 224 currentData.caches[k] = &cache | |
| 225 } | |
| 226 } | |
| 227 | |
| 228 // StagingToCurrent moves a copy of the current data to the staging data. | |
| 229 func StagingFromCurrent() { | |
| 230 currentData.mutex.Lock() | |
| 231 defer currentData.mutex.Unlock() | |
| 232 stagingData.mutex.Lock() | |
| 233 defer stagingData.mutex.Unlock() | |
| 234 | |
| 235 stagingData.caches = map[string]*fuzzReportCache{} | |
| 236 for k, v := range currentData.caches { | |
| 237 cache := fuzzReportCache{} | |
| 238 cache.rawData = cloneReport(v.rawData) | |
| 239 cache.rebuildSortedReports() | |
| 240 stagingData.caches[k] = &cache | |
| 241 } | |
| 242 } | |
| 243 | |
| 244 // StagingCopy returns a fresh copy of the underlying staging data. | |
| 245 func StagingCopy(category string) FuzzReportTree { | |
| 246 stagingData.mutex.Lock() | |
| 247 defer stagingData.mutex.Unlock() | |
| 248 cache, found := stagingData.caches[category] | |
| 249 if !found { | |
| 250 return FuzzReportTree{} | |
| 251 } | |
| 252 return cloneReport(cache.rawData) | |
| 253 } | |
| 254 | |
| 255 // addFuzzReport adds a FuzzReport to a treeReportBuilder's data member | |
| 256 func (r *treeReportBuilder) addFuzzReport(category string, b FuzzReport) { | |
| 257 reportFileName, reportFunctionName, reportLineNumber := extractStacktrac
eInfo(b.DebugStackTrace, b.ReleaseStackTrace) | |
| 258 | |
| 259 cache, found := r.caches[category] | |
| 260 if !found { | |
| 261 cache = &fuzzReportCache{} | |
| 262 r.caches[category] = cache | |
| 263 } | |
| 264 r.mutex.Lock() | |
| 265 defer r.mutex.Unlock() | |
| 266 foundFile, foundFunction, foundLine := cache.makeOrFindRecords(reportFil
eName, reportFunctionName, reportLineNumber) | |
| 267 | |
| 268 foundFile.Count++ | |
| 269 foundFunction.Count++ | |
| 270 foundLine.Count++ | |
| 271 foundLine.Details = foundLine.Details.append(b) | |
| 272 cache.isDirty = true | |
| 273 | |
| 274 } | |
| 275 | |
| 276 // extractStacktraceInfo returns the file name, function name and line number th
at | |
| 277 // a report with the given debug and release stacktrace should be sorted by. | |
| 278 // this tries to read the release stacktrace first, falling back to the debug st
acktrace, | |
| 279 // failling back to Unknown. | |
| 280 func extractStacktraceInfo(debug, release StackTrace) (reportFileName, reportFun
ctionName string, reportLineNumber int) { | |
| 281 reportFileName, reportFunctionName, reportLineNumber = common.UNKNOWN_FI
LE, common.UNKNOWN_FUNCTION, common.UNKNOWN_LINE | |
| 282 | |
| 283 stacktrace := release | |
| 284 if stacktrace.IsEmpty() { | |
| 285 stacktrace = debug | |
| 286 } | |
| 287 if !stacktrace.IsEmpty() { | |
| 288 frame := stacktrace.Frames[0] | |
| 289 reportFileName = frame.PackageName + frame.FileName | |
| 290 reportFunctionName, reportLineNumber = frame.FunctionName, frame
.LineNumber | |
| 291 } | |
| 292 return | |
| 293 } | |
| 294 | |
| 295 // makeOrFindRecords finds the FuzzReportFile, FuzzReportFunction and FuzzReport
LineNumber | |
| 296 // associated with the inputs, creating the structures if needed. | |
| 297 func (c *fuzzReportCache) makeOrFindRecords(reportFileName, reportFunctionName s
tring, reportLineNumber int) (*FileFuzzReport, *FunctionFuzzReport, *LineFuzzRep
ort) { | |
| 298 var foundFile *FileFuzzReport | |
| 299 for i, file := range c.rawData { | |
| 300 if file.FileName == reportFileName { | |
| 301 foundFile = &c.rawData[i] | |
| 302 break | |
| 303 } | |
| 304 } | |
| 305 if foundFile == nil { | |
| 306 c.rawData = append(c.rawData, FileFuzzReport{reportFileName, 0,
nil}) | |
| 307 foundFile = &c.rawData[len(c.rawData)-1] | |
| 308 } | |
| 309 | |
| 310 var foundFunction *FunctionFuzzReport | |
| 311 for i, function := range foundFile.Functions { | |
| 312 if function.FunctionName == reportFunctionName { | |
| 313 foundFunction = &foundFile.Functions[i] | |
| 314 break | |
| 315 } | |
| 316 } | |
| 317 if foundFunction == nil { | |
| 318 foundFile.Functions = append(foundFile.Functions, FunctionFuzzRe
port{reportFunctionName, 0, nil}) | |
| 319 foundFunction = &foundFile.Functions[len(foundFile.Functions)-1] | |
| 320 } | |
| 321 | |
| 322 var foundLine *LineFuzzReport | |
| 323 for i, line := range foundFunction.LineNumbers { | |
| 324 if line.LineNumber == reportLineNumber { | |
| 325 foundLine = &foundFunction.LineNumbers[i] | |
| 326 } | |
| 327 } | |
| 328 if foundLine == nil { | |
| 329 foundFunction.LineNumbers = append(foundFunction.LineNumbers, Li
neFuzzReport{reportLineNumber, 0, nil}) | |
| 330 foundLine = &foundFunction.LineNumbers[len(foundFunction.LineNum
bers)-1] | |
| 331 } | |
| 332 return foundFile, foundFunction, foundLine | |
| 333 } | |
| 334 | |
| 335 // getTreeSortedByTotal gets the detailed FuzzReport for a fuzz category | |
| 336 // sorted by total number of fuzzes. | |
| 337 func (r *treeReportBuilder) getTreeSortedByTotal(category string) FuzzReportTree
{ | |
| 338 cache, found := r.caches[category] | |
| 339 if !found { | |
| 340 glog.Warningf("Could not find report tree for category %s", cate
gory) | |
| 341 return FuzzReportTree{} | |
| 342 } | |
| 343 if cache.isDirty { | |
| 344 r.mutex.Lock() | |
| 345 defer r.mutex.Unlock() | |
| 346 cache.rebuildSortedReports() | |
| 347 } | |
| 348 return cache.FullReport | |
| 349 } | |
| 350 | |
| 351 // rebuildSortedReports creates the sorted reports for a given cache. | |
| 352 func (c *fuzzReportCache) rebuildSortedReports() { | |
| 353 c.FullReport = c.getClonedSortedReport(true) | |
| 354 c.isDirty = false | |
| 355 } | |
| 356 | |
| 357 // getClonedSortedReport makes a newly allocated FuzzReport after running the pa
ssed in function | |
| 358 // on all FuzzReportLineNumber objects in the report. | |
| 359 func (c *fuzzReportCache) getClonedSortedReport(keepDetails bool) FuzzReportTree
{ | |
| 360 report := cloneReport(c.rawData) | |
| 361 sort.Sort(filesTotalSort(report)) | |
| 362 for i := range report { | |
| 363 file := &report[i] | |
| 364 sort.Sort(functionsTotalSort(file.Functions)) | |
| 365 for j := range file.Functions { | |
| 366 function := &file.Functions[j] | |
| 367 sort.Sort(linesTotalSort(function.LineNumbers)) | |
| 368 for k := range function.LineNumbers { | |
| 369 line := &function.LineNumbers[k] | |
| 370 if !keepDetails { | |
| 371 line.Details = nil | |
| 372 } | |
| 373 } | |
| 374 } | |
| 375 } | |
| 376 return report | |
| 377 } | |
| 378 | |
| 379 // cloneReport makes a copy of the input using the gob library. | |
| 380 func cloneReport(data []FileFuzzReport) FuzzReportTree { | |
| 381 var temp bytes.Buffer | |
| 382 enc := gob.NewEncoder(&temp) | |
| 383 dec := gob.NewDecoder(&temp) | |
| 384 | |
| 385 if err := enc.Encode(data); err != nil { | |
| 386 // This should never happen, but log it if it does | |
| 387 glog.Errorf("Error while cloning report: %v", err) | |
| 388 } | |
| 389 var clone FuzzReportTree | |
| 390 if err := dec.Decode(&clone); err != nil { | |
| 391 // This should never happen, but log it if it does | |
| 392 glog.Errorf("Error while cloning report: %v", err) | |
| 393 } | |
| 394 return clone | |
| 395 } | |
| 396 | |
| 397 // Total sort methods - sorts files, functions and lines by Count | |
| 398 type filesTotalSort []FileFuzzReport | |
| 399 | |
| 400 func (r filesTotalSort) Len() int { return len(r) } | |
| 401 func (r filesTotalSort) Swap(i, j int) { r[i], r[j] = r[j], r[i] } | |
| 402 | |
| 403 func (r filesTotalSort) Less(i, j int) bool { | |
| 404 if r[i].Count != r[j].Count { | |
| 405 return r[i].Count > r[j].Count | |
| 406 } | |
| 407 // If they have the same total, sort by name | |
| 408 return r[i].FileName < r[j].FileName | |
| 409 } | |
| 410 | |
| 411 type functionsTotalSort []FunctionFuzzReport | |
| 412 | |
| 413 func (r functionsTotalSort) Len() int { return len(r) } | |
| 414 func (r functionsTotalSort) Swap(i, j int) { r[i], r[j] = r[j], r[i] } | |
| 415 | |
| 416 func (r functionsTotalSort) Less(i, j int) bool { | |
| 417 if r[i].Count != r[j].Count { | |
| 418 return r[i].Count > r[j].Count | |
| 419 } | |
| 420 // If they have the same total, sort by name | |
| 421 return r[i].FunctionName < r[j].FunctionName | |
| 422 } | |
| 423 | |
| 424 type linesTotalSort []LineFuzzReport | |
| 425 | |
| 426 func (r linesTotalSort) Len() int { return len(r) } | |
| 427 func (r linesTotalSort) Swap(i, j int) { r[i], r[j] = r[j], r[i] } | |
| 428 | |
| 429 func (r linesTotalSort) Less(i, j int) bool { | |
| 430 if r[i].Count != r[j].Count { | |
| 431 return r[i].Count > r[j].Count | |
| 432 } | |
| 433 // If they have the same total, sort by line number | |
| 434 return r[i].LineNumber < r[j].LineNumber | |
| 435 } | |
| 436 | |
| 437 func (p SortedFuzzReports) Len() int { return len(p) } | |
| 438 func (p SortedFuzzReports) Less(i, j int) bool { return p[i].FuzzName < p[j].Fuz
zName } | |
| 439 func (p SortedFuzzReports) Swap(i, j int) { p[i], p[j] = p[j], p[i] } | |
| 440 | |
| 441 // append adds b to the already sorted caller, and returns the sorted result. | |
| 442 // Precondition: Caller must be nil or sorted | |
| 443 func (p SortedFuzzReports) append(b FuzzReport) SortedFuzzReports { | |
| 444 s := append(p, b) | |
| 445 | |
| 446 // Google Storage gives us the fuzzes in alphabetical order. Thus, we c
an short circuit | |
| 447 // if the fuzz goes on the end (which is usually does). | |
| 448 // However, we can't always do this because when we load a second batch
of fuzzes, | |
| 449 // those are in alphabetical order, but starting over from 0. | |
| 450 // We want to avoid [a,c,x,z,b,d] where b,d were added from the second b
atch. | |
| 451 if len(s) <= 1 || s.Less(len(s)-2, len(s)-1) { | |
| 452 return s | |
| 453 } | |
| 454 sort.Sort(s) | |
| 455 return s | |
| 456 } | |
| 457 | |
| 458 // containsName returns the FuzzReport and true if a fuzz with the given name is
in the list. | |
| 459 func (p SortedFuzzReports) containsName(fuzzName string) (FuzzReport, bool) { | |
| 460 i := sort.Search(len(p), func(i int) bool { return p[i].FuzzName >= fuzz
Name }) | |
| 461 if i < len(p) && p[i].FuzzName == fuzzName { | |
| 462 return p[i], true | |
| 463 } | |
| 464 return FuzzReport{}, false | |
| 465 } | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 1691893002:
Fuzzer now deduplicates on the analysis side instead of the download side (Closed)
Base URL: https://skia.googlesource.com/buildbot@metrics