| OLD | NEW |
| 1 package backend | 1 package backend |
| 2 | 2 |
| 3 import ( | 3 import ( |
| 4 "fmt" | 4 "fmt" |
| 5 "io/ioutil" | 5 "io/ioutil" |
| 6 "os" | 6 "os" |
| 7 "path/filepath" | 7 "path/filepath" |
| 8 "strings" | 8 "strings" |
| 9 "sync" | 9 "sync" |
| 10 "sync/atomic" | 10 "sync/atomic" |
| (...skipping 66 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 77 | 77 |
| 78 return config.Generator.SkiaVersion, nil | 78 return config.Generator.SkiaVersion, nil |
| 79 } | 79 } |
| 80 | 80 |
| 81 func (p *FuzzPipeline) reanalyzeAndRestart(storageClient *storage.Client, oldRev
ision string) error { | 81 func (p *FuzzPipeline) reanalyzeAndRestart(storageClient *storage.Client, oldRev
ision string) error { |
| 82 // download all bad and grey fuzzes | 82 // download all bad and grey fuzzes |
| 83 badFuzzNames, greyFuzzNames, err := p.downloadAllBadAndGreyFuzzes(oldRev
ision, storageClient) | 83 badFuzzNames, greyFuzzNames, err := p.downloadAllBadAndGreyFuzzes(oldRev
ision, storageClient) |
| 84 if err != nil { | 84 if err != nil { |
| 85 return fmt.Errorf("Problem downloading all previous fuzzes: %s",
err) | 85 return fmt.Errorf("Problem downloading all previous fuzzes: %s",
err) |
| 86 } | 86 } |
| 87 » glog.Infof("There are %d badFuzzNames and %d greyFuzzNames to rescan.",
len(badFuzzNames), len(greyFuzzNames)) | 87 » glog.Infof("There are %d bad fuzzes and %d grey fuzzes to rescan.", len(
badFuzzNames), len(greyFuzzNames)) |
| 88 // This is a soft shutdown, i.e. it waits for aggregator's queues to be
empty | 88 // This is a soft shutdown, i.e. it waits for aggregator's queues to be
empty |
| 89 p.Agg.ShutDown() | 89 p.Agg.ShutDown() |
| 90 |
| 91 if config.Common.ForceReanalysis { |
| 92 glog.Infof("Deleting previous fuzz results") |
| 93 if err := gs.DeleteAllFilesInDir(storageClient, config.GS.Bucket
, fmt.Sprintf("%s/%s/", p.Category, oldRevision), config.Aggregator.NumUploadPro
cesses); err != nil { |
| 94 return fmt.Errorf("Could not delete previous fuzzes: %s"
, err) |
| 95 } |
| 96 } |
| 97 |
| 90 if err := p.Gen.Clear(); err != nil { | 98 if err := p.Gen.Clear(); err != nil { |
| 91 return fmt.Errorf("Could not remove previous afl-fuzz results: %
s", err) | 99 return fmt.Errorf("Could not remove previous afl-fuzz results: %
s", err) |
| 92 } | 100 } |
| 93 | 101 |
| 94 if err := p.Agg.RestartAnalysis(); err != nil { | 102 if err := p.Agg.RestartAnalysis(); err != nil { |
| 95 return fmt.Errorf("Had problem restarting analysis/upload chain:
%s", err) | 103 return fmt.Errorf("Had problem restarting analysis/upload chain:
%s", err) |
| 96 } | 104 } |
| 97 // Reanalyze and reupload the fuzzes, making a bug on regressions. | 105 // Reanalyze and reupload the fuzzes, making a bug on regressions. |
| 98 glog.Infof("Reanalyzing bad fuzzes") | 106 glog.Infof("Reanalyzing bad fuzzes") |
| 99 p.Agg.MakeBugOnBadFuzz = false | 107 p.Agg.MakeBugOnBadFuzz = false |
| (...skipping 32 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 132 completedCounter = 0 | 140 completedCounter = 0 |
| 133 | 141 |
| 134 var wg sync.WaitGroup | 142 var wg sync.WaitGroup |
| 135 for i := 0; i < config.Generator.NumDownloadProcesses; i++ { | 143 for i := 0; i < config.Generator.NumDownloadProcesses; i++ { |
| 136 wg.Add(1) | 144 wg.Add(1) |
| 137 go download(storageClient, toDownload, downloadPath, &wg) | 145 go download(storageClient, toDownload, downloadPath, &wg) |
| 138 } | 146 } |
| 139 | 147 |
| 140 badFilter := func(item *storage.ObjectAttrs) { | 148 badFilter := func(item *storage.ObjectAttrs) { |
| 141 name := item.Name | 149 name := item.Name |
| 142 » » if strings.HasSuffix(name, ".dump") || strings.HasSuffix(name, "
.err") { | 150 » » if strings.Contains(name, ".") { |
| 143 return | 151 return |
| 144 } | 152 } |
| 145 fuzzHash := name[strings.LastIndex(name, "/")+1:] | 153 fuzzHash := name[strings.LastIndex(name, "/")+1:] |
| 146 badFuzzNames = append(badFuzzNames, filepath.Join(downloadPath,
fuzzHash)) | 154 badFuzzNames = append(badFuzzNames, filepath.Join(downloadPath,
fuzzHash)) |
| 147 toDownload <- item.Name | 155 toDownload <- item.Name |
| 148 } | 156 } |
| 149 | 157 |
| 150 greyFilter := func(item *storage.ObjectAttrs) { | 158 greyFilter := func(item *storage.ObjectAttrs) { |
| 151 name := item.Name | 159 name := item.Name |
| 152 » » if strings.HasSuffix(item.Name, ".dump") || strings.HasSuffix(it
em.Name, ".err") { | 160 » » if strings.Contains(name, ".") { |
| 153 return | 161 return |
| 154 } | 162 } |
| 155 fuzzHash := name[strings.LastIndex(name, "/")+1:] | 163 fuzzHash := name[strings.LastIndex(name, "/")+1:] |
| 156 greyFuzzNames = append(greyFuzzNames, filepath.Join(downloadPath
, fuzzHash)) | 164 greyFuzzNames = append(greyFuzzNames, filepath.Join(downloadPath
, fuzzHash)) |
| 157 toDownload <- item.Name | 165 toDownload <- item.Name |
| 158 } | 166 } |
| 159 | 167 |
| 160 if err := gs.AllFilesInDir(storageClient, config.GS.Bucket, fmt.Sprintf(
"%s/%s/bad", p.Category, commitHash), badFilter); err != nil { | 168 if err := gs.AllFilesInDir(storageClient, config.GS.Bucket, fmt.Sprintf(
"%s/%s/bad", p.Category, commitHash), badFilter); err != nil { |
| 161 return nil, nil, fmt.Errorf("Problem getting bad fuzzes: %s", er
r) | 169 return nil, nil, fmt.Errorf("Problem getting bad fuzzes: %s", er
r) |
| 162 } | 170 } |
| (...skipping 29 matching lines...) Expand all Loading... |
| 192 if completedCounter%100 == 0 { | 200 if completedCounter%100 == 0 { |
| 193 glog.Infof("%d fuzzes downloaded", completedCounter) | 201 glog.Infof("%d fuzzes downloaded", completedCounter) |
| 194 } | 202 } |
| 195 } | 203 } |
| 196 } | 204 } |
| 197 | 205 |
| 198 // replaceCurrentSkiaVersionWith puts the oldHash in skia_version/old and the ne
wHash in | 206 // replaceCurrentSkiaVersionWith puts the oldHash in skia_version/old and the ne
wHash in |
| 199 // skia_version/current. It also removes all pending versions. | 207 // skia_version/current. It also removes all pending versions. |
| 200 func (v *VersionUpdater) replaceCurrentSkiaVersionWith(oldHash, newHash string)
error { | 208 func (v *VersionUpdater) replaceCurrentSkiaVersionWith(oldHash, newHash string)
error { |
| 201 // delete all pending requests | 209 // delete all pending requests |
| 202 » if err := gs.DeleteAllFilesInDir(v.storageClient, config.GS.Bucket, "ski
a_version/pending/"); err != nil { | 210 » if err := gs.DeleteAllFilesInDir(v.storageClient, config.GS.Bucket, "ski
a_version/pending/", 1); err != nil { |
| 203 return err | 211 return err |
| 204 } | 212 } |
| 205 » if err := gs.DeleteAllFilesInDir(v.storageClient, config.GS.Bucket, "ski
a_version/current/"); err != nil { | 213 » if err := gs.DeleteAllFilesInDir(v.storageClient, config.GS.Bucket, "ski
a_version/current/", 1); err != nil { |
| 206 return err | 214 return err |
| 207 } | 215 } |
| 208 if err := v.touch(fmt.Sprintf("skia_version/current/%s", newHash)); err
!= nil { | 216 if err := v.touch(fmt.Sprintf("skia_version/current/%s", newHash)); err
!= nil { |
| 209 return err | 217 return err |
| 210 } | 218 } |
| 211 return v.touch(fmt.Sprintf("skia_version/old/%s", oldHash)) | 219 return v.touch(fmt.Sprintf("skia_version/old/%s", oldHash)) |
| 212 } | 220 } |
| 213 | 221 |
| 214 // touch creates an empty file in Google Storage of the given name. | 222 // touch creates an empty file in Google Storage of the given name. |
| 215 func (v *VersionUpdater) touch(file string) error { | 223 func (v *VersionUpdater) touch(file string) error { |
| 216 w := v.storageClient.Bucket(config.GS.Bucket).Object(file).NewWriter(con
text.Background()) | 224 w := v.storageClient.Bucket(config.GS.Bucket).Object(file).NewWriter(con
text.Background()) |
| 217 if err := w.Close(); err != nil { | 225 if err := w.Close(); err != nil { |
| 218 return fmt.Errorf("Could not touch version file %s : %s", file,
err) | 226 return fmt.Errorf("Could not touch version file %s : %s", file,
err) |
| 219 } | 227 } |
| 220 return nil | 228 return nil |
| 221 } | 229 } |
| OLD | NEW |