OLD | NEW |
1 package backend | 1 package backend |
2 | 2 |
3 import ( | 3 import ( |
4 "fmt" | 4 "fmt" |
5 "io/ioutil" | 5 "io/ioutil" |
6 "os" | 6 "os" |
7 "path/filepath" | 7 "path/filepath" |
8 "strings" | 8 "strings" |
9 "sync" | 9 "sync" |
10 "sync/atomic" | 10 "sync/atomic" |
(...skipping 66 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
77 | 77 |
78 return config.Generator.SkiaVersion, nil | 78 return config.Generator.SkiaVersion, nil |
79 } | 79 } |
80 | 80 |
81 func (p *FuzzPipeline) reanalyzeAndRestart(storageClient *storage.Client, oldRev
ision string) error { | 81 func (p *FuzzPipeline) reanalyzeAndRestart(storageClient *storage.Client, oldRev
ision string) error { |
82 // download all bad and grey fuzzes | 82 // download all bad and grey fuzzes |
83 badFuzzNames, greyFuzzNames, err := p.downloadAllBadAndGreyFuzzes(oldRev
ision, storageClient) | 83 badFuzzNames, greyFuzzNames, err := p.downloadAllBadAndGreyFuzzes(oldRev
ision, storageClient) |
84 if err != nil { | 84 if err != nil { |
85 return fmt.Errorf("Problem downloading all previous fuzzes: %s",
err) | 85 return fmt.Errorf("Problem downloading all previous fuzzes: %s",
err) |
86 } | 86 } |
87 » glog.Infof("There are %d badFuzzNames and %d greyFuzzNames to rescan.",
len(badFuzzNames), len(greyFuzzNames)) | 87 » glog.Infof("There are %d bad fuzzes and %d grey fuzzes to rescan.", len(
badFuzzNames), len(greyFuzzNames)) |
88 // This is a soft shutdown, i.e. it waits for aggregator's queues to be
empty | 88 // This is a soft shutdown, i.e. it waits for aggregator's queues to be
empty |
89 p.Agg.ShutDown() | 89 p.Agg.ShutDown() |
| 90 |
| 91 if config.Common.ForceReanalysis { |
| 92 glog.Infof("Deleting previous fuzz results") |
| 93 if err := gs.DeleteAllFilesInDir(storageClient, config.GS.Bucket
, fmt.Sprintf("%s/%s/", p.Category, oldRevision), config.Aggregator.NumUploadPro
cesses); err != nil { |
| 94 return fmt.Errorf("Could not delete previous fuzzes: %s"
, err) |
| 95 } |
| 96 } |
| 97 |
90 if err := p.Gen.Clear(); err != nil { | 98 if err := p.Gen.Clear(); err != nil { |
91 return fmt.Errorf("Could not remove previous afl-fuzz results: %
s", err) | 99 return fmt.Errorf("Could not remove previous afl-fuzz results: %
s", err) |
92 } | 100 } |
93 | 101 |
94 if err := p.Agg.RestartAnalysis(); err != nil { | 102 if err := p.Agg.RestartAnalysis(); err != nil { |
95 return fmt.Errorf("Had problem restarting analysis/upload chain:
%s", err) | 103 return fmt.Errorf("Had problem restarting analysis/upload chain:
%s", err) |
96 } | 104 } |
97 // Reanalyze and reupload the fuzzes, making a bug on regressions. | 105 // Reanalyze and reupload the fuzzes, making a bug on regressions. |
98 glog.Infof("Reanalyzing bad fuzzes") | 106 glog.Infof("Reanalyzing bad fuzzes") |
99 p.Agg.MakeBugOnBadFuzz = false | 107 p.Agg.MakeBugOnBadFuzz = false |
(...skipping 32 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
132 completedCounter = 0 | 140 completedCounter = 0 |
133 | 141 |
134 var wg sync.WaitGroup | 142 var wg sync.WaitGroup |
135 for i := 0; i < config.Generator.NumDownloadProcesses; i++ { | 143 for i := 0; i < config.Generator.NumDownloadProcesses; i++ { |
136 wg.Add(1) | 144 wg.Add(1) |
137 go download(storageClient, toDownload, downloadPath, &wg) | 145 go download(storageClient, toDownload, downloadPath, &wg) |
138 } | 146 } |
139 | 147 |
140 badFilter := func(item *storage.ObjectAttrs) { | 148 badFilter := func(item *storage.ObjectAttrs) { |
141 name := item.Name | 149 name := item.Name |
142 » » if strings.HasSuffix(name, ".dump") || strings.HasSuffix(name, "
.err") { | 150 » » if strings.Contains(name, ".") { |
143 return | 151 return |
144 } | 152 } |
145 fuzzHash := name[strings.LastIndex(name, "/")+1:] | 153 fuzzHash := name[strings.LastIndex(name, "/")+1:] |
146 badFuzzNames = append(badFuzzNames, filepath.Join(downloadPath,
fuzzHash)) | 154 badFuzzNames = append(badFuzzNames, filepath.Join(downloadPath,
fuzzHash)) |
147 toDownload <- item.Name | 155 toDownload <- item.Name |
148 } | 156 } |
149 | 157 |
150 greyFilter := func(item *storage.ObjectAttrs) { | 158 greyFilter := func(item *storage.ObjectAttrs) { |
151 name := item.Name | 159 name := item.Name |
152 » » if strings.HasSuffix(item.Name, ".dump") || strings.HasSuffix(it
em.Name, ".err") { | 160 » » if strings.Contains(name, ".") { |
153 return | 161 return |
154 } | 162 } |
155 fuzzHash := name[strings.LastIndex(name, "/")+1:] | 163 fuzzHash := name[strings.LastIndex(name, "/")+1:] |
156 greyFuzzNames = append(greyFuzzNames, filepath.Join(downloadPath
, fuzzHash)) | 164 greyFuzzNames = append(greyFuzzNames, filepath.Join(downloadPath
, fuzzHash)) |
157 toDownload <- item.Name | 165 toDownload <- item.Name |
158 } | 166 } |
159 | 167 |
160 if err := gs.AllFilesInDir(storageClient, config.GS.Bucket, fmt.Sprintf(
"%s/%s/bad", p.Category, commitHash), badFilter); err != nil { | 168 if err := gs.AllFilesInDir(storageClient, config.GS.Bucket, fmt.Sprintf(
"%s/%s/bad", p.Category, commitHash), badFilter); err != nil { |
161 return nil, nil, fmt.Errorf("Problem getting bad fuzzes: %s", er
r) | 169 return nil, nil, fmt.Errorf("Problem getting bad fuzzes: %s", er
r) |
162 } | 170 } |
(...skipping 29 matching lines...) Expand all Loading... |
192 if completedCounter%100 == 0 { | 200 if completedCounter%100 == 0 { |
193 glog.Infof("%d fuzzes downloaded", completedCounter) | 201 glog.Infof("%d fuzzes downloaded", completedCounter) |
194 } | 202 } |
195 } | 203 } |
196 } | 204 } |
197 | 205 |
198 // replaceCurrentSkiaVersionWith puts the oldHash in skia_version/old and the ne
wHash in | 206 // replaceCurrentSkiaVersionWith puts the oldHash in skia_version/old and the ne
wHash in |
199 // skia_version/current. It also removes all pending versions. | 207 // skia_version/current. It also removes all pending versions. |
200 func (v *VersionUpdater) replaceCurrentSkiaVersionWith(oldHash, newHash string)
error { | 208 func (v *VersionUpdater) replaceCurrentSkiaVersionWith(oldHash, newHash string)
error { |
201 // delete all pending requests | 209 // delete all pending requests |
202 » if err := gs.DeleteAllFilesInDir(v.storageClient, config.GS.Bucket, "ski
a_version/pending/"); err != nil { | 210 » if err := gs.DeleteAllFilesInDir(v.storageClient, config.GS.Bucket, "ski
a_version/pending/", 1); err != nil { |
203 return err | 211 return err |
204 } | 212 } |
205 » if err := gs.DeleteAllFilesInDir(v.storageClient, config.GS.Bucket, "ski
a_version/current/"); err != nil { | 213 » if err := gs.DeleteAllFilesInDir(v.storageClient, config.GS.Bucket, "ski
a_version/current/", 1); err != nil { |
206 return err | 214 return err |
207 } | 215 } |
208 if err := v.touch(fmt.Sprintf("skia_version/current/%s", newHash)); err
!= nil { | 216 if err := v.touch(fmt.Sprintf("skia_version/current/%s", newHash)); err
!= nil { |
209 return err | 217 return err |
210 } | 218 } |
211 return v.touch(fmt.Sprintf("skia_version/old/%s", oldHash)) | 219 return v.touch(fmt.Sprintf("skia_version/old/%s", oldHash)) |
212 } | 220 } |
213 | 221 |
214 // touch creates an empty file in Google Storage of the given name. | 222 // touch creates an empty file in Google Storage of the given name. |
215 func (v *VersionUpdater) touch(file string) error { | 223 func (v *VersionUpdater) touch(file string) error { |
216 w := v.storageClient.Bucket(config.GS.Bucket).Object(file).NewWriter(con
text.Background()) | 224 w := v.storageClient.Bucket(config.GS.Bucket).Object(file).NewWriter(con
text.Background()) |
217 if err := w.Close(); err != nil { | 225 if err := w.Close(); err != nil { |
218 return fmt.Errorf("Could not touch version file %s : %s", file,
err) | 226 return fmt.Errorf("Could not touch version file %s : %s", file,
err) |
219 } | 227 } |
220 return nil | 228 return nil |
221 } | 229 } |
OLD | NEW |