Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(121)

Side by Side Diff: fuzzer/go/backend/version_updater.go

Issue 1668543004: Add AddressSanitizer to fuzzer analysis (Closed) Base URL: https://skia.googlesource.com/buildbot@remove-old-tests
Patch Set: add multi threaded delete Created 4 years, 10 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 package backend 1 package backend
2 2
3 import ( 3 import (
4 "fmt" 4 "fmt"
5 "io/ioutil" 5 "io/ioutil"
6 "os" 6 "os"
7 "path/filepath" 7 "path/filepath"
8 "strings" 8 "strings"
9 "sync" 9 "sync"
10 "sync/atomic" 10 "sync/atomic"
(...skipping 66 matching lines...) Expand 10 before | Expand all | Expand 10 after
77 77
78 return config.Generator.SkiaVersion, nil 78 return config.Generator.SkiaVersion, nil
79 } 79 }
80 80
81 func (p *FuzzPipeline) reanalyzeAndRestart(storageClient *storage.Client, oldRev ision string) error { 81 func (p *FuzzPipeline) reanalyzeAndRestart(storageClient *storage.Client, oldRev ision string) error {
82 // download all bad and grey fuzzes 82 // download all bad and grey fuzzes
83 badFuzzNames, greyFuzzNames, err := p.downloadAllBadAndGreyFuzzes(oldRev ision, storageClient) 83 badFuzzNames, greyFuzzNames, err := p.downloadAllBadAndGreyFuzzes(oldRev ision, storageClient)
84 if err != nil { 84 if err != nil {
85 return fmt.Errorf("Problem downloading all previous fuzzes: %s", err) 85 return fmt.Errorf("Problem downloading all previous fuzzes: %s", err)
86 } 86 }
87 » glog.Infof("There are %d badFuzzNames and %d greyFuzzNames to rescan.", len(badFuzzNames), len(greyFuzzNames)) 87 » glog.Infof("There are %d bad fuzzes and %d grey fuzzes to rescan.", len( badFuzzNames), len(greyFuzzNames))
88 // This is a soft shutdown, i.e. it waits for aggregator's queues to be empty 88 // This is a soft shutdown, i.e. it waits for aggregator's queues to be empty
89 p.Agg.ShutDown() 89 p.Agg.ShutDown()
90
91 if config.Common.ForceReanalysis {
92 glog.Infof("Deleting previous fuzz results")
93 if err := gs.DeleteAllFilesInDir(storageClient, config.GS.Bucket , fmt.Sprintf("%s/%s/", p.Category, oldRevision), config.Aggregator.NumUploadPro cesses); err != nil {
94 return fmt.Errorf("Could not delete previous fuzzes: %s" , err)
95 }
96 }
97
90 if err := p.Gen.Clear(); err != nil { 98 if err := p.Gen.Clear(); err != nil {
91 return fmt.Errorf("Could not remove previous afl-fuzz results: % s", err) 99 return fmt.Errorf("Could not remove previous afl-fuzz results: % s", err)
92 } 100 }
93 101
94 if err := p.Agg.RestartAnalysis(); err != nil { 102 if err := p.Agg.RestartAnalysis(); err != nil {
95 return fmt.Errorf("Had problem restarting analysis/upload chain: %s", err) 103 return fmt.Errorf("Had problem restarting analysis/upload chain: %s", err)
96 } 104 }
97 // Reanalyze and reupload the fuzzes, making a bug on regressions. 105 // Reanalyze and reupload the fuzzes, making a bug on regressions.
98 glog.Infof("Reanalyzing bad fuzzes") 106 glog.Infof("Reanalyzing bad fuzzes")
99 p.Agg.MakeBugOnBadFuzz = false 107 p.Agg.MakeBugOnBadFuzz = false
(...skipping 32 matching lines...) Expand 10 before | Expand all | Expand 10 after
132 completedCounter = 0 140 completedCounter = 0
133 141
134 var wg sync.WaitGroup 142 var wg sync.WaitGroup
135 for i := 0; i < config.Generator.NumDownloadProcesses; i++ { 143 for i := 0; i < config.Generator.NumDownloadProcesses; i++ {
136 wg.Add(1) 144 wg.Add(1)
137 go download(storageClient, toDownload, downloadPath, &wg) 145 go download(storageClient, toDownload, downloadPath, &wg)
138 } 146 }
139 147
140 badFilter := func(item *storage.ObjectAttrs) { 148 badFilter := func(item *storage.ObjectAttrs) {
141 name := item.Name 149 name := item.Name
142 » » if strings.HasSuffix(name, ".dump") || strings.HasSuffix(name, " .err") { 150 » » if strings.Contains(name, ".") {
143 return 151 return
144 } 152 }
145 fuzzHash := name[strings.LastIndex(name, "/")+1:] 153 fuzzHash := name[strings.LastIndex(name, "/")+1:]
146 badFuzzNames = append(badFuzzNames, filepath.Join(downloadPath, fuzzHash)) 154 badFuzzNames = append(badFuzzNames, filepath.Join(downloadPath, fuzzHash))
147 toDownload <- item.Name 155 toDownload <- item.Name
148 } 156 }
149 157
150 greyFilter := func(item *storage.ObjectAttrs) { 158 greyFilter := func(item *storage.ObjectAttrs) {
151 name := item.Name 159 name := item.Name
152 » » if strings.HasSuffix(item.Name, ".dump") || strings.HasSuffix(it em.Name, ".err") { 160 » » if strings.Contains(name, ".") {
153 return 161 return
154 } 162 }
155 fuzzHash := name[strings.LastIndex(name, "/")+1:] 163 fuzzHash := name[strings.LastIndex(name, "/")+1:]
156 greyFuzzNames = append(greyFuzzNames, filepath.Join(downloadPath , fuzzHash)) 164 greyFuzzNames = append(greyFuzzNames, filepath.Join(downloadPath , fuzzHash))
157 toDownload <- item.Name 165 toDownload <- item.Name
158 } 166 }
159 167
160 if err := gs.AllFilesInDir(storageClient, config.GS.Bucket, fmt.Sprintf( "%s/%s/bad", p.Category, commitHash), badFilter); err != nil { 168 if err := gs.AllFilesInDir(storageClient, config.GS.Bucket, fmt.Sprintf( "%s/%s/bad", p.Category, commitHash), badFilter); err != nil {
161 return nil, nil, fmt.Errorf("Problem getting bad fuzzes: %s", er r) 169 return nil, nil, fmt.Errorf("Problem getting bad fuzzes: %s", er r)
162 } 170 }
(...skipping 29 matching lines...) Expand all
192 if completedCounter%100 == 0 { 200 if completedCounter%100 == 0 {
193 glog.Infof("%d fuzzes downloaded", completedCounter) 201 glog.Infof("%d fuzzes downloaded", completedCounter)
194 } 202 }
195 } 203 }
196 } 204 }
197 205
198 // replaceCurrentSkiaVersionWith puts the oldHash in skia_version/old and the ne wHash in 206 // replaceCurrentSkiaVersionWith puts the oldHash in skia_version/old and the ne wHash in
199 // skia_version/current. It also removes all pending versions. 207 // skia_version/current. It also removes all pending versions.
200 func (v *VersionUpdater) replaceCurrentSkiaVersionWith(oldHash, newHash string) error { 208 func (v *VersionUpdater) replaceCurrentSkiaVersionWith(oldHash, newHash string) error {
201 // delete all pending requests 209 // delete all pending requests
202 » if err := gs.DeleteAllFilesInDir(v.storageClient, config.GS.Bucket, "ski a_version/pending/"); err != nil { 210 » if err := gs.DeleteAllFilesInDir(v.storageClient, config.GS.Bucket, "ski a_version/pending/", 1); err != nil {
203 return err 211 return err
204 } 212 }
205 » if err := gs.DeleteAllFilesInDir(v.storageClient, config.GS.Bucket, "ski a_version/current/"); err != nil { 213 » if err := gs.DeleteAllFilesInDir(v.storageClient, config.GS.Bucket, "ski a_version/current/", 1); err != nil {
206 return err 214 return err
207 } 215 }
208 if err := v.touch(fmt.Sprintf("skia_version/current/%s", newHash)); err != nil { 216 if err := v.touch(fmt.Sprintf("skia_version/current/%s", newHash)); err != nil {
209 return err 217 return err
210 } 218 }
211 return v.touch(fmt.Sprintf("skia_version/old/%s", oldHash)) 219 return v.touch(fmt.Sprintf("skia_version/old/%s", oldHash))
212 } 220 }
213 221
214 // touch creates an empty file in Google Storage of the given name. 222 // touch creates an empty file in Google Storage of the given name.
215 func (v *VersionUpdater) touch(file string) error { 223 func (v *VersionUpdater) touch(file string) error {
216 w := v.storageClient.Bucket(config.GS.Bucket).Object(file).NewWriter(con text.Background()) 224 w := v.storageClient.Bucket(config.GS.Bucket).Object(file).NewWriter(con text.Background())
217 if err := w.Close(); err != nil { 225 if err := w.Close(); err != nil {
218 return fmt.Errorf("Could not touch version file %s : %s", file, err) 226 return fmt.Errorf("Could not touch version file %s : %s", file, err)
219 } 227 }
220 return nil 228 return nil
221 } 229 }
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698