Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(55)

Side by Side Diff: golden/go/filediffstore/filediffstore.go

Issue 884943003: Ignore traces (Closed) Base URL: https://skia.googlesource.com/buildbot@master
Patch Set: Created 5 years, 10 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « golden/go/diff/diff.go ('k') | golden/go/skiacorrectness/main.go » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 package filediffstore 1 package filediffstore
2 2
3 import ( 3 import (
4 "bytes" 4 "bytes"
5 "crypto/md5" 5 "crypto/md5"
6 "encoding/base64" 6 "encoding/base64"
7 "encoding/json" 7 "encoding/json"
8 "fmt" 8 "fmt"
9 "image" 9 "image"
10 "io" 10 "io"
(...skipping 94 matching lines...) Expand 10 before | Expand all | Expand 10 after
105 // The GS bucket where images are stored. 105 // The GS bucket where images are stored.
106 gsBucketName string 106 gsBucketName string
107 107
108 // The complete GS URL where images are stored. 108 // The complete GS URL where images are stored.
109 storageBaseDir string 109 storageBaseDir string
110 110
111 // The channels workers pick up tasks from. 111 // The channels workers pick up tasks from.
112 absPathCh chan *WorkerReq 112 absPathCh chan *WorkerReq
113 getCh chan *WorkerReq 113 getCh chan *WorkerReq
114 114
115 » // ignoreableDigests contains the digests that should be ignored. 115 » // unavailableDigests contains the digests that should be ignored.
116 » ignorableDigests map[string]bool 116 » unavailableDigests map[string]bool
117 117
118 » // idChan is a channel to add to ignorableDigests. 118 » // idChan is a channel to add to unavailableDigests.
119 » ignorableChan chan string 119 » unavailableChan chan string
120 120
121 » // Mutex for the ignorable Channel 121 » // unavailableMutex protects unavailableDigests
122 » ignorableMutex sync.Mutex 122 » unavailableMutex sync.Mutex
123 123
124 // Mutexes for ensuring safe access to the different local caches. 124 // Mutexes for ensuring safe access to the different local caches.
125 diffDirLock sync.Mutex 125 diffDirLock sync.Mutex
126 digestDirLock sync.Mutex 126 digestDirLock sync.Mutex
127 } 127 }
128 128
129 // NewFileDiffStore intializes and returns a file based implementation of 129 // NewFileDiffStore intializes and returns a file based implementation of
130 // DiffStore. The optional http.Client is used to make HTTP requests to Google 130 // DiffStore. The optional http.Client is used to make HTTP requests to Google
131 // Storage. If nil is supplied then a default client is used. The baseDir is the 131 // Storage. If nil is supplied then a default client is used. The baseDir is the
132 // local base directory where the DEFAULT_IMG_DIR_NAME, DEFAULT_DIFF_DIR_NAME an d 132 // local base directory where the DEFAULT_IMG_DIR_NAME, DEFAULT_DIFF_DIR_NAME an d
(...skipping 11 matching lines...) Expand all
144 if storageBaseDir == "" { 144 if storageBaseDir == "" {
145 storageBaseDir = DEFAULT_GS_IMG_DIR_NAME 145 storageBaseDir = DEFAULT_GS_IMG_DIR_NAME
146 } 146 }
147 147
148 imageCache, err := lru.New(IMAGE_LRU_CACHE_SIZE) 148 imageCache, err := lru.New(IMAGE_LRU_CACHE_SIZE)
149 if err != nil { 149 if err != nil {
150 return nil, fmt.Errorf("Unable to alloace image LRU cache: %s", err) 150 return nil, fmt.Errorf("Unable to alloace image LRU cache: %s", err)
151 } 151 }
152 152
153 diffCache := cacheFactory("di", DiffMetricsCodec(0)) 153 diffCache := cacheFactory("di", DiffMetricsCodec(0))
154 » ignorableChan := make(chan string, 10) 154 » unavailableChan := make(chan string, 10)
155 155
156 fs := &FileDiffStore{ 156 fs := &FileDiffStore{
157 client: client, 157 client: client,
158 localImgDir: fileutil.Must(fileutil.EnsureDirExists(file path.Join(baseDir, DEFAULT_IMG_DIR_NAME))), 158 localImgDir: fileutil.Must(fileutil.EnsureDirExists(file path.Join(baseDir, DEFAULT_IMG_DIR_NAME))),
159 localDiffDir: fileutil.Must(fileutil.EnsureDirExists(file path.Join(baseDir, DEFAULT_DIFF_DIR_NAME))), 159 localDiffDir: fileutil.Must(fileutil.EnsureDirExists(file path.Join(baseDir, DEFAULT_DIFF_DIR_NAME))),
160 localDiffMetricsDir: fileutil.Must(fileutil.EnsureDirExists(file path.Join(baseDir, DEFAULT_DIFFMETRICS_DIR_NAME))), 160 localDiffMetricsDir: fileutil.Must(fileutil.EnsureDirExists(file path.Join(baseDir, DEFAULT_DIFFMETRICS_DIR_NAME))),
161 localTempFileDir: fileutil.Must(fileutil.EnsureDirExists(file path.Join(baseDir, DEFAULT_TEMPFILE_DIR_NAME))), 161 localTempFileDir: fileutil.Must(fileutil.EnsureDirExists(file path.Join(baseDir, DEFAULT_TEMPFILE_DIR_NAME))),
162 gsBucketName: gsBucketName, 162 gsBucketName: gsBucketName,
163 storageBaseDir: storageBaseDir, 163 storageBaseDir: storageBaseDir,
164 imageCache: imageCache, 164 imageCache: imageCache,
165 diffCache: diffCache, 165 diffCache: diffCache,
166 » » ignorableDigests: map[string]bool{}, 166 » » unavailableDigests: map[string]bool{},
167 » » ignorableChan: ignorableChan, 167 » » unavailableChan: unavailableChan,
168 } 168 }
169 169
170 // TODO(stephana): Clean this up and store digests to ignore in the 170 // TODO(stephana): Clean this up and store digests to ignore in the
171 // database and expose them on the front-end. 171 // database and expose them on the front-end.
172 // This is the hash of the empty, we should ignore this right away. 172 // This is the hash of the empty, we should ignore this right away.
173 » ignorableChan <- "d41d8cd98f00b204e9800998ecf8427e" 173 » unavailableChan <- "d41d8cd98f00b204e9800998ecf8427e"
174 go func() { 174 go func() {
175 var ignoreDigest string 175 var ignoreDigest string
176 for { 176 for {
177 » » » ignoreDigest = <-ignorableChan 177 » » » ignoreDigest = <-unavailableChan
178 func() { 178 func() {
179 » » » » fs.ignorableMutex.Lock() 179 » » » » fs.unavailableMutex.Lock()
180 » » » » defer fs.ignorableMutex.Unlock() 180 » » » » defer fs.unavailableMutex.Unlock()
181 » » » » fs.ignorableDigests[ignoreDigest] = true 181 » » » » fs.unavailableDigests[ignoreDigest] = true
182 }() 182 }()
183 } 183 }
184 }() 184 }()
185 185
186 fs.activateWorkers(workerPoolSize) 186 fs.activateWorkers(workerPoolSize)
187 return fs, nil 187 return fs, nil
188 } 188 }
189 189
190 type WorkerReq struct { 190 type WorkerReq struct {
191 id interface{} 191 id interface{}
(...skipping 203 matching lines...) Expand 10 before | Expand all | Expand 10 after
395 } 395 }
396 if (len(digestsToPaths) + digestErrors) == len(digests) { 396 if (len(digestsToPaths) + digestErrors) == len(digests) {
397 // 4. Return map of digests to paths once all re quests have 397 // 4. Return map of digests to paths once all re quests have
398 // been processed. 398 // been processed.
399 return digestsToPaths 399 return digestsToPaths
400 } 400 }
401 } 401 }
402 } 402 }
403 } 403 }
404 404
405 // IgnorableDigests is part of the diff.DiffStore interface. See details there. 405 // UnavailableDigests is part of the diff.DiffStore interface. See details there .
406 func (fs *FileDiffStore) IgnorableDigests() map[string]bool { 406 func (fs *FileDiffStore) UnavailableDigests() map[string]bool {
407 » fs.ignorableMutex.Lock() 407 » fs.unavailableMutex.Lock()
408 » defer fs.ignorableMutex.Unlock() 408 » defer fs.unavailableMutex.Unlock()
409 » result := make(map[string]bool, len(fs.ignorableDigests)) 409 » result := make(map[string]bool, len(fs.unavailableDigests))
410 » for k, v := range fs.ignorableDigests { 410 » for k, v := range fs.unavailableDigests {
411 result[k] = v 411 result[k] = v
412 } 412 }
413 return result 413 return result
414 } 414 }
415 415
416 func openDiffMetrics(filepath string) (*diff.DiffMetrics, error) { 416 func openDiffMetrics(filepath string) (*diff.DiffMetrics, error) {
417 f, err := ioutil.ReadFile(filepath) 417 f, err := ioutil.ReadFile(filepath)
418 if err != nil { 418 if err != nil {
419 return nil, fmt.Errorf("Failed to open DiffMetrics %s for readin g: %s", filepath, err) 419 return nil, fmt.Errorf("Failed to open DiffMetrics %s for readin g: %s", filepath, err)
420 } 420 }
(...skipping 231 matching lines...) Expand 10 before | Expand all | Expand 10 after
652 if obj, ok := fs.imageCache.Get(d); ok { 652 if obj, ok := fs.imageCache.Get(d); ok {
653 return obj.(image.Image), nil 653 return obj.(image.Image), nil
654 } 654 }
655 img, err = diff.OpenImage(fs.getDigestImagePath(d)) 655 img, err = diff.OpenImage(fs.getDigestImagePath(d))
656 if err == nil { 656 if err == nil {
657 fs.imageCache.Add(d, img) 657 fs.imageCache.Add(d, img)
658 return img, nil 658 return img, nil
659 } 659 }
660 660
661 // Mark the image as ignorable since we were not able to decode it. 661 // Mark the image as ignorable since we were not able to decode it.
662 » fs.ignorableChan <- d 662 » fs.unavailableChan <- d
663 663
664 return nil, fmt.Errorf("Unable to read image for %s: %s", d, err) 664 return nil, fmt.Errorf("Unable to read image for %s: %s", d, err)
665 } 665 }
666 666
667 // getDigestPath returns the filepath where the image corresponding to the 667 // getDigestPath returns the filepath where the image corresponding to the
668 // give digests should be stored. 668 // give digests should be stored.
669 func (fs *FileDiffStore) getDigestImagePath(digest string) string { 669 func (fs *FileDiffStore) getDigestImagePath(digest string) string {
670 return filepath.Join(fs.localImgDir, fmt.Sprintf("%s.%s", digest, IMG_EX TENSION)) 670 return filepath.Join(fs.localImgDir, fmt.Sprintf("%s.%s", digest, IMG_EX TENSION))
671 } 671 }
672 672
673 // getDiffMetricPath returns the filename where the diffmetric should be 673 // getDiffMetricPath returns the filename where the diffmetric should be
674 // cached. 674 // cached.
675 func (fs *FileDiffStore) getDiffMetricPath(baseName string) string { 675 func (fs *FileDiffStore) getDiffMetricPath(baseName string) string {
676 fName := fmt.Sprintf("%s.%s", baseName, DIFFMETRICS_EXTENSION) 676 fName := fmt.Sprintf("%s.%s", baseName, DIFFMETRICS_EXTENSION)
677 return filepath.Join(fs.localDiffMetricsDir, fName) 677 return filepath.Join(fs.localDiffMetricsDir, fName)
678 } 678 }
OLDNEW
« no previous file with comments | « golden/go/diff/diff.go ('k') | golden/go/skiacorrectness/main.go » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698