OLD | NEW |
1 package filediffstore | 1 package filediffstore |
2 | 2 |
3 import ( | 3 import ( |
4 "bytes" | 4 "bytes" |
5 "crypto/md5" | 5 "crypto/md5" |
6 "encoding/base64" | 6 "encoding/base64" |
7 "encoding/json" | 7 "encoding/json" |
8 "fmt" | 8 "fmt" |
9 "image" | 9 "image" |
10 "io" | 10 "io" |
(...skipping 94 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
105 // The GS bucket where images are stored. | 105 // The GS bucket where images are stored. |
106 gsBucketName string | 106 gsBucketName string |
107 | 107 |
108 // The complete GS URL where images are stored. | 108 // The complete GS URL where images are stored. |
109 storageBaseDir string | 109 storageBaseDir string |
110 | 110 |
111 // The channels workers pick up tasks from. | 111 // The channels workers pick up tasks from. |
112 absPathCh chan *WorkerReq | 112 absPathCh chan *WorkerReq |
113 getCh chan *WorkerReq | 113 getCh chan *WorkerReq |
114 | 114 |
115 » // ignoreableDigests contains the digests that should be ignored. | 115 » // unavailableDigests contains the digests that should be ignored. |
116 » ignorableDigests map[string]bool | 116 » unavailableDigests map[string]bool |
117 | 117 |
118 » // idChan is a channel to add to ignorableDigests. | 118 » // idChan is a channel to add to unavailableDigests. |
119 » ignorableChan chan string | 119 » unavailableChan chan string |
120 | 120 |
121 » // Mutex for the ignorable Channel | 121 » // unavailableMutex protects unavailableDigests |
122 » ignorableMutex sync.Mutex | 122 » unavailableMutex sync.Mutex |
123 | 123 |
124 // Mutexes for ensuring safe access to the different local caches. | 124 // Mutexes for ensuring safe access to the different local caches. |
125 diffDirLock sync.Mutex | 125 diffDirLock sync.Mutex |
126 digestDirLock sync.Mutex | 126 digestDirLock sync.Mutex |
127 } | 127 } |
128 | 128 |
129 // NewFileDiffStore intializes and returns a file based implementation of | 129 // NewFileDiffStore intializes and returns a file based implementation of |
130 // DiffStore. The optional http.Client is used to make HTTP requests to Google | 130 // DiffStore. The optional http.Client is used to make HTTP requests to Google |
131 // Storage. If nil is supplied then a default client is used. The baseDir is the | 131 // Storage. If nil is supplied then a default client is used. The baseDir is the |
132 // local base directory where the DEFAULT_IMG_DIR_NAME, DEFAULT_DIFF_DIR_NAME an
d | 132 // local base directory where the DEFAULT_IMG_DIR_NAME, DEFAULT_DIFF_DIR_NAME an
d |
(...skipping 11 matching lines...) Expand all Loading... |
144 if storageBaseDir == "" { | 144 if storageBaseDir == "" { |
145 storageBaseDir = DEFAULT_GS_IMG_DIR_NAME | 145 storageBaseDir = DEFAULT_GS_IMG_DIR_NAME |
146 } | 146 } |
147 | 147 |
148 imageCache, err := lru.New(IMAGE_LRU_CACHE_SIZE) | 148 imageCache, err := lru.New(IMAGE_LRU_CACHE_SIZE) |
149 if err != nil { | 149 if err != nil { |
150 return nil, fmt.Errorf("Unable to alloace image LRU cache: %s",
err) | 150 return nil, fmt.Errorf("Unable to alloace image LRU cache: %s",
err) |
151 } | 151 } |
152 | 152 |
153 diffCache := cacheFactory("di", DiffMetricsCodec(0)) | 153 diffCache := cacheFactory("di", DiffMetricsCodec(0)) |
154 » ignorableChan := make(chan string, 10) | 154 » unavailableChan := make(chan string, 10) |
155 | 155 |
156 fs := &FileDiffStore{ | 156 fs := &FileDiffStore{ |
157 client: client, | 157 client: client, |
158 localImgDir: fileutil.Must(fileutil.EnsureDirExists(file
path.Join(baseDir, DEFAULT_IMG_DIR_NAME))), | 158 localImgDir: fileutil.Must(fileutil.EnsureDirExists(file
path.Join(baseDir, DEFAULT_IMG_DIR_NAME))), |
159 localDiffDir: fileutil.Must(fileutil.EnsureDirExists(file
path.Join(baseDir, DEFAULT_DIFF_DIR_NAME))), | 159 localDiffDir: fileutil.Must(fileutil.EnsureDirExists(file
path.Join(baseDir, DEFAULT_DIFF_DIR_NAME))), |
160 localDiffMetricsDir: fileutil.Must(fileutil.EnsureDirExists(file
path.Join(baseDir, DEFAULT_DIFFMETRICS_DIR_NAME))), | 160 localDiffMetricsDir: fileutil.Must(fileutil.EnsureDirExists(file
path.Join(baseDir, DEFAULT_DIFFMETRICS_DIR_NAME))), |
161 localTempFileDir: fileutil.Must(fileutil.EnsureDirExists(file
path.Join(baseDir, DEFAULT_TEMPFILE_DIR_NAME))), | 161 localTempFileDir: fileutil.Must(fileutil.EnsureDirExists(file
path.Join(baseDir, DEFAULT_TEMPFILE_DIR_NAME))), |
162 gsBucketName: gsBucketName, | 162 gsBucketName: gsBucketName, |
163 storageBaseDir: storageBaseDir, | 163 storageBaseDir: storageBaseDir, |
164 imageCache: imageCache, | 164 imageCache: imageCache, |
165 diffCache: diffCache, | 165 diffCache: diffCache, |
166 » » ignorableDigests: map[string]bool{}, | 166 » » unavailableDigests: map[string]bool{}, |
167 » » ignorableChan: ignorableChan, | 167 » » unavailableChan: unavailableChan, |
168 } | 168 } |
169 | 169 |
170 // TODO(stephana): Clean this up and store digests to ignore in the | 170 // TODO(stephana): Clean this up and store digests to ignore in the |
171 // database and expose them on the front-end. | 171 // database and expose them on the front-end. |
172 // This is the hash of the empty, we should ignore this right away. | 172 // This is the hash of the empty, we should ignore this right away. |
173 » ignorableChan <- "d41d8cd98f00b204e9800998ecf8427e" | 173 » unavailableChan <- "d41d8cd98f00b204e9800998ecf8427e" |
174 go func() { | 174 go func() { |
175 var ignoreDigest string | 175 var ignoreDigest string |
176 for { | 176 for { |
177 » » » ignoreDigest = <-ignorableChan | 177 » » » ignoreDigest = <-unavailableChan |
178 func() { | 178 func() { |
179 » » » » fs.ignorableMutex.Lock() | 179 » » » » fs.unavailableMutex.Lock() |
180 » » » » defer fs.ignorableMutex.Unlock() | 180 » » » » defer fs.unavailableMutex.Unlock() |
181 » » » » fs.ignorableDigests[ignoreDigest] = true | 181 » » » » fs.unavailableDigests[ignoreDigest] = true |
182 }() | 182 }() |
183 } | 183 } |
184 }() | 184 }() |
185 | 185 |
186 fs.activateWorkers(workerPoolSize) | 186 fs.activateWorkers(workerPoolSize) |
187 return fs, nil | 187 return fs, nil |
188 } | 188 } |
189 | 189 |
190 type WorkerReq struct { | 190 type WorkerReq struct { |
191 id interface{} | 191 id interface{} |
(...skipping 203 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
395 } | 395 } |
396 if (len(digestsToPaths) + digestErrors) == len(digests)
{ | 396 if (len(digestsToPaths) + digestErrors) == len(digests)
{ |
397 // 4. Return map of digests to paths once all re
quests have | 397 // 4. Return map of digests to paths once all re
quests have |
398 // been processed. | 398 // been processed. |
399 return digestsToPaths | 399 return digestsToPaths |
400 } | 400 } |
401 } | 401 } |
402 } | 402 } |
403 } | 403 } |
404 | 404 |
405 // IgnorableDigests is part of the diff.DiffStore interface. See details there. | 405 // UnavailableDigests is part of the diff.DiffStore interface. See details there
. |
406 func (fs *FileDiffStore) IgnorableDigests() map[string]bool { | 406 func (fs *FileDiffStore) UnavailableDigests() map[string]bool { |
407 » fs.ignorableMutex.Lock() | 407 » fs.unavailableMutex.Lock() |
408 » defer fs.ignorableMutex.Unlock() | 408 » defer fs.unavailableMutex.Unlock() |
409 » result := make(map[string]bool, len(fs.ignorableDigests)) | 409 » result := make(map[string]bool, len(fs.unavailableDigests)) |
410 » for k, v := range fs.ignorableDigests { | 410 » for k, v := range fs.unavailableDigests { |
411 result[k] = v | 411 result[k] = v |
412 } | 412 } |
413 return result | 413 return result |
414 } | 414 } |
415 | 415 |
416 func openDiffMetrics(filepath string) (*diff.DiffMetrics, error) { | 416 func openDiffMetrics(filepath string) (*diff.DiffMetrics, error) { |
417 f, err := ioutil.ReadFile(filepath) | 417 f, err := ioutil.ReadFile(filepath) |
418 if err != nil { | 418 if err != nil { |
419 return nil, fmt.Errorf("Failed to open DiffMetrics %s for readin
g: %s", filepath, err) | 419 return nil, fmt.Errorf("Failed to open DiffMetrics %s for readin
g: %s", filepath, err) |
420 } | 420 } |
(...skipping 231 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
652 if obj, ok := fs.imageCache.Get(d); ok { | 652 if obj, ok := fs.imageCache.Get(d); ok { |
653 return obj.(image.Image), nil | 653 return obj.(image.Image), nil |
654 } | 654 } |
655 img, err = diff.OpenImage(fs.getDigestImagePath(d)) | 655 img, err = diff.OpenImage(fs.getDigestImagePath(d)) |
656 if err == nil { | 656 if err == nil { |
657 fs.imageCache.Add(d, img) | 657 fs.imageCache.Add(d, img) |
658 return img, nil | 658 return img, nil |
659 } | 659 } |
660 | 660 |
661 // Mark the image as ignorable since we were not able to decode it. | 661 // Mark the image as ignorable since we were not able to decode it. |
662 » fs.ignorableChan <- d | 662 » fs.unavailableChan <- d |
663 | 663 |
664 return nil, fmt.Errorf("Unable to read image for %s: %s", d, err) | 664 return nil, fmt.Errorf("Unable to read image for %s: %s", d, err) |
665 } | 665 } |
666 | 666 |
667 // getDigestPath returns the filepath where the image corresponding to the | 667 // getDigestPath returns the filepath where the image corresponding to the |
668 // give digests should be stored. | 668 // give digests should be stored. |
669 func (fs *FileDiffStore) getDigestImagePath(digest string) string { | 669 func (fs *FileDiffStore) getDigestImagePath(digest string) string { |
670 return filepath.Join(fs.localImgDir, fmt.Sprintf("%s.%s", digest, IMG_EX
TENSION)) | 670 return filepath.Join(fs.localImgDir, fmt.Sprintf("%s.%s", digest, IMG_EX
TENSION)) |
671 } | 671 } |
672 | 672 |
673 // getDiffMetricPath returns the filename where the diffmetric should be | 673 // getDiffMetricPath returns the filename where the diffmetric should be |
674 // cached. | 674 // cached. |
675 func (fs *FileDiffStore) getDiffMetricPath(baseName string) string { | 675 func (fs *FileDiffStore) getDiffMetricPath(baseName string) string { |
676 fName := fmt.Sprintf("%s.%s", baseName, DIFFMETRICS_EXTENSION) | 676 fName := fmt.Sprintf("%s.%s", baseName, DIFFMETRICS_EXTENSION) |
677 return filepath.Join(fs.localDiffMetricsDir, fName) | 677 return filepath.Join(fs.localDiffMetricsDir, fName) |
678 } | 678 } |
OLD | NEW |