| OLD | NEW |
| 1 package filediffstore | 1 package filediffstore |
| 2 | 2 |
| 3 import ( | 3 import ( |
| 4 "bytes" | 4 "bytes" |
| 5 "crypto/md5" | 5 "crypto/md5" |
| 6 "encoding/base64" | 6 "encoding/base64" |
| 7 "encoding/json" | 7 "encoding/json" |
| 8 "fmt" | 8 "fmt" |
| 9 "image" | 9 "image" |
| 10 "io" | 10 "io" |
| (...skipping 94 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 105 // The GS bucket where images are stored. | 105 // The GS bucket where images are stored. |
| 106 gsBucketName string | 106 gsBucketName string |
| 107 | 107 |
| 108 // The complete GS URL where images are stored. | 108 // The complete GS URL where images are stored. |
| 109 storageBaseDir string | 109 storageBaseDir string |
| 110 | 110 |
| 111 // The channels workers pick up tasks from. | 111 // The channels workers pick up tasks from. |
| 112 absPathCh chan *WorkerReq | 112 absPathCh chan *WorkerReq |
| 113 getCh chan *WorkerReq | 113 getCh chan *WorkerReq |
| 114 | 114 |
| 115 » // ignoreableDigests contains the digests that should be ignored. | 115 » // unavailableDigests contains the digests that should be ignored. |
| 116 » ignorableDigests map[string]bool | 116 » unavailableDigests map[string]bool |
| 117 | 117 |
| 118 » // idChan is a channel to add to ignorableDigests. | 118 » // idChan is a channel to add to unavailableDigests. |
| 119 » ignorableChan chan string | 119 » unavailableChan chan string |
| 120 | 120 |
| 121 » // Mutex for the ignorable Channel | 121 » // unavailableMutex protects unavailableDigests |
| 122 » ignorableMutex sync.Mutex | 122 » unavailableMutex sync.Mutex |
| 123 | 123 |
| 124 // Mutexes for ensuring safe access to the different local caches. | 124 // Mutexes for ensuring safe access to the different local caches. |
| 125 diffDirLock sync.Mutex | 125 diffDirLock sync.Mutex |
| 126 digestDirLock sync.Mutex | 126 digestDirLock sync.Mutex |
| 127 } | 127 } |
| 128 | 128 |
| 129 // NewFileDiffStore intializes and returns a file based implementation of | 129 // NewFileDiffStore intializes and returns a file based implementation of |
| 130 // DiffStore. The optional http.Client is used to make HTTP requests to Google | 130 // DiffStore. The optional http.Client is used to make HTTP requests to Google |
| 131 // Storage. If nil is supplied then a default client is used. The baseDir is the | 131 // Storage. If nil is supplied then a default client is used. The baseDir is the |
| 132 // local base directory where the DEFAULT_IMG_DIR_NAME, DEFAULT_DIFF_DIR_NAME an
d | 132 // local base directory where the DEFAULT_IMG_DIR_NAME, DEFAULT_DIFF_DIR_NAME an
d |
| (...skipping 11 matching lines...) Expand all Loading... |
| 144 if storageBaseDir == "" { | 144 if storageBaseDir == "" { |
| 145 storageBaseDir = DEFAULT_GS_IMG_DIR_NAME | 145 storageBaseDir = DEFAULT_GS_IMG_DIR_NAME |
| 146 } | 146 } |
| 147 | 147 |
| 148 imageCache, err := lru.New(IMAGE_LRU_CACHE_SIZE) | 148 imageCache, err := lru.New(IMAGE_LRU_CACHE_SIZE) |
| 149 if err != nil { | 149 if err != nil { |
| 150 return nil, fmt.Errorf("Unable to alloace image LRU cache: %s",
err) | 150 return nil, fmt.Errorf("Unable to alloace image LRU cache: %s",
err) |
| 151 } | 151 } |
| 152 | 152 |
| 153 diffCache := cacheFactory("di", DiffMetricsCodec(0)) | 153 diffCache := cacheFactory("di", DiffMetricsCodec(0)) |
| 154 » ignorableChan := make(chan string, 10) | 154 » unavailableChan := make(chan string, 10) |
| 155 | 155 |
| 156 fs := &FileDiffStore{ | 156 fs := &FileDiffStore{ |
| 157 client: client, | 157 client: client, |
| 158 localImgDir: fileutil.Must(fileutil.EnsureDirExists(file
path.Join(baseDir, DEFAULT_IMG_DIR_NAME))), | 158 localImgDir: fileutil.Must(fileutil.EnsureDirExists(file
path.Join(baseDir, DEFAULT_IMG_DIR_NAME))), |
| 159 localDiffDir: fileutil.Must(fileutil.EnsureDirExists(file
path.Join(baseDir, DEFAULT_DIFF_DIR_NAME))), | 159 localDiffDir: fileutil.Must(fileutil.EnsureDirExists(file
path.Join(baseDir, DEFAULT_DIFF_DIR_NAME))), |
| 160 localDiffMetricsDir: fileutil.Must(fileutil.EnsureDirExists(file
path.Join(baseDir, DEFAULT_DIFFMETRICS_DIR_NAME))), | 160 localDiffMetricsDir: fileutil.Must(fileutil.EnsureDirExists(file
path.Join(baseDir, DEFAULT_DIFFMETRICS_DIR_NAME))), |
| 161 localTempFileDir: fileutil.Must(fileutil.EnsureDirExists(file
path.Join(baseDir, DEFAULT_TEMPFILE_DIR_NAME))), | 161 localTempFileDir: fileutil.Must(fileutil.EnsureDirExists(file
path.Join(baseDir, DEFAULT_TEMPFILE_DIR_NAME))), |
| 162 gsBucketName: gsBucketName, | 162 gsBucketName: gsBucketName, |
| 163 storageBaseDir: storageBaseDir, | 163 storageBaseDir: storageBaseDir, |
| 164 imageCache: imageCache, | 164 imageCache: imageCache, |
| 165 diffCache: diffCache, | 165 diffCache: diffCache, |
| 166 » » ignorableDigests: map[string]bool{}, | 166 » » unavailableDigests: map[string]bool{}, |
| 167 » » ignorableChan: ignorableChan, | 167 » » unavailableChan: unavailableChan, |
| 168 } | 168 } |
| 169 | 169 |
| 170 // TODO(stephana): Clean this up and store digests to ignore in the | 170 // TODO(stephana): Clean this up and store digests to ignore in the |
| 171 // database and expose them on the front-end. | 171 // database and expose them on the front-end. |
| 172 // This is the hash of the empty, we should ignore this right away. | 172 // This is the hash of the empty, we should ignore this right away. |
| 173 » ignorableChan <- "d41d8cd98f00b204e9800998ecf8427e" | 173 » unavailableChan <- "d41d8cd98f00b204e9800998ecf8427e" |
| 174 go func() { | 174 go func() { |
| 175 var ignoreDigest string | 175 var ignoreDigest string |
| 176 for { | 176 for { |
| 177 » » » ignoreDigest = <-ignorableChan | 177 » » » ignoreDigest = <-unavailableChan |
| 178 func() { | 178 func() { |
| 179 » » » » fs.ignorableMutex.Lock() | 179 » » » » fs.unavailableMutex.Lock() |
| 180 » » » » defer fs.ignorableMutex.Unlock() | 180 » » » » defer fs.unavailableMutex.Unlock() |
| 181 » » » » fs.ignorableDigests[ignoreDigest] = true | 181 » » » » fs.unavailableDigests[ignoreDigest] = true |
| 182 }() | 182 }() |
| 183 } | 183 } |
| 184 }() | 184 }() |
| 185 | 185 |
| 186 fs.activateWorkers(workerPoolSize) | 186 fs.activateWorkers(workerPoolSize) |
| 187 return fs, nil | 187 return fs, nil |
| 188 } | 188 } |
| 189 | 189 |
| 190 type WorkerReq struct { | 190 type WorkerReq struct { |
| 191 id interface{} | 191 id interface{} |
| (...skipping 203 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 395 } | 395 } |
| 396 if (len(digestsToPaths) + digestErrors) == len(digests)
{ | 396 if (len(digestsToPaths) + digestErrors) == len(digests)
{ |
| 397 // 4. Return map of digests to paths once all re
quests have | 397 // 4. Return map of digests to paths once all re
quests have |
| 398 // been processed. | 398 // been processed. |
| 399 return digestsToPaths | 399 return digestsToPaths |
| 400 } | 400 } |
| 401 } | 401 } |
| 402 } | 402 } |
| 403 } | 403 } |
| 404 | 404 |
| 405 // IgnorableDigests is part of the diff.DiffStore interface. See details there. | 405 // UnavailableDigests is part of the diff.DiffStore interface. See details there
. |
| 406 func (fs *FileDiffStore) IgnorableDigests() map[string]bool { | 406 func (fs *FileDiffStore) UnavailableDigests() map[string]bool { |
| 407 » fs.ignorableMutex.Lock() | 407 » fs.unavailableMutex.Lock() |
| 408 » defer fs.ignorableMutex.Unlock() | 408 » defer fs.unavailableMutex.Unlock() |
| 409 » result := make(map[string]bool, len(fs.ignorableDigests)) | 409 » result := make(map[string]bool, len(fs.unavailableDigests)) |
| 410 » for k, v := range fs.ignorableDigests { | 410 » for k, v := range fs.unavailableDigests { |
| 411 result[k] = v | 411 result[k] = v |
| 412 } | 412 } |
| 413 return result | 413 return result |
| 414 } | 414 } |
| 415 | 415 |
| 416 func openDiffMetrics(filepath string) (*diff.DiffMetrics, error) { | 416 func openDiffMetrics(filepath string) (*diff.DiffMetrics, error) { |
| 417 f, err := ioutil.ReadFile(filepath) | 417 f, err := ioutil.ReadFile(filepath) |
| 418 if err != nil { | 418 if err != nil { |
| 419 return nil, fmt.Errorf("Failed to open DiffMetrics %s for readin
g: %s", filepath, err) | 419 return nil, fmt.Errorf("Failed to open DiffMetrics %s for readin
g: %s", filepath, err) |
| 420 } | 420 } |
| (...skipping 231 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 652 if obj, ok := fs.imageCache.Get(d); ok { | 652 if obj, ok := fs.imageCache.Get(d); ok { |
| 653 return obj.(image.Image), nil | 653 return obj.(image.Image), nil |
| 654 } | 654 } |
| 655 img, err = diff.OpenImage(fs.getDigestImagePath(d)) | 655 img, err = diff.OpenImage(fs.getDigestImagePath(d)) |
| 656 if err == nil { | 656 if err == nil { |
| 657 fs.imageCache.Add(d, img) | 657 fs.imageCache.Add(d, img) |
| 658 return img, nil | 658 return img, nil |
| 659 } | 659 } |
| 660 | 660 |
| 661 // Mark the image as ignorable since we were not able to decode it. | 661 // Mark the image as ignorable since we were not able to decode it. |
| 662 » fs.ignorableChan <- d | 662 » fs.unavailableChan <- d |
| 663 | 663 |
| 664 return nil, fmt.Errorf("Unable to read image for %s: %s", d, err) | 664 return nil, fmt.Errorf("Unable to read image for %s: %s", d, err) |
| 665 } | 665 } |
| 666 | 666 |
| 667 // getDigestPath returns the filepath where the image corresponding to the | 667 // getDigestPath returns the filepath where the image corresponding to the |
| 668 // give digests should be stored. | 668 // give digests should be stored. |
| 669 func (fs *FileDiffStore) getDigestImagePath(digest string) string { | 669 func (fs *FileDiffStore) getDigestImagePath(digest string) string { |
| 670 return filepath.Join(fs.localImgDir, fmt.Sprintf("%s.%s", digest, IMG_EX
TENSION)) | 670 return filepath.Join(fs.localImgDir, fmt.Sprintf("%s.%s", digest, IMG_EX
TENSION)) |
| 671 } | 671 } |
| 672 | 672 |
| 673 // getDiffMetricPath returns the filename where the diffmetric should be | 673 // getDiffMetricPath returns the filename where the diffmetric should be |
| 674 // cached. | 674 // cached. |
| 675 func (fs *FileDiffStore) getDiffMetricPath(baseName string) string { | 675 func (fs *FileDiffStore) getDiffMetricPath(baseName string) string { |
| 676 fName := fmt.Sprintf("%s.%s", baseName, DIFFMETRICS_EXTENSION) | 676 fName := fmt.Sprintf("%s.%s", baseName, DIFFMETRICS_EXTENSION) |
| 677 return filepath.Join(fs.localDiffMetricsDir, fName) | 677 return filepath.Join(fs.localDiffMetricsDir, fName) |
| 678 } | 678 } |
| OLD | NEW |