cipd: instance cache

client/cipd/client.go

Index: client/cipd/client.go
diff --git a/client/cipd/client.go b/client/cipd/client.go
index 3d2f1f818577be7badd90eda4d21bfa6321dddb4..6eaa4859dfdb1e406f861a027905aa15de5087cd 100644
--- a/client/cipd/client.go
+++ b/client/cipd/client.go
@@ -337,9 +337,9 @@ type ClientOptions struct {
 	// UserAgent is put into User-Agent HTTP header with each request.
 	UserAgent string
 
-	// CacheDir is a directory for shared cache. If empty, tags are cached
-	// inside the site root. If both Root and CacheDir are empty, tag cache
-	// is disabled.
+	// CacheDir is a directory for shared cache. If empty, instances are not
+	// cached and tags are cached inside the site root. If both Root and
+	// CacheDir are empty, tag cache is disabled.
 	CacheDir string
 }
 
@@ -394,6 +394,10 @@ type clientImpl struct {
 	tagCache     *internal.TagCache
 	tagCacheInit sync.Once
 
+	// instanceCache is a file-system based cache of instances.
+	instanceCache     *internal.InstanceCache
+	instanceCacheInit sync.Once
+
 	// authClient is a lazily created http.Client to use for authenticated
 	// requests. Thread safe, but lazily initialized under lock.
 	authClient *http.Client
@@ -487,6 +491,38 @@ func (client *clientImpl) closeTagCache() {
 	client.tagCache = nil
 }
 
+// instanceCachePath returns path to the instance cache directory or "" if
+// instance cache is disabled.
+func (client *clientImpl) instanceCachePath() string {
+	if client.CacheDir == "" {
+		return ""
+	}
+	return filepath.Join(client.CacheDir, "instances")
+}
+
+// getInstanceCache lazy-initializes instanceCache and returns it.
+func (client *clientImpl) getInstanceCache() *internal.InstanceCache {
+	client.instanceCacheInit.Do(func() {
+		if path := client.instanceCachePath(); path != "" {
+			cachePath := local.NewFileSystem(path, client.Logger)
+			client.instanceCache = internal.LoadInstanceCache(cachePath, client.Logger, client.clock.now())
+		}
+	})
+	return client.instanceCache
+}
+
+// closeInstanceCache dumps any changes made to instance cache to disk, if necessary.
+func (client *clientImpl) closeInstanceCache() {
+	if client.instanceCache == nil || !client.instanceCache.Dirty() {
+		return
+	}
+
+	if err := client.instanceCache.Save(); err != nil {
+		client.Logger.Warningf("cipd: failed to save instance cache - %s", err)
+	}
+	client.instanceCache = nil
+}
+
 func (client *clientImpl) FetchACL(packagePath string) ([]PackageACL, error) {
 	return client.remote.fetchACL(packagePath)
 }
@@ -754,15 +790,63 @@ func (client *clientImpl) FetchInstance(pin common.Pin, output io.WriteSeeker) e
 	if err != nil {
 		return err
 	}
+
+	now := client.clock.now()
+
+	cache := client.getInstanceCache()
+	if cache != nil {
+		switch err := cache.Get(pin, output, now); {
+		case os.IsNotExist(err):
+			// output is not corrupted.
+
+		case err != nil:
+			client.Logger.Warningf("cipd: could not get %s from cache - %s", pin, err)
+			// Output may be corrupted. Rewinding back.
+			if _, err := output.Seek(0, 0); err != nil {
+				return err
+			}
+
+		default:
+			client.Logger.Infof("cipd: cache hit for %s", pin)
+			return nil
+		}
+	}
+
 	client.Logger.Infof("cipd: resolving fetch URL for %s", pin)
 	fetchInfo, err := client.remote.fetchInstance(pin)
-	if err == nil {
-		err = client.storage.download(fetchInfo.fetchURL, output)
+	fetchFailed := func(err error) {
+		client.Logger.Errorf("cipd: failed to fetch %s - %s", pin, err)
 	}
 	if err != nil {
-		client.Logger.Errorf("cipd: failed to fetch %s - %s", pin, err)
+		fetchFailed(err)
 		return err
 	}
+
+	if cache == nil {
+		if err := client.storage.download(fetchInfo.fetchURL, output); err != nil {
+			fetchFailed(err)
+			return err
+		}
+	} else {
+		// With cache, we need to download the instance to two places.
+		// Download to the cache and then read from there.
+		err := cache.Put(pin, now, func(f *os.File) error {
+			err := client.storage.download(fetchInfo.fetchURL, f)
+			if err != nil {
+				fetchFailed(err)
+			}
+			return err
+		})
+		if err != nil {
+			return err
+		}
+
+		// Now read it back.
+		if err := cache.Get(pin, output, now); err != nil {
+			return err
+		}
+	}
+
 	client.Logger.Infof("cipd: successfully fetched %s", pin)
 	return nil
 }
@@ -949,6 +1033,7 @@ func (client *clientImpl) Close() {
 	client.lock.Lock()
 	defer client.lock.Unlock()
 	client.closeTagCache()
+	client.closeInstanceCache()
 	client.authClient = nil
 	client.anonClient = nil
 }