cipd: instance cache

client/cipd/client.go

Index: client/cipd/client.go
diff --git a/client/cipd/client.go b/client/cipd/client.go
index d74a6908be9c31ab2d021cc28d3496c383a8d959..ba3f879d89df2dfaab11244f508c9e6d550f268e 100644
--- a/client/cipd/client.go
+++ b/client/cipd/client.go
@@ -337,9 +337,9 @@ type ClientOptions struct {
 	// UserAgent is put into User-Agent HTTP header with each request.
 	UserAgent string
 
-	// CacheDir is a directory for shared cache. If empty, tags are cached
-	// inside the site root. If both Root and CacheDir are empty, tag cache
-	// is disabled.
+	// CacheDir is a directory for shared cache. If empty, instances are not
+	// cached and tags are cached inside the site root. If both Root and
+	// CacheDir are empty, tag cache is disabled.
 	CacheDir string
 }
 
@@ -392,6 +392,10 @@ type clientImpl struct {
 	// tagCacheLock is used to synchronize access to the tag cache file.
 	tagCacheLock sync.Mutex
 
+	// instanceCache is a file-system based cache of instances.
+	instanceCache     *internal.InstanceCache
+	instanceCacheInit sync.Once
+
 	// authClient is a lazily created http.Client to use for authenticated
 	// requests. Thread safe, but lazily initialized under lock.
 	authClient *http.Client
@@ -489,6 +493,20 @@ func (client *clientImpl) withTagCache(f func(*internal.TagCache)) {
 	}
 }
 
+// getInstanceCache lazy-initializes instanceCache and returns it.
+// May return nil if instance cache is disabled.
+func (client *clientImpl) getInstanceCache() *internal.InstanceCache {
+	client.instanceCacheInit.Do(func() {
+		if client.CacheDir == "" {
+			return
+		}
+		path := filepath.Join(client.CacheDir, "instances")
+		fs := local.NewFileSystem(path, client.Logger)
+		client.instanceCache = internal.NewInstanceCache(fs, client.Logger)
+	})
+	return client.instanceCache
+}
+
 func (client *clientImpl) FetchACL(packagePath string) ([]PackageACL, error) {
 	return client.remote.fetchACL(packagePath)
 }
@@ -758,10 +776,59 @@ func (client *clientImpl) FetchInstanceRefs(pin common.Pin, refs []string) ([]Re
 }
 
 func (client *clientImpl) FetchInstance(pin common.Pin, output io.WriteSeeker) error {
-	err := common.ValidatePin(pin)

[Vadim Sh., 2016/04/13 21:49:02]

you lost this check

[nodir, 2016/04/13 23:23:51]

Done.

-	if err != nil {
+	cache := client.getInstanceCache()
+	if cache == nil {
+		return client.fetchInstanceNoCache(pin, output)
+	}
+	return client.fetchInstanceWithCache(pin, cache, output)
+}
+
+func (client *clientImpl) fetchInstanceNoCache(pin common.Pin, output io.WriteSeeker) error {
+	if err := client.remoteFetchInstance(pin, output); err != nil {
 		return err
 	}
+	client.Logger.Infof("cipd: successfully fetched %s", pin)
+	return nil
+}
+
+func (client *clientImpl) fetchInstanceWithCache(pin common.Pin, cache *internal.InstanceCache, output io.WriteSeeker) error {
+	// Try to get the instance from cache.
+	now := client.clock.now()
+	switch err := cache.Get(pin, output, now); {
+	case os.IsNotExist(err):
+		// output is not corrupted.
+
+	case err != nil:
+		client.Logger.Warningf("cipd: could not get %s from cache - %s", pin, err)
+		// Output may be corrupted. Rewinding back.

[Vadim Sh., 2016/04/13 21:49:02]

you still need to truncate the file :(

[nodir, 2016/04/13 23:23:52]

it is not required. I've improved the comment

+		if _, err := output.Seek(0, 0); err != nil {
+			return err
+		}
+
+	default:
+		client.Logger.Debugf("cipd: instance cache hit for %s", pin)
+		return nil
+	}
+
+	return cache.Put(pin, now, func(f *os.File) error {
+		// Fetch to the file.
+		if err := client.remoteFetchInstance(pin, f); err != nil {
+			return err
+		}
+
+		// Copy fetched content to output.
+		if _, err := f.Seek(0, 0); err != nil {
+			return err
+		}
+		if _, err := io.Copy(output, f); err != nil {

[Vadim Sh., 2016/04/13 21:49:02]

eh.. probably good enough for now, but it is a bit wasteful

in most cases 'output' here is actually some temp file that is thrown away after
it is unzipped

it would be good to be able to unzip directly from a file in cache (to avoid
copying the file)

[nodir, 2016/04/13 23:23:52]

Acknowledged.

+			return err
+		}
+		client.Logger.Infof("cipd: successfully fetched %s", pin)
+		return nil
+	})
+}
+
+func (client *clientImpl) remoteFetchInstance(pin common.Pin, output io.WriteSeeker) error {
 	client.Logger.Infof("cipd: resolving fetch URL for %s", pin)
 	fetchInfo, err := client.remote.fetchInstance(pin)
 	if err == nil {
@@ -769,10 +836,8 @@ func (client *clientImpl) FetchInstance(pin common.Pin, output io.WriteSeeker) e
 	}
 	if err != nil {
 		client.Logger.Errorf("cipd: failed to fetch %s - %s", pin, err)
-		return err
 	}
-	client.Logger.Infof("cipd: successfully fetched %s", pin)
-	return nil
+	return err
 }
 
 func (client *clientImpl) FetchAndDeployInstance(pin common.Pin) error {