cipd: instance cache

client/cipd/client.go

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 // Package cipd implements client side of Chrome Infra Package Deployer.
 //
 // Binary package file format (in free form representation):
 //   <binary package> := <zipped data>
 //   <zipped data> := DeterministicZip(<all input files> + <manifest json>)
 //   <manifest json> := File{
@@ skipping 254 matching lines @@
         // instance sorted by tag key and creation timestamp (newest first). If 'tags'
         // is empty, fetches all attached tags, otherwise only ones specified.
         FetchInstanceTags(pin common.Pin, tags []string) ([]TagInfo, error)
 
         // FetchInstanceRefs returns information about refs pointing to the package
         // instance sorted by modification timestamp (newest first). If 'ref' is
         // empty, fetches all refs, otherwise only ones specified.
         FetchInstanceRefs(pin common.Pin, refs []string) ([]RefInfo, error)
 
         // FetchInstance downloads package instance file from the repository.
-        FetchInstance(pin common.Pin, output io.WriteSeeker) error
+        FetchInstance(pin common.Pin, output io.ReadWriteSeeker) error
 
         // FetchAndDeployInstance fetches the package instance and deploys it into
         // the site root. It doesn't check whether the instance is already deployed.
         FetchAndDeployInstance(pin common.Pin) error
 
         // ListPackages returns a list of strings of package names.
         ListPackages(path string, recursive bool) ([]string, error)
 
         // SearchInstances finds all instances with given tag and optionally name and
         // returns their concrete Pins.
@@ skipping 43 matching lines @@
         // RPC requests.
         AuthenticatedClientFactory HTTPClientFactory
 
         // AnonymousClientFactory lazily creates http.Client to use for making
         // requests to storage.
         AnonymousClientFactory HTTPClientFactory
 
         // UserAgent is put into User-Agent HTTP header with each request.
         UserAgent string
 
-        // CacheDir is a directory for shared cache. If empty, tags are cached
-        // inside the site root. If both Root and CacheDir are empty, tag cache
-        // is disabled.
+        // CacheDir is a directory for shared cache. If empty, instances are not
+        // cached and tags are cached inside the site root. If both Root and
+        // CacheDir are empty, tag cache is disabled.
         CacheDir string
 }
 
 // NewClient initializes CIPD client object.
 func NewClient(opts ClientOptions) Client {
         if opts.ServiceURL == "" {
                 opts.ServiceURL = ServiceURL
         }
         if opts.Logger == nil {
                 opts.Logger = logging.Null()
@@ skipping 34 matching lines @@
         storage storage
 
         // deployer knows how to install packages to local file system. Thread safe.
         deployer local.Deployer
 
         // tagCache is used to cache (pkgname, tag) -> instanceID mapping.
         // Thread safe, but lazily initialized under lock.
         tagCache     *internal.TagCache
         tagCacheInit sync.Once
 
+        // instanceCache is a file-system based cache of instances.
+        instanceCache     *internal.InstanceCache
+        instanceCacheInit sync.Once
+
         // authClient is a lazily created http.Client to use for authenticated
         // requests. Thread safe, but lazily initialized under lock.
         authClient *http.Client
 
         // anonClient is a lazily created http.Client to use for anonymous requests.
         // Thread safe, but lazily initialized under lock.
         anonClient *http.Client
 }
 
 // doAuthenticatedHTTPRequest is used by remote implementation to make HTTP calls.
@@ skipping 73 matching lines @@
         out, err := client.tagCache.Save()
         if err == nil {
                 err = fs.EnsureFile(path, out, 0666)
         }
         if err != nil {
                 client.Logger.Warningf("cipd: failed to update tag cache - %s", err)
         }
         client.tagCache = nil
 }
 
+// instanceCachePath returns path to the instance cache directory or "" if
+// instance cache is disabled.
+func (client *clientImpl) instanceCachePath() string {
+        if client.CacheDir == "" {
+                return ""
+        }
+        return filepath.Join(client.CacheDir, "instances")
+}
+
+// getInstanceCache lazy-initializes instanceCache and returns it.
+func (client *clientImpl) getInstanceCache() *internal.InstanceCache {
+        client.instanceCacheInit.Do(func() {
+                if path := client.instanceCachePath(); path != "" {
+                        cachePath := local.NewFileSystem(path, client.Logger)
+                        client.instanceCache = internal.LoadInstanceCache(cachePath, client.Logger, client.clock.now())
+                }
+        })
+        return client.instanceCache
+}
+
+// closeInstanceCache dumps any changes made to instance cache to disk, if necessary.
+func (client *clientImpl) closeInstanceCache() {
+        if client.instanceCache == nil || !client.instanceCache.Dirty() {
+                return
+        }
+
+        if err := client.instanceCache.Save(); err != nil {
+                client.Logger.Warningf("cipd: failed to save instance cache - %s", err)
+        }
+        client.instanceCache = nil
+}
+
 func (client *clientImpl) FetchACL(packagePath string) ([]PackageACL, error) {
         return client.remote.fetchACL(packagePath)
 }
 
 func (client *clientImpl) ModifyACL(packagePath string, changes []PackageACLChange) error {
         return client.remote.modifyACL(packagePath, changes)
 }
 
 func (client *clientImpl) ListPackages(path string, recursive bool) ([]string, error) {
         pkgs, dirs, err := client.remote.listPackages(path, recursive)
@@ skipping 242 matching lines @@
 }
 
 func (client *clientImpl) FetchInstanceRefs(pin common.Pin, refs []string) ([]RefInfo, error) {
         err := common.ValidatePin(pin)
         if err != nil {
                 return nil, err
         }
         return client.remote.fetchRefs(pin, refs)
 }
 
-func (client *clientImpl) FetchInstance(pin common.Pin, output io.WriteSeeker) error {
+func (client *clientImpl) FetchInstance(pin common.Pin, output io.ReadWriteSeeker) error {
         err := common.ValidatePin(pin)
         if err != nil {
                 return err
         }
+
+        now := client.clock.now()
+
+        cache := client.getInstanceCache()
+        if cache != nil {
+                switch err := cache.Get(pin, output, now); {
+                case os.IsNotExist(err):
+                        // output is not corrupted.
+
+                case err != nil:
+                        client.Logger.Warningf("cipd: could not get %s from cache - %s", pin, err)
+                        // Output may be corrupted. Rewinding back.
+                        if _, err := output.Seek(0, 0); err != nil {

[Vadim Sh., 2016/04/11 16:52:02]

well, it may not be enough. We also need to truncate the file to zero length.
Otherwise we may end up with "<valid data>|<garbage left from reading cached
file>".

[nodir, 2016/04/11 22:16:16]

Done.

+                                return err
+                        }
+
+                default:
+                        client.Logger.Infof("cipd: cache hit for %s", pin)
+                        return nil
+                }
+        }
+
         client.Logger.Infof("cipd: resolving fetch URL for %s", pin)
         fetchInfo, err := client.remote.fetchInstance(pin)
-        if err == nil {
-                err = client.storage.download(fetchInfo.fetchURL, output)
+        if err != nil {
+                goto fetchFail
         }
-        if err != nil {
-                client.Logger.Errorf("cipd: failed to fetch %s - %s", pin, err)
-                return err
+
+        if err = client.storage.download(fetchInfo.fetchURL, output); err != nil {
+                goto fetchFail
         }
+
         client.Logger.Infof("cipd: successfully fetched %s", pin)
+
+        if cache != nil {
+                if _, err := output.Seek(0, 0); err != nil {

[Vadim Sh., 2016/04/11 16:52:02]

It may be simpler to always download into cache, and then do io.Copy from the
cache file into output. That way you don't need to change FetchInstance
signature (and no need to truncate files).

[nodir, 2016/04/11 22:16:16]

Done.

+                        client.Logger.Warningf("cipd: could not seek output to the beginning for caching - %s", err)
+                } else if err := cache.Put(pin, output, now); err != nil {
+                        client.Logger.Warningf("cipd: could not cache %s - %s", pin, err)
+                }
+        }
+
         return nil
+
+fetchFail:
+        client.Logger.Errorf("cipd: failed to fetch %s - %s", pin, err)
+        return err
 }
 
 func (client *clientImpl) FetchAndDeployInstance(pin common.Pin) error {
         err := common.ValidatePin(pin)
         if err != nil {
                 return err
         }
 
         // Use temp file for storing package file. Delete it when done.
         var instance local.PackageInstance
@@ skipping 164 matching lines @@
                 client.Logger.Infof("All changes applied.")
                 return actions, nil
         }
         return actions, ErrEnsurePackagesFailed
 }
 
 func (client *clientImpl) Close() {
         client.lock.Lock()
         defer client.lock.Unlock()
         client.closeTagCache()
+        client.closeInstanceCache()
         client.authClient = nil
         client.anonClient = nil
 }
 
 ////////////////////////////////////////////////////////////////////////////////
 // Private structs and interfaces.
 
 type clock interface {
         now() time.Time
         sleep(time.Duration)
@@ skipping 71 matching lines @@
 }
 
 // buildInstanceIDMap builds mapping {package name -> instance ID}.
 func buildInstanceIDMap(pins []common.Pin) map[string]string {
         out := map[string]string{}
         for _, p := range pins {
                 out[p.PackageName] = p.InstanceID
         }
         return out
 }