cipd: instance cache

client/cipd/client.go

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 // Package cipd implements client side of Chrome Infra Package Deployer.
 //
 // Binary package file format (in free form representation):
 //   <binary package> := <zipped data>
 //   <zipped data> := DeterministicZip(<all input files> + <manifest json>)
 //   <manifest json> := File{
@@ skipping 319 matching lines @@
         // RPC requests.
         AuthenticatedClientFactory HTTPClientFactory
 
         // AnonymousClientFactory lazily creates http.Client to use for making
         // requests to storage.
         AnonymousClientFactory HTTPClientFactory
 
         // UserAgent is put into User-Agent HTTP header with each request.
         UserAgent string
 
-        // CacheDir is a directory for shared cache. If empty, tags are cached
-        // inside the site root. If both Root and CacheDir are empty, tag cache
-        // is disabled.
+        // CacheDir is a directory for shared cache. If empty, instances are not
+        // cached and tags are cached inside the site root. If both Root and
+        // CacheDir are empty, tag cache is disabled.
         CacheDir string
 }
 
 // NewClient initializes CIPD client object.
 func NewClient(opts ClientOptions) Client {
         if opts.ServiceURL == "" {
                 opts.ServiceURL = ServiceURL
         }
         if opts.Logger == nil {
                 opts.Logger = logging.Null()
@@ skipping 32 matching lines @@
         // storage knows how to upload and download raw binaries using signed URLs.
         // Thread safe.
         storage storage
 
         // deployer knows how to install packages to local file system. Thread safe.
         deployer local.Deployer
 
         // tagCacheLock is used to synchronize access to the tag cache file.
         tagCacheLock sync.Mutex
 
+        // instanceCache is a file-system based cache of instances.
+        instanceCache     *internal.InstanceCache
+        instanceCacheInit sync.Once
+
         // authClient is a lazily created http.Client to use for authenticated
         // requests. Thread safe, but lazily initialized under lock.
         authClient *http.Client
 
         // anonClient is a lazily created http.Client to use for anonymous requests.
         // Thread safe, but lazily initialized under lock.
         anonClient *http.Client
 }
 
 // doAuthenticatedHTTPRequest is used by remote implementation to make HTTP calls.
@@ skipping 77 matching lines @@
                 if err != nil {
                         client.Logger.Warningf("cipd: failed to update tag cache - %s", err)
                 }
         }
 
         if loadSaveTime > time.Second {
                 client.Logger.Warningf("cipd: loading and saving tag cache with %d entries took %s", cache.Len(), loadSaveTime)
         }
 }
 
+// getInstanceCache lazy-initializes instanceCache and returns it.
+// May return nil if instance cache is disabled.
+func (client *clientImpl) getInstanceCache() *internal.InstanceCache {
+        client.instanceCacheInit.Do(func() {
+                if client.CacheDir == "" {
+                        return
+                }
+                path := filepath.Join(client.CacheDir, "instances")
+                fs := local.NewFileSystem(path, client.Logger)
+                client.instanceCache = internal.NewInstanceCache(fs, client.Logger)
+        })
+        return client.instanceCache
+}
+
 func (client *clientImpl) FetchACL(packagePath string) ([]PackageACL, error) {
         return client.remote.fetchACL(packagePath)
 }
 
 func (client *clientImpl) ModifyACL(packagePath string, changes []PackageACLChange) error {
         return client.remote.modifyACL(packagePath, changes)
 }
 
 func (client *clientImpl) ListPackages(path string, recursive bool) ([]string, error) {
         pkgs, dirs, err := client.remote.listPackages(path, recursive)
@@ skipping 249 matching lines @@
 
 func (client *clientImpl) FetchInstanceRefs(pin common.Pin, refs []string) ([]RefInfo, error) {
         err := common.ValidatePin(pin)
         if err != nil {
                 return nil, err
         }
         return client.remote.fetchRefs(pin, refs)
 }
 
 func (client *clientImpl) FetchInstance(pin common.Pin, output io.WriteSeeker) error {
-        err := common.ValidatePin(pin)

[Vadim Sh., 2016/04/13 21:49:02]

you lost this check

[nodir, 2016/04/13 23:23:51]

Done.

-        if err != nil {
+        cache := client.getInstanceCache()
+        if cache == nil {
+                return client.fetchInstanceNoCache(pin, output)
+        }
+        return client.fetchInstanceWithCache(pin, cache, output)
+}
+
+func (client *clientImpl) fetchInstanceNoCache(pin common.Pin, output io.WriteSeeker) error {
+        if err := client.remoteFetchInstance(pin, output); err != nil {
                 return err
         }
+        client.Logger.Infof("cipd: successfully fetched %s", pin)
+        return nil
+}
+
+func (client *clientImpl) fetchInstanceWithCache(pin common.Pin, cache *internal.InstanceCache, output io.WriteSeeker) error {
+        // Try to get the instance from cache.
+        now := client.clock.now()
+        switch err := cache.Get(pin, output, now); {
+        case os.IsNotExist(err):
+                // output is not corrupted.
+
+        case err != nil:
+                client.Logger.Warningf("cipd: could not get %s from cache - %s", pin, err)
+                // Output may be corrupted. Rewinding back.

[Vadim Sh., 2016/04/13 21:49:02]

you still need to truncate the file :(

[nodir, 2016/04/13 23:23:52]

it is not required. I've improved the comment

+                if _, err := output.Seek(0, 0); err != nil {
+                        return err
+                }
+
+        default:
+                client.Logger.Debugf("cipd: instance cache hit for %s", pin)
+                return nil
+        }
+
+        return cache.Put(pin, now, func(f *os.File) error {
+                // Fetch to the file.
+                if err := client.remoteFetchInstance(pin, f); err != nil {
+                        return err
+                }
+
+                // Copy fetched content to output.
+                if _, err := f.Seek(0, 0); err != nil {
+                        return err
+                }
+                if _, err := io.Copy(output, f); err != nil {

[Vadim Sh., 2016/04/13 21:49:02]

eh.. probably good enough for now, but it is a bit wasteful

in most cases 'output' here is actually some temp file that is thrown away after
it is unzipped

it would be good to be able to unzip directly from a file in cache (to avoid
copying the file)

[nodir, 2016/04/13 23:23:52]

Acknowledged.

+                        return err
+                }
+                client.Logger.Infof("cipd: successfully fetched %s", pin)
+                return nil
+        })
+}
+
+func (client *clientImpl) remoteFetchInstance(pin common.Pin, output io.WriteSeeker) error {
         client.Logger.Infof("cipd: resolving fetch URL for %s", pin)
         fetchInfo, err := client.remote.fetchInstance(pin)
         if err == nil {
                 err = client.storage.download(fetchInfo.fetchURL, output)
         }
         if err != nil {
                 client.Logger.Errorf("cipd: failed to fetch %s - %s", pin, err)
-                return err
         }
-        client.Logger.Infof("cipd: successfully fetched %s", pin)
-        return nil
+        return err
 }
 
 func (client *clientImpl) FetchAndDeployInstance(pin common.Pin) error {
         err := common.ValidatePin(pin)
         if err != nil {
                 return err
         }
 
         // Use temp file for storing package file. Delete it when done.
         var instance local.PackageInstance
@@ skipping 254 matching lines @@
 }
 
 // buildInstanceIDMap builds mapping {package name -> instance ID}.
 func buildInstanceIDMap(pins []common.Pin) map[string]string {
         out := map[string]string{}
         for _, p := range pins {
                 out[p.PackageName] = p.InstanceID
         }
         return out
 }