| OLD | NEW |
| 1 // Copyright 2015 The LUCI Authors. All rights reserved. | 1 // Copyright 2015 The LUCI Authors. All rights reserved. |
| 2 // Use of this source code is governed under the Apache License, Version 2.0 | 2 // Use of this source code is governed under the Apache License, Version 2.0 |
| 3 // that can be found in the LICENSE file. | 3 // that can be found in the LICENSE file. |
| 4 | 4 |
| 5 package gs | 5 package gs |
| 6 | 6 |
| 7 import ( | 7 import ( |
| 8 "fmt" | 8 "fmt" |
| 9 "io" | 9 "io" |
| 10 "net/http" | 10 "net/http" |
| 11 "time" | 11 "time" |
| 12 | 12 |
| 13 "github.com/luci/luci-go/common/errors" | 13 "github.com/luci/luci-go/common/errors" |
| 14 log "github.com/luci/luci-go/common/logging" | 14 log "github.com/luci/luci-go/common/logging" |
| 15 "github.com/luci/luci-go/common/retry" | 15 "github.com/luci/luci-go/common/retry" |
| 16 "github.com/luci/luci-go/common/retry/transient" |
| 16 | 17 |
| 17 gs "cloud.google.com/go/storage" | 18 gs "cloud.google.com/go/storage" |
| 18 "golang.org/x/net/context" | 19 "golang.org/x/net/context" |
| 19 "google.golang.org/api/googleapi" | 20 "google.golang.org/api/googleapi" |
| 20 "google.golang.org/api/option" | 21 "google.golang.org/api/option" |
| 21 ) | 22 ) |
| 22 | 23 |
| 23 var ( | 24 var ( |
| 24 // ReadWriteScopes is the set of scopes needed for read/write Google Sto
rage | 25 // ReadWriteScopes is the set of scopes needed for read/write Google Sto
rage |
| 25 // access. | 26 // access. |
| (...skipping 99 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 125 if err != nil { | 126 if err != nil { |
| 126 return fmt.Errorf("invalid source path: %s", err) | 127 return fmt.Errorf("invalid source path: %s", err) |
| 127 } | 128 } |
| 128 | 129 |
| 129 dstObj, err := c.handleForPath(dst) | 130 dstObj, err := c.handleForPath(dst) |
| 130 if err != nil { | 131 if err != nil { |
| 131 return fmt.Errorf("invalid destination path: %s", err) | 132 return fmt.Errorf("invalid destination path: %s", err) |
| 132 } | 133 } |
| 133 | 134 |
| 134 // First stage: CopyTo | 135 // First stage: CopyTo |
| 135 » err = retry.Retry(c, retry.TransientOnly(retry.Default), func() error { | 136 » err = retry.Retry(c, transient.Only(retry.Default), func() error { |
| 136 if _, err := dstObj.CopierFrom(srcObj).Run(c); err != nil { | 137 if _, err := dstObj.CopierFrom(srcObj).Run(c); err != nil { |
| 137 // The storage library doesn't return gs.ErrObjectNotExi
st when Delete | 138 // The storage library doesn't return gs.ErrObjectNotExi
st when Delete |
| 138 // returns a 404. Catch that explicitly. | 139 // returns a 404. Catch that explicitly. |
| 139 if isNotFoundError(err) { | 140 if isNotFoundError(err) { |
| 140 return err | 141 return err |
| 141 } | 142 } |
| 142 | 143 |
| 143 // Assume all unexpected errors are transient. | 144 // Assume all unexpected errors are transient. |
| 144 » » » return errors.WrapTransient(err) | 145 » » » return transient.Tag.Apply(err) |
| 145 } | 146 } |
| 146 return nil | 147 return nil |
| 147 }, func(err error, d time.Duration) { | 148 }, func(err error, d time.Duration) { |
| 148 log.Fields{ | 149 log.Fields{ |
| 149 log.ErrorKey: err, | 150 log.ErrorKey: err, |
| 150 "delay": d, | 151 "delay": d, |
| 151 "src": src, | 152 "src": src, |
| 152 "dst": dst, | 153 "dst": dst, |
| 153 }.Warningf(c, "Transient error copying GS file. Retrying...") | 154 }.Warningf(c, "Transient error copying GS file. Retrying...") |
| 154 }) | 155 }) |
| (...skipping 14 matching lines...) Expand all Loading... |
| 169 func (c *prodClient) Delete(p Path) error { | 170 func (c *prodClient) Delete(p Path) error { |
| 170 dstObj, err := c.handleForPath(p) | 171 dstObj, err := c.handleForPath(p) |
| 171 if err != nil { | 172 if err != nil { |
| 172 return fmt.Errorf("invalid path: %s", err) | 173 return fmt.Errorf("invalid path: %s", err) |
| 173 } | 174 } |
| 174 | 175 |
| 175 return c.deleteObject(dstObj) | 176 return c.deleteObject(dstObj) |
| 176 } | 177 } |
| 177 | 178 |
| 178 func (c *prodClient) deleteObject(o *gs.ObjectHandle) error { | 179 func (c *prodClient) deleteObject(o *gs.ObjectHandle) error { |
| 179 » return retry.Retry(c, retry.TransientOnly(retry.Default), func() error { | 180 » return retry.Retry(c, transient.Only(retry.Default), func() error { |
| 180 if err := o.Delete(c); err != nil { | 181 if err := o.Delete(c); err != nil { |
| 181 // The storage library doesn't return gs.ErrObjectNotExi
st when Delete | 182 // The storage library doesn't return gs.ErrObjectNotExi
st when Delete |
| 182 // returns a 404. Catch that explicitly. | 183 // returns a 404. Catch that explicitly. |
| 183 if isNotFoundError(err) { | 184 if isNotFoundError(err) { |
| 184 // If the file wasn't found, then the delete "su
cceeded". | 185 // If the file wasn't found, then the delete "su
cceeded". |
| 185 return nil | 186 return nil |
| 186 } | 187 } |
| 187 | 188 |
| 188 // Assume all unexpected errors are transient. | 189 // Assume all unexpected errors are transient. |
| 189 » » » return errors.WrapTransient(err) | 190 » » » return transient.Tag.Apply(err) |
| 190 } | 191 } |
| 191 return nil | 192 return nil |
| 192 }, func(err error, d time.Duration) { | 193 }, func(err error, d time.Duration) { |
| 193 log.Fields{ | 194 log.Fields{ |
| 194 log.ErrorKey: err, | 195 log.ErrorKey: err, |
| 195 "delay": d, | 196 "delay": d, |
| 196 }.Warningf(c, "Transient error deleting file. Retrying...") | 197 }.Warningf(c, "Transient error deleting file. Retrying...") |
| 197 }) | 198 }) |
| 198 } | 199 } |
| 199 | 200 |
| (...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 231 // The storage library doesn't return gs.ErrObjectNotExist when Delete | 232 // The storage library doesn't return gs.ErrObjectNotExist when Delete |
| 232 // returns a 404. Catch that explicitly. | 233 // returns a 404. Catch that explicitly. |
| 233 if t, ok := err.(*googleapi.Error); ok { | 234 if t, ok := err.(*googleapi.Error); ok { |
| 234 switch t.Code { | 235 switch t.Code { |
| 235 case http.StatusNotFound: | 236 case http.StatusNotFound: |
| 236 return true | 237 return true |
| 237 } | 238 } |
| 238 } | 239 } |
| 239 return false | 240 return false |
| 240 } | 241 } |
| OLD | NEW |