Index: go/gs/gs.go |
diff --git a/go/gs/gs.go b/go/gs/gs.go |
index 848c52ca6aaac8758abe8e5683a5f3a4f39b93d0..57f0faf7981280ea45f79aaadfa65cc4d4feb2ae 100644 |
--- a/go/gs/gs.go |
+++ b/go/gs/gs.go |
@@ -7,6 +7,8 @@ import ( |
"net/http" |
"regexp" |
"strings" |
+ "sync" |
+ "sync/atomic" |
"time" |
"github.com/skia-dev/glog" |
@@ -145,20 +147,39 @@ func AllFilesInDir(s *storage.Client, bucket, folder string, callback func(item |
return nil |
} |
-func DeleteAllFilesInDir(s *storage.Client, bucket, folder string) error { |
- var deleteError bool |
+// DeleteAllFilesInDir deletes all the files in a given folder. If processes is set to > 1, |
+// that many go routines will be spun up to delete the file simultaneously. |
+func DeleteAllFilesInDir(s *storage.Client, bucket, folder string, processes int) error { |
+ errCount := int32(0) |
+ var wg sync.WaitGroup |
+ toDelete := make(chan string, 1000) |
+ for i := 0; i < processes; i++ { |
+ go deleteHelper(s, bucket, &wg, toDelete, &errCount) |
+ } |
del := func(item *storage.ObjectAttrs) { |
- if err := s.Bucket(bucket).Object(item.Name).Delete(context.Background()); err != nil { |
- glog.Errorf("Problem deleting gs://%s/%s: %s", bucket, item.Name, err) |
- deleteError = true |
- } |
+ toDelete <- item.Name |
} |
if err := AllFilesInDir(s, bucket, folder, del); err != nil { |
return err |
} |
- if deleteError { |
+ close(toDelete) |
+ wg.Wait() |
+ if errCount > 0 { |
return fmt.Errorf("There were one or more problems when deleting files in folder %q", folder) |
} |
return nil |
} |
+ |
+// deleteHelper spins and waits for work to come in on the toDelete channel. When it does, it |
+// uses the storage client to delete the file from the given bucket. |
+func deleteHelper(s *storage.Client, bucket string, wg *sync.WaitGroup, toDelete <-chan string, errCount *int32) { |
+ wg.Add(1) |
+ defer wg.Done() |
+ for file := range toDelete { |
+ if err := s.Bucket(bucket).Object(file).Delete(context.Background()); err != nil { |
+ glog.Errorf("Problem deleting gs://%s/%s: %s", bucket, file, err) |
+ atomic.AddInt32(errCount, 1) |
+ } |
+ } |
+} |