Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(433)

Side by Side Diff: client/cmd/isolate/exp_archive.go

Issue 2981223003: Reuse blacklisting code in exp_archive. (Closed)
Patch Set: address review comments Created 3 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « client/archiver/directory.go ('k') | client/cmd/isolate/exp_archive_test.go » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 // Copyright 2015 The LUCI Authors. 1 // Copyright 2015 The LUCI Authors.
2 // 2 //
3 // Licensed under the Apache License, Version 2.0 (the "License"); 3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License. 4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at 5 // You may obtain a copy of the License at
6 // 6 //
7 // http://www.apache.org/licenses/LICENSE-2.0 7 // http://www.apache.org/licenses/LICENSE-2.0
8 // 8 //
9 // Unless required by applicable law or agreed to in writing, software 9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS, 10 // distributed under the License is distributed on an "AS IS" BASIS,
(...skipping 12 matching lines...) Expand all
23 "os" 23 "os"
24 "path/filepath" 24 "path/filepath"
25 "strings" 25 "strings"
26 "time" 26 "time"
27 27
28 humanize "github.com/dustin/go-humanize" 28 humanize "github.com/dustin/go-humanize"
29 "github.com/golang/protobuf/proto" 29 "github.com/golang/protobuf/proto"
30 "github.com/maruel/subcommands" 30 "github.com/maruel/subcommands"
31 "golang.org/x/net/context" 31 "golang.org/x/net/context"
32 32
33 "github.com/luci/luci-go/client/internal/common"
33 "github.com/luci/luci-go/client/isolate" 34 "github.com/luci/luci-go/client/isolate"
34 "github.com/luci/luci-go/common/auth" 35 "github.com/luci/luci-go/common/auth"
35 logpb "github.com/luci/luci-go/common/eventlog/proto" 36 logpb "github.com/luci/luci-go/common/eventlog/proto"
36 "github.com/luci/luci-go/common/isolated" 37 "github.com/luci/luci-go/common/isolated"
37 "github.com/luci/luci-go/common/isolatedclient" 38 "github.com/luci/luci-go/common/isolatedclient"
38 ) 39 )
39 40
40 const ( 41 const (
41 // archiveThreshold is the size (in bytes) used to determine whether to add 42 // archiveThreshold is the size (in bytes) used to determine whether to add
42 // files to a tar archive before uploading. Files smaller than this size will 43 // files to a tar archive before uploading. Files smaller than this size will
(...skipping 50 matching lines...) Expand 10 before | Expand all | Expand 10 after
93 totalSize int64 94 totalSize int64
94 } 95 }
95 96
96 func (ig *itemGroup) AddItem(item *Item) { 97 func (ig *itemGroup) AddItem(item *Item) {
97 ig.items = append(ig.items, item) 98 ig.items = append(ig.items, item)
98 ig.totalSize += item.Size 99 ig.totalSize += item.Size
99 } 100 }
100 101
101 // partitioningWalker contains the state necessary to partition isolate deps by handling multiple os.WalkFunc invocations. 102 // partitioningWalker contains the state necessary to partition isolate deps by handling multiple os.WalkFunc invocations.
102 type partitioningWalker struct { 103 type partitioningWalker struct {
103 » // rootDir must be initialized before walkFn is called. 104 » // fsView must be initialized before walkFn is called.
104 » rootDir string 105 » fsView common.FilesystemView
105 106
106 parts partitionedDeps 107 parts partitionedDeps
107 } 108 }
108 109
109 // partitionedDeps contains a list of items to be archived, partitioned into sym links and files categorized by size. 110 // partitionedDeps contains a list of items to be archived, partitioned into sym links and files categorized by size.
110 type partitionedDeps struct { 111 type partitionedDeps struct {
111 links itemGroup 112 links itemGroup
112 filesToArchive itemGroup 113 filesToArchive itemGroup
113 indivFiles itemGroup 114 indivFiles itemGroup
114 } 115 }
115 116
116 // walkFn implements filepath.WalkFunc, for use traversing a directory hierarchy to be isolated. 117 // walkFn implements filepath.WalkFunc, for use traversing a directory hierarchy to be isolated.
117 // It accumulates files in pw.parts, partitioned into symlinks and files categor ized by size. 118 // It accumulates files in pw.parts, partitioned into symlinks and files categor ized by size.
118 func (pw *partitioningWalker) walkFn(path string, info os.FileInfo, err error) e rror { 119 func (pw *partitioningWalker) walkFn(path string, info os.FileInfo, err error) e rror {
119 if err != nil { 120 if err != nil {
120 return err 121 return err
121 } 122 }
123
124 relPath, err := pw.fsView.RelativePath(path)
125 if err != nil {
126 return err
127 }
128
129 if relPath == "" { // empty string indicates skip.
130 return common.WalkFuncSkipFile(info)
131 }
132
122 if info.IsDir() { 133 if info.IsDir() {
123 return nil 134 return nil
124 } 135 }
125 136
126 relPath, err := filepath.Rel(pw.rootDir, path)
127 if err != nil {
128 return err
129 }
130
131 item := &Item{ 137 item := &Item{
132 Path: path, 138 Path: path,
133 RelPath: relPath, 139 RelPath: relPath,
134 Mode: info.Mode(), 140 Mode: info.Mode(),
135 Size: info.Size(), 141 Size: info.Size(),
136 } 142 }
137 143
138 switch { 144 switch {
139 case item.Mode&os.ModeSymlink == os.ModeSymlink: 145 case item.Mode&os.ModeSymlink == os.ModeSymlink:
140 pw.parts.links.AddItem(item) 146 pw.parts.links.AddItem(item)
141 case item.Size < archiveThreshold: 147 case item.Size < archiveThreshold:
142 pw.parts.filesToArchive.AddItem(item) 148 pw.parts.filesToArchive.AddItem(item)
143 default: 149 default:
144 pw.parts.indivFiles.AddItem(item) 150 pw.parts.indivFiles.AddItem(item)
145 } 151 }
146 return nil 152 return nil
147 } 153 }
148 154
149 // partitionDeps walks each of the deps, partioning the results into symlinks an d files categorized by size. 155 // partitionDeps walks each of the deps, partioning the results into symlinks an d files categorized by size.
150 func partitionDeps(deps []string, rootDir string) (partitionedDeps, error) { 156 func partitionDeps(deps []string, rootDir string) (partitionedDeps, error) {
151 » walker := partitioningWalker{rootDir: rootDir} 157 » // TODO(mcgreevy): initialize FilesystemView with blacklist.
158 » fsView, err := common.NewFilesystemView(rootDir, nil)
159 » if err != nil {
160 » » return partitionedDeps{}, err
161 » }
162
163 » walker := partitioningWalker{fsView: fsView}
152 for _, dep := range deps { 164 for _, dep := range deps {
153 // Try to walk dep. If dep is a file (or symlink), the inner fun ction is called exactly once. 165 // Try to walk dep. If dep is a file (or symlink), the inner fun ction is called exactly once.
154 if err := filepath.Walk(filepath.Clean(dep), walker.walkFn); err != nil { 166 if err := filepath.Walk(filepath.Clean(dep), walker.walkFn); err != nil {
155 return partitionedDeps{}, err 167 return partitionedDeps{}, err
156 } 168 }
157 } 169 }
158 return walker.parts, nil 170 return walker.parts, nil
159 } 171 }
160 172
161 // main contains the core logic for experimental archive. 173 // main contains the core logic for experimental archive.
(...skipping 217 matching lines...) Expand 10 before | Expand all | Expand 10 after
379 } 391 }
380 392
381 func hashFile(path string) (isolated.HexDigest, error) { 393 func hashFile(path string) (isolated.HexDigest, error) {
382 f, err := os.Open(path) 394 f, err := os.Open(path)
383 if err != nil { 395 if err != nil {
384 return "", err 396 return "", err
385 } 397 }
386 defer f.Close() 398 defer f.Close()
387 return isolated.Hash(f) 399 return isolated.Hash(f)
388 } 400 }
OLDNEW
« no previous file with comments | « client/archiver/directory.go ('k') | client/cmd/isolate/exp_archive_test.go » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698