| OLD | NEW |
| 1 // Copyright 2015 The LUCI Authors. | 1 // Copyright 2015 The LUCI Authors. |
| 2 // | 2 // |
| 3 // Licensed under the Apache License, Version 2.0 (the "License"); | 3 // Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 // you may not use this file except in compliance with the License. | 4 // you may not use this file except in compliance with the License. |
| 5 // You may obtain a copy of the License at | 5 // You may obtain a copy of the License at |
| 6 // | 6 // |
| 7 // http://www.apache.org/licenses/LICENSE-2.0 | 7 // http://www.apache.org/licenses/LICENSE-2.0 |
| 8 // | 8 // |
| 9 // Unless required by applicable law or agreed to in writing, software | 9 // Unless required by applicable law or agreed to in writing, software |
| 10 // distributed under the License is distributed on an "AS IS" BASIS, | 10 // distributed under the License is distributed on an "AS IS" BASIS, |
| (...skipping 12 matching lines...) Expand all Loading... |
| 23 "os" | 23 "os" |
| 24 "path/filepath" | 24 "path/filepath" |
| 25 "strings" | 25 "strings" |
| 26 "time" | 26 "time" |
| 27 | 27 |
| 28 humanize "github.com/dustin/go-humanize" | 28 humanize "github.com/dustin/go-humanize" |
| 29 "github.com/golang/protobuf/proto" | 29 "github.com/golang/protobuf/proto" |
| 30 "github.com/maruel/subcommands" | 30 "github.com/maruel/subcommands" |
| 31 "golang.org/x/net/context" | 31 "golang.org/x/net/context" |
| 32 | 32 |
| 33 "github.com/luci/luci-go/client/internal/common" |
| 33 "github.com/luci/luci-go/client/isolate" | 34 "github.com/luci/luci-go/client/isolate" |
| 34 "github.com/luci/luci-go/common/auth" | 35 "github.com/luci/luci-go/common/auth" |
| 35 logpb "github.com/luci/luci-go/common/eventlog/proto" | 36 logpb "github.com/luci/luci-go/common/eventlog/proto" |
| 36 "github.com/luci/luci-go/common/isolated" | 37 "github.com/luci/luci-go/common/isolated" |
| 37 "github.com/luci/luci-go/common/isolatedclient" | 38 "github.com/luci/luci-go/common/isolatedclient" |
| 38 ) | 39 ) |
| 39 | 40 |
| 40 const ( | 41 const ( |
| 41 // archiveThreshold is the size (in bytes) used to determine whether to
add | 42 // archiveThreshold is the size (in bytes) used to determine whether to
add |
| 42 // files to a tar archive before uploading. Files smaller than this size
will | 43 // files to a tar archive before uploading. Files smaller than this size
will |
| (...skipping 50 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 93 totalSize int64 | 94 totalSize int64 |
| 94 } | 95 } |
| 95 | 96 |
| 96 func (ig *itemGroup) AddItem(item *Item) { | 97 func (ig *itemGroup) AddItem(item *Item) { |
| 97 ig.items = append(ig.items, item) | 98 ig.items = append(ig.items, item) |
| 98 ig.totalSize += item.Size | 99 ig.totalSize += item.Size |
| 99 } | 100 } |
| 100 | 101 |
| 101 // partitioningWalker contains the state necessary to partition isolate deps by
handling multiple os.WalkFunc invocations. | 102 // partitioningWalker contains the state necessary to partition isolate deps by
handling multiple os.WalkFunc invocations. |
| 102 type partitioningWalker struct { | 103 type partitioningWalker struct { |
| 103 » // rootDir must be initialized before walkFn is called. | 104 » // fsView must be initialized before walkFn is called. |
| 104 » rootDir string | 105 » fsView common.FilesystemView |
| 105 | 106 |
| 106 parts partitionedDeps | 107 parts partitionedDeps |
| 107 } | 108 } |
| 108 | 109 |
| 109 // partitionedDeps contains a list of items to be archived, partitioned into sym
links and files categorized by size. | 110 // partitionedDeps contains a list of items to be archived, partitioned into sym
links and files categorized by size. |
| 110 type partitionedDeps struct { | 111 type partitionedDeps struct { |
| 111 links itemGroup | 112 links itemGroup |
| 112 filesToArchive itemGroup | 113 filesToArchive itemGroup |
| 113 indivFiles itemGroup | 114 indivFiles itemGroup |
| 114 } | 115 } |
| 115 | 116 |
| 116 // walkFn implements filepath.WalkFunc, for use traversing a directory hierarchy
to be isolated. | 117 // walkFn implements filepath.WalkFunc, for use traversing a directory hierarchy
to be isolated. |
| 117 // It accumulates files in pw.parts, partitioned into symlinks and files categor
ized by size. | 118 // It accumulates files in pw.parts, partitioned into symlinks and files categor
ized by size. |
| 118 func (pw *partitioningWalker) walkFn(path string, info os.FileInfo, err error) e
rror { | 119 func (pw *partitioningWalker) walkFn(path string, info os.FileInfo, err error) e
rror { |
| 119 if err != nil { | 120 if err != nil { |
| 120 return err | 121 return err |
| 121 } | 122 } |
| 123 |
| 124 relPath, err := pw.fsView.RelativePath(path) |
| 125 if err != nil { |
| 126 return err |
| 127 } |
| 128 |
| 129 if relPath == "" { // empty string indicates skip. |
| 130 return common.WalkFuncSkipFile(info) |
| 131 } |
| 132 |
| 122 if info.IsDir() { | 133 if info.IsDir() { |
| 123 return nil | 134 return nil |
| 124 } | 135 } |
| 125 | 136 |
| 126 relPath, err := filepath.Rel(pw.rootDir, path) | |
| 127 if err != nil { | |
| 128 return err | |
| 129 } | |
| 130 | |
| 131 item := &Item{ | 137 item := &Item{ |
| 132 Path: path, | 138 Path: path, |
| 133 RelPath: relPath, | 139 RelPath: relPath, |
| 134 Mode: info.Mode(), | 140 Mode: info.Mode(), |
| 135 Size: info.Size(), | 141 Size: info.Size(), |
| 136 } | 142 } |
| 137 | 143 |
| 138 switch { | 144 switch { |
| 139 case item.Mode&os.ModeSymlink == os.ModeSymlink: | 145 case item.Mode&os.ModeSymlink == os.ModeSymlink: |
| 140 pw.parts.links.AddItem(item) | 146 pw.parts.links.AddItem(item) |
| 141 case item.Size < archiveThreshold: | 147 case item.Size < archiveThreshold: |
| 142 pw.parts.filesToArchive.AddItem(item) | 148 pw.parts.filesToArchive.AddItem(item) |
| 143 default: | 149 default: |
| 144 pw.parts.indivFiles.AddItem(item) | 150 pw.parts.indivFiles.AddItem(item) |
| 145 } | 151 } |
| 146 return nil | 152 return nil |
| 147 } | 153 } |
| 148 | 154 |
| 149 // partitionDeps walks each of the deps, partioning the results into symlinks an
d files categorized by size. | 155 // partitionDeps walks each of the deps, partioning the results into symlinks an
d files categorized by size. |
| 150 func partitionDeps(deps []string, rootDir string) (partitionedDeps, error) { | 156 func partitionDeps(deps []string, rootDir string) (partitionedDeps, error) { |
| 151 » walker := partitioningWalker{rootDir: rootDir} | 157 » // TODO(mcgreevy): initialize FilesystemView with blacklist. |
| 158 » fsView, err := common.NewFilesystemView(rootDir, nil) |
| 159 » if err != nil { |
| 160 » » return partitionedDeps{}, err |
| 161 » } |
| 162 |
| 163 » walker := partitioningWalker{fsView: fsView} |
| 152 for _, dep := range deps { | 164 for _, dep := range deps { |
| 153 // Try to walk dep. If dep is a file (or symlink), the inner fun
ction is called exactly once. | 165 // Try to walk dep. If dep is a file (or symlink), the inner fun
ction is called exactly once. |
| 154 if err := filepath.Walk(filepath.Clean(dep), walker.walkFn); err
!= nil { | 166 if err := filepath.Walk(filepath.Clean(dep), walker.walkFn); err
!= nil { |
| 155 return partitionedDeps{}, err | 167 return partitionedDeps{}, err |
| 156 } | 168 } |
| 157 } | 169 } |
| 158 return walker.parts, nil | 170 return walker.parts, nil |
| 159 } | 171 } |
| 160 | 172 |
| 161 // main contains the core logic for experimental archive. | 173 // main contains the core logic for experimental archive. |
| (...skipping 217 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 379 } | 391 } |
| 380 | 392 |
| 381 func hashFile(path string) (isolated.HexDigest, error) { | 393 func hashFile(path string) (isolated.HexDigest, error) { |
| 382 f, err := os.Open(path) | 394 f, err := os.Open(path) |
| 383 if err != nil { | 395 if err != nil { |
| 384 return "", err | 396 return "", err |
| 385 } | 397 } |
| 386 defer f.Close() | 398 defer f.Close() |
| 387 return isolated.Hash(f) | 399 return isolated.Hash(f) |
| 388 } | 400 } |
| OLD | NEW |