OLD | NEW |
(Empty) | |
| 1 // Copyright 2016 The LUCI Authors. All rights reserved. |
| 2 // Use of this source code is governed under the Apache License, Version 2.0 |
| 3 // that can be found in the LICENSE file. |
| 4 |
| 5 package dirwalk |
| 6 |
| 7 import ( |
| 8 "os" |
| 9 "path/filepath" |
| 10 "sync/atomic" |
| 11 |
| 12 "github.com/eapache/channels" |
| 13 ) |
| 14 |
| 15 type fileQueue struct { |
| 16 queued uint64 |
| 17 finished uint64 |
| 18 items channels.Channel |
| 19 waiton chan bool |
| 20 } |
| 21 |
| 22 func (q *fileQueue) add(s string) { |
| 23 atomic.AddUint64(&q.queued, 1) |
| 24 q.items.In() <- s |
| 25 } |
| 26 |
| 27 func (q *fileQueue) done() { |
| 28 atomic.AddUint64(&q.finished, 1) |
| 29 |
| 30 if q.queued == q.finished { |
| 31 q.items.Close() |
| 32 q.waiton <- true |
| 33 } |
| 34 } |
| 35 |
| 36 func (q *fileQueue) wait() { |
| 37 <-q.waiton |
| 38 } |
| 39 |
| 40 func examinePath(queue *fileQueue, callback WalkFunc) { |
| 41 for ipath := range queue.items.Out() { |
| 42 path := ipath.(string) |
| 43 |
| 44 fi, err := os.Stat(path) |
| 45 if err != nil { |
| 46 callback(path, -1, nil, err) |
| 47 return |
| 48 } |
| 49 |
| 50 if fi.IsDir() { |
| 51 d, err := os.Open(path) |
| 52 if err != nil { |
| 53 callback(path, -1, nil, err) |
| 54 } |
| 55 |
| 56 dircontents, err := d.Readdirnames(-1) |
| 57 if err != nil { |
| 58 callback(path, -1, nil, err) |
| 59 } |
| 60 for _, name := range dircontents { |
| 61 fname := filepath.Join(path, name) |
| 62 queue.add(fname) |
| 63 } |
| 64 } else { |
| 65 f, err := os.Open(path) |
| 66 if err != nil { |
| 67 callback(path, -1, nil, err) |
| 68 } else { |
| 69 callback(path, fi.Size(), f, nil) |
| 70 } |
| 71 } |
| 72 queue.done() |
| 73 } |
| 74 } |
| 75 |
| 76 // WalkParallel is a directory walking function which uses multiple threads to |
| 77 // walk a directory tree. |
| 78 // |
| 79 // On the majority of systems testing shows that this function is either |
| 80 // slower than (or at best comparable) the non-parallel version while consuming |
| 81 // many times the resources. |
| 82 // |
| 83 // Linux Kernel versions newer than >4.8 which disable locks in stat path can |
| 84 // make this version faster. |
| 85 // |
| 86 // Use the performance tests to determine the correct walker for your platform |
| 87 // and system! |
| 88 func WalkParallel(root string, callback WalkFunc) { |
| 89 queue := fileQueue{queued: 0, finished: 0, items: channels.NewInfiniteCh
annel(), waiton: make(chan bool)} |
| 90 |
| 91 for w := 0; w <= 10; w++ { |
| 92 go examinePath(&queue, callback) |
| 93 } |
| 94 |
| 95 queue.add(root) |
| 96 queue.wait() |
| 97 } |
OLD | NEW |