OLD | NEW |
---|---|
(Empty) | |
1 // Copyright 2016 The LUCI Authors. All rights reserved. | |
2 // Use of this source code is governed under the Apache License, Version 2.0 | |
3 // that can be found in the LICENSE file. | |
4 | |
5 package main | |
6 | |
7 // Tools for generating test directories. | |
8 | |
9 import ( | |
10 "fmt" | |
11 "log" | |
12 "math/rand" | |
13 "os" | |
14 "path" | |
15 | |
16 "github.com/dustin/go-humanize" | |
17 ) | |
18 | |
19 func min(a uint64, b uint64) uint64 { | |
20 if a > b { | |
21 return b | |
22 } else { | |
M-A Ruel
2016/09/15 14:31:02
}
no need for else
same below
mithro
2016/09/20 12:41:44
Done.
| |
23 return a | |
24 } | |
25 } | |
26 func max(a uint64, b uint64) uint64 { | |
27 if a < b { | |
28 return b | |
29 } else { | |
30 return a | |
31 } | |
32 } | |
33 | |
34 func randChar(r *rand.Rand, runes []rune) rune { | |
35 return runes[r.Intn(len(runes))] | |
36 } | |
37 | |
38 func randStr(r *rand.Rand, length uint64, runes []rune) string { | |
M-A Ruel
2016/09/15 14:31:02
I'd prefer length to be int.
It's a bit confusing
mithro
2016/09/20 12:41:44
Negative length isn't valid though?
M-A Ruel
2016/09/20 16:37:27
uint is sparingly used in Go, it's really only use
| |
39 str := make([]rune, length) | |
40 for i := range str { | |
41 str[i] = randChar(r, runes) | |
42 } | |
43 return string(str) | |
44 } | |
45 | |
46 func randBetween(r *rand.Rand, min uint64, max uint64) uint64 { | |
47 if min == max { | |
48 return min | |
49 } | |
50 return uint64(r.Int63n(int64(max-min))) + min | |
51 } | |
52 | |
53 // FIXME: Maybe some UTF-8 characters? | |
54 var filenameChars = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ 0123456789-") | |
55 | |
56 func filenameRandom(r *rand.Rand, length uint64) string { | |
57 return randStr(r, length, filenameChars) | |
58 } | |
59 | |
60 type DirGen interface { | |
61 Create(seed uint64, num int) | |
62 } | |
63 | |
64 type FileType int | |
M-A Ruel
2016/09/15 14:31:02
I'd prefer to call it Compressibility and have a N
mithro
2016/09/20 12:41:44
I think it is the *type* which is the important fa
| |
65 | |
66 const ( | |
67 FILETYPE_BIN_RAND FileType = iota // Truly random binary data (totally uncompressible) | |
68 FILETYPE_TXT_RAND // Truly random text data (mostly un compressible) | |
69 FILETYPE_BIN_REPEAT // Repeated binary data (compressibl e) | |
70 FILETYPE_TXT_REPEAT // Repeated text data (very compress ible) | |
71 FILETYPE_TXT_LOREM // Lorem Ipsum txt data (very compre ssible) | |
72 | |
73 FILETYPE_MAX | |
74 ) | |
75 | |
76 var FileTypeName []string = []string{ | |
77 "Random Binary", | |
78 "Random Text", | |
79 "Repeated Binary", | |
80 "Repeated Text", | |
81 "Lorem Text", | |
82 } | |
83 | |
84 func (f FileType) String() string { | |
85 return FileTypeName[int(f)] | |
86 } | |
87 | |
88 // FIXME: Maybe some UTF-8 characters? | |
M-A Ruel
2016/09/15 14:31:02
It depends; for file paths, you want UTF-8. For co
| |
89 var textChars = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123 456789-.") | |
90 | |
91 const ( | |
92 BLOCKSIZE uint64 = 1 * 1024 * 1024 // 1 Megabyte | |
93 | |
94 // Maximum 4k long repeated sequences | |
95 SEQUENCE_MINSIZE uint64 = 16 | |
96 SEQUENCE_MAXSIZE uint64 = 4 * 1024 | |
97 ) | |
98 | |
99 func writeFile(r *rand.Rand, filename string, filetype FileType, filesize uint64 ) { | |
100 f, err := os.Create(filename) | |
101 if err != nil { | |
102 log.Fatal(err) | |
M-A Ruel
2016/09/15 14:31:02
return an error, no need to crash.
mithro
2016/09/20 12:41:44
We can't really recover in this case?
We depend o
M-A Ruel
2016/09/20 16:37:27
You return the error, the error is surfaced to the
mithro
2016/09/22 11:19:58
Bubbling that error all the way up to the top seem
| |
103 } | |
104 defer f.Close() | |
105 | |
106 var written uint64 = 0 | |
107 for written < filesize { | |
108 content := make([]byte, min(filesize-written, BLOCKSIZE)) | |
109 | |
110 // Generate a block of content | |
111 switch filetype { | |
112 case FILETYPE_BIN_RAND: | |
M-A Ruel
2016/09/15 14:31:02
You should create a generator so basically:
// G
mithro
2016/09/20 12:41:44
There was a specific reason I didn't do a generato
mithro
2016/09/22 11:19:58
Done.
| |
113 r.Read(content) | |
114 | |
115 case FILETYPE_TXT_RAND: | |
116 // Runes can be multiple bytes long | |
117 for i := 0; i < len(content); { | |
118 bytes := []byte(string(randChar(r, textChars))) | |
119 for j := range bytes { | |
120 content[i+j] = bytes[j] | |
121 } | |
122 i += len(bytes) | |
123 } | |
124 | |
125 case FILETYPE_BIN_REPEAT: | |
126 var sequence []byte = make([]byte, randBetween(r, SEQUEN CE_MINSIZE, SEQUENCE_MAXSIZE)) | |
127 r.Read(sequence) | |
128 | |
129 for i := range content { | |
130 content[i] = sequence[i%len(sequence)] | |
131 } | |
132 | |
133 case FILETYPE_TXT_REPEAT, FILETYPE_TXT_LOREM: | |
134 var sequence []byte | |
135 | |
136 switch filetype { | |
137 case FILETYPE_TXT_REPEAT: | |
138 // FIXME: As runes can be multiple bytes long, t his could technical | |
139 // be longer then SEQUENCE_MAXSIZE, but don't th ink we care? | |
140 sequence = []byte(randStr(r, randBetween(r, SEQU ENCE_MINSIZE, SEQUENCE_MAXSIZE), textChars)) | |
141 case FILETYPE_TXT_LOREM: | |
142 sequence = []byte(lorem) | |
143 } | |
144 | |
145 for i := range content { | |
146 content[i] = sequence[i%len(sequence)] | |
147 } | |
148 } | |
149 f.Write(content) | |
150 written += uint64(len(content)) | |
151 } | |
152 } | |
153 | |
154 const ( | |
155 FILENAME_MINSIZE uint64 = 4 | |
156 FILENAME_MAXSIZE uint64 = 20 | |
157 ) | |
158 | |
159 // Generate num files between (min, max) size | |
160 func GenerateFiles(r *rand.Rand, dir string, num uint64, filesize_min uint64, fi lesize_max uint64) { | |
M-A Ruel
2016/09/15 14:31:02
One thing I care a lot about is to ensure that the
mithro
2016/09/20 12:41:44
This kind of function kind of exists at the bottom
| |
161 for i := uint64(0); i < num; i++ { | |
162 var filename string | |
163 var filepath string | |
164 for true { | |
165 filename = filenameRandom(r, randBetween(r, FILENAME_MIN SIZE, FILENAME_MAXSIZE)) | |
166 filepath = path.Join(dir, filename) | |
167 if _, err := os.Stat(filepath); os.IsNotExist(err) { | |
168 break | |
169 } | |
170 } | |
171 filetype := FileType(r.Intn(int(FILETYPE_MAX))) | |
172 filesize := randBetween(r, filesize_min, filesize_max) | |
173 | |
174 if num < 1000 { | |
175 fmt.Printf("File: %-40s %-20s (%s)\n", filename, filetyp e.String(), humanize.Bytes(filesize)) | |
176 } | |
177 writeFile(r, filepath, filetype, filesize) | |
178 } | |
179 } | |
180 | |
181 // Generate num directories | |
182 func GenerateDirs(r *rand.Rand, dir string, num uint64) []string { | |
183 var result []string | |
184 | |
185 for i := uint64(0); i < num; i++ { | |
186 var dirname string | |
187 var dirpath string | |
188 for true { | |
189 dirname = filenameRandom(r, randBetween(r, FILENAME_MINS IZE, FILENAME_MAXSIZE)) | |
190 dirpath = path.Join(dir, dirname) | |
191 if _, err := os.Stat(dirpath); os.IsNotExist(err) { | |
192 break | |
193 } | |
194 } | |
195 | |
196 if err := os.MkdirAll(dirpath, 0755); err != nil { | |
197 log.Fatal(err) | |
M-A Ruel
2016/09/20 16:37:27
same here.
| |
198 } | |
199 result = append(result, dirpath) | |
200 } | |
201 return result | |
202 } | |
203 | |
204 type FileSettings struct { | |
205 MinNumber uint64 | |
206 MaxNumber uint64 | |
207 MinSize uint64 | |
208 MaxSize uint64 | |
209 } | |
210 | |
211 type DirSettings struct { | |
212 Number []uint64 | |
213 MinFileDepth uint64 | |
214 } | |
215 | |
216 type TreeSettings struct { | |
217 Files []FileSettings | |
218 Dir DirSettings | |
219 } | |
220 | |
221 func generateTreeInternal(r *rand.Rand, dir string, depth uint64, settings *Tree Settings) { | |
222 fmt.Printf("%04d:%s -->\n", depth, dir) | |
223 // Generate the files in this directory | |
224 if depth >= settings.Dir.MinFileDepth { | |
225 for _, files := range settings.Files { | |
226 numfiles := randBetween(r, files.MinNumber, files.MaxNum ber) | |
227 fmt.Printf("%04d:%s: Generating %d files (between %s and %s)\n", depth, dir, numfiles, humanize.Bytes(files.MinSize), humanize.Bytes(fil es.MaxSize)) | |
228 GenerateFiles(r, dir, numfiles, files.MinSize, files.Max Size) | |
229 } | |
230 } | |
231 | |
232 // Generate another depth of directories | |
233 if depth < uint64(len(settings.Dir.Number)) { | |
234 numdirs := settings.Dir.Number[depth] | |
235 fmt.Printf("%04d:%s: Generating %d directories\n", depth, dir, n umdirs) | |
236 for _, childpath := range GenerateDirs(r, dir, numdirs) { | |
237 generateTreeInternal(r, childpath, depth+1, settings) | |
238 } | |
239 } | |
240 fmt.Printf("%04d:%s <--\n", depth, dir) | |
241 } | |
242 | |
243 func GenerateTree(r *rand.Rand, rootdir string, settings *TreeSettings) { | |
244 generateTreeInternal(r, rootdir, 0, settings) | |
245 return | |
246 } | |
OLD | NEW |