OLD | NEW |
---|---|
(Empty) | |
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. | |
palmer
2012/12/06 21:20:17
Putting this back in the tree will (re-)start a fi
| |
2 // Use of this source code is governed by a BSD-style license that can be | |
3 // found in the LICENSE file. | |
4 | |
5 // This program converts the information in | |
6 // transport_security_state_static.json and | |
7 // transport_security_state_static.certs into | |
8 // transport_security_state_static.h. The input files contain information about | |
9 // public key pinning and HTTPS-only sites that is compiled into Chromium. | |
10 | |
11 // Run as: | |
12 // % go run transport_security_state_static_generate.go transport_security_state _static.json transport_security_state_static.certs | |
13 // | |
14 // It will write transport_security_state_static.h | |
15 | |
16 package main | |
17 | |
18 import ( | |
19 "bufio" | |
20 "bytes" | |
21 "crypto/sha1" | |
22 "crypto/x509" | |
23 "encoding/base64" | |
24 "encoding/json" | |
25 "encoding/pem" | |
26 "errors" | |
27 "fmt" | |
28 "io" | |
29 "os" | |
30 "regexp" | |
31 "strings" | |
32 ) | |
33 | |
34 // A pin represents an entry in transport_security_state_static.certs. It's a | |
35 // name associated with a SubjectPublicKeyInfo hash and, optionally, a | |
36 // certificate. | |
37 type pin struct { | |
38 name string | |
39 cert *x509.Certificate | |
40 spkiHash []byte | |
41 spkiHashFunc string // i.e. "sha1" | |
42 } | |
43 | |
44 // preloaded represents the information contained in the | |
45 // transport_security_state_static.json file. This structure and the two | |
46 // following are used by the "json" package to parse the file. See the comments | |
47 // in transport_security_state_static.json for details. | |
48 type preloaded struct { | |
49 Pinsets []pinset `json:"pinsets"` | |
50 Entries []hsts `json:"entries"` | |
51 } | |
52 | |
53 type pinset struct { | |
54 Name string `json:"name"` | |
55 Include []string `json:"static_spki_hashes"` | |
56 Exclude []string `json:"bad_static_spki_hashes"` | |
57 } | |
58 | |
59 type hsts struct { | |
60 Name string `json:"name"` | |
61 Subdomains bool `json:"include_subdomains"` | |
62 Mode string `json:"mode"` | |
63 Pins string `json:"pins"` | |
64 SNIOnly bool `json:"snionly"` | |
65 } | |
66 | |
67 func main() { | |
68 if len(os.Args) != 3 { | |
69 fmt.Fprintf(os.Stderr, "Usage: %s <json file> <certificates file >\n", os.Args[0]) | |
70 os.Exit(1) | |
71 } | |
72 | |
73 if err := process(os.Args[1], os.Args[2]); err != nil { | |
74 fmt.Fprintf(os.Stderr, "Conversion failed: %s\n", err.Error()) | |
75 os.Exit(1) | |
76 } | |
77 } | |
78 | |
79 func process(jsonFileName, certsFileName string) error { | |
80 jsonFile, err := os.Open(jsonFileName) | |
81 if err != nil { | |
82 return fmt.Errorf("failed to open input file: %s\n", err.Error() ) | |
83 } | |
84 defer jsonFile.Close() | |
85 | |
86 jsonBytes, err := removeComments(jsonFile) | |
87 if err != nil { | |
88 return fmt.Errorf("failed to remove comments from JSON: %s\n", e rr.Error()) | |
89 } | |
90 | |
91 var preloaded preloaded | |
92 if err := json.Unmarshal(jsonBytes, &preloaded); err != nil { | |
93 return fmt.Errorf("failed to parse JSON: %s\n", err.Error()) | |
94 } | |
95 | |
96 certsFile, err := os.Open(certsFileName) | |
97 if err != nil { | |
98 return fmt.Errorf("failed to open input file: %s\n", err.Error() ) | |
99 } | |
100 defer certsFile.Close() | |
101 | |
102 pins, err := parseCertsFile(certsFile) | |
103 if err != nil { | |
104 return fmt.Errorf("failed to parse certificates file: %s\n", err ) | |
105 } | |
106 | |
107 if err := checkDuplicatePins(pins); err != nil { | |
108 return err | |
109 } | |
110 | |
111 if err := checkCertsInPinsets(preloaded.Pinsets, pins); err != nil { | |
112 return err | |
113 } | |
114 | |
115 if err := checkNoopEntries(preloaded.Entries); err != nil { | |
116 return err | |
117 } | |
118 | |
119 if err := checkDuplicateEntries(preloaded.Entries); err != nil { | |
120 return err | |
121 } | |
122 | |
123 outFile, err := os.OpenFile("transport_security_state_static.h", os.O_WR ONLY|os.O_CREATE|os.O_TRUNC, 0644) | |
124 if err != nil { | |
125 return err | |
126 } | |
127 defer outFile.Close() | |
128 | |
129 out := bufio.NewWriter(outFile) | |
130 writeHeader(out) | |
131 writeCertsOutput(out, pins) | |
132 writeHSTSOutput(out, preloaded) | |
133 writeFooter(out) | |
134 out.Flush() | |
135 | |
136 return nil | |
137 } | |
138 | |
139 var newLine = []byte("\n") | |
140 var startOfCert = []byte("-----BEGIN CERTIFICATE") | |
141 var endOfCert = []byte("-----END CERTIFICATE") | |
142 var startOfSHA1 = []byte("sha1/") | |
143 | |
144 // nameRegexp matches valid pin names: an uppercase letter followed by zero or | |
145 // more letters and digits. | |
146 var nameRegexp = regexp.MustCompile("[A-Z][a-zA-Z0-9_]*") | |
147 | |
148 // commentRegexp matches lines that optionally start with whitespace | |
149 // followed by "//". | |
150 var commentRegexp = regexp.MustCompile("^[ \t]*//") | |
151 | |
152 // removeComments reads the contents of |r| and removes any lines beginning | |
153 // with optional whitespace followed by "//" | |
154 func removeComments(r io.Reader) ([]byte, error) { | |
155 var buf bytes.Buffer | |
156 in := bufio.NewReader(r) | |
157 | |
158 for { | |
159 line, isPrefix, err := in.ReadLine() | |
160 if isPrefix { | |
161 return nil, errors.New("line too long in JSON") | |
162 } | |
163 if err == io.EOF { | |
164 break | |
165 } | |
166 if err != nil { | |
167 return nil, err | |
168 } | |
169 if commentRegexp.Match(line) { | |
170 continue | |
171 } | |
172 buf.Write(line) | |
173 buf.Write(newLine) | |
174 } | |
175 | |
176 return buf.Bytes(), nil | |
177 } | |
178 | |
179 // parseCertsFile parses |inFile|, in the format of | |
180 // transport_security_state_static.certs. See the comments at the top of that | |
181 // file for details of the format. | |
182 func parseCertsFile(inFile io.Reader) ([]pin, error) { | |
183 const ( | |
184 PRENAME = iota | |
185 POSTNAME = iota | |
186 INCERT = iota | |
187 ) | |
188 | |
189 in := bufio.NewReader(inFile) | |
190 | |
191 lineNo := 0 | |
192 var pemCert []byte | |
193 state := PRENAME | |
194 var name string | |
195 var pins []pin | |
196 | |
197 for { | |
198 lineNo++ | |
199 line, isPrefix, err := in.ReadLine() | |
200 if isPrefix { | |
201 return nil, fmt.Errorf("line %d is too long to process\n ", lineNo) | |
202 } | |
203 if err == io.EOF { | |
204 break | |
205 } | |
206 if err != nil { | |
207 return nil, fmt.Errorf("error reading from input: %s\n", err.Error()) | |
208 } | |
209 | |
210 if len(line) == 0 || line[0] == '#' { | |
211 continue | |
212 } | |
213 | |
214 switch state { | |
215 case PRENAME: | |
216 name = string(line) | |
217 if !nameRegexp.MatchString(name) { | |
218 return nil, fmt.Errorf("invalid name on line %d\ n", lineNo) | |
219 } | |
220 state = POSTNAME | |
221 case POSTNAME: | |
222 switch { | |
223 case bytes.HasPrefix(line, startOfSHA1): | |
224 hash, err := base64.StdEncoding.DecodeString(str ing(line[len(startOfSHA1):])) | |
225 if err != nil { | |
226 return nil, fmt.Errorf("failed to decode hash on line %d: %s\n", lineNo, err) | |
227 } | |
228 if len(hash) != 20 { | |
229 return nil, fmt.Errorf("bad SHA1 hash le ngth on line %d: %s\n", lineNo, err) | |
230 } | |
231 pins = append(pins, pin{ | |
232 name: name, | |
233 spkiHashFunc: "sha1", | |
234 spkiHash: hash, | |
235 }) | |
236 state = PRENAME | |
237 continue | |
238 case bytes.HasPrefix(line, startOfCert): | |
239 pemCert = pemCert[:0] | |
240 pemCert = append(pemCert, line...) | |
241 pemCert = append(pemCert, '\n') | |
242 state = INCERT | |
243 default: | |
244 return nil, fmt.Errorf("line %d, after a name, i s not a hash nor a certificate\n", lineNo) | |
245 } | |
246 case INCERT: | |
247 pemCert = append(pemCert, line...) | |
248 pemCert = append(pemCert, '\n') | |
249 if !bytes.HasPrefix(line, endOfCert) { | |
250 continue | |
251 } | |
252 | |
253 block, _ := pem.Decode(pemCert) | |
254 if block == nil { | |
255 return nil, fmt.Errorf("failed to decode certifi cate ending on line %d\n", lineNo) | |
256 } | |
257 cert, err := x509.ParseCertificate(block.Bytes) | |
258 if err != nil { | |
259 return nil, fmt.Errorf("failed to parse certific ate ending on line %d: %s\n", lineNo, err.Error()) | |
260 } | |
261 certName := cert.Subject.CommonName | |
262 if len(certName) == 0 { | |
263 certName = cert.Subject.Organization[0] + " " + cert.Subject.OrganizationalUnit[0] | |
264 } | |
265 if err := matchNames(certName, name); err != nil { | |
266 return nil, fmt.Errorf("name failure on line %d: %s\n%s -> %s\n", lineNo, err, certName, name) | |
267 } | |
268 h := sha1.New() | |
269 h.Write(cert.RawSubjectPublicKeyInfo) | |
270 pins = append(pins, pin{ | |
271 name: name, | |
272 cert: cert, | |
273 spkiHashFunc: "sha1", | |
274 spkiHash: h.Sum(nil), | |
275 }) | |
276 state = PRENAME | |
277 } | |
278 } | |
279 | |
280 return pins, nil | |
281 } | |
282 | |
283 // matchNames returns true if the given pin name is a reasonable match for the | |
284 // given CN. | |
285 func matchNames(name, v string) error { | |
286 words := strings.Split(name, " ") | |
287 if len(words) == 0 { | |
288 return errors.New("no words in certificate name") | |
289 } | |
290 firstWord := words[0] | |
291 if strings.HasSuffix(firstWord, ",") { | |
292 firstWord = firstWord[:len(firstWord)-1] | |
293 } | |
294 if strings.HasPrefix(firstWord, "*.") { | |
295 firstWord = firstWord[2:] | |
296 } | |
297 if pos := strings.Index(firstWord, "."); pos != -1 { | |
298 firstWord = firstWord[:pos] | |
299 } | |
300 if pos := strings.Index(firstWord, "-"); pos != -1 { | |
301 firstWord = firstWord[:pos] | |
302 } | |
303 if len(firstWord) == 0 { | |
304 return errors.New("first word of certificate name is empty") | |
305 } | |
306 firstWord = strings.ToLower(firstWord) | |
307 lowerV := strings.ToLower(v) | |
308 if !strings.HasPrefix(lowerV, firstWord) { | |
309 return errors.New("the first word of the certificate name isn't a prefix of the variable name") | |
310 } | |
311 | |
312 for i, word := range words { | |
313 if word == "Class" && i+1 < len(words) { | |
314 if strings.Index(v, word+words[i+1]) == -1 { | |
315 return errors.New("class specification doesn't a ppear in the variable name") | |
316 } | |
317 } else if len(word) == 1 && word[0] >= '0' && word[0] <= '9' { | |
318 if strings.Index(v, word) == -1 { | |
319 return errors.New("number doesn't appear in the variable name") | |
320 } | |
321 } else if isImportantWordInCertificateName(word) { | |
322 if strings.Index(v, word) == -1 { | |
323 return errors.New(word + " doesn't appear in the variable name") | |
324 } | |
325 } | |
326 } | |
327 | |
328 return nil | |
329 } | |
330 | |
331 // isImportantWordInCertificateName returns true if w must be found in any | |
332 // corresponding variable name. | |
333 func isImportantWordInCertificateName(w string) bool { | |
334 switch w { | |
335 case "Universal", "Global", "EV", "G1", "G2", "G3", "G4", "G5": | |
336 return true | |
337 } | |
338 return false | |
339 } | |
340 | |
341 // checkDuplicatePins returns an error if any pins have the same name or the sam e hash. | |
342 func checkDuplicatePins(pins []pin) error { | |
343 seenNames := make(map[string]bool) | |
344 seenHashes := make(map[string]string) | |
345 | |
346 for _, pin := range pins { | |
347 if _, ok := seenNames[pin.name]; ok { | |
348 return fmt.Errorf("duplicate name: %s", pin.name) | |
349 } | |
350 seenNames[pin.name] = true | |
351 | |
352 strHash := string(pin.spkiHash) | |
353 if otherName, ok := seenHashes[strHash]; ok { | |
354 return fmt.Errorf("duplicate hash for %s and %s", pin.na me, otherName) | |
355 } | |
356 seenHashes[strHash] = pin.name | |
357 } | |
358 | |
359 return nil | |
360 } | |
361 | |
362 // checkCertsInPinsets returns an error if | |
363 // a) unknown pins are mentioned in |pinsets| | |
364 // b) unused pins are given in |pins| | |
365 // c) a pinset name is used twice | |
366 func checkCertsInPinsets(pinsets []pinset, pins []pin) error { | |
367 pinNames := make(map[string]bool) | |
368 for _, pin := range pins { | |
369 pinNames[pin.name] = true | |
370 } | |
371 | |
372 usedPinNames := make(map[string]bool) | |
373 pinsetNames := make(map[string]bool) | |
374 | |
375 for _, pinset := range pinsets { | |
376 if _, ok := pinsetNames[pinset.Name]; ok { | |
377 return fmt.Errorf("duplicate pinset name: %s", pinset.Na me) | |
378 } | |
379 pinsetNames[pinset.Name] = true | |
380 | |
381 var allPinNames []string | |
382 allPinNames = append(allPinNames, pinset.Include...) | |
383 allPinNames = append(allPinNames, pinset.Exclude...) | |
384 | |
385 for _, pinName := range allPinNames { | |
386 if _, ok := pinNames[pinName]; !ok { | |
387 return fmt.Errorf("unknown pin: %s", pinName) | |
388 } | |
389 usedPinNames[pinName] = true | |
390 } | |
391 } | |
392 | |
393 for pinName := range pinNames { | |
394 if _, ok := usedPinNames[pinName]; !ok { | |
395 return fmt.Errorf("unused pin: %s", pinName) | |
396 } | |
397 } | |
398 | |
399 return nil | |
400 } | |
401 | |
402 func checkNoopEntries(entries []hsts) error { | |
403 for _, e := range entries { | |
404 if len(e.Mode) == 0 && len(e.Pins) == 0 { | |
405 switch e.Name { | |
406 // This entry is deliberately used as an exclusion. | |
407 case "learn.doubleclick.net": | |
408 continue | |
409 default: | |
410 return errors.New("Entry for " + e.Name + " has no mode and no pins") | |
411 } | |
412 } | |
413 } | |
414 | |
415 return nil | |
416 } | |
417 | |
418 func checkDuplicateEntries(entries []hsts) error { | |
419 seen := make(map[string]bool) | |
420 | |
421 for _, e := range entries { | |
422 if _, ok := seen[e.Name]; ok { | |
423 return errors.New("Duplicate entry for " + e.Name) | |
424 } | |
425 seen[e.Name] = true | |
426 } | |
427 | |
428 return nil | |
429 } | |
430 | |
431 func writeHeader(out *bufio.Writer) { | |
432 out.WriteString(`// Copyright (c) 2012 The Chromium Authors. All rights reserved. | |
433 // Use of this source code is governed by a BSD-style license that can be | |
434 // found in the LICENSE file. | |
435 | |
436 // This file is automatically generated by transport_security_state_static_gener ate.go | |
437 | |
438 #ifndef NET_BASE_TRANSPORT_SECURITY_STATE_STATIC_H_ | |
439 #define NET_BASE_TRANSPORT_SECURITY_STATE_STATIC_H_ | |
440 | |
441 `) | |
442 | |
443 } | |
444 | |
445 func writeFooter(out *bufio.Writer) { | |
446 out.WriteString("#endif // NET_BASE_TRANSPORT_SECURITY_STATE_STATIC_H_\n ") | |
447 } | |
448 | |
449 func writeCertsOutput(out *bufio.Writer, pins []pin) { | |
450 out.WriteString(`// These are SubjectPublicKeyInfo hashes for public key pinning. The | |
451 // hashes are SHA1 digests. | |
452 | |
453 `) | |
454 | |
455 for _, pin := range pins { | |
456 fmt.Fprintf(out, "static const char kSPKIHash_%s[] =\n", pin.nam e) | |
457 var s string | |
458 for _,c := range pin.spkiHash { | |
459 s += fmt.Sprintf("\\x%02x", c) | |
460 } | |
461 fmt.Fprintf(out, " \"%s\";\n\n", s) | |
462 } | |
463 } | |
464 | |
465 // uppercaseFirstLetter returns s with the first letter uppercased. | |
466 func uppercaseFirstLetter(s string) string { | |
467 // We need to find the index of the second code-point, which may not be | |
468 // one. | |
469 for i := range s { | |
470 if i == 0 { | |
471 continue | |
472 } | |
473 return strings.ToUpper(s[:i]) + s[i:] | |
474 } | |
475 return strings.ToUpper(s) | |
476 } | |
477 | |
478 func writeListOfPins(w io.Writer, name string, pinNames []string) { | |
479 fmt.Fprintf(w, "static const char* const %s[] = {\n", name) | |
480 for _, pinName := range pinNames { | |
481 fmt.Fprintf(w, " kSPKIHash_%s,\n", pinName) | |
482 } | |
483 fmt.Fprintf(w, " NULL,\n};\n") | |
484 } | |
485 | |
486 // toDNS returns a string converts the domain name |s| into C-escaped, | |
487 // length-prefixed form and also returns the length of the interpreted string. | |
488 // i.e. for an input "example.com" it will return "\\007example\\003com", 13. | |
489 func toDNS(s string) (string, int) { | |
490 labels := strings.Split(s, ".") | |
491 | |
492 var name string | |
493 var l int | |
494 for _, label := range labels { | |
495 if len(label) > 63 { | |
496 panic("DNS label too long") | |
497 } | |
498 name += fmt.Sprintf("\\%03o", len(label)) | |
499 name += label | |
500 l += len(label) + 1 | |
501 } | |
502 l += 1 // For the length of the root label. | |
503 | |
504 return name, l | |
505 } | |
506 | |
507 // domainConstant converts the domain name |s| into a string of the form | |
508 // "DOMAIN_" + uppercase last two labels. | |
509 func domainConstant(s string) string { | |
510 labels := strings.Split(s, ".") | |
511 gtld := strings.ToUpper(labels[len(labels)-1]) | |
512 domain := strings.Replace(strings.ToUpper(labels[len(labels)-2]), "-", " _", -1) | |
513 | |
514 return fmt.Sprintf("DOMAIN_%s_%s", domain, gtld) | |
515 } | |
516 | |
517 func writeHSTSEntry(out *bufio.Writer, entry hsts) { | |
518 dnsName, dnsLen := toDNS(entry.Name) | |
519 domain := "DOMAIN_NOT_PINNED" | |
520 pinsetName := "kNoPins" | |
521 if len(entry.Pins) > 0 { | |
522 pinsetName = fmt.Sprintf("k%sPins", uppercaseFirstLetter(entry.P ins)) | |
523 domain = domainConstant(entry.Name) | |
524 } | |
525 fmt.Fprintf(out, " {%d, %t, \"%s\", %t, %s, %s },\n", dnsLen, entry.Sub domains, dnsName, entry.Mode == "force-https", pinsetName, domain) | |
526 } | |
527 | |
528 func writeHSTSOutput(out *bufio.Writer, hsts preloaded) error { | |
529 out.WriteString(`// The following is static data describing the hosts th at are hardcoded with | |
530 // certificate pins or HSTS information. | |
531 | |
532 // kNoRejectedPublicKeys is a placeholder for when no public keys are rejected. | |
533 static const char* const kNoRejectedPublicKeys[] = { | |
534 NULL, | |
535 }; | |
536 | |
537 `) | |
538 | |
539 for _, pinset := range hsts.Pinsets { | |
540 name := uppercaseFirstLetter(pinset.Name) | |
541 acceptableListName := fmt.Sprintf("k%sAcceptableCerts", name) | |
542 writeListOfPins(out, acceptableListName, pinset.Include) | |
543 | |
544 rejectedListName := "kNoRejectedPublicKeys" | |
545 if len(pinset.Exclude) > 0 { | |
546 rejectedListName = fmt.Sprintf("k%sRejectedCerts", name) | |
547 writeListOfPins(out, rejectedListName, pinset.Exclude) | |
548 } | |
549 fmt.Fprintf(out, `#define k%sPins { \ | |
550 %s, \ | |
551 %s, \ | |
552 } | |
553 | |
554 `, name, acceptableListName, rejectedListName) | |
555 } | |
556 | |
557 out.WriteString(`#define kNoPins {\ | |
558 NULL, NULL, \ | |
559 } | |
560 | |
561 static const struct HSTSPreload kPreloadedSTS[] = { | |
562 `) | |
563 | |
564 for _, entry := range hsts.Entries { | |
565 if entry.SNIOnly { | |
566 continue | |
567 } | |
568 writeHSTSEntry(out, entry) | |
569 } | |
570 | |
571 out.WriteString(`}; | |
572 static const size_t kNumPreloadedSTS = ARRAYSIZE_UNSAFE(kPreloadedSTS); | |
573 | |
574 static const struct HSTSPreload kPreloadedSNISTS[] = { | |
575 `) | |
576 | |
577 for _, entry := range hsts.Entries { | |
578 if !entry.SNIOnly { | |
579 continue | |
580 } | |
581 writeHSTSEntry(out, entry) | |
582 } | |
583 | |
584 out.WriteString(`}; | |
585 static const size_t kNumPreloadedSNISTS = ARRAYSIZE_UNSAFE(kPreloadedSNISTS); | |
586 | |
587 `) | |
588 | |
589 return nil | |
590 } | |
OLD | NEW |