OLD | NEW |
1 # Copyright 2013 The Chromium Authors. All rights reserved. | 1 # Copyright 2013 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 import difflib | 5 import difflib |
6 import hashlib | 6 import hashlib |
7 import os | 7 import os |
8 import re | 8 import re |
9 import sys | 9 import sys |
10 | 10 |
| 11 from util import build_utils |
| 12 |
| 13 if build_utils.COLORAMA_ROOT not in sys.path: |
| 14 sys.path.append(build_utils.COLORAMA_ROOT) |
| 15 import colorama |
| 16 |
11 | 17 |
12 # When set and a difference is detected, a diff of what changed is printed. | 18 # When set and a difference is detected, a diff of what changed is printed. |
13 _PRINT_MD5_DIFFS = int(os.environ.get('PRINT_MD5_DIFFS', 0)) | 19 _PRINT_MD5_DIFFS = int(os.environ.get('PRINT_MD5_DIFFS', 0)) |
14 | 20 |
15 # Used to strip off temp dir prefix. | 21 # Used to strip off temp dir prefix. |
16 _TEMP_DIR_PATTERN = re.compile(r'^/tmp/.*?/') | 22 _TEMP_DIR_PATTERN = re.compile(r'^/tmp/.*?/') |
17 | 23 |
18 | 24 |
19 def CallAndRecordIfStale( | 25 def CallAndRecordIfStale( |
20 function, record_path=None, input_paths=None, input_strings=None, | 26 function, record_path=None, input_paths=None, input_strings=None, |
21 output_paths=None, force=False): | 27 force=False): |
22 """Calls function if outputs are stale. | 28 """Calls function if the md5sum of the input paths/strings has changed. |
23 | 29 |
24 Outputs are considered stale if: | 30 The md5sum of the inputs is compared with the one stored in record_path. If |
25 - any output_paths are missing, or | 31 this has changed (or the record doesn't exist), function will be called and |
26 - the contents of any file within input_paths has changed, or | 32 the new md5sum will be recorded. |
27 - the contents of input_strings has changed. | |
28 | 33 |
29 To debug which files are out-of-date, set the environment variable: | 34 If force is True, the function will be called regardless of whether the |
30 PRINT_MD5_DIFFS=1 | 35 md5sum is out of date. |
31 | |
32 Args: | |
33 function: The function to call. | |
34 record_path: Path to record metadata. | |
35 Defaults to output_paths[0] + '.md5.stamp' | |
36 input_paths: List of paths to calcualte an md5 sum on. | |
37 input_strings: List of strings to record verbatim. | |
38 output_paths: List of output paths. | |
39 force: When True, function is always called. | |
40 """ | 36 """ |
41 assert record_path or output_paths | 37 if not input_paths: |
42 input_paths = input_paths or [] | 38 input_paths = [] |
43 input_strings = input_strings or [] | 39 if not input_strings: |
44 output_paths = output_paths or [] | 40 input_strings = [] |
45 record_path = record_path or output_paths[0] + '.md5.stamp' | |
46 md5_checker = _Md5Checker( | 41 md5_checker = _Md5Checker( |
47 record_path=record_path, | 42 record_path=record_path, |
48 input_paths=input_paths, | 43 input_paths=input_paths, |
49 input_strings=input_strings) | 44 input_strings=input_strings) |
50 | 45 |
51 missing_outputs = [x for x in output_paths if not os.path.exists(x)] | |
52 is_stale = md5_checker.old_digest != md5_checker.new_digest | 46 is_stale = md5_checker.old_digest != md5_checker.new_digest |
53 | 47 if force or is_stale: |
54 if force or missing_outputs or is_stale: | 48 if is_stale and _PRINT_MD5_DIFFS: |
55 if _PRINT_MD5_DIFFS: | 49 print '%sDifference found in %s:%s' % ( |
56 print '=' * 80 | 50 colorama.Fore.YELLOW, record_path, colorama.Fore.RESET) |
57 print 'Difference found in %s:' % record_path | 51 print md5_checker.DescribeDifference() |
58 if missing_outputs: | |
59 print 'Outputs do not exist:\n' + '\n'.join(missing_outputs) | |
60 elif force: | |
61 print 'force=True' | |
62 else: | |
63 print md5_checker.DescribeDifference() | |
64 print '=' * 80 | |
65 function() | 52 function() |
66 md5_checker.Write() | 53 md5_checker.Write() |
67 | 54 |
68 | 55 |
69 def _UpdateMd5ForFile(md5, path, block_size=2**16): | 56 def _UpdateMd5ForFile(md5, path, block_size=2**16): |
70 with open(path, 'rb') as infile: | 57 with open(path, 'rb') as infile: |
71 while True: | 58 while True: |
72 data = infile.read(block_size) | 59 data = infile.read(block_size) |
73 if not data: | 60 if not data: |
74 break | 61 break |
(...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
112 extended_info = [] | 99 extended_info = [] |
113 outer_md5 = hashlib.md5() | 100 outer_md5 = hashlib.md5() |
114 for i in sorted(input_paths): | 101 for i in sorted(input_paths): |
115 inner_md5 = hashlib.md5() | 102 inner_md5 = hashlib.md5() |
116 _UpdateMd5ForPath(inner_md5, i) | 103 _UpdateMd5ForPath(inner_md5, i) |
117 i = _TrimPathPrefix(i) | 104 i = _TrimPathPrefix(i) |
118 extended_info.append(i + '=' + inner_md5.hexdigest()) | 105 extended_info.append(i + '=' + inner_md5.hexdigest()) |
119 # Include the digest in the overall diff, but not the path | 106 # Include the digest in the overall diff, but not the path |
120 outer_md5.update(inner_md5.hexdigest()) | 107 outer_md5.update(inner_md5.hexdigest()) |
121 | 108 |
122 for s in (str(s) for s in input_strings): | 109 for s in input_strings: |
123 outer_md5.update(s) | 110 outer_md5.update(s) |
124 extended_info.append(s) | 111 extended_info.append(s) |
125 | 112 |
126 self.new_digest = outer_md5.hexdigest() | 113 self.new_digest = outer_md5.hexdigest() |
127 self.new_extended_info = extended_info | 114 self.new_extended_info = extended_info |
128 | 115 |
129 self.old_digest = '' | 116 self.old_digest = '' |
130 self.old_extended_info = [] | 117 self.old_extended_info = [] |
131 if os.path.exists(self.record_path): | 118 if os.path.exists(self.record_path): |
132 with open(self.record_path, 'r') as old_record: | 119 with open(self.record_path, 'r') as old_record: |
133 self.old_extended_info = [line.strip() for line in old_record] | 120 self.old_extended_info = [line.strip() for line in old_record] |
134 if self.old_extended_info: | 121 self.old_digest = self.old_extended_info.pop(0) |
135 self.old_digest = self.old_extended_info.pop(0) | |
136 | 122 |
137 def Write(self): | 123 def Write(self): |
138 with open(self.record_path, 'w') as new_record: | 124 with open(self.record_path, 'w') as new_record: |
139 new_record.write(self.new_digest) | 125 new_record.write(self.new_digest) |
140 new_record.write('\n' + '\n'.join(self.new_extended_info) + '\n') | 126 new_record.write('\n' + '\n'.join(self.new_extended_info) + '\n') |
141 | 127 |
142 def DescribeDifference(self): | 128 def DescribeDifference(self): |
143 if self.old_digest == self.new_digest: | 129 if self.old_digest == self.new_digest: |
144 return "There's no difference." | 130 return "There's no difference." |
145 if not self.old_digest: | 131 if not self.old_digest: |
146 return 'Previous stamp file not found.' | 132 return 'Previous stamp file not found.' |
147 if not self.old_extended_info: | 133 if not self.old_extended_info: |
148 return 'Previous stamp file lacks extended info.' | 134 return 'Previous stamp file lacks extended info.' |
149 diff = difflib.unified_diff(self.old_extended_info, self.new_extended_info) | 135 diff = difflib.unified_diff(self.old_extended_info, self.new_extended_info) |
150 return '\n'.join(diff) | 136 return '\n'.join(diff) |
OLD | NEW |