| OLD | NEW |
| 1 # Copyright 2013 The Chromium Authors. All rights reserved. | 1 # Copyright 2013 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 import difflib | 5 import difflib |
| 6 import hashlib | 6 import hashlib |
| 7 import os | 7 import os |
| 8 import re | 8 import re |
| 9 import sys | 9 import sys |
| 10 | 10 |
| 11 from util import build_utils |
| 12 |
| 13 if build_utils.COLORAMA_ROOT not in sys.path: |
| 14 sys.path.append(build_utils.COLORAMA_ROOT) |
| 15 import colorama |
| 16 |
| 11 | 17 |
| 12 # When set and a difference is detected, a diff of what changed is printed. | 18 # When set and a difference is detected, a diff of what changed is printed. |
| 13 _PRINT_MD5_DIFFS = int(os.environ.get('PRINT_MD5_DIFFS', 0)) | 19 _PRINT_MD5_DIFFS = int(os.environ.get('PRINT_MD5_DIFFS', 0)) |
| 14 | 20 |
| 15 # Used to strip off temp dir prefix. | 21 # Used to strip off temp dir prefix. |
| 16 _TEMP_DIR_PATTERN = re.compile(r'^/tmp/.*?/') | 22 _TEMP_DIR_PATTERN = re.compile(r'^/tmp/.*?/') |
| 17 | 23 |
| 18 | 24 |
| 19 def CallAndRecordIfStale( | 25 def CallAndRecordIfStale( |
| 20 function, record_path=None, input_paths=None, input_strings=None, | 26 function, record_path=None, input_paths=None, input_strings=None, |
| 21 output_paths=None, force=False): | 27 force=False): |
| 22 """Calls function if outputs are stale. | 28 """Calls function if the md5sum of the input paths/strings has changed. |
| 23 | 29 |
| 24 Outputs are considered stale if: | 30 The md5sum of the inputs is compared with the one stored in record_path. If |
| 25 - any output_paths are missing, or | 31 this has changed (or the record doesn't exist), function will be called and |
| 26 - the contents of any file within input_paths has changed, or | 32 the new md5sum will be recorded. |
| 27 - the contents of input_strings has changed. | |
| 28 | 33 |
| 29 To debug which files are out-of-date, set the environment variable: | 34 If force is True, the function will be called regardless of whether the |
| 30 PRINT_MD5_DIFFS=1 | 35 md5sum is out of date. |
| 31 | |
| 32 Args: | |
| 33 function: The function to call. | |
| 34 record_path: Path to record metadata. | |
| 35 Defaults to output_paths[0] + '.md5.stamp' | |
| 36 input_paths: List of paths to calcualte an md5 sum on. | |
| 37 input_strings: List of strings to record verbatim. | |
| 38 output_paths: List of output paths. | |
| 39 force: When True, function is always called. | |
| 40 """ | 36 """ |
| 41 assert record_path or output_paths | 37 if not input_paths: |
| 42 input_paths = input_paths or [] | 38 input_paths = [] |
| 43 input_strings = input_strings or [] | 39 if not input_strings: |
| 44 output_paths = output_paths or [] | 40 input_strings = [] |
| 45 record_path = record_path or output_paths[0] + '.md5.stamp' | |
| 46 md5_checker = _Md5Checker( | 41 md5_checker = _Md5Checker( |
| 47 record_path=record_path, | 42 record_path=record_path, |
| 48 input_paths=input_paths, | 43 input_paths=input_paths, |
| 49 input_strings=input_strings) | 44 input_strings=input_strings) |
| 50 | 45 |
| 51 missing_outputs = [x for x in output_paths if not os.path.exists(x)] | |
| 52 is_stale = md5_checker.old_digest != md5_checker.new_digest | 46 is_stale = md5_checker.old_digest != md5_checker.new_digest |
| 53 | 47 if force or is_stale: |
| 54 if force or missing_outputs or is_stale: | 48 if is_stale and _PRINT_MD5_DIFFS: |
| 55 if _PRINT_MD5_DIFFS: | 49 print '%sDifference found in %s:%s' % ( |
| 56 print '=' * 80 | 50 colorama.Fore.YELLOW, record_path, colorama.Fore.RESET) |
| 57 print 'Difference found in %s:' % record_path | 51 print md5_checker.DescribeDifference() |
| 58 if missing_outputs: | |
| 59 print 'Outputs do not exist:\n' + '\n'.join(missing_outputs) | |
| 60 elif force: | |
| 61 print 'force=True' | |
| 62 else: | |
| 63 print md5_checker.DescribeDifference() | |
| 64 print '=' * 80 | |
| 65 function() | 52 function() |
| 66 md5_checker.Write() | 53 md5_checker.Write() |
| 67 | 54 |
| 68 | 55 |
| 69 def _UpdateMd5ForFile(md5, path, block_size=2**16): | 56 def _UpdateMd5ForFile(md5, path, block_size=2**16): |
| 70 with open(path, 'rb') as infile: | 57 with open(path, 'rb') as infile: |
| 71 while True: | 58 while True: |
| 72 data = infile.read(block_size) | 59 data = infile.read(block_size) |
| 73 if not data: | 60 if not data: |
| 74 break | 61 break |
| (...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 112 extended_info = [] | 99 extended_info = [] |
| 113 outer_md5 = hashlib.md5() | 100 outer_md5 = hashlib.md5() |
| 114 for i in sorted(input_paths): | 101 for i in sorted(input_paths): |
| 115 inner_md5 = hashlib.md5() | 102 inner_md5 = hashlib.md5() |
| 116 _UpdateMd5ForPath(inner_md5, i) | 103 _UpdateMd5ForPath(inner_md5, i) |
| 117 i = _TrimPathPrefix(i) | 104 i = _TrimPathPrefix(i) |
| 118 extended_info.append(i + '=' + inner_md5.hexdigest()) | 105 extended_info.append(i + '=' + inner_md5.hexdigest()) |
| 119 # Include the digest in the overall diff, but not the path | 106 # Include the digest in the overall diff, but not the path |
| 120 outer_md5.update(inner_md5.hexdigest()) | 107 outer_md5.update(inner_md5.hexdigest()) |
| 121 | 108 |
| 122 for s in (str(s) for s in input_strings): | 109 for s in input_strings: |
| 123 outer_md5.update(s) | 110 outer_md5.update(s) |
| 124 extended_info.append(s) | 111 extended_info.append(s) |
| 125 | 112 |
| 126 self.new_digest = outer_md5.hexdigest() | 113 self.new_digest = outer_md5.hexdigest() |
| 127 self.new_extended_info = extended_info | 114 self.new_extended_info = extended_info |
| 128 | 115 |
| 129 self.old_digest = '' | 116 self.old_digest = '' |
| 130 self.old_extended_info = [] | 117 self.old_extended_info = [] |
| 131 if os.path.exists(self.record_path): | 118 if os.path.exists(self.record_path): |
| 132 with open(self.record_path, 'r') as old_record: | 119 with open(self.record_path, 'r') as old_record: |
| 133 self.old_extended_info = [line.strip() for line in old_record] | 120 self.old_extended_info = [line.strip() for line in old_record] |
| 134 if self.old_extended_info: | 121 self.old_digest = self.old_extended_info.pop(0) |
| 135 self.old_digest = self.old_extended_info.pop(0) | |
| 136 | 122 |
| 137 def Write(self): | 123 def Write(self): |
| 138 with open(self.record_path, 'w') as new_record: | 124 with open(self.record_path, 'w') as new_record: |
| 139 new_record.write(self.new_digest) | 125 new_record.write(self.new_digest) |
| 140 new_record.write('\n' + '\n'.join(self.new_extended_info) + '\n') | 126 new_record.write('\n' + '\n'.join(self.new_extended_info) + '\n') |
| 141 | 127 |
| 142 def DescribeDifference(self): | 128 def DescribeDifference(self): |
| 143 if self.old_digest == self.new_digest: | 129 if self.old_digest == self.new_digest: |
| 144 return "There's no difference." | 130 return "There's no difference." |
| 145 if not self.old_digest: | 131 if not self.old_digest: |
| 146 return 'Previous stamp file not found.' | 132 return 'Previous stamp file not found.' |
| 147 if not self.old_extended_info: | 133 if not self.old_extended_info: |
| 148 return 'Previous stamp file lacks extended info.' | 134 return 'Previous stamp file lacks extended info.' |
| 149 diff = difflib.unified_diff(self.old_extended_info, self.new_extended_info) | 135 diff = difflib.unified_diff(self.old_extended_info, self.new_extended_info) |
| 150 return '\n'.join(diff) | 136 return '\n'.join(diff) |
| OLD | NEW |