Chromium Code Reviews| OLD | NEW |
|---|---|
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # Copyright 2017 The Chromium Authors. All rights reserved. | 2 # Copyright 2017 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 | 5 |
| 6 """Tool for finding the cause of APK bloat. | 6 """Tool for finding the cause of APK bloat. |
| 7 | 7 |
| 8 Run diagnose_apk_bloat.py -h for detailed usage help. | 8 Run diagnose_apk_bloat.py -h for detailed usage help. |
| 9 """ | 9 """ |
| 10 | 10 |
| 11 import argparse | 11 import argparse |
| 12 import collections | 12 import collections |
| 13 from contextlib import contextmanager | 13 from contextlib import contextmanager |
| 14 import distutils.spawn | 14 import distutils.spawn |
| 15 import json | 15 import json |
| 16 import multiprocessing | 16 import multiprocessing |
| 17 import os | 17 import os |
| 18 import re | 18 import re |
| 19 import shutil | 19 import shutil |
| 20 import subprocess | 20 import subprocess |
| 21 import sys | 21 import sys |
| 22 import tempfile | 22 import tempfile |
| 23 import zipfile | 23 import zipfile |
| 24 | 24 |
| 25 _COMMIT_COUNT_WARN_THRESHOLD = 15 | 25 _COMMIT_COUNT_WARN_THRESHOLD = 15 |
| 26 _ALLOWED_CONSECUTIVE_FAILURES = 2 | 26 _ALLOWED_CONSECUTIVE_FAILURES = 2 |
| 27 _DIFF_DETAILS_LINES_THRESHOLD = 100 | 27 _DIFF_DETAILS_LINES_THRESHOLD = 100 |
| 28 _BUILDER_URL = \ | 28 _BUILDER_URL = \ |
| 29 'https://build.chromium.org/p/chromium.perf/builders/Android%20Builder' | 29 'https://build.chromium.org/p/chromium.perf/builders/Android%20Builder' |
| 30 _CLOUD_OUT_DIR = os.path.join('out', 'Release') | |
| 31 _SRC_ROOT = os.path.abspath( | 30 _SRC_ROOT = os.path.abspath( |
| 32 os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) | 31 os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) |
| 33 _DEFAULT_ARCHIVE_DIR = os.path.join(_SRC_ROOT, 'binary-size-bloat') | 32 _DEFAULT_ARCHIVE_DIR = os.path.join(_SRC_ROOT, 'binary-size-bloat') |
| 34 _DEFAULT_OUT_DIR = os.path.join(_SRC_ROOT, 'out', 'diagnose-apk-bloat') | 33 _DEFAULT_OUT_DIR = os.path.join(_SRC_ROOT, 'out', 'diagnose-apk-bloat') |
| 35 _DEFAULT_TARGET = 'monochrome_public_apk' | 34 _DEFAULT_ANDROID_TARGET = 'monochrome_public_apk' |
| 35 _ANDROID_BUCKET = 'gs://chrome-perf/Android Builder/' | |
| 36 _LINUX_BUCKET = 'gs://chrome-perf/Linux Builder/' | |
| 36 | 37 |
| 37 | 38 |
| 38 _global_restore_checkout_func = None | 39 _global_restore_checkout_func = None |
| 39 | 40 |
| 40 | 41 |
| 41 def _SetRestoreFunc(subrepo): | 42 def _SetRestoreFunc(subrepo): |
| 42 branch = _GitCmd(['rev-parse', '--abbrev-ref', 'HEAD'], subrepo) | 43 branch = _GitCmd(['rev-parse', '--abbrev-ref', 'HEAD'], subrepo) |
| 43 global _global_restore_checkout_func | 44 global _global_restore_checkout_func |
| 44 _global_restore_checkout_func = lambda: _GitCmd(['checkout', branch], subrepo) | 45 _global_restore_checkout_func = lambda: _GitCmd(['checkout', branch], subrepo) |
| 45 | 46 |
| 46 | 47 |
| 47 _DiffResult = collections.namedtuple( | 48 _DiffResult = collections.namedtuple('DiffResult', ['name', 'value', 'units']) |
| 48 'DiffResult', ['name', 'value', 'units']) | |
| 49 | 49 |
| 50 | 50 |
| 51 class BaseDiff(object): | 51 class BaseDiff(object): |
| 52 """Base class capturing binary size diffs.""" | 52 """Base class capturing binary size diffs.""" |
| 53 def __init__(self, name): | 53 def __init__(self, name): |
| 54 self.name = name | 54 self.name = name |
| 55 self.banner = '\n' + '*' * 30 + name + '*' * 30 | 55 self.banner = '\n' + '*' * 30 + name + '*' * 30 |
| 56 | 56 |
| 57 def AppendResults(self, logfile): | 57 def AppendResults(self, logfile): |
| 58 """Print and write diff results to an open |logfile|.""" | 58 """Print and write diff results to an open |logfile|.""" |
| 59 _PrintAndWriteToFile(logfile, self.banner) | 59 _PrintAndWriteToFile(logfile, self.banner) |
| 60 _PrintAndWriteToFile(logfile, 'Summary:') | 60 _PrintAndWriteToFile(logfile, 'Summary:') |
| 61 _PrintAndWriteToFile(logfile, self.Summary()) | 61 _PrintAndWriteToFile(logfile, self.Summary()) |
| 62 _PrintAndWriteToFile(logfile, '\nDetails:') | 62 _PrintAndWriteToFile(logfile, '\nDetails:') |
| 63 _PrintAndWriteToFile(logfile, self.DetailedResults()) | 63 _PrintAndWriteToFile(logfile, self.DetailedResults()) |
| 64 | 64 |
| 65 @property | 65 @property |
| 66 def summary_stat(self): | 66 def summary_stat(self): |
| 67 return None | 67 return None |
| 68 | 68 |
| 69 def Summary(self): | 69 def Summary(self): |
| 70 """A short description that summarizes the source of binary size bloat.""" | 70 """A short description that summarizes the source of binary size bloat.""" |
| 71 raise NotImplementedError() | 71 raise NotImplementedError() |
| 72 | 72 |
| 73 def DetailedResults(self): | 73 def DetailedResults(self): |
| 74 """An iterable description of the cause of binary size bloat.""" | 74 """An iterable description of the cause of binary size bloat.""" |
| 75 raise NotImplementedError() | 75 raise NotImplementedError() |
| 76 | 76 |
| 77 def ProduceDiff(self, archive_dirs): | 77 def ProduceDiff(self, before_dir, after_dir): |
| 78 """Prepare a binary size diff with ready to print results.""" | 78 """Prepare a binary size diff with ready to print results.""" |
| 79 raise NotImplementedError() | 79 raise NotImplementedError() |
| 80 | 80 |
| 81 def RunDiff(self, logfile, archive_dirs): | 81 def RunDiff(self, logfile, before_dir, after_dir): |
| 82 self.ProduceDiff(archive_dirs) | 82 self.ProduceDiff(before_dir, after_dir) |
| 83 self.AppendResults(logfile) | 83 self.AppendResults(logfile) |
| 84 | 84 |
| 85 | 85 |
| 86 class NativeDiff(BaseDiff): | 86 class NativeDiff(BaseDiff): |
| 87 _RE_SUMMARY = re.compile( | 87 _RE_SUMMARY = re.compile( |
| 88 r'.*(Section Sizes .*? object files added, \d+ removed).*', | 88 r'.*(Section Sizes .*? object files added, \d+ removed).*', |
| 89 flags=re.DOTALL) | 89 flags=re.DOTALL) |
| 90 _RE_SUMMARY_STAT = re.compile( | 90 _RE_SUMMARY_STAT = re.compile( |
| 91 r'Section Sizes \(Total=(?P<value>\d+) (?P<units>\w+)\)') | 91 r'Section Sizes \(Total=(?P<value>\d+) (?P<units>\w+)\)') |
| 92 _SUMMARY_STAT_NAME = 'Native Library Delta' | 92 _SUMMARY_STAT_NAME = 'Native Library Delta' |
| (...skipping 11 matching lines...) Expand all Loading... | |
| 104 return _DiffResult( | 104 return _DiffResult( |
| 105 NativeDiff._SUMMARY_STAT_NAME, m.group('value'), m.group('units')) | 105 NativeDiff._SUMMARY_STAT_NAME, m.group('value'), m.group('units')) |
| 106 return None | 106 return None |
| 107 | 107 |
| 108 def DetailedResults(self): | 108 def DetailedResults(self): |
| 109 return self._diff.splitlines() | 109 return self._diff.splitlines() |
| 110 | 110 |
| 111 def Summary(self): | 111 def Summary(self): |
| 112 return NativeDiff._RE_SUMMARY.match(self._diff).group(1) | 112 return NativeDiff._RE_SUMMARY.match(self._diff).group(1) |
| 113 | 113 |
| 114 def ProduceDiff(self, archive_dirs): | 114 def ProduceDiff(self, before_dir, after_dir): |
| 115 size_files = [os.path.join(a, self._size_name) | 115 before_size = os.path.join(before_dir, self._size_name) |
| 116 for a in reversed(archive_dirs)] | 116 after_size = os.path.join(after_dir, self._size_name) |
| 117 cmd = [self._supersize_path, 'diff'] + size_files | 117 cmd = [self._supersize_path, 'diff', before_size, after_size] |
| 118 self._diff = _RunCmd(cmd)[0].replace('{', '{{').replace('}', '}}') | 118 self._diff = _RunCmd(cmd)[0].replace('{', '{{').replace('}', '}}') |
| 119 | 119 |
| 120 | 120 |
| 121 class ResourceSizesDiff(BaseDiff): | 121 class ResourceSizesDiff(BaseDiff): |
| 122 _RESOURCE_SIZES_PATH = os.path.join( | 122 _RESOURCE_SIZES_PATH = os.path.join( |
| 123 _SRC_ROOT, 'build', 'android', 'resource_sizes.py') | 123 _SRC_ROOT, 'build', 'android', 'resource_sizes.py') |
| 124 | 124 |
| 125 def __init__(self, apk_name, slow_options=False): | 125 def __init__(self, apk_name, slow_options=False): |
| 126 self._apk_name = apk_name | 126 self._apk_name = apk_name |
| 127 self._slow_options = slow_options | 127 self._slow_options = slow_options |
| 128 self._diff = None # Set by |ProduceDiff()| | 128 self._diff = None # Set by |ProduceDiff()| |
| 129 super(ResourceSizesDiff, self).__init__('Resource Sizes Diff') | 129 super(ResourceSizesDiff, self).__init__('Resource Sizes Diff') |
| 130 | 130 |
| 131 @property | 131 @property |
| 132 def summary_stat(self): | 132 def summary_stat(self): |
| 133 for s in self._diff: | 133 for s in self._diff: |
| 134 if 'normalized' in s.name: | 134 if 'normalized' in s.name: |
| 135 return s | 135 return s |
| 136 return None | 136 return None |
| 137 | 137 |
| 138 def DetailedResults(self): | 138 def DetailedResults(self): |
| 139 return ['{:>+10,} {} {}'.format(value, units, name) | 139 return ['{:>+10,} {} {}'.format(value, units, name) |
| 140 for name, value, units in self._diff] | 140 for name, value, units in self._diff] |
| 141 | 141 |
| 142 def Summary(self): | 142 def Summary(self): |
| 143 return 'Normalized APK size: {:+,} {}'.format( | 143 return 'Normalized APK size: {:+,} {}'.format( |
| 144 self.summary_stat.value, self.summary_stat.units) | 144 self.summary_stat.value, self.summary_stat.units) |
| 145 | 145 |
| 146 def ProduceDiff(self, archive_dirs): | 146 def ProduceDiff(self, before_dir, after_dir): |
| 147 chartjsons = self._RunResourceSizes(archive_dirs) | 147 before = self._RunResourceSizes(before_dir) |
| 148 after = self._RunResourceSizes(after_dir) | |
| 148 diff = [] | 149 diff = [] |
| 149 with_patch = chartjsons[0]['charts'] | 150 for section, section_dict in after.iteritems(): |
| 150 without_patch = chartjsons[1]['charts'] | |
| 151 for section, section_dict in with_patch.iteritems(): | |
| 152 for subsection, v in section_dict.iteritems(): | 151 for subsection, v in section_dict.iteritems(): |
| 153 # Ignore entries when resource_sizes.py chartjson format has changed. | 152 # Ignore entries when resource_sizes.py chartjson format has changed. |
| 154 if (section not in without_patch or | 153 if (section not in before or |
| 155 subsection not in without_patch[section] or | 154 subsection not in before[section] or |
| 156 v['units'] != without_patch[section][subsection]['units']): | 155 v['units'] != before[section][subsection]['units']): |
| 157 _Print('Found differing dict structures for resource_sizes.py, ' | 156 _Print('Found differing dict structures for resource_sizes.py, ' |
| 158 'skipping {} {}', section, subsection) | 157 'skipping {} {}', section, subsection) |
| 159 else: | 158 else: |
| 160 diff.append( | 159 diff.append( |
| 161 _DiffResult( | 160 _DiffResult( |
| 162 '%s %s' % (section, subsection), | 161 '%s %s' % (section, subsection), |
| 163 v['value'] - without_patch[section][subsection]['value'], | 162 v['value'] - before[section][subsection]['value'], |
| 164 v['units'])) | 163 v['units'])) |
| 165 self._diff = sorted(diff, key=lambda x: abs(x.value), reverse=True) | 164 self._diff = sorted(diff, key=lambda x: abs(x.value), reverse=True) |
| 166 | 165 |
| 167 def _RunResourceSizes(self, archive_dirs): | 166 def _RunResourceSizes(self, archive_dir): |
| 168 chartjsons = [] | 167 apk_path = os.path.join(archive_dir, self._apk_name) |
| 169 for archive_dir in archive_dirs: | 168 chartjson_file = os.path.join(archive_dir, 'results-chart.json') |
| 170 apk_path = os.path.join(archive_dir, self._apk_name) | 169 cmd = [self._RESOURCE_SIZES_PATH, apk_path,'--output-dir', archive_dir, |
| 171 chartjson_file = os.path.join(archive_dir, 'results-chart.json') | 170 '--no-output-dir', '--chartjson'] |
| 172 cmd = [self._RESOURCE_SIZES_PATH, apk_path,'--output-dir', archive_dir, | 171 if self._slow_options: |
| 173 '--no-output-dir', '--chartjson'] | 172 cmd += ['--estimate-patch-size'] |
| 174 if self._slow_options: | 173 else: |
| 175 cmd += ['--estimate-patch-size'] | 174 cmd += ['--no-static-initializer-check'] |
| 176 else: | 175 _RunCmd(cmd) |
| 177 cmd += ['--no-static-initializer-check'] | 176 with open(chartjson_file) as f: |
| 178 _RunCmd(cmd) | 177 chartjson = json.load(f) |
| 179 with open(chartjson_file) as f: | 178 return chartjson['charts'] |
| 180 chartjsons.append(json.load(f)) | |
| 181 return chartjsons | |
| 182 | 179 |
| 183 | 180 |
| 184 class _BuildHelper(object): | 181 class _BuildHelper(object): |
| 185 """Helper class for generating and building targets.""" | 182 """Helper class for generating and building targets.""" |
| 186 def __init__(self, args): | 183 def __init__(self, args): |
| 187 self.cloud = args.cloud | 184 self.cloud = args.cloud |
| 188 self.enable_chrome_android_internal = args.enable_chrome_android_internal | 185 self.enable_chrome_android_internal = args.enable_chrome_android_internal |
| 189 self.extra_gn_args_str = '' | 186 self.extra_gn_args_str = '' |
| 190 self.max_jobs = args.max_jobs | 187 self.max_jobs = args.max_jobs |
| 191 self.max_load_average = args.max_load_average | 188 self.max_load_average = args.max_load_average |
| (...skipping 25 matching lines...) Expand all Loading... | |
| 217 elif 'monochrome' in self.target: | 214 elif 'monochrome' in self.target: |
| 218 return 'lib.unstripped/libmonochrome.so' | 215 return 'lib.unstripped/libmonochrome.so' |
| 219 else: | 216 else: |
| 220 return 'lib.unstripped/libchrome.so' | 217 return 'lib.unstripped/libchrome.so' |
| 221 | 218 |
| 222 @property | 219 @property |
| 223 def abs_main_lib_path(self): | 220 def abs_main_lib_path(self): |
| 224 return os.path.join(self.output_directory, self.main_lib_path) | 221 return os.path.join(self.output_directory, self.main_lib_path) |
| 225 | 222 |
| 226 @property | 223 @property |
| 224 def download_bucket(self): | |
| 225 return _ANDROID_BUCKET if self.IsAndroid() else _LINUX_BUCKET | |
| 226 | |
| 227 @property | |
| 228 def download_output_dir(self): | |
| 229 return 'out/Release' if self.IsAndroid() else 'full-build-linux' | |
| 230 | |
| 231 @property | |
| 227 def map_file_path(self): | 232 def map_file_path(self): |
| 228 return self.main_lib_path + '.map.gz' | 233 return self.main_lib_path + '.map.gz' |
| 229 | 234 |
| 230 @property | 235 @property |
| 231 def size_name(self): | 236 def size_name(self): |
| 232 return os.path.splitext(os.path.basename(self.main_lib_path))[0] + '.size' | 237 return os.path.splitext(os.path.basename(self.main_lib_path))[0] + '.size' |
| 233 | 238 |
| 234 def _SetDefaults(self): | 239 def _SetDefaults(self): |
| 235 has_goma_dir = os.path.exists(os.path.join(os.path.expanduser('~'), 'goma')) | 240 has_goma_dir = os.path.exists(os.path.join(os.path.expanduser('~'), 'goma')) |
| 236 self.use_goma = self.use_goma or has_goma_dir | 241 self.use_goma = self.use_goma or has_goma_dir |
| 237 self.max_load_average = (self.max_load_average or | 242 self.max_load_average = (self.max_load_average or |
| 238 str(multiprocessing.cpu_count())) | 243 str(multiprocessing.cpu_count())) |
| 239 if not self.max_jobs: | 244 if not self.max_jobs: |
| 240 self.max_jobs = '10000' if self.use_goma else '500' | 245 self.max_jobs = '10000' if self.use_goma else '500' |
| 241 | 246 |
| 242 if os.path.exists(os.path.join(os.path.dirname(_SRC_ROOT), 'src-internal')): | 247 if os.path.exists(os.path.join(os.path.dirname(_SRC_ROOT), 'src-internal')): |
| 243 self.extra_gn_args_str = ' is_chrome_branded=true' | 248 self.extra_gn_args_str = ' is_chrome_branded=true' |
| 244 else: | 249 else: |
| 245 self.extra_gn_args_str = (' exclude_unwind_tables=true ' | 250 self.extra_gn_args_str = (' exclude_unwind_tables=true ' |
| 246 'ffmpeg_branding="Chrome" proprietary_codecs=true') | 251 'ffmpeg_branding="Chrome" proprietary_codecs=true') |
| 252 self.target = self.target if self.IsAndroid() else 'chrome' | |
| 247 | 253 |
| 248 def _GenGnCmd(self): | 254 def _GenGnCmd(self): |
| 249 gn_args = 'is_official_build=true symbol_level=1' | 255 gn_args = 'is_official_build=true symbol_level=1' |
| 250 gn_args += ' use_goma=%s' % str(self.use_goma).lower() | 256 gn_args += ' use_goma=%s' % str(self.use_goma).lower() |
| 251 gn_args += ' target_os="%s"' % self.target_os | 257 gn_args += ' target_os="%s"' % self.target_os |
| 252 gn_args += (' enable_chrome_android_internal=%s' % | 258 gn_args += (' enable_chrome_android_internal=%s' % |
| 253 str(self.enable_chrome_android_internal).lower()) | 259 str(self.enable_chrome_android_internal).lower()) |
| 254 gn_args += self.extra_gn_args_str | 260 gn_args += self.extra_gn_args_str |
| 255 return ['gn', 'gen', self.output_directory, '--args=%s' % gn_args] | 261 return ['gn', 'gen', self.output_directory, '--args=%s' % gn_args] |
| 256 | 262 |
| 257 def _GenNinjaCmd(self): | 263 def _GenNinjaCmd(self): |
| 258 cmd = ['ninja', '-C', self.output_directory] | 264 cmd = ['ninja', '-C', self.output_directory] |
| 259 cmd += ['-j', self.max_jobs] if self.max_jobs else [] | 265 cmd += ['-j', self.max_jobs] if self.max_jobs else [] |
| 260 cmd += ['-l', self.max_load_average] if self.max_load_average else [] | 266 cmd += ['-l', self.max_load_average] if self.max_load_average else [] |
| 261 cmd += [self.target] | 267 cmd += [self.target] |
| 262 return cmd | 268 return cmd |
| 263 | 269 |
| 264 def Run(self): | 270 def Run(self): |
| 265 """Run GN gen/ninja build and return the process returncode.""" | 271 """Run GN gen/ninja build and return the process returncode.""" |
| 266 _Print('Building: {}.', self.target) | 272 _Print('Building: {}.', self.target) |
| 267 retcode = _RunCmd( | 273 retcode = _RunCmd( |
| 268 self._GenGnCmd(), print_stdout=True, exit_on_failure=False)[1] | 274 self._GenGnCmd(), print_stdout=True, exit_on_failure=False)[1] |
| 269 if retcode: | 275 if retcode: |
| 270 return retcode | 276 return retcode |
| 271 return _RunCmd( | 277 return _RunCmd( |
| 272 self._GenNinjaCmd(), print_stdout=True, exit_on_failure=False)[1] | 278 self._GenNinjaCmd(), print_stdout=True, exit_on_failure=False)[1] |
| 273 | 279 |
| 280 def DownloadUrl(self, rev): | |
| 281 return self.download_bucket + 'full-build-linux_%s.zip' % rev | |
| 282 | |
| 274 def IsAndroid(self): | 283 def IsAndroid(self): |
| 275 return self.target_os == 'android' | 284 return self.target_os == 'android' |
| 276 | 285 |
| 277 def IsLinux(self): | 286 def IsLinux(self): |
| 278 return self.target_os == 'linux' | 287 return self.target_os == 'linux' |
| 279 | 288 |
| 280 def IsCloud(self): | 289 def IsCloud(self): |
| 281 return self.cloud | 290 return self.cloud |
| 282 | 291 |
| 283 | 292 |
| (...skipping 41 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 325 self.build = build | 334 self.build = build |
| 326 self.build_archives = [_BuildArchive(rev, archive_dir, build, subrepo) | 335 self.build_archives = [_BuildArchive(rev, archive_dir, build, subrepo) |
| 327 for rev in revs] | 336 for rev in revs] |
| 328 self.diffs = diffs | 337 self.diffs = diffs |
| 329 self.subrepo = subrepo | 338 self.subrepo = subrepo |
| 330 self._summary_stats = [] | 339 self._summary_stats = [] |
| 331 | 340 |
| 332 def IterArchives(self): | 341 def IterArchives(self): |
| 333 return iter(self.build_archives) | 342 return iter(self.build_archives) |
| 334 | 343 |
| 335 def MaybeDiff(self, first_id, second_id): | 344 def MaybeDiff(self, before_id, after_id): |
| 336 """Perform diffs given two build archives.""" | 345 """Perform diffs given two build archives.""" |
| 337 archives = [ | 346 before = self.build_archives[before_id] |
| 338 self.build_archives[first_id], self.build_archives[second_id]] | 347 after = self.build_archives[after_id] |
| 339 diff_path = self._DiffFilePath(archives) | 348 diff_path = self._DiffFilePath(before, after) |
| 340 if not self._CanDiff(archives): | 349 if not self._CanDiff(before, after): |
| 341 _Print('Skipping diff for {} due to missing build archives.', diff_path) | 350 _Print('Skipping diff for {} due to missing build archives.', diff_path) |
| 342 return | 351 return |
| 343 | 352 |
| 344 metadata_path = self._DiffMetadataPath(archives) | 353 metadata_path = self._DiffMetadataPath(before, after) |
| 345 metadata = _GenerateMetadata( | 354 metadata = _GenerateMetadata( |
| 346 archives, self.build, metadata_path, self.subrepo) | 355 [before, after], self.build, metadata_path, self.subrepo) |
| 347 if _MetadataExists(metadata): | 356 if _MetadataExists(metadata): |
| 348 _Print('Skipping diff for {} and {}. Matching diff already exists: {}', | 357 _Print('Skipping diff for {} and {}. Matching diff already exists: {}', |
| 349 archives[0].rev, archives[1].rev, diff_path) | 358 before.rev, after.rev, diff_path) |
| 350 else: | 359 else: |
| 351 if os.path.exists(diff_path): | 360 if os.path.exists(diff_path): |
| 352 os.remove(diff_path) | 361 os.remove(diff_path) |
| 353 archive_dirs = [archives[0].dir, archives[1].dir] | |
| 354 with open(diff_path, 'a') as diff_file: | 362 with open(diff_path, 'a') as diff_file: |
| 355 for d in self.diffs: | 363 for d in self.diffs: |
| 356 d.RunDiff(diff_file, archive_dirs) | 364 d.RunDiff(diff_file, before.dir, after.dir) |
| 357 _Print('\nSee detailed diff results here: {}.', diff_path) | 365 _Print('\nSee detailed diff results here: {}.', diff_path) |
| 358 _WriteMetadata(metadata) | 366 _WriteMetadata(metadata) |
| 359 self._AddDiffSummaryStat(archives) | 367 self._AddDiffSummaryStat(before, after) |
| 360 | 368 |
| 361 def Summarize(self): | 369 def Summarize(self): |
| 362 if self._summary_stats: | 370 if self._summary_stats: |
| 363 path = os.path.join(self.archive_dir, 'last_diff_summary.txt') | 371 path = os.path.join(self.archive_dir, 'last_diff_summary.txt') |
| 364 with open(path, 'w') as f: | 372 with open(path, 'w') as f: |
| 365 stats = sorted( | 373 stats = sorted( |
| 366 self._summary_stats, key=lambda x: x[0].value, reverse=True) | 374 self._summary_stats, key=lambda x: x[0].value, reverse=True) |
| 367 _PrintAndWriteToFile(f, '\nDiff Summary') | 375 _PrintAndWriteToFile(f, '\nDiff Summary') |
| 368 for s, before, after in stats: | 376 for s, before, after in stats: |
| 369 _PrintAndWriteToFile(f, '{:>+10} {} {} for range: {}..{}', | 377 _PrintAndWriteToFile(f, '{:>+10} {} {} for range: {}..{}', |
| 370 s.value, s.units, s.name, before, after) | 378 s.value, s.units, s.name, before, after) |
| 371 | 379 |
| 372 def _AddDiffSummaryStat(self, archives): | 380 def _AddDiffSummaryStat(self, before, after): |
| 373 stat = None | 381 stat = None |
| 374 if self.build.IsAndroid(): | 382 if self.build.IsAndroid(): |
| 375 summary_diff_type = ResourceSizesDiff | 383 summary_diff_type = ResourceSizesDiff |
| 376 else: | 384 else: |
| 377 summary_diff_type = NativeDiff | 385 summary_diff_type = NativeDiff |
| 378 for d in self.diffs: | 386 for d in self.diffs: |
| 379 if isinstance(d, summary_diff_type): | 387 if isinstance(d, summary_diff_type): |
| 380 stat = d.summary_stat | 388 stat = d.summary_stat |
| 381 if stat: | 389 if stat: |
| 382 self._summary_stats.append((stat, archives[1].rev, archives[0].rev)) | 390 self._summary_stats.append((stat, before.rev, after.rev)) |
| 383 | 391 |
| 384 def _CanDiff(self, archives): | 392 def _CanDiff(self, before, after): |
| 385 return all(a.Exists() for a in archives) | 393 return before.Exists() and after.Exists() |
| 386 | 394 |
| 387 def _DiffFilePath(self, archives): | 395 def _DiffFilePath(self, before, after): |
| 388 return os.path.join(self._DiffDir(archives), 'diff_results.txt') | 396 return os.path.join(self._DiffDir(before, after), 'diff_results.txt') |
| 389 | 397 |
| 390 def _DiffMetadataPath(self, archives): | 398 def _DiffMetadataPath(self, before, after): |
| 391 return os.path.join(self._DiffDir(archives), 'metadata.txt') | 399 return os.path.join(self._DiffDir(before, after), 'metadata.txt') |
| 392 | 400 |
| 393 def _DiffDir(self, archives): | 401 def _DiffDir(self, before, after): |
| 394 archive_range = '%s..%s' % (archives[1].rev, archives[0].rev) | 402 archive_range = '%s..%s' % (before.rev, after.rev) |
| 395 diff_path = os.path.join(self.archive_dir, 'diffs', archive_range) | 403 diff_path = os.path.join(self.archive_dir, 'diffs', archive_range) |
| 396 _EnsureDirsExist(diff_path) | 404 _EnsureDirsExist(diff_path) |
| 397 return diff_path | 405 return diff_path |
| 398 | 406 |
| 399 | 407 |
| 400 def _EnsureDirsExist(path): | 408 def _EnsureDirsExist(path): |
| 401 if not os.path.exists(path): | 409 if not os.path.exists(path): |
| 402 os.makedirs(path) | 410 os.makedirs(path) |
| 403 | 411 |
| 404 | 412 |
| (...skipping 94 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 499 _GitCmd(['checkout', archive.rev], subrepo) | 507 _GitCmd(['checkout', archive.rev], subrepo) |
| 500 else: | 508 else: |
| 501 # Move to a detached state since gclient sync doesn't work with local | 509 # Move to a detached state since gclient sync doesn't work with local |
| 502 # commits on a branch. | 510 # commits on a branch. |
| 503 _GitCmd(['checkout', '--detach'], subrepo) | 511 _GitCmd(['checkout', '--detach'], subrepo) |
| 504 _GclientSyncCmd(archive.rev, subrepo) | 512 _GclientSyncCmd(archive.rev, subrepo) |
| 505 retcode = build.Run() | 513 retcode = build.Run() |
| 506 return retcode == 0 | 514 return retcode == 0 |
| 507 | 515 |
| 508 | 516 |
| 509 def _GenerateRevList(with_patch, without_patch, all_in_range, subrepo): | 517 def _GenerateRevList(rev, reference_rev, all_in_range, subrepo): |
| 510 """Normalize and optionally generate a list of commits in the given range. | 518 """Normalize and optionally generate a list of commits in the given range. |
| 511 | 519 |
| 512 Returns a list of revisions ordered from newest to oldest. | 520 Returns: |
| 521 A list of revisions ordered from oldest to newest. | |
| 513 """ | 522 """ |
| 514 cmd = ['git', '-C', subrepo, 'merge-base', '--is-ancestor', without_patch, | 523 rev_seq = '%s^..%s' % (reference_rev, rev) |
| 515 with_patch] | |
| 516 _, retcode = _RunCmd(cmd, exit_on_failure=False) | |
| 517 assert not retcode and with_patch != without_patch, ( | |
| 518 'Invalid revision arguments, rev_without_patch (%s) is newer than ' | |
| 519 'rev_with_patch (%s)' % (without_patch, with_patch)) | |
| 520 | |
| 521 rev_seq = '%s^..%s' % (without_patch, with_patch) | |
| 522 stdout = _GitCmd(['rev-list', rev_seq], subrepo) | 524 stdout = _GitCmd(['rev-list', rev_seq], subrepo) |
| 523 all_revs = stdout.splitlines() | 525 all_revs = stdout.splitlines()[::-1] |
| 524 if all_in_range: | 526 if all_in_range: |
| 525 revs = all_revs | 527 revs = all_revs |
| 526 else: | 528 else: |
| 527 revs = [all_revs[0], all_revs[-1]] | 529 revs = [all_revs[0], all_revs[-1]] |
| 528 _VerifyUserAckCommitCount(len(revs)) | 530 if len(revs) >= _COMMIT_COUNT_WARN_THRESHOLD: |
| 531 _VerifyUserAccepts( | |
| 532 'You\'ve provided a commit range that contains %d commits' % len(revs)) | |
| 529 return revs | 533 return revs |
| 530 | 534 |
| 531 | 535 |
| 532 def _VerifyUserAckCommitCount(count): | 536 def _ValidateRevs(rev, reference_rev, subrepo): |
| 533 if count >= _COMMIT_COUNT_WARN_THRESHOLD: | 537 def git_fatal(args, message): |
| 534 _Print('You\'ve provided a commit range that contains {} commits, do you ' | 538 devnull = open(os.devnull, 'wb') |
| 535 'want to proceed? [y/n]', count) | 539 retcode = subprocess.call( |
| 536 if raw_input('> ').lower() != 'y': | 540 ['git', '-C', subrepo] + args, stdout=devnull, stderr=subprocess.STDOUT) |
| 537 _global_restore_checkout_func() | 541 if retcode: |
| 538 sys.exit(1) | 542 _Die(message) |
| 543 | |
| 544 if rev == reference_rev: | |
| 545 _Die('rev and reference-rev cannot be equal') | |
| 546 no_obj_message = ('%s either doesn\'t exist or your local repo is out of ' | |
| 547 'date, try "git fetch origin master"') | |
| 548 git_fatal(['cat-file', '-e', rev], no_obj_message % rev) | |
| 549 git_fatal(['cat-file', '-e', reference_rev], no_obj_message % reference_rev) | |
| 550 git_fatal(['merge-base', '--is-ancestor', reference_rev, rev], | |
| 551 'reference-rev is newer than rev') | |
| 552 return rev, reference_rev | |
| 553 | |
| 554 | |
| 555 def _VerifyUserAccepts(message): | |
| 556 _Print(message + 'Do you want to proceed? [y/n]') | |
| 557 if raw_input('> ').lower() != 'y': | |
| 558 _global_restore_checkout_func() | |
| 559 sys.exit() | |
| 539 | 560 |
| 540 | 561 |
| 541 def _EnsureDirectoryClean(subrepo): | 562 def _EnsureDirectoryClean(subrepo): |
| 542 _Print('Checking source directory') | 563 _Print('Checking source directory') |
| 543 stdout = _GitCmd(['status', '--porcelain'], subrepo) | 564 stdout = _GitCmd(['status', '--porcelain'], subrepo) |
| 544 # Ignore untracked files. | 565 # Ignore untracked files. |
| 545 if stdout and stdout[:2] != '??': | 566 if stdout and stdout[:2] != '??': |
| 546 _Print('Failure: please ensure working directory is clean.') | 567 _Print('Failure: please ensure working directory is clean.') |
| 547 sys.exit() | 568 sys.exit() |
| 548 | 569 |
| (...skipping 17 matching lines...) Expand all Loading... | |
| 566 | 587 |
| 567 download_dir = tempfile.mkdtemp(dir=_SRC_ROOT) | 588 download_dir = tempfile.mkdtemp(dir=_SRC_ROOT) |
| 568 try: | 589 try: |
| 569 _DownloadAndArchive( | 590 _DownloadAndArchive( |
| 570 gsutil_path, archive, download_dir, build, supersize_path) | 591 gsutil_path, archive, download_dir, build, supersize_path) |
| 571 finally: | 592 finally: |
| 572 shutil.rmtree(download_dir) | 593 shutil.rmtree(download_dir) |
| 573 | 594 |
| 574 | 595 |
| 575 def _DownloadAndArchive(gsutil_path, archive, dl_dir, build, supersize_path): | 596 def _DownloadAndArchive(gsutil_path, archive, dl_dir, build, supersize_path): |
| 576 dl_file = 'full-build-linux_%s.zip' % archive.rev | 597 dl_dst = os.path.join(dl_dir, archive.rev) |
| 577 dl_url = 'gs://chrome-perf/Android Builder/%s' % dl_file | |
| 578 dl_dst = os.path.join(dl_dir, dl_file) | |
| 579 _Print('Downloading build artifacts for {}', archive.rev) | 598 _Print('Downloading build artifacts for {}', archive.rev) |
| 580 # gsutil writes stdout and stderr to stderr, so pipe stdout and stderr to | 599 # gsutil writes stdout and stderr to stderr, so pipe stdout and stderr to |
| 581 # sys.stdout. | 600 # sys.stdout. |
| 582 retcode = subprocess.call([gsutil_path, 'cp', dl_url, dl_dir], | 601 retcode = subprocess.call( |
| 583 stdout=sys.stdout, stderr=subprocess.STDOUT) | 602 [gsutil_path, 'cp', build.DownloadUrl(archive.rev), dl_dst], |
| 603 stdout=sys.stdout, stderr=subprocess.STDOUT) | |
| 584 if retcode: | 604 if retcode: |
| 585 _Die('unexpected error while downloading {}. It may no longer exist on ' | 605 _Die('unexpected error while downloading {}. It may no longer exist on ' |
| 586 'the server or it may not have been uploaded yet (check {}). ' | 606 'the server or it may not have been uploaded yet (check {}). ' |
| 587 'Otherwise, you may not have the correct access permissions.', | 607 'Otherwise, you may not have the correct access permissions.', |
| 588 dl_url, _BUILDER_URL) | 608 build.DownloadUrl(archive.rev), _BUILDER_URL) |
| 589 | 609 |
| 590 # Files needed for supersize and resource_sizes. Paths relative to out dir. | 610 # Files needed for supersize and resource_sizes. Paths relative to out dir. |
| 591 to_extract = [build.main_lib_path, build.map_file_path, 'args.gn', | 611 to_extract = [build.main_lib_path, build.map_file_path, 'args.gn'] |
| 592 'build_vars.txt', build.apk_path] | 612 if build.IsAndroid(): |
| 593 extract_dir = os.path.join(os.path.splitext(dl_dst)[0], 'unzipped') | 613 to_extract += ['build_vars.txt', build.apk_path] |
| 614 extract_dir = dl_dst + '_' + 'unzipped' | |
| 594 # Storage bucket stores entire output directory including out/Release prefix. | 615 # Storage bucket stores entire output directory including out/Release prefix. |
| 595 _Print('Extracting build artifacts') | 616 _Print('Extracting build artifacts') |
| 596 with zipfile.ZipFile(dl_dst, 'r') as z: | 617 with zipfile.ZipFile(dl_dst, 'r') as z: |
| 597 _ExtractFiles(to_extract, _CLOUD_OUT_DIR, extract_dir, z) | 618 _ExtractFiles(to_extract, build.download_output_dir, extract_dir, z) |
| 598 dl_out = os.path.join(extract_dir, _CLOUD_OUT_DIR) | 619 dl_out = os.path.join(extract_dir, build.download_output_dir) |
| 599 build.output_directory, output_directory = dl_out, build.output_directory | 620 build.output_directory, output_directory = dl_out, build.output_directory |
| 600 archive.ArchiveBuildResults(supersize_path) | 621 archive.ArchiveBuildResults(supersize_path) |
| 601 build.output_directory = output_directory | 622 build.output_directory = output_directory |
| 602 | 623 |
| 603 | 624 |
| 604 def _ExtractFiles(to_extract, prefix, dst, z): | 625 def _ExtractFiles(to_extract, prefix, dst, z): |
| 605 zip_infos = z.infolist() | 626 zip_infos = z.infolist() |
| 606 assert all(info.filename.startswith(prefix) for info in zip_infos), ( | 627 assert all(info.filename.startswith(prefix) for info in zip_infos), ( |
| 607 'Storage bucket folder structure doesn\'t start with %s' % prefix) | 628 'Storage bucket folder structure doesn\'t start with %s' % prefix) |
| 608 to_extract = [os.path.join(prefix, f) for f in to_extract] | 629 to_extract = [os.path.join(prefix, f) for f in to_extract] |
| (...skipping 84 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 693 choices=['android', 'linux'], | 714 choices=['android', 'linux'], |
| 694 help='target_os gn arg. Default: android.') | 715 help='target_os gn arg. Default: android.') |
| 695 build_group.add_argument('--output-directory', | 716 build_group.add_argument('--output-directory', |
| 696 default=_DEFAULT_OUT_DIR, | 717 default=_DEFAULT_OUT_DIR, |
| 697 help='ninja output directory. ' | 718 help='ninja output directory. ' |
| 698 'Default: %s.' % _DEFAULT_OUT_DIR) | 719 'Default: %s.' % _DEFAULT_OUT_DIR) |
| 699 build_group.add_argument('--enable-chrome-android-internal', | 720 build_group.add_argument('--enable-chrome-android-internal', |
| 700 action='store_true', | 721 action='store_true', |
| 701 help='Allow downstream targets to be built.') | 722 help='Allow downstream targets to be built.') |
| 702 build_group.add_argument('--target', | 723 build_group.add_argument('--target', |
| 703 default=_DEFAULT_TARGET, | 724 default=_DEFAULT_ANDROID_TARGET, |
| 704 help='GN APK target to build. ' | 725 help='GN APK target to build. Ignored for Linux. ' |
| 705 'Default %s.' % _DEFAULT_TARGET) | 726 'Default %s.' % _DEFAULT_ANDROID_TARGET) |
| 706 if len(sys.argv) == 1: | 727 if len(sys.argv) == 1: |
| 707 parser.print_help() | 728 parser.print_help() |
| 708 sys.exit() | 729 sys.exit() |
| 709 args = parser.parse_args() | 730 args = parser.parse_args() |
| 710 build = _BuildHelper(args) | 731 build = _BuildHelper(args) |
| 711 if build.IsCloud(): | 732 if build.IsCloud() and args.subrepo: |
| 712 if build.IsLinux(): | |
| 713 parser.error('--cloud only works for android') | |
| 714 if args.subrepo: | |
| 715 parser.error('--subrepo doesn\'t work with --cloud') | 733 parser.error('--subrepo doesn\'t work with --cloud') |
| 716 | 734 |
| 717 subrepo = args.subrepo or _SRC_ROOT | 735 subrepo = args.subrepo or _SRC_ROOT |
| 718 _EnsureDirectoryClean(subrepo) | 736 _EnsureDirectoryClean(subrepo) |
| 719 _SetRestoreFunc(subrepo) | 737 _SetRestoreFunc(subrepo) |
| 720 revs = _GenerateRevList(args.rev, | 738 if build.IsLinux(): |
| 721 args.reference_rev or args.rev + '^', | 739 _VerifyUserAccepts('Linux diffs are less useful than Android diffs, ' |
|
agrieve
2017/05/02 00:16:52
nit: Worth being more specific here I think. Maybe
estevenson
2017/05/02 16:00:10
Done.
| |
| 722 args.all, | 740 'you should probably be using android.') |
| 723 subrepo) | 741 |
| 742 rev, reference_rev = _ValidateRevs( | |
| 743 args.rev, args.reference_rev or args.rev + '^', subrepo) | |
| 744 revs = _GenerateRevList(rev, reference_rev, args.all, subrepo) | |
| 724 with _TmpCopyBinarySizeDir() as supersize_path: | 745 with _TmpCopyBinarySizeDir() as supersize_path: |
| 725 diffs = [NativeDiff(build.size_name, supersize_path)] | 746 diffs = [NativeDiff(build.size_name, supersize_path)] |
| 726 if build.IsAndroid(): | 747 if build.IsAndroid(): |
| 727 diffs += [ | 748 diffs += [ |
| 728 ResourceSizesDiff( | 749 ResourceSizesDiff( |
| 729 build.apk_name, slow_options=args.include_slow_options) | 750 build.apk_name, slow_options=args.include_slow_options) |
| 730 ] | 751 ] |
| 731 diff_mngr = _DiffArchiveManager( | 752 diff_mngr = _DiffArchiveManager( |
| 732 revs, args.archive_directory, diffs, build, subrepo) | 753 revs, args.archive_directory, diffs, build, subrepo) |
| 733 consecutive_failures = 0 | 754 consecutive_failures = 0 |
| (...skipping 19 matching lines...) Expand all Loading... | |
| 753 if i != 0: | 774 if i != 0: |
| 754 diff_mngr.MaybeDiff(i - 1, i) | 775 diff_mngr.MaybeDiff(i - 1, i) |
| 755 | 776 |
| 756 diff_mngr.Summarize() | 777 diff_mngr.Summarize() |
| 757 | 778 |
| 758 _global_restore_checkout_func() | 779 _global_restore_checkout_func() |
| 759 | 780 |
| 760 if __name__ == '__main__': | 781 if __name__ == '__main__': |
| 761 sys.exit(main()) | 782 sys.exit(main()) |
| 762 | 783 |
| OLD | NEW |