Archive Linux perf builds for manual bisect

scripts/slave/zip_build.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """ Creates a zip file in the staging dir with the result of a compile.
     It can be sent to other machines for testing.
 """
 
 import csv
@@ skipping 148 matching lines @@
     tmp_revision_file.write('%s' % build_revision)
     tmp_revision_file.close()
     chromium_utils.MakeWorldReadable(tmp_revision_file.name)
     dest_path = os.path.join(dirname,
                              chromium_utils.FULL_BUILD_REVISION_FILENAME)
     shutil.move(tmp_revision_file.name, dest_path)
     return dest_path
   except IOError:
     print 'Writing to revision file in %s failed.' % dirname
 
-
 def MakeUnversionedArchive(build_dir, staging_dir, zip_file_list,
-                           zip_file_name):
+                           zip_file_name, strip_symbol=False):
   """Creates an unversioned full build archive.
   Returns the path of the created archive."""
+  strip_files = None
+  if strip_symbol:
+    strip_files = ['chrome', 'nacl_helper']

[stgao, 2016/07/11 19:04:27]

Not sure where this file list should be, but this seems not the best place for
it. It should be close to the logic which determine which files should be
stripped.

[miimnk, 2016/07/11 22:08:48]

There are only two binary files with a significant size ('chrome',
'nacl_helper'). Should I still implement a logic to strip all binary files above
a certain file size? 

[stgao, 2016/07/12 20:40:40]

Stripping big files are good for now.

My comment is more of that the file list should not be added here, because this
script is only for archiving. It should not determine which files to be stripped
with a passed in True/False flag.

Is there any alternative location that the file list could be in?

Maybe archive/perf_test_files.py above?

[miimnk, 2016/07/13 01:12:48]

I stored the strip list in archive/perf_test_files.py and passed to zip_build.py
as a comma separated string of the files that needs to be stripped. Now, it
passes in the file list instead of a True/False flag. 

   (zip_dir, zip_file) = chromium_utils.MakeZip(staging_dir,
                                                zip_file_name,
                                                zip_file_list,
                                                build_dir,
-                                               raise_error=True)
+                                               raise_error=True,
+                                               strip_files=strip_files)
+
   chromium_utils.RemoveDirectory(zip_dir)
   if not os.path.exists(zip_file):
     raise StagingError('Failed to make zip package %s' % zip_file)
   chromium_utils.MakeWorldReadable(zip_file)
 
   # Report the size of the zip file to help catch when it gets too big and
   # can cause bot failures from timeouts during downloads to testers.
   zip_size = os.stat(zip_file)[stat.ST_SIZE]
   print 'Zip file is %ld bytes' % zip_size
 
@@ skipping 64 matching lines @@
   def __init__(self, options):
     def CommaStrParser(val):
       return [f.strip() for f in csv.reader([val]).next()]
     self.inclusions = CommaStrParser(options.include_files)
     self.exclusions = (CommaStrParser(options.exclude_files)
                        + chromium_utils.FileExclusions())
 
     self.regex_whitelist = FileRegexWhitelist(options)
     self.regex_blacklist = FileRegexBlacklist(options)
     self.exclude_unmatched = options.exclude_unmatched
+    self.exclusive_include = options.exclusive_include

[stgao, 2016/07/11 19:04:27]

What's this for? The naming seems a bit confusing to me.

[miimnk, 2016/07/11 22:08:48]

Currently, it is not possible to only include items in the 'include' list. It
goes through a pregenerated regex_whitelist to match other files too. So, I
added a logic to skip the regex_whitelist part. Other names I can think of are
'exclude_unincluded', and 'ignore_regex'. This is different from
'exclude_unmatched' because exclude_unmatched still has to go through including
files in whitelist. 

 
   def __str__(self):
     return '\n  '.join([
         'Zip rules',
         'Inclusions: %s' % self.inclusions,
         'Exclusions: %s' % self.exclusions,
         "Whitelist regex: '%s'" % self.regex_whitelist,
         "Blacklist regex: '%s'" % self.regex_blacklist,
         'Zip unmatched files: %s' % (not self.exclude_unmatched)])
 
   def Match(self, filename):
     if filename in self.inclusions:
       return True
+    # Added to implement exclusive include
+    if self.exclusive_include:
+      return False
     if filename in self.exclusions:
       return False
     if re.match(self.regex_whitelist, filename):
       return True
     if re.match(self.regex_blacklist, filename):
       return False
     return not self.exclude_unmatched
 
 
 def Archive(options):
   build_dir = build_directory.GetBuildOutputDirectory(
       options.src_dir, options.cros_board)
   build_dir = os.path.abspath(os.path.join(build_dir, options.target))
-
   staging_dir = slave_utils.GetStagingDir(options.src_dir)
   chromium_utils.MakeParentDirectoriesWorldReadable(staging_dir)
 
   if not options.build_revision:
     (build_revision, webkit_revision) = slave_utils.GetBuildRevisions(
         options.src_dir, options.webkit_dir, options.revision_dir)
   else:
     build_revision = options.build_revision
     webkit_revision = options.webkit_revision
 
@@ skipping 45 matching lines @@
 
   zip_file_list = [f for f in root_files if path_filter.Match(f)]
 
   # TODO(yzshen): Once we have swarming support ready, we could use it to
   # archive run time dependencies of tests and remove this step.
   mojom_files = _MojomFiles(build_dir, ['.mojom.js', '_mojom.py'])
   print 'Include mojom files: %s' % mojom_files
   zip_file_list.extend(mojom_files)
 
   zip_file = MakeUnversionedArchive(build_dir, staging_dir, zip_file_list,
-                                    unversioned_base_name)
+                                    unversioned_base_name, strip_symbol=options.strip_symbol)
 
   zip_base, zip_ext, versioned_file = MakeVersionedArchive(
       zip_file, version_suffix, options)
 
   prune_limit = 10
   if options.build_url.startswith('gs://'):
     # Don't keep builds lying around when uploading them to google storage.
     prune_limit = 3
   PruneOldArchives(staging_dir, zip_base, zip_ext, prune_limit=prune_limit)
 
@@ skipping 26 matching lines @@
                            help='build target to archive (Debug or Release)')
   option_parser.add_option('--src-dir', default='src',
                            help='path to the top-level sources directory')
   option_parser.add_option('--build-dir', help='ignored')
   option_parser.add_option('--exclude-files', default='',
                            help='Comma separated list of files that should '
                                 'always be excluded from the zip.')
   option_parser.add_option('--include-files', default='',
                            help='Comma separated list of files that should '
                                 'always be included in the zip.')
+  option_parser.add_option('--exclusive_include', action='store_true',
+                           default=False, help='Only include the files in include-files list')
   option_parser.add_option('--master-name', help='Name of the buildbot master.')
   option_parser.add_option('--slave-name', help='Name of the buildbot slave.')
   option_parser.add_option('--build-number', type=int,
                            help='Buildbot build number.')
   option_parser.add_option('--parent-build-number', type=int,
                            help='Buildbot parent build number.')
   option_parser.add_option('--webkit-dir',
                            help='webkit directory path, relative to --src-dir')
   option_parser.add_option('--revision-dir',
                            help='Directory path that shall be used to decide '
@@ skipping 11 matching lines @@
                            help=('Optional URL to which to upload build '
                                  '(overrides build_url factory property)'))
   option_parser.add_option('--cros-board',
                            help=('If building for Chrom[e|ium]OS via the '
                                  'simple chrome workflow, the name of the '
                                  'target CROS board.'))
   option_parser.add_option('--package-dsym-files', action='store_true',
                            default=False, help='Add also dSYM files.')
   option_parser.add_option('--append-deps-patch-sha', action='store_true')
   option_parser.add_option('--gs-acl')
+  option_parser.add_option('--strip_symbol', action='store_true',
+                           default=False, help='Strip symbols from chrome executable.')
   option_parser.add_option('--json-urls',
                            help=('Path to json file containing uploaded '
                                  'archive urls. If this is omitted then '
                                  'the urls will be emitted as buildbot '
                                  'annotations.'))
   chromium_utils.AddPropertiesOptions(option_parser)
 
   options, args = option_parser.parse_args(argv)
 
   if not options.master_name:
@@ skipping 27 matching lines @@
       json.dump(urls, json_file)
   else:  # we need to print buildbot annotations
     if 'storage_url' in urls:
       print '@@@STEP_LINK@download@%s@@@' % urls['storage_url']
     if 'zip_url' in urls:
       print '@@@SET_BUILD_PROPERTY@build_archive_url@"%s"@@@' %  urls['zip_url']
   return 0
 
 if '__main__' == __name__:
   sys.exit(main(sys.argv))