Archive Linux perf builds for manual bisect

scripts/slave/zip_build.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """ Creates a zip file in the staging dir with the result of a compile.
     It can be sent to other machines for testing.
 """
 
 import csv
 import fnmatch
 import glob
 import json
 import optparse
 import os
 import re
 import shutil
 import stat
 import sys
 import tempfile
 
 from common import chromium_utils
 from slave import build_directory
 from slave import slave_utils
 
+STRIP_LIST_LINUX = ['chrome', 'nacl_helper']

[dimu1, 2016/07/07 21:54:11]

Add the list as an argument, and pass this as a argument from the caller of
zip_build.py?

+
+
 class StagingError(Exception): pass
 
 
 def CopyDebugCRT(build_dir):
   # Copy the relevant CRT DLLs to |build_dir|. We copy DLLs from all versions
   # of VS installed to make sure we have the correct CRT version, unused DLLs
   # should not conflict with the others anyways.
   crt_dlls = glob.glob(
       'C:\\Program Files (x86)\\Microsoft Visual Studio *\\VC\\redist\\'
       'Debug_NonRedist\\x86\\Microsoft.*.DebugCRT\\*.dll')
@@ skipping 124 matching lines @@
     tmp_revision_file.close()
     chromium_utils.MakeWorldReadable(tmp_revision_file.name)
     dest_path = os.path.join(dirname,
                              chromium_utils.FULL_BUILD_REVISION_FILENAME)
     shutil.move(tmp_revision_file.name, dest_path)
     return dest_path
   except IOError:
     print 'Writing to revision file in %s failed.' % dirname
 
 
+
+def unzip_strip_zip(zip_file, zip_file_list, strip_list):

[dimu1, 2016/07/07 21:54:11]

Is it possible to strip, then zip? To avoid the zip, unzip, strip, zip step?

+  """Unzips a zip file and strips symbols from designated files and make it into a zip file.
+  Replaces the target file in place.
+
+  Args:
+    zip_file: Directory of the zip file
+    zip_file_list: List of files to be contained in the new zip file
+    strip list: List of file names to strip symbols
+
+  Returns: The path of the written file.
+  """
+  output_dir = os.path.abspath(os.path.join(zip_file, '..'))
+  tmp_dir = os.path.abspath(os.path.join(output_dir, 'tmp'))
+  unzip_dir = os.path.abspath(os.path.join(tmp_dir,
+                              os.path.basename(os.path.splitext(zip_file)[0])))
+  chromium_utils.ExtractZip(zip_file, tmp_dir)
+
+  for file in strip_list:
+    os.system('strip %s/%s' %(unzip_dir, file))
+
+  chromium_utils.RemoveFile(zip_file)
+  (zip_dir, zip_file) = chromium_utils.MakeZip(output_dir,
+                          os.path.splitext(zip_file)[0],
+                          zip_file_list,
+                          unzip_dir
+                          )
+  chromium_utils.RemoveDirectory(tmp_dir)
+  chromium_utils.RemoveDirectory(zip_dir)
+
+
+
 def MakeUnversionedArchive(build_dir, staging_dir, zip_file_list,
-                           zip_file_name):
+                           zip_file_name, strip_symbol=False):
   """Creates an unversioned full build archive.
   Returns the path of the created archive."""
   (zip_dir, zip_file) = chromium_utils.MakeZip(staging_dir,
                                                zip_file_name,
                                                zip_file_list,
                                                build_dir,
                                                raise_error=True)
+
   chromium_utils.RemoveDirectory(zip_dir)
   if not os.path.exists(zip_file):
     raise StagingError('Failed to make zip package %s' % zip_file)
+  if strip_symbol:
+    unzip_strip_zip(zip_file, zip_file_list, STRIP_LIST_LINUX)
   chromium_utils.MakeWorldReadable(zip_file)
 
   # Report the size of the zip file to help catch when it gets too big and
   # can cause bot failures from timeouts during downloads to testers.
   zip_size = os.stat(zip_file)[stat.ST_SIZE]
   print 'Zip file is %ld bytes' % zip_size
 
   return zip_file
 
 
@@ skipping 61 matching lines @@
   def __init__(self, options):
     def CommaStrParser(val):
       return [f.strip() for f in csv.reader([val]).next()]
     self.inclusions = CommaStrParser(options.include_files)
     self.exclusions = (CommaStrParser(options.exclude_files)
                        + chromium_utils.FileExclusions())
 
     self.regex_whitelist = FileRegexWhitelist(options)
     self.regex_blacklist = FileRegexBlacklist(options)
     self.exclude_unmatched = options.exclude_unmatched
+    self.exclusive_include = options.exclusive_include
 
   def __str__(self):
     return '\n  '.join([
         'Zip rules',
         'Inclusions: %s' % self.inclusions,
         'Exclusions: %s' % self.exclusions,
         "Whitelist regex: '%s'" % self.regex_whitelist,
         "Blacklist regex: '%s'" % self.regex_blacklist,
         'Zip unmatched files: %s' % (not self.exclude_unmatched)])
 
   def Match(self, filename):
     if filename in self.inclusions:
       return True
+    # Added to implement exclusive include
+    if self.exclusive_include:
+      return False
     if filename in self.exclusions:
       return False
     if re.match(self.regex_whitelist, filename):
       return True
     if re.match(self.regex_blacklist, filename):
       return False
     return not self.exclude_unmatched
 
 
 def Archive(options):
   build_dir = build_directory.GetBuildOutputDirectory(
       options.src_dir, options.cros_board)
   build_dir = os.path.abspath(os.path.join(build_dir, options.target))
-
   staging_dir = slave_utils.GetStagingDir(options.src_dir)
   chromium_utils.MakeParentDirectoriesWorldReadable(staging_dir)
 
   if not options.build_revision:
     (build_revision, webkit_revision) = slave_utils.GetBuildRevisions(
         options.src_dir, options.webkit_dir, options.revision_dir)
   else:
     build_revision = options.build_revision
     webkit_revision = options.webkit_revision
 
@@ skipping 45 matching lines @@
 
   zip_file_list = [f for f in root_files if path_filter.Match(f)]
 
   # TODO(yzshen): Once we have swarming support ready, we could use it to
   # archive run time dependencies of tests and remove this step.
   mojom_files = _MojomFiles(build_dir, ['.mojom.js', '_mojom.py'])
   print 'Include mojom files: %s' % mojom_files
   zip_file_list.extend(mojom_files)
 
   zip_file = MakeUnversionedArchive(build_dir, staging_dir, zip_file_list,
-                                    unversioned_base_name)
+                                    unversioned_base_name, strip_symbol=options.strip_symbol)
 
   zip_base, zip_ext, versioned_file = MakeVersionedArchive(
       zip_file, version_suffix, options)
 
   prune_limit = 10
   if options.build_url.startswith('gs://'):
     # Don't keep builds lying around when uploading them to google storage.
     prune_limit = 3
   PruneOldArchives(staging_dir, zip_base, zip_ext, prune_limit=prune_limit)
 
@@ skipping 26 matching lines @@
                            help='build target to archive (Debug or Release)')
   option_parser.add_option('--src-dir', default='src',
                            help='path to the top-level sources directory')
   option_parser.add_option('--build-dir', help='ignored')
   option_parser.add_option('--exclude-files', default='',
                            help='Comma separated list of files that should '
                                 'always be excluded from the zip.')
   option_parser.add_option('--include-files', default='',
                            help='Comma separated list of files that should '
                                 'always be included in the zip.')
+  option_parser.add_option('--exclusive_include', action='store_true',
+                           default=False, help='Only include the files in include-files list')
   option_parser.add_option('--master-name', help='Name of the buildbot master.')
   option_parser.add_option('--slave-name', help='Name of the buildbot slave.')
   option_parser.add_option('--build-number', type=int,
                            help='Buildbot build number.')
   option_parser.add_option('--parent-build-number', type=int,
                            help='Buildbot parent build number.')
   option_parser.add_option('--webkit-dir',
                            help='webkit directory path, relative to --src-dir')
   option_parser.add_option('--revision-dir',
                            help='Directory path that shall be used to decide '
@@ skipping 11 matching lines @@
                            help=('Optional URL to which to upload build '
                                  '(overrides build_url factory property)'))
   option_parser.add_option('--cros-board',
                            help=('If building for Chrom[e|ium]OS via the '
                                  'simple chrome workflow, the name of the '
                                  'target CROS board.'))
   option_parser.add_option('--package-dsym-files', action='store_true',
                            default=False, help='Add also dSYM files.')
   option_parser.add_option('--append-deps-patch-sha', action='store_true')
   option_parser.add_option('--gs-acl')
+  option_parser.add_option('--strip_symbol', action='store_true',
+                           default=False, help='Strip symbols from chrome executable.')
   option_parser.add_option('--json-urls',
                            help=('Path to json file containing uploaded '
                                  'archive urls. If this is omitted then '
                                  'the urls will be emitted as buildbot '
                                  'annotations.'))
   chromium_utils.AddPropertiesOptions(option_parser)
 
   options, args = option_parser.parse_args(argv)
 
   if not options.master_name:
@@ skipping 27 matching lines @@
       json.dump(urls, json_file)
   else:  # we need to print buildbot annotations
     if 'storage_url' in urls:
       print '@@@STEP_LINK@download@%s@@@' % urls['storage_url']
     if 'zip_url' in urls:
       print '@@@SET_BUILD_PROPERTY@build_archive_url@"%s"@@@' %  urls['zip_url']
   return 0
 
 if '__main__' == __name__:
   sys.exit(main(sys.argv))