Archive Linux perf builds for manual bisect

scripts/slave/zip_build.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """ Creates a zip file in the staging dir with the result of a compile.
     It can be sent to other machines for testing.
 """
 
 import csv
@@ skipping 148 matching lines @@
     tmp_revision_file.write('%s' % build_revision)
     tmp_revision_file.close()
     chromium_utils.MakeWorldReadable(tmp_revision_file.name)
     dest_path = os.path.join(dirname,
                              chromium_utils.FULL_BUILD_REVISION_FILENAME)
     shutil.move(tmp_revision_file.name, dest_path)
     return dest_path
   except IOError:
     print 'Writing to revision file in %s failed.' % dirname
 
-
 def MakeUnversionedArchive(build_dir, staging_dir, zip_file_list,
-                           zip_file_name):
+                           zip_file_name, strip_files=None):
   """Creates an unversioned full build archive.
   Returns the path of the created archive."""
+  if strip_files:
+    strip_files = [f.strip() for f in csv.reader([strip_files]).next()]

[dtu, 2016/08/05 21:06:19]

might be more readable to do strip_files.split(',') instead of
csv.reader([strip_files]).next()
? Up to you

[miimnk, 2016/08/08 22:41:32]

Done.

   (zip_dir, zip_file) = chromium_utils.MakeZip(staging_dir,
                                                zip_file_name,
                                                zip_file_list,
                                                build_dir,
-                                               raise_error=True)
+                                               raise_error=True,
+                                               strip_files=strip_files)
+
   chromium_utils.RemoveDirectory(zip_dir)
   if not os.path.exists(zip_file):
     raise StagingError('Failed to make zip package %s' % zip_file)
   chromium_utils.MakeWorldReadable(zip_file)
 
   # Report the size of the zip file to help catch when it gets too big and
   # can cause bot failures from timeouts during downloads to testers.
   zip_size = os.stat(zip_file)[stat.ST_SIZE]
   print 'Zip file is %ld bytes' % zip_size
 
@@ skipping 64 matching lines @@
   def __init__(self, options):
     def CommaStrParser(val):
       return [f.strip() for f in csv.reader([val]).next()]
     self.inclusions = CommaStrParser(options.include_files)
     self.exclusions = (CommaStrParser(options.exclude_files)
                        + chromium_utils.FileExclusions())
 
     self.regex_whitelist = FileRegexWhitelist(options)
     self.regex_blacklist = FileRegexBlacklist(options)
     self.exclude_unmatched = options.exclude_unmatched
+    self.ignore_regex = options.ignore_regex
 
   def __str__(self):
     return '\n  '.join([
         'Zip rules',
         'Inclusions: %s' % self.inclusions,
         'Exclusions: %s' % self.exclusions,
         "Whitelist regex: '%s'" % self.regex_whitelist,
         "Blacklist regex: '%s'" % self.regex_blacklist,
-        'Zip unmatched files: %s' % (not self.exclude_unmatched)])
+        'Zip unmatched files: %s' % (not self.exclude_unmatched),
+        'Ignore regex matches: %s' % (self.ignore_regex)])

[dtu, 2016/08/05 21:06:19]

style nit: unnecessary parentheses

[miimnk, 2016/08/08 22:41:32]

Done.

+
 
   def Match(self, filename):
     if filename in self.inclusions:
       return True
     if filename in self.exclusions:
       return False
+    if self.ignore_regex:
+      return False
     if re.match(self.regex_whitelist, filename):
       return True
     if re.match(self.regex_blacklist, filename):
       return False
     return not self.exclude_unmatched
 
 
 def Archive(options):
   build_dir = build_directory.GetBuildOutputDirectory(
       options.src_dir, options.cros_board)
   build_dir = os.path.abspath(os.path.join(build_dir, options.target))
-
   staging_dir = slave_utils.GetStagingDir(options.src_dir)
   chromium_utils.MakeParentDirectoriesWorldReadable(staging_dir)
-
   if not options.build_revision:
     (build_revision, webkit_revision) = slave_utils.GetBuildRevisions(
         options.src_dir, options.webkit_dir, options.revision_dir)
   else:
     build_revision = options.build_revision
     webkit_revision = options.webkit_revision
 
   unversioned_base_name, version_suffix = slave_utils.GetZipFileNames(
       options.master_name,
       options.build_number,
@@ skipping 42 matching lines @@
 
   zip_file_list = [f for f in root_files if path_filter.Match(f)]
 
   # TODO(yzshen): Once we have swarming support ready, we could use it to
   # archive run time dependencies of tests and remove this step.
   mojom_files = _MojomFiles(build_dir, ['.mojom.js', '_mojom.py'])
   print 'Include mojom files: %s' % mojom_files
   zip_file_list.extend(mojom_files)
 
   zip_file = MakeUnversionedArchive(build_dir, staging_dir, zip_file_list,
-                                    unversioned_base_name)
+                                    unversioned_base_name, 
+                                    strip_files=options.strip_files)
 
   zip_base, zip_ext, versioned_file = MakeVersionedArchive(
       zip_file, version_suffix, options)
 
   prune_limit = 10
   if options.build_url.startswith('gs://'):
     # Don't keep builds lying around when uploading them to google storage.
     prune_limit = 3
   PruneOldArchives(staging_dir, zip_base, zip_ext, prune_limit=prune_limit)
 
@@ skipping 26 matching lines @@
                            help='build target to archive (Debug or Release)')
   option_parser.add_option('--src-dir', default='src',
                            help='path to the top-level sources directory')
   option_parser.add_option('--build-dir', help='ignored')
   option_parser.add_option('--exclude-files', default='',
                            help='Comma separated list of files that should '
                                 'always be excluded from the zip.')
   option_parser.add_option('--include-files', default='',
                            help='Comma separated list of files that should '
                                 'always be included in the zip.')
+  option_parser.add_option('--ignore_regex', action='store_true',

[dtu, 2016/08/05 21:06:19]

Command-line args should be separated by - instead of _

[miimnk, 2016/08/08 22:41:30]

Done.

+                           default=False, help='Ignores regex matches')
   option_parser.add_option('--master-name', help='Name of the buildbot master.')
   option_parser.add_option('--slave-name', help='Name of the buildbot slave.')
   option_parser.add_option('--build-number', type=int,
                            help='Buildbot build number.')
   option_parser.add_option('--parent-build-number', type=int,
                            help='Buildbot parent build number.')
   option_parser.add_option('--webkit-dir',
                            help='webkit directory path, relative to --src-dir')
   option_parser.add_option('--revision-dir',
                            help='Directory path that shall be used to decide '
@@ skipping 11 matching lines @@
                            help=('Optional URL to which to upload build '
                                  '(overrides build_url factory property)'))
   option_parser.add_option('--cros-board',
                            help=('If building for Chrom[e|ium]OS via the '
                                  'simple chrome workflow, the name of the '
                                  'target CROS board.'))
   option_parser.add_option('--package-dsym-files', action='store_true',
                            default=False, help='Add also dSYM files.')
   option_parser.add_option('--append-deps-patch-sha', action='store_true')
   option_parser.add_option('--gs-acl')
+  option_parser.add_option('--strip_files', default='',
+                           help='Comma separated list of files that should '
+                                'be stripped of symbols in the zip.')

[dtu, 2016/08/05 21:06:19]

I'd prefer if you convert the argument to a list in this function, rather than
in MakeUnversionedArchive. It's conceptually part of the argument parsing.

[miimnk, 2016/08/08 22:41:31]

Done.

   option_parser.add_option('--json-urls',
                            help=('Path to json file containing uploaded '
                                  'archive urls. If this is omitted then '
                                  'the urls will be emitted as buildbot '
                                  'annotations.'))
   chromium_utils.AddPropertiesOptions(option_parser)
 
   options, args = option_parser.parse_args(argv)
 
   if not options.master_name:
@@ skipping 27 matching lines @@
       json.dump(urls, json_file)
   else:  # we need to print buildbot annotations
     if 'storage_url' in urls:
       print '@@@STEP_LINK@download@%s@@@' % urls['storage_url']
     if 'zip_url' in urls:
       print '@@@SET_BUILD_PROPERTY@build_archive_url@"%s"@@@' %  urls['zip_url']
   return 0
 
 if '__main__' == __name__:
   sys.exit(main(sys.argv))