Enable saving the same file in multiple archives

scripts/common/archive_utils.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Set of common operations/utilities for build archiving."""
 
 import glob
 import os
 import platform
 import re
@@ skipping 60 matching lines @@
   symbol_staging_url = 'http://clients2.google.com/cr/staging_symbol'
 
 
 class FilesCfgParser(object):
   """Class to process a FILES.cfg style listing of build files."""
 
   def __init__(self, files_file, buildtype, arch):
     self._buildtype = buildtype
     self._arch = arch
     self._files_cfg = self._ParseFilesCfg(files_file)
-    self.files_dict = self._FilterFilesCfg()
+    self._files_list = self._FilterFilesCfg()
 
   def _SetArch(self, value):
-    """Set build arch and reset files_dict to reflect new build criteria."""
+    """Set build arch and reset files_list to reflect new build criteria."""
     self._arch = value
-    self.files_dict.clear()
-    self.files_dict.update(self._FilterFilesCfg())
+    del self._files_list[:]
+    self._files_list.extend(self._FilterFilesCfg())
 
   arch = property(fset=_SetArch)
 
   def _SetBuildType(self, value):
-    """Set build type and reset files_dict to reflect new build criteria."""
+    """Set build type and reset files_list to reflect new build criteria."""
     self._buildtype = value
-    self.files_dict.clear()
-    self.files_dict.update(self._FilterFilesCfg())
+    del self._files_list[:]
+    self._files_list.extend(self._FilterFilesCfg())
 
   buildtype = property(fset=_SetBuildType)
 
   def _FilterFilesCfg(self):
-    """Return a dict of file items that match the current build criteria."""
-    files_dict = {}
+    """Return a list of file items that match the current build criteria."""
+    files_list = []
     for fileobj in self._files_cfg:
       if self._buildtype not in fileobj['buildtype']:
         continue
       if not fileobj.get('arch') or self._arch in fileobj['arch']:
-        files_dict[fileobj['filename']] = fileobj
-    return files_dict
+        files_list.append(fileobj)
+    return files_list
 
   @staticmethod
   def _ParseFilesCfg(files_file):
     """Return the dictionary of archive file info read from the given file."""
     if not os.path.exists(files_file):
       raise StagingError('Files list does not exist (%s).' % files_file)
     exec_globals = {'__builtins__': None}
 
     execfile(files_file, exec_globals)
     return exec_globals['FILES']
 
   @classmethod
   def IsDirectArchive(cls, archive_list):
     """Determine if the given archive list should be archived as-is.
 
       An archive list (from ParseArchiveLists) is archived as-is (not added to
       another archive file) iff:
       - There list contains a single file, and
       - That file has the 'direct_archive' flag or its 'archive' name matches
         its 'filename' (an implied 'direct_archive').
     """
     fileobj = archive_list[0]
     return (len(archive_list) == 1 and
             (fileobj['filename'] == fileobj['archive'] or
              fileobj.get('direct_archive')))
 
   def IsOptional(self, filename):
     """Determine if the given filename is marked optional for this config."""
-    return (self.files_dict.get(filename) and self._buildtype in
-            self.files_dict[filename].get('optional', []))
+    has_file = False
+    for fileobj in self._files_list:
+      if fileobj['filename'] == filename:
+        has_file = True
+        if self._buildtype not in fileobj['optional']:
+          return False
+    return has_file

[Michael Moss, 2016/10/11 18:23:32]

Maybe simplify slightly by getting rid of the 'has_file' and just doing:

  for fileobj in self._files_list:
    if fileobj['filename'] == filename and self._buildtype in
fileobj['optional']:
        return True
  return False

[Michael Moss, 2016/10/11 18:47:06]

Ugh, my logic was backwards, ignore. We want it to be required if any instance
of the file is required.

 
   def ParseGroup(self, filegroup):
     """Return the list of filenames in the given group (e.g. "symbols")."""
-    return [fileobj['filename'] for fileobj in self.files_dict.itervalues()
+    return [fileobj['filename'] for fileobj in self._files_list
         if (fileobj.get('filegroup') and filegroup in fileobj.get('filegroup'))
     ]
 
   def ParseArchiveLists(self):
     """Generate a dict of all the file items in all archives."""
     archive_lists = {}
-    for fileobj in self.files_dict.itervalues():
+    for fileobj in self._files_list:
       if fileobj.get('archive'):
         archive_lists.setdefault(fileobj['archive'], []).append(fileobj)
     return archive_lists
 
   def ParseLegacyList(self):
     """Return the list of 'default' filenames.
 
     Default files are either tagged as "default" filegroup or they have no
     filegroup (i.e. legacy entries from before the filegroup field was added.)
     """
     files_list = [
-        fileobj['filename'] for fileobj in self.files_dict.itervalues()
+        fileobj['filename'] for fileobj in self._files_list
         if (not fileobj.get('archive') and
             (not fileobj.get('filegroup') or 'default' in
              fileobj.get('filegroup')))
     ]
     return files_list
 
-

[Michael Moss, 2016/10/11 18:23:32]

Nit: Keep 2 lines before top-level functions/classes.

[chenwilliam, 2016/10/11 19:08:15]

Done.

 def ParseFilesList(files_file, buildtype, arch):
   """DEPRECATED: Determine the list of archive files for a given release.
 
   NOTE: This can be removed after 20.x goes stable (or after scripts/common/
   gets versioned on the official builders like site_config is).
   """
   fparser = FilesCfgParser(files_file, buildtype, arch)
   return fparser.ParseLegacyList()
 
 
@@ skipping 142 matching lines @@
                                                raise_error=not allow_missing)
   if not os.path.exists(zip_file):
     raise StagingError('Failed to make zip package %s' % zip_file)
 
   if os.path.basename(zip_file) != archive_name:
     orig_zip = zip_file
     zip_file = os.path.join(os.path.dirname(orig_zip), archive_name)
     print 'Renaming archive: "%s" -> "%s"' % (orig_zip, zip_file)
     chromium_utils.MoveFile(orig_zip, zip_file)
   return (zip_dir, zip_file)