Reworked devserver so that update images generated are cached in directories named after

autoupdate.py

Index: autoupdate.py
diff --git a/autoupdate.py b/autoupdate.py
index 58455dc453b7085437e91b0fcda595740436dd8e..973087e6006564178a6200611fd2a0316f8d3170 100644
--- a/autoupdate.py
+++ b/autoupdate.py
@@ -4,14 +4,13 @@
 
 from buildutil import BuildObject
 from xml.dom import minidom
+
 import cherrypy
 import os
 import shutil
-import subprocess
-import tempfile
+import socket
 import time
 
-
 def _LogMessage(message):
   cherrypy.log(message, 'UPDATE')
 
@@ -49,7 +48,6 @@ class Autoupdate(BuildObject):
     self.src_image = src_image
     self.vm = vm
     self.board = board
-    self.crosutils = os.path.join(os.path.dirname(__file__), '../../scripts')
 
   def _GetSecondsSinceMidnight(self):
     """Returns the seconds since midnight as a decimal value."""
@@ -86,6 +84,21 @@ class Autoupdate(BuildObject):
       return int(latest_tokens[i]) > int(client_tokens[i])
     return False
 
+  def _UnpackStatefulPartition(self, image_path, stateful_file):
+    """Given an image, unpacks its stateful partition to stateful_file."""
+    image_dir = os.path.dirname(image_path)
+    image_file = os.path.basename(image_path)
+
+    get_offset = '$(cgpt show -b -i 1 %s)' % image_file
+    get_size = '$(cgpt show -s -i 1 %s)' % image_file
+    unpack_command = (
+        'cd %s && '
+        'dd if=%s of=%s bs=512 skip=%s count=%s' % (image_dir, image_file,
+                                                    stateful_file, get_offset,
+                                                    get_size))
+    _LogMessage(unpack_command)
+    return os.system(unpack_command) == 0
+
   def _UnpackZip(self, image_dir):
     """Unpacks an image.zip into a given directory."""
     image = os.path.join(image_dir, self._GetImageName())
@@ -106,22 +119,6 @@ class Autoupdate(BuildObject):
       image_name = 'chromiumos_image.bin'
     return image_name
 
-  def _IsImageNewerThanCached(self, image_path, cached_file_path):
-    """Returns true if the image is newer than the cached image."""
-    if os.path.exists(cached_file_path) and os.path.exists(image_path):
-      _LogMessage('Usable cached image found at %s.' % cached_file_path)
-      return os.path.getmtime(image_path) > os.path.getmtime(cached_file_path)
-    elif not os.path.exists(cached_file_path) and not os.path.exists(image_path):
-      raise Exception('Image does not exist and cached image missing')
-    else:
-      # Only one is missing, figure out which one.
-      if os.path.exists(image_path):
-        _LogMessage('No cached image found - image generation required.')
-        return True
-      else:
-        _LogMessage('Cached image found to serve at %s.' % cached_file_path)
-        return False
-
   def _GetSize(self, update_path):
     """Returns the size of the file given."""
     return os.path.getsize(update_path)
@@ -152,6 +149,11 @@ class Autoupdate(BuildObject):
            update_path)
     return os.popen(cmd).read().rstrip()
 
+  def _GetMd5(self, update_path):
+    """Returns the md5 checksum of the file given."""
+    cmd = ("md5sum %s | awk '{print $1}'" % update_path)
+    return os.popen(cmd).read().rstrip()
+
   def GetUpdatePayload(self, hash, sha256, size, url, is_delta_format):
     """Returns a payload to the client corresponding to a new update.
 
@@ -198,7 +200,7 @@ class Autoupdate(BuildObject):
     """
     return payload % (self._GetSecondsSinceMidnight(), self.app_id)
 
-  def GenerateUpdateFile(self, image_path):
+  def GenerateUpdateFile(self, src_image, image_path, static_image_dir):
     """Generates an update gz given a full path to an image.
 
     Args:
@@ -207,7 +209,7 @@ class Autoupdate(BuildObject):
       Path to created update_payload or None on error.
     """
     image_dir = os.path.dirname(image_path)
-    update_path = os.path.join(image_dir, 'update.gz')
+    update_path = os.path.join(static_image_dir, 'update.gz')

[petkov, 2010/11/13 00:55:18]

Since you're updating this stuff, maybe we should rename "update.gz" to
something that doesn't incorrectly imply this is a gzip file. Maybe just
update.bin or something like that.

[dgarrett, 2010/11/15 21:50:33]

This sounds good to me, but I'm not certain what external code may depend on
that name.

On 2010/11/13 00:55:18, petkov wrote:

     patch_kernel_flag = '--patch_kernel'
     _LogMessage('Generating update image %s' % update_path)
 
@@ -219,7 +221,7 @@ class Autoupdate(BuildObject):
         '%s/cros_generate_update_payload --image="%s" --output="%s" '
         '%s --noold_style --src_image="%s"' % (
             self.scripts_dir, image_path, update_path, patch_kernel_flag,
-            self.src_image))
+            src_image))
     _LogMessage(mkupdate_command)
     if os.system(mkupdate_command) != 0:
       _LogMessage('Failed to create base update file')
@@ -227,75 +229,133 @@ class Autoupdate(BuildObject):
 
     return update_path
 
-  def GenerateStatefulFile(self, image_path):
-    """Generates a stateful update given a full path to an image.
+  def GenerateStatefulFile(self, image_path, static_image_dir):

[petkov, 2010/11/13 00:55:18]

Why are you not calling ./cros_generate_stateful_update_payload any more? There
was intentional effort to channel stateful update generation through a common
script because there're other clients of this -- test lab, network testing lab,
etc.

[dgarrett, 2010/11/15 21:50:33]

Because I was reading the wrong side of a merge as the external changes. I'll
put it back the way it should be.

On 2010/11/13 00:55:18, petkov wrote:

+    """Generates a stateful update gz given a full path to an image.
 
     Args:
       image_path: Full path to image.
     Returns:
-      Path to created stateful update_payload.
-    Raises:
-      A subprocess exception if the update generator fails to generate a
-      stateful payload.
+      Path to created stateful update_payload or None on error.
     """
-    work_dir = os.path.dirname(image_path)
-    output_gz = os.path.join(work_dir, 'stateful.tgz')
-    subprocess.check_call(
-        ['%s/cros_generate_stateful_update_payload' % self.crosutils,
-         '--image=%s' % image_path,
-         '--output_dir=%s' % work_dir,
-        ])
-    return output_gz
-
-  def MoveImagesToStaticDir(self, update_path, stateful_update_path,
-                            static_image_dir):
-    """Moves gz files from their directories to serving directories.
+    stateful_partition_path = os.path.join(os.path.dirname(image_path),
+                                           'stateful.image')
 
-    Args:
-      update_path: full path to main update gz.
-      stateful_update_path: full path to stateful partition gz.
-      static_image_dir: where to put files.
-    Returns:
-      Returns True if the files were moved over successfully.
-    """
-    try:
-      shutil.copy(update_path, static_image_dir)
-      shutil.copy(stateful_update_path, static_image_dir)
-      os.remove(update_path)
-      os.remove(stateful_update_path)
-    except Exception:
-      _LogMessage('Failed to move %s and %s to %s' % (update_path,
-                                                    stateful_update_path,
-                                                    static_image_dir))
-      return False
+    output_file = os.path.join(static_image_dir,
+                               'stateful.image.gz')
+
+    # Unpack to get stateful partition.
+    if self._UnpackStatefulPartition(image_path, stateful_partition_path):
+      cmd = 'gzip --stdout %s > %s' % (stateful_partition_path, output_file)
+      if os.system(cmd) == 0:
+        _LogMessage('Successfully generated %s' % output_file)
+        return output_file
 
-    return True
+    _LogMessage('Failed to create stateful update file')
+    return None
 
-  def GenerateUpdateImage(self, image_path, move_to_static_dir=False,
-                          static_image_dir=None):
-    """Generates an update payload based on the given image_path.
+  def FindCachedUpdateImageSubDir(self, src_image, dest_image):
+    """Given one, or two images for an update, this finds which

[petkov, 2010/11/13 00:55:18]

I think Python style is one summary line. Then followed by a blank line and a
detailed description if necessary.

+       cache directory should hold the udpate files, even if they don't exist yet.

[petkov, 2010/11/13 00:55:18]

80 chars

+       The directory will be inside static_image_dir, and of the form:
+
+       Non-delta updates:
+         cache/12345678
+
+       Delta updates:
+         cache/12345678_12345678
+       """
+
+    # If there is no src, we only have an image file, check image for changes
+    if not src_image:
+      return os.path.join('cache', self._GetMd5(dest_image))
+
+    # If we have src and dest, we are a delta, and check both for changes
+    return os.path.join('cache',
+                        "%s_%s" % (self._GetMd5(src_image),
+                                   self._GetMd5(dest_image)))
+
+  def GenerateUpdateImage(self, src_image, image_path, static_image_dir):
+    """Force generates an update payload based on the given image_path.
 
     Args:
       image_path: full path to the image.
       move_to_static_dir: Moves the files from their dir to the static dir.
       static_image_dir: the directory to move images to after generating.
     Returns:
-      True if the update payload was created successfully.
+      update filename (not directory) on success, or None
     """
+
+    update_file = None
+    stateful_update_file = None
+
+    # Actually do the generation
     _LogMessage('Generating update for image %s' % image_path)
-    update_path = self.GenerateUpdateFile(image_path)
-    if update_path:
-      stateful_update_path = self.GenerateStatefulFile(image_path)
-      if move_to_static_dir:
-        return self.MoveImagesToStaticDir(update_path, stateful_update_path,
+    update_file = self.GenerateUpdateFile(src_image,
+                                          image_path,
                                           static_image_dir)
-        return True
+
+    if update_file:
+      stateful_update_file = self.GenerateStatefulFile(image_path,
+                                                       static_image_dir)
+
+    if update_file and stateful_update_file:
+      return os.path.basename(update_file)
 
     _LogMessage('Failed to generate update')
-    return False
+
+    # Cleanup incomplete files, if they exist
+    if update_file and os.path.exists(update_file):
+      os.remove(update_file)
+
+    if stateful_update_file and os.path.exists(stateful_update_file):
+      os.remove(stateful_update_file)
+
+    return None
+
+  def GenerateUpdateImageWithCache(self, image_path, static_image_dir):
+    """Force generates an update payload based on the given image_path.
+
+    Args:
+      image_path: full path to the image.
+      move_to_static_dir: Moves the files from their dir to the static dir.
+      static_image_dir: the directory to move images to after generating.
+    Returns:
+      update filename (not directory) relative to static_image_dir on success,
+        or None
+    """
+
+    _LogMessage('Generating update for src %s image %s' % (self.src_image,
+                                                           image_path))
+
+    # Which sub_dir of static_image_dir should hold our cached update image
+    cache_sub_dir = self.FindCachedUpdateImageSubDir(self.src_image, image_path)
+    _LogMessage('Caching in sub_dir "%s"' % cache_sub_dir)
+
+    # Check for a cached image to use
+    if os.path.exists(os.path.join(static_image_dir, cache_sub_dir, 'update.gz')):

[petkov, 2010/11/13 00:55:18]

80 chars

+      return os.path.join(cache_sub_dir, 'update.gz')
+
+    full_cache_dir = os.path.join(static_image_dir, cache_sub_dir)
+    src_image = ""
+    dest_image = os.path.join(full_cache_dir, 'dest_image.bin')
+
+    # Create the directory for the cache values
+    if not os.path.exists(full_cache_dir):
+      os.makedirs(full_cache_dir)
+
+    gen_image = self.GenerateUpdateImage(self.src_image,
+                                         image_path,
+                                         full_cache_dir)
+
+    # If the generation worked
+    if gen_image:
+      return os.path.join(cache_sub_dir, gen_image)
+    else:
+      return None
+
 
   def GenerateLatestUpdateImage(self, board_id, client_version,
-                                static_image_dir=None):
+                                static_image_dir):
     """Generates an update using the latest image that has been built.
 
     This will only generate an update if the newest update is newer than that
@@ -306,7 +366,7 @@ class Autoupdate(BuildObject):
       client_version: Current version of the client or 'ForcedUpdate'
       static_image_dir: the directory to move images to after generating.
     Returns:
-      True if the update payload was created successfully.
+      Name of the update image relative to static_image_dir or None
     """
     latest_image_dir = self._GetLatestImageDir(board_id)
     latest_version = self._GetVersionFromDir(latest_image_dir)
@@ -319,15 +379,10 @@ class Autoupdate(BuildObject):
     if client_version != 'ForcedUpdate' and not self._CanUpdate(
         client_version, latest_version):
       _LogMessage('no update')
-      return False
-
-    cached_file_path = os.path.join(static_image_dir, 'update.gz')
-    if (os.path.exists(cached_file_path) and
-        not self._IsImageNewerThanCached(latest_image_path, cached_file_path)):
-      return True
+      return None
 
-    return self.GenerateUpdateImage(latest_image_path, move_to_static_dir=True,
-                                    static_image_dir=static_image_dir)
+    return self.GenerateUpdateImageWithCache(latest_image_path,
+                                             static_image_dir=static_image_dir)
 
   def GenerateImageFromZip(self, static_image_dir):
     """Generates an update from an image zip file.
@@ -339,21 +394,23 @@ class Autoupdate(BuildObject):
     Args:
       static_image_dir: Directory where the zip file exists.
     Returns:
-      True if the update payload was created successfully.
+      Name of the update payload relative to static_image_dir if successful.
     """
     _LogMessage('Preparing to generate update from zip in %s.' % static_image_dir)
     image_path = os.path.join(static_image_dir, self._GetImageName())
-    cached_file_path = os.path.join(static_image_dir, 'update.gz')
     zip_file_path = os.path.join(static_image_dir, 'image.zip')
-    if not self._IsImageNewerThanCached(zip_file_path, cached_file_path):
-      return True
+
+    # XXX Work this into new path

[petkov, 2010/11/13 00:55:18]

Remove this comment along with the commented code? Or change to a TODO.

+    #cached_file_path = os.path.join(static_image_dir, 'update.gz')
+    #if not self._IsImageNewerThanCached(zip_file_path, cached_file_path):
+    #  return 'update.gz'
 
     if not self._UnpackZip(static_image_dir):
       _LogMessage('unzip image.zip failed.')
-      return False
+      return None
 
-    return self.GenerateUpdateImage(image_path, move_to_static_dir=False,
-                                    static_image_dir=None)
+    return self.GenerateUpdateImageWithCache(image_path,
+                                             static_image_dir=static_image_dir)
 
   def ImportFactoryConfigFile(self, filename, validate_checksums=False):
     """Imports a factory-floor server configuration file. The file should
@@ -450,27 +507,25 @@ class Autoupdate(BuildObject):
 
   def GenerateUpdatePayloadForNonFactory(self, board_id, client_version,
                                          static_image_dir):
-    """Generates an update for non-factory and returns True on success."""
-    if self.use_cached and os.path.exists(os.path.join(static_image_dir,
-                                                       'update.gz')):
-      _LogMessage('Using cached image regardless of timestamps.')
-      return True
+    """Generates an update for non-factory and returns file name relative to

[petkov, 2010/11/13 00:55:18]

One line summary. Then description.

+       static_image_dir on success.
+
+       Returns the update file relative to the static_image_dir on success."""
+
+    if self.forced_image:
+      return self.GenerateUpdateImageWithCache(
+          self.forced_image,
+          static_image_dir=static_image_dir)
+    elif self.serve_only:
+      return self.GenerateImageFromZip(static_image_dir)
     else:
-      if self.forced_image:
-        has_built_image = self.GenerateUpdateImage(
-            self.forced_image, move_to_static_dir=True,
-            static_image_dir=static_image_dir)
-        return has_built_image
-      elif self.serve_only:
-        return self.GenerateImageFromZip(static_image_dir)
-      else:
-        if board_id:
-          return self.GenerateLatestUpdateImage(board_id,
-                                                client_version,
-                                                static_image_dir)
+      if board_id:
+        return self.GenerateLatestUpdateImage(board_id,
+                                              client_version,
+                                              static_image_dir)
 
-        _LogMessage('You must set --board for pre-generating latest update.')
-        return False
+      _LogMessage('You must set --board for pre-generating latest update.')
+      return None
 
   def PreGenerateUpdate(self):
     """Pre-generates an update.  Returns True on success."""
@@ -480,8 +535,6 @@ class Autoupdate(BuildObject):
     # Does not work with labels so just use static dir.
     if self.GenerateUpdatePayloadForNonFactory(self.board, '0.0.0.0',
                                                self.static_dir):
-      # Force the devserver to use the pre-generated payload.
-      self.use_cached = True
       _LogMessage('Pre-generated update successfully.')
       return True
     else:
@@ -543,17 +596,19 @@ class Autoupdate(BuildObject):
       if label:
         static_image_dir = os.path.join(static_image_dir, label)
 
-      if self.GenerateUpdatePayloadForNonFactory(board_id, client_version,
-                                                 static_image_dir):
-        filename = os.path.join(static_image_dir, 'update.gz')
+      update_name = self.GenerateUpdatePayloadForNonFactory(board_id,
+                                                            client_version,
+                                                            static_image_dir)
+      if update_name:
+        filename = os.path.join(static_image_dir, update_name)
         hash = self._GetHash(filename)
         sha256 = self._GetSHA256(filename)
         size = self._GetSize(filename)
         is_delta_format = self._IsDeltaFormatFile(filename)
         if label:
-          url = '%s/%s/update.gz' % (static_urlbase, label)
+          url = '%s/%s/%s' % (static_urlbase, label, update_name)
         else:
-          url = '%s/update.gz' % static_urlbase
+          url = '%s/%s' % (static_urlbase, update_name)
 
         _LogMessage('Responding to client to use url %s to get image.' % url)
         return self.GetUpdatePayload(hash, sha256, size, url, is_delta_format)