Add support for preserving on-disk file layout

build_image

Index: build_image
diff --git a/build_image b/build_image
index bd49af5054b2e468f99711d3bceb26d170ed4404..d25fb4e123da517900dc1367ac22885dfe326288 100755
--- a/build_image
+++ b/build_image
@@ -84,6 +84,11 @@ DEFINE_string verity_algorithm "sha1" \
 DEFINE_string oem_customization "" \
   "Path to directory containing OEM partner partition contents"
 
+DEFINE_string base_image "" \
+  "Path to base image for which this image will be derived from"
+DEFINE_boolean zero_freespace ${FLAGS_FALSE} \

[petkov, 2010/12/10 18:33:30]

any downside in defaulting this to TRUE?

+  "Zeroes the freespace to improve compression for delta updates"
+
 # Parse command line.
 FLAGS "$@" || exit 1
 eval set -- "${FLAGS_ARGV}"
@@ -128,6 +133,11 @@ if [ "${FLAGS_installmask}" -eq "${FLAGS_TRUE}" ] ; then
   INSTALL_MASK="${DEFAULT_INSTALL_MASK}"
 fi
 
+# Make sure base image exists before we go too far
+if [ ! -z "${FLAGS_base_image}" ] && [ ! -f "${FLAGS_base_image}" ] ; then

[petkov, 2010/12/10 18:33:30]

if [ -n "${FLAGS_base_image}" ] ...

[sosa, 2010/12/10 19:28:10]

-n instead of ! -z

+  die "Cannot find base image ${FLAGS_base_image}"
+fi
+
 # Reduce the size of factory install shim.
 if [ "${FLAGS_factory_install}" -eq "${FLAGS_TRUE}" ] ||
    [ "${FLAGS_dev_install}" -eq "${FLAGS_TRUE}" ] ; then
@@ -225,10 +235,14 @@ ESP_FS_DIR=${OUTPUT_DIR}/esp
 
 DEVKEYSDIR="/usr/share/vboot/devkeys"
 
+NEW_ROOT_FS=${OUTPUT_DIR}/new.image
+NEW_ROOT_FS_DIR=/tmp/mnew

[sosa, 2010/12/10 19:28:10]

Create a temp dir ... why use the same one everytime?

+
 LOOP_DEV=
 STATEFUL_LOOP_DEV=
 OEM_LOOP_DEV=
 ESP_LOOP_DEV=
+NEW_ROOT_FS_LOOP_DEV=
 
 # ${DEV_IMAGE_ROOT} specifies the location of where developer packages will
 # be installed on the stateful dir.  On a Chromium OS system, this will
@@ -503,6 +517,91 @@ update_base_packages() {
     -s "${STATEFUL_FS_DIR}" -e "${ESP_FS_DIR}"
 }
 
+zero_rootfs_freespace() {
+  # Disable die on error so dd can write until EOF

[sosa, 2010/12/10 19:28:10]

Make this more function-y .. i.e. use $1 $2 etc

+  set +e

[sosa, 2010/12/10 19:28:10]

Don't set +e / -e ... use || instead on the dd line

+
+  local rootfs_dir=$1
+  echo "Zeroing freespace"
+  sudo dd if=/dev/zero of="${rootfs_dir}/filler" 2>/dev/null
+  sudo rm "${rootfs_dir}/filler"
+
+  set -e
+}
+
+align_rootfs_cleanup() {
+  # Disable die on error.
+  set +e
+
+  if [ -n "${NEW_ROOT_FS_LOOP_DEV}" ] ; then
+    sudo umount -d "${NEW_ROOT_FS_DIR}"
+    sudo rmdir "${NEW_ROOT_FS_DIR}"
+    NEW_ROOT_FS_LOOP_DEV=
+  fi
+
+  # Turn die on error back on.
+  set -e
+}
+
+align_rootfs() {
+
+  echo "Aligning rootfs on-disk file layout to base image"

[sosa, 2010/12/10 19:28:10]

info

+
+  # Extract rootfs from base image to output directory.
+  local rootfs_offset=$(partoffset "${FLAGS_base_image}" 3)
+  local rootfs_count=$(partsize "${FLAGS_base_image}" 3)
+  dd if="${FLAGS_base_image}" of="${NEW_ROOT_FS}" bs=512 conv=fsync \
+      skip=${rootfs_offset} count=${rootfs_count} &> /dev/null
+  enable_rw_mount "${NEW_ROOT_FS}"

[sosa, 2010/12/10 19:28:10]

Why is this necessary?  Can't rw just be set in the mount options?

+
+  # Mount the new image.
+  NEW_ROOT_FS_LOOP_DEV=$(sudo losetup -f)

[sosa, 2010/12/10 19:28:10]

Not sure you even have to do this rather than mount use mount -o loop.  There
are some other cases where it is necessary but not here

+  if [ -z "${NEW_ROOT_FS_LOOP_DEV}" ] ; then
+    echo "No free loop device.  Free up a loop device or reboot.  exiting. "
+    exit 1
+  fi
+
+  sudo mkdir -p "${NEW_ROOT_FS_DIR}"
+  sudo losetup "${NEW_ROOT_FS_LOOP_DEV}" "${NEW_ROOT_FS}"
+  sudo mount "${NEW_ROOT_FS_LOOP_DEV}" "${NEW_ROOT_FS_DIR}"
+
+  trap "align_rootfs_cleanup" EXIT
+
+  # Temporarily make immutable files mutable.
+  immutable_files=()

[sosa, 2010/12/10 19:28:10]

local?

+  NEW_ROOT_FS_DIR_ESCAPED=$(echo "${NEW_ROOT_FS_DIR}" | sed 's/\//\\\//g')

[sosa, 2010/12/10 19:28:10]

Why global?

[sosa, 2010/12/10 19:28:10]

Use a diff seperater in your sed than / to make this easier to read

+  local enum_files_cmd='sudo find "${NEW_ROOT_FS_DIR}" -xdev -type f'

[sosa, 2010/12/10 19:28:10]

Why exclude directories and such?

+  eval $enum_files_cmd | sed "s/${NEW_ROOT_FS_DIR_ESCAPED}//" | \

[sosa, 2010/12/10 19:28:10]

{}

+    while read -r FILE ; do
+      immutable=$(sudo lsattr "${NEW_ROOT_FS_DIR}$FILE" | cut -d' ' -f1 | \

[sosa, 2010/12/10 19:28:10]

shouldn't there be spaces in the cut

+                  grep -q i ; echo $?)
+      if [ $immutable -eq 0 ] ; then

[sosa, 2010/12/10 19:28:10]

{}

[sosa, 2010/12/10 19:28:10]

No spaces before ; same elsewhere

+        immutable_files=("${immutable_files[@]}" "${NEW_ROOT_FS_DIR}$FILE")
+        sudo chattr -i "${NEW_ROOT_FS_DIR}$FILE"

[sosa, 2010/12/10 19:28:10]

Little confused ... you set this immmutable twice ... here and below ... why?

+      fi
+    done
+
+  # Copy files over top of base image files to preserve file layout as much
+  # as possible.

[sosa, 2010/12/10 19:28:10]

Layout?  maybe metadata is more accurate

+  sudo rsync -a -H -A -x --force --inplace \
+      "${ROOT_FS_DIR}/" "${NEW_ROOT_FS_DIR}"
+
+  for FILE in ${immutable_files[*]} ; do
+    sudo chattr -i "${NEW_ROOT_FS_DIR}$FILE"
+  done
+
+  # Zero out free space, this makes it more compressible leading to a smaller
+  # delta update.
+  if [ "${FLAGS_zero_freespace}" -eq "${FLAGS_TRUE}" ] ; then
+    zero_rootfs_freespace "${NEW_ROOT_FS_DIR}"
+  fi
+
+  sudo umount -d "${NEW_ROOT_FS_DIR}"
+  sudo rmdir "${NEW_ROOT_FS_DIR}"
+
+  trap - EXIT
+}
+
 create_base_image() {
   local image_name=$1
 
@@ -528,7 +627,7 @@ create_base_image() {
      seek=$((ROOT_SIZE_BYTES + ROOT_HASH_PAD - 1))
   sudo losetup "${LOOP_DEV}" "${ROOT_FS_IMG}"
   # Specify a block size and block count to avoid using the hash pad.
-  sudo mkfs.ext2 -b 4096 "${LOOP_DEV}" "$((ROOT_SIZE_BYTES / 4096))"
+  sudo mkfs.ext2 -O ^has_journal -b 4096 "${LOOP_DEV}" "$((ROOT_SIZE_BYTES / 4096))"
 
   # Tune and mount rootfs.
   DISK_LABEL="C-ROOT"
@@ -683,8 +782,19 @@ create_base_image() {
   # Create an empty esp image to be updated in by update_bootloaders.sh.
   ${SCRIPTS_DIR}/create_esp.sh --to="${ESP_FS_IMG}"
 
+  # Align the rootfs to the previous if there's a base image to align to
+  if [ -n "${FLAGS_base_image}" ] ; then
+    align_rootfs
+  fi
+
   cleanup
 
+  # Copy new aligned image over to of rootfs.img
+  if [ -n "${FLAGS_base_image}" ] ; then
+    rm "${ROOT_FS_IMG}"
+    mv "${NEW_ROOT_FS}" "${ROOT_FS_IMG}"
+  fi
+
   trap delete_prompt EXIT
 
   # Create the GPT-formatted image.