Add support for preserving on-disk file layout

build_image

 #!/bin/bash
 
 # Copyright (c) 2009 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 # Script to build a bootable keyfob-based chromeos system image from within
 # a chromiumos setup. This assumes that all needed packages have been built into
 # the given target's root with binary packages turned on. This script will
 # build the Chrome OS image using only pre-built binary packages.
@@ skipping 66 matching lines @@
 DEFINE_integer verity_depth 1 \
   "Kernel verified boot hash tree depth. Default: 1"
 DEFINE_integer verity_max_ios -1 \
   "Number of outstanding I/O operations dm-verity caps at. Default: -1"
 DEFINE_string verity_algorithm "sha1" \
   "Cryptographic hash algorithm used for kernel vboot. Default : sha1"
 
 DEFINE_string oem_customization "" \
   "Path to directory containing OEM partner partition contents"
 
+DEFINE_string base_image "" \
+  "Path to base image for which this image will be derived from"
+DEFINE_boolean zero_freespace ${FLAGS_FALSE} \

[petkov, 2010/12/10 18:33:30]

any downside in defaulting this to TRUE?

+  "Zeroes the freespace to improve compression for delta updates"
+
 # Parse command line.
 FLAGS "$@" || exit 1
 eval set -- "${FLAGS_ARGV}"
 
 # Only now can we die on error.  shflags functions leak non-zero error codes,
 # so will die prematurely if 'set -e' is specified before now.
 set -e
 
 if [ -z "${FLAGS_board}" ] ; then
   error "--board is required."
@@ skipping 24 matching lines @@
    [ "${FLAGS_dev_install}" -eq "${FLAGS_TRUE}" ] ; then
   die "Incompatible flags: --factory_install and --dev_install cannot be \
 both set to True. Please specify one or none."
 fi
 
 INSTALL_MASK=""
 if [ "${FLAGS_installmask}" -eq "${FLAGS_TRUE}" ] ; then
   INSTALL_MASK="${DEFAULT_INSTALL_MASK}"
 fi
 
+# Make sure base image exists before we go too far
+if [ ! -z "${FLAGS_base_image}" ] && [ ! -f "${FLAGS_base_image}" ] ; then

[petkov, 2010/12/10 18:33:30]

if [ -n "${FLAGS_base_image}" ] ...

[sosa, 2010/12/10 19:28:10]

-n instead of ! -z

+  die "Cannot find base image ${FLAGS_base_image}"
+fi
+
 # Reduce the size of factory install shim.
 if [ "${FLAGS_factory_install}" -eq "${FLAGS_TRUE}" ] ||
    [ "${FLAGS_dev_install}" -eq "${FLAGS_TRUE}" ] ; then
   # Disable --withdev flag when --*_install is set to True. Otherwise, the
   # dev image produced will be based on install shim, rather than a pristine
   # image
   if [ "${FLAGS_withdev}" -eq "${FLAGS_TRUE}" ]; then
     info "Incompatible flags: --withdev and --dev_install or --factory_install \
 cannot be both set to True. Reset --withdev to False."
     FLAGS_withdev=${FLAGS_FALSE}
@@ skipping 77 matching lines @@
 STATEFUL_FS_DIR="${OUTPUT_DIR}/stateful_partition"
 
 OEM_FS_IMG="${OUTPUT_DIR}/partner_partition.image"
 OEM_FS_DIR="${OUTPUT_DIR}/partner_partition"
 
 ESP_FS_IMG=${OUTPUT_DIR}/esp.image
 ESP_FS_DIR=${OUTPUT_DIR}/esp
 
 DEVKEYSDIR="/usr/share/vboot/devkeys"
 
+NEW_ROOT_FS=${OUTPUT_DIR}/new.image
+NEW_ROOT_FS_DIR=/tmp/mnew

[sosa, 2010/12/10 19:28:10]

Create a temp dir ... why use the same one everytime?

+
 LOOP_DEV=
 STATEFUL_LOOP_DEV=
 OEM_LOOP_DEV=
 ESP_LOOP_DEV=
+NEW_ROOT_FS_LOOP_DEV=
 
 # ${DEV_IMAGE_ROOT} specifies the location of where developer packages will
 # be installed on the stateful dir.  On a Chromium OS system, this will
 # translate to /usr/local.
 DEV_IMAGE_ROOT="${STATEFUL_FS_DIR}/dev_image"
 
 # What cross-build are we targeting?
 . "${BOARD_ROOT}/etc/make.conf.board_setup"
 LIBC_VERSION=${LIBC_VERSION:-"2.10.1-r1"}
 
@@ skipping 254 matching lines @@
   # Clean out unused packages
   sudo INSTALL_MASK="${INSTALL_MASK}" ${EMERGE_BOARD_CMD} \
     --root="${ROOT_FS_DIR}" --root-deps=rdeps \
     --usepkgonly --depclean ${EMERGE_JOBS}
 
   trap - EXIT
   ${SCRIPTS_DIR}/mount_gpt_image.sh -u -r "${ROOT_FS_DIR}" \
     -s "${STATEFUL_FS_DIR}" -e "${ESP_FS_DIR}"
 }
 
+zero_rootfs_freespace() {
+  # Disable die on error so dd can write until EOF

[sosa, 2010/12/10 19:28:10]

Make this more function-y .. i.e. use $1 $2 etc

+  set +e

[sosa, 2010/12/10 19:28:10]

Don't set +e / -e ... use || instead on the dd line

+
+  local rootfs_dir=$1
+  echo "Zeroing freespace"
+  sudo dd if=/dev/zero of="${rootfs_dir}/filler" 2>/dev/null
+  sudo rm "${rootfs_dir}/filler"
+
+  set -e
+}
+
+align_rootfs_cleanup() {
+  # Disable die on error.
+  set +e
+
+  if [ -n "${NEW_ROOT_FS_LOOP_DEV}" ] ; then
+    sudo umount -d "${NEW_ROOT_FS_DIR}"
+    sudo rmdir "${NEW_ROOT_FS_DIR}"
+    NEW_ROOT_FS_LOOP_DEV=
+  fi
+
+  # Turn die on error back on.
+  set -e
+}
+
+align_rootfs() {
+
+  echo "Aligning rootfs on-disk file layout to base image"

[sosa, 2010/12/10 19:28:10]

info

+
+  # Extract rootfs from base image to output directory.
+  local rootfs_offset=$(partoffset "${FLAGS_base_image}" 3)
+  local rootfs_count=$(partsize "${FLAGS_base_image}" 3)
+  dd if="${FLAGS_base_image}" of="${NEW_ROOT_FS}" bs=512 conv=fsync \
+      skip=${rootfs_offset} count=${rootfs_count} &> /dev/null
+  enable_rw_mount "${NEW_ROOT_FS}"

[sosa, 2010/12/10 19:28:10]

Why is this necessary?  Can't rw just be set in the mount options?

+
+  # Mount the new image.
+  NEW_ROOT_FS_LOOP_DEV=$(sudo losetup -f)

[sosa, 2010/12/10 19:28:10]

Not sure you even have to do this rather than mount use mount -o loop.  There
are some other cases where it is necessary but not here

+  if [ -z "${NEW_ROOT_FS_LOOP_DEV}" ] ; then
+    echo "No free loop device.  Free up a loop device or reboot.  exiting. "
+    exit 1
+  fi
+
+  sudo mkdir -p "${NEW_ROOT_FS_DIR}"
+  sudo losetup "${NEW_ROOT_FS_LOOP_DEV}" "${NEW_ROOT_FS}"
+  sudo mount "${NEW_ROOT_FS_LOOP_DEV}" "${NEW_ROOT_FS_DIR}"
+
+  trap "align_rootfs_cleanup" EXIT
+
+  # Temporarily make immutable files mutable.
+  immutable_files=()

[sosa, 2010/12/10 19:28:10]

local?

+  NEW_ROOT_FS_DIR_ESCAPED=$(echo "${NEW_ROOT_FS_DIR}" | sed 's/\//\\\//g')

[sosa, 2010/12/10 19:28:10]

Why global?

[sosa, 2010/12/10 19:28:10]

Use a diff seperater in your sed than / to make this easier to read

+  local enum_files_cmd='sudo find "${NEW_ROOT_FS_DIR}" -xdev -type f'

[sosa, 2010/12/10 19:28:10]

Why exclude directories and such?

+  eval $enum_files_cmd | sed "s/${NEW_ROOT_FS_DIR_ESCAPED}//" | \

[sosa, 2010/12/10 19:28:10]

{}

+    while read -r FILE ; do
+      immutable=$(sudo lsattr "${NEW_ROOT_FS_DIR}$FILE" | cut -d' ' -f1 | \

[sosa, 2010/12/10 19:28:10]

shouldn't there be spaces in the cut

+                  grep -q i ; echo $?)
+      if [ $immutable -eq 0 ] ; then

[sosa, 2010/12/10 19:28:10]

{}

[sosa, 2010/12/10 19:28:10]

No spaces before ; same elsewhere

+        immutable_files=("${immutable_files[@]}" "${NEW_ROOT_FS_DIR}$FILE")
+        sudo chattr -i "${NEW_ROOT_FS_DIR}$FILE"

[sosa, 2010/12/10 19:28:10]

Little confused ... you set this immmutable twice ... here and below ... why?

+      fi
+    done
+
+  # Copy files over top of base image files to preserve file layout as much
+  # as possible.

[sosa, 2010/12/10 19:28:10]

Layout?  maybe metadata is more accurate

+  sudo rsync -a -H -A -x --force --inplace \
+      "${ROOT_FS_DIR}/" "${NEW_ROOT_FS_DIR}"
+
+  for FILE in ${immutable_files[*]} ; do
+    sudo chattr -i "${NEW_ROOT_FS_DIR}$FILE"
+  done
+
+  # Zero out free space, this makes it more compressible leading to a smaller
+  # delta update.
+  if [ "${FLAGS_zero_freespace}" -eq "${FLAGS_TRUE}" ] ; then
+    zero_rootfs_freespace "${NEW_ROOT_FS_DIR}"
+  fi
+
+  sudo umount -d "${NEW_ROOT_FS_DIR}"
+  sudo rmdir "${NEW_ROOT_FS_DIR}"
+
+  trap - EXIT
+}
+
 create_base_image() {
   local image_name=$1
 
   trap "cleanup && delete_prompt" EXIT
 
   # Create and format the root file system.
 
   # Check for loop device before creating image.
   LOOP_DEV=$(sudo losetup -f)
   if [ -z "${LOOP_DEV}" ] ; then
     echo "No free loop device.  Free up a loop device or reboot.  exiting. "
     exit 1
   fi
 
   # Create root file system disk image.
   ROOT_SIZE_BYTES=$((1024 * 1024 * ${FLAGS_rootfs_size}))
 
   # Pad out for the hash tree.
   ROOT_HASH_PAD=$((FLAGS_rootfs_hash_pad * 1024 * 1024))
   info "Padding the rootfs image by ${ROOT_HASH_PAD} bytes for hash data"
 
   dd if=/dev/zero of="${ROOT_FS_IMG}" bs=1 count=1 \
      seek=$((ROOT_SIZE_BYTES + ROOT_HASH_PAD - 1))
   sudo losetup "${LOOP_DEV}" "${ROOT_FS_IMG}"
   # Specify a block size and block count to avoid using the hash pad.
-  sudo mkfs.ext2 -b 4096 "${LOOP_DEV}" "$((ROOT_SIZE_BYTES / 4096))"
+  sudo mkfs.ext2 -O ^has_journal -b 4096 "${LOOP_DEV}" "$((ROOT_SIZE_BYTES / 4096))"
 
   # Tune and mount rootfs.
   DISK_LABEL="C-ROOT"
   # Disable checking and minimize metadata differences across builds
   # and wasted reserved space.
   sudo tune2fs -L "${DISK_LABEL}" \
                -U clear \
                -T 20091119110000 \
                -c 0 \
                -i 0 \
@@ skipping 134 matching lines @@
   setup_symlinks_on_root "/usr/local" "/var" "${STATEFUL_FS_DIR}"
 
   # cros_make_image_bootable will clobber vmlinuz.image for x86.
   # Until then, just copy the kernel to vmlinuz.image. It is
   # expected in build_gpt.sh.
   cp "${ROOT_FS_DIR}/boot/vmlinuz" "${OUTPUT_DIR}/vmlinuz.image"
 
   # Create an empty esp image to be updated in by update_bootloaders.sh.
   ${SCRIPTS_DIR}/create_esp.sh --to="${ESP_FS_IMG}"
 
+  # Align the rootfs to the previous if there's a base image to align to
+  if [ -n "${FLAGS_base_image}" ] ; then
+    align_rootfs
+  fi
+
   cleanup
 
+  # Copy new aligned image over to of rootfs.img
+  if [ -n "${FLAGS_base_image}" ] ; then
+    rm "${ROOT_FS_IMG}"
+    mv "${NEW_ROOT_FS}" "${ROOT_FS_IMG}"
+  fi
+
   trap delete_prompt EXIT
 
   # Create the GPT-formatted image.
   ${SCRIPTS_DIR}/build_gpt.sh \
     --arch=${ARCH} \
     --board=${FLAGS_board} \
     --rootfs_partition_size=${FLAGS_rootfs_partition_size} \
     "${OUTPUT_DIR}" \
     "${OUTPUT_DIR}/${image_name}"
 
@@ skipping 94 matching lines @@
   echo "Developer image created as ${DEVELOPER_IMAGE_NAME}"
 fi
 
 print_time_elapsed
 
 echo "To copy to USB keyfob, do something like:"
 echo "  ./image_to_usb.sh --from=${OUTSIDE_OUTPUT_DIR} --to=/dev/sdX"
 echo "To convert to VMWare image, INSIDE the chroot, do something like:"
 echo "  ./image_to_vm.sh --from=${OUTSIDE_OUTPUT_DIR} --board=${BOARD}"
 echo "from the scripts directory where you entered the chroot."