Switch install_packages.sh to re-use the apt cache across runs.

src/scripts/install_packages.sh

 #!/bin/bash
 
 # Copyright (c) 2009 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 # Script to install packages into the target root file system.
 #
 # NOTE: This script should be called by build_image.sh. Do not run this
 # on your own unless you know what you are doing.
@@ skipping 61 matching lines @@
     ! sudo kill $pid  # Preceded by ! to disable ERR trap.
   done
 
   # Sometimes the volatile directory is left mounted and sometimes it is not,
   # so we precede by '!' to disable the ERR trap.
   ! sudo umount "$ROOT_FS_DIR"/lib/modules/2.6.*/volatile/ > /dev/null 2>&1
 
   sudo umount "${ROOT_FS_DIR}/proc"
 }
 
-# Create setup directory and copy over scripts, config files, and locally
-# built packages.
+# Set up repository for locally built packages; these take highest precedence.
 mkdir -p "${SETUP_DIR}/local_packages"
 cp "${FLAGS_build_root}/${FLAGS_target}/local_packages"/* \
   "${SETUP_DIR}/local_packages"
-
-# Set up repository for local packages to install in the rootfs via apt-get.
 cd "$SETUP_DIR"
 dpkg-scanpackages local_packages/ /dev/null | \
    gzip > local_packages/Packages.gz
 cd -
 
 # Create the temporary apt source.list used to install packages.
 APT_SOURCE="${OUTPUT_DIR}/sources.list"
 cat <<EOF > "$APT_SOURCE"
 deb file:"$SETUP_DIR" local_packages/
 deb $FLAGS_server $FLAGS_suite main restricted multiverse universe
 EOF
 
-# Cache directory for APT to use.
-APT_CACHE_DIR="${OUTPUT_DIR}/tmp/cache/"
+# Cache directory for APT to use. This cache is re-used across builds. We
+# rely on the cache to reduce traffic to the hosted repositories.
+APT_CACHE_DIR="${FLAGS_build_root}/apt_cache-${FLAGS_target}/"
 mkdir -p "${APT_CACHE_DIR}/archives/partial"
 
 # Create the apt configuration file. See "man apt.conf"
 NO_MAINTAINER_SCRIPTS=""
 if [ -n "$EXPERIMENTAL_NO_MAINTAINER_SCRIPTS" ]; then
   NO_MAINTAINER_SCRIPTS="Bin { dpkg \"${SCRIPTS_DIR}/dpkg_no_scripts.sh\"; };"
 fi
 APT_PARTS="${OUTPUT_DIR}/apt.conf.d"
 mkdir -p "$APT_PARTS"  # An empty apt.conf.d to avoid other configs.
 export APT_CONFIG="${OUTPUT_DIR}/apt.conf"
 cat <<EOF > "$APT_CONFIG"
 APT
 {
   Install-Recommends "0";
   Install-Suggests "0";
   Get
   {
     Assume-Yes "1";
+    AllowUnauthenticated "1";
   };
 };
 Dir
 {
   $NO_MAINTAINER_SCRIPTS
   Cache "$APT_CACHE_DIR";
   Cache {
     archives "${APT_CACHE_DIR}/archives";
   };
   Etc
@@ skipping 20 matching lines @@
 if [ ! -e "/usr/share/debootstrap/scripts/$FLAGS_suite" ]
 then
   sudo ln -s /usr/share/debootstrap/scripts/jaunty \
     "/usr/share/debootstrap/scripts/$FLAGS_suite"
 fi
 
 if [ -z "$EXPERIMENTAL_NO_DEBOOTSTRAP" -a \
      -z "$EXPERIMENTAL_NO_MAINTAINER_SCRIPTS" ]; then
   # Use debootstrap, which runs maintainer scripts.
   sudo debootstrap --arch=i386 $FLAGS_suite "$ROOT_FS_DIR" "${FLAGS_server}"
+  sudo APT_CONFIG="$APT_CONFIG" DEBIAN_FRONTEND=noninteractive apt-get update
 else
-  # A hack-in-progress that does our own debootstrap equivalent and skips
-  # maintainer scripts.
-
-  # TODO: Replace with a pointer to lool's repo or maybe apt-get --download-only?
-  REPO="${GCLIENT_ROOT}/repo/var/cache/make_local_repo"
+  # We do a rough equivalent to debootstrap that installs the minimal
+  # packages needed to be able to run apt to install the rest. We don't
+  # use debootstrap since it is geared toward having a second stage that
+  # needs to run package maintainer scripts. This is also simpler.
 
   # The set of required packages before apt can take over.
   PACKAGES="debconf libacl1 libattr1 libc6 libgcc1 libselinux1"
 
   # Set of packages that we need to install early so that other packages
   # maintainer scripts can still basically run.
   #
   # login  - So that groupadd will work
   # base-passwd/passwd - So that chmod and useradd/groupadd will work
   # bash - So that scripts can run
   # libpam-runtim/libuuid1 - Not exactly sure why
   EXTRA_PACKAGES="base-files base-passwd bash libpam-runtime libuuid1 login passwd"
 
   # Prep the rootfs to work with dpgk and apt
   sudo mkdir -p "${ROOT_FS_DIR}/var/lib/dpkg/info"
   sudo touch "${ROOT_FS_DIR}/var/lib/dpkg/available"   \
     "${ROOT_FS_DIR}/var/lib/dpkg/diversions"           \
     "${ROOT_FS_DIR}/var/lib/dpkg/status"
   sudo mkdir -p "${ROOT_FS_DIR}/var/lib/apt/lists/partial"  \
     "${ROOT_FS_DIR}/var/lib/dpkg/updates"
 
+  # Download the initial packages into the apt cache if necessary.
+  REPO="${APT_CACHE_DIR}/archives"
+  sudo APT_CONFIG="$APT_CONFIG" DEBIAN_FRONTEND=noninteractive apt-get update
+  sudo APT_CONFIG="$APT_CONFIG" DEBIAN_FRONTEND=noninteractive \
+    apt-get --download-only install $PACKAGES $EXTRA_PACKAGES
+
   i=0
   for p in $PACKAGES $EXTRA_PACKAGES; do
     set +e
     PKG=$(ls "${REPO}"/${p}_*_i386.deb)
     set -e
     if [ -z "$PKG" ]; then
       PKG=$(ls "${REPO}"/${p}_*_all.deb)
     fi
     echo "Installing package: $PKG [$i]"
     sudo "${SCRIPTS_DIR}"/dpkg_no_scripts.sh \
       --root="$ROOT_FS_DIR" --unpack "$PKG"
     i=$((i + 1))
   done
 
+  # Make sure that apt is ready to work. We use --fix-broken to trigger apt
+  # to install additional critical packages. If there are any of these, we
+  # disable the maintainer scripts so they install ok.
+  TMP_FORCE_NO_SCRIPTS="-o=Dir::Bin::dpkg=${SCRIPTS_DIR}/dpkg_no_scripts.sh"
+  sudo APT_CONFIG="$APT_CONFIG" DEBIAN_FRONTEND=noninteractive \
+    apt-get $TMP_FORCE_NO_SCRIPTS --force-yes --fix-broken install
+
   # ----- MAINTAINER SCRIPT FIXUPS -----
 
   # TODO: Remove when we stop having maintainer scripts altogether.
   sudo cp -a /dev/* "${ROOT_FS_DIR}/dev"
 
   # base-passwd
   sudo cp "${ROOT_FS_DIR}/usr/share/base-passwd/passwd.master" \
     "${ROOT_FS_DIR}/etc/passwd"
   sudo cp "${ROOT_FS_DIR}/usr/share/base-passwd/group.master" \
     "${ROOT_FS_DIR}/etc/group"
@@ skipping 12 matching lines @@
   # mawk
   sudo ln -s mawk "${ROOT_FS_DIR}/usr/bin/awk"
 
   # base-files?
   sudo touch "${ROOT_FS_DIR}/etc/fstab"
 
   # sysv-rc needs this
   sudo mkdir -p "${ROOT_FS_DIR}/etc/init.d"
 fi  # EXPERIMENTAL_NO_DEBOOTSTRAP
 
-
 # Set up mounts for working within the rootfs. We copy some basic
 # network information from the host so that maintainer scripts can
 # access the network as needed.
 # TODO: All of this rootfs mount stuff can be removed as soon as we stop
 # running the maintainer scripts on install.
 sudo mount -t proc proc "${ROOT_FS_DIR}/proc"
 sudo cp /etc/hosts "${ROOT_FS_DIR}/etc"
 trap cleanup_rootfs_mounts EXIT
 
-# Make sure that apt is ready to work. We use --fix-broken to trigger apt
-# to install additional critical packages. If there are any of these, we
-# disable the maintainer scripts so they install ok.
-sudo APT_CONFIG="$APT_CONFIG" DEBIAN_FRONTEND=noninteractive apt-get update
-TMP_FORCE_NO_SCRIPTS="-o=Dir::Bin::dpkg=${SCRIPTS_DIR}/dpkg_no_scripts.sh"
-sudo APT_CONFIG="$APT_CONFIG" DEBIAN_FRONTEND=noninteractive \
-  apt-get $TMP_FORCE_NO_SCRIPTS --force-yes --fix-broken install
-
 # Install prod packages
 COMPONENTS=`cat $FLAGS_package_list | grep -v ' *#' | grep -v '^ *$' | sed '/$/{N;s/\n/ /;}'`
 sudo APT_CONFIG="$APT_CONFIG" DEBIAN_FRONTEND=noninteractive \
   apt-get --force-yes install $COMPONENTS
 
 # Create kernel installation configuration to suppress warnings,
 # install the kernel in /boot, and manage symlinks.
 cat <<EOF | sudo dd of="${ROOT_FS_DIR}/etc/kernel-img.conf"
 link_in_boot = yes
 do_symlinks = yes
 minimal_swap = yes
 clobber_modules = yes
 warn_reboot = no
 do_bootloader = no
 do_initrd = yes
 warn_initrd = no
 EOF
 
 # Install the kernel.
 sudo APT_CONFIG="$APT_CONFIG" DEBIAN_FRONTEND=noninteractive \
   apt-get --force-yes install "linux-image-${KERNEL_VERSION}"
 
-# Clean up the apt cache.
-# TODO: The cache was populated by debootstrap, not these installs. Remove
-# this line when we can get debootstrap to stop doing this.
-sudo rm -f "${ROOT_FS_DIR}"/var/cache/apt/archives/*.deb
-# Need to rm read-only created lock files in order for archiving step to work
-sudo rm -rf "$APT_CACHE_DIR"
-
 # List all packages installed so far, since these are what the local
 # repository needs to contain.
 # TODO: Replace with list_installed_packages.sh when it is fixed up.
 dpkg --root="${ROOT_FS_DIR}" -l > \
   "${OUTPUT_DIR}/package_list_installed.txt"
 
 cleanup_rootfs_mounts
 trap - EXIT