Make sbt support latest build workflow

src/scripts/sync_build_test.sh

 #!/bin/bash
 
 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 # Script to sync your checkout, build a Chromium OS image, and test it all
 # with one command.  Can also check out a new Chromium OS checkout and
 # perform a subset of the above operations.
 #
@@ skipping 28 matching lines @@
 #   BUILDBOT_URI - default value for --buildbot_uri
 #   CHROMIUM_REPO - default value for --repo
 #   CHRONOS_PASSWD - default value for --chronos_passwd
 #
 
 # Load common constants.  This should be the first executable line.
 # The path to common.sh should be relative to your script's location.
 . "$(dirname "$0")/common.sh"
 
 
-DEFINE_string top "" \
-    "Root directory of your checkout (defaults to determining from your cwd)"
-DEFINE_string repo "${CHROMIUMOS_REPO}" "gclient repo for chromiumos"
-DEFINE_boolean sync ${FLAGS_TRUE} "Sync the checkout"
-DEFINE_boolean force_make_chroot ${FLAGS_FALSE} "Run make_chroot indep of sync"
+DEFINE_string board "x86-generic" "Board setting"
 DEFINE_boolean build ${FLAGS_TRUE} \
     "Build all code (but not necessarily master image)"
-DEFINE_boolean master ${FLAGS_TRUE} "Master an image from built code"
+DEFINE_boolean build_autotest ${FLAGS_FALSE} "Build autotest"
+DEFINE_string buildbot_uri "${BUILDBOT_URI}" \
+    "Base URI to buildbot build location which contains LATEST file"
+DEFINE_string chronos_passwd "${CHRONOS_PASSWD}" \
+    "Use this as the chronos user passwd (defaults to \$CHRONOS_PASSWD)"
+DEFINE_string chroot "" "Chroot to build/use"
+DEFINE_boolean force_make_chroot ${FLAGS_FALSE} "Run make_chroot indep of sync"
 DEFINE_string grab_buildbot "" \
     "Instead of building, grab this full image.zip URI generated by the \
 buildbot"
-DEFINE_string chronos_passwd "${CHRONOS_PASSWD}" \
-    "Use this as the chronos user passwd (defaults to \$CHRONOS_PASSWD)"
-DEFINE_boolean mod_image_for_test ${FLAGS_FALSE} "Modify the image for testing"
-DEFINE_boolean build_autotest ${FLAGS_FALSE} "Build autotest"
 DEFINE_boolean image_to_live ${FLAGS_FALSE} \
     "Put the resulting image on live instance (requires --remote)"
+DEFINE_string image_to_usb "" \
+    "Treat this device as USB and put the image on it after build"
+# You can set jobs > 1 but then your build may break and you may need
+# to retry.  Setting it to 1 is best for non-interactive sessions.
+DEFINE_boolean interactive ${FLAGS_FALSE} \
+    "Tell user what we plan to do and wait for input to proceed" i
+DEFINE_integer jobs -1 "Concurrent build jobs"
+DEFINE_boolean master ${FLAGS_TRUE} "Master an image from built code"
+DEFINE_boolean mod_image_for_test ${FLAGS_FALSE} "Modify the image for testing"
 DEFINE_string remote "" \
     "Use this hostname/IP for live updating and running tests"
-DEFINE_string image_to_usb "" \
-    "Treat this device as USB and put the image on it after build"
+DEFINE_string repo "${CHROMIUMOS_REPO}" "gclient repo for chromiumos"
+DEFINE_boolean sync ${FLAGS_TRUE} "Sync the checkout"
 DEFINE_string test "" \
     "Test the built image with the given params to run_remote_tests"
-DEFINE_string buildbot_uri "${BUILDBOT_URI}" \
-    "Base URI to buildbot build location which contains LATEST file"
+DEFINE_string top "" \
+    "Root directory of your checkout (defaults to determining from your cwd)"
+DEFINE_boolean withdev ${FLAGS_TRUE} "Build development packages"
 DEFINE_boolean unittest ${FLAGS_TRUE} "Run unit tests"
-DEFINE_boolean interactive ${FLAGS_FALSE} \
-    "Tell user what we plan to do and wait for input to proceed" i
-DEFINE_boolean portage ${FLAGS_TRUE} "Use portage-based build"
-DEFINE_string board "x86-generic" "Board setting (portage)"
-DEFINE_string toolchain "i686-pc-linux-gnu" "Toolchain setting (portage)"
-# You can set build_jobs > 1 but then your build may break and you may need
-# to retry.  Setting it to 1 is best for non-interactive sessions.
-DEFINE_integer build_jobs 1 "Concurrent build jobs (portage)"
+
 
 # Returns a heuristic indicating if we believe this to be a google internal
 # development environment.
 # Returns:
 #   0 if so, 1 otherwise
 function is_google_environment() {
   hostname | egrep -q .google.com\$
   return $?
 }
 
@@ skipping 11 matching lines @@
       test_dir=$(dirname "${test_dir}")
     done
   fi
 
   if [[ -z "${FLAGS_top}" ]]; then
     # Use the top directory based on where this script runs from
     FLAGS_top=$(dirname $(dirname $(dirname $0)))
   fi
 
   # Canonicalize any symlinks
-  FLAGS_top=$(readlink -f "${FLAGS_top}")
+  if [[ -d "${FLAGS_top}" ]]; then
+    FLAGS_top=$(readlink -f "${FLAGS_top}")
+  fi
+
+  if [[ -z "${FLAGS_chroot}" ]]; then
+    FLAGS_chroot="${FLAGS_top}/chroot"
+  fi
+
+  # If chroot does not exist, force making it
+  if [[ ! -d "${FLAGS_chroot}" ]]; then
+    FLAGS_force_make_chroot=${FLAGS_TRUE}
+  fi
 
   if [[ -z "${FLAGS_repo}" ]]; then
     if is_google_environment; then
       FLAGS_repo="ssh://git@chromiumos-git//chromeos"
     else
       FLAGS_repo="http://src.chromium.org/git/chromiumos.git"
     fi
   fi
 
   if [[ -n "${FLAGS_test}" ]]; then
@@ skipping 38 matching lines @@
   fi
 
   if [[ ${FLAGS_mod_image_for_test} -eq ${FLAGS_TRUE} ]]; then
     # Override any specified chronos password with the test one
     local test_file=$(dirname $0)"/mod_for_test_scripts/test_account.passwd"
     FLAGS_chronos_passwd=$(head -1 "${test_file}")
     # Default to building autotests whenever we mod image for test.
     # TODO(kmixter): Make this more efficient by either doing incremental
     # building, or only building if the tests we're running needs to be.
     FLAGS_build_autotest=${FLAGS_TRUE}
-  fi
-
-  if [[ -e "${FLAGS_top}/src/scripts/new_make_env.sh" ]]; then
-    if [[ ${FLAGS_portage} -ne ${FLAGS_TRUE} ]]; then
-      echo "WARNING: It looks like you wanted to pass --portage to "
-      echo "build with the portage method"
-    fi
+    # If you're modding for test, you also want developer packages.
+    FLAGS_withdev=${FLAGS_TRUE}
   fi
 
   if [[ -n "${FLAGS_image_to_usb}" ]]; then
     local device=${FLAGS_image_to_usb#/dev/}
     if [[ -z "${device}" ]]; then
       echo "Expected --image_to_usb option of /dev/* format"
       exit 1
     fi
     local is_removable=$(cat /sys/block/${device}/removable)
     if [[ "${is_removable}" != "1" ]]; then
       echo "Could not verify that ${device} for image_to_usb is removable"
       exit 1
     fi
   fi
 }
 
 
 # Prints a description of what we are doing or did
 function describe_steps() {
   if [[ ${FLAGS_sync} -eq ${FLAGS_TRUE} ]]; then
     echo " * Sync client (gclient sync)"
     if is_google_environment; then
       echo " * Create proper src/scripts/.chromeos_dev"
     fi
   fi
   if [[ ${FLAGS_force_make_chroot} -eq ${FLAGS_TRUE} ]]; then
-    echo " * (Re-)create development chroot (make_chroot.sh)"
+    echo " * Rebuild chroot (make_chroot) in ${FLAGS_chroot}"
   fi
   local set_passwd=${FLAGS_FALSE}
   if [[ ${FLAGS_build} -eq ${FLAGS_TRUE} ]]; then
-    echo " * Build image (build_platform_packages.sh, build_kernel.sh)"
+    local withdev=""
+    local jobs=" single job (slow but safe)"
+    if [[ ${FLAGS_jobs} -gt 1 ]]; then
+      jobs=" ${FLAGS_jobs} jobs (may cause build failure)"
+    fi
+    if [[ ${FLAGS_withdev} -eq ${FLAGS_TRUE} ]]; then
+      withdev=" with dev packages"
+    fi
+    echo " * Build image${withdev}${jobs}"
     set_passwd=${FLAGS_TRUE}
   fi
   if [[ ${FLAGS_master} -eq ${FLAGS_TRUE} ]]; then
-    echo " * Master image (build_image.sh)"
+    echo " * Master image (build_image)"
   fi
   if [[ -n "${FLAGS_grab_buildbot}" ]]; then
     if [[ "${FLAGS_grab_buildbot}" == "LATEST" ]]; then
       echo " * Grab latest buildbot image under ${FLAGS_buildbot_uri}"
     else
       echo " * Grab buildbot image zip at URI ${FLAGS_grab_buildbot}"
     fi
   fi
   if [[ ${FLAGS_mod_image_for_test} -eq ${FLAGS_TRUE} ]]; then
     echo " * Make image able to run tests (mod_image_for_test)"
     set_passwd=${FLAGS_TRUE}
   fi
   if [[ ${set_passwd} -eq ${FLAGS_TRUE} ]]; then
     if [[ -n "${FLAGS_chronos_passwd}" ]]; then
       echo " * Set chronos password to ${FLAGS_chronos_passwd}"
     else
       echo " * Set chronos password randomly"
     fi
   fi
   if [[ ${FLAGS_build_autotest} -eq ${FLAGS_TRUE} ]]; then
-    echo " * Build autotest"
+    echo " * Cross-build autotest client tests (build_autotest)"
   fi
   if [[ -n "${FLAGS_image_to_usb}" ]]; then
     echo " * Write the image to USB device ${FLAGS_image_to_usb}"
   fi
   if [[ ${FLAGS_image_to_live} -eq ${FLAGS_TRUE} ]]; then
     echo " * Reimage live test Chromium OS instance at ${FLAGS_remote}"
   fi
   if [[ -n "${FLAGS_test}" ]]; then
     echo " * Run tests (${FLAGS_test}) on machine at ${FLAGS_remote}"
   fi
@@ skipping 76 matching lines @@
 }
 
 
 # Runs a phase, similar to run_phase, but runs within the chroot.
 # Arguments:
 #   $1 - phase description
 #   $2.. - command/params to run in chroot
 function run_phase_in_chroot() {
   local desc="$1"
   shift
-  run_phase "${desc}" ./enter_chroot.sh -- "$@"
+  run_phase "${desc}" ./enter_chroot.sh "--chroot=${FLAGS_chroot}" -- "$@"
 }
 
 
 # Record start time.
 function set_start_time() {
   START_TIME=$(date '+%s')
 }
 
 
 # Display duration
@@ skipping 18 matching lines @@
     local base_dir=$(dirname $(dirname "${FLAGS_top}"))
     echo <<EOF > src/scripts/.chromeos_dev
 # Use internal chromeos-deb repository
 CHROMEOS_EXT_MIRROR="http://chromeos-deb/ubuntu"
 CHROMEOS_EXT_SUITE="karmic"
 
 # Assume Chrome is checked out nearby
 CHROMEOS_CHROME_DIR="${base_dir}/chrome"
 EOF
   fi
-  if [[ ${FLAGS_portage} -eq ${FLAGS_TRUE} ]]; then
-    chdir_relative src/third_party
-    chromiumos-overlay/chromeos/scripts/setup_source_tree.sh
-  fi
 }
 
 
 function check_rootfs_validity() {
   echo "Checking rootfs validity"
   local device=$(sudo losetup -f)
   local invalid=0
   sudo losetup "${device}" rootfs.image
   sudo mount "${device}" rootfs
   if [[ ! -e rootfs/boot/vmlinuz ]]; then
@@ skipping 52 matching lines @@
 
   # Cache up sudo status
   sudo -v
 
   if [[ ${FLAGS_interactive} -eq ${FLAGS_TRUE} ]]; then
     interactive
   fi
 
   set_start_time
 
+  local withdev_param=""
+  if [[ ${FLAGS_withdev} -eq ${FLAGS_TRUE} ]]; then
+    withdev_param="--withdev"
+  fi
+
+  local jobs_param=""
+  if [[ ${FLAGS_jobs} -gt 1 ]]; then
+    jobs_param="--jobs=${FLAGS_jobs}"
+  fi
+
+  local board_param="--board=${FLAGS_board}"
+
   if [[ ! -e "${FLAGS_top}" ]]; then
     config_new_checkout
   fi
 
   if [[ ${FLAGS_sync} -eq ${FLAGS_TRUE} ]]; then
     sync
-    # The package repository is now potentially out of date, so
-    # reflect that.
-    run_phase "Removing existing package repo" sudo rm -rf repo
-    FLAGS_force_make_chroot=${FLAGS_TRUE}
   fi
 
   if [[ -n "${FLAGS_grab_buildbot}" ]]; then
     grab_buildbot
   fi
 
   if [[ ${FLAGS_force_make_chroot} -eq ${FLAGS_TRUE} ]]; then
     chdir_relative src/scripts
-    if [[ ${FLAGS_portage} -eq ${FLAGS_TRUE} ]]; then
-      run_phase "Replacing chroot" ./new_make_env.sh --replace
-    else
-      run_phase "Replacing chroot" ./make_chroot.sh --replace
-    fi
+    run_phase "Replacing chroot" ./make_chroot --replace \
+        "--chroot=${FLAGS_chroot}" ${jobs_param}
   fi
 
   if [[ ${FLAGS_build} -eq ${FLAGS_TRUE} ]]; then
     chdir_relative src/scripts
-    if [[ ${FLAGS_portage} -eq ${FLAGS_TRUE} ]]; then
-      # Only setup board target if the directory does not exist
-      if [[ ! -d "${DEFAULT_CHROOT_DIR}/build/${FLAGS_board}" ]]; then
-        run_phase_in_chroot "Setting up board target" \
-            ./setup_board "--board=${FLAGS_board}" \
-            "--toolchain=${FLAGS_toolchain}"
-      fi
-      run_phase_in_chroot "Building packages" \
-          ./new_build_pkgs.sh "--board=${FLAGS_board}" \
-          "--jobs=${FLAGS_build_jobs}"
-    else
-      run_phase_in_chroot "Building platform packages and kernel" \
-          "./build_platform_packages.sh && ./build_kernel.sh"
+    # Only setup board target if the directory does not exist
+    if [[ ! -d "${FLAGS_top}/chroot/build/${FLAGS_board}" ]]; then
+      run_phase_in_chroot "Setting up board target" \
+          ./setup_board "${board_param}"
     fi
+    run_phase_in_chroot "Building packages" \
+        ./build_packages "${board_param}" \
+        ${jobs_param} ${withdev_param}
 
     # TODO(kmixter): Enable this once build_tests works, but even
     # then only do it when not cross compiling.
-    if [[ ${FLAGS_portage} -eq ${FLAGS_FALSE} ]]; then
+    if [[ '' ]]; then
       run_phase_in_chroot "Building and running unit tests" \
         "./build_tests.sh && ./run_tests.sh"
     fi
   fi
 
   if [[ ${FLAGS_master} -eq ${FLAGS_TRUE} ]]; then
     chdir_relative src/scripts
     if [[ -n "${FLAGS_chronos_passwd}" ]]; then
       run_phase_in_chroot "Setting default chronos password" \
       ./enter_chroot.sh "echo '${FLAGS_chronos_passwd}' | \
                         ~/trunk/src/scripts/set_shared_user_password.sh"
     fi
-    if [[ ${FLAGS_portage} -eq ${FLAGS_TRUE} ]]; then
-      run_phase_in_chroot "Mastering image" ./new_build_image.sh \
-          "--board=${FLAGS_board}" --replace
-    else
-      run_phase_in_chroot "Mastering image" ./build_image.sh --replace
-    fi
-  fi
-
-  local board_param=""
-  if [[ ${FLAGS_portage} -eq ${FLAGS_TRUE} ]]; then
-    board_param="--board=${FLAGS_board}"
+    run_phase_in_chroot "Mastering image" ./build_image \
+        "${board_param}" --replace ${withdev_param} \
+        ${jobs_param}
   fi
 
   if [[ ${FLAGS_mod_image_for_test} -eq ${FLAGS_TRUE} ]]; then
     chdir_relative src/scripts
     run_phase_in_chroot "Modifying image for test" \
-        "./mod_image_for_test.sh" ${board_param}
+        "./mod_image_for_test.sh" "${board_param}" --yes
   fi
 
   if [[ -n "${FLAGS_image_to_usb}" ]]; then
     chdir_relative src/scripts
     run_phase "Installing image to USB" \
-        ./image_to_usb.sh --yes "--to=${FLAGS_image_to_usb}" ${board_param}
+        ./image_to_usb.sh --yes "--to=${FLAGS_image_to_usb}" "${board_param}"
   fi
 
   if [[ ${FLAGS_image_to_live} -eq ${FLAGS_TRUE} ]]; then
     chdir_relative src/scripts
     run_phase "Re-imaging live Chromium OS machine ${FLAGS_remote}" \
       ./image_to_live.sh "--remote=${FLAGS_remote}" --update_known_hosts
   fi
 
   if [[ ${FLAGS_build_autotest} -eq ${FLAGS_TRUE} ]]; then
     chdir_relative src/scripts
-    run_phase_in_chroot "Building autotest" "./build_autotest.sh" ${board_param}
+    run_phase_in_chroot "Building autotest" "./build_autotest.sh" \
+      --noprompt "${board_param}"
   fi
 
   if [[ -n "${FLAGS_test}" ]]; then
     chdir_relative src/scripts
     # We purposefully do not quote FLAGS_test below as we expect it may
     # have multiple parameters
     run_phase "Running tests on Chromium OS machine ${FLAGS_remote}" \
       ./run_remote_tests.sh "--remote=${FLAGS_remote}" ${FLAGS_test} \
-      ${board_param}
+      "${board_param}"
   fi
 
   echo "Successfully used ${FLAGS_top} to:"
   describe_steps
   show_duration
 }
 
 
 main $@