New LLVM merging bots

tools/llvm/merge-tool.sh

 #!/bin/bash
-# Copyright 2010 The Native Client Authors.  All rights reserved.
-# Use of this source code is governed by a BSD-style license that can
-# be found in the LICENSE file.
+# Copyright 2011 The Native Client Authors.  All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
 #
 ######################################################################
 #
 # This script facilitates merging changes from the LLVM subversion
 # repository into the PNaCl Mercurial repository.
 #
 ######################################################################
 set -o nounset
 set -o errexit
 
 # Make sure this script is run from the right place
 if [[ $(basename $(pwd)) != "native_client" ]] ; then
   echo "ERROR: run this script from the native_client/ dir"
   exit -1
 fi
 
 source tools/llvm/common-tools.sh
+readonly NACL_ROOT="$(pwd)"
+SetScriptPath "${NACL_ROOT}/tools/llvm/merge-tool.sh"
+SetLogDirectory "${NACL_ROOT}/toolchain/hg-log"
+readonly SCRIPT_PATH="$0"
 ######################################################################
 
 # Location of the mercurial repositories
 # These should match the values in utman.sh
 readonly TC_SRC="$(pwd)/hg"
-readonly TC_SRC_LLVM="${TC_SRC}/llvm"
-readonly TC_SRC_LLVM_GCC="${TC_SRC}/llvm-gcc"
+readonly TC_SRC_UPSTREAM="${TC_SRC}/upstream"
 
-# Location of the upstream LLVM repository
-readonly MASTER_LLVM_BASE="http://llvm.org/svn/llvm-project"
-readonly MASTER_LLVM_URL="${MASTER_LLVM_BASE}/llvm/trunk"
-readonly MASTER_LLVM_GCC_URL="${MASTER_LLVM_BASE}/llvm-gcc-4.2/trunk"
+readonly PREDIFF="${TC_SRC}/prediff"
+readonly POSTDIFF="${TC_SRC}/postdiff"
 
-readonly MASTER_LLVM="${TC_SRC}/master-llvm"
-readonly MASTER_LLVM_GCC="${TC_SRC}/master-llvm-gcc"
+# These must be specified in the environment
+readonly MASTER_LLVM=${MASTER_LLVM:-}

[jvoung - send to chromium..., 2011/07/29 00:38:32]

So, on the bots these values are set in .bashrc or something like that? It is
not specified in the invocations found in buildbot_pnacl_merge.sh

Are they "TC_SRC" directories, or are they URLs? Could prefix it with "TC_SRC"
or something if it is a directory of sources.

[pdox, 2011/07/29 00:59:58]

They are the upstream SVN directories, not part of the toolchain source tree.

I'm not sure where they are going to come from. I am waiting on BradN to setup
the new waterfall, which triggers on LLVM svn revisions. Most likely, the
buildbot slave will download the svn repositories and keep it up to date. And
then MASTER_LLVM, MASTER_LLVM_GCC can be hard-coded to a fixed location in
buildbot_pnacl_merge.sh (or PWD?).

I've added a comment to clarify.

+readonly MASTER_LLVM_GCC=${MASTER_LLVM_GCC:-}
 
+# TODO(pdox): Refactor repository checkout into a separate script
+#             so that we don't need to invoke utman.
+utman() {
+  UTMAN_UPSTREAM=true "${NACL_ROOT}"/tools/llvm/utman.sh "$@"
+}
+hg-checkout-upstream() { utman hg-checkout-upstream "$@" ; }
 
-BASEDIR="$(pwd)"
-utman() {
-  pushd "${BASEDIR}" > /dev/null
-  tools/llvm/utman.sh "$@"
-  popd > /dev/null
+check-svn-repos() {
+  if [ -z "${MASTER_LLVM}" ] ||
+     [ -z "${MASTER_LLVM_GCC}" ]; then
+    Fatal "You must set environmental variables MASTER_LLVM and MASTER_LLVM_GCC"
+  fi
 }
-hg-pull-llvm() { utman hg-pull-llvm "$@" ; }
-hg-pull-llvm-gcc() { utman hg-pull-llvm-gcc "$@" ; }
-hg-checkout-llvm() { utman hg-checkout-llvm "$@" ; }
-hg-checkout-llvm-gcc() { utman hg-checkout-llvm-gcc "$@" ; }
 
-#@ all                   - Do LLVM and LLVM-GCC merge (all steps)
-all() {
+#@ auto                  - Non-interactive merge
+auto() {
+  INTERACTIVE_MERGE=false
+  export DISPLAY=""
+  merge-all
+}
+
+#@ manual                - Interactive merge
+manual() {
+  INTERACTIVE_MERGE=true
+  merge-all
+}
+
+#+ merge-all             - Merge all the things
+merge-all() {
+  check-svn-repos
+
+  hg-checkout-upstream
   assert-clean
+  setup-hgrc
 
-  checkout-all
-  choose-revision
+  MERGE_REVISION=$(get-merge-revision)
+  Banner "MERGE REVISION: ${MERGE_REVISION}"
 
-  update-llvm-gcc-vendor
-  merge-llvm-gcc
-  diff-diff-llvm-gcc
+  generate-pre-diff
 
-  update-llvm-vendor
-  merge-llvm
-  diff-diff-llvm
+  commit-vendor
+  hg-merge
+
+  generate-post-diff
+
+  if ${INTERACTIVE_MERGE}; then
+    vim-diff-diff
+  else
+    dump-diff-diff
+  fi
 
   echo "********************************************************************"
   echo "The llvm and llvm-gcc working directories are now in a merged state."
   echo "Before you commit and push, you should build PNaCl and run all tests."
-  echo "For example:"
-  echo "  tools/llvm/utman.sh clean"
-  echo "  tools/llvm/utman.sh all"
-  echo "  tools/llvm/utman.sh test-all"
-  echo "  tests/spec2k/bot_spec.sh 2 <spec-dir>"
-  echo "  tests/spec2k/bot_spec.sh 3 <spec-dir>"
   echo ""
   echo "Expect lots of bugs. You may need to fix and rebuild several times."
   echo "When you are confident all tests are passing, you can commit and push"
-  echo "the merged working directories with:"
+  echo "the merged working directory with:"
   echo "  tools/llvm/merge-tool.sh final-commit"
   echo "********************************************************************"
 }
 
+setup-hgrc() {
+  cp "${NACL_ROOT}/tools/llvm/hgrc" \
+     "${TC_SRC_UPSTREAM}/.hg/hgrc"
+}
+
 assert-clean() {
-  if [ -d "${MASTER_LLVM}" ]; then
-    svn-assert-no-changes "${MASTER_LLVM}"
-  fi
+  svn-assert-no-changes "${MASTER_LLVM}"
+  svn-assert-no-changes "${MASTER_LLVM_GCC}"
 
-  if [ -d "${MASTER_LLVM_GCC}" ]; then
-    svn-assert-no-changes "${MASTER_LLVM_GCC}"
-  fi
-
-  if [ -d "${TC_SRC_LLVM}" ]; then
-    hg-assert-no-changes "${TC_SRC_LLVM}"
-    hg-assert-no-outgoing "${TC_SRC_LLVM}"
-  fi
-
-  if [ -d "${TC_SRC_LLVM_GCC}" ]; then
-    hg-assert-no-changes "${TC_SRC_LLVM_GCC}"
-    hg-assert-no-outgoing "${TC_SRC_LLVM_GCC}"
-  fi
+  hg-assert-no-changes "${TC_SRC_UPSTREAM}"
+  hg-assert-no-outgoing "${TC_SRC_UPSTREAM}"
 }
 
 #@ clean                 - Clean/revert mercurial repositories
 clean() {
   StepBanner "CLEAN - Cleaning repositories"
-  Banner "WARNING: All local changes to hg/llvm and hg/llvm-gcc will be erased"
-  if ! confirm-yes "Are you sure you want to do this?" ; then
-    echo "Cancelled"
-    exit -1
-  fi
-  if ! confirm-yes "Are you really, really sure you want do this?" ; then
-    echo "Cancelled"
-    exit -1
-  fi
-  clean-llvm
-  clean-llvm-gcc
+  clean-upstream
 }
 
-#+ clean-llvm
-clean-llvm() {
-  StepBanner "CLEAN" "Cleaning hg llvm repository"
-  rm -rf "${TC_SRC_LLVM}"
+#+ clean-upstream
+clean-upstream() {
+  StepBanner "CLEAN" "Cleaning hg upstream repository"
+  rm -rf "${TC_SRC_UPSTREAM}"
 }
 
-#+ clean-llvm-gcc
-clean-llvm-gcc() {
-  StepBanner "CLEAN" "Cleaning hg llvm-gcc repository"
-  rm -rf "${TC_SRC_LLVM_GCC}"
-}
-
-#@ checkout-all          - Checkout repositories
-checkout-all() {
-  StepBanner "checkout-all - Checkout all repositories"
-
-  # Checkout LLVM repositories
-  StepBanner "checkout-all" "Checking out SVN repositories"
-  svn-checkout "${MASTER_LLVM_URL}" "${MASTER_LLVM}"
-  svn-checkout "${MASTER_LLVM_GCC_URL}" "${MASTER_LLVM_GCC}"
-
-  # Checkout the hg repositories
-  StepBanner "checkout-all" "Checking out HG repositories"
-  hg-checkout-llvm
-  hg-checkout-llvm-gcc
-}
-
-#+ get-revision       -  Get the current SVN revision
-get-revision() {
+#+ get-merge-revision    - Get the current SVN revision
+get-merge-revision() {
   local llvm_rev=$(svn-get-revision "${MASTER_LLVM}")
   local llvm_gcc_rev=$(svn-get-revision "${MASTER_LLVM_GCC}")
 
   if [ "${llvm_rev}" -ne "${llvm_gcc_rev}" ]; then
     echo -n "Error: Unexpected mismatch between " 1>&2
     echo    "SVN revisions of llvm and llvm-gcc" 1>&2
     exit -1
   fi
   echo "${llvm_rev}"
 }
 
-#@ choose-revision       - Choose LLVM revision
-choose-revision() {
-  StepBanner "choose-revision - Choose LLVM Revision"
-  echo
-  echo "Go to http://google1.osuosl.org:8011/ for LLVM build status"
-  echo
-  local rev
-  while true; do
-    echo -n "Please enter an LLVM revision (or 'tip'): "
-    read rev
-    if [[ "$rev" == "tip" ]]; then
-      break
-    fi
-    if [[ "$rev" =~ ^[0-9]+$ ]]; then
-      break
-    fi
-    echo "Invalid input."
-  done
-
-  # Update the SVN repositories to ${rev}
-  StepBanner "choose-revision" "Updating LLVM repository to ${rev}"
-  svn-update "${MASTER_LLVM}" "${rev}"
-  svn-update "${MASTER_LLVM_GCC}" "${rev}"
+#+ generate-pre-diff     - Generate vendor:pnacl-sfi diff prior to merge
+generate-pre-diff() {
+  spushd "${TC_SRC_UPSTREAM}"
+  hg diff -r vendor:pnacl-sfi &> "${PREDIFF}"
+  spopd
 }
 
-#@ update-llvm-vendor    - Apply update to vendor branch for llvm
-update-llvm-vendor() {
-  local fnid="update-llvm-vendor"
-  StepBanner "${fnid} - Freshen hg 'vendor' branch"
+#+ generate-post-diff    - Generate vendor:pnacl-sfi diff after merge
+generate-post-diff() {
+  spushd "${TC_SRC_UPSTREAM}"
+  hg diff -r vendor &> "${POSTDIFF}"
+  spopd
+}
 
-  StepBanner "${fnid}" "Verifying repository state"
-  svn-assert-no-changes "${MASTER_LLVM}"
-  hg-assert-no-changes "${TC_SRC_LLVM}"
-  hg-assert-no-outgoing "${TC_SRC_LLVM}"
+#@ commit-vendor         - Apply new commit to vendor branch
+commit-vendor() {
+  local stepid="commit-vendor"
+  StepBanner "Committing vendor"
 
-  StepBanner "${fnid}" "hg pull (llvm)"
-  hg-pull-llvm
+  StepBanner "${stepid}" "Switching to hg vendor branch"
 
-  StepBanner "${fnid}" "switch to hg branch vendor (llvm)"
-  hg-update "${TC_SRC_LLVM}" vendor
-  StepBanner "${fnid}" "Delete existing vendor source"
-  rm -rf "${TC_SRC_LLVM}/llvm-trunk"
+  hg-update "${TC_SRC_UPSTREAM}" vendor
 
-  StepBanner "${fnid}" "Exporting svn to hg (llvm)"
-  RunWithLog "${fnid}" \
-    svn export "${MASTER_LLVM}" "${TC_SRC_LLVM}/llvm-trunk"
+  StepBanner "${stepid}" "Exporting svn to hg"
+  rm -rf "${TC_SRC_UPSTREAM}/llvm"
+  svn export "${MASTER_LLVM}" "${TC_SRC_UPSTREAM}/llvm"
 
-  StepBanner "${fnid}" "Updating hg file list (llvm)"
-  spushd "${TC_SRC_LLVM}"
-  RunWithLog "${fnid}" hg add
-  RunWithLog "${fnid}" hg remove -A
+  rm -rf "${TC_SRC_UPSTREAM}/llvm-gcc"
+  svn export "${MASTER_LLVM_GCC}" "${TC_SRC_UPSTREAM}/llvm-gcc"
+
+  StepBanner "Updating hg file list"

[jvoung - send to chromium..., 2011/07/29 00:38:32]

${stepid} ?

[pdox, 2011/07/29 00:59:58]

Done.

+  spushd "${TC_SRC_UPSTREAM}"
+  hg add
+  hg remove -A
   spopd
 
-  hg-status-check LLVM "${TC_SRC_LLVM}"
-
-  local rev=$(get-revision)
-  StepBanner "${fnid} - Commit hg 'vendor' branch (llvm)"
-  hg-commit "${TC_SRC_LLVM}" "Updating vendor to r${rev}"
+  StepBanner "${stepid}" "Committing vendor branch"
+  hg-commit "${TC_SRC_UPSTREAM}" "Updating vendor to r${MERGE_REVISION}"

[jvoung - send to chromium..., 2011/07/29 00:38:32]

Might be better to just $(get-merge-revision) wherever you need it instead of
making it a global and relying on it being set before you get to this point.

[pdox, 2011/07/29 00:59:58]

Technically, I always want to run get-merge-revision at least once, since it
does a sanity check to make sure the revision numbers match.

I've added the setting of MERGE_REVISION to check-svn-repo, and arranged for
check-svn-repo to run before the main command.

 }
 
-#@ update-llvm-gcc-vendor - Apply update to vendor branch for llvm-gcc
-update-llvm-gcc-vendor() {
-  local fnid="update-llvm-gcc-vendor"
-  StepBanner "${fnid}" "hg pull (llvm-gcc)"
-  hg-pull-llvm-gcc
+#@ hg-merge              - Merge and resolve conflicts for llvm
+hg-merge() {
+  StepBanner "hg-merge - Merge and resolve conflicts"
 
-  StepBanner "${fnid}" "Verifying repository state"
-  svn-assert-no-changes "${MASTER_LLVM_GCC}"
-  hg-assert-no-changes "${TC_SRC_LLVM_GCC}"
-  hg-assert-no-outgoing "${TC_SRC_LLVM_GCC}"
+  StepBanner "hg-merge" "Switching to pnacl-sfi branch"
+  hg-update "${TC_SRC_UPSTREAM}" pnacl-sfi
+  hg-assert-no-changes "${TC_SRC_UPSTREAM}"
 
-  StepBanner "${fnid}" "switch to hg branch vendor (llvm-gcc)"
-  hg-update "${TC_SRC_LLVM_GCC}" vendor
-
-  StepBanner "${fnid}" "Delete existing vendor source"
-  rm -rf "${TC_SRC_LLVM_GCC}/llvm-gcc-4.2"
-
-  StepBanner "${fnid}" "Exporting svn to hg (llvm-gcc)"
-  RunWithLog "${fnid}" \
-    svn export "${MASTER_LLVM_GCC}" "${TC_SRC_LLVM_GCC}/llvm-gcc-4.2"
-
-  StepBanner "${fnid}" "Updating hg file list (llvm-gcc)"
-  spushd "${TC_SRC_LLVM_GCC}"
-  RunWithLog "${fnid}" hg add
-  RunWithLog "${fnid}" hg remove -A
-  spopd
-
-  hg-status-check LLVM-GCC "${TC_SRC_LLVM_GCC}"
-
-  local rev=$(get-revision)
-  StepBanner "${fnid} - Commit hg 'vendor' branch (llvm-gcc)"
-  hg-commit "${TC_SRC_LLVM_GCC}" "Updating vendor to r${rev}"
-}
-
-#@ merge-llvm-gcc        - Merge and resolve conflicts for llvm-gcc
-merge-llvm-gcc() {
-  StepBanner "merge-llvm-gcc - Merge and resolve conflicts"
-
-  StepBanner "merge-llvm-gcc" "Switch to pnacl-sfi branch"
-  hg-assert-no-changes "${TC_SRC_LLVM_GCC}"
-  hg-update "${TC_SRC_LLVM_GCC}" pnacl-sfi
-
-  StepBanner "merge-llvm-gcc" "Merging vendor into pnacl-sfi"
-  spushd "${TC_SRC_LLVM_GCC}"
+  StepBanner "hg-merge" "Merging vendor into pnacl-sfi"
+  spushd "${TC_SRC_UPSTREAM}"
   hg merge -r vendor
   spopd
 }
 
-#@ merge-llvm            - Merge and resolve conflicts for llvm
-merge-llvm() {
-  StepBanner "merge-llvm - Merge and resolve conflicts"
-
-  StepBanner "merge-llvm" "Switch to pnacl-sfi branch"
-  hg-assert-no-changes "${TC_SRC_LLVM}"
-  hg-update "${TC_SRC_LLVM}" pnacl-sfi
-
-  StepBanner "merge-llvm" "Merging vendor into pnacl-sfi"
-  spushd "${TC_SRC_LLVM}"
-  hg merge -r vendor
-  spopd
+#@ vim-diff-diff         - Review diff-diff using vim
+vim-diff-diff() {
+  vimdiff "${PREDIFF}" "${POSTDIFF}"
 }
 
-#@ diff-diff-llvm        - Review diff-diff
-diff-diff-llvm() {
-  (
-    echo "Type 'q' to exit less"
-    echo "---------------------------------------------------"
-    tools/llvm/diff-diff.py "${TC_SRC_LLVM}"
-  ) 2>&1 | less
-
-  if ! confirm-yes "Does the diff-diff for LLVM look correct"; then
-    echo "Cancelling."
-    echo "hg repositories remain in uncommitted state."
-    echo "Use 'tools/llvm/merge-tool.sh clean' to clean them"
-    exit -1
-  fi
-}
-
-#@ diff-diff-llvm-gcc    - Review diff-diff
-diff-diff-llvm-gcc() {
-  (
-    echo "Type 'q' to exit less"
-    echo "---------------------------------------------------"
-    tools/llvm/diff-diff.py "${TC_SRC_LLVM_GCC}"
-  ) 2>&1 | less
-
-  if ! confirm-yes "Does the diff-diff for LLVM-GCC look correct"; then
-    echo "Cancelling."
-    echo "hg repositories remain in uncommitted state."
-    echo "Use 'tools/llvm/merge-tool.sh clean' to clean them"
-    exit -1
-  fi
+#@ dump-diff-diff        - Review diff-diff
+dump-diff-diff() {
+  diff "${PREDIFF}" "${POSTDIFF}"
 }
 
 final-commit() {
+  check-svn-repos
+  MERGE_REVISION=$(get-merge-revision)
+
   StepBanner "final-commit" "Committing and pushing merge"

[jvoung - send to chromium..., 2011/07/29 00:38:32]

Could print the MERGE_REVISION somewhere before all the interactive stuff.

[pdox, 2011/07/29 00:59:58]

done, I think? Added the banner to check-svn-repos

-  hg-assert-is-merge "${TC_SRC_LLVM}"
-  hg-assert-is-merge "${TC_SRC_LLVM_GCC}"
-  hg-assert-branch "${TC_SRC_LLVM}" pnacl-sfi
-  hg-assert-branch "${TC_SRC_LLVM_GCC}" pnacl-sfi
+  hg-assert-is-merge "${TC_SRC_UPSTREAM}"
+  hg-assert-branch "${TC_SRC_UPSTREAM}" pnacl-sfi
 
-  Banner "CAUTION: This step will COMMIT and PUSH changes to the repository."
-  echo
-  if ! confirm-yes "Is the merged working directory passing all tests?" ; then
-    echo "Cancelled"
-    exit -1
-  fi
-  if ! confirm-yes "Are you really sure you want to do this?" ; then
-    echo "Cancelled"
-    exit -1
+  if ${INTERACTIVE_MERGE}; then
+    Banner "CAUTION: This step will COMMIT and PUSH changes to the repository."
+    echo
+    if ! confirm-yes "Is the merged working directory passing all tests?" ; then
+      echo "Cancelled"
+      exit -1
+    fi
+    if ! confirm-yes "Are you really sure you want to do this?" ; then
+      echo "Cancelled"
+      exit -1
+    fi
   fi
 
-  local rev=$(get-revision)
+  StepBanner "final-commit" "Committing pnacl-sfi branch"
+  hg-commit "${TC_SRC_UPSTREAM}" "Merged up to r${MERGE_REVISION}"
 
-  StepBanner "final-commit - Committing hg pnacl-sfi branch (llvm)"
-  hg-commit "${TC_SRC_LLVM}" "Merged up to r${rev}"
-
-  StepBanner "final-commit - Committing hg pnacl-sfi branch (llvm-gcc)"
-  hg-commit "${TC_SRC_LLVM_GCC}" "Merged up to r${rev}"
-
-  StepBanner "final-commit - Pushing (llvm)"
-  hg-push "${TC_SRC_LLVM}"
-
-  StepBanner "final-commit - Pushing (llvm-gcc)"
-  hg-push "${TC_SRC_LLVM_GCC}"
-}
-
-#+ hg-status-check <name> <repo>   - Allow the user to check hg status for problems
-hg-status-check() {
-  local name="$1"
-  local repo="$2"
-  spushd "${repo}"
-  (
-    echo "Please verify hg status for ${name}:"
-    echo "Hit q to exit 'less'"
-    echo "---------------------------------------------------"
-    hg status
-    echo "---------------------------------------------------"
-  ) 2>&1 | less
-  spopd
-
-  if ! confirm-yes "Does the status for ${name} look correct"; then
-    echo "Cancelling vendor update."
-    echo "hg repositories remain in uncommitted state."
-    echo "Use 'tools/llvm/merge-tool.sh clean' to clean them"
-    exit -1
-  fi
+  StepBanner "final-commit" "Pushing to hg/upstream"
+  hg-push "${TC_SRC_UPSTREAM}"
 }
 
 #@ help                  - Usage information.
 help() {
   Usage
 }
 
 [ $# = 0 ] && set -- help  # Avoid reference to undefined $1.
 if [ "$(type -t $1)" != "function" ]; then
   echo "ERROR: unknown function '$1'." >&2
   echo "For help, try:"
   echo "    $0 help"
   exit 1
 fi
 
 "$@"