Add blame support to waterfall.sh

tools/valgrind/waterfall.sh

 #!/bin/bash
 
 # Copyright (c) 2010 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 # This script can be used by waterfall sheriffs to fetch the status
 # of Valgrind bots on the memory waterfall and test if their local
 # suppressions match the reports on the waterfall.
 
 set -e
 
 THISDIR=$(dirname "${0}")
 LOGS_DIR=$THISDIR/waterfall.tmp
+WATERFALL_PAGE="http://build.chromium.org/buildbot/memory/builders"

[M-A Ruel, 2010/08/11 13:14:40]

Why is this script bash? I think it's starting to become time to convert it in
python.

Using simplejson.loads( urllib.urlopen(
'http://build.chromium.org/buildbot/memory/json/builders' ) ) looks simpler to
me.

 
 download() {
   # Download a file.
   # $1 = URL to download
   # $2 = Path to the output file
   # {{{1
-  if [ x$(which curl) != x ]; then
-    if ! curl -s -o "$2" "$1" ; then
+  if [ "$(which curl)" != "" ]
+  then
+    if ! curl -s -o "$2" "$1"
+    then
       echo
       echo "Failed to download '$1'... aborting"
       exit 1
     fi
-  elif [ x$(which wget) != x ]; then
-    if ! wget "$1" -O "$2" -q ; then
+  elif [ "$(which wget)" != "" ]
+  then
+    if ! wget "$1" -O "$2" -q
+    then
       echo
       echo "Failed to download '$1'... aborting"
       exit 1
     fi
   else
     echo "Need either curl or wget to download stuff... aborting"
     exit 1
   fi
   # }}}
 }
 
 fetch_logs() {
   # Fetch Valgrind logs from the waterfall {{{1
 
-  WATERFALL_PAGE="http://build.chromium.org/buildbot/memory/builders"
-
   rm -rf "$LOGS_DIR" # Delete old logs
   mkdir "$LOGS_DIR"
 
   echo "Fetching the list of builders..."
   download $WATERFALL_PAGE "$LOGS_DIR/builders"
   SLAVES=$(grep "<a href=\"builders\/" "$LOGS_DIR/builders" | \
            sed "s/.*<a href=\"builders\///" | sed "s/\".*//" | \
            sort | uniq)
 
   for S in $SLAVES
   do
     SLAVE_URL=$WATERFALL_PAGE/$S
-    SLAVE_NAME=$(echo $S | sed "s/%20/ /g" | sed "s/%28/(/g" | sed "s/%29/)/g")
+    SLAVE_NAME=$(echo $S | sed -e "s/%20/ /g" -e "s/%28/(/g" -e "s/%29/)/g")
     echo -n "Fetching builds by slave '${SLAVE_NAME}'"
     download $SLAVE_URL "$LOGS_DIR/slave_${S}"
 
     # We speed up the 'fetch' step by skipping the builds/tests which succeeded.
     # TODO(timurrrr): OTOH, we won't be able to check
     # if some suppression is not used anymore.
     LIST_OF_BUILDS=$(grep "<a href=\"\.\./builders/.*/builds/[0-9]\+.*failed" \
                      "$LOGS_DIR/slave_$S" | grep -v "failed compile" | \
                      sed "s/.*\/builds\///" | sed "s/\".*//" | head -n 2)
 
@@ skipping 16 matching lines @@
     echo " DONE"
   done
   # }}}
 }
 
 match_suppressions() {
   PYTHONPATH=$THISDIR/../python/google \
              python "$THISDIR/test_suppressions.py" "$LOGS_DIR/report_"*
 }
 
+find_blame() {
+  # Return the blame list for the first revision for a suppression occured in
+  # the logs of a given test on a given builder.
+  # $1 - Name of the builder.
+  # $2 - Name of the test.
+  # $3 - Hash of the suppression.
+  # {{{1
+
+  # Validate arguments.
+  if [ "$1" = "" -o "$2" = "" -o "$3" = "" ]
+  then
+    echo "Missing arguments, use waterfall.sh blame <builder> <test> <hash>"
+    exit 1
+  fi
+
+  SLAVE_NAME="$(echo "$1" | sed -e "s/ /%20/g" -e "s/(/%28/g" -e "s/)/%29/g")"
+  if [ ! -f "$LOGS_DIR/slave_$SLAVE_NAME" ]
+  then
+    echo "I don't know about builder '$1'. Did you run waterfall.sh fetch?"
+    exit 1
+  fi
+
+  TEST_NAME="memory%20test%3A%20$(echo "$2" | sed -e "s/ /%20/g")"
+  if ! ls -1 "$LOGS_DIR"/report_${SLAVE_NAME}_*_${TEST_NAME} >/dev/null 2>&1
+  then
+    echo "I don't know about the test '$2'. Did you run waterfall.sh fetch?"
+    exit 1
+  fi
+
+  HASH="#$3#"
+  if ! echo "$3" | grep -q -e '^[0-9A-F]\{8\}$'
+  then
+    echo "The hash '$3' looks strange, it should match ^[0-9A-F]{8}$"
+    exit 1
+  fi
+
+  # Determine the number of the latest build from our cache.
+  LATEST_BUILD=$(grep -l $HASH "$LOGS_DIR"/report_${SLAVE_NAME}_*_${TEST_NAME} |
+      sed "s/.*_\\([0-9]*\\)_${TEST_NAME}/\\1/" | sort -rn | head -n 1)
+
+  # Find a version where this hash does not occur.
+  STEP_SIZE=1
+  FIRST_BUILD=$LATEST_BUILD
+  DOWNLOAD_BASE="$WATERFALL_PAGE/$SLAVE_NAME/builds/"
+  LOG_FILE="/steps/$TEST_NAME/logs/stdio"
+  echo -n "Searching for latest revision without this hash"
+  while true
+  do
+    FIRST_BUILD=$(($FIRST_BUILD - $STEP_SIZE))
+    if [ $STEP_SIZE -lt 128 ]
+    then
+      STEP_SIZE=$(($STEP_SIZE * 2))
+    fi
+    echo -n "."
+    download \
+        "$DOWNLOAD_BASE$FIRST_BUILD$LOG_FILE" \
+        /tmp/waterfall.$$.report
+    if ! grep -ql $HASH /tmp/waterfall.$$.report
+    then
+      break
+    fi
+    LATEST_BUILD=$FIRST_BUILD
+    rm -f /tmp/waterfall.$$.report
+  done
+  rm -f /tmp/waterfall.$$.report
+
+  # Now search backwards for the first version where this hash occurs.
+  while [ $(($FIRST_BUILD + 1)) -ne $LATEST_BUILD ]
+  do
+    if [ $STEP_SIZE -gt 1 ]
+    then
+      STEP_SIZE=$(($STEP_SIZE / 2))
+    fi
+    TEST_BUILD=$(($FIRST_BUILD + $STEP_SIZE))
+    echo -n "."
+    download \
+        "$DOWNLOAD_BASE$TEST_BUILD$LOG_FILE" \
+        /tmp/waterfall.$$.report
+    if ! grep -ql $HASH /tmp/waterfall.$$.report
+    then
+      FIRST_BUILD=$TEST_BUILD
+    else
+      LATEST_BUILD=$TEST_BUILD
+    fi
+    rm -f /tmp/waterfall.$$.report
+  done
+  echo ""
+  echo "Hash $HASH occurs in build #$LATEST_BUILD for the first time."
+  echo -n "Receiving blame list"
+  download "$DOWNLOAD_BASE$LATEST_BUILD" /tmp/waterfall.$$.report
+  echo "."
+  echo -n "Revision: "
+  awk 'BEGIN{FS="[<> ]"}
+      /Got Revision:/{print $7}' /tmp/waterfall.$$.report
+  echo "Blamelist:"
+  awk 'BEGIN{ STATE=0; FS="[<>]" }
+      /Blamelist:/{STATE=1}
+      /<ol>/{if (STATE == 1) STATE=2}
+      /<li>/{if (STATE == 2) print $3}
+      /<\/ol>/{STATE=3}' /tmp/waterfall.$$.report
+  rm -f /tmp/waterfall.$$.report
+  # }}}
+}
+
 if [ "$1" = "fetch" ]
 then
   fetch_logs
 elif [ "$1" = "match" ]
 then
   match_suppressions
+elif [ "$1" = "blame" ]
+then
+  find_blame "$2" "$3" "$4"
 else
   THISNAME=$(basename "${0}")
-  echo "Usage: $THISNAME fetch|match"
+  echo "Usage: $THISNAME fetch|match|blame <builder> <test> <hash>"
   echo "  fetch - Fetch Valgrind logs from the memory waterfall"
   echo "  match - Test the local suppression files against the downloaded logs"
+  echo "  blame - Return the blame list for the revision where the suppression"
+  echo "          <hash> occured for the first time in the log for <test> on"
+  echo "          <builder>"
 fi