Improve waterfall.sh - fetch only reports and failed test names rather then t...

tools/valgrind/waterfall.sh

 #!/bin/bash
 
 # Copyright (c) 2010 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 # This script can be used by waterfall sheriffs to fetch the status
 # of Valgrind bots on the memory waterfall and test if their local
 # suppressions match the reports on the waterfall.
 
@@ skipping 27 matching lines @@
   else
     echo "Need either curl or wget to download stuff... aborting"
     exit 1
   fi
   # }}}
 }
 
 fetch_logs() {
   # Fetch Valgrind logs from the waterfall {{{1
 
+  # TODO(timurrrr,maruel): use JSON, see
+  # http://build.chromium.org/p/chromium.memory/json/help
+
   rm -rf "$LOGS_DIR" # Delete old logs
   mkdir "$LOGS_DIR"
 
   echo "Fetching the list of builders..."
   download $WATERFALL_PAGE "$LOGS_DIR/builders"
   SLAVES=$(grep "<a href=\"builders\/" "$LOGS_DIR/builders" | \
            sed "s/.*<a href=\"builders\///" | sed "s/\".*//" | \
            sort | uniq)
 
   for S in $SLAVES
   do
     SLAVE_URL=$WATERFALL_PAGE/$S
     SLAVE_NAME=$(echo $S | sed -e "s/%20/ /g" -e "s/%28/(/g" -e "s/%29/)/g")
     echo -n "Fetching builds by slave '${SLAVE_NAME}'"
     download $SLAVE_URL "$LOGS_DIR/slave_${S}"
 
     # We speed up the 'fetch' step by skipping the builds/tests which succeeded.
     # TODO(timurrrr): OTOH, we won't be able to check
     # if some suppression is not used anymore.
-    LIST_OF_BUILDS=$(grep "<a href=\"\.\./builders/.*/builds/[0-9]\+.*failed" \
-                     "$LOGS_DIR/slave_$S" | grep -v "failed compile" | \
-                     sed "s/.*\/builds\///" | sed "s/\".*//" | head -n 2)
+    LIST_OF_BUILDS=$(grep "rev.*<a href=\"\.\./builders/.*/builds/[0-9]\+" \
+                     "$LOGS_DIR/slave_$S" | head -n 2 | \
+                     grep "failed" | grep -v "failed compile" | \
+                     sed "s/.*\/builds\///" | sed "s/\".*//")
 
     for BUILD in $LIST_OF_BUILDS
     do
-      BUILD_RESULTS="$LOGS_DIR/slave_${S}_build_${BUILD}"
-      download $SLAVE_URL/builds/$BUILD "$BUILD_RESULTS"
-      LIST_OF_TESTS=$(grep "<a href=\"[0-9]\+/steps/memory.*/logs/stdio\"" \
-                      "$BUILD_RESULTS" | \
-                      sed "s/.*a href=\"//" | sed "s/\".*//")
-      for TEST in $LIST_OF_TESTS
+      # We'll fetch a few tiny URLs now, let's use a temp file.
+      TMPFILE=$(mktemp)
+      download $SLAVE_URL/builds/$BUILD "$TMPFILE"
+
+      REPORT_FILE="$LOGS_DIR/report_${S}_${BUILD}"
+      rm $REPORT_FILE 2>/dev/null || true  # make sure it doesn't exist

[jochen (gone - plz use gerrit), 2011/01/05 12:25:37]

use rm -f instead 2>/dev/null || true

[Timur Iskhodzhanov, 2011/01/17 15:28:26]

Done.

+
+      REPORT_URLS=$(grep -o "[0-9]\+/steps/memory.*/logs/[0-9A-F]\{16\}" \
+                    "$TMPFILE" || true)  # `true` is to succeed on empty output

[jochen (gone - plz use gerrit), 2011/01/05 12:25:37]

why does this need to succeed?

[Timur Iskhodzhanov, 2011/01/17 15:28:26]

There's "set -e" and grep returns failure if no lines has matched the regexp

On 2011/01/05 12:25:37, jochen wrote:

[jochen (gone - plz use gerrit), 2011/01/17 19:54:01]

But you don't use the return value, so it doesn't matter whether it succeeds or
not?

+      FAILED_TESTS=$(grep -o "[0-9]\+/steps/memory.*/logs/[A-Za-z0-9.]\+" \
+                     "$TMPFILE" | grep -v "[0-9A-F]\{16\}" | grep -v "stdio" \
+                     || true)
+
+      for REPORT in $REPORT_URLS
       do
-        REPORT_FILE=$(echo "report_${S}_$TEST" | sed "s/\/logs\/stdio//" | \
-                    sed "s/\/steps//" | sed "s/\//_/g")
-        echo -n "."
-        download $SLAVE_URL/builds/$TEST "$LOGS_DIR/$REPORT_FILE"
-        echo $SLAVE_URL/builds/$TEST >> "$LOGS_DIR/$REPORT_FILE"
+        # Add a hash line with the report so test_suppressions sees the hash.
+        # This is a workaround for http://crbug.com/68233#c1
+        echo "$REPORT" | sed -e "s/.*\//Suppression (error hash=#/" \
+                             -e "s/$/#):/" >> "$REPORT_FILE"
+        download "$SLAVE_URL/builds/$REPORT/text" "$TMPFILE"
+        # `cat -v sed` is a workaround the last "}^M" line getting stripped
+        cat -v "$TMPFILE" | sed "s/\^M$//" >> "$REPORT_FILE"

[jochen (gone - plz use gerrit), 2011/01/05 12:25:37]

if you want to convert CRLF to unix, you should pipe through tr -d '\r'

[Timur Iskhodzhanov, 2011/01/17 15:28:26]

Done.

+        echo >> "$REPORT_FILE"
       done
+
+      for FAILURE in $FAILED_TESTS
+      do
+        download "$SLAVE_URL/builds/$FAILURE/text" "$TMPFILE"
+        # `cat head` is a workaround for http://crbug.com/68233
+        cat "$TMPFILE" | head -n 1 | sed "s/://" | sed "s/^/FAILED:/" \
+          >> "$REPORT_FILE"
+      done
+
+      rm "$TMPFILE"
+      echo $SLAVE_URL/builds/$BUILD >> "$REPORT_FILE"
     done
     echo " DONE"
   done
   # }}}
 }
 
 match_suppressions() {
   PYTHONPATH=$THISDIR/../python/google \
              python "$THISDIR/test_suppressions.py" "$LOGS_DIR/report_"*
 }
 
 match_gtest_excludes() {
   for PLATFORM in "Linux" "Chromium%20Mac" "Chromium%20OS"
   do
     echo
     echo "Test failures on ${PLATFORM}:" | sed "s/%20/ /"
-    grep "\[  FAILED  \] .* ([0-9]\+ ms)" -R "$LOGS_DIR"/*${PLATFORM}* | \
+    grep -h -o "^FAILED:.*" -R "$LOGS_DIR"/*${PLATFORM}* | \
          grep -v "FAILS\|FLAKY" | \
-         sed -e "s/.*%20//" -e "s/_[1-9]\+:/:/" \
-             -e "s/\[  FAILED  \] //" -e "s/ ([0-9]\+ ms)//" -e "s/^/  /"
+         sed -e "s/^FAILED://" -e "s/^/  /"
     # Don't put any operators between "grep | sed" and "RESULT=$PIPESTATUS"
     RESULT=$PIPESTATUS
 
     if [ "$RESULT" == 1 ]
     then
       echo "  None!"
     else
       echo
       echo "  Note: we don't check for failures already excluded locally yet"
       echo "  TODO(timurrrr): don't list tests we've already excluded locally"
@@ skipping 119 matching lines @@
   find_blame "$2" "$3" "$4"
 else
   THISNAME=$(basename "${0}")
   echo "Usage: $THISNAME fetch|match|blame <builder> <test> <hash>"
   echo "  fetch - Fetch Valgrind logs from the memory waterfall"
   echo "  match - Test the local suppression files against the downloaded logs"
   echo "  blame - Return the blame list for the revision where the suppression"
   echo "          <hash> occured for the first time in the log for <test> on"
   echo "          <builder>"
 fi