Add new testing framework along with a few micro-tests.

util/partial_writes_ec.sh

Index: util/partial_writes_ec.sh
diff --git a/util/partial_writes_ec.sh b/util/partial_writes_ec.sh
new file mode 100755
index 0000000000000000000000000000000000000000..47a0eb52825452aaadfd61ef4a65b01948eb39ed
--- /dev/null
+++ b/util/partial_writes_ec.sh
@@ -0,0 +1,322 @@
+#!/bin/sh
+#
+# Copyright (C) 2010 Google Inc.
+# Written by David Hendricks for Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+#
+# ec.sh: This will attempt to stress partial write capabilities for EC firmware
+# ROMs. There are two major parts to the test:
+# 1. Write an alternative firmware image to the "fw" region specified in a
+#    layout file.
+# 2. Stress partial writes using a known pattern in an unused region of the
+#    EC firmware ROM with 128KB of space.
+#
+# REQUIRED ENVIRONMENT VARIABLES
+# ------------------------------
+# Due to the way the test framework copies tests, the environment variables must
+# specify absolute paths to files they reference.
+#
+# $ALT_EC_IMAGE -- Required environment variable
+# This script requires an alternative EC firmware. Most ECs actively read code
+# and data from the firmware ROM during run-time. An example of an EC firmware
+# update process for supported ECs:
+# 1. Enter update mode -- Copy relevant code and data to internal RAM
+# 2. Enable programming interface
+# 3. When host is done programming ROM, issue "exit update mode"
+# 4. Re-load code/data from ROM.
+#
+# Step 4 can cause failure if we attempt to test using patterns. Instead, we
+# must provide the EC with usable code. For blackbox testing, this essentially
+# means replacing the current firmware image with a different image which still
+# works.
+#
+# $LAYOUT_FILE -- Another required environment variable
+# A layout file is required for two reasons:
+# 1. We need a way of knowing which regions are safe to leave in a clobbered
+#    state after exiting flash update mode. They will be labeled "unused".
+# 2. We need a way of knowing where the EC firmware resides since some ECs
+#    boot from bottom of ROM and others from top of ROM. This will be the
+#    region labeled "fw"
+#
+# The following example is for a 1MB ROM with 128KB occupied by EC firmware.
+# The EC in this example loads firmware from the lowest address.
+# 0x000000 0x01ffff fw
+# 0x020000 0x0fffff unused
+
+LOGFILE="${0}.log"
+ZERO_4K="00_4k.bin"
+FF_4K="ff_4k.bin"
+FF_4K_TEXT="ff_4k.txt"
+
+TESTFILE="test.bin"
+
+ec_fail()
+{
+	echo "$1" >> ${LOGFILE}
+	echo "$0: failed" >> ${LOGFILE}
+	exit ${EXIT_FAILURE}
+}
+
+which uuencode > /dev/null
+if [ "$?" != "0" ] ; then
+	ec_fail "uuencode is required to use this script"
+fi
+
+# FIXME: This is a chromium os -ism. Most distros don't strip out "diff".
+which diff > /dev/null
+if [ "$?" != "0" ] ; then
+	ec_fail "diff is required to use this script"
+fi
+
+# FIXME: extra chromium os paranoia
+which printf > /dev/null
+if [ "$?" != "0" ] ; then
+	ec_fail "printf is required to use this script"
+fi
+
+echo "User-provided \$ALT_EC_IMAGE: ${ALT_EC_IMAGE}" >> ${LOGFILE}
+if [ -z "$ALT_EC_IMAGE" ] || [ ! -e "$ALT_EC_IMAGE" ]; then
+	ec_fail "Please provide an alternative EC firmware image using the ALT_EC_IMAGE environment variable."
+fi
+
+echo "User-provided \$LAYOUT_FILE: ${LAYOUT_FILE}" >> ${LOGFILE}
+if [ -z "$LAYOUT_FILE" ] || [ ! -e "$LAYOUT_FILE" ]; then
+	ec_fail "Please provide a layout file using the LAYOUT_FILE environment variable"
+fi
+
+ranges=$(awk '{ if ( $2 == "unused" ) print $1 }' "${LAYOUT_FILE}")
+range_found=0
+for range in $ranges; do
+	start=$(echo $range | awk -F":" '{ print $1 }')
+	end=$(echo $range | awk -F":" '{ print $2 }')
+	len=$((${end} - ${start}))
+
+	echo "Testing if range is usable: ${start}:${end}, len=${len}" >> ${LOGFILE}
+	if [ ${len} -lt $((0x1ffff)) ]; then
+		continue
+	else
+		range_found=1
+		break
+	fi
+done
+
+if [ $range_found -ne 1 ]; then
+	ec_fail "No suitable unused range found"
+else
+	echo "Found usable range: ${start}:${end}, len=${len}"
+fi
+
+# Make 4k worth of 0xff bytes
+echo "begin 640 $FF_4K" > "$FF_4K_TEXT"
+i=0
+while [ $i -le 90 ]; do
+	echo "M____________________________________________________________" >> "$FF_4K_TEXT"
+	i=$((${i} + 1))
+done
+echo "!_P``" >> "$FF_4K_TEXT"
+echo "\`" >> "$FF_4K_TEXT"
+echo "end" >> "$FF_4K_TEXT"
+uudecode -o "$FF_4K" "$FF_4K_TEXT"
+rm -f "$FF_4K_TEXT"
+
+# Make 4k worth of 0x00 bytes
+dd if=/dev/zero of="$ZERO_4K" bs=1 count=4096 2> /dev/null
+echo "ffh pattern written in ${FF_4K}"
+echo "00h pattern written in ${ZERO_4K}"
+
+# Actual tests are performed below.
+#
+NUM_REGIONS=16
+
+# Make a layout - 4K regions on 4K boundaries. This will test basic
+# functionality of erasing and writing specific blocks.
+#FIXME: the layout file is in dec rather than hex. Yuck! Do we care enough
+# to filter the output thru bc or something and convert to hex?
+echo "
+`printf 0x%x $((${start} + 0x00000))`:`printf 0x%x $((${start} + 0x00fff))` 00_0

[Stefan Reinauer, 2010/11/23 19:05:30]

Would it make sense to pack this into a separate function?

[hailfinger, 2010/11/23 20:05:34]

Can you try to keep the diff to the normal partial write test small? This may
require changes in both scripts, but I believe this is worth it.

[Stefan Reinauer, 2010/11/23 21:14:14]

Yepp. It would be nice to see some kind of generic pattern and layout generator
that does something useful. Also maybe the printf stuff could be compressed into
some kind of loop

[dhendrix, 2010/11/24 02:48:14]

I managed to compress this stuff into a loop -- good idea!

Unfortunately, I didn't see an obvious way to break the loop into a function and
decrease complexity at the same time. But it definitely looks a lot better now
with the loops and is less prone to stupid copy/paste errors.

+`printf 0x%x $((${start} + 0x01000))`:`printf 0x%x $((${start} + 0x01fff))` ff_0
+
+`printf 0x%x $((${start} + 0x02000))`:`printf 0x%x $((${start} + 0x02fff))` 00_1
+`printf 0x%x $((${start} + 0x03000))`:`printf 0x%x $((${start} + 0x03fff))` ff_1
+
+`printf 0x%x $((${start} + 0x04000))`:`printf 0x%x $((${start} + 0x04fff))` 00_2
+`printf 0x%x $((${start} + 0x05000))`:`printf 0x%x $((${start} + 0x05fff))` ff_2
+
+`printf 0x%x $((${start} + 0x06000))`:`printf 0x%x $((${start} + 0x06fff))` 00_3
+`printf 0x%x $((${start} + 0x07000))`:`printf 0x%x $((${start} + 0x07fff))` ff_3
+
+`printf 0x%x $((${start} + 0x08000))`:`printf 0x%x $((${start} + 0x08fff))` 00_4
+`printf 0x%x $((${start} + 0x09000))`:`printf 0x%x $((${start} + 0x09fff))` ff_4
+
+`printf 0x%x $((${start} + 0x0a000))`:`printf 0x%x $((${start} + 0x0afff))` 00_5
+`printf 0x%x $((${start} + 0x0b000))`:`printf 0x%x $((${start} + 0x0bfff))` ff_5
+
+`printf 0x%x $((${start} + 0x0c000))`:`printf 0x%x $((${start} + 0x0cfff))` 00_6
+`printf 0x%x $((${start} + 0x0d000))`:`printf 0x%x $((${start} + 0x0dfff))` ff_6
+
+`printf 0x%x $((${start} + 0x0e000))`:`printf 0x%x $((${start} + 0x0efff))` 00_7
+`printf 0x%x $((${start} + 0x0f000))`:`printf 0x%x $((${start} + 0x0ffff))` ff_7
+
+`printf 0x%x $((${start} + 0x10000))`:`printf 0x%x $((${start} + 0x10fff))` 00_8
+`printf 0x%x $((${start} + 0x11000))`:`printf 0x%x $((${start} + 0x11fff))` ff_8
+
+`printf 0x%x $((${start} + 0x12000))`:`printf 0x%x $((${start} + 0x12fff))` 00_9
+`printf 0x%x $((${start} + 0x13000))`:`printf 0x%x $((${start} + 0x13fff))` ff_9
+
+`printf 0x%x $((${start} + 0x14000))`:`printf 0x%x $((${start} + 0x14fff))` 00_10
+`printf 0x%x $((${start} + 0x15000))`:`printf 0x%x $((${start} + 0x15fff))` ff_10
+
+`printf 0x%x $((${start} + 0x16000))`:`printf 0x%x $((${start} + 0x16fff))` 00_11
+`printf 0x%x $((${start} + 0x17000))`:`printf 0x%x $((${start} + 0x17fff))` ff_11
+
+`printf 0x%x $((${start} + 0x18000))`:`printf 0x%x $((${start} + 0x18fff))` 00_12
+`printf 0x%x $((${start} + 0x19000))`:`printf 0x%x $((${start} + 0x19fff))` ff_12
+
+`printf 0x%x $((${start} + 0x1a000))`:`printf 0x%x $((${start} + 0x1afff))` 00_13
+`printf 0x%x $((${start} + 0x1b000))`:`printf 0x%x $((${start} + 0x1bfff))` ff_13
+
+`printf 0x%x $((${start} + 0x1c000))`:`printf 0x%x $((${start} + 0x1cfff))` 00_14
+`printf 0x%x $((${start} + 0x1d000))`:`printf 0x%x $((${start} + 0x1dfff))` ff_14
+
+`printf 0x%x $((${start} + 0x1e000))`:`printf 0x%x $((${start} + 0x1efff))` 00_15
+`printf 0x%x $((${start} + 0x1f000))`:`printf 0x%x $((${start} + 0x1ffff))` ff_15
+" > layout_ec_4k_aligned.txt
+
+cp "${BACKUP}" "$TESTFILE"
+i=0
+while [ $i -lt $NUM_REGIONS ] ; do
+	tmpstr="aligned region ${i} test: "
+	offset=$((${start} + $((${i} * 8192))))
+	dd if=${ZERO_4K} of=${TESTFILE} bs=1 conv=notrunc seek=${offset} 2> /dev/null
+	dd if=${FF_4K} of=${TESTFILE} bs=1 conv=notrunc seek=$((${offset} + 4096)) 2> /dev/null
+
+	./flashrom ${FLASHROM_PARAM} -l layout_ec_4k_aligned.txt -i 00_${i} -i ff_${i} -w "$TESTFILE" 2> /dev/null
+	if [ "$?" != "0" ] ; then
+		ec_fail "failed to flash region"
+	fi
+
+	# download the entire ROM image and use diff to compare to ensure
+	# flashrom logic does not violate user-specified regions
+	flashrom ${FLASHROM_PARAM} -r difftest.bin 2> /dev/null
+	diff -q difftest.bin "$TESTFILE"
+	if [ "$?" != "0" ] ; then
+		ec_fail "failed diff test"
+	fi
+	rm -f difftest.bin
+
+	i=$((${i} + 1))
+	echo "${tmpstr}passed" >> ${LOGFILE}
+done
+
+# Make a layout - 4K regions on 4.5K boundaries. This will help find problems
+# with logic that only operates on part of a block. For example, if a user
+# wishes to re-write a fraction of a block, then:
+# 1. The whole block must be erased.
+# 2. The old content must be restored at unspecified offsets.
+# 3. The new content must be written at specified offsets.
+#
+# Note: The last chunk of 0xff bytes is only 2K as to avoid overrunning a 128KB
+# test region.
+#
+echo "
+`printf 0x%x $((${start} + 0x00800))`:`printf 0x%x $((${start} + 0x017ff))` 00_0
+`printf 0x%x $((${start} + 0x01800))`:`printf 0x%x $((${start} + 0x027ff))` ff_0
+
+`printf 0x%x $((${start} + 0x02800))`:`printf 0x%x $((${start} + 0x037ff))` 00_1
+`printf 0x%x $((${start} + 0x03800))`:`printf 0x%x $((${start} + 0x047ff))` ff_1
+
+`printf 0x%x $((${start} + 0x04800))`:`printf 0x%x $((${start} + 0x057ff))` 00_2
+`printf 0x%x $((${start} + 0x05800))`:`printf 0x%x $((${start} + 0x067ff))` ff_2
+
+`printf 0x%x $((${start} + 0x06800))`:`printf 0x%x $((${start} + 0x077ff))` 00_3
+`printf 0x%x $((${start} + 0x07800))`:`printf 0x%x $((${start} + 0x087ff))` ff_3
+
+`printf 0x%x $((${start} + 0x08800))`:`printf 0x%x $((${start} + 0x097ff))` 00_4
+`printf 0x%x $((${start} + 0x09800))`:`printf 0x%x $((${start} + 0x0a7ff))` ff_4
+
+`printf 0x%x $((${start} + 0x0a800))`:`printf 0x%x $((${start} + 0x0b7ff))` 00_5
+`printf 0x%x $((${start} + 0x0b800))`:`printf 0x%x $((${start} + 0x0c7ff))` ff_5
+
+`printf 0x%x $((${start} + 0x0c800))`:`printf 0x%x $((${start} + 0x0d7ff))` 00_6
+`printf 0x%x $((${start} + 0x0d800))`:`printf 0x%x $((${start} + 0x0e7ff))` ff_6
+
+`printf 0x%x $((${start} + 0x0e800))`:`printf 0x%x $((${start} + 0x0f7ff))` 00_7
+`printf 0x%x $((${start} + 0x0f800))`:`printf 0x%x $((${start} + 0x107ff))` ff_7
+
+`printf 0x%x $((${start} + 0x10800))`:`printf 0x%x $((${start} + 0x117ff))` 00_8
+`printf 0x%x $((${start} + 0x11800))`:`printf 0x%x $((${start} + 0x127ff))` ff_8
+
+`printf 0x%x $((${start} + 0x12800))`:`printf 0x%x $((${start} + 0x137ff))` 00_9
+`printf 0x%x $((${start} + 0x13800))`:`printf 0x%x $((${start} + 0x147ff))` ff_9
+
+`printf 0x%x $((${start} + 0x14800))`:`printf 0x%x $((${start} + 0x157ff))` 00_10
+`printf 0x%x $((${start} + 0x15800))`:`printf 0x%x $((${start} + 0x167ff))` ff_10
+
+`printf 0x%x $((${start} + 0x16800))`:`printf 0x%x $((${start} + 0x177ff))` 00_11
+`printf 0x%x $((${start} + 0x17800))`:`printf 0x%x $((${start} + 0x187ff))` ff_11
+
+`printf 0x%x $((${start} + 0x18800))`:`printf 0x%x $((${start} + 0x197ff))` 00_12
+`printf 0x%x $((${start} + 0x19800))`:`printf 0x%x $((${start} + 0x1a7ff))` ff_12
+
+`printf 0x%x $((${start} + 0x1a800))`:`printf 0x%x $((${start} + 0x1b7ff))` 00_13
+`printf 0x%x $((${start} + 0x1b800))`:`printf 0x%x $((${start} + 0x1c7ff))` ff_13
+
+`printf 0x%x $((${start} + 0x1c800))`:`printf 0x%x $((${start} + 0x1d7ff))` 00_14
+`printf 0x%x $((${start} + 0x1d800))`:`printf 0x%x $((${start} + 0x1e7ff))` ff_14
+
+`printf 0x%x $((${start} + 0x1e800))`:`printf 0x%x $((${start} + 0x1f7ff))` 00_15
+`printf 0x%x $((${start} + 0x1f800))`:`printf 0x%x $((${start} + 0x1ffff))` ff_15
+" > layout_ec_unaligned.txt
+
+# reset the test file and ROM to the original state
+flashrom ${FLASHROM_PARAM} -w "${BACKUP}" > /dev/null
+cp "$BACKUP" "$TESTFILE"
+
+i=0
+while [ $i -lt $NUM_REGIONS ] ; do
+	echo -n "unaligned region ${i} test: "
+
+	offset=$(($((${i} * 8192)) + 2048))
+	# Protect against too long write
+	writelen=4096
+	if [ $((${offset} + 4096 + 4096)) -ge 131072 ]; then
+		writelen=$((131072 - $((${offset} + 4096))))
+		if [ ${writelen} -lt 0 ]; then
+			writelen=0
+		fi
+	fi
+	dd if=${ZERO_4K} of=${TESTFILE} bs=1 conv=notrunc seek=${offset} 2> /dev/null
+	dd if=${FF_4K} of=${TESTFILE} bs=1 conv=notrunc seek=$((${offset} + 4096)) count=writelen 2> /dev/null
+
+	./flashrom ${FLASHROM_PARAM} -l layout_ec_unaligned.txt -i 00_${i} -i ff_${i} -w "$TESTFILE" 2> /dev/null
+	if [ "$?" != "0" ] ; then
+		partial_writes_fail "failed to flash region"
+	fi
+
+	# download the entire ROM image and use diff to compare to ensure
+	# flashrom logic does not violate user-specified regions
+	flashrom ${FLASHROM_PARAM} -r difftest.bin 2> /dev/null
+	diff -q difftest.bin "$TESTFILE"
+	if [ "$?" != "0" ] ; then
+		partial_writes_fail "failed diff test"
+	fi
+	rm -f difftest.bin
+
+	i=$((${i} + 1))
+	echo "passed"
+done
+
+return "$EXIT_SUCCESS"