Add render test results to the results_details webpage.

chrome/test/android/javatests/src/org/chromium/chrome/test/util/RenderUtils.java

Index: chrome/test/android/javatests/src/org/chromium/chrome/test/util/RenderUtils.java
diff --git a/chrome/test/android/javatests/src/org/chromium/chrome/test/util/RenderUtils.java b/chrome/test/android/javatests/src/org/chromium/chrome/test/util/RenderUtils.java
index 85438516e313581640aba653cd1adc18f68dafb5..c0013f27d0072bffcd242371dfcc5d70b197f878 100644
--- a/chrome/test/android/javatests/src/org/chromium/chrome/test/util/RenderUtils.java
+++ b/chrome/test/android/javatests/src/org/chromium/chrome/test/util/RenderUtils.java
@@ -7,6 +7,7 @@ package org.chromium.chrome.test.util;
 import android.app.Activity;
 import android.graphics.Bitmap;
 import android.graphics.BitmapFactory;
+import android.graphics.Color;
 import android.graphics.Point;
 import android.os.Build;
 import android.view.View;
@@ -28,8 +29,12 @@ import java.util.concurrent.Callable;
 public class RenderUtils {
     private static final String TAG = "RenderUtils";
 
+    private static final String DIFF_FOLDER_RELATIVE = "/diffs";
+
     private static final String FAILURE_FOLDER_RELATIVE = "/failures";
 
+    private static final String GOLDEN_FOLDER_RELATIVE = "/goldens";
+
     /**
      * This is a list of devices that we maintain golden images for. If render tests are being run
      * on a device in this list, golden images should exist and their absence is a test failure.
@@ -45,6 +50,11 @@ public class RenderUtils {
      */
     private static final boolean REPORT_ONLY_DO_NOT_FAIL = true;
 
+    /**
+     * How many pixels can be different in an image before counting the images as different.
+     */
+    private static final int PIXEL_DIFF_THRESHOLD = 0;
+
     private enum ComparisonResult { MATCH, MISMATCH, GOLDEN_NOT_FOUND }
 
     /**
@@ -72,11 +82,19 @@ public class RenderUtils {
         private final Activity mActivity;
         private final String mGoldenFolder;
         private final String mTestClass;
+        private String mOutputDirectory;
 
         public ViewRenderer(Activity activity, String goldenFolder, String testClass) {
             mActivity = activity;
             mGoldenFolder = goldenFolder;
             mTestClass = testClass;
+
+            // Render test will output results to subdirectory of |goldenFolder| unless
+            // --render-test-output-dir is passed.
+            mOutputDirectory = CommandLine.getInstance().getSwitchValue("render-test-output-dir");
+            if (mOutputDirectory == null) {
+                mOutputDirectory = mGoldenFolder;
+            }
         }
 
         /**
@@ -93,21 +111,26 @@ public class RenderUtils {
          */
         public void renderAndCompare(final View view, String id) throws IOException {
             // Compare the image against the Golden.
-            Bitmap bitmap = ThreadUtils.runOnUiThreadBlockingNoException(new Callable<Bitmap>() {
-                @Override
-                public Bitmap call() throws Exception {
-                    return UiUtils.generateScaledScreenshot(view, 0, Bitmap.Config.ARGB_8888);
-                }
-            });
+            Bitmap testBitmap =
+                    ThreadUtils.runOnUiThreadBlockingNoException(new Callable<Bitmap>() {
+                        @Override
+                        public Bitmap call() throws Exception {
+                            return UiUtils.generateScaledScreenshot(
+                                    view, 0, Bitmap.Config.ARGB_8888);
+                        }
+                    });
 
             String imagename = imageName(mActivity, mTestClass, id);
-            File goldenFile = createPath(mGoldenFolder, imagename);
-            ComparisonResult result = compareBmpToGolden(bitmap, goldenFile);
 
-            if (REPORT_ONLY_DO_NOT_FAIL && !isGenerateMode()) {
-                Log.d(TAG, "Image comparison for %s: %s", id, result.name());
-                return;
-            }
+            File goldenFile =
+                    createPath(UrlUtils.getIsolatedTestFilePath(mGoldenFolder), imagename);
+            BitmapFactory.Options options = new BitmapFactory.Options();
+            options.inPreferredConfig = testBitmap.getConfig();
+            Bitmap goldenBitmap = BitmapFactory.decodeFile(goldenFile.getAbsolutePath(), options);
+            Bitmap diffBitmap = Bitmap.createBitmap(
+                    testBitmap.getWidth(), testBitmap.getHeight(), testBitmap.getConfig());
+
+            ComparisonResult result = compareBitmapToGolden(testBitmap, goldenBitmap, diffBitmap);
 
             if (result == ComparisonResult.MATCH) {
                 if (isGenerateMode()) {
@@ -116,12 +139,25 @@ public class RenderUtils {
                 return;
             }
 
-            // Save the rendered View.
-            File failureFile = createPath(mGoldenFolder + FAILURE_FOLDER_RELATIVE, imagename);
-            saveBitmap(bitmap, failureFile);
+            File goldenOutputFile =
+                    createPath(mOutputDirectory + GOLDEN_FOLDER_RELATIVE, imagename);
+            saveBitmap(goldenBitmap, goldenOutputFile);

[PEConn, 2017/05/09 08:52:39]

What happens here if the golden isn't found? saveBitmap has a try/catch so we
shouldn't crash but is an image generated?

[mikecase (-- gone --), 2017/05/10 01:05:00]

Changed it so that if golden is not found, it does not attempt to save the
golden or diff images.

+
+            File failureOutputFile =
+                    createPath(mOutputDirectory + FAILURE_FOLDER_RELATIVE, imagename);
+            saveBitmap(testBitmap, failureOutputFile);
+
+            File diffOutputFile = createPath(mOutputDirectory + DIFF_FOLDER_RELATIVE, imagename);
+            saveBitmap(diffBitmap, diffOutputFile);
 
             if (isGenerateMode()) {
-                Log.i(TAG, "%s - generated image saved to %s.", id, failureFile.getAbsolutePath());
+                Log.i(TAG, "%s - generated image saved to %s.", id,
+                        failureOutputFile.getAbsolutePath());
+                return;
+            }
+
+            if (REPORT_ONLY_DO_NOT_FAIL) {
+                Log.d(TAG, "Image comparison for %s: %s", id, result.name());
                 return;
             }
 
@@ -137,8 +173,7 @@ public class RenderUtils {
             } else if (result == ComparisonResult.MISMATCH) {
                 throw new ImageMismatchException(
                         String.format("Image comparison failed on %s. Failure image saved to %s.",
-                                id, failureFile.getAbsolutePath()));
-
+                                id, failureOutputFile.getAbsolutePath()));
             }
         }
     }
@@ -185,14 +220,13 @@ public class RenderUtils {
     }
 
     /**
-     * Convenience method to create a File pointing to |filename| in |folder| in the external
-     * storage directory.
+     * Convenience method to create a File pointing to |filename| in |folder|.
      * @throws IOException
      */
     private static File createPath(String folder, String filename) throws IOException {
-        File path = new File(UrlUtils.getIsolatedTestFilePath(folder));
+        File path = new File(folder);
         if (!path.exists()) {
-            if (!path.mkdir()) {
+            if (!path.mkdirs()) {
                 throw new IOException("Could not create " + path.getAbsolutePath());
             }
         }
@@ -202,12 +236,68 @@ public class RenderUtils {
     /**
      * Returns whether the given |bitmap| is equal to the one stored in |file|.
      */
-    private static ComparisonResult compareBmpToGolden(Bitmap bitmap, File file) {
-        BitmapFactory.Options options = new BitmapFactory.Options();
-        options.inPreferredConfig = bitmap.getConfig();
-        Bitmap golden = BitmapFactory.decodeFile(file.getAbsolutePath(), options);
-
+    private static ComparisonResult compareBitmapToGolden(Bitmap test, Bitmap golden, Bitmap diff) {
         if (golden == null) return ComparisonResult.GOLDEN_NOT_FOUND;
-        return bitmap.sameAs(golden) ? ComparisonResult.MATCH : ComparisonResult.MISMATCH;
+
+        if (golden.getWidth() != test.getWidth() || golden.getHeight() != test.getHeight()) {
+            Log.d(TAG, "Golden bitmap has resolution %d,%d but test bitmap has resolution %d,%d",
+                    golden.getWidth(), golden.getHeight(), test.getWidth(), test.getHeight());
+            return ComparisonResult.MISMATCH;

[PEConn, 2017/05/09 08:52:39]

So if the golden doesn't match the dimensions of the test, we still save the
diff (which I'm assuming is just a blank bitmap the same size as test). This
seems pointless.

[mikecase (-- gone --), 2017/05/10 01:05:00]

Making the diff Bitmap be max_width by max_height. Pixels coords that occur in
one Bitmap but not the other will show up as RED

+        }
+
+        int diffPixelsCount = comparePixels(test, golden, diff);
+
+        if (diffPixelsCount > PIXEL_DIFF_THRESHOLD) {
+            return ComparisonResult.MISMATCH;
+        }
+        return ComparisonResult.MATCH;
+    }
+
+    private static int comparePixels(Bitmap testImage, Bitmap goldenImage, Bitmap diffImage) {
+        return comparePixels(testImage, goldenImage, diffImage, 0);
+    }
+
+    /**
+     * Compares two bitmaps pixel-wise.
+     *
+     * @param testImage Bitmap of test image.
+     *
+     * @param goldenImage Bitmap of golden image.
+     *
+     * @param diffImage This is an output argument. Function will set pixels in the |diffImage| to
+     * either transparent or red depending on whether that pixel differed in the golden and test
+     * bitmaps.
+     *
+     * @param diffThreshold Threshold for when to consider two color values as different. These
+     * values are 8 bit (256) so this threshold value should be in range 0-256.
+     *
+     * @return Returns number of pixels that differ between |goldenImage| and |testImage|
+     */
+    private static int comparePixels(
+            Bitmap testImage, Bitmap goldenImage, Bitmap diffImage, int diffThreshold) {
+        int diffPixels = 0;
+
+        for (int x = 0; x < testImage.getWidth(); x++) {
+            for (int y = 0; y < testImage.getHeight(); y++) {
+                int goldenImageColor = goldenImage.getPixel(x, y);
+                int testImageColor = testImage.getPixel(x, y);
+
+                int redDiff = Math.abs(Color.red(goldenImageColor) - Color.red(testImageColor));
+                int blueDiff =
+                        Math.abs(Color.green(goldenImageColor) - Color.green(testImageColor));
+                int greenDiff = Math.abs(Color.blue(goldenImageColor) - Color.blue(testImageColor));
+                int alphaDiff =
+                        Math.abs(Color.alpha(goldenImageColor) - Color.alpha(testImageColor));
+
+                if (redDiff > diffThreshold || blueDiff > diffThreshold || greenDiff > diffThreshold
+                        || alphaDiff > diffThreshold) {
+                    diffPixels++;
+                    diffImage.setPixel(x, y, Color.RED);
+                } else {
+                    diffImage.setPixel(x, y, Color.TRANSPARENT);
+                }
+            }
+        }
+        return diffPixels;
     }
 }