Fail pixel tests when reference images are missing from cloud storage.

content/test/gpu/gpu_tests/pixel.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 from datetime import datetime
 import glob
 import optparse
 import os
 import re
 
 import cloud_storage_test_base
 import pixel_expectations
 
 from telemetry import test
 from telemetry.core import bitmap
+from telemetry.page import cloud_storage
 from telemetry.page import page_test
 
 test_data_dir = os.path.abspath(os.path.join(
     os.path.dirname(__file__), '..', '..', 'data', 'gpu'))
 
 default_reference_image_dir = os.path.join(test_data_dir, 'gpu_reference')
 
 test_harness_script = r"""
   var domAutomationController = {};
 
@@ skipping 49 matching lines @@
                                                  screenshot):
         # This is the new reference image; there's nothing to compare against.
         ref_png = screenshot
       else:
         # There was a preexisting reference image, so we might as well
         # compare against it.
         ref_png = self._DownloadFromCloudStorage(image_name, page, tab)
     elif self.options.download_refimg_from_cloud_storage:
       # This bot doesn't have the ability to properly generate a
       # reference image, so download it from cloud storage.
-      ref_png = self._DownloadFromCloudStorage(image_name, page, tab)
+      try:
+        ref_png = self._DownloadFromCloudStorage(image_name, page, tab)
+      except cloud_storage.NotFoundError as e:
+        # There is no reference image yet in cloud storage. This
+        # happens when the revision of the test is incremented or when
+        # a new test is added, because the trybots are not allowed to
+        # produce reference images, only the bots on the main
+        # waterfalls. Report this as a failure so the developer has to
+        # take action by explicitly suppressing the failure and
+        # removing the suppression once the reference images have been
+        # generated. Otherwise silent failures could happen for long
+        # periods of time.
+        raise page_test.Failure('Could not find image %s in cloud storage' %
+                                image_name)
     else:
       # Legacy path using on-disk results.
       ref_png = self._GetReferenceImage(self.options.reference_dir,
           image_name, page.revision, screenshot)
 
     # Test new snapshot against existing reference image
     if not ref_png.IsEqual(screenshot, tolerance=2):
       if self.options.test_machine_name:
         self._UploadErrorImagesToCloudStorage(image_name, screenshot, ref_png)
       else:
@@ skipping 48 matching lines @@
         default=default_reference_image_dir)
 
   def CreatePageSet(self, options):
     page_set = super(Pixel, self).CreatePageSet(options)
     for page in page_set.pages:
       page.script_to_evaluate_on_commit = test_harness_script
     return page_set
 
   def CreateExpectations(self, page_set):
     return pixel_expectations.PixelExpectations()