Add an accessibility audit test for WebUI pages

chrome/test/data/webui/test_api.js

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 /**
  * @fileoverview Library providing basic test framework functionality.
  */
 
 /**
  * Namespace for |Test|.
@@ skipping 104 matching lines @@
      */
     testShouldFail: false,
 
     /**
      * Extra libraries to add before loading this test file.
      * @type {Array.<string>}
      */
     extraLibraries: [],
 
     /**
+     * FIXME(aboxhall)
+     */
+    runA11yChecks: true,
+
+    /**
      * Create a new class to handle |messageNames|, assign it to
      * |this.mockHandler|, register its messages and return it.
      * @return {Mock} Mock handler class assigned to |this.mockHandler|.
      */
     makeAndRegisterMockHandler: function(messageNames) {
       var MockClass = makeMockClass(messageNames);
       this.mockHandler = mock(MockClass);
       registerMockMessageCallbacks(this.mockHandler, MockClass);
       return this.mockHandler;
     },
@@ skipping 92 matching lines @@
    * This class is not exported and is available to hold the state of the
    * |currentTestCase| throughout preload and test run.
    * @param {string} name The name of the test case.
    * @param {Test} fixture The fixture object for this test case.
    * @param {Function} body The code to run for the test.
    * @constructor
    */
   function TestCase(name, fixture, body) {
     this.name = name;
     this.fixture = fixture;
+    this.runA11yChecks = fixture.runA11yChecks;
     this.body = body;
   }
 
   TestCase.prototype = {
     /**
      * The name of this test.
      * @type {string}
      */
     name: null,
 
@@ skipping 10 matching lines @@
     body: null,
 
     /**
      * True when the test fixture will run the test later.
      * @type {boolean}
      * @private
      */
     deferred_: false,
 
     /**
+     * FIXME(aboxhall)
+     */
+    runA11yChecks: true,
+
+    /**
      * Called at preload time, proxies to the fixture.
      * @type {Function}
      */
     preLoad: function(name) {
       if (this.fixture)
         this.fixture.preLoad();
     },
 
     /**
      * Called before a test runs.
@@ skipping 301 matching lines @@
   var testIsDone = false;
 
   /**
    * Holds the errors, if any, caught by expects so that the test case can
    * fail. Cleared when results are reported from runTest() or testDone().
    * @type {Array.<Error>}
    */
   var errors = [];
 
   /**
+   * Hold the accessibility warnings and errors, respectively.
+   * Like errors, cleared when results are reported.
+   */
+  var a11y_warnings = [];
+  var a11y_errors = [];
+
+  /**
    * URL to dummy WebUI page for testing framework.
    * @type {string}
    */
   var DUMMY_URL = 'chrome://DummyURL';
 
   /**
    * Resets test state by clearing |errors| and |testIsDone| flags.
    */
   function resetTestState() {
     errors.splice(0, errors.length);
     testIsDone = false;
   }
 
   /**
    * Notifies the running browser test of the test results. Clears |errors|.
    * @param {Array.<boolean, string>=} result When passed, this is used for the
    *     testResult message.
    */
   function testDone(result) {
     if (!testIsDone) {
+      // FIXME(aboxhall): run a11y check if not disabled; combine with result.
+      var accessibilityOk = runAccessibilityAudit_(a11y_errors, a11y_warnings);
+
       testIsDone = true;
       if (currentTestCase) {
         try {
           currentTestCase.tearDown();
         } catch (e) {
           // Caught an exception in tearDown; Register the error and recreate
           // the result if it is passed in.
           errors.push(e);
           if (result)
             result = [false, errorsToMessage([e], result[1])];
         }
         currentTestCase = null;
       }
-      chrome.send('testResult', result ? result : testResult());
+      if (!result)
+        result = testResult();
+      if (!accessibilityOk)
+        result.push(createAccessibilityErrorMessage(a11y_errors,
+                                                    a11y_warnings));
+      chrome.send('testResult', result);
       errors.splice(0, errors.length);
     } else {
       console.warn('testIsDone already');
     }
   }
 
   /**
    * Converts each Error in |errors| to a suitable message, adding them to
    * |message|, and returns the message string.
    * @param {Array.<Error>} errors Array of errors to add to |message|.
@@ skipping 154 matching lines @@
    * Always aborts the current test.
    * @param {string=} message The message to include in the Error thrown.
    * @throws {Error} always.
    */
   function assertNotReached(message) {
     helper.registerCall();
     throw new Error(helper.getCallMessage(message));
   }
 
   /**
+   * FIXME(aboxhall) documentation
+   */
+  function createAccessibilityErrorMessage(a11y_errors, a11y_warnings) {
+    var errorMessage = 'Accessibility ';
+    if (a11y_errors.length > 0)
+      errorMessage += 'Errors ';
+    else
+      errorMessage += 'Warnings ';
+    errorMessage += 'found:\n';
+    for (var i = 0; i < a11y_errors.length; i++)
+      errorMessage += 'Error: ' + a11y_errors[i] + '\n';
+    for (var i = 0; i < a11y_warnings.length; i++)
+      errorMessage += 'Warning: ' + a11y_warnings[i] + '\n';
+    return errorMessage;
+  }
+
+  /**
+   * FIXME(aboxhall) documentation
+   */
+  function assertAccessibilityOk() {
+    helper.registerCall();
+    var a11y_errors = [];
+    var a11y_warnings = [];
+    if (!runAccessibilityAudit_(a11y_errors, a11y_warnings))
+      throw new Error(createAccessibilityErrorMessage(a11y_errors,
+                                                      a11y_warnings));
+  }
+
+  /**
+   * Run an accessibility audit on the current page state.
+   * FIXME(aboxhall) documentation
+   * @type {Function}
+   * @return {boolean} Whether there were any errors or warnings
+   * @private
+   */
+  function runAccessibilityAudit_(errors, warnings) {
+    for (var auditRuleName in axs.AuditRule.specs) {
+      var auditRule = axs.AuditRules.getRule(auditRuleName);
+      if (!auditRule)
+        continue; // Shouldn't happen, but fail silently if it does.
+      if (!auditRule.disabled && !auditRule.requiresConsoleAPI) {
+        var result = auditRule.run();
+          if (result.result == axs.constants.AuditResult.FAIL) {
+          // TODO(aboxhall): more useful error messages (sadly non-trivial)
+          if (auditRule.severity == axs.constants.Severity.Severe)
+            errors.push("Accessibility audit rule " + auditRule.name +
+                        " failed");
+          else
+            warnings.push("Accessibility audit rule " + auditRule.name +
+                          " failed");
+        }
+      }
+    }
+    // TODO(aboxhall): have strict (no errors or warnings) vs non-strict
+    // (warnings ok)
+    // TODO(aboxhall): some kind of info logging for warnings only??
+    return (errors.length == 0 && warnings.length == 0);
+  }
+
+  /**
    * Creates a function based upon a function that thows an exception on
    * failure. The new function stuffs any errors into the |errors| array for
    * checking by runTest. This allows tests to continue running other checks,
    * while failing the overall test if any errors occurrred.
    * @param {Function} assertFunc The function which may throw an Error.
    * @return {function(...*):bool} A function that applies its arguments to
    *     |assertFunc| and returns true if |assertFunc| passes.
    * @see errors
    * @see runTestFunction
    */
@@ skipping 21 matching lines @@
    * @param {boolean} isAsync When false, call testDone() with the test result
    *     otherwise only when assertions are caught.
    * @param {string} testFunction The function name to call.
    * @param {Array} testArguments The arguments to call |testFunction| with.
    * @return {boolean} true always to signal successful execution (but not
    *     necessarily successful results) of this test.
    * @see errors
    * @see runTestFunction
    */
   function runTest(isAsync, testFunction, testArguments) {
+    // FIXME(aboxhall): modify to add a11y phase
     // Avoid eval() if at all possible, since it will not work on pages
     // that have enabled content-security-policy.
     var testBody = this[testFunction];    // global object -- not a method.
     var testName = testFunction;
     if (typeof testBody === "undefined") {
       testBody = eval(testFunction);
       testName = testBody.toString();
     }
     if (testBody != RUN_TEST_F) {
       console.log('Running test ' + testName);
@@ skipping 535 matching lines @@
   exports.callGlobalWithSavedArgs = callGlobalWithSavedArgs;
   exports.expectTrue = createExpect(assertTrue);
   exports.expectFalse = createExpect(assertFalse);
   exports.expectGE = createExpect(assertGE);
   exports.expectGT = createExpect(assertGT);
   exports.expectEquals = createExpect(assertEquals);
   exports.expectLE = createExpect(assertLE);
   exports.expectLT = createExpect(assertLT);
   exports.expectNotEquals = createExpect(assertNotEquals);
   exports.expectNotReached = createExpect(assertNotReached);
+  exports.expectAccessibilityOk = createExpect(assertAccessibilityOk);
   exports.preloadJavascriptLibraries = preloadJavascriptLibraries;
   exports.registerMessageCallback = registerMessageCallback;
   exports.registerMockGlobals = registerMockGlobals;
   exports.registerMockMessageCallbacks = registerMockMessageCallbacks;
   exports.resetTestState = resetTestState;
   exports.runAllActions = runAllActions;
   exports.runAllActionsAsync = runAllActionsAsync;
   exports.runTest = runTest;
   exports.runTestFunction = runTestFunction;
   exports.SaveMockArguments = SaveMockArguments;
   exports.DUMMY_URL = DUMMY_URL;
   exports.TEST = TEST;
   exports.TEST_F = TEST_F;
   exports.GEN = GEN;
   exports.GEN_INCLUDE = GEN_INCLUDE;
   exports.WhenTestDone = WhenTestDone;
 
   // Import the Mock4JS helpers.
   Mock4JS.addMockSupport(exports);
 })(this);