Add an accessibility audit test for WebUI pages

chrome/test/data/webui/test_api.js

Index: chrome/test/data/webui/test_api.js
diff --git a/chrome/test/data/webui/test_api.js b/chrome/test/data/webui/test_api.js
index 03c6721d51d55bf0baf4641caa2e6d9955c31802..ddb8ba77db8230daebeff792f411df814007f958 100644
--- a/chrome/test/data/webui/test_api.js
+++ b/chrome/test/data/webui/test_api.js
@@ -121,6 +121,13 @@ var testing = {};
      */
     extraLibraries: [],
 
+    runA11yChecks : true,
+
+    /**
+     * FIXME(aboxhall)
+     */
+    a11yIssuesAreErrors: true,
+
     /**
      * Create a new class to handle |messageNames|, assign it to
      * |this.mockHandler|, register its messages and return it.
@@ -234,6 +241,8 @@ var testing = {};
   function TestCase(name, fixture, body) {
     this.name = name;
     this.fixture = fixture;
+    this.runA11yChecks = fixture.runA11yChecks;
+    this.a11yIssuesAreErrors = fixture.a11yIssuesAreErrors;
     this.body = body;
   }
 
@@ -263,6 +272,13 @@ var testing = {};
      */
     deferred_: false,
 
+    runA11yChecks: true,
+
+    /**
+     * FIXME(aboxhall)
+     */
+    a11yIssuesAreErrors: true,
+
     /**
      * Called at preload time, proxies to the fixture.
      * @type {Function}
@@ -585,6 +601,13 @@ var testing = {};
   var errors = [];
 
   /**
+   * Hold the accessibility warnings and errors, respectively.
+   * Like errors, cleared when results are reported.
+   */
+  var a11y_warnings = [];
+  var a11y_errors = [];
+
+  /**
    * URL to dummy WebUI page for testing framework.
    * @type {string}
    */
@@ -607,6 +630,20 @@ var testing = {};
     if (!testIsDone) {
       testIsDone = true;
       if (currentTestCase) {
+        if (currentTestCase.runA11yChecks &&
+            !runAccessibilityAudit_(a11y_errors, a11y_warnings)) {
+          if (currentTestCase.a11yIssuesAreErrors) {
+            if (!result)
+              result = testResult();
+            result = [false, accessibilityErrorsToMessage(a11y_errors,
+                                                          a11y_warnings,
+                                                          result[1])];
+          } else { /* a11yIssuesAreErrors */
+            console.warn(accessibilityErrorsToMessage(a11y_errors,
+                                                      a11y_warnings));
+          }
+        }
+
         try {
           currentTestCase.tearDown();
         } catch (e) {
@@ -618,7 +655,9 @@ var testing = {};
         }
         currentTestCase = null;
       }
-      chrome.send('testResult', result ? result : testResult());
+      if (!result)
+        result = testResult();
+      chrome.send('testResult', result);
       errors.splice(0, errors.length);
     } else {
       console.warn('testIsDone already');
@@ -652,9 +691,9 @@ var testing = {};
    */
   function testResult(errorsOk) {
     var result = [true, ''];
-    if (errors.length) {
+    if (errors.length)
       result = [!!errorsOk, errorsToMessage(errors)];
-    }
+
     return result;
   }
 
@@ -792,6 +831,107 @@ var testing = {};
     throw new Error(helper.getCallMessage(message));
   }
 
+  function disableA11yChecks() {
+    if (!currentTestCase)
+      return;
+    currentTestCase.runA11yChecks = false;
+  }
+
+  function enableA11yChecks() {
+    if (!currentTestCase)
+      return;
+    currentTestCase.runA11yChecks = true;
+  }
+
+  function a11yIssuesAreErrors(flag) {
+    if (!currentTestCase)
+      return;
+    currentTestCase.a11yIssuesAreErrors = flag;
+  }
+
+  /**
+   * FIXME(aboxhall) documentation
+   */
+  function accessibilityErrorsToMessage(a11y_errors, a11y_warnings, message) {
+    if (message)
+      message += '\n';
+    else
+      message = '';
+
+    message += 'Accessibility ';
+    if (a11y_errors.length > 0) {
+      message += 'errors ';
+      if (a11y_warnings.length > 0)
+        message += 'and ';
+    }
+    if (a11y_warnings.length > 0)
+      message += 'warnings ';
+    message += 'found:';
+    for (var i = 0; i < a11y_errors.length; i++)
+      message += '\n' + a11y_errors[i];
+    for (var i = 0; i < a11y_warnings.length; i++)
+      message += '\n' + a11y_warnings[i];
+    return message;
+  }
+
+  /**
+   * FIXME(aboxhall): more distinct names for this and above function
+   * FIXME(aboxhall): documentation
+   */
+  function createAccessibilityIssueMessage(rule, result) {
+    if (rule.severity == axs.constants.Severity.Severe)
+      var message = 'Error: '
+    else
+      var message = 'Warning: '
+    message += rule.name + ' failed on the following elements:';
+    for (var i = 0; i < result.elements.length; i++)
+      message += '\n' + result.elements[i];
+    return message;
+  }
+
+  /**
+   * FIXME(aboxhall) documentation
+   */
+  function assertAccessibilityOk() {
+    helper.registerCall();
+    var a11y_errors = [];
+    var a11y_warnings = [];
+    if (!runAccessibilityAudit_(a11y_errors, a11y_warnings))
+      throw new Error(accessibilityErrorsToMessage(a11y_errors,
+                                                   a11y_warnings));
+  }
+
+  /**
+   * Run an accessibility audit on the current page state.
+   * FIXME(aboxhall) documentation
+   * @type {Function}
+   * @return {boolean} Whether there were any errors or warnings
+   * @private
+   */
+  function runAccessibilityAudit_(a11yErrors, a11yWarnings) {
+    for (var auditRuleName in axs.AuditRule.specs) {
+      var auditRule = axs.AuditRules.getRule(auditRuleName);
+      if (!auditRule)
+        continue; // Shouldn't happen, but fail silently if it does.
+
+      if (auditRule.disabled || auditRule.requiresConsoleAPI)
+        continue;
+
+      var result = auditRule.run();
+      if (result.result == axs.constants.AuditResult.FAIL) {
+        // TODO(aboxhall): more useful error messages (sadly non-trivial)
+        if (auditRule.severity == axs.constants.Severity.Severe)
+          a11yErrors.push(createAccessibilityIssueMessage(auditRule, result));
+        else
+          a11yWarnings.push(createAccessibilityIssueMessage(auditRule, result));
+      }
+    }
+    // TODO(aboxhall): have strict (no errors or warnings) vs non-strict
+    // (warnings ok)
+    // TODO(aboxhall): some kind of info logging for warnings only??
+    return (a11yErrors.length == 0 && a11yWarnings.length == 0);
+  }
+
   /**
    * Creates a function based upon a function that thows an exception on
    * failure. The new function stuffs any errors into the |errors| array for
@@ -834,6 +974,7 @@ var testing = {};
    * @see runTestFunction
    */
   function runTest(isAsync, testFunction, testArguments) {
+    // FIXME(aboxhall): modify to add a11y phase
     // Avoid eval() if at all possible, since it will not work on pages
     // that have enabled content-security-policy.
     var testBody = this[testFunction];    // global object -- not a method.
@@ -1377,9 +1518,12 @@ var testing = {};
   exports.assertLT = assertLT;
   exports.assertNotEquals = assertNotEquals;
   exports.assertNotReached = assertNotReached;
+  exports.assertAccessibilityOk = assertAccessibilityOk;
   exports.callFunction = callFunction;
   exports.callFunctionWithSavedArgs = callFunctionWithSavedArgs;
   exports.callGlobalWithSavedArgs = callGlobalWithSavedArgs;
+  exports.disableA11yChecks = disableA11yChecks;
+  exports.enableA11yChecks = enableA11yChecks;
   exports.expectTrue = createExpect(assertTrue);
   exports.expectFalse = createExpect(assertFalse);
   exports.expectGE = createExpect(assertGE);
@@ -1389,6 +1533,7 @@ var testing = {};
   exports.expectLT = createExpect(assertLT);
   exports.expectNotEquals = createExpect(assertNotEquals);
   exports.expectNotReached = createExpect(assertNotReached);
+  exports.expectAccessibilityOk = createExpect(assertAccessibilityOk);
   exports.preloadJavascriptLibraries = preloadJavascriptLibraries;
   exports.registerMessageCallback = registerMessageCallback;
   exports.registerMockGlobals = registerMockGlobals;