Add an accessibility audit test for WebUI pages

chrome/test/data/webui/test_api.js

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 /**
  * @fileoverview Library providing basic test framework functionality.
  */
 
 /**
  * Namespace for |Test|.
@@ skipping 104 matching lines @@
      */
     testShouldFail: false,
 
     /**
      * Extra libraries to add before loading this test file.
      * @type {Array.<string>}
      */
     extraLibraries: [],
 
     /**
+     * Whether to run the accessibility checks.
+     * @type {boolean}
+     */
+    runAccessibilityChecks : true,
+
+    /**
+     * Whether to treat accessibility issues (errors or warnings) as test
+     * failures. If true, any accessibility issues will cause the test to fail.
+     * If false, accessibility issues will cause a console.warn.
+     * Off by default to begin with; as we add the ability to suppress false
+     * positives, we will transition this to true.
+     * @type {boolean}
+     */
+    accessibilityIssuesAreErrors: false,
+
+    /**
      * Create a new class to handle |messageNames|, assign it to
      * |this.mockHandler|, register its messages and return it.
      * @return {Mock} Mock handler class assigned to |this.mockHandler|.
      */
     makeAndRegisterMockHandler: function(messageNames) {
       var MockClass = makeMockClass(messageNames);
       this.mockHandler = mock(MockClass);
       registerMockMessageCallbacks(this.mockHandler, MockClass);
       return this.mockHandler;
     },
@@ skipping 92 matching lines @@
    * This class is not exported and is available to hold the state of the
    * |currentTestCase| throughout preload and test run.
    * @param {string} name The name of the test case.
    * @param {Test} fixture The fixture object for this test case.
    * @param {Function} body The code to run for the test.
    * @constructor
    */
   function TestCase(name, fixture, body) {
     this.name = name;
     this.fixture = fixture;
+    this.runAccessibilityChecks = fixture.runAccessibilityChecks;
+    this.accessibilityIssuesAreErrors = fixture.accessibilityIssuesAreErrors;
     this.body = body;
   }
 
   TestCase.prototype = {
     /**
      * The name of this test.
      * @type {string}
      */
     name: null,
 
@@ skipping 10 matching lines @@
     body: null,
 
     /**
      * True when the test fixture will run the test later.
      * @type {boolean}
      * @private
      */
     deferred_: false,
 
     /**
+     * @see Test.runAccessibilityChecks
+     * TestCase-specific value for |Test.runAccessibilityChecks|. This allows an
+     * individual test case to override the value in the test fixture, using
+     * enableAccessibilityChecks() or disableAccessibilityChecks().
+     * @type {boolean}
+     */
+    runAccessibilityChecks: true,
+
+    /**
+     * @see Test.accessibilityIssuesAreErrors
+     * TestCase-specific value for |Test.accessibilityIssuesAreErrors|. This
+     * allows an  individual test case to override the value in the test
+     * fixture, using accessibilityIssuesAreErrors(flag).
+     * @type {boolean}
+     */
+    accessibilityIssuesAreErrors: true,
+
+    /**
      * Called at preload time, proxies to the fixture.
      * @type {Function}
      */
     preLoad: function(name) {
       if (this.fixture)
         this.fixture.preLoad();
     },
 
     /**
      * Called before a test runs.
@@ skipping 301 matching lines @@
   var testIsDone = false;
 
   /**
    * Holds the errors, if any, caught by expects so that the test case can
    * fail. Cleared when results are reported from runTest() or testDone().
    * @type {Array.<Error>}
    */
   var errors = [];
 
   /**
+   * Holds any accessibility errors found during the accessibility audit.
+   * Like |errors|, cleared when results are reported.
+   * @type {Array.<string>}
+   */
+  var a11yErrors_ = [];
+
+  /**
+   * Holds any accessibility warnings found during the accessibility audit.
+   * Like |errors|, cleared when results are reported.
+   * @type {Array.<string>}
+   */
+  var a11yWarnings_ = [];
+
+  /**
+   * Gets the list of accessibility errors found during the accessibility
+   * audit. Only for use in testing.
+   * @return {Array.<string>}
+   */
+  function getAccessibilityErrors() {
+    return a11yErrors_;
+  };
+
+  /**
+   * Gets the list of accessibility warnings found during the accessibility
+   * audit. Only for use in testing.
+   * @return {Array.<string>}
+   */
+  function getAccessibilityWarnings() {
+    return a11yWarnings_;
+  };
+
+  /**
    * URL to dummy WebUI page for testing framework.
    * @type {string}
    */
   var DUMMY_URL = 'chrome://DummyURL';
 
   /**
    * Resets test state by clearing |errors| and |testIsDone| flags.
    */
   function resetTestState() {
     errors.splice(0, errors.length);
     testIsDone = false;
   }
 
   /**
    * Notifies the running browser test of the test results. Clears |errors|.
    * @param {Array.<boolean, string>=} result When passed, this is used for the
    *     testResult message.
    */
   function testDone(result) {
     if (!testIsDone) {
       testIsDone = true;
       if (currentTestCase) {
+        if (currentTestCase.runAccessibilityChecks) {
+          var a11yOk = runAccessibilityAudit(a11yErrors_, a11yWarnings_);
+          if (!a11yOk) {
+            if (currentTestCase.accessibilityIssuesAreErrors && result) {
+              result = [false, accessibilityAuditReport(a11yErrors_,
+                                                        a11yWarnings_,
+                                                        result[1])];
+            } else {
+              console.warn(accessibilityAuditReport(a11yErrors_,
+                                                    a11yWarnings_));
+            }
+          }
+        }
+
+        if (result)
+          errors.splice(0, errors.length);
         try {
           currentTestCase.tearDown();
         } catch (e) {
           // Caught an exception in tearDown; Register the error and recreate
           // the result if it is passed in.
           errors.push(e);
           if (result)
             result = [false, errorsToMessage([e], result[1])];
         }
+        if (!result)
+          result = testResult();
+        else if (errors.length) {
+          // Add errors generated in tearDown
+          result = [false, errorsToMessage(errors, result[1])];
+        }
+
         currentTestCase = null;
       }
-      chrome.send('testResult', result ? result : testResult());
+      if (!result)
+        result = testResult();
+      chrome.send('testResult', result);
       errors.splice(0, errors.length);
+      a11yErrors_.splice(0, a11yErrors_.length);
+      a11yWarnings_.splice(0, a11yWarnings_.length);
     } else {
       console.warn('testIsDone already');
     }
   }
 
   /**
    * Converts each Error in |errors| to a suitable message, adding them to
    * |message|, and returns the message string.
    * @param {Array.<Error>} errors Array of errors to add to |message|.
    * @param {string?} message When supplied, error messages are appended to it.
@@ skipping 12 matching lines @@
     return message;
   }
 
   /**
    * Returns [success, message] & clears |errors|.
    * @param {boolean} errorsOk When true, errors are ok.
    * @return {Array.<boolean, string>}
    */
   function testResult(errorsOk) {
     var result = [true, ''];
-    if (errors.length) {
+    if (errors.length)
       result = [!!errorsOk, errorsToMessage(errors)];
+
+    if (currentTestCase &&
+        currentTestCase.accessibilityIssuesAreErrors &&
+        (a11yWarnings_.length || a11yErrors_.length)) {
+      result = [!!errorsOk, accessibilityAuditReport(a11yErrors_,
+                                                     a11yWarnings_,
+                                                     result[1])];
     }
     return result;
   }
 
   // Asserts.
   // Use the following assertions to verify a condition within a test.
   // If assertion fails, throw an Error with information pertinent to the test.
 
   /**
    * When |test| !== true, aborts the current test.
@@ skipping 119 matching lines @@
    * Always aborts the current test.
    * @param {string=} message The message to include in the Error thrown.
    * @throws {Error} always.
    */
   function assertNotReached(message) {
     helper.registerCall();
     throw new Error(helper.getCallMessage(message));
   }
 
   /**
+   * Run an accessibility audit on the current page state.
+   * @type {Function}
+   * @return {boolean} Whether there were any errors or warnings
+   * @private
+   */
+  function runAccessibilityAudit(a11yErrors, a11yWarnings) {
+    var auditResults = axs.Audit.run();
+    for (var i = 0; i < auditResults.length; i++) {
+      var auditResult = auditResults[i];
+      if (auditResult.result == axs.constants.AuditResult.FAIL) {
+        var auditRule = auditResult.rule;
+        // TODO(aboxhall): more useful error messages (sadly non-trivial)
+        if (auditRule.severity == axs.constants.Severity.Severe)
+          a11yErrors.push(accessibilityErrorMessage(auditRule, auditResult));
+        else
+          a11yWarnings.push(accessibilityErrorMessage(auditRule, auditResult));
+      }
+    }
+
+    // TODO(aboxhall): have strict (no errors or warnings) vs non-strict
+    // (warnings ok)
+    // TODO(aboxhall): some kind of info logging for warnings only??
+    return (a11yErrors.length == 0 && a11yWarnings.length == 0);
+  }
+
+  function disableAccessibilityChecks() {
+    if (!currentTestCase)
+      return;
+    currentTestCase.runAccessibilityChecks = false;
+  }
+
+  function enableAccessibilityChecks() {
+    if (!currentTestCase)
+      return;
+    currentTestCase.runAccessibilityChecks = true;
+  }
+
+  function accessibilityIssuesAreErrors(flag) {
+    if (!currentTestCase)
+      return;
+    currentTestCase.accessibilityIssuesAreErrors = flag;
+  }
+
+  /**
+   * Concatenates the accessibility error messages in |a11yErrors| and
+   * |a11yWarnings| in to an accessibility report, appends it to the given
+   * |message| and returns the resulting message string.
+   * @param {Array.<string>} a11yErrors The list of accessibility error messages
+   * @param {Array.<string>} a11yWarnings The list of accessibility warning
+   *     messages.
+   * @return {string} |message| + accessibility report.
+   */
+  function accessibilityAuditReport(a11yErrors, a11yWarnings, message) {
+    message = message ? message + '\n' : '';
+    message += '\n*** Begin accessibility audit results ***';
+    message += '\nAn accessibility audit found ';
+
+    if (a11yErrors.length > 0) {
+      message += a11yErrors.length +
+        (a11yErrors.length == 1 ? ' error ' : ' errors ');
+      if (a11yWarnings.length > 0)
+        message += 'and ';
+    }
+    if (a11yWarnings.length > 0) {
+      message += a11yWarnings.length +
+        (a11yWarnings.length == 1 ? ' warning ' : ' warnings ');
+    }
+    message += 'on this page.\n';
+    message += 'For more information, please see ' +
+      'http://chromium.org/developers/accessibility/webui-accessibility-audit';
+
+    for (var i = 0; i < a11yErrors.length; i++)
+      message += '\n\n' + a11yErrors[i];
+
+    for (var i = 0; i < a11yWarnings.length; i++)
+      message += '\n\n' + a11yWarnings[i];
+    message += '\n*** End accessibility audit results ***';
+    return message;
+  }
+
+  /**
+   * Creates an error message for a given accessibility audit rule and
+   * corresponding result.
+   * @param {axs.AuditRule} rule The audit rule which the result is for
+   * @param {Object.<string, (string|Array.<Element>)>} result The result
+   *     object returned from the audit.
+   * @return {string} An error message describing the failure and listing
+   *     up to five elements which failed the audit rule.
+   */
+  function accessibilityErrorMessage(rule, result) {
+    if (rule.severity == axs.constants.Severity.Severe)
+      var message = 'Error: '
+    else
+      var message = 'Warning: '
+    message += rule.name + ' failed on the following ' +
+      (result.elements.length == 1 ? 'element' : 'elements');
+
+    if (result.elements.length == 1)
+      message += ':'
+    else
+      message += ' (1 - ' + Math.min(5, result.elements.length) +
+                 ' of ' + result.elements.length + '):';
+
+    var maxElements = Math.min(result.elements.length, 5);
+    for (var i = 0; i < maxElements; i++)
+      message += '\n' + axs.utils.getQuerySelectorText(result.elements[i]);
+    return message;
+  }
+
+  /**
+   * Asserts that the current page state passes the accessibility audit.
+   */
+  function assertAccessibilityOk() {
+    helper.registerCall();
+    var a11yErrors = [];
+    var a11yWarnings = [];
+    if (!runAccessibilityAudit(a11yErrors, a11yWarnings))
+      throw new Error(accessibilityAuditReport(a11yErrors, a11yWarnings));
+  }
+
+  /**
    * Creates a function based upon a function that thows an exception on
    * failure. The new function stuffs any errors into the |errors| array for
    * checking by runTest. This allows tests to continue running other checks,
    * while failing the overall test if any errors occurrred.
    * @param {Function} assertFunc The function which may throw an Error.
    * @return {function(...*):bool} A function that applies its arguments to
    *     |assertFunc| and returns true if |assertFunc| passes.
    * @see errors
    * @see runTestFunction
    */
@@ skipping 555 matching lines @@
    * @return {RunAllAction} Action for use in will.
    */
   function runAllActionsAsync(whenTestDone) {
     return new RunAllAction(true, whenTestDone,
                             Array.prototype.slice.call(arguments, 1));
   }
 
   // Exports.
   testing.Test = Test;
   exports.testDone = testDone;
+  exports.accessibilityIssuesAreErrors = accessibilityIssuesAreErrors;
   exports.assertTrue = assertTrue;
   exports.assertFalse = assertFalse;
   exports.assertGE = assertGE;
   exports.assertGT = assertGT;
   exports.assertEquals = assertEquals;
   exports.assertLE = assertLE;
   exports.assertLT = assertLT;
   exports.assertNotEquals = assertNotEquals;
   exports.assertNotReached = assertNotReached;
+  exports.assertAccessibilityOk = assertAccessibilityOk;

[dmazzoni, 2012/12/20 22:05:06]

Nit: sort alphabetically

[aboxhall, 2012/12/20 23:01:41]

I agree in principle, but they're actually not in alpha order currently, so I'm
not sure where it (and the analogous expect method) would go best.

   exports.callFunction = callFunction;
   exports.callFunctionWithSavedArgs = callFunctionWithSavedArgs;
   exports.callGlobalWithSavedArgs = callGlobalWithSavedArgs;
+  exports.disableAccessibilityChecks = disableAccessibilityChecks;
+  exports.enableAccessibilityChecks = enableAccessibilityChecks;
   exports.expectTrue = createExpect(assertTrue);
   exports.expectFalse = createExpect(assertFalse);
   exports.expectGE = createExpect(assertGE);
   exports.expectGT = createExpect(assertGT);
   exports.expectEquals = createExpect(assertEquals);
   exports.expectLE = createExpect(assertLE);
   exports.expectLT = createExpect(assertLT);
   exports.expectNotEquals = createExpect(assertNotEquals);
   exports.expectNotReached = createExpect(assertNotReached);
+  exports.expectAccessibilityOk = createExpect(assertAccessibilityOk);
+  exports.getAccessibilityErrors = getAccessibilityErrors;
+  exports.getAccessibilityWarnings = getAccessibilityWarnings;
   exports.preloadJavascriptLibraries = preloadJavascriptLibraries;
   exports.registerMessageCallback = registerMessageCallback;
   exports.registerMockGlobals = registerMockGlobals;
   exports.registerMockMessageCallbacks = registerMockMessageCallbacks;
   exports.resetTestState = resetTestState;
+  exports.runAccessibilityAudit = runAccessibilityAudit;
   exports.runAllActions = runAllActions;
   exports.runAllActionsAsync = runAllActionsAsync;
   exports.runTest = runTest;
   exports.runTestFunction = runTestFunction;
   exports.SaveMockArguments = SaveMockArguments;
   exports.DUMMY_URL = DUMMY_URL;
   exports.TEST = TEST;
   exports.TEST_F = TEST_F;
+  exports.RUNTIME_TEST_F = TEST_F;
   exports.GEN = GEN;
   exports.GEN_INCLUDE = GEN_INCLUDE;
   exports.WhenTestDone = WhenTestDone;
 
   // Import the Mock4JS helpers.
   Mock4JS.addMockSupport(exports);
 })(this);