Add an accessibility audit test for WebUI pages

chrome/test/data/webui/accessibility_audit_browsertest.js

Index: chrome/test/data/webui/accessibility_audit_browsertest.js
diff --git a/chrome/test/data/webui/accessibility_audit_browsertest.js b/chrome/test/data/webui/accessibility_audit_browsertest.js
new file mode 100644
index 0000000000000000000000000000000000000000..360a4647c550e2bbf3f84cedb255ae0f97312484
--- /dev/null
+++ b/chrome/test/data/webui/accessibility_audit_browsertest.js
@@ -0,0 +1,145 @@
+// Copyright (c) 2012 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+function WebUIAccessibilityAuditBrowserTest() {}

[Sheridan Rawlins, 2012/12/18 01:43:35]

JSDoc this class/constructor

[aboxhall, 2012/12/18 19:21:35]

Done.

+
+WebUIAccessibilityAuditBrowserTest.prototype = {
+  __proto__: testing.Test.prototype,
+
+  browsePreload: 'chrome://terms',
+
+  runAccessibilityChecks: true,
+  accessibilityIssuesAreErrors: true,
+
+  // 'a11y' is short for 'accessibility'
+  expectedWarnings: null,

[Sheridan Rawlins, 2012/12/18 01:43:35]

Should these be true/false?

[aboxhall, 2012/12/18 19:21:35]

They are numerical values. Added JSDoc.

+  expectedErrors: null,
+
+  tearDown: function() {
+    if (this.expectedErrors != null)
+      expectEquals(this.expectedErrors, getAccessibilityErrors().length);
+    if (this.expectedWarnings != null)
+      assertEquals(this.expectedWarnings, getAccessibilityWarnings().length);

[Sheridan Rawlins, 2012/12/18 01:43:35]

expect?  Otherwise, it'll skip the call to superclass's tearDown (and fail to
report mock4js errors, if any).

[aboxhall, 2012/12/18 19:21:35]

Ah yes, I was having trouble with expect initially and forgot to change it back.
Thanks.

+    testing.Test.prototype.tearDown.call(this);
+  }
+};
+
+function addAuditFailures() {

[Sheridan Rawlins, 2012/12/18 01:43:35]

jsdoc

[aboxhall, 2012/12/18 19:21:35]

Done.

+  // Contrast ratio
+  var style = document.createElement('style');
+  style.innerText = 'p { color: #ffffff }';
+  document.head.appendChild(style);
+
+  // Bad ARIA role
+  var div = document.createElement('div');
+  div.setAttribute('role', 'not-a-role');
+  document.body.appendChild(div);
+
+  // Unlabelled control
+  var input = document.createElement('input');
+  input.type = 'text';
+  document.body.appendChild(input);
+}
+
+function createMockAudit() {

[Sheridan Rawlins, 2012/12/18 01:43:35]

jsdoc

[aboxhall, 2012/12/18 19:21:35]

Done.

+  function StubAudit() {};
+  StubAudit.prototype.run = function() {};
+  return mock(StubAudit);
+}
+
+function expectAuditWillNotRun() {

[Sheridan Rawlins, 2012/12/18 01:43:35]

jsdoc

[aboxhall, 2012/12/18 19:21:35]

Done.

+  var audit = createMockAudit();
+  audit.expects(never()).run();
+  axs.Audit = audit.proxy();
+}
+
+function expectAuditWillRun(times) {

[Sheridan Rawlins, 2012/12/18 01:43:35]

jsdoc

[aboxhall, 2012/12/18 19:21:35]

Done.

+  var audit = createMockAudit();
+  var realAudit = axs.Audit;
+  var expectedInvocation = audit.expects(exactly(times)).run();
+  var willArgs = [];
+  for (var i = 0; i < times; i++)
+    willArgs.push(callFunction(realAudit.run));
+  ExpectedInvocation.prototype.will.apply(expectedInvocation, willArgs);

[Sheridan Rawlins, 2012/12/18 01:43:35]

I'm a bit confused by this… couldn't you just do something like the following in
the for loop?

expectedInvocation.will(callFunction(realAudit.run));

[aboxhall, 2012/12/18 19:21:35]

Unfortunately, no: will() can only be called once; it destroys all previous
expectations each time it's run. See
https://code.google.com/searchframe#OAMlx_jo-ck/src/chrome/third_party/mock4j...
and
https://code.google.com/searchframe#OAMlx_jo-ck/src/chrome/third_party/mock4j...

[scr, 2012/12/18 23:13:44]

Ok, so how 'bout this after the for loop instead of needing the cumbersome
"apply" syntax?

for(…) …
audit.expects(exactly(times)).run().will(willArgs)

On 2012/12/18 19:21:35, aboxhall wrote:

[aboxhall, 2012/12/19 19:10:05]

will() only takes varargs, not an array - hence the use of apply().

+  axs.Audit = audit.proxy();
+}
+
+TEST_F('WebUIAccessibilityAuditBrowserTest', 'testWithAuditFailures_shouldFail',

[Sheridan Rawlins, 2012/12/18 01:43:35]

regular comment ( // ) what the test intention is.

[aboxhall, 2012/12/18 19:21:35]

Done.

+       function() {
+  expectAuditWillRun(1);
+  addAuditFailures();
+});
+
+TEST_F('WebUIAccessibilityAuditBrowserTest',

[Sheridan Rawlins, 2012/12/18 01:43:35]

// regular comment

[aboxhall, 2012/12/18 19:21:35]

Done.

+       'testWithAuditFailures_a11yChecksDisabled',
+       function() {
+  expectAuditWillNotRun();
+  disableAccessibilityChecks();
+  addAuditFailures();
+});
+
+function WebUIAccessibilityAuditBrowserTest_TestsDisabledInFixture() {}

[Sheridan Rawlins, 2012/12/18 01:43:35]

jsdoc this class/constructor

[aboxhall, 2012/12/18 19:21:35]

Done.

+
+WebUIAccessibilityAuditBrowserTest_TestsDisabledInFixture.prototype = {
+  __proto__: WebUIAccessibilityAuditBrowserTest.prototype,
+
+  runAccessibilityChecks: false,
+  accessibilityIssuesAreErrors: true,
+};
+
+TEST_F('WebUIAccessibilityAuditBrowserTest_TestsDisabledInFixture',

[Sheridan Rawlins, 2012/12/18 01:43:35]

// regular comment

+       'testWithAuditFailures_shouldFail',
+       function() {
+  expectAuditWillRun(1);
+  enableAccessibilityChecks();
+  addAuditFailures();
+});

[Sheridan Rawlins, 2012/12/18 01:43:35]

For kicks, could we add a version of this that does

expectAuditWillRun(2);
...
expectAccessibilityOk();

So we can see it run once with our expect, and once with the tearDown?

[aboxhall, 2012/12/18 19:21:35]

Added at the end, in the fixture with |accessibilityIssuesAreErrors| set to
false, so that it won't cause spurious test passes when the test fails in an
unexpected (due to mock expectations) rather than expected (due to audit
failure) way.

+
+TEST_F('WebUIAccessibilityAuditBrowserTest_TestsDisabledInFixture',

[Sheridan Rawlins, 2012/12/18 01:43:35]

// regular comment

[aboxhall, 2012/12/18 19:21:35]

Done.

+       'testWithAuditFailures_a11yChecksNotEnabled',
+       function() {
+  expectAuditWillNotRun();
+  addAuditFailures();
+});
+
+TEST_F('WebUIAccessibilityAuditBrowserTest_TestsDisabledInFixture',

[Sheridan Rawlins, 2012/12/18 01:43:35]

// regular comment

[aboxhall, 2012/12/18 19:21:35]

Done.

+       'testRunningAuditManually_noErrors',
+       function() {
+  expectAuditWillRun(1);
+  expectAccessibilityOk();
+});
+
+TEST_F('WebUIAccessibilityAuditBrowserTest_TestsDisabledInFixture',

[Sheridan Rawlins, 2012/12/18 01:43:35]

// regular comment

[aboxhall, 2012/12/18 19:21:35]

Done.

+       'testRunningAuditManually_withErrors_shouldFail',
+       function() {
+  expectAuditWillRun(1);
+  addAuditFailures();
+  expectAccessibilityOk();
+});
+
+TEST_F('WebUIAccessibilityAuditBrowserTest_TestsDisabledInFixture',

[Sheridan Rawlins, 2012/12/18 01:43:35]

// regular comment

[aboxhall, 2012/12/18 19:21:35]

Done.

+       'testRunningAuditManuallySeveralTimes', function() {
+  expectAuditWillRun(2);
+  expectAccessibilityOk();
+  expectAccessibilityOk();
+});
+
+function WebUIAccessibilityAuditBrowserTest_IssuesAreWarnings() {}

[Sheridan Rawlins, 2012/12/18 01:43:35]

jsdoc class/constructor

[aboxhall, 2012/12/18 19:21:35]

Done.

+
+WebUIAccessibilityAuditBrowserTest_IssuesAreWarnings.prototype = {
+  __proto__: WebUIAccessibilityAuditBrowserTest.prototype,
+
+  accessibilityIssuesAreErrors: false,
+};
+
+TEST_F('WebUIAccessibilityAuditBrowserTest_IssuesAreWarnings',

[Sheridan Rawlins, 2012/12/18 01:43:35]

// regular comment

[aboxhall, 2012/12/18 19:21:35]

Done.

+       'testWithAuditFailures',
+        function() {
+  expectAuditWillRun(1);
+  this.expectedWarnings = 1;
+  this.expectedErrors = 2;
+  enableAccessibilityChecks();
+  addAuditFailures();
+});
+

[Sheridan Rawlins, 2012/12/18 01:43:35]

Any way to test for warnings being logged?  Maybe in the C++?

Might have to adjust the C++ of WebUIBrowserTest's LogHandler to also save the
logging::LOG_WARNING (or whatever the log level is for console.warn) so it could
be checked by a test in C++.

http://code.google.com/searchframe#OAMlx_jo-ck/src/chrome/browser/ui/webui/we...

[aboxhall, 2012/12/18 19:21:35]

Would it be sufficient to mock out console.warn similarly to axs.Audit.run?

[scr, 2012/12/18 23:13:44]

Sure, that seems reasonable.

On 2012/12/18 19:21:35, aboxhall wrote:

[aboxhall, 2012/12/19 19:10:05]

Done.