Intial checkin of layout test analyzer.

media/tools/layout_tests/layouttests.py

+#!/usr/bin/python
+# Copyright (c) 2011 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""A Module for LayoutTests.

[dennis_jeffrey, 2011/08/25 00:36:46]

We can see this from the file name; is there any other more helpful description
we could give here?

[imasaki1, 2011/08/25 23:57:03]

Done.

+
+Layout tests are stored in Webkit SVN and LayoutTestCaseManager collects these
+layout test cases (including description).
+"""
+
+import copy
+import csv
+import locale
+import pysvn
+import re
+import sys
+import urllib2
+
+
+# Webkit SVN root location.
+DEFAULT_LAYOUTTEST_LOCATION = (
+    'http://svn.webkit.org/repository/webkit/trunk/LayoutTests/')
+
+# When parsing the test HTML file and find test description,

[dennis_jeffrey, 2011/08/25 00:36:46]

'find' --> 'finding the'

[imasaki1, 2011/08/25 23:57:03]

Done.

+# this script tries to find test description using sentences

[dennis_jeffrey, 2011/08/25 00:36:46]

'find test' --> 'find the test'

[imasaki1, 2011/08/25 23:57:03]

Done.

+# starting with these keywords. This is adhoc but it is the only way
+# since there is no standard for writing test description.
+

[dennis_jeffrey, 2011/08/25 00:36:46]

Remove this blank line; the comment above is associated with the statement below

[imasaki1, 2011/08/25 23:57:03]

Done.

+KEYWORDS_FOR_TEST_DESCRIPTION = (
+    ['This tests', 'This test', 'Test that', 'Tests that', 'Test '])

[dennis_jeffrey, 2011/08/25 00:36:46]

I think this list can be shortened.

'This test' implies 'This tests'
'Test' implies 'Tests that' and 'Test that'

[imasaki1, 2011/08/25 23:57:03]

Done.

+
+# If cannot find the keywords, this script tries to find test case
+# description by the following tags.
+TAGS_FOR_TEST_DESCRIPTION = (
+    ['title', 'p', 'div'])

[dennis_jeffrey, 2011/08/25 00:36:46]

Can fit on the previous line?

[imasaki1, 2011/08/25 23:57:03]

Done.

+
+# If cannot find the tags, this script tries to find the test case
+# description in the sentence containing following words.
+KEYWORD_FOR_TEST_DESCRIPTION_FAIL_SAFE = (
+    ['PASSED ', 'PASS:'])
+
+
+class LayoutTests(object):
+  """A class to store test names in layout tests.
+
+  The test names (including regular expression patterns) are read from a CSV
+  file and used for getting layout test names from Webkit SVN.
+  """
+
+  def __init__(self, layouttest_root_path=DEFAULT_LAYOUTTEST_LOCATION,
+               csv_file_path=None):
+    """Initialize LayoutTests

[dennis_jeffrey, 2011/08/25 00:36:46]

Add period at end of line

[imasaki1, 2011/08/25 23:57:03]

Done.

+
+    Args:
+      layouttest_root_path: A location string where Webkit layout tests are
+          stored.
+      csv_file_path: CSV file path. The file contains a list of test names
+          in CSV format.

[dennis_jeffrey, 2011/08/25 00:36:46]

Describe what happens when this parameter is None (the default)

[imasaki1, 2011/08/25 23:57:03]

Done.

+    """
+    if csv_file_path:
+      filter_names = LayoutTests.GetLayoutTestNamesFromCSV(csv_file_path)
+    else:
+      filter_names = []

[dennis_jeffrey, 2011/08/25 00:36:46]

for the above 4 lines:

filter_names = []
if csv_file_path:
  filter_names = LayoutTests.GetLayoutTestNamesFromCSV(csv_file_path)

[imasaki1, 2011/08/25 23:57:03]

Done.

+    parent_location_list = LayoutTests.GetParentDirectoryList(filter_names)
+    if layouttest_root_path.startswith('http://'):
+      name_map = self.GetLayoutTestNamesFromSVN(parent_location_list,
+                                                     layouttest_root_path)

[dennis_jeffrey, 2011/08/25 00:36:46]

indent this line by 5 fewer spaces.

[imasaki1, 2011/08/25 23:57:03]

Done.

+    else:
+      # TODO(imasaki): support other forms such as CSV for reading test names.
+      pass
+    self.name_map = copy.copy(name_map)
+    # Filter names.
+    for lt_name in name_map.keys():
+      match = False
+      for filter_name in filter_names:
+        if re.search(filter_name, lt_name):
+          match = True
+          break
+      if not match:
+        del self.name_map[lt_name]
+    # We get description only for the filtered names.
+    for lt_name in self.name_map.keys():
+      self.name_map[lt_name] = LayoutTests.GetTestDescriptionFromSVN(lt_name)
+
+  @staticmethod
+  def ExtractTestDescription(txt):
+    """Extract the description description from test code in HTML.
+
+    Currently, we have 4 rules described in the code below.
+    example can be fallen into rule 1):

[dennis_jeffrey, 2011/08/25 00:36:46]

(This example falls into rule 1):

[imasaki1, 2011/08/25 23:57:03]

Done.

+    <p>
+    This tests the intrinsic size of a video element is the default
+    300,150 before metadata is loaded, and 0,0 after
+    metadata is loaded for an audio-only file.
+    </p>

[dennis_jeffrey, 2011/08/25 00:36:46]

Indent lines 92-96 a bit since this is an example (to help separate the example
from the text above and below it)

[imasaki1, 2011/08/25 23:57:03]

Done.

+    The strategy is very adhoc since the original test case files
+    (in HTML format) do not have standard way to store test description.
+
+    Args:
+      txt: A HTML text which may or may not contain test description.
+
+    Returns:
+      A string that contains test description. Returns 'UNKNOWN' if the
+          test description is not found.
+    """
+    # (1) Try to find test description contains keywords such as

[dennis_jeffrey, 2011/08/25 00:36:46]

'contains' --> 'that contains'

[imasaki1, 2011/08/25 23:57:03]

Done.

+    #     'test that' and surrounded by p tag.
+    #     We found this is the most common case.

[dennis_jeffrey, 2011/08/25 00:36:46]

'This is the most common case'.

[imasaki1, 2011/08/25 23:57:03]

Done.

+    for keyword in KEYWORDS_FOR_TEST_DESCRIPTION:
+        # Try to find <p> and </p>.

[dennis_jeffrey, 2011/08/25 00:36:46]

Only indent by 2 spaces, not 4, within this function, to keep it consistent with
the indentation used in the rest of this CL.

[imasaki1, 2011/08/25 23:57:03]

Done.

+        pattern = r'<p>(.*' + keyword + '.*)</p>'
+        matches = re.search(pattern, txt)
+        if matches is not None:
+            return matches.group(1).strip()
+
+    # (2) Try to find it by using more generic keywords such as 'PASS' etc.
+    for keyword in KEYWORD_FOR_TEST_DESCRIPTION_FAIL_SAFE:
+        # Try to find new lines.
+        pattern = r'\n(.*' + keyword + '.*)\n'
+        matches = re.search(pattern, txt)
+        if matches is not None:
+            # Remove 'p' tag.
+            text = matches.group(1).strip()
+            return text.replace('<p>', '').replace('</p>', '')
+
+    # (3) Try to find it by using THML tag such as title.

[dennis_jeffrey, 2011/08/25 00:36:46]

'THML' --> 'HTML'

[imasaki1, 2011/08/25 23:57:03]

Done.

+    for tag in TAGS_FOR_TEST_DESCRIPTION:
+        pattern = r'<' + tag + '>(.*)</' + tag + '>'
+        matches = re.search(pattern, txt)
+        if matches is not None:
+            return matches.group(1).strip()
+
+    # (4) Try to find it by using test description and remove 'p' tag.
+    for keyword in KEYWORDS_FOR_TEST_DESCRIPTION:
+        # Try to find <p> and </p>.
+        pattern = r'\n(.*' + keyword + '.*)\n'
+        matches = re.search(pattern, txt)
+        if matches is not None:
+            # Remove 'p' tag.
+            text = matches.group(1).strip()
+            return text.replace('<p>', '').replace('</p>', '')
+
+    # (5) cannot find test description using existing rules.
+    # In this case, just returns 'UNKNOWN'

[dennis_jeffrey, 2011/08/25 00:36:46]

Remove this line.

[imasaki1, 2011/08/25 23:57:03]

Done.

+    return 'UNKNOWN'
+
+  @staticmethod
+  def GetLayoutTestNamesFromSVN(parent_location_list,
+                                layouttest_root_path):
+    """Get LayoutTest names from Webkit SVN.
+
+    Args:
+      parent_location_list: a list of locations of parents directory. This is

[dennis_jeffrey, 2011/08/25 00:36:46]

'parents directory' --> 'parent directories'

[imasaki1, 2011/08/25 23:57:03]

Done.

+          used when getting layout tests using PySVN.list().
+      layouttest_root_path: the root path of the Webkit SVN directory.
+
+    Returns:
+      a map containing test names as keys for de-dupe.
+    """
+    client = pysvn.Client()
+    # Get directory structure in the Webkit SVN.
+    if parent_location_list is None:
+      # Try to get all the layout tests by enabling recursion option.
+      parent_location_list = ['/\S+\.html$']
+      recursion = True
+    else:
+      recursion = False
+    name_map = {}
+    for parent_location in parent_location_list:
+      if parent_location.endswith('/'):
+        file_list = client.list(layouttest_root_path + parent_location,
+                                recurse=recursion)
+        for file_name in file_list:
+          if sys.stdout.isatty():
+            default_encoding = sys.stdout.encoding
+          else:
+            default_encoding = locale.getpreferredencoding()
+          file_name = file_name[0].repos_path.encode(default_encoding)
+          # Remove the word '/truck/LayoutTests'.
+          file_name = file_name.replace('/trunk/LayoutTests/', '')
+          if file_name.endswith('.html'):
+            name_map[file_name] = True
+    return name_map
+
+  @staticmethod
+  def GetLayoutTestNamesFromCSV(csv_file_path):
+    """Get layout test names from CSV file.
+
+    Args:
+      csv_file_path: the path for the CSV file containing test names (including
+          regular expression patterns). The CSV file content has one column and
+          each row contains a test name.
+    """
+    file_object = file(csv_file_path, 'r')
+    reader = csv.reader(file_object)
+    names = []
+    for row in reader:
+      names.append(row[0])

[dennis_jeffrey, 2011/08/25 00:36:46]

For the above 3 lines:

names = [row[0] for row in reader]

[imasaki1, 2011/08/25 23:57:03]

Done.

+    file_object.close()
+    return names
+
+  @staticmethod
+  def GetParentDirectoryList(names):
+    """Get parent directory list from test names.
+
+    Args:
+      names : a list of test names. The test name also have path information as

[dennis_jeffrey, 2011/08/25 00:36:46]

remove the space before the ':'

[dennis_jeffrey, 2011/08/25 00:36:46]

'test name also' -->
'test names also'

[imasaki1, 2011/08/25 23:57:03]

Done.

[imasaki1, 2011/08/25 23:57:03]

Done.

+          well (e.g., media/video-zoom.html).
+    """
+    pd_map = {}
+    for name in names:
+      p_dir = name[0:name.rfind('/') + 1]
+      pd_map[p_dir] = True
+    return list(pd_map.iterkeys())
+
+  def JoinWithTestExpectation(self, test_expections):

[dennis_jeffrey, 2011/08/25 00:36:46]

'test_expections' -->
'test_expectations'

[imasaki1, 2011/08/25 23:57:03]

Done.

+    """Join layout tests with the test expectation file using test name as key.
+
+    Args:
+      test_expectaions: a test expectations object.
+
+    Returns:
+      test_info_map contains test name as key and another map as value. The
+          other map contains test description and the test expectation
+          information which contains keyword (e.g., 'GPU') as key (we do
+          not care about values).

[dennis_jeffrey, 2011/08/25 00:36:46]

If you don't care about values, why not use a list instead of a map?

[imasaki1, 2011/08/25 23:57:03]

But, I have to look the value up several times so I think map is the better
option.

+    """
+    test_info_map = {}
+    for (lt_name, desc) in self.name_map.items():
+      test_info_map[lt_name] = {}
+      test_info_map[lt_name]['desc'] = desc
+      for (te_name, te_info) in (
+          test_expections.all_test_expectation_info.items()):
+        if te_name == lt_name or (
+            te_name in lt_name and te_name.endswith('/')):
+          # Only 1 match (if exist)

[dennis_jeffrey, 2011/08/25 00:36:46]

# Only keep the first match when found.

[imasaki1, 2011/08/25 23:57:03]

Done.

+          test_info_map[lt_name]['te_info'] = te_info
+          break
+    return test_info_map
+
+  @staticmethod
+  def GetTestDescriptionFromSVN(test_location,
+      layouttest_root_path=DEFAULT_LAYOUTTEST_LOCATION):

[dennis_jeffrey, 2011/08/25 00:36:46]

indent this line so it lines up underneath the first parameter in the previous
line

[imasaki1, 2011/08/25 23:57:03]

Done.

+    """Get test description of a layout test from SVN.
+
+    Using urllib2.urlopen(), this method gets the entire HTML and extract its

[dennis_jeffrey, 2011/08/25 00:36:46]

'extract' --> 'extracts'

[imasaki1, 2011/08/25 23:57:03]

Done.

+    test description using |ExtractTestDescription()|.
+
+    Args:
+      test_location: the location of the layout test

[dennis_jeffrey, 2011/08/25 00:36:46]

add period at end of line

[imasaki1, 2011/08/25 23:57:03]

Done.

+      layouttest_root_path: the root path of the Webkit SVN directory.
+
+    Returns:
+      A test description string.
+
+    raises:

[dennis_jeffrey, 2011/08/25 00:36:46]

'raises:' --> 'Raises:'

[imasaki1, 2011/08/25 23:57:03]

Done.

+      A URLError when the layout test is not available.
+    """
+    if test_location.endswith('.html'):
+      resp = urllib2.urlopen(layouttest_root_path + test_location)
+      if resp.code == 200:
+        return LayoutTests.ExtractTestDescription(resp.read())
+    raise URLError

[dennis_jeffrey, 2011/08/25 00:36:46]

Maybe add an informative message to go along with the raised exception?

[imasaki1, 2011/08/25 23:57:03]

Done.