Add ability to create CSV of Stale Test Expectations with list of all bugs and filenames

third_party/WebKit/Tools/Scripts/print-stale-test-expectations-entries

 #!/usr/bin/python
 #
 # Copyright (C) 2013 Google Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 # notice, this list of conditions and the following disclaimer.
@@ skipping 12 matching lines @@
 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 """Prints lists of bug numbers / tests whose bugs haven't been modified recently."""
 
+import csv

[Dirk Pranke, 2016/05/12 02:11:15]

this is no longer needed ...

[nainar, 2016/05/12 04:58:47]

Removed.

 import datetime
 import json
 import optparse
 import re
 import sys
 import time
 import urllib2
 
+from webkitpy.common.host import Host
 from webkitpy.common.system.filesystem import FileSystem
 from webkitpy.common.webkit_finder import WebKitFinder
+from webkitpy.layout_tests.models.test_expectations import TestExpectationParser
 
 google_code_url = 'https://www.googleapis.com/projecthosting/v2/projects/chromium/issues/%s?key=AIzaSyDgCqT1Dt5AZWLHo4QJjyMHaCjhnFacGF0'
 crbug_prefix = 'crbug.com/'
 
+class StaleTestCSV(object):

[Dirk Pranke, 2016/05/12 02:11:15]

this class isn't really adding any value. I would suggest making the changes
below, instead.

[nainar, 2016/05/12 04:58:47]

Done.

+    def __init__(self, filename):
+        self._filename = filename
+
+    def add_to_csv(self, bug_number, test_name):
+        file = open(self._filename, 'a')
+        file.write("%s, %s\n" % (bug_number, test_name))
+
 class StaleTestPrinter(object):
     def __init__(self, options):
         self._days = options.days
+        self._stale_test_csv = StaleTestCSV(options.create_csv)

[Dirk Pranke, 2016/05/12 02:11:15]

change this to self.create_csv = options.create_csv.

Also, there's no harm in making these fields public, really, so I'd remove the
leading underscores.

[nainar, 2016/05/12 04:58:47]

Done.

Done.

 
-    def is_stale(self, bug_number):
+    def is_stale(self, bug_link):
+        bug_number = bug_link.strip(crbug_prefix)
         url = google_code_url % bug_number
         response = urllib2.urlopen(url)
         parsed = json.loads(response.read())
         last_updated = parsed['updated']
         parsed_time = datetime.datetime.strptime(last_updated.split(".")[0]+"UTC", "%Y-%m-%dT%H:%M:%S%Z")
         time_delta = datetime.datetime.now() - parsed_time
-        return time_delta.days > 90
+        return time_delta.days > self._days
 
     def print_stale_tests(self):
-        finder = WebKitFinder(FileSystem())
-        path_to_expectations = finder.path_from_webkit_base('LayoutTests', 'TestExpectations')
-        expectations = open(path_to_expectations)
-
-        for line in expectations:
-            comment_index = line.find("#")
-            if comment_index == -1:
-                comment_index = len(line)
-
-            remaining_string = re.sub(r"\s+", " ", line[:comment_index].strip())
-            if len(remaining_string) == 0:
-                continue
-
-            is_bug_stale = True
-            parts = line.split(' ')
-            for part in parts:
-                if part.startswith(crbug_prefix):
-                    bug_number = part.split('/')[1]
-                    try:
-                        if not self.is_stale(bug_number):
-                            is_bug_stale = False
-                            break;
-                    except urllib2.HTTPError as error:
-                        if error.code == 404:
-                            print '%s%s does not exist.' % (crbug_prefix, bug_number)
-                        elif error.code == 403:
-                            print '%s%s is not accessible. Not able to tell if it\'s stale.' % (crbug_prefix, bug_number)
-                            is_bug_stale = False
-                        else:
-                            raise error
-
-            if is_bug_stale:
-                print line.strip()
+        port = Host().port_factory.get()
+        exps = port.expectations_dict()

[Dirk Pranke, 2016/05/12 02:11:15]

change this to :

  host = Host()
  port = host.port_factory.get()
  exps = port.expectations_dict()
  csv_contents = ''

i.e., save the host pointer and add a new csv_contents var.

[nainar, 2016/05/12 04:58:47]

Done.

+        parser = TestExpectationParser(port, all_tests=(), is_lint_mode=False)
+        for line in parser.parse(*exps.items()[0]):
+            bugs, name = line.bugs, line.name
+            try:
+                if bugs and all(self.is_stale(bug) for bug in bugs):
+                    print line.original_string.strip()
+                    if self._stale_test_csv._filename:
+                        self._stale_test_csv.add_to_csv(bugs[0], name)

[Dirk Pranke, 2016/05/12 02:11:15]

change lines 82-83 to:

csv_contents += "%s, %s\n" % (bugs[0], name)

This builds the csv_contents unconditionally, but that should be fine.

[nainar, 2016/05/12 04:58:47]

Done.

+            except urllib2.HTTPError as error:
+                if error.code == 404:
+                    print 'Does not exist.'
+                elif error.code == 403:
+                    print 'Is not accessible. Not able to tell if it\'s stale.'
+                    is_bug_stale = False
+                else:
+                    print error

[Dirk Pranke, 2016/05/12 02:11:15]

add two lines de-dented out to match lines 76-77 (i.e., after the for loop)

if self.create_csv:
  host.filesystem.write_text_file(self.create_csv, csv_contents)

[nainar, 2016/05/12 04:58:47]

Done.

 
 def main(argv):
     option_parser = optparse.OptionParser()
     option_parser.add_option('--days', type='int', default=90, help='Number of days to consider a bug stale.'),
+    option_parser.add_option('--create-csv', type='string', default=0, help='Generate a CSV of the stale entries as well. Followed by the filename.'),
     options, args = option_parser.parse_args(argv)
 
     printer = StaleTestPrinter(options)
     printer.print_stale_tests()
     return 0
 
 if __name__ == '__main__':
     sys.exit(main(sys.argv[1:]))