Import wpt@f23222137d3fbc14fae1518e64ab3aa9bbf32a7c

third_party/WebKit/LayoutTests/imported/wpt/check_stability.py

 import argparse
 import json
 import logging
 import os
 import re
 import stat
 import subprocess
 import sys
 import tarfile
 import traceback
 import zipfile
 from cStringIO import StringIO
 from collections import defaultdict
 from urlparse import urljoin
 
 import requests
 
+BaseHandler = None
+LogActionFilter = None
+LogHandler = None
+LogLevelFilter = None
+StreamHandler = None
+TbplFormatter = None
+reader = None
+wptcommandline = None
 wptrunner = None
-wptcommandline = None
-reader = None
-LogHandler = None
+
 
 logger = logging.getLogger(os.path.splitext(__file__)[0])
 
 
 def do_delayed_imports():
-    global wptrunner, wptcommandline, reader
-    from wptrunner import wptrunner
-    from wptrunner import wptcommandline
+    global BaseHandler
+    global LogLevelFilter
+    global StreamHandler
+    global TbplFormatter
+    global reader
+    global wptcommandline
+    global wptrunner
     from mozlog import reader
+    from mozlog.formatters import TbplFormatter
+    from mozlog.handlers import BaseHandler, LogLevelFilter, StreamHandler
+    from wptrunner import wptcommandline, wptrunner
     setup_log_handler()
+    setup_action_filter()
 
 
 def setup_logging():
     handler = logging.StreamHandler(sys.stdout)
     formatter = logging.Formatter(logging.BASIC_FORMAT, None)
     handler.setFormatter(formatter)
     logger.addHandler(handler)
     logger.setLevel(logging.DEBUG)
 
 setup_logging()
 
 
+def setup_action_filter():
+    global LogActionFilter
+
+    class LogActionFilter(BaseHandler):
+        """Handler that filters out messages with action of log and a level
+        lower than some specified level.
+
+        :param inner: Handler to use for messages that pass this filter
+        :param level: Minimum log level to process
+        """
+        def __init__(self, inner, actions):
+            BaseHandler.__init__(self, inner)
+            self.inner = inner
+            self.actions = actions
+
+        def __call__(self, item):
+            if item["action"] in self.actions:
+                return self.inner(item)
+
+
 class GitHub(object):
     def __init__(self, org, repo, token):
         self.token = token
         self.headers = {"Accept": "application/vnd.github.v3+json"}
         self.auth = (self.token, "x-oauth-basic")
         self.org = org
         self.repo = repo
         self.base_url = "https://api.github.com/repos/%s/%s/" % (org, repo)
 
     def _headers(self, headers):
@@ skipping 73 matching lines @@
             os.mkdir("profiles")
         with open(os.path.join("profiles", "prefs_general.js"), "wb") as f:
             resp = get("https://hg.mozilla.org/mozilla-central/raw-file/tip/testing/profiles/prefs_general.js")
             f.write(resp.content)
         call("pip", "install", "-r", os.path.join("w3c", "wptrunner", "requirements_firefox.txt"))
 
     def _latest_geckodriver_version(self):
         # This is used rather than an API call to avoid rate limits
         tags = call("git", "ls-remote", "--tags", "--refs",
                     "https://github.com/mozilla/geckodriver.git")
-        logger.debug("Found tags:\n%s" % tags)
         release_re = re.compile(".*refs/tags/v(\d+)\.(\d+)\.(\d+)")
         latest_release = 0
         for item in tags.split("\n"):
             m = release_re.match(item)
             if m:
                 version = [int(item) for item in m.groups()]
                 if version > latest_release:
                     latest_release = version
         assert latest_release != 0
         return "v%s.%s.%s" % tuple(str(item) for item in latest_release)
@@ skipping 227 matching lines @@
     max_widths = reduce(lambda prev, cur: [(len(cur[i]) + 2)
                                            if (len(cur[i]) + 2) > prev[i]
                                            else prev[i]
                                            for i in cols],
                         data,
                         [len(item) + 2 for item in headings])
     log("|%s|" % "|".join(item.center(max_widths[i]) for i, item in enumerate(headings)))
     log("|%s|" % "|".join("-" * max_widths[i] for i in cols))
     for row in data:
         log("|%s|" % "|".join(" %s" % row[i].ljust(max_widths[i] - 1) for i in cols))
+    log("")
 
 
 def write_inconsistent(inconsistent, iterations):
     logger.error("## Unstable results ##\n")
     strings = [(test, subtest if subtest else "", err_string(results, iterations))
                 for test, subtest, results in inconsistent]
     table(["Test", "Subtest", "Results"], strings, logger.error)
 
 
-def write_results(results, iterations):
+def write_results(results, iterations, comment_pr):
     logger.info("## All results ##\n")
     for test, test_results in results.iteritems():
-        logger.info("### %s ###" % test)
+        baseurl = "http://w3c-test.org/submissions"
+        if "https" in os.path.splitext(test)[0].split(".")[1:]:
+            baseurl = "https://w3c-test.org/submissions"
+        pr_number = None
+        if comment_pr:
+            try:
+                pr_number = int(comment_pr)
+            except ValueError:
+                pass
+        if pr_number:
+            logger.info("### [%s](%s/%s%s) ###" % (test, baseurl, pr_number, test))
+        else:
+            logger.info("### %s ###" % test)
         parent = test_results.pop(None)
         strings = [("", err_string(parent, iterations))]
         strings.extend(((subtest if subtest else "", err_string(results, iterations))
                         for subtest, results in test_results.iteritems()))
         table(["Subtest", "Results"], strings, logger.info)
 
 
 def get_parser():
     parser = argparse.ArgumentParser()
     parser.add_argument("--root",
@@ skipping 69 matching lines @@
     browser.install()
     browser.install_webdriver()
 
     kwargs = wptrunner_args(args.root,
                             files_changed,
                             args.iterations,
                             browser)
 
     print >> sys.stderr, "travis_fold:end:browser_setup"
     print >> sys.stderr, "travis_fold:start:running_tests"
+    logger.info("Starting %i test iterations" % args.iterations)
     with open("raw.log", "wb") as log:
         wptrunner.setup_logging(kwargs,
-                                {"tbpl": sys.stdout,
-                                 "raw": log})
+                                {"raw": log})
+        # Setup logging for wptrunner that keeps process output and
+        # warning+ level logs only
+        wptrunner.logger.add_handler(
+            LogActionFilter(
+                LogLevelFilter(
+                    StreamHandler(
+                        sys.stdout,
+                        TbplFormatter()
+                    ),
+                    "WARNING"),
+                ["log", "process_output"]))
+
         wptrunner.run_tests(**kwargs)
 
     with open("raw.log", "rb") as log:
         results, inconsistent = process_results(log, args.iterations)
 
     print >> sys.stderr, "travis_fold:end:running_tests"
 
     if results:
         if inconsistent:
             write_inconsistent(inconsistent, args.iterations)
             retcode = 2
         else:
             logger.info("All results were stable\n")
         print >> sys.stderr, "travis_fold:start:full_results"
-        write_results(results, args.iterations)
+        write_results(results, args.iterations, args.comment_pr)
         print >> sys.stderr, "travis_fold:end:full_results"
     else:
         logger.info("No tests run.")
 
     try:
         if gh_handler:
             gh_handler.send()
     except Exception:
         logger.error(traceback.format_exc())
     return retcode
 
 
 if __name__ == "__main__":
     try:
         retcode = main()
     except:
         raise
     else:
         sys.exit(retcode)