Import wpt@40665266227e475bc4a56884247d8c09d78dfb6a

third_party/WebKit/LayoutTests/external/wpt/check_stability.py

 from __future__ import print_function
 
 import argparse
 import json
 import logging
 import os
 import re
 import stat
 import subprocess
 import sys
 import tarfile
 import traceback
 import zipfile
 from cStringIO import StringIO
 from collections import defaultdict
+from ConfigParser import RawConfigParser
+from io import BytesIO
 from urlparse import urljoin
 from tools.manifest import manifest
 
 import requests
 
 BaseHandler = None
 LogActionFilter = None
 LogHandler = None
 LogLevelFilter = None
 StreamHandler = None
 TbplFormatter = None
 reader = None
 wptcommandline = None
 wptrunner = None
+wpt_root = None
+wptrunner_root = None
 
 logger = logging.getLogger(os.path.splitext(__file__)[0])
 
 
 def do_delayed_imports():
     global BaseHandler
     global LogLevelFilter
     global StreamHandler
     global TbplFormatter
     global reader
@@ skipping 131 matching lines @@
         except Exception:
             self.handleError(record)
 
     def send(self):
         self.github.post_comment(self.pull_number, "\n".join(self.log_data))
         self.log_data = []
 
 
 class Browser(object):
     product = None
+    binary = None
 
     def __init__(self, github_token):
         self.github_token = github_token
 
 
 class Firefox(Browser):
     product = "firefox"
+    binary = "%s/firefox/firefox"
+    platform_ini = "%s/firefox/platform.ini"
 
     def install(self):
-        call("pip", "install", "-r", "w3c/wptrunner/requirements_firefox.txt")
+        call("pip", "install", "-r", os.path.join(wptrunner_root, "requirements_firefox.txt"))
         resp = get("https://archive.mozilla.org/pub/firefox/nightly/latest-mozilla-central/firefox-53.0a1.en-US.linux-x86_64.tar.bz2")
         untar(resp.raw)
 
         if not os.path.exists("profiles"):
             os.mkdir("profiles")
         with open(os.path.join("profiles", "prefs_general.js"), "wb") as f:
             resp = get("https://hg.mozilla.org/mozilla-central/raw-file/tip/testing/profiles/prefs_general.js")
             f.write(resp.content)
-        call("pip", "install", "-r", os.path.join("w3c", "wptrunner", "requirements_firefox.txt"))
+        call("pip", "install", "-r", os.path.join(wptrunner_root, "requirements_firefox.txt"))
 
     def _latest_geckodriver_version(self):
         # This is used rather than an API call to avoid rate limits
         tags = call("git", "ls-remote", "--tags", "--refs",
                     "https://github.com/mozilla/geckodriver.git")
         release_re = re.compile(".*refs/tags/v(\d+)\.(\d+)\.(\d+)")
         latest_release = 0
         for item in tags.split("\n"):
             m = release_re.match(item)
             if m:
                 version = [int(item) for item in m.groups()]
                 if version > latest_release:
                     latest_release = version
         assert latest_release != 0
         return "v%s.%s.%s" % tuple(str(item) for item in latest_release)
 
     def install_webdriver(self):
         version = self._latest_geckodriver_version()
         logger.debug("Latest geckodriver release %s" % version)
         url = "https://github.com/mozilla/geckodriver/releases/download/%s/geckodriver-%s-linux64.tar.gz" % (version, version)
         untar(get(url).raw)
 
+    def version(self, root):
+        """Retrieve the release version of the installed browser."""
+        platform_info = RawConfigParser()
+
+        with open(self.platform_ini % root, "r") as fp:
+            platform_info.readfp(BytesIO(fp.read()))
+            return "BuildID %s; SourceStamp %s" % (
+                platform_info.get("Build", "BuildID"),
+                platform_info.get("Build", "SourceStamp"))
+
     def wptrunner_args(self, root):
         return {
             "product": "firefox",
-            "binary": "%s/firefox/firefox" % root,
+            "binary": self.binary % root,
             "certutil_binary": "certutil",
             "webdriver_binary": "%s/geckodriver" % root,
             "prefs_root": "%s/profiles" % root,
         }
 
 
 class Chrome(Browser):
     product = "chrome"
+    binary = "/usr/bin/google-chrome"
 
     def install(self):
         # Installing the Google Chrome browser requires administrative
         # privileges, so that installation is handled by the invoking script.
 
-        call("pip", "install", "-r", os.path.join("w3c", "wptrunner", "requirements_chrome.txt"))
+        call("pip", "install", "-r", os.path.join(wptrunner_root, "requirements_chrome.txt"))
 
     def install_webdriver(self):
         latest = get("http://chromedriver.storage.googleapis.com/LATEST_RELEASE").text.strip()
         url = "http://chromedriver.storage.googleapis.com/%s/chromedriver_linux64.zip" % latest
         unzip(get(url).raw)
         st = os.stat('chromedriver')
         os.chmod('chromedriver', st.st_mode | stat.S_IEXEC)
 
+    def version(self, root):
+        """Retrieve the release version of the installed browser."""
+        output = call(self.binary, "--version")
+        return re.search(r"[0-9a-z\.]+$", output.strip()).group(0)
+
     def wptrunner_args(self, root):
         return {
             "product": "chrome",
-            "binary": "/usr/bin/google-chrome",
+            "binary": self.binary,
             # Chrome's "sandbox" security feature must be disabled in order to
             # run the browser in OpenVZ environments such as the one provided
             # by TravisCI.
             #
             # Reference: https://github.com/travis-ci/travis-ci/issues/938
             "binary_arg": "--no-sandbox",
             "webdriver_binary": "%s/chromedriver" % root,
             "test_types": ["testharness", "reftest"]
         }
 
@@ skipping 50 matching lines @@
     with zipfile.ZipFile(fileobj) as zip_data:
         for info in zip_data.infolist():
             zip_data.extract(info)
             perm = info.external_attr >> 16 & 0x1FF
             os.chmod(info.filename, perm)
 
 
 def setup_github_logging(args):
     gh_handler = None
     if args.comment_pr:
-        github = GitHub("w3c", "web-platform-tests", args.gh_token, args.product)
+        github = GitHub(args.user, "web-platform-tests", args.gh_token, args.product)
         try:
             pr_number = int(args.comment_pr)
         except ValueError:
             pass
         else:
             gh_handler = GitHubCommentHandler(github, pr_number)
             gh_handler.setLevel(logging.INFO)
             logger.debug("Setting up GitHub logging")
             logger.addHandler(gh_handler)
     else:
         logger.warning("No PR number found; not posting to GitHub")
     return gh_handler
 
 
 class pwd(object):
     def __init__(self, dir):
         self.dir = dir
         self.old_dir = None
 
     def __enter__(self):
         self.old_dir = os.path.abspath(os.curdir)
         os.chdir(self.dir)
 
     def __exit__(self, *args, **kwargs):
         os.chdir(self.old_dir)
         self.old_dir = None
 
 
-def fetch_wpt_master():
-    git = get_git_cmd(os.path.join(os.path.abspath(os.curdir), "w3c", "web-platform-tests"))
-    git("fetch", "https://github.com/w3c/web-platform-tests.git", "master:master")
+def fetch_wpt_master(user):
+    git = get_git_cmd(wpt_root)
+    git("fetch", "https://github.com/%s/web-platform-tests.git" % user, "master:master")
 
 
 def get_sha1():
-    git = get_git_cmd(os.path.join(os.path.abspath(os.curdir), "w3c", "web-platform-tests"))
+    git = get_git_cmd(wpt_root)
     return git("rev-parse", "HEAD").strip()
 
 
 def build_manifest():
-    with pwd(os.path.join(os.path.abspath(os.curdir), "w3c", "web-platform-tests")):
+    with pwd(wpt_root):
         # TODO: Call the manifest code directly
         call("python", "manifest")
 
 
 def install_wptrunner():
-    call("git", "clone", "--depth=1", "https://github.com/w3c/wptrunner.git", "w3c/wptrunner")
-    git = get_git_cmd(os.path.join(os.path.abspath(os.curdir), "w3c", "wptrunner"))
+    call("git", "clone", "--depth=1", "https://github.com/w3c/wptrunner.git", wptrunner_root)
+    git = get_git_cmd(wptrunner_root)
     git("submodule", "update", "--init", "--recursive")
-    call("pip", "install", os.path.join("w3c", "wptrunner"))
+    call("pip", "install", wptrunner_root)
 
 
 def get_files_changed():
     root = os.path.abspath(os.curdir)
-    git = get_git_cmd("%s/w3c/web-platform-tests" % root)
+    git = get_git_cmd(wpt_root)
     branch_point = git("merge-base", "HEAD", "master").strip()
     logger.debug("Branch point from master: %s" % branch_point)
     logger.debug(git("log", "--oneline", "%s.." % branch_point))
     files = git("diff", "--name-only", "-z", "%s.." % branch_point)
     if not files:
         return []
     assert files[-1] == "\0"
-    return ["%s/w3c/web-platform-tests/%s" % (root, item)
+    return ["%s/%s" % (wpt_root, item)
             for item in files[:-1].split("\0")]
 
 
 def get_affected_testfiles(files_changed):
     affected_testfiles = []
     all_tests = set()
     nontests_changed = set(files_changed)
-    repo_root = os.path.abspath(os.path.join(os.path.abspath(os.curdir), "w3c", "web-platform-tests"))
-    manifest_file = os.path.join(repo_root, "MANIFEST.json")
-    for test, _ in manifest.load(repo_root, manifest_file):
-        test_full_path = os.path.join(repo_root, test)
+    manifest_file = os.path.join(wpt_root, "MANIFEST.json")
+    for _, test, _ in manifest.load(wpt_root, manifest_file):
+        test_full_path = os.path.join(wpt_root, test)
         all_tests.add(test_full_path)
         if test_full_path in nontests_changed:
             # Reduce the set of changed files to only non-tests.
             nontests_changed.remove(test_full_path)
     for changedfile_pathname in nontests_changed:
-        changed_file_repo_path = os.path.join(os.path.sep, os.path.relpath(changedfile_pathname, repo_root))
+        changed_file_repo_path = os.path.join(os.path.sep, os.path.relpath(changedfile_pathname, wpt_root))
         os.path.normpath(changed_file_repo_path)
         path_components = changed_file_repo_path.split(os.sep)[1:]
         if len(path_components) < 2:
             # This changed file is in the repo root, so skip it
             # (because it's not part of any test).
             continue
         top_level_subdir = path_components[0]
         if top_level_subdir in ["conformance-checkers", "docs"]:
             continue
         # OK, this changed file is the kind we care about: It's something
         # other than a test (e.g., it's a .js or .json file), and it's
         # somewhere down beneath one of the top-level "spec" directories.
         # So now we try to find any tests that reference it.
-        for root, dirs, fnames in os.walk(os.path.join(repo_root, top_level_subdir)):
+        for root, dirs, fnames in os.walk(os.path.join(wpt_root, top_level_subdir)):
             # Walk top_level_subdir looking for test files containing either the
             # relative filepath or absolute filepatch to the changed file.
             for fname in fnames:
                 testfile_full_path = os.path.join(root, fname)
                 # Skip any test file that's already in files_changed.
                 if testfile_full_path in files_changed:
                     continue
                 # Skip any file that's not a test file.
                 if testfile_full_path not in all_tests:
                     continue
                 with open(testfile_full_path, "r") as fh:
                     file_contents = fh.read()
                     changed_file_relpath = os.path.relpath(changedfile_pathname, root).replace(os.path.sep, "/")
                     if changed_file_relpath in file_contents or changed_file_repo_path.replace(os.path.sep, "/") in file_contents:
                         affected_testfiles.append(testfile_full_path)
     return affected_testfiles
 
 
 def wptrunner_args(root, files_changed, iterations, browser):
     parser = wptcommandline.create_parser([browser.product])
     args = vars(parser.parse_args([]))
-    wpt_root = os.path.join(root, "w3c", "web-platform-tests")
     args.update(browser.wptrunner_args(root))
     args.update({
         "tests_root": wpt_root,
         "metadata_root": wpt_root,
         "repeat": iterations,
-        "config": "%s/w3c/wptrunner/wptrunner.default.ini" % root,
+        "config": "%s//wptrunner.default.ini" % (wptrunner_root),
         "test_list": files_changed,
         "restart_on_unexpected": False,
         "pause_after_test": False
     })
     wptcommandline.check_args(args)
     return args
 
 
 def setup_log_handler():
     global LogHandler
@@ skipping 125 matching lines @@
                         type=int,
                         help="Number of times to run tests")
     parser.add_argument("--gh-token",
                         action="store",
                         default=os.environ.get("GH_TOKEN"),
                         help="OAuth token to use for accessing GitHub api")
     parser.add_argument("--comment-pr",
                         action="store",
                         default=os.environ.get("TRAVIS_PULL_REQUEST"),
                         help="PR to comment on with stability results")
+    parser.add_argument("--user",
+                        action="store",
+                        # Travis docs say do not depend on USER env variable.
+                        # This is a workaround to get what should be the same value
+                        default=os.environ.get("TRAVIS_REPO_SLUG").split('/')[0],
+                        help="Travis user name")
     parser.add_argument("product",
                         action="store",
                         help="Product to run against (`browser-name` or 'browser-name:channel')")
     return parser
 
 
 def main():
+    global wpt_root
+    global wptrunner_root
+
     retcode = 0
     parser = get_parser()
     args = parser.parse_args()
 
+    wpt_root = os.path.abspath(os.curdir)
+    wptrunner_root = os.path.normpath(os.path.join(wpt_root, "..", "wptrunner"))
+
     if not os.path.exists(args.root):
         logger.critical("Root directory %s does not exist" % args.root)
         return 1
 
     os.chdir(args.root)
 
     if args.gh_token:
         gh_handler = setup_github_logging(args)
     else:
         logger.warning("Can't log to GitHub")
         gh_handler = None
 
     browser_name = args.product.split(":")[0]
 
     with TravisFold("browser_setup"):
         logger.info(format_comment_title(args.product))
 
         browser_cls = {"firefox": Firefox,
                        "chrome": Chrome}.get(browser_name)
         if browser_cls is None:
             logger.critical("Unrecognised browser %s" % browser_name)
             return 1
 
-        fetch_wpt_master()
+        fetch_wpt_master(args.user)
 
         head_sha1 = get_sha1()
-        logger.info("Testing revision %s" % head_sha1)
+        logger.info("Testing web-platform-tests at revision %s" % head_sha1)
 
         # For now just pass the whole list of changed files to wptrunner and
         # assume that it will run everything that's actually a test
         files_changed = get_files_changed()
 
         if not files_changed:
             logger.info("No files changed")
             return 0
 
         build_manifest()
         install_wptrunner()
         do_delayed_imports()
 
+        browser = browser_cls(args.gh_token)
+        browser.install()
+        browser.install_webdriver()
+
+        try:
+            version = browser.version(args.root)
+        except Exception, e:
+            version = "unknown (error: %s)" % e
+        logger.info("Using browser at version %s", version)
+
         logger.debug("Files changed:\n%s" % "".join(" * %s\n" % item for item in files_changed))
 
         affected_testfiles = get_affected_testfiles(files_changed)
 
         logger.debug("Affected tests:\n%s" % "".join(" * %s\n" % item for item in affected_testfiles))
 
         files_changed.extend(affected_testfiles)
 
-        browser = browser_cls(args.gh_token)
-
-        browser.install()
-        browser.install_webdriver()
-
         kwargs = wptrunner_args(args.root,
                                 files_changed,
                                 args.iterations,
                                 browser)
 
     with TravisFold("running_tests"):
         logger.info("Starting %i test iterations" % args.iterations)
         with open("raw.log", "wb") as log:
             wptrunner.setup_logging(kwargs,
                                     {"raw": log})
@@ skipping 33 matching lines @@
     return retcode
 
 
 if __name__ == "__main__":
     try:
         retcode = main()
     except:
         raise
     else:
         sys.exit(retcode)