Import wpt@fd6560f225668e933bfb147ee0e20d5971a0d21f

third_party/WebKit/LayoutTests/external/wpt/check_stability.py

 from __future__ import print_function
 
 import argparse
 import json
 import logging
 import os
 import re
 import stat
 import subprocess
 import sys
@@ skipping 71 matching lines @@
         self.name = name
 
     def __enter__(self):
         print("travis_fold:start:%s" % self.name, file=sys.stderr)
 
     def __exit__(self, type, value, traceback):
         print("travis_fold:end:%s" % self.name, file=sys.stderr)
 
 
 class GitHub(object):
-    def __init__(self, org, repo, token, browser):
+    def __init__(self, org, repo, token, product):
         self.token = token
         self.headers = {"Accept": "application/vnd.github.v3+json"}
         self.auth = (self.token, "x-oauth-basic")
         self.org = org
         self.repo = repo
         self.base_url = "https://api.github.com/repos/%s/%s/" % (org, repo)
-        self.browser = browser
+        self.product = product
 
     def _headers(self, headers):
         if headers is None:
             headers = {}
         rv = self.headers.copy()
         rv.update(headers)
         return rv
 
     def post(self, url, data, headers=None):
         logger.debug("POST %s" % url)
@@ skipping 28 matching lines @@
             headers=self._headers(headers),
             auth=self.auth
         )
         resp.raise_for_status()
         return resp
 
     def post_comment(self, issue_number, body):
         user = self.get(urljoin(self.base_url, "/user")).json()
         issue_comments_url = urljoin(self.base_url, "issues/%s/comments" % issue_number)
         comments = self.get(issue_comments_url).json()
-        title_line = "# %s #" % self.browser.title()
+        title_line = format_comment_title(self.product)
         data = {"body": body}
         for comment in comments:
             if (comment["user"]["login"] == user["login"] and
                 comment["body"].startswith(title_line)):
                 comment_url = urljoin(self.base_url, "issues/comments/%s" % comment["id"])
                 self.patch(comment_url, data)
                 break
         else:
             self.post(issue_comments_url, data)
 
-    def releases(self):
-        url = urljoin(self.base_url, "releases/latest")
-        return self.get(url)
-
 
 class GitHubCommentHandler(logging.Handler):
     def __init__(self, github, pull_number):
         logging.Handler.__init__(self)
         self.github = github
         self.pull_number = pull_number
         self.log_data = []
 
     def emit(self, record):
         try:
@@ skipping 57 matching lines @@
             "certutil_binary": "certutil",
             "webdriver_binary": "%s/geckodriver" % root,
             "prefs_root": "%s/profiles" % root,
         }
 
 
 class Chrome(Browser):
     product = "chrome"
 
     def install(self):
-        latest = get("https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2FLAST_CHANGE?alt=media").text.strip()
-        url = "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%%2F%s%%2Fchrome-linux.zip?alt=media" % latest
-        unzip(get(url).raw)
-        logger.debug(call("ls", "-lhrt", "chrome-linux"))
+        # Installing the Google Chrome browser requires administrative
+        # privileges, so that installation is handled by the invoking script.
+
         call("pip", "install", "-r", os.path.join("w3c", "wptrunner", "requirements_chrome.txt"))
 
     def install_webdriver(self):
         latest = get("http://chromedriver.storage.googleapis.com/LATEST_RELEASE").text.strip()
         url = "http://chromedriver.storage.googleapis.com/%s/chromedriver_linux64.zip" % latest
         unzip(get(url).raw)
         st = os.stat('chromedriver')
         os.chmod('chromedriver', st.st_mode | stat.S_IEXEC)
 
     def wptrunner_args(self, root):
         return {
             "product": "chrome",
-            "binary": "%s/chrome-linux/chrome" % root,
+            "binary": "/usr/bin/google-chrome",
+            # Chrome's "sandbox" security feature must be disabled in order to
+            # run the browser in OpenVZ environments such as the one provided
+            # by TravisCI.
+            #
+            # Reference: https://github.com/travis-ci/travis-ci/issues/938
+            "binary_arg": "--no-sandbox",
             "webdriver_binary": "%s/chromedriver" % root,
             "test_types": ["testharness", "reftest"]
         }
 
 
 def get(url):
     logger.debug("GET %s" % url)
     resp = requests.get(url, stream=True)
     resp.raise_for_status()
     return resp
@@ skipping 44 matching lines @@
     with zipfile.ZipFile(fileobj) as zip_data:
         for info in zip_data.infolist():
             zip_data.extract(info)
             perm = info.external_attr >> 16 & 0x1FF
             os.chmod(info.filename, perm)
 
 
 def setup_github_logging(args):
     gh_handler = None
     if args.comment_pr:
-        github = GitHub("w3c", "web-platform-tests", args.gh_token, args.browser)
+        github = GitHub("w3c", "web-platform-tests", args.gh_token, args.product)
         try:
             pr_number = int(args.comment_pr)
         except ValueError:
             pass
         else:
             gh_handler = GitHubCommentHandler(github, pr_number)
             gh_handler.setLevel(logging.INFO)
             logger.debug("Setting up GitHub logging")
             logger.addHandler(gh_handler)
     else:
@@ skipping 153 matching lines @@
     handler = LogHandler()
     reader.handle_log(reader.read(log), handler)
     results = handler.results
     for test, test_results in results.iteritems():
         for subtest, result in test_results.iteritems():
             if is_inconsistent(result, iterations):
                 inconsistent.append((test, subtest, result))
     return results, inconsistent
 
 
+def format_comment_title(product):
+    """Produce a Markdown-formatted string based on a given "product"--a string
+    containing a browser identifier optionally followed by a colon and a
+    release channel. (For example: "firefox" or "chrome:dev".) The generated
+    title string is used both to create new comments and to locate (and
+    subsequently update) previously-submitted comments."""
+    parts = product.split(":")
+    title = parts[0].title()
+
+    if len(parts) > 1:
+       title += " (%s channel)" % parts[1]
+
+    return "# %s #" % title
+
+
 def markdown_adjust(s):
     s = s.replace('\t', u'\\t')
     s = s.replace('\n', u'\\n')
     s = s.replace('\r', u'\\r')
     s = s.replace('`',  u'\\`')
     return s
 
 
 def table(headings, data, log):
     cols = range(len(headings))
@@ skipping 58 matching lines @@
                         type=int,
                         help="Number of times to run tests")
     parser.add_argument("--gh-token",
                         action="store",
                         default=os.environ.get("GH_TOKEN"),
                         help="OAuth token to use for accessing GitHub api")
     parser.add_argument("--comment-pr",
                         action="store",
                         default=os.environ.get("TRAVIS_PULL_REQUEST"),
                         help="PR to comment on with stability results")
-    parser.add_argument("browser",
+    parser.add_argument("product",
                         action="store",
-                        help="Browser to run against")
+                        help="Product to run against (`browser-name` or 'browser-name:channel')")
     return parser
 
 
 def main():
     retcode = 0
     parser = get_parser()
     args = parser.parse_args()
 
     if not os.path.exists(args.root):
         logger.critical("Root directory %s does not exist" % args.root)
         return 1
 
     os.chdir(args.root)
 
     if args.gh_token:
         gh_handler = setup_github_logging(args)
     else:
         logger.warning("Can't log to GitHub")
         gh_handler = None
 
+    browser_name = args.product.split(":")[0]
+
     with TravisFold("browser_setup"):
-        logger.info("# %s #" % args.browser.title())
+        logger.info(format_comment_title(args.product))
 
         browser_cls = {"firefox": Firefox,
-                       "chrome": Chrome}.get(args.browser)
+                       "chrome": Chrome}.get(browser_name)
         if browser_cls is None:
-            logger.critical("Unrecognised browser %s" % args.browser)
+            logger.critical("Unrecognised browser %s" % browser_name)
             return 1
 
         fetch_wpt_master()
 
         head_sha1 = get_sha1()
         logger.info("Testing revision %s" % head_sha1)
 
         # For now just pass the whole list of changed files to wptrunner and
         # assume that it will run everything that's actually a test
         files_changed = get_files_changed()
@@ skipping 65 matching lines @@
     return retcode
 
 
 if __name__ == "__main__":
     try:
         retcode = main()
     except:
         raise
     else:
         sys.exit(retcode)