Add domain checking to returned HTTPResponses

tools/chrome_proxy/webdriver/common.py

 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import argparse
 import json
 import logging
 import os
 import re
 import socket
 import shlex
 import sys
 import time
 import traceback
 import unittest
+import urlparse
 
 sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir,
   os.pardir, 'third_party', 'webdriver', 'pylib'))
 from selenium import webdriver
 from selenium.webdriver.chrome.options import Options
 
 def ParseFlags():
   """Parses the given command line arguments.
 
   Returns:
@@ skipping 20 matching lines @@
     'error streams are buffered during the test run, and output from a '
     'passing test is discarded. Output will always be echoed normally on test '
     'fail or error and is added to the failure messages.', action='store_true')
   parser.add_argument('-c', '--catch', help='Control-C during the test run '
     'waits for the current test to end and then reports all the results so '
     'far. A second Control-C raises the normal KeyboardInterrupt exception.',
     action='store_true')
   parser.add_argument('-f', '--failfast', help='Stop the test run on the first '
     'error or failure.', action='store_true')
   parser.add_argument('--logging_level', choices=['DEBUG', 'INFO', 'WARN',
-    'ERROR', 'CRIT'], default='ERROR', help='The logging verbosity for log '
+    'ERROR', 'CRIT'], default='WARN', help='The logging verbosity for log '
     'messages, printed to stderr. To see stderr logging output during a '
     'successful test run, also pass --disable_buffer. Default=ERROR')
   parser.add_argument('--log_file', help='If given, write logging statements '
     'to the given file instead of stderr.')
   return parser.parse_args(sys.argv[1:])
 
 def GetLogger(name='common'):
   """Creates a Logger instance with the given name and returns it.
 
   If a logger has already been created with the same name, that instance is
@@ skipping 186 matching lines @@
 
   def LoadURL(self, url, timeout=30):
     """Starts Chromium with any arguments previously given and navigates to the
     given URL.
 
     Args:
       url: The URL to navigate to.
       timeout: The time in seconds to load the page before timing out.
     """
     self._url = url
+    if len(urlparse.urlparse(url).netloc) == 0:
+      self._logger.warn('No net_loc given in %s. See RFC 1808', url)

[sclittle, 2016/12/19 22:38:15]

nit: could you make this message a bit clearer? e.g., if an invalid URL was
passed in, you could check this and mention this specifically.

You could also only log when the URL is invalid, since it would be nice to
support calling LoadURL with an encoded "data:" URI.

[Robert Ogden, 2016/12/19 22:56:44]

Done.

     if not self._driver:
       self._StartDriver()
     self._driver.set_page_load_timeout(timeout)
     self._logger.debug('Set page load timeout to %f seconds', timeout)
     self._driver.get(self._url)
     self._logger.debug('Loaded page %s', url)
 
   def ExecuteJavascript(self, script, timeout=30):
     """Executes the given javascript in the browser's current page in an
     anonymous function.
@@ skipping 77 matching lines @@
     all_messages = []
     for log in self._driver.execute('getLog', {'type': 'performance'})['value']:
       message = json.loads(log['message'])['message']
       self._logger.debug('Got Performance log: %s', log['message'])
       if re.match(method_filter, message['method']):
         all_messages.append(message)
     self._logger.info('Got %d performance logs with filter method=%s',
       len(all_messages), method_filter)
     return all_messages
 
-  def GetHTTPResponses(self, include_favicon=False):
+  def GetHTTPResponses(self, include_favicon=False, skip_domainless_pages=True):
     """Parses the Performance Logs and returns a list of HTTPResponse objects.
 
     Use caution when calling this function  multiple times. Only responses
     since the last time this function was called are returned (or since Chrome
     started, whichever is later).
 
     Args:
       include_favicon: A bool that if True will include responses for favicons.
+      skip_domainless_pages: If True, only responses with a net_loc as in RFC
+        1808 will be included. Pages such as about:blank will be skipped.
     Returns:
       A list of HTTPResponse objects, each representing a single completed HTTP
       transaction by Chrome.
     """
     def MakeHTTPResponse(log_dict):
       params = log_dict['params']
       response_dict = params['response']
       http_response_dict = {
         'response_headers': response_dict['headers'] if 'headers' in
           response_dict else {},
         'request_headers': response_dict['requestHeaders'] if 'requestHeaders'
           in response_dict else {},
         'url': response_dict['url'] if 'url' in response_dict else '',
         'protocol': response_dict['protocol'] if 'protocol' in response_dict
           else '',
         'port': response_dict['remotePort'] if 'remotePort' in response_dict
           else -1,
         'status': response_dict['status'] if 'status' in response_dict else -1,
         'request_type': params['type'] if 'type' in params else ''
       }
       return HTTPResponse(**http_response_dict)
     all_responses = []
     for message in self.GetPerformanceLogs():
       response = MakeHTTPResponse(message)
       self._logger.debug('New HTTPResponse: %s', str(response))
       is_favicon = response.url.endswith('favicon.ico')
-      if not is_favicon or include_favicon:
+      has_domain = len(urlparse.urlparse(response.url).netloc) > 0
+      if (not is_favicon or include_favicon) and (not skip_domainless_pages or
+          has_domain):
         all_responses.append(response)
+      else:
+        self._logger.info("Skipping HTTPResponse with url=%s in returned logs.",
+          response.url)
     self._logger.info('%d new HTTPResponse objects found in the logs %s '
       'favicons', len(all_responses), ('including' if include_favicon else
       'not including'))
     return all_responses
 
 class HTTPResponse:
   """This class represents a single HTTP transaction (request and response) by
   Chrome.
 
   This class also includes several convenience functions for ChromeProxy
@@ skipping 114 matching lines @@
     """
     flags = ParseFlags()
     logger = GetLogger()
     logger.debug('Command line args: %s', str(sys.argv))
     logger.info('sys.argv parsed to %s', str(flags))
     # The unittest library uses sys.argv itself and is easily confused by our
     # command line options. Pass it a simpler argv instead, while working in the
     # unittest command line args functionality.
     unittest.main(argv=[sys.argv[0]], verbosity=2, failfast=flags.failfast,
       catchbreak=flags.catch, buffer=(not flags.disable_buffer))