Add domain checking to returned HTTPResponses

tools/chrome_proxy/webdriver/common.py

 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import argparse
 import json
 import logging
 import os
 import re
 import socket
 import shlex
 import sys
 import time
 import traceback
 import unittest
+import urlparse
 
 sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir,
   os.pardir, 'third_party', 'webdriver', 'pylib'))
 from selenium import webdriver
 from selenium.webdriver.chrome.options import Options
 
 def ParseFlags():
   """Parses the given command line arguments.
 
   Returns:
@@ skipping 324 matching lines @@
     all_messages = []
     for log in self._driver.execute('getLog', {'type': 'performance'})['value']:
       message = json.loads(log['message'])['message']
       self._logger.debug('Got Performance log: %s', log['message'])
       if re.match(method_filter, message['method']):
         all_messages.append(message)
     self._logger.info('Got %d performance logs with filter method=%s',
       len(all_messages), method_filter)
     return all_messages
 
-  def GetHTTPResponses(self, include_favicon=False):
+  def GetHTTPResponses(self, include_favicon=False,
+      only_include_loaded_domain=True):

[sclittle, 2016/12/19 19:50:09]

I'm not sure this is the best way to solve the problem of data:// or about:blank
URLs being requested. Could you just filter out all non-http/https URLs by
default, or are there other URLs from other domains that pose problems?

[Robert Ogden, 2016/12/19 20:23:10]

Filtering on presence of the scheme (http,https) would work. The idea behind
domains is not a concrete one, just a thought/fear of unexpected behavior for
test sites that use cross-origin requests.

[sclittle, 2016/12/19 21:17:08]

Hmm - I think I see what you mean, but I'm worried that ignoring the
cross-origin requests could cause more problems than it solves. E.g., suppose
there's an integration test to check if bypass works. If the request that caused
the bypass gets ignored here because it was cross origin, it'll be harder to
debug.

Plus, hopefully it shouldn't be too hard to have our test pages only make
first-party requests. If they do make cross-origin requests, then that's
probably on purpose and we likely don't want to ignore those requests anyways.

WDYT?

[Robert Ogden, 2016/12/19 22:17:46]

SGTM. After looking through the pydoc, I ended up checking the presence of a
net_loc (or domain as I know it) instead of a scheme, since scheme='data' where
url='data:,,' which has been happening a lot recently.

     """Parses the Performance Logs and returns a list of HTTPResponse objects.
 
     Use caution when calling this function  multiple times. Only responses
     since the last time this function was called are returned (or since Chrome
     started, whichever is later).
 
     Args:
       include_favicon: A bool that if True will include responses for favicons.
+      only_include_loaded_domain: If True, only responses with the same domain
+        as the last requested page will be included in the result.
     Returns:
       A list of HTTPResponse objects, each representing a single completed HTTP
       transaction by Chrome.
     """
     def MakeHTTPResponse(log_dict):
       params = log_dict['params']
       response_dict = params['response']
       http_response_dict = {
         'response_headers': response_dict['headers'] if 'headers' in
           response_dict else {},
         'request_headers': response_dict['requestHeaders'] if 'requestHeaders'
           in response_dict else {},
         'url': response_dict['url'] if 'url' in response_dict else '',
         'protocol': response_dict['protocol'] if 'protocol' in response_dict
           else '',
         'port': response_dict['remotePort'] if 'remotePort' in response_dict
           else -1,
         'status': response_dict['status'] if 'status' in response_dict else -1,
         'request_type': params['type'] if 'type' in params else ''
       }
       return HTTPResponse(**http_response_dict)
     all_responses = []
     for message in self.GetPerformanceLogs():
       response = MakeHTTPResponse(message)
       self._logger.debug('New HTTPResponse: %s', str(response))
       is_favicon = response.url.endswith('favicon.ico')
-      if not is_favicon or include_favicon:
+      is_same_loaded_domain = (urlparse.urlparse(response.url).netloc ==
+        urlparse.urlparse(self._url).netloc)
+      if ((not is_favicon or include_favicon) and
+          (not only_include_loaded_domain or is_same_loaded_domain)):
         all_responses.append(response)
+      else:
+        self._logger.info("Skipping HTTPResponse with url=%s in returned logs.",
+          response.url)
     self._logger.info('%d new HTTPResponse objects found in the logs %s '
       'favicons', len(all_responses), ('including' if include_favicon else
       'not including'))
     return all_responses
 
 class HTTPResponse:
   """This class represents a single HTTP transaction (request and response) by
   Chrome.
 
   This class also includes several convenience functions for ChromeProxy
@@ skipping 114 matching lines @@
     """
     flags = ParseFlags()
     logger = GetLogger()
     logger.debug('Command line args: %s', str(sys.argv))
     logger.info('sys.argv parsed to %s', str(flags))
     # The unittest library uses sys.argv itself and is easily confused by our
     # command line options. Pass it a simpler argv instead, while working in the
     # unittest command line args functionality.
     unittest.main(argv=[sys.argv[0]], verbosity=2, failfast=flags.failfast,
       catchbreak=flags.catch, buffer=(not flags.disable_buffer))