Enable CNS to serve files from different port.

media/tools/constrained_network_server/cns.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Constrained Network Server. Serves files with supplied network constraints.
 
 The CNS exposes a web based API allowing network constraints to be imposed on
 file serving.
 
 TODO(dalecurtis): Add some more docs here.
 
 """
 
 import logging
 import mimetypes
 import optparse
 import os
 import signal
 import sys
 import threading
 import time
+import urllib2
+
 import traffic_control
 
 try:
   import cherrypy
 except ImportError:
   print ('CNS requires CherryPy v3 or higher to be installed. Please install\n'
          'and try again. On Linux: sudo apt-get install python-cherrypy3\n')
   sys.exit(1)
 
 # Add webm file types to mimetypes map since cherrypy's default type is text.
@@ skipping 160 matching lines @@
     if no_cache:
       response = cherrypy.response
       response.headers['Pragma'] = 'no-cache'
       response.headers['Cache-Control'] = 'no-cache'
 
     # CherryPy is a bit wonky at detecting parameters, so just make them all
     # optional and validate them ourselves.
     if not f:
       raise cherrypy.HTTPError(400, 'Invalid request. File must be specified.')
 
+    # Check existence early to prevent wasted constraint setup.
+    self._CheckRequestedFileExist(f)
+
+    # If there are no constraints, just serve the file.
+    if bandwidth is None and latency is None and loss is None:
+      return self._ServeFile(f)
+
+    constrained_port = self._GetConstrainedPort(
+        f, bandwidth=bandwidth, latency=latency, loss=loss, new_port=new_port,
+        **kwargs)
+
+    # Build constrained URL using the constrained port and original URL
+    # parameters except the network constraints (bandwidth, latency, and loss).
+    if self._options.local_server_port:
+      constrained_url = self._GetLocalServerURL(f, constrained_port, **kwargs)
+    else:
+      constrained_url = '%s?f=%s&no_cache=%s&%s' % (

[DaleCurtis, 2012/08/07 23:53:57]

This looks similar to the self._GetLocalServerURL() can we collapse these two
methods into one? This code is already kind of ugly, no need to have it twice :)

[shadi, 2012/08/08 22:55:49]

They are not actually that similar, but Done. :-)

+          cherrypy.url().replace(
+              ':%d' % self._options.port, ':%d' % constrained_port),
+          '%s' % f,
+          no_cache,
+          '&'.join(['%s=%s' % (key, kwargs[key]) for key in kwargs]))
+
+    # Redirect request to the constrained port.
+    cherrypy.log('Redirect to %s' % constrained_url)
+    cherrypy.lib.cptools.redirect(constrained_url, internal=False)
+
+  def _CheckRequestedFileExist(self, f):
+    """Checks if the requested file exists, raises HTTPError otherwise."""
+    if self._options.local_server_port:
+      self._CheckFileExistOnLocalServer(f)
+    else:
+      self._CheckFileExistOnServer(f)
+
+  def _CheckFileExistOnServer(self, f):
+    """Checks if requested f exists to be served by this server."""

[DaleCurtis, 2012/08/07 23:53:57]

s/f/file f/

[shadi, 2012/08/08 22:55:49]

Done.

     # Sanitize and check the path to prevent www-root escapes.
     sanitized_path = os.path.abspath(os.path.join(self._options.www_root, f))
     if not sanitized_path.startswith(self._options.www_root):
       raise cherrypy.HTTPError(403, 'Invalid file requested.')
-
-    # Check existence early to prevent wasted constraint setup.
     if not os.path.exists(sanitized_path):
       raise cherrypy.HTTPError(404, 'File not found.')
 
-    # If there are no constraints, just serve the file.
-    if bandwidth is None and latency is None and loss is None:
+  def _CheckFileExistOnLocalServer(self, f):

[DaleCurtis, 2012/08/07 23:53:57]

Hmmm, what is urlopen doing behind the scenes here? This is more expensive than
it's worth if it's actually pulling down the remote file.

If we want to keep this we could require --www-data-dir and --local-server-port
to be specified and just use the original os.path.isfile() check.

[shadi, 2012/08/08 22:55:49]

No it is not pulling down the remote file. It returns a socket file object which
pulls down all the file when a read() is done.
It pulls down the headers and little data. Using urllib allows checking the URL
itself would work and not just that the file exist.

+    """Checks if requested file exists on local server hosting files."""
+    test_url = self._GetLocalServerURL(f, self._options.local_server_port)
+    try:
+      cherrypy.log('Check file exist using URL: %s' % test_url)
+      return urllib2.urlopen(test_url) is not None
+    except Exception:
+      raise cherrypy.HTTPError(404, 'File not found on local server.')
+
+  def _ServeFile(self, f):
+    """Serves the file as an http response."""
+    if self._options.local_server_port:
+      redirect_url = self._GetLocalServerURL(f, self._options.local_server_port)
+      cherrypy.log('Redirect to %s' % redirect_url)
+      cherrypy.lib.cptools.redirect(redirect_url, internal=False)

[DaleCurtis, 2012/08/07 23:53:57]

Do you need to return anything here to keep CherryPy happy with
ServeConstrained?

[shadi, 2012/08/08 22:55:49]

CherryPy has not been complaining :-) 

+    else:
+      sanitized_path = os.path.abspath(os.path.join(self._options.www_root, f))
       return cherrypy.lib.static.serve_file(sanitized_path)
 
+  def _GetLocalServerURL(self, f, port, **kwargs):
+    """Returns a URL for local server to serve the file on given port.
+
+    Args:
+      f: file name to serve on local server. Relative to www_root.
+      port: Local server port (it can be a configured constrained port).
+      kwargs: extra parameteres passed in the URL.
+    """
+    base_url = cherrypy.url().replace('ServeConstrained',
+                                      self._options.www_root)
+    base_url = base_url.replace(':%d' % self._options.port, ':%d' % port)
+    url = '%s/%s' % (base_url, f)
+    extra_args = '&'.join(['%s=%s' % (key, kwargs[key]) for key in kwargs])
+    if extra_args:
+      url += '?%s' % extra_args
+    return url
+
+  def _GetConstrainedPort(self, f=None, bandwidth=None, latency=None, loss=None,
+                          new_port=False, **kwargs):
+    """Creates or gets a port with specified network constraints.
+
+    See ServeConstrained() for more details.
+    """
     # Validate inputs. isdigit() guarantees a natural number.
     bandwidth = self._ParseIntParameter(
         bandwidth, 'Invalid bandwidth constraint.', lambda x: x > 0)
     latency = self._ParseIntParameter(
         latency, 'Invalid latency constraint.', lambda x: x >= 0)
     loss = self._ParseIntParameter(
         loss, 'Invalid loss constraint.', lambda x: x <= 100 and x >= 0)
 
+    redirect_port = self._options.port
+    if self._options.local_server_port:
+      redirect_port = self._options.local_server_port
+
+    start_time = time.time()
     # Allocate a port using the given constraints. If a port with the requested
-    # key is already allocated, it will be reused.
-    #
-    # TODO(dalecurtis): The key cherrypy.request.remote.ip might not be unique
-    # if build slaves are sharing the same VM.
-    start_time = time.time()
+    # key and kwargs already exist then reuse that port.
     constrained_port = self._port_allocator.Get(
-        cherrypy.request.remote.ip, server_port=self._options.port,
+        cherrypy.request.remote.ip, server_port=redirect_port,
         interface=self._options.interface, bandwidth=bandwidth, latency=latency,
         loss=loss, new_port=new_port, file=f, **kwargs)
-    end_time = time.time()
+
+    cherrypy.log('Time to set up port %d = %ssec.' %
+                 (constrained_port, time.time() - start_time))
 
     if not constrained_port:
       raise cherrypy.HTTPError(503, 'Service unavailable. Out of ports.')
-
-    cherrypy.log('Time to set up port %d = %ssec.' %
-                 (constrained_port, end_time - start_time))
-
-    # Build constrained URL using the constrained port and original URL
-    # parameters except the network constraints (bandwidth, latency, and loss).
-    constrained_url = '%s?f=%s&no_cache=%s&%s' % (
-        cherrypy.url().replace(
-            ':%d' % self._options.port, ':%d' % constrained_port),
-        f,
-        no_cache,
-        '&'.join(['%s=%s' % (key, kwargs[key]) for key in kwargs]))
-
-    # Redirect request to the constrained port.
-    cherrypy.lib.cptools.redirect(constrained_url, internal=False)
+    return constrained_port
 
   def _ParseIntParameter(self, param, msg, check):
     """Returns integer value of param and verifies it satisfies the check.
 
     Args:
       param: Parameter name to check.
       msg: Message in error if raised.
       check: Check to verify the parameter value.
 
     Returns:
@@ skipping 30 matching lines @@
                     help=('Interface to setup constraints on. Use lo for a '
                           'local client. Default: %default'))
   parser.add_option('--socket-timeout', type='int',
                     default=cherrypy.server.socket_timeout,
                     help=('Number of seconds before a socket connection times '
                           'out. Default: %default'))
   parser.add_option('--threads', type='int',
                     default=cherrypy._cpserver.Server.thread_pool,
                     help=('Number of threads in the thread pool. Default: '
                           '%default'))
-  parser.add_option('--www-root', default=os.getcwd(),
+  parser.add_option('--www-root', default='',

[DaleCurtis, 2012/08/07 23:53:57]

just remove the default='' ?

[shadi, 2012/08/08 22:55:49]

I actually put this so that www_root == '' when --local-server-port is used.

                     help=('Directory root to serve files from. Defaults to the '
-                          'current directory: %default'))
+                          'current directory (if --local-server-port is not '
+                          'used): %s' % os.getcwd()))
+  parser.add_option('--local-server-port', type='int',
+                    help=('Optional local server port to host files.'))
   parser.add_option('-v', '--verbose', action='store_true', default=False,
                     help='Turn on verbose output.')
 
   options = parser.parse_args()[0]
 
   # Convert port range into the desired tuple format.
   try:
     if isinstance(options.port_range, str):
       options.port_range = [int(port) for port in options.port_range.split(',')]
   except ValueError:
     parser.error('Invalid port range specified.')
 
   if options.expiry_time < 0:
     parser.error('Invalid expiry time specified.')
 
   # Convert the path to an absolute to remove any . or ..
-  options.www_root = os.path.abspath(options.www_root)
+  if not options.local_server_port:
+    if not options.www_root:
+      options.www_root = os.getcwd()
+    options.www_root = os.path.abspath(options.www_root)
 
   # Required so that cherrypy logs do not get propagated to root logger causing
   # the logs to be printed twice.
   cherrypy.log.error_log.propagate = False
   cherrypy.log.access_log.propagate = False
 
   _SetLogger(options.verbose)
 
   return options
 
@@ skipping 31 matching lines @@
   try:
     cherrypy.quickstart(ConstrainedNetworkServer(options, pa))
   finally:
     # Disable Ctrl-C handler to prevent interruption of cleanup.
     signal.signal(signal.SIGINT, lambda signal, frame: None)
     pa.Cleanup(all_ports=True)
 
 
 if __name__ == '__main__':
   Main()