Enable CNS to serve files from different port.

media/tools/constrained_network_server/cns.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Constrained Network Server. Serves files with supplied network constraints.
 
 The CNS exposes a web based API allowing network constraints to be imposed on
 file serving.
 
 TODO(dalecurtis): Add some more docs here.
 
 """
 
 import logging
 from logging import handlers
 import mimetypes
 import optparse
 import os
 import signal
 import sys
 import threading
 import time
+import urllib
+import urllib2
+
 import traffic_control
 
 try:
   import cherrypy
 except ImportError:
   print ('CNS requires CherryPy v3 or higher to be installed. Please install\n'
          'and try again. On Linux: sudo apt-get install python-cherrypy3\n')
   sys.exit(1)
 
 # Add webm file types to mimetypes map since cherrypy's default type is text.
@@ skipping 160 matching lines @@
     if no_cache:
       response = cherrypy.response
       response.headers['Pragma'] = 'no-cache'
       response.headers['Cache-Control'] = 'no-cache'
 
     # CherryPy is a bit wonky at detecting parameters, so just make them all
     # optional and validate them ourselves.
     if not f:
       raise cherrypy.HTTPError(400, 'Invalid request. File must be specified.')
 
+    # Check existence early to prevent wasted constraint setup.
+    self._CheckRequestedFileExist(f)
+
+    # If there are no constraints, just serve the file.
+    if bandwidth is None and latency is None and loss is None:
+      return self._ServeFile(f)
+
+    constrained_port = self._GetConstrainedPort(
+        f, bandwidth=bandwidth, latency=latency, loss=loss, new_port=new_port,
+        **kwargs)
+
+    # Build constrained URL using the constrained port and original URL
+    # parameters except the network constraints (bandwidth, latency, and loss).
+    constrained_url = self._GetServerURL(f, constrained_port,
+                                         no_cache=no_cache, **kwargs)
+
+    # Redirect request to the constrained port.
+    cherrypy.log('Redirect to %s' % constrained_url)
+    cherrypy.lib.cptools.redirect(constrained_url, internal=False)
+
+  def _CheckRequestedFileExist(self, f):
+    """Checks if the requested file exists, raises HTTPError otherwise."""
+    if self._options.local_server_port:
+      self._CheckFileExistOnLocalServer(f)
+    else:
+      self._CheckFileExistOnServer(f)
+
+  def _CheckFileExistOnServer(self, f):
+    """Checks if requested file f exists to be served by this server."""
     # Sanitize and check the path to prevent www-root escapes.
     sanitized_path = os.path.abspath(os.path.join(self._options.www_root, f))
     if not sanitized_path.startswith(self._options.www_root):
       raise cherrypy.HTTPError(403, 'Invalid file requested.')
-
-    # Check existence early to prevent wasted constraint setup.
     if not os.path.exists(sanitized_path):
       raise cherrypy.HTTPError(404, 'File not found.')
 
-    # If there are no constraints, just serve the file.
-    if bandwidth is None and latency is None and loss is None:
+  def _CheckFileExistOnLocalServer(self, f):
+    """Checks if requested file exists on local server hosting files."""
+    test_url = self._GetServerURL(f, self._options.local_server_port)
+    try:
+      cherrypy.log('Check file exist using URL: %s' % test_url)
+      return urllib2.urlopen(test_url) is not None
+    except Exception:
+      raise cherrypy.HTTPError(404, 'File not found on local server.')
+
+  def _ServeFile(self, f):
+    """Serves the file as an http response."""
+    if self._options.local_server_port:
+      redirect_url = self._GetServerURL(f, self._options.local_server_port)
+      cherrypy.log('Redirect to %s' % redirect_url)
+      cherrypy.lib.cptools.redirect(redirect_url, internal=False)
+    else:
+      sanitized_path = os.path.abspath(os.path.join(self._options.www_root, f))
       return cherrypy.lib.static.serve_file(sanitized_path)
 
+  def _GetServerURL(self, f, port, **kwargs):
+    """Returns a URL for local server to serve the file on given port.
+
+    Args:
+      f: file name to serve on local server. Relative to www_root.
+      port: Local server port (it can be a configured constrained port).
+      kwargs: extra parameteres passed in the URL.
+    """
+    base_url = '%s?f=%s' % (cherrypy.url(), f)
+    if self._options.local_server_port:
+      base_url = '%s/%s' % (cherrypy.url().replace('ServeConstrained',

[DaleCurtis, 2012/08/09 17:28:02]

Cleaner as:

base_url = '%s/%s' % (
    cherrypy.url().replace('ServeConstrained', self._options.www_root), f)

Or:
url = cherrypy.url()
if self._options.local_server_port:
  url = url.replace('ServeConstrained', self._options.www_root)

base_url = '%s?f=%s' % (url, f)

I'm confused though, shouldn't www_root always be blank here? Why not just use
url.replace('ServeConstrained', '') ? Then you can remove the default='' below?

[shadi, 2012/08/09 18:26:49]

The www_root used with local-server-port is relative to redirecting url. Example
if www_root = media, then the redirect URL is :
http://localhost:80/media/filename. 

[DaleCurtis, 2012/08/09 18:46:31]

You need to update the --options documentation for that.

+                                                   self._options.www_root),
+                            f)
+    base_url = base_url.replace(':%d' % self._options.port, ':%d' % port)
+    extra_args = urllib.urlencode(kwargs)
+    if extra_args:
+      delimeter = '&'

[DaleCurtis, 2012/08/09 17:28:02]

You can probably just roll the &, ? into the if above, I don't think it should
hurt to have these empty.

[shadi, 2012/08/09 18:26:49]

Done.

+      if self._options.local_server_port:
+        delimeter = '?'
+      base_url += '%s%s' % (delimeter, extra_args)
+    return base_url
+
+  def _GetConstrainedPort(self, f=None, bandwidth=None, latency=None, loss=None,
+                          new_port=False, **kwargs):
+    """Creates or gets a port with specified network constraints.
+
+    See ServeConstrained() for more details.
+    """
     # Validate inputs. isdigit() guarantees a natural number.
     bandwidth = self._ParseIntParameter(
         bandwidth, 'Invalid bandwidth constraint.', lambda x: x > 0)
     latency = self._ParseIntParameter(
         latency, 'Invalid latency constraint.', lambda x: x >= 0)
     loss = self._ParseIntParameter(
         loss, 'Invalid loss constraint.', lambda x: x <= 100 and x >= 0)
 
+    redirect_port = self._options.port
+    if self._options.local_server_port:
+      redirect_port = self._options.local_server_port
+
+    start_time = time.time()
     # Allocate a port using the given constraints. If a port with the requested
-    # key is already allocated, it will be reused.
-    #
-    # TODO(dalecurtis): The key cherrypy.request.remote.ip might not be unique
-    # if build slaves are sharing the same VM.
-    start_time = time.time()
+    # key and kwargs already exist then reuse that port.
     constrained_port = self._port_allocator.Get(
-        cherrypy.request.remote.ip, server_port=self._options.port,
+        cherrypy.request.remote.ip, server_port=redirect_port,
         interface=self._options.interface, bandwidth=bandwidth, latency=latency,
         loss=loss, new_port=new_port, file=f, **kwargs)
-    end_time = time.time()
+
+    cherrypy.log('Time to set up port %d = %.3fsec.' %
+                 (constrained_port, time.time() - start_time))
 
     if not constrained_port:
       raise cherrypy.HTTPError(503, 'Service unavailable. Out of ports.')
-
-    cherrypy.log('Time to set up port %d = %ssec.' %
-                 (constrained_port, end_time - start_time))
-
-    # Build constrained URL using the constrained port and original URL
-    # parameters except the network constraints (bandwidth, latency, and loss).
-    constrained_url = '%s?f=%s&no_cache=%s&%s' % (
-        cherrypy.url().replace(
-            ':%d' % self._options.port, ':%d' % constrained_port),
-        f,
-        no_cache,
-        '&'.join(['%s=%s' % (key, kwargs[key]) for key in kwargs]))
-
-    # Redirect request to the constrained port.
-    cherrypy.lib.cptools.redirect(constrained_url, internal=False)
+    return constrained_port
 
   def _ParseIntParameter(self, param, msg, check):
     """Returns integer value of param and verifies it satisfies the check.
 
     Args:
       param: Parameter name to check.
       msg: Message in error if raised.
       check: Check to verify the parameter value.
 
     Returns:
@@ skipping 30 matching lines @@
                     help=('Interface to setup constraints on. Use lo for a '
                           'local client. Default: %default'))
   parser.add_option('--socket-timeout', type='int',
                     default=cherrypy.server.socket_timeout,
                     help=('Number of seconds before a socket connection times '
                           'out. Default: %default'))
   parser.add_option('--threads', type='int',
                     default=cherrypy._cpserver.Server.thread_pool,
                     help=('Number of threads in the thread pool. Default: '
                           '%default'))
-  parser.add_option('--www-root', default=os.getcwd(),
+  parser.add_option('--www-root', default='',
                     help=('Directory root to serve files from. Defaults to the '
-                          'current directory: %default'))
+                          'current directory (if --local-server-port is not '
+                          'used): %s' % os.getcwd()))
+  parser.add_option('--local-server-port', type='int',
+                    help=('Optional local server port to host files.'))
   parser.add_option('-v', '--verbose', action='store_true', default=False,
                     help='Turn on verbose output.')
 
   options = parser.parse_args()[0]
 
   # Convert port range into the desired tuple format.
   try:
     if isinstance(options.port_range, str):
       options.port_range = [int(port) for port in options.port_range.split(',')]
   except ValueError:
     parser.error('Invalid port range specified.')
 
   if options.expiry_time < 0:
     parser.error('Invalid expiry time specified.')
 
   # Convert the path to an absolute to remove any . or ..
-  options.www_root = os.path.abspath(options.www_root)
+  if not options.local_server_port:
+    if not options.www_root:
+      options.www_root = os.getcwd()
+    options.www_root = os.path.abspath(options.www_root)
 
   _SetLogger(options.verbose)
 
   return options
 
 
 def _SetLogger(verbose):
   file_handler = handlers.RotatingFileHandler('cns.log', 'a', 10000000, 10)
   file_handler.setFormatter(logging.Formatter('[%(threadName)s] %(message)s'))
 
@@ skipping 31 matching lines @@
   try:
     cherrypy.quickstart(ConstrainedNetworkServer(options, pa))
   finally:
     # Disable Ctrl-C handler to prevent interruption of cleanup.
     signal.signal(signal.SIGINT, lambda signal, frame: None)
     pa.Cleanup(all_ports=True)
 
 
 if __name__ == '__main__':
   Main()