Enable CNS to serve files from different port.

media/tools/constrained_network_server/cns.py

Index: media/tools/constrained_network_server/cns.py
diff --git a/media/tools/constrained_network_server/cns.py b/media/tools/constrained_network_server/cns.py
index cf2257f60b5a824b267eefdfb7ec00d4bf67b0a1..46e78bb520be1c0e175fd3449bfe12ff1fbb34b0 100755
--- a/media/tools/constrained_network_server/cns.py
+++ b/media/tools/constrained_network_server/cns.py
@@ -20,6 +20,8 @@ import signal
 import sys
 import threading
 import time
+import urllib2
+
 import traffic_control
 
 try:
@@ -200,19 +202,91 @@ class ConstrainedNetworkServer(object):
     if not f:
       raise cherrypy.HTTPError(400, 'Invalid request. File must be specified.')
 
+    # Check existence early to prevent wasted constraint setup.
+    self._CheckRequestedFileExist(f)
+
+    # If there are no constraints, just serve the file.
+    if bandwidth is None and latency is None and loss is None:
+      return self._ServeFile(f)
+
+    constrained_port = self._GetConstrainedPort(
+        f, bandwidth=bandwidth, latency=latency, loss=loss, new_port=new_port,
+        **kwargs)
+
+    # Build constrained URL using the constrained port and original URL
+    # parameters except the network constraints (bandwidth, latency, and loss).
+    if self._options.local_server_port:
+      constrained_url = self._GetLocalServerURL(f, constrained_port, **kwargs)
+    else:
+      constrained_url = '%s?f=%s&no_cache=%s&%s' % (

[DaleCurtis, 2012/08/07 23:53:57]

This looks similar to the self._GetLocalServerURL() can we collapse these two
methods into one? This code is already kind of ugly, no need to have it twice :)

[shadi, 2012/08/08 22:55:49]

They are not actually that similar, but Done. :-)

+          cherrypy.url().replace(
+              ':%d' % self._options.port, ':%d' % constrained_port),
+          '%s' % f,
+          no_cache,
+          '&'.join(['%s=%s' % (key, kwargs[key]) for key in kwargs]))
+
+    # Redirect request to the constrained port.
+    cherrypy.log('Redirect to %s' % constrained_url)
+    cherrypy.lib.cptools.redirect(constrained_url, internal=False)
+
+  def _CheckRequestedFileExist(self, f):
+    """Checks if the requested file exists, raises HTTPError otherwise."""
+    if self._options.local_server_port:
+      self._CheckFileExistOnLocalServer(f)
+    else:
+      self._CheckFileExistOnServer(f)
+
+  def _CheckFileExistOnServer(self, f):
+    """Checks if requested f exists to be served by this server."""

[DaleCurtis, 2012/08/07 23:53:57]

s/f/file f/

[shadi, 2012/08/08 22:55:49]

Done.

     # Sanitize and check the path to prevent www-root escapes.
     sanitized_path = os.path.abspath(os.path.join(self._options.www_root, f))
     if not sanitized_path.startswith(self._options.www_root):
       raise cherrypy.HTTPError(403, 'Invalid file requested.')
-
-    # Check existence early to prevent wasted constraint setup.
     if not os.path.exists(sanitized_path):
       raise cherrypy.HTTPError(404, 'File not found.')
 
-    # If there are no constraints, just serve the file.
-    if bandwidth is None and latency is None and loss is None:
+  def _CheckFileExistOnLocalServer(self, f):

[DaleCurtis, 2012/08/07 23:53:57]

Hmmm, what is urlopen doing behind the scenes here? This is more expensive than
it's worth if it's actually pulling down the remote file.

If we want to keep this we could require --www-data-dir and --local-server-port
to be specified and just use the original os.path.isfile() check.

[shadi, 2012/08/08 22:55:49]

No it is not pulling down the remote file. It returns a socket file object which
pulls down all the file when a read() is done.
It pulls down the headers and little data. Using urllib allows checking the URL
itself would work and not just that the file exist.

+    """Checks if requested file exists on local server hosting files."""
+    test_url = self._GetLocalServerURL(f, self._options.local_server_port)
+    try:
+      cherrypy.log('Check file exist using URL: %s' % test_url)
+      return urllib2.urlopen(test_url) is not None
+    except Exception:
+      raise cherrypy.HTTPError(404, 'File not found on local server.')
+
+  def _ServeFile(self, f):
+    """Serves the file as an http response."""
+    if self._options.local_server_port:
+      redirect_url = self._GetLocalServerURL(f, self._options.local_server_port)
+      cherrypy.log('Redirect to %s' % redirect_url)
+      cherrypy.lib.cptools.redirect(redirect_url, internal=False)

[DaleCurtis, 2012/08/07 23:53:57]

Do you need to return anything here to keep CherryPy happy with
ServeConstrained?

[shadi, 2012/08/08 22:55:49]

CherryPy has not been complaining :-) 

+    else:
+      sanitized_path = os.path.abspath(os.path.join(self._options.www_root, f))
       return cherrypy.lib.static.serve_file(sanitized_path)
 
+  def _GetLocalServerURL(self, f, port, **kwargs):
+    """Returns a URL for local server to serve the file on given port.
+
+    Args:
+      f: file name to serve on local server. Relative to www_root.
+      port: Local server port (it can be a configured constrained port).
+      kwargs: extra parameteres passed in the URL.
+    """
+    base_url = cherrypy.url().replace('ServeConstrained',
+                                      self._options.www_root)
+    base_url = base_url.replace(':%d' % self._options.port, ':%d' % port)
+    url = '%s/%s' % (base_url, f)
+    extra_args = '&'.join(['%s=%s' % (key, kwargs[key]) for key in kwargs])
+    if extra_args:
+      url += '?%s' % extra_args
+    return url
+
+  def _GetConstrainedPort(self, f=None, bandwidth=None, latency=None, loss=None,
+                          new_port=False, **kwargs):
+    """Creates or gets a port with specified network constraints.
+
+    See ServeConstrained() for more details.
+    """
     # Validate inputs. isdigit() guarantees a natural number.
     bandwidth = self._ParseIntParameter(
         bandwidth, 'Invalid bandwidth constraint.', lambda x: x > 0)
@@ -221,35 +295,24 @@ class ConstrainedNetworkServer(object):
     loss = self._ParseIntParameter(
         loss, 'Invalid loss constraint.', lambda x: x <= 100 and x >= 0)
 
-    # Allocate a port using the given constraints. If a port with the requested
-    # key is already allocated, it will be reused.
-    #
-    # TODO(dalecurtis): The key cherrypy.request.remote.ip might not be unique
-    # if build slaves are sharing the same VM.
+    redirect_port = self._options.port
+    if self._options.local_server_port:
+      redirect_port = self._options.local_server_port
+
     start_time = time.time()
+    # Allocate a port using the given constraints. If a port with the requested
+    # key and kwargs already exist then reuse that port.
     constrained_port = self._port_allocator.Get(
-        cherrypy.request.remote.ip, server_port=self._options.port,
+        cherrypy.request.remote.ip, server_port=redirect_port,
         interface=self._options.interface, bandwidth=bandwidth, latency=latency,
         loss=loss, new_port=new_port, file=f, **kwargs)
-    end_time = time.time()
-
-    if not constrained_port:
-      raise cherrypy.HTTPError(503, 'Service unavailable. Out of ports.')
 
     cherrypy.log('Time to set up port %d = %ssec.' %
-                 (constrained_port, end_time - start_time))
-
-    # Build constrained URL using the constrained port and original URL
-    # parameters except the network constraints (bandwidth, latency, and loss).
-    constrained_url = '%s?f=%s&no_cache=%s&%s' % (
-        cherrypy.url().replace(
-            ':%d' % self._options.port, ':%d' % constrained_port),
-        f,
-        no_cache,
-        '&'.join(['%s=%s' % (key, kwargs[key]) for key in kwargs]))
+                 (constrained_port, time.time() - start_time))
 
-    # Redirect request to the constrained port.
-    cherrypy.lib.cptools.redirect(constrained_url, internal=False)
+    if not constrained_port:
+      raise cherrypy.HTTPError(503, 'Service unavailable. Out of ports.')
+    return constrained_port
 
   def _ParseIntParameter(self, param, msg, check):
     """Returns integer value of param and verifies it satisfies the check.
@@ -300,9 +363,12 @@ def ParseArgs():
                     default=cherrypy._cpserver.Server.thread_pool,
                     help=('Number of threads in the thread pool. Default: '
                           '%default'))
-  parser.add_option('--www-root', default=os.getcwd(),
+  parser.add_option('--www-root', default='',

[DaleCurtis, 2012/08/07 23:53:57]

just remove the default='' ?

[shadi, 2012/08/08 22:55:49]

I actually put this so that www_root == '' when --local-server-port is used.

                     help=('Directory root to serve files from. Defaults to the '
-                          'current directory: %default'))
+                          'current directory (if --local-server-port is not '
+                          'used): %s' % os.getcwd()))
+  parser.add_option('--local-server-port', type='int',
+                    help=('Optional local server port to host files.'))
   parser.add_option('-v', '--verbose', action='store_true', default=False,
                     help='Turn on verbose output.')
 
@@ -319,7 +385,10 @@ def ParseArgs():
     parser.error('Invalid expiry time specified.')
 
   # Convert the path to an absolute to remove any . or ..
-  options.www_root = os.path.abspath(options.www_root)
+  if not options.local_server_port:
+    if not options.www_root:
+      options.www_root = os.getcwd()
+    options.www_root = os.path.abspath(options.www_root)
 
   # Required so that cherrypy logs do not get propagated to root logger causing
   # the logs to be printed twice.