Store each file as "hash/<sha1>", list of hashes as "meta/@<rev>" and serve them at "serve_file/@<r…

gae/main.py

 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import re
 # F0401:  7,0: Unable to import 'webapp2'
 # pylint: disable=F0401
 import webapp2
 
 import cache
 import config
 import content_processor
 import content_type
 import pre_cacher
 import file_reader
 import zip_proxy
 
 current_config = config.DEFAULT
 
 class MainPage(webapp2.RequestHandler):
   def get(self):
     self.response.headers['Content-Type'] = 'text/plain'
     self.response.write(str(current_config))
 
 class Server(webapp2.RequestHandler):
-  def get(self, tag_type, tag, path):
-    cache_key_name = '/res/%s/%s/%s' % (tag_type, tag, path)
-    content = cache.get_content(cache_key_name)
-    helper = config.ConfigHelper(current_config)
+  def get_hash_cache_key(self, rev, path):
+    return '/hashes/%s/%s' % (rev, path)
 
-    if not content:
-      if tag_type == 'rev':
-        meta_file_name = helper.get_revision_path(tag)
-      elif tag_type =='ver':
-        meta_file_name = helper.get_version_path(tag)
-      else:
+  def get_file_hash(self, helper, rev, path):
+    file_hash = cache.get_content(self.get_hash_cache_key(rev, path))
+
+    if not file_hash:
+      meta_file_name = helper.get_meta_path(rev)
+      meta_content = file_reader.read(helper, meta_file_name)
+      if not meta_content:
         self.abort(404)

[mnaganov (inactive), 2015/02/25 12:56:35]

Let's just return null here to make it clear that this function is not a
top-level request handler.

[dgozman, 2015/02/25 14:22:08]

Nice catch! Done.

 
-      meta_content = file_reader.read(helper, meta_file_name)
-      if meta_content:
-        zip_file_name = meta_content.strip(' \t\n')
-      else:
+      for line in meta_content.split('\n'):

[mnaganov (inactive), 2015/02/25 12:56:35]

In case when the path isn't listed in the meta file, we'll be reparsing it again
and again. I guess, we can put a cache entry indicating that the particular meta
file has already been parsed.

[dgozman, 2015/02/25 14:22:08]

Done.

+        index = line.find(':')

[mnaganov (inactive), 2015/02/25 12:56:35]

You just do another split here, so you can just write:

path_and_hash = line.split(':')
if len(path_and_hash) == 2:
  ... use path_and_hash[0] and path_and_hash[1]

[dgozman, 2015/02/25 14:22:08]

Done.

+        if index != -1:
+          line_path = line[(index + 1):]
+          line_hash = line[:index]
+          cache.set_content(self.get_hash_cache_key(rev, line_path), line_hash)
+          if line_path == path:
+            file_hash = line_hash
+
+    return file_hash
+
+  def get(self, tag_type, tag, path):
+    helper = config.ConfigHelper(current_config)
+    content = None
+
+    if tag_type == 'file':
+      file_hash = self.get_file_hash(helper, tag, path)
+      if not file_hash:
         self.abort(404)
-
-      content = zip_proxy.read(helper, zip_file_name, path)
+      content = file_reader.read(helper, helper.get_hash_path(file_hash))
       if not content:
         self.abort(404)
       content = content_processor.process(path, content)
-      cache.set_content(cache_key_name, content)
+
+    else:
+      cache_key_name = '/res/%s/%s/%s' % (tag_type, tag, path)
+      content = cache.get_content(cache_key_name)
+      if not content:
+        if tag_type == 'rev':
+          meta_file_name = helper.get_revision_path(tag)
+        elif tag_type =='ver':
+          meta_file_name = helper.get_version_path(tag)
+        else:
+          self.abort(404)
+
+        meta_content = file_reader.read(helper, meta_file_name)
+        if meta_content:
+          zip_file_name = meta_content.strip(' \t\n')
+        else:
+          self.abort(404)
+
+        content = zip_proxy.read(helper, zip_file_name, path)
+        if not content:
+          self.abort(404)
+        content = content_processor.process(path, content)
+        cache.set_content(cache_key_name, content)
 
     self.response.headers['Content-Type'] = content_type.from_path(path)
     self.response.headers['Access-Control-Allow-Origin'] = '*'
     cache_control = 'public, max-age=%d' % helper.get_max_age()
     self.response.headers['Cache-Control'] = cache_control
     self.response.body = content
 
 class LegacyStatic(Server):
   def get(self, version, path):
     # Map x.x.x.x -> x.x.x.0 to deal with patched releases
     match = re.search('(\d+\.\d+\.\d+\.)\d+', version)
     if match:
       return super(LegacyStatic, self).get('ver', match.group(1) + '0', path)
     else:
       self.abort(404)
 
 app = webapp2.WSGIApplication(
   [('/', MainPage),
    ('/precache_url', pre_cacher.PreCacherFromUrl),
    webapp2.Route('/serve_<tag_type>/<tag>/<path:.*>', Server),
    webapp2.Route('/static/<version>/<path:.*>', LegacyStatic)],
   debug=config.IS_DEV_APP_SERVER)