Store each file as "hash/<sha1>", list of hashes as "meta/@<rev>" and serve them at "serve_file/@<r…

gae/main.py

 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import re
 # F0401:  7,0: Unable to import 'webapp2'
 # pylint: disable=F0401
 import webapp2
 
 import cache
 import config
 import content_processor
 import content_type
 import pre_cacher
 import file_reader
 import zip_proxy
 
 current_config = config.DEFAULT
 
 class MainPage(webapp2.RequestHandler):
   def get(self):
     self.response.headers['Content-Type'] = 'text/plain'
     self.response.write(str(current_config))
 
 class Server(webapp2.RequestHandler):
+  def get_hash_cache_key(self, rev, path):
+    return '/hashes/%s/%s' % (rev, path)
+
+  def get_file_hash(self, helper, rev, path):
+    file_hash = cache.get_content(self.get_hash_cache_key(rev, path))
+
+    if not file_hash:
+      meta_cache_key = '/meta-parsed/%s' % rev
+      if cache.get_content(meta_cache_key):
+        return None
+
+      meta_file_name = helper.get_meta_path(rev)
+      meta_content = file_reader.read(helper, meta_file_name)
+      if not meta_content:
+        return None
+
+      for line in meta_content.split('\n'):
+        hash_and_path = line.split(':')
+        if len(hash_and_path) == 2:
+          (line_hash, line_path) = hash_and_path
+          cache.set_content(self.get_hash_cache_key(rev, line_path), line_hash)
+          if line_path == path:
+            file_hash = line_hash
+      cache.set_content(meta_cache_key, 'parsed')
+
+    return file_hash
+
   def get(self, tag_type, tag, path):
-    cache_key_name = '/res/%s/%s/%s' % (tag_type, tag, path)
-    content = cache.get_content(cache_key_name)
     helper = config.ConfigHelper(current_config)
+    content = None
 
-    if not content:
-      if tag_type == 'rev':
-        meta_file_name = helper.get_revision_path(tag)
-      elif tag_type =='ver':
-        meta_file_name = helper.get_version_path(tag)
-      else:
+    if tag_type == 'file':
+      file_hash = self.get_file_hash(helper, tag, path)
+      if not file_hash:
         self.abort(404)
-
-      meta_content = file_reader.read(helper, meta_file_name)
-      if meta_content:
-        zip_file_name = meta_content.strip(' \t\n')
-      else:
-        self.abort(404)
-
-      content = zip_proxy.read(helper, zip_file_name, path)
+      content = file_reader.read(helper, helper.get_hash_path(file_hash))
       if not content:
         self.abort(404)
       content = content_processor.process(path, content)
-      cache.set_content(cache_key_name, content)
+
+    else:
+      cache_key_name = '/res/%s/%s/%s' % (tag_type, tag, path)
+      content = cache.get_content(cache_key_name)
+      if not content:
+        if tag_type == 'rev':
+          meta_file_name = helper.get_revision_path(tag)
+        elif tag_type =='ver':
+          meta_file_name = helper.get_version_path(tag)
+        else:
+          self.abort(404)
+
+        meta_content = file_reader.read(helper, meta_file_name)
+        if meta_content:
+          zip_file_name = meta_content.strip(' \t\n')
+        else:
+          self.abort(404)
+
+        content = zip_proxy.read(helper, zip_file_name, path)
+        if not content:
+          self.abort(404)
+        content = content_processor.process(path, content)
+        cache.set_content(cache_key_name, content)
 
     self.response.headers['Content-Type'] = content_type.from_path(path)
     self.response.headers['Access-Control-Allow-Origin'] = '*'
     cache_control = 'public, max-age=%d' % helper.get_max_age()
     self.response.headers['Cache-Control'] = cache_control
     self.response.body = content
 
 class LegacyStatic(Server):
   def get(self, version, path):
     # Map x.x.x.x -> x.x.x.0 to deal with patched releases
     match = re.search('(\d+\.\d+\.\d+\.)\d+', version)
     if match:
       return super(LegacyStatic, self).get('ver', match.group(1) + '0', path)
     else:
       self.abort(404)
 
 app = webapp2.WSGIApplication(
   [('/', MainPage),
    ('/precache_url', pre_cacher.PreCacherFromUrl),
    webapp2.Route('/serve_<tag_type>/<tag>/<path:.*>', Server),
    webapp2.Route('/static/<version>/<path:.*>', LegacyStatic)],
   debug=config.IS_DEV_APP_SERVER)