Store each file as "hash/<sha1>", list of hashes as "meta/@<rev>" and serve them at "serve_file/@<r…

gae/main.py

Index: gae/main.py
diff --git a/gae/main.py b/gae/main.py
index e9b427ccf4b140f629772d83337307455d4dc3c1..53263dabe9d094ef2ccc0919b0d63148526c246c 100644
--- a/gae/main.py
+++ b/gae/main.py
@@ -23,30 +23,68 @@ class MainPage(webapp2.RequestHandler):
     self.response.write(str(current_config))
 
 class Server(webapp2.RequestHandler):
-  def get(self, tag_type, tag, path):
-    cache_key_name = '/res/%s/%s/%s' % (tag_type, tag, path)
-    content = cache.get_content(cache_key_name)
-    helper = config.ConfigHelper(current_config)
+  def get_hash_cache_key(self, rev, path):
+    return '/hashes/%s/%s' % (rev, path)
 
-    if not content:
-      if tag_type == 'rev':
-        meta_file_name = helper.get_revision_path(tag)
-      elif tag_type =='ver':
-        meta_file_name = helper.get_version_path(tag)
-      else:
-        self.abort(404)
+  def get_file_hash(self, helper, rev, path):
+    file_hash = cache.get_content(self.get_hash_cache_key(rev, path))
+
+    if not file_hash:
+      meta_cache_key = '/meta-parsed/%s' % rev
+      if cache.get_content(meta_cache_key):
+        return None
 
+      meta_file_name = helper.get_meta_path(rev)
       meta_content = file_reader.read(helper, meta_file_name)
-      if meta_content:
-        zip_file_name = meta_content.strip(' \t\n')
-      else:
-        self.abort(404)
+      if not meta_content:
+        return None
+
+      for line in meta_content.split('\n'):
+        hash_and_path = line.split(':')
+        if len(hash_and_path) == 2:
+          (line_hash, line_path) = hash_and_path
+          cache.set_content(self.get_hash_cache_key(rev, line_path), line_hash)
+          if line_path == path:
+            file_hash = line_hash
+      cache.set_content(meta_cache_key, 'parsed')
+
+    return file_hash
+
+  def get(self, tag_type, tag, path):
+    helper = config.ConfigHelper(current_config)
+    content = None
 
-      content = zip_proxy.read(helper, zip_file_name, path)
+    if tag_type == 'file':
+      file_hash = self.get_file_hash(helper, tag, path)
+      if not file_hash:
+        self.abort(404)
+      content = file_reader.read(helper, helper.get_hash_path(file_hash))
       if not content:
         self.abort(404)
       content = content_processor.process(path, content)
-      cache.set_content(cache_key_name, content)
+
+    else:
+      cache_key_name = '/res/%s/%s/%s' % (tag_type, tag, path)
+      content = cache.get_content(cache_key_name)
+      if not content:
+        if tag_type == 'rev':
+          meta_file_name = helper.get_revision_path(tag)
+        elif tag_type =='ver':
+          meta_file_name = helper.get_version_path(tag)
+        else:
+          self.abort(404)
+
+        meta_content = file_reader.read(helper, meta_file_name)
+        if meta_content:
+          zip_file_name = meta_content.strip(' \t\n')
+        else:
+          self.abort(404)
+
+        content = zip_proxy.read(helper, zip_file_name, path)
+        if not content:
+          self.abort(404)
+        content = content_processor.process(path, content)
+        cache.set_content(cache_key_name, content)
 
     self.response.headers['Content-Type'] = content_type.from_path(path)
     self.response.headers['Access-Control-Allow-Origin'] = '*'