Store each file as "hash/<sha1>", list of hashes as "meta/@<rev>" and serve them at "serve_file/@<r…

gae/main.py

Index: gae/main.py
diff --git a/gae/main.py b/gae/main.py
index e9b427ccf4b140f629772d83337307455d4dc3c1..fc02149ae79b639a9dc0eac1f6dce8be4465b696 100644
--- a/gae/main.py
+++ b/gae/main.py
@@ -23,30 +23,64 @@ class MainPage(webapp2.RequestHandler):
     self.response.write(str(current_config))
 
 class Server(webapp2.RequestHandler):
-  def get(self, tag_type, tag, path):
-    cache_key_name = '/res/%s/%s/%s' % (tag_type, tag, path)
-    content = cache.get_content(cache_key_name)
-    helper = config.ConfigHelper(current_config)
+  def get_hash_cache_key(self, rev, path):
+    return '/hashes/%s/%s' % (rev, path)
 
-    if not content:
-      if tag_type == 'rev':
-        meta_file_name = helper.get_revision_path(tag)
-      elif tag_type =='ver':
-        meta_file_name = helper.get_version_path(tag)
-      else:
-        self.abort(404)
+  def get_file_hash(self, helper, rev, path):
+    file_hash = cache.get_content(self.get_hash_cache_key(rev, path))
 
+    if not file_hash:
+      meta_file_name = helper.get_meta_path(rev)
       meta_content = file_reader.read(helper, meta_file_name)
-      if meta_content:
-        zip_file_name = meta_content.strip(' \t\n')
-      else:
+      if not meta_content:
         self.abort(404)

[mnaganov (inactive), 2015/02/25 12:56:35]

Let's just return null here to make it clear that this function is not a
top-level request handler.

[dgozman, 2015/02/25 14:22:08]

Nice catch! Done.

 
-      content = zip_proxy.read(helper, zip_file_name, path)
+      for line in meta_content.split('\n'):

[mnaganov (inactive), 2015/02/25 12:56:35]

In case when the path isn't listed in the meta file, we'll be reparsing it again
and again. I guess, we can put a cache entry indicating that the particular meta
file has already been parsed.

[dgozman, 2015/02/25 14:22:08]

Done.

+        index = line.find(':')

[mnaganov (inactive), 2015/02/25 12:56:35]

You just do another split here, so you can just write:

path_and_hash = line.split(':')
if len(path_and_hash) == 2:
  ... use path_and_hash[0] and path_and_hash[1]

[dgozman, 2015/02/25 14:22:08]

Done.

+        if index != -1:
+          line_path = line[(index + 1):]
+          line_hash = line[:index]
+          cache.set_content(self.get_hash_cache_key(rev, line_path), line_hash)
+          if line_path == path:
+            file_hash = line_hash
+
+    return file_hash
+
+  def get(self, tag_type, tag, path):
+    helper = config.ConfigHelper(current_config)
+    content = None
+
+    if tag_type == 'file':
+      file_hash = self.get_file_hash(helper, tag, path)
+      if not file_hash:
+        self.abort(404)
+      content = file_reader.read(helper, helper.get_hash_path(file_hash))
       if not content:
         self.abort(404)
       content = content_processor.process(path, content)
-      cache.set_content(cache_key_name, content)
+
+    else:
+      cache_key_name = '/res/%s/%s/%s' % (tag_type, tag, path)
+      content = cache.get_content(cache_key_name)
+      if not content:
+        if tag_type == 'rev':
+          meta_file_name = helper.get_revision_path(tag)
+        elif tag_type =='ver':
+          meta_file_name = helper.get_version_path(tag)
+        else:
+          self.abort(404)
+
+        meta_content = file_reader.read(helper, meta_file_name)
+        if meta_content:
+          zip_file_name = meta_content.strip(' \t\n')
+        else:
+          self.abort(404)
+
+        content = zip_proxy.read(helper, zip_file_name, path)
+        if not content:
+          self.abort(404)
+        content = content_processor.process(path, content)
+        cache.set_content(cache_key_name, content)
 
     self.response.headers['Content-Type'] = content_type.from_path(path)
     self.response.headers['Access-Control-Allow-Origin'] = '*'