Store each file as "hash/<sha1>", list of hashes as "meta/@<rev>" and serve them at "serve_file/@<r…

gae/main.py

 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import re
 # F0401:  7,0: Unable to import 'webapp2'
 # pylint: disable=F0401
 import webapp2
 
 import cache
 import config
 import content_processor
 import content_type
 import pre_cacher
 import file_reader
 import zip_proxy
 
 current_config = config.DEFAULT
 
 class MainPage(webapp2.RequestHandler):
   def get(self):
     self.response.headers['Content-Type'] = 'text/plain'
     self.response.write(str(current_config))
 
 class Server(webapp2.RequestHandler):
+  def get_file_hash(self, meta_content, path):

[mnaganov (inactive), 2015/02/24 14:40:32]

We should probably cache '(tag, path) -> hash' somewhere in memory to avoid
parsing the meta file each time as well as looking up file hashes in linear
time.

[dgozman, 2015/02/25 11:37:13]

Done.

+    for line in meta_content.split('\n'):
+      index = line.find(':')
+      if index != -1 and line[(index + 1):] == path:
+        return line[:index]
+    return None
+
   def get(self, tag_type, tag, path):
     cache_key_name = '/res/%s/%s/%s' % (tag_type, tag, path)
     content = cache.get_content(cache_key_name)
     helper = config.ConfigHelper(current_config)
 
+    if not content and tag_type == 'file':

[mnaganov (inactive), 2015/02/24 14:40:32]

No, we shouldn't cache these resources under 'cache_key_name' at all. The file
reader already caches them using GCE file paths. The reason why we use
'cache_key_name' is because the contents are extracted on the fly from zip
archives. But for resources that we just read directly from GCE this will end up
creating two copies of each resource in the cache.

You should branch on 'tag_type == file' earlier, and just not access cache in
this branch -- file_reader will do caching for you.

[dgozman, 2015/02/25 11:37:13]

Done.

+      meta_file_name = helper.get_meta_path(tag)
+      meta_content = file_reader.read(helper, meta_file_name)
+      if not meta_content:
+        self.abort(404)
+      file_hash = self.get_file_hash(meta_content, path)
+      if not file_hash:
+        self.abort(404)
+      content = file_reader.read(helper, helper.get_hash_path(file_hash))
+      if not content:
+        self.abort(404)
+      content = content_processor.process(path, content)
+      cache.set_content(cache_key_name, content)
+
     if not content:
       if tag_type == 'rev':
         meta_file_name = helper.get_revision_path(tag)
       elif tag_type =='ver':
         meta_file_name = helper.get_version_path(tag)
       else:
         self.abort(404)
 
       meta_content = file_reader.read(helper, meta_file_name)
       if meta_content:
@@ skipping 21 matching lines @@
       return super(LegacyStatic, self).get('ver', match.group(1) + '0', path)
     else:
       self.abort(404)
 
 app = webapp2.WSGIApplication(
   [('/', MainPage),
    ('/precache_url', pre_cacher.PreCacherFromUrl),
    webapp2.Route('/serve_<tag_type>/<tag>/<path:.*>', Server),
    webapp2.Route('/static/<version>/<path:.*>', LegacyStatic)],
   debug=config.IS_DEV_APP_SERVER)