Docserver: Generate a table of extension/app API owners

chrome/common/extensions/docs/server2/owners_data_source.py

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from operator import itemgetter
+import posixpath
+
+from data_source import DataSource
+from docs_server_utils import MarkLast
+from extensions_paths import BROWSER_API_PATHS, BROWSER_CHROME_EXTENSIONS
+from file_system import FileNotFoundError
+from future import All, Future
+
+
+_COMMENT_START_MARKER = '#'
+_OWNERS = 'OWNERS'
+
+
+def _ParseOwnersFile(content):
+  '''Returns a tuple (owners, notes), where
+  |owners| is a list of dicts formed from the owners in |content|,
+  |notes| is a string formed from the comments in |content|.
+  '''
+  if content is None:
+    return [], 'See core extensions owners.'
+  owners = []
+  notes = []
+  for line in content.splitlines():
+    if line == '':
+      continue
+    if line.startswith(_COMMENT_START_MARKER):
+      notes.append(line[len(_COMMENT_START_MARKER):].lstrip())
+    else:
+      # TODO(ahernandez): Mark owners no longer on the project.
+      owners.append({'email': line, 'username': line[:line.find('@')]})
+  MarkLast(owners)
+  return owners, ' '.join(notes)
+
+
+class OwnersDataSource(DataSource):
+  def __init__(self, server_instance, _):
+    self._host_fs = server_instance.host_file_system_provider.GetTrunk()
+    self._cache = server_instance.object_store_creator.Create(OwnersDataSource)
+
+  def _CreateAPIEntry(self, api_name, owners_file):
+    '''Creates a dict with owners information for an API, specified
+    by |owners_file|.
+    '''
+    def create_entry(content):
+      owners, notes = _ParseOwnersFile(content.get(owners_file, None))
+      return {
+        'apiName': api_name,
+        'owners': owners,
+        'notes': notes
+      }
+    return self._host_fs.Read((owners_file,), skip_not_found=True).Then(

[ahernandez, 2014/08/14 22:17:18]

This call is what gets run during the second cron run because of the cache miss
and increments the file system read count.

[not at google - send to devlin, 2014/08/14 22:32:12]

I wonder if it has something to do with the skip_not_found? because the caching
of the file system is supposed to be handling this. If you Read() a file in the
Cron - that's all - then it should be cached for the next time around. I had
thought. This can hardly be the first instance of calling a raw Read from a
cron.

Could you use a compiled fs here now anyway, since you're effectively compiling
OWNERS files? Avoid the problem...

+        create_entry)
+
+  def _CollectOwnersData(self):
+    '''Walks through the file system, collecting owners data from
+    API directories.
+    '''
+    def collect(api_owners):
+      if api_owners is not None:
+        return api_owners
+      api_owners = []
+      for root in BROWSER_API_PATHS:
+        for base, dirs, _ in self._host_fs.Walk(root, depth=1):
+          for dir_ in dirs:
+            owners_file = posixpath.join(root, base, dir_, _OWNERS)
+            api_owners.append(self._CreateAPIEntry(dir_[:-1], owners_file))
+      # Add an entry for the core extensions/apps owners.
+      owners_file = posixpath.join(BROWSER_CHROME_EXTENSIONS, _OWNERS)
+      api_owners.append(self._CreateAPIEntry('Core Extensions/Apps Owners',
+                                             owners_file))
+      def sort(api_owners):
+        api_owners.sort(key=itemgetter('apiName'))
+        self._cache.Set('api_owners', api_owners)
+        return api_owners
+      api_owners_future = All(api_owners).Then(sort)
+      return api_owners_future
+    return self._cache.Get('api_owners').Then(collect)
+    
+  def get(self, key):
+    return {
+      'apis': self._CollectOwnersData()
+    }.get(key).Get()
+
+  def Cron(self):
+    return self._CollectOwnersData()