Docserver overhaul: Gitiles away from me.

chrome/common/extensions/docs/server2/content_providers.py

 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import logging
 import os
 import traceback
 
 from chroot_file_system import ChrootFileSystem
 from content_provider import ContentProvider
 import environment
 from extensions_paths import CONTENT_PROVIDERS, LOCAL_DEBUG_DIR
-from future import Future
-from gitiles_file_system import GitilesFileSystem

[Ken Rockot(use gerrit already), 2015/05/26 00:26:23]

\o/

+from future import All, Future
 from local_file_system import LocalFileSystem
 from third_party.json_schema_compiler.memoize import memoize
 
 
 _IGNORE_MISSING_CONTENT_PROVIDERS = [False]
 
 
 def IgnoreMissingContentProviders(fn):
   '''Decorates |fn| to ignore missing content providers during its run.
   '''
@@ skipping 12 matching lines @@
   chrome/common/extensions/docs/templates/json/content_providers.json for its
   current state and a description of the format.
 
   Returns ContentProvider instances based on how they're configured there.
   '''
 
   def __init__(self,
                object_store_creator,
                compiled_fs_factory,
                host_file_system,
-               github_file_system_provider,

[Ken Rockot(use gerrit already), 2015/05/26 00:26:23]

I removed all the github FS plumbing for two reasons:

 1. It depends on blobstore and other AppEngine-only facilities
 2. Now that we have LocalGitFileSystem, the right (i.e. fast, reliable,
    consistent) way to support a git repo is to mirror it locally.

I didn't bother setting up mirrors for the existing repos because they have file
size issues, and those are conveniently abated by the live site automatically
falling through cache and serving directly from GCS.

                gcs_file_system_provider):
     self._object_store_creator = object_store_creator
     self._compiled_fs_factory = compiled_fs_factory
     self._host_file_system = host_file_system
-    self._github_file_system_provider = github_file_system_provider
     self._gcs_file_system_provider = gcs_file_system_provider
     self._cache = None
 
     # If running the devserver and there is a LOCAL_DEBUG_DIR, we
     # will read the content_provider configuration from there instead
     # of fetching it from Gitiles or patch.
     if environment.IsDevServer() and os.path.exists(LOCAL_DEBUG_DIR):
       local_fs = LocalFileSystem(LOCAL_DEBUG_DIR)
       conf_stat = None
       try:
@@ skipping 78 matching lines @@
         return None
       bucket = gcs_config['bucket']
       if not bucket.startswith('gs://'):
         logging.error('%s: bucket %s should start with gs://' % (name, bucket))
         return None
       bucket = bucket[len('gs://'):]
       file_system = self._gcs_file_system_provider.Create(bucket)
       if 'dir' in gcs_config:
         file_system = ChrootFileSystem(file_system, gcs_config['dir'])
 
-    elif 'github' in config:
-      github_config = config['github']
-      if 'owner' not in github_config or 'repo' not in github_config:
-        logging.error('%s: "github" must provide an "owner" and "repo"' % name)
-        return None
-      file_system = self._github_file_system_provider.Create(
-          github_config['owner'], github_config['repo'])
-      if 'dir' in github_config:
-        file_system = ChrootFileSystem(file_system, github_config['dir'])
-
     else:
       logging.error('%s: content provider type not supported' % name)
       return None
 
     return ContentProvider(name,
                            self._compiled_fs_factory,
                            file_system,
                            self._object_store_creator,
                            default_extensions=default_extensions,
                            supports_templates=supports_templates,
                            supports_zip=supports_zip)
 
-  def GetRefreshPaths(self):
-    return self._GetConfig().keys()
-
-  def Refresh(self, path):
+  def Refresh(self):
     def safe(name, action, callback):
       '''Safely runs |callback| for a ContentProvider called |name| by
       swallowing exceptions and turning them into a None return value. It's
       important to run all ContentProvider Refreshes even if some of them fail.
       '''
       try:
         return callback()
       except:
         if not _IGNORE_MISSING_CONTENT_PROVIDERS[0]:
           logging.error('Error %s Refresh for ContentProvider "%s":\n%s' %
                         (action, name, traceback.format_exc()))
         return None
 
-    config = self._GetConfig()[path]

[Ken Rockot(use gerrit already), 2015/05/26 00:26:23]

No net logic changes, just killing refresh paths

-    provider = self._CreateContentProvider(path, config)
-    future = safe(path,
-                  'initializing',
-                  self._CreateContentProvider(path, config).Refresh)
-    if future is None:
-      return Future(callback=lambda: True)
-    return Future(callback=lambda: safe(path, 'resolving', future.Get))
+    def refresh_provider(path, config):
+      provider = self._CreateContentProvider(path, config)
+      future = safe(path,
+                    'initializing',
+                    self._CreateContentProvider(path, config).Refresh)
+      if future is None:
+        return Future(callback=lambda: True)
+      return Future(callback=lambda: safe(path, 'resolving', future.Get))
+
+    return All(refresh_provider(path, config)
+               for path, config in self._GetConfig().iteritems())