Add support for python pageset

tools/telemetry/telemetry/page/page_set.py

Index: tools/telemetry/telemetry/page/page_set.py
diff --git a/tools/telemetry/telemetry/page/page_set.py b/tools/telemetry/telemetry/page/page_set.py
index f97cb0d69cf900979e158d5dc2fcf0105a6dad92..34d6daacb80d5cfdf0d21d3cc2976382beab3e41 100644
--- a/tools/telemetry/telemetry/page/page_set.py
+++ b/tools/telemetry/telemetry/page/page_set.py
@@ -3,47 +3,65 @@
 # found in the LICENSE file.
 
 import csv
+import imp
+import inspect
 import json
 import logging
 import os
 
+from telemetry.core import camel_case
 from telemetry.page import cloud_storage
 from telemetry.page import page as page_module
 from telemetry.page import page_set_archive_info
 
 
+class PageSetError(Exception):
+  pass
+
+def _RenamePageSubMethod(method_name):
+  assert method_name.startswith("Run")
+  method_name = method_name[3:]
+  return camel_case.ToUnderscore(method_name)
+
 class PageSet(object):
-  def __init__(self, file_path='', attributes=None):
-    self.file_path = file_path
+  def __init__(self, description='', archive_data_file='',
+               credentials_path=None, user_agent_type=None,
+               make_javascript_deterministic=True, wpr_archive_info=None,
+               pages=None, serving_dirs=None):
 
     # These attributes can be set dynamically by the page set.
-    self.description = ''
-    self.archive_data_file = ''
-    self.credentials_path = None
-    self.user_agent_type = None
-    self.make_javascript_deterministic = True
+    self.description = description
+    self.archive_data_file = archive_data_file
+    self.credentials_path = credentials_path
+    self.user_agent_type = user_agent_type
+    self.make_javascript_deterministic = make_javascript_deterministic
+    self.wpr_archive_info = wpr_archive_info
+    if pages:
+      self.pages = pages
+    else:
+      self.pages = []
+    if serving_dirs:
+      self.serving_dirs = serving_dirs
+    else:
+      self.serving_dirs = None
+    # This attribute will be set by FromFile methods
+    self.file_path = ''
     self.navigate_steps = {'action': 'navigate'}
 
-    if attributes:
-      for k, v in attributes.iteritems():
-        setattr(self, k, v)
-
+  def Initialize(self, attributes):

[nduca, 2014/03/05 03:00:57]

_InitializeFromDict(self, dict) and only call from FromDict so that someone
can't do

ps = PageSet(...)
ps.Add(...)
ps.InitializeFromDict({})

[nednguyen, 2014/03/05 20:53:31]

Done.

     # Create a PageSetArchiveInfo object.
     if self.archive_data_file:
       self.wpr_archive_info = page_set_archive_info.PageSetArchiveInfo.FromFile(
           os.path.join(self._base_dir, self.archive_data_file))
-    else:
-      self.wpr_archive_info = None
 
     # Create a Page object for every page.
     self.pages = []
     if attributes and 'pages' in attributes:
       for page_attributes in attributes['pages']:
         url = page_attributes.pop('url')
-
         page = page_module.Page(
             url, self, attributes=page_attributes, base_dir=self._base_dir)
-        self.pages.append(page)
+        self.AddPage(page)
 
     # Prepend _base_dir to our serving dirs.
     # Always use realpath to ensure no duplicates in set.
@@ -84,16 +102,64 @@ class PageSet(object):
             continue
           cloud_storage.GetIfChanged(path)
 
+  def AddPage(self, page):
+    self.pages.append(page)
+
   @classmethod
   def FromFile(cls, file_path):
+    if file_path.endswith('.json'):
+      return cls.FromJSONFile(file_path)
+    elif file_path.endswith('.py'):
+      return cls.FromPythonFile(file_path)
+    else:
+      raise PageSetError("Pageset %s has unsupported file type" % file_path)
+
+  @classmethod

[nduca, 2014/03/05 03:00:57]

@staticmethod? doesn't seem to need cls

+  def FromPythonFile(cls, file_path):
+    module = imp.load_source('page_set_module', file_path)
+    page_set_classes = []
+    for m in dir(module):
+      if m.startswith('PageSet') and m != 'PageSet':
+        page_set_classes.append(getattr(module, m))
+    if len(page_set_classes) != 1:
+      raise PageSetError("Pageset file needs to contain exactly 1 pageset class"
+                         " with prefix 'PageSet'")
+    page_set = page_set_classes[0]()
+    page_set.file_path = file_path
+    for page in page_set.pages:
+      page.page_set = page_set
+      page_class = page.__class__
+
+      for method_name, method in inspect.getmembers(page_class,
+                                          predicate=inspect.ismethod):
+        if method_name.startswith("Run"):
+          args, _, _, _ = inspect.getargspec(method)
+          if not (args[0] == "self" and args[1] == "action_runner"):
+            raise PageSetError("""Definition of Run<...> method of all
+pages in %s must be in the form of def Run<...>(self, action_runner):"""
+                                     % file_path)
+          setattr(page_class, _RenamePageSubMethod(method_name), method)

[nduca, 2014/03/05 03:00:57]

I wonder, how about having the page_test file look for RunX all the time, and
then make the _Initialize method create method objects on the object for the
json backcompat path?

I think the idea would be to make a function that is a closure that runs the
action right away, then add that as an attribute on the class.

AddActionMethodToPageFromDict(page, initial_action_name, action_dict):
   def RunInitialAction(self, action_runner):
     # like the GetCompoundActionsRecursive stuff, but here its just run the
dictionary right awawy

  setattr(page, 'Run' + ToCamelCase(initial_action_name), RunInitialAction)

If thats not quite the exact way shout and I can work with you to figure out the
python voodoo.

+    return page_set
+
+
+  @classmethod
+  def FromJSONFile(cls, file_path):
     with open(file_path, 'r') as f:
       contents = f.read()
     data = json.loads(contents)
     return cls.FromDict(data, file_path)
 
   @classmethod
-  def FromDict(cls, data, file_path):
-    return cls(file_path, data)
+  def FromDict(cls, attributes, file_path=''):
+    page_set = cls()
+    #TODO(nednguyen): all of this is wrong in the new execution model. Add
+    # python code generation logic.
+    page_set.file_path = file_path
+    if attributes:
+      for k, v in attributes.iteritems():
+        setattr(page_set, k, v)
+    page_set.Initialize(attributes)
+    return page_set
 
   @property
   def _base_dir(self):
@@ -126,7 +192,7 @@ class PageSet(object):
 
       for csv_row in csv_reader:
         if csv_row[url_index] in page_set_dict:
-          pages.append(page_set_dict[csv_row[url_index]])
+          self.AddPage(page_set_dict[csv_row[url_index]])
         else:
           raise Exception('Unusable results_file.')
 
@@ -137,6 +203,9 @@ class PageSet(object):
       return None
     return self.wpr_archive_info.WprFilePathForPage(page)
 
+  def AddCustomizeBrowserOptions(self, options):

[nduca, 2014/03/05 03:00:57]

lets try to avoid these and talk about the total set of corner cases that cause
us to need this set of entry points?

+    pass
+
   def __iter__(self):
     return self.pages.__iter__()