tracker/spreadsheet tool: Improve tracker to spreadsheet migration.

src/scripts/tracker_spreadsheet_sync

 #!/usr/bin/env python
 
 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 # For Spreadsheets:
 try:
   from xml.etree import ElementTree
 except ImportError:
@@ skipping 10 matching lines @@
 import gdata.gauth
 import gdata.client
 import gdata.data
 import atom.http_core
 import atom.core
 
 # For this script:
 import getpass
 from optparse import OptionParser
 import pickle
-
+from sets import Set
 
 # Settings
 credentials_store = 'creds.dat'
 
 class Merger(object):
   def __init__(self, ss_key, ss_ws_key, tracker_message, tracker_project,
-               debug):
+               debug, pretend):
     self.ss_key = ss_key
     self.ss_ws_key = ss_ws_key
     self.tracker_message = tracker_message
     self.tracker_project = tracker_project
     self.debug_enabled = debug
+    self.pretend = pretend
     self.user_agent = 'adlr-tracker-spreadsheet-merger'
     self.it_keys = ['id', 'owner', 'status', 'title']
 
   def debug(self, message):
     """Prints message if debug mode is set."""
     if self.debug_enabled:
       print message
 
   def print_feed(self, feed):
     'Handy for debugging'
@@ skipping 56 matching lines @@
     feed = self.gd_client.GetListFeed(self.ss_key, self.ss_ws_key)
     issues = []
     for entry in feed.entry:
       issue = {}
       for key in entry.custom:
         issue[key] = entry.custom[key].text
       issue['__raw_entry'] = entry
       issues.append(issue)
     return issues
 
-  def fetch_tracker_issues(self):
-    """Fetches all issues matching the query and returns them as an array
-    of dictionaries."""
+  def ids_for_spreadsheet_issues(self, ss_issues):
+    """Returns a Set of strings, each string an id from ss_issues"""
+    ret = Set()
+    for ss_issue in ss_issues:
+      ret.add(ss_issue['id'])
+    return ret
+
+  def tracker_issues_for_query_feed(self, feed):
+    """Converts a feed object from a query to a list of tracker issue
+    dictionaries."""
+    issues = []
+    for issue in feed.entry:
+      issue_dict = {}
+      issue_dict['labels'] = [label.text for label in issue.label]
+      issue_dict['id'] = issue.id.text.split('/')[-1]
+      issue_dict['title'] = issue.title.text
+      issue_dict['status'] = issue.status.text
+      if issue.owner:
+        issue_dict['owner'] = issue.owner.username.text
+      issues.append(issue_dict)
+    return issues
+
+  def fetch_tracker_issues(self, ss_issues):
+    """Fetches all relevant issues from traacker and returns them as an array
+    of dictionaries. Relevance is:
+    - has an ID that's in ss_issues, OR
+    - (is Area=Installer AND status is open).
+    Open status is one of: Unconfirmed, Untriaged, Available, Assigned,
+    Started, Upstream"""
     issues = []
     got_results = True
     index = 1
     while got_results:
       query = gdata.projecthosting.client.Query(label='Area-Installer',
                                                 max_results=50,
                                                 start_index=index)
       feed = self.it_client.get_issues('chromium-os', query=query)
       if not feed.entry:
         got_results = False
       index = index + len(feed.entry)
-      for issue in feed.entry:
-        issue_dict = {}
-        issue_dict['labels'] = [label.text for label in issue.label]
-        issue_dict['id'] = issue.id.text.split('/')[-1]
-        issue_dict['title'] = issue.title.text
-        issue_dict['status'] = issue.status.text
-        if issue.owner:
-          issue_dict['owner'] = issue.owner.username.text
-        issues.append(issue_dict)
+      issues.extend(self.tracker_issues_for_query_feed(feed))
+    # Now, remove issues that are open or in ss_issues.
+    ss_ids = self.ids_for_spreadsheet_issues(ss_issues)
+    open_statuses = ['Unconfirmed', 'Untriaged', 'Available', 'Assigned',
+                     'Started', 'Upstream']
+    new_issues = []
+    for issue in issues:
+      if issue['status'] in open_statuses or issue['id'] in ss_ids:
+        new_issues.append(issue)
+      # Remove id from ss_ids, if it's there
+      ss_ids.discard(issue['id'])
+    issues = new_issues
+
+    # Now, for each ss_id that didn't turn up in the query, explicitly add it
+    for id_ in ss_ids:
+      query = gdata.projecthosting.client.Query(issue_id=id_,
+                                                max_results=50,
+                                                start_index=index)
+      feed = self.it_client.get_issues('chromium-os', query=query)
+      if not feed.entry:
+        print 'No result for id', id_
+        continue
+      issues.extend(self.tracker_issues_for_query_feed(feed))
+
     return issues
 
   def store_creds(self):
     """Stores login credentials to disk."""
     obj = {}
     if self.docs_token:
       obj['docs_token'] = self.docs_token
     if self.tracker_token:
       obj['tracker_token'] = self.tracker_token
     if self.tracker_user:
@@ skipping 110 matching lines @@
         continue
       ss_label = None
       if value:
         ss_label = caps_key + value.title()
       t_label = self.label_from_prefix(caps_key, t_issue['labels'])
 
       if t_label is None and ss_label is None:
         # Nothing
         continue
 
-      if (ss_label is None) or (ss_label != t_label):
+      if (t_label is not None) and \
+        ((ss_label is None) or (ss_label != t_label)):
         ret['labels'].append('-' + t_label)
 
-      if (t_label is None) or (t_label != ss_label):
+      if (ss_label is not None) and \
+        ((t_label is None) or (t_label != ss_label)):
         ret['labels'].append(ss_label)
     return ret
 
   def tracker_issue_has_changed(self, t_issue, ss_issue):
     """Returns True iff ss_issue indicates changes in t_issue that need to be
     committed up to the Issue Tracker."""
     if t_issue is None:
       return True
     potential_commit = \
       self.update_spreadsheet_issue_to_tracker_dict(ss_issue, t_issue)
@@ skipping 28 matching lines @@
       ret.append(commit)
     return ret
 
   def fetch_issues(self):
     """Logs into Docs/Tracker, and fetches spreadsheet and tracker issues"""
     print 'Logging into Docs...'
     self.spreadsheet_login()
     print 'Logging into Tracker...'
     self.tracker_login()
 
-    print 'Fetching tracker issues...'
-    t_issues = self.fetch_tracker_issues()
-    self.debug('Tracker issues: %s' % t_issues)
     print 'Fetching spreadsheet issues...'
     ss_issues = self.fetch_spreadsheet_issues()
     self.debug('Spreadsheet issues: %s' % ss_issues)
+    print 'Fetching tracker issues...'
+    t_issues = self.fetch_tracker_issues(ss_issues)
+    self.debug('Tracker issues: %s' % t_issues)
     return (t_issues, ss_issues)
 
   def spreadsheet_to_tracker(self):
     """High-level function to manage migrating data from the spreadsheet
     to Tracker."""
     (t_issues, ss_issues) = self.fetch_issues()
     print 'Calculating deltas...'
     commits = self.spreadsheet_to_tracker_commits(ss_issues, t_issues)
     self.debug('got commits: %s' % commits)
     if not commits:
       print 'No deltas. Done.'
       return
 
     for commit in commits:
       dic = commit['dict']
       labels = dic.get('labels')
       owner = dic.get('owner')
       status = dic.get('status')
 
       if commit['type'] == 'append':
         print 'Creating new tracker issue...'
+        if self.pretend:
+          print '(Skipping because --pretend is set)'
+          continue
         created = self.it_client.add_issue(self.tracker_project,
                                            dic['title'],
                                            self.tracker_message,
                                            self.tracker_user,
                                            labels=labels,
                                            owner=owner,
                                            status=status)
         issue_id = created.id.text.split('/')[-1]
         print 'Created issue with id:', issue_id
         print 'Write id back to spreadsheet row...'
         raw_entry = commit['__ss_issue']['__raw_entry']
         ss_issue = commit['__ss_issue']
         del ss_issue['__raw_entry']
         ss_issue.update({'id': issue_id})
         self.gd_client.UpdateRow(raw_entry, ss_issue)
         print 'Done.'
       else:
         print 'Updating issue with id:', dic['id']
+        if self.pretend:
+          print '(Skipping because --pretend is set)'
+          continue
         self.it_client.update_issue(self.tracker_project,
                                     dic['id'],
                                     self.tracker_user,
                                     comment=self.tracker_message,
                                     status=status,
                                     owner=owner,
                                     labels=labels)
         print 'Done.'
 
   def spreadsheet_issue_for_id(self, issues, id_):
@@ skipping 78 matching lines @@
     ss_commits = self.tracker_to_spreadsheet_commits(t_issues, ss_issues)
     self.debug('commits: %s' % ss_commits)
     if not ss_commits:
       print 'Nothing to commit.'
       return
     print 'Committing...'
     for commit in ss_commits:
       self.debug('Operating on commit: %s' % commit)
       if commit['type'] == 'append':
         print 'Appending new row...'
-        self.gd_client.InsertRow(commit['new_row'], self.ss_key, self.ss_ws_key)
+        if not self.pretend:
+          self.gd_client.InsertRow(commit['new_row'],
+                                   self.ss_key, self.ss_ws_key)
+        else:
+          print '(Skipped because --pretend set)'
       if commit['type'] == 'update':
         print 'Updating row...'
-        self.gd_client.UpdateRow(commit['__raw_entry'], commit['dict'])
+        if not self.pretend:
+          self.gd_client.UpdateRow(commit['__raw_entry'], commit['dict'])
+        else:
+          print '(Skipped because --pretend set)'
       print 'Done.'
 
 def main():
   class PureEpilogOptionParser(OptionParser):
     def format_epilog(self, formatter):
       return self.epilog
 
   parser = PureEpilogOptionParser()
   parser.add_option('-a', '--action', dest='action', metavar='ACTION',
                     help='Action to perform')
   parser.add_option('-d', '--debug', action='store_true', dest='debug',
                     default=False, help='Print debug output.')
   parser.add_option('-m', '--message', dest='message', metavar='TEXT',
                     help='Log message when updating Tracker issues')
+  parser.add_option('-p', '--pretend', action='store_true', dest='pretend',
+                    default=False, help="Don't commit anything.")
   parser.add_option('--ss_key', dest='ss_key', metavar='KEY',
                     help='Spreadsheets key (find with browse action)')
   parser.add_option('--ss_ws_key', dest='ss_ws_key', metavar='KEY',
                     help='Spreadsheets worksheet key (find with browse action)')
   parser.add_option('--tracker_project', dest='tracker_project',
                     metavar='PROJECT',
                     help='Tracker project (default: chromium-os)',
                     default='chromium-os')
   parser.epilog = """Actions:
   browse -- browse spreadsheets to find spreadsheet and worksheet keys.
@@ skipping 43 matching lines @@
 a command-line flag.
 - When creating a new issue on tracker, the owner field isn't set. I (adlr)
 am not sure why. Workaround: If you rerun this script, tho, it will detect
 a delta and update the tracker issue with the owner, which seems to succeed.
 """
 
   (options, args) = parser.parse_args()
 
   merger = Merger(options.ss_key, options.ss_ws_key,
                   options.message, options.tracker_project,
-                  options.debug)
+                  options.debug, options.pretend)
   if options.action == 'browse':
     merger.browse()
   elif options.action == 'ss_to_t':
     if not options.message:
       print 'Error: when updating tracker, -m MESSAGE required.'
       return
     merger.spreadsheet_to_tracker()
   elif options.action == 't_to_ss':
     merger.tracker_to_spreadsheet()
   else:
     raise Exception('Unknown action requested.')
 
 if __name__ == '__main__':
   main()