Moved chromium_status to appengine/

appengine_apps/chromium_status/appengine_module/chromium_status/commit_queue.py

Index: appengine_apps/chromium_status/appengine_module/chromium_status/commit_queue.py
diff --git a/appengine_apps/chromium_status/appengine_module/chromium_status/commit_queue.py b/appengine_apps/chromium_status/appengine_module/chromium_status/commit_queue.py
deleted file mode 100644
index 1350c461527d22b286051b0873f0452ceb0c9bad..0000000000000000000000000000000000000000
--- a/appengine_apps/chromium_status/appengine_module/chromium_status/commit_queue.py
+++ /dev/null
@@ -1,573 +0,0 @@
-# Copyright (c) 2012 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Commit queue status."""
-
-import cgi
-import datetime
-import json
-import logging
-import re
-import sys
-import urllib2
-
-from google.appengine.api import memcache
-from google.appengine.api import users
-from google.appengine.ext import db
-from google.appengine.ext.db import polymodel
-
-from appengine_module.chromium_status.base_page import BasePage
-from appengine_module.chromium_status import utils
-
-
-TRY_SERVER_MAP = (
-  'SUCCESS', 'WARNINGS', 'FAILURE', 'SKIPPED', 'EXCEPTION', 'RETRY',
-)
-
-# Verification name in commit-queue/verification/*.py. Initialized at the bottom
-# of this file.
-EVENT_MAP = {}
-
-
-class Owner(db.Model):
-  """key == email address."""
-  email = db.EmailProperty()
-
-  @staticmethod
-  def to_key(owner):  # pragma: no cover
-    return '<%s>' % owner
-
-
-class PendingCommit(db.Model):
-  """parent is Owner."""
-  created = db.DateTimeProperty()
-  done = db.BooleanProperty(default=False)
-  issue = db.IntegerProperty()
-  patchset = db.IntegerProperty()
-
-  @staticmethod
-  def to_key(issue, patchset, owner):  # pragma: no cover
-    # TODO(maruel): My bad, shouldn't have put owner in the key.
-    return '<%d-%d-%s>' % (issue, patchset, owner)
-
-
-class VerificationEvent(polymodel.PolyModel):
-  """parent is PendingCommit."""
-  created = db.DateTimeProperty(auto_now_add=True)
-  result = db.IntegerProperty()
-  timestamp = db.DateTimeProperty()
-
-  @property
-  def as_html(self):
-    raise NotImplementedError()
-
-
-class TryServerEvent(VerificationEvent):
-  name = 'try server'
-  build = db.IntegerProperty()
-  builder = db.StringProperty()
-  clobber = db.BooleanProperty()
-  job_name = db.StringProperty()
-  revision = db.IntegerProperty()
-  url = db.StringProperty()
-
-  @property
-  def as_html(self):  # pragma: no cover
-    if self.build is not None:
-      out = '<a href="%s">"%s" on %s, build #%s</a>' % (
-          cgi.escape(self.url),
-          cgi.escape(self.job_name),
-          cgi.escape(self.builder),
-          cgi.escape(str(self.build)))
-      if (self.result is not None and
-          0 <= self.result < len(TRY_SERVER_MAP[self.result])):
-        out = '%s - result: %s' % (out, TRY_SERVER_MAP[self.result])
-      return out
-    else:
-      # TODO(maruel): Load the json
-      # ('http://build.chromium.org/p/tryserver.chromium/json/builders/%s/'
-      #  'pendingBuilds') % self.builder and display the rank.
-      return '"%s" on %s (pending)' % (
-          cgi.escape(self.job_name),
-          cgi.escape(self.builder))
-
-  @classmethod
-  def to_key(cls, packet):  # pragma: no cover
-    if not packet.get('builder') or not packet.get('job_name'):
-      return None
-    return '<%s-%s-%s>' % (
-        cls.name, packet['builder'], packet['job_name'])
-
-
-class TryJobRietveldEvent(VerificationEvent):
-  """Same thing as TryServerEvent. Should probably be kept in sync with
-  TryServerEvent.
-
-  It comes from commit-queue/verification/try_job_on_rietveld.py.
-  """
-  name = 'try job rietveld'
-  build = db.IntegerProperty()
-  builder = db.StringProperty()
-  clobber = db.BooleanProperty()
-  job_name = db.StringProperty()
-  # TODO(maruel): Transition all revision properties to string, since it could
-  # be a hash for git commits.
-  revision = db.StringProperty()
-  url = db.StringProperty()
-
-  @property
-  def as_html(self):  # pragma: no cover
-    if self.build is not None:
-      out = '<a href="%s">"%s" on %s, build #%s</a>' % (
-          cgi.escape(self.url),
-          cgi.escape(self.job_name),
-          cgi.escape(self.builder),
-          cgi.escape(str(self.build)))
-      if (self.result is not None and
-          0 <= self.result < len(TRY_SERVER_MAP[self.result])):
-        out = '%s - result: %s' % (out, TRY_SERVER_MAP[self.result])
-      return out
-    else:
-      # TODO(maruel): Load the json
-      # ('http://build.chromium.org/p/tryserver.chromium/json/builders/%s/'
-      #  'pendingBuilds') % self.builder and display the rank.
-      return '"%s" on %s (pending)' % (
-          cgi.escape(self.job_name),
-          cgi.escape(self.builder))
-
-  @classmethod
-  def to_key(cls, packet):  # pragma: no cover
-    if not packet.get('builder') or not packet.get('job_name'):
-      return None
-    return '<%s-%s-%s>' % (
-        cls.name, packet['builder'], packet['job_name'])
-
-
-class PresubmitEvent(VerificationEvent):
-  name = 'presubmit'
-  duration = db.FloatProperty()
-  output = db.TextProperty()
-  timed_out = db.BooleanProperty()
-
-  @property
-  def as_html(self):  # pragma: no cover
-    return '<pre class="output">%s</pre>' % cgi.escape(self.output)
-
-  @classmethod
-  def to_key(cls, _):  # pragma: no cover
-    # There shall be only one PresubmitEvent per PendingCommit.
-    return '<%s>' % cls.name
-
-
-class CommitEvent(VerificationEvent):
-  name = 'commit'
-  output = db.TextProperty()
-  revision = db.IntegerProperty()
-  url = db.StringProperty()
-
-  @property
-  def as_html(self):  # pragma: no cover
-    out = '<pre class="output">%s</pre>' % cgi.escape(self.output)
-    if self.url:
-      out += '<a href="%s">Revision %s</a>' % (
-        cgi.escape(self.url),
-        cgi.escape(str(self.revision)))
-    elif self.revision:
-      out += '<br>Revision %s' % cgi.escape(str(self.revision))
-    return out
-
-  @classmethod
-  def to_key(cls, _):  # pragma: no cover
-    return '<%s>' % cls.name
-
-
-class InitialEvent(VerificationEvent):
-  name = 'initial'
-  revision = db.IntegerProperty()
-
-  @property
-  def as_html(self):  # pragma: no cover
-    return 'Looking at new commit, using revision %s' % (
-        cgi.escape(str(self.revision)))
-
-  @classmethod
-  def to_key(cls, _):  # pragma: no cover
-    return '<%s>' % cls.name
-
-
-class AbortEvent(VerificationEvent):
-  name = 'abort'
-  output = db.TextProperty()
-
-  @property
-  def as_html(self):  # pragma: no cover
-    return '<pre class="output">%s</pre>' % cgi.escape(self.output)
-
-  @classmethod
-  def to_key(cls, _):  # pragma: no cover
-    return '<%s>' % cls.name
-
-
-class WhyNotEvent(VerificationEvent):
-  name = 'why not'
-  message = db.TextProperty()
-
-  @property
-  def as_html(self):  # pragma: no cover
-    return '<pre class="output">%s</pre>' % cgi.escape(self.message)
-
-  @classmethod
-  def to_key(cls, _):  # pragma: no cover
-    return '<%s>' % cls.name
-
-
-def get_owner(owner):  # pragma: no cover
-  """Efficient querying of Owner with memcache."""
-  key = Owner.to_key(owner)
-  obj = memcache.get(key, namespace='Owner')
-  if not obj:
-    obj = Owner.get_or_insert(key_name=key, email=owner)
-    memcache.set(key, obj, time=60*60, namespace='Owner')
-  return obj
-
-
-def get_pending_commit(issue, patchset, owner, timestamp):  # pragma: no cover
-  """Efficient querying of PendingCommit with memcache."""
-  owner_obj = get_owner(owner)
-  key = PendingCommit.to_key(issue, patchset, owner)
-  obj = memcache.get(key, namespace='PendingCommit')
-  if not obj:
-    obj = PendingCommit.get_or_insert(
-        key_name=key, parent=owner_obj, issue=issue, patchset=patchset,
-        owner=owner, created=timestamp)
-    memcache.set(key, obj, time=60*60, namespace='PendingCommit')
-  return obj
-
-
-class CQBasePage(BasePage):
-  """Returns a web page or json data about commit queue state.
-
-  Can filter for everyone, a particular user or a particular issue.
-
-  Query args:
-  - format: can be 'html' or 'json'.
-  - limit: maximum number of elements to result. default is 100.
-  """
-
-  def get(self, *args):  # pragma: no cover
-    query = self._get_query(*args)
-    if not query:
-      # The user probably used /me without being logged.
-      self.redirect(users.create_login_url(self.request.url))
-      return
-    out_format = self.request.get('format', 'html')
-    if out_format == 'json':
-      return self._get_as_json(query)
-    else:
-      return self._get_as_html(query)
-
-  def _get_query(self, owner=None, issue=None,
-        patchset=None):  # pragma: no cover
-    """Returns None on query failure."""
-    query = VerificationEvent.all().order('-timestamp')
-    ancestor = None
-    if owner:
-      owner = self._parse_user(owner)
-      if not owner:
-        return None
-      ancestor = db.Key.from_path('Owner', Owner.to_key(owner))
-
-    if issue:
-      issue = int(issue)
-      if patchset:
-        patchset = int(patchset)
-        pending_key = PendingCommit.to_key(issue, patchset, owner)
-        ancestor = db.Key.from_path(
-            'PendingCommit', pending_key, parent=ancestor)
-      else:
-        # Only show the last object since it's complex to do a OR with multiple
-        # ancestors.
-        ancestor = db.Query(PendingCommit, keys_only=True).filter(
-            'issue =', issue).ancestor(ancestor).order('-created').get()
-
-    if ancestor:
-      query.ancestor(ancestor)
-    return query
-
-  def _get_limit(self):  # pragma: no cover
-    limit = self.request.get('limit')
-    if limit and limit.isdigit():
-      limit = int(limit)
-    else:
-      limit = 100
-    return limit
-
-  def _get_as_json(self, query):  # pragma: no cover
-    self.response.headers['Content-Type'] = 'application/json'
-    self.response.headers['Access-Control-Allow-Origin'] = '*'
-    data = json.dumps([s.AsDict() for s in query.fetch(self._get_limit())])
-    callback = self.request.get('callback')
-    if callback:
-      if re.match(r'^[a-zA-Z$_][a-zA-Z$0-9._]*$', callback):
-        data = '%s(%s);' % (callback, data)
-    self.response.out.write(data)
-
-  def _get_as_html(self, query):  # pragma: no cover
-    raise NotImplementedError()
-
-  def _parse_user(self, user):  # pragma: no cover
-    user = urllib2.unquote(user.strip('/'))
-    if user == 'me':
-      if not self.user:
-        user = None
-      else:
-        user = self.user.email()
-    return user
-
-
-class OwnerStats(object):  # pragma: no cover
-  """CQ usage statistics for a single user."""
-  def __init__(self, now, owner, last_day, last_week, last_month, forever):
-    # Since epoch in float.
-    self.now = now
-    # User instance.
-    self.owner = owner
-    assert all(isinstance(i, PendingCommit) for i in last_day)
-    self.last_day = last_day
-    assert all(isinstance(i, PendingCommit) for i in last_week)
-    self.last_week = last_week
-    assert isinstance(last_month, int)
-    self.last_month = last_month
-    assert isinstance(forever, int)
-    self.forever = forever
-    # Gamify ALL the things!
-    self.points = (
-        len(self.last_day) * 10 +
-        len(self.last_week) * 5 +
-        self.last_month * 2 +
-        self.forever)
-
-
-class OwnerQuery(object):  # pragma: no cover
-  def __init__(self, owner_key, now):
-    self.owner_key = owner_key
-    self.now = now
-    since = lambda x: now - datetime.timedelta(days=x)
-    self._owner = db.get_async(owner_key)
-    self._last_day = self._pendings().filter('created >=', since(1)).run()
-    self._last_week = self._pendings().filter(
-        'created >=', since(7)).filter('created <', since(1)).run()
-    # These block.
-    self.last_month = self._pendings().filter(
-        'created >=', since(30)).count()
-    self.forever = self._pendings(keys_only=True).count()
-
-  def _pendings(self, **kwargs):
-    return PendingCommit.all(**kwargs).ancestor(self.owner_key)
-
-  def to_stats(self):
-    obj = OwnerStats(
-        self.now,
-        self._owner.get_result(),
-        list(self._last_day),
-        list(self._last_week),
-        self.last_month,
-        self.forever)
-    memcache.add(
-        self.owner_key.name(), obj, 2*60*60, namespace='cq_owner_stats')
-    return obj
-
-
-def to_link(pending):  # pragma: no cover
-  return '<a href="/cq/%s/%s/%s">%s</a>' % (
-      pending.parent_key().name()[1:-1],
-      pending.issue,
-      pending.patchset,
-      pending.issue)
-
-
-def get_owner_stats(owner_key, now):  # pragma: no cover
-  """Returns an OnwerStats instance for the Owner."""
-  obj = memcache.get(owner_key.name(), 'cq_owner_stats')
-  if obj:
-    return obj
-  return OwnerQuery(owner_key, now).to_stats()
-
-
-def monthly_top_contributors():  # pragma: no cover
-  """Returns the top monthly contributors as a list of OwnerStats."""
-  obj = memcache.get('monthly', 'cq_top')
-  if not obj:
-    now = datetime.datetime.utcnow()
-    last_pendings = PendingCommit.all(
-        keys_only=True).order('-created').fetch(1000)
-    # Make it use asynchronous queries.
-    obj = [
-        get_owner_stats(o, now) for o in set(p.parent() for p in last_pendings)
-    ]
-    memcache.add('monthly', obj, 2*60*60, namespace='cq_top')
-  return obj
-
-
-class Summary(CQBasePage):  # pragma: no cover
-  def _get_as_html(self, _):
-    owners = []
-    for stats in monthly_top_contributors():
-      data = {
-        'email': stats.owner.email,
-        'last_day': ', '.join(to_link(i) for i in stats.last_day),
-        'last_week': ', '.join(to_link(i) for i in stats.last_week),
-        'last_month': stats.last_month,
-        'forever': stats.forever,
-      }
-      owners.append(data)
-    owners.sort(key=lambda x: -x['last_month'])
-    template_values = self.InitializeTemplate(self.APP_NAME + ' Commit queue')
-    template_values['data'] = owners
-    self.DisplayTemplate('cq_owners.html', template_values, use_cache=True)
-
-
-def ordinal_number(i):
-  if (i % 10) == 1 and (i % 100) != 11:
-    return '%dst' % i
-  elif (i % 10) == 2 and (i % 100) != 12:
-    return '%dnd' % i
-  elif (i % 10) == 3 and (i % 100) != 13:
-    return '%drd' % i
-  else:
-    return '%dth' % i
-
-
-class TopScore(CQBasePage):  # pragma: no cover
-  def _get_as_html(self, _):
-    owners = [
-      {
-        'name': stats.owner.email.split('@', 1)[0].upper(),
-        'points': stats.points,
-      }
-      for stats in monthly_top_contributors()
-    ]
-    owners.sort(key=lambda x: -x['points'])
-    for i in xrange(len(owners)):
-      owners[i]['rank'] = ordinal_number(i + 1)
-    template_values = self.InitializeTemplate(self.APP_NAME + ' Commit queue')
-    template_values['data'] = owners
-    self.DisplayTemplate('cq_top_score.html', template_values, use_cache=True)
-
-
-class User(CQBasePage):  # pragma: no cover
-  def _get_as_html(self, query):
-    pending_commits_events = {}
-    pending_commits = {}
-    for event in query.fetch(self._get_limit()):
-      # Implicitly find PendingCommit's.
-      pending_commit = event.parent()
-      if not pending_commit:
-        logging.warn('Event %s is corrupted, can\'t find %s' % (
-          event.key().id_or_name(), event.parent_key().id_or_name()))
-        continue
-      pending_commits_events.setdefault(pending_commit.key(), []).append(event)
-      pending_commits[pending_commit.key()] = pending_commit
-
-    sorted_data = []
-    for pending_commit in sorted(
-        pending_commits.itervalues(), key=lambda x: x.created, reverse=True):
-      sorted_data.append(
-          (pending_commit,
-            reversed(pending_commits_events[pending_commit.key()])))
-    template_values = self.InitializeTemplate(self.APP_NAME + ' Commit queue')
-    template_values['data'] = sorted_data
-    self.DisplayTemplate('cq_owner.html', template_values, use_cache=True)
-
-
-class Issue(CQBasePage):  # pragma: no cover
-  def _get_as_html(self, query):
-    pending_commits_events = {}
-    pending_commits = {}
-    for event in query.fetch(self._get_limit()):
-      # Implicitly find PendingCommit's.
-      pending_commit = event.parent()
-      if not pending_commit:
-        logging.warn('Event %s is corrupted, can\'t find %s' % (
-          event.key().id_or_name(), event.parent_key().id_or_name()))
-        continue
-      pending_commits_events.setdefault(pending_commit.key(), []).append(event)
-      pending_commits[pending_commit.key()] = pending_commit
-
-    sorted_data = []
-    for pending_commit in sorted(
-        pending_commits.itervalues(), key=lambda x: x.issue):
-      sorted_data.append(
-          (pending_commit,
-            reversed(pending_commits_events[pending_commit.key()])))
-    template_values = self.InitializeTemplate(self.APP_NAME + ' Commit queue')
-    template_values['data'] = sorted_data
-    self.DisplayTemplate('cq_owner.html', template_values, use_cache=True)
-
-
-class Receiver(BasePage):  # pragma: no cover
-  @utils.requires_write_access
-  def post(self):
-    def load_values():
-      for p in self.request.get_all('p'):
-        try:
-          yield json.loads(p)
-        except ValueError:
-          logging.warn('Discarding invalid packet %r' % p)
-
-    count = 0
-    for packet in load_values():
-      cls = EVENT_MAP.get(packet.get('verification'))
-      if (not cls or
-          not isinstance(packet.get('issue'), int) or
-          not isinstance(packet.get('patchset'), int) or
-          not packet.get('timestamp') or
-          not isinstance(packet.get('owner'), basestring)):
-        logging.warning('Ignoring packet %s' % packet)
-        continue
-
-      payload = packet.get('payload', {})
-      # TODO(maruel): Convert the type implicitly, because storing a int into a
-      # FloatProperty or a StringProperty will raise a BadValueError.
-      values = dict(
-          (i, payload[i]) for i in cls.properties()
-          if i not in ('_class', 'pending') and i in payload)
-      # Inject the timestamp.
-      values['timestamp'] = datetime.datetime.utcfromtimestamp(
-          packet['timestamp'])
-      pending = get_pending_commit(
-          packet['issue'], packet['patchset'], packet['owner'],
-          values['timestamp'])
-
-      logging.debug('New packet %s' % cls.__name__)
-      key_name = cls.to_key(values)
-      if not key_name:
-        continue
-
-      # TODO(maruel) Use an async transaction, in batch.
-      obj = cls.get_by_key_name(key_name, parent=pending)
-      # Compare the timestamps. Events could arrive in the reverse order.
-      if not obj or obj.timestamp <= values['timestamp']:
-        # This will override the previous obj if it existed.
-        cls(parent=pending, key_name=key_name, **values).put()
-        count += 1
-      elif obj:
-        logging.warn('Received object out of order')
-
-      # Cache the fact that the change was committed in the PendingCommit.
-      if packet['verification'] == 'commit':
-        pending.done = True
-        pending.put()
-
-    self.response.out.write('%d\n' % count)
-
-
-def bootstrap():  # pragma: no cover
-  # Used by _parse_packet() to find the right model to use from the
-  # 'verification' value of the packet.
-  module = sys.modules[__name__]
-  for i in dir(module):
-    if i.endswith('Event') and i != 'VerificationEvent':
-      obj = getattr(module, i)
-      EVENT_MAP[obj.name] = obj