Add CIPD pin reporting to swarming.

appengine/swarming/server/task_result.py

 # Copyright 2014 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """Task execution result models.
 
 This module doesn't do the scheduling itself. It only describes the entities to
 store tasks results.
 
 - TaskResultSummary represents the overall result for the TaskRequest taking in
@@ skipping 40 matching lines @@
        |id=1 (not stored)| |id=1            |
        +-----------------+ +----------------+
                  ^      ^
                  |      |
     +---------------+  +---------------+
     |TaskOutputChunk|  |TaskOutputChunk| ...
     |id=1           |  |id=2           |
     +---------------+  +---------------+
 """
 
+import collections
 import datetime
 import logging
 import random
 
 from google.appengine.api import datastore_errors
 from google.appengine.datastore import datastore_query
 from google.appengine.ext import ndb
 
 from components import datastore_utils
 from components import utils
@@ skipping 228 matching lines @@
   @property
   def is_valid(self):
     return self.bot_overhead is not None
 
   def _pre_put_hook(self):
     if self.bot_overhead is None:
       raise datastore_errors.BadValueError(
           'PerformanceStats.bot_overhead is required')
 
 
+class CipdPins(ndb.Model):
+  """Specifies which CIPD client and packages were actually installed.
+
+  A part of TaskRunResult.

[M-A Ruel, 2016/08/26 23:19:51]

This is misleading, since this is in TaskResultSummary too.

[iannucci, 2016/08/29 22:08:40]

erp, sorry.

+  """
+  # CIPD package of CIPD client to use.
+  # client_package.package_name and version are provided.
+  # client_package.path will be None.
+  client_package = ndb.LocalStructuredProperty(task_request.CipdPackage)
+
+  # List of packages to install in $CIPD_PATH prior task execution.
+  packages = ndb.LocalStructuredProperty(task_request.CipdPackage,

[M-A Ruel, 2016/08/26 23:19:51]

I'd like the entities to be sorted the same way they are in task_request.py
CipdInput._pre_put_hook()


Also I'd like a _pre_put_hook here to ensure that the items are exactly the same
as in TaskProperties.cipd_input. There shouldn't be asserts in viewing code,
better to assert at DB write time.

[iannucci, 2016/08/29 22:08:40]

yep, absolutely true, don't know what I was thinking...

That said, I don't think this can be _pre_put_hook because we don't have a
reference to the TaskProperties there?

[M-A Ruel, 2016/08/30 02:18:07]

Yes, so the check will have to be done in a validate() or something.

+                                         repeated=True)
+
+
 class _TaskResultCommon(ndb.Model):
   """Contains properties that is common to both TaskRunResult and
   TaskResultSummary.
 
   It is not meant to be instantiated on its own.
 
   TODO(maruel): Overhaul this entity:
   - Get rid of TaskOutput as it is not needed anymore (?)
   """
   # Bot that ran this task.
@@ skipping 47 matching lines @@
   # Children tasks that were triggered by this task. This is set when the task
   # reentrantly creates other Swarming tasks. Note that the task_id is to a
   # TaskResultSummary.
   children_task_ids = ndb.StringProperty(
       validator=_validate_task_summary_id, repeated=True)
 
   # File outputs of the task. Only set if TaskRequest.properties.sources_ref is
   # set. The isolateserver and namespace should match.
   outputs_ref = ndb.LocalStructuredProperty(task_request.FilesRef)
 
+  # The pinned versions of all the CIPD packages used in the task.
+  cipd_pins = ndb.LocalStructuredProperty(CipdPins)
+
   @property
   def can_be_canceled(self):
     """Returns True if the task is in a state that can be canceled."""
     # TOOD(maruel): To be able to add State.RUNNING, the following must be done:
     # task_scheduler.cancel_task() must be strictly a transaction relative to
     # task_scheduler.bot_kill_task() and task_scheduler.bot_update_task().
     #
     # The tricky part is to keep this code performant. On the other hand, all
     # the entities under the transaction (TaskToRun, TaskResultSummary and
     # TaskRunResult) are under the same entity root, so it's definitely
@@ skipping 210 matching lines @@
     if self.deduped_from:
       if self.state != State.COMPLETED:
         raise datastore_errors.BadValueError(
             'state must be COMPLETED on deduped task')
       if self.failure:
         raise datastore_errors.BadValueError(
             'failure can\'t be True on deduped task')
 
     self.children_task_ids = sorted(
         set(self.children_task_ids), key=lambda x: int(x, 16))
 

[M-A Ruel, 2016/08/26 23:19:51]

It should explicitly:
if self.cipd_pins:
  self.cipd_pins._pre_put_hook()

   @classmethod
   def _properties_fixed(cls):
     """Returns all properties with their member name, excluding computed
     properties.
     """
     return [
       prop._code_name for prop in cls._properties.itervalues()
       if not isinstance(prop, ndb.ComputedProperty)
     ]
 
@@ skipping 158 matching lines @@
       return None
     return task_pack.result_summary_key_to_run_result_key(
         self.key, self.try_number)
 
   @property
   def task_id(self):
     return task_pack.pack_result_summary_key(self.key)
 
   def reset_to_pending(self):
     """Resets this entity to pending state."""
+    self.cipd_pins = None
     self.duration = None
     self.exit_code = None
     self.internal_failure = False
     self.outputs_ref = None
     self.started_ts = None
     self.state = State.PENDING
 
   def set_from_run_result(self, run_result, request):
     """Copies all the relevant properties from a TaskRunResult into this
     TaskResultSummary.
@@ skipping 366 matching lines @@
   if tags:
     # Add TaskResultSummary indexes if desired.
     if sort != 'created_ts':
       raise ValueError(
           'Add needed indexes for sort:%s and tags if desired' % sort)
     tags_filter = TaskResultSummary.tags == tags[0]
     for tag in tags[1:]:
       tags_filter = ndb.AND(tags_filter, TaskResultSummary.tags == tag)
     query = query.filter(tags_filter)
   return _filter_query(TaskResultSummary, query, start, end, sort, state)