[Findit] Capture versionized metadata for master_flake_analysis

appengine/findit/model/flake/master_flake_analysis.py

Index: appengine/findit/model/flake/master_flake_analysis.py
diff --git a/appengine/findit/model/flake/master_flake_analysis.py b/appengine/findit/model/flake/master_flake_analysis.py
index 2805c843e5c6f43a3088f709df6e192a2ac079fc..b28016dd64f161f20f3501efe8485d9f17334e3c 100644
--- a/appengine/findit/model/flake/master_flake_analysis.py
+++ b/appengine/findit/model/flake/master_flake_analysis.py
@@ -6,21 +6,16 @@ import base64
 
 from google.appengine.ext import ndb
 
-from model.base_build_model import BaseBuildModel
+from common import time_util
+from model import analysis_status
 from model.base_analysis import BaseAnalysis
-from model.flake.flake_swarming_task import FlakeSwarmingTask
+from model.base_build_model import BaseBuildModel
+from model.versioned_model import VersionedModel
 
 
-class MasterFlakeAnalysis(BaseAnalysis, BaseBuildModel):
+class MasterFlakeAnalysis(BaseAnalysis, BaseBuildModel, VersionedModel):
   """Represents an analysis of a flaky test in a Chromium Waterfall."""
 
-  @staticmethod
-  def _CreateAnalysisId(master_name, builder_name,
-                       build_number, step_name, test_name):
-    encoded_test_name = base64.urlsafe_b64encode(test_name)
-    return '%s/%s/%s/%s/%s' % (master_name, builder_name,
-                               build_number, step_name, encoded_test_name)
-
   @ndb.ComputedProperty
   def step_name(self):
     return self.key.pairs()[0][1].split('/')[3]
@@ -30,29 +25,102 @@ class MasterFlakeAnalysis(BaseAnalysis, BaseBuildModel):
     return base64.urlsafe_b64decode(self.key.pairs()[0][1].split('/')[4])
 
   @staticmethod
-  def _CreateKey(master_name, builder_name, build_number,
-                 step_name, test_name):  # pragma: no cover
-    return ndb.Key('MasterFlakeAnalysis',
-                   MasterFlakeAnalysis._CreateAnalysisId(
-                       master_name, builder_name, build_number,
-                       step_name, test_name))
+  def _CreateAnalysisId(
+      master_name, builder_name, build_number, step_name, test_name):
+    encoded_test_name = base64.urlsafe_b64encode(test_name)
+    return '%s/%s/%s/%s/%s' % (
+        master_name, builder_name, build_number, step_name, encoded_test_name)
 
-  @staticmethod
-  def Create(master_name, builder_name, build_number,
-             step_name, test_name):  # pragma: no cover
-    return MasterFlakeAnalysis(
-        key=MasterFlakeAnalysis._CreateKey(
-            master_name, builder_name, build_number,
-            step_name, test_name))
+  # Arguments number differs from overridden method - pylint: disable=W0221
+  @classmethod
+  def Create(cls, master_name, builder_name, build_number, step_name,
+             test_name):  # pragma: no cover.
+    return super(MasterFlakeAnalysis, cls).Create(
+        MasterFlakeAnalysis._CreateAnalysisId(
+            master_name, builder_name, build_number, step_name, test_name))
 
-  @staticmethod
-  def Get(master_name, builder_name, build_number,
-          step_name, test_name):  # pragma: no cover
-    return MasterFlakeAnalysis._CreateKey(
-        master_name, builder_name, build_number, step_name, test_name).get()
+  # Arguments number differs from overridden method - pylint: disable=W0221
+  @classmethod
+  def GetVersion(cls, master_name, builder_name, build_number, step_name,
+                 test_name, version=None):  # pragma: no cover.
+    return super(MasterFlakeAnalysis, cls).GetVersion(
+        key=MasterFlakeAnalysis._CreateAnalysisId(
+            master_name, builder_name, build_number, step_name, test_name),
+        version=version)
 
-  # List of tested build_numbers and their corresponding success rates.
-  # We need to keep these sorted manually.
-  build_numbers = ndb.IntegerProperty(indexed=False, repeated=True)
-  success_rates = ndb.FloatProperty(indexed=False, repeated=True)
+  def Reset(self):
+    self.created_time = time_util.GetUTCNow()
+    self.status = analysis_status.PENDING
+    self.completed_time = None
+    self.swarming_rerun_results = {}
+    self.error = None
+    self.correct_regression_range = None
+    self.correct_culprit = None
+    self.algorithm_parameters = None
+    self.suspected_flake_build_number = None
+    self.build_numbers = []
+    self.pass_rates = []
+
+  # The UTC timestamp this analysis was requested.
+  created_time = ndb.DateTimeProperty(indexed=True)
+
+  # The UTC timestamp this analysis was completed.
+  completed_time = ndb.DateTimeProperty(indexed=True)
+
+  # A dict containing information about each swarming rerun's results that were
+  # involved in this analysis. The contents of this dict will be used for
+  # metrics, such as the number of cache hits this analysis benefited from, the
+  # number of swarming tasks that were needed end-to-end to find the regressed
+  # build number (if any), etc.
+  #
+  # Example dict:
+  # {
+  #     task_id_1: {

[stgao, 2016/09/30 21:07:31]

Alternative to this is to create a metadata model, and use
LocalStructuredProperty here.
https://cloud.google.com/appengine/docs/python/ndb/entity-property-reference

[lijeffrey, 2016/10/01 01:28:04]

Done.

+  #         'request_time': 2016-09-06 (10:21:26.288) UTC
+  #         'start_time': 2016-09-06 (10:21:26.288) UTC,
+  #         'end_time': 2016-09-06 (10:21:26.288) UTC,
+  #         'build_number': 12345,
+  #         'cache_hit': cache_status (hit/miss/partial),
+  #         'number_of_iterations': 100,
+  #         'number_of_passes': 90,
+  #     },
+  #     task_id_2: {
+  #         ...
+  #     },
+  #     ...
+  # }
+  swarming_rerun_results = ndb.JsonProperty(
+      default={}, indexed=False, compressed=True)
+
+  # Error code and message, if any.
+  error = ndb.JsonProperty(indexed=False)
+
+  # Boolean whether the suspected regression range/build number is correct.
+  correct_regression_range = ndb.BooleanProperty(indexed=True)
+
+  # Boolean whether the suspected CL for found in the regression range
+  # is correct.
+  correct_culprit = ndb.BooleanProperty(indexed=True)
+
+  # The look back algorithm parameters that were used, as specified in Findit's
+  # configuration. For example,
+  # {
+  #     'iterations_to_rerun': 100,
+  #     'lower_flake_threshold': 0.02,
+  #     'max_build_numbers_to_look_back': 500,
+  #     'max_flake_in_a_row': 4,
+  #     'max_stable_in_a_row': 4,
+  #     'upper_flake_threshold': 0.98
+  # }
+  algorithm_parameters = ndb.JsonProperty(indexed=False)
+
+  # The suspected build number to have introduced the flakiness.
   suspected_flake_build_number = ndb.IntegerProperty()
+
+  # The build numbers that were examined to generate this run's flakiness graph.
+  # This list needs to be kept sorted manually.
+  build_numbers = ndb.IntegerProperty(indexed=False, repeated=True)
+
+  # The corresponding pass rates of build number's swarming rerun results.
+  # This list needs to be kept sorted manually.
+  pass_rates = ndb.FloatProperty(indexed=False, repeated=True)