Waterfall components of regression range finder.

appengine/findit/waterfall/flake/recursive_flake_pipeline.py

+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from datetime import datetime
+import logging
+
+from common import appengine_util
+from common import constants
+from common.pipeline_wrapper import BasePipeline
+
+from model import analysis_status
+from model.flake.master_flake_analysis import MasterFlakeAnalysis as MFA

[stgao, 2016/07/09 00:04:33]

I understand the class name is a bit long, but the convention in the code base
is to avoid abbreviations because they could cause confusion.

Same for lines #15 and #17.

[caiw, 2016/07/14 00:59:44]

Done.

+from waterfall.trigger_flake_swarming_task_pipeline import (
+    TriggerFlakeSwarmingTaskPipeline as TFSTP)
+from waterfall.process_flake_swarming_task_result_pipeline import (
+    ProcessFlakeSwarmingTaskResultPipeline as PFSTRP)
+
+
+class RecursiveFlakePipeline(BasePipeline):
+
+  def __init__(self, master_name, builder_name,
+               step_name, build_number, testcase):
+    super(RecursiveFlakePipeline, self).__init__(
+        master_name, builder_name, step_name, build_number, testcase)
+    self.master_name = master_name

[stgao, 2016/07/09 00:04:33]

Why do we need to save these values? they seems not used.

+    self.builder_name = builder_name
+    self.step_name = step_name
+    self.build_number = build_number
+    self.testcase = testcase
+    self.step_future = None
+    self.build_completed = None
+    self.target = None
+
+  # Arguments number differs from overridden method - pylint: disable=W0221
+  def run(self, master_name, builder_name, step_name, build_number,
+          testcase, queue_name=constants.DEFAULT_QUEUE):
+    # Call trigger pipeline (flake style)
+    task_id = yield TFSTP(master_name, builder_name,
+                          build_number, step_name, [testcase])
+    # Pass the trigger pipeline into a process pipeline (flake style)
+    step_future = yield PFSTRP(

[stgao, 2016/07/09 00:04:33]

rename ``step_future`` to "test_result_future``?

[caiw, 2016/07/14 00:59:44]

Done.

+        master_name, builder_name, build_number, step_name, task_id)
+    yield NextBuildNumberPipeline(
+        master_name, builder_name, step_name, testcase, step_future, queue_name)
+
+class NextBuildNumberPipeline(BasePipeline):

[chanli, 2016/07/08 16:58:54]

Any reason you wrote these 2 classes in the same file?

[caiw, 2016/07/14 00:59:44]

Is there a convention of one class per file?  I thought this made sense because
nextbuildnumberpipeline is sort of a helper function of recursiveflakepipeline,
except it has to be a pipeline because of the appengine.

+  # Arguments number differs from overridden method - pylint: disable=W0221
+  # Unused argument - pylint: disable=W0613
+  def run(self, master_name, builder_name, step_name,
+          testcase, step_future, queue_name):
+    # Get MFA success list corresponding to parameters
+    master = MFA.Get(master_name, builder_name, step_name)
+    # From the list, figure out what build_number we should call, if any
+    # This is a placeholder for testing:
+    next_run = False
+    if len(master.build_numbers) < 10:
+      # Placeholder until we develop an algorithm

[stgao, 2016/07/09 00:04:33]

Add a TODO here?

[caiw, 2016/07/14 00:59:44]

Done.

+      next_run = min(master.build_numbers) - 10
+    if next_run:
+      logging.info("Calling recursive pipeline again.")
+      pipeline_job = RecursiveFlakePipeline(
+          master_name, builder_name, step_name,
+          next_run, testcase)
+      pipeline_job.target = appengine_util.GetTargetNameForModule(
+          constants.WATERFALL_BACKEND)
+      pipeline_job.start(queue_name=queue_name)