appengine/findit_copy/.tmp/coverage/appengine_findit_waterfall_trigger_swarming_task_pipeline.html - Issue 2003423003: Merge branch 'blacklist_paths' into only_analyze_top_n_frames

Unified Diff: appengine/findit_copy/.tmp/coverage/appengine_findit_waterfall_trigger_swarming_task_pipeline.html

Issue 2003423003: Merge branch 'blacklist_paths' into only_analyze_top_n_frames Base URL: https://chromium.googlesource.com/infra/infra.git@blacklist_paths

Patch Set: Created 4 years, 7 months ago

Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.

Jump to:

View side-by-side diff with in-line comments

Download patch

« no previous file with comments | « appengine/findit_copy/.tmp/coverage/appengine_findit_waterfall_test_waterfall_config_test.html ('k') | appengine/findit_copy/.tmp/coverage/appengine_findit_waterfall_try_job_pipeline.html » ('j') | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Hide Comments ('s')

Index: appengine/findit_copy/.tmp/coverage/appengine_findit_waterfall_trigger_swarming_task_pipeline.html

diff --git a/appengine/findit_copy/.tmp/coverage/appengine_findit_waterfall_trigger_swarming_task_pipeline.html b/appengine/findit_copy/.tmp/coverage/appengine_findit_waterfall_trigger_swarming_task_pipeline.html

new file mode 100644

index 0000000000000000000000000000000000000000..5fb145a2e6a6fd3692de362b5f354894213b4c83

--- /dev/null

+++ b/appengine/findit_copy/.tmp/coverage/appengine_findit_waterfall_trigger_swarming_task_pipeline.html

@@ -0,0 +1,412 @@

+<!doctype html PUBLIC "-//W3C//DTD html 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">

+<html>

+<head>

+ <meta http-equiv='Content-Type' content='text/html; charset=utf-8'>

+ <meta http-equiv='X-UA-Compatible' content='IE=emulateIE7' />

+ <title>Coverage for appengine/findit/waterfall/trigger_swarming_task_pipeline: 100%</title>

+ <link rel='stylesheet' href='style.css' type='text/css'>

+ <script type='text/javascript' src='jquery.min.js'></script>

+ <script type='text/javascript' src='jquery.hotkeys.js'></script>

+ <script type='text/javascript' src='jquery.isonscreen.js'></script>

+ <script type='text/javascript' src='coverage_html.js'></script>

+ <script type='text/javascript' charset='utf-8'>

+ jQuery(document).ready(coverage.pyfile_ready);

+ </script>

+</head>

+<body id='pyfile'>

+<div id='header'>

+ <div class='content'>

+ <h1>Coverage for appengine/findit/waterfall/trigger_swarming_task_pipeline :

+ 100%

+ </h1>

+ <img id='keyboard_icon' src='keybd_closed.png'>

+ <h2 class='stats'>

+ 68 statements  

+ 68 run

+ 0 missing

+ 6 excluded

+ 0 partial

+ </h2>

+ </div>

+</div>

+<div class='help_panel'>

+ <img id='panel_icon' src='keybd_open.png'>

+Hot-keys on this page

+ <div>

+

+ r

+ m

+ x

+ p   toggle line displays

+

+

+ j

+ k   next/prev highlighted chunk

+

+

+ 0   (zero) top of page

+

+

+ 1   (one) first highlighted chunk

+

+ </div>

+</div>

+<div id='source'>

+ <table cellspacing='0' cellpadding='0'>

+ <tr>

+ <td class='linenos' valign='top'>

+<a href='#n1'>1</a>

+<a href='#n2'>2</a>

+<a href='#n3'>3</a>

+<a href='#n4'>4</a>

+<a href='#n5'>5</a>

+<a href='#n6'>6</a>

+<a href='#n7'>7</a>

+<a href='#n8'>8</a>

+<a href='#n9'>9</a>

+<a href='#n10'>10</a>

+<a href='#n11'>11</a>

+<a href='#n12'>12</a>

+<a href='#n13'>13</a>

+<a href='#n14'>14</a>

+<a href='#n15'>15</a>

+<a href='#n16'>16</a>

+<a href='#n17'>17</a>

+<a href='#n18'>18</a>

+<a href='#n19'>19</a>

+<a href='#n20'>20</a>

+<a href='#n21'>21</a>

+<a href='#n22'>22</a>

+<a href='#n23'>23</a>

+<a href='#n24'>24</a>

+<a href='#n25'>25</a>

+<a href='#n26'>26</a>

+<a href='#n27'>27</a>

+<a href='#n28'>28</a>

+<a href='#n29'>29</a>

+<a href='#n30'>30</a>

+<a href='#n31'>31</a>

+<a href='#n32'>32</a>

+<a href='#n33'>33</a>

+<a href='#n34'>34</a>

+<a href='#n35'>35</a>

+<a href='#n36'>36</a>

+<a href='#n37'>37</a>

+<a href='#n38'>38</a>

+<a href='#n39'>39</a>

+<a href='#n40'>40</a>

+<a href='#n41'>41</a>

+<a href='#n42'>42</a>

+<a href='#n43'>43</a>

+<a href='#n44'>44</a>

+<a href='#n45'>45</a>

+<a href='#n46'>46</a>

+<a href='#n47'>47</a>

+<a href='#n48'>48</a>

+<a href='#n49'>49</a>

+<a href='#n50'>50</a>

+<a href='#n51'>51</a>

+<a href='#n52'>52</a>

+<a href='#n53'>53</a>

+<a href='#n54'>54</a>

+<a href='#n55'>55</a>

+<a href='#n56'>56</a>

+<a href='#n57'>57</a>

+<a href='#n58'>58</a>

+<a href='#n59'>59</a>

+<a href='#n60'>60</a>

+<a href='#n61'>61</a>

+<a href='#n62'>62</a>

+<a href='#n63'>63</a>

+<a href='#n64'>64</a>

+<a href='#n65'>65</a>

+<a href='#n66'>66</a>

+<a href='#n67'>67</a>

+<a href='#n68'>68</a>

+<a href='#n69'>69</a>

+<a href='#n70'>70</a>

+<a href='#n71'>71</a>

+<a href='#n72'>72</a>

+<a href='#n73'>73</a>

+<a href='#n74'>74</a>

+<a href='#n75'>75</a>

+<a href='#n76'>76</a>

+<a href='#n77'>77</a>

+<a href='#n78'>78</a>

+<a href='#n79'>79</a>

+<a href='#n80'>80</a>

+<a href='#n81'>81</a>

+<a href='#n82'>82</a>

+<a href='#n83'>83</a>

+<a href='#n84'>84</a>

+<a href='#n85'>85</a>

+<a href='#n86'>86</a>

+<a href='#n87'>87</a>

+<a href='#n88'>88</a>

+<a href='#n89'>89</a>

+<a href='#n90'>90</a>

+<a href='#n91'>91</a>

+<a href='#n92'>92</a>

+<a href='#n93'>93</a>

+<a href='#n94'>94</a>

+<a href='#n95'>95</a>

+<a href='#n96'>96</a>

+<a href='#n97'>97</a>

+<a href='#n98'>98</a>

+<a href='#n99'>99</a>

+<a href='#n100'>100</a>

+<a href='#n101'>101</a>

+<a href='#n102'>102</a>

+<a href='#n103'>103</a>

+<a href='#n104'>104</a>

+<a href='#n105'>105</a>

+<a href='#n106'>106</a>

+<a href='#n107'>107</a>

+<a href='#n108'>108</a>

+<a href='#n109'>109</a>

+<a href='#n110'>110</a>

+<a href='#n111'>111</a>

+<a href='#n112'>112</a>

+<a href='#n113'>113</a>

+<a href='#n114'>114</a>

+<a href='#n115'>115</a>

+<a href='#n116'>116</a>

+<a href='#n117'>117</a>

+<a href='#n118'>118</a>

+<a href='#n119'>119</a>

+<a href='#n120'>120</a>

+<a href='#n121'>121</a>

+<a href='#n122'>122</a>

+<a href='#n123'>123</a>

+<a href='#n124'>124</a>

+<a href='#n125'>125</a>

+<a href='#n126'>126</a>

+<a href='#n127'>127</a>

+<a href='#n128'>128</a>

+<a href='#n129'>129</a>

+<a href='#n130'>130</a>

+<a href='#n131'>131</a>

+<a href='#n132'>132</a>

+<a href='#n133'>133</a>

+<a href='#n134'>134</a>

+<a href='#n135'>135</a>

+<a href='#n136'>136</a>

+<a href='#n137'>137</a>

+<a href='#n138'>138</a>

+<a href='#n139'>139</a>

+<a href='#n140'>140</a>

+<a href='#n141'>141</a>

+<a href='#n142'>142</a>

+<a href='#n143'>143</a>

+<a href='#n144'>144</a>

+<a href='#n145'>145</a>

+<a href='#n146'>146</a>

+<a href='#n147'>147</a>

+<a href='#n148'>148</a>

+<a href='#n149'>149</a>

+<a href='#n150'>150</a>

+<a href='#n151'>151</a>

+<a href='#n152'>152</a>

+<a href='#n153'>153</a>

+<a href='#n154'>154</a>

+<a href='#n155'>155</a>

+<a href='#n156'>156</a>

+<a href='#n157'>157</a>

+<a href='#n158'>158</a>

+<a href='#n159'>159</a>

+<a href='#n160'>160</a>

+<a href='#n161'>161</a>

+<a href='#n162'>162</a>

+<a href='#n163'>163</a>

+<a href='#n164'>164</a>

+ </td>

+ <td class='text' valign='top'>

+# Use of this source code is governed by a BSD-style license that can be

+# found in the LICENSE file.

+

+import copy

+from datetime import datetime

+import logging

+import time

+

+from google.appengine.ext import ndb

+

+from common.http_client_appengine import HttpClientAppengine as HttpClient

+from model import wf_analysis_status

+from model.wf_swarming_task import WfSwarmingTask

+from pipeline_wrapper import BasePipeline

+from waterfall import swarming_util

+from waterfall import waterfall_config

+

+

+def _GetSwarmingTaskName(ref_task_id):  # pragma: no cover.

+  """Returns a unique task name.

+

+  Have this separate function in order to mock for testing purpose.

+  """

+  return 'findit/deflake/ref_task_id/%s/%s' % (

+      ref_task_id, datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S %f'))

+

+

+def _CreateNewSwarmingTaskRequest(

+    ref_task_id, ref_request, master_name, builder_name, build_number,

+    step_name, tests, iterations):

+  """Returns a SwarmingTaskRequest instance to run the given tests only."""

+  # Make a copy of the referred request and drop or overwrite some fields.

+  new_request = copy.deepcopy(ref_request)

+  new_request.name = _GetSwarmingTaskName(ref_task_id)

+  new_request.parent_task_id = ''

+  new_request.user = ''

+

+  # To force a fresh re-run and ignore cached result of any equivalent run.

+  new_request.idempotent = False

+

+  # Set the gtest_filter to run the given tests only.

+  new_request.extra_args.append('--gtest_repeat=%s' % iterations)

+  new_request.extra_args.append('--test-launcher-retry-limit=0')

+  new_request.extra_args = [

+      a for a in new_request.extra_args if not a.startswith('--gtest_filter')

+  ]

+  new_request.extra_args.append('--gtest_filter=%s' % ':'.join(tests))

+

+  # Remove the env setting for sharding.

+  sharding_settings = ['GTEST_SHARD_INDEX', 'GTEST_TOTAL_SHARDS']

+  new_request.env = [

+      e for e in new_request.env if e['key'] not in sharding_settings

+  ]

+

+  # Reset tags for searching and monitoring.

+  ref_name = swarming_util.GetTagValue(ref_request.tags, 'name')

+  new_request.tags = []

+  new_request.tags.append('purpose:deflake')

+  new_request.tags.append('ref_master:%s' % master_name)

+  new_request.tags.append('ref_buildername:%s' % builder_name)

+  new_request.tags.append('ref_buildnumber:%s' % build_number)

+  new_request.tags.append('ref_stepname:%s' % step_name)

+  new_request.tags.append('ref_task_id:%s' % ref_task_id)

+  new_request.tags.append('ref_name:%s' % ref_name)

+

+  return new_request

+

+

+@ndb.transactional

+def _NeedANewSwarmingTask(master_name, builder_name, build_number, step_name):

+  swarming_task = WfSwarmingTask.Get(

+      master_name, builder_name, build_number, step_name)

+

+  if not swarming_task:

+    swarming_task = WfSwarmingTask.Create(

+        master_name, builder_name, build_number, step_name)

+    swarming_task.status = wf_analysis_status.PENDING

+    swarming_task.put()

+    return True

+  else:

+    # TODO(http://crbug.com/585676): Rerun the Swarming task if it runs into

+    # unexpected infra errors.

+    return False

+

+

+def _GetSwarmingTaskId(master_name, builder_name, build_number, step_name):

+  deadline = time.time() + 5 * 60  # Wait for 5 minutes.

+  while time.time() < deadline:

+    swarming_task = WfSwarmingTask.Get(

+        master_name, builder_name, build_number, step_name)

+

+    if not swarming_task:  # pragma: no cover. Pipeline will retry.

+      raise Exception('Swarming task was deleted unexpectedly!!!')

+

+    if swarming_task.task_id:

+      return swarming_task.task_id

+

+    time.sleep(10)  # Wait for the existing pipeline to start the Swarming task.

+

+  raise Exception('Time out!')  # pragma: no cover. Pipeline will retry.

+

+

+class TriggerSwarmingTaskPipeline(BasePipeline):

+  """A pipeline to trigger a Swarming task to re-run selected tests of a step.

+

+  This pipeline only supports test steps that run on Swarming and support the

+  gtest filter.

+  """

+

+  # Arguments number differs from overridden method - pylint: disable=W0221

+  def run(self, master_name, builder_name, build_number, step_name, tests):

+    """Triggers a new Swarming task to run the given tests.

+

+    Args:

+      master_name (str): The master name.

+      builder_name (str): The builder name.

+      build_number (str): The build number.

+      step_name (str): The failed test step name.

+      tests (list): A list of test cases, eg: ['suite1.test1', 'suite2.test2'].

+

+    Returns:

+      task_id (str): The new Swarming task that re-run the given tests.

+    """

+    # Check if a new Swarming Task is really needed.

+    if not _NeedANewSwarmingTask(

+        master_name, builder_name, build_number, step_name):

+      return _GetSwarmingTaskId(

+          master_name, builder_name, build_number, step_name)

+

+    assert tests

+

+    http_client = HttpClient()

+

+    # 0. Retrieve existing Swarming task ids for the given step.

+    swarming_task_items = swarming_util.ListSwarmingTasksDataByTags(

+        master_name, builder_name, build_number, http_client, step_name)

+    assert len(swarming_task_items) > 0, 'No Swarming task was run.'

+    ref_task_id = swarming_task_items[0]['task_id']

+

+    # 1. Retrieve Swarming task parameters from a given Swarming task id.

+    ref_request = swarming_util.GetSwarmingTaskRequest(

+        ref_task_id, http_client)

+

+    # 2. Update/Overwrite parameters for the re-run.

+    iterations_to_rerun = waterfall_config.GetSwarmingSettings().get(

+        'iterations_to_rerun')

+    new_request = _CreateNewSwarmingTaskRequest(

+        ref_task_id, ref_request, master_name, builder_name, build_number,

+        step_name, tests, iterations_to_rerun)

+

+    # 3. Trigger a new Swarming task to re-run the failed tests.

+    task_id = swarming_util.TriggerSwarmingTask(new_request, http_client)

+

+    # Save the task id.

+    swarming_task = WfSwarmingTask.Get(

+        master_name, builder_name, build_number, step_name)

+    swarming_task.task_id = task_id

+    swarming_task.parameters['tests'] = tests

+    swarming_task.parameters['iterations_to_rerun'] = iterations_to_rerun

+    swarming_task.put()

+

+    logging.info('A Swarming task was triggered:%s', task_id)

+    return task_id

+ </td>

+ </tr>

+ </table>

+</div>

+<div id='footer'>

+ <div class='content'>

+

+ <a class='nav' href='index.html'>« index</a>     <a class='nav' href='http://nedbatchelder.com/code/coverage'>coverage.py v3.7.1</a>

+

+ </div>

+</div>

+</body>

+</html>