Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(61)

Side by Side Diff: appengine/chromium_try_flakes/status/cq_status.py

Issue 1647983002: Fix spelling of the word occurrence (Closed) Base URL: https://chromium.googlesource.com/infra/infra.git@master
Patch Set: Created 4 years, 10 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « appengine/chromium_try_flakes/model/flake.py ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # Copyright 2014 The Chromium Authors. All rights reserved. 1 # Copyright 2014 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 import datetime 5 import datetime
6 import dateutil.parser 6 import dateutil.parser
7 import logging 7 import logging
8 import json 8 import json
9 import time 9 import time
10 import urllib2 10 import urllib2
11 11
12 from google.appengine.api import taskqueue 12 from google.appengine.api import taskqueue
13 from google.appengine.api import urlfetch 13 from google.appengine.api import urlfetch
14 from google.appengine.ext import deferred 14 from google.appengine.ext import deferred
15 from google.appengine.ext import ndb 15 from google.appengine.ext import ndb
16 from google.appengine.runtime import DeadlineExceededError 16 from google.appengine.runtime import DeadlineExceededError
17 from handlers.flake_issues import MIN_REQUIRED_FLAKY_RUNS 17 from handlers.flake_issues import MIN_REQUIRED_FLAKY_RUNS
18 from model.build_run import BuildRun 18 from model.build_run import BuildRun
19 from model.build_run import PatchsetBuilderRuns 19 from model.build_run import PatchsetBuilderRuns
20 from model.fetch_status import FetchStatus 20 from model.fetch_status import FetchStatus
21 from model.flake import Flake 21 from model.flake import Flake
22 from model.flake import FlakeOccurance 22 from model.flake import FlakeOccurrence
23 from model.flake import FlakyRun 23 from model.flake import FlakyRun
24 from status import build_result 24 from status import build_result
25 import time_functions.timestamp 25 import time_functions.timestamp
26 26
27 27
28 @ndb.transactional 28 @ndb.transactional
29 def get_patchset_builder_runs(issue, patchset, master, builder): 29 def get_patchset_builder_runs(issue, patchset, master, builder):
30 patchset_builder_runs_id = PatchsetBuilderRuns.getId(issue, patchset, master, 30 patchset_builder_runs_id = PatchsetBuilderRuns.getId(issue, patchset, master,
31 builder) 31 builder)
32 32
(...skipping 20 matching lines...) Expand all
53 53
54 def is_last_week(date): 54 def is_last_week(date):
55 return (datetime.datetime.utcnow() - date) < datetime.timedelta(weeks=1) 55 return (datetime.datetime.utcnow() - date) < datetime.timedelta(weeks=1)
56 56
57 57
58 def is_last_month(date): 58 def is_last_month(date):
59 return (datetime.datetime.utcnow() - date) < datetime.timedelta(days=31) 59 return (datetime.datetime.utcnow() - date) < datetime.timedelta(days=31)
60 60
61 61
62 # Updates a Flake object, which spans all the instances of one flake, with the 62 # Updates a Flake object, which spans all the instances of one flake, with the
63 # time of an occurance of that flake. 63 # time of an occurrence of that flake.
64 def add_occurance_time_to_flake(flake, occurance_time): # pragma: no cover 64 def add_occurrence_time_to_flake(flake, occurrence_time): # pragma: no cover
65 if occurance_time > flake.last_time_seen: 65 if occurrence_time > flake.last_time_seen:
66 flake.last_time_seen = occurance_time 66 flake.last_time_seen = occurrence_time
67 if is_last_hour(occurance_time): 67 if is_last_hour(occurrence_time):
68 flake.count_hour += 1 68 flake.count_hour += 1
69 flake.last_hour = True 69 flake.last_hour = True
70 if is_last_day(occurance_time): 70 if is_last_day(occurrence_time):
71 flake.count_day += 1 71 flake.count_day += 1
72 flake.last_day = True 72 flake.last_day = True
73 if is_last_week(occurance_time): 73 if is_last_week(occurrence_time):
74 flake.count_week += 1 74 flake.count_week += 1
75 flake.last_week = True 75 flake.last_week = True
76 if is_last_month(occurance_time): 76 if is_last_month(occurrence_time):
77 flake.count_month += 1 77 flake.count_month += 1
78 flake.last_month = True 78 flake.last_month = True
79 flake.count_all += 1 79 flake.count_all += 1
80 80
81 81
82 # Calculate the counters for a Flake object. 82 # Calculate the counters for a Flake object.
83 def update_flake_counters(flake): # pragma: no cover 83 def update_flake_counters(flake): # pragma: no cover
84 occurrences = ndb.get_multi(flake.occurrences) 84 occurrences = ndb.get_multi(flake.occurrences)
85 flake.count_hour = 0 85 flake.count_hour = 0
86 flake.count_day = 0 86 flake.count_day = 0
87 flake.count_week = 0 87 flake.count_week = 0
88 flake.count_month = 0 88 flake.count_month = 0
89 flake.count_all = 0 89 flake.count_all = 0
90 flake.last_hour = False 90 flake.last_hour = False
91 flake.last_day = False 91 flake.last_day = False
92 flake.last_week = False 92 flake.last_week = False
93 flake.last_month = False 93 flake.last_month = False
94 flake.last_time_seen = datetime.datetime.min 94 flake.last_time_seen = datetime.datetime.min
95 for o in occurrences: 95 for o in occurrences:
96 add_occurance_time_to_flake(flake, o.failure_run_time_finished) 96 add_occurrence_time_to_flake(flake, o.failure_run_time_finished)
97 flake.put() 97 flake.put()
98 98
99 99
100 # The following four functions are cron jobs which update the counters for 100 # The following four functions are cron jobs which update the counters for
101 # flakes. To speed things up, we don't update last month/week/day as often as we 101 # flakes. To speed things up, we don't update last month/week/day as often as we
102 # update hourly counters. 102 # update hourly counters.
103 def update_flake_hour_counter(): # pragma: no cover 103 def update_flake_hour_counter(): # pragma: no cover
104 query = Flake.query().filter(Flake.last_hour == True) 104 query = Flake.query().filter(Flake.last_hour == True)
105 for flake in query: 105 for flake in query:
106 update_flake_counters(flake) 106 update_flake_counters(flake)
(...skipping 42 matching lines...) Expand 10 before | Expand all | Expand 10 after
149 @ndb.transactional(xg=True) # pylint: disable=no-value-for-parameter 149 @ndb.transactional(xg=True) # pylint: disable=no-value-for-parameter
150 def add_failure_to_flake(name, flaky_run): 150 def add_failure_to_flake(name, flaky_run):
151 flake = Flake.get_by_id(name) 151 flake = Flake.get_by_id(name)
152 if not flake: 152 if not flake:
153 flake = Flake(name=name, id=name, last_time_seen=datetime.datetime.min) 153 flake = Flake(name=name, id=name, last_time_seen=datetime.datetime.min)
154 flake.put() 154 flake.put()
155 155
156 flake.occurrences.append(flaky_run.key) 156 flake.occurrences.append(flaky_run.key)
157 157
158 flaky_run_time = flaky_run.failure_run.get().time_finished 158 flaky_run_time = flaky_run.failure_run.get().time_finished
159 add_occurance_time_to_flake(flake, flaky_run_time) 159 add_occurrence_time_to_flake(flake, flaky_run_time)
160 160
161 flake.put() 161 flake.put()
162 162
163 # see examples: 163 # see examples:
164 # compile http://build.chromium.org/p/tryserver.chromium.mac/json/builders/ 164 # compile http://build.chromium.org/p/tryserver.chromium.mac/json/builders/
165 # mac_chromium_compile_dbg/builds/11167?as_text=1 165 # mac_chromium_compile_dbg/builds/11167?as_text=1
166 # gtest http://build.chromium.org/p/tryserver.chromium.win/json/builders/ 166 # gtest http://build.chromium.org/p/tryserver.chromium.win/json/builders/
167 # win_chromium_x64_rel_swarming/builds/4357?as_text=1 167 # win_chromium_x64_rel_swarming/builds/4357?as_text=1
168 # TODO(jam): get specific problem with compile so we can use that as name 168 # TODO(jam): get specific problem with compile so we can use that as name
169 # TODO(jam): It's unfortunate to have to parse this html. Can we get it from 169 # TODO(jam): It's unfortunate to have to parse this html. Can we get it from
(...skipping 116 matching lines...) Expand 10 before | Expand all | Expand 10 after
286 steps_to_ignore.append(other_step['name']) 286 steps_to_ignore.append(other_step['name'])
287 287
288 for step in failed_steps: 288 for step in failed_steps:
289 step_name = step['name'] 289 step_name = step['name']
290 if step_name in steps_to_ignore: 290 if step_name in steps_to_ignore:
291 continue 291 continue
292 flakes = get_flakes(step) 292 flakes = get_flakes(step)
293 if not flakes: 293 if not flakes:
294 continue 294 continue
295 for flake in flakes: 295 for flake in flakes:
296 flake_occurance = FlakeOccurance(name=step_name, failure=flake) 296 flake_occurrence = FlakeOccurrence(name=step_name, failure=flake)
297 flaky_run.flakes.append(flake_occurance) 297 flaky_run.flakes.append(flake_occurrence)
298 298
299 add_failure_to_flake(flake, flaky_run) 299 add_failure_to_flake(flake, flaky_run)
300 flaky_run.put() 300 flaky_run.put()
301 301
302 302
303 def get_int_value(properties, key): 303 def get_int_value(properties, key):
304 if not key in properties: 304 if not key in properties:
305 raise ValueError('key not found') 305 raise ValueError('key not found')
306 value = properties[key] 306 value = properties[key]
307 if type(value) == type(list()): 307 if type(value) == type(list()):
(...skipping 232 matching lines...) Expand 10 before | Expand all | Expand 10 after
540 else: 540 else:
541 fetch_status.begin = begin 541 fetch_status.begin = begin
542 fetch_status.end = end 542 fetch_status.end = end
543 fetch_status.cursor = cursor 543 fetch_status.cursor = cursor
544 fetch_status.put() 544 fetch_status.put()
545 545
546 if not more: 546 if not more:
547 return # finish the cron job and wait for next iteration 547 return # finish the cron job and wait for next iteration
548 except urllib2.URLError, e: 548 except urllib2.URLError, e:
549 logging.warning('Failed to fetch CQ status: %s', e.reason) 549 logging.warning('Failed to fetch CQ status: %s', e.reason)
OLDNEW
« no previous file with comments | « appengine/chromium_try_flakes/model/flake.py ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698