Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(381)

Side by Side Diff: appengine/findit/crash/crash_pipeline.py

Issue 2432203003: [Predator] Run predator. (Closed)
Patch Set: . Created 4 years, 1 month ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | appengine/findit/crash/culprit.py » ('j') | appengine/findit/crash/culprit.py » ('J')
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # Copyright 2016 The Chromium Authors. All rights reserved. 1 # Copyright 2016 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 import copy 5 import copy
6 import json 6 import json
7 import logging 7 import logging
8 8
9 from common import appengine_util 9 from common import appengine_util
10 from common import pubsub_util 10 from common import pubsub_util
11 from common import time_util 11 from common import time_util
12 from common.http_client_appengine import HttpClientAppengine 12 from common.http_client_appengine import HttpClientAppengine
13 from common.pipeline_wrapper import BasePipeline 13 from common.pipeline_wrapper import BasePipeline
14 from common.pipeline_wrapper import pipeline 14 from common.pipeline_wrapper import pipeline
15 from crash import findit_for_chromecrash 15 from crash import findit_for_chromecrash
16 from crash import findit_for_clusterfuzz 16 from crash import findit_for_clusterfuzz
17 from crash.type_enums import CrashClient 17 from crash.type_enums import CrashClient
18 from lib.gitiles import gitiles_repository 18 from lib.gitiles import gitiles_repository
19 from model import analysis_status 19 from model import analysis_status
20 20
21 21
22 # TODO(http://crbug.com/659346): write complete coverage tests for this. 22 # TODO(http://crbug.com/659346): write complete coverage tests for this.
23 def FinditForClientID(client_id): # pragma: no cover 23 def FinditForClientID(client_id, repository): # pragma: no cover
24 """Construct a Findit object from a client id string specifying the class. 24 """Construct a Findit object from a client id string specifying the class.
25 25
26 We cannot pass Findit objects to the various methods in 26 We cannot pass Findit objects to the various methods in
27 ``crash.crash_pipeline``, because they are not JSON serializable. For 27 ``crash.crash_pipeline``, because they are not JSON serializable. For
28 now, we just serialize Findit objects as their ``client_id``, and then 28 now, we just serialize Findit objects as their ``client_id``, and then
29 use this function to reconstruct them. Alas, this means we will lose 29 use this function to reconstruct them. Alas, this means we will lose
30 various other information stored in the Findit object (i.e., stuff that 30 various other information stored in the Findit object (i.e., stuff that
31 comes from CrashConfig); which could lead to some hard-to-diagnose 31 comes from CrashConfig); which could lead to some hard-to-diagnose
32 coherency bugs, since the new Findit object will be based on the 32 coherency bugs, since the new Findit object will be based on the
33 CrashConfig at the time it's constructed, which may be different 33 CrashConfig at the time it's constructed, which may be different
34 than the CrashConfig at the time the previous Findit object was 34 than the CrashConfig at the time the previous Findit object was
35 constructed. In the future we should fix all this to serialize Findit 35 constructed. In the future we should fix all this to serialize Findit
36 objects in a more robust way. 36 objects in a more robust way.
37 """ 37 """
38 assert isinstance(client_id, (str, unicode)), ( 38 assert isinstance(client_id, (str, unicode)), (
39 'FinditForClientID: expected string or unicode, but got %s' 39 'FinditForClientID: expected string or unicode, but got %s'
40 % client_id.__class__.__name__) 40 % client_id.__class__.__name__)
41 # TODO(wrengr): it'd be nice to replace this with a single lookup in 41 # TODO(wrengr): it'd be nice to replace this with a single lookup in
42 # a dict; but that's buggy for some unknown reason. 42 # a dict; but that's buggy for some unknown reason.
43 if client_id == CrashClient.FRACAS: 43 if client_id == CrashClient.FRACAS:
44 cls = findit_for_chromecrash.FinditForFracas 44 cls = findit_for_chromecrash.FinditForFracas
45 elif client_id == CrashClient.CRACAS: 45 elif client_id == CrashClient.CRACAS:
46 cls = findit_for_chromecrash.FinditForCracas 46 cls = findit_for_chromecrash.FinditForCracas
47 elif client_id == CrashClient.CLUSTERFUZZ: 47 elif client_id == CrashClient.CLUSTERFUZZ:
48 cls = findit_for_clusterfuzz.FinditForClusterfuzz 48 cls = findit_for_clusterfuzz.FinditForClusterfuzz
49 else: 49 else:
50 raise ValueError('FinditForClientID: ' 50 raise ValueError('FinditForClientID: '
51 'unknown or unsupported client %s' % client_id) 51 'unknown or unsupported client %s' % client_id)
52 52
53 return cls( 53 return cls(repository, CrashWrapperPipeline)
54 gitiles_repository.GitilesRepository(http_client=HttpClientAppengine()),
55 CrashWrapperPipeline)
56 54
57 55
58 # Some notes about the classes below, for people who are not 56 # Some notes about the classes below, for people who are not
59 # familiar with AppEngine. The thing that really kicks everything off 57 # familiar with AppEngine. The thing that really kicks everything off
60 # is ``CrashWrapperPipeline.run``. However, an important thing to bear in 58 # is ``CrashWrapperPipeline.run``. However, an important thing to bear in
61 # mind is that whatever arguments are passed to that method will also 59 # mind is that whatever arguments are passed to that method will also
62 # be passed to the ``run`` method on whatever objects it yields. Thus, 60 # be passed to the ``run`` method on whatever objects it yields. Thus,
63 # all the ``run`` methods across these different classes must have 61 # all the ``run`` methods across these different classes must have
64 # the same type. In practice, we end up passing all the arguments to the 62 # the same type. In practice, we end up passing all the arguments to the
65 # constructors, because we need to have the fields around for logging 63 # constructors, because we need to have the fields around for logging
66 # (e.g., in ``finalized``); thus, there's nothing that needs to be passed 64 # (e.g., in ``finalized``); thus, there's nothing that needs to be passed
67 # to ``run``. Another thing to bear in mind is that whatever objects 65 # to ``run``. Another thing to bear in mind is that whatever objects
68 # ``CrashWrapperPipeline.run`` yields must be JSON-serializable. The base 66 # ``CrashWrapperPipeline.run`` yields must be JSON-serializable. The base
69 # class handles most of that for us, so the force of this constraint is 67 # class handles most of that for us, so the force of this constraint is
70 # that all the arguments to the constructors for those classes must be 68 # that all the arguments to the constructors for those classes must be
71 # JSON-serializable. Thus, we cannot actually pass a Findit object to 69 # JSON-serializable. Thus, we cannot actually pass a Findit object to
72 # the constructor, but rather must pass only the ``client_id`` (or whatever 70 # the constructor, but rather must pass only the ``client_id`` (or whatever
73 # JSON dict) and then reconstruct the Findit object from that. Moreover, 71 # JSON dict) and then reconstruct the Findit object from that. Moreover,
74 # the ``run`` method and the ``finalized`` method will be run in different 72 # the ``run`` method and the ``finalized`` method will be run in different
75 # processes, so we will actually end up reconstructing the Findit object 73 # processes, so we will actually end up reconstructing the Findit object
76 # twice. Thus, we shouldn't store anything in the pipeline objects outside 74 # twice. Thus, we shouldn't store anything in the pipeline objects outside
77 # of what their constructors store. 75 # of what their constructors store.
78 76
79 class CrashBasePipeline(BasePipeline): 77 class CrashBasePipeline(BasePipeline):
80 def __init__(self, client_id, crash_identifiers): 78 def __init__(self, client_id, crash_identifiers):
81 super(CrashBasePipeline, self).__init__(client_id, crash_identifiers) 79 super(CrashBasePipeline, self).__init__(client_id, crash_identifiers)
82 self._crash_identifiers = crash_identifiers 80 self._crash_identifiers = crash_identifiers
83 self._findit = FinditForClientID(client_id) 81 self._findit = FinditForClientID(
82 client_id,
83 gitiles_repository.GitilesRepository(http_client=HttpClientAppengine()))
84 84
85 @property 85 @property
86 def client_id(self): # pragma: no cover 86 def client_id(self): # pragma: no cover
87 return self._findit.client_id 87 return self._findit.client_id
88 88
89 def run(self, *args, **kwargs): 89 def run(self, *args, **kwargs):
90 raise NotImplementedError() 90 raise NotImplementedError()
91 91
92 92
93 class CrashAnalysisPipeline(CrashBasePipeline): 93 class CrashAnalysisPipeline(CrashBasePipeline):
(...skipping 89 matching lines...) Expand 10 before | Expand all | Expand 10 after
183 self._crash_identifiers = crash_identifiers 183 self._crash_identifiers = crash_identifiers
184 self._client_id = client_id 184 self._client_id = client_id
185 185
186 # TODO(http://crbug.com/659346): write coverage tests. 186 # TODO(http://crbug.com/659346): write coverage tests.
187 # Arguments number differs from overridden method - pylint: disable=W0221 187 # Arguments number differs from overridden method - pylint: disable=W0221
188 def run(self): # pragma: no cover 188 def run(self): # pragma: no cover
189 run_analysis = yield CrashAnalysisPipeline( 189 run_analysis = yield CrashAnalysisPipeline(
190 self._client_id, self._crash_identifiers) 190 self._client_id, self._crash_identifiers)
191 with pipeline.After(run_analysis): 191 with pipeline.After(run_analysis):
192 yield PublishResultPipeline(self._client_id, self._crash_identifiers) 192 yield PublishResultPipeline(self._client_id, self._crash_identifiers)
OLDNEW
« no previous file with comments | « no previous file | appengine/findit/crash/culprit.py » ('j') | appengine/findit/crash/culprit.py » ('J')

Powered by Google App Engine
This is Rietveld 408576698