OLD | NEW |
---|---|
1 # Copyright 2016 The Chromium Authors. All rights reserved. | 1 # Copyright 2016 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 import copy | 5 import copy |
6 import json | 6 import json |
7 import logging | 7 import logging |
8 | 8 |
9 from crash import monitoring | 9 from crash import monitoring |
10 | 10 |
11 from common import appengine_util | 11 from common import appengine_util |
12 from common import pubsub_util | 12 from common import pubsub_util |
13 from common.http_client_appengine import HttpClientAppengine | 13 from common.http_client_appengine import HttpClientAppengine |
14 from common.pipeline_wrapper import BasePipeline | 14 from common.pipeline_wrapper import BasePipeline |
15 from common.pipeline_wrapper import pipeline | 15 from common.pipeline_wrapper import pipeline |
16 from crash import findit_for_chromecrash | 16 from crash import findit_for_chromecrash |
17 from crash import findit_for_clusterfuzz | 17 from crash import findit_for_clusterfuzz |
18 from crash.type_enums import CrashClient | 18 from crash.type_enums import CrashClient |
19 from libs import time_util | 19 from libs import time_util |
20 from gae_libs.gitiles import cached_gitiles_repository | 20 from gae_libs.gitiles import cached_gitiles_repository |
21 from model import analysis_status | 21 from model import analysis_status |
22 | 22 |
23 | 23 |
24 # TODO(http://crbug.com/659346): write complete coverage tests for this. | 24 # TODO(http://crbug.com/659346): write complete coverage tests for this. |
25 def FinditForClientID(client_id, repository): # pragma: no cover | 25 def FinditForClientID(client_id, get_repository): # pragma: no cover |
26 """Construct a Findit object from a client id string specifying the class. | 26 """Construct a Findit object from a client id string specifying the class. |
27 | 27 |
28 We cannot pass Findit objects to the various methods in | 28 We cannot pass Findit objects to the various methods in |
29 ``crash.crash_pipeline``, because they are not JSON serializable. For | 29 ``crash.crash_pipeline``, because they are not JSON serializable. For |
30 now, we just serialize Findit objects as their ``client_id``, and then | 30 now, we just serialize Findit objects as their ``client_id``, and then |
31 use this function to reconstruct them. Alas, this means we will lose | 31 use this function to reconstruct them. Alas, this means we will lose |
32 various other information stored in the Findit object (i.e., stuff that | 32 various other information stored in the Findit object (i.e., stuff that |
33 comes from CrashConfig); which could lead to some hard-to-diagnose | 33 comes from CrashConfig); which could lead to some hard-to-diagnose |
34 coherency bugs, since the new Findit object will be based on the | 34 coherency bugs, since the new Findit object will be based on the |
35 CrashConfig at the time it's constructed, which may be different | 35 CrashConfig at the time it's constructed, which may be different |
36 than the CrashConfig at the time the previous Findit object was | 36 than the CrashConfig at the time the previous Findit object was |
37 constructed. In the future we should fix all this to serialize Findit | 37 constructed. In the future we should fix all this to serialize Findit |
38 objects in a more robust way. | 38 objects in a more robust way. |
39 """ | 39 """ |
40 assert isinstance(client_id, (str, unicode)), ( | 40 assert isinstance(client_id, (str, unicode)), ( |
41 'FinditForClientID: expected string or unicode, but got %s' | 41 'FinditForClientID: expected string or unicode, but got %s' |
42 % client_id.__class__.__name__) | 42 % client_id.__class__.__name__) |
43 # TODO(wrengr): it'd be nice to replace this with a single lookup in | 43 # TODO(wrengr): it'd be nice to replace this with a single lookup in |
44 # a dict; but that's buggy for some unknown reason. | 44 # a dict; but that's buggy for some unknown reason. |
45 if client_id == CrashClient.FRACAS: | 45 if client_id == CrashClient.FRACAS: |
46 cls = findit_for_chromecrash.FinditForFracas | 46 cls = findit_for_chromecrash.FinditForFracas |
47 elif client_id == CrashClient.CRACAS: # pragma: no cover | 47 elif client_id == CrashClient.CRACAS: # pragma: no cover |
48 cls = findit_for_chromecrash.FinditForCracas | 48 cls = findit_for_chromecrash.FinditForCracas |
49 elif client_id == CrashClient.CLUSTERFUZZ: # pragma: no cover | 49 elif client_id == CrashClient.CLUSTERFUZZ: # pragma: no cover |
50 cls = findit_for_clusterfuzz.FinditForClusterfuzz | 50 cls = findit_for_clusterfuzz.FinditForClusterfuzz |
51 else: # pragma: no cover | 51 else: # pragma: no cover |
52 raise ValueError('FinditForClientID: ' | 52 raise ValueError('FinditForClientID: ' |
53 'unknown or unsupported client %s' % client_id) | 53 'unknown or unsupported client %s' % client_id) |
54 | 54 |
55 return cls(repository) | 55 return cls(get_repository) |
56 | 56 |
57 | 57 |
58 # Some notes about the classes below, for people who are not familiar | 58 # Some notes about the classes below, for people who are not familiar |
59 # with AppEngine pipelines: | 59 # with AppEngine pipelines: |
60 # | 60 # |
61 # The pipeline library is designed in a strange way which requires that | 61 # The pipeline library is designed in a strange way which requires that |
62 # all the ``__init__`` and ``run`` methods in this file take the exact | 62 # all the ``__init__`` and ``run`` methods in this file take the exact |
63 # same arguments. This arises from the interaction between a few | 63 # same arguments. This arises from the interaction between a few |
64 # different constraints. First, for any given pipeline, its ``__init__`` | 64 # different constraints. First, for any given pipeline, its ``__init__`` |
65 # and ``run`` must take the same arguments. Second, all the objects that | 65 # and ``run`` must take the same arguments. Second, all the objects that |
(...skipping 14 matching lines...) Expand all Loading... | |
80 # serialization. Thus, we cannot pass a ``Findit`` object directly to | 80 # serialization. Thus, we cannot pass a ``Findit`` object directly to |
81 # any of the methods here, but rather must instead pass the ``client_id`` | 81 # any of the methods here, but rather must instead pass the ``client_id`` |
82 # (or whatever JSON dict), and then reconstruct the ``Findit`` object | 82 # (or whatever JSON dict), and then reconstruct the ``Findit`` object |
83 # from that data. | 83 # from that data. |
84 # | 84 # |
85 # Moreover, the ``run`` and ``finalized`` methods are executed in separate | 85 # Moreover, the ``run`` and ``finalized`` methods are executed in separate |
86 # processes, so we'll actually end up reconstructing the ``Findit`` object | 86 # processes, so we'll actually end up reconstructing the ``Findit`` object |
87 # twice. This also means ``run`` can't store anything in the pipeline | 87 # twice. This also means ``run`` can't store anything in the pipeline |
88 # object and expect it to still be available in the ``finalized`` method. | 88 # object and expect it to still be available in the ``finalized`` method. |
89 | 89 |
90 def _CachedGitilesRepositoryFactory(repo_url): # pragma: no cover | |
Sharu Jiang
2016/12/28 23:05:12
what's the reason for ``pragma: no cover`` here?
wrengr
2016/12/29 20:57:31
Apparently it's never called by the unit tests ¯\_
| |
91 return cached_gitiles_repository.CachedGitilesRepository( | |
92 HttpClientAppengine(), repo_url) | |
93 | |
94 | |
90 class CrashBasePipeline(BasePipeline): | 95 class CrashBasePipeline(BasePipeline): |
91 def __init__(self, client_id, crash_identifiers): | 96 def __init__(self, client_id, crash_identifiers): |
92 super(CrashBasePipeline, self).__init__(client_id, crash_identifiers) | 97 super(CrashBasePipeline, self).__init__(client_id, crash_identifiers) |
93 self._crash_identifiers = crash_identifiers | 98 self._crash_identifiers = crash_identifiers |
94 self._findit = FinditForClientID( | 99 self._findit = FinditForClientID(client_id, _CachedGitilesRepositoryFactory) |
95 client_id, | |
96 cached_gitiles_repository.CachedGitilesRepository( | |
97 HttpClientAppengine())) | |
98 | 100 |
99 @property | 101 @property |
100 def client_id(self): # pragma: no cover | 102 def client_id(self): # pragma: no cover |
101 return self._findit.client_id | 103 return self._findit.client_id |
102 | 104 |
103 def run(self, *args, **kwargs): | 105 def run(self, *args, **kwargs): |
104 raise NotImplementedError() | 106 raise NotImplementedError() |
105 | 107 |
106 | 108 |
107 class CrashAnalysisPipeline(CrashBasePipeline): | 109 class CrashAnalysisPipeline(CrashBasePipeline): |
(...skipping 122 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
230 N.B., due to the structure of AppEngine pipelines, this method must | 232 N.B., due to the structure of AppEngine pipelines, this method must |
231 accept the same arguments as are passed to ``__init__``; however, | 233 accept the same arguments as are passed to ``__init__``; however, |
232 because they were already passed to ``__init__`` there's no use in | 234 because they were already passed to ``__init__`` there's no use in |
233 recieving them here. Thus, we discard all the arguments to this method | 235 recieving them here. Thus, we discard all the arguments to this method |
234 (except for ``self``, naturally). | 236 (except for ``self``, naturally). |
235 """ | 237 """ |
236 run_analysis = yield CrashAnalysisPipeline( | 238 run_analysis = yield CrashAnalysisPipeline( |
237 self._client_id, self._crash_identifiers) | 239 self._client_id, self._crash_identifiers) |
238 with pipeline.After(run_analysis): | 240 with pipeline.After(run_analysis): |
239 yield PublishResultPipeline(self._client_id, self._crash_identifiers) | 241 yield PublishResultPipeline(self._client_id, self._crash_identifiers) |
OLD | NEW |