Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(1088)

Side by Side Diff: appengine/findit/crash/crash_pipeline.py

Issue 2605943002: Removing the mutation in the factories for getting dep repositories (Closed)
Patch Set: Created 3 years, 11 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 # Copyright 2016 The Chromium Authors. All rights reserved. 1 # Copyright 2016 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 import copy 5 import copy
6 import json 6 import json
7 import logging 7 import logging
8 8
9 from crash import monitoring 9 from crash import monitoring
10 10
11 from common import appengine_util 11 from common import appengine_util
12 from common import pubsub_util 12 from common import pubsub_util
13 from common.http_client_appengine import HttpClientAppengine 13 from common.http_client_appengine import HttpClientAppengine
14 from common.pipeline_wrapper import BasePipeline 14 from common.pipeline_wrapper import BasePipeline
15 from common.pipeline_wrapper import pipeline 15 from common.pipeline_wrapper import pipeline
16 from crash import findit_for_chromecrash 16 from crash import findit_for_chromecrash
17 from crash import findit_for_clusterfuzz 17 from crash import findit_for_clusterfuzz
18 from crash.type_enums import CrashClient 18 from crash.type_enums import CrashClient
19 from libs import time_util 19 from libs import time_util
20 from gae_libs.gitiles import cached_gitiles_repository 20 from gae_libs.gitiles import cached_gitiles_repository
21 from model import analysis_status 21 from model import analysis_status
22 22
23 23
24 # TODO(http://crbug.com/659346): write complete coverage tests for this. 24 # TODO(http://crbug.com/659346): write complete coverage tests for this.
25 def FinditForClientID(client_id, repository): # pragma: no cover 25 def FinditForClientID(client_id, get_repository): # pragma: no cover
26 """Construct a Findit object from a client id string specifying the class. 26 """Construct a Findit object from a client id string specifying the class.
27 27
28 We cannot pass Findit objects to the various methods in 28 We cannot pass Findit objects to the various methods in
29 ``crash.crash_pipeline``, because they are not JSON serializable. For 29 ``crash.crash_pipeline``, because they are not JSON serializable. For
30 now, we just serialize Findit objects as their ``client_id``, and then 30 now, we just serialize Findit objects as their ``client_id``, and then
31 use this function to reconstruct them. Alas, this means we will lose 31 use this function to reconstruct them. Alas, this means we will lose
32 various other information stored in the Findit object (i.e., stuff that 32 various other information stored in the Findit object (i.e., stuff that
33 comes from CrashConfig); which could lead to some hard-to-diagnose 33 comes from CrashConfig); which could lead to some hard-to-diagnose
34 coherency bugs, since the new Findit object will be based on the 34 coherency bugs, since the new Findit object will be based on the
35 CrashConfig at the time it's constructed, which may be different 35 CrashConfig at the time it's constructed, which may be different
36 than the CrashConfig at the time the previous Findit object was 36 than the CrashConfig at the time the previous Findit object was
37 constructed. In the future we should fix all this to serialize Findit 37 constructed. In the future we should fix all this to serialize Findit
38 objects in a more robust way. 38 objects in a more robust way.
39 """ 39 """
40 assert isinstance(client_id, (str, unicode)), ( 40 assert isinstance(client_id, (str, unicode)), (
41 'FinditForClientID: expected string or unicode, but got %s' 41 'FinditForClientID: expected string or unicode, but got %s'
42 % client_id.__class__.__name__) 42 % client_id.__class__.__name__)
43 # TODO(wrengr): it'd be nice to replace this with a single lookup in 43 # TODO(wrengr): it'd be nice to replace this with a single lookup in
44 # a dict; but that's buggy for some unknown reason. 44 # a dict; but that's buggy for some unknown reason.
45 if client_id == CrashClient.FRACAS: 45 if client_id == CrashClient.FRACAS:
46 cls = findit_for_chromecrash.FinditForFracas 46 cls = findit_for_chromecrash.FinditForFracas
47 elif client_id == CrashClient.CRACAS: # pragma: no cover 47 elif client_id == CrashClient.CRACAS: # pragma: no cover
48 cls = findit_for_chromecrash.FinditForCracas 48 cls = findit_for_chromecrash.FinditForCracas
49 elif client_id == CrashClient.CLUSTERFUZZ: # pragma: no cover 49 elif client_id == CrashClient.CLUSTERFUZZ: # pragma: no cover
50 cls = findit_for_clusterfuzz.FinditForClusterfuzz 50 cls = findit_for_clusterfuzz.FinditForClusterfuzz
51 else: # pragma: no cover 51 else: # pragma: no cover
52 raise ValueError('FinditForClientID: ' 52 raise ValueError('FinditForClientID: '
53 'unknown or unsupported client %s' % client_id) 53 'unknown or unsupported client %s' % client_id)
54 54
55 return cls(repository) 55 return cls(get_repository)
56 56
57 57
58 # Some notes about the classes below, for people who are not familiar 58 # Some notes about the classes below, for people who are not familiar
59 # with AppEngine pipelines: 59 # with AppEngine pipelines:
60 # 60 #
61 # The pipeline library is designed in a strange way which requires that 61 # The pipeline library is designed in a strange way which requires that
62 # all the ``__init__`` and ``run`` methods in this file take the exact 62 # all the ``__init__`` and ``run`` methods in this file take the exact
63 # same arguments. This arises from the interaction between a few 63 # same arguments. This arises from the interaction between a few
64 # different constraints. First, for any given pipeline, its ``__init__`` 64 # different constraints. First, for any given pipeline, its ``__init__``
65 # and ``run`` must take the same arguments. Second, all the objects that 65 # and ``run`` must take the same arguments. Second, all the objects that
(...skipping 14 matching lines...) Expand all
80 # serialization. Thus, we cannot pass a ``Findit`` object directly to 80 # serialization. Thus, we cannot pass a ``Findit`` object directly to
81 # any of the methods here, but rather must instead pass the ``client_id`` 81 # any of the methods here, but rather must instead pass the ``client_id``
82 # (or whatever JSON dict), and then reconstruct the ``Findit`` object 82 # (or whatever JSON dict), and then reconstruct the ``Findit`` object
83 # from that data. 83 # from that data.
84 # 84 #
85 # Moreover, the ``run`` and ``finalized`` methods are executed in separate 85 # Moreover, the ``run`` and ``finalized`` methods are executed in separate
86 # processes, so we'll actually end up reconstructing the ``Findit`` object 86 # processes, so we'll actually end up reconstructing the ``Findit`` object
87 # twice. This also means ``run`` can't store anything in the pipeline 87 # twice. This also means ``run`` can't store anything in the pipeline
88 # object and expect it to still be available in the ``finalized`` method. 88 # object and expect it to still be available in the ``finalized`` method.
89 89
90 def _CachedGitilesRepositoryFactory(repo_url): # pragma: no cover
Sharu Jiang 2016/12/28 23:05:12 what's the reason for ``pragma: no cover`` here?
wrengr 2016/12/29 20:57:31 Apparently it's never called by the unit tests ¯\_
91 return cached_gitiles_repository.CachedGitilesRepository(
92 HttpClientAppengine(), repo_url)
93
94
90 class CrashBasePipeline(BasePipeline): 95 class CrashBasePipeline(BasePipeline):
91 def __init__(self, client_id, crash_identifiers): 96 def __init__(self, client_id, crash_identifiers):
92 super(CrashBasePipeline, self).__init__(client_id, crash_identifiers) 97 super(CrashBasePipeline, self).__init__(client_id, crash_identifiers)
93 self._crash_identifiers = crash_identifiers 98 self._crash_identifiers = crash_identifiers
94 self._findit = FinditForClientID( 99 self._findit = FinditForClientID(client_id, _CachedGitilesRepositoryFactory)
95 client_id,
96 cached_gitiles_repository.CachedGitilesRepository(
97 HttpClientAppengine()))
98 100
99 @property 101 @property
100 def client_id(self): # pragma: no cover 102 def client_id(self): # pragma: no cover
101 return self._findit.client_id 103 return self._findit.client_id
102 104
103 def run(self, *args, **kwargs): 105 def run(self, *args, **kwargs):
104 raise NotImplementedError() 106 raise NotImplementedError()
105 107
106 108
107 class CrashAnalysisPipeline(CrashBasePipeline): 109 class CrashAnalysisPipeline(CrashBasePipeline):
(...skipping 122 matching lines...) Expand 10 before | Expand all | Expand 10 after
230 N.B., due to the structure of AppEngine pipelines, this method must 232 N.B., due to the structure of AppEngine pipelines, this method must
231 accept the same arguments as are passed to ``__init__``; however, 233 accept the same arguments as are passed to ``__init__``; however,
232 because they were already passed to ``__init__`` there's no use in 234 because they were already passed to ``__init__`` there's no use in
233 recieving them here. Thus, we discard all the arguments to this method 235 recieving them here. Thus, we discard all the arguments to this method
234 (except for ``self``, naturally). 236 (except for ``self``, naturally).
235 """ 237 """
236 run_analysis = yield CrashAnalysisPipeline( 238 run_analysis = yield CrashAnalysisPipeline(
237 self._client_id, self._crash_identifiers) 239 self._client_id, self._crash_identifiers)
238 with pipeline.After(run_analysis): 240 with pipeline.After(run_analysis):
239 yield PublishResultPipeline(self._client_id, self._crash_identifiers) 241 yield PublishResultPipeline(self._client_id, self._crash_identifiers)
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698