| OLD | NEW |
| 1 # Copyright 2016 The Chromium Authors. All rights reserved. | 1 # Copyright 2016 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 import copy | 5 import copy |
| 6 import json | 6 import json |
| 7 import logging | 7 import logging |
| 8 | 8 |
| 9 from common import appengine_util | 9 from common import appengine_util |
| 10 from common import pubsub_util | 10 from common import pubsub_util |
| 11 from common import time_util | 11 from common import time_util |
| 12 from common.http_client_appengine import HttpClientAppengine | 12 from common.http_client_appengine import HttpClientAppengine |
| 13 from common.pipeline_wrapper import BasePipeline | 13 from common.pipeline_wrapper import BasePipeline |
| 14 from common.pipeline_wrapper import pipeline | 14 from common.pipeline_wrapper import pipeline |
| 15 from crash import findit_for_chromecrash | 15 from crash import findit_for_chromecrash |
| 16 from crash import findit_for_clusterfuzz | 16 from crash import findit_for_clusterfuzz |
| 17 from crash.type_enums import CrashClient | 17 from crash.type_enums import CrashClient |
| 18 from lib.gitiles import gitiles_repository | 18 from lib.gitiles import gitiles_repository |
| 19 from model import analysis_status | 19 from model import analysis_status |
| 20 | 20 |
| 21 | 21 |
| 22 # TODO(http://crbug.com/659346): write complete coverage tests for this. | 22 # TODO(http://crbug.com/659346): write complete coverage tests for this. |
| 23 def FinditForClientID(client_id): # pragma: no cover | 23 def FinditForClientID(client_id, repository): # pragma: no cover |
| 24 """Construct a Findit object from a client id string specifying the class. | 24 """Construct a Findit object from a client id string specifying the class. |
| 25 | 25 |
| 26 We cannot pass Findit objects to the various methods in | 26 We cannot pass Findit objects to the various methods in |
| 27 ``crash.crash_pipeline``, because they are not JSON serializable. For | 27 ``crash.crash_pipeline``, because they are not JSON serializable. For |
| 28 now, we just serialize Findit objects as their ``client_id``, and then | 28 now, we just serialize Findit objects as their ``client_id``, and then |
| 29 use this function to reconstruct them. Alas, this means we will lose | 29 use this function to reconstruct them. Alas, this means we will lose |
| 30 various other information stored in the Findit object (i.e., stuff that | 30 various other information stored in the Findit object (i.e., stuff that |
| 31 comes from CrashConfig); which could lead to some hard-to-diagnose | 31 comes from CrashConfig); which could lead to some hard-to-diagnose |
| 32 coherency bugs, since the new Findit object will be based on the | 32 coherency bugs, since the new Findit object will be based on the |
| 33 CrashConfig at the time it's constructed, which may be different | 33 CrashConfig at the time it's constructed, which may be different |
| 34 than the CrashConfig at the time the previous Findit object was | 34 than the CrashConfig at the time the previous Findit object was |
| 35 constructed. In the future we should fix all this to serialize Findit | 35 constructed. In the future we should fix all this to serialize Findit |
| 36 objects in a more robust way. | 36 objects in a more robust way. |
| 37 """ | 37 """ |
| 38 assert isinstance(client_id, (str, unicode)), ( | 38 assert isinstance(client_id, (str, unicode)), ( |
| 39 'FinditForClientID: expected string or unicode, but got %s' | 39 'FinditForClientID: expected string or unicode, but got %s' |
| 40 % client_id.__class__.__name__) | 40 % client_id.__class__.__name__) |
| 41 # TODO(wrengr): it'd be nice to replace this with a single lookup in | 41 # TODO(wrengr): it'd be nice to replace this with a single lookup in |
| 42 # a dict; but that's buggy for some unknown reason. | 42 # a dict; but that's buggy for some unknown reason. |
| 43 if client_id == CrashClient.FRACAS: | 43 if client_id == CrashClient.FRACAS: |
| 44 cls = findit_for_chromecrash.FinditForFracas | 44 cls = findit_for_chromecrash.FinditForFracas |
| 45 elif client_id == CrashClient.CRACAS: | 45 elif client_id == CrashClient.CRACAS: |
| 46 cls = findit_for_chromecrash.FinditForCracas | 46 cls = findit_for_chromecrash.FinditForCracas |
| 47 elif client_id == CrashClient.CLUSTERFUZZ: | 47 elif client_id == CrashClient.CLUSTERFUZZ: |
| 48 cls = findit_for_clusterfuzz.FinditForClusterfuzz | 48 cls = findit_for_clusterfuzz.FinditForClusterfuzz |
| 49 else: | 49 else: |
| 50 raise ValueError('FinditForClientID: ' | 50 raise ValueError('FinditForClientID: ' |
| 51 'unknown or unsupported client %s' % client_id) | 51 'unknown or unsupported client %s' % client_id) |
| 52 | 52 |
| 53 return cls( | 53 return cls(repository, CrashWrapperPipeline) |
| 54 gitiles_repository.GitilesRepository(http_client=HttpClientAppengine()), | |
| 55 CrashWrapperPipeline) | |
| 56 | 54 |
| 57 | 55 |
| 58 # Some notes about the classes below, for people who are not | 56 # Some notes about the classes below, for people who are not |
| 59 # familiar with AppEngine. The thing that really kicks everything off | 57 # familiar with AppEngine. The thing that really kicks everything off |
| 60 # is ``CrashWrapperPipeline.run``. However, an important thing to bear in | 58 # is ``CrashWrapperPipeline.run``. However, an important thing to bear in |
| 61 # mind is that whatever arguments are passed to that method will also | 59 # mind is that whatever arguments are passed to that method will also |
| 62 # be passed to the ``run`` method on whatever objects it yields. Thus, | 60 # be passed to the ``run`` method on whatever objects it yields. Thus, |
| 63 # all the ``run`` methods across these different classes must have | 61 # all the ``run`` methods across these different classes must have |
| 64 # the same type. In practice, we end up passing all the arguments to the | 62 # the same type. In practice, we end up passing all the arguments to the |
| 65 # constructors, because we need to have the fields around for logging | 63 # constructors, because we need to have the fields around for logging |
| 66 # (e.g., in ``finalized``); thus, there's nothing that needs to be passed | 64 # (e.g., in ``finalized``); thus, there's nothing that needs to be passed |
| 67 # to ``run``. Another thing to bear in mind is that whatever objects | 65 # to ``run``. Another thing to bear in mind is that whatever objects |
| 68 # ``CrashWrapperPipeline.run`` yields must be JSON-serializable. The base | 66 # ``CrashWrapperPipeline.run`` yields must be JSON-serializable. The base |
| 69 # class handles most of that for us, so the force of this constraint is | 67 # class handles most of that for us, so the force of this constraint is |
| 70 # that all the arguments to the constructors for those classes must be | 68 # that all the arguments to the constructors for those classes must be |
| 71 # JSON-serializable. Thus, we cannot actually pass a Findit object to | 69 # JSON-serializable. Thus, we cannot actually pass a Findit object to |
| 72 # the constructor, but rather must pass only the ``client_id`` (or whatever | 70 # the constructor, but rather must pass only the ``client_id`` (or whatever |
| 73 # JSON dict) and then reconstruct the Findit object from that. Moreover, | 71 # JSON dict) and then reconstruct the Findit object from that. Moreover, |
| 74 # the ``run`` method and the ``finalized`` method will be run in different | 72 # the ``run`` method and the ``finalized`` method will be run in different |
| 75 # processes, so we will actually end up reconstructing the Findit object | 73 # processes, so we will actually end up reconstructing the Findit object |
| 76 # twice. Thus, we shouldn't store anything in the pipeline objects outside | 74 # twice. Thus, we shouldn't store anything in the pipeline objects outside |
| 77 # of what their constructors store. | 75 # of what their constructors store. |
| 78 | 76 |
| 79 class CrashBasePipeline(BasePipeline): | 77 class CrashBasePipeline(BasePipeline): |
| 80 def __init__(self, client_id, crash_identifiers): | 78 def __init__(self, client_id, crash_identifiers): |
| 81 super(CrashBasePipeline, self).__init__(client_id, crash_identifiers) | 79 super(CrashBasePipeline, self).__init__(client_id, crash_identifiers) |
| 82 self._crash_identifiers = crash_identifiers | 80 self._crash_identifiers = crash_identifiers |
| 83 self._findit = FinditForClientID(client_id) | 81 self._findit = FinditForClientID( |
| 82 client_id, |
| 83 gitiles_repository.GitilesRepository(http_client=HttpClientAppengine())) |
| 84 | 84 |
| 85 @property | 85 @property |
| 86 def client_id(self): # pragma: no cover | 86 def client_id(self): # pragma: no cover |
| 87 return self._findit.client_id | 87 return self._findit.client_id |
| 88 | 88 |
| 89 def run(self, *args, **kwargs): | 89 def run(self, *args, **kwargs): |
| 90 raise NotImplementedError() | 90 raise NotImplementedError() |
| 91 | 91 |
| 92 | 92 |
| 93 class CrashAnalysisPipeline(CrashBasePipeline): | 93 class CrashAnalysisPipeline(CrashBasePipeline): |
| (...skipping 89 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 183 self._crash_identifiers = crash_identifiers | 183 self._crash_identifiers = crash_identifiers |
| 184 self._client_id = client_id | 184 self._client_id = client_id |
| 185 | 185 |
| 186 # TODO(http://crbug.com/659346): write coverage tests. | 186 # TODO(http://crbug.com/659346): write coverage tests. |
| 187 # Arguments number differs from overridden method - pylint: disable=W0221 | 187 # Arguments number differs from overridden method - pylint: disable=W0221 |
| 188 def run(self): # pragma: no cover | 188 def run(self): # pragma: no cover |
| 189 run_analysis = yield CrashAnalysisPipeline( | 189 run_analysis = yield CrashAnalysisPipeline( |
| 190 self._client_id, self._crash_identifiers) | 190 self._client_id, self._crash_identifiers) |
| 191 with pipeline.After(run_analysis): | 191 with pipeline.After(run_analysis): |
| 192 yield PublishResultPipeline(self._client_id, self._crash_identifiers) | 192 yield PublishResultPipeline(self._client_id, self._crash_identifiers) |
| OLD | NEW |