[Findit] Reorganizing findit_for_*.py

appengine/findit/handlers/crash/test/crash_handler_test.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import base64
 import json
 import os
 import re
 
 import webapp2
 import webtest
 
 from crash import crash_pipeline
+from crash.findit import Findit
 from crash.test.crash_testcase import CrashTestCase
 from handlers.crash import crash_handler
 
 
 class CrashHandlerTest(CrashTestCase):
   app_module = webapp2.WSGIApplication([
       ('/_ah/push-handlers/crash/fracas', crash_handler.CrashHandler),
   ], debug=True)
 
-  def _MockScheduleNewAnalysisForCrash(self, requested_crashes):
-    def Mocked_ScheduleNewAnalysisForCrash(*crash_data, **_):
-      requested_crashes.append(crash_data)
-    self.mock(crash_pipeline, 'ScheduleNewAnalysisForCrash',
-              Mocked_ScheduleNewAnalysisForCrash)
-
   def testAnalysisScheduled(self):
     requested_crashes = []
-    self._MockScheduleNewAnalysisForCrash(requested_crashes)
+    # TODO(wrengr): the old test mocked the function directly, but since
+    # we moved it to a method that's impossible. So how to we make sure
+    # things get dispatched to this mock? It doesn't look like it was
+    # called from anywhere...
+    class _MockFindit(Findit):
+      def ScheduleNewAnalysis(self, crash_data, **_):
+        requested_crashes.append(crash_data)
+
     self.mock_current_user(user_email='test@chromium.org', is_admin=True)
 
     client_id = 'fracas'
     channel = 'supported_channel'
     platform = 'supported_platform'
     signature = 'signature/here'
     stack_trace = 'frame1\nframe2\nframe3'
     chrome_version = '50.2500.0.0'
-    historic_metadata = [{'chrome_version': '50.2500.0.0', 'cpm': 0.6}]
+    historical_metadata = [{'chrome_version': '50.2500.0.0', 'cpm': 0.6}]
 
     crash_identifiers = {
         'chrome_version': chrome_version,
         'signature': signature,
         'channel': channel,
         'platform': platform,
         'process_type': 'renderer'
     }
 
     request_json_data = {
         'message': {
             'data': base64.b64encode(json.dumps({
                 'customized_data': {
-                    'channel': 'supported_channel',
-                    'historical_metadata': [
-                        {
-                            'chrome_version': '50.2500.0.0',
-                            'cpm': 0.6
-                        },
-                    ]
+                    'channel': channel,
+                    'historical_metadata': historical_metadata,
                 },
-                'chrome_version': '50.2500.0.0',
-                'signature': 'signature/here',
-                'client_id': 'fracas',
-                'platform': 'supported_platform',
+                'chrome_version': chrome_version,
+                'signature': signature,
+                'client_id': client_id,
+                'platform': platform,
                 'crash_identifiers': {
-                    'chrome_version': '50.2500.0.0',
-                    'signature': 'signature/here',
-                    'channel': 'supported_channel',
-                    'platform': 'supported_platform',
-                    'process_type': 'renderer'
+                    'chrome_version': chrome_version,
+                    'signature': signature,
+                    'channel': channel,
+                    'platform': platform,
+                    'process_type': 'renderer',
                 },
-                'stack_trace': 'frame1\nframe2\nframe3'
+                'stack_trace': stack_trace,
             })),
             'message_id': 'id',
         },
         'subscription': 'subscription',
     }
 
     self.test_app.post_json('/_ah/push-handlers/crash/fracas',
                             request_json_data)
 
     self.assertEqual(1, len(requested_crashes))

[wrengr, 2016/10/18 23:28:10]

The mocking of ScheduleNewAnalysis isn't working here. An example stacktrace:

  File
"/usr/local/google/home/wrengr/chromium-srcs/infra/appengine/findit/handlers/crash/test/crash_handler_test.py",
line 80, in testAnalysisScheduled
    self.assertEqual(1, len(requested_crashes))
  File "/usr/lib/python2.7/unittest/case.py", line 515, in assertEqual
    assertion_func(first, second, msg=msg)
  File "/usr/lib/python2.7/unittest/case.py", line 508, in _baseAssertEqual
    raise self.failureException(msg)
AssertionError: 1 != 0

==== captured logging output ====
LOG:
INFO: Config
CrashConfig<component_classifier=JsonProperty('component_classifier',
default={}, compressed=True), cracas=JsonProperty('cracas', default={}),
fracas=JsonProperty('fracas', default={}),
project_classifier=JsonProperty('project_classifier', default={}),
updated_by=StringProperty('updated_by', indexed=False),
updated_ts=DateTimeProperty('updated_ts', indexed=False, auto_now=True)> was
updated by admin
INFO: Processing message id from subscription subscription.
INFO: Crash data is {"stack_trace": "frame1\nframe2\nframe3",
"crash_identifiers": {"platform": "supported_platform", "signature":
"signature/here", "chrome_version": "50.2500.0.0", "process_type": "renderer",
"channel": "supported_channel"}, "customized_data": {"channel":
"supported_channel", "historical_metadata": [{"chrome_version": "50.2500.0.0",
"cpm": 0.6}]}, "platform": "supported_platform", "client_id": "fracas",
"signature": "signature/here", "chrome_version": "50.2500.0.0"}
WARNING: Failed to find spikes in history data [{u'chrome_version':
u'50.2500.0.0', u'cpm': 0.6}]
WARNING: suspended generator transaction(context.py:1018) raised
KeyError('crashed_version')
ERROR: Failed to process crash message
Traceback (most recent call last):
  File
"/usr/local/google/home/wrengr/chromium-srcs/infra/appengine/findit/handlers/crash/crash_handler.py",
line 126, in HandlePost
    queue_name=constants.CRASH_ANALYSIS_QUEUE[client_id])
  File
"/usr/local/google/home/wrengr/chromium-srcs/infra/appengine/findit/crash/findit.py",
line 141, in ScheduleNewAnalysis
    if not self._NeedsNewAnalysis(crash_data):
  File
"/usr/local/google/home/wrengr/chromium-srcs/google_appengine/google/appengine/ext/ndb/utils.py",
line 197, in inner_wrapper
    return wrapped_decorator(func, args, kwds, **options)
  File
"/usr/local/google/home/wrengr/chromium-srcs/google_appengine/google/appengine/ext/ndb/model.py",
line 3834, in transactional
    func, args, kwds, **options).get_result()
  File
"/usr/local/google/home/wrengr/chromium-srcs/google_appengine/google/appengine/ext/ndb/tasklets.py",
line 383, in get_result
    self.check_success()
  File
"/usr/local/google/home/wrengr/chromium-srcs/google_appengine/google/appengine/ext/ndb/tasklets.py",
line 430, in _help_tasklet_along
    value = gen.send(val)
  File
"/usr/local/google/home/wrengr/chromium-srcs/google_appengine/google/appengine/ext/ndb/context.py",
line 1014, in transaction
    result = callback()
  File
"/usr/local/google/home/wrengr/chromium-srcs/google_appengine/google/appengine/ext/ndb/model.py",
line 3842, in <lambda>
    return transaction_async(lambda: func(*args, **kwds), **options)
  File
"/usr/local/google/home/wrengr/chromium-srcs/infra/appengine/findit/crash/findit_for_chromecrash.py",
line 89, in _NeedsNewAnalysis
    self._InitializeAnalysis(model, crash_data)
  File
"/usr/local/google/home/wrengr/chromium-srcs/infra/appengine/findit/crash/findit_for_chromecrash.py",
line 59, in _InitializeAnalysis
    super(FinditForChromeCrash, self)._InitializeAnalysis(model, crash_data)
  File
"/usr/local/google/home/wrengr/chromium-srcs/infra/appengine/findit/crash/findit.py",
line 117, in _InitializeAnalysis
    model.crashed_version = crash_data['crashed_version']
KeyError: 'crashed_version'
INFO: {"message": {"data":
"eyJzdGFja190cmFjZSI6ICJmcmFtZTFcbmZyYW1lMlxuZnJhbWUzIiwgImNyYXNoX2lkZW50aWZpZXJzIjogeyJwbGF0Zm9ybSI6ICJzdXBwb3J0ZWRfcGxhdGZvcm0iLCAicHJvY2Vzc190eXBlIjogInJlbmRlcmVyIiwgImNocm9tZV92ZXJzaW9uIjogIjUwLjI1MDAuMC4wIiwgImNoYW5uZWwiOiAic3VwcG9ydGVkX2NoYW5uZWwiLCAic2lnbmF0dXJlIjogInNpZ25hdHVyZS9oZXJlIn0sICJjaHJvbWVfdmVyc2lvbiI6ICI1MC4yNTAwLjAuMCIsICJwbGF0Zm9ybSI6ICJzdXBwb3J0ZWRfcGxhdGZvcm0iLCAiY2xpZW50X2lkIjogImZyYWNhcyIsICJzaWduYXR1cmUiOiAic2lnbmF0dXJlL2hlcmUiLCAiY3VzdG9taXplZF9kYXRhIjogeyJjaGFubmVsIjogInN1cHBvcnRlZF9jaGFubmVsIiwgImhpc3RvcmljYWxfbWV0YWRhdGEiOiBbeyJjaHJvbWVfdmVyc2lvbiI6ICI1MC4yNTAwLjAuMCIsICJjcG0iOiAwLjZ9XX19",
"message_id": "id"}, "subscription": "subscription"}

STDOUT:
(empty)

STDERR:
(empty)

[Sharu Jiang, 2016/10/19 00:53:07]

"/usr/local/google/home/wrengr/chromium-srcs/infra/appengine/findit/crash/findit.py",
line 117, in _InitializeAnalysis
    model.crashed_version = crash_data['crashed_version']

It's 'chrome_version' in crash_data

[wrengr, 2016/10/19 18:05:16]

Except that it's 'crashed_version' for everyone else who calls
ScheduleNewAnalysis (e.g., in crash_pipeline_test.py)

[Sharu Jiang, 2016/10/19 21:03:09]

Right, the name is not consistent..., however changing name in CrashAnalysis
will cause some legacy data issue, and changing name in client message will
cause some cross-team back and forth, so we just live with it :)

     self.assertEqual(
         (crash_identifiers, chrome_version, signature, client_id,
          platform, stack_trace, {'channel': channel,
-                                 'historical_metadata': historic_metadata}),
+                                 'historical_metadata': historical_metadata}),
         requested_crashes[0])