Open Source Monorail

appengine/monorail/framework/test/emailfmt_test.py

Index: appengine/monorail/framework/test/emailfmt_test.py
diff --git a/appengine/monorail/framework/test/emailfmt_test.py b/appengine/monorail/framework/test/emailfmt_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..af8a7001bbeb483a862f9dab3452593d6650f0a9
--- /dev/null
+++ b/appengine/monorail/framework/test/emailfmt_test.py
@@ -0,0 +1,726 @@
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is govered by a BSD-style
+# license that can be found in the LICENSE file or at
+# https://developers.google.com/open-source/licenses/bsd
+
+"""Tests for monorail.framework.emailfmt."""
+
+import unittest
+
+import settings
+from framework import emailfmt
+from framework import framework_views
+from proto import project_pb2
+from testing import testing_helpers
+
+from google.appengine.api import apiproxy_stub_map
+
+
+class EmailFmtTest(unittest.TestCase):
+
+  @unittest.skipIf('memcache' not in
+                   apiproxy_stub_map.apiproxy._APIProxyStubMap__stub_map,
+                   'memcache api proxy will not be found')
+  def testValidateReferencesHeader(self):
+    project = project_pb2.Project()
+    project.project_name = 'open-open'
+    subject = 'slipped disk'
+    expected = emailfmt.MakeMessageID(
+        'jrobbins@gmail.com', subject,
+        '%s@%s' % (project.project_name, emailfmt.MailDomain()))
+    self.assertTrue(
+        emailfmt.ValidateReferencesHeader(
+            expected, project, 'jrobbins@gmail.com', subject))
+
+    self.assertFalse(
+        emailfmt.ValidateReferencesHeader(
+            expected, project, 'jrobbins@gmail.com', 'something else'))
+
+    self.assertFalse(
+        emailfmt.ValidateReferencesHeader(
+            expected, project, 'someoneelse@gmail.com', subject))
+
+    project.project_name = 'other-project'
+    self.assertFalse(
+        emailfmt.ValidateReferencesHeader(
+            expected, project, 'jrobbins@gmail.com', subject))
+
+  def testParseEmailMessage(self):
+    msg = testing_helpers.MakeMessage(testing_helpers.HEADER_LINES, 'awesome!')
+
+    (from_addr, to_addrs, cc_addrs, references, subject,
+     body) = emailfmt.ParseEmailMessage(msg)
+
+    self.assertEqual('user@example.com', from_addr)
+    self.assertEqual(['proj@monorail.example.com'], to_addrs)
+    self.assertEqual(['ningerso@chromium.org'], cc_addrs)
+    # Expected msg-id was generated from a previous known-good test run.
+    self.assertEqual(['<0=969704940193871313=13442892928193434663='
+                      'proj@monorail.example.com>'],
+                     references)
+    self.assertEqual('Issue 123 in proj: broken link', subject)
+    self.assertEqual('awesome!', body)
+
+    references_header = ('References', '<1234@foo.com> <5678@bar.com>')
+    msg = testing_helpers.MakeMessage(
+        testing_helpers.HEADER_LINES + [references_header], 'awesome!')
+    (from_addr, to_addrs, cc_addrs, references, subject,
+     body) = emailfmt.ParseEmailMessage(msg)
+    self.assertItemsEqual(
+        ['<5678@bar.com>',
+         '<0=969704940193871313=13442892928193434663='
+         'proj@monorail.example.com>',
+         '<1234@foo.com>'],
+        references)
+
+  def testParseEmailMessage_Bulk(self):
+    for precedence in ['Bulk', 'Junk']:
+      msg = testing_helpers.MakeMessage(
+          testing_helpers.HEADER_LINES + [('Precedence', precedence)],
+          'I am on vacation!')
+
+      (from_addr, to_addrs, cc_addrs, in_reply_to, subject,
+       body) = emailfmt.ParseEmailMessage(msg)
+
+      self.assertEqual('', from_addr)
+      self.assertEqual([], to_addrs)
+      self.assertEqual([], cc_addrs)
+      self.assertEqual('', in_reply_to)
+      self.assertEqual('', subject)
+      self.assertEqual('', body)
+
+  def testExtractAddrs(self):
+    header_val = ''
+    self.assertEqual(
+        [], emailfmt._ExtractAddrs(header_val))
+
+    header_val = 'J. Robbins <a@b.com>, c@d.com,\n Nick "Name" Dude <e@f.com>'
+    self.assertEqual(
+        ['a@b.com', 'c@d.com', 'e@f.com'],
+        emailfmt._ExtractAddrs(header_val))
+
+    header_val = ('hot: J. O\'Robbins <a@b.com>; '
+                  'cool: "friendly" <e.g-h@i-j.k-L.com>')
+    self.assertEqual(
+        ['a@b.com', 'e.g-h@i-j.k-L.com'],
+        emailfmt._ExtractAddrs(header_val))
+
+  def CheckIdentifiedValues(
+      self, project_addr, subject, expected_project_name, expected_local_id):
+    """Testing helper function to check 3 results against expected values."""
+    project_name, local_id = emailfmt.IdentifyProjectAndIssue(
+        project_addr, subject)
+    self.assertEqual(expected_project_name, project_name)
+    self.assertEqual(expected_local_id, local_id)
+
+  def testIdentifyProjectAndIssues(self):
+    self.CheckIdentifiedValues(
+        'proj@monorail.example.com',
+        'Issue 123 in proj: the dogs wont eat the dogfood',
+        'proj', 123)
+
+    self.CheckIdentifiedValues(
+        'Proj@MonoRail.Example.Com',
+        'Issue 123 in proj: the dogs wont eat the dogfood',
+        'proj', 123)
+
+    self.CheckIdentifiedValues(
+        'proj-4-u@test-example3.com',
+        'Issue 123 in proj-4-u: this one goes to: 11',
+        'proj-4-u', 123)
+
+    self.CheckIdentifiedValues(
+        'night@monorail.example.com',
+        'Issue 451 in day: something is fishy',
+        None, 451)
+
+    self.CheckIdentifiedValues(
+        'no_reply@chromium.org',
+        'Issue 234 in project foo: ignore this one',
+        None, None)
+
+  def testStripSubjectPrefixes(self):
+    self.assertEqual(
+        '',
+        emailfmt._StripSubjectPrefixes(''))
+
+    self.assertEqual(
+        'this is it',
+        emailfmt._StripSubjectPrefixes('this is it'))
+
+    self.assertEqual(
+        'this is it',
+        emailfmt._StripSubjectPrefixes('re: this is it'))
+
+    self.assertEqual(
+        'this is it',
+        emailfmt._StripSubjectPrefixes('Re: Fwd: aw:this is it'))
+
+    self.assertEqual(
+        'This - . IS it',
+        emailfmt._StripSubjectPrefixes('This - . IS it'))
+
+
+class MailDomainTest(unittest.TestCase):
+
+  def testTrivialCases(self):
+    self.assertEqual(
+        'testbed-test.appspotmail.com',
+        emailfmt.MailDomain())
+
+
+class NoReplyAddressTest(unittest.TestCase):
+
+  def testNoCommenter(self):
+    self.assertEqual(
+        'no_reply@testbed-test.appspotmail.com',
+        emailfmt.NoReplyAddress())
+
+  def testWithCommenter(self):
+    commenter_view = framework_views.UserView(111L, 'user@example.com', True)
+    self.assertEqual(
+        'user@example.com via Monorail <no_reply@testbed-test.appspotmail.com>',
+        emailfmt.NoReplyAddress(
+            commenter_view=commenter_view, reveal_addr=True))
+
+  def testObscuredCommenter(self):
+    commenter_view = framework_views.UserView(111L, 'user@example.com', True)
+    self.assertEqual(
+        'u...@example.com via Monorail <no_reply@testbed-test.appspotmail.com>',
+        emailfmt.NoReplyAddress(
+            commenter_view=commenter_view, reveal_addr=False))
+
+
+class FormatFromAddrTest(unittest.TestCase):
+
+  def setUp(self):
+    self.project = project_pb2.Project(project_name='proj')
+
+  def testNoCommenter(self):
+    self.assertEqual(settings.send_email_as,
+                     emailfmt.FormatFromAddr(self.project))
+
+  def testNoCommenterWithNoReply(self):
+    self.assertEqual(settings.send_noreply_email_as,
+                     emailfmt.FormatFromAddr(self.project, can_reply_to=False))
+
+  def testWithCommenter(self):
+    commenter_view = framework_views.UserView(111L, 'user@example.com', True)
+    self.assertEqual(
+        'user@example.com via Monorail <%s>' % settings.send_email_as,
+        emailfmt.FormatFromAddr(
+            self.project, commenter_view=commenter_view, reveal_addr=True))
+
+  def testObscuredCommenter(self):
+    commenter_view = framework_views.UserView(111L, 'user@example.com', True)
+    self.assertEqual(
+        'u...@example.com via Monorail <%s>' % settings.send_email_as,
+        emailfmt.FormatFromAddr(
+            self.project, commenter_view=commenter_view, reveal_addr=False))
+
+  def testServiceAccountCommenter(self):
+    johndoe_bot = '123456789@developer.gserviceaccount.com'
+    commenter_view = framework_views.UserView(111L, johndoe_bot, True)
+    self.assertEqual(
+        ('johndoe@example.com via Monorail <%s>' % settings.send_email_as),
+        emailfmt.FormatFromAddr(
+            self.project, commenter_view=commenter_view, reveal_addr=False))
+
+
+class NormalizeHeaderWhitespaceTest(unittest.TestCase):
+
+  def testTrivialCases(self):
+    self.assertEqual(
+        '',
+        emailfmt.NormalizeHeader(''))
+
+    self.assertEqual(
+        '',
+        emailfmt.NormalizeHeader(' \t\n'))
+
+    self.assertEqual(
+        'a',
+        emailfmt.NormalizeHeader('a'))
+
+    self.assertEqual(
+        'a b',
+        emailfmt.NormalizeHeader(' a  b '))
+
+  def testLongSummary(self):
+    big_string = 'x' * 500
+    self.assertEqual(
+        big_string[:emailfmt.MAX_HEADER_CHARS_CONSIDERED],
+        emailfmt.NormalizeHeader(big_string))
+
+    big_string = 'x y ' * 500
+    self.assertEqual(
+        big_string[:emailfmt.MAX_HEADER_CHARS_CONSIDERED],
+        emailfmt.NormalizeHeader(big_string))
+
+    big_string = 'x   ' * 100
+    self.assertEqual(
+        'x ' * 99 + 'x',
+        emailfmt.NormalizeHeader(big_string))
+
+  def testNormalCase(self):
+    self.assertEqual(
+        '[a] b: c d',
+        emailfmt.NormalizeHeader('[a]  b:\tc\n\td'))
+
+
+class MakeMessageIDTest(unittest.TestCase):
+
+  @unittest.skipIf('memcache' not in
+                   apiproxy_stub_map.apiproxy._APIProxyStubMap__stub_map,
+                   'memcache api proxy will not be found')
+  def testMakeMessageIDTest(self):
+    message_id = emailfmt.MakeMessageID(
+        'to@to.com', 'subject', 'from@from.com')
+    self.assertTrue(message_id.startswith('<0='))
+    self.assertEqual('testbed-test.appspotmail.com>',
+                     message_id.split('@')[-1])
+
+    settings.mail_domain = None
+    message_id = emailfmt.MakeMessageID(
+        'to@to.com', 'subject', 'from@from.com')
+    self.assertTrue(message_id.startswith('<0='))
+    self.assertEqual('testbed-test.appspotmail.com>',
+                     message_id.split('@')[-1])
+
+    message_id = emailfmt.MakeMessageID(
+        'to@to.com', 'subject', 'from@from.com')
+    self.assertTrue(message_id.startswith('<0='))
+    self.assertEqual('testbed-test.appspotmail.com>',
+                     message_id.split('@')[-1])
+
+    message_id_ws_1 = emailfmt.MakeMessageID(
+        'to@to.com',
+        'this is a very long subject that is sure to be wordwrapped by gmail',
+        'from@from.com')
+    message_id_ws_2 = emailfmt.MakeMessageID(
+        'to@to.com',
+        'this is a  very   long subject   that \n\tis sure to be '
+        'wordwrapped \t\tby gmail',
+        'from@from.com')
+    self.assertEqual(message_id_ws_1, message_id_ws_2)
+
+
+class GetReferencesTest(unittest.TestCase):
+
+  def testNotPartOfThread(self):
+    refs = emailfmt.GetReferences(
+        'a@a.com', 'hi', None, emailfmt.NoReplyAddress())
+    self.assertEqual(0, len(refs))
+
+  @unittest.skipIf('memcache' not in
+                   apiproxy_stub_map.apiproxy._APIProxyStubMap__stub_map,
+                   'memcache api proxy will not be found')
+  def testAnywhereInThread(self):
+    refs = emailfmt.GetReferences(
+        'a@a.com', 'hi', 0, emailfmt.NoReplyAddress())
+    self.assertTrue(len(refs))
+    self.assertTrue(refs.startswith('<0='))
+
+
+class StripQuotedTextTest(unittest.TestCase):
+
+  def CheckExpected(self, expected_output, test_input):
+    actual_output = emailfmt.StripQuotedText(test_input)
+    self.assertEqual(expected_output, actual_output)
+
+  def testAllNewText(self):
+    self.CheckExpected('', '')
+    self.CheckExpected('', '\n')
+    self.CheckExpected('', '\n\n')
+    self.CheckExpected('new', 'new')
+    self.CheckExpected('new', '\nnew\n')
+    self.CheckExpected('new\ntext', '\nnew\ntext\n')
+    self.CheckExpected('new\n\ntext', '\nnew\n\ntext\n')
+
+  def testQuotedLines(self):
+    self.CheckExpected(
+        ('new\n'
+         'text'),
+        ('new\n'
+         'text\n'
+         '\n'
+         '> something you said\n'
+         '> that took two lines'))
+
+    self.CheckExpected(
+        ('new\n'
+         'text'),
+        ('new\n'
+         'text\n'
+         '\n'
+         '> something you said\n'
+         '> that took two lines'))
+
+    self.CheckExpected(
+        ('new\n'
+         'text'),
+        ('> something you said\n'
+         '> that took two lines\n'
+         'new\n'
+         'text\n'
+         '\n'))
+
+    self.CheckExpected(
+        ('newtext'),
+        ('> something you said\n'
+         '> that took two lines\n'
+         'newtext'))
+
+    self.CheckExpected(
+        ('new\n'
+         'text'),
+        ('new\n'
+         'text\n'
+         '\n'
+         '> something you said\n'
+         '> > in response to some other junk'))
+
+    self.CheckExpected(
+        ('new\n'
+         '\n'
+         'text'),
+        ('new\n'
+         '\n'
+         '> something you said\n'
+         '> > in response to some other junk\n'
+         '\n'
+         'text\n'))
+
+    self.CheckExpected(
+        ('new\n'
+         '\n'
+         'text'),
+        ('new\n'
+         'On Mon, Jan 1, 2023, So-and-so <so@and-so.com> Wrote:\n'
+         '> something you said\n'
+         '> > in response to some other junk\n'
+         '\n'
+         'text\n'))
+
+    self.CheckExpected(
+        ('new\n'
+         '\n'
+         'text'),
+        ('new\n'
+         'On Mon, Jan 1, 2023, So-and-so <so@and-so.com> Wrote:\n'
+         '\n'
+         '> something you said\n'
+         '> > in response to some other junk\n'
+         '\n'
+         'text\n'))
+
+    self.CheckExpected(
+        ('new\n'
+         '\n'
+         'text'),
+        ('new\n'
+         'On Mon, Jan 1, 2023, user@example.com via Monorail\n'
+         '<monorail@chromium.com> Wrote:\n'
+         '\n'
+         '> something you said\n'
+         '> > in response to some other junk\n'
+         '\n'
+         'text\n'))
+
+    self.CheckExpected(
+        ('new\n'
+         '\n'
+         'text'),
+        ('new\n'
+         'On Jan 14, 2016 6:19 AM, "user@example.com via Monorail" <\n'
+         'monorail@chromium.com> Wrote:\n'
+         '\n'
+         '> something you said\n'
+         '> > in response to some other junk\n'
+         '\n'
+         'text\n'))
+
+    self.CheckExpected(
+        ('new\n'
+         '\n'
+         'text'),
+        ('new\n'
+         'On Jan 14, 2016 6:19 AM, "user@example.com via Monorail" <\n'
+         'monorail@monorail-prod.appspotmail.com> wrote:\n'
+         '\n'
+         '> something you said\n'
+         '> > in response to some other junk\n'
+         '\n'
+         'text\n'))
+
+    self.CheckExpected(
+        ('new\n'
+         '\n'
+         'text'),
+        ('new\n'
+         'On Mon, Jan 1, 2023, So-and-so so@and-so.com wrote:\n'
+         '\n'
+         '> something you said\n'
+         '> > in response to some other junk\n'
+         '\n'
+         'text\n'))
+
+    self.CheckExpected(
+        ('new\n'
+         '\n'
+         'text'),
+        ('new\n'
+         'On Wed, Sep 8, 2010 at 6:56 PM, So =AND= <so@gmail.com>wrote:\n'
+         '\n'
+         '> something you said\n'
+         '> > in response to some other junk\n'
+         '\n'
+         'text\n'))
+
+    self.CheckExpected(
+        ('new\n'
+         '\n'
+         'text'),
+        ('new\n'
+         'On Mon, Jan 1, 2023, So-and-so <so@and-so.com> Wrote:\n'
+         '\n'
+         '> something you said\n'
+         '> > in response to some other junk\n'
+         '\n'
+         'text\n'))
+
+    self.CheckExpected(
+        ('new\n'
+         '\n'
+         'text'),
+        ('new\n'
+         'project-name@testbed-test.appspotmail.com wrote:\n'
+         '\n'
+         '> something you said\n'
+         '> > in response to some other junk\n'
+         '\n'
+         'text\n'))
+
+    self.CheckExpected(
+        ('new\n'
+         '\n'
+         'text'),
+        ('new\n'
+         'project-name@testbed-test.appspotmail.com a \xc3\xa9crit :\n'
+         '\n'
+         '> something you said\n'
+         '> > in response to some other junk\n'
+         '\n'
+         'text\n'))
+
+    self.CheckExpected(
+        ('new\n'
+         '\n'
+         'text'),
+        ('new\n'
+         'project.domain.com@testbed-test.appspotmail.com a \xc3\xa9crit :\n'
+         '\n'
+         '> something you said\n'
+         '> > in response to some other junk\n'
+         '\n'
+         'text\n'))
+
+    self.CheckExpected(
+        ('new\n'
+         '\n'
+         'text'),
+        ('new\n'
+         '2023/01/4 <so@and-so.com>\n'
+         '\n'
+         '> something you said\n'
+         '> > in response to some other junk\n'
+         '\n'
+         'text\n'))
+
+    self.CheckExpected(
+        ('new\n'
+         '\n'
+         'text'),
+        ('new\n'
+         '2023/01/4 <so-and@so.com>\n'
+         '\n'
+         '> something you said\n'
+         '> > in response to some other junk\n'
+         '\n'
+         'text\n'))
+
+  def testBoundaryLines(self):
+
+    self.CheckExpected(
+        ('new'),
+        ('new\n'
+         '---- forwarded message ======\n'
+         '\n'
+         'something you said\n'
+         '> in response to some other junk\n'
+         '\n'
+         'text\n'))
+
+    self.CheckExpected(
+        ('new'),
+        ('new\n'
+         '-----Original Message-----\n'
+         '\n'
+         'something you said\n'
+         '> in response to some other junk\n'
+         '\n'
+         'text\n'))
+
+    self.CheckExpected(
+        ('new'),
+        ('new\n'
+         '\n'
+         'Updates:\n'
+         '\tStatus: Fixed\n'
+         '\n'
+         'notification text\n'))
+
+    self.CheckExpected(
+        ('new'),
+        ('new\n'
+         '\n'
+         'Comment #1 on issue 9 by username: Is there ...'
+         'notification text\n'))
+
+  def testSignatures(self):
+
+    self.CheckExpected(
+        ('new\n'
+         'text'),
+        ('new\n'
+         'text\n'
+         '-- \n'
+         'Name\n'
+         'phone\n'
+         'funny quote, or legal disclaimers\n'))
+
+    self.CheckExpected(
+        ('new\n'
+         'text'),
+        ('new\n'
+         'text\n'
+         '--\n'
+         'Name\n'
+         'phone\n'
+         'funny quote, or legal disclaimers\n'))
+
+    self.CheckExpected(
+        ('new\n'
+         'text'),
+        ('new\n'
+         'text\n'
+         '--\n'
+         'Name\n'
+         'ginormous signature\n'
+         'phone\n'
+         'address\n'
+         'address\n'
+         'address\n'
+         'homepage\n'
+         'social network A\n'
+         'social network B\n'
+         'social network C\n'
+         'funny quote\n'
+         '4 lines about why email should be short\n'
+         'legal disclaimers\n'))
+
+    self.CheckExpected(
+        ('new\n'
+         'text'),
+        ('new\n'
+         'text\n'
+         '_______________\n'
+         'Name\n'
+         'phone\n'
+         'funny quote, or legal disclaimers\n'))
+
+    self.CheckExpected(
+        ('new\n'
+         'text'),
+        ('new\n'
+         'text\n'
+         '\n'
+         'Thanks,\n'
+         'Name\n'
+         '\n'
+         '_______________\n'
+         'Name\n'
+         'phone\n'
+         'funny quote, or legal disclaimers\n'))
+
+    self.CheckExpected(
+        ('new\n'
+         'text'),
+        ('new\n'
+         'text\n'
+         '\n'
+         'Thanks,\n'
+         'Name'))
+
+    self.CheckExpected(
+        ('new\n'
+         'text'),
+        ('new\n'
+         'text\n'
+         '\n'
+         'Cheers,\n'
+         'Name'))
+
+    self.CheckExpected(
+        ('new\n'
+         'text'),
+        ('new\n'
+         'text\n'
+         '\n'
+         'Regards\n'
+         'Name'))
+
+    self.CheckExpected(
+        ('new\n'
+         'text'),
+        ('new\n'
+         'text\n'
+         '\n'
+         'best regards'))
+
+    self.CheckExpected(
+        ('new\n'
+         'text'),
+        ('new\n'
+         'text\n'
+         '\n'
+         'THX'))
+
+    self.CheckExpected(
+        ('new\n'
+         'text'),
+        ('new\n'
+         'text\n'
+         '\n'
+         'Thank you,\n'
+         'Name'))
+
+    self.CheckExpected(
+        ('new\n'
+         'text'),
+        ('new\n'
+         'text\n'
+         '\n'
+         'Sent from my iPhone'))
+
+    self.CheckExpected(
+        ('new\n'
+         'text'),
+        ('new\n'
+         'text\n'
+         '\n'
+         'Sent from my iPod'))
+
+
+if __name__ == '__main__':
+  unittest.main()