Added orphaned compiled Python file checks.

tests/presubmit_unittest.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Unit tests for presubmit_support.py and presubmit_canned_checks.py."""
 
 # pylint: disable=E1101,E1103
 
+import __builtin__
+
+import contextlib
 import functools
 import itertools
 import logging
 import os
 import StringIO
 import sys
 import time
 import unittest
 
 _ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ skipping 106 matching lines @@
 """
 
   def setUp(self):
     SuperMoxTestBase.setUp(self)
     class FakeChange(object):
       def __init__(self, obj):
         self._root = obj.fake_root_dir
       def RepositoryRoot(self):
         return self._root
 
+    self.UnMock(presubmit.os, 'walk')

[szager1, 2014/10/28 17:08:14]

Can you group this together with the other presubmit.os mock overrides below? 
Bonus points for using the same mechanism for all of them.

[dnj, 2014/10/28 18:37:35]

Must have been late - didn't realize SuperMoxTestBase already mocks this
function.

     self.mox.StubOutWithMock(presubmit, 'random')
     self.mox.StubOutWithMock(presubmit, 'warn')
+    self.mox.StubOutWithMock(presubmit.os, 'walk')
     presubmit._ASKED_FOR_FEEDBACK = False
     self.fake_root_dir = self.RootDir()
     self.fake_change = FakeChange(self)
 
     # Special mocks.
     def MockAbsPath(f):
       return f
     def MockChdir(f):
       return None
     # SuperMoxTestBase already mock these but simplify our life.
@@ skipping 22 matching lines @@
       'GitChange', 'InputApi', 'ListRelevantPresubmitFiles', 'Main',
       'NonexistantCannedCheckFilter', 'OutputApi', 'ParseFiles',
       'PresubmitFailure', 'PresubmitExecuter', 'PresubmitOutput', 'ScanSubDirs',
       'SvnAffectedFile', 'SvnChange', 'cPickle', 'cpplint', 'cStringIO',
       'contextlib', 'canned_check_filter', 'fix_encoding', 'fnmatch',
       'gclient_utils', 'glob', 'inspect', 'json', 'load_files', 'logging',
       'marshal', 'normpath', 'optparse', 'os', 'owners', 'pickle',
       'presubmit_canned_checks', 'random', 're', 'rietveld', 'scm',
       'subprocess', 'sys', 'tempfile', 'time', 'traceback', 'types', 'unittest',
       'urllib2', 'warn', 'multiprocessing', 'DoGetTryMasters',
-      'GetTryMastersExecuter', 'itertools',
+      'GetTryMastersExecuter', 'itertools', 'CleanOrphanedCompiledPython',
+      'IsCompiledPython',
     ]
     # If this test fails, you should add the relevant test.
     self.compareMembers(presubmit, members)
 
   def testCannedCheckFilter(self):
     canned = presubmit.presubmit_canned_checks
     orig = canned.CheckOwners
     with presubmit.canned_check_filter(['CheckOwners']):
       self.assertNotEqual(canned.CheckOwners, orig)
       self.assertEqual(canned.CheckOwners(None, None), [])
@@ skipping 484 matching lines @@
     inherit_path = presubmit.os.path.join(self.fake_root_dir,
                                           self._INHERIT_SETTINGS)
     presubmit.os.path.isfile(inherit_path).AndReturn(False)
     presubmit.os.path.isfile(root_path).AndReturn(True)
     presubmit.os.path.isfile(haspresubmit_path).AndReturn(True)
     presubmit.gclient_utils.FileRead(root_path,
                                      'rU').AndReturn(self.presubmit_text)
     presubmit.gclient_utils.FileRead(haspresubmit_path,
                                      'rU').AndReturn(self.presubmit_text)
     presubmit.random.randint(0, 4).AndReturn(1)
+    presubmit.os.walk(self.fake_root_dir).AndReturn([])
     self.mox.ReplayAll()
 
     input_buf = StringIO.StringIO('y\n')
     change = presubmit.Change(
         'mychange',
         '\n'.join(description_lines),
         self.fake_root_dir,
         files,
         0,
         0,
@@ skipping 20 matching lines @@
     for _ in range(2):
       presubmit.os.path.isfile(inherit_path).AndReturn(False)
       presubmit.os.path.isfile(presubmit_path).AndReturn(True)
       presubmit.os.path.isfile(haspresubmit_path).AndReturn(True)
       presubmit.gclient_utils.FileRead(presubmit_path, 'rU'
           ).AndReturn(self.presubmit_text)
       presubmit.gclient_utils.FileRead(haspresubmit_path, 'rU'
           ).AndReturn(self.presubmit_text)
     presubmit.random.randint(0, 4).AndReturn(1)
     presubmit.random.randint(0, 4).AndReturn(1)
+    presubmit.os.walk(self.fake_root_dir).AndReturn([])
+    presubmit.os.walk(self.fake_root_dir).AndReturn([])
     self.mox.ReplayAll()
 
     input_buf = StringIO.StringIO('n\n')  # say no to the warning
     change = presubmit.Change(
         'mychange',
         '\n'.join(description_lines),
         self.fake_root_dir,
         files,
         0,
         0,
@@ skipping 26 matching lines @@
     inherit_path = presubmit.os.path.join(self.fake_root_dir,
                                           self._INHERIT_SETTINGS)
     presubmit.os.path.isfile(inherit_path).AndReturn(False)
     presubmit.os.path.isfile(presubmit_path).AndReturn(True)
     presubmit.os.path.isfile(haspresubmit_path).AndReturn(True)
     presubmit.gclient_utils.FileRead(presubmit_path, 'rU'
                                      ).AndReturn(self.presubmit_text)
     presubmit.gclient_utils.FileRead(haspresubmit_path, 'rU').AndReturn(
         self.presubmit_text)
     presubmit.random.randint(0, 4).AndReturn(1)
+    presubmit.os.walk(self.fake_root_dir).AndReturn([])
     self.mox.ReplayAll()
 
     change = presubmit.Change(
         'mychange',
         '\n'.join(description_lines),
         self.fake_root_dir,
         files,
         0,
         0,
         None)
@@ skipping 21 matching lines @@
 """
     inherit_path = presubmit.os.path.join(self.fake_root_dir,
                                           self._INHERIT_SETTINGS)
     presubmit.os.path.isfile(inherit_path).AndReturn(False)
     presubmit.os.path.isfile(join(self.fake_root_dir, 'PRESUBMIT.py')
         ).AndReturn(False)
     presubmit.os.path.isfile(join(self.fake_root_dir,
                                   'haspresubmit',
                                   'PRESUBMIT.py')).AndReturn(False)
     presubmit.random.randint(0, 4).AndReturn(0)
+    presubmit.os.walk(self.fake_root_dir).AndReturn([])
     self.mox.ReplayAll()
 
     input_buf = StringIO.StringIO('y\n')
     # Always fail.
     change = presubmit.Change(
         'mychange',
         '\n'.join(description_lines),
         self.fake_root_dir,
         files,
         0,
@@ skipping 62 matching lines @@
     return [output_api.PresubmitError('Tag parsing failed. 5 ' +
                                       input_api.change.FullDescriptionText())]
   return [output_api.PresubmitNotifyResult(input_api.change.tags['STORY'])]
 def CheckChangeOnCommit(input_api, output_api):
   raise Exception("Test error")
 """
     presubmit.random.randint(0, 4).AndReturn(1)
     inherit_path = presubmit.os.path.join(self.fake_root_dir,
                                           self._INHERIT_SETTINGS)
     presubmit.os.path.isfile(inherit_path).AndReturn(False)
+    presubmit.os.walk(self.fake_root_dir).AndReturn([])
     self.mox.ReplayAll()
 
     output = StringIO.StringIO()
     input_buf = StringIO.StringIO('y\n')
     change = presubmit.Change(
         'foo',
         'Blah Blah\n\nSTORY=http://tracker.com/42\nBUG=boo\n',
         self.fake_root_dir,
         None,
         0,
@@ skipping 264 matching lines @@
         'presubmit_unittest.py: error: For unversioned directory, <files> is '
         'not optional.\n')
     self.mox.ReplayAll()
 
     try:
       presubmit.Main(['--root', self.fake_root_dir])
       self.fail()
     except SystemExit, e:
       self.assertEquals(2, e.code)
 
+  def testCleanOrphanedCompiledPython(self):
+    self.mox.StubOutWithMock(presubmit, 'IsCompiledPython')
+    py_path = os.path.join(self.fake_root_dir, 'myfile.py')
+    pyc_path = os.path.join(self.fake_root_dir, 'myfile.pyc')
+
+    presubmit.os.walk(self.fake_root_dir).AndReturn([
+        (self.fake_root_dir, [], ['myfile.pyc']),
+    ])
+    presubmit.os.path.isfile(py_path).AndReturn(False)
+    presubmit.IsCompiledPython(pyc_path).AndReturn(True)

[szager1, 2014/10/28 17:08:14]

Probably should also test by adding a mock invocation for os.remove and calling
CleanOrphanedCompiledPython with 'True'.

[dnj, 2014/10/28 18:37:35]

I'm actually going to switch gears here - have one "False doesn't remove files"
test and leave the rest test the "True" code path, asserting correct "remove"
invocations.

+    self.mox.ReplayAll()
+
+    output = StringIO.StringIO()
+    presubmit.CleanOrphanedCompiledPython(output, self.fake_root_dir, False)
+
+  def testCleanNonOrphanedCompiledPython(self):
+    self.mox.StubOutWithMock(presubmit, 'IsCompiledPython')
+    py_path = os.path.join(self.fake_root_dir, 'myfile.py')
+
+    presubmit.os.walk(self.fake_root_dir).AndReturn([
+        (self.fake_root_dir, [], ['myfile.pyc', 'myfile.py']),
+    ])
+    presubmit.os.path.isfile(py_path).AndReturn(True)
+    self.mox.ReplayAll()
+
+    output = StringIO.StringIO()
+    presubmit.CleanOrphanedCompiledPython(output, self.fake_root_dir, False)
+
+  def testCleanOrphanedCompiledPythonAvoidsOtherFiles(self):
+    self.mox.StubOutWithMock(presubmit, 'IsCompiledPython')
+
+    presubmit.os.walk(self.fake_root_dir).AndReturn([
+        (self.fake_root_dir, [], ['otherfile.txt']),
+    ])
+    self.mox.ReplayAll()
+
+    output = StringIO.StringIO()
+    presubmit.CleanOrphanedCompiledPython(output, self.fake_root_dir, False)
+
+  def testIsCompiledPython(self):
+    self.mox.StubOutWithMock(__builtin__, 'open')
+    pyc_path = os.path.join(self.fake_root_dir, 'testfile.pyc')
+    open(pyc_path, 'rb').AndReturn(contextlib.closing(
+        StringIO.StringIO('\x00\x00\x0d\x0a\x00\x00\x00\x00')))
+    self.mox.ReplayAll()
+
+    self.assertTrue(presubmit.IsCompiledPython(pyc_path))
+
+  def testIsCompiledPythonFailsForInvalidMagic(self):
+    self.mox.StubOutWithMock(__builtin__, 'open')
+    pyc_path = os.path.join(self.fake_root_dir, 'testfile.pyc')
+    open(pyc_path, 'rb').AndReturn(contextlib.closing(
+        StringIO.StringIO('\xca\xfe\xba\xbe\x00\x00\x00\x00')))
+    self.mox.ReplayAll()
+
+    self.assertFalse(presubmit.IsCompiledPython(pyc_path))
+
+  def testIsCompiledPythonFailsForInvalidSize(self):
+    self.mox.StubOutWithMock(__builtin__, 'open')
+    pyc_path = os.path.join(self.fake_root_dir, 'testfile.pyc')
+    open(pyc_path, 'rb').AndReturn(contextlib.closing(
+        StringIO.StringIO('\x00\x00\x0d\x0a\x00\x00\x00')))
+    self.mox.ReplayAll()
+
+    self.assertFalse(presubmit.IsCompiledPython(pyc_path))
+
 
 class InputApiUnittest(PresubmitTestsBase):
   """Tests presubmit.InputApi."""
   def testMembersChanged(self):
     self.mox.ReplayAll()
     members = [
       'AbsoluteLocalPaths', 'AffectedFiles', 'AffectedSourceFiles',
       'AffectedTextFiles',
       'DEFAULT_BLACK_LIST', 'DEFAULT_WHITE_LIST',
       'DepotToLocalPath', 'FilterSourceFile', 'LocalPaths',
@@ skipping 1731 matching lines @@
         owners_check=False)
     self.assertEqual(1, len(results))
     self.assertEqual(
         'Found line ending with white spaces in:', results[0]._message)
     self.checkstdout('')
 
 
 if __name__ == '__main__':
   import unittest
   unittest.main()