[android] Switch test scripts to use last modified time instead of md5sum when checking dependencie…

build/android/pylib/android_commands.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Provides an interface to communicate with the device via the adb command.
 
 Assumes adb binary is currently on system path.
 """
 
 import collections
@@ skipping 43 matching lines @@
 
 # Keycode "enum" suitable for passing to AndroidCommands.SendKey().
 KEYCODE_HOME = 3
 KEYCODE_BACK = 4
 KEYCODE_DPAD_UP = 19
 KEYCODE_DPAD_DOWN = 20
 KEYCODE_DPAD_RIGHT = 22
 KEYCODE_ENTER = 66
 KEYCODE_MENU = 82
 
-MD5SUM_DEVICE_FOLDER = constants.TEST_EXECUTABLE_DIR + '/md5sum/'
-MD5SUM_DEVICE_PATH = MD5SUM_DEVICE_FOLDER + 'md5sum_bin'
-MD5SUM_LD_LIBRARY_PATH = 'LD_LIBRARY_PATH=%s' % MD5SUM_DEVICE_FOLDER
-
 def GetEmulators():
   """Returns a list of emulators.  Does not filter by status (e.g. offline).
 
   Both devices starting with 'emulator' will be returned in below output:
 
     * daemon not running. starting it now on port 5037 *
     * daemon started successfully *
     List of devices attached
     027c10494100b4d7        device
     emulator-5554   offline
@@ skipping 91 matching lines @@
         lastmod -= utc_delta
       files[filename] = (int(file_match.group('size')), lastmod)
   return files
 
 
 def _ComputeFileListHash(md5sum_output):
   """Returns a list of MD5 strings from the provided md5sum output."""
   return [line.split('  ')[0] for line in md5sum_output]
 
 
+def _ComputeFileListTimesModified(timemodified_output):
+  """Returns a list of tuples from the provided timemodified output.
+
+  Args:
+    timemodified_output: output directly from timemodified binary.
+
+  Returns:
+    List of namedtuples (time, path).
+  """
+  TimeAndPath = collections.namedtuple('TimeAndPath', ['time', 'path'])
+  split_lines = [line.split('  ') for line in timemodified_output]
+  assert all(len(s) == 2 for s in split_lines), 'Invalid timemodified output'
+  return [TimeAndPath._make(s) for s in split_lines]
+
+
 def _HasAdbPushSucceeded(command_output):
   """Returns whether adb push has succeeded from the provided output."""
   # TODO(frankf): We should look at the return code instead of the command
   # output for many of the commands in this file.
   if not command_output:
     return True
   # Success looks like this: "3035 KB/s (12512056 bytes in 4.025s)"
   # Errors look like this: "failed to copy  ... "
   if not re.search('^[0-9]', command_output.splitlines()[-1]):
     logging.critical('PUSH FAILED: ' + command_output)
@@ skipping 29 matching lines @@
       self._adb.SetTargetSerial(device)
     self._device = device
     self._logcat = None
     self.logcat_process = None
     self._logcat_tmpoutfile = None
     self._pushed_files = []
     self._device_utc_offset = None
     self._potential_push_size = 0
     self._actual_push_size = 0
     self._md5sum_build_dir = ''
+    self._timemodified_build_dir = ''
     self._external_storage = ''
     self._util_wrapper = ''
 
   def _LogShell(self, cmd):
     """Logs the adb shell command."""
     if self._device:
       device_repr = self._device[-4:]
     else:
       device_repr = '????'
     logging.info('[%s]> %s', device_repr, cmd)
@@ skipping 467 matching lines @@
 
     Args:
       local_path: Path (file or directory) on the host.
       device_path: Path on the device.
       ignore_paths: If False, both the md5sum and the relative paths/names of
           files must match. If True, only the md5sum must match.
 
     Returns:
       True if the md5sums match.
     """
+    MD5SUM_DEVICE_FOLDER = constants.TEST_EXECUTABLE_DIR + '/md5sum/'
+    MD5SUM_DEVICE_PATH = MD5SUM_DEVICE_FOLDER + 'md5sum_bin'
+    MD5SUM_LD_LIBRARY_PATH = 'LD_LIBRARY_PATH=%s' % MD5SUM_DEVICE_FOLDER
     if not self._md5sum_build_dir:
       default_build_type = os.environ.get('BUILD_TYPE', 'Debug')
       build_dir = '%s/%s/' % (
           cmd_helper.OutDirectory().get(), default_build_type)
       md5sum_dist_path = '%s/md5sum_dist' % build_dir
       if not os.path.exists(md5sum_dist_path):
         build_dir = '%s/Release/' % cmd_helper.OutDirectory().get()
         md5sum_dist_path = '%s/md5sum_dist' % build_dir
         assert os.path.exists(md5sum_dist_path), 'Please build md5sum.'
       command = 'push %s %s' % (md5sum_dist_path, MD5SUM_DEVICE_FOLDER)
       assert _HasAdbPushSucceeded(self._adb.SendCommand(command))
       self._md5sum_build_dir = build_dir
 
     hashes_on_device = _ComputeFileListHash(
         self.RunShellCommand(MD5SUM_LD_LIBRARY_PATH + ' ' + self._util_wrapper +
             ' ' + MD5SUM_DEVICE_PATH + ' ' + device_path))
     assert os.path.exists(local_path), 'Local path not found %s' % local_path
     md5sum_output = cmd_helper.GetCmdOutput(
         ['%s/md5sum_bin_host' % self._md5sum_build_dir, local_path])
     hashes_on_host = _ComputeFileListHash(md5sum_output.splitlines())
 
     if ignore_paths:
       hashes_on_device = [h.split()[0] for h in hashes_on_device]
       hashes_on_host = [h.split()[0] for h in hashes_on_host]
 
     return hashes_on_device == hashes_on_host
 
+  def AreFilesUnsynchronized(self, local_path, device_path):

[bulach, 2013/07/15 18:35:20]

nit: make this internal, "_" prefix

[craigdh, 2013/07/15 19:06:50]

Done.

+    """Compares the time modified of a local path against a device path.

[bulach, 2013/07/15 18:35:20]

could probably add a note saying that files transferred via "adb push" will keep
the host's modified time.

[craigdh, 2013/07/15 19:06:50]

Done.

+
+    Args:
+      local_path: Path (file or directory) on the host.
+      device_path: Path (file or directory) on the device.
+
+    Returns:
+      True if the time the files were modified match.
+    """
+    TIMEMODIFIED_DEVICE_DIRECTORY = (constants.TEST_EXECUTABLE_DIR +
+                                     '/timemodified/')
+    TIMEMODIFIED_DEVICE_PATH = (TIMEMODIFIED_DEVICE_DIRECTORY +
+                                'timemodified_bin')
+    TIMEMODIFIED_LD_LIBRARY_PATH = ('LD_LIBRARY_PATH=%s' %
+                                    TIMEMODIFIED_DEVICE_DIRECTORY)
+    assert os.path.exists(local_path), 'Local path not found %s' % local_path
+
+    if not self._timemodified_build_dir:
+      default_build_type = os.environ.get('BUILD_TYPE', 'Debug')
+      build_dir = '%s/%s/' % (
+          cmd_helper.OutDirectory().get(), default_build_type)
+      host_timemodified_dist_path = '%s/timemodified_dist' % build_dir
+      if not os.path.exists(host_timemodified_dist_path):
+        build_dir = '%s/Release/' % cmd_helper.OutDirectory().get()
+        host_timemodified_dist_path = '%s/timemodified_dist' % build_dir
+        assert os.path.exists(host_timemodified_dist_path), 'Build timemodified'
+      command = 'push %s %s' % (host_timemodified_dist_path,
+                                TIMEMODIFIED_DEVICE_DIRECTORY)
+      assert _HasAdbPushSucceeded(self._adb.SendCommand(command))
+      self._timemodified_build_dir = build_dir
+
+    self._pushed_files.append(device_path)
+    device_timemodified_output = self.RunShellCommand(
+        TIMEMODIFIED_LD_LIBRARY_PATH + ' ' + self._util_wrapper + ' ' +
+        TIMEMODIFIED_DEVICE_PATH + ' ' + device_path)
+    device_time_tuples = _ComputeFileListTimesModified(
+        device_timemodified_output)
+    assert os.path.exists(local_path), 'Local path not found %s' % local_path
+    host_timemodified_output = cmd_helper.GetCmdOutput(
+        ['%s/timemodified_bin_host' % self._timemodified_build_dir, local_path])
+    host_time_tuples = _ComputeFileListTimesModified(
+        host_timemodified_output.splitlines())
+
+    if len(device_time_tuples) < len(host_time_tuples):
+      return True
+
+    # Ignore extra files on the device.
+    if len(device_time_tuples) > len(host_time_tuples):
+      host_files = [os.path.relpath(os.path.normpath(p.path),
+                    os.path.normpath(local_path)) for p in host_time_tuples]
+
+      def _host_has(fname):
+        return any(path in fname for path in host_files)
+
+      times_on_device = [t.time for t in device_time_tuples if
+                         _host_has(t.path)]
+    else:
+      times_on_device = [t.time for t in device_time_tuples]
+
+    # Compare timestamps between host and device files.
+    times_on_host = [t.time for t in host_time_tuples]
+    times_on_device.sort()
+    times_on_host.sort()
+    return any(int(dtime) < int(htime) for dtime, htime in

[bulach, 2013/07/15 18:35:20]

should we check for != instead? i.e., a file modified in the device should
probably be pushed again?

[craigdh, 2013/07/15 19:06:50]

Done.

+        zip(times_on_device, times_on_host))
+
   def PushIfNeeded(self, local_path, device_path):
     """Pushes |local_path| to |device_path|.
 
     Works for files and directories. This method skips copying any paths in
-    |test_data_paths| that already exist on the device with the same hash.
+    |test_data_paths| that already exist on the device with the same modified
+    time.
 
     All pushed files can be removed by calling RemovePushedFiles().
     """
     assert os.path.exists(local_path), 'Local path not found %s' % local_path
     size = int(cmd_helper.GetCmdOutput(['du', '-sb', local_path]).split()[0])
     self._pushed_files.append(device_path)
     self._potential_push_size += size
 
-    if self.CheckMd5Sum(local_path, device_path):
+    if not self.AreFilesUnsynchronized(local_path, device_path):
       return
 
     self._actual_push_size += size
     # They don't match, so remove everything first and then create it.
     if os.path.isdir(local_path):
       self.RunShellCommand('rm -r %s' % device_path, timeout_time=2 * 60)
       self.RunShellCommand('mkdir -p %s' % device_path)
 
     # NOTE: We can't use adb_interface.Push() because it hardcodes a timeout of
     # 60 seconds which isn't sufficient for a lot of users of this method.
@@ skipping 531 matching lines @@
     host_dir = os.path.dirname(host_file)
     if not os.path.exists(host_dir):
       os.makedirs(host_dir)
     device_file = '%s/screenshot.png' % self.GetExternalStorage()
     self.RunShellCommand('/system/bin/screencap -p %s' % device_file)
     assert self._adb.Pull(device_file, host_file)
     assert os.path.exists(host_file)
 
   def SetUtilWrapper(self, util_wrapper):
     """Sets a wrapper prefix to be used when running a locally-built
-    binary on the device (ex.: md5sum_bin).
+    binary on the device (ex.: timemodified_bin).
     """
     self._util_wrapper = util_wrapper
 
   def RunInstrumentationTest(self, test, test_package, instr_args, timeout):
     """Runs a single instrumentation test.
 
     Args:
       test: Test class/method.
       test_package: Package name of test apk.
       instr_args: Extra key/value to pass to am instrument.
@@ skipping 49 matching lines @@
   """
   def __init__(self, output):
     self._output = output
 
   def write(self, data):
     data = data.replace('\r\r\n', '\n')
     self._output.write(data)
 
   def flush(self):
     self._output.flush()