Add new Android test runner command to handle linker tests.

build/android/pylib/linker/test_case.py

+# Copyright 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Base class for linker-specific test cases.
+
+   The custom dynamic linker can only be tested through a custom test case
+   for various technical reasons:
+
+     - It's an 'invisible feature', i.e. it doesn't expose a new API or
+       behaviour, all it does is save RAM when loading native libraries.
+
+     - Checking that it works correctly requires several things that do not
+       fit the existing GTest-based and instrumentation-based tests:
+
+         - Native test code needs to be run in both the browser and renderer
+           process at the same time just after loading native libraries, in
+           a completely asynchronous way.
+
+         - Each test case requires restarting a whole new application process
+           with a different command-line.
+
+         - Enabling test support in the Linker code requires building a special
+           APK with a flag to activate special test-only support code in the
+           Linker code itself.
+
+       Host-driven tests have also been tried, but since they're really
+       sub-classes of instrumentation tests, they didn't work well either.
+
+   To build and run the linker tests, do the following:
+
+     ninja -C out/Debug content_linker_test_apk
+     build/android/test_runner.py linker
+
+   The core of the checks performed here are pretty simple:
+
+     - Clear the logcat and start recording with an appropriate set of filters.
+     - Create the command-line appropriate for the test-case.
+     - Start the activity (always forcing a cold start).
+     - Every second, look at the current content of the filtered logcat lines
+       and look for instances of the following:
+
+          BROWSER_LINKER_TEST: <status>
+          RENDERER_LINKER_TEST: <status>
+
+       where <status> can be either FAIL or SUCCESS. These lines can appear
+       in any order in the logcat. Once both browser and renderer status are
+       found, stop the loop. Otherwise timeout after 30 seconds.
+
+       Note that there can be other lines beginning with BROWSER_LINKER_TEST:
+       and RENDERER_LINKER_TEST:, but are not followed by a <status> code.
+
+     - The test case passes if the <status> for both the browser and renderer
+       process are SUCCESS. Otherwise its a fail.
+"""
+
+import logging
+import os
+import StringIO
+import subprocess
+import tempfile
+import time
+
+from pylib import android_commands
+from pylib.base import base_test_result
+
+_LINKER_TAG = 'ContentLinkerTest'
+_PACKAGE_NAME='org.chromium.content_linker_test_apk'
+_ACTIVITY_NAME='.ContentLinkerTestActivity'
+_COMMAND_LINE_FILE='/data/local/tmp/content-linker-test-command-line'
+
+# Logcat filters used during each test. Only the 'chromium' one is really
+# needed, but the logs are added to the TestResult in case of error, and
+# it is handy to have the 'content_android_linker' ones as well when
+# troubleshooting.
+_LOGCAT_FILTERS = [ '*:s', 'chromium:v', 'content_android_linker:v' ]
+
+
+def _CheckPrefixedTestStatus(logcat, prefix):

[frankf, 2013/10/02 18:16:02]

Are you not using regex in case logcat is very large?

[digit1, 2013/10/03 09:16:00]

The logcat won't be very large, due to the filters being used here and the fact
that this only collects traces at program startup. I've changed it to a regex
though.

+  """Parse the content of |logcat| for some text that begins
+     with |prefix| and is followed by 'FAIL' or 'SUCCESS'.
+
+  Args:
+    logcat: A string to parse. Can include line separators.
+    prefix: The status prefix
+
+  Returns:
+    A tuple, result[0] is True is a line was found, then

[frankf, 2013/10/02 18:16:02]

is -> if

[digit1, 2013/10/03 09:16:00]

Done.

+    result[1] will be True for 'SUCCESS' and False for 'Fail'
+  """
+  start = 0
+  while True:
+    n = logcat.find(prefix, start)
+    if n < 0:
+      return (False, None)
+
+    n += len(prefix)
+    if logcat.find('FAIL', n) == n:
+      return (True, False)
+    if logcat.find('SUCCESS', n) == n:
+      return (True, True)
+
+    start = n
+
+
+def _CheckLinkerTestStatus(logcat):
+  """Parse the content of |logcat| and checks for both a browser and
+     renderer status line.
+
+  Args:
+    logcat: A string to parse. Can include line separators.
+
+  Returns:
+    A tuple, result[0] is True if there is a complete match, then
+    result[1] and result[2] will be True or False to reflect the
+    test status for the browser and renderer processes, respectively.
+  """
+  browser_found, browser_success = _CheckPrefixedTestStatus(
+      logcat, 'BROWSER_LINKER_TEST: ')
+  renderer_found, renderer_success = _CheckPrefixedTestStatus(
+      logcat, 'RENDERER_LINKER_TEST: ')
+
+  if browser_found and renderer_found:
+    return (True, browser_success, renderer_success)
+  return (False, None, None)

[frankf, 2013/10/02 18:16:02]

duplicate of line 128

[digit1, 2013/10/03 09:16:00]

Done.

+
+  # Didn't find anything.
+  return (False, None, None)
+
+
+def _CreateCommandLineFileOnDevice(adb, cmd_line):

[frankf, 2013/10/02 18:16:02]

you can use flag_changer.py 

[digit1, 2013/10/03 09:16:00]

Done.

+  """Create a command-line file on the device.
+  Args:
+    adb: An AndroidCommands instance to communicate with the device.
+    cmd_line: The command-line as a string.
+  """
+  command_line_file = tempfile.NamedTemporaryFile()
+  command_line_file.write(cmd_line)
+  command_line_file.flush()
+  adb.PushIfNeeded(command_line_file.name, _COMMAND_LINE_FILE)
+
+
+class LinkerTestCase(object):
+  """Base class for linker test cases."""
+
+  def __init__(self, test_name, is_low_memory=False):
+    """Create a test case initialized to run |test_name|.
+
+    Args:
+      test_name: The name of the method to run as the test.
+      is_low_memory: True to simulate a low-memory device, False otherwise.
+    """
+    self.test_name = test_name
+    class_name = self.__class__.__name__
+    self.qualified_name = '%s.%s' % (class_name, self.test_name)
+    # Use tagged_name when creating results, so that we can identify linker
+    # tests in the overall results.

[frankf, 2013/10/02 18:16:02]

isn't class_name sufficient for this. Note the linker tests would run as a
separate step on the bots since they have a different invocations
(test_runner.py linker ...), so there wouldn't be a mix up.

[digit1, 2013/10/03 09:16:00]

The class name itself won't be sufficient, but self.qualified name can be. I've
changed the code to use that instead. (FWIW, this part also came originally from
host_driven.TestCase).

+    self.tagged_name = '%s_%s' % (_LINKER_TAG, self.test_name)
+    self.is_low_memory = is_low_memory
+
+  def Run(self, device):
+    margin = 8
+    print "[ %-*s ] %s" % (margin, 'RUN', self.tagged_name)

[frankf, 2013/10/02 18:16:02]

single quotes for consistancy

[digit1, 2013/10/03 09:16:00]

Done.

+    logging.info('Running linker test: %s', self.tagged_name)
+    adb = android_commands.AndroidCommands(device)
+
+    # 1. Write command-line file with appropriate options.
+    command_line = ''
+    if self.is_low_memory:
+      command_line = '--low-memory-device'
+    _CreateCommandLineFileOnDevice(adb, command_line)
+
+    # 2. Start recording logcat with appropriate filters.
+    adb.StartRecordingLogcat(clear=True, filters=_LOGCAT_FILTERS)
+
+    try:
+      # 3. Force-start activity.
+      adb.StartActivity(package=_PACKAGE_NAME,
+                        activity=_ACTIVITY_NAME,
+                        force_stop=True)
+
+      # 4. Wait up to 30 seconds until the linker test status is in the logcat.
+      max_tries = 30
+      num_tries = 0
+      found = False
+      logcat = None
+      while num_tries < max_tries:
+        time.sleep(1)
+        num_tries += 1
+        found, browser_ok, renderer_ok = _CheckLinkerTestStatus(
+            adb.GetCurrentRecordedLogcat())
+        if found:
+          break
+
+    finally:
+      # Ensure the ADB polling process is always killed when
+      # the script is interrupted by the user with Ctrl-C.
+      logs = adb.StopRecordingLogcat()
+
+    results = base_test_result.TestRunResults()
+
+    if num_tries >= max_tries:
+      # Timeout
+      print '[ %*s ] %s' % (margin, 'TIMEOUT', self.tagged_name)
+      results.AddResult(
+          base_test_result.BaseTestResult(
+              self.test_name,
+              base_test_result.ResultType.TIMEOUT,
+              logs))
+    elif browser_ok and renderer_ok:
+      # Passed
+      logging.info(
+          'Logcat start ---------------------------------\n%s' +
+          'Logcat end -----------------------------------', logs)
+      print '[ %*s ] %s' % (margin, 'OK', self.tagged_name)
+      results.AddResult(
+          base_test_result.BaseTestResult(
+              self.test_name,
+              base_test_result.ResultType.PASS))
+    else:
+      print '[ %*s ] %s' % (margin, 'FAILED', self.tagged_name)
+      # Failed
+      results.AddResult(
+          base_test_result.BaseTestResult(
+              self.test_name,
+              base_test_result.ResultType.FAIL,
+              logs))
+
+    return results
+
+  def __str__(self):
+    return self.tagged_name
+
+  def __repr__(self):
+    return self.tagged_name