Reland: Make stack tool works for Monochrome

third_party/android_platform/development/scripts/stack_core.py

Index: third_party/android_platform/development/scripts/stack_core.py
diff --git a/third_party/android_platform/development/scripts/stack_core.py b/third_party/android_platform/development/scripts/stack_core.py
index ddaaec26561bf969757ae8b1280f44066834454f..1dceb2ef66b795b236b8c2f0f9914454622a61c2 100755
--- a/third_party/android_platform/development/scripts/stack_core.py
+++ b/third_party/android_platform/development/scripts/stack_core.py
@@ -21,11 +21,16 @@ import logging
 import multiprocessing
 import os
 import re
+import struct
 import subprocess
 import time
+import zipfile
 
 import symbol
 
+
+from pylib import constants
+
 UNKNOWN = '<unknown>'
 HEAP = '[heap]'
 STACK = '[stack]'
@@ -33,8 +38,9 @@ _DEFAULT_JOBS=8
 _CHUNK_SIZE = 1000
 
 _BASE_APK = 'base.apk'
-_LIBCHROME_SO = 'libchrome.so'
+_FALLBACK_SO = 'libchrome.so'
 
+_ABI_LINE = re.compile('ABI: \'(?P<abi>[a-z0-9A-Z]+)\'')
 _PROCESS_INFO_LINE = re.compile('(pid: [0-9]+, tid: [0-9]+.*)')
 # Same as above, but used to extract the pid.
 _PROCESS_INFO_PID = re.compile('pid: ([0-9]+)')
@@ -92,6 +98,11 @@ code_line = re.compile('(.*)[ \t]*[a-f0-9]' + _WIDTH + '[ \t]*[a-f0-9]' + _WIDTH
                        '[ \t]*[a-f0-9]' + _WIDTH + '[ \t]*[a-f0-9]' + _WIDTH +
                        '[ \t]*[a-f0-9]' + _WIDTH + '[ \t]*[ \r\n]')  # pylint: disable-msg=C6310
 
+# This pattern is used to find shared library offset in APK.
+# Example:
+#    (offset 0x568000)
+_SHARED_LIB_OFFSET_IN_APK = re.compile(' \(offset 0x(?P<offset>[0-9a-f]{0,16})\)$')
+
 def PrintTraceLines(trace_lines):
   """Print back trace."""
   maxlen = max(map(lambda tl: len(tl[1]), trace_lines))
@@ -144,17 +155,34 @@ def PrintDivider():
   print
   print '-----------------------------------------------------\n'
 
-def ConvertTrace(lines, load_vaddrs, more_info):
+def ConvertTrace(lines, load_vaddrs, more_info, fallback_monochrome, arch_defined):
   """Convert strings containing native crash to a stack."""
+  if fallback_monochrome:
+    global _FALLBACK_SO
+    _FALLBACK_SO = 'libmonochrome.so'
   start = time.time()
 
   chunks = [lines[i: i+_CHUNK_SIZE] for i in xrange(0, len(lines), _CHUNK_SIZE)]
   pool = multiprocessing.Pool(processes=_DEFAULT_JOBS)
-  useful_log = itertools.chain(*pool.map(PreProcessLog(load_vaddrs), chunks))
+  results = pool.map(PreProcessLog(load_vaddrs), chunks)
+  useful_log = []
+  so_dirs = []
+  for result in results:
+    useful_log += result[0]
+    so_dirs += result[1]
   pool.close()
   pool.join()
   end = time.time()
   logging.debug('Finished processing. Elapsed time: %.4fs', (end - start))
+  if so_dirs:
+    UpdateLibrarySearchPath(so_dirs)
+
+  # if arch isn't defined in command line, find it from log
+  if not arch_defined:
+    arch = FindAbi(useful_log)
+    if arch:
+      print ('Find ABI:' + arch)
+      symbol.ARCH = arch
 
   ResolveCrashSymbol(list(useful_log), more_info)
   end = time.time()
@@ -169,6 +197,37 @@ class PreProcessLog:
       load_vaddrs: LOAD segment min_vaddrs keyed on mapped executable
     """
     self._load_vaddrs = load_vaddrs;
+    # This is mapping from apk's offset to shared libraries.
+    self._shared_libraries_mapping = dict()
+    # The list of directires in which instead of default output dir,
+    # the shared libraries is found.
+    self._so_dirs = []
+
+  def _DetectSharedLibrary(self, lib, symbol_present):
+    """Detect the possible shared library from the mapping offset of APK
+    Return:
+       the shared library in APK if only one is found.
+    """
+    offset_match = _SHARED_LIB_OFFSET_IN_APK.match(symbol_present)
+    if not offset_match:
+      return
+    offset = offset_match.group('offset')
+    key = '%s:%s' % (lib, offset)
+    if self._shared_libraries_mapping.has_key(key):
+      soname = self._shared_libraries_mapping[key]
+    else:
+      soname, host_so = FindSharedLibraryFromAPKs(constants.GetOutDirectory(),
+                                                  int(offset, 16))
+      if soname:
+        self._shared_libraries_mapping[key] = soname
+        so_dir = os.path.dirname(host_so)
+        # Store the directory if it is not the default output dir, so
+        # we can update library search path in main process.
+        if not os.path.samefile(constants.GetOutDirectory(), so_dir):
+          self._so_dirs.append(so_dir)
+        print ("Detected: %s is %s which is loaded directly from APK."
+                % (host_so, soname))
+    return soname
 
   def _AdjustAddress(self, address, lib):
     """Add the vaddr of the library's first LOAD segment to address.
@@ -205,18 +264,25 @@ class PreProcessLog:
           or _DALVIK_JNI_THREAD_LINE.search(line)
           or _DALVIK_NATIVE_THREAD_LINE.search(line)
           or _LOG_FATAL_LINE.search(line)
-          or _DEBUG_TRACE_LINE.match(line)
+          or _DEBUG_TRACE_LINE.search(line)
+          or _ABI_LINE.search(line)
           or _JAVA_STDERR_LINE.search(line)):
         useful_log.append(line)
         continue
 
       match = _TRACE_LINE.match(line)
       if match:
-        # If the trace line suggests a direct load from APK, replace the
-        # APK name with libchrome.so. Current load from APK supports only
-        # single library load, so it must be libchrome.so that was loaded
-        # in this way.
-        line = line.replace('/' + _BASE_APK, '/' + _LIBCHROME_SO)
+        lib, symbol_present = match.group('lib', 'symbol_present')
+        if os.path.splitext(lib)[1] == '.apk':
+          soname = self._DetectSharedLibrary(lib, symbol_present)
+          if soname:
+            line = line.replace('/' + os.path.basename(lib), '/' + soname)
+          else:
+            # If the trace line suggests a direct load from APK, replace the
+            # APK name with _FALLBACK_SO.
+            line = line.replace('/' + _BASE_APK, '/' + _FALLBACK_SO)
+            logging.debug("Can't detect shared library in APK, fallback to" +
+                          " library " + _FALLBACK_SO)
         # For trace lines specifically, the address may need to be adjusted
         # to account for relocation packing. This is because debuggerd on
         # pre-M platforms does not understand non-zero vaddr LOAD segments.
@@ -231,7 +297,7 @@ class PreProcessLog:
         continue
       if _VALUE_LINE.match(line):
         useful_log.append(line)
-    return useful_log
+    return useful_log, self._so_dirs
 
 def ResolveCrashSymbol(lines, more_info):
   """Convert unicode strings which contains native crash to a stack
@@ -385,3 +451,98 @@ def ResolveCrashSymbol(lines, more_info):
   if pid != -1 and pid in java_stderr_by_pid:
     java_lines = java_stderr_by_pid[pid]
   PrintOutput(trace_lines, value_lines, java_lines, more_info)
+
+def UpdateLibrarySearchPath(so_dirs):
+  # All dirs in so_dirs must be same, since a dir represents the cpu arch.
+  so_dir = set(so_dirs)
+  so_dir_len = len(so_dir)
+  if so_dir_len > 0:
+    if so_dir_len > 1:
+      raise Exception("Found different so dirs, they are %s", repr(so_dir))
+    else:
+      search_path = so_dir.pop()
+      print "Search libraries in " + search_path
+      symbol.SetSecondaryAbiOutputPath(search_path)
+
+def GetUncompressedSharedLibraryFromAPK(apkname, offset):
+  """Check if there is uncompressed shared library at specifc offset of APK."""
+  FILE_NAME_LEN_OFFSET = 26
+  FILE_NAME_OFFSET = 30
+  soname = ""
+  sosize = 0
+  with zipfile.ZipFile(apkname, 'r') as apk:
+    for infoList in apk.infolist():
+      filename, file_extension = os.path.splitext(infoList.filename)
+      if (file_extension == '.so' and
+           infoList.file_size == infoList.compress_size):
+        with open(apkname, 'rb') as f:
+          f.seek(infoList.header_offset + FILE_NAME_LEN_OFFSET)
+          file_name_len = struct.unpack('H', f.read(2))[0]
+          extra_field_len = struct.unpack('H', f.read(2))[0]
+          file_offset = (infoList.header_offset + FILE_NAME_OFFSET +
+                         file_name_len + extra_field_len)
+          f.seek(file_offset)
+          if offset == file_offset and f.read(4) == "\x7fELF":
+            soname = infoList.filename
+            sosize = infoList.file_size
+            break
+  return soname, sosize
+
+def GetSharedLibraryInHost(soname, sosize, dirs):
+  """Find the shared library in given host directories by comparing
+     the name and size.
+  Return:
+     the shared libray in host if found.
+  """
+  for dir in dirs:
+    host_so_file = os.path.join(dir, os.path.basename(soname))
+    if not os.path.isfile(host_so_file):
+      continue
+    if sosize == os.stat(host_so_file).st_size:
+      logging.debug("%s match to the one in APK" % host_so_file)
+      return host_so_file
+
+def FindSharedLibraryFromAPKs(out_dir, offset):
+  """Find the shared library at the specifc offset of APK.
+  Return:
+     a pair of library which is in apk and host respectively
+     only if one library is found, otherwise, it means detecting
+     library failed.
+  """
+  apk_dir = os.path.join(out_dir, "apks")
+  if not os.path.isdir(apk_dir):
+    return (None, None)
+
+  apks = [os.path.join(apk_dir, f) for f in os.listdir(apk_dir)
+              if (os.path.isfile(os.path.join(apk_dir, f)) and
+                   os.path.splitext(f)[1] == '.apk')]
+  shared_libraries = []
+  for apk in apks:
+    soname, sosize = GetUncompressedSharedLibraryFromAPK(apk, offset)
+    if soname == "":
+      continue
+    # These are possible directores for the shared libraries,
+    # Beside out_dir, others are possible secondary abi output dir
+    # for Monochrome.
+    dirs = [out_dir,
+           os.path.join(out_dir, "arm_v8_arm"),
+           os.path.join(out_dir, "mips_v8_mips"),
+           os.path.join(out_dir, "x86"),
+           ]
+    host_so_file = GetSharedLibraryInHost(soname, sosize, dirs)
+    if host_so_file:
+      shared_libraries += [(soname, host_so_file)]
+  # If there are more than one libraries found, it means detecting
+  # library failed.
+  number_of_library = len(shared_libraries)
+  if number_of_library == 1:
+    return shared_libraries[0]
+  elif number_of_library > 1:
+    print "More than one libraries could be loaded from APK."
+  return (None, None)
+
+def FindAbi(lines):
+  for line in lines:
+    match = _ABI_LINE.search(line)
+    if match:
+      return match.group('abi')