[polymer] Merge of master into polymer10-migration

telemetry/telemetry/internal/backends/chrome/desktop_browser_backend.py

 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import datetime
 import glob
 import heapq
 import logging
 import os
 import os.path
@@ skipping 96 matching lines @@
     super(DesktopBrowserBackend, self).__init__(
         desktop_platform_backend,
         supports_tab_control=not is_content_shell,
         supports_extensions=not is_content_shell,
         browser_options=browser_options)
 
     # Initialize fields so that an explosion during init doesn't break in Close.
     self._proc = None
     self._tmp_profile_dir = None
     self._tmp_output_file = None
+    self._most_recent_symbolized_minidump_paths = set([])
 
     self._executable = executable
     if not self._executable:
       raise Exception('Cannot create browser, no executable found!')
 
     assert not flash_path or os.path.exists(flash_path)
     self._flash_path = flash_path
 
     self._is_content_shell = is_content_shell
 
     extensions_to_load = browser_options.extensions_to_load
 
     if len(extensions_to_load) > 0 and is_content_shell:
       raise browser_backend.ExtensionsNotSupportedException(
           'Content shell does not support extensions.')
 
     self._browser_directory = browser_directory
     self._port = None
     self._tmp_minidump_dir = tempfile.mkdtemp()
-    if self.browser_options.enable_logging:
+    if self.is_logging_enabled:
       self._log_file_path = os.path.join(tempfile.mkdtemp(), 'chrome.log')
     else:
       self._log_file_path = None
 
     self._SetupProfile()
 
   @property
+  def is_logging_enabled(self):
+    return self.browser_options.logging_verbosity in [
+        self.browser_options.NON_VERBOSE_LOGGING,
+        self.browser_options.VERBOSE_LOGGING]
+
+  @property
   def log_file_path(self):
     return self._log_file_path
 
   @property
   def supports_uploading_logs(self):
     return (self.browser_options.logs_cloud_bucket and self.log_file_path and
             os.path.isfile(self.log_file_path))
 
   def _SetupProfile(self):
     if not self.browser_options.dont_override_profile:
@@ skipping 107 matching lines @@
   def Start(self):
     assert not self._proc, 'Must call Close() before Start()'
 
     args = [self._executable]
     args.extend(self.GetBrowserStartupArgs())
     if self.browser_options.startup_url:
       args.append(self.browser_options.startup_url)
     env = os.environ.copy()
     env['CHROME_HEADLESS'] = '1'  # Don't upload minidumps.
     env['BREAKPAD_DUMP_LOCATION'] = self._tmp_minidump_dir
-    if self.browser_options.enable_logging:
+    if self.is_logging_enabled:
       sys.stderr.write(
         'Chrome log file will be saved in %s\n' % self.log_file_path)
       env['CHROME_LOG_FILE'] = self.log_file_path
     logging.info('Starting Chrome %s', args)
     if not self.browser_options.show_stdout:
       self._tmp_output_file = tempfile.NamedTemporaryFile('w', 0)
       self._proc = subprocess.Popen(
           args, stdout=self._tmp_output_file, stderr=subprocess.STDOUT, env=env)
     else:
       self._proc = subprocess.Popen(args, env=env)
@@ skipping 37 matching lines @@
         # recursive call to this function.
         print >> sys.stderr, "Can't get standard output with --show-stdout"
       return ''
     self._tmp_output_file.flush()
     try:
       with open(self._tmp_output_file.name) as f:
         return f.read()
     except IOError:
       return ''
 
-  def _GetMostRecentCrashpadMinidump(self):
+  def _GetAllCrashpadMinidumps(self):
     os_name = self.browser.platform.GetOSName()
     arch_name = self.browser.platform.GetArchName()
     try:
       crashpad_database_util = binary_manager.FetchPath(
           'crashpad_database_util', arch_name, os_name)
       if not crashpad_database_util:
         logging.warning('No crashpad_database_util found')
         return None
     except dependency_manager.NoPathFoundError:
       logging.warning('No path to crashpad_database_util found')
@@ skipping 38 matching lines @@
     # Include the last report.
     if report_dict:
       try:
         report_time = ParseCrashpadDateTime(report_dict['Creation time'])
         report_path = report_dict['Path'].strip()
         reports_list.append((report_time, report_path))
       except (ValueError, KeyError) as e:
         logging.warning('Crashpad report expected valid keys'
                           ' "Path" and "Creation time": %s', e)
 
+    return reports_list
+
+  def _GetMostRecentCrashpadMinidump(self):
+    reports_list = self._GetAllCrashpadMinidumps()
     if reports_list:
       _, most_recent_report_path = max(reports_list)
       return most_recent_report_path
 
     return None
 
+  def _GetBreakPadMinidumpPaths(self):
+    return glob.glob(os.path.join(self._tmp_minidump_dir, '*.dmp'))
+
   def _GetMostRecentMinidump(self):
     # Crashpad dump layout will be the standard eventually, check it first.
     most_recent_dump = self._GetMostRecentCrashpadMinidump()
 
     # Typical breakpad format is simply dump files in a folder.
     if not most_recent_dump:
       logging.info('No minidump found via crashpad_database_util')
-      dumps = glob.glob(os.path.join(self._tmp_minidump_dir, '*.dmp'))
+      dumps = self._GetBreakPadMinidumpPaths()
       if dumps:
         most_recent_dump = heapq.nlargest(1, dumps, os.path.getmtime)[0]
         if most_recent_dump:
           logging.info('Found minidump via globbing in minidump dir')
 
     # As a sanity check, make sure the crash dump is recent.
     if (most_recent_dump and
         os.path.getmtime(most_recent_dump) < (time.time() - (5 * 60))):
       logging.warning('Crash dump is older than 5 minutes. May not be correct.')
 
@@ skipping 16 matching lines @@
     else:
       return False
 
   def _GetStackFromMinidump(self, minidump):
     os_name = self.browser.platform.GetOSName()
     if os_name == 'win':
       cdb = self._GetCdbPath()
       if not cdb:
         logging.warning('cdb.exe not found.')
         return None
+      # Include all the threads' stacks ("~*kb30") in addition to the
+      # ostensibly crashed stack associated with the exception context
+      # record (".ecxr;kb30"). Note that stack dumps, including that
+      # for the crashed thread, may not be as precise as the one
+      # starting from the exception context record.
+      # Specify kb instead of k in order to get four arguments listed, for
+      # easier diagnosis from stacks.
       output = subprocess.check_output([cdb, '-y', self._browser_directory,
-                                        '-c', '.ecxr;k30;q', '-z', minidump])
+                                        '-c', '.ecxr;kb30;~*kb30;q',
+                                        '-z', minidump])
       # cdb output can start the stack with "ChildEBP", "Child-SP", and possibly
       # other things we haven't seen yet. If we can't find the start of the
       # stack, include output from the beginning.
       stack_start = 0
       stack_start_match = re.search("^Child(?:EBP|-SP)", output, re.MULTILINE)
       if stack_start_match:
         stack_start = stack_start_match.start()
       stack_end = output.find('quit:')
       return output[stack_start:stack_end]
 
@@ skipping 33 matching lines @@
   def GetStackTrace(self):
     """Returns a stack trace if a valid minidump is found, will return a tuple
        (valid, output) where valid will be True if a valid minidump was found
        and output will contain either an error message or the attempt to
        symbolize the minidump if one was found.
     """
     most_recent_dump = self._GetMostRecentMinidump()
     if not most_recent_dump:
       return (False, 'No crash dump found.')
     logging.info('Minidump found: %s' % most_recent_dump)
-    stack = self._GetStackFromMinidump(most_recent_dump)
+    return self._InternalSymbolizeMinidump(most_recent_dump)
+
+  def GetMostRecentMinidumpPath(self):
+    return self._GetMostRecentMinidump()
+
+  def GetAllMinidumpPaths(self):
+    reports_list = self._GetAllCrashpadMinidumps()
+    if reports_list:
+      return [report[1] for report in reports_list]
+    else:
+      logging.info('No minidump found via crashpad_database_util')
+      dumps = self._GetBreakPadMinidumpPaths()
+      if dumps:
+        logging.info('Found minidump via globbing in minidump dir')
+        return dumps
+      return None
+
+  def GetAllUnsymbolizedMinidumpPaths(self):
+    minidump_paths = set(self.GetAllMinidumpPaths())
+    # If we have already symbolized paths remove them from the list
+    unsymbolized_paths = (minidump_paths
+      - self._most_recent_symbolized_minidump_paths)
+    return list(unsymbolized_paths)
+
+  def SymbolizeMinidump(self, minidump_path):
+    return self._InternalSymbolizeMinidump(minidump_path)
+
+  def _InternalSymbolizeMinidump(self, minidump_path):
+    stack = self._GetStackFromMinidump(minidump_path)
     if not stack:
-      cloud_storage_link = self._UploadMinidumpToCloudStorage(most_recent_dump)
+      cloud_storage_link = self._UploadMinidumpToCloudStorage(minidump_path)
       error_message = ('Failed to symbolize minidump. Raw stack is uploaded to'
                        ' cloud storage: %s.' % cloud_storage_link)
       return (False, error_message)
 
+    self._most_recent_symbolized_minidump_paths.add(minidump_path)
     return (True, stack)
 
   def __del__(self):
     self.Close()
 
   def _TryCooperativeShutdown(self):
     if self.browser.platform.IsCooperativeShutdownSupported():
       # Ideally there would be a portable, cooperative shutdown
       # mechanism for the browser. This seems difficult to do
       # correctly for all embedders of the content API. The only known
@@ skipping 41 matching lines @@
         shutil.rmtree(self._tmp_profile_dir, ignore_errors=True)
         self._tmp_profile_dir = None
 
     if self._tmp_output_file:
       self._tmp_output_file.close()
       self._tmp_output_file = None
 
     if self._tmp_minidump_dir:
       shutil.rmtree(self._tmp_minidump_dir, ignore_errors=True)
       self._tmp_minidump_dir = None