Enable support for coredump archiving on windows

tools/utils.py

 # Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 # for details. All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
 # This file contains a set of utilities functions used by other Python-based
 # scripts.
 
 import commands
+import contextlib
 import datetime
 import glob
 import imp
 import json
 import os
 import platform
 import re
 import shutil
 import subprocess
 import sys
@@ skipping 580 matching lines @@
 #     https://github.com/dart-lang/sdk/wiki/The-checked-in-SDK-in-tools
 def CheckedInSdkPath():
   # We don't use the normal macos, linux, win32 directory names here, instead,
   # we use the names that the download_from_google_storage script uses.
   osdict = {'Darwin':'mac', 'Linux':'linux', 'Windows':'win'}
   system = platform.system()
   try:
     osname = osdict[system]
   except KeyError:
     print >>sys.stderr, ('WARNING: platform "%s" not supported') % (system)
-    return None;
+    return None
   tools_dir = os.path.dirname(os.path.realpath(__file__))
   return os.path.join(tools_dir,
                       'sdks',
                       osname,
                       'dart-sdk')
 
 
 def CheckedInSdkExecutable():
   name = 'dart'
   if IsWindows():
@@ skipping 22 matching lines @@
                                'canary.dart')
   try:
     with open(os.devnull, 'wb') as silent_sink:
       if 0 == subprocess.call([executable, canary_script], stdout=silent_sink):
         return True
   except OSError as e:
     pass
   return False
 
 
+def CheckLinuxCoreDumpPattern(fatal=False):
+  core_pattern_file = '/proc/sys/kernel/core_pattern'
+  core_pattern = open(core_pattern_file).read()
+
+  expected_core_pattern = 'core.%p'
+  if core_pattern.strip() != expected_core_pattern:
+    if fatal:
+      message = ('Invalid core_pattern configuration. '
+          'The configuration of core dump handling is *not* correct for '
+          'a buildbot. The content of {0} must be "{1}" instead of "{2}".'
+          .format(core_pattern_file, expected_core_pattern, core_pattern))
+      raise Exception(message)
+    else:
+      return False
+  return True
+
+
 class TempDir(object):
   def __init__(self, prefix=''):
     self._temp_dir = None
     self._prefix = prefix
 
   def __enter__(self):
     self._temp_dir = tempfile.mkdtemp(self._prefix)
     return self._temp_dir
 
   def __exit__(self, *_):
     shutil.rmtree(self._temp_dir, ignore_errors=True)
 
 class ChangedWorkingDirectory(object):
   def __init__(self, working_directory):
     self._working_directory = working_directory
 
   def __enter__(self):
     self._old_cwd = os.getcwd()
     print "Enter directory = ", self._working_directory
     os.chdir(self._working_directory)
 
   def __exit__(self, *_):
     print "Enter directory = ", self._old_cwd
     os.chdir(self._old_cwd)
 
-class CoreDump(object):
-  def __init__(self, test, core, binary):
+
+class UnexpectedCrash(object):
+  def __init__(self, test, pid, binary):
     self.test = test
-    self.core = core
+    self.pid = pid 
     self.binary = binary
 
   def __str__(self):
-    return "%s: %s %s" % (self.test, self.binary, self.core)
+    return "%s: %s %s" % (self.test, self.binary, self.pid)
 
-class CoreDumpArchiver(object):
+
+class PosixCoredumpEnabler(object):
+  def __init__(self):
+    self._old_limits = None
+
+  def __enter__(self):
+    self._old_limits = resource.getrlimit(resource.RLIMIT_CORE)
+
+    # Bump core limits to unlimited if core_pattern is correctly configured.
+    if CheckLinuxCoreDumpPattern(fatal=False):
+      resource.setrlimit(resource.RLIMIT_CORE, (-1, -1))
+
+  def __exit__(self, *_):
+    resource.setrlimit(resource.RLIMIT_CORE, self._old_limits)
+    CheckLinuxCoreDumpPattern(fatal=True)
+
+class WindowsCoredumpEnabler(object):
+  """Configure Windows Error Reporting to store crash dumps.
+
+  The documentation can be found here:
+  https://msdn.microsoft.com/en-us/library/windows/desktop/bb787181.aspx
+  """
+
+  WINDOWS_COREDUMP_FOLDER = r'crashes'
+
+  WER_NAME = r'SOFTWARE\Microsoft\Windows\Windows Error Reporting'
+  WER_LOCALDUMPS_NAME = r'%s\LocalDumps' % WER_NAME
+  IMGEXEC_NAME = (r'SOFTWARE\Microsoft\Windows NT\CurrentVersion'
+                  r'\Image File Execution Options\WerFault.exe')
+
+  def __init__(self):
+    # Depending on wether we're in cygwin or not we use a different import.

[Vyacheslav Egorov (Google), 2017/02/14 15:24:07]

whether

[kustermann, 2017/02/14 15:34:28]

Done.

+    try:
+      import winreg
+    except ImportError:
+      import _winreg as winreg
+    self.winreg = winreg
+
+  def __enter__(self):
+    # We want 32 and 64 bit coredumps to land in the same coredump directory.
+    for sam in [self.winreg.KEY_WOW64_64KEY, self.winreg.KEY_WOW64_32KEY]:
+      wer = None
+      wer_localdumps = None
+      try:
+        # In case WerFault.exe was prevented from executing, we fix it here.
+        # TODO(kustermann): Remove this once https://crbug.com/691971 is fixed.
+        self._prune_existing_key(
+            self.winreg.HKEY_LOCAL_MACHINE, self.IMGEXEC_NAME, sam)
+
+        # Create (or open) the WER keys.
+        wer = self.winreg.CreateKeyEx(

[Vyacheslav Egorov (Google), 2017/02/14 15:24:07]

maybe 

with CreateKeyEx() as wer: 
  ...

[kustermann, 2017/02/14 15:34:28]

Done.

+            self.winreg.HKEY_LOCAL_MACHINE, self.WER_NAME, 0,
+            self.winreg.KEY_ALL_ACCESS | sam)
+        wer_localdumps = self.winreg.CreateKeyEx(
+            self.winreg.HKEY_LOCAL_MACHINE, self.WER_LOCALDUMPS_NAME, 0,
+            self.winreg.KEY_ALL_ACCESS | sam)
+
+        # Prevent any modal UI dialog & disable normal windows error reporting.
+        # TODO(kustermann): Remove this once https://crbug.com/691971 is fixed.
+        self.winreg.SetValueEx(wer, "DontShowUI", 0, self.winreg.REG_DWORD, 1)
+        self.winreg.SetValueEx(wer, "Disabled", 0, self.winreg.REG_DWORD, 1)
+
+        coredump_folder = os.path.join(
+            os.getcwd(), WindowsCoredumpEnabler.WINDOWS_COREDUMP_FOLDER)
+
+        # Create the directory which will contain the dumps
+        if not os.path.exists(coredump_folder):
+          os.mkdir(coredump_folder)
+
+        # Do full dumps (not just mini dumps), keep max 100 dumps and specify
+        # folder.
+        self.winreg.SetValueEx(
+            wer_localdumps, "DumpType", 0, self.winreg.REG_DWORD, 2)
+        self.winreg.SetValueEx(
+            wer_localdumps, "DumpCount", 0, self.winreg.REG_DWORD, 200)
+        self.winreg.SetValueEx(
+            wer_localdumps, "DumpFolder", 0, self.winreg.REG_EXPAND_SZ,
+            coredump_folder)
+      finally:
+        if wer:
+          wer.Close()
+        if wer_localdumps:
+          wer_localdumps.Close()
+
+  def __exit__(self, *_):
+    # We remove the local dumps settings we have afte running the tests.

[Vyacheslav Egorov (Google), 2017/02/14 15:24:07]

after

[kustermann, 2017/02/14 15:34:28]

Done.

+    for sam in [self.winreg.KEY_WOW64_64KEY, self.winreg.KEY_WOW64_32KEY]:
+      wer_localdumps = self.winreg.CreateKeyEx(
+          self.winreg.HKEY_LOCAL_MACHINE, self.WER_LOCALDUMPS_NAME, 0,
+          self.winreg.KEY_ALL_ACCESS | sam)
+      try:

[Vyacheslav Egorov (Google), 2017/02/14 15:24:07]

maybe

with ... as wer_localdumps:
 ...

[kustermann, 2017/02/14 15:34:28]

Done.

+        self.winreg.DeleteValue(wer_localdumps, 'DumpType')
+        self.winreg.DeleteValue(wer_localdumps, 'DumpCount')
+        self.winreg.DeleteValue(wer_localdumps, 'DumpFolder')
+      finally:
+        wer_localdumps.Close()
+
+  def _prune_existing_key(self, key, subkey, wowbit):
+    handle = None
+
+    # If the open fails, the key doesn't exist and it's fine.
+    try:
+      handle = self.winreg.OpenKey(
+          key, subkey, 0, self.winreg.KEY_READ | wowbit)
+    except OSError:
+      pass
+
+    # If the key exists then we delete it. If the deletion does not work, we
+    # let the exception through.
+    if handle:
+      handle.Close()
+      self.winreg.DeleteKeyEx(key, subkey, wowbit, 0)
+
+class BaseCoreDumpArchiver(object):
   """This class reads coredumps file written by UnexpectedCrashDumpArchiver
   into the current working directory and uploads all cores and binaries
   listed in it into Cloud Storage (see tools/testing/dart/test_progress.dart).
   """
 
-  def __init__(self, args):
-    self._enabled = '--copy-coredumps' in args and GuessOS() == 'linux'
-    self._search_dir = os.getcwd()
+  # test.dart will write a line for each unexpected crash into this file.
+  _UNEXPECTED_CRASHES_FILE = "unexpected-crashes"
+
+  def __init__(self):
     self._bucket = 'dart-temp-crash-archive'
-    self._old_limits = None
+    self._binaries_dir = os.getcwd()
 
   def __enter__(self):
-    if not self._enabled:
-      return
-
-    # Cleanup any stale coredumps
+    # Cleanup any stale files
     if self._cleanup():
       print "WARNING: Found and removed stale coredumps"
 
-    self._old_limits = resource.getrlimit(resource.RLIMIT_CORE)
-
-    # Bump core limits to unlimited if core_pattern is correctly configured.
-    if self._check_core_dump_pattern(fatal=False):
-      resource.setrlimit(resource.RLIMIT_CORE, (-1, -1))
-
   def __exit__(self, *_):
-    if not self._enabled:
-      return
-
     try:
-      # Restore old core limit.
-      resource.setrlimit(resource.RLIMIT_CORE, self._old_limits)
-
-      # Check that kernel was correctly configured to use core.%p
-      # core_pattern.
-      self._check_core_dump_pattern(fatal=True)
-
-      coredumps = self._find_coredumps()
-      if coredumps:
+      crashes = self._find_unexpected_crashes()
+      if crashes:
         # If we get a ton of crashes, only archive 10 dumps.
-        archive_coredumps = coredumps[:10]
-        print 'Archiving coredumps:'
-        for core in archive_coredumps:
-          print '----> %s' % core
+        archive_crashes = crashes[:10]
+        print 'Archiving coredumps for crash (if possible):'
+        for crash in archive_crashes:
+          print '----> %s' % crash
 
         sys.stdout.flush()
-        self._archive(archive_coredumps)
+        self._archive(archive_crashes)
 
     finally:
       self._cleanup()
 
-  def _cleanup(self):
-    found = False
-    for core in glob.glob(os.path.join(self._search_dir, 'core.*')):
-      found = True
-      os.unlink(core)
-    for binary in glob.glob(os.path.join(self._search_dir, 'binary.*')):
-      found = True
-      os.unlink(binary)
-    try:
-      os.unlink(os.path.join(self._search_dir, 'coredumps'))
-      found = True
-    except:
-      pass
-
-    return found
-
-  def _find_coredumps(self):
-    """Load coredumps file. Each line has the following format:
-
-              test-name,core-file,binary-file
-    """
-    try:
-      with open('coredumps') as f:
-        return [CoreDump(*ln.strip('\n').split(',')) for ln in f.readlines()]
-    except:
-      return []
-
-  def _archive(self, coredumps):
+  def _archive(self, crashes):
     files = set()
-    for core in coredumps:
-      files.add(core.core)
-      files.add(core.binary)
+    missing = []
+    for crash in crashes:
+      files.add(crash.binary)
+      core = self._find_coredump_file(crash)
+      if core:
+        files.add(core)
+      else:
+        missing.append(crash)
     self._upload(files)
+    if missing:
+      raise Exception('Missing crash dumps for: %s' % ', '.join(missing))
 
   def _upload(self, files):
     bot_utils = GetBotUtils()
     gsutil = bot_utils.GSUtil()
     storage_path = '%s/%s/' % (self._bucket, uuid.uuid4())
     gs_prefix = 'gs://%s' % storage_path
     http_prefix = 'https://storage.cloud.google.com/%s' % storage_path
 
     print '\n--- Uploading into %s (%s) ---' % (gs_prefix, http_prefix)
     for file in files:
@@ skipping 17 matching lines @@
 
       try:
         gsutil.upload(tarname, gs_url)
         print '+++ Uploaded %s (%s)' % (gs_url, http_url)
       except Exception as error:
         print '!!! Failed to upload %s, error: %s' % (tarname, error)
 
       os.unlink(tarname)
     print '--- Done ---\n'
 
-  def _check_core_dump_pattern(self, fatal=False):
-    core_pattern_file = '/proc/sys/kernel/core_pattern'
-    core_pattern = open(core_pattern_file).read()
+  def _find_unexpected_crashes(self):
+    """Load coredumps file. Each line has the following format:
 
-    expected_core_pattern = 'core.%p'
-    if core_pattern.strip() != expected_core_pattern:
-      if fatal:
-        message = ('Invalid core_pattern configuration. '
-            'The configuration of core dump handling is *not* correct for '
-            'a buildbot. The content of {0} must be "{1}" instead of "{2}".'
-            .format(core_pattern_file, expected_core_pattern, core_pattern))
-        raise Exception(message)
-      else:
-        return False
-    return True
+        test-name,pid,binary-file
+    """
+    try:
+      with open(BaseCoreDumpArchiver._UNEXPECTED_CRASHES_FILE) as f:
+        return [UnexpectedCrash(*ln.strip('\n').split(',')) for ln in f.readlines()]
+    except:
+      return []
+
+  def _cleanup(self):
+    found = False
+    if os.path.exists(BaseCoreDumpArchiver._UNEXPECTED_CRASHES_FILE):
+      os.unlink(BaseCoreDumpArchiver._UNEXPECTED_CRASHES_FILE)
+      found = True
+    for binary in glob.glob(os.path.join(self._binaries_dir, 'binary.*')):
+      found = True
+      os.unlink(binary)
+    return found
+
+class LinuxCoreDumpArchiver(BaseCoreDumpArchiver):
+  def __init__(self):
+    super(self.__class__, self).__init__()
+    self._search_dir = os.getcwd()
+
+  def _cleanup(self):
+    found = super(self.__class__, self)._cleanup()
+    for core in glob.glob(os.path.join(self._search_dir, 'core.*')):
+      found = True
+      os.unlink(core)
+    return found
+
+  def _find_coredump_file(self, crash):
+    core_filename = os.path.join(self._search_dir, 'core.%s' % crash.pid)
+    if os.path.exists(core_filename):
+      return core_filename
+
+class WindowsCoreDumpArchiver(BaseCoreDumpArchiver):
+  def __init__(self):
+    super(self.__class__, self).__init__()
+    self._search_dir = os.path.join(
+        os.getcwd(), WindowsCoredumpEnabler.WINDOWS_COREDUMP_FOLDER)
+
+  def _cleanup(self):
+    found = super(self.__class__, self)._cleanup()
+    for core in glob.glob(os.path.join(self._search_dir, '*')):
+      found = True
+      os.unlink(core)
+    return found
+
+  def _find_coredump_file(self, crash):
+    pattern = os.path.join(self._search_dir, '*.%s.*' % crash.pid)
+    for core_filename in glob.glob(pattern):
+      return core_filename
+
+@contextlib.contextmanager
+def NooptCoreDumpArchiver():
+  yield
+
+
+def CoreDumpArchiver(args):
+  enabled = '--copy-coredumps' in args
+
+  if not enabled:
+    return NooptCoreDumpArchiver()
+
+  osname = GuessOS()
+  if osname == 'linux':
+    return contextlib.nested(PosixCoredumpEnabler(),
+                             LinuxCoreDumpArchiver())
+  elif osname == 'win32':
+    return contextlib.nested(WindowsCoredumpEnabler(),
+                             WindowsCoreDumpArchiver())
+  else:
+    # We don't have support for MacOS yet.
+    assert osname == 'macos'
+    return NooptCoreDumpArchiver()
 
 if __name__ == "__main__":
   import sys
   Main()