Enable support for coredump archiving on windows

tools/utils.py

Index: tools/utils.py
diff --git a/tools/utils.py b/tools/utils.py
index 87ee9075e51f6f8cb749008076bab3a14717bcd5..6949c23fc1d96f6e85aee25105c074cfade2adb5 100644
--- a/tools/utils.py
+++ b/tools/utils.py
@@ -6,6 +6,7 @@
 # scripts.
 
 import commands
+import contextlib
 import datetime
 import glob
 import imp
@@ -606,7 +607,7 @@ def CheckedInSdkPath():
     osname = osdict[system]
   except KeyError:
     print >>sys.stderr, ('WARNING: platform "%s" not supported') % (system)
-    return None;
+    return None
   tools_dir = os.path.dirname(os.path.realpath(__file__))
   return os.path.join(tools_dir,
                       'sdks',
@@ -649,6 +650,23 @@ def CheckedInSdkCheckExecutable():
   return False
 
 
+def CheckLinuxCoreDumpPattern(fatal=False):
+  core_pattern_file = '/proc/sys/kernel/core_pattern'
+  core_pattern = open(core_pattern_file).read()
+
+  expected_core_pattern = 'core.%p'
+  if core_pattern.strip() != expected_core_pattern:
+    if fatal:
+      message = ('Invalid core_pattern configuration. '
+          'The configuration of core dump handling is *not* correct for '
+          'a buildbot. The content of {0} must be "{1}" instead of "{2}".'
+          .format(core_pattern_file, expected_core_pattern, core_pattern))
+      raise Exception(message)
+    else:
+      return False
+  return True
+
+
 class TempDir(object):
   def __init__(self, prefix=''):
     self._temp_dir = None
@@ -674,100 +692,176 @@ class ChangedWorkingDirectory(object):
     print "Enter directory = ", self._old_cwd
     os.chdir(self._old_cwd)
 
-class CoreDump(object):
-  def __init__(self, test, core, binary):
+
+class UnexpectedCrash(object):
+  def __init__(self, test, pid, binary):
     self.test = test
-    self.core = core
+    self.pid = pid 
     self.binary = binary
 
   def __str__(self):
-    return "%s: %s %s" % (self.test, self.binary, self.core)
+    return "%s: %s %s" % (self.test, self.binary, self.pid)
 
-class CoreDumpArchiver(object):
-  """This class reads coredumps file written by UnexpectedCrashDumpArchiver
-  into the current working directory and uploads all cores and binaries
-  listed in it into Cloud Storage (see tools/testing/dart/test_progress.dart).
-  """
 
-  def __init__(self, args):
-    self._enabled = '--copy-coredumps' in args and GuessOS() == 'linux'
-    self._search_dir = os.getcwd()
-    self._bucket = 'dart-temp-crash-archive'
+class PosixCoredumpEnabler(object):
+  def __init__(self):
     self._old_limits = None
 
   def __enter__(self):
-    if not self._enabled:
-      return
-
-    # Cleanup any stale coredumps
-    if self._cleanup():
-      print "WARNING: Found and removed stale coredumps"
-
     self._old_limits = resource.getrlimit(resource.RLIMIT_CORE)
 
     # Bump core limits to unlimited if core_pattern is correctly configured.
-    if self._check_core_dump_pattern(fatal=False):
+    if CheckLinuxCoreDumpPattern(fatal=False):
       resource.setrlimit(resource.RLIMIT_CORE, (-1, -1))
 
   def __exit__(self, *_):
-    if not self._enabled:
-      return
+    resource.setrlimit(resource.RLIMIT_CORE, self._old_limits)
+    CheckLinuxCoreDumpPattern(fatal=True)
 
-    try:
-      # Restore old core limit.
-      resource.setrlimit(resource.RLIMIT_CORE, self._old_limits)
+class WindowsCoredumpEnabler(object):
+  """Configure Windows Error Reporting to store crash dumps.
 
-      # Check that kernel was correctly configured to use core.%p
-      # core_pattern.
-      self._check_core_dump_pattern(fatal=True)
+  The documentation can be found here:
+  https://msdn.microsoft.com/en-us/library/windows/desktop/bb787181.aspx
+  """
 
-      coredumps = self._find_coredumps()
-      if coredumps:
-        # If we get a ton of crashes, only archive 10 dumps.
-        archive_coredumps = coredumps[:10]
-        print 'Archiving coredumps:'
-        for core in archive_coredumps:
-          print '----> %s' % core
+  WINDOWS_COREDUMP_FOLDER = r'crashes'
 
-        sys.stdout.flush()
-        self._archive(archive_coredumps)
+  WER_NAME = r'SOFTWARE\Microsoft\Windows\Windows Error Reporting'
+  WER_LOCALDUMPS_NAME = r'%s\LocalDumps' % WER_NAME
+  IMGEXEC_NAME = (r'SOFTWARE\Microsoft\Windows NT\CurrentVersion'
+                  r'\Image File Execution Options\WerFault.exe')
 
-    finally:
-      self._cleanup()
+  def __init__(self):
+    # Depending on wether we're in cygwin or not we use a different import.

[Vyacheslav Egorov (Google), 2017/02/14 15:24:07]

whether

[kustermann, 2017/02/14 15:34:28]

Done.

+    try:
+      import winreg
+    except ImportError:
+      import _winreg as winreg
+    self.winreg = winreg
 
-  def _cleanup(self):
-    found = False
-    for core in glob.glob(os.path.join(self._search_dir, 'core.*')):
-      found = True
-      os.unlink(core)
-    for binary in glob.glob(os.path.join(self._search_dir, 'binary.*')):
-      found = True
-      os.unlink(binary)
+  def __enter__(self):
+    # We want 32 and 64 bit coredumps to land in the same coredump directory.
+    for sam in [self.winreg.KEY_WOW64_64KEY, self.winreg.KEY_WOW64_32KEY]:
+      wer = None
+      wer_localdumps = None
+      try:
+        # In case WerFault.exe was prevented from executing, we fix it here.
+        # TODO(kustermann): Remove this once https://crbug.com/691971 is fixed.
+        self._prune_existing_key(
+            self.winreg.HKEY_LOCAL_MACHINE, self.IMGEXEC_NAME, sam)
+
+        # Create (or open) the WER keys.
+        wer = self.winreg.CreateKeyEx(

[Vyacheslav Egorov (Google), 2017/02/14 15:24:07]

maybe 

with CreateKeyEx() as wer: 
  ...

[kustermann, 2017/02/14 15:34:28]

Done.

+            self.winreg.HKEY_LOCAL_MACHINE, self.WER_NAME, 0,
+            self.winreg.KEY_ALL_ACCESS | sam)
+        wer_localdumps = self.winreg.CreateKeyEx(
+            self.winreg.HKEY_LOCAL_MACHINE, self.WER_LOCALDUMPS_NAME, 0,
+            self.winreg.KEY_ALL_ACCESS | sam)
+
+        # Prevent any modal UI dialog & disable normal windows error reporting.
+        # TODO(kustermann): Remove this once https://crbug.com/691971 is fixed.
+        self.winreg.SetValueEx(wer, "DontShowUI", 0, self.winreg.REG_DWORD, 1)
+        self.winreg.SetValueEx(wer, "Disabled", 0, self.winreg.REG_DWORD, 1)
+
+        coredump_folder = os.path.join(
+            os.getcwd(), WindowsCoredumpEnabler.WINDOWS_COREDUMP_FOLDER)
+
+        # Create the directory which will contain the dumps
+        if not os.path.exists(coredump_folder):
+          os.mkdir(coredump_folder)
+
+        # Do full dumps (not just mini dumps), keep max 100 dumps and specify
+        # folder.
+        self.winreg.SetValueEx(
+            wer_localdumps, "DumpType", 0, self.winreg.REG_DWORD, 2)
+        self.winreg.SetValueEx(
+            wer_localdumps, "DumpCount", 0, self.winreg.REG_DWORD, 200)
+        self.winreg.SetValueEx(
+            wer_localdumps, "DumpFolder", 0, self.winreg.REG_EXPAND_SZ,
+            coredump_folder)
+      finally:
+        if wer:
+          wer.Close()
+        if wer_localdumps:
+          wer_localdumps.Close()
+
+  def __exit__(self, *_):
+    # We remove the local dumps settings we have afte running the tests.

[Vyacheslav Egorov (Google), 2017/02/14 15:24:07]

after

[kustermann, 2017/02/14 15:34:28]

Done.

+    for sam in [self.winreg.KEY_WOW64_64KEY, self.winreg.KEY_WOW64_32KEY]:
+      wer_localdumps = self.winreg.CreateKeyEx(
+          self.winreg.HKEY_LOCAL_MACHINE, self.WER_LOCALDUMPS_NAME, 0,
+          self.winreg.KEY_ALL_ACCESS | sam)
+      try:

[Vyacheslav Egorov (Google), 2017/02/14 15:24:07]

maybe

with ... as wer_localdumps:
 ...

[kustermann, 2017/02/14 15:34:28]

Done.

+        self.winreg.DeleteValue(wer_localdumps, 'DumpType')
+        self.winreg.DeleteValue(wer_localdumps, 'DumpCount')
+        self.winreg.DeleteValue(wer_localdumps, 'DumpFolder')
+      finally:
+        wer_localdumps.Close()
+
+  def _prune_existing_key(self, key, subkey, wowbit):
+    handle = None
+
+    # If the open fails, the key doesn't exist and it's fine.
     try:
-      os.unlink(os.path.join(self._search_dir, 'coredumps'))
-      found = True
-    except:
+      handle = self.winreg.OpenKey(
+          key, subkey, 0, self.winreg.KEY_READ | wowbit)
+    except OSError:
       pass
 
-    return found
+    # If the key exists then we delete it. If the deletion does not work, we
+    # let the exception through.
+    if handle:
+      handle.Close()
+      self.winreg.DeleteKeyEx(key, subkey, wowbit, 0)
 
-  def _find_coredumps(self):
-    """Load coredumps file. Each line has the following format:
+class BaseCoreDumpArchiver(object):
+  """This class reads coredumps file written by UnexpectedCrashDumpArchiver
+  into the current working directory and uploads all cores and binaries
+  listed in it into Cloud Storage (see tools/testing/dart/test_progress.dart).
+  """
 
-              test-name,core-file,binary-file
-    """
+  # test.dart will write a line for each unexpected crash into this file.
+  _UNEXPECTED_CRASHES_FILE = "unexpected-crashes"
+
+  def __init__(self):
+    self._bucket = 'dart-temp-crash-archive'
+    self._binaries_dir = os.getcwd()
+
+  def __enter__(self):
+    # Cleanup any stale files
+    if self._cleanup():
+      print "WARNING: Found and removed stale coredumps"
+
+  def __exit__(self, *_):
     try:
-      with open('coredumps') as f:
-        return [CoreDump(*ln.strip('\n').split(',')) for ln in f.readlines()]
-    except:
-      return []
+      crashes = self._find_unexpected_crashes()
+      if crashes:
+        # If we get a ton of crashes, only archive 10 dumps.
+        archive_crashes = crashes[:10]
+        print 'Archiving coredumps for crash (if possible):'
+        for crash in archive_crashes:
+          print '----> %s' % crash
 
-  def _archive(self, coredumps):
+        sys.stdout.flush()
+        self._archive(archive_crashes)
+
+    finally:
+      self._cleanup()
+
+  def _archive(self, crashes):
     files = set()
-    for core in coredumps:
-      files.add(core.core)
-      files.add(core.binary)
+    missing = []
+    for crash in crashes:
+      files.add(crash.binary)
+      core = self._find_coredump_file(crash)
+      if core:
+        files.add(core)
+      else:
+        missing.append(crash)
     self._upload(files)
+    if missing:
+      raise Exception('Missing crash dumps for: %s' % ', '.join(missing))
 
   def _upload(self, files):
     bot_utils = GetBotUtils()
@@ -805,21 +899,84 @@ class CoreDumpArchiver(object):
       os.unlink(tarname)
     print '--- Done ---\n'
 
-  def _check_core_dump_pattern(self, fatal=False):
-    core_pattern_file = '/proc/sys/kernel/core_pattern'
-    core_pattern = open(core_pattern_file).read()
-
-    expected_core_pattern = 'core.%p'
-    if core_pattern.strip() != expected_core_pattern:
-      if fatal:
-        message = ('Invalid core_pattern configuration. '
-            'The configuration of core dump handling is *not* correct for '
-            'a buildbot. The content of {0} must be "{1}" instead of "{2}".'
-            .format(core_pattern_file, expected_core_pattern, core_pattern))
-        raise Exception(message)
-      else:
-        return False
-    return True
+  def _find_unexpected_crashes(self):
+    """Load coredumps file. Each line has the following format:
+
+        test-name,pid,binary-file
+    """
+    try:
+      with open(BaseCoreDumpArchiver._UNEXPECTED_CRASHES_FILE) as f:
+        return [UnexpectedCrash(*ln.strip('\n').split(',')) for ln in f.readlines()]
+    except:
+      return []
+
+  def _cleanup(self):
+    found = False
+    if os.path.exists(BaseCoreDumpArchiver._UNEXPECTED_CRASHES_FILE):
+      os.unlink(BaseCoreDumpArchiver._UNEXPECTED_CRASHES_FILE)
+      found = True
+    for binary in glob.glob(os.path.join(self._binaries_dir, 'binary.*')):
+      found = True
+      os.unlink(binary)
+    return found
+
+class LinuxCoreDumpArchiver(BaseCoreDumpArchiver):
+  def __init__(self):
+    super(self.__class__, self).__init__()
+    self._search_dir = os.getcwd()
+
+  def _cleanup(self):
+    found = super(self.__class__, self)._cleanup()
+    for core in glob.glob(os.path.join(self._search_dir, 'core.*')):
+      found = True
+      os.unlink(core)
+    return found
+
+  def _find_coredump_file(self, crash):
+    core_filename = os.path.join(self._search_dir, 'core.%s' % crash.pid)
+    if os.path.exists(core_filename):
+      return core_filename
+
+class WindowsCoreDumpArchiver(BaseCoreDumpArchiver):
+  def __init__(self):
+    super(self.__class__, self).__init__()
+    self._search_dir = os.path.join(
+        os.getcwd(), WindowsCoredumpEnabler.WINDOWS_COREDUMP_FOLDER)
+
+  def _cleanup(self):
+    found = super(self.__class__, self)._cleanup()
+    for core in glob.glob(os.path.join(self._search_dir, '*')):
+      found = True
+      os.unlink(core)
+    return found
+
+  def _find_coredump_file(self, crash):
+    pattern = os.path.join(self._search_dir, '*.%s.*' % crash.pid)
+    for core_filename in glob.glob(pattern):
+      return core_filename
+
+@contextlib.contextmanager
+def NooptCoreDumpArchiver():
+  yield
+
+
+def CoreDumpArchiver(args):
+  enabled = '--copy-coredumps' in args
+
+  if not enabled:
+    return NooptCoreDumpArchiver()
+
+  osname = GuessOS()
+  if osname == 'linux':
+    return contextlib.nested(PosixCoredumpEnabler(),
+                             LinuxCoreDumpArchiver())
+  elif osname == 'win32':
+    return contextlib.nested(WindowsCoredumpEnabler(),
+                             WindowsCoreDumpArchiver())
+  else:
+    # We don't have support for MacOS yet.
+    assert osname == 'macos'
+    return NooptCoreDumpArchiver()
 
 if __name__ == "__main__":
   import sys