Only compare allocation output in predictable mode.

tools/testrunner/local/execution.py

 # Copyright 2012 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 47 matching lines @@
   output = commands.Execute(job.command, job.verbose, job.timeout)
   return (job.id, output, time.time() - start_time)
 
 class Runner(object):
 
   def __init__(self, suites, progress_indicator, context):
     self.datapath = os.path.join("out", "testrunner_data")
     self.perf_data_manager = perfdata.PerfDataManager(self.datapath)
     self.perfdata = self.perf_data_manager.GetStore(context.arch, context.mode)
     self.perf_failures = False
+    self.printed_allocations = False
     self.tests = [ t for s in suites for t in s.tests ]
     if not context.no_sorting:
       for t in self.tests:
         t.duration = self.perfdata.FetchPerfData(t) or 1.0
       self.tests.sort(key=lambda t: t.duration, reverse=True)
     self._CommonInit(len(self.tests), progress_indicator, context)
 
   def _CommonInit(self, num_tests, progress_indicator, context):
     self.indicator = progress_indicator
     progress_indicator.runner = self
@@ skipping 64 matching lines @@
     # as unexpected even if it flakily passes in order to include it in the
     # output.
     self.indicator.HasRun(test, has_unexpected_output or test.run > 1)
     if has_unexpected_output:
       # Rerun test failures after the indicator has processed the results.
       self._MaybeRerun(pool, test)
     # Update the perf database if the test succeeded.
     return not has_unexpected_output
 
   def _ProcessTestPredictable(self, test, result, pool):
+    def HasDifferentAllocations(output1, output2):
+      def AllocationStr(stdout):
+        for line in reversed((stdout or "").splitlines()):
+          if line.startswith("### Allocations = "):
+            self.printed_allocations = True
+            return line
+        return ""
+      return (AllocationStr(output1.stdout) != AllocationStr(output2.stdout))
+
     # Always pass the test duration for the database update.
     test.duration = result[2]
     if test.run == 1 and result[1].HasTimedOut():
       # If we get a timeout in the first run, we are already in an
       # unpredictable state. Just report it as a failure and don't rerun.
       self.indicator.AboutToRun(test)
       test.output = result[1]
       self.remaining -= 1
       self.failed.append(test)
       self.indicator.HasRun(test, True)
-    if test.run > 1 and test.output != result[1]:
-      # From the second run on, check for differences. If a difference is
-      # found, call the indicator twice to report both tests. All runs of each
-      # test are counted as one for the statistic.
+    if test.run > 1 and HasDifferentAllocations(test.output, result[1]):
+      # From the second run on, check for different allocations. If a
+      # difference is found, call the indicator twice to report both tests.
+      # All runs of each test are counted as one for the statistic.
       self.indicator.AboutToRun(test)
       self.remaining -= 1
       self.failed.append(test)
       self.indicator.HasRun(test, True)
       self.indicator.AboutToRun(test)
       test.output = result[1]
       self.indicator.HasRun(test, True)
     elif test.run >= 3:
       # No difference on the third run -> report a success.
       self.indicator.AboutToRun(test)
@@ skipping 50 matching lines @@
       pool.terminate()
       self._RunPerfSafe(lambda: self.perf_data_manager.close())
       if self.perf_failures:
         # Nuke perf data in case of failures. This might not work on windows as
         # some files might still be open.
         print "Deleting perf test data due to db corruption."
         shutil.rmtree(self.datapath)
     if queued_exception:
       raise queued_exception
 
+    # Make sure that any allocations were printed in predictable mode.
+    assert not self.context.predictable or self.printed_allocations
 
   def GetCommand(self, test):
     d8testflag = []
     shell = test.suite.shell()
     if shell == "d8":
       d8testflag = ["--test"]
     if utils.IsWindows():
       shell += ".exe"
     cmd = (self.context.command_prefix +
            [os.path.abspath(os.path.join(self.context.shell_dir, shell))] +
            d8testflag +
            ["--random-seed=%s" % self.context.random_seed] +
            test.suite.GetFlagsForTestCase(test, self.context) +
            self.context.extra_flags)
     return cmd
 
 
 class BreakNowException(Exception):
   def __init__(self, value):
     self.value = value
   def __str__(self):
     return repr(self.value)