Added update_test_expectations script to remove non-flaky test expectations.

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/port/test.py

 # Copyright (C) 2010 Google Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 # notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 # copyright notice, this list of conditions and the following disclaimer
@@ skipping 340 matching lines @@
             add_file(test, '-expected.wav', test.expected_audio)
             continue
         add_file(test, '-expected.txt', test.expected_text)
         add_file(test, '-expected.png', test.expected_image)
 
     filesystem.write_text_file(filesystem.join(LAYOUT_TEST_DIR, 'virtual', 'virtual_passes', 'passes', 'args-expected.txt'), 'args-txt --virtual-arg')
     # Clear the list of written files so that we can watch what happens during testing.
     filesystem.clear_written_files()
 
 
 class TestPort(Port):

[qyearsley, 2016/03/21 21:45:25]

Question, unrelated to this CL:

Do you know what a TestPort represents? Does it represent one layout test on one
platform? (or multiple platforms, or multiple tests)?

[Dirk Pranke, 2016/03/29 22:39:01]

A TestPort is a fake implementation of the Port class, used for unit testing the
webkitpy implementations.

FakePort would probably be a better name.

[bokan, 2016/04/05 12:28:56]

I'm not sure. TestPort is a testing version of the Port class. My impression is
that it might have related to the WebKit notion of a port. e.g Chromium was a
port, Safari was a port, etc. In this case, I think different ports would have
their own builders and infrastructure. That's my best guess though. Ojan or Dirk
might know more.

     port_name = 'test'
     default_port_name = 'test-mac-mac10.10'
 
     # TODO(wkorman): The below constant is legacy code and is only referenced by a unit test. Find the modern way to do
     # the same thing that test is doing and delete this.
     #
     # A list of platform names sufficient to cover all the baselines.
     # The list should be sorted so that a later platform  will reuse
     # an earlier platform's baselines if they are the same (e.g.,
     # 'mac10.10' should precede 'mac10.9').
     ALL_BASELINE_VARIANTS = (
         'test-linux-trusty', 'test-linux-precise', 'test-linux-x86',
         'test-mac-mac10.11', 'test-mac-mac10.10',
         'test-win-win10', 'test-win-win7'
     )
 
     FALLBACK_PATHS = {
         'win7':        ['test-win-win7', 'test-win-win10'],
         'win10':       ['test-win-win10'],
         'mac10.10':    ['test-mac-mac10.10', 'test-mac-mac10.11'],
         'mac10.11':    ['test-mac-mac10.11'],
         'trusty':      ['test-linux-trusty', 'test-win-win7'],
         'precise':     ['test-linux-precise', 'test-linux-trusty', 'test-win-win7'],
         'linux32':     ['test-linux-x86', 'test-linux-precise', 'test-linux-trusty', 'test-win-win7'],
     }
 
+    _mock_all_systems = (('mac10.10', 'x86'),
+                         ('mac10.11', 'x86'),
+                         ('win7', 'x86'),
+                         ('win10', 'x86'),
+                         ('linux32', 'x86'),
+                         ('precise', 'x86_64'),
+                         ('trusty', 'x86_64'))
+
+    _mock_all_build_types = ('debug', 'release')
+
+    """To avoid surprises when introducing new macros, these are intentionally fixed in time."""
+    _mock_configuration_specifier_macros = {
+        'mac': ['mac10.10', 'mac10.11'],
+        'win': ['win7', 'win10'],
+        'linux': ['linux32', 'precise', 'trusty']
+    }

[Dirk Pranke, 2016/03/29 22:39:01]

Nit: I would leave out the "_mock" part of these names. These aren't actually
mocks (they're fakes), and even calling them "_fake" doesn't add much, since the
whole class is a fake.

[bokan, 2016/04/05 12:28:56]

Done.

+
     @classmethod
     def determine_full_port_name(cls, host, options, port_name):
         if port_name == 'test':
             return TestPort.default_port_name
         return port_name
 
     def __init__(self, host, port_name=None, **kwargs):
         Port.__init__(self, host, port_name or TestPort.default_port_name, **kwargs)
         self._tests = unit_test_list()
         self._flakes = set()
@@ skipping 131 matching lines @@
         # every configuration on every system.
         test_configurations = []
         for version, architecture in self._all_systems():
             for build_type in self._all_build_types():
                 test_configurations.append(TestConfiguration(
                     version=version,
                     architecture=architecture,
                     build_type=build_type))
         return test_configurations
 
+    def set_all_systems(self, all_systems):
+        self._mock_all_systems = all_systems
+
     def _all_systems(self):
-        return (('mac10.10', 'x86'),
-                ('mac10.11', 'x86'),
-                ('win7', 'x86'),
-                ('win10', 'x86'),
-                ('linux32', 'x86'),
-                ('precise', 'x86_64'),
-                ('trusty', 'x86_64'))
+        return self._mock_all_systems

[Dirk Pranke, 2016/03/29 22:39:01]

Why not just make these things public instance variables? It's not very pythonic
to use setters and getters.

[bokan, 2016/04/05 12:28:56]

Done.

+
+    def set_all_build_types(self, all_build_types):
+        self._mock_all_build_types = all_build_types
 
     def _all_build_types(self):
-        return ('debug', 'release')
+        return self._mock_all_build_types
+
+    def set_configuration_specifier_macros(self, macros):
+        self._mock_configuration_specifier_macros = macros
 
     def configuration_specifier_macros(self):
-        """To avoid surprises when introducing new macros, these are intentionally fixed in time."""
-        return {
-            'mac': ['mac10.10', 'mac10.11'],
-            'win': ['win7', 'win10'],
-            'linux': ['linux32', 'precise', 'trusty']
-        }
+        return self._mock_configuration_specifier_macros
 
     def virtual_test_suites(self):
         return [
             VirtualTestSuite(prefix='virtual_passes', base='passes', args=['--virtual-arg']),
             VirtualTestSuite(prefix='skipped', base='failures/expected', args=['--virtual-arg2']),
             VirtualTestSuite(prefix='references_use_default_args', base='passes/reftest.html',
                              args=['--virtual-arg'], references_use_default_args=True),
         ]
 
 
@@ skipping 87 matching lines @@
         else:
             image = test.actual_image
         return DriverOutput(actual_text, image, test.actual_checksum, audio,
             crash=(crash or web_process_crash), crashed_process_name=crashed_process_name,
             crashed_pid=crashed_pid, crash_log=crash_log,
             test_time=time.time() - start_time, timeout=test.timeout, error=test.error, pid=self.pid,
             leak=test.leak)
 
     def stop(self):
         self.started = False