[Password manager Python tests] Re-arrange tests

components/test/data/password_manager/automated_tests/run_tests.py

 # -*- coding: utf-8 -*-
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Encapsulates running tests defined in tests.py.
 
 Running this script requires passing --config-path with a path to a config file
 of the following structure:
 
@@ skipping 49 matching lines @@
     self.profile_path = tempfile.mkdtemp()
     results = tempfile.NamedTemporaryFile(delete=False)
     self.results_path = results.name
     results.close()
     self.test_cmd = test_cmd + ["--profile-path", self.profile_path,
                                 "--save-path", self.results_path]
     self.test_name = test_name
     # TODO(vabr): Ideally we would replace timeout with something allowing
     # calling tests directly inside Python, and working on other platforms.
     #
-    # The website test runs in two passes, each pass has an internal
+    # The website test runs multiple scenarios, each one has an internal
     # timeout of 200s for waiting (see |remaining_time_to_wait| and
-    # Wait() in websitetest.py). Accounting for some more time spent on
-    # the non-waiting execution, 300 seconds should be the upper bound on
-    # the runtime of one pass, thus 600 seconds for the whole test.
-    self.test_cmd = ["timeout", "600"] + self.test_cmd
+    # Wait() in websitetest.py). Expecting that not every scenario should
+    # take 200s, the maximum time allocated for all of them is 300s.
+    self.test_cmd = ["timeout", "300"] + self.test_cmd
 
     self.logger.log(SCRIPT_DEBUG,
                     "TestRunner set up for test %s, command '%s', "
                     "profile path %s, results file %s",
                     self.test_name, self.test_cmd, self.profile_path,
                     self.results_path)
 
     self.runner_process = None
     # The tests can be flaky. This is why we try to rerun up to 3 times.
     self.max_test_runs_left = 3
@@ skipping 17 matching lines @@
       self.logger.log(SCRIPT_DEBUG, "Test %s passed", self.test_name)
       return True, []
     if self.max_test_runs_left == 0:
       self.logger.log(SCRIPT_DEBUG, "Test %s failed", self.test_name)
       return False, self.failures
     self._run_test()
     return None
 
   def _check_if_test_passed(self):
     """Returns True if and only if the test passed."""
+
+    success = False
     if os.path.isfile(self.results_path):
       with open(self.results_path, "r") as results:
-        count = 0  # Count the number of successful tests.
+        # TODO(vabr): Parse the results to make sure all scenarios succeeded
+        # instead of hard-coding here the number of tests scenarios from
+        # test.py:main.
+        NUMBER_OF_TEST_SCENARIOS = 3
+        passed_scenarios = 0
         for line in results:
           self.failures.append(line)
-          count += line.count("successful='True'")
+          passed_scenarios += line.count("successful='True'")
+          success = passed_scenarios == NUMBER_OF_TEST_SCENARIOS
+          if success:
+            break
 
-      # There is only two tests running for every website: the prompt and
-      # the normal test. If both of the tests were successful, the tests
-      # would be stopped for the current website.
-      self.logger.log(SCRIPT_DEBUG, "Test run of %s: %s",
-                      self.test_name, "pass" if count == 2 else "fail")
-      if count == 2:
-        return True
-    return False
+    self.logger.log(
+        SCRIPT_DEBUG,
+        "Test run of {0} succeded: {1}".format(self.test_name, success))
+    return success
 
   def _run_test(self):
     """Executes the command to run the test."""
     with open(self.results_path, "w"):
       pass  # Just clear the results file.
     shutil.rmtree(path=self.profile_path, ignore_errors=True)
     self.max_test_runs_left -= 1
     self.logger.log(SCRIPT_DEBUG, "Run of test %s started", self.test_name)
     self.runner_process = subprocess.Popen(self.test_cmd)
 
@@ skipping 87 matching lines @@
 def main():
   parser = argparse.ArgumentParser()
   parser.add_argument("config_path", metavar="N",
                       help="Path to the config.ini file.")
   args = parser.parse_args()
   run_tests(args.config_path)
 
 
 if __name__ == "__main__":
   main()