Automatic password manager tests now rely on being killed by the bot on timeout

components/test/data/password_manager/run_tests.py

 # -*- coding: utf-8 -*-
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """This file allows the bots to be easily configure and run the tests."""
 
 import argparse
-import os
-import tempfile
 
 from environment import Environment
 import tests
 
+def printResults(results):
+  """Print the test results.
+
+  Args:
+    results: A list of the TestResults that are going to be printed.
+  """
+  for result in results:
+    if result.successful:
+      print (
+          "The test %s for %s was successful."
+          % (result.test_type, result.name))
+    else:
+      print (
+          "The test %s for %s has failed:\n%s"
+          % (result.test_type, result.name, result.message))
+
 if __name__ == "__main__":
   parser = argparse.ArgumentParser(
       description="Password Manager automated tests runner help.")
   parser.add_argument(
       "--chrome-path", action="store", dest="chrome_path",
       help="Set the chrome path (required).", nargs=1, required=True)
   parser.add_argument(
       "--chromedriver-path", action="store", dest="chromedriver_path",
       help="Set the chromedriver path (required).", nargs=1, required=True)
   parser.add_argument(
       "--profile-path", action="store", dest="profile_path",
       help="Set the profile path (required). You just need to choose a "
            "temporary empty folder. If the folder is not empty all its content "
            "is going to be removed.",
       nargs=1, required=True)
   parser.add_argument(
       "--passwords-path", action="store", dest="passwords_path",
       help="Set the usernames/passwords path (required).", nargs=1,
       required=True)
   parser.add_argument("--save-path", action="store", nargs=1, dest="save_path",
       help="Write the results in a file.", required=True)
   args = parser.parse_args()
 
   environment = Environment('', '', '', None, False)
   tests.Tests(environment)
 
-  xml = open(args.save_path[0],"w")
-  xml.write("<xml>")
-  try:
-    results = tempfile.NamedTemporaryFile(
-        dir=os.path.join(tempfile.gettempdir()), delete=False)
-    results_path = results.name
-    results.close()
+  xml = "<xml>"
 
-    full_path = os.path.realpath(__file__)
-    tests_dir = os.path.dirname(full_path)
-    tests_path = os.path.join(tests_dir, "tests.py")
+  all_successful = True
+  for websitetest in environment.websitetests:
+    successful = False
+    # The tests can be flaky. This is why we try to rerun up to 3 times.
+    for x in range(0, 3):
+      # This message needs to be printed out to let the trybot know that the

[vabr (Chromium), 2014/07/22 15:41:45]

typo: know -> knows

[rchtara, 2014/07/22 16:18:47]

http://www.oxfordlearnersdictionaries.com/definition/english/let_1 (let
somebody/something do something)
:)

[vabr (Chromium), 2014/07/22 16:32:04]

You are right, I missed the "let"! :)
Never mind, then.

+      # tests are still working, so they should not be killed.
+      print "Running tests for %s " % websitetest.name
+      tests_results = []
+      # Exceptions thrown by this test need to be ignored so that the other
+      # tests can run without any problem.
+      try:
+        # Run the test without enable-automatic-password-saving to check whether
+        # or not the prompt is shown in the way we expected.
+        tests_results += tests.RunTests(
+            args.chrome_path[0],
+            args.chromedriver_path[0],
+            args.profile_path[0],
+            args.passwords_path[0],
+            False,
+            10, # Debug log level.
+            True,
+            None,
+            tests.TypeOfTestedWebsites.LIST_OF_TESTS,
+            [websitetest.name])
+      except Exception:

[vabr (Chromium), 2014/07/22 15:41:45]

Could you print the textual description of the exception?
except Exception as e:
  print str(e)
(also below)

[rchtara, 2014/07/22 16:18:47]

Done.

+        pass
 
-    for websitetest in environment.websitetests:
-      # The tests can be flaky. This is why we try to rerun up to 3 times.
-      for x in range(0, 3):
-        # TODO(rchtara): Using "pkill" is just temporary until a better,
-        # platform-independent solution is found.
-        os.system("pkill chrome")
-        try:
-          os.remove(results_path)
-        except Exception:
-          pass
-        # TODO(rchtara): Using "timeout is just temporary until a better,
-        # platform-independent solution is found.
+      # Exceptions thrown by this test need to be ignored so that the other
+      # tests can run without any problem.
+      try:
+        # Run the test with enable-automatic-password-saving to check whether
+        # or not the password is stored in the the way we expected.
+        tests_results += tests.RunTests(
+            args.chrome_path[0],
+            args.chromedriver_path[0],
+            args.profile_path[0],
+            args.passwords_path[0],
+            True,
+            10, # Debug log level.
+            True,
+            None,
+            tests.TypeOfTestedWebsites.LIST_OF_TESTS,
+            [websitetest.name])
+      except Exception:
+        pass
 
-        # The website test runs in two passes, each pass has an internal
-        # timeout of 200s for waiting (see |remaining_time_to_wait| and
-        # Wait() in websitetest.py). Accounting for some more time spent on
-        # the non-waiting execution, 300 seconds should be the upper bound on
-        # the runtime of one pass, thus 600 seconds for the whole test.
-        os.system("timeout 600 python %s %s --chrome-path %s "
-            "--chromedriver-path %s --passwords-path %s --profile-path %s "
-            "--save-path %s" %
-            (tests_path, websitetest.name, args.chrome_path[0],
-             args.chromedriver_path[0], args.passwords_path[0],
-             args.profile_path[0], results_path))
-        if os.path.isfile(results_path):
-          results = open(results_path, "r")
-          count = 0 # Count the number of successful tests.
-          for line in results:
-            xml.write(line)
-            count += line.count("successful='True'")
-          results.close()
-          # There is only two tests running for every website: the prompt and
-          # the normal test. If both of the tests were successful, the tests
-          # would be stopped for the current website.
-          if count == 2:
-            break
-        else:
-          xml.write("<result><test name='%s' type='prompt' successful='false'>"
-              "</test><test name='%s' type='normal' successful='false'></test>"
-              "</result>" % (websitetest.name, websitetest.name))
-  finally:
-    try:
-      os.remove(results_path)
-    except Exception:
-      pass
+      if (len(tests_results) > 1 and tests_results[0].successful and
+          tests_results[1].successful):
+        successful = True
+        xml += tests.resultsToXml(tests_results)

[vabr (Chromium), 2014/07/22 15:41:45]

You currently duplicate here the code from lines 127 and 128.
If you remove lines 107-109, and instead put the lines
if successful:
  break
after line 128, you remove that duplicity.

[rchtara, 2014/07/22 16:18:47]

Done.

+        printResults(tests_results)
+        break
 
-  xml.write("</xml>")
-  xml.close()
+      elif len(tests_results) == 1:
+        failed_test = "prompt"
+        if tests_results[0].type == "prompt":
+          failed_test = "normal"
+        tests_results.append(
+            environment.TestResult(websitetest.name, failed_test, False,
+                "An expected error has occurred."))

[vabr (Chromium), 2014/07/22 15:41:45]

expected -> unexpected?
(also below)

[rchtara, 2014/07/22 16:18:47]

Done.

+
+      elif len(tests_results) == 0::

[vabr (Chromium), 2014/07/22 15:41:45]

typo: :: -> :

[rchtara, 2014/07/22 16:18:47]

Done.

+        tests_results.append(
+            environment.TestResult(websitetest.name, "prompt", False,
+                "An expected error has occurred."))
+        tests_results.append(
+            environment.TestResult(websitetest.name, "normal", False,
+                "An expected error has occurred."))
+
+      xml += tests.resultsToXml(tests_results)
+      printResults(tests_results)
+
+    all_successful = all_successful and successful
+
+  xml += "</xml>"
+
+  # |xml| should be sent to the dashboard, once it is ready.
+
+  json = open(args.save_path[0], "w")
+  json.write("{'successful': '%s'}" % all_successful)
+  json.close()