Allow the test bots to easily run the password manager python tests.

components/test/data/password_manager/run_tests.py

+# -*- coding: utf-8 -*-
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""This file allows the bots to be easily configure and run the tests."""
+
+import argparse
+import os
+import tempfile
+
+from environment import Environment
+import tests
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser(
+      description="Password Manager automated tests runner help.")
+  parser.add_argument(
+      "--chrome-path", action="store", dest="chrome_path",
+      help="Set the chrome path (required).", nargs=1, required=True)
+  parser.add_argument(
+      "--chromedriver-path", action="store", dest="chromedriver_path",
+      help="Set the chromedriver path (required).", nargs=1, required=True)
+  parser.add_argument(
+      "--profile-path", action="store", dest="profile_path",
+      help="Set the profile path (required). You just need to choose a "
+           "temporary empty folder. If the folder is not empty all its content "
+           "is going to be removed.",
+      nargs=1, required=True)
+  parser.add_argument(
+      "--passwords-path", action="store", dest="passwords_path",
+      help="Set the usernames/passwords path (required).", nargs=1,
+      required=True)
+  parser.add_argument("--save-path", action="store", nargs=1, dest="save_path",
+      help="Write the results in a file.", required=True)
+  args = parser.parse_args()
+
+  environment = Environment('', '', '', None, False)
+  tests.Tests(environment)
+
+  xml = open(args.save_path[0],"w")
+  xml.write("<xml>")
+  try:
+    results = tempfile.NamedTemporaryFile(
+        dir=os.path.join(tempfile.gettempdir()), delete=False)
+    results_path = results.name
+    results.close()
+
+    full_path = os.path.realpath(__file__)
+    tests_dir = os.path.dirname(full_path)
+    tests_path = os.path.join(tests_dir, "tests.py")
+
+    for websitetest in environment.websitetests:
+      # The tests are not robust and could some time fail for reasons that are
+      # not related to the password manager (e.g., unexpected chrome crash).

[vabr (Chromium), 2014/07/17 09:15:44]

I still oppose to mentioning "unexpected chrome crash" as an accepted reason for
failing tests.

(Re-running tests because they are flaky is OK and done with other tests too.)

Please reformulate the comment.

[rchtara, 2014/07/17 14:05:17]

Done.

+      # This is why we need to run them many times to reduce the effect of such
+      # problems.
+      for x in range(0, 3):
+        os.system("pkill -f chrome")
+        try:
+          os.remove(results_path)
+        except OSError:
+          pass
+        # Two tests are going to be run here: the normal test and the prompt
+        # test. Each of these tests have a |remaining_time_to_wait| timeout

[vabr (Chromium), 2014/07/17 09:15:44]

nit: I'd suggest a slight rephrasing to indicate that detailed knowledge of the
internal tests logic is not required. How about:
"The website test runs in two passes, each pass has an internal timeout for 200s
for waiting (see |remaining_time_to_wait| and Wait() in websitetest.py).
Accounting for some more time spent on the non-waiting execution, 300 seconds
should be the upper bound on the runtime of one pass, thus 600 seconds for the
whole test."

[rchtara, 2014/07/17 14:05:18]

Done.

+        # limit set in websitetest.py to 200 seconds that does not include
+        # everything, only time spent in Wait() calls. So we need 300 seconds
+        # for every test and 600 seconds for both of the tests.
+        os.system("timeout 600 python %s %s --chrome-path %s "

[vabr (Chromium), 2014/07/17 09:15:44]

Please add a "TODO(rchtara): " comment explaining that using "timeout" is just
temporary until a better, platform-independent solution is found.

[rchtara, 2014/07/17 14:05:17]

Done.

+            "--chromedriver-path %s --passwords-path %s --profile-path %s "
+            "--save-path %s" %
+            (tests_path, websitetest.name, args.chrome_path[0],
+             args.chromedriver_path[0], args.passwords_path[0],
+             args.profile_path[0], results_path))
+        if os.path.isfile(results_path):
+          results = open(results_path, "r")
+          count = 0 # Count the number of successful tests.
+          for line in results:
+            xml.write(line)
+            count += line.count("successful='True'")
+          results.close()
+          # There is only two tests running for every website: the prompt and
+          # the normal test. If both of the tests were successful, the tests
+          # would be stopped for the current website.
+          if count == 2:
+            break
+        else:
+          xml.write("<result><test name='%s' type='prompt' successful='false'>"
+              "</test><test name='%s' type='normal' successful='false'></test>"
+              "</result>" % (websitetest.name, websitetest.name))
+  finally:
+    try:
+      os.remove(results_path)
+    except Exception:
+      pass
+
+  xml.write("</xml>")
+  xml.close()