Allow the test bots to easily run the password manager python tests.

components/test/data/password_manager/run_tests.py

+# -*- coding: utf-8 -*-
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""This file allows the bots to be easily configure and run the tests."""
+
+import argparse
+import os
+import tempfile
+
+from environment import Environment
+import tests
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser(
+      description="Password Manager automated tests runner help.")
+  parser.add_argument(
+      "--chrome-path", action="store", dest="chrome_path",
+      help="Set the chrome path (required).", nargs=1, required=True)
+  parser.add_argument(
+      "--chromedriver-path", action="store", dest="chromedriver_path",
+      help="Set the chromedriver path (required).", nargs=1, required=True)
+  parser.add_argument(
+      "--profile-path", action="store", dest="profile_path",
+      help="Set the profile path (required). You just need to choose a "
+           "temporary empty folder. If the folder is not empty all its content "
+           "is going to be removed.",
+      nargs=1, required=True)
+  parser.add_argument(
+      "--passwords-path", action="store", dest="passwords_path",
+      help="Set the usernames/passwords path (required).", nargs=1,
+      required=True)
+  parser.add_argument("--save-path", action="store", nargs=1, dest="save_path",
+      help="Write the results in a file.", required=True)
+  args = parser.parse_args()
+
+  environment = Environment('', '', '', None, False)
+  tests.Tests(environment)
+
+  xml = open(args.save_path[0],"w")
+  xml.write("<xml>")
+  try:
+    results = tempfile.NamedTemporaryFile(
+        dir=os.path.join(tempfile.gettempdir()),
+        delete=False)

[vabr (Chromium), 2014/07/16 08:48:44]

nit: That's not what I meant -- the arguments do fit all on one line, don't
they?
results = tempfile.NamedTemporaryFile(
    dir=os.path.join(tempfile.gettempdir()), delete=False)

[rchtara, 2014/07/16 16:31:58]

Done.

+    results_path = results.name
+    results.close()
+
+    full_path = os.path.realpath(__file__)
+    tests_dir = os.path.dirname(full_path)
+    tests_path = os.path.join(tests_dir, "tests.py")
+
+    for websitetest in environment.websitetests:
+      # The tests are not robust and could some time fail for reasons that are
+      # not related to the password manager(ex: unexpected chrome crash). This

[vabr (Chromium), 2014/07/16 08:48:44]

I don't think we should tolerate unexpected chrome crashes. If Chrome crashes,
that test needs to be disabled and a bug for that filed.

[vabr (Chromium), 2014/07/16 08:48:44]

nits:
(1) space before "("
(2) ex: -> e.g.,

[rchtara, 2014/07/16 16:31:58]

Done.

+      # is why we need to run them many times to reduce the effect of such

[vabr (Chromium), 2014/07/16 08:48:44]

I can see 2 issues with the current approach:
(1) All tests are retried, regardless of their success.
(2) There seems to be no filtering of the results for tests which both fail and
succeed (during different runs). Would a test failing at some run make the
overall result look like failed?

I'm afraid, there needs to be a more clever logic to handle this, which keeps
re-running only failed tests (at most 3 times), and marks the whole test job as
failed if and only if there are still some failed runs in the last round.

[rchtara, 2014/07/16 16:31:59]

Done.

+      # problems.
+      for x in range(0, 3):
+        os.system("pkill -f chrome")
+        try:
+          os.remove(results_path)
+        except OSError:
+          pass
+        # The test could sometimes be blocked waiting for a page to be loaded
+        # or an HTML element to appear. This why we need to run them with a
+        # timeout to stop the test in this case. The timeout is 200 seconds and
+        # it's more than enough to run the test.

[vabr (Chromium), 2014/07/16 08:48:44]

"it's more than enough" -- that's a bold claim. If we had been confident it's
more than enough, we should have made it shorter.
Instead, I suggest referring to the |remaining_time_to_wait| timeout limit set
in websitetest.py, and increasing the number below back to 300 seconds -- the
remaining_time_to_wait timeout is 200, and it does not include everything, only
time spent in Wait() calls.

[rchtara, 2014/07/16 16:31:59]

Done.

+        os.system("timeout 200 python %s %s --chrome-path %s "
+            "--chromedriver-path %s --passwords-path %s --profile-path %s "
+            "--save-path %s" %
+            (tests_path, websitetest.name, args.chrome_path[0],
+             args.chromedriver_path[0], args.passwords_path[0],
+             args.profile_path[0], results_path))
+        if os.path.isfile(results_path):
+          results = open(results_path, "r")
+          for line in results:
+            xml.write(line)
+          results.close()
+        else:
+          xml.write("<result><test name='%s' type='prompt' successful='false'>"
+              "</test><test name='%s' type='normal' successful='false'></test>"
+              "</result>" % (websitetest.name, websitetest.name))
+  except Exception:

[vabr (Chromium), 2014/07/16 08:48:44]

Are you sure you want to flush all exceptions here?
This will make it impossible to understand what happened in the try block.
Did you mean to use a finally clause for removing the results instead?
try:
  #all the test execution here
finally:
  os.remove(results_path)

[rchtara, 2014/07/16 16:31:59]

Done.

+    pass
+  try:
+    os.remove(results_path)
+  except Exception:
+    pass
+
+  xml.write("</xml>")
+  xml.close()