Automatic password manager tests now rely on being killed by the bot on timeout

components/test/data/password_manager/run_tests.py

 # -*- coding: utf-8 -*-
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """This file allows the bots to be easily configure and run the tests."""
 
 import argparse
-import os
-import tempfile
+import urllib2

[vabr (Chromium), 2014/07/22 07:54:27]

This also needs to be deleted if |response| and }request| are (see the comment
at the end).

[rchtara, 2014/07/22 12:49:36]

Done.

 
 from environment import Environment
 import tests
 
 if __name__ == "__main__":
   parser = argparse.ArgumentParser(
       description="Password Manager automated tests runner help.")
   parser.add_argument(
       "--chrome-path", action="store", dest="chrome_path",
       help="Set the chrome path (required).", nargs=1, required=True)
@@ skipping 10 matching lines @@
       "--passwords-path", action="store", dest="passwords_path",
       help="Set the usernames/passwords path (required).", nargs=1,
       required=True)
   parser.add_argument("--save-path", action="store", nargs=1, dest="save_path",
       help="Write the results in a file.", required=True)
   args = parser.parse_args()
 
   environment = Environment('', '', '', None, False)
   tests.Tests(environment)
 
-  xml = open(args.save_path[0],"w")
-  xml.write("<xml>")
-  try:
-    results = tempfile.NamedTemporaryFile(
-        dir=os.path.join(tempfile.gettempdir()), delete=False)
-    results_path = results.name
-    results.close()
+  xml = "<xml>"
 
-    full_path = os.path.realpath(__file__)
-    tests_dir = os.path.dirname(full_path)
-    tests_path = os.path.join(tests_dir, "tests.py")
+  all_successful = True
+  for websitetest in environment.websitetests:
+    successful = False
+    # The tests can be flaky. This is why we try to rerun up to 3 times.
+    for x in range(0, 3):
+      print "Running tests for %s " % websitetest.name

[vabr (Chromium), 2014/07/22 07:54:28]

nit: Please add a comment explaining that this message needs to be printed out
to let the trybot know that the tests are still working, so they should not be
killed.

[rchtara, 2014/07/22 12:49:36]

Done.

 
-    for websitetest in environment.websitetests:
-      # The tests can be flaky. This is why we try to rerun up to 3 times.
-      for x in range(0, 3):
-        # TODO(rchtara): Using "pkill" is just temporary until a better,
-        # platform-independent solution is found.
-        os.system("pkill chrome")
-        try:
-          os.remove(results_path)
-        except Exception:
-          pass
-        # TODO(rchtara): Using "timeout is just temporary until a better,
-        # platform-independent solution is found.
+      # Run the test without enable-automatic-password-saving to check whether
+      # or not the prompt is shown in the way we expected.
+      tests_results = tests.RunTests(
+          args.chrome_path[0],
+          args.chromedriver_path[0],
+          args.profile_path[0],
+          args.passwords_path[0],
+          False,
+          10, # Debug log level.
+          True,
+          None,
+          tests.TypeOfTestedWebsites.LIST_OF_TESTS,
+          [websitetest.name])
 
-        # The website test runs in two passes, each pass has an internal
-        # timeout of 200s for waiting (see |remaining_time_to_wait| and
-        # Wait() in websitetest.py). Accounting for some more time spent on
-        # the non-waiting execution, 300 seconds should be the upper bound on
-        # the runtime of one pass, thus 600 seconds for the whole test.
-        os.system("timeout 600 python %s %s --chrome-path %s "
-            "--chromedriver-path %s --passwords-path %s --profile-path %s "
-            "--save-path %s" %
-            (tests_path, websitetest.name, args.chrome_path[0],
-             args.chromedriver_path[0], args.passwords_path[0],
-             args.profile_path[0], results_path))
-        if os.path.isfile(results_path):
-          results = open(results_path, "r")
-          count = 0 # Count the number of successful tests.
-          for line in results:
-            xml.write(line)
-            count += line.count("successful='True'")
-          results.close()
-          # There is only two tests running for every website: the prompt and
-          # the normal test. If both of the tests were successful, the tests
-          # would be stopped for the current website.
-          if count == 2:
-            break
-        else:
-          xml.write("<result><test name='%s' type='prompt' successful='false'>"
-              "</test><test name='%s' type='normal' successful='false'></test>"
-              "</result>" % (websitetest.name, websitetest.name))
-  finally:
-    try:
-      os.remove(results_path)
-    except Exception:
-      pass
+      # Run the test with enable-automatic-password-saving to check whether
+      # or not the password is stored in the the way we expected.
+      tests_results += tests.RunTests(
+          args.chrome_path[0],
+          args.chromedriver_path[0],
+          args.profile_path[0],
+          args.passwords_path[0],
+          True,
+          10, # Debug log level.
+          True,
+          None,
+          tests.TypeOfTestedWebsites.LIST_OF_TESTS,
+          [websitetest.name])
 
-  xml.write("</xml>")
-  xml.close()
+      if len(tests_results) > 1:
+        xml = xml + tests.resultsToXml(tests_results)

[vabr (Chromium), 2014/07/22 07:54:28]

xml += ...

(also below)

[rchtara, 2014/07/22 12:49:36]

Done.

+        if tests_results[0].successful and tests_results[1].successful:
+          successful = True
+          break
+      else:

[vabr (Chromium), 2014/07/22 07:54:27]

The "else" branch scraps the test results, even if there is one. That might be
confusing: Imagine someone would suspect they might have broken the "prompt"
version, but they will see an empty failure for both.
Please make sure to report all results we got, and only enter dummy failure logs
for a particular test if we really did not get any results for it.

[rchtara, 2014/07/22 12:49:36]

Done.

+        xml = xml + (
+            "<result><test name='%s' type='prompt' successful='false'>"
+            "</test><test name='%s' type='normal' successful='false'></test>"
+            "</result>" % (websitetest.name, websitetest.name))
+
+    all_successful = all_successful and successful
+
+  xml = xml + "</xml>"
+
+  request = urllib2.Request("http://dashbord", data=xml,

[vabr (Chromium), 2014/07/22 07:54:27]

Please do not include code which is not working. Until the endpoint (dashboard)
is ready, the script should not send anything to it.
Please delete |request| and |response| for now. You might leave a comment saying
that |xml| should be sent to the dashboard, once it is ready.

[rchtara, 2014/07/22 12:49:36]

Done.

+                            headers={'Content-Type': 'application/xml'})
+  response = urllib2.urlopen(request).read()
+  json = open(args.save_path[0], "w")

[vabr (Chromium), 2014/07/22 07:54:27]

When I run a test in Rietveld, and want to see a result, I can always click on
the "stdio" link for every step (see, e.g.,
http://build.chromium.org/p/tryserver.blink/builders/mac_blink_rel/builds/15722)
to see the output. I understand that that's what the test outputs to its
standard output. Could you please confirm that, and if so, make sure to print
the information you also store in the XML format as plain text to stdout, so
that even without the dashboard we an already see the results?

[rchtara, 2014/07/22 12:49:36]

Yes I confirm that.
The new patch allows to print the results to stdio.

+  json.write("{'successful': '%s'}" % all_successful)
+  json.close()