Automatic password manager tests now rely on being killed by the bot on timeout

components/test/data/password_manager/run_tests.py

 # -*- coding: utf-8 -*-
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """This file allows the bots to be easily configure and run the tests."""
 
 import argparse
 import os
 import tempfile
@@ skipping 35 matching lines @@
     results_path = results.name
     results.close()
 
     full_path = os.path.realpath(__file__)
     tests_dir = os.path.dirname(full_path)
     tests_path = os.path.join(tests_dir, "tests.py")
 
     for websitetest in environment.websitetests:
       # The tests can be flaky. This is why we try to rerun up to 3 times.
       for x in range(0, 3):
-        # TODO(rchtara): Using "pkill" is just temporary until a better,
-        # platform-independent solution is found.
-        os.system("pkill chrome")
         try:
           os.remove(results_path)
         except Exception:
           pass
-        # TODO(rchtara): Using "timeout is just temporary until a better,
-        # platform-independent solution is found.
 
-        # The website test runs in two passes, each pass has an internal
-        # timeout of 200s for waiting (see |remaining_time_to_wait| and
-        # Wait() in websitetest.py). Accounting for some more time spent on
-        # the non-waiting execution, 300 seconds should be the upper bound on
-        # the runtime of one pass, thus 600 seconds for the whole test.
-        os.system("timeout 600 python %s %s --chrome-path %s "
-            "--chromedriver-path %s --passwords-path %s --profile-path %s "
-            "--save-path %s" %
-            (tests_path, websitetest.name, args.chrome_path[0],
-             args.chromedriver_path[0], args.passwords_path[0],
-             args.profile_path[0], results_path))
+        print "Running tests for %s " % websitetest.name
+
+        # Run the test without enable-automatic-password-saving to check whether
+        # or not the prompt is shown in the way we expected.
+        tests_results = tests.RunTests(args.chrome_path[0],
+                                 args.chromedriver_path[0],
+                                 args.profile_path[0],
+                                 args.passwords_path[0],
+                                 False,
+                                 10, # Debug log level.
+                                 True,
+                                 None,
+                                 tests.TypeOfTestedWebsites.LIST_OF_TESTS,
+                                 [websitetest.name])
+
+        # Run the test with enable-automatic-password-saving to check whether
+        # or not the password is stored in the the way we expected.
+        tests_results += tests.RunTests(args.chrome_path[0],
+                                 args.chromedriver_path[0],
+                                 args.profile_path[0],
+                                 args.passwords_path[0],
+                                 True,
+                                 10, # Debug log level.
+                                 True,
+                                 None,
+                                 tests.TypeOfTestedWebsites.LIST_OF_TESTS,
+                                 [websitetest.name])
+
+        tests.saveResults(tests_results, results_path)

[vabr (Chromium), 2014/07/21 08:39:52]

Why would you save the tests to a file, and immediately read them back?
Could you now get rid of the results file altogether?

[rchtara, 2014/07/21 15:12:40]

I wait for the answer of Aaron answer to know if it's possible to read  the json
from a file (The tests log has many things that are not in json format, and I
dont want to change that, so if someone wants to run the tests on it's own
machine, he should get a plain text log instead of json)
There is now two kind of result data:
*the first one is xml that holds all details about tests and it's going to be
send to appengine (not saved)
* the second is a very simple json file for the recipe (saved)

         if os.path.isfile(results_path):
           results = open(results_path, "r")
           count = 0 # Count the number of successful tests.
           for line in results:
             xml.write(line)
             count += line.count("successful='True'")
           results.close()
           # There is only two tests running for every website: the prompt and
           # the normal test. If both of the tests were successful, the tests
           # would be stopped for the current website.
           if count == 2:
             break
         else:
           xml.write("<result><test name='%s' type='prompt' successful='false'>"
               "</test><test name='%s' type='normal' successful='false'></test>"
               "</result>" % (websitetest.name, websitetest.name))
   finally:
     try:
       os.remove(results_path)
     except Exception:
       pass
 
   xml.write("</xml>")
   xml.close()