Added support for versioning of layout test results of run-webkit-tests runs

Tools/Scripts/webkitpy/layout_tests/controllers/manager.py

 # Copyright (C) 2010 Google Inc. All rights reserved.
 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 # notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
@@ skipping 23 matching lines @@
 create a final report.
 """
 
 import datetime
 import json
 import logging
 import random
 import sys
 import time
 
+from webkitpy.common.checkout.scm.detection import SCMDetector

[Dirk Pranke, 2014/06/17 17:58:58]

I think this import is unneeded.

[patro, 2014/07/15 10:36:56]

Done.

 from webkitpy.common.net.file_uploader import FileUploader
+from webkitpy.common.system.filesystem import FileSystem

[Dirk Pranke, 2014/06/17 17:58:58]

this should be unneeded as well.

[patro, 2014/07/15 10:36:56]

Done.

 from webkitpy.layout_tests.controllers.layout_test_finder import LayoutTestFinder
 from webkitpy.layout_tests.controllers.layout_test_runner import LayoutTestRunner
 from webkitpy.layout_tests.controllers.test_result_writer import TestResultWriter
 from webkitpy.layout_tests.layout_package import json_results_generator
 from webkitpy.layout_tests.models import test_expectations
 from webkitpy.layout_tests.models import test_failures
 from webkitpy.layout_tests.models import test_run_results
 from webkitpy.layout_tests.models.test_input import TestInput
 
 _log = logging.getLogger(__name__)
@@ skipping 15 matching lines @@
         Args:
           port: an object implementing port-specific
           options: a dictionary of command line options
           printer: a Printer object to record updates to.
         """
         self._port = port
         self._filesystem = port.host.filesystem
         self._options = options
         self._printer = printer
         self._expectations = None
+        # Actions related to archiving of the results.
+        self._is_scm_initialized = False
+        self._scm = None

[Dirk Pranke, 2014/06/17 17:58:58]

These scm fields don't seem to be used?

[patro, 2014/07/15 10:36:57]

Done.

+        self.ARCHIVED_PATH = None
+        self.REVISION_INFO = ''
 
         self.HTTP_SUBDIR = 'http' + port.TEST_PATH_SEPARATOR
         self.PERF_SUBDIR = 'perf'
         self.WEBSOCKET_SUBDIR = 'websocket' + port.TEST_PATH_SEPARATOR
         self.LAYOUT_TESTS_DIRECTORY = 'LayoutTests'
         self._http_server_started = False
         self._websockets_server_started = False
 
         self._results_directory = self._port.results_directory()
         self._finder = LayoutTestFinder(self._port, self._options)
@@ skipping 54 matching lines @@
     def _test_is_expected_missing(self, test_file):
         expectations = self._expectations.model().get_expectations(test_file)
         return test_expectations.MISSING in expectations or test_expectations.NEEDS_REBASELINE in expectations or test_expectations.NEEDS_MANUAL_REBASELINE in expectations
 
     def _test_is_slow(self, test_file):
         return test_expectations.SLOW in self._expectations.model().get_expectations(test_file)
 
     def needs_servers(self, test_names):
         return any(self._test_requires_lock(test_name) for test_name in test_names)
 
+    def _rename_results_folder_if_required(self):
+        try:
+            timestamp = time.strftime("%d-%m-%Y-%H-%M-%S", time.gmtime(self._filesystem.mtime(self._filesystem.join(self._results_directory, "results.html"))))
+            archived_name = ''.join((self._filesystem.basename(self._results_directory), "_", timestamp))
+            archived_path = self._filesystem.join(self._filesystem.dirname(self._results_directory), archived_name)
+            self._filesystem.move(self._results_directory, archived_path)
+            return archived_path

[Dirk Pranke, 2014/06/17 17:58:58]

I don't know that we should assume we can write outside of
self._results_directory. We should consider other options here.

[patro, 2014/07/15 10:36:57]

If the archived results are enabled by default then we can create a sub
directory in self._results_directory for each archived results. The current
directory structure is maintained as it is still an experimental feature.

+        except OSError, e:
+            # It might be possible that results.html was not generated in previous run, because the test
+            # run was interrupted even before testing started. In those cases, don't archive the folder.
+            # Simply override the current folder contents with new results.
+            import errno
+            if e.errno == errno.EEXIST:
+                print "Found no result.html in the folder, ignoring this result and overwritten the folder..."
+                _log.info("Found no result.html in the folder, ignoring this result and overwritten the folder...")
+                return None
+
     def _set_up_run(self, test_names):
         self._printer.write_update("Checking build ...")
         if self._options.build:
             exit_code = self._port.check_build(self.needs_servers(test_names), self._printer)
             if exit_code:
                 _log.error("Build check failed")
                 return exit_code
 
         # This must be started before we check the system dependencies,
         # since the helper may do things to make the setup correct.
         if self._options.pixel_tests:
             self._printer.write_update("Starting pixel test helper ...")
             self._port.start_helper()
 
         # Check that the system dependencies (themes, fonts, ...) are correct.
         if not self._options.nocheck_sys_deps:
             self._printer.write_update("Checking system dependencies ...")
             exit_code = self._port.check_sys_deps(self.needs_servers(test_names))
             if exit_code:
                 self._port.stop_helper()
                 return exit_code
 
         if self._options.clobber_old_results:
             self._clobber_old_results()
 
+        # FIXME:: Here the condition has to be (archive_prev_results) and (total_archived_results < MAX_ARCHIVE_RESULTS)
+        if self._options.enable_versioned_results and self._filesystem.exists(self._results_directory):
+            # Get revision info for storing in archived dir.
+            self.REVISION_INFO = ''
+            # Rename the existing results folder for archiving.
+            self.ARCHIVED_PATH = self._rename_results_folder_if_required()
+
         # Create the output directory if it doesn't already exist.
         self._port.host.filesystem.maybe_make_directory(self._results_directory)
 
         self._port.setup_test_run()
         return test_run_results.OK_EXIT_STATUS
 
     def run(self, args):
         """Run the tests and return a RunDetails object with the results."""
         start_time = time.time()
         self._printer.write_update("Collecting tests ...")
@@ skipping 53 matching lines @@
             self._clean_up_run()
 
         # Some crash logs can take a long time to be written out so look
         # for new logs after the test run finishes.
         self._printer.write_update("looking for new crash logs")
         self._look_for_new_crash_logs(initial_results, start_time)
         if retry_results:
             self._look_for_new_crash_logs(retry_results, start_time)
 
         _log.debug("summarizing results")
-        summarized_full_results = test_run_results.summarize_results(self._port, self._expectations, initial_results, retry_results, enabled_pixel_tests_in_retry)
+        summarized_full_results = test_run_results.summarize_results(self._port, self._expectations, initial_results, retry_results, enabled_pixel_tests_in_retry, False, self.ARCHIVED_PATH)

[Dirk Pranke, 2014/06/17 17:58:57]

see comment in test_run_results.

[patro, 2014/07/15 10:36:57]

Done.

         summarized_failing_results = test_run_results.summarize_results(self._port, self._expectations, initial_results, retry_results, enabled_pixel_tests_in_retry, only_include_failing=True)
 
         exit_code = summarized_failing_results['num_regressions']
         if exit_code > test_run_results.MAX_FAILURES_EXIT_STATUS:
             _log.warning('num regressions (%d) exceeds max exit status (%d)' %
                          (exit_code, test_run_results.MAX_FAILURES_EXIT_STATUS))
             exit_code = test_run_results.MAX_FAILURES_EXIT_STATUS
 
         if not self._options.dry_run:
             self._write_json_files(summarized_full_results, summarized_failing_results, initial_results)
@@ skipping 174 matching lines @@
             return int(worker_name.split('/')[1]) if worker_name else -1
 
         stats = {}
         for result in initial_results.results_by_name.values():
             if result.type != test_expectations.SKIP:
                 stats[result.test_name] = {'results': (_worker_number(result.worker_name), result.test_number, result.pid, int(result.test_run_time * 1000), int(result.total_run_time * 1000))}
         stats_trie = {}
         for name, value in stats.iteritems():
             json_results_generator.add_path_to_trie(name, value, stats_trie)
         return stats_trie