blink/run-webkit-tests: Write layout test environment to json file.

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py

 # Copyright (C) 2010 Google Inc. All rights reserved.
 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 # notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
@@ skipping 21 matching lines @@
 This includes finding tests to run, reading the test expectations,
 starting the required helper servers, deciding the order and way to
 run the tests, retrying fails tests and collecting the test results,
 including crash logs, and mismatches with expectations.
 
 The Manager object has a constructor and one main method called run.
 """
 
 import json
 import logging
+import os
+import platform
 import random
 import re
 import sys
 import time
+import types
 
 from webkitpy.common.net.file_uploader import FileUploader
 from webkitpy.layout_tests.controllers.layout_test_finder import LayoutTestFinder
 from webkitpy.layout_tests.controllers.layout_test_runner import LayoutTestRunner
 from webkitpy.layout_tests.controllers.test_result_writer import TestResultWriter
 from webkitpy.layout_tests.layout_package import json_results_generator
 from webkitpy.layout_tests.models import test_expectations
 from webkitpy.layout_tests.models import test_failures
 from webkitpy.layout_tests.models import test_run_results
 from webkitpy.layout_tests.models.test_input import TestInput
@@ skipping 14 matching lines @@
         """Initialize test runner data structures.
 
         Args:
           port: An object implementing platform-specific functionality.
           options: An options argument which contains command line options.
           printer: A Printer object to record updates to.
         """
         self._port = port
         self._filesystem = port.host.filesystem
         self._options = options
+        self._args = []
         self._printer = printer
         self._expectations = None
 
         self.HTTP_SUBDIR = 'http' + port.TEST_PATH_SEPARATOR
         self.INSPECTOR_SUBDIR = 'inspector' + port.TEST_PATH_SEPARATOR
         self.PERF_SUBDIR = 'perf'
         self.WEBSOCKET_SUBDIR = 'websocket' + port.TEST_PATH_SEPARATOR
         self.LAYOUT_TESTS_DIRECTORY = 'LayoutTests'
         self.ARCHIVED_RESULTS_LIMIT = 25
         self._http_server_started = False
         self._wptserve_started = False
         self._websockets_server_started = False
 
         self._random_seed = None
         self._results_directory = self._port.results_directory()
         self._finder = LayoutTestFinder(self._port, self._options)
         self._runner = LayoutTestRunner(self._options, self._port, self._printer, self._results_directory, self._test_is_slow)
 
+    def _env(self):
+        env = {
+            'args': self._args,
+            # os.environ needs to be converted to a dict otherwise it can't be
+            # json serialized
+            'env': dict(os.environ),

[Dirk Pranke, 2016/06/21 21:57:54]

The args and environ are dumped to stderr as part of the logging run. Do we
really need to write them to a file as well? (though you should make sure the
random seed is logged).

[mithro, 2016/06/22 08:42:24]

Does the logging output get saved with the test results somewhere? 

The aim here is to provide as much information as possible (in a computer
readable version) to allow a person to reproduce the layout test results.

Ideally the env information would include content_shell build information in the
future too.

[Dirk Pranke, 2016/06/23 23:57:26]

Well, the logging output is on the builder alongside the links to the test
results. I suppose
it depends on where you're getting the test results from, but for me for
triaging infra/harness
type issues it's almost always way more useful to have the log than the test
results.
This information is also available in the build logs. I wouldn't try to
duplicate that
information in the results archives (though the chromium build hash is already
in the 
JSON test results).

If you were going to save more data, I would save it in the fulL_results.json,
rather than
creating another json file.

+            # self._options also isn't a dictionary object, so convert it too.
+            'options': self._options.__dict__,

[chrishall, 2016/08/08 04:32:57]

Minor nitpick:

above to convert environ we used `dict(os.environ)`,
why are we here using `self._options.__dict___` ?

could we instead use `dict(self._options)` for consistency ?

+            'platform': {},
+        }
+
+        # Capture the information python knows
+        for func in dir(platform):

[Dirk Pranke, 2016/06/21 21:57:54]

Is platform actually defined somewhere? I'm also actually leery of logging what
I'm guessing you're trying to log here. I don't know when you would actually
need this information?

[mithro, 2016/06/22 08:42:24]

platform is a standard library module -
https://docs.python.org/2/library/platform.html

The information I'm interested here include processor + architecture, operating
system version, python information.

[Dirk Pranke, 2016/06/23 23:57:26]

I was actually referring to the symbol itself. For whatever reason, I didn't see
the "import platform"
in your patch before.

Most of this information is already available in the build logs. The stuff that
isn't isn't historically
very useful, and I'm not generally a fan of logging everything because it might
be useful.

[chrishall, 2016/08/08 04:32:57]

If our goal is to produce an easy-to-consume JSON file then it may be worthwhile
trying to make this step process the input a little more - rather than slurping
everything and requiring later steps to massage the data.

For example it would be nice to have a key 'os_name' and a value under that of
'windows'.


Platform is a little gross in that it seems to expose each operating system's
information in slightly different ways, I have code to deal with this - see
below.

This is particularly ugly when dealing with mac versioning (special case) - see
my 'mac_ver' code - linked below.

I implemented similar things for OS metric gathering which captures:
os name
os version
os arch
python arch

Which you can find in:

https://codereview.chromium.org/2106953006/diff/140001/infra/services/sysmon/...

+            f = getattr(platform, func)
+            if func.startswith("_") or func in ("popen",) or isinstance(f, types.ModuleType):
+                continue
+            if callable(f):
+                try:
+                    env['platform'][func] = f()
+                except Exception:
+                    pass
+            else:
+                env['platform'][func] = f
+
+        if self._random_seed:
+            env['random-seed'] = self._random_seed
+
+        return env
+
     def run(self, args):
         """Run the tests and return a RunDetails object with the results."""
         start_time = time.time()
         self._printer.write_update("Collecting tests ...")
         running_all_tests = False
         try:
+            self._args = args
             paths, test_names, running_all_tests = self._collect_tests(args)
         except IOError:
             # This is raised if --test-list doesn't exist
             return test_run_results.RunDetails(exit_code=test_run_results.NO_TESTS_EXIT_STATUS)
 
         self._printer.write_update("Parsing expectations ...")
         self._expectations = test_expectations.TestExpectations(self._port, test_names)
 
         tests_to_run, tests_to_skip = self._prepare_lists(paths, test_names)
         self._printer.print_found(len(test_names), len(tests_to_run), self._options.repeat_each, self._options.iterations)
@@ skipping 382 matching lines @@
 
     def _tests_to_retry(self, run_results):
         # TODO(ojan): This should also check that result.type != test_expectations.MISSING since retrying missing expectations is silly.
         # But that's a bit tricky since we only consider the last retry attempt for the count of unexpected regressions.
         return [result.test_name for result in run_results.unexpected_results_by_name.values(
         ) if result.type != test_expectations.PASS]
 
     def _write_json_files(self, summarized_full_results, summarized_failing_results, initial_results, running_all_tests):
         _log.debug("Writing JSON files in %s." % self._results_directory)
 
+        # Write the environment the test ran under into env.json file
+        env_json_path = self._filesystem.join(self._results_directory, "env.json")
+        json_results_generator.write_json(self._filesystem, self._env(), env_json_path)

[Dirk Pranke, 2016/06/23 23:57:26]

see comment above about just using the existing json file.

+
         # FIXME: Upload stats.json to the server and delete times_ms.
         times_trie = json_results_generator.test_timings_trie(initial_results.results_by_name.values())
         times_json_path = self._filesystem.join(self._results_directory, "times_ms.json")
         json_results_generator.write_json(self._filesystem, times_trie, times_json_path)
 
         # Save out the times data so we can use it for --fastest in the future.
         if running_all_tests:
             bot_test_times_path = self._port.bot_test_times_path()
             self._filesystem.maybe_make_directory(self._filesystem.dirname(bot_test_times_path))
             json_results_generator.write_json(self._filesystem, times_trie, bot_test_times_path)
@@ skipping 61 matching lines @@
 
         stats = {}
         for result in initial_results.results_by_name.values():
             if result.type != test_expectations.SKIP:
                 stats[result.test_name] = {'results': (_worker_number(result.worker_name), result.test_number, result.pid, int(
                     result.test_run_time * 1000), int(result.total_run_time * 1000))}
         stats_trie = {}
         for name, value in stats.iteritems():
             json_results_generator.add_path_to_trie(name, value, stats_trie)
         return stats_trie