Keep track of output_snippet bytes and drop output snippets if output is too large

base/test/launcher/test_results_tracker.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "base/test/launcher/test_results_tracker.h"
 
 #include <stddef.h>
 
 #include <utility>
 
@@ skipping 283 matching lines @@
              "had unknown result");
 
   fprintf(stdout, "End of the summary.\n");
   fflush(stdout);
 }
 
 void TestResultsTracker::AddGlobalTag(const std::string& tag) {
   global_tags_.insert(tag);
 }
 
+// Sets output_snippet-related fields in test_result_value.
+// Does not take ownership of test_result_value.
+// Returns the number of bytes of snippet data that were added to
+// test_result_value.
+int SetSnippets(StringPiece output_snippet,
+                DictionaryValue* test_result_value) {
+  int num_bytes = 0;
+  // Always include the raw version (base64-encoded so that it can be safely
+  // JSON-serialized - there are no guarantees about character encoding of the
+  // snippet). This can be very useful piece of information when debugging a
+  // test failure related to character encoding.
+  std::string base64;
+  Base64Encode(output_snippet, &base64);
+  test_result_value->SetString("output_snippet_base64", base64);
+  num_bytes += base64.size();
+
+  bool lossless_snippet = false;
+  if (IsStringUTF8(output_snippet)) {
+    lossless_snippet = true;
+  } else {
+    output_snippet = "<non-UTF-8 snippet, see output_snippet_base64>";
+  }
+
+  test_result_value->SetString("output_snippet", output_snippet);
+  num_bytes += output_snippet.size();
+
+  // TODO(phajdan.jr): Fix typo in JSON key (losless -> lossless)
+  // making sure not to break any consumers of this data.
+  test_result_value->SetBoolean("losless_snippet", lossless_snippet);
+
+  return num_bytes;
+}
+
+// This is the maximum output.json which can be handled by
+// build/scripts/slave/recipe_modules/swarming/resources/collect_gtest_task.py
+// SaveSummaryAsJSON does not strictly ensure that its output is below this
+// threshold, but it will attempt to avoid exceeding this maximum by dropping
+// output_snippets as the total output approaches this threshold.
+const int kMaxSummaryJSON = 100 * 1024 * 1024;
+
 bool TestResultsTracker::SaveSummaryAsJSON(
     const FilePath& path,
     const std::vector<std::string>& additional_tags) const {
   std::unique_ptr<DictionaryValue> summary_root(new DictionaryValue);
 
   std::unique_ptr<ListValue> global_tags(new ListValue);
   for (const auto& global_tag : global_tags_) {
     global_tags->AppendString(global_tag);
   }
   for (const auto& tag : additional_tags) {
     global_tags->AppendString(tag);
   }
   summary_root->Set("global_tags", std::move(global_tags));
 
   std::unique_ptr<ListValue> all_tests(new ListValue);
   for (const auto& test : all_tests_) {
     all_tests->AppendString(test);
   }
   summary_root->Set("all_tests", std::move(all_tests));
 
   std::unique_ptr<ListValue> disabled_tests(new ListValue);
   for (const auto& disabled_test : disabled_tests_) {
     disabled_tests->AppendString(disabled_test);
   }
   summary_root->Set("disabled_tests", std::move(disabled_tests));
 
   std::unique_ptr<ListValue> per_iteration_data(new ListValue);
 
+  // Keep track of the number of bytes used by the contents of output snippets.
+  // Once we use up half of the maxium output size, we will start dropping
+  // snippets.
+  int snippet_bytes = 0;
+  int snippet_byte_threshold = kMaxSummaryJSON / 2;
+
   for (int i = 0; i <= iteration_; i++) {
     std::unique_ptr<DictionaryValue> current_iteration_data(
         new DictionaryValue);
 
     for (PerIterationData::ResultsMap::const_iterator j =
              per_iteration_data_[i].results.begin();
          j != per_iteration_data_[i].results.end();
          ++j) {
       std::unique_ptr<ListValue> test_results(new ListValue);
 
       for (size_t k = 0; k < j->second.test_results.size(); k++) {
         const TestResult& test_result = j->second.test_results[k];
 
         std::unique_ptr<DictionaryValue> test_result_value(new DictionaryValue);
 
         test_result_value->SetString("status", test_result.StatusAsString());
         test_result_value->SetInteger(
             "elapsed_time_ms",
             static_cast<int>(test_result.elapsed_time.InMilliseconds()));
 
-        bool lossless_snippet = false;
-        if (IsStringUTF8(test_result.output_snippet)) {
-          test_result_value->SetString(
-              "output_snippet", test_result.output_snippet);
-          lossless_snippet = true;
-        } else {
-          test_result_value->SetString(
-              "output_snippet",
-              "<non-UTF-8 snippet, see output_snippet_base64>");
-        }
+        StringPiece snippet = snippet_bytes > snippet_byte_threshold
+                                  ? "lengthy output elided"

[Paweł Hajdan Jr., 2016/12/22 14:18:39]

I'm concerned this seems to result in a silent failure. It seems very easy we
might run with effectively no output snippets until someone discovers a latent
issue by accident.

I'd strongly advocate surfacing this condition in some way, possibly by failing
the test - see e.g. https://codereview.chromium.org/2406243004 .

Another way we could do is further tighten per-test output limit, such that even
with largest test binary we can't exceed max limit. See
https://groups.google.com/a/chromium.org/d/msg/chromium-dev/ymxI-AaZ7-o/TgUWT...
.

We could also consider getting 64-bit python on the bots.

For further context, please read https://goto.google.com/epoll .

+                                  : test_result.output_snippet;
+        snippet_bytes += SetSnippets(snippet, test_result_value.get());
 
-        // TODO(phajdan.jr): Fix typo in JSON key (losless -> lossless)
-        // making sure not to break any consumers of this data.
-        test_result_value->SetBoolean("losless_snippet", lossless_snippet);
-
-        // Also include the raw version (base64-encoded so that it can be safely
-        // JSON-serialized - there are no guarantees about character encoding
-        // of the snippet). This can be very useful piece of information when
-        // debugging a test failure related to character encoding.
-        std::string base64_output_snippet;
-        Base64Encode(test_result.output_snippet, &base64_output_snippet);
-        test_result_value->SetString("output_snippet_base64",
-                                     base64_output_snippet);
         test_results->Append(std::move(test_result_value));
       }
 
       current_iteration_data->SetWithoutPathExpansion(j->first,
                                                       std::move(test_results));
     }
     per_iteration_data->Append(std::move(current_iteration_data));
   }
   summary_root->Set("per_iteration_data", std::move(per_iteration_data));
 
@@ skipping 78 matching lines @@
 TestResultsTracker::PerIterationData::PerIterationData() {
 }
 
 TestResultsTracker::PerIterationData::PerIterationData(
     const PerIterationData& other) = default;
 
 TestResultsTracker::PerIterationData::~PerIterationData() {
 }
 
 }  // namespace base