json: Improve error handling and adding ability to limit log output.

recipe_engine/util.py

Index: recipe_engine/util.py
diff --git a/recipe_engine/util.py b/recipe_engine/util.py
index e7497073fab6b4d9dbfbe56da10fc497ef621d5a..fbbf1b6a11ecf4a7e72742b7c0747c6cbcd97261 100644
--- a/recipe_engine/util.py
+++ b/recipe_engine/util.py
@@ -2,6 +2,8 @@
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
+from collections import deque
+
 import contextlib
 import datetime
 import functools
@@ -309,3 +311,58 @@ def map_defer_exceptions(fn, it, *exc_types):
     with mexc_builder.catch(*exc_types):
       fn(e)
   mexc_builder.raise_if_any()
+
+
+def snip_output_lines(lines_data, max_bytes):
+  """Snip the middle out of very large data file.
+
+  The output will be roughly max_bytes in size with the first half of the bytes
+  showing the start of the data and the second half of the bytes showing the
+  end of the data. If any data was removed, an indicator of
+  '...\\n<snip>\\n... ' will be placed where the removed data was like so;
+
+    no more than half max_bytes
+    of starting data
+    <snip>
+    no more than half max_bytes of
+    data at the end of the file.
+
+
+  Args:
+    lines_data (list of strings): Lines of the file.
+    max_bytes (int): Maximum amount of bytes.
+  """
+  lines_deque = deque(lines_data)
+
+  max_bytes = int(max_bytes / 2)
+
+  start_lines = []
+  start_count = max_bytes
+  while lines_deque and start_count > 0:
+    line = lines_deque.popleft()
+    start_data = line[:start_count]
+    start_count -= len(start_data)
+    if len(line) == len(start_data):
+      start_lines.append(line)
+    else:
+      start_lines.append('%s ...' % start_data)
+      lines_deque.appendleft(line[len(start_data):])
+      break
+
+  end_lines = []
+  end_count = max_bytes
+  while lines_deque and end_count > 0:
+    line = lines_deque.pop()
+    end_data = line[-end_count:]
+    end_count -= len(end_data)
+    if len(line) == len(end_data):
+      end_lines.append(line)
+    else:
+      end_lines.append('... %s' % end_data)
+      lines_deque.append(line[:-len(end_data)])
+      break
+
+  if not lines_deque:
+    return lines_data
+  else:
+    return start_lines + ['<snip>'] + end_lines[::-1]