json: Improve error handling and adding ability to limit log output.

recipe_engine/util.py

 # Copyright 2013 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
+from collections import deque
+
 import contextlib
 import datetime
 import functools
 import logging
 import os
 import sys
 import traceback
 import time
 import urllib
 
@@ skipping 287 matching lines @@
     it (iterable): An iterable to traverse.
     exc_types (list): An optional list of specific exception types to defer.
         If empty, Exception will be used. Any Exceptions not referenced by this
         list will skip deferring and be immediately raised.
   """
   mexc_builder = MultiException.Builder()
   for e in it:
     with mexc_builder.catch(*exc_types):
       fn(e)
   mexc_builder.raise_if_any()
+
+
+def snip_output_lines(lines_data, max_bytes):
+  """Snip the middle out of very large data file.
+
+  The output will be roughly max_bytes in size with the first half of the bytes
+  showing the start of the data and the second half of the bytes showing the
+  end of the data. If any data was removed, an indicator of
+  '...\\n<snip>\\n... ' will be placed where the removed data was like so;
+
+    no more than half max_bytes
+    of starting data
+    <snip>
+    no more than half max_bytes of
+    data at the end of the file.
+
+
+  Args:
+    lines_data (list of strings): Lines of the file.
+    max_bytes (int): Maximum amount of bytes.
+  """
+  lines_deque = deque(lines_data)
+
+  max_bytes = int(max_bytes / 2)
+
+  start_lines = []
+  start_count = max_bytes
+  while lines_deque and start_count > 0:
+    line = lines_deque.popleft()
+    start_data = line[:start_count]
+    start_count -= len(start_data)
+    if len(line) == len(start_data):
+      start_lines.append(line)
+    else:
+      start_lines.append('%s ...' % start_data)
+      lines_deque.appendleft(line[len(start_data):])
+      break
+
+  end_lines = []
+  end_count = max_bytes
+  while lines_deque and end_count > 0:
+    line = lines_deque.pop()
+    end_data = line[-end_count:]
+    end_count -= len(end_data)
+    if len(line) == len(end_data):
+      end_lines.append(line)
+    else:
+      end_lines.append('... %s' % end_data)
+      lines_deque.append(line[:-len(end_data)])
+      break
+
+  if not lines_deque:
+    return lines_data
+  else:
+    return start_lines + ['<snip>'] + end_lines[::-1]