Make blink_trybot recipe work on windows

scripts/common/annotator.py

 #!/usr/bin/env python
 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Contains generating and parsing systems of the Chromium Buildbot Annotator.
 
 When executed as a script, this reads step name / command pairs from a file and
 executes those lines while annotating the output. The input is json:
 
 [{"name": "step_name", "cmd": ["command", "arg1", "arg2"]},
  {"name": "step_name2", "cmd": ["command2", "arg1"]}]
 
 """
 
+import itertools
 import json
 import optparse
 import os
 import re
+import subprocess
 import sys
+import threading
 import traceback
 
-from common import chromium_utils
-
 
 def emit(line, stream, flush_before=None):
   if flush_before:
     flush_before.flush()
-  print >> stream, '\n' + line
+  print >> stream
+  # WinDOS can only handle 64kb of output to the console at a time, per process.
+  if sys.platform.startswith('win'):
+    lim = 2**15
+    while line:
+      to_print, line = line[:lim], line[lim:]
+      stream.write(to_print)
+  else:
+    print >> stream, line
   stream.flush()
 
 
 class StepCommands(object):
   """Class holding step commands. Intended to be subclassed."""
   def __init__(self, stream, flush_before):
     self.stream = stream
     self.flush_before = flush_before
 
   def emit(self, line):
@@ skipping 366 matching lines @@
 
   # For error reporting.
   step_dict = locals().copy()
   step_dict.pop('kwargs')
   step_dict.pop('stream')
   step_dict.update(kwargs)
 
   for step_name in (seed_steps or []):
     stream.seed_step(step_name)
 
-  filter_obj = None
-  if not allow_subannotations:
-    class AnnotationFilter(chromium_utils.RunCommandFilter):
-      # Missing __init__
-      # Method could be a function (but not really since it's an override)
-      # pylint: disable=W0232,R0201
-      def FilterLine(self, line):
-        return line.replace('@@@', '###')
-    filter_obj = AnnotationFilter()
-
   ret = None
   with stream.step(name) as s:
     print_step(step_dict)
     try:
-      ret = chromium_utils.RunCommand(command=cmd,
-                                      cwd=cwd,
-                                      env=_merge_envs(os.environ, env),
-                                      filter_obj=filter_obj,
-                                      print_cmd=False)
+      proc = subprocess.Popen(
+          cmd,
+          env=_merge_envs(os.environ, env),
+          cwd=cwd,
+          stdout=subprocess.PIPE,
+          stderr=subprocess.PIPE)
+
+      outlock = threading.Lock()
+      def filter_lines(lock, allow_subannotations, inhandle, outhandle):
+        while True:
+          line = inhandle.readline()
+          if not line:
+            break
+          if not allow_subannotations:
+            if line.startswith('@@@'):
+              line = '###'+line[3:]

[agable, 2013/06/25 15:55:28]

This only replaces the leading @@@ with ###, whereas the old filter replaced all
@@@ with ###. The functionality is the same but the old one looked better.

[iannucci, 2013/06/25 21:22:09]

I'll do you one better. I changed it to just print a '!' before the line. Now we
don't have to copy any strings at all. Yay.

+          lock.acquire()
+          try:
+            outhandle.write(line)
+            outhandle.flush()
+          finally:
+            lock.release()
+
+      outthread = threading.Thread(
+          target=filter_lines,
+          args=(outlock, allow_subannotations, proc.stdout, sys.stdout))
+      errthread = threading.Thread(
+          target=filter_lines,
+          args=(outlock, allow_subannotations, proc.stderr, sys.stderr))
+      outthread.start()
+      errthread.start()
+      proc.wait()
+      outthread.join()
+      errthread.join()
+      ret = proc.returncode
     except OSError:
       # File wasn't found, error will be reported to stream when the exception
       # crosses the context manager.
       ret = -1
       raise
     if ret > 0:
       stream.step_cursor(stream.current_step)
       print 'step returned non-zero exit code: %d' % ret
       s.step_failure()
     if followup_fn:
@@ skipping 37 matching lines @@
   usage = '%s <command list file or - for stdin>' % sys.argv[0]
   parser = optparse.OptionParser(usage=usage)
   _, args = parser.parse_args()
   if not args:
     parser.error('Must specify an input filename.')
   if len(args) > 1:
     parser.error('Too many arguments specified.')
 
   steps = []
 
+  def force_list_str(lst):
+    ret = []
+    for v in lst:
+      if isinstance(v, basestring):
+        v = str(v)

[agable, 2013/06/25 15:55:28]

this isn't very unicode() friendly :(

[iannucci, 2013/06/25 21:22:09]

No, but we shouldn't have unicode in our annotator dictionaries. For some
reason, windows is hellbent on decoding ascii json as unicode(). 

In particular, subprocess hates unicode arguments for Popen (at least on 2.6),
and python hates unicode keys for dictionaries in kwargs (like when we **step).

+      elif isinstance(v, list):
+        v = force_list_str(v)
+      ret.append(v)
+    return ret
+
+  def force_dict_strs(obj):
+    ret = {}
+    for k, v in obj.iteritems():
+      if isinstance(v, basestring):
+        v = str(v)
+      elif isinstance(v, list):
+        v = force_list_str(v)
+      ret[str(k)] = v

[agable, 2013/06/25 15:55:28]

elif isinstance(v, dict):?

[iannucci, 2013/06/25 21:22:09]

Oops. Fixed.

+    return ret
+
   if args[0] == '-':
-    steps.extend(json.load(sys.stdin))
+    steps.extend(json.load(sys.stdin, object_hook=force_dict_strs))
   else:
     with open(args[0], 'rb') as f:
-      steps.extend(json.load(f))
+      steps.extend(json.load(f, object_hook=force_dict_strs))
 
   return 1 if run_steps(steps, False)[0] else 0
 
 
 if __name__ == '__main__':
   sys.exit(main())