Work around unicode issues by re-encoding properties as utf-8

recipes.py

 #!/usr/bin/env python
 # Copyright 2015 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """Tool to interact with recipe repositories.
 
 This tool operates on the nearest ancestor directory containing an
 infra/config/recipes.cfg.
 """
@@ skipping 164 matching lines @@
 
     with stream_engine.make_step_stream('Failure reason') as s:
       with s.new_log_stream('reason') as l:
         l.write_split(f.human_reason)
 
     return 1
 
   return 0
 
 
+def strip_unicode(obj):

[iannucci, 2017/04/06 18:10:12]

Could we move this function to e.g. util.py and then import+use it from the json
recipe module, so that it's not defined in two places? Then if we have to fix a
bug in it, we don't get diverging behaviors.

+  if isinstance(obj, unicode):
+    return obj.encode('utf-8', 'replace')
+
+  if isinstance(obj, list):
+    return map(strip_unicode, obj)
+
+  if isinstance(obj, dict):
+    new_obj = type(obj)(
+        (strip_unicode(k), strip_unicode(v)) for k, v in obj.iteritems() )
+    return new_obj
+
+  return obj
+
+
 def run(package_deps, args, op_args):
   from recipe_engine import run as recipe_run
   from recipe_engine import loader
   from recipe_engine import step_runner
   from recipe_engine import stream
   from recipe_engine import stream_logdog
 
   def get_properties_from_args(args):
     properties = dict(x.split('=', 1) for x in args)
     for key, val in properties.iteritems():
@@ skipping 30 matching lines @@
     properties = get_properties_from_json(args.properties)
   elif args.properties_file:
     properties = get_properties_from_file(args.properties_file)
   elif op_properties is not None:
     properties = op_properties
   else:
     properties = arg_properties
 
   properties['recipe'] = args.recipe
 
+  # Work around unicode issues by re-encoding properties as utf-8.
+  properties = strip_unicode(properties)
+  reload(sys)

[iannucci, 2017/04/06 18:10:12]

Now that I think about this, I would actually do this even earlier, maybe at
line 20 in this file? This needs to apply in all modes of recipe engine
execution, not just run.

+  sys.setdefaultencoding('UTF8')
+
   os.environ['PYTHONUNBUFFERED'] = '1'
   os.environ['PYTHONIOENCODING'] = 'UTF-8'
 
   _, config_file = get_package_config(args)
   universe_view = loader.UniverseView(
       loader.RecipeUniverse(
           package_deps, config_file), package_deps.root_package)
 
   workdir = (args.workdir or
       os.path.join(os.path.dirname(os.path.realpath(__file__)), 'workdir'))
@@ skipping 517 matching lines @@
 
   if not isinstance(ret, int):
     if ret is None:
       ret = 0
     else:
       print >> sys.stderr, ret
       ret = 1
   sys.stdout.flush()
   sys.stderr.flush()
   os._exit(ret)