Work around unicode issues by re-encoding properties as utf-8

recipes.py

 #!/usr/bin/env python
 # Copyright 2015 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """Tool to interact with recipe repositories.
 
 This tool operates on the nearest ancestor directory containing an
 infra/config/recipes.cfg.
 """
 
 import argparse
 import json
 import logging
 import os
 import shutil
 import subprocess
 import sys
 import tempfile
 
+reload(sys)

[iannucci, 2017/04/06 18:15:59]

Comment:

# This is necessary to ensure that str literals are by-default assumed to hold
utf-8. It also makes the
# implicit str(unicode(...)) act like unicode(...).encode('utf-8'), rather than
unicode(...).encode('ascii')

[Paweł Hajdan Jr., 2017/04/06 18:18:41]

Done.

+sys.setdefaultencoding('UTF8')
+
 ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
 sys.path.insert(0, ROOT_DIR)
 
 from recipe_engine import env
 from recipe_engine import arguments_pb2
+from recipe_engine import util as recipe_util
 from google.protobuf import json_format as jsonpb
 
 
 def get_package_config(args):
   from recipe_engine import package
 
   assert args.package, 'No recipe config (--package) given.'
   assert os.path.exists(args.package), (
       'Given recipes config file %s does not exist.' % args.package)
   return (
@@ skipping 189 matching lines @@
     properties = get_properties_from_json(args.properties)
   elif args.properties_file:
     properties = get_properties_from_file(args.properties_file)
   elif op_properties is not None:
     properties = op_properties
   else:
     properties = arg_properties
 
   properties['recipe'] = args.recipe
 
+  properties = recipe_util.strip_unicode(properties)
+
   os.environ['PYTHONUNBUFFERED'] = '1'
   os.environ['PYTHONIOENCODING'] = 'UTF-8'
 
   _, config_file = get_package_config(args)
   universe_view = loader.UniverseView(
       loader.RecipeUniverse(
           package_deps, config_file), package_deps.root_package)
 
   workdir = (args.workdir or
       os.path.join(os.path.dirname(os.path.realpath(__file__)), 'workdir'))
@@ skipping 517 matching lines @@
 
   if not isinstance(ret, int):
     if ret is None:
       ret = 0
     else:
       print >> sys.stderr, ret
       ret = 1
   sys.stdout.flush()
   sys.stderr.flush()
   os._exit(ret)