Add support for repeated keys in TaskRequest.

appengine/swarming/message_conversion.py

 # Copyright 2015 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """This module facilitates conversion from dictionaries to ProtoRPC messages.
 
 Given a dictionary whose keys' names and values' types comport with the
 fields defined for a protorpc.messages.Message subclass, this module tries to
 generate a Message instance that corresponds to the provided dictionary. The
 "normal" use case is for ndb.Models which need to be represented as a
 ProtoRPC.
 """
 
 import datetime
 import json
 
 import swarming_rpcs
 
 from components import utils
 from server import task_pack
 from server import task_request
 from server import task_result
 
 
 ### Private API.
 
 
-def _string_pairs_from_dict(dictionary):
+def _string_pairs_from_list_of_tuples(dictionary):
   return [
     swarming_rpcs.StringPair(key=k, value=v)
-    for k, v in sorted((dictionary or {}).iteritems())
+    for k, v in sorted((dictionary or []))
   ]
 
 
+def _string_pairs_from_dict(dictionary):
+  return _string_pairs_from_list_of_tuples((dictionary or {}).iteritems())
+
+
 def _string_list_pairs_from_dict(dictionary):
   return [
     swarming_rpcs.StringListPair(key=k, value=v)
     for k, v in sorted((dictionary or {}).iteritems())
   ]
 
 
 def _ndb_to_rpc(cls, entity, **overrides):
   members = (f.name for f in cls.all_fields())
   kwargs = {m: getattr(entity, m) for m in members if not m in overrides}
@@ skipping 73 matching lines @@
     inputs_ref = _ndb_to_rpc(
         swarming_rpcs.FilesRef, entity.properties.inputs_ref)
 
   props = entity.properties
   properties = _ndb_to_rpc(
       swarming_rpcs.TaskProperties,
       props,
       caches=[_ndb_to_rpc(swarming_rpcs.CacheEntry, c) for c in props.caches],
       cipd_input=cipd_input,
       secret_bytes='<REDACTED>' if props.has_secret_bytes else None,
-      dimensions=_string_pairs_from_dict(props.dimensions),
+      dimensions=_string_pairs_from_list_of_tuples(props.dimensions),
       env=_string_pairs_from_dict(props.env),
       inputs_ref=inputs_ref)
 
   return _ndb_to_rpc(
       swarming_rpcs.TaskRequest,
       entity,
       authenticated=entity.authenticated.to_bytes(),
       properties=properties)
 
 
@@ skipping 25 matching lines @@
         ])
 
   inputs_ref = None
   if props.inputs_ref:
     inputs_ref = _rpc_to_ndb(task_request.FilesRef, props.inputs_ref)
 
   secret_bytes = None
   if props.secret_bytes:
     secret_bytes = task_request.SecretBytes(secret_bytes=props.secret_bytes)
 
-  if len(set(i.key for i in props.dimensions)) != len(props.dimensions):
-    raise ValueError('same dimension key cannot be specified twice')
+  # keys with ':' would corrupt the data. Rest of validation is done in
+  # TaskProperties.
+  if any(':' in i.key for i in props.dimensions):
+    raise ValueError('dimension key cannot contain ":"')
+  # Repeated keys would be lost. Rest of validation is done in TaskProperties.
   if len(set(i.key for i in props.env)) != len(props.env):
     raise ValueError('same environment variable key cannot be specified twice')
 
   properties = _rpc_to_ndb(
       task_request.TaskProperties,
       props,
       caches=[_rpc_to_ndb(task_request.CacheEntry, c) for c in props.caches],
       cipd_input=cipd_input,
       # Passing command=None is supported at API level but not at NDB level.
       command=props.command or [],
       has_secret_bytes=secret_bytes is not None,
       secret_bytes=None, # ignore this, it's handled out of band
       dimensions=None,
-      dimensions_dict={i.key: i.value for i in props.dimensions},
-      dimensions_flat=[],
+      dimensions_flat=sorted(
+          u'%s:%s' % (i.key, i.value) for i in props.dimensions),
       env={i.key: i.value for i in props.env},
       inputs_ref=inputs_ref)
 
   req = _rpc_to_ndb(
       task_request.TaskRequest,
       msg,
       created_ts=now,
       expiration_ts=now+datetime.timedelta(seconds=msg.expiration_secs),
       # It is set in task_request.init_new_request().
       authenticated=None,
@@ skipping 56 matching lines @@
     kwargs['run_id'] = entity.task_id
   else:
     assert entity.__class__ is task_result.TaskResultSummary, entity
     kwargs['properties_hash'] = (
         entity.properties_hash.encode('hex')
         if entity.properties_hash else None)
     # This returns the right value for deduped tasks too.
     k = entity.run_result_key
     kwargs['run_id'] = task_pack.pack_run_result_key(k) if k else None
   return _ndb_to_rpc(swarming_rpcs.TaskResult, entity, **kwargs)