cipd: registerPackage method implementation.

appengine/chrome_infra_packages/cipd/api.py

Index: appengine/chrome_infra_packages/cipd/api.py
diff --git a/appengine/chrome_infra_packages/cipd/api.py b/appengine/chrome_infra_packages/cipd/api.py
new file mode 100644
index 0000000000000000000000000000000000000000..51d46deb445e8f084b309c1d01b759e764935e1f
--- /dev/null
+++ b/appengine/chrome_infra_packages/cipd/api.py
@@ -0,0 +1,210 @@
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Cloud Endpoints API for Package Repository service."""
+
+import endpoints
+
+from protorpc import message_types
+from protorpc import messages
+from protorpc import remote
+
+from components import auth
+from components import utils
+
+from . import acl
+from . import impl
+
+
+# This is used by endpoints indirectly.
+package = 'cipd'
+
+
+class InstanceMetadata(messages.Message):
+  """Description of how the package instance was built and registered."""
+  date = messages.StringField(1, required=True)
+  hostname = messages.StringField(2, required=True)
+  user = messages.StringField(3, required=True)
+
+  # Output only fields.
+  registered_by = messages.StringField(4, required=False)
+  registered_ts = messages.IntegerField(5, required=False)
+
+
+def metadata_from_entity(ent):
+  """PackageInstanceMetadata entity -> InstanceMetadata message."""
+  return InstanceMetadata(
+      date=ent.date,
+      hostname=ent.hostname,
+      user=ent.user,
+      registered_by=ent.registered_by.to_bytes(),
+      registered_ts=utils.datetime_to_timestamp(ent.registered_ts))
+
+
+def metadata_to_entity(msg):
+  """InstanceMetadata message -> PackageInstanceMetadata entity."""
+  return impl.PackageInstanceMetadata(
+      date=msg.date,
+      hostname=msg.hostname,
+      user=msg.user)
+
+
+class Signature(messages.Message):
+  """Single signature. Each package instance can have multiple signatures.
+
+  See also SignatureBlock struct in infra/tools/cipd/common.go.
+  """
+  hash_algo = messages.StringField(1, required=True)
+  digest = messages.BytesField(2, required=True)
+  signature_algo = messages.StringField(3, required=True)
+  signature_key = messages.StringField(4, required=True)
+  signature = messages.BytesField(5, required=True)
+
+  # Output only fields.
+  added_by = messages.StringField(6, required=False)
+  added_ts = messages.IntegerField(7, required=False)
+
+
+def signature_from_entity(ent):
+  """PackageInstanceSignature entity -> Signature message."""
+  return Signature(
+      hash_algo=ent.hash_algo,
+      digest=ent.digest,
+      signature_algo=ent.signature_algo,
+      signature_key=ent.signature_key,
+      signature=ent.signature,
+      added_by=ent.added_by.to_bytes(),
+      added_ts=utils.datetime_to_timestamp(ent.added_ts))
+
+
+def signature_to_entity(msg):
+  """Signature message -> PackageInstanceSignature entity."""
+  return impl.PackageInstanceSignature(
+      hash_algo=msg.hash_algo,
+      digest=msg.digest,
+      signature_algo=msg.signature_algo,
+      signature_key=msg.signature_key,
+      signature=msg.signature)
+
+
+class RegisterPackageRequest(messages.Message):
+  """Request to add a new package instance if it is not yet present.
+
+  Instance metadata is recorded only if package instance is not yet present.
+  Signatures are appended to the list of signatures (even for existing package).
+
+  Callers are expected to execute following protocol:
+    1. Attempt to register a package instance by callling registerPackage(msg).

[nodir, 2014/12/30 22:54:00]

typo: calllling

[Vadim Sh., 2014/12/31 01:27:35]

Done.

+    2. On UPLOAD_FIRST response, upload package data and finalize the upload by
+       using upload_session_id and upload_url and calling cas.finishUpload.
+    3. Once upload is finalized, call registerPackage(msg) again.
+  """
+  package_name = messages.StringField(1, required=True)
+  instance_id = messages.StringField(2, required=True)
+  metadata = messages.MessageField(InstanceMetadata, 3, required=True)
+  signatures = messages.MessageField(Signature, 4, repeated=True)
+
+
+class RegisterPackageResponse(messages.Message):
+  """Results of registerPackage call.
+
+  upload_session_id and upload_url (if present) can be used with CAS service
+  (finishUpload call in particular).
+  """
+  class Status(messages.Enum):

[nodir, 2014/12/30 22:54:00]

Blank line after """

[Vadim Sh., 2014/12/31 01:27:35]

Done.

+    # Package instance successfully registered.
+    REGISTERED = 1
+    # Such package instance already exists. It is not an error.
+    ALREADY_REGISTERED = 2
+    # Package data has to be upload to CAS first.
+    UPLOAD_FIRST = 3
+    # Some unexpected fatal error happened.
+    ERROR = 4
+
+  # Status of this operation, defines what other fields to expect.
+  status = messages.EnumField(
+      'RegisterPackageResponse.Status', 1, required=True)

[nodir, 2014/12/30 22:54:00]

Why not EnumField(Status, ...) ?

[Vadim Sh., 2014/12/31 01:27:35]

Done. For some reason I though it does like local classes.

+
+  # For REGISTERED or ALREADY_REGISTERED a current metadata of package instance.
+  metadata = messages.MessageField(InstanceMetadata, 2, required=False)
+
+  # For UPLOAD_FIRST status, a unique identifier of the upload operation.
+  upload_session_id = messages.StringField(3, required=False)
+  # For UPLOAD_FIRST status, URL to PUT file to via resumable upload protocol.
+  upload_url = messages.StringField(4, required=False)
+
+  # For ERROR status, a error message.
+  error_message = messages.StringField(5, required=False)
+
+
+@auth.endpoints_api(
+    name='repo',
+    version='v1',
+    title='Package Repository API')
+class PackageRepositoryApi(remote.Service):
+  """Package Repository API."""
+
+  @auth.endpoints_method(
+      RegisterPackageRequest,
+      RegisterPackageResponse,
+      http_method='POST',
+      name='registerPackage')
+  @auth.require(lambda: not auth.get_current_identity().is_anonymous)
+  def register_package(self, request):
+    """Registers a new package instance in the repository."""
+    if not impl.is_valid_package_name(request.package_name):
+      raise endpoints.BadRequestException('Invalid package name')
+    if not impl.is_valid_instance_id(request.instance_id):
+      raise endpoints.BadRequestException('Invalid instance ID')

[nodir, 2014/12/30 22:54:00]

Consider returning different error reasons. A program would have hard time
distinguishing different error responses

[Vadim Sh., 2014/12/31 01:27:35]

Client will do client side validation. Checks here are mostly "asserts".

+

[nodir, 2014/12/30 22:54:00]

metadata is not validated? To me, either it should be arbitrary key-value pairs,
or it is semantic and validated

[Vadim Sh., 2014/12/31 01:27:35]

I was planing to add very few required FYI only fields. Custom key-value paris
will be implemented as part of labels system.

In fact, f*ck it, I'll remove notion of "metadata" for now and implement it
later together with labels.

[nodir, 2015/01/02 19:13:03]

Acknowledged.

+    caller = auth.get_current_identity()
+    if not acl.can_register_package(request.package_name, caller):
+      raise auth.AuthorizationError()
+
+    service = impl.get_repo_service()
+    if service is None:
+      raise endpoints.InternalServerErrorException('Service is not configured')
+
+    # Metadata proto -> entity.
+    metadata = metadata_to_entity(request.metadata)
+    metadata.registered_by = caller
+    metadata.registered_ts = utils.utcnow()
+
+    # Signature list proto -> entity.
+    signatures = []
+    for sig in request.signatures:
+      ent = signature_to_entity(sig)
+      ent.added_by = caller
+      ent.added_ts = utils.utcnow()

[nodir, 2014/12/30 22:54:00]

call utcnow once per request

[Vadim Sh., 2014/12/31 01:27:35]

Done.

+      signatures.append(ent)
+
+    # Already registered? Just attach any new signatures.
+    pkg = service.get_instance(request.package_name, request.instance_id)
+    if pkg is not None:
+      service.add_signatures(
+          request.package_name, request.instance_id, signatures)

[nodir, 2014/12/30 22:54:00]

I think you should not add signatures

[Vadim Sh., 2014/12/31 01:27:35]

Then there's no way to resign a package. I mean it can be done as a separate API
call probably, but it would complicate the client.

I'd like to keep it as is for now and maybe later implement a separate call,
when client implementation is more clear.

+      return RegisterPackageResponse(
+          status=RegisterPackageResponse.Status.ALREADY_REGISTERED,
+          metadata=metadata_from_entity(pkg.metadata))
+
+    # Need to upload to CAS first? Open an upload session. Caller must use
+    # CASServiceApi to finish the upload and then call registerPackage again.
+    if not service.is_data_uploaded(request.package_name, request.instance_id):
+      upload_url, upload_session_id = service.create_upload_session(

[Vadim Sh., 2014/12/30 02:22:26]

It makes 'cas' service and 'repo' service a bit tangled :( The problem is, CAS
service doesn't know anything about authorization (e.g. who is allowed to
upload) since ACL is formulated in terms of package names (e.g. package
"infra/*" are allowed to be modified by chrome-infra-team group) that CAS
service has no idea about.

There are two ways to make it work:
1) Hide generic 'initiateUpload' CAS call and expose a package repository
specific one (with proper authorization). That is what I do in this CL.
2) Make generic initiateUpload call accept some sort of nonce that grants an
upload of specific item, and make registerPackage check ACL and generate that
nonce. It makes services more decoupled, but introduces a round trip in the
protocol and makes client more knowledgeable.  

¯\_(ツ)_/¯

+          request.package_name, request.instance_id, caller)
+      return RegisterPackageResponse(
+          status=RegisterPackageResponse.Status.UPLOAD_FIRST,
+          upload_session_id=upload_session_id,
+          upload_url=upload_url)
+
+    # Package data is in the store. Make an entity.
+    pkg, registered = service.register_instance(
+        request.package_name, request.instance_id, metadata, signatures)
+    if registered:
+      status = RegisterPackageResponse.Status.REGISTERED
+    else:  # pragma: no cover
+      status = RegisterPackageResponse.Status.ALREADY_REGISTERED
+    return RegisterPackageResponse(
+        status=status,
+        metadata=metadata_from_entity(pkg.metadata))