Split secure command channel and untrusted application channel

src/trusted/service_runtime/sel_ldr.c

Index: src/trusted/service_runtime/sel_ldr.c
diff --git a/src/trusted/service_runtime/sel_ldr.c b/src/trusted/service_runtime/sel_ldr.c
index cb750c704121672b400a6082fd126ace3b90038d..74c4ccf83e9354bd13ad00a902a03dca649e39cf 100644
--- a/src/trusted/service_runtime/sel_ldr.c
+++ b/src/trusted/service_runtime/sel_ldr.c
@@ -158,6 +158,9 @@ int NaClAppWithSyscallTableCtor(struct NaClApp               *nap,
   nap->service_address = NULL;
   nap->bootstrap_channel = NULL;
   nap->secure_service = NULL;
+
+  nap->num_secure_channel = 0;
+
   nap->manifest_proxy = NULL;
   nap->kernel_service = NULL;
   nap->resource_phase = NACL_RESOURCE_PHASE_START;
@@ -1267,77 +1270,6 @@ void NaClBlockIfCommandChannelExists(struct NaClApp *nap) {
   }
 }
 
-
-/*
- * Secure command channels.
- */
-
-struct NaClSecureService {
-  struct NaClSimpleService      base;
-  struct NaClApp                *nap;
-};
-
-struct NaClSimpleServiceVtbl const kNaClSecureServiceVtbl;
-
-
-struct NaClConnectionHandler {
-  struct NaClConnectionHandler  *next;
-
-  /* used by NaClSimpleRevServiceClient's ClientCallback fn */
-  void                          (*handler)(
-      void                        *state,
-      struct NaClThreadInterface  *tif,
-      struct NaClDesc             *conn);
-  void                          *state;
-};
-
-struct NaClSecureReverseClient {
-  struct NaClSimpleRevClient        base;
-  struct NaClApp                    *nap;
-
-  struct NaClMutex                  mu;
-
-  struct NaClConnectionHandler      *queue_head;
-  struct NaClConnectionHandler      **queue_insert;
-};
-
-struct NaClSimpleRevClientVtbl const kNaClSecureReverseClientVtbl;
-
-
-int NaClSecureServiceCtor(struct NaClSecureService          *self,
-                          struct NaClSrpcHandlerDesc const  *srpc_handlers,
-                          struct NaClApp                    *nap) {
-  NaClLog(4,
-          "Entered NaClSecureServiceCtor: self 0x%"NACL_PRIxPTR"\n",
-          (uintptr_t) self);
-  if (NACL_FI_ERROR_COND(
-          "NaClSecureServiceCtor__NaClSimpleServiceWithSocketCtor",
-          !NaClSimpleServiceWithSocketCtor(
-              &self->base,
-              srpc_handlers,
-              NaClThreadInterfaceThreadFactory,
-              (void *) NULL,
-              nap->service_port,
-              nap->service_address))) {
-    goto failure_simple_ctor;
-  }
-  self->nap = nap;
-
-  NACL_VTBL(NaClRefCount, self) =
-      (struct NaClRefCountVtbl *) &kNaClSecureServiceVtbl;
-  return 1;
- failure_simple_ctor:
-  return 0;
-}
-
-void NaClSecureServiceDtor(struct NaClRefCount *vself) {
-  struct NaClSecureService *self = (struct NaClSecureService *) vself;
-
-  NACL_VTBL(NaClRefCount, self) = (struct NaClRefCountVtbl const *)
-      &kNaClSimpleServiceVtbl;
-  (*NACL_VTBL(NaClRefCount, self)->Dtor)(vself);
-}
-
 /*
  * The first connection is performed by this callback handler.  This
  * spawns a client thread that will bootstrap the other connections by
@@ -1397,14 +1329,6 @@ static void NaClSecureReverseClientCallback(
   NaClLog(4, "Leaving NaClSecureReverseClientCallback\n");
 }
 
-/* fwd */
-int NaClSecureReverseClientCtor(
-    struct NaClSecureReverseClient *self,
-    void                            (*client_callback)(
-        void *, struct NaClThreadInterface *, struct NaClDesc *),
-    void                            *state,
-    struct NaClApp                  *nap);
-
 static void NaClSecureReverseClientSetup(struct NaClSrpcRpc     *rpc,
                                          struct NaClSrpcArg     **in_args,
                                          struct NaClSrpcArg     **out_args,
@@ -1473,269 +1397,64 @@ done:
   NaClLog(4, "Leaving NaClSecureReverseClientSetup\n");
 }
 
-/*
- * Only called at startup and thereafter by the reverse secure
- * channel, with |self| locked.
- */
-static
-int NaClSecureReverseClientInsertHandler_mu(
-    struct NaClSecureReverseClient  *self,
-    void                            (*h)(void *,
-                                         struct NaClThreadInterface *,
-                                         struct NaClDesc *),
-    void                            *d) {
-  struct NaClConnectionHandler *entry;
+struct NaClSrpcHandlerDesc const secure_handlers[]; /* fwd */
 
-  NaClLog(4,
-          ("NaClSecureReverseClientInsertHandler_mu: h 0x%"NACL_PRIxPTR","
-           " d 0x%"NACL_PRIxPTR"\n"),
-          (uintptr_t) h,
-          (uintptr_t) d);
-  entry = (struct NaClConnectionHandler *) malloc(sizeof *entry);
-  if (NULL == entry) {
-    return 0;
-  }
-  entry->handler = h;
-  entry->state = d;
-  entry->next = (struct NaClConnectionHandler *) NULL;
-  *self->queue_insert = entry;
-  self->queue_insert = &entry->next;
-
-  return 1;
-}
-
-/*
- * Caller must set up handler before issuing connection request RPC on
- * nap->reverse_channel, since otherwise the connection handler queue
- * may be empty and the connect code would abort.  Because the connect
- * doesn't wait for a handler, we don't need a condvar.
- *
- * We do not need to serialize on the handlers, since the
- * RPC-server/IMC-client implementation should not distinguish one
- * connection from another: it is okay for two handlers to be
- * inserted, and two connection request RPCs to be preformed
- * (sequentially, since they are over a single channel), and have the
- * server side spawn threads that asynchronously connect twice, in the
- * "incorrect" order, etc.
- */
-int NaClSecureReverseClientInsertHandler(
-    struct NaClSecureReverseClient *self,
-    void                            (*handler)(
-        void                        *handlr_state,
-        struct NaClThreadInterface  *thread_if,
-        struct NaClDesc             *new_conn),
-    void                            *handler_state) {
-  int retval;
-
-  NaClXMutexLock(&self->mu);
-  retval = NaClSecureReverseClientInsertHandler_mu(self,
-                                                   handler, handler_state);
-  NaClXMutexUnlock(&self->mu);
-  return retval;
-}
+static void NaClSecureCommandServiceSetupRpc(struct NaClSrpcRpc     *rpc,
+                                             struct NaClSrpcArg     **in_args,
+                                             struct NaClSrpcArg     **out_args,
+                                             struct NaClSrpcClosure *done) {
+  struct NaClApp                  *nap =
+      (struct NaClApp *) rpc->channel->server_instance_data;
+  struct NaClSecureService        *secure_service;
+  struct NaClDesc                 *pair[2];

[bsy, 2012/09/25 18:37:10]

plz make the pair be part of the NaClApp object as discussed, so that we only
create one pair and spin off one thread for all additional command channels.

[Petr Hosek, 2012/09/25 23:09:11]

Separated secure command channel and untrusted app channel as discussed.

 
-static
-struct NaClConnectionHandler *NaClSecureReverseClientPopHandler(
-    struct NaClSecureReverseClient *self) {
-  struct NaClConnectionHandler *head;
+  UNREFERENCED_PARAMETER(in_args);
+  NaClLog(4, "Entered NaClSecureCommandServiceSetupRpc\n");
 
-  NaClLog(4, "Entered NaClSecureReverseClientPopHandler, acquiring lock\n");
-  NaClXMutexLock(&self->mu);
-  NaClLog(4, "NaClSecureReverseClientPopHandler, got lock\n");
-  head = self->queue_head;
-  if (NULL == head) {
-    NaClLog(LOG_FATAL,
-            "NaClSecureReverseClientPopHandler:  empty handler queue\n");
-  }
-  if (NULL == (self->queue_head = head->next)) {
-    NaClLog(4, "NaClSecureReverseClientPopHandler, last elt patch up\n");
-    self->queue_insert = &self->queue_head;
+  if (0 != NaClCommonDescMakeBoundSock(pair)) {
+    NaClLog(LOG_FATAL, "Cound not create service socket\n");
   }
-  NaClLog(4, "NaClSecureReverseClientPopHandler, unlocking\n");
-  NaClXMutexUnlock(&self->mu);
-
-  head->next = NULL;
-  NaClLog(4,
-          ("Leaving NaClSecureReverseClientPopHandler:"
-           " returning %"NACL_PRIxPTR"\n"),
-          (uintptr_t) head);
-  return head;
-}
-
-static
-void NaClSecureReverseClientInternalCallback(
-    void                        *state,
-    struct NaClThreadInterface  *tif,
-    struct NaClDesc             *conn) {
-  struct NaClSecureReverseClient *self =
-      (struct NaClSecureReverseClient *) state;
-  struct NaClConnectionHandler *hand_ptr;
-
-  UNREFERENCED_PARAMETER(tif);
-  NaClLog(4, "Entered NaClSecureReverseClientInternalCallback\n");
-  hand_ptr = NaClSecureReverseClientPopHandler(self);
-  NaClLog(4, " got callback object %"NACL_PRIxPTR"\n", (uintptr_t) hand_ptr);
-  NaClLog(4,
-          " callback:0x%"NACL_PRIxPTR"(0x%"NACL_PRIxPTR",0x%"NACL_PRIxPTR")\n",
-          (uintptr_t) hand_ptr->handler,
-          (uintptr_t) hand_ptr->state,
-          (uintptr_t) conn);
-  (*hand_ptr->handler)(hand_ptr->state, tif, conn);
-  NaClLog(4, "NaClSecureReverseClientInternalCallback: freeing memory\n");
-  free(hand_ptr);
-  NaClLog(4, "Leaving NaClSecureReverseClientInternalCallback\n");
-}
-
-/*
- * Require an initial connection handler in the Ctor, so that it's
- * obvious that a reverse client needs to accept an IMC connection
- * from the server to get things bootstrapped.
- */
-int NaClSecureReverseClientCtor(
-    struct NaClSecureReverseClient  *self,
-    void                            (*client_callback)(
-        void *, struct NaClThreadInterface*, struct NaClDesc *),
-    void                            *state,
-    struct NaClApp                  *nap) {
   NaClLog(4,
-          ("Entered NaClSecureReverseClientCtor, self 0x%"NACL_PRIxPTR","
-           " nap 0x%"NACL_PRIxPTR"\n"),
-          (uintptr_t) self,
-          (uintptr_t) nap);
-  if (!NaClSimpleRevClientCtor(&self->base,
-                               NaClSecureReverseClientInternalCallback,
-                               (void *) self,
-                               NaClThreadInterfaceThreadFactory,
-                               (void *) NULL)) {
-    goto failure_simple_ctor;
-  }
-  NaClLog(4, "NaClSecureReverseClientCtor: Mutex\n");
-  if (!NaClMutexCtor(&self->mu)) {
-    goto failure_mutex_ctor;
-  }
-  self->nap = nap;
-  self->queue_head = (struct NaClConnectionHandler *) NULL;
-  self->queue_insert = &self->queue_head;
-
-  NACL_VTBL(NaClRefCount, self) =
-      (struct NaClRefCountVtbl *) &kNaClSecureReverseClientVtbl;
+          "got bound socket at 0x%08"NACL_PRIxPTR", "
+          "addr at 0x%08"NACL_PRIxPTR"\n",
+          (uintptr_t) pair[0],
+          (uintptr_t) pair[1]);
 
-  NaClLog(4, "NaClSecureReverseClientCtor: InsertHandler\n");
-  if (!NaClSecureReverseClientInsertHandler(self,
-                                            client_callback,
-                                            state)) {
-    goto failure_handler_insert;
+  secure_service = (struct NaClSecureService *) malloc(
+      sizeof *secure_service);
+  if (NULL == secure_service) {
+    NaClLog(LOG_ERROR, "Out of memory for secure command channel\n");
+    rpc->result = NACL_SRPC_RESULT_APP_ERROR;
+    goto done;
   }
-
-  NaClLog(4, "Leaving NaClSecureReverseClientCtor\n");
-  return 1;
-
- failure_handler_insert:
-  NaClLog(4, "NaClSecureReverseClientCtor: InsertHandler failed\n");
-  NACL_VTBL(NaClRefCount, self) =
-      (struct NaClRefCountVtbl *) &kNaClSimpleRevClientVtbl;
-
-  self->nap = NULL;
-  self->queue_insert = (struct NaClConnectionHandler **) NULL;
-  NaClMutexDtor(&self->mu);
-
- failure_mutex_ctor:
-  NaClLog(4, "NaClSecureReverseClientCtor: Mutex failed\n");
-  (*NACL_VTBL(NaClRefCount, self)->Dtor)((struct NaClRefCount *) self);
- failure_simple_ctor:
-  NaClLog(4, "Leaving NaClSecureReverseClientCtor\n");
-  return 0;
-}
-
-void NaClSecureReverseClientDtor(struct NaClRefCount *vself) {
-  struct NaClSecureReverseClient *self =
-      (struct NaClSecureReverseClient *) vself;
-
-  struct NaClConnectionHandler  *entry;
-  struct NaClConnectionHandler  *next;
-
-  for (entry = self->queue_head; NULL != entry; entry = next) {
-    next = entry->next;
-    free(entry);
+  if (!NaClSecureServiceCtor(secure_service,
+                             secure_handlers,
+                             nap,
+                             pair[0],
+                             pair[1])) {
+    NaClLog(LOG_ERROR, "NaClSecureServiceCtor failed\n");
+    free(secure_service);
+    rpc->result = NACL_SRPC_RESULT_APP_ERROR;
+    goto done;
   }
-  NaClMutexDtor(&self->mu);
-
-  NACL_VTBL(NaClRefCount, self) = (struct NaClRefCountVtbl const *)
-      &kNaClSimpleRevClientVtbl;
-  (*NACL_VTBL(NaClRefCount, self)->Dtor)(vself);
-}
-
-int NaClSecureServiceConnectionFactory(
-    struct NaClSimpleService            *vself,
-    struct NaClDesc                     *conn,
-    struct NaClSimpleServiceConnection  **out) {
-  struct NaClSecureService *self =
-      (struct NaClSecureService *) vself;
-
-  /* our instance_data is not connection specific */
-  return NaClSimpleServiceConnectionFactoryWithInstanceData(
-      vself, conn, (void *) self->nap, out);
-}
-
-int NaClSecureServiceAcceptAndSpawnHandler(struct NaClSimpleService *vself) {
-  int rv;
+  out_args[0]->u.hval = NaClDescRef(pair[1]);
+  rpc->result = NACL_SRPC_RESULT_OK;
 
-  NaClLog(4,
-          "NaClSecureServiceAcceptAndSpawnHandler: invoking base class vfn\n");
-  rv = (*kNaClSimpleServiceVtbl.AcceptAndSpawnHandler)(vself);
-  if (0 != rv) {
+  NaClLog(4, "NaClSecureCommandChannel: starting service thread\n");
+  if (!NaClSimpleServiceStartServiceThread((struct NaClSimpleService *)
+                                           secure_service)) {
     NaClLog(LOG_FATAL,
-            "Secure channel AcceptAndSpawnHandler returned %d\n",
-            rv);
+            "Could not start secure command channel service thread\n");
   }
-  NaClThreadExit(0);
-  /*
-   * NOTREACHED The port is now to be used by untrusted code: all
-   * subsequent connections are handled there.
-   */
-  return rv;
-}
 
-void NaClSecureServiceRpcHandler(struct NaClSimpleService           *vself,
-                                 struct NaClSimpleServiceConnection *vconn) {
-
-  NaClLog(4, "NaClSecureChannelThread started\n");
-  (*kNaClSimpleServiceVtbl.RpcHandler)(vself, vconn);
-  NaClLog(4, "NaClSecureChannelThread: channel closed, exiting.\n");
-  NaClExit(0);
+done:
+  (*done->Run)(done);
+  NaClLog(4, "Leaving NaClSecureCommandServiceSetupRpc\n");
 }
 
-struct NaClSimpleServiceVtbl const kNaClSecureServiceVtbl = {
-  {
-    NaClSecureServiceDtor,
-  },
-  NaClSecureServiceConnectionFactory,
-  NaClSimpleServiceAcceptConnection,
-  NaClSecureServiceAcceptAndSpawnHandler,
-  NaClSecureServiceRpcHandler,
-};
-
-struct NaClSimpleRevClientVtbl const kNaClSecureReverseClientVtbl = {
-  {
-    NaClSecureReverseClientDtor,
-  },
-};
-
-
 void NaClSecureCommandChannel(struct NaClApp *nap) {
   struct NaClSecureService *secure_command_server;
 
-  static struct NaClSrpcHandlerDesc const secure_handlers[] = {
-    { "hard_shutdown::", NaClSecureChannelShutdownRpc, },
-    { "start_module::i", NaClSecureChannelStartModuleRpc, },
-    { "log:is:", NaClSecureChannelLog, },
-    { "load_module:hs:", NaClLoadModuleRpc, },
-    { "load_irt:h:", NaClLoadIrtRpc, },
-    { "reverse_setup::h", NaClSecureReverseClientSetup, },
-    /* add additional calls here.  upcall set up?  start module signal? */
-    { (char const *) NULL, (NaClSrpcMethod) NULL, },
-  };
-
   NaClLog(4, "Entered NaClSecureCommandChannel\n");
 
   secure_command_server = (struct NaClSecureService *) malloc(
@@ -1746,7 +1465,10 @@ void NaClSecureCommandChannel(struct NaClApp *nap) {
   }
   if (NACL_FI_ERROR_COND("NaClSecureCommandChannel__NaClSecureServiceCtor",
                          !NaClSecureServiceCtor(secure_command_server,
-                                                secure_handlers, nap))) {
+                                                secure_handlers,
+                                                nap,
+                                                nap->service_port,
+                                                nap->service_address))) {
     NaClLog(LOG_FATAL, "NaClSecureServiceCtor failed\n");
   }
   nap->secure_service = secure_command_server;
@@ -1763,6 +1485,43 @@ void NaClSecureCommandChannel(struct NaClApp *nap) {
   NaClLog(4, "Leaving NaClSecureCommandChannel\n");
 }
 
+struct NaClSrpcHandlerDesc const secure_handlers[] = {
+  { "hard_shutdown::", NaClSecureChannelShutdownRpc, },
+  { "start_module::i", NaClSecureChannelStartModuleRpc, },
+  { "log:is:", NaClSecureChannelLog, },
+  { "load_module:hs:", NaClLoadModuleRpc, },
+  { "load_irt:h:", NaClLoadIrtRpc, },
+  { "reverse_setup::h", NaClSecureReverseClientSetup, },
+  { "command_setup::h", NaClSecureCommandServiceSetupRpc, },
+  /* add additional calls here.  upcall set up?  start module signal? */
+  { (char const *) NULL, (NaClSrpcMethod) NULL, },
+};
+
+void NaClSecureChannelCountIncr(struct NaClApp *nap) {
+  NaClLog(4, "NaClSecureChannelCountIncr\n");
+  NaClXMutexLock(&nap->mu);
+  if (0 == ++nap->num_secure_channel) {
+    NaClLog(LOG_FATAL,
+            "NaClSecureChannelCountIncr: "
+            "secure channel count overflow!\n");
+  }
+  NaClXMutexUnlock(&nap->mu);
+}
+
+void NaClSecureChannelCountDecr(struct NaClApp *nap) {
+  NaClLog(4, "NaClSecureChannelCountDecr\n");
+  NaClXMutexLock(&nap->mu);
+  if (0 == nap->num_secure_channel) {
+    NaClLog(4,
+            "NaClSecureChannelCountDecr: "
+            "no more secure channels, exiting.\n");
+    NaClExit(0);
+  }
+  if (0 == --nap->num_secure_channel) {
+    NaClXCondVarBroadcast(&nap->cv);
+  }
+  NaClXMutexUnlock(&nap->mu);
+}
 
 /*
  * It is fine to have multiple I/O operations read from memory in Write