Non-SFI mode: Add Linux asynchronous signal support

src/nonsfi/linux/irt_signal_handling.c

+/*
+ * Copyright (c) 2015 The Native Client Authors. All rights reserved.
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ */
+
+#include <errno.h>
+#include <pthread.h>
+#include <signal.h>
+#include <stdint.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "native_client/src/include/elf_constants.h"
+#include "native_client/src/include/nacl/nacl_exception.h"
+#include "native_client/src/include/nacl_macros.h"
+#include "native_client/src/nonsfi/linux/irt_signal_handling.h"
+#include "native_client/src/nonsfi/linux/linux_sys_private.h"
+#include "native_client/src/nonsfi/linux/linux_syscall_defines.h"
+#include "native_client/src/nonsfi/linux/linux_syscall_structs.h"
+#include "native_client/src/nonsfi/linux/nonsfi_signal.h"
+#include "native_client/src/public/linux_syscalls/sys/syscall.h"
+#include "native_client/src/public/nonsfi/irt_signal_handling.h"
+#include "native_client/src/untrusted/irt/irt.h"
+
+typedef struct compat_sigaltstack {
+  uint32_t ss_sp;
+  int32_t ss_flags;
+  uint32_t ss_size;
+} linux_stack_t;
+

[Mark Seaborn, 2015/08/12 01:43:07]

Nit: be consistent about whether you have 1 or 2 empty lines between top-level
decl

[Luis Héctor Chávez, 2015/08/12 22:14:26]

Done.

+
+/* From linux/arch/arm/include/asm/ucontext.h */
+struct linux_ucontext_t {

[Mark Seaborn, 2015/08/12 01:43:07]

You've moved a lot of this from src/nonsfi/linux/irt_exception_handling.c.  How
about moving the rest of irt_exception_handling.c into this file too?

Then that will show up as renaming irt_exception_handling.c to
irt_signal_handling.c.  More definitions can stay as "static".  There's a lot of
overlap between these two pieces of functionality -- I don't think having two
separate files is necessary.

[Luis Héctor Chávez, 2015/08/12 22:14:26]

Done.

+  uint32_t uc_flags;
+  uint32_t uc_link;
+  linux_stack_t uc_stack;
+  linux_mcontext_t uc_mcontext;
+  linux_sigset_t uc_sigmask;
+  /* More data follows which we don't care about. */
+};
+
+pthread_mutex_t g_signal_handler_mutex = PTHREAD_MUTEX_INITIALIZER;
+static NaClExceptionHandler g_signal_handler_function_pointer = NULL;
+static int g_signal_handler_initialized = 0;
+static int g_tgid = 0;
+static int g_main_tid;
+
+static void machine_context_to_register(const linux_mcontext_t *mctx,
+                                        NaClUserRegisterState *dest) {
+#if defined(__i386__)
+#define COPY_REG(A) dest->e##A = mctx->A
+  COPY_REG(ax);
+  COPY_REG(cx);
+  COPY_REG(dx);
+  COPY_REG(bx);
+  COPY_REG(bp);
+  COPY_REG(si);
+  COPY_REG(di);
+#undef COPY_REG
+  dest->stack_ptr = mctx->sp;
+  dest->prog_ctr = mctx->ip;
+  dest->flags = mctx->flags;
+#elif defined(__arm__)
+#define COPY_REG(A) dest->A = mctx->arm_##A
+  COPY_REG(r0);
+  COPY_REG(r1);
+  COPY_REG(r2);
+  COPY_REG(r3);
+  COPY_REG(r4);
+  COPY_REG(r5);
+  COPY_REG(r6);
+  COPY_REG(r7);
+  COPY_REG(r8);
+  COPY_REG(r9);
+  COPY_REG(r10);
+  COPY_REG(r11);
+  COPY_REG(r12);
+#undef COPY_REG
+  dest->stack_ptr = mctx->arm_sp;
+  dest->lr = mctx->arm_lr;
+  dest->prog_ctr = mctx->arm_pc;
+  dest->cpsr = mctx->arm_cpsr;
+#else
+# error Unsupported architecture
+#endif
+}
+
+void exception_frame_from_signal_context(
+    struct NonSfiExceptionFrame *frame,
+    const void *raw_ctx) {
+  const struct linux_ucontext_t *uctx = (struct linux_ucontext_t *) raw_ctx;
+  const linux_mcontext_t *mctx = &uctx->uc_mcontext;
+  frame->context.size = (((uintptr_t) (&frame->portable + 1))
+                         - (uintptr_t) &frame->context);
+  frame->context.portable_context_offset = ((uintptr_t) &frame->portable
+                                            - (uintptr_t) &frame->context);
+  frame->context.portable_context_size = sizeof(frame->portable);
+  frame->context.regs_size = sizeof(frame->context.regs);
+
+  memset(frame->context.reserved, 0, sizeof(frame->context.reserved));
+  machine_context_to_register(mctx, &frame->context.regs);
+  frame->portable.prog_ctr = frame->context.regs.prog_ctr;
+  frame->portable.stack_ptr = frame->context.regs.stack_ptr;
+
+#if defined(__i386__)
+  frame->context.arch = EM_386;
+  frame->portable.frame_ptr = frame->context.regs.ebp;
+#elif defined(__arm__)
+  frame->context.arch = EM_ARM;
+  /* R11 is frame pointer in ARM mode, R8 is frame pointer in thumb mode. */
+  frame->portable.frame_ptr = frame->context.regs.r11;
+#else
+# error Unsupported architecture
+#endif
+}
+
+/* A replacement of sigreturn. It does not restore the signal mask. */
+static void __attribute__((noreturn))
+nonsfi_restore_context(const linux_mcontext_t *mctx) {
+
+#if defined(__i386__)
+
+#define OFFSET(name) \
+  [name] "i" (offsetof(linux_mcontext_t, name))
+#define RESTORE_SEGMENT(name) \
+  "mov %c[" #name "](%%eax), %%" #name "\n"
+#define RESTORE(name) \
+  "movl %c[" #name "](%%eax), %%e" #name "\n"
+
+  __asm__ __volatile__(
+      /* Restore floating-point environment */

[Mark Seaborn, 2015/08/12 01:43:07]

What about restoring SSE/AVX registers?  I think they're all non-callee-saved.

I would suggest you do this as a follow-on change.

[Luis Héctor Chávez, 2015/08/12 22:14:26]

Acknowledged.

+      "mov %c[fpstate](%%eax), %%ecx\n"
+      "fldenv (%%ecx)\n"
+
+      /* Restore all segment registers */
+      RESTORE_SEGMENT(gs)
+      RESTORE_SEGMENT(fs)
+      RESTORE_SEGMENT(es)
+      RESTORE_SEGMENT(ds)
+
+      /*
+       * Restore most of the other registers.
+       */
+      RESTORE(di)
+      RESTORE(si)
+      RESTORE(bp)
+      RESTORE(bx)
+
+      /*
+       * Prepare the last registers. eip *must* be one slot above the original
+       * stack, since that is the only way eip and esp can be simultaneously
+       * restored. Here, we are using ecx as the pseudo stack pointer, and edx
+       * as a scratch register. Once the stack is laid out the way we want it to
+       * be, restore edx and eax last.
+       */
+      "mov %c[sp](%%eax), %%ecx\n"
+      "mov %c[ip](%%eax), %%edx\n"
+      "mov %%edx, -4(%%ecx)\n"
+      "mov %c[flags](%%eax), %%edx\n"
+      "mov %%edx, -8(%%ecx)\n"
+      "mov %c[cx](%%eax), %%edx\n"
+      "mov %%edx, -12(%%ecx)\n"
+      RESTORE(dx)
+      RESTORE(ax)
+      "lea -12(%%ecx), %%esp\n"
+
+      /*
+       * Finally pop ecx off the stack, restore the processor flags, and return
+       * to simultaneously restore esp and eip.
+       */
+      "pop %%ecx\n"
+      "popf\n"
+      "ret\n"
+      :
+      : "a" (mctx),
+        OFFSET(gs),
+        OFFSET(fs),
+        OFFSET(es),
+        OFFSET(ds),
+        OFFSET(di),
+        OFFSET(si),
+        OFFSET(bp),
+        OFFSET(sp),
+        OFFSET(bx),
+        OFFSET(dx),
+        OFFSET(cx),
+        OFFSET(ax),
+        OFFSET(ip),
+        OFFSET(flags),
+        OFFSET(fpstate)
+  );
+
+#undef OFFSET
+#undef RESTORE
+#undef RESTORE_SEGMENT
+
+#elif defined(__arm__)
+
+#define OFFSET(name) \
+  [name] "I" (offsetof(linux_mcontext_t, arm_ ## name) - \
+              offsetof(linux_mcontext_t, arm_r0))
+
+  register uint32_t a14 __asm__("r14") = (uint32_t) &mctx->arm_r0;
+
+  __asm__ __volatile__(
+      /* Restore flags */

[Mark Seaborn, 2015/08/12 01:43:07]

What about FP registers too?  Some of them are non-callee-saved.

I would suggest you do this as a follow-on change.

[Luis Héctor Chávez, 2015/08/12 22:14:26]

Acknowledged.

+      "ldr r0, [r14, %[cpsr]]\n"
+      "msr APSR_nzcvqg, r0\n"
+
+      /* Restore general-purpose registers */
+      "ldmia r14, {r0-r10}\n"

[Mark Seaborn, 2015/08/12 01:43:07]

How about "{r0-pc}"?  Then you shouldn't need the further code below.  On ARM
it's really easy to restore all the general purpose registers (+ PC) in one
instruction.

[Luis Héctor Chávez, 2015/08/12 22:14:26]

The docs mentioned two deprecations for this instruction:

* SP must not be in the list
* If PC is in the list, LR must not be in the list.

GCC prints a warning but the test passes. FWIW, glibc (longjmp) and the kernel
(sigreturn) have a similar implementation.

If we are OK with the warning, I can change it to {r0-pc}. Stress testing did
not show anything wrong.

[Mark Seaborn, 2015/08/13 21:17:50]

Can you give me a pointer to which docs say that, and/or what they say?
Do you mean glibc and Linux use "ldm ... {r0-pc}"?
I think if Linux and glibc use {r0-pc}, that would be preferred.  What is the
warning?  Did you find a way to turn it off?

[Luis Héctor Chávez, 2015/08/13 22:48:12]

http://infocenter.arm.com/help/topic/com.arm.doc.dui0489i/DUI0489I_arm_assemb...
page 3-85 (187 in the .pdf)

"In 32-bit Thumb instructions:
• the SP cannot be in the list
• the PC and LR cannot both be in the list
• there must be two or more registers in the list

ARM load instructions can have SP and PC in the reglist but these instructions
that include SP
in the reglist or both PC and LR in the reglist are deprecated in ARMv6T2 and
above."
glibc in ARM splits the loading of the registers in several steps. It only uses
ldmia for a few registers and then the rest are loaded individually.

In Linux the example I looked at was that arch/arm/kernel/entry-header.S has two
implementations for svc_exit: one for non-Thumb2 that does use ldmia {r0-pc} and
another one for Thumb2 where "stm/ldm rd, {sp, lr} is not available" where they
do something similar to what I'm doing (although they just load r0-r12 and then
deal with the rest of the registers in a much more straightforward fashion).
pnacl-translate issues the following warning:

<inline asm>:3:1: warning: use of SP in the list is deprecated
ldmia r14, {r0-pc}

I'm not very familiar with clang and had no success in finding a way to silence
it.

[Mark Seaborn, 2015/08/17 23:55:56]

OK, you can leave the ARM code as it is.  Could you just add a brief comment
saying why the code doesn't use "ldmia r14, {r0-pc}", please?

[Luis Héctor Chávez, 2015/08/18 02:53:31]

Done.

+
+      /*
+       * Copy r11, r12, lr, and pc just before the original sp.
+       * r12 will work as a temporary sp. r11 will be the scratch register, and
+       * will be restored just before moving sp.
+       */
+      "ldr r12, [r14, %[sp]]\n"
+
+      "ldr r11, [r14, %[pc]]\n"
+      "stmdb r12!, {r11}\n"
+      "ldr r11, [r14, %[lr]]\n"
+      "stmdb r12!, {r11}\n"
+      "ldr r11, [r14, %[r12]]\n"
+      "stmdb r12!, {r11}\n"
+      "ldr r11, [r14, %[r11]]\n"
+      "mov sp, r12\n"
+
+      /*
+       * Restore r12, lr, and pc. sp will point to the correct location once
+       * we're done.
+       */
+      "pop {r12, lr}\n"
+      "pop {pc}\n"
+      :
+      : "r" (a14),
+        OFFSET(cpsr),
+        OFFSET(r11),
+        OFFSET(r12),
+        OFFSET(sp),
+        OFFSET(lr),
+        OFFSET(pc)
+  );
+
+#undef OFFSET
+
+#else
+# error Unsupported architecture
+#endif
+
+  /* Should never reach this. */
+  __builtin_trap();
+}
+
+static __attribute__((noreturn))
+void restore_context(void *raw_ctx) {
+  const struct linux_ucontext_t *uctx = (struct linux_ucontext_t *) raw_ctx;
+  const linux_mcontext_t *mctx = &uctx->uc_mcontext;
+  nonsfi_restore_context(mctx);
+}
+
+static void signal_catch(int sig, linux_siginfo_t *info, void *uc) {
+  if (g_signal_handler_function_pointer) {
+    struct NonSfiExceptionFrame exception_frame;
+    exception_frame_from_signal_context(&exception_frame, uc);
+    g_signal_handler_function_pointer(&exception_frame.context);
+  }
+  restore_context(uc);
+}
+
+static void nonsfi_install_signal_handler_locked() {
+  struct linux_sigaction sa;
+
+  memset(&sa, 0, sizeof(sa));
+  sa.sa_sigaction = signal_catch;
+
+  /*
+   * User signal handler can be recursively interrupted to avoid having
+   * to allow sigreturn/sigprocmask.
+   */
+  sa.sa_flags = LINUX_SA_SIGINFO | LINUX_SA_NODEFER;
+  sigset_t *mask = (sigset_t*)&sa.sa_mask;
+  sigemptyset(mask);
+
+  /*
+   * Install a single handler.  Multiple signals can be multiplexed in
+   * userspace.
+   */
+  if (linux_sigaction(LINUX_SIGUSR1, &sa, NULL) != 0)
+    abort();
+}
+
+void nonsfi_initialize_signal_handler_locked() {
+  if (g_signal_handler_initialized)
+    return;
+  nonsfi_install_exception_handler_locked();
+  nonsfi_install_signal_handler_locked();
+  g_tgid = getpid();

[Mark Seaborn, 2015/08/12 01:43:07]

You should check for errors in these two syscalls, since they could easily be
blocked by seccomp-bpf.

[Luis Héctor Chávez, 2015/08/12 22:14:26]

Added an abort() on failure for both (although the purpose of this function is
that it should be called before seccomp-bpf is enabled, since
nonsfi_install_.*_locked() above call sigaction()).

+  g_main_tid = syscall(__NR_gettid);
+  g_signal_handler_initialized = 1;
+}
+
+/*
+ * Initialize signal handlers before entering sandbox.
+ */
+void nonsfi_initialize_signal_handler() {
+  if (pthread_mutex_lock(&g_signal_handler_mutex) != 0)
+    abort();
+  nonsfi_initialize_signal_handler_locked();
+  if (pthread_mutex_unlock(&g_signal_handler_mutex) != 0)
+    abort();
+}
+
+int nacl_signal_set_handler(NaClIrtSignalHandler handler) {
+  if (pthread_mutex_lock(&g_signal_handler_mutex) != 0)
+    abort();
+  nonsfi_initialize_signal_handler_locked();
+  g_signal_handler_function_pointer = handler;
+  if (pthread_mutex_unlock(&g_signal_handler_mutex) != 0)
+    abort();
+  return 0;
+}
+
+int nacl_signal_send_async_signal(nacl_irt_tid_t tid) {
+  if (!g_signal_handler_initialized)
+    return ESRCH;
+  if (tid == 0)
+    tid = g_main_tid;
+  if (linux_tgkill(g_tgid, tid, LINUX_SIGUSR1) == -1)
+    return errno;
+  return 0;
+}